diff --git a/.github/workflows/ci-timings.yml b/.github/workflows/ci-timings.yml index 73a9ba4b97..74f27af32c 100644 --- a/.github/workflows/ci-timings.yml +++ b/.github/workflows/ci-timings.yml @@ -12,12 +12,13 @@ concurrency: permissions: contents: read actions: read + pull-requests: read jobs: report: name: Collect recent completed workflow attempts runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 steps: # Consume only API data and execute batch code from the trusted default branch. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -27,6 +28,18 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc + - name: Cache npm package downloads + id: npm-cache + continue-on-error: true + uses: ./.github/actions/npm-cache + - name: Diagnose missing and startup-failed PR CI + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + CI_REPOSITORY: ${{ github.repository }} + CI_REPOSITORY_ID: ${{ github.repository_id }} + CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node scripts/ci/pr-ci-health.mjs - name: Record queue, execution, and matched step timing continue-on-error: true env: @@ -44,3 +57,28 @@ jobs: if-no-files-found: error overwrite: true retention-days: 14 + - name: Install dependencies for static failure attribution + id: failure-deps + if: ${{ !cancelled() && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }} + continue-on-error: true + run: npm ci + - name: Attribute supported static failures against main + if: ${{ !cancelled() && steps.failure-deps.outcome == 'success' && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + CI_REPOSITORY: ${{ github.repository }} + CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node scripts/ci/failure-report.mjs + - name: Upload CI health and failure diagnostics + if: ${{ !cancelled() }} + continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ci-health-diagnostics + path: | + .cache/ci-pr-health/ + .cache/ci-failures/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8da32ee270..b35b6a6e18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,12 @@ jobs: - name: Validate workflow syntax and expressions if: needs.changes.outputs.workflows == 'true' run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 -shellcheck= -pyflakes= + - name: Record static task source and execution identity + continue-on-error: true + if: needs.changes.outputs.code == 'true' + env: + CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: npm run ci:static-context - if: needs.changes.outputs.code == 'true' run: npm run check:ci - if: needs.changes.outputs.code == 'true' @@ -167,6 +173,34 @@ jobs: if-no-files-found: ignore retention-days: 7 + test-source: + name: Source tests + needs: changes + if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache + - run: npm ci + - name: Run source tests without waiting for build + run: npm run test:fast -- --reporter=default --reporter=blob + env: + VITEST_BLOB_LABEL: source + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ !cancelled() }} + with: + name: vitest-report-source + path: .vitest/blob/* + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + test-shard: name: Test shard (${{ matrix.shard }}) needs: [changes, build] @@ -178,7 +212,9 @@ jobs: # every shard's diagnostics and coverage. Passing PRs run the full suite. fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }} matrix: - shard: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6] + # Fast feedback spends the same six test jobs on one source group and + # five built groups; full qualification retains six complete shards. + shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/5","2/5","3/5","4/5","5/5"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 @@ -197,15 +233,16 @@ jobs: run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }} env: REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - - name: Run complete portable suite for fast feedback + - name: Run built-dependent tests for fast feedback if: needs.changes.outputs.tier == 'fast' - run: npm run test:run -- --reporter=default --reporter=blob --shard=${{ matrix.shard }} + run: npm run test:built -- --reporter=default --reporter=blob --shard=${{ matrix.shard }} env: + VITEST_BLOB_LABEL: built REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ !cancelled() }} with: - name: vitest-report-${{ strategy.job-index }} + name: vitest-report-built-${{ strategy.job-index }} path: .vitest/blob/* include-hidden-files: true if-no-files-found: error @@ -219,11 +256,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - cache: npm + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache - run: npm ci - name: Run reverse tools tests with coverage run: npm run test:coverage:reverse-tools @@ -316,7 +355,7 @@ jobs: test: name: Portable tests - needs: [changes, test-shard] + needs: [changes, test-source, test-shard] if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }} runs-on: ubuntu-latest timeout-minutes: 10 @@ -340,7 +379,9 @@ jobs: - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 if: >- !cancelled() && needs.changes.outputs.code == 'true' && - (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure') + (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' || + (needs.changes.outputs.tier == 'fast' && + (needs.test-source.result == 'success' || needs.test-source.result == 'failure'))) with: path: .vitest/blob pattern: vitest-report-* @@ -368,7 +409,8 @@ jobs: if: >- !cancelled() && needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' && - (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure') + (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' || + needs.test-source.result == 'success' || needs.test-source.result == 'failure') run: >- npm exec -- vitest --merge-reports --reporter=default @@ -381,6 +423,8 @@ jobs: if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-timings.json') != '' env: SHARD_RESULT: ${{ needs.test-shard.result }} + SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }} + SOURCE_RESULT: ${{ needs.test-source.result }} MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} CI_VERIFY_TEST_INVENTORY: "true" CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} @@ -413,7 +457,13 @@ jobs: env: CODE_REQUIRED: ${{ needs.changes.outputs.code }} SHARD_RESULT: ${{ needs.test-shard.result }} - run: test "${CODE_REQUIRED}" != "true" || test "${SHARD_RESULT}" = "success" + SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }} + SOURCE_RESULT: ${{ needs.test-source.result }} + run: | + if [ "${CODE_REQUIRED}" = "true" ]; then + test "${SHARD_RESULT}" = "success" || exit 1 + test "${SOURCE_REQUIRED}" != "true" || test "${SOURCE_RESULT}" = "success" + fi conformance-fixtures: name: Source-built conformance fixtures @@ -847,6 +897,7 @@ jobs: build, reverse-tools-quality, test-shard, + test-source, sqlite-runtime, test, conformance-fixtures, diff --git a/.github/workflows/real-cutter.yml b/.github/workflows/real-cutter.yml index 8567779991..91121f2073 100644 --- a/.github/workflows/real-cutter.yml +++ b/.github/workflows/real-cutter.yml @@ -13,8 +13,8 @@ jobs: name: Build Windows bridge security boundary runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - name: Install the cross compiler used by the Windows native lane @@ -44,11 +44,13 @@ jobs: - platform: Windows runner: [self-hosted, cutter-upstream, Windows] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - cache: npm + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache - run: npm ci - if: matrix.platform == 'Windows' uses: actions/download-artifact@v4 diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index b4cf2e4ade..dd562d8137 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -63,6 +63,7 @@ export default defineConfig({ { text: "Ghidra DOS analysis", link: "/ghidra-dos" }, { text: "Ghidra raw binary", link: "/raw-binary" }, { text: "Ghidra NativeAOT", link: "/ghidra-nativeaot" }, + { text: "Ghidra PowerPC", link: "/ghidra-ppc" }, { text: "Ghidra semantics", link: "/ghidra-provider" }, ], }, diff --git a/docs/ghidra-ppc.md b/docs/ghidra-ppc.md new file mode 100644 index 0000000000..0341583785 --- /dev/null +++ b/docs/ghidra-ppc.md @@ -0,0 +1,90 @@ +# PowerPC ELF analysis through Ghidra + +This lane extends native ELF analysis to 32-bit big-endian PowerPC; it does +not execute PowerPC code. It is the first milestone of +[#1741](https://github.com/morluto/rea/issues/1741). GameCube/Wii DOL +executables, PPC64 and other PowerPC variants are separate follow-ups. + +## Admission boundary + +ELF `EM_PPC` (20) is retained as the provider-neutral `powerpc` family. The +target also retains class, byte order, `e_type` and raw `e_flags`. Recognition +does not mean every provider or PowerPC variant is supported. `EM_PPC64` (21) +is not recognized as this family. + +The Ghidra lane admits ELF32 `ET_EXEC`, big endian. ELF64, little-endian, +relocatable and shared objects are refused with a specific unsupported +constraint. `EF_PPC_EMB` declares an embedded variant and is refused; the +remaining `e_flags` bits only declare relocation conventions and do not change +instruction decoding. + +## Bound language + +Ghidra's ELF loader selects `PowerPC:BE:32:e500:default` for a plain `EM_PPC` +header with `e_flags` 0. e500 is the embedded Book-E variant whose SPE unit +replaces the classic floating-point unit, so it misinterprets ordinary +750-family code. Nothing in the header distinguishes the two. + +The lane therefore imports with `PowerPC:BE:32:default` and compiler spec +`default`, commits that choice in the analysis profile as +`language_selection: "provider-pinned-v1"`, and checks the language Ghidra +reports after import. A different language set through +`REA_GHIDRA_LANGUAGE_ID` (and optionally `REA_GHIDRA_COMPILER_SPEC_ID`) +replaces the lane default and is admitted against the installed language +catalog as usual; for example `PowerPC:BE:32:e500` for genuine e500 code. + +PowerPC header facts are committed in the analysis profile so interpretation +changes invalidate profile-bound snapshots. The support-lane identity is +versioned independently of file identity. Evidence, lifecycle outputs and saved +snapshots preserve the `powerpc` family. Hopper and IDA adapters do not +implicitly gain PowerPC support. Windows Ghidra P0 remains PE x86/x86-64 only. + +## Use + +With a caller-installed Ghidra and compatible JDK on a supported Linux/macOS host: + +```bash +rea analyze /absolute/path/to/ppc.elf --provider ghidra --json +rea function /absolute/path/to/ppc.elf function_name --provider ghidra --json +``` + +MCP uses the existing tools, including `open_binary` with an absolute path and +`provider_id: "ghidra"`. No PowerPC-specific tool catalog or emulator is +introduced. + +## Verification + +The source-owned freestanding fixture is `tests/conformance/c/ppc.c`. The +optional cross-target lane requires a caller-supplied Clang with the PowerPC +target, LLD, GNU `readelf`, Ghidra and its compatible JDK; ordinary host-native +checks do not acquire these tools. `REA_PPC_CLANG` selects Clang and +`REA_PPC_READELF` selects GNU readelf (for example +`/opt/homebrew/opt/binutils/bin/readelf` on macOS). These are test-time +prerequisites, not REA runtime providers. + +```bash +GHIDRA_INSTALL_DIR=/absolute/path/to/ghidra npm run verify:ghidra:ppc +``` + +The lane compiles the fixture for a 750-family core (`-mcpu=750`) and never +executes it. GNU readelf independently supplies header facts and symbol +addresses, compared with the production REA target resolver before provider +startup. Its version, bounded command output and selected tool path are +retained in the verification report. The parser deliberately recognizes only +this fixture's GNU output, rejecting missing, ambiguous or unexpected fields +rather than substituting defaults. + +Existing public MCP operations check Ghidra's **observed** language and +compiler specification and loaded-file hashes; an e500 import fails the lane. +The source-owned `ppc-probe.S` supplies fixed `addi` and conditional-branch +encodings: verification checks bytes, length, immediate and direct destination, +independent of decompiler variable names. Global and string bytes are checked at +independently reported symbol addresses. + +CLI/MCP discovery and decompilation, direct calls, target/profile/Evidence +identity, snapshot round-trip and final process cleanup remain covered. +Pseudocode presence is a liveness check, not proof of semantic equivalence. +Instruction decoding and reader agreement do not establish runtime or complete +floating-point semantics. Parser/profile tests are narrower checks, not +substitutes for this real-provider run. Report the actual lanes run and leave +unexecuted verification pending. diff --git a/docs/managed-code-analysis.md b/docs/managed-code-analysis.md index 9f86551d7b..05782b4a2e 100644 --- a/docs/managed-code-analysis.md +++ b/docs/managed-code-analysis.md @@ -209,19 +209,27 @@ native/runtime questions retain their own authority and coverage. Names and semantic labels do not establish durable member identity. Comparison preserves observed differences, structural inferences, and unknowns separately. -Cross-version matching first prefers exact raw CIL/signature identity, with -equal decoded fingerprints. Equal decoded fingerprints alone cannot establish -an exact match: they omit switch targets and collapse distinct NaN encodings. -When that does not match, it pairs an exact declared type, method name, and raw -signature before trying structural body shape. The exact-signature key uses names only -as part of that full tuple; names alone never select a pair. Duplicate tuples -remain ambiguous rather than being paired by token order. Fields follow the -same exact tiers. The raw signature is exact whether or not REA decoded it, so -undecoded signatures pair by that tuple too, but they never enter structural -rounds. An unmatched member is therefore reported as `unknown`, not added or -removed, when its own signature was not decoded or when the other side has an -unpaired member, one-sided or ambiguous, with the same declared type and name -whose signature was not decoded. +Cross-version matching first pairs members of the same declared type and name: +methods by exact CIL/signature identity, then by raw signature, then by +structural body shape; fields by raw signature. Exact CIL/signature identity +requires equal raw CIL digests and decoded fingerprints. Equal decoded +fingerprints alone cannot establish an exact match: they omit switch targets +and collapse distinct NaN encodings. Only then may a remap pair a +renamed or moved member across declared types or names: a method by exact +CIL/signature identity or structural body shape, a field by exact decoded +signature. A remap never takes a member whose own declared type and name remain +unpaired on the other side, so a member is never taken from its own changed +counterpart. A member whose declaring type is unresolved could belong to any +type, so it pairs by name only when no other member shares its exact +CIL/signature key, never by structural shape alone, and any remap group it +joins stays ambiguous. The exact-signature key uses names only as part of that +full tuple; names alone never select a pair. Duplicate tuples remain ambiguous +rather than being paired by token order. The raw signature is exact whether or +not REA decoded it, so undecoded signatures pair by that tuple too, but they +never enter structural rounds. An unmatched member is therefore reported as +`unknown`, not added or removed, when its own signature was not decoded or when +the other side has an unpaired member, one-sided or ambiguous, with the same +declared type and name whose signature was not decoded. For a matched method, an unavailable or partial body makes body-shape facets unknown while preserving observed signature differences. For complete bodies, diff --git a/docs/testing.md b/docs/testing.md index b543fe89f7..aa8d2163ff 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -280,8 +280,15 @@ focused suite also exercises affected MCP routes and the Python Cutter bridge. Contract, integration, and real-provider lanes complement these focused tests; simulated providers do not establish real-provider behavior. -CI runs six native Vitest shards without retries. Each shard emits a blob -report; the merge job produces JUnit and JSON diagnostics, a module timing +Fast CI starts the source-only projects after classification, independently of +build and static checks, and runs five built-dependent shards after the build. +The six test jobs still select every portable test. Full qualification keeps six +complete Vitest shards with coverage. Neither tier retries tests. Canonical +project ownership lives in `scripts/lib/test-projects.mjs`, shared by Vitest, +CI group execution, and development-test preparation. A test importing compiled +verifier code belongs to the built group even if its directory is conformance. +Source and built blobs have separate labels to prevent merge collisions. +Each test job emits a blob report; the merge job produces JUnit and JSON diagnostics, a module timing report, plus aggregate coverage during full qualification. It retains available reports even when a shard fails, uploads available diagnostics, and writes the slowest files to the workflow summary. Partial reports are marked diagnostic @@ -316,6 +323,30 @@ portable test shard. These limits leave headroom for cold setup and full coverag while preventing a hung job from holding the active main run for hours. They do not bound runner queue waiting. Timeouts still fail the required gate. +The separate `CI timings` workflow also diagnoses missing, startup-failed, +queued/running, approval-required, canceled, and failed PR CI through bounded, +read-only API scans. Incomplete pagination or changing identity stays unknown. +It never closes/reopens PRs, reruns workflows, posts comments, or changes gates. +The timing manifest retains each run's event and head branch, including explicit +unknowns, so PR runs cannot consume the eligible-main comparison budget. +Its static failure report compares supported lint/duplication diagnostics with +failed main runs using exact source, ancestry, task hashes, and toolchain proof. +A match describes those diagnostics only; it does not establish that every +failure in a job came from main. Duplication reports retain the actual checker +exit result, so passing duplication checks preserve lint failures and caller +thresholds remain owned by the producer. Missing checker outcomes leave +those duplication diagnostics unclassified. Missing historical receipts remain unknown. +The reporter executes only trusted default-branch code and parses whitelisted +JSON artifact members without executing artifact content. Reports and limits +are retained in `ci-health-diagnostics`; its ten-minute job limit covers the +existing four-minute timing collection plus the bounded diagnostic stages. + +Shard-count reduction remains deferred: the available successful timing sample +covers an older inventory, and file phase weights are not job wall time. The +one-source/five-built split retains six test jobs; it makes no hosted speedup +claim. Current complete timings and completed candidate hosted runs are needed +before enabling a smaller adaptive matrix. + The separate `CI timings` workflow batches recent completed CI and Release runs hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and a normalized point-in-time report for each collected workflow attempt, plus a diff --git a/package.json b/package.json index dff275eb44..5ccf53c994 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,9 @@ "precheck:pr": "npm run deps:check", "check:pr": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos", "precheck:ci": "npm run deps:check", - "check:ci": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos && npm run verify:architecture-guards", + "check:ci": "node scripts/run-exclusive.mjs artifacts npm run check:ci:tasks && npm run verify:architecture-guards", + "check:ci:tasks": "turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos", + "ci:static-context": "node scripts/ci/static-task-context.mjs", "build:cached": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills", "build:termux": "node scripts/run-exclusive.mjs artifacts npm run build:termux:run", "build:termux:run": "npm run compile:termux && npm run artifacts:skills", @@ -138,7 +140,8 @@ "test:run": "vitest run", "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v", "test:local": "node scripts/run-development-tests.mjs local", - "test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation", + "test:fast": "node scripts/ci/test-group.mjs source", + "test:built": "node scripts/ci/test-group.mjs built", "pretest:boundary": "npm run test:prepare", "test:boundary": "vitest run --project boundary --project cli-boundary --project process-boundary --project mcp-boundary --project process-global", "pretest:mcp": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills artifacts:mcp-catalog", @@ -183,6 +186,7 @@ "verify:ghidra": "npm run build:cached && node scripts/verify-real-ghidra.mjs", "verify:ghidra:seeds": "npm run build:cached && node scripts/verify/ghidra/seeds.mjs", "verify:ghidra:mips": "npm run build:cached && node scripts/verify-real-ghidra-mips.mjs", + "verify:ghidra:ppc": "npm run build:cached && node scripts/verify-real-ghidra-ppc.mjs", "verify:ghidra:riscv": "npm run build:cached && node scripts/verify-real-ghidra-riscv.mjs", "verify:ghidra:6502": "npm run build:cached && node scripts/verify-real-ghidra-6502.mjs", "verify:ghidra:mips-shared": "npm run build:cached && node scripts/verify-real-ghidra-mips-shared.mjs", diff --git a/scripts/ci/artifacts.d.mts b/scripts/ci/artifacts.d.mts new file mode 100644 index 0000000000..93cf2f7013 --- /dev/null +++ b/scripts/ci/artifacts.d.mts @@ -0,0 +1,67 @@ +import type { NormalizedWorkflowRun } from "./workflow-run.mjs"; +import type { StaticTaskContext } from "./static-task-context.mjs"; +export type { StaticTaskContext } from "./static-task-context.mjs"; + +export interface StaticDiagnostic { + readonly kind: "lint" | "jscpd"; + readonly path?: string; + readonly paths?: readonly string[]; + readonly code?: string; + readonly message?: string; + readonly severity?: string; +} + +export function classifyFailure(input: { + repository: string; + defaultBranch: string; + run: Record & NormalizedWorkflowRun; + job: Record; + diagnostics: readonly StaticDiagnostic[]; + context: StaticTaskContext; + mainRun: Record & NormalizedWorkflowRun; + mainJob: Record; + mainDiagnostics: readonly StaticDiagnostic[]; + mainContext: StaticTaskContext; + executionSourcesVerified: boolean; + ancestry: { status: string; mergeBaseSha: string | null }; + blobs: Readonly>; +}): { + classification: "matching-main-diagnostics" | "unknown"; + reason: string; +}; + +export function diagnosticPaths(item: StaticDiagnostic): readonly string[]; +export function isFailedConclusion( + value: unknown, +): value is "failure" | "timed_out"; +export function isEligibleMainRun( + run: Record, + defaultBranch: string, + repository: string, +): boolean; +export function pullRequestBaseSha( + run: Record, + repository: string, +): string | null; +export function safeSha(value: unknown): string | null; +export function validRepoPath(value: unknown): value is string; +export function selectStaticCheckArtifact( + page: unknown, + run: { id?: string | number; head_sha?: string }, + runAttempt: number, +): unknown | null; + +export function readStaticCheckArtifact(options: { + repository: string; + artifactId: number | string; + token: string; + fetchImpl?: typeof fetch; +}): Promise<{ + context: StaticTaskContext; + diagnostics: readonly StaticDiagnostic[]; +}>; + +export function parseStaticArchive(archivePath: string): Promise<{ + context: StaticTaskContext; + diagnostics: readonly StaticDiagnostic[]; +}>; diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjs new file mode 100644 index 0000000000..250a15872c --- /dev/null +++ b/scripts/ci/artifacts.mjs @@ -0,0 +1,442 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join } from "node:path"; +import { promisify } from "node:util"; + +import { parseStaticTaskContext } from "./static-task-context.mjs"; + +const execute = promisify(execFile); +const MAX_ARCHIVE_BYTES = 32 * 1024 * 1024; +const MAX_MEMBER_BYTES = 4 * 1024 * 1024; +const MAX_DIAGNOSTICS = 10_000; +const API_ROOT = "https://api.github.com/repos"; +const SHA = /^[0-9a-f]{40}$/iu; +const TASK_HASH = /^[a-f0-9]{16}$/u; + +/** Compare parsed static diagnostics only after source, task, host, and ancestry proof. */ +export function classifyFailure({ + repository, + defaultBranch, + run, + job, + diagnostics, + context, + mainRun, + mainJob, + mainDiagnostics, + mainContext, + executionSourcesVerified, + ancestry, + blobs, +}) { + const failedGate = firstFailure([ + () => [ + !isCompletedFailure(run) || + job?.status !== "completed" || + !isFailedConclusion(job?.conclusion), + "run_or_job_not_a_completed_failure", + ], + () => [ + !isEligiblePullRun(run, repository) || + !isEligibleMainRun(mainRun, defaultBranch, repository), + "run_event_or_repository_identity_unverified", + ], + () => [ + job?.name !== mainJob?.name || job?.name !== "Static checks", + "failed_job_not_the_same_static_gate", + ], + () => [ + !Array.isArray(diagnostics) || + diagnostics.length === 0 || + !Array.isArray(mainDiagnostics), + "structured_diagnostics_missing_or_invalid", + ], + () => [ + !validContext(context) || !validContext(mainContext), + "static_task_context_missing_or_invalid", + ], + () => [ + executionSourcesVerified !== true, + "checkout_sources_not_verified_against_api_commits", + ], + () => [ + !sameExecutionContext(context, mainContext), + "task_or_toolchain_identity_differs", + ], + () => [ + ancestry?.status !== "ahead" && ancestry?.status !== "identical", + "main_head_not_proven_ancestor_of_tested_base", + ], + () => [ + ancestry?.mergeBaseSha !== mainRun?.head_sha, + "main_head_is_not_the_verified_merge_base", + ], + () => [ + !sameTaskHashes(diagnostics, context, mainContext), + "owning_task_hash_differs", + ], + ]); + if (failedGate) return outcome("unknown", failedGate.reason); + const keys = diagnostics.map(diagnosticKey); + const mainKeys = new Set(mainDiagnostics.map(diagnosticKey).filter(Boolean)); + if (keys.some((key) => key === null)) + return outcome("unknown", "diagnostic_shape_not_supported"); + if (!keys.every((key) => mainKeys.has(key))) + return outcome( + "unknown", + "main_run_does_not_contain_the_same_structured_diagnostic", + ); + const blobStatus = blobIdentityStatus(diagnostics, blobs); + if (blobStatus !== "identical") + return outcome( + "unknown", + blobStatus === "different" + ? "diagnostic_source_content_not_identical_at_main_and_tested_base" + : "diagnostic_source_blob_identity_unavailable", + ); + return outcome( + "matching-main-diagnostics", + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + ); +} + +export function diagnosticPaths(item) { + return item.kind === "jscpd" ? item.paths : [item.path]; +} +export function isFailedConclusion(value) { + return value === "failure" || value === "timed_out"; +} +export function isEligibleMainRun(run, defaultBranch, repository) { + return ( + isEligibleStaticRun(run) && + run.repository.full_name === repository && + run.head_branch === defaultBranch && + ["push", "schedule", "workflow_dispatch"].includes(run.event) + ); +} +export function pullRequestBaseSha(run, repository) { + if ( + run?.event !== "pull_request" || + run.repository?.full_name !== repository || + !Array.isArray(run.pull_requests) || + run.pull_requests.length !== 1 + ) + return null; + const base = run.pull_requests[0].base; + return base?.repo?.full_name === repository && SHA.test(base?.sha ?? "") + ? base.sha + : null; +} + +export function safeSha(value) { + return typeof value === "string" && SHA.test(value) + ? value.toLowerCase() + : null; +} +export function selectStaticCheckArtifact(page, run, runAttempt) { + if (runAttempt !== 1) return null; + if ( + !Array.isArray(page?.artifacts) || + !Number.isSafeInteger(page.total_count) || + page.total_count !== page.artifacts.length || + page.artifacts.length >= 100 + ) + return null; + const matches = page.artifacts.filter( + (artifact) => artifact.name === "static-check-reports", + ); + if (matches.length !== 1) return null; + const [artifact] = matches; + if ( + artifact.expired !== false || + String(artifact.workflow_run?.id) !== String(run.id) || + artifact.workflow_run?.head_sha !== run.head_sha + ) + return null; + return artifact; +} +export function validRepoPath(value) { + return ( + typeof value === "string" && + value.length > 0 && + !isAbsolute(value) && + !value.includes("\\") && + value + .split("/") + .every((part) => part !== "" && part !== "." && part !== "..") && + !value.startsWith("/") + ); +} + +/** Download one named static report archive and parse only whitelisted JSON members. */ +export async function readStaticCheckArtifact({ + repository, + artifactId, + token, + fetchImpl = fetch, +}) { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? "")) + throw new Error("artifact repository identity is invalid"); + if (!token) throw new Error("artifact download requires a GitHub token"); + const url = `${API_ROOT}/${repository}/actions/artifacts/${positiveId(artifactId)}/zip`; + let response = await fetchImpl(url, { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28", + }, + signal: AbortSignal.timeout(10_000), + redirect: "manual", + }); + if ([301, 302, 303, 307, 308].includes(response.status)) { + const location = response.headers.get("location"); + if (!location || new URL(location).protocol !== "https:") + throw new Error("static_artifact_redirect_invalid"); + response = await fetchImpl(location, { + signal: AbortSignal.timeout(10_000), + redirect: "error", + }); + } + if (!response.ok) throw new Error(`github_http_${response.status}`); + const bytes = await readLimitedBytes(response, MAX_ARCHIVE_BYTES); + const directory = await mkdtemp(join(tmpdir(), "rea-ci-static-artifact-")); + const archive = join(directory, "artifact.zip"); + try { + await writeFile(archive, bytes, { flag: "wx" }); + return await parseStaticArchive(archive); + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +/** Accept only the three named producer outputs; no archive paths are written to disk. */ +export async function parseStaticArchive(archivePath) { + const { stdout: listing } = await execute("unzip", ["-Z1", archivePath], { + encoding: "utf8", + maxBuffer: 128 * 1024, + timeout: 5_000, + }); + const allowed = new Set([ + "oxlint.json", + "jscpd/jscpd-report.json", + "static-task-context.json", + ]); + const members = new Map(); + for (const name of listing.split(/\r?\n/u).filter(Boolean)) { + const prefix = ".cache/static-checks/"; + const member = name.startsWith(prefix) ? name.slice(prefix.length) : name; + if (!allowed.has(member)) continue; + if (members.has(member)) + throw new Error("static artifact contains duplicate report members"); + const { stdout } = await execute("unzip", ["-p", archivePath, name], { + encoding: "buffer", + maxBuffer: MAX_MEMBER_BYTES, + timeout: 5_000, + }); + members.set(member, JSON.parse(stdout.toString("utf8"))); + } + const context = parseStaticTaskContext( + members.get("static-task-context.json"), + ); + const diagnostics = normalizeDiagnostics( + members.get("oxlint.json"), + members.get("jscpd/jscpd-report.json"), + ); + if (diagnostics.length > MAX_DIAGNOSTICS) + throw new Error("static diagnostic count exceeds the reporting limit"); + return { context, diagnostics }; +} + +function normalizeDiagnostics(lint, duplicate) { + return [...normalizeLint(lint), ...normalizeDuplicates(duplicate)]; +} + +function normalizeLint(report) { + if (report === undefined) return []; + if (!Array.isArray(report?.diagnostics)) + throw new Error("oxlint diagnostics are malformed"); + return report.diagnostics + .filter((item) => item.severity === "error") + .map((item) => { + if ( + typeof item.filename !== "string" || + !repoPath(item.filename) || + typeof item.code !== "string" || + typeof item.message !== "string" + ) + throw new Error("oxlint error diagnostic is malformed"); + return { + kind: "lint", + path: item.filename, + code: item.code, + message: item.message, + severity: "error", + }; + }); +} + +function normalizeDuplicates(report) { + if (report === undefined) return []; + if ( + !Array.isArray(report?.duplicates) || + !Number.isFinite(report.statistics?.total?.percentage) || + report.statistics.total.percentage < 0 + ) + throw new Error("jscpd report is malformed"); + // The producer owns the checker outcome; do not copy or infer its threshold. + const exitCode = report.reaCheck?.exitCode; + if (exitCode === undefined || exitCode === 0) return []; + if (!Number.isSafeInteger(exitCode) || exitCode < 0) + throw new Error("jscpd checker outcome is malformed"); + if (exitCode !== 1) return []; + return report.duplicates.map((item) => { + const paths = [item.firstFile?.name, item.secondFile?.name].sort(); + if (paths.some((path) => !repoPath(path))) + throw new Error("jscpd duplicate paths are malformed"); + return { kind: "jscpd", paths }; + }); +} + +async function readLimitedBytes(response, limit) { + const reader = response.body?.getReader(); + if (!reader) throw new Error("static artifact response has no body"); + const chunks = []; + let size = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) { + await reader.cancel(); + throw new Error("static artifact exceeds the download limit"); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const result = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; +} + +function repoPath(value) { + return ( + typeof value === "string" && + value.length > 0 && + !value.startsWith("/") && + !value.includes("\\") && + value + .split("/") + .every((part) => part !== "" && part !== "." && part !== "..") + ); +} + +function positiveId(value) { + const id = Number(value); + if (!Number.isSafeInteger(id) || id < 1) + throw new Error("artifact ID must be a positive integer"); + return String(id); +} + +function diagnosticKey(item) { + if ( + item?.kind === "lint" && + validRepoPath(item.path) && + typeof item.code === "string" && + typeof item.message === "string" && + item.severity === "error" + ) + return JSON.stringify([ + item.kind, + item.path, + item.code, + item.message, + item.severity, + ]); + if ( + item?.kind === "jscpd" && + Array.isArray(item.paths) && + item.paths.length === 2 && + item.paths.every(validRepoPath) + ) + return JSON.stringify([item.kind, ...[...item.paths].sort()]); + return null; +} +function blobIdentityStatus(diagnostics, blobs) { + const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))]; + if (paths.some((path) => blobs?.[path] === false)) return "different"; + return paths.length > 0 && paths.every((path) => blobs?.[path] === true) + ? "identical" + : "unavailable"; +} +function sameTaskHashes(diagnostics, left, right) { + return diagnostics.every((diagnostic) => { + const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint:built"; + return ( + TASK_HASH.test(left.taskHashes[task] ?? "") && + left.taskHashes[task] === right.taskHashes[task] + ); + }); +} +function validContext(value) { + return ( + value?.version === 1 && + SHA.test(value.sourceSha ?? "") && + value.sourceClean === true && + typeof value.nodeVersion === "string" && + typeof value.npmVersion === "string" && + typeof value.turboVersion === "string" && + typeof value.platform === "string" && + typeof value.architecture === "string" && + value.taskHashes && + typeof value.taskHashes === "object" && + !Array.isArray(value.taskHashes) + ); +} +function sameExecutionContext(left, right) { + return [ + "nodeVersion", + "npmVersion", + "turboVersion", + "platform", + "architecture", + ].every((key) => left[key] === right[key]); +} +function isEligiblePullRun(run, repository) { + return ( + isEligibleStaticRun(run) && pullRequestBaseSha(run, repository) !== null + ); +} +function isEligibleStaticRun(run) { + return ( + isRecord(run) && + run.status === "completed" && + isFailedConclusion(run.conclusion) && + SHA.test(run.head_sha ?? "") && + typeof run.repository?.full_name === "string" && + run.workflowPath?.file === ".github/workflows/ci.yml" + ); +} +function isCompletedFailure(run) { + return run?.status === "completed" && isFailedConclusion(run.conclusion); +} +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function firstFailure(checks) { + for (const check of checks) { + const [failed, reason] = check(); + if (failed) return { reason }; + } + return null; +} +function outcome(classification, reason) { + return { classification, reason }; +} diff --git a/scripts/ci/failure-report.d.mts b/scripts/ci/failure-report.d.mts new file mode 100644 index 0000000000..47ac65745d --- /dev/null +++ b/scripts/ci/failure-report.d.mts @@ -0,0 +1,159 @@ +import type { NormalizedWorkflowRun } from "./workflow-run.mjs"; +import type { StaticDiagnostic } from "./artifacts.mjs"; +export type { StaticDiagnostic } from "./artifacts.mjs"; +import type { StaticTaskContext } from "./static-task-context.mjs"; +export type { StaticTaskContext } from "./static-task-context.mjs"; +export interface CiRun { + readonly id?: string | number; + readonly run_attempt?: number; + readonly status?: string; + readonly conclusion?: string | null; + readonly event?: string; + readonly path?: string; + readonly head_sha?: string; + readonly head_branch?: string; + readonly repository?: { readonly full_name?: string }; + readonly pull_requests?: readonly { + readonly base?: { + readonly sha?: string; + readonly repo?: { readonly full_name?: string }; + }; + }[]; +} + +export interface CiJob { + readonly name?: string; + readonly status?: string; + readonly conclusion?: string | null; + readonly run_id?: number; + readonly run_attempt?: number; +} + +export interface StaticEvidence { + readonly context: StaticTaskContext; + readonly diagnostics: readonly StaticDiagnostic[]; +} + +export interface CiFailureReport { + readonly version: 1; + readonly observedAt: string; + readonly repository: string; + readonly defaultBranch: string; + readonly informationalOnly: true; + readonly gateEffect: string; + readonly boundedInput: { + readonly candidateRuns: number; + readonly limit: number; + readonly mainCandidateLimit: number; + readonly mainCandidateRuns: number; + readonly mainRunLookupLimit: number; + readonly manifestTruncated: boolean; + }; + readonly counts: Readonly< + Record<"matching-main-diagnostics" | "unknown", number> + >; + readonly failures: readonly { + readonly runId: number | string; + readonly runAttempt: number; + readonly classification: "matching-main-diagnostics" | "unknown"; + readonly reason: string; + readonly candidateFailures?: readonly { + readonly runId: number | string; + readonly runAttempt: number; + readonly reason: string; + }[]; + readonly runHead?: string | null; + readonly event?: string | null; + readonly job?: string; + readonly conclusion?: string | null; + }[]; +} + +export interface FailureReportApi { + getRun(repository: string, id: string | number): Promise; + getJobs( + repository: string, + id: string | number, + attempt: number, + ): Promise; + getStaticEvidence( + repository: string, + run: CiRun, + job: CiJob, + ): Promise; + getAncestry( + repository: string, + ancestor: string, + descendant: string, + ): Promise<{ status: string; mergeBaseSha: string | null }>; + verifyExecutionSource( + repository: string, + run: CiRun, + context: StaticTaskContext, + ): Promise; + getBlobIdentity( + repository: string, + ancestor: string, + descendant: string, + paths: readonly string[], + ): Promise>>; +} + +export function classifyFailure(input: { + repository: string; + defaultBranch: string; + run: CiRun & NormalizedWorkflowRun; + job: CiJob; + diagnostics: readonly StaticDiagnostic[]; + context: StaticTaskContext; + mainRun: CiRun & NormalizedWorkflowRun; + mainJob: CiJob; + mainDiagnostics: readonly StaticDiagnostic[]; + mainContext: StaticTaskContext; + executionSourcesVerified: boolean; + ancestry: { status: string; mergeBaseSha: string | null }; + blobs: Readonly>; +}): { + classification: "matching-main-diagnostics" | "unknown"; + reason: string; +}; + +export function reportCiFailures(options: { + repository: string; + defaultBranch: string; + batchManifest: { + readonly repository: string; + readonly runs: readonly { + readonly id: number | string; + readonly runAttempt: number; + readonly workflow: string; + readonly event?: string | null; + readonly headBranch?: string | null; + readonly conclusion: string | null; + readonly status: string; + }[]; + readonly truncation?: readonly unknown[]; + readonly coverage?: { readonly exhaustive?: boolean }; + readonly selection?: { readonly exhaustive?: boolean }; + }; + api: FailureReportApi; + outputDirectory?: string; + summaryPath?: string; + observedAt?: string; +}): Promise; + +export function createGitHubApi(options?: { + token?: string; + fetchImpl?: typeof fetch; + apiRequest?: ( + method: string, + endpoint: string, + budget: { requests: number; startedAt: number; limitMs: number }, + ) => Promise; +}): FailureReportApi; + +export function writeFailureReport( + report: CiFailureReport, + outputDirectory?: string, + summaryPath?: string, +): Promise<{ jsonPath: string; markdownPath: string }>; diff --git a/scripts/ci/failure-report.mjs b/scripts/ci/failure-report.mjs new file mode 100644 index 0000000000..12364c59f1 --- /dev/null +++ b/scripts/ci/failure-report.mjs @@ -0,0 +1,533 @@ +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + classifyFailure, + diagnosticPaths, + isEligibleMainRun, + isFailedConclusion, + pullRequestBaseSha, + readStaticCheckArtifact, + safeSha, + selectStaticCheckArtifact, + validRepoPath, +} from "./artifacts.mjs"; +import { createGhApiRequest } from "./github-api.mjs"; +import { normalizeWorkflowRun } from "./workflow-run.mjs"; + +const OUTPUT_DIRECTORY = join(".cache", "ci-failures"); +const BATCH_DIRECTORY = join(".cache", "ci-job-timings"); +const MAX_JSON_BYTES = 8 * 1024 * 1024; +const MAX_FAILED_RUNS = 20; +const MAX_MAIN_CANDIDATES = 5; +const MAX_MAIN_RUN_LOOKUPS = 100; +const MAX_PATHS_PER_COMPARISON = 50; +const MAX_API_REQUESTS = 250; + +export { classifyFailure } from "./artifacts.mjs"; + +/** Uses only failed CI candidates present in the supplied bounded timing manifest. */ +export async function reportCiFailures({ + repository, + defaultBranch, + batchManifest, + api, + outputDirectory = OUTPUT_DIRECTORY, + summaryPath, + observedAt = new Date().toISOString(), +}) { + validateRepository(repository); + if (typeof defaultBranch !== "string" || defaultBranch.length === 0) + throw new Error("defaultBranch is required"); + if ( + !batchManifest || + batchManifest.repository !== repository || + !Array.isArray(batchManifest.runs) + ) + throw new Error("timing manifest repository or runs are invalid"); + + const failedRuns = batchManifest.runs.filter( + (entry) => + entry.workflow === "CI" && + entry.status === "collected" && + isFailedConclusion(entry.conclusion), + ); + const prRuns = failedRuns.filter((entry) => entry.event === "pull_request"); + const candidates = prRuns.slice(0, MAX_FAILED_RUNS); + const mainEntries = failedRuns.filter( + (entry) => + entry.headBranch === defaultBranch && + ["push", "schedule", "workflow_dispatch"].includes(entry.event), + ); + const lookupRun = verifiedRunLookup(repository, api); + const mainCandidates = []; + for (const entry of mainEntries.slice(0, MAX_MAIN_RUN_LOOKUPS)) { + try { + const run = await lookupRun(entry); + if (isEligibleMainRun(run, defaultBranch, repository)) + mainCandidates.push({ + run, + getEvidence: mainEvidenceLookup(repository, api, entry, run), + }); + if (mainCandidates.length === MAX_MAIN_CANDIDATES) break; + } catch { + // Unverified metadata cannot consume the eligible-main candidate budget. + } + } + const entries = []; + for (const entry of candidates) { + try { + entries.push( + ...(await analyzeFailedRun({ + repository, + defaultBranch, + entry, + run: await lookupRun(entry), + mainCandidates, + api, + })), + ); + } catch (error) { + entries.push({ + runId: entry.id, + runAttempt: entry.runAttempt, + classification: "unknown", + reason: errorCode(error), + }); + } + } + const report = { + version: 1, + observedAt, + repository, + defaultBranch, + informationalOnly: true, + gateEffect: "none; this report never changes CI conclusions", + boundedInput: { + candidateRuns: candidates.length, + limit: MAX_FAILED_RUNS, + mainCandidateLimit: MAX_MAIN_CANDIDATES, + mainCandidateRuns: mainCandidates.length, + mainRunLookupLimit: MAX_MAIN_RUN_LOOKUPS, + manifestTruncated: + prRuns.length > MAX_FAILED_RUNS || + batchManifest.truncation?.length > 0 || + batchManifest.selection?.exhaustive === false || + batchManifest.coverage?.exhaustive === false, + }, + counts: Object.fromEntries( + ["matching-main-diagnostics", "unknown"].map((category) => [ + category, + entries.filter((item) => item.classification === category).length, + ]), + ), + failures: entries, + }; + await writeFailureReport(report, outputDirectory, summaryPath); + return report; +} + +async function analyzeFailedRun({ + repository, + defaultBranch, + entry, + run, + mainCandidates, + api, +}) { + const jobs = await api.getJobs(repository, entry.id, entry.runAttempt); + if (jobs.some((job) => !hasJobIdentity(job, entry))) + throw new Error("workflow_job_identity_mismatch"); + const failedJobs = jobs.filter( + (job) => job.status === "completed" && isFailedConclusion(job.conclusion), + ); + if (failedJobs.length === 0) + return [ + { + runId: entry.id, + runAttempt: entry.runAttempt, + classification: "unknown", + reason: "failed_jobs_not_present_in_api_snapshot", + }, + ]; + const failures = []; + for (const job of failedJobs) { + let classification; + try { + const evidence = await api.getStaticEvidence(repository, run, job); + classification = await attributeToMain({ + repository, + defaultBranch, + run, + job, + evidence, + mainCandidates, + api, + }); + } catch (error) { + classification = outcome("unknown", errorCode(error)); + } + failures.push({ + runId: entry.id, + runAttempt: entry.runAttempt, + runHead: safeSha(run.head_sha), + event: run.event ?? null, + job: job.name, + conclusion: job.conclusion, + ...classification, + }); + } + return failures; +} + +async function attributeToMain({ + repository, + defaultBranch, + run, + job, + evidence, + mainCandidates, + api, +}) { + const baseSha = pullRequestBaseSha(run, repository); + if (baseSha === null || evidence === null) + return outcome("unknown", "no_matching_verified_main_failure"); + let result = outcome("unknown", "no_matching_verified_main_failure"); + const candidateFailures = []; + for (const candidate of mainCandidates) { + try { + const main = await compareCandidate({ + repository, + defaultBranch, + run, + job, + evidence, + baseSha, + candidate, + api, + }); + if (main !== null) result = main; + if (main?.classification === "matching-main-diagnostics") break; + } catch (error) { + candidateFailures.push({ + runId: candidate.run.id, + runAttempt: candidate.run.run_attempt, + reason: errorCode(error), + }); + } + } + return { + ...result, + ...(candidateFailures.length > 0 ? { candidateFailures } : {}), + }; +} + +async function compareCandidate({ + repository, + defaultBranch, + run, + job, + evidence, + baseSha, + candidate, + api, +}) { + const mainRun = candidate.run; + const main = await candidate.getEvidence(job.name); + if (!main?.evidence) return null; + const { job: mainJob, evidence: mainEvidence } = main; + const ancestry = await api.getAncestry(repository, mainRun.head_sha, baseSha); + const paths = [ + ...new Set(evidence.diagnostics.flatMap(diagnosticPaths)), + ].slice(0, MAX_PATHS_PER_COMPARISON); + const blobs = await api.getBlobIdentity( + repository, + mainRun.head_sha, + baseSha, + paths, + ); + const executionSourcesVerified = + (await api.verifyExecutionSource(repository, run, evidence.context)) && + (await api.verifyExecutionSource( + repository, + mainRun, + mainEvidence.context, + )); + return classifyFailure({ + repository, + defaultBranch, + run, + job, + diagnostics: evidence.diagnostics, + context: evidence.context, + mainRun, + mainJob, + mainDiagnostics: mainEvidence.diagnostics, + mainContext: mainEvidence.context, + executionSourcesVerified, + ancestry, + blobs, + }); +} + +// Each verified main run/attempt owns its lazy jobs and parsed evidence for this +// report only. Cache nulls and rejected reads too; retries cannot spend the budget +// repeatedly on an immutable observation that was unavailable or invalid. +function mainEvidenceLookup(repository, api, entry, run) { + let jobs; + const evidence = new Map(); + return (name) => { + jobs ??= Promise.resolve().then(async () => { + const items = await api.getJobs(repository, entry.id, entry.runAttempt); + return items.some((item) => !hasJobIdentity(item, entry)) ? [] : items; + }); + if (!evidence.has(name)) + evidence.set( + name, + jobs.then(async (items) => { + const job = items.find( + (item) => + item.name === name && + item.status === "completed" && + isFailedConclusion(item.conclusion), + ); + if (!job) return null; + return { + job, + evidence: await api.getStaticEvidence(repository, run, job), + }; + }), + ); + return evidence.get(name); + }; +} + +// Selection and attribution share one immutable API observation per run/attempt. +function verifiedRunLookup(repository, api) { + const runs = new Map(); + return (entry) => { + const key = `${positiveId(entry.id)}:${positiveId(entry.runAttempt)}`; + if (!runs.has(key)) + runs.set( + key, + Promise.resolve().then(async () => { + const run = normalizeWorkflowRun( + await api.getRun(repository, entry.id), + ); + if (!hasRunIdentity(run, entry)) + throw new Error("workflow_run_identity_mismatch"); + return run; + }), + ); + return runs.get(key); + }; +} + +function hasRunIdentity(run, entry) { + return ( + String(run?.id) === String(entry.id) && + Number(run.run_attempt) === Number(entry.runAttempt) && + run.workflowPath?.file === ".github/workflows/ci.yml" + ); +} +function hasJobIdentity(job, entry) { + return ( + String(job.run_id) === String(entry.id) && + Number(job.run_attempt) === Number(entry.runAttempt) + ); +} + +export function createGitHubApi({ + token = process.env.GH_TOKEN, + fetchImpl = fetch, + apiRequest = createGhApiRequest(), +} = {}) { + if (!token) throw new Error("GH_TOKEN is required"); + const budget = { requests: 0, startedAt: Date.now(), limitMs: 60_000 }; + const request = async (endpoint) => { + if (budget.requests >= MAX_API_REQUESTS) + throw new Error("github_api_request_budget_exhausted"); + budget.requests += 1; + return await apiRequest("GET", endpoint, budget); + }; + return { + async getRun(repository, id) { + return await request( + `repos/${repository}/actions/runs/${positiveId(id)}`, + ); + }, + async getJobs(repository, id, attempt) { + const page = await request( + `repos/${repository}/actions/runs/${positiveId(id)}/attempts/${positiveId(attempt)}/jobs?per_page=100&page=1`, + ); + if (!Array.isArray(page.jobs) || page.jobs.length >= 100) + throw new Error("job_list_incomplete_or_malformed"); + return page.jobs; + }, + async getStaticEvidence(repository, run, job) { + if (job.name !== "Static checks") return null; + if (run.run_attempt !== 1) return null; + const page = await request( + `repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`, + ); + const artifact = selectStaticCheckArtifact(page, run, run.run_attempt); + if (!artifact) return null; + return await readStaticCheckArtifact({ + repository, + artifactId: artifact.id, + token, + fetchImpl, + }); + }, + async getAncestry(repository, ancestor, descendant) { + const comparison = await request( + `repos/${repository}/compare/${safeSha(ancestor)}...${safeSha(descendant)}`, + ); + return { + status: comparison.status, + mergeBaseSha: safeSha(comparison.merge_base_commit?.sha), + }; + }, + async verifyExecutionSource(repository, run, context) { + const sourceSha = safeSha(context.sourceSha); + if (!sourceSha) return false; + const pr = + run.event === "pull_request" + ? pullRequestBaseSha(run, repository) + : null; + if ( + run.event === "pull_request" && + (pr === null || context.prHeadSha !== run.head_sha) + ) + return false; + const commit = await request(`repos/${repository}/commits/${sourceSha}`); + if (safeSha(commit.sha) !== sourceSha) return false; + if (sourceSha === run.head_sha) return true; + if ( + pr === null || + !Array.isArray(commit.parents) || + commit.parents.length !== 2 + ) + return false; + return ( + safeSha(commit.parents[0]?.sha) === pr && + safeSha(commit.parents[1]?.sha) === run.head_sha + ); + }, + async getBlobIdentity(repository, ancestor, descendant, paths) { + const result = {}; + for (const path of paths) { + if (!validRepoPath(path)) continue; + const [left, right] = await Promise.all([ + getContentBlobSha(repository, ancestor, path, request), + getContentBlobSha(repository, descendant, path, request), + ]); + result[path] = left === right; + } + return result; + }, + }; +} + +async function getContentBlobSha(repository, ref, path, request) { + const encodedPath = path.split("/").map(encodeURIComponent).join("/"); + const value = await request( + `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`, + ); + const sha = safeSha(value?.sha); + if (value?.type !== "file" || sha === null) + throw new Error("github_content_blob_identity_unavailable"); + return sha; +} + +function outcome(classification, reason) { + return { classification, reason }; +} +function validateRepository(value) { + if ( + typeof value !== "string" || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value) + ) + throw new Error("repository must be owner/name"); +} +function positiveId(value) { + if (!Number.isSafeInteger(Number(value)) || Number(value) < 1) + throw new Error( + "GitHub run and attempt IDs must be positive safe integers", + ); + return String(Number(value)); +} +function errorCode(error) { + return error instanceof Error ? error.message.slice(0, 160) : "unknown_error"; +} + +export async function writeFailureReport( + report, + outputDirectory = OUTPUT_DIRECTORY, + summaryPath, +) { + await mkdir(outputDirectory, { recursive: true }); + const jsonPath = join(outputDirectory, "report.json"); + const markdownPath = join(outputDirectory, "report.md"); + const matchedDiagnostics = report.failures.filter( + (item) => item.classification === "matching-main-diagnostics", + ); + const lines = [ + "# CI failure attribution", + "", + "Informational only. This report never changes a CI conclusion or required gate.", + "", + `Observed ${report.observedAt}; ${String(matchedDiagnostics.length)} of ${String(report.failures.length)} failure reports contain supported diagnostics matching a failed static task on an ancestor main revision. This does not attribute other failures in the same job. Unknown means evidence was incomplete or did not match.`, + "", + ]; + for (const item of report.failures) + lines.push( + `- Run ${item.runId} attempt ${item.runAttempt}${item.job ? `, ${item.job}` : ""}: **${item.classification}** — ${item.reason}.`, + ); + if (report.failures.length === 0) + lines.push( + "- No completed failed CI attempts were present in the supplied timing snapshot.", + ); + await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + await writeFile(markdownPath, `${lines.join("\n")}\n`, "utf8"); + if (summaryPath) + await appendFile(summaryPath, `${lines.join("\n")}\n`, "utf8"); + return { jsonPath, markdownPath }; +} + +async function main() { + const repository = process.env.CI_REPOSITORY; + const defaultBranch = process.env.CI_DEFAULT_BRANCH; + const batchPath = + process.env.CI_TIMING_MANIFEST ?? + join(BATCH_DIRECTORY, "batch-manifest.json"); + const batchManifest = JSON.parse(await readBoundedFile(batchPath)); + const api = createGitHubApi(); + await reportCiFailures({ + repository, + defaultBranch, + batchManifest, + api, + outputDirectory: process.env.CI_FAILURES_DIRECTORY ?? OUTPUT_DIRECTORY, + summaryPath: process.env.GITHUB_STEP_SUMMARY, + }); +} +async function readBoundedFile(path) { + const bytes = await readFile(path); + if (bytes.byteLength > MAX_JSON_BYTES) + throw new Error("timing manifest exceeds size limit"); + return bytes.toString("utf8"); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/ci/github-api.d.mts b/scripts/ci/github-api.d.mts new file mode 100644 index 0000000000..67e157e6de --- /dev/null +++ b/scripts/ci/github-api.d.mts @@ -0,0 +1,16 @@ +export interface GitHubApiBudget { + requests: number; + startedAt: number; + limitMs: number; +} + +export type GitHubApiRequest = ( + method: string, + endpoint: string, + budget: GitHubApiBudget, +) => Promise; + +export function createGhApiRequest(options?: { + env?: NodeJS.ProcessEnv; + now?: () => number; +}): GitHubApiRequest; diff --git a/scripts/ci/github-api.mjs b/scripts/ci/github-api.mjs new file mode 100644 index 0000000000..9567bf93bc --- /dev/null +++ b/scripts/ci/github-api.mjs @@ -0,0 +1,33 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const MAX_BUFFER = 8 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 10_000; + +/** Create a bounded GitHub CLI JSON transport with GET-only authority. */ +export function createGhApiRequest({ env = process.env, now = Date.now } = {}) { + return async (method, endpoint, budget) => { + if (method !== "GET") + throw new Error( + "GitHub API transport supports read-only GET requests only", + ); + const remaining = budget.limitMs - (now() - budget.startedAt); + if (remaining <= 0) throw new Error("lookup_time_budget_exhausted"); + const { stdout } = await execFileAsync( + "gh", + ["api", "--method", "GET", endpoint], + { + env, + encoding: "utf8", + maxBuffer: MAX_BUFFER, + timeout: Math.min(REQUEST_TIMEOUT_MS, remaining), + }, + ); + try { + return JSON.parse(stdout); + } catch (cause) { + throw new Error("github_api_malformed_json", { cause }); + } + }; +} diff --git a/scripts/ci/job-timings-batch.mjs b/scripts/ci/job-timings-batch.mjs index 842e884f0d..f2dca7143c 100644 --- a/scripts/ci/job-timings-batch.mjs +++ b/scripts/ci/job-timings-batch.mjs @@ -103,6 +103,9 @@ export async function collectJobTimingsBatch({ id: run.id, runAttempt: run.run_attempt, workflow: workflow.name, + event: typeof run.event === "string" ? run.event : null, + headBranch: + typeof run.head_branch === "string" ? run.head_branch : null, conclusion: run.conclusion ?? null, createdAt: run.created_at ?? null, }); diff --git a/scripts/ci/pr-ci-health.d.mts b/scripts/ci/pr-ci-health.d.mts new file mode 100644 index 0000000000..0db2454bdd --- /dev/null +++ b/scripts/ci/pr-ci-health.d.mts @@ -0,0 +1,95 @@ +export interface PrCiHead { + repositoryId: number | null; + repository?: string | null; + ref: string | null; + sha: string | null; +} + +export interface PrCiPullRequest { + number: number | null; + draft?: boolean; + identityValid: boolean; + mergeable?: boolean; + head: PrCiHead; + [key: string]: unknown; +} + +export interface PrCiWorkflowRun { + id?: number; + run_attempt?: number; + event?: string; + head_branch?: string; + head_repository?: { id?: number | null } | null; + head_sha?: string; + status?: string; + conclusion?: string | null; + created_at?: string; + html_url?: string; + pull_requests?: Array<{ + number?: number; + head?: { sha?: string; repo?: { id?: number | null } } | null; + }>; + [key: string]: unknown; +} + +export interface PrCiClassification { + category: string; + reason: string; + runs: Array>; +} + +export interface PrCiHealthReport { + schemaVersion: 1; + observedAt: string; + repository: string; + repositoryId: number | null; + defaultBranch: string; + lookbackDays: number; + scan: { + status: "complete" | "partial"; + cutoff: string; + pullRequestPages: number; + workflowRunPages: number; + pullRequestInventoryComplete: boolean; + workflowRunInventoryComplete: boolean; + pullRequestDetailsFetched: number; + pullRequestDetailsLimit: number; + apiRequests: number; + apiRequestLimit: number; + errors: string[]; + }; + counts: Record; + pullRequests: Array; +} + +export function collectPrCiHealth(options: { + repository: string; + repositoryId?: number | string | null; + defaultBranch: string; + lookbackDays?: number; + now?: number; + apiRequest?: GitHubApiRequest; +}): Promise; + +export function classifyPrCi(input: { + pull: PrCiPullRequest; + runs: Array<{ run: PrCiWorkflowRun; basis: string }>; + runsComplete: boolean; + pullRequestInventoryComplete: boolean; + ambiguousRuns: Array<{ run: PrCiWorkflowRun; basis: string }>; +}): PrCiClassification; + +export function matchRunToPullRequest(input: { + run: PrCiWorkflowRun; + pull: PrCiPullRequest; + identityCounts: Map; + inventoryComplete?: boolean; +}): { kind: "none" } | { kind: "ambiguous" } | { kind: "match"; basis: string }; + +export function writePrCiHealthReport( + report: PrCiHealthReport, + outputDirectory?: string, +): Promise<{ jsonPath: string; markdownPath: string }>; + +export function renderMarkdownSummary(report: PrCiHealthReport): string; +import type { GitHubApiRequest } from "./github-api.mjs"; diff --git a/scripts/ci/pr-ci-health.mjs b/scripts/ci/pr-ci-health.mjs new file mode 100644 index 0000000000..d312dcecd9 --- /dev/null +++ b/scripts/ci/pr-ci-health.mjs @@ -0,0 +1,718 @@ +import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { createGhApiRequest } from "./github-api.mjs"; + +const OUTPUT_DIRECTORY = join(".cache", "ci-pr-health"); +const PAGE_SIZE = 100; +const MAX_PULL_REQUEST_PAGES = 3; +const MAX_WORKFLOW_RUN_PAGES = 5; +const MAX_PULL_REQUEST_DETAILS = 40; +const MAX_API_REQUESTS = + MAX_PULL_REQUEST_PAGES + MAX_WORKFLOW_RUN_PAGES + MAX_PULL_REQUEST_DETAILS; +const LOOKBACK_DAYS = 14; +const MAX_LOOKBACK_DAYS = 30; +const MAX_TOTAL_TIME_MS = 60_000; +const SHA_PATTERN = /^[0-9a-f]{40}$/iu; +const COMPLETED_CATEGORIES = new Map([ + ["failure", "completed_failure"], + ["timed_out", "completed_failure"], + ["cancelled", "completed_canceled"], + ["success", "completed_success"], +]); + +/** Read workflow/PR state and conservatively distinguish absent CI from unknown. */ +export async function collectPrCiHealth({ + repository, + repositoryId, + defaultBranch, + lookbackDays = LOOKBACK_DAYS, + now = Date.now(), + apiRequest = createGhApiRequest(), +}) { + const identity = validateInputs({ + repository, + repositoryId, + defaultBranch, + lookbackDays, + now, + }); + const observedAt = new Date(now).toISOString(); + const cutoff = now - identity.lookbackDays * 24 * 60 * 60 * 1_000; + const snapshot = await collectApiSnapshot({ apiRequest, identity, cutoff }); + const classified = classifyPullRequests(snapshot); + const { pulls, runs, budget } = snapshot; + const errors = [pulls.error, runs.error, ...snapshot.detailsErrors].filter( + Boolean, + ); + return { + schemaVersion: 1, + observedAt, + repository: identity.repository, + repositoryId: identity.repositoryId, + defaultBranch: identity.defaultBranch, + lookbackDays: identity.lookbackDays, + scan: { + status: + errors.length === 0 && pulls.complete && runs.complete + ? "complete" + : "partial", + cutoff: new Date(cutoff).toISOString(), + pullRequestPages: pulls.pages, + workflowRunPages: runs.pages, + pullRequestInventoryComplete: pulls.complete, + workflowRunInventoryComplete: runs.complete, + pullRequestDetailsFetched: snapshot.detailsFetched, + pullRequestDetailsLimit: MAX_PULL_REQUEST_DETAILS, + apiRequests: budget.requests, + apiRequestLimit: MAX_API_REQUESTS, + errors, + }, + counts: countCategories(classified), + pullRequests: classified, + }; +} + +async function collectApiSnapshot({ apiRequest, identity, cutoff }) { + const budget = { + requests: 0, + startedAt: Date.now(), + limitMs: MAX_TOTAL_TIME_MS, + }; + const requestGet = async (endpoint) => { + if (budget.requests >= MAX_API_REQUESTS) + throw new Error("lookup_request_budget_exhausted"); + if (Date.now() - budget.startedAt >= budget.limitMs) + throw new Error("lookup_time_budget_exhausted"); + budget.requests += 1; + return await apiRequest("GET", endpoint, budget); + }; + const pulls = await listOpenPullRequests({ + requestGet, + identity, + }); + const candidates = pulls.items.filter( + (pull) => + !pull.draft && + Number.isFinite(Date.parse(pull.createdAt ?? "")) && + Date.parse(pull.createdAt ?? "") >= cutoff && + pull.base.ref === identity.defaultBranch && + (identity.repositoryId === null || + pull.base.repositoryId === identity.repositoryId), + ); + const runs = await listRecentWorkflowRuns({ + requestGet, + identity, + cutoff, + }); + + const detailResult = await resolveMergeability({ + candidates, + requestGet, + identity, + }); + return { + budget, + pulls, + runs, + candidates, + detailsFetched: detailResult.fetched, + detailsErrors: detailResult.errors, + }; +} + +function classifyPullRequests(snapshot) { + const { candidates, pulls, runs } = snapshot; + const identityCounts = countHeadIdentities(pulls.items); + return candidates.map((pull) => { + const relevant = []; + const ambiguousRuns = []; + for (const run of runs.items) { + const match = matchRunToPullRequest({ + run, + pull, + identityCounts, + inventoryComplete: pulls.complete, + }); + if (match.kind === "match") relevant.push({ run, basis: match.basis }); + if (match.kind === "ambiguous") + ambiguousRuns.push({ run, basis: "ambiguous_head_identity" }); + } + return { + ...pull, + ci: classifyPrCi({ + pull, + runs: relevant, + runsComplete: runs.complete, + pullRequestInventoryComplete: pulls.complete, + ambiguousRuns, + }), + }; + }); +} + +async function resolveMergeability({ candidates, requestGet, identity }) { + let detailsFetched = 0; + const errors = new Set(); + for (const pull of candidates) { + if (pull.mergeable !== undefined) continue; + if (detailsFetched >= MAX_PULL_REQUEST_DETAILS) { + errors.add("pull_request_mergeability_lookup_budget_exhausted"); + continue; + } + const result = await getPullRequestDetails({ requestGet, identity, pull }); + detailsFetched += 1; + if (result.error !== null) { + errors.add(result.error); + continue; + } + const { detail } = result; + const identityChanged = + detail.headSha !== pull.head.sha || + detail.headRef !== pull.head.ref || + detail.headRepositoryId !== pull.head.repositoryId || + detail.baseRef !== pull.base.ref || + detail.baseRepositoryId !== pull.base.repositoryId; + if (identityChanged) { + pull.identityValid = false; + pull.identityChangedDuringScan = true; + errors.add("pull_request_identity_changed_during_scan"); + } else { + pull.mergeable = detail.mergeable; + pull.detailFetched = true; + if (detail.mergeable === undefined) + errors.add("pull_request_mergeability_unknown"); + } + } + return { fetched: detailsFetched, errors: [...errors] }; +} + +function countCategories(pulls) { + const categories = [ + "missing", + "startup_failure", + "queued_pending_executing", + "approval_required", + "completed_success", + "completed_failure", + "completed_canceled", + "completed_other", + "merge_conflict", + "unknown", + "deferred", + ]; + return Object.fromEntries( + categories.map((category) => [ + category, + pulls.filter(({ ci }) => ci.category === category).length, + ]), + ); +} + +/** Pure status classifier; missing requires complete, identity-safe inventories. */ +export function classifyPrCi({ + pull, + runs, + runsComplete, + pullRequestInventoryComplete, + ambiguousRuns, +}) { + if (!pull.identityValid) + return result("unknown", "pull_request_head_identity_invalid", []); + const evidence = [...runs, ...ambiguousRuns]; + if (pull.mergeable === false) + return result("merge_conflict", "github_reports_merge_conflict", evidence); + // The newest collected run is not proven current until pagination completes: + // an older-created run outside this snapshot may have been retried recently. + if (!runsComplete) + return result( + evidence.length > 0 ? "unknown" : "deferred", + evidence.length > 0 + ? "workflow_run_inventory_incomplete" + : "lookup_inventory_incomplete", + evidence, + ); + const selection = newestRun(runs); + if (!selection.known) + return result("unknown", "workflow_run_order_unknown", evidence); + const latest = selection.latest; + if ( + ambiguousRuns.some((candidate) => { + const order = latest ? compareRunEvidence(candidate, latest) : null; + return order === null || order >= 0; + }) + ) + return result("unknown", "workflow_run_identity_ambiguous", evidence); + if (latest !== undefined) { + const run = latest.run; + if (run.conclusion === "startup_failure") + return result("startup_failure", "workflow_startup_failure", evidence); + if (run.status === "waiting" || run.conclusion === "action_required") + return result( + "approval_required", + "workflow_waiting_for_approval_or_gate", + evidence, + ); + if (["queued", "requested", "pending", "in_progress"].includes(run.status)) + return result( + "queued_pending_executing", + `workflow_${run.status}`, + evidence, + ); + if (run.status === "completed") + return result( + COMPLETED_CATEGORIES.get(run.conclusion) ?? "completed_other", + run.conclusion === "success" + ? "workflow_succeeded" + : `workflow_${run.conclusion ?? "unknown_conclusion"}`, + evidence, + ); + return result("unknown", "workflow_status_unrecognized", evidence); + } + if (!pullRequestInventoryComplete) + return result("deferred", "lookup_inventory_incomplete", []); + if (pull.mergeable !== true) + return result("unknown", "pull_request_mergeability_unknown", []); + return result("missing", "no_pull_request_event_run_for_current_head", []); +} + +/** Match only a pull_request event proven to belong to this immutable PR head. */ +export function matchRunToPullRequest({ + run, + pull, + identityCounts, + inventoryComplete = true, +}) { + if (run?.event !== "pull_request") return { kind: "none" }; + if (!pull.identityValid || !isRecord(run)) return { kind: "none" }; + const association = matchExplicitPullAssociation(run, pull); + if (association === "match") + return { kind: "match", basis: "event_pr_number_and_head_sha" }; + if (association === "ambiguous") return { kind: "ambiguous" }; + if (association === "other" || !sameHeadRepositoryAndRef(run, pull)) + return { kind: "none" }; + if (!uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete)) + return { kind: "ambiguous" }; + if (isSha(run.head_sha) && run.head_sha.toLowerCase() === pull.head.sha) + return { kind: "match", basis: "event_unique_head_ref_and_repository_id" }; + return { kind: "none" }; +} + +function matchExplicitPullAssociation(run, pull) { + const associations = Array.isArray(run.pull_requests) + ? run.pull_requests + : []; + if (associations.length === 0) return "absent"; + const matchingNumber = associations.filter( + (item) => item?.number === pull.number, + ); + if (matchingNumber.length === 0) return "other"; + const exactHead = matchingNumber.find( + (item) => item?.head?.sha === pull.head.sha, + ); + if (exactHead === undefined) { + return matchingNumber.some( + (item) => + typeof item?.head?.sha === "string" && item.head.sha !== pull.head.sha, + ) + ? "other" + : "absent"; + } + const associatedHeadRepositoryId = positiveInteger(exactHead.head?.repo?.id); + const runHeadRepositoryId = positiveInteger(run.head_repository?.id); + if (associatedHeadRepositoryId === null || runHeadRepositoryId === null) + return "ambiguous"; + if ( + associatedHeadRepositoryId !== pull.head.repositoryId || + runHeadRepositoryId !== pull.head.repositoryId + ) + return "other"; + const baseRefMatches = + typeof exactHead.base?.ref !== "string" || + exactHead.base.ref === pull.base.ref; + const baseRepositoryId = positiveInteger(exactHead.base?.repo?.id); + const baseRepositoryMatches = + baseRepositoryId === null || baseRepositoryId === pull.base.repositoryId; + return baseRefMatches && baseRepositoryMatches ? "match" : "other"; +} + +function sameHeadRepositoryAndRef(run, pull) { + return ( + run.head_branch === pull.head.ref && + run.head_repository?.id === pull.head.repositoryId + ); +} + +function uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete) { + if (!inventoryComplete) return false; + const key = headIdentityKey(pull.head.repositoryId, pull.head.ref); + const unknownRepositoryKey = headIdentityKey(null, pull.head.ref); + return ( + identityCounts.get(key) === 1 && + (identityCounts.get(unknownRepositoryKey) ?? 0) === 0 + ); +} + +function normalizePullRequest(raw, identity) { + const number = positiveInteger(raw?.number); + const base = normalizeBase(raw?.base); + const head = normalizeHead(raw?.head); + const draft = raw?.draft === true; + return { + number, + draft, + createdAt: typeof raw?.created_at === "string" ? raw.created_at : null, + url: typeof raw?.html_url === "string" ? raw.html_url : null, + base, + head, + mergeable: typeof raw?.mergeable === "boolean" ? raw.mergeable : undefined, + identityValid: hasValidPullIdentity({ raw, identity, number, base, head }), + }; +} + +function normalizeBase(raw) { + return { + ref: typeof raw?.ref === "string" ? raw.ref : null, + repositoryId: positiveInteger(raw?.repo?.id), + }; +} + +function normalizeHead(raw) { + return { + repositoryId: positiveInteger(raw?.repo?.id), + repository: + typeof raw?.repo?.full_name === "string" ? raw.repo.full_name : null, + ref: typeof raw?.ref === "string" ? raw.ref : null, + sha: + typeof raw?.sha === "string" && isSha(raw.sha) + ? raw.sha.toLowerCase() + : null, + }; +} + +function hasValidPullIdentity({ raw, identity, number, base, head }) { + return ( + number !== null && + typeof raw?.draft === "boolean" && + base.ref === identity.defaultBranch && + (identity.repositoryId === null || + base.repositoryId === identity.repositoryId) && + head.repositoryId !== null && + head.sha !== null && + head.ref !== null + ); +} + +function countHeadIdentities(pulls) { + const counts = new Map(); + for (const pull of pulls) { + if (typeof pull.head.ref !== "string") continue; + const key = headIdentityKey(pull.head.repositoryId, pull.head.ref); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return counts; +} + +function headIdentityKey(repositoryId, ref) { + return `${String(repositoryId)}\u0000${ref}`; +} + +function newestRun(runs) { + let latest = runs[0]; + for (const candidate of runs) { + if (latest && (compareRunEvidence(candidate, latest) ?? 0) > 0) + latest = candidate; + } + const known = runs.every((candidate) => { + if (candidate === latest) return true; + const order = compareRunEvidence(candidate, latest); + return order !== null && order <= 0; + }); + return { known, latest }; +} + +// Attempt ordering is meaningful within one run; different run IDs are not clocks. +function compareRunEvidence(left, right) { + const leftRun = left.run; + const rightRun = right.run; + const id = positiveInteger(leftRun.id); + if (id !== null && id === positiveInteger(rightRun.id)) { + const leftAttempt = positiveInteger(leftRun.run_attempt); + const rightAttempt = positiveInteger(rightRun.run_attempt); + if (leftAttempt === null || rightAttempt === null) return null; + if (leftAttempt !== rightAttempt) return leftAttempt - rightAttempt; + return leftRun.status === rightRun.status && + leftRun.conclusion === rightRun.conclusion + ? 0 + : null; + } + // A retry may start long after its original creation; without an attempt + // clock, creation dates only order first attempts from different runs. + if ( + positiveInteger(leftRun.run_attempt) !== 1 || + positiveInteger(rightRun.run_attempt) !== 1 + ) + return null; + const time = + Date.parse(leftRun.created_at ?? "") - + Date.parse(rightRun.created_at ?? ""); + return Number.isFinite(time) && time !== 0 ? time : null; +} + +function result(category, reason, runs) { + return { + category, + reason, + runs: runs.map(({ run, basis }) => ({ + id: positiveInteger(run.id), + attempt: positiveInteger(run.run_attempt), + status: typeof run.status === "string" ? run.status : null, + conclusion: typeof run.conclusion === "string" ? run.conclusion : null, + headSha: isSha(run.head_sha) ? run.head_sha.toLowerCase() : null, + createdAt: typeof run.created_at === "string" ? run.created_at : null, + url: typeof run.html_url === "string" ? run.html_url : null, + identityBasis: basis, + })), + }; +} + +async function listOpenPullRequests({ requestGet, identity }) { + const items = []; + let complete = false; + let pages = 0; + let error = null; + for (let page = 1; page <= MAX_PULL_REQUEST_PAGES; page += 1) { + try { + const endpoint = `repos/${identity.repository}/pulls?state=open&sort=created&direction=desc&per_page=${PAGE_SIZE}&page=${page}`; + const response = await requestGet(endpoint); + if (!Array.isArray(response)) + throw new Error("pull_request_page_malformed"); + pages += 1; + const normalized = response.map((raw) => + normalizePullRequest(raw, identity), + ); + if ( + normalized.some( + (pull) => !Number.isFinite(Date.parse(pull.createdAt ?? "")), + ) + ) + throw new Error("pull_request_page_malformed"); + items.push(...normalized); + if (response.length < PAGE_SIZE) { + complete = true; + break; + } + } catch (cause) { + error = safeErrorCode(cause); + break; + } + } + return { items, pages, complete, error }; +} + +async function listRecentWorkflowRuns({ requestGet, identity, cutoff }) { + const items = []; + let complete = false; + let pages = 0; + let error = null; + for (let page = 1; page <= MAX_WORKFLOW_RUN_PAGES; page += 1) { + try { + const cutoffDate = new Date(cutoff).toISOString().slice(0, 10); + const endpoint = `repos/${identity.repository}/actions/workflows/ci.yml/runs?event=pull_request&created=%3E%3D${cutoffDate}&per_page=${PAGE_SIZE}&page=${page}`; + const response = await requestGet(endpoint); + if (!isRecord(response) || !Array.isArray(response.workflow_runs)) + throw new Error("workflow_run_page_malformed"); + pages += 1; + if (!response.workflow_runs.every(isRecord)) + throw new Error("workflow_run_page_malformed"); + items.push(...response.workflow_runs); + if (response.workflow_runs.length < PAGE_SIZE) { + complete = true; + break; + } + } catch (cause) { + error = safeErrorCode(cause); + break; + } + } + return { items, pages, complete, error }; +} + +async function getPullRequestDetails({ requestGet, identity, pull }) { + if (pull.number === null) + return { detail: null, error: "pull_request_detail_identity_missing" }; + try { + const response = await requestGet( + `repos/${identity.repository}/pulls/${pull.number}`, + ); + if (!isRecord(response)) + return { detail: null, error: "pull_request_detail_malformed" }; + return { + detail: { + mergeable: + typeof response.mergeable === "boolean" + ? response.mergeable + : undefined, + headSha: isSha(response.head?.sha) + ? response.head.sha.toLowerCase() + : null, + headRef: + typeof response.head?.ref === "string" ? response.head.ref : null, + headRepositoryId: positiveInteger(response.head?.repo?.id), + baseRef: + typeof response.base?.ref === "string" ? response.base.ref : null, + baseRepositoryId: positiveInteger(response.base?.repo?.id), + }, + error: null, + }; + } catch (cause) { + return { detail: null, error: safeErrorCode(cause) }; + } +} + +function validateInputs({ + repository, + repositoryId, + defaultBranch, + lookbackDays, + now, +}) { + if ( + typeof repository !== "string" || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository) + ) + throw new Error( + "CI_REPOSITORY must be an owner/name repository identifier", + ); + if ( + typeof defaultBranch !== "string" || + defaultBranch.length === 0 || + /[\r\n]/u.test(defaultBranch) + ) + throw new Error("CI_DEFAULT_BRANCH must be a nonempty branch name"); + const parsedRepositoryId = + repositoryId === undefined || repositoryId === null || repositoryId === "" + ? null + : positiveInteger(repositoryId); + if ( + repositoryId !== undefined && + repositoryId !== null && + repositoryId !== "" && + parsedRepositoryId === null + ) + throw new Error("CI_REPOSITORY_ID must be a positive safe integer"); + if ( + !Number.isInteger(lookbackDays) || + lookbackDays < 1 || + lookbackDays > MAX_LOOKBACK_DAYS + ) + throw new Error( + `lookbackDays must be an integer from 1 to ${String(MAX_LOOKBACK_DAYS)}`, + ); + if (!Number.isFinite(now)) throw new Error("now must be a finite timestamp"); + return { + repository, + repositoryId: parsedRepositoryId, + defaultBranch, + lookbackDays, + }; +} + +function positiveInteger(value) { + const number = typeof value === "number" ? value : Number(value); + return Number.isSafeInteger(number) && number > 0 ? number : null; +} + +function isSha(value) { + return typeof value === "string" && SHA_PATTERN.test(value); +} + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function safeErrorCode(cause) { + if (cause?.message === "lookup_time_budget_exhausted") + return "lookup_time_budget_exhausted"; + if (cause?.message === "lookup_request_budget_exhausted") + return "lookup_request_budget_exhausted"; + if (cause?.message === "github_api_malformed_json") + return "github_api_malformed_json"; + if (cause?.message?.endsWith("_malformed")) return cause.message; + if (cause?.code === "ETIMEDOUT" || cause?.killed) return "github_api_timeout"; + return "github_api_request_failed"; +} + +export async function writePrCiHealthReport( + report, + outputDirectory = OUTPUT_DIRECTORY, +) { + const jsonPath = join(outputDirectory, "report.json"); + const markdownPath = join(outputDirectory, "summary.md"); + await mkdir(outputDirectory, { recursive: true }); + await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + await writeFile(markdownPath, renderMarkdownSummary(report), "utf8"); + return { jsonPath, markdownPath }; +} + +export function renderMarkdownSummary(report) { + const rows = report.pullRequests.map( + (pull) => + `| #${String(pull.number ?? "?")} | ${escapeCell(pull.head.repository ?? "unknown")} | ${escapeCell(pull.head.ref ?? "unknown")} | \`${pull.head.sha ?? "unknown"}\` | ${pull.ci.category} | ${escapeCell(pull.ci.reason)} |`, + ); + return [ + "# Pull request CI health snapshot", + "", + `Observed ${report.observedAt} for \`${report.repository}\`, target branch \`${escapeCell(report.defaultBranch)}\`; recent window ${String(report.lookbackDays)} days. Scan: **${report.scan.status}**.`, + "", + "This is read-only point-in-time telemetry. Missing is reported only when PR and workflow-run inventories are complete and the current immutable PR head identity is unambiguous. Incomplete lookups remain deferred or unknown.", + "", + `API pages: ${String(report.scan.pullRequestPages)} PR, ${String(report.scan.workflowRunPages)} workflow runs; ${String(report.scan.apiRequests)} bounded requests. Errors: ${report.scan.errors.length === 0 ? "none" : report.scan.errors.join(", ")}.`, + "", + "| PR | Head repository | Head ref | Head SHA | CI classification | Evidence |", + "| ---: | --- | --- | --- | --- | --- |", + ...rows, + "", + "Categories: missing, startup failure, queued/pending/executing, approval required, completed outcomes, merge conflict, unknown, and deferred.", + "", + ].join("\n"); +} + +function escapeCell(value) { + return String(value) + .replaceAll("|", "\\|") + .replaceAll("\n", " ") + .replaceAll("\r", " "); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + const report = await collectPrCiHealth({ + repository: process.env.CI_REPOSITORY, + repositoryId: process.env.CI_REPOSITORY_ID, + defaultBranch: process.env.CI_DEFAULT_BRANCH, + }); + const paths = await writePrCiHealthReport( + report, + process.env.CI_PR_HEALTH_DIRECTORY ?? OUTPUT_DIRECTORY, + ); + if (process.env.GITHUB_STEP_SUMMARY) + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + renderMarkdownSummary(report), + "utf8", + ); + process.stdout.write( + `Wrote ${report.pullRequests.length} recent PR CI records to ${paths.jsonPath}\n`, + ); + } catch (cause) { + process.stderr.write( + `PR CI health telemetry failed: ${cause instanceof Error ? cause.message : "unknown_error"}\n`, + ); + process.exitCode = 1; + } +} diff --git a/scripts/ci/qualification.mjs b/scripts/ci/qualification.mjs index ab47c168f2..8dfb3f20c9 100644 --- a/scripts/ci/qualification.mjs +++ b/scripts/ci/qualification.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { normalizeWorkflowRun } from "./workflow-run.mjs"; const ARTIFACT_NAME = "ci-full-qualification-v1"; const MAX_AGE_MS = 24 * 60 * 60 * 1000; @@ -119,8 +120,8 @@ function admit() { // Artifact metadata narrows the search before run lookups. Never scan an // unbounded history or treat a missing receipt as a successful qualification. for (const artifact of candidates.slice(0, 3)) { - const run = api( - `repos/${repository}/actions/runs/${artifact.workflow_run.id}`, + const run = normalizeWorkflowRun( + api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`), ); // An old run can upload a new artifact when only its gate is rerun. // Bound the original run age as well so retries cannot renew stale proof. @@ -133,7 +134,7 @@ function admit() { run.id === artifact.workflow_run.id && run.status === "completed" && run.conclusion === "success" && - run.path === ".github/workflows/ci.yml" && + run.workflowPath?.file === ".github/workflows/ci.yml" && ["schedule", "workflow_dispatch"].includes(run.event) && run.head_sha === head && run.head_branch === branch && diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index 20fbac23fd..1839a11498 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -30,7 +30,11 @@ if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); const required = reused ? ["changes"] : ["changes", "static", "test"]; const lanes = { - code: ["build", "test-shard"], + code: [ + "build", + "test-shard", + ...(outputs.tier === "fast" ? ["test-source"] : []), + ], reverse_tools: ["reverse-tools-quality"], runtime: ["windows-native-build", "package-e2e"], package: ["windows-curated"], diff --git a/scripts/ci/static-check-report.mjs b/scripts/ci/static-check-report.mjs index 04460b2c20..eecaf750af 100644 --- a/scripts/ci/static-check-report.mjs +++ b/scripts/ci/static-check-report.mjs @@ -133,6 +133,8 @@ try { for (const clone of report.duplicates) for (const file of [clone.firstFile, clone.secondFile]) file.name = relative(process.cwd(), file.name).split(sep).join("/"); + // Record the actual checker result, including caller-selected thresholds. + report.reaCheck = { exitCode: result.status }; await writeFile(reportPath, JSON.stringify(report, null, 2)); const { sources, clones, percentage } = report.statistics.total; console.log( diff --git a/scripts/ci/static-task-context.d.mts b/scripts/ci/static-task-context.d.mts new file mode 100644 index 0000000000..164756b15d --- /dev/null +++ b/scripts/ci/static-task-context.d.mts @@ -0,0 +1,18 @@ +export interface StaticTaskContext { + readonly version: 1; + readonly sourceSha: string; + readonly sourceClean: boolean; + readonly prHeadSha?: string; + readonly nodeVersion: string; + readonly npmVersion: string; + readonly turboVersion: string; + readonly platform: string; + readonly architecture: string; + readonly taskHashes: Readonly>; +} +export function parseStaticTaskContext(value: unknown): StaticTaskContext; +export function recordStaticTaskContext(options?: { + root?: string; + npmCli?: string; + outputDirectory?: string; +}): Promise; diff --git a/scripts/ci/static-task-context.mjs b/scripts/ci/static-task-context.mjs new file mode 100644 index 0000000000..879c568d19 --- /dev/null +++ b/scripts/ci/static-task-context.mjs @@ -0,0 +1,104 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { z } from "zod"; + +const execute = promisify(execFile); +const contextSchema = z.object({ + version: z.literal(1), + sourceSha: z.string().regex(/^[a-f0-9]{40}$/u), + sourceClean: z.boolean(), + prHeadSha: z + .string() + .regex(/^[a-f0-9]{40}$/u) + .optional(), + nodeVersion: z.string().min(1), + npmVersion: z.string().min(1), + turboVersion: z.string().min(1), + platform: z.string().min(1), + architecture: z.string().min(1), + taskHashes: z.record(z.string(), z.string().regex(/^[a-f0-9]{16}$/u)), +}); + +/** Parse the producing check graph's receipt; incomplete provenance is unknown. */ +export const parseStaticTaskContext = (value) => contextSchema.parse(value); + +/** Retain only named task hashes and tool/source identity, never the environment. */ +export async function recordStaticTaskContext({ + root = process.cwd(), + npmCli = process.env.npm_execpath, + outputDirectory = join(root, ".cache/static-checks"), +} = {}) { + if (typeof npmCli !== "string" || !npmCli) + throw new Error( + "Run static context recording through npm run ci:static-context", + ); + const options = { + cwd: root, + encoding: "utf8", + timeout: 30_000, + maxBuffer: 32 * 1024 * 1024, + }; + const [graph, source, status, npmVersion, turbo] = await Promise.all([ + execute( + process.execPath, + [npmCli, "run", "--silent", "check:ci:tasks", "--", "--dry=json"], + options, + ), + execute("git", ["rev-parse", "HEAD"], options), + execute( + "git", + ["status", "--porcelain", "--untracked-files=normal"], + options, + ), + execute(process.execPath, [npmCli, "--version"], options), + readFile(join(root, "node_modules/turbo/package.json"), "utf8"), + ]); + const tasks = z + .object({ + tasks: z + .array( + z.object({ + task: z.string().min(1), + hash: z.string().regex(/^[a-f0-9]{16}$/u), + }), + ) + .min(1), + }) + .parse(JSON.parse(graph.stdout)).tasks; + if (new Set(tasks.map(({ task }) => task)).size !== tasks.length) + throw new Error("Static graph contains duplicate task identities"); + const context = parseStaticTaskContext({ + version: 1, + sourceSha: source.stdout.trim(), + sourceClean: status.stdout.length === 0, + ...(process.env.CI_PR_HEAD_SHA + ? { prHeadSha: process.env.CI_PR_HEAD_SHA } + : {}), + nodeVersion: process.version, + npmVersion: npmVersion.stdout.trim(), + turboVersion: z.object({ version: z.string() }).parse(JSON.parse(turbo)) + .version, + platform: process.platform, + architecture: process.arch, + taskHashes: Object.fromEntries(tasks.map(({ task, hash }) => [task, hash])), + }); + if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== context.sourceSha) + throw new Error( + "Static context source does not match the workflow revision", + ); + await mkdir(outputDirectory, { recursive: true }); + await writeFile( + join(outputDirectory, "static-task-context.json"), + JSON.stringify(context, null, 2) + "\n", + ); + return context; +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) + await recordStaticTaskContext(); diff --git a/scripts/ci/test-group.mjs b/scripts/ci/test-group.mjs new file mode 100644 index 0000000000..7d99175b4e --- /dev/null +++ b/scripts/ci/test-group.mjs @@ -0,0 +1,46 @@ +import { spawn } from "node:child_process"; +import { resolve } from "node:path"; + +import { + BUILT_TEST_PROJECTS, + SOURCE_TEST_PROJECTS, +} from "../lib/test-projects.mjs"; + +const [group, ...arguments_] = process.argv.slice(2); +const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[ + group +]; +if (!projects) throw new Error("Expected source or built test group"); +// The group owns project selection; callers can request reporters or shards. +if ( + arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument)) +) + throw new Error("Test groups cannot override their project inventory"); +const child = spawn( + process.execPath, + [ + resolve("node_modules/vitest/vitest.mjs"), + "run", + ...projects.flatMap(({ name }) => ["--project", name]), + ...arguments_, + ], + { stdio: "inherit" }, +); +let shutdownSignal; +for (const signal of ["SIGINT", "SIGTERM"]) + process.on(signal, () => { + shutdownSignal ??= signal; + child.kill(signal); + }); +child.once("error", (error) => { + console.error(error.message); + process.exitCode = 1; +}); +child.once("close", (code) => { + process.exitCode = + shutdownSignal === "SIGINT" + ? 130 + : shutdownSignal === "SIGTERM" + ? 143 + : (code ?? 1); +}); diff --git a/scripts/ci/test-history.mjs b/scripts/ci/test-history.mjs index 4386f48af3..14dbb582b4 100644 --- a/scripts/ci/test-history.mjs +++ b/scripts/ci/test-history.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { normalizeWorkflowRun } from "./workflow-run.mjs"; const ARTIFACT_NAME = "test-timings"; const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; @@ -68,8 +69,8 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) { .slice(0, MAX_CANDIDATES); for (const artifact of candidates) { - const run = api( - `repos/${repository}/actions/runs/${artifact.workflow_run.id}`, + const run = normalizeWorkflowRun( + api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`), ); const runAge = now - Date.parse(run?.created_at); if ( @@ -80,7 +81,7 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) { run.id === artifact.workflow_run.id && run.status === "completed" && run.conclusion === "success" && - run.path === ".github/workflows/ci.yml" && + run.workflowPath?.file === ".github/workflows/ci.yml" && ["schedule", "workflow_dispatch"].includes(run.event) && run.head_branch === branch && run.head_repository?.full_name === repository diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 035a019908..7f85c6ec5b 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -41,9 +41,20 @@ const files = report.files.map(({ name, durationMs, state }) => { throw new Error(`Test report path is outside the checkout: ${name}`); return { name: path.split(sep).join("/"), durationMs, state }; }); -const complete = process.env.SHARD_RESULT === "success"; +// Keep group outcomes intact: an unavailable group is not a completed failure. +const groupResults = [ + process.env.SHARD_RESULT, + ...(process.env.SOURCE_REQUIRED === "true" + ? [process.env.SOURCE_RESULT] + : []), +]; +const groupsCompleted = groupResults.every((state) => + ["success", "failure"].includes(state), +); const passed = - complete && process.env.MERGE_RESULT === "success" && report.success; + groupResults.every((state) => state === "success") && + process.env.MERGE_RESULT === "success" && + report.success; // Successful shards and merged reports must account for the complete portable // inventory. Coverage thresholds alone cannot prove that every file ran. const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true"; @@ -69,7 +80,7 @@ if (process.env.CI_TEST_SELECTION) { comparison.complete && report.unhandledErrors === 0 && ["passed", "failed"].includes(report.reason) && - ["success", "failure"].includes(process.env.SHARD_RESULT) && + groupsCompleted && ["success", "failure"].includes(process.env.MERGE_RESULT), executionSha: process.env.GITHUB_SHA ?? null, prHeadSha: process.env.CI_PR_HEAD_SHA ?? null, @@ -91,7 +102,9 @@ if (passed && verifyInventory) { ); } const summary = [ - complete ? "## Merged test diagnostics" : "## Partial test diagnostics", + groupsCompleted + ? "## Merged test diagnostics" + : "## Partial test diagnostics", "", ...(selection ? [ @@ -99,11 +112,14 @@ const summary = [ "", `Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`, `Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`, - "The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", + "This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", "", ] : []), `Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`, + ...(process.env.SOURCE_REQUIRED === "true" + ? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`] + : []), passed ? "Successful timing history is available." : "This report is diagnostic only and is not a successful timing baseline.", diff --git a/scripts/ci/workflow-run.d.mts b/scripts/ci/workflow-run.d.mts new file mode 100644 index 0000000000..b0e6263780 --- /dev/null +++ b/scripts/ci/workflow-run.d.mts @@ -0,0 +1,10 @@ +export interface WorkflowPath { + readonly file: string; + readonly ref: string | null; +} +export interface NormalizedWorkflowRun { + readonly workflowPath: WorkflowPath | null; +} +export function normalizeWorkflowRun>( + value: T, +): Omit & NormalizedWorkflowRun; diff --git a/scripts/ci/workflow-run.mjs b/scripts/ci/workflow-run.mjs new file mode 100644 index 0000000000..8c33214bf0 --- /dev/null +++ b/scripts/ci/workflow-run.mjs @@ -0,0 +1,15 @@ +/** Keep the API path as evidence and parse its optional ref at the boundary. */ +export function normalizeWorkflowRun(value) { + if (typeof value !== "object" || value === null || Array.isArray(value)) + throw new Error("workflow_run_malformed"); + const match = + typeof value.path === "string" && !/[\u0000-\u001f\u007f]/u.test(value.path) + ? /^(\.github\/workflows\/[^/\\@\s]+\.ya?ml)(?:@([^\s]+))?$/u.exec( + value.path, + ) + : null; + return { + ...value, + workflowPath: match ? { file: match[1], ref: match[2] ?? null } : null, + }; +} diff --git a/scripts/lib/browser-nested-source-map-e2e.mjs b/scripts/lib/browser-nested-source-map-e2e.mjs index ff2af88632..d7bac5ee22 100644 --- a/scripts/lib/browser-nested-source-map-e2e.mjs +++ b/scripts/lib/browser-nested-source-map-e2e.mjs @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { TraceMap, eachMapping } from "@jridgewell/trace-mapping"; import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js"; +import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js"; import { CdpConnection } from "../../dist/browser/CdpConnection.js"; import { parseEvidence } from "../../dist/domain/evidence.js"; import { traceSourceMap } from "../../dist/javascript/sourceMaps/TraceSourceMap.js"; @@ -89,10 +90,12 @@ async function verifyCase(executable, entrypoint, scenario) { fixture = await startRuntimeFixtureBrowser(executable, site.origin); const endpoint = `http://127.0.0.1:${fixture.port}`; const provider = new CdpBrowserProvider(); - const targets = await provider.listTargets({ - cdp_endpoint: endpoint, - allowed_origins: [site.origin], - }); + const targets = await provider.listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: [site.origin], + }), + ); if (!targets.ok) throw targets.error; const target = targets.value.targets[0]; assert.ok(target, "Owned fixture target missing"); diff --git a/scripts/lib/browser-runtime-e2e.mjs b/scripts/lib/browser-runtime-e2e.mjs index c4cb4f09f6..b35cde09ab 100644 --- a/scripts/lib/browser-runtime-e2e.mjs +++ b/scripts/lib/browser-runtime-e2e.mjs @@ -8,6 +8,7 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js"; +import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js"; import { CdpConnection } from "../../dist/browser/CdpConnection.js"; import { parseEvidence } from "../../dist/domain/evidence.js"; import { webExecutionSchema } from "../../dist/domain/webExecution.js"; @@ -45,10 +46,12 @@ export async function verifyBrowserRuntime( try { fixture = await startRuntimeFixtureBrowser(executable, site.origin); const endpoint = `http://127.0.0.1:${fixture.port}`; - const targets = await new CdpBrowserProvider().listTargets({ - cdp_endpoint: endpoint, - allowed_origins: [site.origin], - }); + const targets = await new CdpBrowserProvider().listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: [site.origin], + }), + ); if (!targets.ok) throw targets.error; const target = targets.value.targets[0]; assert.ok(target, "owned fixture page target missing"); diff --git a/scripts/lib/development-tests.mjs b/scripts/lib/development-tests.mjs index 8ddebe115b..6aaf10768f 100644 --- a/scripts/lib/development-tests.mjs +++ b/scripts/lib/development-tests.mjs @@ -1,17 +1,4 @@ -const sourceProjects = [ - "domain", - "services", - "adapters", - "composition", - "conformance", - "evaluation", -]; - -const sourceTest = (path) => - path.startsWith("src/") || - ["composition", "conformance", "evaluation"].some((project) => - path.startsWith(`tests/${project}/`), - ); +import { SOURCE_TEST_PROJECTS, projectForTest } from "./test-projects.mjs"; /** Parse explicit test paths and an optional branch-comparison base. */ export const parseDevelopmentTestRequest = (mode, arguments_) => { @@ -35,10 +22,9 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => { path.split("/").some((part) => part === ".." || part === ".") ) throw new Error(`Expected an exact repository test path: ${argument}`); - if (mode !== "focused" && !sourceTest(path)) - throw new Error( - `Use test:focused for compiled boundary tests: ${path}`, - ); + const project = projectForTest(path); + if (mode !== "focused" && project.group !== "source") + throw new Error(`Use test:focused for built test projects: ${path}`); paths.push(path); } } @@ -72,9 +58,14 @@ export const developmentTestPlan = (request, baseCommit) => { !/^[a-f0-9]{40,64}$/u.test(baseCommit ?? "") ) throw new Error("Changed tests require a resolved Git merge base"); - const projects = request.mode === "focused" ? [] : sourceProjects; + const projects = + request.mode === "focused" + ? [] + : SOURCE_TEST_PROJECTS.map(({ name }) => name); return { - needsBuild: request.paths.some((path) => !sourceTest(path)), + needsBuild: request.paths.some( + (path) => projectForTest(path).group === "built", + ), artifactTasks: Object.entries(artifactConsumers).flatMap( ([task, consumers]) => request.paths.some((path) => consumers.includes(path)) ? [task] : [], diff --git a/scripts/lib/test-projects.d.mts b/scripts/lib/test-projects.d.mts new file mode 100644 index 0000000000..a9d6ae3563 --- /dev/null +++ b/scripts/lib/test-projects.d.mts @@ -0,0 +1,10 @@ +export interface TestProjectDefinition { + readonly name: string; + readonly group: "source" | "built"; + readonly include: readonly string[]; + readonly exclude?: readonly string[]; +} + +export const SOURCE_TEST_PROJECTS: readonly TestProjectDefinition[]; +export const BUILT_TEST_PROJECTS: readonly TestProjectDefinition[]; +export function projectForTest(testPath: string): TestProjectDefinition; diff --git a/scripts/lib/test-projects.mjs b/scripts/lib/test-projects.mjs new file mode 100644 index 0000000000..68fb90dbd0 --- /dev/null +++ b/scripts/lib/test-projects.mjs @@ -0,0 +1,107 @@ +import { matchesGlob } from "node:path"; + +/** Canonical file ownership shared by Vitest and development-test planning. */ +export const SOURCE_TEST_PROJECTS = [ + { + name: "domain", + group: "source", + include: ["src/{contracts,domain}/**/*.test.ts"], + }, + { + name: "services", + group: "source", + include: ["src/application/**/*.test.ts"], + }, + { + name: "adapters", + group: "source", + include: [ + "src/*.test.ts", + "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", + ], + }, + { + name: "composition", + group: "source", + include: ["tests/composition/**/*.test.ts"], + }, + { + name: "conformance", + group: "source", + include: ["tests/conformance/**/*.test.ts"], + exclude: ["tests/conformance/providers/hopper/realHopperSearch.test.ts"], + }, + { + name: "evaluation", + group: "source", + include: ["tests/evaluation/**/*.test.ts"], + }, +]; + +export const BUILT_TEST_PROJECTS = [ + { + name: "boundary", + group: "built", + include: [ + "tests/boundary/**/*.test.ts", + "tests/conformance/providers/hopper/realHopperSearch.test.ts", + ], + exclude: [ + "tests/boundary/cli/**/*.test.ts", + "tests/boundary/mcp/**/*.test.ts", + "tests/boundary/process/**/*.test.ts", + ], + }, + { + name: "cli-boundary", + group: "built", + include: ["tests/boundary/cli/**/*.test.ts"], + }, + { + name: "process-boundary", + group: "built", + include: ["tests/boundary/process/**/*.test.ts"], + }, + { + name: "mcp-boundary", + group: "built", + include: ["tests/boundary/mcp/**/*.test.ts"], + }, + { + name: "acceptance", + group: "built", + include: ["tests/acceptance/**/*.test.ts"], + }, + { + name: "process-global", + group: "built", + include: ["tests/process-global/**/*.test.ts"], + }, +]; + +const ALL_TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]; + +/** Return the unique Vitest owner for an exact repository-relative test path. */ +export const projectForTest = (testPath) => { + if ( + typeof testPath !== "string" || + testPath.startsWith("/") || + testPath.includes("\\") || + testPath.split("/").some((part) => part === ".." || part === ".") || + !testPath.endsWith(".test.ts") + ) + throw new Error(`Expected a normalized repository test path: ${testPath}`); + + const owners = ALL_TEST_PROJECTS.filter( + ({ include, exclude = [] }) => + include.some((pattern) => matchesGlob(testPath, pattern)) && + !exclude.some((pattern) => matchesGlob(testPath, pattern)), + ); + if (owners.length !== 1) + throw new Error( + owners.length === 0 + ? `No test project owns ${testPath}` + : `Multiple test projects own ${testPath}: ${owners.map(({ name }) => name).join(", ")}`, + ); + return owners[0]; +}; diff --git a/scripts/verify-real-ghidra-ppc.mjs b/scripts/verify-real-ghidra-ppc.mjs new file mode 100644 index 0000000000..591d32b2b5 --- /dev/null +++ b/scripts/verify-real-ghidra-ppc.mjs @@ -0,0 +1,282 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; +import { + inspectPowerPcReadelf, + assertPowerPcLoadedImage, + assertPowerPcProbe, + assertPowerPcGlobal, +} from "../tests/conformance/ghidra/ppc-checks.mjs"; +import { parseEvidence } from "../dist/domain/evidence.js"; +import { parseAnalysisSnapshot } from "../dist/domain/analysisSnapshot.js"; +import { functionDossierSchema } from "../dist/domain/hopperValues.js"; +import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js"; +import { buildPowerPcFixture } from "../tests/conformance/ghidra/ppc-fixture.mjs"; +import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; + +// Optional cross-target lane. No tool acquisition and no target execution. +if (process.argv.length !== 2) + throw new Error("Usage: node scripts/verify-real-ghidra-ppc.mjs"); +if (!["linux", "darwin"].includes(process.platform)) + throw new Error( + "PowerPC verification requires a supported Linux/macOS Ghidra host", + ); +if (process.env.GHIDRA_INSTALL_DIR === undefined) + throw new Error("Missing prerequisite: caller-supplied GHIDRA_INSTALL_DIR"); +const installation = inspectGhidraInstallation({ + environment: process.env, + installDir: process.env.GHIDRA_INSTALL_DIR, + ...(process.env.JAVA_HOME === undefined + ? {} + : { javaHome: process.env.JAVA_HOME }), +}); +if (installation.status !== "available") + throw new Error( + `Ghidra prerequisite unavailable: ${JSON.stringify(installation)}`, + ); +const execute = promisify(execFile); +const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url)); +const run = createVerifierRun(); +const workspace = await mkdtemp(join(tmpdir(), "rea-ppc-proof-")); +const runtime = join(workspace, "runtime"); +await mkdir(runtime); +const env = { + PATH: process.env.PATH ?? "/usr/bin:/bin", + TMPDIR: runtime, + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "ghidra", + REA_PROCESS_RUN_ID: run.run_id, + GHIDRA_INSTALL_DIR: installation.installDir, + JAVA_HOME: installation.javaHome, + HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper", +}; +for (const setting of ["GHIDRA_HEADLESS_MAXMEM", "GHIDRA_MAXMEM"]) { + if (process.env[setting] !== undefined) env[setting] = process.env[setting]; +} +const reports = []; +let primaryFailure; +try { + { + const fixture = await buildPowerPcFixture(workspace); + const independent = await inspectPowerPcReadelf(fixture.path); + const resolved = await parseBinaryTarget(fixture.path); + if (!resolved.ok) throw resolved.error; + assert.deepEqual( + resolved.value.powerpc, + independent.powerpc, + "REA ELF interpretation disagrees with GNU readelf", + ); + assert.equal(resolved.value.sha256, fixture.sha256); + const snapshotPath = join(workspace, "ppc.snapshot.json"); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env, + stderr: "pipe", + }); + const client = new Client({ name: "rea-real-ppc-proof", version: "1" }); + let stderr = ""; + transport.stderr?.on("data", (chunk) => { + stderr = (stderr + chunk.toString()).slice(-65536); + }); + const call = async (name, args = {}) => { + const reply = await client.callTool( + { name, arguments: args }, + undefined, + { timeout: 180000 }, + ); + return requireMcpOperationResult(reply, name); + }; + try { + await client.connect(transport); + const target = await call("open_binary", { + path: fixture.path, + provider_id: "ghidra", + }); + assert.equal(target.architecture, "powerpc"); + assert.equal(target.sha256, fixture.sha256); + // ELF load-image verification is not the DOS comparison lane. Its retained + // observations still expose the actual Ghidra language and loaded bytes; + // the plain loader would have chosen e500 for this header. + const image = await call("inspect_native_load_image"); + assertPowerPcLoadedImage(image.observations, fixture.sha256); + const procedures = await call("list_procedures"); + const entry = procedures.find((p) => p.value === "rea_ppc_entry"); + const leaf = procedures.find((p) => p.value === "rea_ppc_leaf"); + assert.ok(entry, "Entry function missing from real Ghidra inventory"); + assert.ok(leaf, "Leaf function missing from real Ghidra inventory"); + assert.equal( + BigInt(entry.address), + BigInt(independent.symbols.rea_ppc_entry), + ); + assert.equal( + BigInt(leaf.address), + BigInt(independent.symbols.rea_ppc_leaf), + ); + const probe = independent.symbols.rea_ppc_probe; + assert.ok( + procedures.some( + (p) => + p.value === "rea_ppc_probe" && BigInt(p.address) === BigInt(probe), + ), + ); + const move = await call("inspect_native_instruction", { address: probe }); + const branch = await call("inspect_native_instruction", { + address: `0x${(BigInt(probe) + 8n).toString(16)}`, + }); + assertPowerPcProbe(move, branch, probe); + const global = await call("read_bytes", { + address: independent.symbols.rea_ppc_global, + length: 4, + }); + assert.equal(global.returned_bytes, 4); + assertPowerPcGlobal(global.bytes_hex); + const marker = await call("read_bytes", { + address: independent.symbols.rea_ppc_marker, + length: 29, + }); + assert.equal( + marker.bytes_hex, + Buffer.from("rea-ppc-source-owned-fixture\0").toString("hex"), + ); + const callees = await call("procedure_callees", { + procedure: entry.address, + }); + assert.ok( + callees.includes(leaf.address), + "Expected direct call not recovered", + ); + const assembly = await call("procedure_assembly", { + procedure: entry.address, + }); + assert.ok(assembly.trim().length > 0); + const dossier = functionDossierSchema.parse( + await call("analyze_function", { procedure: entry.address }), + ); + assert.ok(dossier.pseudocode.trim().length > 0); + const mapping = await call("address_to_file_offset", { + address: entry.address, + }); + const memory = await call("read_bytes", { + address: entry.address, + length: 16, + }); + const sourceBytes = await readFile(fixture.path); + assert.equal(memory.returned_bytes, 16); + assert.equal( + memory.bytes_hex, + sourceBytes + .subarray(mapping.file_offset, mapping.file_offset + 16) + .toString("hex"), + ); + const saved = await call("close_binary", { + snapshot_path: snapshotPath, + }); + assert.ok(saved.evidence_records > 0); + const snapshot = parseAnalysisSnapshot( + JSON.parse(await readFile(snapshotPath, "utf8")), + ); + assert.equal(snapshot.target.architecture, "powerpc"); + assert.equal( + snapshot.binding.analysis_profile.parameters.language_id, + "PowerPC:BE:32:default", + ); + assert.ok(snapshot.evidence_bundle.records.length > 0); + for (const evidence of snapshot.evidence_bundle.records) + assert.equal(parseEvidence(evidence).subject?.architecture, "powerpc"); + await call("open_binary", { + path: fixture.path, + provider_id: "ghidra", + snapshot_path: snapshotPath, + }); + assert.deepEqual(await call("list_procedures"), procedures); + await call("close_binary"); + + // Same public operation through CLI, not a private injected provider. + const cli = await execute( + process.execPath, + [ + entrypoint, + "function", + fixture.path, + "rea_ppc_entry", + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 }, + ); + const evidence = parseEvidence(JSON.parse(cli.stdout)); + assert.equal(evidence.subject?.architecture, "powerpc"); + assert.equal(evidence.subject?.digest.sha256, fixture.sha256); + const cliDossier = functionDossierSchema.parse( + evidence.normalized_result, + ); + assert.equal(cliDossier.procedure.address, dossier.procedure.address); + assert.ok(cliDossier.pseudocode.trim().length > 0); + assert.equal( + createHash("sha256") + .update(await readFile(fixture.path)) + .digest("hex"), + fixture.sha256, + "Analysis modified the source executable", + ); + reports.push({ + ...fixture, + independent_reader: independent, + effective_language: image.observations.language_id, + effective_compiler_spec: image.observations.compiler_spec_id, + checks: [ + "independent ELF header and symbols", + "loaded Ghidra identity and bound language", + "fixed instruction bytes and immediate", + "conditional branch destination", + "global word and marker bytes", + "CLI/MCP and snapshot identity", + ], + procedures: procedures.length, + status: "passed", + }); + } catch (cause) { + throw new Error(`PowerPC verification failed; MCP stderr: ${stderr}`, { + cause, + }); + } finally { + try { + await client.close(); + } finally { + await transport.close(); + } + } + } +} catch (cause) { + primaryFailure = cause; +} +const completed = await completeVerifierRun(run); +if ( + completed.process_lineage.status !== "verified" || + completed.process_lineage.descendants.length !== 0 +) + throw new Error( + `Cleanup unverified; retained diagnostic workspace: ${workspace}`, + { cause: primaryFailure }, + ); +if (primaryFailure !== undefined) + throw new Error( + `PowerPC verification failed; retained workspace: ${workspace}`, + { + cause: primaryFailure, + }, + ); +await rm(workspace, { recursive: true, force: true }); +console.log(JSON.stringify({ ...completed, fixtures: reports }, null, 2)); diff --git a/scripts/verify-windows-native-inspection.mjs b/scripts/verify-windows-native-inspection.mjs index e5f807da31..5b3c2e7260 100644 --- a/scripts/verify-windows-native-inspection.mjs +++ b/scripts/verify-windows-native-inspection.mjs @@ -62,8 +62,9 @@ assert.equal(first.retryable,true); assert.match(first.reason,/Delete owned object by handle failed.*Win32 5/s); const [name,...extra]=readdirSync(temp);assert.ok(name);assert.equal(extra.length,0); const path=join(temp,name);const identity=statSync(path,{bigint:true}); -assert.throws(()=>renameSync(path,path+'-replacement')); -assert.equal(statSync(path,{bigint:true}).ino,identity.ino); +// Ownership follows the retained NTFS handle even when the directory moves. +const moved=path+'-renamed';renameSync(path,moved); +assert.equal(statSync(moved,{bigint:true}).ino,identity.ino); if(process.argv[2]!=='teardown'){ const second=systemWindowsNativeAuthority();assert.equal(second.available,true); assert.equal(readdirSync(temp).length,0); diff --git a/src/application/HermesConfigurationYaml.ts b/src/application/HermesConfigurationYaml.ts index 8d245da37e..61eb0dfdef 100644 --- a/src/application/HermesConfigurationYaml.ts +++ b/src/application/HermesConfigurationYaml.ts @@ -1,7 +1,7 @@ import { isScalar, - parse, parseDocument, + type Document, type ParsedNode, type ScalarTag, type Tags, @@ -56,9 +56,28 @@ const HERMES_YAML_OPTIONS = { a.source === b.source), }; -/** Parse a Hermes `config.yaml`. */ -export const parseHermesConfigurationYaml = (text: string): unknown => - parse(text, HERMES_YAML_OPTIONS); +// A comment-only document has no root; a bare marker has an unwritten null +// scalar. Explicit null values, tags and anchors remain actual values. +const isEmptyYamlRoot = (root: Document["contents"]): boolean => + root === null || + (isScalar(root) && + root.value === null && + root.source === "" && + root.tag === undefined && + root.anchor === undefined); + +const parseHermesDocument = (text: string) => { + const document = parseDocument(text, HERMES_YAML_OPTIONS); + const error = document.errors[0]; + if (error !== undefined) throw error; + return document; +}; + +/** Parse a Hermes `config.yaml`, preserving the distinction between no root and explicit null. */ +export const parseHermesConfigurationYaml = (text: string): unknown => { + const document = parseHermesDocument(text); + return isEmptyYamlRoot(document.contents) ? {} : document.toJS(); +}; /** Set or delete one value in a Hermes `config.yaml`, keeping the rest as written. */ export const editHermesConfigurationYaml = ( @@ -66,7 +85,16 @@ export const editHermesConfigurationYaml = ( path: readonly string[], value: unknown, ): string => { - const document = parseDocument(text, HERMES_YAML_OPTIONS); + // A plain Document, so a bare `---` root can be replaced by a new map. + const document: Document = parseHermesDocument(text); + const root = document.contents; + if (isScalar(root) && isEmptyYamlRoot(root)) { + const map = document.createNode({}); + map.commentBefore = [root.commentBefore, root.comment] + .filter((comment) => comment !== undefined) + .join("\n"); + document.contents = map; + } if (value === undefined) document.deleteIn(path); else document.setIn(path, value); // Editing REA must not reflow unrelated user prompts or other YAML scalars. diff --git a/src/application/MacHopper.ts b/src/application/MacHopper.ts index a211e9f2dd..858e7fe489 100644 --- a/src/application/MacHopper.ts +++ b/src/application/MacHopper.ts @@ -15,6 +15,8 @@ import { promisify } from "node:util"; import { z } from "zod"; import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js"; +import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js"; +import { macDeveloperIdRequirement } from "../process/MacCodeSignature.js"; import { downloadHopperPackage, HOPPER_DOWNLOAD_BYTES, @@ -24,29 +26,10 @@ const execFileAsync = promisify(execFile); const RELEASES_URL = "https://www.hopperapp.com/include/files-api.php?request=releases&public=true"; const DOWNLOAD_PREFIX = "https://www.hopperapp.com:443/downloader/public/"; - -/** - * Apple publisher identity pinned for the official Hopper macOS bundle. The - * team identifier was captured from `Hopper-6.6.0-demo.dmg` at - * `https://www.hopperapp.com:443/downloader/public/Hopper-6.6.0-demo.dmg` - * (34846271 bytes, SHA-1 0be530f038249742590992199c7b042ea20ca955) with - * `codesign -dvvv`, which reported - * `Authority=Developer ID Application: Cryptic Apps (2AMA2753NF)`. - */ -export const HOPPER_PUBLISHER = Object.freeze({ - bundleIdentifier: "com.cryptic-apps.hopper-web-4", - teamIdentifier: "2AMA2753NF", -}); - -/** - * Code requirement enforcing the bundle identifier and signing team together. - * The identifier is read from the sealed CodeDirectory, so it cannot be changed - * by editing `Info.plist` the way the previous string comparison could. - */ -const HOPPER_PUBLISHER_REQUIREMENT = - `= identifier "${HOPPER_PUBLISHER.bundleIdentifier}"` + - ` and anchor apple generic` + - ` and certificate leaf[subject.OU] = "${HOPPER_PUBLISHER.teamIdentifier}"`; +const HOPPER_PUBLISHER_REQUIREMENT = macDeveloperIdRequirement( + HOPPER_PUBLISHER.bundleIdentifier, + HOPPER_PUBLISHER.teamIdentifier, +); /** Outcome of reading a mounted bundle's signature and publisher identity. */ export type MacHopperBundleVerification = diff --git a/src/application/SetupInstallFailure.test.ts b/src/application/SetupInstallFailure.test.ts index 980e221510..d8ba27bdb9 100644 --- a/src/application/SetupInstallFailure.test.ts +++ b/src/application/SetupInstallFailure.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { HOPPER_PUBLISHER } from "./MacHopper.js"; +import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js"; import { setupInstallFailure } from "./SetupInstallFailure.js"; describe("macOS Hopper setup failure projection", () => { diff --git a/src/application/SetupInstallFailure.ts b/src/application/SetupInstallFailure.ts index 0b332fa25f..944a603430 100644 --- a/src/application/SetupInstallFailure.ts +++ b/src/application/SetupInstallFailure.ts @@ -1,4 +1,4 @@ -import { HOPPER_PUBLISHER } from "./MacHopper.js"; +import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js"; export type SetupFailureCode = | "unsupported_host" diff --git a/src/contracts/managed/managedWorkflowToolContracts.ts b/src/contracts/managed/managedWorkflowToolContracts.ts index 519c2e1259..18d78a3102 100644 --- a/src/contracts/managed/managedWorkflowToolContracts.ts +++ b/src/contracts/managed/managedWorkflowToolContracts.ts @@ -98,7 +98,7 @@ export const MANAGED_WORKFLOW_TOOL_CONTRACTS = [ name: "compare_managed_members", ...toolContractMetadata("compare_managed_members"), description: - "Compare two authenticated inspect_managed_members Evidence records using unique-only tiers: exact CIL/signature, exact declared type, name, and raw signature (decoded or not), and decoded structural method shape. Names alone are never a matching basis; a member whose signature was not decoded, or whose same-named counterpart's was not, stays unknown rather than added or removed. Metadata tokens remain build-local coordinates bound to each artifact SHA-256 and MVID, and the tuple digest does not itself remap them.", + "Compare two authenticated inspect_managed_members Evidence records using unique-only tiers: exact declared type, name, and CIL/signature; exact declared type, name, and raw signature (decoded or not); exact declared type, name, and decoded structural method shape; then exact CIL/signature or decoded structural method shape alone, which never takes a member whose own declared type and name remain unpaired on the other side or whose declaring type is unresolved. Names alone are never a matching basis; a member whose signature was not decoded, or whose same-named counterpart's was not, stays unknown rather than added or removed. Metadata tokens remain build-local coordinates bound to each artifact SHA-256 and MVID, and the tuple digest does not itself remap them.", kind: "application", inputSchema: compareManagedMembersReferenceInputSchema, outputSchema: comparisonOutputSchema, diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index 3231a2da94..5939d458a6 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -957,11 +957,11 @@ it("keeps the complete unique set when it fits the byte budget", () => { it("does not impose a message-count cutoff below the byte budget", () => { const messages = Array.from( { length: 100 }, - (_, index) => `message ${index}`, + (_, index) => `message ${String(index).padStart(2, "0")}`, ); expect( boundedViewMessages(messages, `ev_${"a".repeat(64)}`, "limitations"), - ).toEqual(messages); + ).toEqual(messages.toSorted()); }); it("bounds repeated view messages even when fewer than 64 source strings are huge", () => { diff --git a/src/domain/binaryTarget.powerpc.test.ts b/src/domain/binaryTarget.powerpc.test.ts new file mode 100644 index 0000000000..f32b4f3674 --- /dev/null +++ b/src/domain/binaryTarget.powerpc.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { parseExecutableHeader } from "./binaryTarget.js"; + +const header = (little: boolean, bits: 32 | 64 = 32, machine = 20): Buffer => { + const bytes = Buffer.alloc(bits === 32 ? 52 : 64); + bytes.set([0x7f, 0x45, 0x4c, 0x46, bits === 32 ? 1 : 2, little ? 1 : 2, 1]); + if (little) { + bytes.writeUInt16LE(2, 16); + bytes.writeUInt16LE(machine, 18); + bytes.writeUInt32LE(1, 20); + bytes.writeUInt16LE(bytes.length, bits === 32 ? 40 : 52); + } else { + bytes.writeUInt16BE(2, 16); + bytes.writeUInt16BE(machine, 18); + bytes.writeUInt32BE(1, 20); + bytes.writeUInt16BE(bytes.length, bits === 32 ? 40 : 52); + } + return bytes; +}; + +describe("PowerPC ELF header boundaries", () => { + it.each([false, true])( + "retains source endian and flags (little=%s)", + (little) => { + const parsed = parseExecutableHeader(header(little), "x64"); + if (!parsed.ok) throw new Error(parsed.error); + expect(parsed.value).toEqual({ + format: "elf", + architecture: "powerpc", + availableArchitectures: ["powerpc"], + powerpc: { + elfClass: 32, + byteOrder: little ? "little" : "big", + type: 2, + flags: 0, + }, + }); + }, + ); + + it("reads ELF64 flags at their own offset without claiming support", () => { + const bytes = header(false, 64); + bytes.writeUInt32BE(0x80000000, 48); + const parsed = parseExecutableHeader(bytes, "x64"); + if (!parsed.ok) throw new Error(parsed.error); + expect(parsed.value).toMatchObject({ + architecture: "powerpc", + powerpc: { elfClass: 64, byteOrder: "big", flags: 0x80000000 }, + }); + }); + + it("does not recognize EM_PPC64 as the 32-bit family", () => { + expect(parseExecutableHeader(header(false, 64, 21), "x64")).toEqual({ + ok: false, + error: "unsupported ELF architecture", + }); + }); +}); diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index 4e668e4605..54798e1d14 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -5,6 +5,7 @@ import type { MipsElfMetadata, PeCodeViewRecord, RiscvElfMetadata, + PowerPcElfMetadata, UniversalMachOSlice, } from "./binaryTargetTypes.js"; import { err, ok, type Result } from "./result.js"; @@ -30,6 +31,12 @@ export type ExecutableMetadata = readonly availableArchitectures: readonly "riscv"[]; readonly riscv: RiscvElfMetadata; } + | { + readonly format: "elf"; + readonly architecture: "powerpc"; + readonly availableArchitectures: readonly "powerpc"[]; + readonly powerpc: PowerPcElfMetadata; + } | { readonly format: "mach-o" | "elf" | "dos-mz" | "dos-com"; readonly architecture: HeaderArchitecture; @@ -201,7 +208,11 @@ const parseElf = ( const machine = little ? bytes.readUInt16LE(18) : bytes.readUInt16BE(18); const architecture = elfArchitecture(machine); if (architecture === undefined) return err("unsupported ELF architecture"); - if (architecture === "mips" || architecture === "riscv") { + if ( + architecture === "mips" || + architecture === "riscv" || + architecture === "powerpc" + ) { const flagsOffset = elfClass === 32 ? 36 : 48; const metadata: ElfFamilyMetadata = { elfClass, @@ -211,6 +222,13 @@ const parseElf = ( ? bytes.readUInt32LE(flagsOffset) : bytes.readUInt32BE(flagsOffset), }; + if (architecture === "powerpc") + return ok({ + format: "elf", + architecture, + availableArchitectures: [architecture], + powerpc: metadata, + }); return architecture === "mips" ? ok({ format: "elf", @@ -532,6 +550,8 @@ const elfArchitecture = ( return "x86_64"; case 8: // EM_MIPS: ISA, ABI and byte order remain separate ELF facts. return "mips"; + case 20: // EM_PPC; byte order and processor variant remain ELF declarations. + return "powerpc"; case 40: return "arm"; case 183: diff --git a/src/domain/binaryTargetTypes.ts b/src/domain/binaryTargetTypes.ts index 8c16612bca..2cd6ae2078 100644 --- a/src/domain/binaryTargetTypes.ts +++ b/src/domain/binaryTargetTypes.ts @@ -9,6 +9,7 @@ export const BINARY_ARCHITECTURES = [ "mips", "6502", "riscv", + "powerpc", ] as const; export type BinaryArchitecture = (typeof BINARY_ARCHITECTURES)[number]; @@ -36,6 +37,7 @@ export interface PeCodeViewRecord { export const ELF_METADATA_ARCHITECTURES = [ "mips", "riscv", + "powerpc", ] as const satisfies readonly BinaryArchitecture[]; export type ElfMetadataArchitecture = (typeof ELF_METADATA_ARCHITECTURES)[number]; @@ -90,6 +92,9 @@ export interface MipsAbiFlags { */ export type RiscvElfMetadata = ElfFamilyMetadata; +/** Raw PowerPC declarations; family identity does not select a processor variant. */ +export type PowerPcElfMetadata = ElfFamilyMetadata; + /** Byte range of the selected slice inside its universal Mach-O container. */ export interface UniversalMachOSlice { readonly offset: number; @@ -140,6 +145,15 @@ export type BinaryTarget = readonly executableRole?: never; readonly managed?: never; }) + | (BinaryTargetIdentity & { + readonly kind: "executable"; + readonly format: "elf"; + readonly architecture: "powerpc"; + readonly availableArchitectures: readonly "powerpc"[]; + readonly powerpc: PowerPcElfMetadata; + readonly executableRole?: never; + readonly managed?: never; + }) | (ExecutableTarget & { readonly format: "mach-o" | "elf" | "dos-mz" | "dos-com"; readonly universalSlice?: never; diff --git a/src/domain/hopperPublisher.ts b/src/domain/hopperPublisher.ts new file mode 100644 index 0000000000..6c212ea64d --- /dev/null +++ b/src/domain/hopperPublisher.ts @@ -0,0 +1,12 @@ +/** + * Apple publisher identity pinned for the official Hopper macOS bundle. The + * team identifier was captured from `Hopper-6.6.0-demo.dmg` at + * `https://www.hopperapp.com:443/downloader/public/Hopper-6.6.0-demo.dmg` + * (34846271 bytes, SHA-1 0be530f038249742590992199c7b042ea20ca955) with + * `codesign -dvvv`, which reported + * `Authority=Developer ID Application: Cryptic Apps (2AMA2753NF)`. + */ +export const HOPPER_PUBLISHER = Object.freeze({ + bundleIdentifier: "com.cryptic-apps.hopper-web-4", + teamIdentifier: "2AMA2753NF", +}); diff --git a/src/domain/managed/managedMemberComparisonCoverage.ts b/src/domain/managed/managedMemberComparisonCoverage.ts index 85da3b4eba..fce7b25fb4 100644 --- a/src/domain/managed/managedMemberComparisonCoverage.ts +++ b/src/domain/managed/managedMemberComparisonCoverage.ts @@ -107,8 +107,8 @@ export const comparisonLimitations = ( ): string[] => { const limitations: string[] = [ "Metadata tokens are build-local coordinates; matched pairs are remaps, not persistent identities.", - "Methods pair by exact CIL/signature, then declared type, name, and signature, then structural shape; names alone are never a matching basis.", - "Fields pair by exact signature, then declared type, name, and signature; an undecoded signature is compared only by its raw bytes, and an unmatched member with one stays unknown.", + "Methods pair within one declared type and name by exact CIL/signature, then signature, then structural shape; only then may exact CIL/signature or structural shape alone remap across declared types or names, never taking a member whose own declared type and name remain unpaired on the other side or whose declaring type is unresolved. Names alone are never a matching basis.", + "Fields pair by declared type, name, and signature, then remap by exact decoded signature alone under the same rule; an undecoded signature is compared only by its raw bytes, and an unmatched member with one stays unknown.", ]; if (left.coverage.state !== "complete" || right.coverage.state !== "complete") limitations.push("At least one managed member observation is partial."); diff --git a/src/domain/managed/managedMemberComparisonItems.ts b/src/domain/managed/managedMemberComparisonItems.ts index 26331be083..61bfd7c180 100644 --- a/src/domain/managed/managedMemberComparisonItems.ts +++ b/src/domain/managed/managedMemberComparisonItems.ts @@ -5,6 +5,7 @@ import type { ManagedMemberComparisonResult } from "./managedMemberComparison.js import { type ManagedFieldMatches, type ManagedMethodMatches, + identityOf, sha256, } from "./managedMemberComparisonMatch.js"; @@ -211,33 +212,36 @@ const absenceLimitations = ( `signature-not-decoded: The member's ${member.signature.parse_status} signature was compared only by exact declared type, name, and raw bytes, so a changed or renamed counterpart could not be sought.`, ]), ...(member.signature.parse_status === "decoded" && - opposite.oppositeUndecodedNames.has(declaredName(member)) + opposite.oppositeUndecodedNames.size > 0 && + opposite.oppositeUndecodedNames.has(identityOf(member)) ? [ `counterpart-signature-not-decoded: The ${opposite.opposite} ${opposite.inventory} inventory has an unpaired member with the same declared type and name whose signature was not decoded, so it may be this member with a changed signature.`, ] : []), ]; -const declaredName = (member: Method | Field): string => - JSON.stringify([member.declaring_type, member.name]); - /** * Declared names of unpaired members whose signature was not decoded: one-sided * members and candidates of ambiguous groups, which also remain unpaired. */ +interface Unpaired { + readonly item: Method | Field; + readonly identity: string; +} + const undecodedOneSidedNames = (members: { - readonly leftOnly: readonly { readonly item: Method | Field }[]; - readonly rightOnly: readonly { readonly item: Method | Field }[]; + readonly leftOnly: readonly Unpaired[]; + readonly rightOnly: readonly Unpaired[]; readonly ambiguous: readonly { - readonly left: readonly { readonly item: Method | Field }[]; - readonly right: readonly { readonly item: Method | Field }[]; + readonly left: readonly Unpaired[]; + readonly right: readonly Unpaired[]; }[]; }): UndecodedOneSidedNames => { - const names = (side: readonly { readonly item: Method | Field }[]) => + const names = (side: readonly Unpaired[]) => new Set( side .filter(({ item }) => item.signature.parse_status !== "decoded") - .map(({ item }) => declaredName(item)), + .map(({ identity }) => identity), ); return { left: names([ @@ -300,18 +304,20 @@ const fieldOnlyItem = ( }; /** - * Name the identity key an ambiguous group shares. The exact-signature key - * already includes the declared type and name, so names cannot separate it. + * Name only evidence shared by the whole group. Exact-signature ambiguities + * include both declared-identity collisions and unresolved raw-signature choices. */ const ambiguityLimitation = ( members: "methods" | "fields", basis: MethodItem["match"]["basis"], ): string => basis === "exact-signature" - ? `Multiple managed ${members} share the same declared type, name, and raw signature; REA did not guess a token remap.` - : members === "methods" - ? "Multiple managed methods share the same non-name identity key; REA did not guess a token remap." - : "Multiple managed fields share the same signature; REA did not guess a token remap from names."; + ? `Multiple managed ${members} share the same raw signature; REA did not guess declared ownership or a token remap.` + : basis === "exact-il-signature" + ? "Multiple managed methods share the same CIL and signature; REA did not guess a token remap." + : members === "methods" + ? "Multiple managed methods share the same structural shape; REA did not guess a token remap." + : "Multiple managed fields share the same signature; REA did not guess a token remap."; export const buildMethodItems = ( matches: ManagedMethodMatches, diff --git a/src/domain/managed/managedMemberComparisonMatch.ts b/src/domain/managed/managedMemberComparisonMatch.ts index 90106a17cd..2cbe32a4a6 100644 --- a/src/domain/managed/managedMemberComparisonMatch.ts +++ b/src/domain/managed/managedMemberComparisonMatch.ts @@ -28,8 +28,11 @@ type ConcreteMatchBasis = Exclude; interface Keyed { readonly item: Item; + /** Declared type and name; an unresolved type is null, the name remains. */ + readonly identity: string; readonly exactKey: string | null; readonly signatureKey: string | null; + readonly rawSignatureKey: string; readonly structuralKey: string | null; } @@ -48,6 +51,8 @@ interface Ambiguous { const keyMethod = (item: Method): Keyed => ({ item, + identity: identityOf(item), + rawSignatureKey: item.signature.raw_sha256, exactKey: item.body.status === "present" && item.body.il_sha256 !== null && @@ -99,6 +104,8 @@ const keyMethod = (item: Method): Keyed => ({ const keyField = (item: Field): Keyed => ({ item, + identity: identityOf(item), + rawSignatureKey: item.signature.raw_sha256, exactKey: item.signature.parse_status === "decoded" ? sha256(["field-exact", item.signature.raw_sha256]) @@ -126,15 +133,34 @@ const matchMethods = ( matchByKeys({ left, right, - exactBasis: "exact-il-signature", - fallbackBases: [ + rounds: [ + { + basis: "exact-il-signature", + key: declaredAs(({ exactKey }) => exactKey), + }, { basis: "exact-signature", key: ({ signatureKey }) => signatureKey, + competitionKey: ({ rawSignatureKey }) => rawSignatureKey, + }, + { + basis: "structural-method-shape", + // An unresolved owner has no declared shape match to prove. + key: declaredAs((keyed) => + keyed.item.declaring_type === null ? null : keyed.structuralKey, + ), + competitionKey: ({ structuralKey }) => structuralKey, + }, + { + basis: "exact-il-signature", + key: ({ exactKey }) => exactKey, + remap: true, }, { basis: "structural-method-shape", key: ({ structuralKey }) => structuralKey, + competitionKey: ({ structuralKey }) => structuralKey, + remap: true, }, ], }); @@ -146,11 +172,20 @@ const matchFields = ( matchByKeys({ left, right, - exactBasis: "field-signature", - fallbackBases: [ + rounds: [ + { + basis: "field-signature", + key: declaredAs(({ exactKey }) => exactKey), + }, { basis: "exact-signature", key: ({ signatureKey }) => signatureKey, + competitionKey: ({ rawSignatureKey }) => rawSignatureKey, + }, + { + basis: "field-signature", + key: ({ exactKey }) => exactKey, + remap: true, }, ], }); @@ -158,21 +193,47 @@ const matchFields = ( export type ManagedMethodMatches = ReturnType; export type ManagedFieldMatches = ReturnType; +interface MatchRound { + readonly basis: ConcreteMatchBasis; + readonly key: (item: Keyed) => string | null; + /** Name-free evidence paired with the retained member name for contesting. */ + readonly competitionKey?: (item: Keyed) => string | null; + /** + * The key omits the declared type, so a pair may join different declared + * identities; same-identity pairs were taken by earlier rounds. Contention + * still retains the member name, which unresolved owners keep as evidence. + * A member whose own identity is still unpaired on the other side does not + * take part, and a group with a member of an unresolved type stays ambiguous. + */ + readonly remap?: true; +} + interface MatchByKeysInput { readonly left: readonly Keyed[]; readonly right: readonly Keyed[]; - readonly exactBasis: ConcreteMatchBasis; - readonly fallbackBases: readonly { - readonly basis: ConcreteMatchBasis; - readonly key: (item: Keyed) => string | null; - }[]; + readonly rounds: readonly MatchRound[]; } -const matchByKeys = ({ +export const identityOf = ({ + declaring_type, + name, +}: { + readonly declaring_type: string | null; + readonly name: string; +}): string => sha256([declaring_type, name]); + +/** Narrow a key to members of one declared type and name. */ +const declaredAs = + (key: (item: Keyed) => string | null) => + (item: Keyed): string | null => { + const value = key(item); + return value === null ? null : sha256([item.identity, value]); + }; + +const matchByKeys = ({ left, right, - exactBasis, - fallbackBases, + rounds, }: MatchByKeysInput): { readonly pairs: readonly MatchedPair[]; readonly ambiguous: readonly Ambiguous[]; @@ -181,33 +242,58 @@ const matchByKeys = ({ } => { const usedLeft = new Set>(); const usedRight = new Set>(); + const pairedLeft = new Set>(); + const pairedRight = new Set>(); const pairs: MatchedPair[] = []; const ambiguous: Ambiguous[] = []; - const rounds: readonly { - readonly basis: ConcreteMatchBasis; - readonly key: (item: Keyed) => string | null; - }[] = [ - { basis: exactBasis, key: ({ exactKey }) => exactKey }, - ...fallbackBases, - ]; for (const round of rounds) { + const competitionKey = round.competitionKey ?? ((item) => item.exactKey); + const unpairedLeft = unpairedOf(left, pairedLeft); + const unpairedRight = unpairedOf(right, pairedRight); + const declarable = ownerlessDeclarable( + unpairedLeft, + unpairedRight, + competitionKey, + ); + const openLeft = left.filter((item) => !usedLeft.has(item)); + const openRight = right.filter((item) => !usedRight.has(item)); + // Only open candidates can justify overriding a held declared identity: + // consumed ambiguous rows remain identity barriers, but cannot join a + // later remap group to establish its ambiguity. + const remapContested = + round.remap === true + ? ownerlessContested([openLeft, openRight], competitionKey) + : new Set(); const leftGroups = groupBy( - left.filter((item) => !usedLeft.has(item)), + round.remap === true + ? remappable(openLeft, unpairedRight, competitionKey, remapContested) + : openLeft.filter(declarable), round.key, ); const rightGroups = groupBy( - right.filter((item) => !usedRight.has(item)), + round.remap === true + ? remappable(openRight, unpairedLeft, competitionKey, remapContested) + : openRight.filter(declarable), round.key, ); for (const [key, leftItems] of leftGroups) { const rightItems = rightGroups.get(key); if (rightItems === undefined) continue; - if (leftItems.length === 1 && rightItems.length === 1) { + // An unresolved owner may be either member's own counterpart, so a + // remap group it joins stays ambiguous. + const refused = + round.remap === true && + [...leftItems, ...rightItems].some( + ({ item }) => item.declaring_type === null, + ); + if (!refused && leftItems.length === 1 && rightItems.length === 1) { const [leftItem] = leftItems; const [rightItem] = rightItems; if (leftItem === undefined || rightItem === undefined) continue; usedLeft.add(leftItem); usedRight.add(rightItem); + pairedLeft.add(leftItem); + pairedRight.add(rightItem); pairs.push({ left: leftItem, right: rightItem, @@ -230,6 +316,22 @@ const matchByKeys = ({ } } } + // Contention can prevent a declared signature pair even when CIL/shape + // evidence cannot form a later remap. Held ambiguities still contest + // ownership; only proven pairs leave that evidence pool. Group only open + // candidates so held members retain their original ambiguity and provenance. + for (const group of unresolvedSignatureAmbiguities( + left.filter((item) => !usedLeft.has(item)), + right.filter((item) => !usedRight.has(item)), + ownerlessContested( + [unpairedOf(left, pairedLeft), unpairedOf(right, pairedRight)], + (member) => member.rawSignatureKey, + ), + )) { + for (const item of group.left) usedLeft.add(item); + for (const item of group.right) usedRight.add(item); + ambiguous.push(group); + } return { pairs, ambiguous, @@ -238,6 +340,132 @@ const matchByKeys = ({ }; }; +/** Open signature choices contested by unresolved owners, including held rows. */ +const unresolvedSignatureAmbiguities = ( + left: readonly Keyed[], + right: readonly Keyed[], + contested: ReadonlySet, +): Ambiguous[] => { + const key = (member: Keyed): string => + competitionIdentityKey(member, member.rawSignatureKey) ?? ""; + const leftGroups = groupBy(left, key); + const rightGroups = groupBy(right, key); + const groups: Ambiguous[] = []; + for (const [signature, leftItems] of leftGroups) { + const rightItems = rightGroups.get(signature); + if (!contested.has(signature) || rightItems === undefined) continue; + groups.push({ + left: leftItems, + right: rightItems, + basis: "exact-signature", + }); + } + return groups; +}; + +interface Declared { + readonly declaring_type: string | null; + readonly name: string; +} + +const unpairedOf = ( + items: readonly Keyed[], + paired: ReadonlySet>, +): Keyed[] => items.filter((item) => !paired.has(item)); + +/** + * Round-specific evidence keys an unresolved owner competes for, paired with + * its retained member name. Another same-name member on the same side can be + * the unresolved row's owner; a different name cannot. Contested candidates + * stay in remap rounds rather than pairing by declared identity. + */ +const competitionIdentityKey = ( + member: Keyed, + evidenceKey: string | null, +): string | null => + evidenceKey === null ? null : JSON.stringify([member.item.name, evidenceKey]); + +const ownerlessContested = ( + sides: readonly (readonly Keyed[])[], + keyOf: (item: Keyed) => string | null, +): ReadonlySet => { + const contested = new Set(); + for (const side of sides) { + const seen = new Map(); + const ownerless = new Set(); + for (const member of side) { + const key = competitionIdentityKey(member, keyOf(member)); + if (key === null) continue; + seen.set(key, (seen.get(key) ?? 0) + 1); + if (member.item.declaring_type === null) ownerless.add(key); + } + for (const key of ownerless) + if ((seen.get(key) ?? 0) > 1) contested.add(key); + } + return contested; +}; + +/** + * Members a remap round may group: an unresolved owner, which keeps its group + * ambiguous; a member whose selected key an unresolved owner competes for, so the + * group that preserved the base matcher's ambiguity can form; or no member of + * the same declared identity left unpaired on the other side, whether + * one-sided or in an ambiguous group. The sets are taken before grouping, so + * no group order decides a remap. + */ +const remappable = ( + items: readonly Keyed[], + others: readonly Keyed[], + keyOf: (item: Keyed) => string | null, + contested: ReadonlySet, +): Keyed[] => { + const kept = new Set(others.map(({ identity }) => identity)); + return items.filter((member) => { + const key = keyOf(member); + return ( + (key !== null && + contested.has(competitionIdentityKey(member, key) ?? "")) || + member.item.declaring_type === null || + !kept.has(member.identity) + ); + }); +}; + +/** + * Whether a member may pair in a declared-identity round. A member with an + * unresolved owner may, only when no other member on either side shares its + * selected evidence key: its owner might be any of them, so a shared key + * leaves it to the remap rounds, which keep such a group ambiguous. The + * resolved members that share such a key stay out too, so they cannot pair and + * consume the candidate the ownerless member might own. + */ +const ownerlessDeclarable = ( + left: readonly Keyed[], + right: readonly Keyed[], + keyOf: (item: Keyed) => string | null, +): ((member: Keyed) => boolean) => { + const count = (side: readonly Keyed[]) => { + const seen = new Map(); + for (const member of side) { + const key = keyOf(member); + if (key !== null) seen.set(key, (seen.get(key) ?? 0) + 1); + } + return seen; + }; + const leftSeen = count(left); + const rightSeen = count(right); + const contested = ownerlessContested([left, right], keyOf); + return (member) => { + const key = keyOf(member); + return key !== null && + contested.has(competitionIdentityKey(member, key) ?? "") + ? false + : member.item.declaring_type !== null || + key === null || + ((leftSeen.get(key) ?? 0) <= 1 && (rightSeen.get(key) ?? 0) <= 1); + }; +}; + const groupBy = ( items: readonly Item[], keyOf: (item: Item) => string | null, @@ -266,7 +494,7 @@ const observedBy = * the other side's selection stays one-sided, so a type selection never yields * a remap to a member of a type the other side did not inspect. */ -const matchObserved = ( +const matchObserved = ( left: readonly Item[], right: readonly Item[], sides: { diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index d4bd5a4330..8175b776b0 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -21,6 +21,10 @@ import { ghidraRiscvLanguageOverride, ghidraRiscvProfileParameters, } from "./GhidraRiscvProfile.js"; +import { + ghidraPowerPcLanguage, + ghidraPowerPcProfileParameters, +} from "./GhidraPowerPcProfile.js"; import { ghidraRawBinaryLanguage } from "./GhidraRawBinaryLanguage.js"; import { @@ -44,7 +48,9 @@ import { export const ghidraPinnedLanguage = ( target: BinaryTarget, ): GhidraLanguageOverride | undefined => - ghidraMipsLanguageOverride(target) ?? ghidraRiscvLanguageOverride(target); + ghidraMipsLanguageOverride(target) ?? + ghidraRiscvLanguageOverride(target) ?? + ghidraPowerPcLanguage(target); /** Resolve version-bound, deterministic semantics before Ghidra imports a target. */ export const resolveGhidraAnalysisProfile = async ( @@ -155,6 +161,7 @@ export const resolveGhidraAnalysisProfile = async ( const pinnedLanguage = ghidraPinnedLanguage(target); if ( pinnedLanguage !== undefined && + target.architecture !== "powerpc" && languageOverride !== undefined && (languageOverride.languageId !== pinnedLanguage.languageId || (languageOverride.compilerSpecId !== undefined && @@ -277,6 +284,7 @@ export const resolveGhidraAnalysisProfile = async ( ...analyzerParameters, ...ghidraMipsProfileParameters(target, languageOverride), ...ghidraRiscvProfileParameters(target), + ...ghidraPowerPcProfileParameters(target), ...(pspExtension?.ok === true ? { mips_support_lane: "psp-elf32-exec-eabi32-allegrex-v2", diff --git a/src/ghidra/GhidraPowerPcProfile.test.ts b/src/ghidra/GhidraPowerPcProfile.test.ts new file mode 100644 index 0000000000..831a47f3bc --- /dev/null +++ b/src/ghidra/GhidraPowerPcProfile.test.ts @@ -0,0 +1,166 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { parseConfig } from "../config/parseConfig.js"; +import type { + BinaryTarget, + PowerPcElfMetadata, +} from "../domain/binaryTargetTypes.js"; +import { silentLogger } from "../logger.js"; +import { HopperProvider } from "../hopper/HopperProvider.js"; +import { hopperLoaderArgsForTarget } from "../hopper/HopperAnalysisProfile.js"; +import { IdaProvider } from "../ida/IdaProvider.js"; +import { + ghidraPinnedLanguage, + ghidraProfileLanguageOverride, +} from "./GhidraAnalysisProfile.js"; +import { GhidraProvider } from "./GhidraProvider.js"; +import type { GhidraInstallationHost } from "./GhidraInstallation.js"; + +// Abbreviated from Ghidra 12.1.4 Processors/PowerPC/data/languages/ppc.ldefs. +const PPC_LDEFS = ` + + + + + + + + +`; + +// Fake installation inspection tests admission/profile contracts, not Ghidra. +const host: GhidraInstallationHost = { + platform: "linux", + architecture: "x64", + readText: () => "application.version=12.1.4\n", + executable: () => true, + probeJava: () => ({ + version: "21.0.11", + major: 21, + home: "/jdk-21", + bits: 64, + runtime: "jdk", + }), +}; +const configFor = (environment: NodeJS.ProcessEnv) => { + const parsed = parseConfig({ GHIDRA_INSTALL_DIR: "/ghidra", ...environment }); + if (!parsed.ok) throw parsed.error; + return parsed.value; +}; +const config = configFor({}); +const target = (change: Partial = {}): BinaryTarget => ({ + path: "/fixture.elf", + sha256: "a".repeat(64), + kind: "executable", + format: "elf", + architecture: "powerpc", + availableArchitectures: ["powerpc"], + powerpc: { elfClass: 32, byteOrder: "big", type: 2, flags: 0, ...change }, +}); + +describe("bounded Ghidra PowerPC admission", () => { + it("binds the generic language instead of the loader's e500 choice", async () => { + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + expect(ghidra.inspectTargetSupport(target()).status).toBe("supported"); + const resolved = await ghidra.resolveAnalysisProfile(target()); + if (!resolved.ok) throw resolved.error; + const parameters = resolved.value.profile?.parameters ?? {}; + expect(parameters).toMatchObject({ + architecture: "powerpc", + language_id: "PowerPC:BE:32:default", + compiler_spec_id: "default", + language_selection: "provider-pinned-v1", + powerpc_elf: { elf_class: 32, byte_order: "big", type: 2, flags: 0 }, + powerpc_support_lane: "elf32-be-exec-generic-v1", + }); + expect(ghidraPinnedLanguage(target())).toEqual({ + languageId: "PowerPC:BE:32:default", + compilerSpecId: "default", + }); + expect(ghidraProfileLanguageOverride(parameters)).toEqual( + ghidraPinnedLanguage(target()), + ); + }); + + it("lets a configured language replace the lane default", async () => { + const installDir = await createTestTempDirectory("rea-ghidra-ppc-"); + const languages = join( + installDir, + "Ghidra/Processors/PowerPC/data/languages", + ); + await mkdir(languages, { recursive: true }); + await writeFile(join(languages, "ppc.ldefs"), PPC_LDEFS); + const ghidra = new GhidraProvider( + configFor({ + GHIDRA_INSTALL_DIR: installDir, + REA_GHIDRA_LANGUAGE_ID: "PowerPC:BE:32:e500", + }), + silentLogger, + {}, + host, + ); + const resolved = await ghidra.resolveAnalysisProfile(target()); + if (!resolved.ok) throw resolved.error; + const parameters = resolved.value.profile?.parameters ?? {}; + expect(parameters).toMatchObject({ + language_id: "PowerPC:BE:32:e500", + language_selection: "configured-v1", + }); + expect(ghidraProfileLanguageOverride(parameters)).toEqual({ + languageId: "PowerPC:BE:32:e500", + }); + }); + + it("commits header flags so a changed interpretation changes the profile", async () => { + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + const left = await ghidra.resolveAnalysisProfile(target()); + const right = await ghidra.resolveAnalysisProfile( + target({ flags: 0x00010000 }), + ); + if (!left.ok) throw left.error; + if (!right.ok) throw right.error; + expect(left.value.profile?.digest).not.toBe(right.value.profile?.digest); + }); + + it("does not widen Windows P0", () => { + const windows = new GhidraProvider( + config, + silentLogger, + {}, + { ...host, platform: "win32" }, + ); + expect(windows.inspectTargetSupport(target()).status).toBe("unsupported"); + }); + + it.each([ + [{ elfClass: 64 }, "ELF32"], + [{ byteOrder: "little" }, "big-endian"], + [{ type: 1 }, "ET_EXEC"], + [{ type: 3 }, "ET_EXEC"], + [{ flags: 0x80000000 }, "EF_PPC_EMB"], + ] satisfies [Partial, string][])( + "refuses an unverified PowerPC interpretation %j", + async (change, reason) => { + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + const value = target(change); + expect(ghidra.inspectTargetSupport(value)).toMatchObject({ + status: "unsupported", + reason: expect.stringContaining(reason), + }); + const resolved = await ghidra.resolveAnalysisProfile(value); + expect(resolved.ok).toBe(false); + }, + ); + + it("keeps PowerPC out of unverified Hopper and IDA adapter lanes", () => { + const hopper = new HopperProvider(config, silentLogger, {}, "linux"); + expect(hopper.inspectTargetSupport(target()).status).toBe("unsupported"); + expect(hopperLoaderArgsForTarget(target()).ok).toBe(false); + const ida = new IdaProvider(config, {}); + expect(ida.inspectTargetSupport(target()).status).toBe("unsupported"); + }); +}); diff --git a/src/ghidra/GhidraPowerPcProfile.ts b/src/ghidra/GhidraPowerPcProfile.ts new file mode 100644 index 0000000000..c1655f2523 --- /dev/null +++ b/src/ghidra/GhidraPowerPcProfile.ts @@ -0,0 +1,67 @@ +import type { GhidraLanguageOverride } from "../config/ghidraLanguageOverride.js"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import type { JsonValue } from "../domain/jsonValue.js"; + +/** + * Generic 32-bit big-endian PowerPC. Ghidra's ELF loader selects the e500 + * (Book-E/SPE) variant for a plain EM_PPC header, so the lane binds this + * language instead of accepting the loader's choice. + */ +export const GHIDRA_POWERPC_LANGUAGE: GhidraLanguageOverride = { + languageId: "PowerPC:BE:32:default", + compilerSpecId: "default", +}; + +/** Keep family recognition separate from the deliberately bounded Ghidra lane. */ +export const ghidraPowerPcUnsupportedReason = ( + target: BinaryTarget, +): string | null => { + if (target.kind !== "executable" || target.architecture !== "powerpc") + return null; + const metadata = target.powerpc; + if (target.format !== "elf" || metadata === undefined) + return "PowerPC analysis requires ELF header metadata; a family name alone does not select a processor variant."; + if (metadata.elfClass !== 32) + return "This Ghidra PowerPC lane supports ELF32 only; PPC64 remains unsupported."; + if (metadata.byteOrder !== "big") + return "This Ghidra PowerPC lane supports big-endian ELF32 only; little-endian PowerPC needs separate verification."; + if (metadata.type !== 2) + return "This Ghidra PowerPC lane supports ET_EXEC only; relocatable objects and shared images need separate load verification."; + // EF_PPC_EMB (0x80000000) marks embedded variants; other bits only declare + // relocation conventions that do not change instruction decoding. + if ((metadata.flags & 0x80000000) !== 0) + return "EF_PPC_EMB declares an embedded PowerPC variant that needs a separately supported processor profile."; + return null; +}; + +/** + * The language the lane imports with, unless REA_GHIDRA_LANGUAGE_ID already + * selected one through the committed profile. + */ +export const ghidraPowerPcLanguage = ( + target: BinaryTarget, +): GhidraLanguageOverride | undefined => + target.kind === "executable" && target.architecture === "powerpc" + ? GHIDRA_POWERPC_LANGUAGE + : undefined; + +/** Commit source ELF interpretation without confusing target ISA with host CPU. */ +export const ghidraPowerPcProfileParameters = ( + target: BinaryTarget, +): Readonly> => { + if ( + target.kind !== "executable" || + target.architecture !== "powerpc" || + target.powerpc === undefined + ) + return {}; + return { + powerpc_elf: { + elf_class: target.powerpc.elfClass, + byte_order: target.powerpc.byteOrder, + type: target.powerpc.type, + flags: target.powerpc.flags, + }, + powerpc_support_lane: "elf32-be-exec-generic-v1", + }; +}; diff --git a/src/ghidra/GhidraProvider.ts b/src/ghidra/GhidraProvider.ts index 0a1cde0f07..b0ee8787d4 100644 --- a/src/ghidra/GhidraProvider.ts +++ b/src/ghidra/GhidraProvider.ts @@ -68,6 +68,7 @@ const SUPPORTED_ARCHITECTURES = new Set([ "mips", "6502", "riscv", + "powerpc", ]); /** Ghidra candidate backed by an isolated ephemeral headless import. */ diff --git a/src/ghidra/GhidraPspProfile.ts b/src/ghidra/GhidraPspProfile.ts index 17260d038e..3550d62ab2 100644 --- a/src/ghidra/GhidraPspProfile.ts +++ b/src/ghidra/GhidraPspProfile.ts @@ -1,5 +1,6 @@ import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { ghidraMipsUnsupportedReason } from "./GhidraMipsProfile.js"; +import { ghidraPowerPcUnsupportedReason } from "./GhidraPowerPcProfile.js"; import { ghidraRiscvUnsupportedReason } from "./GhidraRiscvProfile.js"; /** Allegrex's ELF machine declaration, not a filename or inferred instruction. */ @@ -14,6 +15,8 @@ export const isGhidraPspTarget = (target: BinaryTarget): boolean => export const ghidraProcessorUnsupportedReason = ( target: BinaryTarget, ): string | null => { + if (target.kind === "executable" && target.architecture === "powerpc") + return ghidraPowerPcUnsupportedReason(target); if (target.kind === "executable" && target.architecture === "riscv") return ghidraRiscvUnsupportedReason(target); if ( diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index 043d394a4b..3f83867bf9 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -33,6 +33,7 @@ import { } from "./HopperTargetLease.js"; import type { HopperTargetLease } from "./HopperTargetLease.js"; import { HOPPER_SHUTDOWN_TIMEOUT_MS } from "./HopperSessionValues.js"; +import { verifyMacHopperNativeHelper } from "./HopperMacPublisher.js"; const execFileAsync = promisify(execFile); const HOPPER_BACKGROUND_STARTUP_MS = 5_000; @@ -47,6 +48,8 @@ export interface BridgeSession { /** Process handle returned by a bridge launcher. */ export type BridgeLaunch = { + /** Verified publisher and sealed native-helper identity; omission is unknown. */ + readonly nativeHelperVerified?: true; /** Only literal native-helper arguments support a pre-handoff denial conclusion. */ readonly authorizationDenialBeforeHandoff?: true; /** Release application/target exclusivity only after verified document and process cleanup. */ @@ -106,6 +109,7 @@ export type HopperApplicationLauncherOptions = }); export interface HopperApplicationLauncherDependencies { + readonly verifyNativeHelper?: typeof verifyMacHopperNativeHelper; readonly platform?: NodeJS.Platform; readonly acquireTargetLease?: typeof acquireHopperTargetLease; readonly acquireApplicationLease?: typeof acquireLinuxHopperApplicationLease; @@ -213,6 +217,13 @@ export class HopperApplicationLauncher implements BridgeLauncher { if (signal?.aborted === true) return err(new HopperCancelledError()); const display = await this.#privateDisplay(signal); if (!display.ok) return err(display.error); + const nativeHelperVerified = + (this.dependencies.platform ?? process.platform) === "darwin" && + this.options.launchMode === "native" && + (await ( + this.dependencies.verifyNativeHelper ?? verifyMacHopperNativeHelper + )(this.options.launcherPath, this.options.environment, signal)) === + "verified"; const prepared = await prepareHopperApplication( this.options.launcherPath, this.options.environment, @@ -257,6 +268,9 @@ export class HopperApplicationLauncher implements BridgeLauncher { ...ownedLauncher, providerLifetime: "external-application", shutdownMode: "bridge-request", + ...(nativeHelperVerified + ? { nativeHelperVerified: true as const } + : {}), // The vendor helper interpolates these values into AppleScript // literals without escaping. Preserve them, but withhold the // pre-handoff conclusion when they can alter that script. diff --git a/src/hopper/HopperAnalysisProfile.ts b/src/hopper/HopperAnalysisProfile.ts index a4848533f3..79ba3ab50b 100644 --- a/src/hopper/HopperAnalysisProfile.ts +++ b/src/hopper/HopperAnalysisProfile.ts @@ -110,6 +110,7 @@ const HOPPER_DECLINED_ARCHITECTURE_LABELS: Record< > = { mips: "MIPS", riscv: "RISC-V", + powerpc: "PowerPC", }; /** Derive Hopper's complete non-interactive CLI loader selection. */ diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index 8196adcee7..2943839f43 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -59,6 +59,7 @@ import { import { connectHopperSocketOnce } from "./HopperSocketConnection.js"; import { hopperLauncherFailureDiagnostic, + hopperLauncherReportedApplicationNotFound, nativeHopperLaunchWasDenied, } from "./HopperProcessDiagnostic.js"; import { DEFAULT_HOPPER_STARTUP_TIMEOUT_MS } from "../config/providerStartupTimeout.js"; @@ -768,6 +769,17 @@ export class HopperClient { launcherFailure, }); } + if (hopperLauncherReportedApplicationNotFound(launcherFailure)) { + // Only the verified native helper's Spotlight exit precedes any target + // handoff. Wrappers can forward the same message after opening a document. + if (this.#resources.launch?.nativeHelperVerified === true) + this.#resources.shutdownConfirmed = true; + return new HopperStartError({ + userMessage: + "Hopper's launcher could not find Hopper because it relies on Spotlight. Enable Spotlight indexing for the volume that contains Hopper (for example: sudo mdutil -i on /System/Volumes/Data), wait for indexing to finish, then retry opening the target.", + launcherFailure, + }); + } return new HopperStartError({ userMessage: "Hopper's launcher helper failed before bridge readiness. Review details.launcher and complete Hopper setup before retrying.", diff --git a/src/hopper/HopperMacPublisher.ts b/src/hopper/HopperMacPublisher.ts new file mode 100644 index 0000000000..f60852915e --- /dev/null +++ b/src/hopper/HopperMacPublisher.ts @@ -0,0 +1,36 @@ +import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js"; +import { macDeveloperIdRequirement } from "../process/MacCodeSignature.js"; +import { execFileOutput } from "../process/ExecFileOutput.js"; + +/** Failure or unavailable verification leaves custom/native provenance unknown. */ +export const verifyMacHopperNativeHelper = async ( + launcherPath: string, + environment: Readonly, + signal?: AbortSignal, +): Promise<"verified" | "unknown"> => { + // codesign treats a bare name as a local file, while spawn searches PATH. + // Without an explicit path they need not identify the same executable. + if (!launcherPath.includes("/")) return "unknown"; + try { + await execFileOutput( + "/usr/bin/codesign", + [ + "--verify", + "--strict", + `-R${macDeveloperIdRequirement("hopper", HOPPER_PUBLISHER.teamIdentifier)}`, + launcherPath, + ], + { + env: environment, + ...(signal === undefined ? {} : { signal }), + timeout: 5_000, + stopSignal: "SIGKILL", + }, + ); + return "verified"; + } catch (cause: unknown) { + // Optional provenance: a failed check must never authorize lease release. + void cause; + return "unknown"; + } +}; diff --git a/src/hopper/HopperProcessDiagnostic.test.ts b/src/hopper/HopperProcessDiagnostic.test.ts index 2fe1c4cc9f..6828c7762c 100644 --- a/src/hopper/HopperProcessDiagnostic.test.ts +++ b/src/hopper/HopperProcessDiagnostic.test.ts @@ -1,7 +1,10 @@ import { expect, it } from "vitest"; import type { HopperLauncherOutcome } from "../domain/hopperErrors.js"; -import { nativeHopperLaunchWasDenied } from "./HopperProcessDiagnostic.js"; +import { + hopperLauncherReportedApplicationNotFound, + nativeHopperLaunchWasDenied, +} from "./HopperProcessDiagnostic.js"; // Captured from Hopper 6.1.0-demo and 6.6.0-demo's unmodified helpers with native // AppleScript error -1743 before open dispatch. The target script never ran. @@ -23,6 +26,8 @@ it("recognizes the native helper's complete authorization rejection", () => { }); it.each([ + { stdout: { text: "wrapper output\n", bytes: 15, retained_bytes: 15 } }, + { stdout: { text: "", bytes: 1, retained_bytes: 0 } }, { output_closed: false }, { diagnostic_truncated: true }, { signal: "SIGTERM", exit_code: null }, @@ -52,3 +57,41 @@ it.each([ expect(nativeHopperLaunchWasDenied({ ...denied, ...change })).toBe(false); }, ); + +// Captured from Hopper 6's unmodified helper with Spotlight indexing disabled. +const notFound: HopperLauncherOutcome = { + exit_code: 3, + signal: null, + stdout: { text: "", bytes: 0, retained_bytes: 0 }, + stderr: { + text: "Hopper has not been found.\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable.\n", + bytes: 121, + retained_bytes: 121, + }, + output_closed: true, + diagnostic_truncated: false, +}; + +it("recognizes the native helper's complete Spotlight not-found report", () => { + expect(hopperLauncherReportedApplicationNotFound(notFound)).toBe(true); + expect(nativeHopperLaunchWasDenied(notFound)).toBe(false); +}); + +it.each([ + { stdout: { text: "wrapper output\n", bytes: 15, retained_bytes: 15 } }, + { stdout: { text: "", bytes: 1, retained_bytes: 0 } }, + { output_closed: false }, + { diagnostic_truncated: true }, + { signal: "SIGTERM", exit_code: null }, + { exit_code: 2 }, + { + stderr: { + ...notFound.stderr, + text: `${notFound.stderr.text}another failure\n`, + }, + }, +])("rejects other not-found diagnostic shapes: %j", (change) => { + expect( + hopperLauncherReportedApplicationNotFound({ ...notFound, ...change }), + ).toBe(false); +}); diff --git a/src/hopper/HopperProcessDiagnostic.ts b/src/hopper/HopperProcessDiagnostic.ts index cf16fb9bb8..47348d8817 100644 --- a/src/hopper/HopperProcessDiagnostic.ts +++ b/src/hopper/HopperProcessDiagnostic.ts @@ -6,6 +6,14 @@ import type { ProviderProcessDiagnostic } from "../process/ProviderProcess.js"; import { parseLinuxPrivateDisplayDiagnostic } from "./LinuxPrivateDisplayDiagnostic.js"; import type { HopperLauncherOutcome } from "../domain/hopperErrors.js"; +/** Recognize launcher stderr signatures only with complete, empty stdout. */ +const hasCompleteLauncherOutput = (outcome: HopperLauncherOutcome): boolean => + outcome.output_closed && + !outcome.diagnostic_truncated && + outcome.stdout.text === "" && + outcome.stdout.bytes === 0 && + outcome.stdout.retained_bytes === 0; + /** * Hopper's native helper sends activate before its open-document Apple event. * Recognize its explicit authorization rejection. The launcher must separately @@ -18,12 +26,27 @@ export const nativeHopperLaunchWasDenied = ( outcome !== undefined && outcome.exit_code === 2 && outcome.signal === null && - outcome.output_closed && - !outcome.diagnostic_truncated && + hasCompleteLauncherOutput(outcome) && /^Cannot launch Hopper!\r?\nNot authorized to send Apple events to [^\r\n]+\.\r?\n?$/u.test( outcome.stderr.text, ); +/** + * Recognize the launcher's complete Spotlight diagnostic for actionable setup + * guidance. Custom launchers may forward this report after a handoff, so the + * report does not establish whether a document exists or release ownership. + */ +export const hopperLauncherReportedApplicationNotFound = ( + outcome: HopperLauncherOutcome | undefined, +): outcome is HopperLauncherOutcome => + outcome !== undefined && + outcome.exit_code === 3 && + outcome.signal === null && + hasCompleteLauncherOutput(outcome) && + /^Hopper has not been found\.\r?\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable\.\r?\n?$/u.test( + outcome.stderr.text, + ); + /** Extract only a matching Linux adapter failure record. */ export const hopperLauncherFailureDiagnostic = ( event: Extract, diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index aec8c20829..f9826575b9 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -126,7 +126,11 @@ export class HopperProvider implements AnalysisProviderCandidate { status: "unsupported", code: "architecture_unsupported", reason: `REA's Hopper adapter does not admit ${ - target.architecture === "mips" ? "MIPS" : "RISC-V" + target.architecture === "mips" + ? "MIPS" + : target.architecture === "riscv" + ? "RISC-V" + : "PowerPC" } targets.`, diagnostics, }; diff --git a/src/ida/IdaProvider.ts b/src/ida/IdaProvider.ts index 09011caa57..5ce924fcd1 100644 --- a/src/ida/IdaProvider.ts +++ b/src/ida/IdaProvider.ts @@ -113,7 +113,11 @@ export class IdaProvider implements AnalysisProviderCandidate { status: "unsupported", code: "architecture_unsupported", reason: `REA's IDA adapter has no verified ${ - target.architecture === "mips" ? "MIPS" : "RISC-V" + target.architecture === "mips" + ? "MIPS" + : target.architecture === "riscv" + ? "RISC-V" + : "PowerPC" } lane. Select a supported Ghidra profile instead.`, diagnostics, }; diff --git a/src/process/MacCodeSignature.ts b/src/process/MacCodeSignature.ts new file mode 100644 index 0000000000..92cd7f86ff --- /dev/null +++ b/src/process/MacCodeSignature.ts @@ -0,0 +1,7 @@ +/** Require a sealed identifier and Apple Developer ID signing team together. */ +export const macDeveloperIdRequirement = ( + identifier: string, + teamIdentifier: string, +): string => + `= identifier "${identifier}" and anchor apple generic` + + ` and certificate leaf[subject.OU] = "${teamIdentifier}"`; diff --git a/tests/acceptance/providers/hopperStartupRecovery.test.ts b/tests/acceptance/providers/hopperStartupRecovery.test.ts index 7fb039d4d9..bd4a2d0f1e 100644 --- a/tests/acceptance/providers/hopperStartupRecovery.test.ts +++ b/tests/acceptance/providers/hopperStartupRecovery.test.ts @@ -50,7 +50,42 @@ test( }); const mcp = new Client({ name: "hopper-literal-boundary", version: "1" }); let runtime: string | undefined; - onTestFinished(async () => { + try { + await mcp.connect(transport); + const call = (name: string, arguments_: Record = {}) => + mcp.callTool({ name, arguments: arguments_ }); + expect((await call("open_binary", { path: target })).isError).not.toBe( + true, + ); + const first = parseMcpToolError(await call("current_document")); + runtime = dirname(await readFile(receipt, "utf8")); + expect(first.error).toMatchObject({ + code: "cleanup_incomplete", + details: { + diagnostics: { + primary_error: { + code: "provider_unavailable", + details: { + launcher: { + exit_code: 2, + stderr: { + text: expect.stringContaining( + "Not authorized to send Apple events", + ), + }, + }, + }, + }, + }, + }, + }); + await access(`${receipt}.document-created`); + await access(runtime); + expect(parseMcpToolError(await call("list_procedures"))).toEqual(first); + expect(parseMcpToolError(await call("close_binary")).error.code).toBe( + "cleanup_incomplete", + ); + } finally { // SDK close can return immediately after SIGKILL. Wait for its actual // child-close event before recovering this process's retained lease. let serverExited: Promise | undefined; @@ -77,41 +112,7 @@ test( }); if (!acquired.acquired) throw new Error("Fixture lease remained live"); await acquired.lease.release(); - }); - await mcp.connect(transport); - const call = (name: string, arguments_: Record = {}) => - mcp.callTool({ name, arguments: arguments_ }); - expect((await call("open_binary", { path: target })).isError).not.toBe( - true, - ); - const first = parseMcpToolError(await call("current_document")); - runtime = dirname(await readFile(receipt, "utf8")); - expect(first.error).toMatchObject({ - code: "cleanup_incomplete", - details: { - diagnostics: { - primary_error: { - code: "provider_unavailable", - details: { - launcher: { - exit_code: 2, - stderr: { - text: expect.stringContaining( - "Not authorized to send Apple events", - ), - }, - }, - }, - }, - }, - }, - }); - await access(`${receipt}.document-created`); - await access(runtime); - expect(parseMcpToolError(await call("list_procedures"))).toEqual(first); - expect(parseMcpToolError(await call("close_binary")).error.code).toBe( - "cleanup_incomplete", - ); + } } }, ); @@ -201,13 +202,13 @@ test("recovers a denied Hopper launch through the same stdio MCP session and CLI ).toBe(false); }); -test( - "preserves a timed-out handoff and closes only a late bridge with matching run identity", +test.for(["handoff", "not-found"] as const)( + "preserves an unconfirmed %s handoff and closes only a late bridge with matching run identity", { timeout: 60_000 }, - async ({ workspace }) => { + async (launchMode, { workspace }) => { const target = workspace.path("target"); await copyFile("/usr/bin/true", target); - const mode = await workspace.write("mode", "handoff"); + const mode = await workspace.write("mode", launchMode); const receipt = workspace.path("bootstrap-path"); const transport = new StdioClientTransport({ command: process.execPath, @@ -257,17 +258,35 @@ test( details: { diagnostics: { primary_error: { - code: "provider_timeout", - details: { timeout_ms: 2000, launcher: { exit_code: 0 } }, + code: + launchMode === "handoff" + ? "provider_timeout" + : "provider_unavailable", + details: + launchMode === "handoff" + ? { timeout_ms: 2000, launcher: { exit_code: 0 } } + : { stage: "launch", launcher: { exit_code: 3 } }, }, }, }, }); + if (launchMode === "not-found") { + await access(`${receipt}.document-created`); + expect(initial.error.details).toMatchObject({ + diagnostics: { + primary_error: { message: expect.stringContaining("Spotlight") }, + }, + }); + } expect(parseMcpToolError(await call("list_procedures"))).toEqual(initial); const status = z.object({ result: z.object({ provider_operation_health: z.object({ - state: z.literal("not_started"), + // A timeout observes incomplete startup; a failed launcher without + // confirmed ownership leaves the provider's state unknown. + state: z.literal( + launchMode === "handoff" ? "not_started" : "unknown", + ), stage: z.literal("launch"), }), }), @@ -289,7 +308,7 @@ test( const token = setting("REA_TOKEN"); const runId = setting("REA_RUN_ID"); runtime = dirname(socket); - const lateBridge = async (identity: string, credential = token) => { + const lateBridge = (identity: string, credential = token) => { const child = spawn( process.execPath, [ @@ -301,14 +320,14 @@ test( { stdio: "ignore" }, ); children.push(child); - await vi.waitFor(() => access(socket)); return child; }; for (const [identity, credential] of [ ["different-run", token], [runId, "different-token"], ] as const) { - const wrong = await lateBridge(identity, credential); + const wrong = lateBridge(identity, credential); + await vi.waitFor(() => access(socket)); expect(parseMcpToolError(await call("close_binary")).error.code).toBe( "cleanup_incomplete", ); @@ -319,7 +338,9 @@ test( } const closing = call("close_binary"); await delay(150); - const owned = await lateBridge(runId); + // A verified close may remove the socket before the fixture can poll it. + // The successful close and child exit establish the owned bridge outcome. + const owned = lateBridge(runId); expect((await closing).isError).not.toBe(true); await vi.waitFor(() => expect(owned.exitCode).toBe(0)); await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); diff --git a/tests/boundary/cli/developmentTests.test.ts b/tests/boundary/cli/developmentTests.test.ts index 2987206c87..7becdd4e7e 100644 --- a/tests/boundary/cli/developmentTests.test.ts +++ b/tests/boundary/cli/developmentTests.test.ts @@ -45,6 +45,7 @@ const fixture = async () => { for (const path of [ "scripts/run-development-tests.mjs", "scripts/lib/development-tests.mjs", + "scripts/lib/test-projects.mjs", ]) await copyFile(join(process.cwd(), path), join(root, path)); await symlink( diff --git a/tests/boundary/cli/hopperLaunchFailureCli.test.ts b/tests/boundary/cli/hopperLaunchFailureCli.test.ts index 476f58f6be..9a4f55b7cb 100644 --- a/tests/boundary/cli/hopperLaunchFailureCli.test.ts +++ b/tests/boundary/cli/hopperLaunchFailureCli.test.ts @@ -1,6 +1,6 @@ import { chmod, copyFile, rm } from "node:fs/promises"; -import { describe, expect, onTestFinished } from "vitest"; +import { describe, expect } from "vitest"; import { z } from "zod"; import { workspaceCliTest } from "../../support/cli/workspaceCliFixture.js"; @@ -39,7 +39,57 @@ describe("Hopper launcher failure through the one-shot CLI", () => { const targetPath = workspace.path("probe"); await copyFile("/usr/bin/true", targetPath); const loaderArgs = ["--probe"]; - onTestFinished(async () => { + try { + for (const attempt of ["first", "retry"]) { + const result = await cli.run({ + arguments: [ + "analyze", + targetPath, + "--provider", + "hopper", + "--json", + ], + environment: { + ...workspace.environment, + HOPPER_LAUNCHER_PATH: launcherPath, + HOPPER_LOADER_ARGS_JSON: JSON.stringify(loaderArgs), + REA_LOG_LEVEL: "silent", + }, + timeoutMs: 45_000, + }); + const retained = RetainedRuntime.safeParse(result.json); + if (retained.success) + await rm( + retained.data.details.diagnostics.cleanup_error.details + .diagnostics.runtime_root, + { recursive: true, force: true }, + ); + + expect(result.exitCode, attempt).toBe(1); + expect(result.json, attempt).toMatchObject({ + code: "cleanup_incomplete", + details: { + cleanup: "incomplete", + diagnostics: { + primary_error: { + code: "provider_unavailable", + details: { + stage: "launch", + launcher: { + exit_code: 2, + stderr: { + text: expect.stringContaining( + "Document state is unknown", + ), + }, + }, + }, + }, + }, + }, + }); + } + } finally { const recovered = await acquireHopperTargetLease({ targetPath, targetKind: "executable", @@ -48,50 +98,6 @@ describe("Hopper launcher failure through the one-shot CLI", () => { }); expect(recovered.acquired).toBe(true); if (recovered.acquired) await recovered.lease.release(); - }); - - for (const attempt of ["first", "retry"]) { - const result = await cli.run({ - arguments: ["analyze", targetPath, "--provider", "hopper", "--json"], - environment: { - ...workspace.environment, - HOPPER_LAUNCHER_PATH: launcherPath, - HOPPER_LOADER_ARGS_JSON: JSON.stringify(loaderArgs), - REA_LOG_LEVEL: "silent", - }, - timeoutMs: 45_000, - }); - const retained = RetainedRuntime.safeParse(result.json); - if (retained.success) - await rm( - retained.data.details.diagnostics.cleanup_error.details.diagnostics - .runtime_root, - { recursive: true, force: true }, - ); - - expect(result.exitCode, attempt).toBe(1); - expect(result.json, attempt).toMatchObject({ - code: "cleanup_incomplete", - details: { - cleanup: "incomplete", - diagnostics: { - primary_error: { - code: "provider_unavailable", - details: { - stage: "launch", - launcher: { - exit_code: 2, - stderr: { - text: expect.stringContaining( - "Document state is unknown", - ), - }, - }, - }, - }, - }, - }, - }); } }, ); diff --git a/tests/boundary/filesystem/ciCachePolicy.test.ts b/tests/boundary/filesystem/ciCachePolicy.test.ts index 47e4f86dc4..4dea36ae5c 100644 --- a/tests/boundary/filesystem/ciCachePolicy.test.ts +++ b/tests/boundary/filesystem/ciCachePolicy.test.ts @@ -226,10 +226,6 @@ it("bounds the rolling static snapshot with Turbo age and size eviction", async .object({ cacheMaxAge: z.string(), cacheMaxSize: z.string(), - tasks: z.record( - z.string(), - z.object({ inputs: z.array(z.unknown()).optional() }), - ), }) .parse(JSON.parse(await readFile(join(repository, "turbo.json"), "utf8"))); // The static job restores the previous snapshot by prefix and saves the @@ -248,5 +244,4 @@ it("bounds the rolling static snapshot with Turbo age and size eviction", async unit === "TB" ? value * 1024 * 1024 : unit === "GB" ? value * 1024 : value; expect(megabytes).toBeGreaterThan(0); expect(megabytes).toBeLessThanOrEqual(1024); - expect(turbo.tasks["knip"]?.inputs).toContain("vitest.config.ts"); }); diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.ts new file mode 100644 index 0000000000..c2a282906c --- /dev/null +++ b/tests/boundary/filesystem/ciFailureReport.test.ts @@ -0,0 +1,435 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs"; + +import { + readStaticCheckArtifact, + selectStaticCheckArtifact, +} from "../../../scripts/ci/artifacts.mjs"; +import { + classifyFailure, + createGitHubApi, + reportCiFailures, + type CiRun, + type CiJob, + type FailureReportApi, +} from "../../../scripts/ci/failure-report.mjs"; + +const sha = (digit: string) => digit.repeat(40); +const diagnostic = { + kind: "lint", + path: "src/example.ts", + code: "no-debugger", + message: "Unexpected debugger statement.", + severity: "error", +} as const; +const context = ( + sourceSha: string, + taskHash = "a".repeat(16), + prHeadSha?: string, +) => ({ + version: 1 as const, + sourceSha, + ...(prHeadSha ? { prHeadSha } : {}), + sourceClean: true, + nodeVersion: "v24.18.0", + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: "linux", + architecture: "x64", + taskHashes: { "lint:built": taskHash }, +}); +const run = (overrides: Partial = {}) => + normalizeWorkflowRun({ + id: 10, + run_attempt: 1, + status: "completed", + conclusion: "failure", + event: "pull_request", + path: ".github/workflows/ci.yml@main", + head_sha: sha("a"), + head_branch: "feature", + repository: { full_name: "owner/rea" }, + pull_requests: [ + { base: { sha: sha("b"), repo: { full_name: "owner/rea" } } }, + ], + ...overrides, + }); +const job = (overrides: Partial = {}) => ({ + name: "Static checks", + status: "completed", + conclusion: "failure", + ...overrides, +}); + +const comparison = (): Parameters[0] => ({ + repository: "owner/rea", + defaultBranch: "main", + run: run(), + job: job(), + diagnostics: [diagnostic], + context: context(sha("a")), + mainRun: run({ + id: 20, + event: "push", + head_sha: sha("c"), + head_branch: "main", + pull_requests: [], + }), + mainJob: job(), + mainDiagnostics: [diagnostic], + mainContext: context(sha("c")), + executionSourcesVerified: true, + ancestry: { status: "ahead", mergeBaseSha: sha("c") }, + blobs: { "src/example.ts": true }, +}); + +it("attributes only an identical structured lint failure with matching task, context, ancestry, and source blob", () => { + const result = classifyFailure(comparison()); + expect(result).toEqual({ + classification: "matching-main-diagnostics", + reason: + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + }); +}); + +it.each([ + ["task hash", { mainContext: context(sha("c"), "b".repeat(16)) }], + ["dirty source", { context: { ...context(sha("a")), sourceClean: false } }], + [ + "toolchain", + { mainContext: { ...context(sha("c")), turboVersion: "other" } }, + ], + [ + "source sha", + { context: context(sha("d")), executionSourcesVerified: false }, + ], + [ + "diagnostic", + { mainDiagnostics: [{ ...diagnostic, message: "different message" }] }, + ], + ["content", { blobs: { "src/example.ts": false } }], + ["ancestry", { ancestry: { status: "diverged", mergeBaseSha: sha("e") } }], + ["canceled job", { job: job({ conclusion: "cancelled" }) }], + ["not a static job", { job: job({ name: "Test shard (1)" }) }], +] as const)("does not attribute on mismatched %s", (_label, overrides) => { + const result = classifyFailure({ ...comparison(), ...overrides }); + expect(result.classification).toBe("unknown"); +}); + +it.each([ + [{}, "diagnostic_source_blob_identity_unavailable"], + [ + { "src/example.ts": false }, + "diagnostic_source_content_not_identical_at_main_and_tested_base", + ], +] as const)( + "distinguishes unavailable blob identity from verified differences (%s)", + (blobs, reason) => { + expect(classifyFailure({ ...comparison(), blobs })).toEqual({ + classification: "unknown", + reason, + }); + }, +); + +it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => { + const directory = await createTestTempDirectory("rea-ci-failure-test-"); + const calls: string[] = []; + const manifest = { + repository: "owner/rea", + truncation: [], + coverage: { exhaustive: false }, + runs: [ + { + id: 10, + runAttempt: 1, + workflow: "CI", + event: "pull_request", + conclusion: "failure", + status: "collected", + }, + { + id: 11, + runAttempt: 1, + workflow: "CI", + event: "pull_request", + conclusion: "success", + status: "collected", + }, + ], + }; + const failedRun = run(); + const api: FailureReportApi = { + async getRun(_repository, id) { + calls.push(`run:${id}`); + return failedRun; + }, + async getJobs(_repository, id) { + calls.push(`jobs:${id}`); + return [{ ...job(), run_id: Number(id), run_attempt: 1 }]; + }, + async getStaticEvidence() { + calls.push("artifact"); + return null; + }, + async getAncestry() { + throw new Error("must not compare without evidence"); + }, + async verifyExecutionSource() { + throw new Error("must not verify source without evidence"); + }, + async getBlobIdentity() { + throw new Error("must not inspect blobs without evidence"); + }, + }; + const report = await reportCiFailures({ + repository: "owner/rea", + defaultBranch: "main", + batchManifest: manifest, + api, + outputDirectory: directory, + }); + expect(calls).toEqual(["run:10", "jobs:10", "artifact"]); + expect(report).toMatchObject({ + informationalOnly: true, + gateEffect: "none; this report never changes CI conclusions", + counts: { unknown: 1 }, + }); + const json = JSON.parse( + await readFile(join(directory, "report.json"), "utf8"), + ); + const markdown = await readFile(join(directory, "report.md"), "utf8"); + expect(json.failures[0]).toMatchObject({ + classification: "unknown", + reason: "no_matching_verified_main_failure", + }); + expect(markdown).toContain("Informational only"); + expect(markdown).toContain("Unknown means evidence was incomplete"); +}); + +it.each([ + [22, false], + [6, true], +] as const)( + "finds verified main evidence after %s newer PR runs (extra incorrect main hint: %s)", + async (prCount, incorrectMainHints) => { + const directory = await createTestTempDirectory("rea-ci-main-proof-"); + const mainSha = sha("c"); + const prHeadSha = sha("a"); + const mergeSha = sha("d"); + const mainRun = run({ + id: 20, + event: "push", + head_sha: mainSha, + head_branch: "main", + pull_requests: [], + }); + const prRun = run({ + head_sha: prHeadSha, + pull_requests: [ + { base: { sha: mainSha, repo: { full_name: "owner/rea" } } }, + ], + }); + const evidence = (sourceSha: string, prHead?: string) => ({ + context: context(sourceSha, undefined, prHead), + diagnostics: [diagnostic], + }); + const lookups: number[] = []; + const api: FailureReportApi = { + async getRun(_repository, id) { + lookups.push(Number(id)); + return Number(id) === 20 ? mainRun : { ...prRun, id }; + }, + async getJobs(_repository, id) { + return [{ ...job(), run_id: Number(id), run_attempt: 1 }]; + }, + async getStaticEvidence(_repository, selectedRun) { + return Number(selectedRun.id) === 20 + ? evidence(mainSha) + : evidence(mergeSha, prHeadSha); + }, + async verifyExecutionSource(_repository, selectedRun, sourceContext) { + return selectedRun.event === "pull_request" + ? sourceContext.sourceSha === mergeSha && + sourceContext.prHeadSha === selectedRun.head_sha + : sourceContext.sourceSha === selectedRun.head_sha; + }, + async getAncestry() { + return { status: "ahead", mergeBaseSha: mainSha }; + }, + async getBlobIdentity() { + return { "src/example.ts": true }; + }, + }; + const result = await reportCiFailures({ + repository: "owner/rea", + defaultBranch: "main", + batchManifest: { + repository: "owner/rea", + runs: [ + ...(incorrectMainHints + ? [ + { + id: 30, + runAttempt: 1, + workflow: "CI", + event: "push", + headBranch: "main", + conclusion: "failure", + status: "collected", + }, + ] + : []), + ...Array.from({ length: prCount }, (_, index) => ({ + id: 100 + index, + runAttempt: 1, + workflow: "CI", + event: "pull_request", + headBranch: "feature", + conclusion: "failure", + status: "collected", + })), + { + id: 20, + runAttempt: 1, + workflow: "CI", + event: "push", + headBranch: "main", + conclusion: "failure", + status: "collected", + }, + ], + }, + api, + outputDirectory: directory, + }); + expect(result.failures[0]).toMatchObject({ + runId: 100, + classification: "matching-main-diagnostics", + reason: + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + }); + expect(lookups).toHaveLength( + Math.min(prCount, 20) + 1 + Number(incorrectMainHints), + ); + expect(lookups[0]).toBe(incorrectMainHints ? 30 : 20); + expect(new Set(lookups).size).toBe(lookups.length); + expect(result.boundedInput.mainCandidateRuns).toBe(1); + expect(result.failures).toHaveLength(Math.min(prCount, 20)); + }, +); + +it("proves checkout merge parents in base-then-PR-head order and rejects false identities", async () => { + const baseSha = sha("b"); + const headSha = sha("a"); + const mergeSha = sha("c"); + const prRun = run({ + head_sha: headSha, + pull_requests: [ + { base: { sha: baseSha, repo: { full_name: "owner/rea" } } }, + ], + }); + let parents = [baseSha, headSha]; + const calls: string[] = []; + const api = createGitHubApi({ + token: "fixture-token", + apiRequest: async (_method, endpoint) => { + calls.push(endpoint); + return { + sha: mergeSha, + parents: parents.map((parentSha) => ({ sha: parentSha })), + }; + }, + }); + const mergeContext = context(mergeSha, undefined, headSha); + expect( + await api.verifyExecutionSource("owner/rea", prRun, mergeContext), + ).toBe(true); + expect(calls).toEqual([`repos/owner/rea/commits/${mergeSha}`]); + + parents = [headSha, baseSha]; + expect( + await api.verifyExecutionSource("owner/rea", prRun, mergeContext), + ).toBe(false); + expect( + await api.verifyExecutionSource("owner/rea", prRun, { + ...mergeContext, + prHeadSha: sha("e"), + }), + ).toBe(false); +}); + +it("uses static artifacts only when listing is complete, unique, and tied to this run attempt", () => { + const artifact = { + id: 12, + name: "static-check-reports", + expired: false, + workflow_run: { id: 10, head_sha: sha("a") }, + }; + const runRecord = { id: 10, head_sha: sha("a") }; + expect( + selectStaticCheckArtifact( + { total_count: 1, artifacts: [artifact] }, + runRecord, + 1, + ), + ).toBe(artifact); + expect( + selectStaticCheckArtifact( + { total_count: 2, artifacts: [artifact] }, + runRecord, + 1, + ), + ).toBeNull(); + expect( + selectStaticCheckArtifact( + { total_count: 2, artifacts: [artifact, artifact] }, + runRecord, + 1, + ), + ).toBeNull(); + expect( + selectStaticCheckArtifact( + { + total_count: 1, + artifacts: [ + { ...artifact, workflow_run: { id: 11, head_sha: sha("a") } }, + ], + }, + runRecord, + 1, + ), + ).toBeNull(); + expect( + selectStaticCheckArtifact( + { total_count: 1, artifacts: [artifact] }, + runRecord, + 2, + ), + ).toBeNull(); +}); + +it("rejects artifact redirects that could send the bearer token to a non-HTTPS host", async () => { + const calls: string[] = []; + await expect( + readStaticCheckArtifact({ + repository: "owner/rea", + artifactId: 12, + token: "fixture-token", + fetchImpl: (async (url: string | URL | Request) => { + calls.push(String(url)); + return new Response(null, { + status: 302, + headers: { location: "http://untrusted.example/artifact.zip" }, + }); + }) as typeof fetch, + }), + ).rejects.toThrow("static_artifact_redirect_invalid"); + expect(calls).toEqual([ + "https://api.github.com/repos/owner/rea/actions/artifacts/12/zip", + ]); +}); diff --git a/tests/boundary/filesystem/ciFailureReportBudget.test.ts b/tests/boundary/filesystem/ciFailureReportBudget.test.ts new file mode 100644 index 0000000000..1caacde165 --- /dev/null +++ b/tests/boundary/filesystem/ciFailureReportBudget.test.ts @@ -0,0 +1,407 @@ +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + reportCiFailures, + createGitHubApi, + type CiRun, + type FailureReportApi, + type StaticDiagnostic, + type StaticEvidence, +} from "../../../scripts/ci/failure-report.mjs"; + +const repository = "owner/rea"; +const mainSha = "c".repeat(40); +const prSha = "a".repeat(40); +const diagnostic: StaticDiagnostic = { + kind: "lint", + path: "src/example.ts", + code: "no-debugger", + message: "Unexpected debugger statement.", + severity: "error", +}; +const evidence = ( + run: CiRun, + diagnostics: readonly StaticDiagnostic[], +): StaticEvidence => ({ + context: { + version: 1, + sourceSha: run.head_sha!, + sourceClean: true, + nodeVersion: "v24.18.0", + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: "linux", + architecture: "x64", + taskHashes: { "lint:built": "a".repeat(16) }, + }, + diagnostics, +}); +const entry = (id: number, event = "pull_request", runAttempt = 1) => ({ + id, + runAttempt, + workflow: "CI", + event, + headBranch: event === "pull_request" ? "feature" : "main", + status: "collected", + conclusion: "failure", +}); + +function fixture(mainIds: readonly number[], prCount: number) { + const jobs: number[] = []; + const artifacts: number[] = []; + const paths: (readonly string[])[] = []; + const runs: Array< + Omit, "event"> & { event?: string } + > = [ + ...mainIds.map((id) => entry(id, "schedule")), + ...Array.from({ length: prCount }, (_, index) => entry(100 + index)), + ]; + const api: FailureReportApi = { + async getRun(_repository, id) { + const selected = runs.find((run) => run.id === Number(id))!; + return { + id, + run_attempt: selected.runAttempt, + status: "completed", + conclusion: "failure", + ...(selected.event ? { event: selected.event } : {}), + head_branch: selected.headBranch, + path: ".github/workflows/ci.yml@main", + head_sha: mainIds.includes(Number(id)) ? mainSha : prSha, + repository: { full_name: repository }, + pull_requests: + selected.event === "pull_request" + ? [{ base: { sha: mainSha, repo: { full_name: repository } } }] + : [], + }; + }, + async getJobs(_repository, id, attempt) { + jobs.push(Number(id)); + return [ + { + name: "Static checks", + status: "completed", + conclusion: "failure", + run_id: Number(id), + run_attempt: attempt, + }, + ]; + }, + async getStaticEvidence(_repository, run) { + artifacts.push(Number(run.id)); + return evidence(run, [diagnostic]); + }, + async getAncestry() { + return { status: "identical", mergeBaseSha: mainSha }; + }, + async getBlobIdentity(_repository, _ancestor, _descendant, requested) { + paths.push(requested); + return Object.fromEntries(requested.map((path) => [path, true])); + }, + async verifyExecutionSource() { + return true; + }, + }; + return { api, runs, jobs, artifacts, paths }; +} + +async function report(input: ReturnType) { + return await reportCiFailures({ + repository, + defaultBranch: "main", + batchManifest: { repository, runs: input.runs }, + api: input.api, + outputDirectory: await createTestTempDirectory("rea-ci-budget-"), + }); +} + +it("selects PR failures before the cap even after a day of scheduled main failures", async () => { + const input = fixture( + Array.from({ length: 24 }, (_, index) => 20 + index), + 1, + ); + input.runs.push({ + id: 200, + runAttempt: 1, + workflow: "CI", + headBranch: "feature", + status: "collected", + conclusion: "failure", + }); + const result = await report(input); + expect(result.failures).toMatchObject([ + { runId: 100, classification: "matching-main-diagnostics" }, + ]); + expect(result.boundedInput).toMatchObject({ + candidateRuns: 1, + mainCandidateRuns: 5, + manifestTruncated: false, + }); + expect(input.jobs).toEqual([100, 20]); +}); + +it("reuses five main job/artifact observations across twenty completed PR comparisons", async () => { + const input = fixture([20, 21, 22, 23, 24], 20); + const load = input.api.getStaticEvidence; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + return Number(args[1].id) < 24 + ? evidence(args[1], [{ ...diagnostic, message: "different" }]) + : result; + }; + const result = await report(input); + expect(result.counts).toEqual({ + "matching-main-diagnostics": 20, + unknown: 0, + }); + expect(input.paths).toHaveLength(100); + expect(input.jobs).toHaveLength(25); + expect(input.artifacts).toHaveLength(25); + for (const id of [20, 21, 22, 23, 24]) { + expect(input.jobs.filter((item) => item === id)).toHaveLength(1); + expect(input.artifacts.filter((item) => item === id)).toHaveLength(1); + } +}); + +it.each(["missing", "artifact error", "job error", "wrong job attempt"])( + "reuses unavailable main evidence without retries: %s", + async (failure) => { + const input = fixture([20], 3); + const load = input.api.getStaticEvidence; + const jobs = input.api.getJobs; + input.api.getJobs = async (...args) => { + const result = await jobs(...args); + if (Number(args[1]) !== 20) return result; + if (failure === "job error") throw new Error("job_snapshot_unavailable"); + return failure === "wrong job attempt" + ? result.map((job) => ({ ...job, run_attempt: 2 })) + : result; + }; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + if (Number(args[1].id) !== 20) return result; + if (failure === "artifact error") throw new Error("artifact_unavailable"); + return null; + }; + const result = await report(input); + expect(result.counts.unknown).toBe(3); + expect(input.jobs.filter((id) => id === 20)).toHaveLength(1); + expect(input.artifacts.filter((id) => id === 20)).toHaveLength( + failure === "job error" || failure === "wrong job attempt" ? 0 : 1, + ); + }, +); + +it.each([1, 2])( + "refreshes unavailable main evidence in the next report (attempt %s)", + async (attempt) => { + const input = fixture([20], 1); + expect((await report(input)).counts["matching-main-diagnostics"]).toBe(1); + input.runs[0]!.runAttempt = attempt; + const load = input.api.getStaticEvidence; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + return Number(args[1].id) === 20 ? null : result; + }; + expect((await report(input)).counts.unknown).toBe(1); + expect(input.jobs.filter((id) => id === 20)).toHaveLength(2); + expect(input.artifacts.filter((id) => id === 20)).toHaveLength(2); + }, +); + +it.each([1, 51])( + "fetches only PR paths with %s PR diagnostics and noisy main evidence", + async (count) => { + const input = fixture([20], 1); + const load = input.api.getStaticEvidence; + const diagnostics = Array.from({ length: count }, (_, index) => ({ + ...diagnostic, + path: `src/pr-${index}.ts`, + })); + input.api.getStaticEvidence = async (...args) => { + await load(...args); + const noise = + Number(args[1].id) === 20 + ? Array.from({ length: 60 }, (_, index) => ({ + ...diagnostic, + path: `src/main-${index}.ts`, + })) + : []; + return evidence(args[1], [...noise, ...diagnostics]); + }; + const result = await report(input); + expect(input.paths).toEqual([ + diagnostics.slice(0, 50).map((item) => item.path), + ]); + expect(result.failures[0]!.classification).toBe( + count === 1 ? "matching-main-diagnostics" : "unknown", + ); + if (count > 50) + expect(result.failures[0]!.reason).toBe( + "diagnostic_source_blob_identity_unavailable", + ); + }, +); + +it.each(["jobs", "artifact", "comparison"])( + "continues after an isolated main %s failure and preserves its provenance", + async (failure) => { + const input = fixture([20, 21], 3); + const jobs = input.api.getJobs; + const load = input.api.getStaticEvidence; + input.api.getJobs = async (...args) => { + const result = await jobs(...args); + if (Number(args[1]) === 20 && failure === "jobs") + throw new Error("main_jobs_unavailable"); + return result; + }; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + if (Number(args[1].id) === 20 && failure === "artifact") + throw new Error("main_artifact_unavailable"); + return Number(args[1].id) === 20 && failure === "comparison" + ? { + ...result!, + context: { ...result!.context, sourceSha: "d".repeat(40) }, + } + : result; + }; + input.api.verifyExecutionSource = async (_repository, run, context) => { + if (Number(run.id) === 20 && context.sourceSha !== run.head_sha) + throw new Error("main_comparison_unavailable"); + return true; + }; + const result = await report(input); + expect(result.counts).toEqual({ + "matching-main-diagnostics": 3, + unknown: 0, + }); + for (const item of result.failures) { + expect(item.candidateFailures).toEqual([ + { + runId: 20, + runAttempt: 1, + reason: `main_${failure}_unavailable`, + }, + ]); + } + expect(input.jobs.filter((id) => id === 20)).toHaveLength(1); + expect(input.jobs.filter((id) => id === 21)).toHaveLength(1); + expect(input.artifacts.filter((id) => id === 20)).toHaveLength( + failure === "jobs" ? 0 : 1, + ); + expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1); + }, +); + +it.each([0, 1, 2])( + "preserves every failed job when artifact analysis fails at position %s", + async (position) => { + const input = fixture([20], 1); + const jobs = [ + { name: "Python producer", conclusion: "failure" }, + { name: "Test shard (1)", conclusion: "timed_out" }, + ]; + jobs.splice(position, 0, { name: "Static checks", conclusion: "failure" }); + input.api.getJobs = async (_repository, id, attempt) => + jobs.map((job) => ({ + ...job, + status: "completed", + run_id: Number(id), + run_attempt: attempt, + })); + input.api.getStaticEvidence = async (_repository, _run, job) => { + if (job.name === "Static checks") + throw new Error("static_artifact_unavailable"); + return null; + }; + const result = await report(input); + expect(result.counts.unknown).toBe(3); + expect(result.failures).toEqual( + jobs.map((job) => ({ + runId: 100, + runAttempt: 1, + runHead: prSha, + event: "pull_request", + job: job.name, + conclusion: job.conclusion, + classification: "unknown", + reason: + job.name === "Static checks" + ? "static_artifact_unavailable" + : "no_matching_verified_main_failure", + })), + ); + }, +); + +it("keeps unavailable job inventories at run scope", async () => { + const input = fixture([], 1); + input.api.getJobs = async () => { + throw new Error("job_inventory_unavailable"); + }; + expect((await report(input)).failures).toEqual([ + { + runId: 100, + runAttempt: 1, + classification: "unknown", + reason: "job_inventory_unavailable", + }, + ]); +}); + +it.each([ + ["request", "github_api_temporarily_unavailable"], + ["budget", "github_api_request_budget_exhausted"], + ["missing sha", "github_content_blob_identity_unavailable"], + ["invalid sha", "github_content_blob_identity_unavailable"], + ["directory", "github_content_blob_identity_unavailable"], +] as const)( + "retains unavailable blob evidence as a comparison error: %s", + async (failure, reason) => { + const input = fixture([20], 1); + const contentApi = createGitHubApi({ + token: "fixture-token", + apiRequest: async () => { + if (failure === "request") throw new Error(reason); + if (failure === "missing sha") return { type: "file" }; + if (failure === "invalid sha") + return { type: "file", sha: "not-a-sha" }; + if (failure === "directory") return []; + return { type: "file", sha: "e".repeat(40) }; + }, + }); + if (failure === "budget") { + for (let id = 1; id <= 250; id++) await contentApi.getRun(repository, id); + } + input.api.getBlobIdentity = contentApi.getBlobIdentity; + const result = await report(input); + expect(result.failures).toMatchObject([ + { + job: "Static checks", + classification: "unknown", + reason: "no_matching_verified_main_failure", + candidateFailures: [{ runId: 20, runAttempt: 1, reason }], + }, + ]); + }, +); + +it.each([true, false])( + "reports blob equality only from verified file SHAs (equal: %s)", + async (equal) => { + const api = createGitHubApi({ + token: "fixture-token", + apiRequest: async (_method, endpoint) => ({ + type: "file", + sha: + equal || endpoint.includes(`ref=${mainSha}`) + ? "e".repeat(40) + : "f".repeat(40), + }), + }); + expect( + await api.getBlobIdentity(repository, mainSha, prSha, ["src/example.ts"]), + ).toEqual({ "src/example.ts": equal }); + }, +); diff --git a/tests/boundary/filesystem/ciFeedback.test.ts b/tests/boundary/filesystem/ciFeedback.test.ts new file mode 100644 index 0000000000..c61cb0d92b --- /dev/null +++ b/tests/boundary/filesystem/ciFeedback.test.ts @@ -0,0 +1,313 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; + +import { + parseStaticTaskContext, + recordStaticTaskContext, +} from "../../../scripts/ci/static-task-context.mjs"; +import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const scopeFlags = async () => + z + .object({ CI_FLAGS: z.array(z.string()) }) + .parse( + await import( + new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href + ), + ).CI_FLAGS; +const workflow = async () => + z + .object({ + jobs: z.record( + z.string(), + z + .object({ + needs: z.union([z.string(), z.array(z.string())]).optional(), + if: z.string().optional(), + steps: z + .array( + z + .object({ + run: z.string().optional(), + env: z.record(z.string(), z.string()).optional(), + }) + .passthrough(), + ) + .optional(), + }) + .passthrough(), + ), + }) + .parse( + parse( + await readFile( + new URL("../../../.github/workflows/ci.yml", import.meta.url), + "utf8", + ), + ), + ).jobs; + +it("starts source feedback before build and requires both groups without waiving missing jobs", async () => { + const jobs = await workflow(); + expect( + jobs.static?.steps?.find( + (step) => step.run === "npm run ci:static-context", + )?.["continue-on-error"], + ).toBe(true); + expect(jobs["test-source"]?.needs).toBe("changes"); + expect(jobs["test-shard"]?.needs).toEqual(["changes", "build"]); + expect( + jobs["test-source"]?.steps?.some( + (step) => + step.run === "npm run test:fast -- --reporter=default --reporter=blob", + ), + ).toBe(true); + const required = fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ); + const flags = await scopeFlags(); + for (const tier of ["fast", "full"]) { + for (const result of [ + "success", + "failure", + "cancelled", + "skipped", + "missing", + ]) { + const needs = Object.fromEntries( + Object.keys(jobs) + .filter((name) => name !== "required") + .map((name) => [name, { result: "success" }]), + ); + const outputs = { + ...Object.fromEntries( + flags.map((flag) => [flag, tier === "full" ? "true" : "false"]), + ), + tier, + code: "true", + }; + const context: Record< + string, + { result: string; outputs?: Record } + > = { ...needs, changes: { result: "success", outputs } }; + if (result === "missing") delete context["test-source"]; + else context["test-source"] = { result }; + const run = execute(process.execPath, [required], { + env: { + ...process.env, + CI_JOBS: JSON.stringify(context), + CI_QUALIFICATION_DIRECTORY: "", + }, + }); + if ( + result === "success" || + (tier === "full" && ["skipped", "missing"].includes(result)) + ) + await expect(run).resolves.toMatchObject({ stderr: "" }); + else + await expect(run).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("test-source"), + }); + } + } +}); + +it("records the actual static task graph with explicit source identity and no ambient environment", async () => { + const directory = await createTestTempDirectory("rea-ci-context-"); + const context = await recordStaticTaskContext({ outputDirectory: directory }); + const head = await execute("git", ["rev-parse", "HEAD"]); + expect(context.sourceSha).toBe(head.stdout.trim()); + expect(context.taskHashes["lint:built"]).toMatch(/^[a-f0-9]{16}$/u); + expect(context.taskHashes.jscpd).toMatch(/^[a-f0-9]{16}$/u); + expect(context).not.toHaveProperty("environmentVariables"); + expect(context).not.toHaveProperty("env"); + expect( + JSON.parse(await readFile(`${directory}/static-task-context.json`, "utf8")), + ).toEqual(context); + for (const invalid of [ + undefined, + {}, + { ...context, sourceSha: "bad" }, + { ...context, prHeadSha: "bad" }, + { ...context, taskHashes: { jscpd: "stale" } }, + ]) + expect(() => parseStaticTaskContext(invalid)).toThrow(); +}); + +it("rejects mutation methods at the shared GitHub transport before invoking the CLI", async () => { + const request = createGhApiRequest({ env: { PATH: "" } }); + await expect( + request("POST", "repos/owner/repo/actions/runs/1/rerun", { + requests: 0, + startedAt: Date.now(), + limitMs: 1000, + }), + ).rejects.toThrow("read-only GET"); +}); + +it("collects basic CI telemetry before optional dependency installation can fail", async () => { + const { jobs } = z + .object({ + jobs: z.object({ + report: z.object({ + steps: z.array( + z.object({ + id: z.string().optional(), + run: z.string().optional(), + if: z.string().optional(), + "continue-on-error": z.boolean().optional(), + }), + ), + }), + }), + }) + .parse( + parse( + await readFile( + new URL("../../../.github/workflows/ci-timings.yml", import.meta.url), + "utf8", + ), + ), + ); + const steps = jobs.report.steps; + const installation = steps.findIndex((step) => step.id === "failure-deps"); + for (const script of ["pr-ci-health.mjs", "job-timings-batch.mjs"]) { + const collection = steps.findIndex( + (step) => step.run === `node scripts/ci/${script}`, + ); + expect(collection).toBeGreaterThanOrEqual(0); + expect(installation).toBeGreaterThan(collection); + } + expect(steps[installation]).toMatchObject({ + run: "npm ci", + "continue-on-error": true, + }); + expect( + steps.find((step) => step.id === "npm-cache")?.["continue-on-error"], + ).toBe(true); + expect( + steps.find((step) => step.run === "node scripts/ci/failure-report.mjs")?.if, + ).toContain("steps.failure-deps.outcome == 'success'"); +}); + +it("merges source and built reports exactly once using the workflow's shard matrix", async () => { + const jobs = await workflow(); + const matrix = z + .object({ + matrix: z.object({ shard: z.string().regex(/^\$\{\{.*\}\}$/u) }), + }) + .parse(jobs["test-shard"]?.strategy).matrix.shard; + const shards = z + .array(z.string().regex(/^\d+\/\d+$/u)) + .min(1) + .parse( + runInNewContext( + matrix.slice(3, -2), + { + fromJSON: JSON.parse, + needs: { changes: { outputs: { tier: "fast" } } }, + }, + { timeout: 1_000 }, + ), + ); + const labels = Object.fromEntries( + ["source", "built"].map((group) => [ + group, + z + .string() + .min(1) + .parse( + jobs[group === "source" ? "test-source" : "test-shard"]?.steps?.find( + (step) => step.env?.VITEST_BLOB_LABEL, + )?.env?.VITEST_BLOB_LABEL, + ), + ]), + ); + const root = await createTestTempDirectory("rea-ci-group-merge-"); + const config = join(root, "vitest.config.mjs"); + await writeFile( + config, + `export default { test: { maxWorkers: 1, projects: [${["source", "built"].map((name) => `{ test: { name: "${name}", globals: true, include: ["${name}*.test.js"] } }`).join(",")}] } };`, + ); + const files = [ + "source.test.js", + ...Array.from( + { length: shards.length }, + (_, index) => `built-${index}.test.js`, + ), + ]; + for (const file of files) + await writeFile( + join(root, file), + 'it("runs its selected group", () => expect(true).toBe(true));', + ); + const vitest = fileURLToPath( + new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url), + ); + const blobs = new Map(); + const runs = [ + { group: "source", shard: [] }, + ...shards.map((shard) => ({ group: "built", shard: [`--shard=${shard}`] })), + ]; + for (const { group, shard } of runs) { + await execute( + process.execPath, + [ + vitest, + "run", + "--config", + config, + "--project", + group, + "--reporter=blob", + ...shard, + ], + { + cwd: root, + env: { ...process.env, VITEST_BLOB_LABEL: labels[group]! }, + timeout: 20_000, + }, + ); + for (const filename of await readdir(join(root, ".vitest/blob"))) { + expect(blobs.has(filename)).toBe(false); + blobs.set(filename, await readFile(join(root, ".vitest/blob", filename))); + } + // Separate hosted jobs start with independent report directories. + await rm(join(root, ".vitest/blob"), { recursive: true }); + } + expect(blobs.size).toBe(runs.length); + await mkdir(join(root, ".vitest/blob"), { recursive: true }); + for (const [filename, bytes] of blobs) + await writeFile(join(root, ".vitest/blob", filename), bytes); + await execute( + process.execPath, + [ + vitest, + "--config", + config, + "--merge-reports", + "--reporter=json", + "--outputFile.json=merged.json", + ], + { cwd: root, timeout: 20_000 }, + ); + const merged = z + .object({ + success: z.literal(true), + testResults: z.array(z.object({ name: z.string() })), + }) + .parse(JSON.parse(await readFile(join(root, "merged.json"), "utf8"))); + expect( + merged.testResults.map(({ name }) => name.split("/").at(-1)).sort(), + ).toEqual(files.toSorted()); +}); diff --git a/tests/boundary/filesystem/ciJobTimingsBatch.test.ts b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts index 6c2658c000..e0dcecc824 100644 --- a/tests/boundary/filesystem/ciJobTimingsBatch.test.ts +++ b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts @@ -18,6 +18,8 @@ it("collects the latest attempt for recent completed CI and Release runs into a workflow_runs: [ { id: 42, + event: "push", + head_branch: "main", run_attempt: 3, status: "completed", conclusion: "failure", @@ -43,51 +45,9 @@ it("collects the latest attempt for recent completed CI and Release runs into a }, ], }, - [ - { - jobs: [ - { - name: "Release lane", - status: "completed", - conclusion: "success", - created_at: recent, - started_at: recent, - completed_at: new Date(Date.parse(recent) + 50_000).toISOString(), - steps: [], - }, - ], - }, - ], - [ - { - jobs: [ - { - name: "Failed lane", - status: "completed", - conclusion: "failure", - created_at: recent, - started_at: recent, - completed_at: new Date(Date.parse(recent) + 55_000).toISOString(), - steps: [], - }, - ], - }, - ], - [ - { - jobs: [ - { - name: "Cancelled lane", - status: "completed", - conclusion: "cancelled", - created_at: recent, - started_at: recent, - completed_at: new Date(Date.parse(recent) + 30_000).toISOString(), - steps: [], - }, - ], - }, - ], + completedJobs("Release lane", "success", recent, 50), + completedJobs("Failed lane", "failure", recent, 55), + completedJobs("Cancelled lane", "cancelled", recent, 30), ]); const result = await runBatch(fixture); const manifest = JSON.parse( @@ -116,8 +76,20 @@ it("collects the latest attempt for recent completed CI and Release runs into a expect(manifest.errors).toEqual([]); expect(manifest.runs).toMatchObject([ { id: 43, runAttempt: 2, conclusion: "success" }, - { id: 42, runAttempt: 3, conclusion: "failure" }, - { id: 41, runAttempt: 1, conclusion: "cancelled" }, + { + id: 42, + runAttempt: 3, + conclusion: "failure", + event: "push", + headBranch: "main", + }, + { + id: 41, + runAttempt: 1, + conclusion: "cancelled", + event: null, + headBranch: null, + }, ]); expect(manifest.runs).toEqual( expect.arrayContaining([ @@ -234,6 +206,31 @@ it("retains a partial manifest when its collection budget is exhausted", async ( expect(manifest.truncation.join(" ")).toContain("Time budget"); }); +function completedJobs( + name: string, + conclusion: string, + startedAt: string, + seconds: number, +) { + return [ + { + jobs: [ + { + name, + status: "completed", + conclusion, + created_at: startedAt, + started_at: startedAt, + completed_at: new Date( + Date.parse(startedAt) + seconds * 1000, + ).toISOString(), + steps: [], + }, + ], + }, + ]; +} + async function createFixture(responses: unknown[]) { const directory = await createTestTempDirectory("rea-ci-job-timings-batch-"); const fakeBin = join(directory, "bin"); diff --git a/tests/boundary/filesystem/ciPrHealth.test.ts b/tests/boundary/filesystem/ciPrHealth.test.ts new file mode 100644 index 0000000000..756fb47489 --- /dev/null +++ b/tests/boundary/filesystem/ciPrHealth.test.ts @@ -0,0 +1,617 @@ +import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import { delimiter, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + classifyPrCi, + collectPrCiHealth, + matchRunToPullRequest, + writePrCiHealthReport, +} from "../../../scripts/ci/pr-ci-health.mjs"; +import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs"; +import type { + PrCiPullRequest, + PrCiWorkflowRun, +} from "../../../scripts/ci/pr-ci-health.mjs"; +import { executeWorkflowFixture } from "../../support/workflowGit.js"; + +const reporter = fileURLToPath( + new URL("../../../scripts/ci/pr-ci-health.mjs", import.meta.url), +); +const observedAt = Date.parse("2026-10-14T00:00:00Z"); +const headSha = "a".repeat(40); +const mergeSha = "b".repeat(40); + +it("classifies workflow states without confusing absent, pending, and completed CI", () => { + const pull = validPull(); + const cases = [ + [ + "startup_failure", + run({ status: "completed", conclusion: "startup_failure" }), + ], + [ + "queued_pending_executing", + run({ status: "in_progress", conclusion: null }), + ], + ["approval_required", run({ status: "waiting", conclusion: null })], + ["completed_failure", run({ status: "completed", conclusion: "failure" })], + [ + "completed_canceled", + run({ status: "completed", conclusion: "cancelled" }), + ], + ["completed_success", run({ status: "completed", conclusion: "success" })], + ["completed_other", run({ status: "completed", conclusion: "neutral" })], + [ + "completed_other", + run({ status: "completed", conclusion: "constructor" }), + ], + ] as const; + for (const [category, evidence] of cases) { + for (const runsComplete of [true, false]) { + const result = classifyPrCi({ + pull, + runs: [ + { + run: { ...evidence, run_attempt: 1 }, + basis: "event_pr_number_and_head_sha", + }, + ], + runsComplete, + pullRequestInventoryComplete: true, + ambiguousRuns: [], + }); + expect(result).toMatchObject({ + category: runsComplete ? category : "unknown", + ...(runsComplete + ? {} + : { reason: "workflow_run_inventory_incomplete" }), + runs: [ + { + id: 77, + attempt: 1, + status: evidence.status, + conclusion: evidence.conclusion, + }, + ], + }); + } + } + for (const runsComplete of [true, false]) { + expect( + classifyPrCi({ + pull: { ...pull, mergeable: false }, + runs: [], + runsComplete, + pullRequestInventoryComplete: true, + ambiguousRuns: [], + }).category, + ).toBe("merge_conflict"); + } +}); + +it("requires complete inventories, known mergeability, and immutable head identity to report missing", () => { + const pull = validPull({ mergeable: true }); + const classify = (overrides = {}) => + classifyPrCi({ + pull, + runs: [], + runsComplete: true, + pullRequestInventoryComplete: true, + ambiguousRuns: [], + ...overrides, + }); + + expect(classify().category).toBe("missing"); + expect(classify({ runsComplete: false })).toMatchObject({ + category: "deferred", + reason: "lookup_inventory_incomplete", + }); + expect(classify({ pullRequestInventoryComplete: false }).category).toBe( + "deferred", + ); + expect( + classify({ + ambiguousRuns: [{ run: run(), basis: "ambiguous_head_identity" }], + }).category, + ).toBe("unknown"); + expect(classify({ pull: { ...pull, mergeable: undefined } }).category).toBe( + "unknown", + ); + expect(classify({ pull: { ...pull, identityValid: false } }).category).toBe( + "unknown", + ); +}); + +it("accepts event PR/head evidence for merge-SHA runs and refuses ambiguous branch identities", () => { + const pull = validPull(); + const associatedMergeRun = run({ + head_sha: mergeSha, + pull_requests: [ + { + number: pull.number!, + head: { sha: headSha, repo: { id: pull.head.repositoryId } }, + }, + ], + }); + expect( + matchRunToPullRequest({ + run: associatedMergeRun, + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }), + ).toEqual({ kind: "match", basis: "event_pr_number_and_head_sha" }); + expect( + matchRunToPullRequest({ + run: run({ + pull_requests: [ + { number: pull.number!, head: { sha: "c".repeat(40) } }, + ], + }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("none"); + + const branchRun = run({ pull_requests: [] }); + expect( + matchRunToPullRequest({ + run: branchRun, + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + inventoryComplete: false, + }).kind, + ).toBe("ambiguous"); + expect( + matchRunToPullRequest({ + run: branchRun, + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }), + ).toEqual({ + kind: "match", + basis: "event_unique_head_ref_and_repository_id", + }); + expect( + matchRunToPullRequest({ + run: run({ head_sha: mergeSha, pull_requests: [] }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("none"); + expect( + matchRunToPullRequest({ + run: run({ + pull_requests: [{ number: pull.number!, head: { sha: headSha } }], + }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("ambiguous"); + expect( + matchRunToPullRequest({ + run: branchRun, + pull, + identityCounts: new Map([["73\u0000feature", 2]]), + }).kind, + ).toBe("ambiguous"); + expect( + matchRunToPullRequest({ + run: run({ event: "workflow_dispatch" }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("none"); +}); + +it("scans target-branch PRs through public gh GET requests and writes JSON and Markdown", async () => { + const fixture = await createFixture(); + const calls: string[] = []; + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (method, endpoint) => { + calls.push(`${method} ${endpoint}`); + if (method !== "GET") throw new Error("fixture rejected a non-GET request"); + if (endpoint.includes("/pulls?")) + return [ + rawPull(42, { mergeable: true }), + rawPull(43, { mergeable: false }), + rawPull(44, { + base: { ref: "release", repo: { id: 11 } }, + mergeable: true, + }), + rawPull(45, { draft: true, mergeable: true }), + ]; + if (endpoint.includes("/actions/workflows/ci.yml/runs?")) + return { + total_count: 1, + workflow_runs: [ + run({ + id: 9001, + event: "pull_request", + status: "completed", + conclusion: "failure", + head_sha: mergeSha, + pull_requests: [ + { + number: 42, + head: { sha: headSha, repo: { id: 73 } }, + }, + ], + }), + ], + }; + throw new Error(`unexpected GET endpoint: ${endpoint}`); + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(calls).toHaveLength(2); + expect(calls.every((call) => call.startsWith("GET "))).toBe(true); + expect(calls[0]).toContain("state=open"); + expect(calls[0]).not.toContain("base="); + expect(calls[1]).toContain("event=pull_request"); + expect(report.pullRequests).toHaveLength(2); + expect(report.pullRequests.map(({ number }) => number)).toEqual([42, 43]); + expect(report.pullRequests.map(({ ci }) => ci.category)).toEqual([ + "completed_failure", + "merge_conflict", + ]); + expect(report.scan).toMatchObject({ + status: "complete", + pullRequestInventoryComplete: true, + workflowRunInventoryComplete: true, + apiRequests: 2, + }); + + const output = join(fixture.directory, ".cache", "ci-pr-health"); + const paths = await writePrCiHealthReport(report, output); + const written = JSON.parse(await readFile(paths.jsonPath, "utf8")); + const markdown = await readFile(paths.markdownPath, "utf8"); + expect(written.counts.completed_failure).toBe(1); + expect(markdown).toContain("# Pull request CI health snapshot"); + expect(markdown).toContain("#42"); + expect(markdown).toContain("completed_failure"); +}); + +it("counts older open PRs when deciding whether a branch/ref fallback is unique", async () => { + const olderPull = rawPull(63, { + created_at: "2026-09-01T00:00:00Z", + head: { ...rawPull(63).head, ref: "shared-feature" }, + }); + const currentPull = rawPull(64, { + mergeable: true, + head: { ...olderPull.head }, + }); + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [currentPull, olderPull]; + if (endpoint.includes("/actions/workflows/ci.yml/runs?")) + return { + workflow_runs: [ + run({ + head_branch: "shared-feature", + pull_requests: [], + }), + ], + }; + throw new Error(`unexpected GET endpoint: ${endpoint}`); + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(report.scan.pullRequestInventoryComplete).toBe(true); + expect(report.pullRequests.map(({ number }) => number)).toEqual([64]); + expect(report.pullRequests[0]?.ci).toMatchObject({ + category: "unknown", + reason: "workflow_run_identity_ambiguous", + }); +}); + +it.each([false, true])( + "keeps page-capped scans uncertain (observed success: %s)", + async (observedSuccess) => { + const pull = rawPull(62, { mergeable: true }); + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [pull]; + const page = Number( + new URL(`https://fixture.invalid/${endpoint}`).searchParams.get("page"), + ); + return { + workflow_runs: Array.from({ length: 100 }, (_item, index) => + observedSuccess && page === 1 && index === 0 + ? run({ + run_attempt: 1, + pull_requests: [ + { number: 62, head: { sha: headSha, repo: { id: 73 } } }, + ], + }) + : run({ + id: page * 100 + index, + head_branch: "different-branch", + head_repository: { id: 999 }, + pull_requests: [], + }), + ), + }; + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(report.scan.workflowRunPages).toBe(5); + expect(report.scan.workflowRunInventoryComplete).toBe(false); + const firstPull = report.pullRequests[0]; + if (firstPull === undefined) throw new Error("fixture PR was not scanned"); + expect(firstPull.ci).toMatchObject({ + category: observedSuccess ? "unknown" : "deferred", + reason: observedSuccess + ? "workflow_run_inventory_incomplete" + : "lookup_inventory_incomplete", + }); + expect(firstPull.ci.runs).toHaveLength(observedSuccess ? 1 : 0); + }, +); + +it("keeps the result unknown when the head changes between API snapshots", async () => { + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [rawPull(63)]; + if (endpoint.includes("/actions/workflows/ci.yml/runs?")) + return { workflow_runs: [] }; + return { + ...rawPull(63, { mergeable: true }), + head: { + ...rawPull(63).head, + sha: mergeSha, + }, + }; + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(report.pullRequests[0]).toMatchObject({ + identityChangedDuringScan: true, + ci: { category: "unknown", reason: "pull_request_head_identity_invalid" }, + }); + expect(report.scan).toMatchObject({ + status: "partial", + errors: ["pull_request_identity_changed_during_scan"], + }); +}); + +it("public CLI uses only GET and produces a report from fixture GitHub responses", async () => { + const fixture = await createFixture(); + const ghCalls = join(fixture.directory, "gh-calls.jsonl"); + const ghResponse = join(fixture.directory, "gh-response.json"); + const fakeGh = join(fixture.fakeBin, "gh"); + await writeFile( + ghResponse, + JSON.stringify([ + [rawPull(51, { mergeable: true })], + { total_count: 0, workflow_runs: [] }, + ]), + ); + await writeFile( + fakeGh, + `#!/usr/bin/env node\nconst fs=require("node:fs");\nconst args=process.argv.slice(2);\nfs.appendFileSync(process.env.CI_GH_CALLS, JSON.stringify(args)+"\\n");\nif(args[0]!=="api"||args[1]!=="--method"||args[2]!=="GET")process.exit(41);\nconst page=Number(new URL("https://fixture.invalid/"+args.at(-1)).searchParams.get("page"));\nconst responses=JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSE,"utf8"));\nprocess.stdout.write(JSON.stringify(responses[page===1&&args.at(-1).includes("/pulls?")?0:1]));\n`, + ); + await chmod(fakeGh, 0o755); + const outputDirectory = join(fixture.directory, ".cache", "ci-pr-health"); + await executeWorkflowFixture(process.execPath, [reporter], { + cwd: fixture.directory, + env: { + ...process.env, + PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`, + CI_REPOSITORY: "owner/rea", + CI_REPOSITORY_ID: "11", + CI_DEFAULT_BRANCH: "main", + CI_PR_HEALTH_DIRECTORY: outputDirectory, + CI_GH_CALLS: ghCalls, + CI_GH_RESPONSE: ghResponse, + }, + }); + const calls = (await readFile(ghCalls, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(calls).toHaveLength(2); + expect( + calls.every( + (args: string[]) => args[1] === "--method" && args[2] === "GET", + ), + ).toBe(true); + const report = JSON.parse( + await readFile(join(outputDirectory, "report.json"), "utf8"), + ); + expect(report.pullRequests[0].ci.category).toBe("missing"); + await expect( + readFile(join(outputDirectory, "summary.md"), "utf8"), + ).resolves.toContain("missing"); +}); + +it("rejects non-GET methods before launching the GitHub client", async () => { + const fixture = await createFixture(); + const request = createGhApiRequest({ + env: { + ...process.env, + PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`, + CI_GH_CALLS: join(fixture.directory, "should-not-exist"), + }, + }); + await expect( + request("POST", "repos/owner/rea/pulls", { + requests: 0, + startedAt: observedAt, + limitMs: 1_000, + }), + ).rejects.toThrow("read-only GET"); + await expect( + readFile(join(fixture.directory, "should-not-exist")), + ).rejects.toMatchObject({ code: "ENOENT" }); +}); + +function validPull(overrides: Partial = {}): PrCiPullRequest { + return { + number: 42, + identityValid: true, + mergeable: true, + head: { repositoryId: 73, ref: "feature", sha: headSha }, + ...overrides, + }; +} + +it.each([ + ["newer run", { id: 78, created_at: "2026-10-13T01:00:00Z" }, "unknown"], + [ + "older run", + { id: 78, created_at: "2026-10-12T23:00:00Z" }, + "completed_success", + ], + ["equal timestamps", { id: 78 }, "unknown"], + [ + "retry of an older run", + { id: 78, created_at: "2026-10-12T23:00:00Z", run_attempt: 2 }, + "unknown", + ], + ["unknown timestamp", { id: 78, created_at: "invalid" }, "unknown"], + ["newer attempt", { run_attempt: 3 }, "unknown"], + ["older attempt", { run_attempt: 1 }, "completed_success"], +] as const)( + "retains ambiguity evidence for a %s without reporting a stale completion", + (_label, overrides, category) => { + const currentAttempt = "id" in overrides ? 1 : 2; + const result = classifyPrCi({ + pull: validPull(), + runs: [ + { + run: run({ run_attempt: currentAttempt }), + basis: "event_pr_number_and_head_sha", + }, + ], + ambiguousRuns: [ + { + run: run({ run_attempt: currentAttempt, ...overrides }), + basis: "ambiguous_head_identity", + }, + ], + runsComplete: true, + pullRequestInventoryComplete: true, + }); + expect(result.category).toBe(category); + expect(result.runs).toHaveLength(2); + }, +); + +it("does not let an older proven run hide a newer run with an unverified repository ID", async () => { + const pull = rawPull(42); + const older = run({ + id: 70, + created_at: "2026-10-13T00:00:00Z", + pull_requests: [{ number: 42, head: { sha: headSha, repo: { id: 73 } } }], + }); + const newer = run({ + id: 71, + created_at: "2026-10-13T01:00:00Z", + status: "in_progress", + conclusion: null, + pull_requests: [{ number: 42, head: { sha: headSha } }], + }); + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest: async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [pull]; + if (endpoint.endsWith("/pulls/42")) return { ...pull, mergeable: true }; + return { workflow_runs: [newer, older] }; + }, + }); + expect(report.pullRequests[0]?.ci).toMatchObject({ + category: "unknown", + reason: "workflow_run_identity_ambiguous", + }); + expect(report.pullRequests[0]?.ci.runs).toHaveLength(2); +}); + +it("keeps a proven newest run even when two older matched runs have equal timestamps", () => { + const result = classifyPrCi({ + pull: validPull(), + runs: [ + run({ id: 70, run_attempt: 1 }), + run({ id: 71, run_attempt: 1, conclusion: "failure" }), + run({ + id: 72, + run_attempt: 1, + created_at: "2026-10-13T01:00:00Z", + status: "in_progress", + conclusion: null, + }), + ].map((run) => ({ run, basis: "event_pr_number_and_head_sha" })), + ambiguousRuns: [], + runsComplete: true, + pullRequestInventoryComplete: true, + }); + expect(result.category).toBe("queued_pending_executing"); + expect(result.runs).toHaveLength(3); +}); + +function run(overrides: Partial = {}): PrCiWorkflowRun { + return { + id: 77, + event: "pull_request", + head_branch: "feature", + head_repository: { id: 73 }, + head_sha: headSha, + status: "completed", + conclusion: "success", + created_at: "2026-10-13T00:00:00Z", + pull_requests: [], + ...overrides, + }; +} + +function rawPull(number: number, overrides: Record = {}) { + return { + number, + draft: false, + created_at: "2026-10-13T00:00:00Z", + html_url: `https://github.com/owner/rea/pull/${String(number)}`, + base: { ref: "main", repo: { id: 11 } }, + head: { + ref: `feature-${String(number)}`, + sha: headSha, + repo: { id: 73, full_name: "contributor/rea" }, + }, + ...overrides, + }; +} + +async function createFixture() { + const directory = await createTestTempDirectory("rea-ci-pr-health-"); + const fakeBin = join(directory, "bin"); + await mkdir(fakeBin); + return { directory, fakeBin }; +} diff --git a/tests/boundary/filesystem/ciQualification.test.ts b/tests/boundary/filesystem/ciQualification.test.ts index b6a5a1e1a4..673b09d34c 100644 --- a/tests/boundary/filesystem/ciQualification.test.ts +++ b/tests/boundary/filesystem/ciQualification.test.ts @@ -44,7 +44,7 @@ function run(overrides: Record = {}) { id: 7, status: "completed", conclusion: "success", - path: ".github/workflows/ci.yml", + path: ".github/workflows/ci.yml@main", event: "schedule", created_at: new Date(now - 120_000).toISOString(), head_sha: validSha, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index d7ac1418ed..d2191eb01c 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -121,7 +121,11 @@ it("exports every canonical scope with read authority for admission and keeps te ); expect(timings.on.workflow_run).toBeUndefined(); expect(timings.on.schedule).toHaveLength(1); - expect(timings.permissions).toEqual({ contents: "read", actions: "read" }); + expect(timings.permissions).toEqual({ + contents: "read", + actions: "read", + "pull-requests": "read", + }); const checkout = timings.jobs.report.steps.find((step) => step.uses?.startsWith("actions/checkout@"), ); @@ -154,6 +158,7 @@ it("holds native and provider fan-out behind static validation while portable fe "static", "build", "test-shard", + "test-source", "test", "docs", "required", @@ -369,7 +374,7 @@ it("keeps fast feedback separate from complete qualification and release proof", // release their concurrency slot without waiting for another runner. expect(ci.jobs.required?.if).toBe("${{ !cancelled() }}"); const shards = ci.jobs["test-shard"]?.steps ?? []; - const fast = shards.find((step) => step.run?.includes("npm run test:run")); + const fast = shards.find((step) => step.run?.includes("npm run test:built")); const full = shards.find((step) => step.run?.includes("test:ci:shard:run")); expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'"); expect(fast?.run).toContain("--reporter=blob"); diff --git a/tests/boundary/filesystem/ciStaticArtifacts.test.ts b/tests/boundary/filesystem/ciStaticArtifacts.test.ts new file mode 100644 index 0000000000..45915f4504 --- /dev/null +++ b/tests/boundary/filesystem/ciStaticArtifacts.test.ts @@ -0,0 +1,99 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +import { parseStaticArchive } from "../../../scripts/ci/artifacts.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const producer = fileURLToPath( + new URL("../../../scripts/ci/static-check-report.mjs", import.meta.url), +); + +it.each([3.5, 100, 0])( + "retains real lint failures when the duplication threshold is %s", + async (threshold) => { + const root = await createTestTempDirectory("rea-ci-static-archive-"); + await mkdir(join(root, "src")); + await mkdir(join(root, "tests")); + await writeFile( + join(root, ".oxlintrc.json"), + JSON.stringify({ rules: { "no-debugger": "error" } }), + ); + await writeFile( + join(root, ".jscpd.json"), + JSON.stringify({ threshold, minLines: 5, minTokens: 50 }), + ); + await writeFile( + join(root, "src/example.ts"), + "debugger;\nexport const example = true;\n", + ); + const duplicate = + "export function compute(value: number) {\n" + + Array.from({ length: 20 }, (_, index) => ` value += ${index};\n`).join( + "", + ) + + " return value;\n}\n"; + for (const name of threshold === 3.5 ? [] : ["first", "second"]) + await writeFile(join(root, `src/${name}.ts`), duplicate); + await expect( + execute(process.execPath, [producer, "lint", "src/example.ts"], { + cwd: root, + }), + ).rejects.toMatchObject({ code: 1 }); + const duplication = execute(process.execPath, [producer, "duplicates"], { + cwd: root, + }); + if (threshold === 0) + await expect(duplication).rejects.toMatchObject({ code: 1 }); + else await duplication; + const reports = join(root, ".cache/static-checks"); + expect( + JSON.parse( + await readFile(join(reports, "jscpd/jscpd-report.json"), "utf8"), + ), + ).toMatchObject({ reaCheck: { exitCode: threshold === 0 ? 1 : 0 } }); + await writeFile( + join(reports, "static-task-context.json"), + JSON.stringify({ + version: 1, + sourceSha: "a".repeat(40), + sourceClean: true, + nodeVersion: process.version, + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: process.platform, + architecture: process.arch, + taskHashes: { "lint:built": "a".repeat(16), jscpd: "b".repeat(16) }, + }), + ); + const archive = join(root, "reports.zip"); + await execute( + "zip", + [ + "-q", + archive, + "oxlint.json", + "jscpd/jscpd-report.json", + "static-task-context.json", + ], + { cwd: reports }, + ); + const evidence = await parseStaticArchive(archive); + expect(evidence.diagnostics.filter(({ kind }) => kind === "lint")).toEqual([ + expect.objectContaining({ + kind: "lint", + path: "src/example.ts", + severity: "error", + }), + ]); + const duplicates = evidence.diagnostics.filter( + ({ kind }) => kind === "jscpd", + ); + if (threshold === 0) expect(duplicates.length).toBeGreaterThan(0); + else expect(duplicates).toEqual([]); + }, +); diff --git a/tests/boundary/filesystem/ciTestHistory.test.ts b/tests/boundary/filesystem/ciTestHistory.test.ts index 411e9ece9d..84950d49c5 100644 --- a/tests/boundary/filesystem/ciTestHistory.test.ts +++ b/tests/boundary/filesystem/ciTestHistory.test.ts @@ -35,7 +35,7 @@ function run(id: number, overrides: Record = {}) { id, status: "completed", conclusion: "success", - path: ".github/workflows/ci.yml", + path: ".github/workflows/ci.yml@main", event: "schedule", created_at: new Date(now - 120_000).toISOString(), head_branch: "main", diff --git a/tests/boundary/filesystem/ciTestProjects.test.ts b/tests/boundary/filesystem/ciTestProjects.test.ts new file mode 100644 index 0000000000..f0394c5bbf --- /dev/null +++ b/tests/boundary/filesystem/ciTestProjects.test.ts @@ -0,0 +1,75 @@ +import { execFileSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { + BUILT_TEST_PROJECTS, + SOURCE_TEST_PROJECTS, + projectForTest, +} from "../../../scripts/lib/test-projects.mjs"; +import { + developmentTestPlan, + parseDevelopmentTestRequest, +} from "../../../scripts/lib/development-tests.mjs"; + +const repository = process.cwd(); + +describe("canonical test-project ownership", () => { + it("assigns every tracked test file exactly once to source or built", () => { + const files = execFileSync("git", ["ls-files", "*.test.ts"], { + cwd: repository, + encoding: "utf8", + }) + .trim() + .split("\n") + .filter(Boolean); + const grouped = new Map(); + for (const file of files) { + const project = projectForTest(file); + grouped.set(project.group, [...(grouped.get(project.group) ?? []), file]); + } + + expect(grouped.get("source")).not.toBeUndefined(); + expect(grouped.get("built")).not.toBeUndefined(); + expect( + (grouped.get("source")?.length ?? 0) + + (grouped.get("built")?.length ?? 0), + ).toBe(files.length); + expect( + SOURCE_TEST_PROJECTS.reduce( + (count, project) => count + project.include.length, + 0, + ), + ).toBeGreaterThan(0); + expect( + BUILT_TEST_PROJECTS.every((project) => project.group === "built"), + ).toBe(true); + expect(() => projectForTest("tests/unowned/example.test.ts")).toThrow( + "No test project owns", + ); + }); + + it("owns the Hopper real-provider conformance test as built and requires preparation", () => { + const testPath = + "tests/conformance/providers/hopper/realHopperSearch.test.ts"; + expect(projectForTest(testPath)).toMatchObject({ + name: "boundary", + group: "built", + }); + expect(() => parseDevelopmentTestRequest("local", [testPath])).toThrow( + `Use test:focused for built test projects: ${testPath}`, + ); + const request = parseDevelopmentTestRequest("focused", [testPath]); + expect(developmentTestPlan(request)).toMatchObject({ needsBuild: true }); + }); + + it("uses canonical selectors in Vitest while retaining independent runner policy", async () => { + const config = await readFile(join(repository, "vitest.config.ts"), "utf8"); + expect(config).toContain( + "[...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]", + ); + expect(config).toContain("PROJECT_POLICIES"); + expect(config).not.toContain('"src/{contracts,domain}/**/*.test.ts"'); + }); +}); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 3e2aadcd45..6a59424480 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -3,6 +3,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { runInNewContext } from "node:vm"; import { expect, it } from "vitest"; import { parse } from "yaml"; @@ -48,9 +49,7 @@ const workflow = async () => it("merges available failed-shard diagnostics and skips unavailable shard reports", async () => { const steps = (await workflow()).test!.steps; const download = steps.find( - (step) => - step.uses?.includes("actions/download-artifact") && - step.with?.pattern === "vitest-report-*", + (step) => step.with?.pattern === "vitest-report-*", ); const merge = steps.findIndex((step) => step.run?.includes("test:ci:merge")); const upload = steps.findIndex((step) => step.with?.name === "test-results"); @@ -59,8 +58,7 @@ it("merges available failed-shard diagnostics and skips unavailable shard report ); expect(merge).toBeGreaterThan(0); expect(download?.if).toContain("!cancelled()"); - expect(download?.if).toContain("needs.test-shard.result == 'success'"); - expect(download?.if).toContain("needs.test-shard.result == 'failure'"); + expect(upload).toBeGreaterThan(merge); expect(gate).toBeGreaterThan(upload); const merges = steps.filter((step) => step.run?.includes("--reporter=json")); @@ -70,8 +68,6 @@ it("merges available failed-shard diagnostics and skips unavailable shard report "--reporter=./scripts/ci/test-timing-reporter.mjs", ); expect(step.if).toContain("!cancelled()"); - expect(step.if).toContain("needs.test-shard.result == 'success'"); - expect(step.if).toContain("needs.test-shard.result == 'failure'"); } expect(steps[upload]?.with?.path).toContain("test-timings.json"); expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true }); @@ -86,12 +82,23 @@ it("merges available failed-shard diagnostics and skips unavailable shard report ); const command = steps[gate]?.run; if (command === undefined) throw new Error("Missing shard result gate"); - for (const result of ["failure", "cancelled", "skipped", ""]) { - await expect( - execute("bash", ["-c", command], { - env: { ...process.env, CODE_REQUIRED: "true", SHARD_RESULT: result }, - }), - ).rejects.toMatchObject({ code: 1 }); + for (const tier of ["fast", "full"]) { + for (const shard of ["success", "failure", "cancelled", "skipped", ""]) { + for (const source of ["success", "failure", "cancelled", "skipped", ""]) { + const check = execute("bash", ["-c", command], { + env: { + ...process.env, + CODE_REQUIRED: "true", + SHARD_RESULT: shard, + SOURCE_REQUIRED: String(tier === "fast"), + SOURCE_RESULT: source, + }, + }); + if (shard === "success" && (tier !== "fast" || source === "success")) + await expect(check).resolves.toMatchObject({ stderr: "" }); + else await expect(check).rejects.toMatchObject({ code: 1 }); + } + } } await expect( execute("bash", ["-c", command], { @@ -100,6 +107,80 @@ it("merges available failed-shard diagnostics and skips unavailable shard report ).resolves.toMatchObject({ stderr: "" }); }); +// These workflow conditions use the shared boolean/string expression subset. +// Evaluate the checked-in expressions so failure-state coverage cannot drift +// into a separate test implementation of the collection policy. +function conditionValue( + condition: string | undefined, + needs: Record, + canceled: boolean, +) { + if (!condition) throw new Error("Missing report collection condition"); + return runInNewContext( + condition + .replaceAll("needs.test-shard", 'needs["test-shard"]') + .replaceAll("needs.test-source", 'needs["test-source"]'), + { needs, cancelled: () => canceled }, + { timeout: 1_000 }, + ); +} + +it("collects either fast test group while keeping full reports tied to full shards", async () => { + const steps = (await workflow()).test.steps; + const download = steps.find( + (step) => step.with?.pattern === "vitest-report-*", + ); + const fast = steps.find( + (step) => step.name === "Merge fast-suite JUnit and timing reports", + ); + const full = steps.find( + (step) => step.name === "Merge coverage, JUnit, and timing reports", + ); + for (const tier of ["fast", "full"]) { + for (const shard of ["success", "failure", "cancelled", "skipped", ""]) { + for (const source of ["success", "failure", "cancelled", "skipped", ""]) { + for (const [code, canceled] of [ + ["true", false], + ["true", true], + ["false", false], + ["false", true], + ] as const) { + const needs = { + changes: { outputs: { code, tier } }, + "test-shard": { result: shard }, + "test-source": { result: source }, + }; + const builtAvailable = ["success", "failure"].includes(shard); + const sourceAvailable = ["success", "failure"].includes(source); + const enabled = code === "true" && !canceled; + const available = + builtAvailable || (tier === "fast" && sourceAvailable); + const expected = enabled + ? { + download: available, + fast: tier === "fast" && available, + full: tier === "full" && builtAvailable, + } + : { download: false, fast: false, full: false }; + expect(conditionValue(download?.if, needs, canceled)).toBe( + expected.download, + ); + expect(conditionValue(fast?.if, needs, canceled)).toBe(expected.fast); + expect(conditionValue(full?.if, needs, canceled)).toBe(expected.full); + } + } + } + } + const summary = steps.find((step) => + step.run?.includes("scripts/ci/test-report.mjs"), + ); + expect(summary?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}"); + expect(summary?.env?.SOURCE_RESULT).toBe("${{ needs.test-source.result }}"); + expect(summary?.env?.SOURCE_REQUIRED).toBe( + "${{ needs.changes.outputs.tier == 'fast' }}", + ); +}); + it("keeps diagnosis available after merge failure and publishes only complete successful history", async () => { const steps = (await workflow()).test!.steps; const diagnosis = steps.find((step) => step.with?.name === "test-results"); @@ -220,7 +301,12 @@ it("exports portable timing history and removes it on a failed, partial, or inva { name: join(root, "tests/fast.test.ts"), durationMs: 10 }, ], }; - const run = async (value: unknown, shards = "success", merge = "success") => { + const run = async ( + value: unknown, + shards = "success", + merge = "success", + source?: string, + ) => { await writeFile(join(root, "test-timings.json"), JSON.stringify(value)); await writeFile(summary, ""); return execute(process.execPath, [fileURLToPath(script)], { @@ -229,6 +315,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva ...process.env, GITHUB_STEP_SUMMARY: summary, SHARD_RESULT: shards, + SOURCE_REQUIRED: String(source !== undefined), + SOURCE_RESULT: source ?? "", MERGE_RESULT: merge, }, }); @@ -256,6 +344,24 @@ it("exports portable timing history and removes it on a failed, partial, or inva await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" }); expect(await readFile(summary, "utf8")).toContain("diagnostic only"); } + await run(report, "success", "success", "success"); + expect(await readFile(history, "utf8")).toContain('"success":true'); + for (const [shards, source] of [ + ["skipped", "success"], + ["success", "skipped"], + ["success", "cancelled"], + ["success", "failure"], + ["success", ""], + ]) { + await run(report); + await run(report, shards, "success", source); + await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" }); + const text = await readFile(summary, "utf8"); + expect(text).toContain(`Source result: ${source}.`); + expect(text).toContain("diagnostic only"); + if (shards === "skipped" || source !== "failure") + expect(text).toContain("Partial test diagnostics"); + } await run(report); await expect( run({ @@ -352,6 +458,21 @@ it("recovers complete module costs from real blob reports without executing test expect(file.durationMs).toBeCloseTo( Object.values(file.phases).reduce((sum, value) => sum + value, 0), ); + const partial = await execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { + ...process.env, + SHARD_RESULT: "skipped", + SOURCE_REQUIRED: "true", + SOURCE_RESULT: "success", + MERGE_RESULT: "success", + }, + }); + expect(partial.stdout).toContain("Partial test diagnostics"); + expect(partial.stdout).toContain("sample.test.js"); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); await execute(process.execPath, [fileURLToPath(script)], { cwd: root, env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" }, @@ -420,13 +541,15 @@ it("records failed omissions from full reports without treating partial or unhan state: index === 1 ? "failed" : "passed", })), }; - const run = async (report: unknown) => { + const run = async (report: unknown, source = "failure") => { await writeFile(join(root, "test-timings.json"), JSON.stringify(report)); await execute(process.execPath, [fileURLToPath(script)], { cwd: root, env: { ...process.env, SHARD_RESULT: "failure", + SOURCE_REQUIRED: "true", + SOURCE_RESULT: source, MERGE_RESULT: "failure", CI_VERIFY_TEST_INVENTORY: "true", CI_TEST_SELECTION: JSON.stringify( @@ -458,6 +581,8 @@ it("records failed omissions from full reports without treating partial or unhan { ...timing, reason: "interrupted" }, ]) expect(await run(report)).toMatchObject({ complete: false }); + for (const source of ["skipped", "cancelled", ""]) + expect(await run(timing, source)).toMatchObject({ complete: false }); await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({ code: 1, }); diff --git a/tests/boundary/filesystem/ciWorkflowRun.test.ts b/tests/boundary/filesystem/ciWorkflowRun.test.ts new file mode 100644 index 0000000000..cbebb5c4d1 --- /dev/null +++ b/tests/boundary/filesystem/ciWorkflowRun.test.ts @@ -0,0 +1,54 @@ +import { expect, it } from "vitest"; +import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs"; + +it.each([ + [".github/workflows/ci.yml", ".github/workflows/ci.yml", null], + [".github/workflows/ci.yml@main", ".github/workflows/ci.yml", "main"], + [ + ".github/workflows/ci.yml@refs/heads/release/v1", + ".github/workflows/ci.yml", + "refs/heads/release/v1", + ], + [ + ".github/workflows/ci.yml@feature@v1", + ".github/workflows/ci.yml", + "feature@v1", + ], + [ + ".github/workflows/release.yaml@main", + ".github/workflows/release.yaml", + "main", + ], +])( + "preserves API path %s while separating its file identity and ref", + (path, file, ref) => { + const raw = { id: 10, path, head_sha: "a".repeat(40) }; + const parsed = normalizeWorkflowRun(raw); + expect(parsed).toEqual({ ...raw, workflowPath: { file, ref } }); + expect(raw).not.toHaveProperty("workflowPath"); + }, +); + +it.each([ + undefined, + null, + 42, + "", + ".github/workflows/ci.yml@", + ".github/workflows/ci.yml@main\n", + ".github/workflows/ci.yml@main\0", + "../.github/workflows/ci.yml", + ".github/workflows/../ci.yml@main", + ".github/workflows/ci.yml-extra@main", + "https://example.test/ci.yml@main", +])( + "keeps malformed workflow path %s unknown instead of inventing a file identity", + (path) => { + const parsed = normalizeWorkflowRun({ + path, + workflowPath: { file: ".github/workflows/ci.yml", ref: "main" }, + }); + expect(parsed.path).toBe(path); + expect(parsed.workflowPath).toBeNull(); + }, +); diff --git a/tests/boundary/filesystem/hermesClientConfiguration.test.ts b/tests/boundary/filesystem/hermesClientConfiguration.test.ts index 218573d5c4..7bd72a6767 100644 --- a/tests/boundary/filesystem/hermesClientConfiguration.test.ts +++ b/tests/boundary/filesystem/hermesClientConfiguration.test.ts @@ -114,18 +114,34 @@ describe("Hermes YAML shape diagnostics", () => { "!!omap [{rea: {command: rea, args: [mcp]}}]", "!!timestamp 2001-12-15", "!!binary cmVh", - ].flatMap((value) => [ - { - original: `${value}\n`, - problem: "The configuration root must be an object", - }, - { - original: `mcp_servers: ${value}\n`, - problem: '"mcp_servers" must be an object', - }, - ]), + ] + .flatMap((value) => [ + { + original: `${value}\n`, + problem: "The configuration root must be an object", + }, + { + original: `mcp_servers: ${value}\n`, + problem: '"mcp_servers" must be an object', + }, + ]) + .concat( + // An explicit null root is a value, not an empty file. + [ + "null\n", + "~\n", + "# Hermes configuration\nnull\n", + "--- ~\n", + "--- !!null\n", + '!!null ""\n', + "--- &empty\n", + ].map((original) => ({ + original, + problem: "The configuration root must be an object", + })), + ), )( - "blocks setup and uninstall before changes for tagged non-record YAML: $original", + "blocks setup and uninstall before changes for tagged or null non-record YAML: $original", async ({ original, problem }) => { const { home, client } = await hermesClient(); const cursor = join(home, ".cursor", "mcp.json"); @@ -281,7 +297,36 @@ mcp_servers: }); }); - it.each(["", "\n"])( + // Comments and a bare document marker parse to a null document, like an + // empty file, and are kept ahead of the new registration. +}); + +describe("Hermes YAML empty configurations", () => { + it("rejects multiple YAML documents before treating an empty first root as absent", async () => { + const { client } = await hermesClient(); + const original = + "---\n---\nmcp_servers:\n rea:\n command: rea\n args: [mcp]\n"; + await writeFile(client.configPath, original); + expect(await inspectClientConfiguration(client, {}, command)).toMatchObject( + { status: "invalid" }, + ); + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "failed", + reason: "readback", + }); + expect(await readFile(client.configPath, "utf8")).toBe(original); + await expect( + readFile(`${client.configPath}.rea.backup`, "utf8"), + ).rejects.toThrow(); + }); + + it.each([ + "", + "\n", + "# Hermes configuration\n", + "---\n", + "---\n# Hermes configuration\n", + ])( "configures an empty config.yaml %j as a new document", async (original) => { const { home, client } = await hermesClient(); @@ -289,7 +334,10 @@ mcp_servers: expect( (await configureClientConfiguration(client, {}, command)).status, ).toBe("configured"); - expect(parseYaml(await readFile(client.configPath, "utf8"))).toEqual({ + const text = await readFile(client.configPath, "utf8"); + const kept = original.trimEnd(); + expect(text.slice(0, kept.length)).toBe(kept); + expect(parseYaml(text)).toEqual({ mcp_servers: { rea: { command: command[0], args: command.slice(1), enabled: true }, }, diff --git a/tests/boundary/filesystem/staticCheckReports.test.ts b/tests/boundary/filesystem/staticCheckReports.test.ts index 7d022da72d..10ef066744 100644 --- a/tests/boundary/filesystem/staticCheckReports.test.ts +++ b/tests/boundary/filesystem/staticCheckReports.test.ts @@ -49,7 +49,7 @@ const lintDiagnostics = async (root: string) => ), ).diagnostics; -it("includes every actually linted file in both cached lint task inputs", () => { +it("covers the lint inventory and root tool configs in resolved cache inputs", () => { const scan = spawnSync( process.execPath, [join(repository, "node_modules/oxlint/bin/oxlint"), "--debug=files"], @@ -69,6 +69,7 @@ it("includes every actually linted file in both cached lint task inputs", () => "run", "lint", "lint:built", + "knip", "--dry=json", ], { @@ -99,6 +100,14 @@ it("includes every actually linted file in both cached lint task inputs", () => files.filter((file) => !Object.hasOwn(task.inputs, file)), `${task.task} omits actual checker inputs`, ).toEqual([]); + const knip = tasks.find(({ task }) => task === "knip"); + expect(knip).toBeDefined(); + expect( + files.filter( + (file) => !file.includes("/") && !Object.hasOwn(knip!.inputs, file), + ), + "knip omits root tool configuration inputs", + ).toEqual([]); }); it("summarizes warnings without losing their complete diagnostic records", async () => { diff --git a/src/application/javascript/nativeTypeBindings.test.ts b/tests/boundary/javascript/nativeTypeBindings.test.ts similarity index 77% rename from src/application/javascript/nativeTypeBindings.test.ts rename to tests/boundary/javascript/nativeTypeBindings.test.ts index 6fcf8791e2..b0ec74d819 100644 --- a/src/application/javascript/nativeTypeBindings.test.ts +++ b/tests/boundary/javascript/nativeTypeBindings.test.ts @@ -1,27 +1,16 @@ -import { mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { afterEach, expect, it } from "vitest"; +import { expect, it } from "vitest"; -import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; -import type { JavaScriptApplicationAnalysisResult } from "../../domain/javascript/javascriptApplicationAnalysis.js"; -import { buildJavaScriptNativeHandoffs } from "../../domain/javascript/javascriptNativeHandoff.js"; -import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js"; - -const fixtures: string[] = []; - -afterEach(async () => { - await Promise.all( - fixtures - .splice(0) - .map((path) => rm(path, { recursive: true, force: true })), - ); -}); +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import type { JavaScriptApplicationAnalysisResult } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { buildJavaScriptNativeHandoffs } from "../../../src/domain/javascript/javascriptNativeHandoff.js"; +import { analyzeJavaScriptApplication } from "../../../src/application/javascript/JavaScriptApplicationService.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const analyzeFixture = async (source: string) => { - const fixture = await mkdtemp(join(tmpdir(), "rea-native-types-")); - fixtures.push(fixture); + const fixture = await createTestTempDirectory("rea-native-types-"); const scope = new ArtifactResourceScope(); try { await writeFile(join(fixture, "main.ts"), `${source}\n`); diff --git a/tests/boundary/providers/RizinInferiorTty.test.ts b/tests/boundary/providers/RizinInferiorTty.test.ts index 4c19760d56..61774b192e 100644 --- a/tests/boundary/providers/RizinInferiorTty.test.ts +++ b/tests/boundary/providers/RizinInferiorTty.test.ts @@ -215,7 +215,10 @@ process.stdin.on("data", (chunk) => { ); execFileSync(process.execPath, ["--check", provider]); await chmod(provider, 0o700); - manager = new RizinDebugSessionManager({ REA_RIZIN_COMMAND: provider }); + manager = new RizinDebugSessionManager({ + ...process.env, + REA_RIZIN_COMMAND: provider, + }); const started = await manager.start({ path: "/tmp/fake-target", inferior_tty: tty, @@ -576,7 +579,7 @@ process.stdin.on("data", (chunk) => { await chmod(provider, 0o700); let providerStderr = ""; manager = new RizinDebugSessionManager( - { REA_RIZIN_COMMAND: provider }, + { ...process.env, REA_RIZIN_COMMAND: provider }, { spawnProcess: async (spawnOptions) => { const launched = await spawnOwnedProviderProcess(spawnOptions); diff --git a/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts b/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts index 06e725dc18..b236b21ebe 100644 --- a/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts @@ -15,6 +15,7 @@ import type { } from "../../../../src/hopper/BridgeLauncher.js"; import { HopperApplicationLauncher } from "../../../../src/hopper/BridgeLauncher.js"; import { HopperClient } from "../../../../src/hopper/HopperClient.js"; +import { acquireHopperTargetLease } from "../../../../src/hopper/HopperTargetLease.js"; import { HopperFixtureLauncher } from "./hopperClient.fixture.js"; // Failure assertions keep the external document unknown. Teardown then publishes @@ -118,6 +119,86 @@ it("does not equate an owned native helper exit with an external GUI exit", asyn }); }); +it.skipIf(process.platform !== "darwin")( + "releases the runtime and target lease after a provenance-verified Spotlight exit, then retries", + async () => { + const directory = await createTestTempDirectory( + "rea-hopper-native-not-found-", + ); + const targetPath = join(directory, "target"); + await writeFile(targetPath, "fixture target"); + const loaderArgs = [ + "-e", + 'process.stderr.write("Hopper has not been found.\\nPlease make sure that Spotlight is active in order for this tool to find Hopper\'s executable.\\n", () => process.exit(3))', + "--", + ]; + // Simulate a verified native helper at the launcher provenance boundary; + // publisher verification itself has real codesign boundary coverage. + const native = new HopperApplicationLauncher( + { + environment: {}, + launcherPath: process.execPath, + targetPath, + targetKind: "executable", + loaderArgs, + bridgeScriptPath: "/rea/hopper_bridge.py", + launchMode: "native", + }, + { verifyNativeHelper: async () => "verified" }, + ); + const retry = new HopperFixtureLauncher(); + let session: BridgeSession | undefined; + const client = new HopperClient({ + launcher: { + async launch(input, options) { + if (session !== undefined) return retry.launch(input); + session = input; + const launched = await native.launch(input, options); + expect(launched).toMatchObject({ + ok: true, + value: { nativeHelperVerified: true }, + }); + return launched; + }, + }, + startupTimeoutMs: 10_000, + }); + onTestFinished(async () => { + const closed = await client.close(); + if (!closed.ok) throw closed.error; + }); + const failed = await client.start(); + expect(failed).toMatchObject({ + ok: false, + error: { _tag: "HopperStartError", cleanupIncomplete: false }, + }); + if (failed.ok) throw new Error("Expected Spotlight failure"); + expect(projectAnalysisError(failed.error)).toMatchObject({ + code: "provider_unavailable", + message: expect.stringContaining("Spotlight"), + details: { launcher: { exit_code: 3 } }, + }); + if (session === undefined) throw new Error("Missing launch session"); + await expect(access(session.directory)).rejects.toMatchObject({ + code: "ENOENT", + }); + const lease = await acquireHopperTargetLease({ + targetPath, + targetKind: "executable", + loaderArgs, + runId: "spotlight-retry-proof", + }); + if (!lease.acquired) + throw new Error("Spotlight failure retained the target lease"); + await lease.lease.release(); + expect(await client.start()).toMatchObject({ ok: true }); + expect(await client.callTool("echo", { value: "retried" })).toEqual({ + ok: true, + value: { value: "retried" }, + }); + }, +); + it.each(["exit", "signal", "exit75"] as const)( "reports a failed non-owning launcher %s without waiting for bridge timeout or inventing a GUI exit", async (termination) => { diff --git a/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts b/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts index 7866b978b0..5ffabdd59a 100644 --- a/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts +++ b/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts @@ -47,7 +47,7 @@ describe("managed member comparison", () => { }), ]); expect(result.limitations).toContain( - "Methods pair by exact CIL/signature, then declared type, name, and signature, then structural shape; names alone are never a matching basis.", + "Methods pair within one declared type and name by exact CIL/signature, then signature, then structural shape; only then may exact CIL/signature or structural shape alone remap across declared types or names, never taking a member whose own declared type and name remain unpaired on the other side or whose declaring type is unresolved. Names alone are never a matching basis.", ); expect( managedMemberComparisonResultSchema.safeParse({ @@ -339,10 +339,12 @@ describe("managed member comparison uncertainty", () => { const rightField = right.result.fields[0]; expect(rightField).toBeDefined(); if (rightField === undefined) return; + // Both right fields are renamed, so neither left field keeps its declared + // identity and only the shared signature links them. const duplicatedRight = { ...right.result, fields: [ - rightField, + { ...rightField, name: "moved" }, { ...rightField, token: "0x04000002", @@ -365,6 +367,9 @@ describe("managed member comparison uncertainty", () => { candidate_left_tokens: ["0x04000001", "0x04000002"], candidate_right_tokens: ["0x04000001", "0x04000002"], }), + limitations: [ + "Multiple managed fields share the same signature; REA did not guess a token remap.", + ], }), ]); }); @@ -582,7 +587,7 @@ describe("managed member comparison of ambiguous undecoded signatures", () => { basis: "exact-il-signature", }), limitations: [ - "Multiple managed methods share the same non-name identity key; REA did not guess a token remap.", + "Multiple managed methods share the same CIL and signature; REA did not guess a token remap.", ], }), expect.objectContaining({ @@ -670,6 +675,891 @@ describe("managed member comparison of selected types", () => { }); }); +describe("managed member comparison of one signature in several types", () => { + it("pairs a file with itself when two types hold identical members", () => { + const inspected = withTwin( + inspect(buildManagedPeFixture(), "/tmp/twin.dll"), + ); + const result = compareManagedMembers(inspected, inspected); + + expect(result.matching.ambiguous).toBe(0); + expect(result.summary).toMatchObject({ + unchanged: 4, + changed: 0, + added: 0, + removed: 0, + unknown: 0, + }); + for (const item of [...result.methods, ...result.fields]) + expect(item.right?.token).toBe(item.left?.token); + }); + + it.each([1, 2])( + "keeps a member with its own changed counterpart beside %i identical members of other types", + (twins) => { + const left = inspect(buildManagedPeFixture(), "/tmp/own-before.dll"); + const changed = withTwin( + inspect( + buildManagedPeFixture({ + ilBody: Buffer.from([ + 0x32, 0x02, 0x7b, 0x02, 0x00, 0x00, 0x04, 0x28, 0x02, 0x00, 0x00, + 0x0a, 0x2a, + ]), + }), + "/tmp/own-after.dll", + ), + left, + twins, + ); + const result = compareManagedMembers(left, changed); + + expect(result.methods).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: "changed", + left: expect.objectContaining({ + declaring_type: "Fixture.Program", + }), + right: expect.objectContaining({ + declaring_type: "Fixture.Program", + }), + match: expect.objectContaining({ basis: "exact-signature" }), + dimensions: expect.arrayContaining(["cil"]), + }), + expect.objectContaining({ + status: "added", + right: expect.objectContaining({ declaring_type: "Other.Twin" }), + }), + ]), + ); + expect(result.fields).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: "unchanged", + left: expect.objectContaining({ + declaring_type: "Fixture.Program", + }), + right: expect.objectContaining({ + declaring_type: "Fixture.Program", + }), + }), + expect.objectContaining({ + status: "added", + right: expect.objectContaining({ declaring_type: "Other.Twin" }), + }), + ]), + ); + expect(result.summary).toMatchObject({ unknown: 0, removed: 0 }); + }, + ); + + it("does not remap a member whose own identity remains on the other side", () => { + const left = inspect(buildManagedPeFixture(), "/tmp/sig-before.dll"); + const changed = withTwin( + inspect( + buildManagedPeFixture({ + methodSignature: Buffer.from([0x00, 0x00, 0x08]), + fieldSignature: Buffer.from([0x06, 0x0a]), + }), + "/tmp/sig-after.dll", + ), + left, + ); + // Either side may hold the member whose own identity is left behind. + const forward = compareManagedMembers(left, changed); + const backward = compareManagedMembers(changed, left); + + for (const item of [ + ...forward.methods, + ...forward.fields, + ...backward.methods, + ...backward.fields, + ]) + expect(item.match.status).not.toBe("matched"); + expect(forward.summary).toMatchObject({ removed: 2, added: 4 }); + expect(backward.summary).toMatchObject({ removed: 4, added: 2 }); + }); + + it("does not remap by structural shape a method whose own identity remains on the other side", () => { + const left = inspect(buildManagedPeFixture(), "/tmp/shape-before.dll"); + const changed = withTwin( + inspect( + buildManagedPeFixture({ + methodSignature: Buffer.from([0x00, 0x00, 0x08]), + }), + "/tmp/shape-after.dll", + ), + left, + ); + const [own, twin] = changed.result.methods; + if (own === undefined || twin === undefined) + throw new Error("Expected two methods"); + // Same body shape, different CIL digest: only the structural round links + // the left method to the twin. + const result = compareManagedMembers(left, { + ...changed, + result: { + ...changed.result, + methods: [ + own, + { + ...twin, + body: { ...twin.body, normalized_il_sha256: "0".repeat(64) }, + }, + ], + }, + }); + + for (const item of result.methods) + expect(item.match.status).not.toBe("matched"); + }); +}); + +describe("managed member comparison remaps around kept identities", () => { + it("does not hide a member whose own identity remains in an ambiguous remap group", () => { + const left = inspect(buildManagedPeFixture(), "/tmp/group-before.dll"); + const changed = withTwin( + inspect( + buildManagedPeFixture({ + methodSignature: Buffer.from([0x00, 0x00, 0x08]), + fieldSignature: Buffer.from([0x06, 0x0a]), + }), + "/tmp/group-after.dll", + ), + left, + 2, + ); + const result = compareManagedMembers(left, changed); + + expect(result.matching.ambiguous).toBe(0); + expect(result.summary).toMatchObject({ removed: 2, added: 6 }); + }); + + it("remaps to the one candidate whose own identity is not kept elsewhere", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/kept-base.dll"); + const changedSignature = inspect( + buildManagedPeFixture({ + methodSignature: Buffer.from([0x00, 0x00, 0x08]), + }), + "/tmp/kept-signature.dll", + ); + // Left: Fixture.Program.Main, plus Other.Twin.Main with another signature. + const left = withTwin(base, changedSignature); + // Right: Other.Twin.Main and Other.Twin3.Main, both identical to the + // left Fixture.Program.Main, which the right no longer has. + const twins = withTwin(base, base, 2); + const right = { + ...twins, + result: { ...twins.result, methods: twins.result.methods.slice(1) }, + }; + const result = compareManagedMembers(left, right); + + expect(result.methods).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + left: expect.objectContaining({ declaring_type: "Fixture.Program" }), + right: expect.objectContaining({ declaring_type: "Other.Twin3" }), + match: expect.objectContaining({ + status: "matched", + basis: "exact-il-signature", + }), + }), + ]), + ); + expect( + result.methods.filter(({ match }) => match.status === "ambiguous"), + ).toEqual([]); + }); + + it("names the shared structural shape of an ambiguous remap group", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/shape-group.dll"); + const twins = withTwin(base, base, 2); + // Two right methods share the left method's shape but not its CIL digest, + // and the right no longer declares the left method's identity. + const right = { + ...twins, + result: { + ...twins.result, + methods: twins.result.methods.slice(1).map((method) => ({ + ...method, + body: { ...method.body, normalized_il_sha256: "0".repeat(64) }, + })), + fields: [], + }, + }; + const left = { ...base, result: { ...base.result, fields: [] } }; + const result = compareManagedMembers(left, right); + + expect(result.methods).toEqual([ + expect.objectContaining({ + match: expect.objectContaining({ + status: "ambiguous", + basis: "structural-method-shape", + }), + limitations: [ + "Multiple managed methods share the same structural shape; REA did not guess a token remap.", + ], + }), + ]); + }); +}); + +describe("managed member comparison of unresolved and held identities", () => { + it("does not remap past an unresolved member that may be the counterpart", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/orphan-copy.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const left = { ...base, result: { ...base.result, fields: [] } }; + // The right keeps Main with an unresolved owner, and an identical copy + // in another type; either could be the left Main. + const right = { + ...base, + result: { + ...base.result, + methods: [ + { ...method, declaring_type: null, declaring_type_token: null }, + { + ...method, + declaring_type: "Other.Twin", + declaring_type_token: "0x02000003", + name: "Copy", + token: "0x06000002", + }, + ], + fields: [], + }, + }; + + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) { + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + } + }); +}); + +describe("managed member comparison of raw-signature contention", () => { + it("contests same-signature methods even when their bodies differ", () => { + const base = inspect( + buildManagedPeFixture(), + "/tmp/ownerless-signatures.dll", + ); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const changedBody = (hash: string, stack: number) => ({ + ...method.body, + normalized_il_sha256: hash.repeat(64), + max_stack: stack, + }); + const withOwnerless = { + ...base, + result: { + ...base.result, + fields: [], + methods: [ + { ...method, body: changedBody("a", 1) }, + { + ...method, + token: "0x06000002", + declaring_type: null, + declaring_type_token: null, + body: changedBody("b", 2), + }, + ], + }, + }; + const resolvedOnly = { + ...base, + result: { + ...base.result, + fields: [], + methods: [{ ...method, body: changedBody("c", 3) }], + }, + }; + for (const result of [ + compareManagedMembers(withOwnerless, resolvedOnly), + compareManagedMembers(resolvedOnly, withOwnerless), + ]) { + expect(result.methods).toHaveLength(1); + const [item] = result.methods; + if (item === undefined) + throw new Error("Expected uncertain signature group"); + expect(item).toMatchObject({ + status: "unknown", + match: { status: "ambiguous", basis: "exact-signature" }, + }); + expect(result.summary).toMatchObject({ + added: 0, + removed: 0, + unknown: 1, + }); + expect( + item.match.candidate_left_tokens.length + + item.match.candidate_right_tokens.length, + ).toBe(3); + } + }); + + it("contests ownerless undecoded fields by raw signature", () => { + const base = inspect( + buildManagedPeFixture({ fieldSignature: Buffer.from([0x06, 0x7f]) }), + "/tmp/ownerless-field-signatures.dll", + ); + const [field] = base.result.fields; + if (field === undefined) throw new Error("Expected a fixture field"); + expect(field.signature.parse_status).toBe("unsupported"); + const withOwnerless = { + ...base, + result: { + ...base.result, + methods: [], + fields: [ + field, + { + ...field, + token: "0x04000002", + declaring_type: null, + declaring_type_token: null, + }, + ], + }, + }; + const resolvedOnly = { + ...base, + result: { ...base.result, methods: [], fields: [field] }, + }; + for (const result of [ + compareManagedMembers(withOwnerless, resolvedOnly), + compareManagedMembers(resolvedOnly, withOwnerless), + ]) + expect( + result.fields.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + }); +}); + +describe("managed member comparison of consumed ownerless ambiguity", () => { + it("keeps a consumed ownerless ambiguity from authorizing a later remap", () => { + const base = inspect( + buildManagedPeFixture(), + "/tmp/consumed-ownerless.dll", + ); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const candidate = ( + raw: string, + token: string, + owner: string | null, + name: string, + stack: number, + ) => ({ + ...method, + token, + declaring_type: owner, + declaring_type_token: owner === null ? null : method.declaring_type_token, + name, + signature: { ...method.signature, raw_sha256: raw.repeat(64) }, + body: { + ...method.body, + normalized_il_sha256: raw.repeat(64), + max_stack: stack, + }, + }); + const left = { + ...base, + result: { + ...base.result, + fields: [], + methods: [ + candidate("a", "0x06000001", "T", "M", 1), + candidate("b", "0x06000002", null, "M", 1), + candidate("b", "0x06000003", "D", "D", 2), + ], + }, + }; + const right = { + ...base, + result: { + ...base.result, + fields: [], + methods: [ + candidate("c", "0x06000001", "U", "N", 1), + candidate("b", "0x06000002", null, "M", 1), + candidate("d", "0x06000003", "T", "M", 3), + ], + }, + }; + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) { + expect(result.matching.ambiguous).toBeGreaterThan(0); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + } + }); +}); + +describe("managed member comparison of retained ownership contention", () => { + it("keeps changed counterparts unknown after an ownerless candidate is held", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/held-ownerless.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const body = (hash: string, stack: number) => ({ + ...method.body, + il_sha256: hash.repeat(64), + normalized_il_sha256: hash.repeat(64), + max_stack: stack, + }); + const ownerless = { + ...method, + token: "0x06000002", + declaring_type: null, + declaring_type_token: null, + body: body("b", 2), + }; + const left = { + ...base, + result: { + ...base.result, + fields: [], + methods: [{ ...method, body: body("a", 1) }, ownerless], + }, + }; + const right = { + ...base, + result: { + ...base.result, + fields: [], + methods: [ + { ...method, body: body("c", 3) }, + { + ...method, + token: "0x06000002", + declaring_type: "Other.Twin", + name: "Copy", + body: ownerless.body, + }, + ], + }, + }; + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) { + expect(result.summary).toEqual({ + unchanged: 0, + changed: 0, + added: 0, + removed: 0, + unknown: 2, + }); + expect(result.methods).toEqual([ + expect.objectContaining({ + status: "unknown", + match: { + status: "ambiguous", + basis: "exact-il-signature", + confidence: "unknown", + candidate_left_tokens: ["0x06000002"], + candidate_right_tokens: ["0x06000002"], + }, + }), + expect.objectContaining({ + status: "unknown", + match: { + status: "ambiguous", + basis: "exact-signature", + confidence: "unknown", + candidate_left_tokens: ["0x06000001"], + candidate_right_tokens: ["0x06000001"], + }, + }), + ]); + } + }); +}); + +describe("managed member comparison of ownerless structural competitors", () => { + it("keeps structural competitors of an ownerless method ambiguous in both directions", () => { + const base = inspect( + buildManagedPeFixture(), + "/tmp/ownerless-shape-competitors.dll", + ); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const withSignature = (raw: string, token: string) => ({ + ...method, + token, + signature: { ...method.signature, raw_sha256: raw.repeat(64) }, + }); + const withOwnerless = { + ...base, + result: { + ...base.result, + methods: [ + withSignature("a", "0x06000001"), + { + ...withSignature("b", "0x06000002"), + declaring_type: null, + declaring_type_token: null, + }, + ], + fields: [], + }, + }; + const resolvedOnly = { + ...base, + result: { + ...base.result, + methods: [withSignature("c", "0x06000001")], + fields: [], + }, + }; + for (const result of [ + compareManagedMembers(withOwnerless, resolvedOnly), + compareManagedMembers(resolvedOnly, withOwnerless), + ]) { + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + } + }); +}); + +describe("managed member comparison of ownerless members", () => { + it("keeps an ownerless member ambiguous beside a resolved copy, as main does", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-copy.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + const left = { + ...base, + result: { ...base.result, methods: [ownerless], fields: [] }, + }; + // The right keeps the ownerless Foo and a resolved copy with the same + // CIL and signature; the left Foo's owner may be either one. + const right = { + ...base, + result: { + ...base.result, + methods: [ownerless, { ...method, token: "0x06000002" }], + fields: [], + }, + }; + + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) { + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + } + }); + + it("still pairs a lone ownerless member with its lone counterpart", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-lone.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + const side = { + ...base, + result: { ...base.result, methods: [ownerless], fields: [] }, + }; + + const result = compareManagedMembers(side, side); + + expect(result.matching.ambiguous).toBe(0); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toHaveLength(1); + }); + + it("keeps a resolved copy out of the declared round when an ownerless competitor is on the left, as main does", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-left.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + // The left holds the resolved Foo and an ownerless Foo with the same CIL + // and signature; the right holds only the resolved Foo. The ownerless + // Foo's owner may be the resolved one, so the three stay ambiguous. + const withOwnerless = { + ...base, + result: { ...base.result, methods: [method, ownerless], fields: [] }, + }; + const resolvedOnly = { + ...base, + result: { ...base.result, methods: [method], fields: [] }, + }; + + const result = compareManagedMembers(withOwnerless, resolvedOnly); + + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + }); + + it("keeps a resolved copy out of the declared round when an ownerless competitor is on the right, as main does", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-right.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + const withOwnerless = { + ...base, + result: { ...base.result, methods: [method, ownerless], fields: [] }, + }; + const resolvedOnly = { + ...base, + result: { ...base.result, methods: [method], fields: [] }, + }; + + const result = compareManagedMembers(resolvedOnly, withOwnerless); + + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + }); + + it("keeps an ownerless method ambiguous beside a resolved method of the same shape", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-shape.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + // Raw signature digests differ, so only the decoded shape links them. + const reshaped = (raw: string, token: string) => ({ + signature: { ...method.signature, raw_sha256: raw.repeat(64) }, + token, + }); + const left = { + ...base, + result: { ...base.result, methods: [ownerless], fields: [] }, + }; + const right = { + ...base, + result: { + ...base.result, + methods: [ + { ...ownerless, ...reshaped("e", "0x06000001") }, + { ...method, name: "Bar", ...reshaped("d", "0x06000002") }, + ], + fields: [], + }, + }; + const result = compareManagedMembers(left, right); + + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + }); +}); + +describe("managed member comparison of unresolved owners", () => { + it("does not pair members of unresolved types under different names", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/orphan-names.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const orphan = { declaring_type: null, declaring_type_token: null }; + const side = (name: string) => ({ + ...base, + result: { + ...base.result, + methods: [{ ...method, ...orphan, name }], + fields: [], + }, + }); + const result = compareManagedMembers(side("Main"), side("Renamed")); + + for (const item of result.methods) + expect(item.match.status).not.toBe("matched"); + }); + + it("keeps ownerless members that share a body ambiguous, as main does", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/orphan-self.dll"); + const [method] = base.result.methods; + const [field] = base.result.fields; + if (method === undefined || field === undefined) + throw new Error("Expected a fixture method and field"); + const orphan = { declaring_type: null, declaring_type_token: null }; + const inspected = { + ...base, + result: { + ...base.result, + methods: [ + { ...method, ...orphan, name: "p" }, + { ...method, ...orphan, name: "q", token: "0x06000002" }, + ], + fields: [ + { ...field, ...orphan, name: "a" }, + { ...field, ...orphan, name: "b", token: "0x04000002" }, + ], + }, + }; + const result = compareManagedMembers(inspected, inspected); + + // Their owners are unknown, so `p` may be `q`; names alone cannot tell. + expect(result.matching.ambiguous).toBe(2); + expect( + [...result.methods, ...result.fields].filter( + ({ match }) => match.status === "matched", + ), + ).toEqual([]); + }); + + it("does not remap a member whose own identity is in an ambiguous group", () => { + const undecoded = { methodSignature: Buffer.from([0x00, 0x00, 0x7f]) }; + const decoded = inspect(buildManagedPeFixture(), "/tmp/held-decoded.dll"); + const held = inspect(buildManagedPeFixture(undecoded), "/tmp/held.dll"); + const [undecodedMethod] = held.result.methods; + const [decodedMethod] = decoded.result.methods; + if (undecodedMethod === undefined || decodedMethod === undefined) + throw new Error("Expected fixture methods"); + // The right keeps two identical undecoded Fixture.Program.Main, which + // stay ambiguous, and a copy of the decoded Main in another type. + const left = { + ...held, + result: { + ...held.result, + methods: [{ ...undecodedMethod, token: "0x06000002" }, decodedMethod], + }, + }; + const right = { + ...held, + result: { + ...held.result, + methods: [ + undecodedMethod, + { ...undecodedMethod, token: "0x06000002" }, + { + ...decodedMethod, + declaring_type: "Other.Twin", + declaring_type_token: "0x02000003", + token: "0x06000003", + }, + ], + }, + }; + + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + }); + + it("does not remap a member whose declaring type is unresolved", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/orphan.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const left = { + ...base, + result: { + ...base.result, + methods: [ + { ...method, declaring_type: null, declaring_type_token: null }, + ], + fields: [], + }, + }; + const right = { + ...base, + result: { + ...base.result, + methods: [ + { ...method, declaring_type: "Other.Twin", token: "0x06000002" }, + ], + fields: [], + }, + }; + + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) + for (const item of result.methods) + expect(item.match.status).not.toBe("matched"); + }); +}); + +/** + * Add copies of the fixture's method and field, each pair declared by another + * type, to an inspection. The copies come from `source`, so they can carry + * the bytes of a different build than the inspection they join. + */ +const withTwin = ( + inspected: ReturnType, + source: ReturnType = inspected, + count = 1, +): ReturnType => { + const [method] = source.result.methods; + const [field] = source.result.fields; + if (method === undefined || field === undefined) + throw new Error("Expected a fixture method and field"); + const rows = Array.from({ length: count }, (_, index) => index + 2); + const twin = (row: number) => ({ + declaring_type: row === 2 ? "Other.Twin" : `Other.Twin${String(row)}`, + declaring_type_token: `0x0200000${String(row + 1)}`, + }); + return { + ...inspected, + result: { + ...inspected.result, + methods: [ + ...inspected.result.methods, + ...rows.map((row) => ({ + ...method, + ...twin(row), + token: `0x0600000${String(row)}`, + })), + ], + fields: [ + ...inspected.result.fields, + ...rows.map((row) => ({ + ...field, + ...twin(row), + token: `0x0400000${String(row)}`, + })), + ], + }, + }; +}; + const inspect = (bytes: Buffer, path: string, typeNames?: string[]) => { const target = managedPeFixtureTarget(bytes, path); const result = inspectManagedMembersBytes( diff --git a/tests/boundary/setup/macosHopperPublisherVerification.test.ts b/tests/boundary/setup/macosHopperPublisherVerification.test.ts index 29b696e85a..dcb85cca5e 100644 --- a/tests/boundary/setup/macosHopperPublisherVerification.test.ts +++ b/tests/boundary/setup/macosHopperPublisherVerification.test.ts @@ -6,10 +6,11 @@ import { promisify } from "node:util"; import { describe, expect, it } from "vitest"; import { - HOPPER_PUBLISHER, macHopperInstallDisclosure, systemMacHopperInstallHost, } from "../../../src/application/MacHopper.js"; +import { verifyMacHopperNativeHelper } from "../../../src/hopper/HopperMacPublisher.js"; +import { HOPPER_PUBLISHER } from "../../../src/domain/hopperPublisher.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const run = promisify(execFile); @@ -94,6 +95,24 @@ describe("macOS Hopper publisher verification", () => { ); }); + itDarwin( + "withholds native-helper provenance from custom signed or modified executables", + async () => { + const root = await createTestTempDirectory( + "rea-hopper-helper-publisher-", + ); + const app = await buildBundle(root, "hopper"); + const executable = join(app, ...executableSegments); + await expect( + verifyMacHopperNativeHelper(executable, process.env), + ).resolves.toBe("unknown"); + await appendFile(executable, "modified\n"); + await expect( + verifyMacHopperNativeHelper(executable, process.env), + ).resolves.toBe("unknown"); + }, + ); + // Opt-in positive control: point this at a mounted official Hopper bundle, // for example `hdiutil attach -readonly -nobrowse Hopper-6.6.0-demo.dmg`. // It proves the pinned requirement is satisfiable by the real publisher, so @@ -103,6 +122,12 @@ describe("macOS Hopper publisher verification", () => { "accepts the officially signed Hopper application", async () => { await expect(host().validBundle(realApp ?? "")).resolves.toBe("verified"); + await expect( + verifyMacHopperNativeHelper( + join(realApp ?? "", ...executableSegments), + process.env, + ), + ).resolves.toBe("verified"); }, ); }); diff --git a/tests/composition/analysis-sessions/powerpc-identity.test.ts b/tests/composition/analysis-sessions/powerpc-identity.test.ts new file mode 100644 index 0000000000..3dd1d374a2 --- /dev/null +++ b/tests/composition/analysis-sessions/powerpc-identity.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import type { BinaryTarget } from "../../../src/domain/binaryTargetTypes.js"; +import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; +import { + createAnalysisSnapshotEntry, + parseAnalysisSnapshot, + serializeAnalysisSnapshot, + snapshotBinding, + snapshotTarget, + snapshotMatchesProfile, +} from "../../../src/domain/analysisSnapshot.js"; +import { sessionOutputSchemas } from "../../../src/contracts/toolOutputSchemaGroups.js"; + +const target: BinaryTarget = { + path: "/source-owned.elf", + sha256: "a".repeat(64), + kind: "executable", + format: "elf", + architecture: "powerpc", + availableArchitectures: ["powerpc"], + powerpc: { elfClass: 32, byteOrder: "big", type: 2, flags: 0 }, +}; +const provider = { id: "ghidra", name: "Ghidra", version: "12.1.4" }; +const profile = createAnalysisProfile(provider, { + powerpc_elf: { elf_class: 32, byte_order: "big", type: 2, flags: 0 }, +}); + +describe("PowerPC identity through lifecycle, Evidence and snapshot consumers", () => { + it("round-trips query bindings without relaxing integrity", () => { + const result = [{ address: "0x100101b0", value: "rea_ppc_entry" }]; + const parameters = {}; + const evidence = createEvidence(target, provider, { + operation: "list_procedures", + parameters, + result, + analysisProfile: profile, + }); + const binding = snapshotBinding(profile); + const entry = createAnalysisSnapshotEntry({ + target: snapshotTarget(target), + binding, + operation: "list_procedures", + parameters, + execution: { + result, + rawResult: null, + provider, + analysisProfile: profile, + limitations: [], + locations: [], + subject: target, + }, + }); + const snapshot = { + target: snapshotTarget(target), + binding, + entries: [entry], + workflow_entries: [], + evidence_bundle: createEvidenceBundle([evidence]), + }; + const restored = parseAnalysisSnapshot( + JSON.parse(serializeAnalysisSnapshot(snapshot)), + ); + expect(restored.target.architecture).toBe("powerpc"); + expect(restored.evidence_bundle.records[0]?.subject?.architecture).toBe( + "powerpc", + ); + expect(restored.entries[0]?.execution.subject?.architecture).toBe( + "powerpc", + ); + expect( + sessionOutputSchemas.open_binary.parse({ result: target }).result + .architecture, + ).toBe("powerpc"); + expect(() => + parseEvidence({ + ...evidence, + subject: { ...evidence.subject, architecture: "x86" }, + }), + ).toThrow("semantic identifier"); + const changed = createAnalysisProfile(provider, { + powerpc_elf: { + elf_class: 32, + byte_order: "big", + type: 2, + flags: 0x00010000, + }, + }); + expect(snapshotMatchesProfile(binding, changed)).toBe(false); + }); +}); diff --git a/tests/conformance/c/ppc.c b/tests/conformance/c/ppc.c new file mode 100644 index 0000000000..3f62ca57e6 --- /dev/null +++ b/tests/conformance/c/ppc.c @@ -0,0 +1,14 @@ +/* Freestanding, source-owned static-analysis fixture. Never executed by REA. */ +volatile unsigned int rea_ppc_global = 7; +const char rea_ppc_marker[] = "rea-ppc-source-owned-fixture"; + +__attribute__((noinline)) unsigned int rea_ppc_leaf(unsigned int value) +{ + return value * 3u + rea_ppc_global; +} + +__attribute__((noinline)) unsigned int rea_ppc_entry(unsigned int selector) +{ + unsigned int adjusted = selector > 10u ? selector : selector + 1u; + return rea_ppc_leaf(adjusted) + (unsigned char)rea_ppc_marker[0]; +} diff --git a/tests/conformance/ghidra/ppc-checks.d.mts b/tests/conformance/ghidra/ppc-checks.d.mts new file mode 100644 index 0000000000..38609de454 --- /dev/null +++ b/tests/conformance/ghidra/ppc-checks.d.mts @@ -0,0 +1,29 @@ +import type { PowerPcElfMetadata } from "../../../src/domain/binaryTargetTypes.js"; + +export interface PowerPcReadelfFacts { + powerpc: PowerPcElfMetadata; + symbols: Record< + | "rea_ppc_entry" + | "rea_ppc_leaf" + | "rea_ppc_probe" + | "rea_ppc_global" + | "rea_ppc_marker", + string + >; +} +export function parsePowerPcReadelf(text: string): PowerPcReadelfFacts; +export function inspectPowerPcReadelf( + path: string, +): Promise< + PowerPcReadelfFacts & { command: string; version: string; raw: string } +>; +export function assertPowerPcLoadedImage( + observations: unknown, + sha256: string, +): void; +export function assertPowerPcProbe( + move: unknown, + branch: unknown, + address: string, +): void; +export function assertPowerPcGlobal(bytes: string): void; diff --git a/tests/conformance/ghidra/ppc-checks.mjs b/tests/conformance/ghidra/ppc-checks.mjs new file mode 100644 index 0000000000..1136f2602e --- /dev/null +++ b/tests/conformance/ghidra/ppc-checks.mjs @@ -0,0 +1,132 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execute = promisify(execFile); +// The lane binds this language; Ghidra's ELF loader alone selects e500. +const LANGUAGE = "PowerPC:BE:32:default"; +const wordHex = (value) => { + const bytes = Buffer.alloc(4); + bytes.writeUInt32BE(value); + return bytes.toString("hex"); +}; + +/** Parse only the documented GNU readelf fields needed by our tiny fixture. */ +export function parsePowerPcReadelf(text) { + const field = (expression, label) => { + const matches = [...text.matchAll(expression)]; + assert.equal(matches.length, 1, `Missing or ambiguous readelf ${label}`); + return matches[0][1].trim(); + }; + assert.equal(field(/^\s*Class:\s*(.+)$/gmu, "class"), "ELF32"); + assert.equal( + field(/^\s*Data:.*?,\s*(little|big) endian\s*$/gmu, "byte order"), + "big", + ); + assert.match(field(/^\s*Type:\s*(.+)$/gmu, "type"), /^EXEC\b/u); + assert.equal(field(/^\s*Machine:\s*(.+)$/gmu, "machine"), "PowerPC"); + const flags = Number(field(/^\s*Flags:\s*(0x[\da-f]+)\s*$/gmu, "flags")); + assert.equal(flags, 0, "Fixture ELF flags differ from build intent"); + const symbols = {}; + for (const name of [ + "rea_ppc_entry", + "rea_ppc_leaf", + "rea_ppc_probe", + "rea_ppc_global", + "rea_ppc_marker", + ]) { + const value = field( + new RegExp( + `^\\s*\\d+:\\s+([\\da-f]+)\\s+\\d+\\s+(?:FUNC|OBJECT)\\s+GLOBAL\\s+\\S+\\s+\\d+\\s+${name}\\s*$`, + "gmu", + ), + `symbol ${name}`, + ); + symbols[name] = `0x${BigInt(`0x${value}`).toString(16)}`; + } + return { + powerpc: { elfClass: 32, byteOrder: "big", type: 2, flags }, + symbols, + }; +} + +/** Independent reader is a bounded test prerequisite, never an REA provider. */ +export async function inspectPowerPcReadelf(path) { + const command = process.env.REA_PPC_READELF ?? "readelf"; + const options = { + env: { ...process.env, LC_ALL: "C" }, + timeout: 10000, + maxBuffer: 1024 * 1024, + }; + const version = await execute(command, ["--version"], options); + assert.match( + version.stdout, + /^GNU readelf\b/u, + "This lane requires GNU readelf", + ); + const result = await execute(command, ["-h", "-s", "-W", path], options); + assert.equal( + result.stderr.trim(), + "", + `readelf diagnostics: ${result.stderr}`, + ); + return { + ...parsePowerPcReadelf(result.stdout), + command, + version: version.stdout.split("\n")[0], + raw: result.stdout, + }; +} + +/** Check effective Ghidra observations, not the requested analysis profile. */ +export function assertPowerPcLoadedImage(observations, sha256) { + assert.equal(observations.language_id, LANGUAGE); + assert.equal(observations.compiler_spec_id, "default"); + assert.ok( + observations.source_files.some( + (file) => + file.original_sha256 === sha256 && file.modified_sha256 === sha256, + ), + "Ghidra loaded source identity changed", + ); +} + +/** Verify the exact constant/conditional destination in ppc-probe.S. */ +export function assertPowerPcProbe(move, branch, address) { + const start = BigInt(address); + for (const [instruction, offset, word] of [ + [move, 0n, 0x38641234], + [branch, 8n, 0x41820008], + ]) { + assert.equal(instruction.status, "decoded"); + assert.equal(BigInt(instruction.address), start + offset); + assert.equal(instruction.architecture, LANGUAGE); + assert.equal(instruction.length, 4); + assert.equal(instruction.bytes, wordHex(word)); + } + assert.match(move.mnemonic, /^ADDI$/iu); + assert.ok( + move.operands.some((operand) => + operand.components.some( + (part) => + part.kind === "immediate" && + part.value !== null && + BigInt(part.value) === 0x1234n, + ), + ), + "ADDI immediate was not recovered", + ); + assert.match(branch.mnemonic, /^BEQ$/iu); + assert.equal(branch.flow.kind, "jump"); + assert.equal(branch.flow.conditional, true); + assert.equal(branch.flow.computed, false); + assert.deepEqual( + branch.flow.direct_destinations.map((value) => BigInt(value)), + [start + 16n], + ); +} + +/** Four bytes must encode the known source-owned global in big-endian order. */ +export function assertPowerPcGlobal(bytes) { + assert.equal(bytes, wordHex(7)); +} diff --git a/tests/conformance/ghidra/ppc-checks.test.ts b/tests/conformance/ghidra/ppc-checks.test.ts new file mode 100644 index 0000000000..6328ce0292 --- /dev/null +++ b/tests/conformance/ghidra/ppc-checks.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it } from "vitest"; +import { + assertPowerPcGlobal, + assertPowerPcLoadedImage, + assertPowerPcProbe, + parsePowerPcReadelf, +} from "./ppc-checks.mjs"; + +// GNU readelf 2.47 output for the source-owned Clang 23 / LLD fixture. +// This tests the reader boundary, not Ghidra; the real lane rebuilds it. +const readelf = `ELF Header: + Magic: 7f 45 4c 46 01 02 01 00 00 00 00 00 00 00 00 00 + Class: ELF32 + Data: 2's complement, big endian + Version: 1 (current) + OS/ABI: UNIX - System V + ABI Version: 0 + Type: EXEC (Executable file) + Machine: PowerPC + Version: 0x1 + Entry point address: 0x100101b0 + Start of program headers: 52 (bytes into file) + Start of section headers: 2188 (bytes into file) + Flags: 0x0 + Size of this header: 52 (bytes) + Size of program headers: 32 (bytes) + Number of program headers: 6 + Size of section headers: 40 (bytes) + Number of section headers: 18 + Section header string table index: 16 + +Symbol table '.symtab' contains 7 entries: + Num: Value Size Type Bind Vis Ndx Name + 0: 00000000 0 NOTYPE LOCAL DEFAULT UND + 1: 00000000 0 FILE LOCAL DEFAULT ABS ppc.c + 2: 10010180 48 FUNC GLOBAL DEFAULT 4 rea_ppc_leaf + 3: 1002023c 4 OBJECT GLOBAL DEFAULT 5 rea_ppc_global + 4: 100101b0 116 FUNC GLOBAL DEFAULT 4 rea_ppc_entry + 5: 100000f4 29 OBJECT GLOBAL DEFAULT 1 rea_ppc_marker + 6: 10010224 24 FUNC GLOBAL DEFAULT 4 rea_ppc_probe +`; +const sha256 = "a".repeat(64); +const image = { + language_id: "PowerPC:BE:32:default", + compiler_spec_id: "default", + source_files: [{ original_sha256: sha256, modified_sha256: sha256 }], +}; +const move = { + address: "0x10010224", + status: "decoded", + architecture: "PowerPC:BE:32:default", + length: 4, + bytes: "38641234", + mnemonic: "addi", + operands: [ + { index: 2, components: [{ kind: "immediate", value: "0x1234" }] }, + ], +}; +const branch = { + address: "0x1001022c", + status: "decoded", + architecture: "PowerPC:BE:32:default", + length: 4, + bytes: "41820008", + mnemonic: "beq", + flow: { + kind: "jump", + conditional: true, + computed: false, + direct_destinations: ["0x10010234"], + }, +}; + +describe("PowerPC real-verifier independent expectations", () => { + it("reads GNU header and defined symbols without a Ghidra-derived oracle", () => { + const parsed = parsePowerPcReadelf(readelf); + expect(parsed.powerpc).toEqual({ + elfClass: 32, + byteOrder: "big", + type: 2, + flags: 0, + }); + expect(parsed.symbols.rea_ppc_probe).toBe("0x10010224"); + }); + + it.each([ + ["wrong class", "ELF32", "ELF64"], + ["wrong byte order", "big endian", "little endian"], + [ + "wrong machine", + "Machine: PowerPC", + "Machine: PowerPC64", + ], + ["wrong object type", "EXEC (Executable file)", "DYN (Shared object file)"], + [ + "wrong flags", + "Flags: 0x0", + "Flags: 0x80000000", + ], + ["missing symbol", "rea_ppc_probe", "unrelated_probe"], + ])( + "rejects %s instead of returning a guessed fixture interpretation", + (_name, from, to) => { + expect(() => parsePowerPcReadelf(readelf.replace(from, to))).toThrow(); + }, + ); + + it("rejects ambiguous producer fields", () => { + expect(() => parsePowerPcReadelf(`${readelf}\nClass: ELF32\n`)).toThrow( + /ambiguous/u, + ); + }); + + it("checks the effective loaded image, including Ghidra's e500 default", () => { + assertPowerPcLoadedImage(image, sha256); + expect(() => + assertPowerPcLoadedImage( + { ...image, language_id: "PowerPC:BE:32:e500" }, + sha256, + ), + ).toThrow(); + expect(() => + assertPowerPcLoadedImage( + { ...image, compiler_spec_id: "macosx" }, + sha256, + ), + ).toThrow(); + expect(() => + assertPowerPcLoadedImage( + { + ...image, + source_files: [ + { original_sha256: sha256, modified_sha256: "b".repeat(64) }, + ], + }, + sha256, + ), + ).toThrow(); + }); + + it("checks exact probe bytes, immediate and conditional destination", () => { + assertPowerPcProbe(move, branch, "0x10010224"); + }); + + it("rejects plausible nonempty but incorrect instruction observations", () => { + expect(() => + assertPowerPcProbe({ ...move, bytes: "38641235" }, branch, "0x10010224"), + ).toThrow(); + expect(() => + assertPowerPcProbe({ ...move, length: 2 }, branch, "0x10010224"), + ).toThrow(); + expect(() => + assertPowerPcProbe( + { + ...move, + operands: [ + { index: 2, components: [{ kind: "immediate", value: "0x1235" }] }, + ], + }, + branch, + "0x10010224", + ), + ).toThrow(); + expect(() => + assertPowerPcProbe( + move, + { + ...branch, + flow: { ...branch.flow, direct_destinations: ["0x10010230"] }, + }, + "0x10010224", + ), + ).toThrow(); + expect(() => + assertPowerPcProbe( + move, + { ...branch, flow: { ...branch.flow, conditional: false } }, + "0x10010224", + ), + ).toThrow(); + }); + + it("checks the big-endian global word", () => { + assertPowerPcGlobal("00000007"); + expect(() => assertPowerPcGlobal("07000000")).toThrow(); + }); +}); diff --git a/tests/conformance/ghidra/ppc-fixture.mjs b/tests/conformance/ghidra/ppc-fixture.mjs new file mode 100644 index 0000000000..202c0dacf6 --- /dev/null +++ b/tests/conformance/ghidra/ppc-fixture.mjs @@ -0,0 +1,62 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const execute = promisify(execFile); + +/** Build a source-owned freestanding ELF; never execute its target code. */ +export async function buildPowerPcFixture(directory) { + const compiler = process.env.REA_PPC_CLANG ?? "clang"; + const target = "powerpc-unknown-linux-gnu"; + const source = fileURLToPath(new URL("../c/ppc.c", import.meta.url)); + const probe = fileURLToPath(new URL("./ppc-probe.S", import.meta.url)); + const path = join(directory, `${target}.elf`); + const args = [ + `--target=${target}`, + // A 750-family core with the classic FPU, unlike the e500 SPE variant. + "-mcpu=750", + "-fno-pic", + "-fuse-ld=lld", + "-nostdlib", + "-static", + "-O0", + "-g", + "-Wl,-e,rea_ppc_entry", + "-Wl,--build-id=none", + source, + probe, + "-o", + path, + ]; + const version = await execute(compiler, ["--version"], { timeout: 10000 }); + await execute(compiler, args, { timeout: 60000 }); + const bytes = await readFile(path); + if ( + bytes.length < 52 || + bytes.toString("hex", 0, 4) !== "7f454c46" || + bytes[4] !== 1 || + bytes[5] !== 2 || + bytes.readUInt16BE(16) !== 2 || + bytes.readUInt16BE(18) !== 20 || + bytes.readUInt32BE(36) !== 0 + ) + throw new Error( + "Compiler did not produce the declared ELF32 big-endian PowerPC fixture", + ); + return { + path, + flags: bytes.readUInt32BE(36), + sha256: createHash("sha256").update(bytes).digest("hex"), + source_sha256: createHash("sha256") + .update(await readFile(source)) + .digest("hex"), + probe_sha256: createHash("sha256") + .update(await readFile(probe)) + .digest("hex"), + compiler: version.stdout.trim(), + arguments: args, + }; +} diff --git a/tests/conformance/ghidra/ppc-probe.S b/tests/conformance/ghidra/ppc-probe.S new file mode 100644 index 0000000000..f67793942b --- /dev/null +++ b/tests/conformance/ghidra/ppc-probe.S @@ -0,0 +1,17 @@ +/* Source-owned 32-bit PowerPC instructions with fixed, reviewable encodings. + * No target execution. The conditional branch at +8 targets +16. + */ +.text +.p2align 2 +.globl rea_ppc_probe +.type rea_ppc_probe, @function +rea_ppc_probe: + addi 3, 4, 0x1234 + cmpwi 0, 4, 0 + beq 0, .Lzero + blr +.Lzero: + li 3, 7 + blr +.size rea_ppc_probe, .-rea_ppc_probe +.section .note.GNU-stack,"",@progbits diff --git a/tests/fixtures/hopperLaunchAdmission.mjs b/tests/fixtures/hopperLaunchAdmission.mjs index f7c81d9bfa..6a5033d6b9 100644 --- a/tests/fixtures/hopperLaunchAdmission.mjs +++ b/tests/fixtures/hopperLaunchAdmission.mjs @@ -19,6 +19,13 @@ if (mode === "denied" || mode === "denied-after-handoff") { process.exit(2); }, ); +} else if (mode === "not-found") { + // A custom wrapper can forward the native diagnostic after a handoff. + await writeFile(`${receiptPath}.document-created`, "unconfirmed"); + process.stderr.write( + "Hopper has not been found.\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable.\n", + () => process.exit(3), + ); } else if (mode === "handoff") { process.exit(0); } else if (mode === "bridge") { diff --git a/turbo.json b/turbo.json index ffee2a0704..35e7ece044 100644 --- a/turbo.json +++ b/turbo.json @@ -63,9 +63,7 @@ "!docs/.vitepress/dist/**", "!website/_site/**", "!experiments/analysis-efficiency/results/**", - "vitest.reverse-tools.config.ts", - "vitest.reverse-tools-semantics.config.ts", - "vitest.config.ts" + "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}" ] }, "lint:built": { @@ -86,9 +84,7 @@ "!docs/.vitepress/dist/**", "!website/_site/**", "!experiments/analysis-efficiency/results/**", - "vitest.reverse-tools.config.ts", - "vitest.reverse-tools-semantics.config.ts", - "vitest.config.ts" + "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}" ] }, "format:check": {}, @@ -104,8 +100,8 @@ "tests/**/*.ts", "tsconfig.json", "tsconfig.build.json", - "vitest.config.ts", - "docs/.vitepress/config.ts" + "docs/.vitepress/config.ts", + "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}" ] }, "jscpd": { diff --git a/vitest.config.ts b/vitest.config.ts index a8bf6b0454..2c034beb2a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,6 +3,10 @@ import { availableParallelism, tmpdir } from "node:os"; import { join } from "node:path"; import { defineConfig } from "vitest/config"; +import { + BUILT_TEST_PROJECTS, + SOURCE_TEST_PROJECTS, +} from "./scripts/lib/test-projects.mjs"; import { CiTestSequencer } from "./tests/ciTestSequencer.js"; @@ -28,57 +32,32 @@ const COVERAGE_SHARD = process.argv.some((argument) => // Keep local and CI concurrency bounded by the runner rather than command locks. const MAX_TEST_WORKERS = Math.min(2, availableParallelism()); -const TEST_PROJECTS = [ - { - name: "domain", - include: ["src/{contracts,domain}/**/*.test.ts"], +const TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]; +const PROJECT_POLICIES = { + domain: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Domain and contract modules are pure and own no process-global state, so // per-file module isolation adds startup cost without protecting a seam. isolate: false, }, - { - name: "services", - include: ["src/application/**/*.test.ts"], + services: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Service tests use recording ports and own no process-global state. Share // their module graph so file startup does not dominate the lane. isolate: false, }, - { - name: "adapters", - include: [ - "src/*.test.ts", - "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", - ], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "composition", - include: ["tests/composition/**/*.test.ts"], + adapters: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + composition: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Composition tests use provider-neutral sessions and test-scoped cleanup; // sharing their immutable module graph avoids per-file startup work. isolate: false, }, - { - name: "boundary", - include: ["tests/boundary/**/*.test.ts"], - exclude: [ - "tests/boundary/cli/**/*.test.ts", - "tests/boundary/mcp/**/*.test.ts", - "tests/boundary/process/**/*.test.ts", - ], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "cli-boundary", - include: ["tests/boundary/cli/**/*.test.ts"], + boundary: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + "cli-boundary": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS, // CLI tests create fresh command instances or subprocesses and own their @@ -88,54 +67,36 @@ const TEST_PROJECTS = [ // before their two heaps compete with the real CLI child processes. execArgv: ["--max-old-space-size=384"], }, - { - // Process-tree observations share host sampling resources; keep this small - // lane serial while the independent projects run together. - name: "process-boundary", - include: ["tests/boundary/process/**/*.test.ts"], + "process-boundary": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS, + // Process-tree observations share host sampling resources; keep this small + // lane serial while the independent projects run together. fileParallelism: false, sequence: { groupOrder: 1 }, }, - { - name: "mcp-boundary", - include: ["tests/boundary/mcp/**/*.test.ts"], + "mcp-boundary": { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Each file creates and closes an explicit in-memory MCP session. Reuse the // immutable server graph so module startup does not outweigh transport work. isolate: false, }, - { - name: "acceptance", - include: ["tests/acceptance/**/*.test.ts"], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "process-global", - include: ["tests/process-global/**/*.test.ts"], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "conformance", - include: ["tests/conformance/**/*.test.ts"], + acceptance: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + "process-global": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + conformance: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Contract cases use recording providers and own no process-global state. isolate: false, }, - { - name: "evaluation", - include: ["tests/evaluation/**/*.test.ts"], + evaluation: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Evaluator tests inspect deterministic inputs without mutating shared state. isolate: false, }, -]; +}; const ZERO_COVERAGE_THRESHOLDS = { statements: 0, @@ -156,10 +117,21 @@ const ZERO_COVERAGE_THRESHOLDS = { }, }; -const projects = TEST_PROJECTS.map((project) => ({ - extends: true as const, - test: project, -})); +const projects = TEST_PROJECTS.map((project) => { + const policy = + PROJECT_POLICIES[project.name as keyof typeof PROJECT_POLICIES]; + if (!policy) + throw new Error(`Missing test runner policy for ${project.name}`); + return { + extends: true as const, + test: { + name: project.name, + include: [...project.include], + ...(project.exclude ? { exclude: [...project.exclude] } : {}), + ...policy, + }, + }; +}); export default defineConfig({ test: {