From 1342eddd03ff4f50cf87484d36fcd72b698d80af Mon Sep 17 00:00:00 2001 From: morconsultoriadigital-afk Date: Tue, 13 Oct 2026 18:45:41 -0300 Subject: [PATCH 01/14] test(analysis-view): order the message-cutoff fixture lexicographically (#3018) boundedViewMessages returns uniqueSorted output, so the 100-message fixture came back in lexicographic order ("message 59", "message 6", "message 60") while the assertion compared against generation order. Pad the index to a fixed width so both orders coincide and the test checks what it means to: that no count-based cutoff applies below the byte budget. Co-authored-by: Claude Opus 5 --- src/domain/analysisView/analysisView.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index 3231a2da94..3c14636d72 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -957,7 +957,7 @@ it("keeps the complete unique set when it fits the byte budget", () => { it("does not impose a message-count cutoff below the byte budget", () => { const messages = Array.from( { length: 100 }, - (_, index) => `message ${index}`, + (_, index) => `message ${String(index).padStart(2, "0")}`, ); expect( boundedViewMessages(messages, `ev_${"a".repeat(64)}`, "limitations"), From aa6f74af04c06d3d09ef69d2292f05f74e93cf1b Mon Sep 17 00:00:00 2001 From: Jeremy Levartovsky <140487034+JayOfTheKeyboard@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:48:05 +1100 Subject: [PATCH 02/14] fix(setup): configure a comment-only Hermes config.yaml (#2096) * fix(setup): configure a comment-only Hermes config.yaml comments, or only a `---` marker, also parses to a null YAML document. Setup still rejected it as malformed ("expected record, received null") and reported Hermes as invalid. Treat a null Hermes document as holding no settings, as #1983 does for an empty file. The serializer already edits a comment-only document in place. A bare `---` document holds a null scalar that takes no keys, so replace it with a map and keep the comments written after the marker. * fix(setup): keep rejecting an explicit null Hermes root Only a document with no root, or a bare --- marker whose null scalar has no source text, holds no settings. An explicit null or ~ is a value, so setup and uninstall reject it as before, the same as any other non-map root. * test(setup): split the Hermes empty-configuration tests under the function length limit The Hermes YAML registration block reached 159 lines, past oxlint's max-lines-per-function limit of 150. Move the empty-document cases and the multiple-document rejection into their own block. * fix(setup): type the Hermes edit document as a plain Document parseDocument with options returns a Document.Parsed, whose contents accept only parsed nodes, so assigning the new map for a bare --- root failed typecheck (TS2322). Annotate the edited document as Document, as the code did before it moved into HermesConfigurationYaml.ts. * fix(ci): isolate Linux Hopper process fixture inventory --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/application/HermesConfigurationYaml.ts | 38 ++++++++-- .../hermesClientConfiguration.test.ts | 74 +++++++++++++++---- 2 files changed, 94 insertions(+), 18 deletions(-) diff --git a/src/application/HermesConfigurationYaml.ts b/src/application/HermesConfigurationYaml.ts index 8d245da37e..61eb0dfdef 100644 --- a/src/application/HermesConfigurationYaml.ts +++ b/src/application/HermesConfigurationYaml.ts @@ -1,7 +1,7 @@ import { isScalar, - parse, parseDocument, + type Document, type ParsedNode, type ScalarTag, type Tags, @@ -56,9 +56,28 @@ const HERMES_YAML_OPTIONS = { a.source === b.source), }; -/** Parse a Hermes `config.yaml`. */ -export const parseHermesConfigurationYaml = (text: string): unknown => - parse(text, HERMES_YAML_OPTIONS); +// A comment-only document has no root; a bare marker has an unwritten null +// scalar. Explicit null values, tags and anchors remain actual values. +const isEmptyYamlRoot = (root: Document["contents"]): boolean => + root === null || + (isScalar(root) && + root.value === null && + root.source === "" && + root.tag === undefined && + root.anchor === undefined); + +const parseHermesDocument = (text: string) => { + const document = parseDocument(text, HERMES_YAML_OPTIONS); + const error = document.errors[0]; + if (error !== undefined) throw error; + return document; +}; + +/** Parse a Hermes `config.yaml`, preserving the distinction between no root and explicit null. */ +export const parseHermesConfigurationYaml = (text: string): unknown => { + const document = parseHermesDocument(text); + return isEmptyYamlRoot(document.contents) ? {} : document.toJS(); +}; /** Set or delete one value in a Hermes `config.yaml`, keeping the rest as written. */ export const editHermesConfigurationYaml = ( @@ -66,7 +85,16 @@ export const editHermesConfigurationYaml = ( path: readonly string[], value: unknown, ): string => { - const document = parseDocument(text, HERMES_YAML_OPTIONS); + // A plain Document, so a bare `---` root can be replaced by a new map. + const document: Document = parseHermesDocument(text); + const root = document.contents; + if (isScalar(root) && isEmptyYamlRoot(root)) { + const map = document.createNode({}); + map.commentBefore = [root.commentBefore, root.comment] + .filter((comment) => comment !== undefined) + .join("\n"); + document.contents = map; + } if (value === undefined) document.deleteIn(path); else document.setIn(path, value); // Editing REA must not reflow unrelated user prompts or other YAML scalars. diff --git a/tests/boundary/filesystem/hermesClientConfiguration.test.ts b/tests/boundary/filesystem/hermesClientConfiguration.test.ts index 218573d5c4..7bd72a6767 100644 --- a/tests/boundary/filesystem/hermesClientConfiguration.test.ts +++ b/tests/boundary/filesystem/hermesClientConfiguration.test.ts @@ -114,18 +114,34 @@ describe("Hermes YAML shape diagnostics", () => { "!!omap [{rea: {command: rea, args: [mcp]}}]", "!!timestamp 2001-12-15", "!!binary cmVh", - ].flatMap((value) => [ - { - original: `${value}\n`, - problem: "The configuration root must be an object", - }, - { - original: `mcp_servers: ${value}\n`, - problem: '"mcp_servers" must be an object', - }, - ]), + ] + .flatMap((value) => [ + { + original: `${value}\n`, + problem: "The configuration root must be an object", + }, + { + original: `mcp_servers: ${value}\n`, + problem: '"mcp_servers" must be an object', + }, + ]) + .concat( + // An explicit null root is a value, not an empty file. + [ + "null\n", + "~\n", + "# Hermes configuration\nnull\n", + "--- ~\n", + "--- !!null\n", + '!!null ""\n', + "--- &empty\n", + ].map((original) => ({ + original, + problem: "The configuration root must be an object", + })), + ), )( - "blocks setup and uninstall before changes for tagged non-record YAML: $original", + "blocks setup and uninstall before changes for tagged or null non-record YAML: $original", async ({ original, problem }) => { const { home, client } = await hermesClient(); const cursor = join(home, ".cursor", "mcp.json"); @@ -281,7 +297,36 @@ mcp_servers: }); }); - it.each(["", "\n"])( + // Comments and a bare document marker parse to a null document, like an + // empty file, and are kept ahead of the new registration. +}); + +describe("Hermes YAML empty configurations", () => { + it("rejects multiple YAML documents before treating an empty first root as absent", async () => { + const { client } = await hermesClient(); + const original = + "---\n---\nmcp_servers:\n rea:\n command: rea\n args: [mcp]\n"; + await writeFile(client.configPath, original); + expect(await inspectClientConfiguration(client, {}, command)).toMatchObject( + { status: "invalid" }, + ); + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "failed", + reason: "readback", + }); + expect(await readFile(client.configPath, "utf8")).toBe(original); + await expect( + readFile(`${client.configPath}.rea.backup`, "utf8"), + ).rejects.toThrow(); + }); + + it.each([ + "", + "\n", + "# Hermes configuration\n", + "---\n", + "---\n# Hermes configuration\n", + ])( "configures an empty config.yaml %j as a new document", async (original) => { const { home, client } = await hermesClient(); @@ -289,7 +334,10 @@ mcp_servers: expect( (await configureClientConfiguration(client, {}, command)).status, ).toBe("configured"); - expect(parseYaml(await readFile(client.configPath, "utf8"))).toEqual({ + const text = await readFile(client.configPath, "utf8"); + const kept = original.trimEnd(); + expect(text.slice(0, kept.length)).toBe(kept); + expect(parseYaml(text)).toEqual({ mcp_servers: { rea: { command: command[0], args: command.slice(1), enabled: true }, }, From 6510d0d57f9d37da866fdf4fdf9d3146f974fd28 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:07:08 +0800 Subject: [PATCH 03/14] perf(ci): start source feedback early and diagnose stalled runs (#3013) * perf(ci): start source feedback early and diagnose stalled runs * fix(ci): verify workflow identities before main comparison limits * fix(ci): preserve diagnostics across passing checks and install failures * fix(ci): preserve ambiguous workflow run ordering * fix(ci): preserve failure-report candidate and request budgets * fix(ci): retain candidate errors while checking later main evidence * fix(ci): retain partial reports and require complete health scans * test(ci): prune stale fixtures and duplicated assumptions --- .github/workflows/ci-timings.yml | 40 +- .github/workflows/ci.yml | 73 +- .github/workflows/real-cutter.yml | 12 +- docs/testing.md | 35 +- package.json | 7 +- scripts/ci/artifacts.d.mts | 67 ++ scripts/ci/artifacts.mjs | 436 +++++++++++ scripts/ci/failure-report.d.mts | 159 ++++ scripts/ci/failure-report.mjs | 529 +++++++++++++ scripts/ci/github-api.d.mts | 16 + scripts/ci/github-api.mjs | 33 + scripts/ci/job-timings-batch.mjs | 3 + scripts/ci/pr-ci-health.d.mts | 95 +++ scripts/ci/pr-ci-health.mjs | 718 ++++++++++++++++++ scripts/ci/qualification.mjs | 7 +- scripts/ci/required.mjs | 6 +- scripts/ci/static-check-report.mjs | 2 + scripts/ci/static-task-context.d.mts | 18 + scripts/ci/static-task-context.mjs | 104 +++ scripts/ci/test-group.mjs | 46 ++ scripts/ci/test-history.mjs | 7 +- scripts/ci/test-report.mjs | 26 +- scripts/ci/workflow-run.d.mts | 10 + scripts/ci/workflow-run.mjs | 15 + scripts/lib/development-tests.mjs | 31 +- scripts/lib/test-projects.d.mts | 10 + scripts/lib/test-projects.mjs | 107 +++ src/domain/analysisView/analysisView.test.ts | 2 +- tests/boundary/cli/developmentTests.test.ts | 1 + .../filesystem/ciFailureReport.test.ts | 419 ++++++++++ .../filesystem/ciFailureReportBudget.test.ts | 290 +++++++ tests/boundary/filesystem/ciFeedback.test.ts | 313 ++++++++ .../filesystem/ciJobTimingsBatch.test.ts | 91 ++- tests/boundary/filesystem/ciPrHealth.test.ts | 617 +++++++++++++++ .../filesystem/ciQualification.test.ts | 2 +- .../filesystem/ciRequiredLanes.test.ts | 9 +- .../filesystem/ciStaticArtifacts.test.ts | 99 +++ .../boundary/filesystem/ciTestHistory.test.ts | 2 +- .../filesystem/ciTestProjects.test.ts | 75 ++ .../boundary/filesystem/ciTestReports.test.ts | 155 +++- .../boundary/filesystem/ciWorkflowRun.test.ts | 54 ++ turbo.json | 12 +- vitest.config.ts | 100 +-- 43 files changed, 4661 insertions(+), 192 deletions(-) create mode 100644 scripts/ci/artifacts.d.mts create mode 100644 scripts/ci/artifacts.mjs create mode 100644 scripts/ci/failure-report.d.mts create mode 100644 scripts/ci/failure-report.mjs create mode 100644 scripts/ci/github-api.d.mts create mode 100644 scripts/ci/github-api.mjs create mode 100644 scripts/ci/pr-ci-health.d.mts create mode 100644 scripts/ci/pr-ci-health.mjs create mode 100644 scripts/ci/static-task-context.d.mts create mode 100644 scripts/ci/static-task-context.mjs create mode 100644 scripts/ci/test-group.mjs create mode 100644 scripts/ci/workflow-run.d.mts create mode 100644 scripts/ci/workflow-run.mjs create mode 100644 scripts/lib/test-projects.d.mts create mode 100644 scripts/lib/test-projects.mjs create mode 100644 tests/boundary/filesystem/ciFailureReport.test.ts create mode 100644 tests/boundary/filesystem/ciFailureReportBudget.test.ts create mode 100644 tests/boundary/filesystem/ciFeedback.test.ts create mode 100644 tests/boundary/filesystem/ciPrHealth.test.ts create mode 100644 tests/boundary/filesystem/ciStaticArtifacts.test.ts create mode 100644 tests/boundary/filesystem/ciTestProjects.test.ts create mode 100644 tests/boundary/filesystem/ciWorkflowRun.test.ts diff --git a/.github/workflows/ci-timings.yml b/.github/workflows/ci-timings.yml index 73a9ba4b97..74f27af32c 100644 --- a/.github/workflows/ci-timings.yml +++ b/.github/workflows/ci-timings.yml @@ -12,12 +12,13 @@ concurrency: permissions: contents: read actions: read + pull-requests: read jobs: report: name: Collect recent completed workflow attempts runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 steps: # Consume only API data and execute batch code from the trusted default branch. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -27,6 +28,18 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc + - name: Cache npm package downloads + id: npm-cache + continue-on-error: true + uses: ./.github/actions/npm-cache + - name: Diagnose missing and startup-failed PR CI + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + CI_REPOSITORY: ${{ github.repository }} + CI_REPOSITORY_ID: ${{ github.repository_id }} + CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node scripts/ci/pr-ci-health.mjs - name: Record queue, execution, and matched step timing continue-on-error: true env: @@ -44,3 +57,28 @@ jobs: if-no-files-found: error overwrite: true retention-days: 14 + - name: Install dependencies for static failure attribution + id: failure-deps + if: ${{ !cancelled() && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }} + continue-on-error: true + run: npm ci + - name: Attribute supported static failures against main + if: ${{ !cancelled() && steps.failure-deps.outcome == 'success' && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + CI_REPOSITORY: ${{ github.repository }} + CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node scripts/ci/failure-report.mjs + - name: Upload CI health and failure diagnostics + if: ${{ !cancelled() }} + continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ci-health-diagnostics + path: | + .cache/ci-pr-health/ + .cache/ci-failures/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8da32ee270..b35b6a6e18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,12 @@ jobs: - name: Validate workflow syntax and expressions if: needs.changes.outputs.workflows == 'true' run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 -shellcheck= -pyflakes= + - name: Record static task source and execution identity + continue-on-error: true + if: needs.changes.outputs.code == 'true' + env: + CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: npm run ci:static-context - if: needs.changes.outputs.code == 'true' run: npm run check:ci - if: needs.changes.outputs.code == 'true' @@ -167,6 +173,34 @@ jobs: if-no-files-found: ignore retention-days: 7 + test-source: + name: Source tests + needs: changes + if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache + - run: npm ci + - name: Run source tests without waiting for build + run: npm run test:fast -- --reporter=default --reporter=blob + env: + VITEST_BLOB_LABEL: source + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ !cancelled() }} + with: + name: vitest-report-source + path: .vitest/blob/* + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + test-shard: name: Test shard (${{ matrix.shard }}) needs: [changes, build] @@ -178,7 +212,9 @@ jobs: # every shard's diagnostics and coverage. Passing PRs run the full suite. fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }} matrix: - shard: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6] + # Fast feedback spends the same six test jobs on one source group and + # five built groups; full qualification retains six complete shards. + shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/5","2/5","3/5","4/5","5/5"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 @@ -197,15 +233,16 @@ jobs: run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }} env: REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - - name: Run complete portable suite for fast feedback + - name: Run built-dependent tests for fast feedback if: needs.changes.outputs.tier == 'fast' - run: npm run test:run -- --reporter=default --reporter=blob --shard=${{ matrix.shard }} + run: npm run test:built -- --reporter=default --reporter=blob --shard=${{ matrix.shard }} env: + VITEST_BLOB_LABEL: built REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ !cancelled() }} with: - name: vitest-report-${{ strategy.job-index }} + name: vitest-report-built-${{ strategy.job-index }} path: .vitest/blob/* include-hidden-files: true if-no-files-found: error @@ -219,11 +256,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - cache: npm + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache - run: npm ci - name: Run reverse tools tests with coverage run: npm run test:coverage:reverse-tools @@ -316,7 +355,7 @@ jobs: test: name: Portable tests - needs: [changes, test-shard] + needs: [changes, test-source, test-shard] if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }} runs-on: ubuntu-latest timeout-minutes: 10 @@ -340,7 +379,9 @@ jobs: - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 if: >- !cancelled() && needs.changes.outputs.code == 'true' && - (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure') + (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' || + (needs.changes.outputs.tier == 'fast' && + (needs.test-source.result == 'success' || needs.test-source.result == 'failure'))) with: path: .vitest/blob pattern: vitest-report-* @@ -368,7 +409,8 @@ jobs: if: >- !cancelled() && needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' && - (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure') + (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' || + needs.test-source.result == 'success' || needs.test-source.result == 'failure') run: >- npm exec -- vitest --merge-reports --reporter=default @@ -381,6 +423,8 @@ jobs: if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-timings.json') != '' env: SHARD_RESULT: ${{ needs.test-shard.result }} + SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }} + SOURCE_RESULT: ${{ needs.test-source.result }} MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} CI_VERIFY_TEST_INVENTORY: "true" CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} @@ -413,7 +457,13 @@ jobs: env: CODE_REQUIRED: ${{ needs.changes.outputs.code }} SHARD_RESULT: ${{ needs.test-shard.result }} - run: test "${CODE_REQUIRED}" != "true" || test "${SHARD_RESULT}" = "success" + SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }} + SOURCE_RESULT: ${{ needs.test-source.result }} + run: | + if [ "${CODE_REQUIRED}" = "true" ]; then + test "${SHARD_RESULT}" = "success" || exit 1 + test "${SOURCE_REQUIRED}" != "true" || test "${SOURCE_RESULT}" = "success" + fi conformance-fixtures: name: Source-built conformance fixtures @@ -847,6 +897,7 @@ jobs: build, reverse-tools-quality, test-shard, + test-source, sqlite-runtime, test, conformance-fixtures, diff --git a/.github/workflows/real-cutter.yml b/.github/workflows/real-cutter.yml index 8567779991..91121f2073 100644 --- a/.github/workflows/real-cutter.yml +++ b/.github/workflows/real-cutter.yml @@ -13,8 +13,8 @@ jobs: name: Build Windows bridge security boundary runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - name: Install the cross compiler used by the Windows native lane @@ -44,11 +44,13 @@ jobs: - platform: Windows runner: [self-hosted, cutter-upstream, Windows] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - cache: npm + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache - run: npm ci - if: matrix.platform == 'Windows' uses: actions/download-artifact@v4 diff --git a/docs/testing.md b/docs/testing.md index b543fe89f7..aa8d2163ff 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -280,8 +280,15 @@ focused suite also exercises affected MCP routes and the Python Cutter bridge. Contract, integration, and real-provider lanes complement these focused tests; simulated providers do not establish real-provider behavior. -CI runs six native Vitest shards without retries. Each shard emits a blob -report; the merge job produces JUnit and JSON diagnostics, a module timing +Fast CI starts the source-only projects after classification, independently of +build and static checks, and runs five built-dependent shards after the build. +The six test jobs still select every portable test. Full qualification keeps six +complete Vitest shards with coverage. Neither tier retries tests. Canonical +project ownership lives in `scripts/lib/test-projects.mjs`, shared by Vitest, +CI group execution, and development-test preparation. A test importing compiled +verifier code belongs to the built group even if its directory is conformance. +Source and built blobs have separate labels to prevent merge collisions. +Each test job emits a blob report; the merge job produces JUnit and JSON diagnostics, a module timing report, plus aggregate coverage during full qualification. It retains available reports even when a shard fails, uploads available diagnostics, and writes the slowest files to the workflow summary. Partial reports are marked diagnostic @@ -316,6 +323,30 @@ portable test shard. These limits leave headroom for cold setup and full coverag while preventing a hung job from holding the active main run for hours. They do not bound runner queue waiting. Timeouts still fail the required gate. +The separate `CI timings` workflow also diagnoses missing, startup-failed, +queued/running, approval-required, canceled, and failed PR CI through bounded, +read-only API scans. Incomplete pagination or changing identity stays unknown. +It never closes/reopens PRs, reruns workflows, posts comments, or changes gates. +The timing manifest retains each run's event and head branch, including explicit +unknowns, so PR runs cannot consume the eligible-main comparison budget. +Its static failure report compares supported lint/duplication diagnostics with +failed main runs using exact source, ancestry, task hashes, and toolchain proof. +A match describes those diagnostics only; it does not establish that every +failure in a job came from main. Duplication reports retain the actual checker +exit result, so passing duplication checks preserve lint failures and caller +thresholds remain owned by the producer. Missing checker outcomes leave +those duplication diagnostics unclassified. Missing historical receipts remain unknown. +The reporter executes only trusted default-branch code and parses whitelisted +JSON artifact members without executing artifact content. Reports and limits +are retained in `ci-health-diagnostics`; its ten-minute job limit covers the +existing four-minute timing collection plus the bounded diagnostic stages. + +Shard-count reduction remains deferred: the available successful timing sample +covers an older inventory, and file phase weights are not job wall time. The +one-source/five-built split retains six test jobs; it makes no hosted speedup +claim. Current complete timings and completed candidate hosted runs are needed +before enabling a smaller adaptive matrix. + The separate `CI timings` workflow batches recent completed CI and Release runs hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and a normalized point-in-time report for each collected workflow attempt, plus a diff --git a/package.json b/package.json index dff275eb44..e09e51366c 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,9 @@ "precheck:pr": "npm run deps:check", "check:pr": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos", "precheck:ci": "npm run deps:check", - "check:ci": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos && npm run verify:architecture-guards", + "check:ci": "node scripts/run-exclusive.mjs artifacts npm run check:ci:tasks && npm run verify:architecture-guards", + "check:ci:tasks": "turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos", + "ci:static-context": "node scripts/ci/static-task-context.mjs", "build:cached": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills", "build:termux": "node scripts/run-exclusive.mjs artifacts npm run build:termux:run", "build:termux:run": "npm run compile:termux && npm run artifacts:skills", @@ -138,7 +140,8 @@ "test:run": "vitest run", "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v", "test:local": "node scripts/run-development-tests.mjs local", - "test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation", + "test:fast": "node scripts/ci/test-group.mjs source", + "test:built": "node scripts/ci/test-group.mjs built", "pretest:boundary": "npm run test:prepare", "test:boundary": "vitest run --project boundary --project cli-boundary --project process-boundary --project mcp-boundary --project process-global", "pretest:mcp": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills artifacts:mcp-catalog", diff --git a/scripts/ci/artifacts.d.mts b/scripts/ci/artifacts.d.mts new file mode 100644 index 0000000000..93cf2f7013 --- /dev/null +++ b/scripts/ci/artifacts.d.mts @@ -0,0 +1,67 @@ +import type { NormalizedWorkflowRun } from "./workflow-run.mjs"; +import type { StaticTaskContext } from "./static-task-context.mjs"; +export type { StaticTaskContext } from "./static-task-context.mjs"; + +export interface StaticDiagnostic { + readonly kind: "lint" | "jscpd"; + readonly path?: string; + readonly paths?: readonly string[]; + readonly code?: string; + readonly message?: string; + readonly severity?: string; +} + +export function classifyFailure(input: { + repository: string; + defaultBranch: string; + run: Record & NormalizedWorkflowRun; + job: Record; + diagnostics: readonly StaticDiagnostic[]; + context: StaticTaskContext; + mainRun: Record & NormalizedWorkflowRun; + mainJob: Record; + mainDiagnostics: readonly StaticDiagnostic[]; + mainContext: StaticTaskContext; + executionSourcesVerified: boolean; + ancestry: { status: string; mergeBaseSha: string | null }; + blobs: Readonly>; +}): { + classification: "matching-main-diagnostics" | "unknown"; + reason: string; +}; + +export function diagnosticPaths(item: StaticDiagnostic): readonly string[]; +export function isFailedConclusion( + value: unknown, +): value is "failure" | "timed_out"; +export function isEligibleMainRun( + run: Record, + defaultBranch: string, + repository: string, +): boolean; +export function pullRequestBaseSha( + run: Record, + repository: string, +): string | null; +export function safeSha(value: unknown): string | null; +export function validRepoPath(value: unknown): value is string; +export function selectStaticCheckArtifact( + page: unknown, + run: { id?: string | number; head_sha?: string }, + runAttempt: number, +): unknown | null; + +export function readStaticCheckArtifact(options: { + repository: string; + artifactId: number | string; + token: string; + fetchImpl?: typeof fetch; +}): Promise<{ + context: StaticTaskContext; + diagnostics: readonly StaticDiagnostic[]; +}>; + +export function parseStaticArchive(archivePath: string): Promise<{ + context: StaticTaskContext; + diagnostics: readonly StaticDiagnostic[]; +}>; diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjs new file mode 100644 index 0000000000..9a7cc1c218 --- /dev/null +++ b/scripts/ci/artifacts.mjs @@ -0,0 +1,436 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join } from "node:path"; +import { promisify } from "node:util"; + +import { parseStaticTaskContext } from "./static-task-context.mjs"; + +const execute = promisify(execFile); +const MAX_ARCHIVE_BYTES = 32 * 1024 * 1024; +const MAX_MEMBER_BYTES = 4 * 1024 * 1024; +const MAX_DIAGNOSTICS = 10_000; +const API_ROOT = "https://api.github.com/repos"; +const SHA = /^[0-9a-f]{40}$/iu; +const TASK_HASH = /^[a-f0-9]{16}$/u; + +/** Compare parsed static diagnostics only after source, task, host, and ancestry proof. */ +export function classifyFailure({ + repository, + defaultBranch, + run, + job, + diagnostics, + context, + mainRun, + mainJob, + mainDiagnostics, + mainContext, + executionSourcesVerified, + ancestry, + blobs, +}) { + const failedGate = firstFailure([ + () => [ + !isCompletedFailure(run) || + job?.status !== "completed" || + !isFailedConclusion(job?.conclusion), + "run_or_job_not_a_completed_failure", + ], + () => [ + !isEligiblePullRun(run, repository) || + !isEligibleMainRun(mainRun, defaultBranch, repository), + "run_event_or_repository_identity_unverified", + ], + () => [ + job?.name !== mainJob?.name || job?.name !== "Static checks", + "failed_job_not_the_same_static_gate", + ], + () => [ + !Array.isArray(diagnostics) || + diagnostics.length === 0 || + !Array.isArray(mainDiagnostics), + "structured_diagnostics_missing_or_invalid", + ], + () => [ + !validContext(context) || !validContext(mainContext), + "static_task_context_missing_or_invalid", + ], + () => [ + executionSourcesVerified !== true, + "checkout_sources_not_verified_against_api_commits", + ], + () => [ + !sameExecutionContext(context, mainContext), + "task_or_toolchain_identity_differs", + ], + () => [ + ancestry?.status !== "ahead" && ancestry?.status !== "identical", + "main_head_not_proven_ancestor_of_tested_base", + ], + () => [ + ancestry?.mergeBaseSha !== mainRun?.head_sha, + "main_head_is_not_the_verified_merge_base", + ], + () => [ + !sameTaskHashes(diagnostics, context, mainContext), + "owning_task_hash_differs", + ], + ]); + if (failedGate) return outcome("unknown", failedGate.reason); + const keys = diagnostics.map(diagnosticKey); + const mainKeys = new Set(mainDiagnostics.map(diagnosticKey).filter(Boolean)); + if (keys.some((key) => key === null)) + return outcome("unknown", "diagnostic_shape_not_supported"); + if (!keys.every((key) => mainKeys.has(key))) + return outcome( + "unknown", + "main_run_does_not_contain_the_same_structured_diagnostic", + ); + if (!sameBlobEvidence(diagnostics, blobs)) + return outcome( + "unknown", + "diagnostic_source_content_not_identical_at_main_and_tested_base", + ); + return outcome( + "matching-main-diagnostics", + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + ); +} + +export function diagnosticPaths(item) { + return item.kind === "jscpd" ? item.paths : [item.path]; +} +export function isFailedConclusion(value) { + return value === "failure" || value === "timed_out"; +} +export function isEligibleMainRun(run, defaultBranch, repository) { + return ( + isEligibleStaticRun(run) && + run.repository.full_name === repository && + run.head_branch === defaultBranch && + ["push", "schedule", "workflow_dispatch"].includes(run.event) + ); +} +export function pullRequestBaseSha(run, repository) { + if ( + run?.event !== "pull_request" || + run.repository?.full_name !== repository || + !Array.isArray(run.pull_requests) || + run.pull_requests.length !== 1 + ) + return null; + const base = run.pull_requests[0].base; + return base?.repo?.full_name === repository && SHA.test(base?.sha ?? "") + ? base.sha + : null; +} + +export function safeSha(value) { + return typeof value === "string" && SHA.test(value) + ? value.toLowerCase() + : null; +} +export function selectStaticCheckArtifact(page, run, runAttempt) { + if (runAttempt !== 1) return null; + if ( + !Array.isArray(page?.artifacts) || + !Number.isSafeInteger(page.total_count) || + page.total_count !== page.artifacts.length || + page.artifacts.length >= 100 + ) + return null; + const matches = page.artifacts.filter( + (artifact) => artifact.name === "static-check-reports", + ); + if (matches.length !== 1) return null; + const [artifact] = matches; + if ( + artifact.expired !== false || + String(artifact.workflow_run?.id) !== String(run.id) || + artifact.workflow_run?.head_sha !== run.head_sha + ) + return null; + return artifact; +} +export function validRepoPath(value) { + return ( + typeof value === "string" && + value.length > 0 && + !isAbsolute(value) && + !value.includes("\\") && + value + .split("/") + .every((part) => part !== "" && part !== "." && part !== "..") && + !value.startsWith("/") + ); +} + +/** Download one named static report archive and parse only whitelisted JSON members. */ +export async function readStaticCheckArtifact({ + repository, + artifactId, + token, + fetchImpl = fetch, +}) { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? "")) + throw new Error("artifact repository identity is invalid"); + if (!token) throw new Error("artifact download requires a GitHub token"); + const url = `${API_ROOT}/${repository}/actions/artifacts/${positiveId(artifactId)}/zip`; + let response = await fetchImpl(url, { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28", + }, + signal: AbortSignal.timeout(10_000), + redirect: "manual", + }); + if ([301, 302, 303, 307, 308].includes(response.status)) { + const location = response.headers.get("location"); + if (!location || new URL(location).protocol !== "https:") + throw new Error("static_artifact_redirect_invalid"); + response = await fetchImpl(location, { + signal: AbortSignal.timeout(10_000), + redirect: "error", + }); + } + if (!response.ok) throw new Error(`github_http_${response.status}`); + const bytes = await readLimitedBytes(response, MAX_ARCHIVE_BYTES); + const directory = await mkdtemp(join(tmpdir(), "rea-ci-static-artifact-")); + const archive = join(directory, "artifact.zip"); + try { + await writeFile(archive, bytes, { flag: "wx" }); + return await parseStaticArchive(archive); + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +/** Accept only the three named producer outputs; no archive paths are written to disk. */ +export async function parseStaticArchive(archivePath) { + const { stdout: listing } = await execute("unzip", ["-Z1", archivePath], { + encoding: "utf8", + maxBuffer: 128 * 1024, + timeout: 5_000, + }); + const allowed = new Set([ + "oxlint.json", + "jscpd/jscpd-report.json", + "static-task-context.json", + ]); + const members = new Map(); + for (const name of listing.split(/\r?\n/u).filter(Boolean)) { + const prefix = ".cache/static-checks/"; + const member = name.startsWith(prefix) ? name.slice(prefix.length) : name; + if (!allowed.has(member)) continue; + if (members.has(member)) + throw new Error("static artifact contains duplicate report members"); + const { stdout } = await execute("unzip", ["-p", archivePath, name], { + encoding: "buffer", + maxBuffer: MAX_MEMBER_BYTES, + timeout: 5_000, + }); + members.set(member, JSON.parse(stdout.toString("utf8"))); + } + const context = parseStaticTaskContext( + members.get("static-task-context.json"), + ); + const diagnostics = normalizeDiagnostics( + members.get("oxlint.json"), + members.get("jscpd/jscpd-report.json"), + ); + if (diagnostics.length > MAX_DIAGNOSTICS) + throw new Error("static diagnostic count exceeds the reporting limit"); + return { context, diagnostics }; +} + +function normalizeDiagnostics(lint, duplicate) { + return [...normalizeLint(lint), ...normalizeDuplicates(duplicate)]; +} + +function normalizeLint(report) { + if (report === undefined) return []; + if (!Array.isArray(report?.diagnostics)) + throw new Error("oxlint diagnostics are malformed"); + return report.diagnostics + .filter((item) => item.severity === "error") + .map((item) => { + if ( + typeof item.filename !== "string" || + !repoPath(item.filename) || + typeof item.code !== "string" || + typeof item.message !== "string" + ) + throw new Error("oxlint error diagnostic is malformed"); + return { + kind: "lint", + path: item.filename, + code: item.code, + message: item.message, + severity: "error", + }; + }); +} + +function normalizeDuplicates(report) { + if (report === undefined) return []; + if ( + !Array.isArray(report?.duplicates) || + !Number.isFinite(report.statistics?.total?.percentage) || + report.statistics.total.percentage < 0 + ) + throw new Error("jscpd report is malformed"); + // The producer owns the checker outcome; do not copy or infer its threshold. + const exitCode = report.reaCheck?.exitCode; + if (exitCode === undefined || exitCode === 0) return []; + if (!Number.isSafeInteger(exitCode) || exitCode < 0) + throw new Error("jscpd checker outcome is malformed"); + if (exitCode !== 1) return []; + return report.duplicates.map((item) => { + const paths = [item.firstFile?.name, item.secondFile?.name].sort(); + if (paths.some((path) => !repoPath(path))) + throw new Error("jscpd duplicate paths are malformed"); + return { kind: "jscpd", paths }; + }); +} + +async function readLimitedBytes(response, limit) { + const reader = response.body?.getReader(); + if (!reader) throw new Error("static artifact response has no body"); + const chunks = []; + let size = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) { + await reader.cancel(); + throw new Error("static artifact exceeds the download limit"); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const result = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; +} + +function repoPath(value) { + return ( + typeof value === "string" && + value.length > 0 && + !value.startsWith("/") && + !value.includes("\\") && + value + .split("/") + .every((part) => part !== "" && part !== "." && part !== "..") + ); +} + +function positiveId(value) { + const id = Number(value); + if (!Number.isSafeInteger(id) || id < 1) + throw new Error("artifact ID must be a positive integer"); + return String(id); +} + +function diagnosticKey(item) { + if ( + item?.kind === "lint" && + validRepoPath(item.path) && + typeof item.code === "string" && + typeof item.message === "string" && + item.severity === "error" + ) + return JSON.stringify([ + item.kind, + item.path, + item.code, + item.message, + item.severity, + ]); + if ( + item?.kind === "jscpd" && + Array.isArray(item.paths) && + item.paths.length === 2 && + item.paths.every(validRepoPath) + ) + return JSON.stringify([item.kind, ...[...item.paths].sort()]); + return null; +} +function sameBlobEvidence(diagnostics, blobs) { + const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))]; + return paths.length > 0 && paths.every((path) => blobs?.[path] === true); +} +function sameTaskHashes(diagnostics, left, right) { + return diagnostics.every((diagnostic) => { + const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint:built"; + return ( + TASK_HASH.test(left.taskHashes[task] ?? "") && + left.taskHashes[task] === right.taskHashes[task] + ); + }); +} +function validContext(value) { + return ( + value?.version === 1 && + SHA.test(value.sourceSha ?? "") && + value.sourceClean === true && + typeof value.nodeVersion === "string" && + typeof value.npmVersion === "string" && + typeof value.turboVersion === "string" && + typeof value.platform === "string" && + typeof value.architecture === "string" && + value.taskHashes && + typeof value.taskHashes === "object" && + !Array.isArray(value.taskHashes) + ); +} +function sameExecutionContext(left, right) { + return [ + "nodeVersion", + "npmVersion", + "turboVersion", + "platform", + "architecture", + ].every((key) => left[key] === right[key]); +} +function isEligiblePullRun(run, repository) { + return ( + isEligibleStaticRun(run) && pullRequestBaseSha(run, repository) !== null + ); +} +function isEligibleStaticRun(run) { + return ( + isRecord(run) && + run.status === "completed" && + isFailedConclusion(run.conclusion) && + SHA.test(run.head_sha ?? "") && + typeof run.repository?.full_name === "string" && + run.workflowPath?.file === ".github/workflows/ci.yml" + ); +} +function isCompletedFailure(run) { + return run?.status === "completed" && isFailedConclusion(run.conclusion); +} +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function firstFailure(checks) { + for (const check of checks) { + const [failed, reason] = check(); + if (failed) return { reason }; + } + return null; +} +function outcome(classification, reason) { + return { classification, reason }; +} diff --git a/scripts/ci/failure-report.d.mts b/scripts/ci/failure-report.d.mts new file mode 100644 index 0000000000..47ac65745d --- /dev/null +++ b/scripts/ci/failure-report.d.mts @@ -0,0 +1,159 @@ +import type { NormalizedWorkflowRun } from "./workflow-run.mjs"; +import type { StaticDiagnostic } from "./artifacts.mjs"; +export type { StaticDiagnostic } from "./artifacts.mjs"; +import type { StaticTaskContext } from "./static-task-context.mjs"; +export type { StaticTaskContext } from "./static-task-context.mjs"; +export interface CiRun { + readonly id?: string | number; + readonly run_attempt?: number; + readonly status?: string; + readonly conclusion?: string | null; + readonly event?: string; + readonly path?: string; + readonly head_sha?: string; + readonly head_branch?: string; + readonly repository?: { readonly full_name?: string }; + readonly pull_requests?: readonly { + readonly base?: { + readonly sha?: string; + readonly repo?: { readonly full_name?: string }; + }; + }[]; +} + +export interface CiJob { + readonly name?: string; + readonly status?: string; + readonly conclusion?: string | null; + readonly run_id?: number; + readonly run_attempt?: number; +} + +export interface StaticEvidence { + readonly context: StaticTaskContext; + readonly diagnostics: readonly StaticDiagnostic[]; +} + +export interface CiFailureReport { + readonly version: 1; + readonly observedAt: string; + readonly repository: string; + readonly defaultBranch: string; + readonly informationalOnly: true; + readonly gateEffect: string; + readonly boundedInput: { + readonly candidateRuns: number; + readonly limit: number; + readonly mainCandidateLimit: number; + readonly mainCandidateRuns: number; + readonly mainRunLookupLimit: number; + readonly manifestTruncated: boolean; + }; + readonly counts: Readonly< + Record<"matching-main-diagnostics" | "unknown", number> + >; + readonly failures: readonly { + readonly runId: number | string; + readonly runAttempt: number; + readonly classification: "matching-main-diagnostics" | "unknown"; + readonly reason: string; + readonly candidateFailures?: readonly { + readonly runId: number | string; + readonly runAttempt: number; + readonly reason: string; + }[]; + readonly runHead?: string | null; + readonly event?: string | null; + readonly job?: string; + readonly conclusion?: string | null; + }[]; +} + +export interface FailureReportApi { + getRun(repository: string, id: string | number): Promise; + getJobs( + repository: string, + id: string | number, + attempt: number, + ): Promise; + getStaticEvidence( + repository: string, + run: CiRun, + job: CiJob, + ): Promise; + getAncestry( + repository: string, + ancestor: string, + descendant: string, + ): Promise<{ status: string; mergeBaseSha: string | null }>; + verifyExecutionSource( + repository: string, + run: CiRun, + context: StaticTaskContext, + ): Promise; + getBlobIdentity( + repository: string, + ancestor: string, + descendant: string, + paths: readonly string[], + ): Promise>>; +} + +export function classifyFailure(input: { + repository: string; + defaultBranch: string; + run: CiRun & NormalizedWorkflowRun; + job: CiJob; + diagnostics: readonly StaticDiagnostic[]; + context: StaticTaskContext; + mainRun: CiRun & NormalizedWorkflowRun; + mainJob: CiJob; + mainDiagnostics: readonly StaticDiagnostic[]; + mainContext: StaticTaskContext; + executionSourcesVerified: boolean; + ancestry: { status: string; mergeBaseSha: string | null }; + blobs: Readonly>; +}): { + classification: "matching-main-diagnostics" | "unknown"; + reason: string; +}; + +export function reportCiFailures(options: { + repository: string; + defaultBranch: string; + batchManifest: { + readonly repository: string; + readonly runs: readonly { + readonly id: number | string; + readonly runAttempt: number; + readonly workflow: string; + readonly event?: string | null; + readonly headBranch?: string | null; + readonly conclusion: string | null; + readonly status: string; + }[]; + readonly truncation?: readonly unknown[]; + readonly coverage?: { readonly exhaustive?: boolean }; + readonly selection?: { readonly exhaustive?: boolean }; + }; + api: FailureReportApi; + outputDirectory?: string; + summaryPath?: string; + observedAt?: string; +}): Promise; + +export function createGitHubApi(options?: { + token?: string; + fetchImpl?: typeof fetch; + apiRequest?: ( + method: string, + endpoint: string, + budget: { requests: number; startedAt: number; limitMs: number }, + ) => Promise; +}): FailureReportApi; + +export function writeFailureReport( + report: CiFailureReport, + outputDirectory?: string, + summaryPath?: string, +): Promise<{ jsonPath: string; markdownPath: string }>; diff --git a/scripts/ci/failure-report.mjs b/scripts/ci/failure-report.mjs new file mode 100644 index 0000000000..2ce6d813c5 --- /dev/null +++ b/scripts/ci/failure-report.mjs @@ -0,0 +1,529 @@ +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + classifyFailure, + diagnosticPaths, + isEligibleMainRun, + isFailedConclusion, + pullRequestBaseSha, + readStaticCheckArtifact, + safeSha, + selectStaticCheckArtifact, + validRepoPath, +} from "./artifacts.mjs"; +import { createGhApiRequest } from "./github-api.mjs"; +import { normalizeWorkflowRun } from "./workflow-run.mjs"; + +const OUTPUT_DIRECTORY = join(".cache", "ci-failures"); +const BATCH_DIRECTORY = join(".cache", "ci-job-timings"); +const MAX_JSON_BYTES = 8 * 1024 * 1024; +const MAX_FAILED_RUNS = 20; +const MAX_MAIN_CANDIDATES = 5; +const MAX_MAIN_RUN_LOOKUPS = 100; +const MAX_PATHS_PER_COMPARISON = 50; +const MAX_API_REQUESTS = 250; + +export { classifyFailure } from "./artifacts.mjs"; + +/** Uses only failed CI candidates present in the supplied bounded timing manifest. */ +export async function reportCiFailures({ + repository, + defaultBranch, + batchManifest, + api, + outputDirectory = OUTPUT_DIRECTORY, + summaryPath, + observedAt = new Date().toISOString(), +}) { + validateRepository(repository); + if (typeof defaultBranch !== "string" || defaultBranch.length === 0) + throw new Error("defaultBranch is required"); + if ( + !batchManifest || + batchManifest.repository !== repository || + !Array.isArray(batchManifest.runs) + ) + throw new Error("timing manifest repository or runs are invalid"); + + const failedRuns = batchManifest.runs.filter( + (entry) => + entry.workflow === "CI" && + entry.status === "collected" && + isFailedConclusion(entry.conclusion), + ); + const prRuns = failedRuns.filter((entry) => entry.event === "pull_request"); + const candidates = prRuns.slice(0, MAX_FAILED_RUNS); + const mainEntries = failedRuns.filter( + (entry) => + entry.headBranch === defaultBranch && + ["push", "schedule", "workflow_dispatch"].includes(entry.event), + ); + const lookupRun = verifiedRunLookup(repository, api); + const mainCandidates = []; + for (const entry of mainEntries.slice(0, MAX_MAIN_RUN_LOOKUPS)) { + try { + const run = await lookupRun(entry); + if (isEligibleMainRun(run, defaultBranch, repository)) + mainCandidates.push({ + run, + getEvidence: mainEvidenceLookup(repository, api, entry, run), + }); + if (mainCandidates.length === MAX_MAIN_CANDIDATES) break; + } catch { + // Unverified metadata cannot consume the eligible-main candidate budget. + } + } + const entries = []; + for (const entry of candidates) { + try { + entries.push( + ...(await analyzeFailedRun({ + repository, + defaultBranch, + entry, + run: await lookupRun(entry), + mainCandidates, + api, + })), + ); + } catch (error) { + entries.push({ + runId: entry.id, + runAttempt: entry.runAttempt, + classification: "unknown", + reason: errorCode(error), + }); + } + } + const report = { + version: 1, + observedAt, + repository, + defaultBranch, + informationalOnly: true, + gateEffect: "none; this report never changes CI conclusions", + boundedInput: { + candidateRuns: candidates.length, + limit: MAX_FAILED_RUNS, + mainCandidateLimit: MAX_MAIN_CANDIDATES, + mainCandidateRuns: mainCandidates.length, + mainRunLookupLimit: MAX_MAIN_RUN_LOOKUPS, + manifestTruncated: + prRuns.length > MAX_FAILED_RUNS || + batchManifest.truncation?.length > 0 || + batchManifest.selection?.exhaustive === false || + batchManifest.coverage?.exhaustive === false, + }, + counts: Object.fromEntries( + ["matching-main-diagnostics", "unknown"].map((category) => [ + category, + entries.filter((item) => item.classification === category).length, + ]), + ), + failures: entries, + }; + await writeFailureReport(report, outputDirectory, summaryPath); + return report; +} + +async function analyzeFailedRun({ + repository, + defaultBranch, + entry, + run, + mainCandidates, + api, +}) { + const jobs = await api.getJobs(repository, entry.id, entry.runAttempt); + if (jobs.some((job) => !hasJobIdentity(job, entry))) + throw new Error("workflow_job_identity_mismatch"); + const failedJobs = jobs.filter( + (job) => job.status === "completed" && isFailedConclusion(job.conclusion), + ); + if (failedJobs.length === 0) + return [ + { + runId: entry.id, + runAttempt: entry.runAttempt, + classification: "unknown", + reason: "failed_jobs_not_present_in_api_snapshot", + }, + ]; + const failures = []; + for (const job of failedJobs) { + const evidence = await api.getStaticEvidence(repository, run, job); + const classification = await attributeToMain({ + repository, + defaultBranch, + run, + job, + evidence, + mainCandidates, + api, + }); + failures.push({ + runId: entry.id, + runAttempt: entry.runAttempt, + runHead: safeSha(run.head_sha), + event: run.event ?? null, + job: job.name, + conclusion: job.conclusion, + ...classification, + }); + } + return failures; +} + +async function attributeToMain({ + repository, + defaultBranch, + run, + job, + evidence, + mainCandidates, + api, +}) { + const baseSha = pullRequestBaseSha(run, repository); + if (baseSha === null || evidence === null) + return outcome("unknown", "no_matching_verified_main_failure"); + let result = outcome("unknown", "no_matching_verified_main_failure"); + const candidateFailures = []; + for (const candidate of mainCandidates) { + try { + const main = await compareCandidate({ + repository, + defaultBranch, + run, + job, + evidence, + baseSha, + candidate, + api, + }); + if (main !== null) result = main; + if (main?.classification === "matching-main-diagnostics") break; + } catch (error) { + candidateFailures.push({ + runId: candidate.run.id, + runAttempt: candidate.run.run_attempt, + reason: errorCode(error), + }); + } + } + return { + ...result, + ...(candidateFailures.length > 0 ? { candidateFailures } : {}), + }; +} + +async function compareCandidate({ + repository, + defaultBranch, + run, + job, + evidence, + baseSha, + candidate, + api, +}) { + const mainRun = candidate.run; + const main = await candidate.getEvidence(job.name); + if (!main?.evidence) return null; + const { job: mainJob, evidence: mainEvidence } = main; + const ancestry = await api.getAncestry(repository, mainRun.head_sha, baseSha); + const paths = [ + ...new Set(evidence.diagnostics.flatMap(diagnosticPaths)), + ].slice(0, MAX_PATHS_PER_COMPARISON); + const blobs = await api.getBlobIdentity( + repository, + mainRun.head_sha, + baseSha, + paths, + ); + const executionSourcesVerified = + (await api.verifyExecutionSource(repository, run, evidence.context)) && + (await api.verifyExecutionSource( + repository, + mainRun, + mainEvidence.context, + )); + return classifyFailure({ + repository, + defaultBranch, + run, + job, + diagnostics: evidence.diagnostics, + context: evidence.context, + mainRun, + mainJob, + mainDiagnostics: mainEvidence.diagnostics, + mainContext: mainEvidence.context, + executionSourcesVerified, + ancestry, + blobs, + }); +} + +// Each verified main run/attempt owns its lazy jobs and parsed evidence for this +// report only. Cache nulls and rejected reads too; retries cannot spend the budget +// repeatedly on an immutable observation that was unavailable or invalid. +function mainEvidenceLookup(repository, api, entry, run) { + let jobs; + const evidence = new Map(); + return (name) => { + jobs ??= Promise.resolve().then(async () => { + const items = await api.getJobs(repository, entry.id, entry.runAttempt); + return items.some((item) => !hasJobIdentity(item, entry)) ? [] : items; + }); + if (!evidence.has(name)) + evidence.set( + name, + jobs.then(async (items) => { + const job = items.find( + (item) => + item.name === name && + item.status === "completed" && + isFailedConclusion(item.conclusion), + ); + if (!job) return null; + return { + job, + evidence: await api.getStaticEvidence(repository, run, job), + }; + }), + ); + return evidence.get(name); + }; +} + +// Selection and attribution share one immutable API observation per run/attempt. +function verifiedRunLookup(repository, api) { + const runs = new Map(); + return (entry) => { + const key = `${positiveId(entry.id)}:${positiveId(entry.runAttempt)}`; + if (!runs.has(key)) + runs.set( + key, + Promise.resolve().then(async () => { + const run = normalizeWorkflowRun( + await api.getRun(repository, entry.id), + ); + if (!hasRunIdentity(run, entry)) + throw new Error("workflow_run_identity_mismatch"); + return run; + }), + ); + return runs.get(key); + }; +} + +function hasRunIdentity(run, entry) { + return ( + String(run?.id) === String(entry.id) && + Number(run.run_attempt) === Number(entry.runAttempt) && + run.workflowPath?.file === ".github/workflows/ci.yml" + ); +} +function hasJobIdentity(job, entry) { + return ( + String(job.run_id) === String(entry.id) && + Number(job.run_attempt) === Number(entry.runAttempt) + ); +} + +export function createGitHubApi({ + token = process.env.GH_TOKEN, + fetchImpl = fetch, + apiRequest = createGhApiRequest(), +} = {}) { + if (!token) throw new Error("GH_TOKEN is required"); + const budget = { requests: 0, startedAt: Date.now(), limitMs: 60_000 }; + const request = async (endpoint) => { + if (budget.requests >= MAX_API_REQUESTS) + throw new Error("github_api_request_budget_exhausted"); + budget.requests += 1; + return await apiRequest("GET", endpoint, budget); + }; + return { + async getRun(repository, id) { + return await request( + `repos/${repository}/actions/runs/${positiveId(id)}`, + ); + }, + async getJobs(repository, id, attempt) { + const page = await request( + `repos/${repository}/actions/runs/${positiveId(id)}/attempts/${positiveId(attempt)}/jobs?per_page=100&page=1`, + ); + if (!Array.isArray(page.jobs) || page.jobs.length >= 100) + throw new Error("job_list_incomplete_or_malformed"); + return page.jobs; + }, + async getStaticEvidence(repository, run, job) { + if (job.name !== "Static checks") return null; + if (run.run_attempt !== 1) return null; + const page = await request( + `repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`, + ); + const artifact = selectStaticCheckArtifact(page, run, run.run_attempt); + if (!artifact) return null; + return await readStaticCheckArtifact({ + repository, + artifactId: artifact.id, + token, + fetchImpl, + }); + }, + async getAncestry(repository, ancestor, descendant) { + const comparison = await request( + `repos/${repository}/compare/${safeSha(ancestor)}...${safeSha(descendant)}`, + ); + return { + status: comparison.status, + mergeBaseSha: safeSha(comparison.merge_base_commit?.sha), + }; + }, + async verifyExecutionSource(repository, run, context) { + const sourceSha = safeSha(context.sourceSha); + if (!sourceSha) return false; + const pr = + run.event === "pull_request" + ? pullRequestBaseSha(run, repository) + : null; + if ( + run.event === "pull_request" && + (pr === null || context.prHeadSha !== run.head_sha) + ) + return false; + const commit = await request(`repos/${repository}/commits/${sourceSha}`); + if (safeSha(commit.sha) !== sourceSha) return false; + if (sourceSha === run.head_sha) return true; + if ( + pr === null || + !Array.isArray(commit.parents) || + commit.parents.length !== 2 + ) + return false; + return ( + safeSha(commit.parents[0]?.sha) === pr && + safeSha(commit.parents[1]?.sha) === run.head_sha + ); + }, + async getBlobIdentity(repository, ancestor, descendant, paths) { + const result = {}; + for (const path of paths) { + if (!validRepoPath(path)) continue; + const [left, right] = await Promise.all([ + getContent(repository, ancestor, path, request), + getContent(repository, descendant, path, request), + ]); + result[path] = left !== null && left === right; + } + return result; + }, + }; +} + +async function getContent(repository, ref, path, request) { + try { + const encodedPath = path.split("/").map(encodeURIComponent).join("/"); + const value = await request( + `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`, + ); + return typeof value.sha === "string" ? value.sha : null; + } catch { + return null; + } +} + +function outcome(classification, reason) { + return { classification, reason }; +} +function validateRepository(value) { + if ( + typeof value !== "string" || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value) + ) + throw new Error("repository must be owner/name"); +} +function positiveId(value) { + if (!Number.isSafeInteger(Number(value)) || Number(value) < 1) + throw new Error( + "GitHub run and attempt IDs must be positive safe integers", + ); + return String(Number(value)); +} +function errorCode(error) { + return error instanceof Error ? error.message.slice(0, 160) : "unknown_error"; +} + +export async function writeFailureReport( + report, + outputDirectory = OUTPUT_DIRECTORY, + summaryPath, +) { + await mkdir(outputDirectory, { recursive: true }); + const jsonPath = join(outputDirectory, "report.json"); + const markdownPath = join(outputDirectory, "report.md"); + const matchedDiagnostics = report.failures.filter( + (item) => item.classification === "matching-main-diagnostics", + ); + const lines = [ + "# CI failure attribution", + "", + "Informational only. This report never changes a CI conclusion or required gate.", + "", + `Observed ${report.observedAt}; ${String(matchedDiagnostics.length)} of ${String(report.failures.length)} failure reports contain supported diagnostics matching a failed static task on an ancestor main revision. This does not attribute other failures in the same job. Unknown means evidence was incomplete or did not match.`, + "", + ]; + for (const item of report.failures) + lines.push( + `- Run ${item.runId} attempt ${item.runAttempt}${item.job ? `, ${item.job}` : ""}: **${item.classification}** — ${item.reason}.`, + ); + if (report.failures.length === 0) + lines.push( + "- No completed failed CI attempts were present in the supplied timing snapshot.", + ); + await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + await writeFile(markdownPath, `${lines.join("\n")}\n`, "utf8"); + if (summaryPath) + await appendFile(summaryPath, `${lines.join("\n")}\n`, "utf8"); + return { jsonPath, markdownPath }; +} + +async function main() { + const repository = process.env.CI_REPOSITORY; + const defaultBranch = process.env.CI_DEFAULT_BRANCH; + const batchPath = + process.env.CI_TIMING_MANIFEST ?? + join(BATCH_DIRECTORY, "batch-manifest.json"); + const batchManifest = JSON.parse(await readBoundedFile(batchPath)); + const api = createGitHubApi(); + await reportCiFailures({ + repository, + defaultBranch, + batchManifest, + api, + outputDirectory: process.env.CI_FAILURES_DIRECTORY ?? OUTPUT_DIRECTORY, + summaryPath: process.env.GITHUB_STEP_SUMMARY, + }); +} +async function readBoundedFile(path) { + const bytes = await readFile(path); + if (bytes.byteLength > MAX_JSON_BYTES) + throw new Error("timing manifest exceeds size limit"); + return bytes.toString("utf8"); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/ci/github-api.d.mts b/scripts/ci/github-api.d.mts new file mode 100644 index 0000000000..67e157e6de --- /dev/null +++ b/scripts/ci/github-api.d.mts @@ -0,0 +1,16 @@ +export interface GitHubApiBudget { + requests: number; + startedAt: number; + limitMs: number; +} + +export type GitHubApiRequest = ( + method: string, + endpoint: string, + budget: GitHubApiBudget, +) => Promise; + +export function createGhApiRequest(options?: { + env?: NodeJS.ProcessEnv; + now?: () => number; +}): GitHubApiRequest; diff --git a/scripts/ci/github-api.mjs b/scripts/ci/github-api.mjs new file mode 100644 index 0000000000..9567bf93bc --- /dev/null +++ b/scripts/ci/github-api.mjs @@ -0,0 +1,33 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const MAX_BUFFER = 8 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 10_000; + +/** Create a bounded GitHub CLI JSON transport with GET-only authority. */ +export function createGhApiRequest({ env = process.env, now = Date.now } = {}) { + return async (method, endpoint, budget) => { + if (method !== "GET") + throw new Error( + "GitHub API transport supports read-only GET requests only", + ); + const remaining = budget.limitMs - (now() - budget.startedAt); + if (remaining <= 0) throw new Error("lookup_time_budget_exhausted"); + const { stdout } = await execFileAsync( + "gh", + ["api", "--method", "GET", endpoint], + { + env, + encoding: "utf8", + maxBuffer: MAX_BUFFER, + timeout: Math.min(REQUEST_TIMEOUT_MS, remaining), + }, + ); + try { + return JSON.parse(stdout); + } catch (cause) { + throw new Error("github_api_malformed_json", { cause }); + } + }; +} diff --git a/scripts/ci/job-timings-batch.mjs b/scripts/ci/job-timings-batch.mjs index 842e884f0d..f2dca7143c 100644 --- a/scripts/ci/job-timings-batch.mjs +++ b/scripts/ci/job-timings-batch.mjs @@ -103,6 +103,9 @@ export async function collectJobTimingsBatch({ id: run.id, runAttempt: run.run_attempt, workflow: workflow.name, + event: typeof run.event === "string" ? run.event : null, + headBranch: + typeof run.head_branch === "string" ? run.head_branch : null, conclusion: run.conclusion ?? null, createdAt: run.created_at ?? null, }); diff --git a/scripts/ci/pr-ci-health.d.mts b/scripts/ci/pr-ci-health.d.mts new file mode 100644 index 0000000000..0db2454bdd --- /dev/null +++ b/scripts/ci/pr-ci-health.d.mts @@ -0,0 +1,95 @@ +export interface PrCiHead { + repositoryId: number | null; + repository?: string | null; + ref: string | null; + sha: string | null; +} + +export interface PrCiPullRequest { + number: number | null; + draft?: boolean; + identityValid: boolean; + mergeable?: boolean; + head: PrCiHead; + [key: string]: unknown; +} + +export interface PrCiWorkflowRun { + id?: number; + run_attempt?: number; + event?: string; + head_branch?: string; + head_repository?: { id?: number | null } | null; + head_sha?: string; + status?: string; + conclusion?: string | null; + created_at?: string; + html_url?: string; + pull_requests?: Array<{ + number?: number; + head?: { sha?: string; repo?: { id?: number | null } } | null; + }>; + [key: string]: unknown; +} + +export interface PrCiClassification { + category: string; + reason: string; + runs: Array>; +} + +export interface PrCiHealthReport { + schemaVersion: 1; + observedAt: string; + repository: string; + repositoryId: number | null; + defaultBranch: string; + lookbackDays: number; + scan: { + status: "complete" | "partial"; + cutoff: string; + pullRequestPages: number; + workflowRunPages: number; + pullRequestInventoryComplete: boolean; + workflowRunInventoryComplete: boolean; + pullRequestDetailsFetched: number; + pullRequestDetailsLimit: number; + apiRequests: number; + apiRequestLimit: number; + errors: string[]; + }; + counts: Record; + pullRequests: Array; +} + +export function collectPrCiHealth(options: { + repository: string; + repositoryId?: number | string | null; + defaultBranch: string; + lookbackDays?: number; + now?: number; + apiRequest?: GitHubApiRequest; +}): Promise; + +export function classifyPrCi(input: { + pull: PrCiPullRequest; + runs: Array<{ run: PrCiWorkflowRun; basis: string }>; + runsComplete: boolean; + pullRequestInventoryComplete: boolean; + ambiguousRuns: Array<{ run: PrCiWorkflowRun; basis: string }>; +}): PrCiClassification; + +export function matchRunToPullRequest(input: { + run: PrCiWorkflowRun; + pull: PrCiPullRequest; + identityCounts: Map; + inventoryComplete?: boolean; +}): { kind: "none" } | { kind: "ambiguous" } | { kind: "match"; basis: string }; + +export function writePrCiHealthReport( + report: PrCiHealthReport, + outputDirectory?: string, +): Promise<{ jsonPath: string; markdownPath: string }>; + +export function renderMarkdownSummary(report: PrCiHealthReport): string; +import type { GitHubApiRequest } from "./github-api.mjs"; diff --git a/scripts/ci/pr-ci-health.mjs b/scripts/ci/pr-ci-health.mjs new file mode 100644 index 0000000000..d312dcecd9 --- /dev/null +++ b/scripts/ci/pr-ci-health.mjs @@ -0,0 +1,718 @@ +import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { createGhApiRequest } from "./github-api.mjs"; + +const OUTPUT_DIRECTORY = join(".cache", "ci-pr-health"); +const PAGE_SIZE = 100; +const MAX_PULL_REQUEST_PAGES = 3; +const MAX_WORKFLOW_RUN_PAGES = 5; +const MAX_PULL_REQUEST_DETAILS = 40; +const MAX_API_REQUESTS = + MAX_PULL_REQUEST_PAGES + MAX_WORKFLOW_RUN_PAGES + MAX_PULL_REQUEST_DETAILS; +const LOOKBACK_DAYS = 14; +const MAX_LOOKBACK_DAYS = 30; +const MAX_TOTAL_TIME_MS = 60_000; +const SHA_PATTERN = /^[0-9a-f]{40}$/iu; +const COMPLETED_CATEGORIES = new Map([ + ["failure", "completed_failure"], + ["timed_out", "completed_failure"], + ["cancelled", "completed_canceled"], + ["success", "completed_success"], +]); + +/** Read workflow/PR state and conservatively distinguish absent CI from unknown. */ +export async function collectPrCiHealth({ + repository, + repositoryId, + defaultBranch, + lookbackDays = LOOKBACK_DAYS, + now = Date.now(), + apiRequest = createGhApiRequest(), +}) { + const identity = validateInputs({ + repository, + repositoryId, + defaultBranch, + lookbackDays, + now, + }); + const observedAt = new Date(now).toISOString(); + const cutoff = now - identity.lookbackDays * 24 * 60 * 60 * 1_000; + const snapshot = await collectApiSnapshot({ apiRequest, identity, cutoff }); + const classified = classifyPullRequests(snapshot); + const { pulls, runs, budget } = snapshot; + const errors = [pulls.error, runs.error, ...snapshot.detailsErrors].filter( + Boolean, + ); + return { + schemaVersion: 1, + observedAt, + repository: identity.repository, + repositoryId: identity.repositoryId, + defaultBranch: identity.defaultBranch, + lookbackDays: identity.lookbackDays, + scan: { + status: + errors.length === 0 && pulls.complete && runs.complete + ? "complete" + : "partial", + cutoff: new Date(cutoff).toISOString(), + pullRequestPages: pulls.pages, + workflowRunPages: runs.pages, + pullRequestInventoryComplete: pulls.complete, + workflowRunInventoryComplete: runs.complete, + pullRequestDetailsFetched: snapshot.detailsFetched, + pullRequestDetailsLimit: MAX_PULL_REQUEST_DETAILS, + apiRequests: budget.requests, + apiRequestLimit: MAX_API_REQUESTS, + errors, + }, + counts: countCategories(classified), + pullRequests: classified, + }; +} + +async function collectApiSnapshot({ apiRequest, identity, cutoff }) { + const budget = { + requests: 0, + startedAt: Date.now(), + limitMs: MAX_TOTAL_TIME_MS, + }; + const requestGet = async (endpoint) => { + if (budget.requests >= MAX_API_REQUESTS) + throw new Error("lookup_request_budget_exhausted"); + if (Date.now() - budget.startedAt >= budget.limitMs) + throw new Error("lookup_time_budget_exhausted"); + budget.requests += 1; + return await apiRequest("GET", endpoint, budget); + }; + const pulls = await listOpenPullRequests({ + requestGet, + identity, + }); + const candidates = pulls.items.filter( + (pull) => + !pull.draft && + Number.isFinite(Date.parse(pull.createdAt ?? "")) && + Date.parse(pull.createdAt ?? "") >= cutoff && + pull.base.ref === identity.defaultBranch && + (identity.repositoryId === null || + pull.base.repositoryId === identity.repositoryId), + ); + const runs = await listRecentWorkflowRuns({ + requestGet, + identity, + cutoff, + }); + + const detailResult = await resolveMergeability({ + candidates, + requestGet, + identity, + }); + return { + budget, + pulls, + runs, + candidates, + detailsFetched: detailResult.fetched, + detailsErrors: detailResult.errors, + }; +} + +function classifyPullRequests(snapshot) { + const { candidates, pulls, runs } = snapshot; + const identityCounts = countHeadIdentities(pulls.items); + return candidates.map((pull) => { + const relevant = []; + const ambiguousRuns = []; + for (const run of runs.items) { + const match = matchRunToPullRequest({ + run, + pull, + identityCounts, + inventoryComplete: pulls.complete, + }); + if (match.kind === "match") relevant.push({ run, basis: match.basis }); + if (match.kind === "ambiguous") + ambiguousRuns.push({ run, basis: "ambiguous_head_identity" }); + } + return { + ...pull, + ci: classifyPrCi({ + pull, + runs: relevant, + runsComplete: runs.complete, + pullRequestInventoryComplete: pulls.complete, + ambiguousRuns, + }), + }; + }); +} + +async function resolveMergeability({ candidates, requestGet, identity }) { + let detailsFetched = 0; + const errors = new Set(); + for (const pull of candidates) { + if (pull.mergeable !== undefined) continue; + if (detailsFetched >= MAX_PULL_REQUEST_DETAILS) { + errors.add("pull_request_mergeability_lookup_budget_exhausted"); + continue; + } + const result = await getPullRequestDetails({ requestGet, identity, pull }); + detailsFetched += 1; + if (result.error !== null) { + errors.add(result.error); + continue; + } + const { detail } = result; + const identityChanged = + detail.headSha !== pull.head.sha || + detail.headRef !== pull.head.ref || + detail.headRepositoryId !== pull.head.repositoryId || + detail.baseRef !== pull.base.ref || + detail.baseRepositoryId !== pull.base.repositoryId; + if (identityChanged) { + pull.identityValid = false; + pull.identityChangedDuringScan = true; + errors.add("pull_request_identity_changed_during_scan"); + } else { + pull.mergeable = detail.mergeable; + pull.detailFetched = true; + if (detail.mergeable === undefined) + errors.add("pull_request_mergeability_unknown"); + } + } + return { fetched: detailsFetched, errors: [...errors] }; +} + +function countCategories(pulls) { + const categories = [ + "missing", + "startup_failure", + "queued_pending_executing", + "approval_required", + "completed_success", + "completed_failure", + "completed_canceled", + "completed_other", + "merge_conflict", + "unknown", + "deferred", + ]; + return Object.fromEntries( + categories.map((category) => [ + category, + pulls.filter(({ ci }) => ci.category === category).length, + ]), + ); +} + +/** Pure status classifier; missing requires complete, identity-safe inventories. */ +export function classifyPrCi({ + pull, + runs, + runsComplete, + pullRequestInventoryComplete, + ambiguousRuns, +}) { + if (!pull.identityValid) + return result("unknown", "pull_request_head_identity_invalid", []); + const evidence = [...runs, ...ambiguousRuns]; + if (pull.mergeable === false) + return result("merge_conflict", "github_reports_merge_conflict", evidence); + // The newest collected run is not proven current until pagination completes: + // an older-created run outside this snapshot may have been retried recently. + if (!runsComplete) + return result( + evidence.length > 0 ? "unknown" : "deferred", + evidence.length > 0 + ? "workflow_run_inventory_incomplete" + : "lookup_inventory_incomplete", + evidence, + ); + const selection = newestRun(runs); + if (!selection.known) + return result("unknown", "workflow_run_order_unknown", evidence); + const latest = selection.latest; + if ( + ambiguousRuns.some((candidate) => { + const order = latest ? compareRunEvidence(candidate, latest) : null; + return order === null || order >= 0; + }) + ) + return result("unknown", "workflow_run_identity_ambiguous", evidence); + if (latest !== undefined) { + const run = latest.run; + if (run.conclusion === "startup_failure") + return result("startup_failure", "workflow_startup_failure", evidence); + if (run.status === "waiting" || run.conclusion === "action_required") + return result( + "approval_required", + "workflow_waiting_for_approval_or_gate", + evidence, + ); + if (["queued", "requested", "pending", "in_progress"].includes(run.status)) + return result( + "queued_pending_executing", + `workflow_${run.status}`, + evidence, + ); + if (run.status === "completed") + return result( + COMPLETED_CATEGORIES.get(run.conclusion) ?? "completed_other", + run.conclusion === "success" + ? "workflow_succeeded" + : `workflow_${run.conclusion ?? "unknown_conclusion"}`, + evidence, + ); + return result("unknown", "workflow_status_unrecognized", evidence); + } + if (!pullRequestInventoryComplete) + return result("deferred", "lookup_inventory_incomplete", []); + if (pull.mergeable !== true) + return result("unknown", "pull_request_mergeability_unknown", []); + return result("missing", "no_pull_request_event_run_for_current_head", []); +} + +/** Match only a pull_request event proven to belong to this immutable PR head. */ +export function matchRunToPullRequest({ + run, + pull, + identityCounts, + inventoryComplete = true, +}) { + if (run?.event !== "pull_request") return { kind: "none" }; + if (!pull.identityValid || !isRecord(run)) return { kind: "none" }; + const association = matchExplicitPullAssociation(run, pull); + if (association === "match") + return { kind: "match", basis: "event_pr_number_and_head_sha" }; + if (association === "ambiguous") return { kind: "ambiguous" }; + if (association === "other" || !sameHeadRepositoryAndRef(run, pull)) + return { kind: "none" }; + if (!uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete)) + return { kind: "ambiguous" }; + if (isSha(run.head_sha) && run.head_sha.toLowerCase() === pull.head.sha) + return { kind: "match", basis: "event_unique_head_ref_and_repository_id" }; + return { kind: "none" }; +} + +function matchExplicitPullAssociation(run, pull) { + const associations = Array.isArray(run.pull_requests) + ? run.pull_requests + : []; + if (associations.length === 0) return "absent"; + const matchingNumber = associations.filter( + (item) => item?.number === pull.number, + ); + if (matchingNumber.length === 0) return "other"; + const exactHead = matchingNumber.find( + (item) => item?.head?.sha === pull.head.sha, + ); + if (exactHead === undefined) { + return matchingNumber.some( + (item) => + typeof item?.head?.sha === "string" && item.head.sha !== pull.head.sha, + ) + ? "other" + : "absent"; + } + const associatedHeadRepositoryId = positiveInteger(exactHead.head?.repo?.id); + const runHeadRepositoryId = positiveInteger(run.head_repository?.id); + if (associatedHeadRepositoryId === null || runHeadRepositoryId === null) + return "ambiguous"; + if ( + associatedHeadRepositoryId !== pull.head.repositoryId || + runHeadRepositoryId !== pull.head.repositoryId + ) + return "other"; + const baseRefMatches = + typeof exactHead.base?.ref !== "string" || + exactHead.base.ref === pull.base.ref; + const baseRepositoryId = positiveInteger(exactHead.base?.repo?.id); + const baseRepositoryMatches = + baseRepositoryId === null || baseRepositoryId === pull.base.repositoryId; + return baseRefMatches && baseRepositoryMatches ? "match" : "other"; +} + +function sameHeadRepositoryAndRef(run, pull) { + return ( + run.head_branch === pull.head.ref && + run.head_repository?.id === pull.head.repositoryId + ); +} + +function uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete) { + if (!inventoryComplete) return false; + const key = headIdentityKey(pull.head.repositoryId, pull.head.ref); + const unknownRepositoryKey = headIdentityKey(null, pull.head.ref); + return ( + identityCounts.get(key) === 1 && + (identityCounts.get(unknownRepositoryKey) ?? 0) === 0 + ); +} + +function normalizePullRequest(raw, identity) { + const number = positiveInteger(raw?.number); + const base = normalizeBase(raw?.base); + const head = normalizeHead(raw?.head); + const draft = raw?.draft === true; + return { + number, + draft, + createdAt: typeof raw?.created_at === "string" ? raw.created_at : null, + url: typeof raw?.html_url === "string" ? raw.html_url : null, + base, + head, + mergeable: typeof raw?.mergeable === "boolean" ? raw.mergeable : undefined, + identityValid: hasValidPullIdentity({ raw, identity, number, base, head }), + }; +} + +function normalizeBase(raw) { + return { + ref: typeof raw?.ref === "string" ? raw.ref : null, + repositoryId: positiveInteger(raw?.repo?.id), + }; +} + +function normalizeHead(raw) { + return { + repositoryId: positiveInteger(raw?.repo?.id), + repository: + typeof raw?.repo?.full_name === "string" ? raw.repo.full_name : null, + ref: typeof raw?.ref === "string" ? raw.ref : null, + sha: + typeof raw?.sha === "string" && isSha(raw.sha) + ? raw.sha.toLowerCase() + : null, + }; +} + +function hasValidPullIdentity({ raw, identity, number, base, head }) { + return ( + number !== null && + typeof raw?.draft === "boolean" && + base.ref === identity.defaultBranch && + (identity.repositoryId === null || + base.repositoryId === identity.repositoryId) && + head.repositoryId !== null && + head.sha !== null && + head.ref !== null + ); +} + +function countHeadIdentities(pulls) { + const counts = new Map(); + for (const pull of pulls) { + if (typeof pull.head.ref !== "string") continue; + const key = headIdentityKey(pull.head.repositoryId, pull.head.ref); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return counts; +} + +function headIdentityKey(repositoryId, ref) { + return `${String(repositoryId)}\u0000${ref}`; +} + +function newestRun(runs) { + let latest = runs[0]; + for (const candidate of runs) { + if (latest && (compareRunEvidence(candidate, latest) ?? 0) > 0) + latest = candidate; + } + const known = runs.every((candidate) => { + if (candidate === latest) return true; + const order = compareRunEvidence(candidate, latest); + return order !== null && order <= 0; + }); + return { known, latest }; +} + +// Attempt ordering is meaningful within one run; different run IDs are not clocks. +function compareRunEvidence(left, right) { + const leftRun = left.run; + const rightRun = right.run; + const id = positiveInteger(leftRun.id); + if (id !== null && id === positiveInteger(rightRun.id)) { + const leftAttempt = positiveInteger(leftRun.run_attempt); + const rightAttempt = positiveInteger(rightRun.run_attempt); + if (leftAttempt === null || rightAttempt === null) return null; + if (leftAttempt !== rightAttempt) return leftAttempt - rightAttempt; + return leftRun.status === rightRun.status && + leftRun.conclusion === rightRun.conclusion + ? 0 + : null; + } + // A retry may start long after its original creation; without an attempt + // clock, creation dates only order first attempts from different runs. + if ( + positiveInteger(leftRun.run_attempt) !== 1 || + positiveInteger(rightRun.run_attempt) !== 1 + ) + return null; + const time = + Date.parse(leftRun.created_at ?? "") - + Date.parse(rightRun.created_at ?? ""); + return Number.isFinite(time) && time !== 0 ? time : null; +} + +function result(category, reason, runs) { + return { + category, + reason, + runs: runs.map(({ run, basis }) => ({ + id: positiveInteger(run.id), + attempt: positiveInteger(run.run_attempt), + status: typeof run.status === "string" ? run.status : null, + conclusion: typeof run.conclusion === "string" ? run.conclusion : null, + headSha: isSha(run.head_sha) ? run.head_sha.toLowerCase() : null, + createdAt: typeof run.created_at === "string" ? run.created_at : null, + url: typeof run.html_url === "string" ? run.html_url : null, + identityBasis: basis, + })), + }; +} + +async function listOpenPullRequests({ requestGet, identity }) { + const items = []; + let complete = false; + let pages = 0; + let error = null; + for (let page = 1; page <= MAX_PULL_REQUEST_PAGES; page += 1) { + try { + const endpoint = `repos/${identity.repository}/pulls?state=open&sort=created&direction=desc&per_page=${PAGE_SIZE}&page=${page}`; + const response = await requestGet(endpoint); + if (!Array.isArray(response)) + throw new Error("pull_request_page_malformed"); + pages += 1; + const normalized = response.map((raw) => + normalizePullRequest(raw, identity), + ); + if ( + normalized.some( + (pull) => !Number.isFinite(Date.parse(pull.createdAt ?? "")), + ) + ) + throw new Error("pull_request_page_malformed"); + items.push(...normalized); + if (response.length < PAGE_SIZE) { + complete = true; + break; + } + } catch (cause) { + error = safeErrorCode(cause); + break; + } + } + return { items, pages, complete, error }; +} + +async function listRecentWorkflowRuns({ requestGet, identity, cutoff }) { + const items = []; + let complete = false; + let pages = 0; + let error = null; + for (let page = 1; page <= MAX_WORKFLOW_RUN_PAGES; page += 1) { + try { + const cutoffDate = new Date(cutoff).toISOString().slice(0, 10); + const endpoint = `repos/${identity.repository}/actions/workflows/ci.yml/runs?event=pull_request&created=%3E%3D${cutoffDate}&per_page=${PAGE_SIZE}&page=${page}`; + const response = await requestGet(endpoint); + if (!isRecord(response) || !Array.isArray(response.workflow_runs)) + throw new Error("workflow_run_page_malformed"); + pages += 1; + if (!response.workflow_runs.every(isRecord)) + throw new Error("workflow_run_page_malformed"); + items.push(...response.workflow_runs); + if (response.workflow_runs.length < PAGE_SIZE) { + complete = true; + break; + } + } catch (cause) { + error = safeErrorCode(cause); + break; + } + } + return { items, pages, complete, error }; +} + +async function getPullRequestDetails({ requestGet, identity, pull }) { + if (pull.number === null) + return { detail: null, error: "pull_request_detail_identity_missing" }; + try { + const response = await requestGet( + `repos/${identity.repository}/pulls/${pull.number}`, + ); + if (!isRecord(response)) + return { detail: null, error: "pull_request_detail_malformed" }; + return { + detail: { + mergeable: + typeof response.mergeable === "boolean" + ? response.mergeable + : undefined, + headSha: isSha(response.head?.sha) + ? response.head.sha.toLowerCase() + : null, + headRef: + typeof response.head?.ref === "string" ? response.head.ref : null, + headRepositoryId: positiveInteger(response.head?.repo?.id), + baseRef: + typeof response.base?.ref === "string" ? response.base.ref : null, + baseRepositoryId: positiveInteger(response.base?.repo?.id), + }, + error: null, + }; + } catch (cause) { + return { detail: null, error: safeErrorCode(cause) }; + } +} + +function validateInputs({ + repository, + repositoryId, + defaultBranch, + lookbackDays, + now, +}) { + if ( + typeof repository !== "string" || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository) + ) + throw new Error( + "CI_REPOSITORY must be an owner/name repository identifier", + ); + if ( + typeof defaultBranch !== "string" || + defaultBranch.length === 0 || + /[\r\n]/u.test(defaultBranch) + ) + throw new Error("CI_DEFAULT_BRANCH must be a nonempty branch name"); + const parsedRepositoryId = + repositoryId === undefined || repositoryId === null || repositoryId === "" + ? null + : positiveInteger(repositoryId); + if ( + repositoryId !== undefined && + repositoryId !== null && + repositoryId !== "" && + parsedRepositoryId === null + ) + throw new Error("CI_REPOSITORY_ID must be a positive safe integer"); + if ( + !Number.isInteger(lookbackDays) || + lookbackDays < 1 || + lookbackDays > MAX_LOOKBACK_DAYS + ) + throw new Error( + `lookbackDays must be an integer from 1 to ${String(MAX_LOOKBACK_DAYS)}`, + ); + if (!Number.isFinite(now)) throw new Error("now must be a finite timestamp"); + return { + repository, + repositoryId: parsedRepositoryId, + defaultBranch, + lookbackDays, + }; +} + +function positiveInteger(value) { + const number = typeof value === "number" ? value : Number(value); + return Number.isSafeInteger(number) && number > 0 ? number : null; +} + +function isSha(value) { + return typeof value === "string" && SHA_PATTERN.test(value); +} + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function safeErrorCode(cause) { + if (cause?.message === "lookup_time_budget_exhausted") + return "lookup_time_budget_exhausted"; + if (cause?.message === "lookup_request_budget_exhausted") + return "lookup_request_budget_exhausted"; + if (cause?.message === "github_api_malformed_json") + return "github_api_malformed_json"; + if (cause?.message?.endsWith("_malformed")) return cause.message; + if (cause?.code === "ETIMEDOUT" || cause?.killed) return "github_api_timeout"; + return "github_api_request_failed"; +} + +export async function writePrCiHealthReport( + report, + outputDirectory = OUTPUT_DIRECTORY, +) { + const jsonPath = join(outputDirectory, "report.json"); + const markdownPath = join(outputDirectory, "summary.md"); + await mkdir(outputDirectory, { recursive: true }); + await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + await writeFile(markdownPath, renderMarkdownSummary(report), "utf8"); + return { jsonPath, markdownPath }; +} + +export function renderMarkdownSummary(report) { + const rows = report.pullRequests.map( + (pull) => + `| #${String(pull.number ?? "?")} | ${escapeCell(pull.head.repository ?? "unknown")} | ${escapeCell(pull.head.ref ?? "unknown")} | \`${pull.head.sha ?? "unknown"}\` | ${pull.ci.category} | ${escapeCell(pull.ci.reason)} |`, + ); + return [ + "# Pull request CI health snapshot", + "", + `Observed ${report.observedAt} for \`${report.repository}\`, target branch \`${escapeCell(report.defaultBranch)}\`; recent window ${String(report.lookbackDays)} days. Scan: **${report.scan.status}**.`, + "", + "This is read-only point-in-time telemetry. Missing is reported only when PR and workflow-run inventories are complete and the current immutable PR head identity is unambiguous. Incomplete lookups remain deferred or unknown.", + "", + `API pages: ${String(report.scan.pullRequestPages)} PR, ${String(report.scan.workflowRunPages)} workflow runs; ${String(report.scan.apiRequests)} bounded requests. Errors: ${report.scan.errors.length === 0 ? "none" : report.scan.errors.join(", ")}.`, + "", + "| PR | Head repository | Head ref | Head SHA | CI classification | Evidence |", + "| ---: | --- | --- | --- | --- | --- |", + ...rows, + "", + "Categories: missing, startup failure, queued/pending/executing, approval required, completed outcomes, merge conflict, unknown, and deferred.", + "", + ].join("\n"); +} + +function escapeCell(value) { + return String(value) + .replaceAll("|", "\\|") + .replaceAll("\n", " ") + .replaceAll("\r", " "); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + const report = await collectPrCiHealth({ + repository: process.env.CI_REPOSITORY, + repositoryId: process.env.CI_REPOSITORY_ID, + defaultBranch: process.env.CI_DEFAULT_BRANCH, + }); + const paths = await writePrCiHealthReport( + report, + process.env.CI_PR_HEALTH_DIRECTORY ?? OUTPUT_DIRECTORY, + ); + if (process.env.GITHUB_STEP_SUMMARY) + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + renderMarkdownSummary(report), + "utf8", + ); + process.stdout.write( + `Wrote ${report.pullRequests.length} recent PR CI records to ${paths.jsonPath}\n`, + ); + } catch (cause) { + process.stderr.write( + `PR CI health telemetry failed: ${cause instanceof Error ? cause.message : "unknown_error"}\n`, + ); + process.exitCode = 1; + } +} diff --git a/scripts/ci/qualification.mjs b/scripts/ci/qualification.mjs index ab47c168f2..8dfb3f20c9 100644 --- a/scripts/ci/qualification.mjs +++ b/scripts/ci/qualification.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { normalizeWorkflowRun } from "./workflow-run.mjs"; const ARTIFACT_NAME = "ci-full-qualification-v1"; const MAX_AGE_MS = 24 * 60 * 60 * 1000; @@ -119,8 +120,8 @@ function admit() { // Artifact metadata narrows the search before run lookups. Never scan an // unbounded history or treat a missing receipt as a successful qualification. for (const artifact of candidates.slice(0, 3)) { - const run = api( - `repos/${repository}/actions/runs/${artifact.workflow_run.id}`, + const run = normalizeWorkflowRun( + api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`), ); // An old run can upload a new artifact when only its gate is rerun. // Bound the original run age as well so retries cannot renew stale proof. @@ -133,7 +134,7 @@ function admit() { run.id === artifact.workflow_run.id && run.status === "completed" && run.conclusion === "success" && - run.path === ".github/workflows/ci.yml" && + run.workflowPath?.file === ".github/workflows/ci.yml" && ["schedule", "workflow_dispatch"].includes(run.event) && run.head_sha === head && run.head_branch === branch && diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index 20fbac23fd..1839a11498 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -30,7 +30,11 @@ if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); const required = reused ? ["changes"] : ["changes", "static", "test"]; const lanes = { - code: ["build", "test-shard"], + code: [ + "build", + "test-shard", + ...(outputs.tier === "fast" ? ["test-source"] : []), + ], reverse_tools: ["reverse-tools-quality"], runtime: ["windows-native-build", "package-e2e"], package: ["windows-curated"], diff --git a/scripts/ci/static-check-report.mjs b/scripts/ci/static-check-report.mjs index 04460b2c20..eecaf750af 100644 --- a/scripts/ci/static-check-report.mjs +++ b/scripts/ci/static-check-report.mjs @@ -133,6 +133,8 @@ try { for (const clone of report.duplicates) for (const file of [clone.firstFile, clone.secondFile]) file.name = relative(process.cwd(), file.name).split(sep).join("/"); + // Record the actual checker result, including caller-selected thresholds. + report.reaCheck = { exitCode: result.status }; await writeFile(reportPath, JSON.stringify(report, null, 2)); const { sources, clones, percentage } = report.statistics.total; console.log( diff --git a/scripts/ci/static-task-context.d.mts b/scripts/ci/static-task-context.d.mts new file mode 100644 index 0000000000..164756b15d --- /dev/null +++ b/scripts/ci/static-task-context.d.mts @@ -0,0 +1,18 @@ +export interface StaticTaskContext { + readonly version: 1; + readonly sourceSha: string; + readonly sourceClean: boolean; + readonly prHeadSha?: string; + readonly nodeVersion: string; + readonly npmVersion: string; + readonly turboVersion: string; + readonly platform: string; + readonly architecture: string; + readonly taskHashes: Readonly>; +} +export function parseStaticTaskContext(value: unknown): StaticTaskContext; +export function recordStaticTaskContext(options?: { + root?: string; + npmCli?: string; + outputDirectory?: string; +}): Promise; diff --git a/scripts/ci/static-task-context.mjs b/scripts/ci/static-task-context.mjs new file mode 100644 index 0000000000..879c568d19 --- /dev/null +++ b/scripts/ci/static-task-context.mjs @@ -0,0 +1,104 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { z } from "zod"; + +const execute = promisify(execFile); +const contextSchema = z.object({ + version: z.literal(1), + sourceSha: z.string().regex(/^[a-f0-9]{40}$/u), + sourceClean: z.boolean(), + prHeadSha: z + .string() + .regex(/^[a-f0-9]{40}$/u) + .optional(), + nodeVersion: z.string().min(1), + npmVersion: z.string().min(1), + turboVersion: z.string().min(1), + platform: z.string().min(1), + architecture: z.string().min(1), + taskHashes: z.record(z.string(), z.string().regex(/^[a-f0-9]{16}$/u)), +}); + +/** Parse the producing check graph's receipt; incomplete provenance is unknown. */ +export const parseStaticTaskContext = (value) => contextSchema.parse(value); + +/** Retain only named task hashes and tool/source identity, never the environment. */ +export async function recordStaticTaskContext({ + root = process.cwd(), + npmCli = process.env.npm_execpath, + outputDirectory = join(root, ".cache/static-checks"), +} = {}) { + if (typeof npmCli !== "string" || !npmCli) + throw new Error( + "Run static context recording through npm run ci:static-context", + ); + const options = { + cwd: root, + encoding: "utf8", + timeout: 30_000, + maxBuffer: 32 * 1024 * 1024, + }; + const [graph, source, status, npmVersion, turbo] = await Promise.all([ + execute( + process.execPath, + [npmCli, "run", "--silent", "check:ci:tasks", "--", "--dry=json"], + options, + ), + execute("git", ["rev-parse", "HEAD"], options), + execute( + "git", + ["status", "--porcelain", "--untracked-files=normal"], + options, + ), + execute(process.execPath, [npmCli, "--version"], options), + readFile(join(root, "node_modules/turbo/package.json"), "utf8"), + ]); + const tasks = z + .object({ + tasks: z + .array( + z.object({ + task: z.string().min(1), + hash: z.string().regex(/^[a-f0-9]{16}$/u), + }), + ) + .min(1), + }) + .parse(JSON.parse(graph.stdout)).tasks; + if (new Set(tasks.map(({ task }) => task)).size !== tasks.length) + throw new Error("Static graph contains duplicate task identities"); + const context = parseStaticTaskContext({ + version: 1, + sourceSha: source.stdout.trim(), + sourceClean: status.stdout.length === 0, + ...(process.env.CI_PR_HEAD_SHA + ? { prHeadSha: process.env.CI_PR_HEAD_SHA } + : {}), + nodeVersion: process.version, + npmVersion: npmVersion.stdout.trim(), + turboVersion: z.object({ version: z.string() }).parse(JSON.parse(turbo)) + .version, + platform: process.platform, + architecture: process.arch, + taskHashes: Object.fromEntries(tasks.map(({ task, hash }) => [task, hash])), + }); + if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== context.sourceSha) + throw new Error( + "Static context source does not match the workflow revision", + ); + await mkdir(outputDirectory, { recursive: true }); + await writeFile( + join(outputDirectory, "static-task-context.json"), + JSON.stringify(context, null, 2) + "\n", + ); + return context; +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) + await recordStaticTaskContext(); diff --git a/scripts/ci/test-group.mjs b/scripts/ci/test-group.mjs new file mode 100644 index 0000000000..7d99175b4e --- /dev/null +++ b/scripts/ci/test-group.mjs @@ -0,0 +1,46 @@ +import { spawn } from "node:child_process"; +import { resolve } from "node:path"; + +import { + BUILT_TEST_PROJECTS, + SOURCE_TEST_PROJECTS, +} from "../lib/test-projects.mjs"; + +const [group, ...arguments_] = process.argv.slice(2); +const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[ + group +]; +if (!projects) throw new Error("Expected source or built test group"); +// The group owns project selection; callers can request reporters or shards. +if ( + arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument)) +) + throw new Error("Test groups cannot override their project inventory"); +const child = spawn( + process.execPath, + [ + resolve("node_modules/vitest/vitest.mjs"), + "run", + ...projects.flatMap(({ name }) => ["--project", name]), + ...arguments_, + ], + { stdio: "inherit" }, +); +let shutdownSignal; +for (const signal of ["SIGINT", "SIGTERM"]) + process.on(signal, () => { + shutdownSignal ??= signal; + child.kill(signal); + }); +child.once("error", (error) => { + console.error(error.message); + process.exitCode = 1; +}); +child.once("close", (code) => { + process.exitCode = + shutdownSignal === "SIGINT" + ? 130 + : shutdownSignal === "SIGTERM" + ? 143 + : (code ?? 1); +}); diff --git a/scripts/ci/test-history.mjs b/scripts/ci/test-history.mjs index 4386f48af3..14dbb582b4 100644 --- a/scripts/ci/test-history.mjs +++ b/scripts/ci/test-history.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { normalizeWorkflowRun } from "./workflow-run.mjs"; const ARTIFACT_NAME = "test-timings"; const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; @@ -68,8 +69,8 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) { .slice(0, MAX_CANDIDATES); for (const artifact of candidates) { - const run = api( - `repos/${repository}/actions/runs/${artifact.workflow_run.id}`, + const run = normalizeWorkflowRun( + api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`), ); const runAge = now - Date.parse(run?.created_at); if ( @@ -80,7 +81,7 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) { run.id === artifact.workflow_run.id && run.status === "completed" && run.conclusion === "success" && - run.path === ".github/workflows/ci.yml" && + run.workflowPath?.file === ".github/workflows/ci.yml" && ["schedule", "workflow_dispatch"].includes(run.event) && run.head_branch === branch && run.head_repository?.full_name === repository diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 035a019908..7f85c6ec5b 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -41,9 +41,20 @@ const files = report.files.map(({ name, durationMs, state }) => { throw new Error(`Test report path is outside the checkout: ${name}`); return { name: path.split(sep).join("/"), durationMs, state }; }); -const complete = process.env.SHARD_RESULT === "success"; +// Keep group outcomes intact: an unavailable group is not a completed failure. +const groupResults = [ + process.env.SHARD_RESULT, + ...(process.env.SOURCE_REQUIRED === "true" + ? [process.env.SOURCE_RESULT] + : []), +]; +const groupsCompleted = groupResults.every((state) => + ["success", "failure"].includes(state), +); const passed = - complete && process.env.MERGE_RESULT === "success" && report.success; + groupResults.every((state) => state === "success") && + process.env.MERGE_RESULT === "success" && + report.success; // Successful shards and merged reports must account for the complete portable // inventory. Coverage thresholds alone cannot prove that every file ran. const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true"; @@ -69,7 +80,7 @@ if (process.env.CI_TEST_SELECTION) { comparison.complete && report.unhandledErrors === 0 && ["passed", "failed"].includes(report.reason) && - ["success", "failure"].includes(process.env.SHARD_RESULT) && + groupsCompleted && ["success", "failure"].includes(process.env.MERGE_RESULT), executionSha: process.env.GITHUB_SHA ?? null, prHeadSha: process.env.CI_PR_HEAD_SHA ?? null, @@ -91,7 +102,9 @@ if (passed && verifyInventory) { ); } const summary = [ - complete ? "## Merged test diagnostics" : "## Partial test diagnostics", + groupsCompleted + ? "## Merged test diagnostics" + : "## Partial test diagnostics", "", ...(selection ? [ @@ -99,11 +112,14 @@ const summary = [ "", `Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`, `Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`, - "The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", + "This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", "", ] : []), `Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`, + ...(process.env.SOURCE_REQUIRED === "true" + ? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`] + : []), passed ? "Successful timing history is available." : "This report is diagnostic only and is not a successful timing baseline.", diff --git a/scripts/ci/workflow-run.d.mts b/scripts/ci/workflow-run.d.mts new file mode 100644 index 0000000000..b0e6263780 --- /dev/null +++ b/scripts/ci/workflow-run.d.mts @@ -0,0 +1,10 @@ +export interface WorkflowPath { + readonly file: string; + readonly ref: string | null; +} +export interface NormalizedWorkflowRun { + readonly workflowPath: WorkflowPath | null; +} +export function normalizeWorkflowRun>( + value: T, +): Omit & NormalizedWorkflowRun; diff --git a/scripts/ci/workflow-run.mjs b/scripts/ci/workflow-run.mjs new file mode 100644 index 0000000000..8c33214bf0 --- /dev/null +++ b/scripts/ci/workflow-run.mjs @@ -0,0 +1,15 @@ +/** Keep the API path as evidence and parse its optional ref at the boundary. */ +export function normalizeWorkflowRun(value) { + if (typeof value !== "object" || value === null || Array.isArray(value)) + throw new Error("workflow_run_malformed"); + const match = + typeof value.path === "string" && !/[\u0000-\u001f\u007f]/u.test(value.path) + ? /^(\.github\/workflows\/[^/\\@\s]+\.ya?ml)(?:@([^\s]+))?$/u.exec( + value.path, + ) + : null; + return { + ...value, + workflowPath: match ? { file: match[1], ref: match[2] ?? null } : null, + }; +} diff --git a/scripts/lib/development-tests.mjs b/scripts/lib/development-tests.mjs index 8ddebe115b..6aaf10768f 100644 --- a/scripts/lib/development-tests.mjs +++ b/scripts/lib/development-tests.mjs @@ -1,17 +1,4 @@ -const sourceProjects = [ - "domain", - "services", - "adapters", - "composition", - "conformance", - "evaluation", -]; - -const sourceTest = (path) => - path.startsWith("src/") || - ["composition", "conformance", "evaluation"].some((project) => - path.startsWith(`tests/${project}/`), - ); +import { SOURCE_TEST_PROJECTS, projectForTest } from "./test-projects.mjs"; /** Parse explicit test paths and an optional branch-comparison base. */ export const parseDevelopmentTestRequest = (mode, arguments_) => { @@ -35,10 +22,9 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => { path.split("/").some((part) => part === ".." || part === ".") ) throw new Error(`Expected an exact repository test path: ${argument}`); - if (mode !== "focused" && !sourceTest(path)) - throw new Error( - `Use test:focused for compiled boundary tests: ${path}`, - ); + const project = projectForTest(path); + if (mode !== "focused" && project.group !== "source") + throw new Error(`Use test:focused for built test projects: ${path}`); paths.push(path); } } @@ -72,9 +58,14 @@ export const developmentTestPlan = (request, baseCommit) => { !/^[a-f0-9]{40,64}$/u.test(baseCommit ?? "") ) throw new Error("Changed tests require a resolved Git merge base"); - const projects = request.mode === "focused" ? [] : sourceProjects; + const projects = + request.mode === "focused" + ? [] + : SOURCE_TEST_PROJECTS.map(({ name }) => name); return { - needsBuild: request.paths.some((path) => !sourceTest(path)), + needsBuild: request.paths.some( + (path) => projectForTest(path).group === "built", + ), artifactTasks: Object.entries(artifactConsumers).flatMap( ([task, consumers]) => request.paths.some((path) => consumers.includes(path)) ? [task] : [], diff --git a/scripts/lib/test-projects.d.mts b/scripts/lib/test-projects.d.mts new file mode 100644 index 0000000000..a9d6ae3563 --- /dev/null +++ b/scripts/lib/test-projects.d.mts @@ -0,0 +1,10 @@ +export interface TestProjectDefinition { + readonly name: string; + readonly group: "source" | "built"; + readonly include: readonly string[]; + readonly exclude?: readonly string[]; +} + +export const SOURCE_TEST_PROJECTS: readonly TestProjectDefinition[]; +export const BUILT_TEST_PROJECTS: readonly TestProjectDefinition[]; +export function projectForTest(testPath: string): TestProjectDefinition; diff --git a/scripts/lib/test-projects.mjs b/scripts/lib/test-projects.mjs new file mode 100644 index 0000000000..68fb90dbd0 --- /dev/null +++ b/scripts/lib/test-projects.mjs @@ -0,0 +1,107 @@ +import { matchesGlob } from "node:path"; + +/** Canonical file ownership shared by Vitest and development-test planning. */ +export const SOURCE_TEST_PROJECTS = [ + { + name: "domain", + group: "source", + include: ["src/{contracts,domain}/**/*.test.ts"], + }, + { + name: "services", + group: "source", + include: ["src/application/**/*.test.ts"], + }, + { + name: "adapters", + group: "source", + include: [ + "src/*.test.ts", + "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", + ], + }, + { + name: "composition", + group: "source", + include: ["tests/composition/**/*.test.ts"], + }, + { + name: "conformance", + group: "source", + include: ["tests/conformance/**/*.test.ts"], + exclude: ["tests/conformance/providers/hopper/realHopperSearch.test.ts"], + }, + { + name: "evaluation", + group: "source", + include: ["tests/evaluation/**/*.test.ts"], + }, +]; + +export const BUILT_TEST_PROJECTS = [ + { + name: "boundary", + group: "built", + include: [ + "tests/boundary/**/*.test.ts", + "tests/conformance/providers/hopper/realHopperSearch.test.ts", + ], + exclude: [ + "tests/boundary/cli/**/*.test.ts", + "tests/boundary/mcp/**/*.test.ts", + "tests/boundary/process/**/*.test.ts", + ], + }, + { + name: "cli-boundary", + group: "built", + include: ["tests/boundary/cli/**/*.test.ts"], + }, + { + name: "process-boundary", + group: "built", + include: ["tests/boundary/process/**/*.test.ts"], + }, + { + name: "mcp-boundary", + group: "built", + include: ["tests/boundary/mcp/**/*.test.ts"], + }, + { + name: "acceptance", + group: "built", + include: ["tests/acceptance/**/*.test.ts"], + }, + { + name: "process-global", + group: "built", + include: ["tests/process-global/**/*.test.ts"], + }, +]; + +const ALL_TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]; + +/** Return the unique Vitest owner for an exact repository-relative test path. */ +export const projectForTest = (testPath) => { + if ( + typeof testPath !== "string" || + testPath.startsWith("/") || + testPath.includes("\\") || + testPath.split("/").some((part) => part === ".." || part === ".") || + !testPath.endsWith(".test.ts") + ) + throw new Error(`Expected a normalized repository test path: ${testPath}`); + + const owners = ALL_TEST_PROJECTS.filter( + ({ include, exclude = [] }) => + include.some((pattern) => matchesGlob(testPath, pattern)) && + !exclude.some((pattern) => matchesGlob(testPath, pattern)), + ); + if (owners.length !== 1) + throw new Error( + owners.length === 0 + ? `No test project owns ${testPath}` + : `Multiple test projects own ${testPath}: ${owners.map(({ name }) => name).join(", ")}`, + ); + return owners[0]; +}; diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index 3c14636d72..5939d458a6 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -961,7 +961,7 @@ it("does not impose a message-count cutoff below the byte budget", () => { ); expect( boundedViewMessages(messages, `ev_${"a".repeat(64)}`, "limitations"), - ).toEqual(messages); + ).toEqual(messages.toSorted()); }); it("bounds repeated view messages even when fewer than 64 source strings are huge", () => { diff --git a/tests/boundary/cli/developmentTests.test.ts b/tests/boundary/cli/developmentTests.test.ts index 2987206c87..7becdd4e7e 100644 --- a/tests/boundary/cli/developmentTests.test.ts +++ b/tests/boundary/cli/developmentTests.test.ts @@ -45,6 +45,7 @@ const fixture = async () => { for (const path of [ "scripts/run-development-tests.mjs", "scripts/lib/development-tests.mjs", + "scripts/lib/test-projects.mjs", ]) await copyFile(join(process.cwd(), path), join(root, path)); await symlink( diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.ts new file mode 100644 index 0000000000..26feb3ee29 --- /dev/null +++ b/tests/boundary/filesystem/ciFailureReport.test.ts @@ -0,0 +1,419 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs"; + +import { + readStaticCheckArtifact, + selectStaticCheckArtifact, +} from "../../../scripts/ci/artifacts.mjs"; +import { + classifyFailure, + createGitHubApi, + reportCiFailures, + type CiRun, + type CiJob, + type FailureReportApi, +} from "../../../scripts/ci/failure-report.mjs"; + +const sha = (digit: string) => digit.repeat(40); +const diagnostic = { + kind: "lint", + path: "src/example.ts", + code: "no-debugger", + message: "Unexpected debugger statement.", + severity: "error", +} as const; +const context = ( + sourceSha: string, + taskHash = "a".repeat(16), + prHeadSha?: string, +) => ({ + version: 1 as const, + sourceSha, + ...(prHeadSha ? { prHeadSha } : {}), + sourceClean: true, + nodeVersion: "v24.18.0", + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: "linux", + architecture: "x64", + taskHashes: { "lint:built": taskHash }, +}); +const run = (overrides: Partial = {}) => + normalizeWorkflowRun({ + id: 10, + run_attempt: 1, + status: "completed", + conclusion: "failure", + event: "pull_request", + path: ".github/workflows/ci.yml@main", + head_sha: sha("a"), + head_branch: "feature", + repository: { full_name: "owner/rea" }, + pull_requests: [ + { base: { sha: sha("b"), repo: { full_name: "owner/rea" } } }, + ], + ...overrides, + }); +const job = (overrides: Partial = {}) => ({ + name: "Static checks", + status: "completed", + conclusion: "failure", + ...overrides, +}); + +const comparison = (): Parameters[0] => ({ + repository: "owner/rea", + defaultBranch: "main", + run: run(), + job: job(), + diagnostics: [diagnostic], + context: context(sha("a")), + mainRun: run({ + id: 20, + event: "push", + head_sha: sha("c"), + head_branch: "main", + pull_requests: [], + }), + mainJob: job(), + mainDiagnostics: [diagnostic], + mainContext: context(sha("c")), + executionSourcesVerified: true, + ancestry: { status: "ahead", mergeBaseSha: sha("c") }, + blobs: { "src/example.ts": true }, +}); + +it("attributes only an identical structured lint failure with matching task, context, ancestry, and source blob", () => { + const result = classifyFailure(comparison()); + expect(result).toEqual({ + classification: "matching-main-diagnostics", + reason: + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + }); +}); + +it.each([ + ["task hash", { mainContext: context(sha("c"), "b".repeat(16)) }], + ["dirty source", { context: { ...context(sha("a")), sourceClean: false } }], + [ + "toolchain", + { mainContext: { ...context(sha("c")), turboVersion: "other" } }, + ], + [ + "source sha", + { context: context(sha("d")), executionSourcesVerified: false }, + ], + [ + "diagnostic", + { mainDiagnostics: [{ ...diagnostic, message: "different message" }] }, + ], + ["content", { blobs: { "src/example.ts": false } }], + ["ancestry", { ancestry: { status: "diverged", mergeBaseSha: sha("e") } }], + ["canceled job", { job: job({ conclusion: "cancelled" }) }], + ["not a static job", { job: job({ name: "Test shard (1)" }) }], +] as const)("does not attribute on mismatched %s", (_label, overrides) => { + const result = classifyFailure({ ...comparison(), ...overrides }); + expect(result.classification).toBe("unknown"); +}); + +it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => { + const directory = await createTestTempDirectory("rea-ci-failure-test-"); + const calls: string[] = []; + const manifest = { + repository: "owner/rea", + truncation: [], + coverage: { exhaustive: false }, + runs: [ + { + id: 10, + runAttempt: 1, + workflow: "CI", + event: "pull_request", + conclusion: "failure", + status: "collected", + }, + { + id: 11, + runAttempt: 1, + workflow: "CI", + event: "pull_request", + conclusion: "success", + status: "collected", + }, + ], + }; + const failedRun = run(); + const api: FailureReportApi = { + async getRun(_repository, id) { + calls.push(`run:${id}`); + return failedRun; + }, + async getJobs(_repository, id) { + calls.push(`jobs:${id}`); + return [{ ...job(), run_id: Number(id), run_attempt: 1 }]; + }, + async getStaticEvidence() { + calls.push("artifact"); + return null; + }, + async getAncestry() { + throw new Error("must not compare without evidence"); + }, + async verifyExecutionSource() { + throw new Error("must not verify source without evidence"); + }, + async getBlobIdentity() { + throw new Error("must not inspect blobs without evidence"); + }, + }; + const report = await reportCiFailures({ + repository: "owner/rea", + defaultBranch: "main", + batchManifest: manifest, + api, + outputDirectory: directory, + }); + expect(calls).toEqual(["run:10", "jobs:10", "artifact"]); + expect(report).toMatchObject({ + informationalOnly: true, + gateEffect: "none; this report never changes CI conclusions", + counts: { unknown: 1 }, + }); + const json = JSON.parse( + await readFile(join(directory, "report.json"), "utf8"), + ); + const markdown = await readFile(join(directory, "report.md"), "utf8"); + expect(json.failures[0]).toMatchObject({ + classification: "unknown", + reason: "no_matching_verified_main_failure", + }); + expect(markdown).toContain("Informational only"); + expect(markdown).toContain("Unknown means evidence was incomplete"); +}); + +it.each([ + [22, false], + [6, true], +] as const)( + "finds verified main evidence after %s newer PR runs (extra incorrect main hint: %s)", + async (prCount, incorrectMainHints) => { + const directory = await createTestTempDirectory("rea-ci-main-proof-"); + const mainSha = sha("c"); + const prHeadSha = sha("a"); + const mergeSha = sha("d"); + const mainRun = run({ + id: 20, + event: "push", + head_sha: mainSha, + head_branch: "main", + pull_requests: [], + }); + const prRun = run({ + head_sha: prHeadSha, + pull_requests: [ + { base: { sha: mainSha, repo: { full_name: "owner/rea" } } }, + ], + }); + const evidence = (sourceSha: string, prHead?: string) => ({ + context: context(sourceSha, undefined, prHead), + diagnostics: [diagnostic], + }); + const lookups: number[] = []; + const api: FailureReportApi = { + async getRun(_repository, id) { + lookups.push(Number(id)); + return Number(id) === 20 ? mainRun : { ...prRun, id }; + }, + async getJobs(_repository, id) { + return [{ ...job(), run_id: Number(id), run_attempt: 1 }]; + }, + async getStaticEvidence(_repository, selectedRun) { + return Number(selectedRun.id) === 20 + ? evidence(mainSha) + : evidence(mergeSha, prHeadSha); + }, + async verifyExecutionSource(_repository, selectedRun, sourceContext) { + return selectedRun.event === "pull_request" + ? sourceContext.sourceSha === mergeSha && + sourceContext.prHeadSha === selectedRun.head_sha + : sourceContext.sourceSha === selectedRun.head_sha; + }, + async getAncestry() { + return { status: "ahead", mergeBaseSha: mainSha }; + }, + async getBlobIdentity() { + return { "src/example.ts": true }; + }, + }; + const result = await reportCiFailures({ + repository: "owner/rea", + defaultBranch: "main", + batchManifest: { + repository: "owner/rea", + runs: [ + ...(incorrectMainHints + ? [ + { + id: 30, + runAttempt: 1, + workflow: "CI", + event: "push", + headBranch: "main", + conclusion: "failure", + status: "collected", + }, + ] + : []), + ...Array.from({ length: prCount }, (_, index) => ({ + id: 100 + index, + runAttempt: 1, + workflow: "CI", + event: "pull_request", + headBranch: "feature", + conclusion: "failure", + status: "collected", + })), + { + id: 20, + runAttempt: 1, + workflow: "CI", + event: "push", + headBranch: "main", + conclusion: "failure", + status: "collected", + }, + ], + }, + api, + outputDirectory: directory, + }); + expect(result.failures[0]).toMatchObject({ + runId: 100, + classification: "matching-main-diagnostics", + reason: + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + }); + expect(lookups).toHaveLength( + Math.min(prCount, 20) + 1 + Number(incorrectMainHints), + ); + expect(lookups[0]).toBe(incorrectMainHints ? 30 : 20); + expect(new Set(lookups).size).toBe(lookups.length); + expect(result.boundedInput.mainCandidateRuns).toBe(1); + expect(result.failures).toHaveLength(Math.min(prCount, 20)); + }, +); + +it("proves checkout merge parents in base-then-PR-head order and rejects false identities", async () => { + const baseSha = sha("b"); + const headSha = sha("a"); + const mergeSha = sha("c"); + const prRun = run({ + head_sha: headSha, + pull_requests: [ + { base: { sha: baseSha, repo: { full_name: "owner/rea" } } }, + ], + }); + let parents = [baseSha, headSha]; + const calls: string[] = []; + const api = createGitHubApi({ + token: "fixture-token", + apiRequest: async (_method, endpoint) => { + calls.push(endpoint); + return { + sha: mergeSha, + parents: parents.map((parentSha) => ({ sha: parentSha })), + }; + }, + }); + const mergeContext = context(mergeSha, undefined, headSha); + expect( + await api.verifyExecutionSource("owner/rea", prRun, mergeContext), + ).toBe(true); + expect(calls).toEqual([`repos/owner/rea/commits/${mergeSha}`]); + + parents = [headSha, baseSha]; + expect( + await api.verifyExecutionSource("owner/rea", prRun, mergeContext), + ).toBe(false); + expect( + await api.verifyExecutionSource("owner/rea", prRun, { + ...mergeContext, + prHeadSha: sha("e"), + }), + ).toBe(false); +}); + +it("uses static artifacts only when listing is complete, unique, and tied to this run attempt", () => { + const artifact = { + id: 12, + name: "static-check-reports", + expired: false, + workflow_run: { id: 10, head_sha: sha("a") }, + }; + const runRecord = { id: 10, head_sha: sha("a") }; + expect( + selectStaticCheckArtifact( + { total_count: 1, artifacts: [artifact] }, + runRecord, + 1, + ), + ).toBe(artifact); + expect( + selectStaticCheckArtifact( + { total_count: 2, artifacts: [artifact] }, + runRecord, + 1, + ), + ).toBeNull(); + expect( + selectStaticCheckArtifact( + { total_count: 2, artifacts: [artifact, artifact] }, + runRecord, + 1, + ), + ).toBeNull(); + expect( + selectStaticCheckArtifact( + { + total_count: 1, + artifacts: [ + { ...artifact, workflow_run: { id: 11, head_sha: sha("a") } }, + ], + }, + runRecord, + 1, + ), + ).toBeNull(); + expect( + selectStaticCheckArtifact( + { total_count: 1, artifacts: [artifact] }, + runRecord, + 2, + ), + ).toBeNull(); +}); + +it("rejects artifact redirects that could send the bearer token to a non-HTTPS host", async () => { + const calls: string[] = []; + await expect( + readStaticCheckArtifact({ + repository: "owner/rea", + artifactId: 12, + token: "fixture-token", + fetchImpl: (async (url: string | URL | Request) => { + calls.push(String(url)); + return new Response(null, { + status: 302, + headers: { location: "http://untrusted.example/artifact.zip" }, + }); + }) as typeof fetch, + }), + ).rejects.toThrow("static_artifact_redirect_invalid"); + expect(calls).toEqual([ + "https://api.github.com/repos/owner/rea/actions/artifacts/12/zip", + ]); +}); diff --git a/tests/boundary/filesystem/ciFailureReportBudget.test.ts b/tests/boundary/filesystem/ciFailureReportBudget.test.ts new file mode 100644 index 0000000000..92fe3e4aec --- /dev/null +++ b/tests/boundary/filesystem/ciFailureReportBudget.test.ts @@ -0,0 +1,290 @@ +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + reportCiFailures, + type CiRun, + type FailureReportApi, + type StaticDiagnostic, + type StaticEvidence, +} from "../../../scripts/ci/failure-report.mjs"; + +const repository = "owner/rea"; +const mainSha = "c".repeat(40); +const prSha = "a".repeat(40); +const diagnostic: StaticDiagnostic = { + kind: "lint", + path: "src/example.ts", + code: "no-debugger", + message: "Unexpected debugger statement.", + severity: "error", +}; +const evidence = ( + run: CiRun, + diagnostics: readonly StaticDiagnostic[], +): StaticEvidence => ({ + context: { + version: 1, + sourceSha: run.head_sha!, + sourceClean: true, + nodeVersion: "v24.18.0", + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: "linux", + architecture: "x64", + taskHashes: { "lint:built": "a".repeat(16) }, + }, + diagnostics, +}); +const entry = (id: number, event = "pull_request", runAttempt = 1) => ({ + id, + runAttempt, + workflow: "CI", + event, + headBranch: event === "pull_request" ? "feature" : "main", + status: "collected", + conclusion: "failure", +}); + +function fixture(mainIds: readonly number[], prCount: number) { + const jobs: number[] = []; + const artifacts: number[] = []; + const paths: (readonly string[])[] = []; + const runs: Array< + Omit, "event"> & { event?: string } + > = [ + ...mainIds.map((id) => entry(id, "schedule")), + ...Array.from({ length: prCount }, (_, index) => entry(100 + index)), + ]; + const api: FailureReportApi = { + async getRun(_repository, id) { + const selected = runs.find((run) => run.id === Number(id))!; + return { + id, + run_attempt: selected.runAttempt, + status: "completed", + conclusion: "failure", + ...(selected.event ? { event: selected.event } : {}), + head_branch: selected.headBranch, + path: ".github/workflows/ci.yml@main", + head_sha: mainIds.includes(Number(id)) ? mainSha : prSha, + repository: { full_name: repository }, + pull_requests: + selected.event === "pull_request" + ? [{ base: { sha: mainSha, repo: { full_name: repository } } }] + : [], + }; + }, + async getJobs(_repository, id, attempt) { + jobs.push(Number(id)); + return [ + { + name: "Static checks", + status: "completed", + conclusion: "failure", + run_id: Number(id), + run_attempt: attempt, + }, + ]; + }, + async getStaticEvidence(_repository, run) { + artifacts.push(Number(run.id)); + return evidence(run, [diagnostic]); + }, + async getAncestry() { + return { status: "identical", mergeBaseSha: mainSha }; + }, + async getBlobIdentity(_repository, _ancestor, _descendant, requested) { + paths.push(requested); + return Object.fromEntries(requested.map((path) => [path, true])); + }, + async verifyExecutionSource() { + return true; + }, + }; + return { api, runs, jobs, artifacts, paths }; +} + +async function report(input: ReturnType) { + return await reportCiFailures({ + repository, + defaultBranch: "main", + batchManifest: { repository, runs: input.runs }, + api: input.api, + outputDirectory: await createTestTempDirectory("rea-ci-budget-"), + }); +} + +it("selects PR failures before the cap even after a day of scheduled main failures", async () => { + const input = fixture( + Array.from({ length: 24 }, (_, index) => 20 + index), + 1, + ); + input.runs.push({ + id: 200, + runAttempt: 1, + workflow: "CI", + headBranch: "feature", + status: "collected", + conclusion: "failure", + }); + const result = await report(input); + expect(result.failures).toMatchObject([ + { runId: 100, classification: "matching-main-diagnostics" }, + ]); + expect(result.boundedInput).toMatchObject({ + candidateRuns: 1, + mainCandidateRuns: 5, + manifestTruncated: false, + }); + expect(input.jobs).toEqual([100, 20]); +}); + +it("reuses five main job/artifact observations across twenty completed PR comparisons", async () => { + const input = fixture([20, 21, 22, 23, 24], 20); + const load = input.api.getStaticEvidence; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + return Number(args[1].id) < 24 + ? evidence(args[1], [{ ...diagnostic, message: "different" }]) + : result; + }; + const result = await report(input); + expect(result.counts).toEqual({ + "matching-main-diagnostics": 20, + unknown: 0, + }); + expect(input.paths).toHaveLength(100); + expect(input.jobs).toHaveLength(25); + expect(input.artifacts).toHaveLength(25); + for (const id of [20, 21, 22, 23, 24]) { + expect(input.jobs.filter((item) => item === id)).toHaveLength(1); + expect(input.artifacts.filter((item) => item === id)).toHaveLength(1); + } +}); + +it.each(["missing", "artifact error", "job error", "wrong job attempt"])( + "reuses unavailable main evidence without retries: %s", + async (failure) => { + const input = fixture([20], 3); + const load = input.api.getStaticEvidence; + const jobs = input.api.getJobs; + input.api.getJobs = async (...args) => { + const result = await jobs(...args); + if (Number(args[1]) !== 20) return result; + if (failure === "job error") throw new Error("job_snapshot_unavailable"); + return failure === "wrong job attempt" + ? result.map((job) => ({ ...job, run_attempt: 2 })) + : result; + }; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + if (Number(args[1].id) !== 20) return result; + if (failure === "artifact error") throw new Error("artifact_unavailable"); + return null; + }; + const result = await report(input); + expect(result.counts.unknown).toBe(3); + expect(input.jobs.filter((id) => id === 20)).toHaveLength(1); + expect(input.artifacts.filter((id) => id === 20)).toHaveLength( + failure === "job error" || failure === "wrong job attempt" ? 0 : 1, + ); + }, +); + +it.each([1, 2])( + "refreshes unavailable main evidence in the next report (attempt %s)", + async (attempt) => { + const input = fixture([20], 1); + expect((await report(input)).counts["matching-main-diagnostics"]).toBe(1); + input.runs[0]!.runAttempt = attempt; + const load = input.api.getStaticEvidence; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + return Number(args[1].id) === 20 ? null : result; + }; + expect((await report(input)).counts.unknown).toBe(1); + expect(input.jobs.filter((id) => id === 20)).toHaveLength(2); + expect(input.artifacts.filter((id) => id === 20)).toHaveLength(2); + }, +); + +it.each([1, 51])( + "fetches only PR paths with %s PR diagnostics and noisy main evidence", + async (count) => { + const input = fixture([20], 1); + const load = input.api.getStaticEvidence; + const diagnostics = Array.from({ length: count }, (_, index) => ({ + ...diagnostic, + path: `src/pr-${index}.ts`, + })); + input.api.getStaticEvidence = async (...args) => { + await load(...args); + const noise = + Number(args[1].id) === 20 + ? Array.from({ length: 60 }, (_, index) => ({ + ...diagnostic, + path: `src/main-${index}.ts`, + })) + : []; + return evidence(args[1], [...noise, ...diagnostics]); + }; + const result = await report(input); + expect(input.paths).toEqual([ + diagnostics.slice(0, 50).map((item) => item.path), + ]); + expect(result.failures[0]!.classification).toBe( + count === 1 ? "matching-main-diagnostics" : "unknown", + ); + }, +); + +it.each(["jobs", "artifact", "comparison"])( + "continues after an isolated main %s failure and preserves its provenance", + async (failure) => { + const input = fixture([20, 21], 3); + const jobs = input.api.getJobs; + const load = input.api.getStaticEvidence; + input.api.getJobs = async (...args) => { + const result = await jobs(...args); + if (Number(args[1]) === 20 && failure === "jobs") + throw new Error("main_jobs_unavailable"); + return result; + }; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + if (Number(args[1].id) === 20 && failure === "artifact") + throw new Error("main_artifact_unavailable"); + return Number(args[1].id) === 20 && failure === "comparison" + ? { + ...result!, + context: { ...result!.context, sourceSha: "d".repeat(40) }, + } + : result; + }; + input.api.verifyExecutionSource = async (_repository, run, context) => { + if (Number(run.id) === 20 && context.sourceSha !== run.head_sha) + throw new Error("main_comparison_unavailable"); + return true; + }; + const result = await report(input); + expect(result.counts).toEqual({ + "matching-main-diagnostics": 3, + unknown: 0, + }); + for (const item of result.failures) { + expect(item.candidateFailures).toEqual([ + { + runId: 20, + runAttempt: 1, + reason: `main_${failure}_unavailable`, + }, + ]); + } + expect(input.jobs.filter((id) => id === 20)).toHaveLength(1); + expect(input.jobs.filter((id) => id === 21)).toHaveLength(1); + expect(input.artifacts.filter((id) => id === 20)).toHaveLength( + failure === "jobs" ? 0 : 1, + ); + expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1); + }, +); diff --git a/tests/boundary/filesystem/ciFeedback.test.ts b/tests/boundary/filesystem/ciFeedback.test.ts new file mode 100644 index 0000000000..c61cb0d92b --- /dev/null +++ b/tests/boundary/filesystem/ciFeedback.test.ts @@ -0,0 +1,313 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; + +import { + parseStaticTaskContext, + recordStaticTaskContext, +} from "../../../scripts/ci/static-task-context.mjs"; +import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const scopeFlags = async () => + z + .object({ CI_FLAGS: z.array(z.string()) }) + .parse( + await import( + new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href + ), + ).CI_FLAGS; +const workflow = async () => + z + .object({ + jobs: z.record( + z.string(), + z + .object({ + needs: z.union([z.string(), z.array(z.string())]).optional(), + if: z.string().optional(), + steps: z + .array( + z + .object({ + run: z.string().optional(), + env: z.record(z.string(), z.string()).optional(), + }) + .passthrough(), + ) + .optional(), + }) + .passthrough(), + ), + }) + .parse( + parse( + await readFile( + new URL("../../../.github/workflows/ci.yml", import.meta.url), + "utf8", + ), + ), + ).jobs; + +it("starts source feedback before build and requires both groups without waiving missing jobs", async () => { + const jobs = await workflow(); + expect( + jobs.static?.steps?.find( + (step) => step.run === "npm run ci:static-context", + )?.["continue-on-error"], + ).toBe(true); + expect(jobs["test-source"]?.needs).toBe("changes"); + expect(jobs["test-shard"]?.needs).toEqual(["changes", "build"]); + expect( + jobs["test-source"]?.steps?.some( + (step) => + step.run === "npm run test:fast -- --reporter=default --reporter=blob", + ), + ).toBe(true); + const required = fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ); + const flags = await scopeFlags(); + for (const tier of ["fast", "full"]) { + for (const result of [ + "success", + "failure", + "cancelled", + "skipped", + "missing", + ]) { + const needs = Object.fromEntries( + Object.keys(jobs) + .filter((name) => name !== "required") + .map((name) => [name, { result: "success" }]), + ); + const outputs = { + ...Object.fromEntries( + flags.map((flag) => [flag, tier === "full" ? "true" : "false"]), + ), + tier, + code: "true", + }; + const context: Record< + string, + { result: string; outputs?: Record } + > = { ...needs, changes: { result: "success", outputs } }; + if (result === "missing") delete context["test-source"]; + else context["test-source"] = { result }; + const run = execute(process.execPath, [required], { + env: { + ...process.env, + CI_JOBS: JSON.stringify(context), + CI_QUALIFICATION_DIRECTORY: "", + }, + }); + if ( + result === "success" || + (tier === "full" && ["skipped", "missing"].includes(result)) + ) + await expect(run).resolves.toMatchObject({ stderr: "" }); + else + await expect(run).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("test-source"), + }); + } + } +}); + +it("records the actual static task graph with explicit source identity and no ambient environment", async () => { + const directory = await createTestTempDirectory("rea-ci-context-"); + const context = await recordStaticTaskContext({ outputDirectory: directory }); + const head = await execute("git", ["rev-parse", "HEAD"]); + expect(context.sourceSha).toBe(head.stdout.trim()); + expect(context.taskHashes["lint:built"]).toMatch(/^[a-f0-9]{16}$/u); + expect(context.taskHashes.jscpd).toMatch(/^[a-f0-9]{16}$/u); + expect(context).not.toHaveProperty("environmentVariables"); + expect(context).not.toHaveProperty("env"); + expect( + JSON.parse(await readFile(`${directory}/static-task-context.json`, "utf8")), + ).toEqual(context); + for (const invalid of [ + undefined, + {}, + { ...context, sourceSha: "bad" }, + { ...context, prHeadSha: "bad" }, + { ...context, taskHashes: { jscpd: "stale" } }, + ]) + expect(() => parseStaticTaskContext(invalid)).toThrow(); +}); + +it("rejects mutation methods at the shared GitHub transport before invoking the CLI", async () => { + const request = createGhApiRequest({ env: { PATH: "" } }); + await expect( + request("POST", "repos/owner/repo/actions/runs/1/rerun", { + requests: 0, + startedAt: Date.now(), + limitMs: 1000, + }), + ).rejects.toThrow("read-only GET"); +}); + +it("collects basic CI telemetry before optional dependency installation can fail", async () => { + const { jobs } = z + .object({ + jobs: z.object({ + report: z.object({ + steps: z.array( + z.object({ + id: z.string().optional(), + run: z.string().optional(), + if: z.string().optional(), + "continue-on-error": z.boolean().optional(), + }), + ), + }), + }), + }) + .parse( + parse( + await readFile( + new URL("../../../.github/workflows/ci-timings.yml", import.meta.url), + "utf8", + ), + ), + ); + const steps = jobs.report.steps; + const installation = steps.findIndex((step) => step.id === "failure-deps"); + for (const script of ["pr-ci-health.mjs", "job-timings-batch.mjs"]) { + const collection = steps.findIndex( + (step) => step.run === `node scripts/ci/${script}`, + ); + expect(collection).toBeGreaterThanOrEqual(0); + expect(installation).toBeGreaterThan(collection); + } + expect(steps[installation]).toMatchObject({ + run: "npm ci", + "continue-on-error": true, + }); + expect( + steps.find((step) => step.id === "npm-cache")?.["continue-on-error"], + ).toBe(true); + expect( + steps.find((step) => step.run === "node scripts/ci/failure-report.mjs")?.if, + ).toContain("steps.failure-deps.outcome == 'success'"); +}); + +it("merges source and built reports exactly once using the workflow's shard matrix", async () => { + const jobs = await workflow(); + const matrix = z + .object({ + matrix: z.object({ shard: z.string().regex(/^\$\{\{.*\}\}$/u) }), + }) + .parse(jobs["test-shard"]?.strategy).matrix.shard; + const shards = z + .array(z.string().regex(/^\d+\/\d+$/u)) + .min(1) + .parse( + runInNewContext( + matrix.slice(3, -2), + { + fromJSON: JSON.parse, + needs: { changes: { outputs: { tier: "fast" } } }, + }, + { timeout: 1_000 }, + ), + ); + const labels = Object.fromEntries( + ["source", "built"].map((group) => [ + group, + z + .string() + .min(1) + .parse( + jobs[group === "source" ? "test-source" : "test-shard"]?.steps?.find( + (step) => step.env?.VITEST_BLOB_LABEL, + )?.env?.VITEST_BLOB_LABEL, + ), + ]), + ); + const root = await createTestTempDirectory("rea-ci-group-merge-"); + const config = join(root, "vitest.config.mjs"); + await writeFile( + config, + `export default { test: { maxWorkers: 1, projects: [${["source", "built"].map((name) => `{ test: { name: "${name}", globals: true, include: ["${name}*.test.js"] } }`).join(",")}] } };`, + ); + const files = [ + "source.test.js", + ...Array.from( + { length: shards.length }, + (_, index) => `built-${index}.test.js`, + ), + ]; + for (const file of files) + await writeFile( + join(root, file), + 'it("runs its selected group", () => expect(true).toBe(true));', + ); + const vitest = fileURLToPath( + new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url), + ); + const blobs = new Map(); + const runs = [ + { group: "source", shard: [] }, + ...shards.map((shard) => ({ group: "built", shard: [`--shard=${shard}`] })), + ]; + for (const { group, shard } of runs) { + await execute( + process.execPath, + [ + vitest, + "run", + "--config", + config, + "--project", + group, + "--reporter=blob", + ...shard, + ], + { + cwd: root, + env: { ...process.env, VITEST_BLOB_LABEL: labels[group]! }, + timeout: 20_000, + }, + ); + for (const filename of await readdir(join(root, ".vitest/blob"))) { + expect(blobs.has(filename)).toBe(false); + blobs.set(filename, await readFile(join(root, ".vitest/blob", filename))); + } + // Separate hosted jobs start with independent report directories. + await rm(join(root, ".vitest/blob"), { recursive: true }); + } + expect(blobs.size).toBe(runs.length); + await mkdir(join(root, ".vitest/blob"), { recursive: true }); + for (const [filename, bytes] of blobs) + await writeFile(join(root, ".vitest/blob", filename), bytes); + await execute( + process.execPath, + [ + vitest, + "--config", + config, + "--merge-reports", + "--reporter=json", + "--outputFile.json=merged.json", + ], + { cwd: root, timeout: 20_000 }, + ); + const merged = z + .object({ + success: z.literal(true), + testResults: z.array(z.object({ name: z.string() })), + }) + .parse(JSON.parse(await readFile(join(root, "merged.json"), "utf8"))); + expect( + merged.testResults.map(({ name }) => name.split("/").at(-1)).sort(), + ).toEqual(files.toSorted()); +}); diff --git a/tests/boundary/filesystem/ciJobTimingsBatch.test.ts b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts index 6c2658c000..e0dcecc824 100644 --- a/tests/boundary/filesystem/ciJobTimingsBatch.test.ts +++ b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts @@ -18,6 +18,8 @@ it("collects the latest attempt for recent completed CI and Release runs into a workflow_runs: [ { id: 42, + event: "push", + head_branch: "main", run_attempt: 3, status: "completed", conclusion: "failure", @@ -43,51 +45,9 @@ it("collects the latest attempt for recent completed CI and Release runs into a }, ], }, - [ - { - jobs: [ - { - name: "Release lane", - status: "completed", - conclusion: "success", - created_at: recent, - started_at: recent, - completed_at: new Date(Date.parse(recent) + 50_000).toISOString(), - steps: [], - }, - ], - }, - ], - [ - { - jobs: [ - { - name: "Failed lane", - status: "completed", - conclusion: "failure", - created_at: recent, - started_at: recent, - completed_at: new Date(Date.parse(recent) + 55_000).toISOString(), - steps: [], - }, - ], - }, - ], - [ - { - jobs: [ - { - name: "Cancelled lane", - status: "completed", - conclusion: "cancelled", - created_at: recent, - started_at: recent, - completed_at: new Date(Date.parse(recent) + 30_000).toISOString(), - steps: [], - }, - ], - }, - ], + completedJobs("Release lane", "success", recent, 50), + completedJobs("Failed lane", "failure", recent, 55), + completedJobs("Cancelled lane", "cancelled", recent, 30), ]); const result = await runBatch(fixture); const manifest = JSON.parse( @@ -116,8 +76,20 @@ it("collects the latest attempt for recent completed CI and Release runs into a expect(manifest.errors).toEqual([]); expect(manifest.runs).toMatchObject([ { id: 43, runAttempt: 2, conclusion: "success" }, - { id: 42, runAttempt: 3, conclusion: "failure" }, - { id: 41, runAttempt: 1, conclusion: "cancelled" }, + { + id: 42, + runAttempt: 3, + conclusion: "failure", + event: "push", + headBranch: "main", + }, + { + id: 41, + runAttempt: 1, + conclusion: "cancelled", + event: null, + headBranch: null, + }, ]); expect(manifest.runs).toEqual( expect.arrayContaining([ @@ -234,6 +206,31 @@ it("retains a partial manifest when its collection budget is exhausted", async ( expect(manifest.truncation.join(" ")).toContain("Time budget"); }); +function completedJobs( + name: string, + conclusion: string, + startedAt: string, + seconds: number, +) { + return [ + { + jobs: [ + { + name, + status: "completed", + conclusion, + created_at: startedAt, + started_at: startedAt, + completed_at: new Date( + Date.parse(startedAt) + seconds * 1000, + ).toISOString(), + steps: [], + }, + ], + }, + ]; +} + async function createFixture(responses: unknown[]) { const directory = await createTestTempDirectory("rea-ci-job-timings-batch-"); const fakeBin = join(directory, "bin"); diff --git a/tests/boundary/filesystem/ciPrHealth.test.ts b/tests/boundary/filesystem/ciPrHealth.test.ts new file mode 100644 index 0000000000..756fb47489 --- /dev/null +++ b/tests/boundary/filesystem/ciPrHealth.test.ts @@ -0,0 +1,617 @@ +import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import { delimiter, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + classifyPrCi, + collectPrCiHealth, + matchRunToPullRequest, + writePrCiHealthReport, +} from "../../../scripts/ci/pr-ci-health.mjs"; +import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs"; +import type { + PrCiPullRequest, + PrCiWorkflowRun, +} from "../../../scripts/ci/pr-ci-health.mjs"; +import { executeWorkflowFixture } from "../../support/workflowGit.js"; + +const reporter = fileURLToPath( + new URL("../../../scripts/ci/pr-ci-health.mjs", import.meta.url), +); +const observedAt = Date.parse("2026-10-14T00:00:00Z"); +const headSha = "a".repeat(40); +const mergeSha = "b".repeat(40); + +it("classifies workflow states without confusing absent, pending, and completed CI", () => { + const pull = validPull(); + const cases = [ + [ + "startup_failure", + run({ status: "completed", conclusion: "startup_failure" }), + ], + [ + "queued_pending_executing", + run({ status: "in_progress", conclusion: null }), + ], + ["approval_required", run({ status: "waiting", conclusion: null })], + ["completed_failure", run({ status: "completed", conclusion: "failure" })], + [ + "completed_canceled", + run({ status: "completed", conclusion: "cancelled" }), + ], + ["completed_success", run({ status: "completed", conclusion: "success" })], + ["completed_other", run({ status: "completed", conclusion: "neutral" })], + [ + "completed_other", + run({ status: "completed", conclusion: "constructor" }), + ], + ] as const; + for (const [category, evidence] of cases) { + for (const runsComplete of [true, false]) { + const result = classifyPrCi({ + pull, + runs: [ + { + run: { ...evidence, run_attempt: 1 }, + basis: "event_pr_number_and_head_sha", + }, + ], + runsComplete, + pullRequestInventoryComplete: true, + ambiguousRuns: [], + }); + expect(result).toMatchObject({ + category: runsComplete ? category : "unknown", + ...(runsComplete + ? {} + : { reason: "workflow_run_inventory_incomplete" }), + runs: [ + { + id: 77, + attempt: 1, + status: evidence.status, + conclusion: evidence.conclusion, + }, + ], + }); + } + } + for (const runsComplete of [true, false]) { + expect( + classifyPrCi({ + pull: { ...pull, mergeable: false }, + runs: [], + runsComplete, + pullRequestInventoryComplete: true, + ambiguousRuns: [], + }).category, + ).toBe("merge_conflict"); + } +}); + +it("requires complete inventories, known mergeability, and immutable head identity to report missing", () => { + const pull = validPull({ mergeable: true }); + const classify = (overrides = {}) => + classifyPrCi({ + pull, + runs: [], + runsComplete: true, + pullRequestInventoryComplete: true, + ambiguousRuns: [], + ...overrides, + }); + + expect(classify().category).toBe("missing"); + expect(classify({ runsComplete: false })).toMatchObject({ + category: "deferred", + reason: "lookup_inventory_incomplete", + }); + expect(classify({ pullRequestInventoryComplete: false }).category).toBe( + "deferred", + ); + expect( + classify({ + ambiguousRuns: [{ run: run(), basis: "ambiguous_head_identity" }], + }).category, + ).toBe("unknown"); + expect(classify({ pull: { ...pull, mergeable: undefined } }).category).toBe( + "unknown", + ); + expect(classify({ pull: { ...pull, identityValid: false } }).category).toBe( + "unknown", + ); +}); + +it("accepts event PR/head evidence for merge-SHA runs and refuses ambiguous branch identities", () => { + const pull = validPull(); + const associatedMergeRun = run({ + head_sha: mergeSha, + pull_requests: [ + { + number: pull.number!, + head: { sha: headSha, repo: { id: pull.head.repositoryId } }, + }, + ], + }); + expect( + matchRunToPullRequest({ + run: associatedMergeRun, + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }), + ).toEqual({ kind: "match", basis: "event_pr_number_and_head_sha" }); + expect( + matchRunToPullRequest({ + run: run({ + pull_requests: [ + { number: pull.number!, head: { sha: "c".repeat(40) } }, + ], + }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("none"); + + const branchRun = run({ pull_requests: [] }); + expect( + matchRunToPullRequest({ + run: branchRun, + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + inventoryComplete: false, + }).kind, + ).toBe("ambiguous"); + expect( + matchRunToPullRequest({ + run: branchRun, + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }), + ).toEqual({ + kind: "match", + basis: "event_unique_head_ref_and_repository_id", + }); + expect( + matchRunToPullRequest({ + run: run({ head_sha: mergeSha, pull_requests: [] }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("none"); + expect( + matchRunToPullRequest({ + run: run({ + pull_requests: [{ number: pull.number!, head: { sha: headSha } }], + }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("ambiguous"); + expect( + matchRunToPullRequest({ + run: branchRun, + pull, + identityCounts: new Map([["73\u0000feature", 2]]), + }).kind, + ).toBe("ambiguous"); + expect( + matchRunToPullRequest({ + run: run({ event: "workflow_dispatch" }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("none"); +}); + +it("scans target-branch PRs through public gh GET requests and writes JSON and Markdown", async () => { + const fixture = await createFixture(); + const calls: string[] = []; + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (method, endpoint) => { + calls.push(`${method} ${endpoint}`); + if (method !== "GET") throw new Error("fixture rejected a non-GET request"); + if (endpoint.includes("/pulls?")) + return [ + rawPull(42, { mergeable: true }), + rawPull(43, { mergeable: false }), + rawPull(44, { + base: { ref: "release", repo: { id: 11 } }, + mergeable: true, + }), + rawPull(45, { draft: true, mergeable: true }), + ]; + if (endpoint.includes("/actions/workflows/ci.yml/runs?")) + return { + total_count: 1, + workflow_runs: [ + run({ + id: 9001, + event: "pull_request", + status: "completed", + conclusion: "failure", + head_sha: mergeSha, + pull_requests: [ + { + number: 42, + head: { sha: headSha, repo: { id: 73 } }, + }, + ], + }), + ], + }; + throw new Error(`unexpected GET endpoint: ${endpoint}`); + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(calls).toHaveLength(2); + expect(calls.every((call) => call.startsWith("GET "))).toBe(true); + expect(calls[0]).toContain("state=open"); + expect(calls[0]).not.toContain("base="); + expect(calls[1]).toContain("event=pull_request"); + expect(report.pullRequests).toHaveLength(2); + expect(report.pullRequests.map(({ number }) => number)).toEqual([42, 43]); + expect(report.pullRequests.map(({ ci }) => ci.category)).toEqual([ + "completed_failure", + "merge_conflict", + ]); + expect(report.scan).toMatchObject({ + status: "complete", + pullRequestInventoryComplete: true, + workflowRunInventoryComplete: true, + apiRequests: 2, + }); + + const output = join(fixture.directory, ".cache", "ci-pr-health"); + const paths = await writePrCiHealthReport(report, output); + const written = JSON.parse(await readFile(paths.jsonPath, "utf8")); + const markdown = await readFile(paths.markdownPath, "utf8"); + expect(written.counts.completed_failure).toBe(1); + expect(markdown).toContain("# Pull request CI health snapshot"); + expect(markdown).toContain("#42"); + expect(markdown).toContain("completed_failure"); +}); + +it("counts older open PRs when deciding whether a branch/ref fallback is unique", async () => { + const olderPull = rawPull(63, { + created_at: "2026-09-01T00:00:00Z", + head: { ...rawPull(63).head, ref: "shared-feature" }, + }); + const currentPull = rawPull(64, { + mergeable: true, + head: { ...olderPull.head }, + }); + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [currentPull, olderPull]; + if (endpoint.includes("/actions/workflows/ci.yml/runs?")) + return { + workflow_runs: [ + run({ + head_branch: "shared-feature", + pull_requests: [], + }), + ], + }; + throw new Error(`unexpected GET endpoint: ${endpoint}`); + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(report.scan.pullRequestInventoryComplete).toBe(true); + expect(report.pullRequests.map(({ number }) => number)).toEqual([64]); + expect(report.pullRequests[0]?.ci).toMatchObject({ + category: "unknown", + reason: "workflow_run_identity_ambiguous", + }); +}); + +it.each([false, true])( + "keeps page-capped scans uncertain (observed success: %s)", + async (observedSuccess) => { + const pull = rawPull(62, { mergeable: true }); + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [pull]; + const page = Number( + new URL(`https://fixture.invalid/${endpoint}`).searchParams.get("page"), + ); + return { + workflow_runs: Array.from({ length: 100 }, (_item, index) => + observedSuccess && page === 1 && index === 0 + ? run({ + run_attempt: 1, + pull_requests: [ + { number: 62, head: { sha: headSha, repo: { id: 73 } } }, + ], + }) + : run({ + id: page * 100 + index, + head_branch: "different-branch", + head_repository: { id: 999 }, + pull_requests: [], + }), + ), + }; + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(report.scan.workflowRunPages).toBe(5); + expect(report.scan.workflowRunInventoryComplete).toBe(false); + const firstPull = report.pullRequests[0]; + if (firstPull === undefined) throw new Error("fixture PR was not scanned"); + expect(firstPull.ci).toMatchObject({ + category: observedSuccess ? "unknown" : "deferred", + reason: observedSuccess + ? "workflow_run_inventory_incomplete" + : "lookup_inventory_incomplete", + }); + expect(firstPull.ci.runs).toHaveLength(observedSuccess ? 1 : 0); + }, +); + +it("keeps the result unknown when the head changes between API snapshots", async () => { + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [rawPull(63)]; + if (endpoint.includes("/actions/workflows/ci.yml/runs?")) + return { workflow_runs: [] }; + return { + ...rawPull(63, { mergeable: true }), + head: { + ...rawPull(63).head, + sha: mergeSha, + }, + }; + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(report.pullRequests[0]).toMatchObject({ + identityChangedDuringScan: true, + ci: { category: "unknown", reason: "pull_request_head_identity_invalid" }, + }); + expect(report.scan).toMatchObject({ + status: "partial", + errors: ["pull_request_identity_changed_during_scan"], + }); +}); + +it("public CLI uses only GET and produces a report from fixture GitHub responses", async () => { + const fixture = await createFixture(); + const ghCalls = join(fixture.directory, "gh-calls.jsonl"); + const ghResponse = join(fixture.directory, "gh-response.json"); + const fakeGh = join(fixture.fakeBin, "gh"); + await writeFile( + ghResponse, + JSON.stringify([ + [rawPull(51, { mergeable: true })], + { total_count: 0, workflow_runs: [] }, + ]), + ); + await writeFile( + fakeGh, + `#!/usr/bin/env node\nconst fs=require("node:fs");\nconst args=process.argv.slice(2);\nfs.appendFileSync(process.env.CI_GH_CALLS, JSON.stringify(args)+"\\n");\nif(args[0]!=="api"||args[1]!=="--method"||args[2]!=="GET")process.exit(41);\nconst page=Number(new URL("https://fixture.invalid/"+args.at(-1)).searchParams.get("page"));\nconst responses=JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSE,"utf8"));\nprocess.stdout.write(JSON.stringify(responses[page===1&&args.at(-1).includes("/pulls?")?0:1]));\n`, + ); + await chmod(fakeGh, 0o755); + const outputDirectory = join(fixture.directory, ".cache", "ci-pr-health"); + await executeWorkflowFixture(process.execPath, [reporter], { + cwd: fixture.directory, + env: { + ...process.env, + PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`, + CI_REPOSITORY: "owner/rea", + CI_REPOSITORY_ID: "11", + CI_DEFAULT_BRANCH: "main", + CI_PR_HEALTH_DIRECTORY: outputDirectory, + CI_GH_CALLS: ghCalls, + CI_GH_RESPONSE: ghResponse, + }, + }); + const calls = (await readFile(ghCalls, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(calls).toHaveLength(2); + expect( + calls.every( + (args: string[]) => args[1] === "--method" && args[2] === "GET", + ), + ).toBe(true); + const report = JSON.parse( + await readFile(join(outputDirectory, "report.json"), "utf8"), + ); + expect(report.pullRequests[0].ci.category).toBe("missing"); + await expect( + readFile(join(outputDirectory, "summary.md"), "utf8"), + ).resolves.toContain("missing"); +}); + +it("rejects non-GET methods before launching the GitHub client", async () => { + const fixture = await createFixture(); + const request = createGhApiRequest({ + env: { + ...process.env, + PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`, + CI_GH_CALLS: join(fixture.directory, "should-not-exist"), + }, + }); + await expect( + request("POST", "repos/owner/rea/pulls", { + requests: 0, + startedAt: observedAt, + limitMs: 1_000, + }), + ).rejects.toThrow("read-only GET"); + await expect( + readFile(join(fixture.directory, "should-not-exist")), + ).rejects.toMatchObject({ code: "ENOENT" }); +}); + +function validPull(overrides: Partial = {}): PrCiPullRequest { + return { + number: 42, + identityValid: true, + mergeable: true, + head: { repositoryId: 73, ref: "feature", sha: headSha }, + ...overrides, + }; +} + +it.each([ + ["newer run", { id: 78, created_at: "2026-10-13T01:00:00Z" }, "unknown"], + [ + "older run", + { id: 78, created_at: "2026-10-12T23:00:00Z" }, + "completed_success", + ], + ["equal timestamps", { id: 78 }, "unknown"], + [ + "retry of an older run", + { id: 78, created_at: "2026-10-12T23:00:00Z", run_attempt: 2 }, + "unknown", + ], + ["unknown timestamp", { id: 78, created_at: "invalid" }, "unknown"], + ["newer attempt", { run_attempt: 3 }, "unknown"], + ["older attempt", { run_attempt: 1 }, "completed_success"], +] as const)( + "retains ambiguity evidence for a %s without reporting a stale completion", + (_label, overrides, category) => { + const currentAttempt = "id" in overrides ? 1 : 2; + const result = classifyPrCi({ + pull: validPull(), + runs: [ + { + run: run({ run_attempt: currentAttempt }), + basis: "event_pr_number_and_head_sha", + }, + ], + ambiguousRuns: [ + { + run: run({ run_attempt: currentAttempt, ...overrides }), + basis: "ambiguous_head_identity", + }, + ], + runsComplete: true, + pullRequestInventoryComplete: true, + }); + expect(result.category).toBe(category); + expect(result.runs).toHaveLength(2); + }, +); + +it("does not let an older proven run hide a newer run with an unverified repository ID", async () => { + const pull = rawPull(42); + const older = run({ + id: 70, + created_at: "2026-10-13T00:00:00Z", + pull_requests: [{ number: 42, head: { sha: headSha, repo: { id: 73 } } }], + }); + const newer = run({ + id: 71, + created_at: "2026-10-13T01:00:00Z", + status: "in_progress", + conclusion: null, + pull_requests: [{ number: 42, head: { sha: headSha } }], + }); + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest: async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [pull]; + if (endpoint.endsWith("/pulls/42")) return { ...pull, mergeable: true }; + return { workflow_runs: [newer, older] }; + }, + }); + expect(report.pullRequests[0]?.ci).toMatchObject({ + category: "unknown", + reason: "workflow_run_identity_ambiguous", + }); + expect(report.pullRequests[0]?.ci.runs).toHaveLength(2); +}); + +it("keeps a proven newest run even when two older matched runs have equal timestamps", () => { + const result = classifyPrCi({ + pull: validPull(), + runs: [ + run({ id: 70, run_attempt: 1 }), + run({ id: 71, run_attempt: 1, conclusion: "failure" }), + run({ + id: 72, + run_attempt: 1, + created_at: "2026-10-13T01:00:00Z", + status: "in_progress", + conclusion: null, + }), + ].map((run) => ({ run, basis: "event_pr_number_and_head_sha" })), + ambiguousRuns: [], + runsComplete: true, + pullRequestInventoryComplete: true, + }); + expect(result.category).toBe("queued_pending_executing"); + expect(result.runs).toHaveLength(3); +}); + +function run(overrides: Partial = {}): PrCiWorkflowRun { + return { + id: 77, + event: "pull_request", + head_branch: "feature", + head_repository: { id: 73 }, + head_sha: headSha, + status: "completed", + conclusion: "success", + created_at: "2026-10-13T00:00:00Z", + pull_requests: [], + ...overrides, + }; +} + +function rawPull(number: number, overrides: Record = {}) { + return { + number, + draft: false, + created_at: "2026-10-13T00:00:00Z", + html_url: `https://github.com/owner/rea/pull/${String(number)}`, + base: { ref: "main", repo: { id: 11 } }, + head: { + ref: `feature-${String(number)}`, + sha: headSha, + repo: { id: 73, full_name: "contributor/rea" }, + }, + ...overrides, + }; +} + +async function createFixture() { + const directory = await createTestTempDirectory("rea-ci-pr-health-"); + const fakeBin = join(directory, "bin"); + await mkdir(fakeBin); + return { directory, fakeBin }; +} diff --git a/tests/boundary/filesystem/ciQualification.test.ts b/tests/boundary/filesystem/ciQualification.test.ts index b6a5a1e1a4..673b09d34c 100644 --- a/tests/boundary/filesystem/ciQualification.test.ts +++ b/tests/boundary/filesystem/ciQualification.test.ts @@ -44,7 +44,7 @@ function run(overrides: Record = {}) { id: 7, status: "completed", conclusion: "success", - path: ".github/workflows/ci.yml", + path: ".github/workflows/ci.yml@main", event: "schedule", created_at: new Date(now - 120_000).toISOString(), head_sha: validSha, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index d7ac1418ed..d2191eb01c 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -121,7 +121,11 @@ it("exports every canonical scope with read authority for admission and keeps te ); expect(timings.on.workflow_run).toBeUndefined(); expect(timings.on.schedule).toHaveLength(1); - expect(timings.permissions).toEqual({ contents: "read", actions: "read" }); + expect(timings.permissions).toEqual({ + contents: "read", + actions: "read", + "pull-requests": "read", + }); const checkout = timings.jobs.report.steps.find((step) => step.uses?.startsWith("actions/checkout@"), ); @@ -154,6 +158,7 @@ it("holds native and provider fan-out behind static validation while portable fe "static", "build", "test-shard", + "test-source", "test", "docs", "required", @@ -369,7 +374,7 @@ it("keeps fast feedback separate from complete qualification and release proof", // release their concurrency slot without waiting for another runner. expect(ci.jobs.required?.if).toBe("${{ !cancelled() }}"); const shards = ci.jobs["test-shard"]?.steps ?? []; - const fast = shards.find((step) => step.run?.includes("npm run test:run")); + const fast = shards.find((step) => step.run?.includes("npm run test:built")); const full = shards.find((step) => step.run?.includes("test:ci:shard:run")); expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'"); expect(fast?.run).toContain("--reporter=blob"); diff --git a/tests/boundary/filesystem/ciStaticArtifacts.test.ts b/tests/boundary/filesystem/ciStaticArtifacts.test.ts new file mode 100644 index 0000000000..45915f4504 --- /dev/null +++ b/tests/boundary/filesystem/ciStaticArtifacts.test.ts @@ -0,0 +1,99 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +import { parseStaticArchive } from "../../../scripts/ci/artifacts.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const producer = fileURLToPath( + new URL("../../../scripts/ci/static-check-report.mjs", import.meta.url), +); + +it.each([3.5, 100, 0])( + "retains real lint failures when the duplication threshold is %s", + async (threshold) => { + const root = await createTestTempDirectory("rea-ci-static-archive-"); + await mkdir(join(root, "src")); + await mkdir(join(root, "tests")); + await writeFile( + join(root, ".oxlintrc.json"), + JSON.stringify({ rules: { "no-debugger": "error" } }), + ); + await writeFile( + join(root, ".jscpd.json"), + JSON.stringify({ threshold, minLines: 5, minTokens: 50 }), + ); + await writeFile( + join(root, "src/example.ts"), + "debugger;\nexport const example = true;\n", + ); + const duplicate = + "export function compute(value: number) {\n" + + Array.from({ length: 20 }, (_, index) => ` value += ${index};\n`).join( + "", + ) + + " return value;\n}\n"; + for (const name of threshold === 3.5 ? [] : ["first", "second"]) + await writeFile(join(root, `src/${name}.ts`), duplicate); + await expect( + execute(process.execPath, [producer, "lint", "src/example.ts"], { + cwd: root, + }), + ).rejects.toMatchObject({ code: 1 }); + const duplication = execute(process.execPath, [producer, "duplicates"], { + cwd: root, + }); + if (threshold === 0) + await expect(duplication).rejects.toMatchObject({ code: 1 }); + else await duplication; + const reports = join(root, ".cache/static-checks"); + expect( + JSON.parse( + await readFile(join(reports, "jscpd/jscpd-report.json"), "utf8"), + ), + ).toMatchObject({ reaCheck: { exitCode: threshold === 0 ? 1 : 0 } }); + await writeFile( + join(reports, "static-task-context.json"), + JSON.stringify({ + version: 1, + sourceSha: "a".repeat(40), + sourceClean: true, + nodeVersion: process.version, + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: process.platform, + architecture: process.arch, + taskHashes: { "lint:built": "a".repeat(16), jscpd: "b".repeat(16) }, + }), + ); + const archive = join(root, "reports.zip"); + await execute( + "zip", + [ + "-q", + archive, + "oxlint.json", + "jscpd/jscpd-report.json", + "static-task-context.json", + ], + { cwd: reports }, + ); + const evidence = await parseStaticArchive(archive); + expect(evidence.diagnostics.filter(({ kind }) => kind === "lint")).toEqual([ + expect.objectContaining({ + kind: "lint", + path: "src/example.ts", + severity: "error", + }), + ]); + const duplicates = evidence.diagnostics.filter( + ({ kind }) => kind === "jscpd", + ); + if (threshold === 0) expect(duplicates.length).toBeGreaterThan(0); + else expect(duplicates).toEqual([]); + }, +); diff --git a/tests/boundary/filesystem/ciTestHistory.test.ts b/tests/boundary/filesystem/ciTestHistory.test.ts index 411e9ece9d..84950d49c5 100644 --- a/tests/boundary/filesystem/ciTestHistory.test.ts +++ b/tests/boundary/filesystem/ciTestHistory.test.ts @@ -35,7 +35,7 @@ function run(id: number, overrides: Record = {}) { id, status: "completed", conclusion: "success", - path: ".github/workflows/ci.yml", + path: ".github/workflows/ci.yml@main", event: "schedule", created_at: new Date(now - 120_000).toISOString(), head_branch: "main", diff --git a/tests/boundary/filesystem/ciTestProjects.test.ts b/tests/boundary/filesystem/ciTestProjects.test.ts new file mode 100644 index 0000000000..f0394c5bbf --- /dev/null +++ b/tests/boundary/filesystem/ciTestProjects.test.ts @@ -0,0 +1,75 @@ +import { execFileSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { + BUILT_TEST_PROJECTS, + SOURCE_TEST_PROJECTS, + projectForTest, +} from "../../../scripts/lib/test-projects.mjs"; +import { + developmentTestPlan, + parseDevelopmentTestRequest, +} from "../../../scripts/lib/development-tests.mjs"; + +const repository = process.cwd(); + +describe("canonical test-project ownership", () => { + it("assigns every tracked test file exactly once to source or built", () => { + const files = execFileSync("git", ["ls-files", "*.test.ts"], { + cwd: repository, + encoding: "utf8", + }) + .trim() + .split("\n") + .filter(Boolean); + const grouped = new Map(); + for (const file of files) { + const project = projectForTest(file); + grouped.set(project.group, [...(grouped.get(project.group) ?? []), file]); + } + + expect(grouped.get("source")).not.toBeUndefined(); + expect(grouped.get("built")).not.toBeUndefined(); + expect( + (grouped.get("source")?.length ?? 0) + + (grouped.get("built")?.length ?? 0), + ).toBe(files.length); + expect( + SOURCE_TEST_PROJECTS.reduce( + (count, project) => count + project.include.length, + 0, + ), + ).toBeGreaterThan(0); + expect( + BUILT_TEST_PROJECTS.every((project) => project.group === "built"), + ).toBe(true); + expect(() => projectForTest("tests/unowned/example.test.ts")).toThrow( + "No test project owns", + ); + }); + + it("owns the Hopper real-provider conformance test as built and requires preparation", () => { + const testPath = + "tests/conformance/providers/hopper/realHopperSearch.test.ts"; + expect(projectForTest(testPath)).toMatchObject({ + name: "boundary", + group: "built", + }); + expect(() => parseDevelopmentTestRequest("local", [testPath])).toThrow( + `Use test:focused for built test projects: ${testPath}`, + ); + const request = parseDevelopmentTestRequest("focused", [testPath]); + expect(developmentTestPlan(request)).toMatchObject({ needsBuild: true }); + }); + + it("uses canonical selectors in Vitest while retaining independent runner policy", async () => { + const config = await readFile(join(repository, "vitest.config.ts"), "utf8"); + expect(config).toContain( + "[...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]", + ); + expect(config).toContain("PROJECT_POLICIES"); + expect(config).not.toContain('"src/{contracts,domain}/**/*.test.ts"'); + }); +}); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 3e2aadcd45..6a59424480 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -3,6 +3,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { runInNewContext } from "node:vm"; import { expect, it } from "vitest"; import { parse } from "yaml"; @@ -48,9 +49,7 @@ const workflow = async () => it("merges available failed-shard diagnostics and skips unavailable shard reports", async () => { const steps = (await workflow()).test!.steps; const download = steps.find( - (step) => - step.uses?.includes("actions/download-artifact") && - step.with?.pattern === "vitest-report-*", + (step) => step.with?.pattern === "vitest-report-*", ); const merge = steps.findIndex((step) => step.run?.includes("test:ci:merge")); const upload = steps.findIndex((step) => step.with?.name === "test-results"); @@ -59,8 +58,7 @@ it("merges available failed-shard diagnostics and skips unavailable shard report ); expect(merge).toBeGreaterThan(0); expect(download?.if).toContain("!cancelled()"); - expect(download?.if).toContain("needs.test-shard.result == 'success'"); - expect(download?.if).toContain("needs.test-shard.result == 'failure'"); + expect(upload).toBeGreaterThan(merge); expect(gate).toBeGreaterThan(upload); const merges = steps.filter((step) => step.run?.includes("--reporter=json")); @@ -70,8 +68,6 @@ it("merges available failed-shard diagnostics and skips unavailable shard report "--reporter=./scripts/ci/test-timing-reporter.mjs", ); expect(step.if).toContain("!cancelled()"); - expect(step.if).toContain("needs.test-shard.result == 'success'"); - expect(step.if).toContain("needs.test-shard.result == 'failure'"); } expect(steps[upload]?.with?.path).toContain("test-timings.json"); expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true }); @@ -86,12 +82,23 @@ it("merges available failed-shard diagnostics and skips unavailable shard report ); const command = steps[gate]?.run; if (command === undefined) throw new Error("Missing shard result gate"); - for (const result of ["failure", "cancelled", "skipped", ""]) { - await expect( - execute("bash", ["-c", command], { - env: { ...process.env, CODE_REQUIRED: "true", SHARD_RESULT: result }, - }), - ).rejects.toMatchObject({ code: 1 }); + for (const tier of ["fast", "full"]) { + for (const shard of ["success", "failure", "cancelled", "skipped", ""]) { + for (const source of ["success", "failure", "cancelled", "skipped", ""]) { + const check = execute("bash", ["-c", command], { + env: { + ...process.env, + CODE_REQUIRED: "true", + SHARD_RESULT: shard, + SOURCE_REQUIRED: String(tier === "fast"), + SOURCE_RESULT: source, + }, + }); + if (shard === "success" && (tier !== "fast" || source === "success")) + await expect(check).resolves.toMatchObject({ stderr: "" }); + else await expect(check).rejects.toMatchObject({ code: 1 }); + } + } } await expect( execute("bash", ["-c", command], { @@ -100,6 +107,80 @@ it("merges available failed-shard diagnostics and skips unavailable shard report ).resolves.toMatchObject({ stderr: "" }); }); +// These workflow conditions use the shared boolean/string expression subset. +// Evaluate the checked-in expressions so failure-state coverage cannot drift +// into a separate test implementation of the collection policy. +function conditionValue( + condition: string | undefined, + needs: Record, + canceled: boolean, +) { + if (!condition) throw new Error("Missing report collection condition"); + return runInNewContext( + condition + .replaceAll("needs.test-shard", 'needs["test-shard"]') + .replaceAll("needs.test-source", 'needs["test-source"]'), + { needs, cancelled: () => canceled }, + { timeout: 1_000 }, + ); +} + +it("collects either fast test group while keeping full reports tied to full shards", async () => { + const steps = (await workflow()).test.steps; + const download = steps.find( + (step) => step.with?.pattern === "vitest-report-*", + ); + const fast = steps.find( + (step) => step.name === "Merge fast-suite JUnit and timing reports", + ); + const full = steps.find( + (step) => step.name === "Merge coverage, JUnit, and timing reports", + ); + for (const tier of ["fast", "full"]) { + for (const shard of ["success", "failure", "cancelled", "skipped", ""]) { + for (const source of ["success", "failure", "cancelled", "skipped", ""]) { + for (const [code, canceled] of [ + ["true", false], + ["true", true], + ["false", false], + ["false", true], + ] as const) { + const needs = { + changes: { outputs: { code, tier } }, + "test-shard": { result: shard }, + "test-source": { result: source }, + }; + const builtAvailable = ["success", "failure"].includes(shard); + const sourceAvailable = ["success", "failure"].includes(source); + const enabled = code === "true" && !canceled; + const available = + builtAvailable || (tier === "fast" && sourceAvailable); + const expected = enabled + ? { + download: available, + fast: tier === "fast" && available, + full: tier === "full" && builtAvailable, + } + : { download: false, fast: false, full: false }; + expect(conditionValue(download?.if, needs, canceled)).toBe( + expected.download, + ); + expect(conditionValue(fast?.if, needs, canceled)).toBe(expected.fast); + expect(conditionValue(full?.if, needs, canceled)).toBe(expected.full); + } + } + } + } + const summary = steps.find((step) => + step.run?.includes("scripts/ci/test-report.mjs"), + ); + expect(summary?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}"); + expect(summary?.env?.SOURCE_RESULT).toBe("${{ needs.test-source.result }}"); + expect(summary?.env?.SOURCE_REQUIRED).toBe( + "${{ needs.changes.outputs.tier == 'fast' }}", + ); +}); + it("keeps diagnosis available after merge failure and publishes only complete successful history", async () => { const steps = (await workflow()).test!.steps; const diagnosis = steps.find((step) => step.with?.name === "test-results"); @@ -220,7 +301,12 @@ it("exports portable timing history and removes it on a failed, partial, or inva { name: join(root, "tests/fast.test.ts"), durationMs: 10 }, ], }; - const run = async (value: unknown, shards = "success", merge = "success") => { + const run = async ( + value: unknown, + shards = "success", + merge = "success", + source?: string, + ) => { await writeFile(join(root, "test-timings.json"), JSON.stringify(value)); await writeFile(summary, ""); return execute(process.execPath, [fileURLToPath(script)], { @@ -229,6 +315,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva ...process.env, GITHUB_STEP_SUMMARY: summary, SHARD_RESULT: shards, + SOURCE_REQUIRED: String(source !== undefined), + SOURCE_RESULT: source ?? "", MERGE_RESULT: merge, }, }); @@ -256,6 +344,24 @@ it("exports portable timing history and removes it on a failed, partial, or inva await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" }); expect(await readFile(summary, "utf8")).toContain("diagnostic only"); } + await run(report, "success", "success", "success"); + expect(await readFile(history, "utf8")).toContain('"success":true'); + for (const [shards, source] of [ + ["skipped", "success"], + ["success", "skipped"], + ["success", "cancelled"], + ["success", "failure"], + ["success", ""], + ]) { + await run(report); + await run(report, shards, "success", source); + await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" }); + const text = await readFile(summary, "utf8"); + expect(text).toContain(`Source result: ${source}.`); + expect(text).toContain("diagnostic only"); + if (shards === "skipped" || source !== "failure") + expect(text).toContain("Partial test diagnostics"); + } await run(report); await expect( run({ @@ -352,6 +458,21 @@ it("recovers complete module costs from real blob reports without executing test expect(file.durationMs).toBeCloseTo( Object.values(file.phases).reduce((sum, value) => sum + value, 0), ); + const partial = await execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { + ...process.env, + SHARD_RESULT: "skipped", + SOURCE_REQUIRED: "true", + SOURCE_RESULT: "success", + MERGE_RESULT: "success", + }, + }); + expect(partial.stdout).toContain("Partial test diagnostics"); + expect(partial.stdout).toContain("sample.test.js"); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); await execute(process.execPath, [fileURLToPath(script)], { cwd: root, env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" }, @@ -420,13 +541,15 @@ it("records failed omissions from full reports without treating partial or unhan state: index === 1 ? "failed" : "passed", })), }; - const run = async (report: unknown) => { + const run = async (report: unknown, source = "failure") => { await writeFile(join(root, "test-timings.json"), JSON.stringify(report)); await execute(process.execPath, [fileURLToPath(script)], { cwd: root, env: { ...process.env, SHARD_RESULT: "failure", + SOURCE_REQUIRED: "true", + SOURCE_RESULT: source, MERGE_RESULT: "failure", CI_VERIFY_TEST_INVENTORY: "true", CI_TEST_SELECTION: JSON.stringify( @@ -458,6 +581,8 @@ it("records failed omissions from full reports without treating partial or unhan { ...timing, reason: "interrupted" }, ]) expect(await run(report)).toMatchObject({ complete: false }); + for (const source of ["skipped", "cancelled", ""]) + expect(await run(timing, source)).toMatchObject({ complete: false }); await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({ code: 1, }); diff --git a/tests/boundary/filesystem/ciWorkflowRun.test.ts b/tests/boundary/filesystem/ciWorkflowRun.test.ts new file mode 100644 index 0000000000..cbebb5c4d1 --- /dev/null +++ b/tests/boundary/filesystem/ciWorkflowRun.test.ts @@ -0,0 +1,54 @@ +import { expect, it } from "vitest"; +import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs"; + +it.each([ + [".github/workflows/ci.yml", ".github/workflows/ci.yml", null], + [".github/workflows/ci.yml@main", ".github/workflows/ci.yml", "main"], + [ + ".github/workflows/ci.yml@refs/heads/release/v1", + ".github/workflows/ci.yml", + "refs/heads/release/v1", + ], + [ + ".github/workflows/ci.yml@feature@v1", + ".github/workflows/ci.yml", + "feature@v1", + ], + [ + ".github/workflows/release.yaml@main", + ".github/workflows/release.yaml", + "main", + ], +])( + "preserves API path %s while separating its file identity and ref", + (path, file, ref) => { + const raw = { id: 10, path, head_sha: "a".repeat(40) }; + const parsed = normalizeWorkflowRun(raw); + expect(parsed).toEqual({ ...raw, workflowPath: { file, ref } }); + expect(raw).not.toHaveProperty("workflowPath"); + }, +); + +it.each([ + undefined, + null, + 42, + "", + ".github/workflows/ci.yml@", + ".github/workflows/ci.yml@main\n", + ".github/workflows/ci.yml@main\0", + "../.github/workflows/ci.yml", + ".github/workflows/../ci.yml@main", + ".github/workflows/ci.yml-extra@main", + "https://example.test/ci.yml@main", +])( + "keeps malformed workflow path %s unknown instead of inventing a file identity", + (path) => { + const parsed = normalizeWorkflowRun({ + path, + workflowPath: { file: ".github/workflows/ci.yml", ref: "main" }, + }); + expect(parsed.path).toBe(path); + expect(parsed.workflowPath).toBeNull(); + }, +); diff --git a/turbo.json b/turbo.json index ffee2a0704..35e7ece044 100644 --- a/turbo.json +++ b/turbo.json @@ -63,9 +63,7 @@ "!docs/.vitepress/dist/**", "!website/_site/**", "!experiments/analysis-efficiency/results/**", - "vitest.reverse-tools.config.ts", - "vitest.reverse-tools-semantics.config.ts", - "vitest.config.ts" + "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}" ] }, "lint:built": { @@ -86,9 +84,7 @@ "!docs/.vitepress/dist/**", "!website/_site/**", "!experiments/analysis-efficiency/results/**", - "vitest.reverse-tools.config.ts", - "vitest.reverse-tools-semantics.config.ts", - "vitest.config.ts" + "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}" ] }, "format:check": {}, @@ -104,8 +100,8 @@ "tests/**/*.ts", "tsconfig.json", "tsconfig.build.json", - "vitest.config.ts", - "docs/.vitepress/config.ts" + "docs/.vitepress/config.ts", + "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}" ] }, "jscpd": { diff --git a/vitest.config.ts b/vitest.config.ts index a8bf6b0454..2c034beb2a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,6 +3,10 @@ import { availableParallelism, tmpdir } from "node:os"; import { join } from "node:path"; import { defineConfig } from "vitest/config"; +import { + BUILT_TEST_PROJECTS, + SOURCE_TEST_PROJECTS, +} from "./scripts/lib/test-projects.mjs"; import { CiTestSequencer } from "./tests/ciTestSequencer.js"; @@ -28,57 +32,32 @@ const COVERAGE_SHARD = process.argv.some((argument) => // Keep local and CI concurrency bounded by the runner rather than command locks. const MAX_TEST_WORKERS = Math.min(2, availableParallelism()); -const TEST_PROJECTS = [ - { - name: "domain", - include: ["src/{contracts,domain}/**/*.test.ts"], +const TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]; +const PROJECT_POLICIES = { + domain: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Domain and contract modules are pure and own no process-global state, so // per-file module isolation adds startup cost without protecting a seam. isolate: false, }, - { - name: "services", - include: ["src/application/**/*.test.ts"], + services: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Service tests use recording ports and own no process-global state. Share // their module graph so file startup does not dominate the lane. isolate: false, }, - { - name: "adapters", - include: [ - "src/*.test.ts", - "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", - ], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "composition", - include: ["tests/composition/**/*.test.ts"], + adapters: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + composition: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Composition tests use provider-neutral sessions and test-scoped cleanup; // sharing their immutable module graph avoids per-file startup work. isolate: false, }, - { - name: "boundary", - include: ["tests/boundary/**/*.test.ts"], - exclude: [ - "tests/boundary/cli/**/*.test.ts", - "tests/boundary/mcp/**/*.test.ts", - "tests/boundary/process/**/*.test.ts", - ], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "cli-boundary", - include: ["tests/boundary/cli/**/*.test.ts"], + boundary: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + "cli-boundary": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS, // CLI tests create fresh command instances or subprocesses and own their @@ -88,54 +67,36 @@ const TEST_PROJECTS = [ // before their two heaps compete with the real CLI child processes. execArgv: ["--max-old-space-size=384"], }, - { - // Process-tree observations share host sampling resources; keep this small - // lane serial while the independent projects run together. - name: "process-boundary", - include: ["tests/boundary/process/**/*.test.ts"], + "process-boundary": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS, + // Process-tree observations share host sampling resources; keep this small + // lane serial while the independent projects run together. fileParallelism: false, sequence: { groupOrder: 1 }, }, - { - name: "mcp-boundary", - include: ["tests/boundary/mcp/**/*.test.ts"], + "mcp-boundary": { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Each file creates and closes an explicit in-memory MCP session. Reuse the // immutable server graph so module startup does not outweigh transport work. isolate: false, }, - { - name: "acceptance", - include: ["tests/acceptance/**/*.test.ts"], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "process-global", - include: ["tests/process-global/**/*.test.ts"], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "conformance", - include: ["tests/conformance/**/*.test.ts"], + acceptance: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + "process-global": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + conformance: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Contract cases use recording providers and own no process-global state. isolate: false, }, - { - name: "evaluation", - include: ["tests/evaluation/**/*.test.ts"], + evaluation: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Evaluator tests inspect deterministic inputs without mutating shared state. isolate: false, }, -]; +}; const ZERO_COVERAGE_THRESHOLDS = { statements: 0, @@ -156,10 +117,21 @@ const ZERO_COVERAGE_THRESHOLDS = { }, }; -const projects = TEST_PROJECTS.map((project) => ({ - extends: true as const, - test: project, -})); +const projects = TEST_PROJECTS.map((project) => { + const policy = + PROJECT_POLICIES[project.name as keyof typeof PROJECT_POLICIES]; + if (!policy) + throw new Error(`Missing test runner policy for ${project.name}`); + return { + extends: true as const, + test: { + name: project.name, + include: [...project.include], + ...(project.exclude ? { exclude: [...project.exclude] } : {}), + ...policy, + }, + }; +}); export default defineConfig({ test: { From 4d98123279c5a6f5442c458307f5a3ec678fd49e Mon Sep 17 00:00:00 2001 From: Bruno Costa Candia <55243348+bruno-candia@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:08:31 -0300 Subject: [PATCH 04/14] fix(hopper): recover from a Spotlight not-found launch (#2795) * fix(hopper): recover from a Spotlight not-found launch Hopper's native launcher locates the application through a fixed Spotlight query before any Apple event. With indexing disabled it exits 3 with a not-found report, but REA treated the external document as unconfirmed, retained the runtime and failed every later open_binary until the server restarted. Recognize the complete report as a pre-handoff failure, confirm no document exists, and tell the caller to enable Spotlight indexing. Co-Authored-By: Claude Opus 5.5 * fix(hopper): require empty stdout for native launch signatures * fix(hopper): retain ownership after unverified Spotlight reports --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/hopper/HopperClient.ts | 8 ++++ src/hopper/HopperProcessDiagnostic.test.ts | 48 ++++++++++++++++++- src/hopper/HopperProcessDiagnostic.ts | 27 ++++++++++- .../providers/hopperStartupRecovery.test.ts | 24 +++++++--- tests/fixtures/hopperLaunchAdmission.mjs | 6 +++ 5 files changed, 104 insertions(+), 9 deletions(-) diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index 8196adcee7..8af42774cc 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -59,6 +59,7 @@ import { import { connectHopperSocketOnce } from "./HopperSocketConnection.js"; import { hopperLauncherFailureDiagnostic, + hopperLauncherReportedApplicationNotFound, nativeHopperLaunchWasDenied, } from "./HopperProcessDiagnostic.js"; import { DEFAULT_HOPPER_STARTUP_TIMEOUT_MS } from "../config/providerStartupTimeout.js"; @@ -768,6 +769,13 @@ export class HopperClient { launcherFailure, }); } + if (hopperLauncherReportedApplicationNotFound(launcherFailure)) { + return new HopperStartError({ + userMessage: + "Hopper's launcher could not find Hopper because it relies on Spotlight. Enable Spotlight indexing for the volume that contains Hopper (for example: sudo mdutil -i on /System/Volumes/Data), wait for indexing to finish, then retry opening the target.", + launcherFailure, + }); + } return new HopperStartError({ userMessage: "Hopper's launcher helper failed before bridge readiness. Review details.launcher and complete Hopper setup before retrying.", diff --git a/src/hopper/HopperProcessDiagnostic.test.ts b/src/hopper/HopperProcessDiagnostic.test.ts index 2fe1c4cc9f..ac4ee75af6 100644 --- a/src/hopper/HopperProcessDiagnostic.test.ts +++ b/src/hopper/HopperProcessDiagnostic.test.ts @@ -1,7 +1,10 @@ import { expect, it } from "vitest"; import type { HopperLauncherOutcome } from "../domain/hopperErrors.js"; -import { nativeHopperLaunchWasDenied } from "./HopperProcessDiagnostic.js"; +import { + hopperLauncherReportedApplicationNotFound, + nativeHopperLaunchWasDenied, +} from "./HopperProcessDiagnostic.js"; // Captured from Hopper 6.1.0-demo and 6.6.0-demo's unmodified helpers with native // AppleScript error -1743 before open dispatch. The target script never ran. @@ -23,6 +26,8 @@ it("recognizes the native helper's complete authorization rejection", () => { }); it.each([ + { stdout: { text: "wrapper output\n", bytes: 15, retained_bytes: 15 } }, + { stdout: { text: "", bytes: 1, retained_bytes: 0 } }, { output_closed: false }, { diagnostic_truncated: true }, { signal: "SIGTERM", exit_code: null }, @@ -52,3 +57,44 @@ it.each([ expect(nativeHopperLaunchWasDenied({ ...denied, ...change })).toBe(false); }, ); + +// Captured from Hopper 6's unmodified helper with Spotlight indexing disabled. +const notFound: HopperLauncherOutcome = { + exit_code: 3, + signal: null, + stdout: { text: "", bytes: 0, retained_bytes: 0 }, + stderr: { + text: "Hopper has not been found.\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable.\n", + bytes: 121, + retained_bytes: 121, + }, + output_closed: true, + diagnostic_truncated: false, +}; + +it("recognizes the native helper's complete Spotlight not-found report", () => { + expect(hopperLauncherReportedApplicationNotFound(notFound)).toBe(true); + expect(nativeHopperLaunchWasDenied(notFound)).toBe(false); +}); + +it.each([ + { stdout: { text: "wrapper output\n", bytes: 15, retained_bytes: 15 } }, + { stdout: { text: "", bytes: 1, retained_bytes: 0 } }, + { output_closed: false }, + { diagnostic_truncated: true }, + { signal: "SIGTERM", exit_code: null }, + { exit_code: 2 }, + { + stderr: { + ...notFound.stderr, + text: `${notFound.stderr.text}another failure\n`, + }, + }, +])( + "rejects other not-found diagnostic shapes: %j", + (change) => { + expect( + hopperLauncherReportedApplicationNotFound({ ...notFound, ...change }), + ).toBe(false); + }, +); diff --git a/src/hopper/HopperProcessDiagnostic.ts b/src/hopper/HopperProcessDiagnostic.ts index cf16fb9bb8..47348d8817 100644 --- a/src/hopper/HopperProcessDiagnostic.ts +++ b/src/hopper/HopperProcessDiagnostic.ts @@ -6,6 +6,14 @@ import type { ProviderProcessDiagnostic } from "../process/ProviderProcess.js"; import { parseLinuxPrivateDisplayDiagnostic } from "./LinuxPrivateDisplayDiagnostic.js"; import type { HopperLauncherOutcome } from "../domain/hopperErrors.js"; +/** Recognize launcher stderr signatures only with complete, empty stdout. */ +const hasCompleteLauncherOutput = (outcome: HopperLauncherOutcome): boolean => + outcome.output_closed && + !outcome.diagnostic_truncated && + outcome.stdout.text === "" && + outcome.stdout.bytes === 0 && + outcome.stdout.retained_bytes === 0; + /** * Hopper's native helper sends activate before its open-document Apple event. * Recognize its explicit authorization rejection. The launcher must separately @@ -18,12 +26,27 @@ export const nativeHopperLaunchWasDenied = ( outcome !== undefined && outcome.exit_code === 2 && outcome.signal === null && - outcome.output_closed && - !outcome.diagnostic_truncated && + hasCompleteLauncherOutput(outcome) && /^Cannot launch Hopper!\r?\nNot authorized to send Apple events to [^\r\n]+\.\r?\n?$/u.test( outcome.stderr.text, ); +/** + * Recognize the launcher's complete Spotlight diagnostic for actionable setup + * guidance. Custom launchers may forward this report after a handoff, so the + * report does not establish whether a document exists or release ownership. + */ +export const hopperLauncherReportedApplicationNotFound = ( + outcome: HopperLauncherOutcome | undefined, +): outcome is HopperLauncherOutcome => + outcome !== undefined && + outcome.exit_code === 3 && + outcome.signal === null && + hasCompleteLauncherOutput(outcome) && + /^Hopper has not been found\.\r?\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable\.\r?\n?$/u.test( + outcome.stderr.text, + ); + /** Extract only a matching Linux adapter failure record. */ export const hopperLauncherFailureDiagnostic = ( event: Extract, diff --git a/tests/acceptance/providers/hopperStartupRecovery.test.ts b/tests/acceptance/providers/hopperStartupRecovery.test.ts index 7fb039d4d9..65882a86bf 100644 --- a/tests/acceptance/providers/hopperStartupRecovery.test.ts +++ b/tests/acceptance/providers/hopperStartupRecovery.test.ts @@ -201,13 +201,13 @@ test("recovers a denied Hopper launch through the same stdio MCP session and CLI ).toBe(false); }); -test( - "preserves a timed-out handoff and closes only a late bridge with matching run identity", +test.each(["handoff", "not-found"] as const)( + "preserves an unconfirmed %s handoff and closes only a late bridge with matching run identity", { timeout: 60_000 }, - async ({ workspace }) => { + async (launchMode, { workspace }) => { const target = workspace.path("target"); await copyFile("/usr/bin/true", target); - const mode = await workspace.write("mode", "handoff"); + const mode = await workspace.write("mode", launchMode); const receipt = workspace.path("bootstrap-path"); const transport = new StdioClientTransport({ command: process.execPath, @@ -257,12 +257,24 @@ test( details: { diagnostics: { primary_error: { - code: "provider_timeout", - details: { timeout_ms: 2000, launcher: { exit_code: 0 } }, + code: + launchMode === "handoff" + ? "provider_timeout" + : "provider_unavailable", + details: + launchMode === "handoff" + ? { timeout_ms: 2000, launcher: { exit_code: 0 } } + : { stage: "launch", launcher: { exit_code: 3 } }, }, }, }, }); + if (launchMode === "not-found") + expect(initial.error.details).toMatchObject({ + diagnostics: { + primary_error: { message: expect.stringContaining("Spotlight") }, + }, + }); expect(parseMcpToolError(await call("list_procedures"))).toEqual(initial); const status = z.object({ result: z.object({ diff --git a/tests/fixtures/hopperLaunchAdmission.mjs b/tests/fixtures/hopperLaunchAdmission.mjs index f7c81d9bfa..bb80ad4489 100644 --- a/tests/fixtures/hopperLaunchAdmission.mjs +++ b/tests/fixtures/hopperLaunchAdmission.mjs @@ -19,6 +19,12 @@ if (mode === "denied" || mode === "denied-after-handoff") { process.exit(2); }, ); +} else if (mode === "not-found") { + // The native helper's Spotlight lookup fails before any Apple event. + process.stderr.write( + "Hopper has not been found.\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable.\n", + () => process.exit(3), + ); } else if (mode === "handoff") { process.exit(0); } else if (mode === "bridge") { From 4acda94b76203cae15363bdf65be5b0e1390395b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:19:47 +0800 Subject: [PATCH 05/14] fix(hopper): recover provenance-verified Spotlight exits Verify the native helper's sealed identifier and pinned Apple signing team in BridgeLauncher, then carry that provenance to HopperClient. Only a verified native helper's complete not-found exit confirms no document handoff, allowing runtime and target-lease cleanup and retry; wrappers retain unknown document state. Share publisher identity with setup through the domain contract. Correct fixture teardown ordering and parameterized MCP coverage. Validation: 54 focused tests passed across boundary, CLI, MCP acceptance, diagnostics and installer coverage (one Linux-only case skipped); check:fast passed. Real codesign verification accepted the installed official Hopper helper; Spotlight failures and bridge recovery used process fixtures. --- src/application/MacHopper.ts | 29 ++---- src/application/SetupInstallFailure.test.ts | 2 +- src/application/SetupInstallFailure.ts | 2 +- src/domain/hopperPublisher.ts | 12 +++ src/hopper/BridgeLauncher.ts | 14 +++ src/hopper/HopperClient.ts | 4 + src/hopper/HopperMacPublisher.ts | 36 +++++++ src/process/MacCodeSignature.ts | 7 ++ .../providers/hopperStartupRecovery.test.ts | 83 ++++++++-------- .../cli/hopperLaunchFailureCli.test.ts | 98 ++++++++++--------- .../hopper/hopperClient.helperFailure.test.ts | 81 +++++++++++++++ .../macosHopperPublisherVerification.test.ts | 27 ++++- tests/fixtures/hopperLaunchAdmission.mjs | 3 +- 13 files changed, 286 insertions(+), 112 deletions(-) create mode 100644 src/domain/hopperPublisher.ts create mode 100644 src/hopper/HopperMacPublisher.ts create mode 100644 src/process/MacCodeSignature.ts diff --git a/src/application/MacHopper.ts b/src/application/MacHopper.ts index a211e9f2dd..858e7fe489 100644 --- a/src/application/MacHopper.ts +++ b/src/application/MacHopper.ts @@ -15,6 +15,8 @@ import { promisify } from "node:util"; import { z } from "zod"; import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js"; +import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js"; +import { macDeveloperIdRequirement } from "../process/MacCodeSignature.js"; import { downloadHopperPackage, HOPPER_DOWNLOAD_BYTES, @@ -24,29 +26,10 @@ const execFileAsync = promisify(execFile); const RELEASES_URL = "https://www.hopperapp.com/include/files-api.php?request=releases&public=true"; const DOWNLOAD_PREFIX = "https://www.hopperapp.com:443/downloader/public/"; - -/** - * Apple publisher identity pinned for the official Hopper macOS bundle. The - * team identifier was captured from `Hopper-6.6.0-demo.dmg` at - * `https://www.hopperapp.com:443/downloader/public/Hopper-6.6.0-demo.dmg` - * (34846271 bytes, SHA-1 0be530f038249742590992199c7b042ea20ca955) with - * `codesign -dvvv`, which reported - * `Authority=Developer ID Application: Cryptic Apps (2AMA2753NF)`. - */ -export const HOPPER_PUBLISHER = Object.freeze({ - bundleIdentifier: "com.cryptic-apps.hopper-web-4", - teamIdentifier: "2AMA2753NF", -}); - -/** - * Code requirement enforcing the bundle identifier and signing team together. - * The identifier is read from the sealed CodeDirectory, so it cannot be changed - * by editing `Info.plist` the way the previous string comparison could. - */ -const HOPPER_PUBLISHER_REQUIREMENT = - `= identifier "${HOPPER_PUBLISHER.bundleIdentifier}"` + - ` and anchor apple generic` + - ` and certificate leaf[subject.OU] = "${HOPPER_PUBLISHER.teamIdentifier}"`; +const HOPPER_PUBLISHER_REQUIREMENT = macDeveloperIdRequirement( + HOPPER_PUBLISHER.bundleIdentifier, + HOPPER_PUBLISHER.teamIdentifier, +); /** Outcome of reading a mounted bundle's signature and publisher identity. */ export type MacHopperBundleVerification = diff --git a/src/application/SetupInstallFailure.test.ts b/src/application/SetupInstallFailure.test.ts index 980e221510..d8ba27bdb9 100644 --- a/src/application/SetupInstallFailure.test.ts +++ b/src/application/SetupInstallFailure.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { HOPPER_PUBLISHER } from "./MacHopper.js"; +import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js"; import { setupInstallFailure } from "./SetupInstallFailure.js"; describe("macOS Hopper setup failure projection", () => { diff --git a/src/application/SetupInstallFailure.ts b/src/application/SetupInstallFailure.ts index 0b332fa25f..944a603430 100644 --- a/src/application/SetupInstallFailure.ts +++ b/src/application/SetupInstallFailure.ts @@ -1,4 +1,4 @@ -import { HOPPER_PUBLISHER } from "./MacHopper.js"; +import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js"; export type SetupFailureCode = | "unsupported_host" diff --git a/src/domain/hopperPublisher.ts b/src/domain/hopperPublisher.ts new file mode 100644 index 0000000000..6c212ea64d --- /dev/null +++ b/src/domain/hopperPublisher.ts @@ -0,0 +1,12 @@ +/** + * Apple publisher identity pinned for the official Hopper macOS bundle. The + * team identifier was captured from `Hopper-6.6.0-demo.dmg` at + * `https://www.hopperapp.com:443/downloader/public/Hopper-6.6.0-demo.dmg` + * (34846271 bytes, SHA-1 0be530f038249742590992199c7b042ea20ca955) with + * `codesign -dvvv`, which reported + * `Authority=Developer ID Application: Cryptic Apps (2AMA2753NF)`. + */ +export const HOPPER_PUBLISHER = Object.freeze({ + bundleIdentifier: "com.cryptic-apps.hopper-web-4", + teamIdentifier: "2AMA2753NF", +}); diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index 043d394a4b..3f83867bf9 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -33,6 +33,7 @@ import { } from "./HopperTargetLease.js"; import type { HopperTargetLease } from "./HopperTargetLease.js"; import { HOPPER_SHUTDOWN_TIMEOUT_MS } from "./HopperSessionValues.js"; +import { verifyMacHopperNativeHelper } from "./HopperMacPublisher.js"; const execFileAsync = promisify(execFile); const HOPPER_BACKGROUND_STARTUP_MS = 5_000; @@ -47,6 +48,8 @@ export interface BridgeSession { /** Process handle returned by a bridge launcher. */ export type BridgeLaunch = { + /** Verified publisher and sealed native-helper identity; omission is unknown. */ + readonly nativeHelperVerified?: true; /** Only literal native-helper arguments support a pre-handoff denial conclusion. */ readonly authorizationDenialBeforeHandoff?: true; /** Release application/target exclusivity only after verified document and process cleanup. */ @@ -106,6 +109,7 @@ export type HopperApplicationLauncherOptions = }); export interface HopperApplicationLauncherDependencies { + readonly verifyNativeHelper?: typeof verifyMacHopperNativeHelper; readonly platform?: NodeJS.Platform; readonly acquireTargetLease?: typeof acquireHopperTargetLease; readonly acquireApplicationLease?: typeof acquireLinuxHopperApplicationLease; @@ -213,6 +217,13 @@ export class HopperApplicationLauncher implements BridgeLauncher { if (signal?.aborted === true) return err(new HopperCancelledError()); const display = await this.#privateDisplay(signal); if (!display.ok) return err(display.error); + const nativeHelperVerified = + (this.dependencies.platform ?? process.platform) === "darwin" && + this.options.launchMode === "native" && + (await ( + this.dependencies.verifyNativeHelper ?? verifyMacHopperNativeHelper + )(this.options.launcherPath, this.options.environment, signal)) === + "verified"; const prepared = await prepareHopperApplication( this.options.launcherPath, this.options.environment, @@ -257,6 +268,9 @@ export class HopperApplicationLauncher implements BridgeLauncher { ...ownedLauncher, providerLifetime: "external-application", shutdownMode: "bridge-request", + ...(nativeHelperVerified + ? { nativeHelperVerified: true as const } + : {}), // The vendor helper interpolates these values into AppleScript // literals without escaping. Preserve them, but withhold the // pre-handoff conclusion when they can alter that script. diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index 8af42774cc..2943839f43 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -770,6 +770,10 @@ export class HopperClient { }); } if (hopperLauncherReportedApplicationNotFound(launcherFailure)) { + // Only the verified native helper's Spotlight exit precedes any target + // handoff. Wrappers can forward the same message after opening a document. + if (this.#resources.launch?.nativeHelperVerified === true) + this.#resources.shutdownConfirmed = true; return new HopperStartError({ userMessage: "Hopper's launcher could not find Hopper because it relies on Spotlight. Enable Spotlight indexing for the volume that contains Hopper (for example: sudo mdutil -i on /System/Volumes/Data), wait for indexing to finish, then retry opening the target.", diff --git a/src/hopper/HopperMacPublisher.ts b/src/hopper/HopperMacPublisher.ts new file mode 100644 index 0000000000..f60852915e --- /dev/null +++ b/src/hopper/HopperMacPublisher.ts @@ -0,0 +1,36 @@ +import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js"; +import { macDeveloperIdRequirement } from "../process/MacCodeSignature.js"; +import { execFileOutput } from "../process/ExecFileOutput.js"; + +/** Failure or unavailable verification leaves custom/native provenance unknown. */ +export const verifyMacHopperNativeHelper = async ( + launcherPath: string, + environment: Readonly, + signal?: AbortSignal, +): Promise<"verified" | "unknown"> => { + // codesign treats a bare name as a local file, while spawn searches PATH. + // Without an explicit path they need not identify the same executable. + if (!launcherPath.includes("/")) return "unknown"; + try { + await execFileOutput( + "/usr/bin/codesign", + [ + "--verify", + "--strict", + `-R${macDeveloperIdRequirement("hopper", HOPPER_PUBLISHER.teamIdentifier)}`, + launcherPath, + ], + { + env: environment, + ...(signal === undefined ? {} : { signal }), + timeout: 5_000, + stopSignal: "SIGKILL", + }, + ); + return "verified"; + } catch (cause: unknown) { + // Optional provenance: a failed check must never authorize lease release. + void cause; + return "unknown"; + } +}; diff --git a/src/process/MacCodeSignature.ts b/src/process/MacCodeSignature.ts new file mode 100644 index 0000000000..92cd7f86ff --- /dev/null +++ b/src/process/MacCodeSignature.ts @@ -0,0 +1,7 @@ +/** Require a sealed identifier and Apple Developer ID signing team together. */ +export const macDeveloperIdRequirement = ( + identifier: string, + teamIdentifier: string, +): string => + `= identifier "${identifier}" and anchor apple generic` + + ` and certificate leaf[subject.OU] = "${teamIdentifier}"`; diff --git a/tests/acceptance/providers/hopperStartupRecovery.test.ts b/tests/acceptance/providers/hopperStartupRecovery.test.ts index 65882a86bf..842fdaa077 100644 --- a/tests/acceptance/providers/hopperStartupRecovery.test.ts +++ b/tests/acceptance/providers/hopperStartupRecovery.test.ts @@ -50,7 +50,42 @@ test( }); const mcp = new Client({ name: "hopper-literal-boundary", version: "1" }); let runtime: string | undefined; - onTestFinished(async () => { + try { + await mcp.connect(transport); + const call = (name: string, arguments_: Record = {}) => + mcp.callTool({ name, arguments: arguments_ }); + expect((await call("open_binary", { path: target })).isError).not.toBe( + true, + ); + const first = parseMcpToolError(await call("current_document")); + runtime = dirname(await readFile(receipt, "utf8")); + expect(first.error).toMatchObject({ + code: "cleanup_incomplete", + details: { + diagnostics: { + primary_error: { + code: "provider_unavailable", + details: { + launcher: { + exit_code: 2, + stderr: { + text: expect.stringContaining( + "Not authorized to send Apple events", + ), + }, + }, + }, + }, + }, + }, + }); + await access(`${receipt}.document-created`); + await access(runtime); + expect(parseMcpToolError(await call("list_procedures"))).toEqual(first); + expect(parseMcpToolError(await call("close_binary")).error.code).toBe( + "cleanup_incomplete", + ); + } finally { // SDK close can return immediately after SIGKILL. Wait for its actual // child-close event before recovering this process's retained lease. let serverExited: Promise | undefined; @@ -77,41 +112,7 @@ test( }); if (!acquired.acquired) throw new Error("Fixture lease remained live"); await acquired.lease.release(); - }); - await mcp.connect(transport); - const call = (name: string, arguments_: Record = {}) => - mcp.callTool({ name, arguments: arguments_ }); - expect((await call("open_binary", { path: target })).isError).not.toBe( - true, - ); - const first = parseMcpToolError(await call("current_document")); - runtime = dirname(await readFile(receipt, "utf8")); - expect(first.error).toMatchObject({ - code: "cleanup_incomplete", - details: { - diagnostics: { - primary_error: { - code: "provider_unavailable", - details: { - launcher: { - exit_code: 2, - stderr: { - text: expect.stringContaining( - "Not authorized to send Apple events", - ), - }, - }, - }, - }, - }, - }, - }); - await access(`${receipt}.document-created`); - await access(runtime); - expect(parseMcpToolError(await call("list_procedures"))).toEqual(first); - expect(parseMcpToolError(await call("close_binary")).error.code).toBe( - "cleanup_incomplete", - ); + } } }, ); @@ -201,7 +202,7 @@ test("recovers a denied Hopper launch through the same stdio MCP session and CLI ).toBe(false); }); -test.each(["handoff", "not-found"] as const)( +test.for(["handoff", "not-found"] as const)( "preserves an unconfirmed %s handoff and closes only a late bridge with matching run identity", { timeout: 60_000 }, async (launchMode, { workspace }) => { @@ -269,17 +270,21 @@ test.each(["handoff", "not-found"] as const)( }, }, }); - if (launchMode === "not-found") + if (launchMode === "not-found") { + await access(`${receipt}.document-created`); expect(initial.error.details).toMatchObject({ diagnostics: { primary_error: { message: expect.stringContaining("Spotlight") }, }, }); + } expect(parseMcpToolError(await call("list_procedures"))).toEqual(initial); const status = z.object({ result: z.object({ provider_operation_health: z.object({ - state: z.literal("not_started"), + state: z.literal( + launchMode === "handoff" ? "not_started" : "unknown", + ), stage: z.literal("launch"), }), }), diff --git a/tests/boundary/cli/hopperLaunchFailureCli.test.ts b/tests/boundary/cli/hopperLaunchFailureCli.test.ts index 476f58f6be..9a4f55b7cb 100644 --- a/tests/boundary/cli/hopperLaunchFailureCli.test.ts +++ b/tests/boundary/cli/hopperLaunchFailureCli.test.ts @@ -1,6 +1,6 @@ import { chmod, copyFile, rm } from "node:fs/promises"; -import { describe, expect, onTestFinished } from "vitest"; +import { describe, expect } from "vitest"; import { z } from "zod"; import { workspaceCliTest } from "../../support/cli/workspaceCliFixture.js"; @@ -39,7 +39,57 @@ describe("Hopper launcher failure through the one-shot CLI", () => { const targetPath = workspace.path("probe"); await copyFile("/usr/bin/true", targetPath); const loaderArgs = ["--probe"]; - onTestFinished(async () => { + try { + for (const attempt of ["first", "retry"]) { + const result = await cli.run({ + arguments: [ + "analyze", + targetPath, + "--provider", + "hopper", + "--json", + ], + environment: { + ...workspace.environment, + HOPPER_LAUNCHER_PATH: launcherPath, + HOPPER_LOADER_ARGS_JSON: JSON.stringify(loaderArgs), + REA_LOG_LEVEL: "silent", + }, + timeoutMs: 45_000, + }); + const retained = RetainedRuntime.safeParse(result.json); + if (retained.success) + await rm( + retained.data.details.diagnostics.cleanup_error.details + .diagnostics.runtime_root, + { recursive: true, force: true }, + ); + + expect(result.exitCode, attempt).toBe(1); + expect(result.json, attempt).toMatchObject({ + code: "cleanup_incomplete", + details: { + cleanup: "incomplete", + diagnostics: { + primary_error: { + code: "provider_unavailable", + details: { + stage: "launch", + launcher: { + exit_code: 2, + stderr: { + text: expect.stringContaining( + "Document state is unknown", + ), + }, + }, + }, + }, + }, + }, + }); + } + } finally { const recovered = await acquireHopperTargetLease({ targetPath, targetKind: "executable", @@ -48,50 +98,6 @@ describe("Hopper launcher failure through the one-shot CLI", () => { }); expect(recovered.acquired).toBe(true); if (recovered.acquired) await recovered.lease.release(); - }); - - for (const attempt of ["first", "retry"]) { - const result = await cli.run({ - arguments: ["analyze", targetPath, "--provider", "hopper", "--json"], - environment: { - ...workspace.environment, - HOPPER_LAUNCHER_PATH: launcherPath, - HOPPER_LOADER_ARGS_JSON: JSON.stringify(loaderArgs), - REA_LOG_LEVEL: "silent", - }, - timeoutMs: 45_000, - }); - const retained = RetainedRuntime.safeParse(result.json); - if (retained.success) - await rm( - retained.data.details.diagnostics.cleanup_error.details.diagnostics - .runtime_root, - { recursive: true, force: true }, - ); - - expect(result.exitCode, attempt).toBe(1); - expect(result.json, attempt).toMatchObject({ - code: "cleanup_incomplete", - details: { - cleanup: "incomplete", - diagnostics: { - primary_error: { - code: "provider_unavailable", - details: { - stage: "launch", - launcher: { - exit_code: 2, - stderr: { - text: expect.stringContaining( - "Document state is unknown", - ), - }, - }, - }, - }, - }, - }, - }); } }, ); diff --git a/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts b/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts index 06e725dc18..b236b21ebe 100644 --- a/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts @@ -15,6 +15,7 @@ import type { } from "../../../../src/hopper/BridgeLauncher.js"; import { HopperApplicationLauncher } from "../../../../src/hopper/BridgeLauncher.js"; import { HopperClient } from "../../../../src/hopper/HopperClient.js"; +import { acquireHopperTargetLease } from "../../../../src/hopper/HopperTargetLease.js"; import { HopperFixtureLauncher } from "./hopperClient.fixture.js"; // Failure assertions keep the external document unknown. Teardown then publishes @@ -118,6 +119,86 @@ it("does not equate an owned native helper exit with an external GUI exit", asyn }); }); +it.skipIf(process.platform !== "darwin")( + "releases the runtime and target lease after a provenance-verified Spotlight exit, then retries", + async () => { + const directory = await createTestTempDirectory( + "rea-hopper-native-not-found-", + ); + const targetPath = join(directory, "target"); + await writeFile(targetPath, "fixture target"); + const loaderArgs = [ + "-e", + 'process.stderr.write("Hopper has not been found.\\nPlease make sure that Spotlight is active in order for this tool to find Hopper\'s executable.\\n", () => process.exit(3))', + "--", + ]; + // Simulate a verified native helper at the launcher provenance boundary; + // publisher verification itself has real codesign boundary coverage. + const native = new HopperApplicationLauncher( + { + environment: {}, + launcherPath: process.execPath, + targetPath, + targetKind: "executable", + loaderArgs, + bridgeScriptPath: "/rea/hopper_bridge.py", + launchMode: "native", + }, + { verifyNativeHelper: async () => "verified" }, + ); + const retry = new HopperFixtureLauncher(); + let session: BridgeSession | undefined; + const client = new HopperClient({ + launcher: { + async launch(input, options) { + if (session !== undefined) return retry.launch(input); + session = input; + const launched = await native.launch(input, options); + expect(launched).toMatchObject({ + ok: true, + value: { nativeHelperVerified: true }, + }); + return launched; + }, + }, + startupTimeoutMs: 10_000, + }); + onTestFinished(async () => { + const closed = await client.close(); + if (!closed.ok) throw closed.error; + }); + const failed = await client.start(); + expect(failed).toMatchObject({ + ok: false, + error: { _tag: "HopperStartError", cleanupIncomplete: false }, + }); + if (failed.ok) throw new Error("Expected Spotlight failure"); + expect(projectAnalysisError(failed.error)).toMatchObject({ + code: "provider_unavailable", + message: expect.stringContaining("Spotlight"), + details: { launcher: { exit_code: 3 } }, + }); + if (session === undefined) throw new Error("Missing launch session"); + await expect(access(session.directory)).rejects.toMatchObject({ + code: "ENOENT", + }); + const lease = await acquireHopperTargetLease({ + targetPath, + targetKind: "executable", + loaderArgs, + runId: "spotlight-retry-proof", + }); + if (!lease.acquired) + throw new Error("Spotlight failure retained the target lease"); + await lease.lease.release(); + expect(await client.start()).toMatchObject({ ok: true }); + expect(await client.callTool("echo", { value: "retried" })).toEqual({ + ok: true, + value: { value: "retried" }, + }); + }, +); + it.each(["exit", "signal", "exit75"] as const)( "reports a failed non-owning launcher %s without waiting for bridge timeout or inventing a GUI exit", async (termination) => { diff --git a/tests/boundary/setup/macosHopperPublisherVerification.test.ts b/tests/boundary/setup/macosHopperPublisherVerification.test.ts index 29b696e85a..dcb85cca5e 100644 --- a/tests/boundary/setup/macosHopperPublisherVerification.test.ts +++ b/tests/boundary/setup/macosHopperPublisherVerification.test.ts @@ -6,10 +6,11 @@ import { promisify } from "node:util"; import { describe, expect, it } from "vitest"; import { - HOPPER_PUBLISHER, macHopperInstallDisclosure, systemMacHopperInstallHost, } from "../../../src/application/MacHopper.js"; +import { verifyMacHopperNativeHelper } from "../../../src/hopper/HopperMacPublisher.js"; +import { HOPPER_PUBLISHER } from "../../../src/domain/hopperPublisher.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const run = promisify(execFile); @@ -94,6 +95,24 @@ describe("macOS Hopper publisher verification", () => { ); }); + itDarwin( + "withholds native-helper provenance from custom signed or modified executables", + async () => { + const root = await createTestTempDirectory( + "rea-hopper-helper-publisher-", + ); + const app = await buildBundle(root, "hopper"); + const executable = join(app, ...executableSegments); + await expect( + verifyMacHopperNativeHelper(executable, process.env), + ).resolves.toBe("unknown"); + await appendFile(executable, "modified\n"); + await expect( + verifyMacHopperNativeHelper(executable, process.env), + ).resolves.toBe("unknown"); + }, + ); + // Opt-in positive control: point this at a mounted official Hopper bundle, // for example `hdiutil attach -readonly -nobrowse Hopper-6.6.0-demo.dmg`. // It proves the pinned requirement is satisfiable by the real publisher, so @@ -103,6 +122,12 @@ describe("macOS Hopper publisher verification", () => { "accepts the officially signed Hopper application", async () => { await expect(host().validBundle(realApp ?? "")).resolves.toBe("verified"); + await expect( + verifyMacHopperNativeHelper( + join(realApp ?? "", ...executableSegments), + process.env, + ), + ).resolves.toBe("verified"); }, ); }); diff --git a/tests/fixtures/hopperLaunchAdmission.mjs b/tests/fixtures/hopperLaunchAdmission.mjs index bb80ad4489..6a5033d6b9 100644 --- a/tests/fixtures/hopperLaunchAdmission.mjs +++ b/tests/fixtures/hopperLaunchAdmission.mjs @@ -20,7 +20,8 @@ if (mode === "denied" || mode === "denied-after-handoff") { }, ); } else if (mode === "not-found") { - // The native helper's Spotlight lookup fails before any Apple event. + // A custom wrapper can forward the native diagnostic after a handoff. + await writeFile(`${receiptPath}.document-created`, "unconfirmed"); process.stderr.write( "Hopper has not been found.\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable.\n", () => process.exit(3), From 3ae02f6a458e25e56b82386e89fb56eae5249bff Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:13:22 +0800 Subject: [PATCH 06/14] fix(verification): parse browser endpoints before provider calls --- scripts/lib/browser-nested-source-map-e2e.mjs | 11 +++++++---- scripts/lib/browser-runtime-e2e.mjs | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/lib/browser-nested-source-map-e2e.mjs b/scripts/lib/browser-nested-source-map-e2e.mjs index ff2af88632..d7bac5ee22 100644 --- a/scripts/lib/browser-nested-source-map-e2e.mjs +++ b/scripts/lib/browser-nested-source-map-e2e.mjs @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { TraceMap, eachMapping } from "@jridgewell/trace-mapping"; import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js"; +import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js"; import { CdpConnection } from "../../dist/browser/CdpConnection.js"; import { parseEvidence } from "../../dist/domain/evidence.js"; import { traceSourceMap } from "../../dist/javascript/sourceMaps/TraceSourceMap.js"; @@ -89,10 +90,12 @@ async function verifyCase(executable, entrypoint, scenario) { fixture = await startRuntimeFixtureBrowser(executable, site.origin); const endpoint = `http://127.0.0.1:${fixture.port}`; const provider = new CdpBrowserProvider(); - const targets = await provider.listTargets({ - cdp_endpoint: endpoint, - allowed_origins: [site.origin], - }); + const targets = await provider.listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: [site.origin], + }), + ); if (!targets.ok) throw targets.error; const target = targets.value.targets[0]; assert.ok(target, "Owned fixture target missing"); diff --git a/scripts/lib/browser-runtime-e2e.mjs b/scripts/lib/browser-runtime-e2e.mjs index c4cb4f09f6..b35cde09ab 100644 --- a/scripts/lib/browser-runtime-e2e.mjs +++ b/scripts/lib/browser-runtime-e2e.mjs @@ -8,6 +8,7 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js"; +import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js"; import { CdpConnection } from "../../dist/browser/CdpConnection.js"; import { parseEvidence } from "../../dist/domain/evidence.js"; import { webExecutionSchema } from "../../dist/domain/webExecution.js"; @@ -45,10 +46,12 @@ export async function verifyBrowserRuntime( try { fixture = await startRuntimeFixtureBrowser(executable, site.origin); const endpoint = `http://127.0.0.1:${fixture.port}`; - const targets = await new CdpBrowserProvider().listTargets({ - cdp_endpoint: endpoint, - allowed_origins: [site.origin], - }); + const targets = await new CdpBrowserProvider().listTargets( + listBrowserTargetsInputSchema.parse({ + cdp_endpoint: endpoint, + allowed_origins: [site.origin], + }), + ); if (!targets.ok) throw targets.error; const target = targets.value.targets[0]; assert.ok(target, "owned fixture page target missing"); From f0e86997aa5337f2229d228ea3cb94cb5d1903e5 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:13:23 +0800 Subject: [PATCH 07/14] test(providers): correct executable and retained-owner expectations --- scripts/verify-windows-native-inspection.mjs | 5 +++-- tests/boundary/providers/RizinInferiorTty.test.ts | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/verify-windows-native-inspection.mjs b/scripts/verify-windows-native-inspection.mjs index e5f807da31..5b3c2e7260 100644 --- a/scripts/verify-windows-native-inspection.mjs +++ b/scripts/verify-windows-native-inspection.mjs @@ -62,8 +62,9 @@ assert.equal(first.retryable,true); assert.match(first.reason,/Delete owned object by handle failed.*Win32 5/s); const [name,...extra]=readdirSync(temp);assert.ok(name);assert.equal(extra.length,0); const path=join(temp,name);const identity=statSync(path,{bigint:true}); -assert.throws(()=>renameSync(path,path+'-replacement')); -assert.equal(statSync(path,{bigint:true}).ino,identity.ino); +// Ownership follows the retained NTFS handle even when the directory moves. +const moved=path+'-renamed';renameSync(path,moved); +assert.equal(statSync(moved,{bigint:true}).ino,identity.ino); if(process.argv[2]!=='teardown'){ const second=systemWindowsNativeAuthority();assert.equal(second.available,true); assert.equal(readdirSync(temp).length,0); diff --git a/tests/boundary/providers/RizinInferiorTty.test.ts b/tests/boundary/providers/RizinInferiorTty.test.ts index 4c19760d56..61774b192e 100644 --- a/tests/boundary/providers/RizinInferiorTty.test.ts +++ b/tests/boundary/providers/RizinInferiorTty.test.ts @@ -215,7 +215,10 @@ process.stdin.on("data", (chunk) => { ); execFileSync(process.execPath, ["--check", provider]); await chmod(provider, 0o700); - manager = new RizinDebugSessionManager({ REA_RIZIN_COMMAND: provider }); + manager = new RizinDebugSessionManager({ + ...process.env, + REA_RIZIN_COMMAND: provider, + }); const started = await manager.start({ path: "/tmp/fake-target", inferior_tty: tty, @@ -576,7 +579,7 @@ process.stdin.on("data", (chunk) => { await chmod(provider, 0o700); let providerStderr = ""; manager = new RizinDebugSessionManager( - { REA_RIZIN_COMMAND: provider }, + { ...process.env, REA_RIZIN_COMMAND: provider }, { spawnProcess: async (spawnOptions) => { const launched = await spawnOwnedProviderProcess(spawnOptions); From 70c238e376aff3c4ff16f8c6e8474c61e8e7fa89 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:16:00 +0800 Subject: [PATCH 08/14] test(ci): verify resolved cache inputs instead of glob spelling --- tests/boundary/filesystem/ciCachePolicy.test.ts | 5 ----- tests/boundary/filesystem/staticCheckReports.test.ts | 11 ++++++++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/boundary/filesystem/ciCachePolicy.test.ts b/tests/boundary/filesystem/ciCachePolicy.test.ts index 47e4f86dc4..4dea36ae5c 100644 --- a/tests/boundary/filesystem/ciCachePolicy.test.ts +++ b/tests/boundary/filesystem/ciCachePolicy.test.ts @@ -226,10 +226,6 @@ it("bounds the rolling static snapshot with Turbo age and size eviction", async .object({ cacheMaxAge: z.string(), cacheMaxSize: z.string(), - tasks: z.record( - z.string(), - z.object({ inputs: z.array(z.unknown()).optional() }), - ), }) .parse(JSON.parse(await readFile(join(repository, "turbo.json"), "utf8"))); // The static job restores the previous snapshot by prefix and saves the @@ -248,5 +244,4 @@ it("bounds the rolling static snapshot with Turbo age and size eviction", async unit === "TB" ? value * 1024 * 1024 : unit === "GB" ? value * 1024 : value; expect(megabytes).toBeGreaterThan(0); expect(megabytes).toBeLessThanOrEqual(1024); - expect(turbo.tasks["knip"]?.inputs).toContain("vitest.config.ts"); }); diff --git a/tests/boundary/filesystem/staticCheckReports.test.ts b/tests/boundary/filesystem/staticCheckReports.test.ts index 7d022da72d..10ef066744 100644 --- a/tests/boundary/filesystem/staticCheckReports.test.ts +++ b/tests/boundary/filesystem/staticCheckReports.test.ts @@ -49,7 +49,7 @@ const lintDiagnostics = async (root: string) => ), ).diagnostics; -it("includes every actually linted file in both cached lint task inputs", () => { +it("covers the lint inventory and root tool configs in resolved cache inputs", () => { const scan = spawnSync( process.execPath, [join(repository, "node_modules/oxlint/bin/oxlint"), "--debug=files"], @@ -69,6 +69,7 @@ it("includes every actually linted file in both cached lint task inputs", () => "run", "lint", "lint:built", + "knip", "--dry=json", ], { @@ -99,6 +100,14 @@ it("includes every actually linted file in both cached lint task inputs", () => files.filter((file) => !Object.hasOwn(task.inputs, file)), `${task.task} omits actual checker inputs`, ).toEqual([]); + const knip = tasks.find(({ task }) => task === "knip"); + expect(knip).toBeDefined(); + expect( + files.filter( + (file) => !file.includes("/") && !Object.hasOwn(knip!.inputs, file), + ), + "knip omits root tool configuration inputs", + ).toEqual([]); }); it("summarizes warnings without losing their complete diagnostic records", async () => { From 0161a593e520db550d5e76288124d520870aa2cd Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:16:01 +0800 Subject: [PATCH 09/14] style(hopper): format startup diagnostic expectations --- src/hopper/HopperProcessDiagnostic.test.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/hopper/HopperProcessDiagnostic.test.ts b/src/hopper/HopperProcessDiagnostic.test.ts index ac4ee75af6..6828c7762c 100644 --- a/src/hopper/HopperProcessDiagnostic.test.ts +++ b/src/hopper/HopperProcessDiagnostic.test.ts @@ -90,11 +90,8 @@ it.each([ text: `${notFound.stderr.text}another failure\n`, }, }, -])( - "rejects other not-found diagnostic shapes: %j", - (change) => { - expect( - hopperLauncherReportedApplicationNotFound({ ...notFound, ...change }), - ).toBe(false); - }, -); +])("rejects other not-found diagnostic shapes: %j", (change) => { + expect( + hopperLauncherReportedApplicationNotFound({ ...notFound, ...change }), + ).toBe(false); +}); From 22552525d54f349571c1e9d43d9535abea3df9b7 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:21:10 +0800 Subject: [PATCH 10/14] fix(ci): preserve per-job findings and unavailable blob evidence --- scripts/ci/artifacts.mjs | 14 ++- scripts/ci/failure-report.mjs | 50 ++++---- .../filesystem/ciFailureReport.test.ts | 16 +++ .../filesystem/ciFailureReportBudget.test.ts | 117 ++++++++++++++++++ 4 files changed, 170 insertions(+), 27 deletions(-) diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjs index 9a7cc1c218..250a15872c 100644 --- a/scripts/ci/artifacts.mjs +++ b/scripts/ci/artifacts.mjs @@ -87,10 +87,13 @@ export function classifyFailure({ "unknown", "main_run_does_not_contain_the_same_structured_diagnostic", ); - if (!sameBlobEvidence(diagnostics, blobs)) + const blobStatus = blobIdentityStatus(diagnostics, blobs); + if (blobStatus !== "identical") return outcome( "unknown", - "diagnostic_source_content_not_identical_at_main_and_tested_base", + blobStatus === "different" + ? "diagnostic_source_content_not_identical_at_main_and_tested_base" + : "diagnostic_source_blob_identity_unavailable", ); return outcome( "matching-main-diagnostics", @@ -366,9 +369,12 @@ function diagnosticKey(item) { return JSON.stringify([item.kind, ...[...item.paths].sort()]); return null; } -function sameBlobEvidence(diagnostics, blobs) { +function blobIdentityStatus(diagnostics, blobs) { const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))]; - return paths.length > 0 && paths.every((path) => blobs?.[path] === true); + if (paths.some((path) => blobs?.[path] === false)) return "different"; + return paths.length > 0 && paths.every((path) => blobs?.[path] === true) + ? "identical" + : "unavailable"; } function sameTaskHashes(diagnostics, left, right) { return diagnostics.every((diagnostic) => { diff --git a/scripts/ci/failure-report.mjs b/scripts/ci/failure-report.mjs index 2ce6d813c5..12364c59f1 100644 --- a/scripts/ci/failure-report.mjs +++ b/scripts/ci/failure-report.mjs @@ -153,16 +153,21 @@ async function analyzeFailedRun({ ]; const failures = []; for (const job of failedJobs) { - const evidence = await api.getStaticEvidence(repository, run, job); - const classification = await attributeToMain({ - repository, - defaultBranch, - run, - job, - evidence, - mainCandidates, - api, - }); + let classification; + try { + const evidence = await api.getStaticEvidence(repository, run, job); + classification = await attributeToMain({ + repository, + defaultBranch, + run, + job, + evidence, + mainCandidates, + api, + }); + } catch (error) { + classification = outcome("unknown", errorCode(error)); + } failures.push({ runId: entry.id, runAttempt: entry.runAttempt, @@ -415,26 +420,25 @@ export function createGitHubApi({ for (const path of paths) { if (!validRepoPath(path)) continue; const [left, right] = await Promise.all([ - getContent(repository, ancestor, path, request), - getContent(repository, descendant, path, request), + getContentBlobSha(repository, ancestor, path, request), + getContentBlobSha(repository, descendant, path, request), ]); - result[path] = left !== null && left === right; + result[path] = left === right; } return result; }, }; } -async function getContent(repository, ref, path, request) { - try { - const encodedPath = path.split("/").map(encodeURIComponent).join("/"); - const value = await request( - `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`, - ); - return typeof value.sha === "string" ? value.sha : null; - } catch { - return null; - } +async function getContentBlobSha(repository, ref, path, request) { + const encodedPath = path.split("/").map(encodeURIComponent).join("/"); + const value = await request( + `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`, + ); + const sha = safeSha(value?.sha); + if (value?.type !== "file" || sha === null) + throw new Error("github_content_blob_identity_unavailable"); + return sha; } function outcome(classification, reason) { diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.ts index 26feb3ee29..c2a282906c 100644 --- a/tests/boundary/filesystem/ciFailureReport.test.ts +++ b/tests/boundary/filesystem/ciFailureReport.test.ts @@ -120,6 +120,22 @@ it.each([ expect(result.classification).toBe("unknown"); }); +it.each([ + [{}, "diagnostic_source_blob_identity_unavailable"], + [ + { "src/example.ts": false }, + "diagnostic_source_content_not_identical_at_main_and_tested_base", + ], +] as const)( + "distinguishes unavailable blob identity from verified differences (%s)", + (blobs, reason) => { + expect(classifyFailure({ ...comparison(), blobs })).toEqual({ + classification: "unknown", + reason, + }); + }, +); + it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => { const directory = await createTestTempDirectory("rea-ci-failure-test-"); const calls: string[] = []; diff --git a/tests/boundary/filesystem/ciFailureReportBudget.test.ts b/tests/boundary/filesystem/ciFailureReportBudget.test.ts index 92fe3e4aec..1caacde165 100644 --- a/tests/boundary/filesystem/ciFailureReportBudget.test.ts +++ b/tests/boundary/filesystem/ciFailureReportBudget.test.ts @@ -2,6 +2,7 @@ import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { reportCiFailures, + createGitHubApi, type CiRun, type FailureReportApi, type StaticDiagnostic, @@ -235,6 +236,10 @@ it.each([1, 51])( expect(result.failures[0]!.classification).toBe( count === 1 ? "matching-main-diagnostics" : "unknown", ); + if (count > 50) + expect(result.failures[0]!.reason).toBe( + "diagnostic_source_blob_identity_unavailable", + ); }, ); @@ -288,3 +293,115 @@ it.each(["jobs", "artifact", "comparison"])( expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1); }, ); + +it.each([0, 1, 2])( + "preserves every failed job when artifact analysis fails at position %s", + async (position) => { + const input = fixture([20], 1); + const jobs = [ + { name: "Python producer", conclusion: "failure" }, + { name: "Test shard (1)", conclusion: "timed_out" }, + ]; + jobs.splice(position, 0, { name: "Static checks", conclusion: "failure" }); + input.api.getJobs = async (_repository, id, attempt) => + jobs.map((job) => ({ + ...job, + status: "completed", + run_id: Number(id), + run_attempt: attempt, + })); + input.api.getStaticEvidence = async (_repository, _run, job) => { + if (job.name === "Static checks") + throw new Error("static_artifact_unavailable"); + return null; + }; + const result = await report(input); + expect(result.counts.unknown).toBe(3); + expect(result.failures).toEqual( + jobs.map((job) => ({ + runId: 100, + runAttempt: 1, + runHead: prSha, + event: "pull_request", + job: job.name, + conclusion: job.conclusion, + classification: "unknown", + reason: + job.name === "Static checks" + ? "static_artifact_unavailable" + : "no_matching_verified_main_failure", + })), + ); + }, +); + +it("keeps unavailable job inventories at run scope", async () => { + const input = fixture([], 1); + input.api.getJobs = async () => { + throw new Error("job_inventory_unavailable"); + }; + expect((await report(input)).failures).toEqual([ + { + runId: 100, + runAttempt: 1, + classification: "unknown", + reason: "job_inventory_unavailable", + }, + ]); +}); + +it.each([ + ["request", "github_api_temporarily_unavailable"], + ["budget", "github_api_request_budget_exhausted"], + ["missing sha", "github_content_blob_identity_unavailable"], + ["invalid sha", "github_content_blob_identity_unavailable"], + ["directory", "github_content_blob_identity_unavailable"], +] as const)( + "retains unavailable blob evidence as a comparison error: %s", + async (failure, reason) => { + const input = fixture([20], 1); + const contentApi = createGitHubApi({ + token: "fixture-token", + apiRequest: async () => { + if (failure === "request") throw new Error(reason); + if (failure === "missing sha") return { type: "file" }; + if (failure === "invalid sha") + return { type: "file", sha: "not-a-sha" }; + if (failure === "directory") return []; + return { type: "file", sha: "e".repeat(40) }; + }, + }); + if (failure === "budget") { + for (let id = 1; id <= 250; id++) await contentApi.getRun(repository, id); + } + input.api.getBlobIdentity = contentApi.getBlobIdentity; + const result = await report(input); + expect(result.failures).toMatchObject([ + { + job: "Static checks", + classification: "unknown", + reason: "no_matching_verified_main_failure", + candidateFailures: [{ runId: 20, runAttempt: 1, reason }], + }, + ]); + }, +); + +it.each([true, false])( + "reports blob equality only from verified file SHAs (equal: %s)", + async (equal) => { + const api = createGitHubApi({ + token: "fixture-token", + apiRequest: async (_method, endpoint) => ({ + type: "file", + sha: + equal || endpoint.includes(`ref=${mainSha}`) + ? "e".repeat(40) + : "f".repeat(40), + }), + }); + expect( + await api.getBlobIdentity(repository, mainSha, prSha, ["src/example.ts"]), + ).toEqual({ "src/example.ts": equal }); + }, +); From cd740a6764d24e466f31613c2cef3368c3f64cba Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:23:08 +0800 Subject: [PATCH 11/14] test(hopper): fix fixture context and cleanup assumptions --- .../acceptance/providers/hopperStartupRecovery.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/acceptance/providers/hopperStartupRecovery.test.ts b/tests/acceptance/providers/hopperStartupRecovery.test.ts index 842fdaa077..f35ec68d66 100644 --- a/tests/acceptance/providers/hopperStartupRecovery.test.ts +++ b/tests/acceptance/providers/hopperStartupRecovery.test.ts @@ -306,7 +306,7 @@ test.for(["handoff", "not-found"] as const)( const token = setting("REA_TOKEN"); const runId = setting("REA_RUN_ID"); runtime = dirname(socket); - const lateBridge = async (identity: string, credential = token) => { + const lateBridge = (identity: string, credential = token) => { const child = spawn( process.execPath, [ @@ -318,14 +318,14 @@ test.for(["handoff", "not-found"] as const)( { stdio: "ignore" }, ); children.push(child); - await vi.waitFor(() => access(socket)); return child; }; for (const [identity, credential] of [ ["different-run", token], [runId, "different-token"], ] as const) { - const wrong = await lateBridge(identity, credential); + const wrong = lateBridge(identity, credential); + await vi.waitFor(() => access(socket)); expect(parseMcpToolError(await call("close_binary")).error.code).toBe( "cleanup_incomplete", ); @@ -336,7 +336,9 @@ test.for(["handoff", "not-found"] as const)( } const closing = call("close_binary"); await delay(150); - const owned = await lateBridge(runId); + // A verified close may remove the socket before the fixture can poll it. + // The successful close and child exit establish the owned bridge outcome. + const owned = lateBridge(runId); expect((await closing).isError).not.toBe(true); await vi.waitFor(() => expect(owned.exitCode).toBe(0)); await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); From 7e9fb6f1bbea8aad1704627d5961d88067680051 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:32:39 +0800 Subject: [PATCH 12/14] test(javascript): run native type bindings with compiled boundary prerequisites --- .../javascript/nativeTypeBindings.test.ts | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) rename {src/application => tests/boundary}/javascript/nativeTypeBindings.test.ts (77%) diff --git a/src/application/javascript/nativeTypeBindings.test.ts b/tests/boundary/javascript/nativeTypeBindings.test.ts similarity index 77% rename from src/application/javascript/nativeTypeBindings.test.ts rename to tests/boundary/javascript/nativeTypeBindings.test.ts index 6fcf8791e2..b0ec74d819 100644 --- a/src/application/javascript/nativeTypeBindings.test.ts +++ b/tests/boundary/javascript/nativeTypeBindings.test.ts @@ -1,27 +1,16 @@ -import { mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { afterEach, expect, it } from "vitest"; +import { expect, it } from "vitest"; -import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; -import type { JavaScriptApplicationAnalysisResult } from "../../domain/javascript/javascriptApplicationAnalysis.js"; -import { buildJavaScriptNativeHandoffs } from "../../domain/javascript/javascriptNativeHandoff.js"; -import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js"; - -const fixtures: string[] = []; - -afterEach(async () => { - await Promise.all( - fixtures - .splice(0) - .map((path) => rm(path, { recursive: true, force: true })), - ); -}); +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import type { JavaScriptApplicationAnalysisResult } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { buildJavaScriptNativeHandoffs } from "../../../src/domain/javascript/javascriptNativeHandoff.js"; +import { analyzeJavaScriptApplication } from "../../../src/application/javascript/JavaScriptApplicationService.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const analyzeFixture = async (source: string) => { - const fixture = await mkdtemp(join(tmpdir(), "rea-native-types-")); - fixtures.push(fixture); + const fixture = await createTestTempDirectory("rea-native-types-"); const scope = new ArtifactResourceScope(); try { await writeFile(join(fixture, "main.ts"), `${source}\n`); From c86b09a1095ffd360cd8851ff20a7b6aeb19b198 Mon Sep 17 00:00:00 2001 From: Ariel Fikru Avicenna <71917936+arielfikru@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:35:55 +0700 Subject: [PATCH 13/14] feat(ghidra): admit 32-bit big-endian PowerPC ELF (#1747) * feat(ghidra): admit 32-bit big-endian PowerPC ELF Recognize EM_PPC as a provider-neutral `powerpc` family that keeps its ELF class, byte order, type and flags, and admit ELF32 big-endian ET_EXEC through a bounded Ghidra lane. ELF64, little-endian, relocatable/shared objects and EF_PPC_EMB are refused with a specific reason; Hopper and IDA do not gain PowerPC support, and Windows Ghidra P0 is unchanged. Ghidra's ELF loader selects PowerPC:BE:32:e500:default for a plain EM_PPC header, which misreads 750-family code. The lane binds PowerPC:BE:32:default/default in the profile (language_selection "admitted-lane-v1"), imports with it and checks the reported language. A language set through REA_GHIDRA_LANGUAGE_ID still replaces the lane default. Add a source-owned fixture, fixed-encoding probe, GNU readelf cross-check and the optional `verify:ghidra:ppc` real-provider lane, plus docs/ghidra-ppc.md. Refs #1741 * style(ghidra): format shared PowerPC profile integration * fix(ghidra): preserve caller-selected PowerPC languages --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/.vitepress/config.ts | 1 + docs/ghidra-ppc.md | 90 ++++++ package.json | 1 + scripts/verify-real-ghidra-ppc.mjs | 282 ++++++++++++++++++ src/domain/binaryTarget.powerpc.test.ts | 58 ++++ src/domain/binaryTarget.ts | 22 +- src/domain/binaryTargetTypes.ts | 14 + src/ghidra/GhidraAnalysisProfile.ts | 10 +- src/ghidra/GhidraPowerPcProfile.test.ts | 166 +++++++++++ src/ghidra/GhidraPowerPcProfile.ts | 67 +++++ src/ghidra/GhidraProvider.ts | 1 + src/ghidra/GhidraPspProfile.ts | 3 + src/hopper/HopperAnalysisProfile.ts | 1 + src/hopper/HopperProvider.ts | 6 +- src/ida/IdaProvider.ts | 6 +- .../powerpc-identity.test.ts | 93 ++++++ tests/conformance/c/ppc.c | 14 + tests/conformance/ghidra/ppc-checks.d.mts | 29 ++ tests/conformance/ghidra/ppc-checks.mjs | 132 ++++++++ tests/conformance/ghidra/ppc-checks.test.ts | 187 ++++++++++++ tests/conformance/ghidra/ppc-fixture.mjs | 62 ++++ tests/conformance/ghidra/ppc-probe.S | 17 ++ 22 files changed, 1258 insertions(+), 4 deletions(-) create mode 100644 docs/ghidra-ppc.md create mode 100644 scripts/verify-real-ghidra-ppc.mjs create mode 100644 src/domain/binaryTarget.powerpc.test.ts create mode 100644 src/ghidra/GhidraPowerPcProfile.test.ts create mode 100644 src/ghidra/GhidraPowerPcProfile.ts create mode 100644 tests/composition/analysis-sessions/powerpc-identity.test.ts create mode 100644 tests/conformance/c/ppc.c create mode 100644 tests/conformance/ghidra/ppc-checks.d.mts create mode 100644 tests/conformance/ghidra/ppc-checks.mjs create mode 100644 tests/conformance/ghidra/ppc-checks.test.ts create mode 100644 tests/conformance/ghidra/ppc-fixture.mjs create mode 100644 tests/conformance/ghidra/ppc-probe.S diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index b4cf2e4ade..dd562d8137 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -63,6 +63,7 @@ export default defineConfig({ { text: "Ghidra DOS analysis", link: "/ghidra-dos" }, { text: "Ghidra raw binary", link: "/raw-binary" }, { text: "Ghidra NativeAOT", link: "/ghidra-nativeaot" }, + { text: "Ghidra PowerPC", link: "/ghidra-ppc" }, { text: "Ghidra semantics", link: "/ghidra-provider" }, ], }, diff --git a/docs/ghidra-ppc.md b/docs/ghidra-ppc.md new file mode 100644 index 0000000000..0341583785 --- /dev/null +++ b/docs/ghidra-ppc.md @@ -0,0 +1,90 @@ +# PowerPC ELF analysis through Ghidra + +This lane extends native ELF analysis to 32-bit big-endian PowerPC; it does +not execute PowerPC code. It is the first milestone of +[#1741](https://github.com/morluto/rea/issues/1741). GameCube/Wii DOL +executables, PPC64 and other PowerPC variants are separate follow-ups. + +## Admission boundary + +ELF `EM_PPC` (20) is retained as the provider-neutral `powerpc` family. The +target also retains class, byte order, `e_type` and raw `e_flags`. Recognition +does not mean every provider or PowerPC variant is supported. `EM_PPC64` (21) +is not recognized as this family. + +The Ghidra lane admits ELF32 `ET_EXEC`, big endian. ELF64, little-endian, +relocatable and shared objects are refused with a specific unsupported +constraint. `EF_PPC_EMB` declares an embedded variant and is refused; the +remaining `e_flags` bits only declare relocation conventions and do not change +instruction decoding. + +## Bound language + +Ghidra's ELF loader selects `PowerPC:BE:32:e500:default` for a plain `EM_PPC` +header with `e_flags` 0. e500 is the embedded Book-E variant whose SPE unit +replaces the classic floating-point unit, so it misinterprets ordinary +750-family code. Nothing in the header distinguishes the two. + +The lane therefore imports with `PowerPC:BE:32:default` and compiler spec +`default`, commits that choice in the analysis profile as +`language_selection: "provider-pinned-v1"`, and checks the language Ghidra +reports after import. A different language set through +`REA_GHIDRA_LANGUAGE_ID` (and optionally `REA_GHIDRA_COMPILER_SPEC_ID`) +replaces the lane default and is admitted against the installed language +catalog as usual; for example `PowerPC:BE:32:e500` for genuine e500 code. + +PowerPC header facts are committed in the analysis profile so interpretation +changes invalidate profile-bound snapshots. The support-lane identity is +versioned independently of file identity. Evidence, lifecycle outputs and saved +snapshots preserve the `powerpc` family. Hopper and IDA adapters do not +implicitly gain PowerPC support. Windows Ghidra P0 remains PE x86/x86-64 only. + +## Use + +With a caller-installed Ghidra and compatible JDK on a supported Linux/macOS host: + +```bash +rea analyze /absolute/path/to/ppc.elf --provider ghidra --json +rea function /absolute/path/to/ppc.elf function_name --provider ghidra --json +``` + +MCP uses the existing tools, including `open_binary` with an absolute path and +`provider_id: "ghidra"`. No PowerPC-specific tool catalog or emulator is +introduced. + +## Verification + +The source-owned freestanding fixture is `tests/conformance/c/ppc.c`. The +optional cross-target lane requires a caller-supplied Clang with the PowerPC +target, LLD, GNU `readelf`, Ghidra and its compatible JDK; ordinary host-native +checks do not acquire these tools. `REA_PPC_CLANG` selects Clang and +`REA_PPC_READELF` selects GNU readelf (for example +`/opt/homebrew/opt/binutils/bin/readelf` on macOS). These are test-time +prerequisites, not REA runtime providers. + +```bash +GHIDRA_INSTALL_DIR=/absolute/path/to/ghidra npm run verify:ghidra:ppc +``` + +The lane compiles the fixture for a 750-family core (`-mcpu=750`) and never +executes it. GNU readelf independently supplies header facts and symbol +addresses, compared with the production REA target resolver before provider +startup. Its version, bounded command output and selected tool path are +retained in the verification report. The parser deliberately recognizes only +this fixture's GNU output, rejecting missing, ambiguous or unexpected fields +rather than substituting defaults. + +Existing public MCP operations check Ghidra's **observed** language and +compiler specification and loaded-file hashes; an e500 import fails the lane. +The source-owned `ppc-probe.S` supplies fixed `addi` and conditional-branch +encodings: verification checks bytes, length, immediate and direct destination, +independent of decompiler variable names. Global and string bytes are checked at +independently reported symbol addresses. + +CLI/MCP discovery and decompilation, direct calls, target/profile/Evidence +identity, snapshot round-trip and final process cleanup remain covered. +Pseudocode presence is a liveness check, not proof of semantic equivalence. +Instruction decoding and reader agreement do not establish runtime or complete +floating-point semantics. Parser/profile tests are narrower checks, not +substitutes for this real-provider run. Report the actual lanes run and leave +unexecuted verification pending. diff --git a/package.json b/package.json index e09e51366c..5ccf53c994 100644 --- a/package.json +++ b/package.json @@ -186,6 +186,7 @@ "verify:ghidra": "npm run build:cached && node scripts/verify-real-ghidra.mjs", "verify:ghidra:seeds": "npm run build:cached && node scripts/verify/ghidra/seeds.mjs", "verify:ghidra:mips": "npm run build:cached && node scripts/verify-real-ghidra-mips.mjs", + "verify:ghidra:ppc": "npm run build:cached && node scripts/verify-real-ghidra-ppc.mjs", "verify:ghidra:riscv": "npm run build:cached && node scripts/verify-real-ghidra-riscv.mjs", "verify:ghidra:6502": "npm run build:cached && node scripts/verify-real-ghidra-6502.mjs", "verify:ghidra:mips-shared": "npm run build:cached && node scripts/verify-real-ghidra-mips-shared.mjs", diff --git a/scripts/verify-real-ghidra-ppc.mjs b/scripts/verify-real-ghidra-ppc.mjs new file mode 100644 index 0000000000..591d32b2b5 --- /dev/null +++ b/scripts/verify-real-ghidra-ppc.mjs @@ -0,0 +1,282 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; +import { + inspectPowerPcReadelf, + assertPowerPcLoadedImage, + assertPowerPcProbe, + assertPowerPcGlobal, +} from "../tests/conformance/ghidra/ppc-checks.mjs"; +import { parseEvidence } from "../dist/domain/evidence.js"; +import { parseAnalysisSnapshot } from "../dist/domain/analysisSnapshot.js"; +import { functionDossierSchema } from "../dist/domain/hopperValues.js"; +import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js"; +import { buildPowerPcFixture } from "../tests/conformance/ghidra/ppc-fixture.mjs"; +import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; + +// Optional cross-target lane. No tool acquisition and no target execution. +if (process.argv.length !== 2) + throw new Error("Usage: node scripts/verify-real-ghidra-ppc.mjs"); +if (!["linux", "darwin"].includes(process.platform)) + throw new Error( + "PowerPC verification requires a supported Linux/macOS Ghidra host", + ); +if (process.env.GHIDRA_INSTALL_DIR === undefined) + throw new Error("Missing prerequisite: caller-supplied GHIDRA_INSTALL_DIR"); +const installation = inspectGhidraInstallation({ + environment: process.env, + installDir: process.env.GHIDRA_INSTALL_DIR, + ...(process.env.JAVA_HOME === undefined + ? {} + : { javaHome: process.env.JAVA_HOME }), +}); +if (installation.status !== "available") + throw new Error( + `Ghidra prerequisite unavailable: ${JSON.stringify(installation)}`, + ); +const execute = promisify(execFile); +const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url)); +const run = createVerifierRun(); +const workspace = await mkdtemp(join(tmpdir(), "rea-ppc-proof-")); +const runtime = join(workspace, "runtime"); +await mkdir(runtime); +const env = { + PATH: process.env.PATH ?? "/usr/bin:/bin", + TMPDIR: runtime, + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "ghidra", + REA_PROCESS_RUN_ID: run.run_id, + GHIDRA_INSTALL_DIR: installation.installDir, + JAVA_HOME: installation.javaHome, + HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper", +}; +for (const setting of ["GHIDRA_HEADLESS_MAXMEM", "GHIDRA_MAXMEM"]) { + if (process.env[setting] !== undefined) env[setting] = process.env[setting]; +} +const reports = []; +let primaryFailure; +try { + { + const fixture = await buildPowerPcFixture(workspace); + const independent = await inspectPowerPcReadelf(fixture.path); + const resolved = await parseBinaryTarget(fixture.path); + if (!resolved.ok) throw resolved.error; + assert.deepEqual( + resolved.value.powerpc, + independent.powerpc, + "REA ELF interpretation disagrees with GNU readelf", + ); + assert.equal(resolved.value.sha256, fixture.sha256); + const snapshotPath = join(workspace, "ppc.snapshot.json"); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env, + stderr: "pipe", + }); + const client = new Client({ name: "rea-real-ppc-proof", version: "1" }); + let stderr = ""; + transport.stderr?.on("data", (chunk) => { + stderr = (stderr + chunk.toString()).slice(-65536); + }); + const call = async (name, args = {}) => { + const reply = await client.callTool( + { name, arguments: args }, + undefined, + { timeout: 180000 }, + ); + return requireMcpOperationResult(reply, name); + }; + try { + await client.connect(transport); + const target = await call("open_binary", { + path: fixture.path, + provider_id: "ghidra", + }); + assert.equal(target.architecture, "powerpc"); + assert.equal(target.sha256, fixture.sha256); + // ELF load-image verification is not the DOS comparison lane. Its retained + // observations still expose the actual Ghidra language and loaded bytes; + // the plain loader would have chosen e500 for this header. + const image = await call("inspect_native_load_image"); + assertPowerPcLoadedImage(image.observations, fixture.sha256); + const procedures = await call("list_procedures"); + const entry = procedures.find((p) => p.value === "rea_ppc_entry"); + const leaf = procedures.find((p) => p.value === "rea_ppc_leaf"); + assert.ok(entry, "Entry function missing from real Ghidra inventory"); + assert.ok(leaf, "Leaf function missing from real Ghidra inventory"); + assert.equal( + BigInt(entry.address), + BigInt(independent.symbols.rea_ppc_entry), + ); + assert.equal( + BigInt(leaf.address), + BigInt(independent.symbols.rea_ppc_leaf), + ); + const probe = independent.symbols.rea_ppc_probe; + assert.ok( + procedures.some( + (p) => + p.value === "rea_ppc_probe" && BigInt(p.address) === BigInt(probe), + ), + ); + const move = await call("inspect_native_instruction", { address: probe }); + const branch = await call("inspect_native_instruction", { + address: `0x${(BigInt(probe) + 8n).toString(16)}`, + }); + assertPowerPcProbe(move, branch, probe); + const global = await call("read_bytes", { + address: independent.symbols.rea_ppc_global, + length: 4, + }); + assert.equal(global.returned_bytes, 4); + assertPowerPcGlobal(global.bytes_hex); + const marker = await call("read_bytes", { + address: independent.symbols.rea_ppc_marker, + length: 29, + }); + assert.equal( + marker.bytes_hex, + Buffer.from("rea-ppc-source-owned-fixture\0").toString("hex"), + ); + const callees = await call("procedure_callees", { + procedure: entry.address, + }); + assert.ok( + callees.includes(leaf.address), + "Expected direct call not recovered", + ); + const assembly = await call("procedure_assembly", { + procedure: entry.address, + }); + assert.ok(assembly.trim().length > 0); + const dossier = functionDossierSchema.parse( + await call("analyze_function", { procedure: entry.address }), + ); + assert.ok(dossier.pseudocode.trim().length > 0); + const mapping = await call("address_to_file_offset", { + address: entry.address, + }); + const memory = await call("read_bytes", { + address: entry.address, + length: 16, + }); + const sourceBytes = await readFile(fixture.path); + assert.equal(memory.returned_bytes, 16); + assert.equal( + memory.bytes_hex, + sourceBytes + .subarray(mapping.file_offset, mapping.file_offset + 16) + .toString("hex"), + ); + const saved = await call("close_binary", { + snapshot_path: snapshotPath, + }); + assert.ok(saved.evidence_records > 0); + const snapshot = parseAnalysisSnapshot( + JSON.parse(await readFile(snapshotPath, "utf8")), + ); + assert.equal(snapshot.target.architecture, "powerpc"); + assert.equal( + snapshot.binding.analysis_profile.parameters.language_id, + "PowerPC:BE:32:default", + ); + assert.ok(snapshot.evidence_bundle.records.length > 0); + for (const evidence of snapshot.evidence_bundle.records) + assert.equal(parseEvidence(evidence).subject?.architecture, "powerpc"); + await call("open_binary", { + path: fixture.path, + provider_id: "ghidra", + snapshot_path: snapshotPath, + }); + assert.deepEqual(await call("list_procedures"), procedures); + await call("close_binary"); + + // Same public operation through CLI, not a private injected provider. + const cli = await execute( + process.execPath, + [ + entrypoint, + "function", + fixture.path, + "rea_ppc_entry", + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 }, + ); + const evidence = parseEvidence(JSON.parse(cli.stdout)); + assert.equal(evidence.subject?.architecture, "powerpc"); + assert.equal(evidence.subject?.digest.sha256, fixture.sha256); + const cliDossier = functionDossierSchema.parse( + evidence.normalized_result, + ); + assert.equal(cliDossier.procedure.address, dossier.procedure.address); + assert.ok(cliDossier.pseudocode.trim().length > 0); + assert.equal( + createHash("sha256") + .update(await readFile(fixture.path)) + .digest("hex"), + fixture.sha256, + "Analysis modified the source executable", + ); + reports.push({ + ...fixture, + independent_reader: independent, + effective_language: image.observations.language_id, + effective_compiler_spec: image.observations.compiler_spec_id, + checks: [ + "independent ELF header and symbols", + "loaded Ghidra identity and bound language", + "fixed instruction bytes and immediate", + "conditional branch destination", + "global word and marker bytes", + "CLI/MCP and snapshot identity", + ], + procedures: procedures.length, + status: "passed", + }); + } catch (cause) { + throw new Error(`PowerPC verification failed; MCP stderr: ${stderr}`, { + cause, + }); + } finally { + try { + await client.close(); + } finally { + await transport.close(); + } + } + } +} catch (cause) { + primaryFailure = cause; +} +const completed = await completeVerifierRun(run); +if ( + completed.process_lineage.status !== "verified" || + completed.process_lineage.descendants.length !== 0 +) + throw new Error( + `Cleanup unverified; retained diagnostic workspace: ${workspace}`, + { cause: primaryFailure }, + ); +if (primaryFailure !== undefined) + throw new Error( + `PowerPC verification failed; retained workspace: ${workspace}`, + { + cause: primaryFailure, + }, + ); +await rm(workspace, { recursive: true, force: true }); +console.log(JSON.stringify({ ...completed, fixtures: reports }, null, 2)); diff --git a/src/domain/binaryTarget.powerpc.test.ts b/src/domain/binaryTarget.powerpc.test.ts new file mode 100644 index 0000000000..f32b4f3674 --- /dev/null +++ b/src/domain/binaryTarget.powerpc.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { parseExecutableHeader } from "./binaryTarget.js"; + +const header = (little: boolean, bits: 32 | 64 = 32, machine = 20): Buffer => { + const bytes = Buffer.alloc(bits === 32 ? 52 : 64); + bytes.set([0x7f, 0x45, 0x4c, 0x46, bits === 32 ? 1 : 2, little ? 1 : 2, 1]); + if (little) { + bytes.writeUInt16LE(2, 16); + bytes.writeUInt16LE(machine, 18); + bytes.writeUInt32LE(1, 20); + bytes.writeUInt16LE(bytes.length, bits === 32 ? 40 : 52); + } else { + bytes.writeUInt16BE(2, 16); + bytes.writeUInt16BE(machine, 18); + bytes.writeUInt32BE(1, 20); + bytes.writeUInt16BE(bytes.length, bits === 32 ? 40 : 52); + } + return bytes; +}; + +describe("PowerPC ELF header boundaries", () => { + it.each([false, true])( + "retains source endian and flags (little=%s)", + (little) => { + const parsed = parseExecutableHeader(header(little), "x64"); + if (!parsed.ok) throw new Error(parsed.error); + expect(parsed.value).toEqual({ + format: "elf", + architecture: "powerpc", + availableArchitectures: ["powerpc"], + powerpc: { + elfClass: 32, + byteOrder: little ? "little" : "big", + type: 2, + flags: 0, + }, + }); + }, + ); + + it("reads ELF64 flags at their own offset without claiming support", () => { + const bytes = header(false, 64); + bytes.writeUInt32BE(0x80000000, 48); + const parsed = parseExecutableHeader(bytes, "x64"); + if (!parsed.ok) throw new Error(parsed.error); + expect(parsed.value).toMatchObject({ + architecture: "powerpc", + powerpc: { elfClass: 64, byteOrder: "big", flags: 0x80000000 }, + }); + }); + + it("does not recognize EM_PPC64 as the 32-bit family", () => { + expect(parseExecutableHeader(header(false, 64, 21), "x64")).toEqual({ + ok: false, + error: "unsupported ELF architecture", + }); + }); +}); diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index 4e668e4605..54798e1d14 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -5,6 +5,7 @@ import type { MipsElfMetadata, PeCodeViewRecord, RiscvElfMetadata, + PowerPcElfMetadata, UniversalMachOSlice, } from "./binaryTargetTypes.js"; import { err, ok, type Result } from "./result.js"; @@ -30,6 +31,12 @@ export type ExecutableMetadata = readonly availableArchitectures: readonly "riscv"[]; readonly riscv: RiscvElfMetadata; } + | { + readonly format: "elf"; + readonly architecture: "powerpc"; + readonly availableArchitectures: readonly "powerpc"[]; + readonly powerpc: PowerPcElfMetadata; + } | { readonly format: "mach-o" | "elf" | "dos-mz" | "dos-com"; readonly architecture: HeaderArchitecture; @@ -201,7 +208,11 @@ const parseElf = ( const machine = little ? bytes.readUInt16LE(18) : bytes.readUInt16BE(18); const architecture = elfArchitecture(machine); if (architecture === undefined) return err("unsupported ELF architecture"); - if (architecture === "mips" || architecture === "riscv") { + if ( + architecture === "mips" || + architecture === "riscv" || + architecture === "powerpc" + ) { const flagsOffset = elfClass === 32 ? 36 : 48; const metadata: ElfFamilyMetadata = { elfClass, @@ -211,6 +222,13 @@ const parseElf = ( ? bytes.readUInt32LE(flagsOffset) : bytes.readUInt32BE(flagsOffset), }; + if (architecture === "powerpc") + return ok({ + format: "elf", + architecture, + availableArchitectures: [architecture], + powerpc: metadata, + }); return architecture === "mips" ? ok({ format: "elf", @@ -532,6 +550,8 @@ const elfArchitecture = ( return "x86_64"; case 8: // EM_MIPS: ISA, ABI and byte order remain separate ELF facts. return "mips"; + case 20: // EM_PPC; byte order and processor variant remain ELF declarations. + return "powerpc"; case 40: return "arm"; case 183: diff --git a/src/domain/binaryTargetTypes.ts b/src/domain/binaryTargetTypes.ts index 8c16612bca..2cd6ae2078 100644 --- a/src/domain/binaryTargetTypes.ts +++ b/src/domain/binaryTargetTypes.ts @@ -9,6 +9,7 @@ export const BINARY_ARCHITECTURES = [ "mips", "6502", "riscv", + "powerpc", ] as const; export type BinaryArchitecture = (typeof BINARY_ARCHITECTURES)[number]; @@ -36,6 +37,7 @@ export interface PeCodeViewRecord { export const ELF_METADATA_ARCHITECTURES = [ "mips", "riscv", + "powerpc", ] as const satisfies readonly BinaryArchitecture[]; export type ElfMetadataArchitecture = (typeof ELF_METADATA_ARCHITECTURES)[number]; @@ -90,6 +92,9 @@ export interface MipsAbiFlags { */ export type RiscvElfMetadata = ElfFamilyMetadata; +/** Raw PowerPC declarations; family identity does not select a processor variant. */ +export type PowerPcElfMetadata = ElfFamilyMetadata; + /** Byte range of the selected slice inside its universal Mach-O container. */ export interface UniversalMachOSlice { readonly offset: number; @@ -140,6 +145,15 @@ export type BinaryTarget = readonly executableRole?: never; readonly managed?: never; }) + | (BinaryTargetIdentity & { + readonly kind: "executable"; + readonly format: "elf"; + readonly architecture: "powerpc"; + readonly availableArchitectures: readonly "powerpc"[]; + readonly powerpc: PowerPcElfMetadata; + readonly executableRole?: never; + readonly managed?: never; + }) | (ExecutableTarget & { readonly format: "mach-o" | "elf" | "dos-mz" | "dos-com"; readonly universalSlice?: never; diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index d4bd5a4330..8175b776b0 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -21,6 +21,10 @@ import { ghidraRiscvLanguageOverride, ghidraRiscvProfileParameters, } from "./GhidraRiscvProfile.js"; +import { + ghidraPowerPcLanguage, + ghidraPowerPcProfileParameters, +} from "./GhidraPowerPcProfile.js"; import { ghidraRawBinaryLanguage } from "./GhidraRawBinaryLanguage.js"; import { @@ -44,7 +48,9 @@ import { export const ghidraPinnedLanguage = ( target: BinaryTarget, ): GhidraLanguageOverride | undefined => - ghidraMipsLanguageOverride(target) ?? ghidraRiscvLanguageOverride(target); + ghidraMipsLanguageOverride(target) ?? + ghidraRiscvLanguageOverride(target) ?? + ghidraPowerPcLanguage(target); /** Resolve version-bound, deterministic semantics before Ghidra imports a target. */ export const resolveGhidraAnalysisProfile = async ( @@ -155,6 +161,7 @@ export const resolveGhidraAnalysisProfile = async ( const pinnedLanguage = ghidraPinnedLanguage(target); if ( pinnedLanguage !== undefined && + target.architecture !== "powerpc" && languageOverride !== undefined && (languageOverride.languageId !== pinnedLanguage.languageId || (languageOverride.compilerSpecId !== undefined && @@ -277,6 +284,7 @@ export const resolveGhidraAnalysisProfile = async ( ...analyzerParameters, ...ghidraMipsProfileParameters(target, languageOverride), ...ghidraRiscvProfileParameters(target), + ...ghidraPowerPcProfileParameters(target), ...(pspExtension?.ok === true ? { mips_support_lane: "psp-elf32-exec-eabi32-allegrex-v2", diff --git a/src/ghidra/GhidraPowerPcProfile.test.ts b/src/ghidra/GhidraPowerPcProfile.test.ts new file mode 100644 index 0000000000..831a47f3bc --- /dev/null +++ b/src/ghidra/GhidraPowerPcProfile.test.ts @@ -0,0 +1,166 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { parseConfig } from "../config/parseConfig.js"; +import type { + BinaryTarget, + PowerPcElfMetadata, +} from "../domain/binaryTargetTypes.js"; +import { silentLogger } from "../logger.js"; +import { HopperProvider } from "../hopper/HopperProvider.js"; +import { hopperLoaderArgsForTarget } from "../hopper/HopperAnalysisProfile.js"; +import { IdaProvider } from "../ida/IdaProvider.js"; +import { + ghidraPinnedLanguage, + ghidraProfileLanguageOverride, +} from "./GhidraAnalysisProfile.js"; +import { GhidraProvider } from "./GhidraProvider.js"; +import type { GhidraInstallationHost } from "./GhidraInstallation.js"; + +// Abbreviated from Ghidra 12.1.4 Processors/PowerPC/data/languages/ppc.ldefs. +const PPC_LDEFS = ` + + + + + + + + +`; + +// Fake installation inspection tests admission/profile contracts, not Ghidra. +const host: GhidraInstallationHost = { + platform: "linux", + architecture: "x64", + readText: () => "application.version=12.1.4\n", + executable: () => true, + probeJava: () => ({ + version: "21.0.11", + major: 21, + home: "/jdk-21", + bits: 64, + runtime: "jdk", + }), +}; +const configFor = (environment: NodeJS.ProcessEnv) => { + const parsed = parseConfig({ GHIDRA_INSTALL_DIR: "/ghidra", ...environment }); + if (!parsed.ok) throw parsed.error; + return parsed.value; +}; +const config = configFor({}); +const target = (change: Partial = {}): BinaryTarget => ({ + path: "/fixture.elf", + sha256: "a".repeat(64), + kind: "executable", + format: "elf", + architecture: "powerpc", + availableArchitectures: ["powerpc"], + powerpc: { elfClass: 32, byteOrder: "big", type: 2, flags: 0, ...change }, +}); + +describe("bounded Ghidra PowerPC admission", () => { + it("binds the generic language instead of the loader's e500 choice", async () => { + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + expect(ghidra.inspectTargetSupport(target()).status).toBe("supported"); + const resolved = await ghidra.resolveAnalysisProfile(target()); + if (!resolved.ok) throw resolved.error; + const parameters = resolved.value.profile?.parameters ?? {}; + expect(parameters).toMatchObject({ + architecture: "powerpc", + language_id: "PowerPC:BE:32:default", + compiler_spec_id: "default", + language_selection: "provider-pinned-v1", + powerpc_elf: { elf_class: 32, byte_order: "big", type: 2, flags: 0 }, + powerpc_support_lane: "elf32-be-exec-generic-v1", + }); + expect(ghidraPinnedLanguage(target())).toEqual({ + languageId: "PowerPC:BE:32:default", + compilerSpecId: "default", + }); + expect(ghidraProfileLanguageOverride(parameters)).toEqual( + ghidraPinnedLanguage(target()), + ); + }); + + it("lets a configured language replace the lane default", async () => { + const installDir = await createTestTempDirectory("rea-ghidra-ppc-"); + const languages = join( + installDir, + "Ghidra/Processors/PowerPC/data/languages", + ); + await mkdir(languages, { recursive: true }); + await writeFile(join(languages, "ppc.ldefs"), PPC_LDEFS); + const ghidra = new GhidraProvider( + configFor({ + GHIDRA_INSTALL_DIR: installDir, + REA_GHIDRA_LANGUAGE_ID: "PowerPC:BE:32:e500", + }), + silentLogger, + {}, + host, + ); + const resolved = await ghidra.resolveAnalysisProfile(target()); + if (!resolved.ok) throw resolved.error; + const parameters = resolved.value.profile?.parameters ?? {}; + expect(parameters).toMatchObject({ + language_id: "PowerPC:BE:32:e500", + language_selection: "configured-v1", + }); + expect(ghidraProfileLanguageOverride(parameters)).toEqual({ + languageId: "PowerPC:BE:32:e500", + }); + }); + + it("commits header flags so a changed interpretation changes the profile", async () => { + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + const left = await ghidra.resolveAnalysisProfile(target()); + const right = await ghidra.resolveAnalysisProfile( + target({ flags: 0x00010000 }), + ); + if (!left.ok) throw left.error; + if (!right.ok) throw right.error; + expect(left.value.profile?.digest).not.toBe(right.value.profile?.digest); + }); + + it("does not widen Windows P0", () => { + const windows = new GhidraProvider( + config, + silentLogger, + {}, + { ...host, platform: "win32" }, + ); + expect(windows.inspectTargetSupport(target()).status).toBe("unsupported"); + }); + + it.each([ + [{ elfClass: 64 }, "ELF32"], + [{ byteOrder: "little" }, "big-endian"], + [{ type: 1 }, "ET_EXEC"], + [{ type: 3 }, "ET_EXEC"], + [{ flags: 0x80000000 }, "EF_PPC_EMB"], + ] satisfies [Partial, string][])( + "refuses an unverified PowerPC interpretation %j", + async (change, reason) => { + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + const value = target(change); + expect(ghidra.inspectTargetSupport(value)).toMatchObject({ + status: "unsupported", + reason: expect.stringContaining(reason), + }); + const resolved = await ghidra.resolveAnalysisProfile(value); + expect(resolved.ok).toBe(false); + }, + ); + + it("keeps PowerPC out of unverified Hopper and IDA adapter lanes", () => { + const hopper = new HopperProvider(config, silentLogger, {}, "linux"); + expect(hopper.inspectTargetSupport(target()).status).toBe("unsupported"); + expect(hopperLoaderArgsForTarget(target()).ok).toBe(false); + const ida = new IdaProvider(config, {}); + expect(ida.inspectTargetSupport(target()).status).toBe("unsupported"); + }); +}); diff --git a/src/ghidra/GhidraPowerPcProfile.ts b/src/ghidra/GhidraPowerPcProfile.ts new file mode 100644 index 0000000000..c1655f2523 --- /dev/null +++ b/src/ghidra/GhidraPowerPcProfile.ts @@ -0,0 +1,67 @@ +import type { GhidraLanguageOverride } from "../config/ghidraLanguageOverride.js"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import type { JsonValue } from "../domain/jsonValue.js"; + +/** + * Generic 32-bit big-endian PowerPC. Ghidra's ELF loader selects the e500 + * (Book-E/SPE) variant for a plain EM_PPC header, so the lane binds this + * language instead of accepting the loader's choice. + */ +export const GHIDRA_POWERPC_LANGUAGE: GhidraLanguageOverride = { + languageId: "PowerPC:BE:32:default", + compilerSpecId: "default", +}; + +/** Keep family recognition separate from the deliberately bounded Ghidra lane. */ +export const ghidraPowerPcUnsupportedReason = ( + target: BinaryTarget, +): string | null => { + if (target.kind !== "executable" || target.architecture !== "powerpc") + return null; + const metadata = target.powerpc; + if (target.format !== "elf" || metadata === undefined) + return "PowerPC analysis requires ELF header metadata; a family name alone does not select a processor variant."; + if (metadata.elfClass !== 32) + return "This Ghidra PowerPC lane supports ELF32 only; PPC64 remains unsupported."; + if (metadata.byteOrder !== "big") + return "This Ghidra PowerPC lane supports big-endian ELF32 only; little-endian PowerPC needs separate verification."; + if (metadata.type !== 2) + return "This Ghidra PowerPC lane supports ET_EXEC only; relocatable objects and shared images need separate load verification."; + // EF_PPC_EMB (0x80000000) marks embedded variants; other bits only declare + // relocation conventions that do not change instruction decoding. + if ((metadata.flags & 0x80000000) !== 0) + return "EF_PPC_EMB declares an embedded PowerPC variant that needs a separately supported processor profile."; + return null; +}; + +/** + * The language the lane imports with, unless REA_GHIDRA_LANGUAGE_ID already + * selected one through the committed profile. + */ +export const ghidraPowerPcLanguage = ( + target: BinaryTarget, +): GhidraLanguageOverride | undefined => + target.kind === "executable" && target.architecture === "powerpc" + ? GHIDRA_POWERPC_LANGUAGE + : undefined; + +/** Commit source ELF interpretation without confusing target ISA with host CPU. */ +export const ghidraPowerPcProfileParameters = ( + target: BinaryTarget, +): Readonly> => { + if ( + target.kind !== "executable" || + target.architecture !== "powerpc" || + target.powerpc === undefined + ) + return {}; + return { + powerpc_elf: { + elf_class: target.powerpc.elfClass, + byte_order: target.powerpc.byteOrder, + type: target.powerpc.type, + flags: target.powerpc.flags, + }, + powerpc_support_lane: "elf32-be-exec-generic-v1", + }; +}; diff --git a/src/ghidra/GhidraProvider.ts b/src/ghidra/GhidraProvider.ts index 0a1cde0f07..b0ee8787d4 100644 --- a/src/ghidra/GhidraProvider.ts +++ b/src/ghidra/GhidraProvider.ts @@ -68,6 +68,7 @@ const SUPPORTED_ARCHITECTURES = new Set([ "mips", "6502", "riscv", + "powerpc", ]); /** Ghidra candidate backed by an isolated ephemeral headless import. */ diff --git a/src/ghidra/GhidraPspProfile.ts b/src/ghidra/GhidraPspProfile.ts index 17260d038e..3550d62ab2 100644 --- a/src/ghidra/GhidraPspProfile.ts +++ b/src/ghidra/GhidraPspProfile.ts @@ -1,5 +1,6 @@ import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { ghidraMipsUnsupportedReason } from "./GhidraMipsProfile.js"; +import { ghidraPowerPcUnsupportedReason } from "./GhidraPowerPcProfile.js"; import { ghidraRiscvUnsupportedReason } from "./GhidraRiscvProfile.js"; /** Allegrex's ELF machine declaration, not a filename or inferred instruction. */ @@ -14,6 +15,8 @@ export const isGhidraPspTarget = (target: BinaryTarget): boolean => export const ghidraProcessorUnsupportedReason = ( target: BinaryTarget, ): string | null => { + if (target.kind === "executable" && target.architecture === "powerpc") + return ghidraPowerPcUnsupportedReason(target); if (target.kind === "executable" && target.architecture === "riscv") return ghidraRiscvUnsupportedReason(target); if ( diff --git a/src/hopper/HopperAnalysisProfile.ts b/src/hopper/HopperAnalysisProfile.ts index a4848533f3..79ba3ab50b 100644 --- a/src/hopper/HopperAnalysisProfile.ts +++ b/src/hopper/HopperAnalysisProfile.ts @@ -110,6 +110,7 @@ const HOPPER_DECLINED_ARCHITECTURE_LABELS: Record< > = { mips: "MIPS", riscv: "RISC-V", + powerpc: "PowerPC", }; /** Derive Hopper's complete non-interactive CLI loader selection. */ diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index aec8c20829..f9826575b9 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -126,7 +126,11 @@ export class HopperProvider implements AnalysisProviderCandidate { status: "unsupported", code: "architecture_unsupported", reason: `REA's Hopper adapter does not admit ${ - target.architecture === "mips" ? "MIPS" : "RISC-V" + target.architecture === "mips" + ? "MIPS" + : target.architecture === "riscv" + ? "RISC-V" + : "PowerPC" } targets.`, diagnostics, }; diff --git a/src/ida/IdaProvider.ts b/src/ida/IdaProvider.ts index 09011caa57..5ce924fcd1 100644 --- a/src/ida/IdaProvider.ts +++ b/src/ida/IdaProvider.ts @@ -113,7 +113,11 @@ export class IdaProvider implements AnalysisProviderCandidate { status: "unsupported", code: "architecture_unsupported", reason: `REA's IDA adapter has no verified ${ - target.architecture === "mips" ? "MIPS" : "RISC-V" + target.architecture === "mips" + ? "MIPS" + : target.architecture === "riscv" + ? "RISC-V" + : "PowerPC" } lane. Select a supported Ghidra profile instead.`, diagnostics, }; diff --git a/tests/composition/analysis-sessions/powerpc-identity.test.ts b/tests/composition/analysis-sessions/powerpc-identity.test.ts new file mode 100644 index 0000000000..3dd1d374a2 --- /dev/null +++ b/tests/composition/analysis-sessions/powerpc-identity.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; +import type { BinaryTarget } from "../../../src/domain/binaryTargetTypes.js"; +import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; +import { + createAnalysisSnapshotEntry, + parseAnalysisSnapshot, + serializeAnalysisSnapshot, + snapshotBinding, + snapshotTarget, + snapshotMatchesProfile, +} from "../../../src/domain/analysisSnapshot.js"; +import { sessionOutputSchemas } from "../../../src/contracts/toolOutputSchemaGroups.js"; + +const target: BinaryTarget = { + path: "/source-owned.elf", + sha256: "a".repeat(64), + kind: "executable", + format: "elf", + architecture: "powerpc", + availableArchitectures: ["powerpc"], + powerpc: { elfClass: 32, byteOrder: "big", type: 2, flags: 0 }, +}; +const provider = { id: "ghidra", name: "Ghidra", version: "12.1.4" }; +const profile = createAnalysisProfile(provider, { + powerpc_elf: { elf_class: 32, byte_order: "big", type: 2, flags: 0 }, +}); + +describe("PowerPC identity through lifecycle, Evidence and snapshot consumers", () => { + it("round-trips query bindings without relaxing integrity", () => { + const result = [{ address: "0x100101b0", value: "rea_ppc_entry" }]; + const parameters = {}; + const evidence = createEvidence(target, provider, { + operation: "list_procedures", + parameters, + result, + analysisProfile: profile, + }); + const binding = snapshotBinding(profile); + const entry = createAnalysisSnapshotEntry({ + target: snapshotTarget(target), + binding, + operation: "list_procedures", + parameters, + execution: { + result, + rawResult: null, + provider, + analysisProfile: profile, + limitations: [], + locations: [], + subject: target, + }, + }); + const snapshot = { + target: snapshotTarget(target), + binding, + entries: [entry], + workflow_entries: [], + evidence_bundle: createEvidenceBundle([evidence]), + }; + const restored = parseAnalysisSnapshot( + JSON.parse(serializeAnalysisSnapshot(snapshot)), + ); + expect(restored.target.architecture).toBe("powerpc"); + expect(restored.evidence_bundle.records[0]?.subject?.architecture).toBe( + "powerpc", + ); + expect(restored.entries[0]?.execution.subject?.architecture).toBe( + "powerpc", + ); + expect( + sessionOutputSchemas.open_binary.parse({ result: target }).result + .architecture, + ).toBe("powerpc"); + expect(() => + parseEvidence({ + ...evidence, + subject: { ...evidence.subject, architecture: "x86" }, + }), + ).toThrow("semantic identifier"); + const changed = createAnalysisProfile(provider, { + powerpc_elf: { + elf_class: 32, + byte_order: "big", + type: 2, + flags: 0x00010000, + }, + }); + expect(snapshotMatchesProfile(binding, changed)).toBe(false); + }); +}); diff --git a/tests/conformance/c/ppc.c b/tests/conformance/c/ppc.c new file mode 100644 index 0000000000..3f62ca57e6 --- /dev/null +++ b/tests/conformance/c/ppc.c @@ -0,0 +1,14 @@ +/* Freestanding, source-owned static-analysis fixture. Never executed by REA. */ +volatile unsigned int rea_ppc_global = 7; +const char rea_ppc_marker[] = "rea-ppc-source-owned-fixture"; + +__attribute__((noinline)) unsigned int rea_ppc_leaf(unsigned int value) +{ + return value * 3u + rea_ppc_global; +} + +__attribute__((noinline)) unsigned int rea_ppc_entry(unsigned int selector) +{ + unsigned int adjusted = selector > 10u ? selector : selector + 1u; + return rea_ppc_leaf(adjusted) + (unsigned char)rea_ppc_marker[0]; +} diff --git a/tests/conformance/ghidra/ppc-checks.d.mts b/tests/conformance/ghidra/ppc-checks.d.mts new file mode 100644 index 0000000000..38609de454 --- /dev/null +++ b/tests/conformance/ghidra/ppc-checks.d.mts @@ -0,0 +1,29 @@ +import type { PowerPcElfMetadata } from "../../../src/domain/binaryTargetTypes.js"; + +export interface PowerPcReadelfFacts { + powerpc: PowerPcElfMetadata; + symbols: Record< + | "rea_ppc_entry" + | "rea_ppc_leaf" + | "rea_ppc_probe" + | "rea_ppc_global" + | "rea_ppc_marker", + string + >; +} +export function parsePowerPcReadelf(text: string): PowerPcReadelfFacts; +export function inspectPowerPcReadelf( + path: string, +): Promise< + PowerPcReadelfFacts & { command: string; version: string; raw: string } +>; +export function assertPowerPcLoadedImage( + observations: unknown, + sha256: string, +): void; +export function assertPowerPcProbe( + move: unknown, + branch: unknown, + address: string, +): void; +export function assertPowerPcGlobal(bytes: string): void; diff --git a/tests/conformance/ghidra/ppc-checks.mjs b/tests/conformance/ghidra/ppc-checks.mjs new file mode 100644 index 0000000000..1136f2602e --- /dev/null +++ b/tests/conformance/ghidra/ppc-checks.mjs @@ -0,0 +1,132 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execute = promisify(execFile); +// The lane binds this language; Ghidra's ELF loader alone selects e500. +const LANGUAGE = "PowerPC:BE:32:default"; +const wordHex = (value) => { + const bytes = Buffer.alloc(4); + bytes.writeUInt32BE(value); + return bytes.toString("hex"); +}; + +/** Parse only the documented GNU readelf fields needed by our tiny fixture. */ +export function parsePowerPcReadelf(text) { + const field = (expression, label) => { + const matches = [...text.matchAll(expression)]; + assert.equal(matches.length, 1, `Missing or ambiguous readelf ${label}`); + return matches[0][1].trim(); + }; + assert.equal(field(/^\s*Class:\s*(.+)$/gmu, "class"), "ELF32"); + assert.equal( + field(/^\s*Data:.*?,\s*(little|big) endian\s*$/gmu, "byte order"), + "big", + ); + assert.match(field(/^\s*Type:\s*(.+)$/gmu, "type"), /^EXEC\b/u); + assert.equal(field(/^\s*Machine:\s*(.+)$/gmu, "machine"), "PowerPC"); + const flags = Number(field(/^\s*Flags:\s*(0x[\da-f]+)\s*$/gmu, "flags")); + assert.equal(flags, 0, "Fixture ELF flags differ from build intent"); + const symbols = {}; + for (const name of [ + "rea_ppc_entry", + "rea_ppc_leaf", + "rea_ppc_probe", + "rea_ppc_global", + "rea_ppc_marker", + ]) { + const value = field( + new RegExp( + `^\\s*\\d+:\\s+([\\da-f]+)\\s+\\d+\\s+(?:FUNC|OBJECT)\\s+GLOBAL\\s+\\S+\\s+\\d+\\s+${name}\\s*$`, + "gmu", + ), + `symbol ${name}`, + ); + symbols[name] = `0x${BigInt(`0x${value}`).toString(16)}`; + } + return { + powerpc: { elfClass: 32, byteOrder: "big", type: 2, flags }, + symbols, + }; +} + +/** Independent reader is a bounded test prerequisite, never an REA provider. */ +export async function inspectPowerPcReadelf(path) { + const command = process.env.REA_PPC_READELF ?? "readelf"; + const options = { + env: { ...process.env, LC_ALL: "C" }, + timeout: 10000, + maxBuffer: 1024 * 1024, + }; + const version = await execute(command, ["--version"], options); + assert.match( + version.stdout, + /^GNU readelf\b/u, + "This lane requires GNU readelf", + ); + const result = await execute(command, ["-h", "-s", "-W", path], options); + assert.equal( + result.stderr.trim(), + "", + `readelf diagnostics: ${result.stderr}`, + ); + return { + ...parsePowerPcReadelf(result.stdout), + command, + version: version.stdout.split("\n")[0], + raw: result.stdout, + }; +} + +/** Check effective Ghidra observations, not the requested analysis profile. */ +export function assertPowerPcLoadedImage(observations, sha256) { + assert.equal(observations.language_id, LANGUAGE); + assert.equal(observations.compiler_spec_id, "default"); + assert.ok( + observations.source_files.some( + (file) => + file.original_sha256 === sha256 && file.modified_sha256 === sha256, + ), + "Ghidra loaded source identity changed", + ); +} + +/** Verify the exact constant/conditional destination in ppc-probe.S. */ +export function assertPowerPcProbe(move, branch, address) { + const start = BigInt(address); + for (const [instruction, offset, word] of [ + [move, 0n, 0x38641234], + [branch, 8n, 0x41820008], + ]) { + assert.equal(instruction.status, "decoded"); + assert.equal(BigInt(instruction.address), start + offset); + assert.equal(instruction.architecture, LANGUAGE); + assert.equal(instruction.length, 4); + assert.equal(instruction.bytes, wordHex(word)); + } + assert.match(move.mnemonic, /^ADDI$/iu); + assert.ok( + move.operands.some((operand) => + operand.components.some( + (part) => + part.kind === "immediate" && + part.value !== null && + BigInt(part.value) === 0x1234n, + ), + ), + "ADDI immediate was not recovered", + ); + assert.match(branch.mnemonic, /^BEQ$/iu); + assert.equal(branch.flow.kind, "jump"); + assert.equal(branch.flow.conditional, true); + assert.equal(branch.flow.computed, false); + assert.deepEqual( + branch.flow.direct_destinations.map((value) => BigInt(value)), + [start + 16n], + ); +} + +/** Four bytes must encode the known source-owned global in big-endian order. */ +export function assertPowerPcGlobal(bytes) { + assert.equal(bytes, wordHex(7)); +} diff --git a/tests/conformance/ghidra/ppc-checks.test.ts b/tests/conformance/ghidra/ppc-checks.test.ts new file mode 100644 index 0000000000..6328ce0292 --- /dev/null +++ b/tests/conformance/ghidra/ppc-checks.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it } from "vitest"; +import { + assertPowerPcGlobal, + assertPowerPcLoadedImage, + assertPowerPcProbe, + parsePowerPcReadelf, +} from "./ppc-checks.mjs"; + +// GNU readelf 2.47 output for the source-owned Clang 23 / LLD fixture. +// This tests the reader boundary, not Ghidra; the real lane rebuilds it. +const readelf = `ELF Header: + Magic: 7f 45 4c 46 01 02 01 00 00 00 00 00 00 00 00 00 + Class: ELF32 + Data: 2's complement, big endian + Version: 1 (current) + OS/ABI: UNIX - System V + ABI Version: 0 + Type: EXEC (Executable file) + Machine: PowerPC + Version: 0x1 + Entry point address: 0x100101b0 + Start of program headers: 52 (bytes into file) + Start of section headers: 2188 (bytes into file) + Flags: 0x0 + Size of this header: 52 (bytes) + Size of program headers: 32 (bytes) + Number of program headers: 6 + Size of section headers: 40 (bytes) + Number of section headers: 18 + Section header string table index: 16 + +Symbol table '.symtab' contains 7 entries: + Num: Value Size Type Bind Vis Ndx Name + 0: 00000000 0 NOTYPE LOCAL DEFAULT UND + 1: 00000000 0 FILE LOCAL DEFAULT ABS ppc.c + 2: 10010180 48 FUNC GLOBAL DEFAULT 4 rea_ppc_leaf + 3: 1002023c 4 OBJECT GLOBAL DEFAULT 5 rea_ppc_global + 4: 100101b0 116 FUNC GLOBAL DEFAULT 4 rea_ppc_entry + 5: 100000f4 29 OBJECT GLOBAL DEFAULT 1 rea_ppc_marker + 6: 10010224 24 FUNC GLOBAL DEFAULT 4 rea_ppc_probe +`; +const sha256 = "a".repeat(64); +const image = { + language_id: "PowerPC:BE:32:default", + compiler_spec_id: "default", + source_files: [{ original_sha256: sha256, modified_sha256: sha256 }], +}; +const move = { + address: "0x10010224", + status: "decoded", + architecture: "PowerPC:BE:32:default", + length: 4, + bytes: "38641234", + mnemonic: "addi", + operands: [ + { index: 2, components: [{ kind: "immediate", value: "0x1234" }] }, + ], +}; +const branch = { + address: "0x1001022c", + status: "decoded", + architecture: "PowerPC:BE:32:default", + length: 4, + bytes: "41820008", + mnemonic: "beq", + flow: { + kind: "jump", + conditional: true, + computed: false, + direct_destinations: ["0x10010234"], + }, +}; + +describe("PowerPC real-verifier independent expectations", () => { + it("reads GNU header and defined symbols without a Ghidra-derived oracle", () => { + const parsed = parsePowerPcReadelf(readelf); + expect(parsed.powerpc).toEqual({ + elfClass: 32, + byteOrder: "big", + type: 2, + flags: 0, + }); + expect(parsed.symbols.rea_ppc_probe).toBe("0x10010224"); + }); + + it.each([ + ["wrong class", "ELF32", "ELF64"], + ["wrong byte order", "big endian", "little endian"], + [ + "wrong machine", + "Machine: PowerPC", + "Machine: PowerPC64", + ], + ["wrong object type", "EXEC (Executable file)", "DYN (Shared object file)"], + [ + "wrong flags", + "Flags: 0x0", + "Flags: 0x80000000", + ], + ["missing symbol", "rea_ppc_probe", "unrelated_probe"], + ])( + "rejects %s instead of returning a guessed fixture interpretation", + (_name, from, to) => { + expect(() => parsePowerPcReadelf(readelf.replace(from, to))).toThrow(); + }, + ); + + it("rejects ambiguous producer fields", () => { + expect(() => parsePowerPcReadelf(`${readelf}\nClass: ELF32\n`)).toThrow( + /ambiguous/u, + ); + }); + + it("checks the effective loaded image, including Ghidra's e500 default", () => { + assertPowerPcLoadedImage(image, sha256); + expect(() => + assertPowerPcLoadedImage( + { ...image, language_id: "PowerPC:BE:32:e500" }, + sha256, + ), + ).toThrow(); + expect(() => + assertPowerPcLoadedImage( + { ...image, compiler_spec_id: "macosx" }, + sha256, + ), + ).toThrow(); + expect(() => + assertPowerPcLoadedImage( + { + ...image, + source_files: [ + { original_sha256: sha256, modified_sha256: "b".repeat(64) }, + ], + }, + sha256, + ), + ).toThrow(); + }); + + it("checks exact probe bytes, immediate and conditional destination", () => { + assertPowerPcProbe(move, branch, "0x10010224"); + }); + + it("rejects plausible nonempty but incorrect instruction observations", () => { + expect(() => + assertPowerPcProbe({ ...move, bytes: "38641235" }, branch, "0x10010224"), + ).toThrow(); + expect(() => + assertPowerPcProbe({ ...move, length: 2 }, branch, "0x10010224"), + ).toThrow(); + expect(() => + assertPowerPcProbe( + { + ...move, + operands: [ + { index: 2, components: [{ kind: "immediate", value: "0x1235" }] }, + ], + }, + branch, + "0x10010224", + ), + ).toThrow(); + expect(() => + assertPowerPcProbe( + move, + { + ...branch, + flow: { ...branch.flow, direct_destinations: ["0x10010230"] }, + }, + "0x10010224", + ), + ).toThrow(); + expect(() => + assertPowerPcProbe( + move, + { ...branch, flow: { ...branch.flow, conditional: false } }, + "0x10010224", + ), + ).toThrow(); + }); + + it("checks the big-endian global word", () => { + assertPowerPcGlobal("00000007"); + expect(() => assertPowerPcGlobal("07000000")).toThrow(); + }); +}); diff --git a/tests/conformance/ghidra/ppc-fixture.mjs b/tests/conformance/ghidra/ppc-fixture.mjs new file mode 100644 index 0000000000..202c0dacf6 --- /dev/null +++ b/tests/conformance/ghidra/ppc-fixture.mjs @@ -0,0 +1,62 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const execute = promisify(execFile); + +/** Build a source-owned freestanding ELF; never execute its target code. */ +export async function buildPowerPcFixture(directory) { + const compiler = process.env.REA_PPC_CLANG ?? "clang"; + const target = "powerpc-unknown-linux-gnu"; + const source = fileURLToPath(new URL("../c/ppc.c", import.meta.url)); + const probe = fileURLToPath(new URL("./ppc-probe.S", import.meta.url)); + const path = join(directory, `${target}.elf`); + const args = [ + `--target=${target}`, + // A 750-family core with the classic FPU, unlike the e500 SPE variant. + "-mcpu=750", + "-fno-pic", + "-fuse-ld=lld", + "-nostdlib", + "-static", + "-O0", + "-g", + "-Wl,-e,rea_ppc_entry", + "-Wl,--build-id=none", + source, + probe, + "-o", + path, + ]; + const version = await execute(compiler, ["--version"], { timeout: 10000 }); + await execute(compiler, args, { timeout: 60000 }); + const bytes = await readFile(path); + if ( + bytes.length < 52 || + bytes.toString("hex", 0, 4) !== "7f454c46" || + bytes[4] !== 1 || + bytes[5] !== 2 || + bytes.readUInt16BE(16) !== 2 || + bytes.readUInt16BE(18) !== 20 || + bytes.readUInt32BE(36) !== 0 + ) + throw new Error( + "Compiler did not produce the declared ELF32 big-endian PowerPC fixture", + ); + return { + path, + flags: bytes.readUInt32BE(36), + sha256: createHash("sha256").update(bytes).digest("hex"), + source_sha256: createHash("sha256") + .update(await readFile(source)) + .digest("hex"), + probe_sha256: createHash("sha256") + .update(await readFile(probe)) + .digest("hex"), + compiler: version.stdout.trim(), + arguments: args, + }; +} diff --git a/tests/conformance/ghidra/ppc-probe.S b/tests/conformance/ghidra/ppc-probe.S new file mode 100644 index 0000000000..f67793942b --- /dev/null +++ b/tests/conformance/ghidra/ppc-probe.S @@ -0,0 +1,17 @@ +/* Source-owned 32-bit PowerPC instructions with fixed, reviewable encodings. + * No target execution. The conditional branch at +8 targets +16. + */ +.text +.p2align 2 +.globl rea_ppc_probe +.type rea_ppc_probe, @function +rea_ppc_probe: + addi 3, 4, 0x1234 + cmpwi 0, 4, 0 + beq 0, .Lzero + blr +.Lzero: + li 3, 7 + blr +.size rea_ppc_probe, .-rea_ppc_probe +.section .note.GNU-stack,"",@progbits From 1ebbc3b3714a83f10950eaa09bee5d4490bd572e Mon Sep 17 00:00:00 2001 From: Jeremy Levartovsky <140487034+JayOfTheKeyboard@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:36:43 +1100 Subject: [PATCH 14/14] fix(managed): pair members by declared identity before name-free remaps (#2116) * fix(managed): pair members by declared identity before name-free remaps The first matching round keyed methods on signature and normalized CIL, and fields on signature, without the declared type or name. Identical members of different types fell into one ambiguous group, so a file compared with itself reported unknowns, and a lone match across types was reported as an unchanged exact pair while the member's own counterpart went unpaired. Pair by declared type, name, and exact CIL/signature first, then by declared type, name, and raw signature. The name-free CIL/signature, field-signature, and structural rounds now run last, as remaps, and skip a pair across declared identities while either member's own type and name remain unpaired on the other side. Nested types that share a bare name still share one declared identity, so their identical members stay ambiguous in a self-comparison. * fix(managed): keep members with kept identities out of remap rounds The cross-identity check applied only to one-to-one groups. A member whose own declared type and name were still unpaired on the other side could fall into a larger remap group and be reported ambiguous, and a blocked candidate could make an otherwise unique remap ambiguous. A remap round now drops such members before grouping, along with members whose declaring type is unresolved. Same-identity structural pairs get their own round before the remaps. Each member's declared identity is computed once. Ambiguity limitations now name the evidence a group shares, which stays true whether the round keyed on names or not. * fix(managed): keep names in unresolved identities and hold ambiguous ones A member of an unresolved declaring type had no declared identity, so the declared-identity rounds keyed it by CIL or signature alone. Two unresolved members with different names paired as an exact match, and a file compared with itself reported unresolved members ambiguous. The declared identity now always includes the name, with a null type for an unresolved one. Remap rounds still leave such members out. A remap round now also leaves out a member whose own identity sits in an ambiguous group on the other side, not only one left one-sided. The undecoded-counterpart check uses the same declared identity. * fix(managed): keep remap groups with unresolved owners ambiguous Remap rounds dropped members of an unresolved declaring type before grouping. Such a member can be another member's own counterpart, so a resolved member could remap to a copy in another type while its unresolved counterpart was reported added. Unresolved members now join remap grouping, and a group that contains one stays ambiguous instead of pairing. The undecoded-counterpart set reuses each member's computed identity. * fix(managed): keep ownerless members out of declared rounds when shared A member whose declaring type did not resolve has a declared identity of a null type and its name. The declared rounds paired two such same-named members even when the other side also held a resolved member with the same CIL and signature, which could be the owner. The old name-free first round reported that group as ambiguous. An ownerless member now enters the declared rounds only when no other member on either side shares its exact CIL/signature key, and never the declared structural round. Otherwise it reaches the remap rounds, which keep any group with an unresolved member ambiguous. * fix(managed): keep competitors of an ownerless member out of the declared round When an ownerless member shares its exact CIL/signature key with another member on the same side, its owner may be that member. The resolved members holding the key now stay out of the declared round too, so they cannot pair and consume the candidate the ownerless member may own, and the remap round keeps the whole group ambiguous as main does. A lone ownerless member still pairs with its lone counterpart. * test(managed): split the ownerless comparison tests under the function length limit The ownerless describe block reached 181 lines, past oxlint's max-lines-per-function limit of 150. Move the structural-competitor case into its own block. * fix(managed): contest signatures and remap only open candidates * fix(managed): retain blocked raw-signature choices as unknown * fix(managed): keep member names in signature contention * fix(managed): scope residual signature contention by name once ownerlessContested applies competitionIdentityKey itself, but unresolvedSignatureAmbiguities passed it keys that were already name-scoped, so the contested set never matched the groups and the same-signature, different-body case reported additions and removals instead of one unknown group. Pass the raw signature key. * fix(managed): constrain matchObserved to declared members matchObserved passes its items to matchByKeys, which needs Declared, so its looser { declaring_type } bound failed typecheck (TS2344, TS2322). * fix(managed): retain held ownership contention for residual matches * fix(test): bind workspace fixture in Hopper launch cases * fix(ci): prepare worker tests and preserve Hopper cleanup ordering --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/managed-code-analysis.md | 34 +- .../managed/managedWorkflowToolContracts.ts | 2 +- .../managedMemberComparisonCoverage.ts | 4 +- .../managed/managedMemberComparisonItems.ts | 38 +- .../managed/managedMemberComparisonMatch.ts | 274 +++++- .../providers/hopperStartupRecovery.test.ts | 2 + ...anagedMemberComparisonFromMetadata.test.ts | 896 +++++++++++++++++- 7 files changed, 1192 insertions(+), 58 deletions(-) diff --git a/docs/managed-code-analysis.md b/docs/managed-code-analysis.md index 9f86551d7b..05782b4a2e 100644 --- a/docs/managed-code-analysis.md +++ b/docs/managed-code-analysis.md @@ -209,19 +209,27 @@ native/runtime questions retain their own authority and coverage. Names and semantic labels do not establish durable member identity. Comparison preserves observed differences, structural inferences, and unknowns separately. -Cross-version matching first prefers exact raw CIL/signature identity, with -equal decoded fingerprints. Equal decoded fingerprints alone cannot establish -an exact match: they omit switch targets and collapse distinct NaN encodings. -When that does not match, it pairs an exact declared type, method name, and raw -signature before trying structural body shape. The exact-signature key uses names only -as part of that full tuple; names alone never select a pair. Duplicate tuples -remain ambiguous rather than being paired by token order. Fields follow the -same exact tiers. The raw signature is exact whether or not REA decoded it, so -undecoded signatures pair by that tuple too, but they never enter structural -rounds. An unmatched member is therefore reported as `unknown`, not added or -removed, when its own signature was not decoded or when the other side has an -unpaired member, one-sided or ambiguous, with the same declared type and name -whose signature was not decoded. +Cross-version matching first pairs members of the same declared type and name: +methods by exact CIL/signature identity, then by raw signature, then by +structural body shape; fields by raw signature. Exact CIL/signature identity +requires equal raw CIL digests and decoded fingerprints. Equal decoded +fingerprints alone cannot establish an exact match: they omit switch targets +and collapse distinct NaN encodings. Only then may a remap pair a +renamed or moved member across declared types or names: a method by exact +CIL/signature identity or structural body shape, a field by exact decoded +signature. A remap never takes a member whose own declared type and name remain +unpaired on the other side, so a member is never taken from its own changed +counterpart. A member whose declaring type is unresolved could belong to any +type, so it pairs by name only when no other member shares its exact +CIL/signature key, never by structural shape alone, and any remap group it +joins stays ambiguous. The exact-signature key uses names only as part of that +full tuple; names alone never select a pair. Duplicate tuples remain ambiguous +rather than being paired by token order. The raw signature is exact whether or +not REA decoded it, so undecoded signatures pair by that tuple too, but they +never enter structural rounds. An unmatched member is therefore reported as +`unknown`, not added or removed, when its own signature was not decoded or when +the other side has an unpaired member, one-sided or ambiguous, with the same +declared type and name whose signature was not decoded. For a matched method, an unavailable or partial body makes body-shape facets unknown while preserving observed signature differences. For complete bodies, diff --git a/src/contracts/managed/managedWorkflowToolContracts.ts b/src/contracts/managed/managedWorkflowToolContracts.ts index 519c2e1259..18d78a3102 100644 --- a/src/contracts/managed/managedWorkflowToolContracts.ts +++ b/src/contracts/managed/managedWorkflowToolContracts.ts @@ -98,7 +98,7 @@ export const MANAGED_WORKFLOW_TOOL_CONTRACTS = [ name: "compare_managed_members", ...toolContractMetadata("compare_managed_members"), description: - "Compare two authenticated inspect_managed_members Evidence records using unique-only tiers: exact CIL/signature, exact declared type, name, and raw signature (decoded or not), and decoded structural method shape. Names alone are never a matching basis; a member whose signature was not decoded, or whose same-named counterpart's was not, stays unknown rather than added or removed. Metadata tokens remain build-local coordinates bound to each artifact SHA-256 and MVID, and the tuple digest does not itself remap them.", + "Compare two authenticated inspect_managed_members Evidence records using unique-only tiers: exact declared type, name, and CIL/signature; exact declared type, name, and raw signature (decoded or not); exact declared type, name, and decoded structural method shape; then exact CIL/signature or decoded structural method shape alone, which never takes a member whose own declared type and name remain unpaired on the other side or whose declaring type is unresolved. Names alone are never a matching basis; a member whose signature was not decoded, or whose same-named counterpart's was not, stays unknown rather than added or removed. Metadata tokens remain build-local coordinates bound to each artifact SHA-256 and MVID, and the tuple digest does not itself remap them.", kind: "application", inputSchema: compareManagedMembersReferenceInputSchema, outputSchema: comparisonOutputSchema, diff --git a/src/domain/managed/managedMemberComparisonCoverage.ts b/src/domain/managed/managedMemberComparisonCoverage.ts index 85da3b4eba..fce7b25fb4 100644 --- a/src/domain/managed/managedMemberComparisonCoverage.ts +++ b/src/domain/managed/managedMemberComparisonCoverage.ts @@ -107,8 +107,8 @@ export const comparisonLimitations = ( ): string[] => { const limitations: string[] = [ "Metadata tokens are build-local coordinates; matched pairs are remaps, not persistent identities.", - "Methods pair by exact CIL/signature, then declared type, name, and signature, then structural shape; names alone are never a matching basis.", - "Fields pair by exact signature, then declared type, name, and signature; an undecoded signature is compared only by its raw bytes, and an unmatched member with one stays unknown.", + "Methods pair within one declared type and name by exact CIL/signature, then signature, then structural shape; only then may exact CIL/signature or structural shape alone remap across declared types or names, never taking a member whose own declared type and name remain unpaired on the other side or whose declaring type is unresolved. Names alone are never a matching basis.", + "Fields pair by declared type, name, and signature, then remap by exact decoded signature alone under the same rule; an undecoded signature is compared only by its raw bytes, and an unmatched member with one stays unknown.", ]; if (left.coverage.state !== "complete" || right.coverage.state !== "complete") limitations.push("At least one managed member observation is partial."); diff --git a/src/domain/managed/managedMemberComparisonItems.ts b/src/domain/managed/managedMemberComparisonItems.ts index 26331be083..61bfd7c180 100644 --- a/src/domain/managed/managedMemberComparisonItems.ts +++ b/src/domain/managed/managedMemberComparisonItems.ts @@ -5,6 +5,7 @@ import type { ManagedMemberComparisonResult } from "./managedMemberComparison.js import { type ManagedFieldMatches, type ManagedMethodMatches, + identityOf, sha256, } from "./managedMemberComparisonMatch.js"; @@ -211,33 +212,36 @@ const absenceLimitations = ( `signature-not-decoded: The member's ${member.signature.parse_status} signature was compared only by exact declared type, name, and raw bytes, so a changed or renamed counterpart could not be sought.`, ]), ...(member.signature.parse_status === "decoded" && - opposite.oppositeUndecodedNames.has(declaredName(member)) + opposite.oppositeUndecodedNames.size > 0 && + opposite.oppositeUndecodedNames.has(identityOf(member)) ? [ `counterpart-signature-not-decoded: The ${opposite.opposite} ${opposite.inventory} inventory has an unpaired member with the same declared type and name whose signature was not decoded, so it may be this member with a changed signature.`, ] : []), ]; -const declaredName = (member: Method | Field): string => - JSON.stringify([member.declaring_type, member.name]); - /** * Declared names of unpaired members whose signature was not decoded: one-sided * members and candidates of ambiguous groups, which also remain unpaired. */ +interface Unpaired { + readonly item: Method | Field; + readonly identity: string; +} + const undecodedOneSidedNames = (members: { - readonly leftOnly: readonly { readonly item: Method | Field }[]; - readonly rightOnly: readonly { readonly item: Method | Field }[]; + readonly leftOnly: readonly Unpaired[]; + readonly rightOnly: readonly Unpaired[]; readonly ambiguous: readonly { - readonly left: readonly { readonly item: Method | Field }[]; - readonly right: readonly { readonly item: Method | Field }[]; + readonly left: readonly Unpaired[]; + readonly right: readonly Unpaired[]; }[]; }): UndecodedOneSidedNames => { - const names = (side: readonly { readonly item: Method | Field }[]) => + const names = (side: readonly Unpaired[]) => new Set( side .filter(({ item }) => item.signature.parse_status !== "decoded") - .map(({ item }) => declaredName(item)), + .map(({ identity }) => identity), ); return { left: names([ @@ -300,18 +304,20 @@ const fieldOnlyItem = ( }; /** - * Name the identity key an ambiguous group shares. The exact-signature key - * already includes the declared type and name, so names cannot separate it. + * Name only evidence shared by the whole group. Exact-signature ambiguities + * include both declared-identity collisions and unresolved raw-signature choices. */ const ambiguityLimitation = ( members: "methods" | "fields", basis: MethodItem["match"]["basis"], ): string => basis === "exact-signature" - ? `Multiple managed ${members} share the same declared type, name, and raw signature; REA did not guess a token remap.` - : members === "methods" - ? "Multiple managed methods share the same non-name identity key; REA did not guess a token remap." - : "Multiple managed fields share the same signature; REA did not guess a token remap from names."; + ? `Multiple managed ${members} share the same raw signature; REA did not guess declared ownership or a token remap.` + : basis === "exact-il-signature" + ? "Multiple managed methods share the same CIL and signature; REA did not guess a token remap." + : members === "methods" + ? "Multiple managed methods share the same structural shape; REA did not guess a token remap." + : "Multiple managed fields share the same signature; REA did not guess a token remap."; export const buildMethodItems = ( matches: ManagedMethodMatches, diff --git a/src/domain/managed/managedMemberComparisonMatch.ts b/src/domain/managed/managedMemberComparisonMatch.ts index 90106a17cd..2cbe32a4a6 100644 --- a/src/domain/managed/managedMemberComparisonMatch.ts +++ b/src/domain/managed/managedMemberComparisonMatch.ts @@ -28,8 +28,11 @@ type ConcreteMatchBasis = Exclude; interface Keyed { readonly item: Item; + /** Declared type and name; an unresolved type is null, the name remains. */ + readonly identity: string; readonly exactKey: string | null; readonly signatureKey: string | null; + readonly rawSignatureKey: string; readonly structuralKey: string | null; } @@ -48,6 +51,8 @@ interface Ambiguous { const keyMethod = (item: Method): Keyed => ({ item, + identity: identityOf(item), + rawSignatureKey: item.signature.raw_sha256, exactKey: item.body.status === "present" && item.body.il_sha256 !== null && @@ -99,6 +104,8 @@ const keyMethod = (item: Method): Keyed => ({ const keyField = (item: Field): Keyed => ({ item, + identity: identityOf(item), + rawSignatureKey: item.signature.raw_sha256, exactKey: item.signature.parse_status === "decoded" ? sha256(["field-exact", item.signature.raw_sha256]) @@ -126,15 +133,34 @@ const matchMethods = ( matchByKeys({ left, right, - exactBasis: "exact-il-signature", - fallbackBases: [ + rounds: [ + { + basis: "exact-il-signature", + key: declaredAs(({ exactKey }) => exactKey), + }, { basis: "exact-signature", key: ({ signatureKey }) => signatureKey, + competitionKey: ({ rawSignatureKey }) => rawSignatureKey, + }, + { + basis: "structural-method-shape", + // An unresolved owner has no declared shape match to prove. + key: declaredAs((keyed) => + keyed.item.declaring_type === null ? null : keyed.structuralKey, + ), + competitionKey: ({ structuralKey }) => structuralKey, + }, + { + basis: "exact-il-signature", + key: ({ exactKey }) => exactKey, + remap: true, }, { basis: "structural-method-shape", key: ({ structuralKey }) => structuralKey, + competitionKey: ({ structuralKey }) => structuralKey, + remap: true, }, ], }); @@ -146,11 +172,20 @@ const matchFields = ( matchByKeys({ left, right, - exactBasis: "field-signature", - fallbackBases: [ + rounds: [ + { + basis: "field-signature", + key: declaredAs(({ exactKey }) => exactKey), + }, { basis: "exact-signature", key: ({ signatureKey }) => signatureKey, + competitionKey: ({ rawSignatureKey }) => rawSignatureKey, + }, + { + basis: "field-signature", + key: ({ exactKey }) => exactKey, + remap: true, }, ], }); @@ -158,21 +193,47 @@ const matchFields = ( export type ManagedMethodMatches = ReturnType; export type ManagedFieldMatches = ReturnType; +interface MatchRound { + readonly basis: ConcreteMatchBasis; + readonly key: (item: Keyed) => string | null; + /** Name-free evidence paired with the retained member name for contesting. */ + readonly competitionKey?: (item: Keyed) => string | null; + /** + * The key omits the declared type, so a pair may join different declared + * identities; same-identity pairs were taken by earlier rounds. Contention + * still retains the member name, which unresolved owners keep as evidence. + * A member whose own identity is still unpaired on the other side does not + * take part, and a group with a member of an unresolved type stays ambiguous. + */ + readonly remap?: true; +} + interface MatchByKeysInput { readonly left: readonly Keyed[]; readonly right: readonly Keyed[]; - readonly exactBasis: ConcreteMatchBasis; - readonly fallbackBases: readonly { - readonly basis: ConcreteMatchBasis; - readonly key: (item: Keyed) => string | null; - }[]; + readonly rounds: readonly MatchRound[]; } -const matchByKeys = ({ +export const identityOf = ({ + declaring_type, + name, +}: { + readonly declaring_type: string | null; + readonly name: string; +}): string => sha256([declaring_type, name]); + +/** Narrow a key to members of one declared type and name. */ +const declaredAs = + (key: (item: Keyed) => string | null) => + (item: Keyed): string | null => { + const value = key(item); + return value === null ? null : sha256([item.identity, value]); + }; + +const matchByKeys = ({ left, right, - exactBasis, - fallbackBases, + rounds, }: MatchByKeysInput): { readonly pairs: readonly MatchedPair[]; readonly ambiguous: readonly Ambiguous[]; @@ -181,33 +242,58 @@ const matchByKeys = ({ } => { const usedLeft = new Set>(); const usedRight = new Set>(); + const pairedLeft = new Set>(); + const pairedRight = new Set>(); const pairs: MatchedPair[] = []; const ambiguous: Ambiguous[] = []; - const rounds: readonly { - readonly basis: ConcreteMatchBasis; - readonly key: (item: Keyed) => string | null; - }[] = [ - { basis: exactBasis, key: ({ exactKey }) => exactKey }, - ...fallbackBases, - ]; for (const round of rounds) { + const competitionKey = round.competitionKey ?? ((item) => item.exactKey); + const unpairedLeft = unpairedOf(left, pairedLeft); + const unpairedRight = unpairedOf(right, pairedRight); + const declarable = ownerlessDeclarable( + unpairedLeft, + unpairedRight, + competitionKey, + ); + const openLeft = left.filter((item) => !usedLeft.has(item)); + const openRight = right.filter((item) => !usedRight.has(item)); + // Only open candidates can justify overriding a held declared identity: + // consumed ambiguous rows remain identity barriers, but cannot join a + // later remap group to establish its ambiguity. + const remapContested = + round.remap === true + ? ownerlessContested([openLeft, openRight], competitionKey) + : new Set(); const leftGroups = groupBy( - left.filter((item) => !usedLeft.has(item)), + round.remap === true + ? remappable(openLeft, unpairedRight, competitionKey, remapContested) + : openLeft.filter(declarable), round.key, ); const rightGroups = groupBy( - right.filter((item) => !usedRight.has(item)), + round.remap === true + ? remappable(openRight, unpairedLeft, competitionKey, remapContested) + : openRight.filter(declarable), round.key, ); for (const [key, leftItems] of leftGroups) { const rightItems = rightGroups.get(key); if (rightItems === undefined) continue; - if (leftItems.length === 1 && rightItems.length === 1) { + // An unresolved owner may be either member's own counterpart, so a + // remap group it joins stays ambiguous. + const refused = + round.remap === true && + [...leftItems, ...rightItems].some( + ({ item }) => item.declaring_type === null, + ); + if (!refused && leftItems.length === 1 && rightItems.length === 1) { const [leftItem] = leftItems; const [rightItem] = rightItems; if (leftItem === undefined || rightItem === undefined) continue; usedLeft.add(leftItem); usedRight.add(rightItem); + pairedLeft.add(leftItem); + pairedRight.add(rightItem); pairs.push({ left: leftItem, right: rightItem, @@ -230,6 +316,22 @@ const matchByKeys = ({ } } } + // Contention can prevent a declared signature pair even when CIL/shape + // evidence cannot form a later remap. Held ambiguities still contest + // ownership; only proven pairs leave that evidence pool. Group only open + // candidates so held members retain their original ambiguity and provenance. + for (const group of unresolvedSignatureAmbiguities( + left.filter((item) => !usedLeft.has(item)), + right.filter((item) => !usedRight.has(item)), + ownerlessContested( + [unpairedOf(left, pairedLeft), unpairedOf(right, pairedRight)], + (member) => member.rawSignatureKey, + ), + )) { + for (const item of group.left) usedLeft.add(item); + for (const item of group.right) usedRight.add(item); + ambiguous.push(group); + } return { pairs, ambiguous, @@ -238,6 +340,132 @@ const matchByKeys = ({ }; }; +/** Open signature choices contested by unresolved owners, including held rows. */ +const unresolvedSignatureAmbiguities = ( + left: readonly Keyed[], + right: readonly Keyed[], + contested: ReadonlySet, +): Ambiguous[] => { + const key = (member: Keyed): string => + competitionIdentityKey(member, member.rawSignatureKey) ?? ""; + const leftGroups = groupBy(left, key); + const rightGroups = groupBy(right, key); + const groups: Ambiguous[] = []; + for (const [signature, leftItems] of leftGroups) { + const rightItems = rightGroups.get(signature); + if (!contested.has(signature) || rightItems === undefined) continue; + groups.push({ + left: leftItems, + right: rightItems, + basis: "exact-signature", + }); + } + return groups; +}; + +interface Declared { + readonly declaring_type: string | null; + readonly name: string; +} + +const unpairedOf = ( + items: readonly Keyed[], + paired: ReadonlySet>, +): Keyed[] => items.filter((item) => !paired.has(item)); + +/** + * Round-specific evidence keys an unresolved owner competes for, paired with + * its retained member name. Another same-name member on the same side can be + * the unresolved row's owner; a different name cannot. Contested candidates + * stay in remap rounds rather than pairing by declared identity. + */ +const competitionIdentityKey = ( + member: Keyed, + evidenceKey: string | null, +): string | null => + evidenceKey === null ? null : JSON.stringify([member.item.name, evidenceKey]); + +const ownerlessContested = ( + sides: readonly (readonly Keyed[])[], + keyOf: (item: Keyed) => string | null, +): ReadonlySet => { + const contested = new Set(); + for (const side of sides) { + const seen = new Map(); + const ownerless = new Set(); + for (const member of side) { + const key = competitionIdentityKey(member, keyOf(member)); + if (key === null) continue; + seen.set(key, (seen.get(key) ?? 0) + 1); + if (member.item.declaring_type === null) ownerless.add(key); + } + for (const key of ownerless) + if ((seen.get(key) ?? 0) > 1) contested.add(key); + } + return contested; +}; + +/** + * Members a remap round may group: an unresolved owner, which keeps its group + * ambiguous; a member whose selected key an unresolved owner competes for, so the + * group that preserved the base matcher's ambiguity can form; or no member of + * the same declared identity left unpaired on the other side, whether + * one-sided or in an ambiguous group. The sets are taken before grouping, so + * no group order decides a remap. + */ +const remappable = ( + items: readonly Keyed[], + others: readonly Keyed[], + keyOf: (item: Keyed) => string | null, + contested: ReadonlySet, +): Keyed[] => { + const kept = new Set(others.map(({ identity }) => identity)); + return items.filter((member) => { + const key = keyOf(member); + return ( + (key !== null && + contested.has(competitionIdentityKey(member, key) ?? "")) || + member.item.declaring_type === null || + !kept.has(member.identity) + ); + }); +}; + +/** + * Whether a member may pair in a declared-identity round. A member with an + * unresolved owner may, only when no other member on either side shares its + * selected evidence key: its owner might be any of them, so a shared key + * leaves it to the remap rounds, which keep such a group ambiguous. The + * resolved members that share such a key stay out too, so they cannot pair and + * consume the candidate the ownerless member might own. + */ +const ownerlessDeclarable = ( + left: readonly Keyed[], + right: readonly Keyed[], + keyOf: (item: Keyed) => string | null, +): ((member: Keyed) => boolean) => { + const count = (side: readonly Keyed[]) => { + const seen = new Map(); + for (const member of side) { + const key = keyOf(member); + if (key !== null) seen.set(key, (seen.get(key) ?? 0) + 1); + } + return seen; + }; + const leftSeen = count(left); + const rightSeen = count(right); + const contested = ownerlessContested([left, right], keyOf); + return (member) => { + const key = keyOf(member); + return key !== null && + contested.has(competitionIdentityKey(member, key) ?? "") + ? false + : member.item.declaring_type !== null || + key === null || + ((leftSeen.get(key) ?? 0) <= 1 && (rightSeen.get(key) ?? 0) <= 1); + }; +}; + const groupBy = ( items: readonly Item[], keyOf: (item: Item) => string | null, @@ -266,7 +494,7 @@ const observedBy = * the other side's selection stays one-sided, so a type selection never yields * a remap to a member of a type the other side did not inspect. */ -const matchObserved = ( +const matchObserved = ( left: readonly Item[], right: readonly Item[], sides: { diff --git a/tests/acceptance/providers/hopperStartupRecovery.test.ts b/tests/acceptance/providers/hopperStartupRecovery.test.ts index f35ec68d66..bd4a2d0f1e 100644 --- a/tests/acceptance/providers/hopperStartupRecovery.test.ts +++ b/tests/acceptance/providers/hopperStartupRecovery.test.ts @@ -282,6 +282,8 @@ test.for(["handoff", "not-found"] as const)( const status = z.object({ result: z.object({ provider_operation_health: z.object({ + // A timeout observes incomplete startup; a failed launcher without + // confirmed ownership leaves the provider's state unknown. state: z.literal( launchMode === "handoff" ? "not_started" : "unknown", ), diff --git a/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts b/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts index 7866b978b0..5ffabdd59a 100644 --- a/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts +++ b/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts @@ -47,7 +47,7 @@ describe("managed member comparison", () => { }), ]); expect(result.limitations).toContain( - "Methods pair by exact CIL/signature, then declared type, name, and signature, then structural shape; names alone are never a matching basis.", + "Methods pair within one declared type and name by exact CIL/signature, then signature, then structural shape; only then may exact CIL/signature or structural shape alone remap across declared types or names, never taking a member whose own declared type and name remain unpaired on the other side or whose declaring type is unresolved. Names alone are never a matching basis.", ); expect( managedMemberComparisonResultSchema.safeParse({ @@ -339,10 +339,12 @@ describe("managed member comparison uncertainty", () => { const rightField = right.result.fields[0]; expect(rightField).toBeDefined(); if (rightField === undefined) return; + // Both right fields are renamed, so neither left field keeps its declared + // identity and only the shared signature links them. const duplicatedRight = { ...right.result, fields: [ - rightField, + { ...rightField, name: "moved" }, { ...rightField, token: "0x04000002", @@ -365,6 +367,9 @@ describe("managed member comparison uncertainty", () => { candidate_left_tokens: ["0x04000001", "0x04000002"], candidate_right_tokens: ["0x04000001", "0x04000002"], }), + limitations: [ + "Multiple managed fields share the same signature; REA did not guess a token remap.", + ], }), ]); }); @@ -582,7 +587,7 @@ describe("managed member comparison of ambiguous undecoded signatures", () => { basis: "exact-il-signature", }), limitations: [ - "Multiple managed methods share the same non-name identity key; REA did not guess a token remap.", + "Multiple managed methods share the same CIL and signature; REA did not guess a token remap.", ], }), expect.objectContaining({ @@ -670,6 +675,891 @@ describe("managed member comparison of selected types", () => { }); }); +describe("managed member comparison of one signature in several types", () => { + it("pairs a file with itself when two types hold identical members", () => { + const inspected = withTwin( + inspect(buildManagedPeFixture(), "/tmp/twin.dll"), + ); + const result = compareManagedMembers(inspected, inspected); + + expect(result.matching.ambiguous).toBe(0); + expect(result.summary).toMatchObject({ + unchanged: 4, + changed: 0, + added: 0, + removed: 0, + unknown: 0, + }); + for (const item of [...result.methods, ...result.fields]) + expect(item.right?.token).toBe(item.left?.token); + }); + + it.each([1, 2])( + "keeps a member with its own changed counterpart beside %i identical members of other types", + (twins) => { + const left = inspect(buildManagedPeFixture(), "/tmp/own-before.dll"); + const changed = withTwin( + inspect( + buildManagedPeFixture({ + ilBody: Buffer.from([ + 0x32, 0x02, 0x7b, 0x02, 0x00, 0x00, 0x04, 0x28, 0x02, 0x00, 0x00, + 0x0a, 0x2a, + ]), + }), + "/tmp/own-after.dll", + ), + left, + twins, + ); + const result = compareManagedMembers(left, changed); + + expect(result.methods).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: "changed", + left: expect.objectContaining({ + declaring_type: "Fixture.Program", + }), + right: expect.objectContaining({ + declaring_type: "Fixture.Program", + }), + match: expect.objectContaining({ basis: "exact-signature" }), + dimensions: expect.arrayContaining(["cil"]), + }), + expect.objectContaining({ + status: "added", + right: expect.objectContaining({ declaring_type: "Other.Twin" }), + }), + ]), + ); + expect(result.fields).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: "unchanged", + left: expect.objectContaining({ + declaring_type: "Fixture.Program", + }), + right: expect.objectContaining({ + declaring_type: "Fixture.Program", + }), + }), + expect.objectContaining({ + status: "added", + right: expect.objectContaining({ declaring_type: "Other.Twin" }), + }), + ]), + ); + expect(result.summary).toMatchObject({ unknown: 0, removed: 0 }); + }, + ); + + it("does not remap a member whose own identity remains on the other side", () => { + const left = inspect(buildManagedPeFixture(), "/tmp/sig-before.dll"); + const changed = withTwin( + inspect( + buildManagedPeFixture({ + methodSignature: Buffer.from([0x00, 0x00, 0x08]), + fieldSignature: Buffer.from([0x06, 0x0a]), + }), + "/tmp/sig-after.dll", + ), + left, + ); + // Either side may hold the member whose own identity is left behind. + const forward = compareManagedMembers(left, changed); + const backward = compareManagedMembers(changed, left); + + for (const item of [ + ...forward.methods, + ...forward.fields, + ...backward.methods, + ...backward.fields, + ]) + expect(item.match.status).not.toBe("matched"); + expect(forward.summary).toMatchObject({ removed: 2, added: 4 }); + expect(backward.summary).toMatchObject({ removed: 4, added: 2 }); + }); + + it("does not remap by structural shape a method whose own identity remains on the other side", () => { + const left = inspect(buildManagedPeFixture(), "/tmp/shape-before.dll"); + const changed = withTwin( + inspect( + buildManagedPeFixture({ + methodSignature: Buffer.from([0x00, 0x00, 0x08]), + }), + "/tmp/shape-after.dll", + ), + left, + ); + const [own, twin] = changed.result.methods; + if (own === undefined || twin === undefined) + throw new Error("Expected two methods"); + // Same body shape, different CIL digest: only the structural round links + // the left method to the twin. + const result = compareManagedMembers(left, { + ...changed, + result: { + ...changed.result, + methods: [ + own, + { + ...twin, + body: { ...twin.body, normalized_il_sha256: "0".repeat(64) }, + }, + ], + }, + }); + + for (const item of result.methods) + expect(item.match.status).not.toBe("matched"); + }); +}); + +describe("managed member comparison remaps around kept identities", () => { + it("does not hide a member whose own identity remains in an ambiguous remap group", () => { + const left = inspect(buildManagedPeFixture(), "/tmp/group-before.dll"); + const changed = withTwin( + inspect( + buildManagedPeFixture({ + methodSignature: Buffer.from([0x00, 0x00, 0x08]), + fieldSignature: Buffer.from([0x06, 0x0a]), + }), + "/tmp/group-after.dll", + ), + left, + 2, + ); + const result = compareManagedMembers(left, changed); + + expect(result.matching.ambiguous).toBe(0); + expect(result.summary).toMatchObject({ removed: 2, added: 6 }); + }); + + it("remaps to the one candidate whose own identity is not kept elsewhere", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/kept-base.dll"); + const changedSignature = inspect( + buildManagedPeFixture({ + methodSignature: Buffer.from([0x00, 0x00, 0x08]), + }), + "/tmp/kept-signature.dll", + ); + // Left: Fixture.Program.Main, plus Other.Twin.Main with another signature. + const left = withTwin(base, changedSignature); + // Right: Other.Twin.Main and Other.Twin3.Main, both identical to the + // left Fixture.Program.Main, which the right no longer has. + const twins = withTwin(base, base, 2); + const right = { + ...twins, + result: { ...twins.result, methods: twins.result.methods.slice(1) }, + }; + const result = compareManagedMembers(left, right); + + expect(result.methods).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + left: expect.objectContaining({ declaring_type: "Fixture.Program" }), + right: expect.objectContaining({ declaring_type: "Other.Twin3" }), + match: expect.objectContaining({ + status: "matched", + basis: "exact-il-signature", + }), + }), + ]), + ); + expect( + result.methods.filter(({ match }) => match.status === "ambiguous"), + ).toEqual([]); + }); + + it("names the shared structural shape of an ambiguous remap group", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/shape-group.dll"); + const twins = withTwin(base, base, 2); + // Two right methods share the left method's shape but not its CIL digest, + // and the right no longer declares the left method's identity. + const right = { + ...twins, + result: { + ...twins.result, + methods: twins.result.methods.slice(1).map((method) => ({ + ...method, + body: { ...method.body, normalized_il_sha256: "0".repeat(64) }, + })), + fields: [], + }, + }; + const left = { ...base, result: { ...base.result, fields: [] } }; + const result = compareManagedMembers(left, right); + + expect(result.methods).toEqual([ + expect.objectContaining({ + match: expect.objectContaining({ + status: "ambiguous", + basis: "structural-method-shape", + }), + limitations: [ + "Multiple managed methods share the same structural shape; REA did not guess a token remap.", + ], + }), + ]); + }); +}); + +describe("managed member comparison of unresolved and held identities", () => { + it("does not remap past an unresolved member that may be the counterpart", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/orphan-copy.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const left = { ...base, result: { ...base.result, fields: [] } }; + // The right keeps Main with an unresolved owner, and an identical copy + // in another type; either could be the left Main. + const right = { + ...base, + result: { + ...base.result, + methods: [ + { ...method, declaring_type: null, declaring_type_token: null }, + { + ...method, + declaring_type: "Other.Twin", + declaring_type_token: "0x02000003", + name: "Copy", + token: "0x06000002", + }, + ], + fields: [], + }, + }; + + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) { + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + } + }); +}); + +describe("managed member comparison of raw-signature contention", () => { + it("contests same-signature methods even when their bodies differ", () => { + const base = inspect( + buildManagedPeFixture(), + "/tmp/ownerless-signatures.dll", + ); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const changedBody = (hash: string, stack: number) => ({ + ...method.body, + normalized_il_sha256: hash.repeat(64), + max_stack: stack, + }); + const withOwnerless = { + ...base, + result: { + ...base.result, + fields: [], + methods: [ + { ...method, body: changedBody("a", 1) }, + { + ...method, + token: "0x06000002", + declaring_type: null, + declaring_type_token: null, + body: changedBody("b", 2), + }, + ], + }, + }; + const resolvedOnly = { + ...base, + result: { + ...base.result, + fields: [], + methods: [{ ...method, body: changedBody("c", 3) }], + }, + }; + for (const result of [ + compareManagedMembers(withOwnerless, resolvedOnly), + compareManagedMembers(resolvedOnly, withOwnerless), + ]) { + expect(result.methods).toHaveLength(1); + const [item] = result.methods; + if (item === undefined) + throw new Error("Expected uncertain signature group"); + expect(item).toMatchObject({ + status: "unknown", + match: { status: "ambiguous", basis: "exact-signature" }, + }); + expect(result.summary).toMatchObject({ + added: 0, + removed: 0, + unknown: 1, + }); + expect( + item.match.candidate_left_tokens.length + + item.match.candidate_right_tokens.length, + ).toBe(3); + } + }); + + it("contests ownerless undecoded fields by raw signature", () => { + const base = inspect( + buildManagedPeFixture({ fieldSignature: Buffer.from([0x06, 0x7f]) }), + "/tmp/ownerless-field-signatures.dll", + ); + const [field] = base.result.fields; + if (field === undefined) throw new Error("Expected a fixture field"); + expect(field.signature.parse_status).toBe("unsupported"); + const withOwnerless = { + ...base, + result: { + ...base.result, + methods: [], + fields: [ + field, + { + ...field, + token: "0x04000002", + declaring_type: null, + declaring_type_token: null, + }, + ], + }, + }; + const resolvedOnly = { + ...base, + result: { ...base.result, methods: [], fields: [field] }, + }; + for (const result of [ + compareManagedMembers(withOwnerless, resolvedOnly), + compareManagedMembers(resolvedOnly, withOwnerless), + ]) + expect( + result.fields.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + }); +}); + +describe("managed member comparison of consumed ownerless ambiguity", () => { + it("keeps a consumed ownerless ambiguity from authorizing a later remap", () => { + const base = inspect( + buildManagedPeFixture(), + "/tmp/consumed-ownerless.dll", + ); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const candidate = ( + raw: string, + token: string, + owner: string | null, + name: string, + stack: number, + ) => ({ + ...method, + token, + declaring_type: owner, + declaring_type_token: owner === null ? null : method.declaring_type_token, + name, + signature: { ...method.signature, raw_sha256: raw.repeat(64) }, + body: { + ...method.body, + normalized_il_sha256: raw.repeat(64), + max_stack: stack, + }, + }); + const left = { + ...base, + result: { + ...base.result, + fields: [], + methods: [ + candidate("a", "0x06000001", "T", "M", 1), + candidate("b", "0x06000002", null, "M", 1), + candidate("b", "0x06000003", "D", "D", 2), + ], + }, + }; + const right = { + ...base, + result: { + ...base.result, + fields: [], + methods: [ + candidate("c", "0x06000001", "U", "N", 1), + candidate("b", "0x06000002", null, "M", 1), + candidate("d", "0x06000003", "T", "M", 3), + ], + }, + }; + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) { + expect(result.matching.ambiguous).toBeGreaterThan(0); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + } + }); +}); + +describe("managed member comparison of retained ownership contention", () => { + it("keeps changed counterparts unknown after an ownerless candidate is held", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/held-ownerless.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const body = (hash: string, stack: number) => ({ + ...method.body, + il_sha256: hash.repeat(64), + normalized_il_sha256: hash.repeat(64), + max_stack: stack, + }); + const ownerless = { + ...method, + token: "0x06000002", + declaring_type: null, + declaring_type_token: null, + body: body("b", 2), + }; + const left = { + ...base, + result: { + ...base.result, + fields: [], + methods: [{ ...method, body: body("a", 1) }, ownerless], + }, + }; + const right = { + ...base, + result: { + ...base.result, + fields: [], + methods: [ + { ...method, body: body("c", 3) }, + { + ...method, + token: "0x06000002", + declaring_type: "Other.Twin", + name: "Copy", + body: ownerless.body, + }, + ], + }, + }; + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) { + expect(result.summary).toEqual({ + unchanged: 0, + changed: 0, + added: 0, + removed: 0, + unknown: 2, + }); + expect(result.methods).toEqual([ + expect.objectContaining({ + status: "unknown", + match: { + status: "ambiguous", + basis: "exact-il-signature", + confidence: "unknown", + candidate_left_tokens: ["0x06000002"], + candidate_right_tokens: ["0x06000002"], + }, + }), + expect.objectContaining({ + status: "unknown", + match: { + status: "ambiguous", + basis: "exact-signature", + confidence: "unknown", + candidate_left_tokens: ["0x06000001"], + candidate_right_tokens: ["0x06000001"], + }, + }), + ]); + } + }); +}); + +describe("managed member comparison of ownerless structural competitors", () => { + it("keeps structural competitors of an ownerless method ambiguous in both directions", () => { + const base = inspect( + buildManagedPeFixture(), + "/tmp/ownerless-shape-competitors.dll", + ); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const withSignature = (raw: string, token: string) => ({ + ...method, + token, + signature: { ...method.signature, raw_sha256: raw.repeat(64) }, + }); + const withOwnerless = { + ...base, + result: { + ...base.result, + methods: [ + withSignature("a", "0x06000001"), + { + ...withSignature("b", "0x06000002"), + declaring_type: null, + declaring_type_token: null, + }, + ], + fields: [], + }, + }; + const resolvedOnly = { + ...base, + result: { + ...base.result, + methods: [withSignature("c", "0x06000001")], + fields: [], + }, + }; + for (const result of [ + compareManagedMembers(withOwnerless, resolvedOnly), + compareManagedMembers(resolvedOnly, withOwnerless), + ]) { + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + } + }); +}); + +describe("managed member comparison of ownerless members", () => { + it("keeps an ownerless member ambiguous beside a resolved copy, as main does", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-copy.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + const left = { + ...base, + result: { ...base.result, methods: [ownerless], fields: [] }, + }; + // The right keeps the ownerless Foo and a resolved copy with the same + // CIL and signature; the left Foo's owner may be either one. + const right = { + ...base, + result: { + ...base.result, + methods: [ownerless, { ...method, token: "0x06000002" }], + fields: [], + }, + }; + + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) { + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + } + }); + + it("still pairs a lone ownerless member with its lone counterpart", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-lone.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + const side = { + ...base, + result: { ...base.result, methods: [ownerless], fields: [] }, + }; + + const result = compareManagedMembers(side, side); + + expect(result.matching.ambiguous).toBe(0); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toHaveLength(1); + }); + + it("keeps a resolved copy out of the declared round when an ownerless competitor is on the left, as main does", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-left.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + // The left holds the resolved Foo and an ownerless Foo with the same CIL + // and signature; the right holds only the resolved Foo. The ownerless + // Foo's owner may be the resolved one, so the three stay ambiguous. + const withOwnerless = { + ...base, + result: { ...base.result, methods: [method, ownerless], fields: [] }, + }; + const resolvedOnly = { + ...base, + result: { ...base.result, methods: [method], fields: [] }, + }; + + const result = compareManagedMembers(withOwnerless, resolvedOnly); + + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + }); + + it("keeps a resolved copy out of the declared round when an ownerless competitor is on the right, as main does", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-right.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + const withOwnerless = { + ...base, + result: { ...base.result, methods: [method, ownerless], fields: [] }, + }; + const resolvedOnly = { + ...base, + result: { ...base.result, methods: [method], fields: [] }, + }; + + const result = compareManagedMembers(resolvedOnly, withOwnerless); + + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + expect(result.summary).toMatchObject({ added: 0, removed: 0 }); + }); + + it("keeps an ownerless method ambiguous beside a resolved method of the same shape", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-shape.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const ownerless = { + ...method, + declaring_type: null, + declaring_type_token: null, + }; + // Raw signature digests differ, so only the decoded shape links them. + const reshaped = (raw: string, token: string) => ({ + signature: { ...method.signature, raw_sha256: raw.repeat(64) }, + token, + }); + const left = { + ...base, + result: { ...base.result, methods: [ownerless], fields: [] }, + }; + const right = { + ...base, + result: { + ...base.result, + methods: [ + { ...ownerless, ...reshaped("e", "0x06000001") }, + { ...method, name: "Bar", ...reshaped("d", "0x06000002") }, + ], + fields: [], + }, + }; + const result = compareManagedMembers(left, right); + + expect(result.matching.ambiguous).toBe(1); + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + }); +}); + +describe("managed member comparison of unresolved owners", () => { + it("does not pair members of unresolved types under different names", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/orphan-names.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const orphan = { declaring_type: null, declaring_type_token: null }; + const side = (name: string) => ({ + ...base, + result: { + ...base.result, + methods: [{ ...method, ...orphan, name }], + fields: [], + }, + }); + const result = compareManagedMembers(side("Main"), side("Renamed")); + + for (const item of result.methods) + expect(item.match.status).not.toBe("matched"); + }); + + it("keeps ownerless members that share a body ambiguous, as main does", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/orphan-self.dll"); + const [method] = base.result.methods; + const [field] = base.result.fields; + if (method === undefined || field === undefined) + throw new Error("Expected a fixture method and field"); + const orphan = { declaring_type: null, declaring_type_token: null }; + const inspected = { + ...base, + result: { + ...base.result, + methods: [ + { ...method, ...orphan, name: "p" }, + { ...method, ...orphan, name: "q", token: "0x06000002" }, + ], + fields: [ + { ...field, ...orphan, name: "a" }, + { ...field, ...orphan, name: "b", token: "0x04000002" }, + ], + }, + }; + const result = compareManagedMembers(inspected, inspected); + + // Their owners are unknown, so `p` may be `q`; names alone cannot tell. + expect(result.matching.ambiguous).toBe(2); + expect( + [...result.methods, ...result.fields].filter( + ({ match }) => match.status === "matched", + ), + ).toEqual([]); + }); + + it("does not remap a member whose own identity is in an ambiguous group", () => { + const undecoded = { methodSignature: Buffer.from([0x00, 0x00, 0x7f]) }; + const decoded = inspect(buildManagedPeFixture(), "/tmp/held-decoded.dll"); + const held = inspect(buildManagedPeFixture(undecoded), "/tmp/held.dll"); + const [undecodedMethod] = held.result.methods; + const [decodedMethod] = decoded.result.methods; + if (undecodedMethod === undefined || decodedMethod === undefined) + throw new Error("Expected fixture methods"); + // The right keeps two identical undecoded Fixture.Program.Main, which + // stay ambiguous, and a copy of the decoded Main in another type. + const left = { + ...held, + result: { + ...held.result, + methods: [{ ...undecodedMethod, token: "0x06000002" }, decodedMethod], + }, + }; + const right = { + ...held, + result: { + ...held.result, + methods: [ + undecodedMethod, + { ...undecodedMethod, token: "0x06000002" }, + { + ...decodedMethod, + declaring_type: "Other.Twin", + declaring_type_token: "0x02000003", + token: "0x06000003", + }, + ], + }, + }; + + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) + expect( + result.methods.filter(({ match }) => match.status === "matched"), + ).toEqual([]); + }); + + it("does not remap a member whose declaring type is unresolved", () => { + const base = inspect(buildManagedPeFixture(), "/tmp/orphan.dll"); + const [method] = base.result.methods; + if (method === undefined) throw new Error("Expected a fixture method"); + const left = { + ...base, + result: { + ...base.result, + methods: [ + { ...method, declaring_type: null, declaring_type_token: null }, + ], + fields: [], + }, + }; + const right = { + ...base, + result: { + ...base.result, + methods: [ + { ...method, declaring_type: "Other.Twin", token: "0x06000002" }, + ], + fields: [], + }, + }; + + for (const result of [ + compareManagedMembers(left, right), + compareManagedMembers(right, left), + ]) + for (const item of result.methods) + expect(item.match.status).not.toBe("matched"); + }); +}); + +/** + * Add copies of the fixture's method and field, each pair declared by another + * type, to an inspection. The copies come from `source`, so they can carry + * the bytes of a different build than the inspection they join. + */ +const withTwin = ( + inspected: ReturnType, + source: ReturnType = inspected, + count = 1, +): ReturnType => { + const [method] = source.result.methods; + const [field] = source.result.fields; + if (method === undefined || field === undefined) + throw new Error("Expected a fixture method and field"); + const rows = Array.from({ length: count }, (_, index) => index + 2); + const twin = (row: number) => ({ + declaring_type: row === 2 ? "Other.Twin" : `Other.Twin${String(row)}`, + declaring_type_token: `0x0200000${String(row + 1)}`, + }); + return { + ...inspected, + result: { + ...inspected.result, + methods: [ + ...inspected.result.methods, + ...rows.map((row) => ({ + ...method, + ...twin(row), + token: `0x0600000${String(row)}`, + })), + ], + fields: [ + ...inspected.result.fields, + ...rows.map((row) => ({ + ...field, + ...twin(row), + token: `0x0400000${String(row)}`, + })), + ], + }, + }; +}; + const inspect = (bytes: Buffer, path: string, typeNames?: string[]) => { const target = managedPeFixtureTarget(bytes, path); const result = inspectManagedMembersBytes(