chore(docs): synchronize main for browser skill review

Integrate the verified main revision without rewriting the contributor's history.

Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-13 22:40:54 +00:00
94 changed files with 7779 additions and 432 deletions
+39 -1
View File
@@ -12,12 +12,13 @@ concurrency:
permissions:
contents: read
actions: read
pull-requests: read
jobs:
report:
name: Collect recent completed workflow attempts
runs-on: ubuntu-latest
timeout-minutes: 5
timeout-minutes: 10
steps:
# Consume only API data and execute batch code from the trusted default branch.
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
@@ -27,6 +28,18 @@ jobs:
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- name: Cache npm package downloads
id: npm-cache
continue-on-error: true
uses: ./.github/actions/npm-cache
- name: Diagnose missing and startup-failed PR CI
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
CI_REPOSITORY: ${{ github.repository }}
CI_REPOSITORY_ID: ${{ github.repository_id }}
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: node scripts/ci/pr-ci-health.mjs
- name: Record queue, execution, and matched step timing
continue-on-error: true
env:
@@ -44,3 +57,28 @@ jobs:
if-no-files-found: error
overwrite: true
retention-days: 14
- name: Install dependencies for static failure attribution
id: failure-deps
if: ${{ !cancelled() && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }}
continue-on-error: true
run: npm ci
- name: Attribute supported static failures against main
if: ${{ !cancelled() && steps.failure-deps.outcome == 'success' && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }}
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
CI_REPOSITORY: ${{ github.repository }}
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
run: node scripts/ci/failure-report.mjs
- name: Upload CI health and failure diagnostics
if: ${{ !cancelled() }}
continue-on-error: true
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: ci-health-diagnostics
path: |
.cache/ci-pr-health/
.cache/ci-failures/
include-hidden-files: true
if-no-files-found: warn
retention-days: 14
+62 -11
View File
@@ -139,6 +139,12 @@ jobs:
- name: Validate workflow syntax and expressions
if: needs.changes.outputs.workflows == 'true'
run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 -shellcheck= -pyflakes=
- name: Record static task source and execution identity
continue-on-error: true
if: needs.changes.outputs.code == 'true'
env:
CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: npm run ci:static-context
- if: needs.changes.outputs.code == 'true'
run: npm run check:ci
- if: needs.changes.outputs.code == 'true'
@@ -167,6 +173,34 @@ jobs:
if-no-files-found: ignore
retention-days: 7
test-source:
name: Source tests
needs: changes
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- name: Cache npm package downloads
id: npm-cache
uses: ./.github/actions/npm-cache
- run: npm ci
- name: Run source tests without waiting for build
run: npm run test:fast -- --reporter=default --reporter=blob
env:
VITEST_BLOB_LABEL: source
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ !cancelled() }}
with:
name: vitest-report-source
path: .vitest/blob/*
include-hidden-files: true
if-no-files-found: error
retention-days: 7
test-shard:
name: Test shard (${{ matrix.shard }})
needs: [changes, build]
@@ -178,7 +212,9 @@ jobs:
# every shard's diagnostics and coverage. Passing PRs run the full suite.
fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }}
matrix:
shard: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6]
# Fast feedback spends the same six test jobs on one source group and
# five built groups; full qualification retains six complete shards.
shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/5","2/5","3/5","4/5","5/5"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
@@ -197,15 +233,16 @@ jobs:
run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
env:
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
- name: Run complete portable suite for fast feedback
- name: Run built-dependent tests for fast feedback
if: needs.changes.outputs.tier == 'fast'
run: npm run test:run -- --reporter=default --reporter=blob --shard=${{ matrix.shard }}
run: npm run test:built -- --reporter=default --reporter=blob --shard=${{ matrix.shard }}
env:
VITEST_BLOB_LABEL: built
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ !cancelled() }}
with:
name: vitest-report-${{ strategy.job-index }}
name: vitest-report-built-${{ strategy.job-index }}
path: .vitest/blob/*
include-hidden-files: true
if-no-files-found: error
@@ -219,11 +256,13 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
cache: npm
- name: Cache npm package downloads
id: npm-cache
uses: ./.github/actions/npm-cache
- run: npm ci
- name: Run reverse tools tests with coverage
run: npm run test:coverage:reverse-tools
@@ -316,7 +355,7 @@ jobs:
test:
name: Portable tests
needs: [changes, test-shard]
needs: [changes, test-source, test-shard]
if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }}
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -340,7 +379,9 @@ jobs:
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
if: >-
!cancelled() && needs.changes.outputs.code == 'true' &&
(needs.test-shard.result == 'success' || needs.test-shard.result == 'failure')
(needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' ||
(needs.changes.outputs.tier == 'fast' &&
(needs.test-source.result == 'success' || needs.test-source.result == 'failure')))
with:
path: .vitest/blob
pattern: vitest-report-*
@@ -368,7 +409,8 @@ jobs:
if: >-
!cancelled() && needs.changes.outputs.code == 'true' &&
needs.changes.outputs.tier == 'fast' &&
(needs.test-shard.result == 'success' || needs.test-shard.result == 'failure')
(needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' ||
needs.test-source.result == 'success' || needs.test-source.result == 'failure')
run: >-
npm exec -- vitest --merge-reports
--reporter=default
@@ -381,6 +423,8 @@ jobs:
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-timings.json') != ''
env:
SHARD_RESULT: ${{ needs.test-shard.result }}
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
SOURCE_RESULT: ${{ needs.test-source.result }}
MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }}
CI_VERIFY_TEST_INVENTORY: "true"
CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }}
@@ -413,7 +457,13 @@ jobs:
env:
CODE_REQUIRED: ${{ needs.changes.outputs.code }}
SHARD_RESULT: ${{ needs.test-shard.result }}
run: test "${CODE_REQUIRED}" != "true" || test "${SHARD_RESULT}" = "success"
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
SOURCE_RESULT: ${{ needs.test-source.result }}
run: |
if [ "${CODE_REQUIRED}" = "true" ]; then
test "${SHARD_RESULT}" = "success" || exit 1
test "${SOURCE_REQUIRED}" != "true" || test "${SOURCE_RESULT}" = "success"
fi
conformance-fixtures:
name: Source-built conformance fixtures
@@ -847,6 +897,7 @@ jobs:
build,
reverse-tools-quality,
test-shard,
test-source,
sqlite-runtime,
test,
conformance-fixtures,
+7 -5
View File
@@ -13,8 +13,8 @@ jobs:
name: Build Windows bridge security boundary
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- name: Install the cross compiler used by the Windows native lane
@@ -44,11 +44,13 @@ jobs:
- platform: Windows
runner: [self-hosted, cutter-upstream, Windows]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
cache: npm
- name: Cache npm package downloads
id: npm-cache
uses: ./.github/actions/npm-cache
- run: npm ci
- if: matrix.platform == 'Windows'
uses: actions/download-artifact@v4
+1
View File
@@ -63,6 +63,7 @@ export default defineConfig({
{ text: "Ghidra DOS analysis", link: "/ghidra-dos" },
{ text: "Ghidra raw binary", link: "/raw-binary" },
{ text: "Ghidra NativeAOT", link: "/ghidra-nativeaot" },
{ text: "Ghidra PowerPC", link: "/ghidra-ppc" },
{ text: "Ghidra semantics", link: "/ghidra-provider" },
],
},
+90
View File
@@ -0,0 +1,90 @@
# PowerPC ELF analysis through Ghidra
This lane extends native ELF analysis to 32-bit big-endian PowerPC; it does
not execute PowerPC code. It is the first milestone of
[#1741](https://github.com/morluto/rea/issues/1741). GameCube/Wii DOL
executables, PPC64 and other PowerPC variants are separate follow-ups.
## Admission boundary
ELF `EM_PPC` (20) is retained as the provider-neutral `powerpc` family. The
target also retains class, byte order, `e_type` and raw `e_flags`. Recognition
does not mean every provider or PowerPC variant is supported. `EM_PPC64` (21)
is not recognized as this family.
The Ghidra lane admits ELF32 `ET_EXEC`, big endian. ELF64, little-endian,
relocatable and shared objects are refused with a specific unsupported
constraint. `EF_PPC_EMB` declares an embedded variant and is refused; the
remaining `e_flags` bits only declare relocation conventions and do not change
instruction decoding.
## Bound language
Ghidra's ELF loader selects `PowerPC:BE:32:e500:default` for a plain `EM_PPC`
header with `e_flags` 0. e500 is the embedded Book-E variant whose SPE unit
replaces the classic floating-point unit, so it misinterprets ordinary
750-family code. Nothing in the header distinguishes the two.
The lane therefore imports with `PowerPC:BE:32:default` and compiler spec
`default`, commits that choice in the analysis profile as
`language_selection: "provider-pinned-v1"`, and checks the language Ghidra
reports after import. A different language set through
`REA_GHIDRA_LANGUAGE_ID` (and optionally `REA_GHIDRA_COMPILER_SPEC_ID`)
replaces the lane default and is admitted against the installed language
catalog as usual; for example `PowerPC:BE:32:e500` for genuine e500 code.
PowerPC header facts are committed in the analysis profile so interpretation
changes invalidate profile-bound snapshots. The support-lane identity is
versioned independently of file identity. Evidence, lifecycle outputs and saved
snapshots preserve the `powerpc` family. Hopper and IDA adapters do not
implicitly gain PowerPC support. Windows Ghidra P0 remains PE x86/x86-64 only.
## Use
With a caller-installed Ghidra and compatible JDK on a supported Linux/macOS host:
```bash
rea analyze /absolute/path/to/ppc.elf --provider ghidra --json
rea function /absolute/path/to/ppc.elf function_name --provider ghidra --json
```
MCP uses the existing tools, including `open_binary` with an absolute path and
`provider_id: "ghidra"`. No PowerPC-specific tool catalog or emulator is
introduced.
## Verification
The source-owned freestanding fixture is `tests/conformance/c/ppc.c`. The
optional cross-target lane requires a caller-supplied Clang with the PowerPC
target, LLD, GNU `readelf`, Ghidra and its compatible JDK; ordinary host-native
checks do not acquire these tools. `REA_PPC_CLANG` selects Clang and
`REA_PPC_READELF` selects GNU readelf (for example
`/opt/homebrew/opt/binutils/bin/readelf` on macOS). These are test-time
prerequisites, not REA runtime providers.
```bash
GHIDRA_INSTALL_DIR=/absolute/path/to/ghidra npm run verify:ghidra:ppc
```
The lane compiles the fixture for a 750-family core (`-mcpu=750`) and never
executes it. GNU readelf independently supplies header facts and symbol
addresses, compared with the production REA target resolver before provider
startup. Its version, bounded command output and selected tool path are
retained in the verification report. The parser deliberately recognizes only
this fixture's GNU output, rejecting missing, ambiguous or unexpected fields
rather than substituting defaults.
Existing public MCP operations check Ghidra's **observed** language and
compiler specification and loaded-file hashes; an e500 import fails the lane.
The source-owned `ppc-probe.S` supplies fixed `addi` and conditional-branch
encodings: verification checks bytes, length, immediate and direct destination,
independent of decompiler variable names. Global and string bytes are checked at
independently reported symbol addresses.
CLI/MCP discovery and decompilation, direct calls, target/profile/Evidence
identity, snapshot round-trip and final process cleanup remain covered.
Pseudocode presence is a liveness check, not proof of semantic equivalence.
Instruction decoding and reader agreement do not establish runtime or complete
floating-point semantics. Parser/profile tests are narrower checks, not
substitutes for this real-provider run. Report the actual lanes run and leave
unexecuted verification pending.
+21 -13
View File
@@ -209,19 +209,27 @@ native/runtime questions retain their own authority and coverage.
Names and semantic labels do not establish durable member identity. Comparison
preserves observed differences, structural inferences, and unknowns separately.
Cross-version matching first prefers exact raw CIL/signature identity, with
equal decoded fingerprints. Equal decoded fingerprints alone cannot establish
an exact match: they omit switch targets and collapse distinct NaN encodings.
When that does not match, it pairs an exact declared type, method name, and raw
signature before trying structural body shape. The exact-signature key uses names only
as part of that full tuple; names alone never select a pair. Duplicate tuples
remain ambiguous rather than being paired by token order. Fields follow the
same exact tiers. The raw signature is exact whether or not REA decoded it, so
undecoded signatures pair by that tuple too, but they never enter structural
rounds. An unmatched member is therefore reported as `unknown`, not added or
removed, when its own signature was not decoded or when the other side has an
unpaired member, one-sided or ambiguous, with the same declared type and name
whose signature was not decoded.
Cross-version matching first pairs members of the same declared type and name:
methods by exact CIL/signature identity, then by raw signature, then by
structural body shape; fields by raw signature. Exact CIL/signature identity
requires equal raw CIL digests and decoded fingerprints. Equal decoded
fingerprints alone cannot establish an exact match: they omit switch targets
and collapse distinct NaN encodings. Only then may a remap pair a
renamed or moved member across declared types or names: a method by exact
CIL/signature identity or structural body shape, a field by exact decoded
signature. A remap never takes a member whose own declared type and name remain
unpaired on the other side, so a member is never taken from its own changed
counterpart. A member whose declaring type is unresolved could belong to any
type, so it pairs by name only when no other member shares its exact
CIL/signature key, never by structural shape alone, and any remap group it
joins stays ambiguous. The exact-signature key uses names only as part of that
full tuple; names alone never select a pair. Duplicate tuples remain ambiguous
rather than being paired by token order. The raw signature is exact whether or
not REA decoded it, so undecoded signatures pair by that tuple too, but they
never enter structural rounds. An unmatched member is therefore reported as
`unknown`, not added or removed, when its own signature was not decoded or when
the other side has an unpaired member, one-sided or ambiguous, with the same
declared type and name whose signature was not decoded.
For a matched method, an unavailable or partial body makes body-shape facets
unknown while preserving observed signature differences. For complete bodies,
+33 -2
View File
@@ -280,8 +280,15 @@ focused suite also exercises affected MCP routes and the Python Cutter bridge.
Contract, integration, and real-provider lanes complement these focused tests;
simulated providers do not establish real-provider behavior.
CI runs six native Vitest shards without retries. Each shard emits a blob
report; the merge job produces JUnit and JSON diagnostics, a module timing
Fast CI starts the source-only projects after classification, independently of
build and static checks, and runs five built-dependent shards after the build.
The six test jobs still select every portable test. Full qualification keeps six
complete Vitest shards with coverage. Neither tier retries tests. Canonical
project ownership lives in `scripts/lib/test-projects.mjs`, shared by Vitest,
CI group execution, and development-test preparation. A test importing compiled
verifier code belongs to the built group even if its directory is conformance.
Source and built blobs have separate labels to prevent merge collisions.
Each test job emits a blob report; the merge job produces JUnit and JSON diagnostics, a module timing
report, plus aggregate coverage during full qualification. It retains available
reports even when a shard fails, uploads available diagnostics, and writes the
slowest files to the workflow summary. Partial reports are marked diagnostic
@@ -316,6 +323,30 @@ portable test shard. These limits leave headroom for cold setup and full coverag
while preventing a hung job from holding the active main run for hours. They do
not bound runner queue waiting. Timeouts still fail the required gate.
The separate `CI timings` workflow also diagnoses missing, startup-failed,
queued/running, approval-required, canceled, and failed PR CI through bounded,
read-only API scans. Incomplete pagination or changing identity stays unknown.
It never closes/reopens PRs, reruns workflows, posts comments, or changes gates.
The timing manifest retains each run's event and head branch, including explicit
unknowns, so PR runs cannot consume the eligible-main comparison budget.
Its static failure report compares supported lint/duplication diagnostics with
failed main runs using exact source, ancestry, task hashes, and toolchain proof.
A match describes those diagnostics only; it does not establish that every
failure in a job came from main. Duplication reports retain the actual checker
exit result, so passing duplication checks preserve lint failures and caller
thresholds remain owned by the producer. Missing checker outcomes leave
those duplication diagnostics unclassified. Missing historical receipts remain unknown.
The reporter executes only trusted default-branch code and parses whitelisted
JSON artifact members without executing artifact content. Reports and limits
are retained in `ci-health-diagnostics`; its ten-minute job limit covers the
existing four-minute timing collection plus the bounded diagnostic stages.
Shard-count reduction remains deferred: the available successful timing sample
covers an older inventory, and file phase weights are not job wall time. The
one-source/five-built split retains six test jobs; it makes no hosted speedup
claim. Current complete timings and completed candidate hosted runs are needed
before enabling a smaller adaptive matrix.
The separate `CI timings` workflow batches recent completed CI and Release runs
hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and
a normalized point-in-time report for each collected workflow attempt, plus a
+6 -2
View File
@@ -118,7 +118,9 @@
"precheck:pr": "npm run deps:check",
"check:pr": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos",
"precheck:ci": "npm run deps:check",
"check:ci": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos && npm run verify:architecture-guards",
"check:ci": "node scripts/run-exclusive.mjs artifacts npm run check:ci:tasks && npm run verify:architecture-guards",
"check:ci:tasks": "turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos",
"ci:static-context": "node scripts/ci/static-task-context.mjs",
"build:cached": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills",
"build:termux": "node scripts/run-exclusive.mjs artifacts npm run build:termux:run",
"build:termux:run": "npm run compile:termux && npm run artifacts:skills",
@@ -138,7 +140,8 @@
"test:run": "vitest run",
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v",
"test:local": "node scripts/run-development-tests.mjs local",
"test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation",
"test:fast": "node scripts/ci/test-group.mjs source",
"test:built": "node scripts/ci/test-group.mjs built",
"pretest:boundary": "npm run test:prepare",
"test:boundary": "vitest run --project boundary --project cli-boundary --project process-boundary --project mcp-boundary --project process-global",
"pretest:mcp": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills artifacts:mcp-catalog",
@@ -183,6 +186,7 @@
"verify:ghidra": "npm run build:cached && node scripts/verify-real-ghidra.mjs",
"verify:ghidra:seeds": "npm run build:cached && node scripts/verify/ghidra/seeds.mjs",
"verify:ghidra:mips": "npm run build:cached && node scripts/verify-real-ghidra-mips.mjs",
"verify:ghidra:ppc": "npm run build:cached && node scripts/verify-real-ghidra-ppc.mjs",
"verify:ghidra:riscv": "npm run build:cached && node scripts/verify-real-ghidra-riscv.mjs",
"verify:ghidra:6502": "npm run build:cached && node scripts/verify-real-ghidra-6502.mjs",
"verify:ghidra:mips-shared": "npm run build:cached && node scripts/verify-real-ghidra-mips-shared.mjs",
+67
View File
@@ -0,0 +1,67 @@
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
import type { StaticTaskContext } from "./static-task-context.mjs";
export type { StaticTaskContext } from "./static-task-context.mjs";
export interface StaticDiagnostic {
readonly kind: "lint" | "jscpd";
readonly path?: string;
readonly paths?: readonly string[];
readonly code?: string;
readonly message?: string;
readonly severity?: string;
}
export function classifyFailure(input: {
repository: string;
defaultBranch: string;
run: Record<string, unknown> & NormalizedWorkflowRun;
job: Record<string, unknown>;
diagnostics: readonly StaticDiagnostic[];
context: StaticTaskContext;
mainRun: Record<string, unknown> & NormalizedWorkflowRun;
mainJob: Record<string, unknown>;
mainDiagnostics: readonly StaticDiagnostic[];
mainContext: StaticTaskContext;
executionSourcesVerified: boolean;
ancestry: { status: string; mergeBaseSha: string | null };
blobs: Readonly<Record<string, boolean>>;
}): {
classification: "matching-main-diagnostics" | "unknown";
reason: string;
};
export function diagnosticPaths(item: StaticDiagnostic): readonly string[];
export function isFailedConclusion(
value: unknown,
): value is "failure" | "timed_out";
export function isEligibleMainRun(
run: Record<string, unknown>,
defaultBranch: string,
repository: string,
): boolean;
export function pullRequestBaseSha(
run: Record<string, unknown>,
repository: string,
): string | null;
export function safeSha(value: unknown): string | null;
export function validRepoPath(value: unknown): value is string;
export function selectStaticCheckArtifact(
page: unknown,
run: { id?: string | number; head_sha?: string },
runAttempt: number,
): unknown | null;
export function readStaticCheckArtifact(options: {
repository: string;
artifactId: number | string;
token: string;
fetchImpl?: typeof fetch;
}): Promise<{
context: StaticTaskContext;
diagnostics: readonly StaticDiagnostic[];
}>;
export function parseStaticArchive(archivePath: string): Promise<{
context: StaticTaskContext;
diagnostics: readonly StaticDiagnostic[];
}>;
+442
View File
@@ -0,0 +1,442 @@
import { execFile } from "node:child_process";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { isAbsolute, join } from "node:path";
import { promisify } from "node:util";
import { parseStaticTaskContext } from "./static-task-context.mjs";
const execute = promisify(execFile);
const MAX_ARCHIVE_BYTES = 32 * 1024 * 1024;
const MAX_MEMBER_BYTES = 4 * 1024 * 1024;
const MAX_DIAGNOSTICS = 10_000;
const API_ROOT = "https://api.github.com/repos";
const SHA = /^[0-9a-f]{40}$/iu;
const TASK_HASH = /^[a-f0-9]{16}$/u;
/** Compare parsed static diagnostics only after source, task, host, and ancestry proof. */
export function classifyFailure({
repository,
defaultBranch,
run,
job,
diagnostics,
context,
mainRun,
mainJob,
mainDiagnostics,
mainContext,
executionSourcesVerified,
ancestry,
blobs,
}) {
const failedGate = firstFailure([
() => [
!isCompletedFailure(run) ||
job?.status !== "completed" ||
!isFailedConclusion(job?.conclusion),
"run_or_job_not_a_completed_failure",
],
() => [
!isEligiblePullRun(run, repository) ||
!isEligibleMainRun(mainRun, defaultBranch, repository),
"run_event_or_repository_identity_unverified",
],
() => [
job?.name !== mainJob?.name || job?.name !== "Static checks",
"failed_job_not_the_same_static_gate",
],
() => [
!Array.isArray(diagnostics) ||
diagnostics.length === 0 ||
!Array.isArray(mainDiagnostics),
"structured_diagnostics_missing_or_invalid",
],
() => [
!validContext(context) || !validContext(mainContext),
"static_task_context_missing_or_invalid",
],
() => [
executionSourcesVerified !== true,
"checkout_sources_not_verified_against_api_commits",
],
() => [
!sameExecutionContext(context, mainContext),
"task_or_toolchain_identity_differs",
],
() => [
ancestry?.status !== "ahead" && ancestry?.status !== "identical",
"main_head_not_proven_ancestor_of_tested_base",
],
() => [
ancestry?.mergeBaseSha !== mainRun?.head_sha,
"main_head_is_not_the_verified_merge_base",
],
() => [
!sameTaskHashes(diagnostics, context, mainContext),
"owning_task_hash_differs",
],
]);
if (failedGate) return outcome("unknown", failedGate.reason);
const keys = diagnostics.map(diagnosticKey);
const mainKeys = new Set(mainDiagnostics.map(diagnosticKey).filter(Boolean));
if (keys.some((key) => key === null))
return outcome("unknown", "diagnostic_shape_not_supported");
if (!keys.every((key) => mainKeys.has(key)))
return outcome(
"unknown",
"main_run_does_not_contain_the_same_structured_diagnostic",
);
const blobStatus = blobIdentityStatus(diagnostics, blobs);
if (blobStatus !== "identical")
return outcome(
"unknown",
blobStatus === "different"
? "diagnostic_source_content_not_identical_at_main_and_tested_base"
: "diagnostic_source_blob_identity_unavailable",
);
return outcome(
"matching-main-diagnostics",
"supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
);
}
export function diagnosticPaths(item) {
return item.kind === "jscpd" ? item.paths : [item.path];
}
export function isFailedConclusion(value) {
return value === "failure" || value === "timed_out";
}
export function isEligibleMainRun(run, defaultBranch, repository) {
return (
isEligibleStaticRun(run) &&
run.repository.full_name === repository &&
run.head_branch === defaultBranch &&
["push", "schedule", "workflow_dispatch"].includes(run.event)
);
}
export function pullRequestBaseSha(run, repository) {
if (
run?.event !== "pull_request" ||
run.repository?.full_name !== repository ||
!Array.isArray(run.pull_requests) ||
run.pull_requests.length !== 1
)
return null;
const base = run.pull_requests[0].base;
return base?.repo?.full_name === repository && SHA.test(base?.sha ?? "")
? base.sha
: null;
}
export function safeSha(value) {
return typeof value === "string" && SHA.test(value)
? value.toLowerCase()
: null;
}
export function selectStaticCheckArtifact(page, run, runAttempt) {
if (runAttempt !== 1) return null;
if (
!Array.isArray(page?.artifacts) ||
!Number.isSafeInteger(page.total_count) ||
page.total_count !== page.artifacts.length ||
page.artifacts.length >= 100
)
return null;
const matches = page.artifacts.filter(
(artifact) => artifact.name === "static-check-reports",
);
if (matches.length !== 1) return null;
const [artifact] = matches;
if (
artifact.expired !== false ||
String(artifact.workflow_run?.id) !== String(run.id) ||
artifact.workflow_run?.head_sha !== run.head_sha
)
return null;
return artifact;
}
export function validRepoPath(value) {
return (
typeof value === "string" &&
value.length > 0 &&
!isAbsolute(value) &&
!value.includes("\\") &&
value
.split("/")
.every((part) => part !== "" && part !== "." && part !== "..") &&
!value.startsWith("/")
);
}
/** Download one named static report archive and parse only whitelisted JSON members. */
export async function readStaticCheckArtifact({
repository,
artifactId,
token,
fetchImpl = fetch,
}) {
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? ""))
throw new Error("artifact repository identity is invalid");
if (!token) throw new Error("artifact download requires a GitHub token");
const url = `${API_ROOT}/${repository}/actions/artifacts/${positiveId(artifactId)}/zip`;
let response = await fetchImpl(url, {
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28",
},
signal: AbortSignal.timeout(10_000),
redirect: "manual",
});
if ([301, 302, 303, 307, 308].includes(response.status)) {
const location = response.headers.get("location");
if (!location || new URL(location).protocol !== "https:")
throw new Error("static_artifact_redirect_invalid");
response = await fetchImpl(location, {
signal: AbortSignal.timeout(10_000),
redirect: "error",
});
}
if (!response.ok) throw new Error(`github_http_${response.status}`);
const bytes = await readLimitedBytes(response, MAX_ARCHIVE_BYTES);
const directory = await mkdtemp(join(tmpdir(), "rea-ci-static-artifact-"));
const archive = join(directory, "artifact.zip");
try {
await writeFile(archive, bytes, { flag: "wx" });
return await parseStaticArchive(archive);
} finally {
await rm(directory, { recursive: true, force: true });
}
}
/** Accept only the three named producer outputs; no archive paths are written to disk. */
export async function parseStaticArchive(archivePath) {
const { stdout: listing } = await execute("unzip", ["-Z1", archivePath], {
encoding: "utf8",
maxBuffer: 128 * 1024,
timeout: 5_000,
});
const allowed = new Set([
"oxlint.json",
"jscpd/jscpd-report.json",
"static-task-context.json",
]);
const members = new Map();
for (const name of listing.split(/\r?\n/u).filter(Boolean)) {
const prefix = ".cache/static-checks/";
const member = name.startsWith(prefix) ? name.slice(prefix.length) : name;
if (!allowed.has(member)) continue;
if (members.has(member))
throw new Error("static artifact contains duplicate report members");
const { stdout } = await execute("unzip", ["-p", archivePath, name], {
encoding: "buffer",
maxBuffer: MAX_MEMBER_BYTES,
timeout: 5_000,
});
members.set(member, JSON.parse(stdout.toString("utf8")));
}
const context = parseStaticTaskContext(
members.get("static-task-context.json"),
);
const diagnostics = normalizeDiagnostics(
members.get("oxlint.json"),
members.get("jscpd/jscpd-report.json"),
);
if (diagnostics.length > MAX_DIAGNOSTICS)
throw new Error("static diagnostic count exceeds the reporting limit");
return { context, diagnostics };
}
function normalizeDiagnostics(lint, duplicate) {
return [...normalizeLint(lint), ...normalizeDuplicates(duplicate)];
}
function normalizeLint(report) {
if (report === undefined) return [];
if (!Array.isArray(report?.diagnostics))
throw new Error("oxlint diagnostics are malformed");
return report.diagnostics
.filter((item) => item.severity === "error")
.map((item) => {
if (
typeof item.filename !== "string" ||
!repoPath(item.filename) ||
typeof item.code !== "string" ||
typeof item.message !== "string"
)
throw new Error("oxlint error diagnostic is malformed");
return {
kind: "lint",
path: item.filename,
code: item.code,
message: item.message,
severity: "error",
};
});
}
function normalizeDuplicates(report) {
if (report === undefined) return [];
if (
!Array.isArray(report?.duplicates) ||
!Number.isFinite(report.statistics?.total?.percentage) ||
report.statistics.total.percentage < 0
)
throw new Error("jscpd report is malformed");
// The producer owns the checker outcome; do not copy or infer its threshold.
const exitCode = report.reaCheck?.exitCode;
if (exitCode === undefined || exitCode === 0) return [];
if (!Number.isSafeInteger(exitCode) || exitCode < 0)
throw new Error("jscpd checker outcome is malformed");
if (exitCode !== 1) return [];
return report.duplicates.map((item) => {
const paths = [item.firstFile?.name, item.secondFile?.name].sort();
if (paths.some((path) => !repoPath(path)))
throw new Error("jscpd duplicate paths are malformed");
return { kind: "jscpd", paths };
});
}
async function readLimitedBytes(response, limit) {
const reader = response.body?.getReader();
if (!reader) throw new Error("static artifact response has no body");
const chunks = [];
let size = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
size += value.byteLength;
if (size > limit) {
await reader.cancel();
throw new Error("static artifact exceeds the download limit");
}
chunks.push(value);
}
} finally {
reader.releaseLock();
}
const result = new Uint8Array(size);
let offset = 0;
for (const chunk of chunks) {
result.set(chunk, offset);
offset += chunk.byteLength;
}
return result;
}
function repoPath(value) {
return (
typeof value === "string" &&
value.length > 0 &&
!value.startsWith("/") &&
!value.includes("\\") &&
value
.split("/")
.every((part) => part !== "" && part !== "." && part !== "..")
);
}
function positiveId(value) {
const id = Number(value);
if (!Number.isSafeInteger(id) || id < 1)
throw new Error("artifact ID must be a positive integer");
return String(id);
}
function diagnosticKey(item) {
if (
item?.kind === "lint" &&
validRepoPath(item.path) &&
typeof item.code === "string" &&
typeof item.message === "string" &&
item.severity === "error"
)
return JSON.stringify([
item.kind,
item.path,
item.code,
item.message,
item.severity,
]);
if (
item?.kind === "jscpd" &&
Array.isArray(item.paths) &&
item.paths.length === 2 &&
item.paths.every(validRepoPath)
)
return JSON.stringify([item.kind, ...[...item.paths].sort()]);
return null;
}
function blobIdentityStatus(diagnostics, blobs) {
const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))];
if (paths.some((path) => blobs?.[path] === false)) return "different";
return paths.length > 0 && paths.every((path) => blobs?.[path] === true)
? "identical"
: "unavailable";
}
function sameTaskHashes(diagnostics, left, right) {
return diagnostics.every((diagnostic) => {
const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint:built";
return (
TASK_HASH.test(left.taskHashes[task] ?? "") &&
left.taskHashes[task] === right.taskHashes[task]
);
});
}
function validContext(value) {
return (
value?.version === 1 &&
SHA.test(value.sourceSha ?? "") &&
value.sourceClean === true &&
typeof value.nodeVersion === "string" &&
typeof value.npmVersion === "string" &&
typeof value.turboVersion === "string" &&
typeof value.platform === "string" &&
typeof value.architecture === "string" &&
value.taskHashes &&
typeof value.taskHashes === "object" &&
!Array.isArray(value.taskHashes)
);
}
function sameExecutionContext(left, right) {
return [
"nodeVersion",
"npmVersion",
"turboVersion",
"platform",
"architecture",
].every((key) => left[key] === right[key]);
}
function isEligiblePullRun(run, repository) {
return (
isEligibleStaticRun(run) && pullRequestBaseSha(run, repository) !== null
);
}
function isEligibleStaticRun(run) {
return (
isRecord(run) &&
run.status === "completed" &&
isFailedConclusion(run.conclusion) &&
SHA.test(run.head_sha ?? "") &&
typeof run.repository?.full_name === "string" &&
run.workflowPath?.file === ".github/workflows/ci.yml"
);
}
function isCompletedFailure(run) {
return run?.status === "completed" && isFailedConclusion(run.conclusion);
}
function isRecord(value) {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function firstFailure(checks) {
for (const check of checks) {
const [failed, reason] = check();
if (failed) return { reason };
}
return null;
}
function outcome(classification, reason) {
return { classification, reason };
}
+159
View File
@@ -0,0 +1,159 @@
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
import type { StaticDiagnostic } from "./artifacts.mjs";
export type { StaticDiagnostic } from "./artifacts.mjs";
import type { StaticTaskContext } from "./static-task-context.mjs";
export type { StaticTaskContext } from "./static-task-context.mjs";
export interface CiRun {
readonly id?: string | number;
readonly run_attempt?: number;
readonly status?: string;
readonly conclusion?: string | null;
readonly event?: string;
readonly path?: string;
readonly head_sha?: string;
readonly head_branch?: string;
readonly repository?: { readonly full_name?: string };
readonly pull_requests?: readonly {
readonly base?: {
readonly sha?: string;
readonly repo?: { readonly full_name?: string };
};
}[];
}
export interface CiJob {
readonly name?: string;
readonly status?: string;
readonly conclusion?: string | null;
readonly run_id?: number;
readonly run_attempt?: number;
}
export interface StaticEvidence {
readonly context: StaticTaskContext;
readonly diagnostics: readonly StaticDiagnostic[];
}
export interface CiFailureReport {
readonly version: 1;
readonly observedAt: string;
readonly repository: string;
readonly defaultBranch: string;
readonly informationalOnly: true;
readonly gateEffect: string;
readonly boundedInput: {
readonly candidateRuns: number;
readonly limit: number;
readonly mainCandidateLimit: number;
readonly mainCandidateRuns: number;
readonly mainRunLookupLimit: number;
readonly manifestTruncated: boolean;
};
readonly counts: Readonly<
Record<"matching-main-diagnostics" | "unknown", number>
>;
readonly failures: readonly {
readonly runId: number | string;
readonly runAttempt: number;
readonly classification: "matching-main-diagnostics" | "unknown";
readonly reason: string;
readonly candidateFailures?: readonly {
readonly runId: number | string;
readonly runAttempt: number;
readonly reason: string;
}[];
readonly runHead?: string | null;
readonly event?: string | null;
readonly job?: string;
readonly conclusion?: string | null;
}[];
}
export interface FailureReportApi {
getRun(repository: string, id: string | number): Promise<CiRun>;
getJobs(
repository: string,
id: string | number,
attempt: number,
): Promise<readonly CiJob[]>;
getStaticEvidence(
repository: string,
run: CiRun,
job: CiJob,
): Promise<StaticEvidence | null>;
getAncestry(
repository: string,
ancestor: string,
descendant: string,
): Promise<{ status: string; mergeBaseSha: string | null }>;
verifyExecutionSource(
repository: string,
run: CiRun,
context: StaticTaskContext,
): Promise<boolean>;
getBlobIdentity(
repository: string,
ancestor: string,
descendant: string,
paths: readonly string[],
): Promise<Readonly<Record<string, boolean>>>;
}
export function classifyFailure(input: {
repository: string;
defaultBranch: string;
run: CiRun & NormalizedWorkflowRun;
job: CiJob;
diagnostics: readonly StaticDiagnostic[];
context: StaticTaskContext;
mainRun: CiRun & NormalizedWorkflowRun;
mainJob: CiJob;
mainDiagnostics: readonly StaticDiagnostic[];
mainContext: StaticTaskContext;
executionSourcesVerified: boolean;
ancestry: { status: string; mergeBaseSha: string | null };
blobs: Readonly<Record<string, boolean>>;
}): {
classification: "matching-main-diagnostics" | "unknown";
reason: string;
};
export function reportCiFailures(options: {
repository: string;
defaultBranch: string;
batchManifest: {
readonly repository: string;
readonly runs: readonly {
readonly id: number | string;
readonly runAttempt: number;
readonly workflow: string;
readonly event?: string | null;
readonly headBranch?: string | null;
readonly conclusion: string | null;
readonly status: string;
}[];
readonly truncation?: readonly unknown[];
readonly coverage?: { readonly exhaustive?: boolean };
readonly selection?: { readonly exhaustive?: boolean };
};
api: FailureReportApi;
outputDirectory?: string;
summaryPath?: string;
observedAt?: string;
}): Promise<CiFailureReport>;
export function createGitHubApi(options?: {
token?: string;
fetchImpl?: typeof fetch;
apiRequest?: (
method: string,
endpoint: string,
budget: { requests: number; startedAt: number; limitMs: number },
) => Promise<unknown>;
}): FailureReportApi;
export function writeFailureReport(
report: CiFailureReport,
outputDirectory?: string,
summaryPath?: string,
): Promise<{ jsonPath: string; markdownPath: string }>;
+533
View File
@@ -0,0 +1,533 @@
import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import {
classifyFailure,
diagnosticPaths,
isEligibleMainRun,
isFailedConclusion,
pullRequestBaseSha,
readStaticCheckArtifact,
safeSha,
selectStaticCheckArtifact,
validRepoPath,
} from "./artifacts.mjs";
import { createGhApiRequest } from "./github-api.mjs";
import { normalizeWorkflowRun } from "./workflow-run.mjs";
const OUTPUT_DIRECTORY = join(".cache", "ci-failures");
const BATCH_DIRECTORY = join(".cache", "ci-job-timings");
const MAX_JSON_BYTES = 8 * 1024 * 1024;
const MAX_FAILED_RUNS = 20;
const MAX_MAIN_CANDIDATES = 5;
const MAX_MAIN_RUN_LOOKUPS = 100;
const MAX_PATHS_PER_COMPARISON = 50;
const MAX_API_REQUESTS = 250;
export { classifyFailure } from "./artifacts.mjs";
/** Uses only failed CI candidates present in the supplied bounded timing manifest. */
export async function reportCiFailures({
repository,
defaultBranch,
batchManifest,
api,
outputDirectory = OUTPUT_DIRECTORY,
summaryPath,
observedAt = new Date().toISOString(),
}) {
validateRepository(repository);
if (typeof defaultBranch !== "string" || defaultBranch.length === 0)
throw new Error("defaultBranch is required");
if (
!batchManifest ||
batchManifest.repository !== repository ||
!Array.isArray(batchManifest.runs)
)
throw new Error("timing manifest repository or runs are invalid");
const failedRuns = batchManifest.runs.filter(
(entry) =>
entry.workflow === "CI" &&
entry.status === "collected" &&
isFailedConclusion(entry.conclusion),
);
const prRuns = failedRuns.filter((entry) => entry.event === "pull_request");
const candidates = prRuns.slice(0, MAX_FAILED_RUNS);
const mainEntries = failedRuns.filter(
(entry) =>
entry.headBranch === defaultBranch &&
["push", "schedule", "workflow_dispatch"].includes(entry.event),
);
const lookupRun = verifiedRunLookup(repository, api);
const mainCandidates = [];
for (const entry of mainEntries.slice(0, MAX_MAIN_RUN_LOOKUPS)) {
try {
const run = await lookupRun(entry);
if (isEligibleMainRun(run, defaultBranch, repository))
mainCandidates.push({
run,
getEvidence: mainEvidenceLookup(repository, api, entry, run),
});
if (mainCandidates.length === MAX_MAIN_CANDIDATES) break;
} catch {
// Unverified metadata cannot consume the eligible-main candidate budget.
}
}
const entries = [];
for (const entry of candidates) {
try {
entries.push(
...(await analyzeFailedRun({
repository,
defaultBranch,
entry,
run: await lookupRun(entry),
mainCandidates,
api,
})),
);
} catch (error) {
entries.push({
runId: entry.id,
runAttempt: entry.runAttempt,
classification: "unknown",
reason: errorCode(error),
});
}
}
const report = {
version: 1,
observedAt,
repository,
defaultBranch,
informationalOnly: true,
gateEffect: "none; this report never changes CI conclusions",
boundedInput: {
candidateRuns: candidates.length,
limit: MAX_FAILED_RUNS,
mainCandidateLimit: MAX_MAIN_CANDIDATES,
mainCandidateRuns: mainCandidates.length,
mainRunLookupLimit: MAX_MAIN_RUN_LOOKUPS,
manifestTruncated:
prRuns.length > MAX_FAILED_RUNS ||
batchManifest.truncation?.length > 0 ||
batchManifest.selection?.exhaustive === false ||
batchManifest.coverage?.exhaustive === false,
},
counts: Object.fromEntries(
["matching-main-diagnostics", "unknown"].map((category) => [
category,
entries.filter((item) => item.classification === category).length,
]),
),
failures: entries,
};
await writeFailureReport(report, outputDirectory, summaryPath);
return report;
}
async function analyzeFailedRun({
repository,
defaultBranch,
entry,
run,
mainCandidates,
api,
}) {
const jobs = await api.getJobs(repository, entry.id, entry.runAttempt);
if (jobs.some((job) => !hasJobIdentity(job, entry)))
throw new Error("workflow_job_identity_mismatch");
const failedJobs = jobs.filter(
(job) => job.status === "completed" && isFailedConclusion(job.conclusion),
);
if (failedJobs.length === 0)
return [
{
runId: entry.id,
runAttempt: entry.runAttempt,
classification: "unknown",
reason: "failed_jobs_not_present_in_api_snapshot",
},
];
const failures = [];
for (const job of failedJobs) {
let classification;
try {
const evidence = await api.getStaticEvidence(repository, run, job);
classification = await attributeToMain({
repository,
defaultBranch,
run,
job,
evidence,
mainCandidates,
api,
});
} catch (error) {
classification = outcome("unknown", errorCode(error));
}
failures.push({
runId: entry.id,
runAttempt: entry.runAttempt,
runHead: safeSha(run.head_sha),
event: run.event ?? null,
job: job.name,
conclusion: job.conclusion,
...classification,
});
}
return failures;
}
async function attributeToMain({
repository,
defaultBranch,
run,
job,
evidence,
mainCandidates,
api,
}) {
const baseSha = pullRequestBaseSha(run, repository);
if (baseSha === null || evidence === null)
return outcome("unknown", "no_matching_verified_main_failure");
let result = outcome("unknown", "no_matching_verified_main_failure");
const candidateFailures = [];
for (const candidate of mainCandidates) {
try {
const main = await compareCandidate({
repository,
defaultBranch,
run,
job,
evidence,
baseSha,
candidate,
api,
});
if (main !== null) result = main;
if (main?.classification === "matching-main-diagnostics") break;
} catch (error) {
candidateFailures.push({
runId: candidate.run.id,
runAttempt: candidate.run.run_attempt,
reason: errorCode(error),
});
}
}
return {
...result,
...(candidateFailures.length > 0 ? { candidateFailures } : {}),
};
}
async function compareCandidate({
repository,
defaultBranch,
run,
job,
evidence,
baseSha,
candidate,
api,
}) {
const mainRun = candidate.run;
const main = await candidate.getEvidence(job.name);
if (!main?.evidence) return null;
const { job: mainJob, evidence: mainEvidence } = main;
const ancestry = await api.getAncestry(repository, mainRun.head_sha, baseSha);
const paths = [
...new Set(evidence.diagnostics.flatMap(diagnosticPaths)),
].slice(0, MAX_PATHS_PER_COMPARISON);
const blobs = await api.getBlobIdentity(
repository,
mainRun.head_sha,
baseSha,
paths,
);
const executionSourcesVerified =
(await api.verifyExecutionSource(repository, run, evidence.context)) &&
(await api.verifyExecutionSource(
repository,
mainRun,
mainEvidence.context,
));
return classifyFailure({
repository,
defaultBranch,
run,
job,
diagnostics: evidence.diagnostics,
context: evidence.context,
mainRun,
mainJob,
mainDiagnostics: mainEvidence.diagnostics,
mainContext: mainEvidence.context,
executionSourcesVerified,
ancestry,
blobs,
});
}
// Each verified main run/attempt owns its lazy jobs and parsed evidence for this
// report only. Cache nulls and rejected reads too; retries cannot spend the budget
// repeatedly on an immutable observation that was unavailable or invalid.
function mainEvidenceLookup(repository, api, entry, run) {
let jobs;
const evidence = new Map();
return (name) => {
jobs ??= Promise.resolve().then(async () => {
const items = await api.getJobs(repository, entry.id, entry.runAttempt);
return items.some((item) => !hasJobIdentity(item, entry)) ? [] : items;
});
if (!evidence.has(name))
evidence.set(
name,
jobs.then(async (items) => {
const job = items.find(
(item) =>
item.name === name &&
item.status === "completed" &&
isFailedConclusion(item.conclusion),
);
if (!job) return null;
return {
job,
evidence: await api.getStaticEvidence(repository, run, job),
};
}),
);
return evidence.get(name);
};
}
// Selection and attribution share one immutable API observation per run/attempt.
function verifiedRunLookup(repository, api) {
const runs = new Map();
return (entry) => {
const key = `${positiveId(entry.id)}:${positiveId(entry.runAttempt)}`;
if (!runs.has(key))
runs.set(
key,
Promise.resolve().then(async () => {
const run = normalizeWorkflowRun(
await api.getRun(repository, entry.id),
);
if (!hasRunIdentity(run, entry))
throw new Error("workflow_run_identity_mismatch");
return run;
}),
);
return runs.get(key);
};
}
function hasRunIdentity(run, entry) {
return (
String(run?.id) === String(entry.id) &&
Number(run.run_attempt) === Number(entry.runAttempt) &&
run.workflowPath?.file === ".github/workflows/ci.yml"
);
}
function hasJobIdentity(job, entry) {
return (
String(job.run_id) === String(entry.id) &&
Number(job.run_attempt) === Number(entry.runAttempt)
);
}
export function createGitHubApi({
token = process.env.GH_TOKEN,
fetchImpl = fetch,
apiRequest = createGhApiRequest(),
} = {}) {
if (!token) throw new Error("GH_TOKEN is required");
const budget = { requests: 0, startedAt: Date.now(), limitMs: 60_000 };
const request = async (endpoint) => {
if (budget.requests >= MAX_API_REQUESTS)
throw new Error("github_api_request_budget_exhausted");
budget.requests += 1;
return await apiRequest("GET", endpoint, budget);
};
return {
async getRun(repository, id) {
return await request(
`repos/${repository}/actions/runs/${positiveId(id)}`,
);
},
async getJobs(repository, id, attempt) {
const page = await request(
`repos/${repository}/actions/runs/${positiveId(id)}/attempts/${positiveId(attempt)}/jobs?per_page=100&page=1`,
);
if (!Array.isArray(page.jobs) || page.jobs.length >= 100)
throw new Error("job_list_incomplete_or_malformed");
return page.jobs;
},
async getStaticEvidence(repository, run, job) {
if (job.name !== "Static checks") return null;
if (run.run_attempt !== 1) return null;
const page = await request(
`repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`,
);
const artifact = selectStaticCheckArtifact(page, run, run.run_attempt);
if (!artifact) return null;
return await readStaticCheckArtifact({
repository,
artifactId: artifact.id,
token,
fetchImpl,
});
},
async getAncestry(repository, ancestor, descendant) {
const comparison = await request(
`repos/${repository}/compare/${safeSha(ancestor)}...${safeSha(descendant)}`,
);
return {
status: comparison.status,
mergeBaseSha: safeSha(comparison.merge_base_commit?.sha),
};
},
async verifyExecutionSource(repository, run, context) {
const sourceSha = safeSha(context.sourceSha);
if (!sourceSha) return false;
const pr =
run.event === "pull_request"
? pullRequestBaseSha(run, repository)
: null;
if (
run.event === "pull_request" &&
(pr === null || context.prHeadSha !== run.head_sha)
)
return false;
const commit = await request(`repos/${repository}/commits/${sourceSha}`);
if (safeSha(commit.sha) !== sourceSha) return false;
if (sourceSha === run.head_sha) return true;
if (
pr === null ||
!Array.isArray(commit.parents) ||
commit.parents.length !== 2
)
return false;
return (
safeSha(commit.parents[0]?.sha) === pr &&
safeSha(commit.parents[1]?.sha) === run.head_sha
);
},
async getBlobIdentity(repository, ancestor, descendant, paths) {
const result = {};
for (const path of paths) {
if (!validRepoPath(path)) continue;
const [left, right] = await Promise.all([
getContentBlobSha(repository, ancestor, path, request),
getContentBlobSha(repository, descendant, path, request),
]);
result[path] = left === right;
}
return result;
},
};
}
async function getContentBlobSha(repository, ref, path, request) {
const encodedPath = path.split("/").map(encodeURIComponent).join("/");
const value = await request(
`repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
);
const sha = safeSha(value?.sha);
if (value?.type !== "file" || sha === null)
throw new Error("github_content_blob_identity_unavailable");
return sha;
}
function outcome(classification, reason) {
return { classification, reason };
}
function validateRepository(value) {
if (
typeof value !== "string" ||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)
)
throw new Error("repository must be owner/name");
}
function positiveId(value) {
if (!Number.isSafeInteger(Number(value)) || Number(value) < 1)
throw new Error(
"GitHub run and attempt IDs must be positive safe integers",
);
return String(Number(value));
}
function errorCode(error) {
return error instanceof Error ? error.message.slice(0, 160) : "unknown_error";
}
export async function writeFailureReport(
report,
outputDirectory = OUTPUT_DIRECTORY,
summaryPath,
) {
await mkdir(outputDirectory, { recursive: true });
const jsonPath = join(outputDirectory, "report.json");
const markdownPath = join(outputDirectory, "report.md");
const matchedDiagnostics = report.failures.filter(
(item) => item.classification === "matching-main-diagnostics",
);
const lines = [
"# CI failure attribution",
"",
"Informational only. This report never changes a CI conclusion or required gate.",
"",
`Observed ${report.observedAt}; ${String(matchedDiagnostics.length)} of ${String(report.failures.length)} failure reports contain supported diagnostics matching a failed static task on an ancestor main revision. This does not attribute other failures in the same job. Unknown means evidence was incomplete or did not match.`,
"",
];
for (const item of report.failures)
lines.push(
`- Run ${item.runId} attempt ${item.runAttempt}${item.job ? `, ${item.job}` : ""}: **${item.classification}** — ${item.reason}.`,
);
if (report.failures.length === 0)
lines.push(
"- No completed failed CI attempts were present in the supplied timing snapshot.",
);
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
await writeFile(markdownPath, `${lines.join("\n")}\n`, "utf8");
if (summaryPath)
await appendFile(summaryPath, `${lines.join("\n")}\n`, "utf8");
return { jsonPath, markdownPath };
}
async function main() {
const repository = process.env.CI_REPOSITORY;
const defaultBranch = process.env.CI_DEFAULT_BRANCH;
const batchPath =
process.env.CI_TIMING_MANIFEST ??
join(BATCH_DIRECTORY, "batch-manifest.json");
const batchManifest = JSON.parse(await readBoundedFile(batchPath));
const api = createGitHubApi();
await reportCiFailures({
repository,
defaultBranch,
batchManifest,
api,
outputDirectory: process.env.CI_FAILURES_DIRECTORY ?? OUTPUT_DIRECTORY,
summaryPath: process.env.GITHUB_STEP_SUMMARY,
});
}
async function readBoundedFile(path) {
const bytes = await readFile(path);
if (bytes.byteLength > MAX_JSON_BYTES)
throw new Error("timing manifest exceeds size limit");
return bytes.toString("utf8");
}
if (
process.argv[1] &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
) {
try {
await main();
} catch (error) {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
}
}
+16
View File
@@ -0,0 +1,16 @@
export interface GitHubApiBudget {
requests: number;
startedAt: number;
limitMs: number;
}
export type GitHubApiRequest = (
method: string,
endpoint: string,
budget: GitHubApiBudget,
) => Promise<unknown>;
export function createGhApiRequest(options?: {
env?: NodeJS.ProcessEnv;
now?: () => number;
}): GitHubApiRequest;
+33
View File
@@ -0,0 +1,33 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
const execFileAsync = promisify(execFile);
const MAX_BUFFER = 8 * 1024 * 1024;
const REQUEST_TIMEOUT_MS = 10_000;
/** Create a bounded GitHub CLI JSON transport with GET-only authority. */
export function createGhApiRequest({ env = process.env, now = Date.now } = {}) {
return async (method, endpoint, budget) => {
if (method !== "GET")
throw new Error(
"GitHub API transport supports read-only GET requests only",
);
const remaining = budget.limitMs - (now() - budget.startedAt);
if (remaining <= 0) throw new Error("lookup_time_budget_exhausted");
const { stdout } = await execFileAsync(
"gh",
["api", "--method", "GET", endpoint],
{
env,
encoding: "utf8",
maxBuffer: MAX_BUFFER,
timeout: Math.min(REQUEST_TIMEOUT_MS, remaining),
},
);
try {
return JSON.parse(stdout);
} catch (cause) {
throw new Error("github_api_malformed_json", { cause });
}
};
}
+3
View File
@@ -103,6 +103,9 @@ export async function collectJobTimingsBatch({
id: run.id,
runAttempt: run.run_attempt,
workflow: workflow.name,
event: typeof run.event === "string" ? run.event : null,
headBranch:
typeof run.head_branch === "string" ? run.head_branch : null,
conclusion: run.conclusion ?? null,
createdAt: run.created_at ?? null,
});
+95
View File
@@ -0,0 +1,95 @@
export interface PrCiHead {
repositoryId: number | null;
repository?: string | null;
ref: string | null;
sha: string | null;
}
export interface PrCiPullRequest {
number: number | null;
draft?: boolean;
identityValid: boolean;
mergeable?: boolean;
head: PrCiHead;
[key: string]: unknown;
}
export interface PrCiWorkflowRun {
id?: number;
run_attempt?: number;
event?: string;
head_branch?: string;
head_repository?: { id?: number | null } | null;
head_sha?: string;
status?: string;
conclusion?: string | null;
created_at?: string;
html_url?: string;
pull_requests?: Array<{
number?: number;
head?: { sha?: string; repo?: { id?: number | null } } | null;
}>;
[key: string]: unknown;
}
export interface PrCiClassification {
category: string;
reason: string;
runs: Array<Record<string, unknown>>;
}
export interface PrCiHealthReport {
schemaVersion: 1;
observedAt: string;
repository: string;
repositoryId: number | null;
defaultBranch: string;
lookbackDays: number;
scan: {
status: "complete" | "partial";
cutoff: string;
pullRequestPages: number;
workflowRunPages: number;
pullRequestInventoryComplete: boolean;
workflowRunInventoryComplete: boolean;
pullRequestDetailsFetched: number;
pullRequestDetailsLimit: number;
apiRequests: number;
apiRequestLimit: number;
errors: string[];
};
counts: Record<string, number>;
pullRequests: Array<PrCiPullRequest & { ci: PrCiClassification }>;
}
export function collectPrCiHealth(options: {
repository: string;
repositoryId?: number | string | null;
defaultBranch: string;
lookbackDays?: number;
now?: number;
apiRequest?: GitHubApiRequest;
}): Promise<PrCiHealthReport>;
export function classifyPrCi(input: {
pull: PrCiPullRequest;
runs: Array<{ run: PrCiWorkflowRun; basis: string }>;
runsComplete: boolean;
pullRequestInventoryComplete: boolean;
ambiguousRuns: Array<{ run: PrCiWorkflowRun; basis: string }>;
}): PrCiClassification;
export function matchRunToPullRequest(input: {
run: PrCiWorkflowRun;
pull: PrCiPullRequest;
identityCounts: Map<string, number>;
inventoryComplete?: boolean;
}): { kind: "none" } | { kind: "ambiguous" } | { kind: "match"; basis: string };
export function writePrCiHealthReport(
report: PrCiHealthReport,
outputDirectory?: string,
): Promise<{ jsonPath: string; markdownPath: string }>;
export function renderMarkdownSummary(report: PrCiHealthReport): string;
import type { GitHubApiRequest } from "./github-api.mjs";
+718
View File
@@ -0,0 +1,718 @@
import { appendFile, mkdir, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { createGhApiRequest } from "./github-api.mjs";
const OUTPUT_DIRECTORY = join(".cache", "ci-pr-health");
const PAGE_SIZE = 100;
const MAX_PULL_REQUEST_PAGES = 3;
const MAX_WORKFLOW_RUN_PAGES = 5;
const MAX_PULL_REQUEST_DETAILS = 40;
const MAX_API_REQUESTS =
MAX_PULL_REQUEST_PAGES + MAX_WORKFLOW_RUN_PAGES + MAX_PULL_REQUEST_DETAILS;
const LOOKBACK_DAYS = 14;
const MAX_LOOKBACK_DAYS = 30;
const MAX_TOTAL_TIME_MS = 60_000;
const SHA_PATTERN = /^[0-9a-f]{40}$/iu;
const COMPLETED_CATEGORIES = new Map([
["failure", "completed_failure"],
["timed_out", "completed_failure"],
["cancelled", "completed_canceled"],
["success", "completed_success"],
]);
/** Read workflow/PR state and conservatively distinguish absent CI from unknown. */
export async function collectPrCiHealth({
repository,
repositoryId,
defaultBranch,
lookbackDays = LOOKBACK_DAYS,
now = Date.now(),
apiRequest = createGhApiRequest(),
}) {
const identity = validateInputs({
repository,
repositoryId,
defaultBranch,
lookbackDays,
now,
});
const observedAt = new Date(now).toISOString();
const cutoff = now - identity.lookbackDays * 24 * 60 * 60 * 1_000;
const snapshot = await collectApiSnapshot({ apiRequest, identity, cutoff });
const classified = classifyPullRequests(snapshot);
const { pulls, runs, budget } = snapshot;
const errors = [pulls.error, runs.error, ...snapshot.detailsErrors].filter(
Boolean,
);
return {
schemaVersion: 1,
observedAt,
repository: identity.repository,
repositoryId: identity.repositoryId,
defaultBranch: identity.defaultBranch,
lookbackDays: identity.lookbackDays,
scan: {
status:
errors.length === 0 && pulls.complete && runs.complete
? "complete"
: "partial",
cutoff: new Date(cutoff).toISOString(),
pullRequestPages: pulls.pages,
workflowRunPages: runs.pages,
pullRequestInventoryComplete: pulls.complete,
workflowRunInventoryComplete: runs.complete,
pullRequestDetailsFetched: snapshot.detailsFetched,
pullRequestDetailsLimit: MAX_PULL_REQUEST_DETAILS,
apiRequests: budget.requests,
apiRequestLimit: MAX_API_REQUESTS,
errors,
},
counts: countCategories(classified),
pullRequests: classified,
};
}
async function collectApiSnapshot({ apiRequest, identity, cutoff }) {
const budget = {
requests: 0,
startedAt: Date.now(),
limitMs: MAX_TOTAL_TIME_MS,
};
const requestGet = async (endpoint) => {
if (budget.requests >= MAX_API_REQUESTS)
throw new Error("lookup_request_budget_exhausted");
if (Date.now() - budget.startedAt >= budget.limitMs)
throw new Error("lookup_time_budget_exhausted");
budget.requests += 1;
return await apiRequest("GET", endpoint, budget);
};
const pulls = await listOpenPullRequests({
requestGet,
identity,
});
const candidates = pulls.items.filter(
(pull) =>
!pull.draft &&
Number.isFinite(Date.parse(pull.createdAt ?? "")) &&
Date.parse(pull.createdAt ?? "") >= cutoff &&
pull.base.ref === identity.defaultBranch &&
(identity.repositoryId === null ||
pull.base.repositoryId === identity.repositoryId),
);
const runs = await listRecentWorkflowRuns({
requestGet,
identity,
cutoff,
});
const detailResult = await resolveMergeability({
candidates,
requestGet,
identity,
});
return {
budget,
pulls,
runs,
candidates,
detailsFetched: detailResult.fetched,
detailsErrors: detailResult.errors,
};
}
function classifyPullRequests(snapshot) {
const { candidates, pulls, runs } = snapshot;
const identityCounts = countHeadIdentities(pulls.items);
return candidates.map((pull) => {
const relevant = [];
const ambiguousRuns = [];
for (const run of runs.items) {
const match = matchRunToPullRequest({
run,
pull,
identityCounts,
inventoryComplete: pulls.complete,
});
if (match.kind === "match") relevant.push({ run, basis: match.basis });
if (match.kind === "ambiguous")
ambiguousRuns.push({ run, basis: "ambiguous_head_identity" });
}
return {
...pull,
ci: classifyPrCi({
pull,
runs: relevant,
runsComplete: runs.complete,
pullRequestInventoryComplete: pulls.complete,
ambiguousRuns,
}),
};
});
}
async function resolveMergeability({ candidates, requestGet, identity }) {
let detailsFetched = 0;
const errors = new Set();
for (const pull of candidates) {
if (pull.mergeable !== undefined) continue;
if (detailsFetched >= MAX_PULL_REQUEST_DETAILS) {
errors.add("pull_request_mergeability_lookup_budget_exhausted");
continue;
}
const result = await getPullRequestDetails({ requestGet, identity, pull });
detailsFetched += 1;
if (result.error !== null) {
errors.add(result.error);
continue;
}
const { detail } = result;
const identityChanged =
detail.headSha !== pull.head.sha ||
detail.headRef !== pull.head.ref ||
detail.headRepositoryId !== pull.head.repositoryId ||
detail.baseRef !== pull.base.ref ||
detail.baseRepositoryId !== pull.base.repositoryId;
if (identityChanged) {
pull.identityValid = false;
pull.identityChangedDuringScan = true;
errors.add("pull_request_identity_changed_during_scan");
} else {
pull.mergeable = detail.mergeable;
pull.detailFetched = true;
if (detail.mergeable === undefined)
errors.add("pull_request_mergeability_unknown");
}
}
return { fetched: detailsFetched, errors: [...errors] };
}
function countCategories(pulls) {
const categories = [
"missing",
"startup_failure",
"queued_pending_executing",
"approval_required",
"completed_success",
"completed_failure",
"completed_canceled",
"completed_other",
"merge_conflict",
"unknown",
"deferred",
];
return Object.fromEntries(
categories.map((category) => [
category,
pulls.filter(({ ci }) => ci.category === category).length,
]),
);
}
/** Pure status classifier; missing requires complete, identity-safe inventories. */
export function classifyPrCi({
pull,
runs,
runsComplete,
pullRequestInventoryComplete,
ambiguousRuns,
}) {
if (!pull.identityValid)
return result("unknown", "pull_request_head_identity_invalid", []);
const evidence = [...runs, ...ambiguousRuns];
if (pull.mergeable === false)
return result("merge_conflict", "github_reports_merge_conflict", evidence);
// The newest collected run is not proven current until pagination completes:
// an older-created run outside this snapshot may have been retried recently.
if (!runsComplete)
return result(
evidence.length > 0 ? "unknown" : "deferred",
evidence.length > 0
? "workflow_run_inventory_incomplete"
: "lookup_inventory_incomplete",
evidence,
);
const selection = newestRun(runs);
if (!selection.known)
return result("unknown", "workflow_run_order_unknown", evidence);
const latest = selection.latest;
if (
ambiguousRuns.some((candidate) => {
const order = latest ? compareRunEvidence(candidate, latest) : null;
return order === null || order >= 0;
})
)
return result("unknown", "workflow_run_identity_ambiguous", evidence);
if (latest !== undefined) {
const run = latest.run;
if (run.conclusion === "startup_failure")
return result("startup_failure", "workflow_startup_failure", evidence);
if (run.status === "waiting" || run.conclusion === "action_required")
return result(
"approval_required",
"workflow_waiting_for_approval_or_gate",
evidence,
);
if (["queued", "requested", "pending", "in_progress"].includes(run.status))
return result(
"queued_pending_executing",
`workflow_${run.status}`,
evidence,
);
if (run.status === "completed")
return result(
COMPLETED_CATEGORIES.get(run.conclusion) ?? "completed_other",
run.conclusion === "success"
? "workflow_succeeded"
: `workflow_${run.conclusion ?? "unknown_conclusion"}`,
evidence,
);
return result("unknown", "workflow_status_unrecognized", evidence);
}
if (!pullRequestInventoryComplete)
return result("deferred", "lookup_inventory_incomplete", []);
if (pull.mergeable !== true)
return result("unknown", "pull_request_mergeability_unknown", []);
return result("missing", "no_pull_request_event_run_for_current_head", []);
}
/** Match only a pull_request event proven to belong to this immutable PR head. */
export function matchRunToPullRequest({
run,
pull,
identityCounts,
inventoryComplete = true,
}) {
if (run?.event !== "pull_request") return { kind: "none" };
if (!pull.identityValid || !isRecord(run)) return { kind: "none" };
const association = matchExplicitPullAssociation(run, pull);
if (association === "match")
return { kind: "match", basis: "event_pr_number_and_head_sha" };
if (association === "ambiguous") return { kind: "ambiguous" };
if (association === "other" || !sameHeadRepositoryAndRef(run, pull))
return { kind: "none" };
if (!uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete))
return { kind: "ambiguous" };
if (isSha(run.head_sha) && run.head_sha.toLowerCase() === pull.head.sha)
return { kind: "match", basis: "event_unique_head_ref_and_repository_id" };
return { kind: "none" };
}
function matchExplicitPullAssociation(run, pull) {
const associations = Array.isArray(run.pull_requests)
? run.pull_requests
: [];
if (associations.length === 0) return "absent";
const matchingNumber = associations.filter(
(item) => item?.number === pull.number,
);
if (matchingNumber.length === 0) return "other";
const exactHead = matchingNumber.find(
(item) => item?.head?.sha === pull.head.sha,
);
if (exactHead === undefined) {
return matchingNumber.some(
(item) =>
typeof item?.head?.sha === "string" && item.head.sha !== pull.head.sha,
)
? "other"
: "absent";
}
const associatedHeadRepositoryId = positiveInteger(exactHead.head?.repo?.id);
const runHeadRepositoryId = positiveInteger(run.head_repository?.id);
if (associatedHeadRepositoryId === null || runHeadRepositoryId === null)
return "ambiguous";
if (
associatedHeadRepositoryId !== pull.head.repositoryId ||
runHeadRepositoryId !== pull.head.repositoryId
)
return "other";
const baseRefMatches =
typeof exactHead.base?.ref !== "string" ||
exactHead.base.ref === pull.base.ref;
const baseRepositoryId = positiveInteger(exactHead.base?.repo?.id);
const baseRepositoryMatches =
baseRepositoryId === null || baseRepositoryId === pull.base.repositoryId;
return baseRefMatches && baseRepositoryMatches ? "match" : "other";
}
function sameHeadRepositoryAndRef(run, pull) {
return (
run.head_branch === pull.head.ref &&
run.head_repository?.id === pull.head.repositoryId
);
}
function uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete) {
if (!inventoryComplete) return false;
const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
const unknownRepositoryKey = headIdentityKey(null, pull.head.ref);
return (
identityCounts.get(key) === 1 &&
(identityCounts.get(unknownRepositoryKey) ?? 0) === 0
);
}
function normalizePullRequest(raw, identity) {
const number = positiveInteger(raw?.number);
const base = normalizeBase(raw?.base);
const head = normalizeHead(raw?.head);
const draft = raw?.draft === true;
return {
number,
draft,
createdAt: typeof raw?.created_at === "string" ? raw.created_at : null,
url: typeof raw?.html_url === "string" ? raw.html_url : null,
base,
head,
mergeable: typeof raw?.mergeable === "boolean" ? raw.mergeable : undefined,
identityValid: hasValidPullIdentity({ raw, identity, number, base, head }),
};
}
function normalizeBase(raw) {
return {
ref: typeof raw?.ref === "string" ? raw.ref : null,
repositoryId: positiveInteger(raw?.repo?.id),
};
}
function normalizeHead(raw) {
return {
repositoryId: positiveInteger(raw?.repo?.id),
repository:
typeof raw?.repo?.full_name === "string" ? raw.repo.full_name : null,
ref: typeof raw?.ref === "string" ? raw.ref : null,
sha:
typeof raw?.sha === "string" && isSha(raw.sha)
? raw.sha.toLowerCase()
: null,
};
}
function hasValidPullIdentity({ raw, identity, number, base, head }) {
return (
number !== null &&
typeof raw?.draft === "boolean" &&
base.ref === identity.defaultBranch &&
(identity.repositoryId === null ||
base.repositoryId === identity.repositoryId) &&
head.repositoryId !== null &&
head.sha !== null &&
head.ref !== null
);
}
function countHeadIdentities(pulls) {
const counts = new Map();
for (const pull of pulls) {
if (typeof pull.head.ref !== "string") continue;
const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
counts.set(key, (counts.get(key) ?? 0) + 1);
}
return counts;
}
function headIdentityKey(repositoryId, ref) {
return `${String(repositoryId)}\u0000${ref}`;
}
function newestRun(runs) {
let latest = runs[0];
for (const candidate of runs) {
if (latest && (compareRunEvidence(candidate, latest) ?? 0) > 0)
latest = candidate;
}
const known = runs.every((candidate) => {
if (candidate === latest) return true;
const order = compareRunEvidence(candidate, latest);
return order !== null && order <= 0;
});
return { known, latest };
}
// Attempt ordering is meaningful within one run; different run IDs are not clocks.
function compareRunEvidence(left, right) {
const leftRun = left.run;
const rightRun = right.run;
const id = positiveInteger(leftRun.id);
if (id !== null && id === positiveInteger(rightRun.id)) {
const leftAttempt = positiveInteger(leftRun.run_attempt);
const rightAttempt = positiveInteger(rightRun.run_attempt);
if (leftAttempt === null || rightAttempt === null) return null;
if (leftAttempt !== rightAttempt) return leftAttempt - rightAttempt;
return leftRun.status === rightRun.status &&
leftRun.conclusion === rightRun.conclusion
? 0
: null;
}
// A retry may start long after its original creation; without an attempt
// clock, creation dates only order first attempts from different runs.
if (
positiveInteger(leftRun.run_attempt) !== 1 ||
positiveInteger(rightRun.run_attempt) !== 1
)
return null;
const time =
Date.parse(leftRun.created_at ?? "") -
Date.parse(rightRun.created_at ?? "");
return Number.isFinite(time) && time !== 0 ? time : null;
}
function result(category, reason, runs) {
return {
category,
reason,
runs: runs.map(({ run, basis }) => ({
id: positiveInteger(run.id),
attempt: positiveInteger(run.run_attempt),
status: typeof run.status === "string" ? run.status : null,
conclusion: typeof run.conclusion === "string" ? run.conclusion : null,
headSha: isSha(run.head_sha) ? run.head_sha.toLowerCase() : null,
createdAt: typeof run.created_at === "string" ? run.created_at : null,
url: typeof run.html_url === "string" ? run.html_url : null,
identityBasis: basis,
})),
};
}
async function listOpenPullRequests({ requestGet, identity }) {
const items = [];
let complete = false;
let pages = 0;
let error = null;
for (let page = 1; page <= MAX_PULL_REQUEST_PAGES; page += 1) {
try {
const endpoint = `repos/${identity.repository}/pulls?state=open&sort=created&direction=desc&per_page=${PAGE_SIZE}&page=${page}`;
const response = await requestGet(endpoint);
if (!Array.isArray(response))
throw new Error("pull_request_page_malformed");
pages += 1;
const normalized = response.map((raw) =>
normalizePullRequest(raw, identity),
);
if (
normalized.some(
(pull) => !Number.isFinite(Date.parse(pull.createdAt ?? "")),
)
)
throw new Error("pull_request_page_malformed");
items.push(...normalized);
if (response.length < PAGE_SIZE) {
complete = true;
break;
}
} catch (cause) {
error = safeErrorCode(cause);
break;
}
}
return { items, pages, complete, error };
}
async function listRecentWorkflowRuns({ requestGet, identity, cutoff }) {
const items = [];
let complete = false;
let pages = 0;
let error = null;
for (let page = 1; page <= MAX_WORKFLOW_RUN_PAGES; page += 1) {
try {
const cutoffDate = new Date(cutoff).toISOString().slice(0, 10);
const endpoint = `repos/${identity.repository}/actions/workflows/ci.yml/runs?event=pull_request&created=%3E%3D${cutoffDate}&per_page=${PAGE_SIZE}&page=${page}`;
const response = await requestGet(endpoint);
if (!isRecord(response) || !Array.isArray(response.workflow_runs))
throw new Error("workflow_run_page_malformed");
pages += 1;
if (!response.workflow_runs.every(isRecord))
throw new Error("workflow_run_page_malformed");
items.push(...response.workflow_runs);
if (response.workflow_runs.length < PAGE_SIZE) {
complete = true;
break;
}
} catch (cause) {
error = safeErrorCode(cause);
break;
}
}
return { items, pages, complete, error };
}
async function getPullRequestDetails({ requestGet, identity, pull }) {
if (pull.number === null)
return { detail: null, error: "pull_request_detail_identity_missing" };
try {
const response = await requestGet(
`repos/${identity.repository}/pulls/${pull.number}`,
);
if (!isRecord(response))
return { detail: null, error: "pull_request_detail_malformed" };
return {
detail: {
mergeable:
typeof response.mergeable === "boolean"
? response.mergeable
: undefined,
headSha: isSha(response.head?.sha)
? response.head.sha.toLowerCase()
: null,
headRef:
typeof response.head?.ref === "string" ? response.head.ref : null,
headRepositoryId: positiveInteger(response.head?.repo?.id),
baseRef:
typeof response.base?.ref === "string" ? response.base.ref : null,
baseRepositoryId: positiveInteger(response.base?.repo?.id),
},
error: null,
};
} catch (cause) {
return { detail: null, error: safeErrorCode(cause) };
}
}
function validateInputs({
repository,
repositoryId,
defaultBranch,
lookbackDays,
now,
}) {
if (
typeof repository !== "string" ||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
)
throw new Error(
"CI_REPOSITORY must be an owner/name repository identifier",
);
if (
typeof defaultBranch !== "string" ||
defaultBranch.length === 0 ||
/[\r\n]/u.test(defaultBranch)
)
throw new Error("CI_DEFAULT_BRANCH must be a nonempty branch name");
const parsedRepositoryId =
repositoryId === undefined || repositoryId === null || repositoryId === ""
? null
: positiveInteger(repositoryId);
if (
repositoryId !== undefined &&
repositoryId !== null &&
repositoryId !== "" &&
parsedRepositoryId === null
)
throw new Error("CI_REPOSITORY_ID must be a positive safe integer");
if (
!Number.isInteger(lookbackDays) ||
lookbackDays < 1 ||
lookbackDays > MAX_LOOKBACK_DAYS
)
throw new Error(
`lookbackDays must be an integer from 1 to ${String(MAX_LOOKBACK_DAYS)}`,
);
if (!Number.isFinite(now)) throw new Error("now must be a finite timestamp");
return {
repository,
repositoryId: parsedRepositoryId,
defaultBranch,
lookbackDays,
};
}
function positiveInteger(value) {
const number = typeof value === "number" ? value : Number(value);
return Number.isSafeInteger(number) && number > 0 ? number : null;
}
function isSha(value) {
return typeof value === "string" && SHA_PATTERN.test(value);
}
function isRecord(value) {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function safeErrorCode(cause) {
if (cause?.message === "lookup_time_budget_exhausted")
return "lookup_time_budget_exhausted";
if (cause?.message === "lookup_request_budget_exhausted")
return "lookup_request_budget_exhausted";
if (cause?.message === "github_api_malformed_json")
return "github_api_malformed_json";
if (cause?.message?.endsWith("_malformed")) return cause.message;
if (cause?.code === "ETIMEDOUT" || cause?.killed) return "github_api_timeout";
return "github_api_request_failed";
}
export async function writePrCiHealthReport(
report,
outputDirectory = OUTPUT_DIRECTORY,
) {
const jsonPath = join(outputDirectory, "report.json");
const markdownPath = join(outputDirectory, "summary.md");
await mkdir(outputDirectory, { recursive: true });
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
await writeFile(markdownPath, renderMarkdownSummary(report), "utf8");
return { jsonPath, markdownPath };
}
export function renderMarkdownSummary(report) {
const rows = report.pullRequests.map(
(pull) =>
`| #${String(pull.number ?? "?")} | ${escapeCell(pull.head.repository ?? "unknown")} | ${escapeCell(pull.head.ref ?? "unknown")} | \`${pull.head.sha ?? "unknown"}\` | ${pull.ci.category} | ${escapeCell(pull.ci.reason)} |`,
);
return [
"# Pull request CI health snapshot",
"",
`Observed ${report.observedAt} for \`${report.repository}\`, target branch \`${escapeCell(report.defaultBranch)}\`; recent window ${String(report.lookbackDays)} days. Scan: **${report.scan.status}**.`,
"",
"This is read-only point-in-time telemetry. Missing is reported only when PR and workflow-run inventories are complete and the current immutable PR head identity is unambiguous. Incomplete lookups remain deferred or unknown.",
"",
`API pages: ${String(report.scan.pullRequestPages)} PR, ${String(report.scan.workflowRunPages)} workflow runs; ${String(report.scan.apiRequests)} bounded requests. Errors: ${report.scan.errors.length === 0 ? "none" : report.scan.errors.join(", ")}.`,
"",
"| PR | Head repository | Head ref | Head SHA | CI classification | Evidence |",
"| ---: | --- | --- | --- | --- | --- |",
...rows,
"",
"Categories: missing, startup failure, queued/pending/executing, approval required, completed outcomes, merge conflict, unknown, and deferred.",
"",
].join("\n");
}
function escapeCell(value) {
return String(value)
.replaceAll("|", "\\|")
.replaceAll("\n", " ")
.replaceAll("\r", " ");
}
if (
process.argv[1] !== undefined &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
) {
try {
const report = await collectPrCiHealth({
repository: process.env.CI_REPOSITORY,
repositoryId: process.env.CI_REPOSITORY_ID,
defaultBranch: process.env.CI_DEFAULT_BRANCH,
});
const paths = await writePrCiHealthReport(
report,
process.env.CI_PR_HEALTH_DIRECTORY ?? OUTPUT_DIRECTORY,
);
if (process.env.GITHUB_STEP_SUMMARY)
await appendFile(
process.env.GITHUB_STEP_SUMMARY,
renderMarkdownSummary(report),
"utf8",
);
process.stdout.write(
`Wrote ${report.pullRequests.length} recent PR CI records to ${paths.jsonPath}\n`,
);
} catch (cause) {
process.stderr.write(
`PR CI health telemetry failed: ${cause instanceof Error ? cause.message : "unknown_error"}\n`,
);
process.exitCode = 1;
}
}
+4 -3
View File
@@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { normalizeWorkflowRun } from "./workflow-run.mjs";
const ARTIFACT_NAME = "ci-full-qualification-v1";
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
@@ -119,8 +120,8 @@ function admit() {
// Artifact metadata narrows the search before run lookups. Never scan an
// unbounded history or treat a missing receipt as a successful qualification.
for (const artifact of candidates.slice(0, 3)) {
const run = api(
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
const run = normalizeWorkflowRun(
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
);
// An old run can upload a new artifact when only its gate is rerun.
// Bound the original run age as well so retries cannot renew stale proof.
@@ -133,7 +134,7 @@ function admit() {
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
run.path === ".github/workflows/ci.yml" &&
run.workflowPath?.file === ".github/workflows/ci.yml" &&
["schedule", "workflow_dispatch"].includes(run.event) &&
run.head_sha === head &&
run.head_branch === branch &&
+5 -1
View File
@@ -30,7 +30,11 @@ if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
throw new Error("Full qualification requires every canonical CI scope");
const required = reused ? ["changes"] : ["changes", "static", "test"];
const lanes = {
code: ["build", "test-shard"],
code: [
"build",
"test-shard",
...(outputs.tier === "fast" ? ["test-source"] : []),
],
reverse_tools: ["reverse-tools-quality"],
runtime: ["windows-native-build", "package-e2e"],
package: ["windows-curated"],
+2
View File
@@ -133,6 +133,8 @@ try {
for (const clone of report.duplicates)
for (const file of [clone.firstFile, clone.secondFile])
file.name = relative(process.cwd(), file.name).split(sep).join("/");
// Record the actual checker result, including caller-selected thresholds.
report.reaCheck = { exitCode: result.status };
await writeFile(reportPath, JSON.stringify(report, null, 2));
const { sources, clones, percentage } = report.statistics.total;
console.log(
+18
View File
@@ -0,0 +1,18 @@
export interface StaticTaskContext {
readonly version: 1;
readonly sourceSha: string;
readonly sourceClean: boolean;
readonly prHeadSha?: string;
readonly nodeVersion: string;
readonly npmVersion: string;
readonly turboVersion: string;
readonly platform: string;
readonly architecture: string;
readonly taskHashes: Readonly<Record<string, string>>;
}
export function parseStaticTaskContext(value: unknown): StaticTaskContext;
export function recordStaticTaskContext(options?: {
root?: string;
npmCli?: string;
outputDirectory?: string;
}): Promise<StaticTaskContext>;
+104
View File
@@ -0,0 +1,104 @@
import { execFile } from "node:child_process";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { z } from "zod";
const execute = promisify(execFile);
const contextSchema = z.object({
version: z.literal(1),
sourceSha: z.string().regex(/^[a-f0-9]{40}$/u),
sourceClean: z.boolean(),
prHeadSha: z
.string()
.regex(/^[a-f0-9]{40}$/u)
.optional(),
nodeVersion: z.string().min(1),
npmVersion: z.string().min(1),
turboVersion: z.string().min(1),
platform: z.string().min(1),
architecture: z.string().min(1),
taskHashes: z.record(z.string(), z.string().regex(/^[a-f0-9]{16}$/u)),
});
/** Parse the producing check graph's receipt; incomplete provenance is unknown. */
export const parseStaticTaskContext = (value) => contextSchema.parse(value);
/** Retain only named task hashes and tool/source identity, never the environment. */
export async function recordStaticTaskContext({
root = process.cwd(),
npmCli = process.env.npm_execpath,
outputDirectory = join(root, ".cache/static-checks"),
} = {}) {
if (typeof npmCli !== "string" || !npmCli)
throw new Error(
"Run static context recording through npm run ci:static-context",
);
const options = {
cwd: root,
encoding: "utf8",
timeout: 30_000,
maxBuffer: 32 * 1024 * 1024,
};
const [graph, source, status, npmVersion, turbo] = await Promise.all([
execute(
process.execPath,
[npmCli, "run", "--silent", "check:ci:tasks", "--", "--dry=json"],
options,
),
execute("git", ["rev-parse", "HEAD"], options),
execute(
"git",
["status", "--porcelain", "--untracked-files=normal"],
options,
),
execute(process.execPath, [npmCli, "--version"], options),
readFile(join(root, "node_modules/turbo/package.json"), "utf8"),
]);
const tasks = z
.object({
tasks: z
.array(
z.object({
task: z.string().min(1),
hash: z.string().regex(/^[a-f0-9]{16}$/u),
}),
)
.min(1),
})
.parse(JSON.parse(graph.stdout)).tasks;
if (new Set(tasks.map(({ task }) => task)).size !== tasks.length)
throw new Error("Static graph contains duplicate task identities");
const context = parseStaticTaskContext({
version: 1,
sourceSha: source.stdout.trim(),
sourceClean: status.stdout.length === 0,
...(process.env.CI_PR_HEAD_SHA
? { prHeadSha: process.env.CI_PR_HEAD_SHA }
: {}),
nodeVersion: process.version,
npmVersion: npmVersion.stdout.trim(),
turboVersion: z.object({ version: z.string() }).parse(JSON.parse(turbo))
.version,
platform: process.platform,
architecture: process.arch,
taskHashes: Object.fromEntries(tasks.map(({ task, hash }) => [task, hash])),
});
if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== context.sourceSha)
throw new Error(
"Static context source does not match the workflow revision",
);
await mkdir(outputDirectory, { recursive: true });
await writeFile(
join(outputDirectory, "static-task-context.json"),
JSON.stringify(context, null, 2) + "\n",
);
return context;
}
if (
process.argv[1] &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
)
await recordStaticTaskContext();
+46
View File
@@ -0,0 +1,46 @@
import { spawn } from "node:child_process";
import { resolve } from "node:path";
import {
BUILT_TEST_PROJECTS,
SOURCE_TEST_PROJECTS,
} from "../lib/test-projects.mjs";
const [group, ...arguments_] = process.argv.slice(2);
const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[
group
];
if (!projects) throw new Error("Expected source or built test group");
// The group owns project selection; callers can request reporters or shards.
if (
arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument))
)
throw new Error("Test groups cannot override their project inventory");
const child = spawn(
process.execPath,
[
resolve("node_modules/vitest/vitest.mjs"),
"run",
...projects.flatMap(({ name }) => ["--project", name]),
...arguments_,
],
{ stdio: "inherit" },
);
let shutdownSignal;
for (const signal of ["SIGINT", "SIGTERM"])
process.on(signal, () => {
shutdownSignal ??= signal;
child.kill(signal);
});
child.once("error", (error) => {
console.error(error.message);
process.exitCode = 1;
});
child.once("close", (code) => {
process.exitCode =
shutdownSignal === "SIGINT"
? 130
: shutdownSignal === "SIGTERM"
? 143
: (code ?? 1);
});
+4 -3
View File
@@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
import { appendFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { normalizeWorkflowRun } from "./workflow-run.mjs";
const ARTIFACT_NAME = "test-timings";
const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
@@ -68,8 +69,8 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
.slice(0, MAX_CANDIDATES);
for (const artifact of candidates) {
const run = api(
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
const run = normalizeWorkflowRun(
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
);
const runAge = now - Date.parse(run?.created_at);
if (
@@ -80,7 +81,7 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
run.path === ".github/workflows/ci.yml" &&
run.workflowPath?.file === ".github/workflows/ci.yml" &&
["schedule", "workflow_dispatch"].includes(run.event) &&
run.head_branch === branch &&
run.head_repository?.full_name === repository
+21 -5
View File
@@ -41,9 +41,20 @@ const files = report.files.map(({ name, durationMs, state }) => {
throw new Error(`Test report path is outside the checkout: ${name}`);
return { name: path.split(sep).join("/"), durationMs, state };
});
const complete = process.env.SHARD_RESULT === "success";
// Keep group outcomes intact: an unavailable group is not a completed failure.
const groupResults = [
process.env.SHARD_RESULT,
...(process.env.SOURCE_REQUIRED === "true"
? [process.env.SOURCE_RESULT]
: []),
];
const groupsCompleted = groupResults.every((state) =>
["success", "failure"].includes(state),
);
const passed =
complete && process.env.MERGE_RESULT === "success" && report.success;
groupResults.every((state) => state === "success") &&
process.env.MERGE_RESULT === "success" &&
report.success;
// Successful shards and merged reports must account for the complete portable
// inventory. Coverage thresholds alone cannot prove that every file ran.
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
@@ -69,7 +80,7 @@ if (process.env.CI_TEST_SELECTION) {
comparison.complete &&
report.unhandledErrors === 0 &&
["passed", "failed"].includes(report.reason) &&
["success", "failure"].includes(process.env.SHARD_RESULT) &&
groupsCompleted &&
["success", "failure"].includes(process.env.MERGE_RESULT),
executionSha: process.env.GITHUB_SHA ?? null,
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
@@ -91,7 +102,9 @@ if (passed && verifyInventory) {
);
}
const summary = [
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
groupsCompleted
? "## Merged test diagnostics"
: "## Partial test diagnostics",
"",
...(selection
? [
@@ -99,11 +112,14 @@ const summary = [
"",
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
"The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
"This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
"",
]
: []),
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
...(process.env.SOURCE_REQUIRED === "true"
? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`]
: []),
passed
? "Successful timing history is available."
: "This report is diagnostic only and is not a successful timing baseline.",
+10
View File
@@ -0,0 +1,10 @@
export interface WorkflowPath {
readonly file: string;
readonly ref: string | null;
}
export interface NormalizedWorkflowRun {
readonly workflowPath: WorkflowPath | null;
}
export function normalizeWorkflowRun<T extends Record<string, unknown>>(
value: T,
): Omit<T, "workflowPath"> & NormalizedWorkflowRun;
+15
View File
@@ -0,0 +1,15 @@
/** Keep the API path as evidence and parse its optional ref at the boundary. */
export function normalizeWorkflowRun(value) {
if (typeof value !== "object" || value === null || Array.isArray(value))
throw new Error("workflow_run_malformed");
const match =
typeof value.path === "string" && !/[\u0000-\u001f\u007f]/u.test(value.path)
? /^(\.github\/workflows\/[^/\\@\s]+\.ya?ml)(?:@([^\s]+))?$/u.exec(
value.path,
)
: null;
return {
...value,
workflowPath: match ? { file: match[1], ref: match[2] ?? null } : null,
};
}
@@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { TraceMap, eachMapping } from "@jridgewell/trace-mapping";
import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js";
import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { parseEvidence } from "../../dist/domain/evidence.js";
import { traceSourceMap } from "../../dist/javascript/sourceMaps/TraceSourceMap.js";
@@ -89,10 +90,12 @@ async function verifyCase(executable, entrypoint, scenario) {
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const provider = new CdpBrowserProvider();
const targets = await provider.listTargets({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
});
const targets = await provider.listTargets(
listBrowserTargetsInputSchema.parse({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
}),
);
if (!targets.ok) throw targets.error;
const target = targets.value.targets[0];
assert.ok(target, "Owned fixture target missing");
+7 -4
View File
@@ -8,6 +8,7 @@ import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js";
import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { parseEvidence } from "../../dist/domain/evidence.js";
import { webExecutionSchema } from "../../dist/domain/webExecution.js";
@@ -45,10 +46,12 @@ export async function verifyBrowserRuntime(
try {
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const targets = await new CdpBrowserProvider().listTargets({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
});
const targets = await new CdpBrowserProvider().listTargets(
listBrowserTargetsInputSchema.parse({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
}),
);
if (!targets.ok) throw targets.error;
const target = targets.value.targets[0];
assert.ok(target, "owned fixture page target missing");
+11 -20
View File
@@ -1,17 +1,4 @@
const sourceProjects = [
"domain",
"services",
"adapters",
"composition",
"conformance",
"evaluation",
];
const sourceTest = (path) =>
path.startsWith("src/") ||
["composition", "conformance", "evaluation"].some((project) =>
path.startsWith(`tests/${project}/`),
);
import { SOURCE_TEST_PROJECTS, projectForTest } from "./test-projects.mjs";
/** Parse explicit test paths and an optional branch-comparison base. */
export const parseDevelopmentTestRequest = (mode, arguments_) => {
@@ -35,10 +22,9 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => {
path.split("/").some((part) => part === ".." || part === ".")
)
throw new Error(`Expected an exact repository test path: ${argument}`);
if (mode !== "focused" && !sourceTest(path))
throw new Error(
`Use test:focused for compiled boundary tests: ${path}`,
);
const project = projectForTest(path);
if (mode !== "focused" && project.group !== "source")
throw new Error(`Use test:focused for built test projects: ${path}`);
paths.push(path);
}
}
@@ -72,9 +58,14 @@ export const developmentTestPlan = (request, baseCommit) => {
!/^[a-f0-9]{40,64}$/u.test(baseCommit ?? "")
)
throw new Error("Changed tests require a resolved Git merge base");
const projects = request.mode === "focused" ? [] : sourceProjects;
const projects =
request.mode === "focused"
? []
: SOURCE_TEST_PROJECTS.map(({ name }) => name);
return {
needsBuild: request.paths.some((path) => !sourceTest(path)),
needsBuild: request.paths.some(
(path) => projectForTest(path).group === "built",
),
artifactTasks: Object.entries(artifactConsumers).flatMap(
([task, consumers]) =>
request.paths.some((path) => consumers.includes(path)) ? [task] : [],
+10
View File
@@ -0,0 +1,10 @@
export interface TestProjectDefinition {
readonly name: string;
readonly group: "source" | "built";
readonly include: readonly string[];
readonly exclude?: readonly string[];
}
export const SOURCE_TEST_PROJECTS: readonly TestProjectDefinition[];
export const BUILT_TEST_PROJECTS: readonly TestProjectDefinition[];
export function projectForTest(testPath: string): TestProjectDefinition;
+107
View File
@@ -0,0 +1,107 @@
import { matchesGlob } from "node:path";
/** Canonical file ownership shared by Vitest and development-test planning. */
export const SOURCE_TEST_PROJECTS = [
{
name: "domain",
group: "source",
include: ["src/{contracts,domain}/**/*.test.ts"],
},
{
name: "services",
group: "source",
include: ["src/application/**/*.test.ts"],
},
{
name: "adapters",
group: "source",
include: [
"src/*.test.ts",
"src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
],
},
{
name: "composition",
group: "source",
include: ["tests/composition/**/*.test.ts"],
},
{
name: "conformance",
group: "source",
include: ["tests/conformance/**/*.test.ts"],
exclude: ["tests/conformance/providers/hopper/realHopperSearch.test.ts"],
},
{
name: "evaluation",
group: "source",
include: ["tests/evaluation/**/*.test.ts"],
},
];
export const BUILT_TEST_PROJECTS = [
{
name: "boundary",
group: "built",
include: [
"tests/boundary/**/*.test.ts",
"tests/conformance/providers/hopper/realHopperSearch.test.ts",
],
exclude: [
"tests/boundary/cli/**/*.test.ts",
"tests/boundary/mcp/**/*.test.ts",
"tests/boundary/process/**/*.test.ts",
],
},
{
name: "cli-boundary",
group: "built",
include: ["tests/boundary/cli/**/*.test.ts"],
},
{
name: "process-boundary",
group: "built",
include: ["tests/boundary/process/**/*.test.ts"],
},
{
name: "mcp-boundary",
group: "built",
include: ["tests/boundary/mcp/**/*.test.ts"],
},
{
name: "acceptance",
group: "built",
include: ["tests/acceptance/**/*.test.ts"],
},
{
name: "process-global",
group: "built",
include: ["tests/process-global/**/*.test.ts"],
},
];
const ALL_TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS];
/** Return the unique Vitest owner for an exact repository-relative test path. */
export const projectForTest = (testPath) => {
if (
typeof testPath !== "string" ||
testPath.startsWith("/") ||
testPath.includes("\\") ||
testPath.split("/").some((part) => part === ".." || part === ".") ||
!testPath.endsWith(".test.ts")
)
throw new Error(`Expected a normalized repository test path: ${testPath}`);
const owners = ALL_TEST_PROJECTS.filter(
({ include, exclude = [] }) =>
include.some((pattern) => matchesGlob(testPath, pattern)) &&
!exclude.some((pattern) => matchesGlob(testPath, pattern)),
);
if (owners.length !== 1)
throw new Error(
owners.length === 0
? `No test project owns ${testPath}`
: `Multiple test projects own ${testPath}: ${owners.map(({ name }) => name).join(", ")}`,
);
return owners[0];
};
+282
View File
@@ -0,0 +1,282 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdtemp, mkdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
import {
inspectPowerPcReadelf,
assertPowerPcLoadedImage,
assertPowerPcProbe,
assertPowerPcGlobal,
} from "../tests/conformance/ghidra/ppc-checks.mjs";
import { parseEvidence } from "../dist/domain/evidence.js";
import { parseAnalysisSnapshot } from "../dist/domain/analysisSnapshot.js";
import { functionDossierSchema } from "../dist/domain/hopperValues.js";
import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js";
import { buildPowerPcFixture } from "../tests/conformance/ghidra/ppc-fixture.mjs";
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
// Optional cross-target lane. No tool acquisition and no target execution.
if (process.argv.length !== 2)
throw new Error("Usage: node scripts/verify-real-ghidra-ppc.mjs");
if (!["linux", "darwin"].includes(process.platform))
throw new Error(
"PowerPC verification requires a supported Linux/macOS Ghidra host",
);
if (process.env.GHIDRA_INSTALL_DIR === undefined)
throw new Error("Missing prerequisite: caller-supplied GHIDRA_INSTALL_DIR");
const installation = inspectGhidraInstallation({
environment: process.env,
installDir: process.env.GHIDRA_INSTALL_DIR,
...(process.env.JAVA_HOME === undefined
? {}
: { javaHome: process.env.JAVA_HOME }),
});
if (installation.status !== "available")
throw new Error(
`Ghidra prerequisite unavailable: ${JSON.stringify(installation)}`,
);
const execute = promisify(execFile);
const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url));
const run = createVerifierRun();
const workspace = await mkdtemp(join(tmpdir(), "rea-ppc-proof-"));
const runtime = join(workspace, "runtime");
await mkdir(runtime);
const env = {
PATH: process.env.PATH ?? "/usr/bin:/bin",
TMPDIR: runtime,
REA_LOG_LEVEL: "silent",
REA_ANALYSIS_PROVIDER: "ghidra",
REA_PROCESS_RUN_ID: run.run_id,
GHIDRA_INSTALL_DIR: installation.installDir,
JAVA_HOME: installation.javaHome,
HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper",
};
for (const setting of ["GHIDRA_HEADLESS_MAXMEM", "GHIDRA_MAXMEM"]) {
if (process.env[setting] !== undefined) env[setting] = process.env[setting];
}
const reports = [];
let primaryFailure;
try {
{
const fixture = await buildPowerPcFixture(workspace);
const independent = await inspectPowerPcReadelf(fixture.path);
const resolved = await parseBinaryTarget(fixture.path);
if (!resolved.ok) throw resolved.error;
assert.deepEqual(
resolved.value.powerpc,
independent.powerpc,
"REA ELF interpretation disagrees with GNU readelf",
);
assert.equal(resolved.value.sha256, fixture.sha256);
const snapshotPath = join(workspace, "ppc.snapshot.json");
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env,
stderr: "pipe",
});
const client = new Client({ name: "rea-real-ppc-proof", version: "1" });
let stderr = "";
transport.stderr?.on("data", (chunk) => {
stderr = (stderr + chunk.toString()).slice(-65536);
});
const call = async (name, args = {}) => {
const reply = await client.callTool(
{ name, arguments: args },
undefined,
{ timeout: 180000 },
);
return requireMcpOperationResult(reply, name);
};
try {
await client.connect(transport);
const target = await call("open_binary", {
path: fixture.path,
provider_id: "ghidra",
});
assert.equal(target.architecture, "powerpc");
assert.equal(target.sha256, fixture.sha256);
// ELF load-image verification is not the DOS comparison lane. Its retained
// observations still expose the actual Ghidra language and loaded bytes;
// the plain loader would have chosen e500 for this header.
const image = await call("inspect_native_load_image");
assertPowerPcLoadedImage(image.observations, fixture.sha256);
const procedures = await call("list_procedures");
const entry = procedures.find((p) => p.value === "rea_ppc_entry");
const leaf = procedures.find((p) => p.value === "rea_ppc_leaf");
assert.ok(entry, "Entry function missing from real Ghidra inventory");
assert.ok(leaf, "Leaf function missing from real Ghidra inventory");
assert.equal(
BigInt(entry.address),
BigInt(independent.symbols.rea_ppc_entry),
);
assert.equal(
BigInt(leaf.address),
BigInt(independent.symbols.rea_ppc_leaf),
);
const probe = independent.symbols.rea_ppc_probe;
assert.ok(
procedures.some(
(p) =>
p.value === "rea_ppc_probe" && BigInt(p.address) === BigInt(probe),
),
);
const move = await call("inspect_native_instruction", { address: probe });
const branch = await call("inspect_native_instruction", {
address: `0x${(BigInt(probe) + 8n).toString(16)}`,
});
assertPowerPcProbe(move, branch, probe);
const global = await call("read_bytes", {
address: independent.symbols.rea_ppc_global,
length: 4,
});
assert.equal(global.returned_bytes, 4);
assertPowerPcGlobal(global.bytes_hex);
const marker = await call("read_bytes", {
address: independent.symbols.rea_ppc_marker,
length: 29,
});
assert.equal(
marker.bytes_hex,
Buffer.from("rea-ppc-source-owned-fixture\0").toString("hex"),
);
const callees = await call("procedure_callees", {
procedure: entry.address,
});
assert.ok(
callees.includes(leaf.address),
"Expected direct call not recovered",
);
const assembly = await call("procedure_assembly", {
procedure: entry.address,
});
assert.ok(assembly.trim().length > 0);
const dossier = functionDossierSchema.parse(
await call("analyze_function", { procedure: entry.address }),
);
assert.ok(dossier.pseudocode.trim().length > 0);
const mapping = await call("address_to_file_offset", {
address: entry.address,
});
const memory = await call("read_bytes", {
address: entry.address,
length: 16,
});
const sourceBytes = await readFile(fixture.path);
assert.equal(memory.returned_bytes, 16);
assert.equal(
memory.bytes_hex,
sourceBytes
.subarray(mapping.file_offset, mapping.file_offset + 16)
.toString("hex"),
);
const saved = await call("close_binary", {
snapshot_path: snapshotPath,
});
assert.ok(saved.evidence_records > 0);
const snapshot = parseAnalysisSnapshot(
JSON.parse(await readFile(snapshotPath, "utf8")),
);
assert.equal(snapshot.target.architecture, "powerpc");
assert.equal(
snapshot.binding.analysis_profile.parameters.language_id,
"PowerPC:BE:32:default",
);
assert.ok(snapshot.evidence_bundle.records.length > 0);
for (const evidence of snapshot.evidence_bundle.records)
assert.equal(parseEvidence(evidence).subject?.architecture, "powerpc");
await call("open_binary", {
path: fixture.path,
provider_id: "ghidra",
snapshot_path: snapshotPath,
});
assert.deepEqual(await call("list_procedures"), procedures);
await call("close_binary");
// Same public operation through CLI, not a private injected provider.
const cli = await execute(
process.execPath,
[
entrypoint,
"function",
fixture.path,
"rea_ppc_entry",
"--provider",
"ghidra",
"--json",
],
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
);
const evidence = parseEvidence(JSON.parse(cli.stdout));
assert.equal(evidence.subject?.architecture, "powerpc");
assert.equal(evidence.subject?.digest.sha256, fixture.sha256);
const cliDossier = functionDossierSchema.parse(
evidence.normalized_result,
);
assert.equal(cliDossier.procedure.address, dossier.procedure.address);
assert.ok(cliDossier.pseudocode.trim().length > 0);
assert.equal(
createHash("sha256")
.update(await readFile(fixture.path))
.digest("hex"),
fixture.sha256,
"Analysis modified the source executable",
);
reports.push({
...fixture,
independent_reader: independent,
effective_language: image.observations.language_id,
effective_compiler_spec: image.observations.compiler_spec_id,
checks: [
"independent ELF header and symbols",
"loaded Ghidra identity and bound language",
"fixed instruction bytes and immediate",
"conditional branch destination",
"global word and marker bytes",
"CLI/MCP and snapshot identity",
],
procedures: procedures.length,
status: "passed",
});
} catch (cause) {
throw new Error(`PowerPC verification failed; MCP stderr: ${stderr}`, {
cause,
});
} finally {
try {
await client.close();
} finally {
await transport.close();
}
}
}
} catch (cause) {
primaryFailure = cause;
}
const completed = await completeVerifierRun(run);
if (
completed.process_lineage.status !== "verified" ||
completed.process_lineage.descendants.length !== 0
)
throw new Error(
`Cleanup unverified; retained diagnostic workspace: ${workspace}`,
{ cause: primaryFailure },
);
if (primaryFailure !== undefined)
throw new Error(
`PowerPC verification failed; retained workspace: ${workspace}`,
{
cause: primaryFailure,
},
);
await rm(workspace, { recursive: true, force: true });
console.log(JSON.stringify({ ...completed, fixtures: reports }, null, 2));
+3 -2
View File
@@ -62,8 +62,9 @@ assert.equal(first.retryable,true);
assert.match(first.reason,/Delete owned object by handle failed.*Win32 5/s);
const [name,...extra]=readdirSync(temp);assert.ok(name);assert.equal(extra.length,0);
const path=join(temp,name);const identity=statSync(path,{bigint:true});
assert.throws(()=>renameSync(path,path+'-replacement'));
assert.equal(statSync(path,{bigint:true}).ino,identity.ino);
// Ownership follows the retained NTFS handle even when the directory moves.
const moved=path+'-renamed';renameSync(path,moved);
assert.equal(statSync(moved,{bigint:true}).ino,identity.ino);
if(process.argv[2]!=='teardown'){
const second=systemWindowsNativeAuthority();assert.equal(second.available,true);
assert.equal(readdirSync(temp).length,0);
+33 -5
View File
@@ -1,7 +1,7 @@
import {
isScalar,
parse,
parseDocument,
type Document,
type ParsedNode,
type ScalarTag,
type Tags,
@@ -56,9 +56,28 @@ const HERMES_YAML_OPTIONS = {
a.source === b.source),
};
/** Parse a Hermes `config.yaml`. */
export const parseHermesConfigurationYaml = (text: string): unknown =>
parse(text, HERMES_YAML_OPTIONS);
// A comment-only document has no root; a bare marker has an unwritten null
// scalar. Explicit null values, tags and anchors remain actual values.
const isEmptyYamlRoot = (root: Document["contents"]): boolean =>
root === null ||
(isScalar(root) &&
root.value === null &&
root.source === "" &&
root.tag === undefined &&
root.anchor === undefined);
const parseHermesDocument = (text: string) => {
const document = parseDocument(text, HERMES_YAML_OPTIONS);
const error = document.errors[0];
if (error !== undefined) throw error;
return document;
};
/** Parse a Hermes `config.yaml`, preserving the distinction between no root and explicit null. */
export const parseHermesConfigurationYaml = (text: string): unknown => {
const document = parseHermesDocument(text);
return isEmptyYamlRoot(document.contents) ? {} : document.toJS();
};
/** Set or delete one value in a Hermes `config.yaml`, keeping the rest as written. */
export const editHermesConfigurationYaml = (
@@ -66,7 +85,16 @@ export const editHermesConfigurationYaml = (
path: readonly string[],
value: unknown,
): string => {
const document = parseDocument(text, HERMES_YAML_OPTIONS);
// A plain Document, so a bare `---` root can be replaced by a new map.
const document: Document = parseHermesDocument(text);
const root = document.contents;
if (isScalar(root) && isEmptyYamlRoot(root)) {
const map = document.createNode({});
map.commentBefore = [root.commentBefore, root.comment]
.filter((comment) => comment !== undefined)
.join("\n");
document.contents = map;
}
if (value === undefined) document.deleteIn(path);
else document.setIn(path, value);
// Editing REA must not reflow unrelated user prompts or other YAML scalars.
+6 -23
View File
@@ -15,6 +15,8 @@ import { promisify } from "node:util";
import { z } from "zod";
import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js";
import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js";
import { macDeveloperIdRequirement } from "../process/MacCodeSignature.js";
import {
downloadHopperPackage,
HOPPER_DOWNLOAD_BYTES,
@@ -24,29 +26,10 @@ const execFileAsync = promisify(execFile);
const RELEASES_URL =
"https://www.hopperapp.com/include/files-api.php?request=releases&public=true";
const DOWNLOAD_PREFIX = "https://www.hopperapp.com:443/downloader/public/";
/**
* Apple publisher identity pinned for the official Hopper macOS bundle. The
* team identifier was captured from `Hopper-6.6.0-demo.dmg` at
* `https://www.hopperapp.com:443/downloader/public/Hopper-6.6.0-demo.dmg`
* (34846271 bytes, SHA-1 0be530f038249742590992199c7b042ea20ca955) with
* `codesign -dvvv`, which reported
* `Authority=Developer ID Application: Cryptic Apps (2AMA2753NF)`.
*/
export const HOPPER_PUBLISHER = Object.freeze({
bundleIdentifier: "com.cryptic-apps.hopper-web-4",
teamIdentifier: "2AMA2753NF",
});
/**
* Code requirement enforcing the bundle identifier and signing team together.
* The identifier is read from the sealed CodeDirectory, so it cannot be changed
* by editing `Info.plist` the way the previous string comparison could.
*/
const HOPPER_PUBLISHER_REQUIREMENT =
`= identifier "${HOPPER_PUBLISHER.bundleIdentifier}"` +
` and anchor apple generic` +
` and certificate leaf[subject.OU] = "${HOPPER_PUBLISHER.teamIdentifier}"`;
const HOPPER_PUBLISHER_REQUIREMENT = macDeveloperIdRequirement(
HOPPER_PUBLISHER.bundleIdentifier,
HOPPER_PUBLISHER.teamIdentifier,
);
/** Outcome of reading a mounted bundle's signature and publisher identity. */
export type MacHopperBundleVerification =
+1 -1
View File
@@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";
import { HOPPER_PUBLISHER } from "./MacHopper.js";
import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js";
import { setupInstallFailure } from "./SetupInstallFailure.js";
describe("macOS Hopper setup failure projection", () => {
+1 -1
View File
@@ -1,4 +1,4 @@
import { HOPPER_PUBLISHER } from "./MacHopper.js";
import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js";
export type SetupFailureCode =
| "unsupported_host"
@@ -98,7 +98,7 @@ export const MANAGED_WORKFLOW_TOOL_CONTRACTS = [
name: "compare_managed_members",
...toolContractMetadata("compare_managed_members"),
description:
"Compare two authenticated inspect_managed_members Evidence records using unique-only tiers: exact CIL/signature, exact declared type, name, and raw signature (decoded or not), and decoded structural method shape. Names alone are never a matching basis; a member whose signature was not decoded, or whose same-named counterpart's was not, stays unknown rather than added or removed. Metadata tokens remain build-local coordinates bound to each artifact SHA-256 and MVID, and the tuple digest does not itself remap them.",
"Compare two authenticated inspect_managed_members Evidence records using unique-only tiers: exact declared type, name, and CIL/signature; exact declared type, name, and raw signature (decoded or not); exact declared type, name, and decoded structural method shape; then exact CIL/signature or decoded structural method shape alone, which never takes a member whose own declared type and name remain unpaired on the other side or whose declaring type is unresolved. Names alone are never a matching basis; a member whose signature was not decoded, or whose same-named counterpart's was not, stays unknown rather than added or removed. Metadata tokens remain build-local coordinates bound to each artifact SHA-256 and MVID, and the tuple digest does not itself remap them.",
kind: "application",
inputSchema: compareManagedMembersReferenceInputSchema,
outputSchema: comparisonOutputSchema,
+2 -2
View File
@@ -957,11 +957,11 @@ it("keeps the complete unique set when it fits the byte budget", () => {
it("does not impose a message-count cutoff below the byte budget", () => {
const messages = Array.from(
{ length: 100 },
(_, index) => `message ${index}`,
(_, index) => `message ${String(index).padStart(2, "0")}`,
);
expect(
boundedViewMessages(messages, `ev_${"a".repeat(64)}`, "limitations"),
).toEqual(messages);
).toEqual(messages.toSorted());
});
it("bounds repeated view messages even when fewer than 64 source strings are huge", () => {
+58
View File
@@ -0,0 +1,58 @@
import { describe, expect, it } from "vitest";
import { parseExecutableHeader } from "./binaryTarget.js";
const header = (little: boolean, bits: 32 | 64 = 32, machine = 20): Buffer => {
const bytes = Buffer.alloc(bits === 32 ? 52 : 64);
bytes.set([0x7f, 0x45, 0x4c, 0x46, bits === 32 ? 1 : 2, little ? 1 : 2, 1]);
if (little) {
bytes.writeUInt16LE(2, 16);
bytes.writeUInt16LE(machine, 18);
bytes.writeUInt32LE(1, 20);
bytes.writeUInt16LE(bytes.length, bits === 32 ? 40 : 52);
} else {
bytes.writeUInt16BE(2, 16);
bytes.writeUInt16BE(machine, 18);
bytes.writeUInt32BE(1, 20);
bytes.writeUInt16BE(bytes.length, bits === 32 ? 40 : 52);
}
return bytes;
};
describe("PowerPC ELF header boundaries", () => {
it.each([false, true])(
"retains source endian and flags (little=%s)",
(little) => {
const parsed = parseExecutableHeader(header(little), "x64");
if (!parsed.ok) throw new Error(parsed.error);
expect(parsed.value).toEqual({
format: "elf",
architecture: "powerpc",
availableArchitectures: ["powerpc"],
powerpc: {
elfClass: 32,
byteOrder: little ? "little" : "big",
type: 2,
flags: 0,
},
});
},
);
it("reads ELF64 flags at their own offset without claiming support", () => {
const bytes = header(false, 64);
bytes.writeUInt32BE(0x80000000, 48);
const parsed = parseExecutableHeader(bytes, "x64");
if (!parsed.ok) throw new Error(parsed.error);
expect(parsed.value).toMatchObject({
architecture: "powerpc",
powerpc: { elfClass: 64, byteOrder: "big", flags: 0x80000000 },
});
});
it("does not recognize EM_PPC64 as the 32-bit family", () => {
expect(parseExecutableHeader(header(false, 64, 21), "x64")).toEqual({
ok: false,
error: "unsupported ELF architecture",
});
});
});
+21 -1
View File
@@ -5,6 +5,7 @@ import type {
MipsElfMetadata,
PeCodeViewRecord,
RiscvElfMetadata,
PowerPcElfMetadata,
UniversalMachOSlice,
} from "./binaryTargetTypes.js";
import { err, ok, type Result } from "./result.js";
@@ -30,6 +31,12 @@ export type ExecutableMetadata =
readonly availableArchitectures: readonly "riscv"[];
readonly riscv: RiscvElfMetadata;
}
| {
readonly format: "elf";
readonly architecture: "powerpc";
readonly availableArchitectures: readonly "powerpc"[];
readonly powerpc: PowerPcElfMetadata;
}
| {
readonly format: "mach-o" | "elf" | "dos-mz" | "dos-com";
readonly architecture: HeaderArchitecture;
@@ -201,7 +208,11 @@ const parseElf = (
const machine = little ? bytes.readUInt16LE(18) : bytes.readUInt16BE(18);
const architecture = elfArchitecture(machine);
if (architecture === undefined) return err("unsupported ELF architecture");
if (architecture === "mips" || architecture === "riscv") {
if (
architecture === "mips" ||
architecture === "riscv" ||
architecture === "powerpc"
) {
const flagsOffset = elfClass === 32 ? 36 : 48;
const metadata: ElfFamilyMetadata = {
elfClass,
@@ -211,6 +222,13 @@ const parseElf = (
? bytes.readUInt32LE(flagsOffset)
: bytes.readUInt32BE(flagsOffset),
};
if (architecture === "powerpc")
return ok({
format: "elf",
architecture,
availableArchitectures: [architecture],
powerpc: metadata,
});
return architecture === "mips"
? ok({
format: "elf",
@@ -532,6 +550,8 @@ const elfArchitecture = (
return "x86_64";
case 8: // EM_MIPS: ISA, ABI and byte order remain separate ELF facts.
return "mips";
case 20: // EM_PPC; byte order and processor variant remain ELF declarations.
return "powerpc";
case 40:
return "arm";
case 183:
+14
View File
@@ -9,6 +9,7 @@ export const BINARY_ARCHITECTURES = [
"mips",
"6502",
"riscv",
"powerpc",
] as const;
export type BinaryArchitecture = (typeof BINARY_ARCHITECTURES)[number];
@@ -36,6 +37,7 @@ export interface PeCodeViewRecord {
export const ELF_METADATA_ARCHITECTURES = [
"mips",
"riscv",
"powerpc",
] as const satisfies readonly BinaryArchitecture[];
export type ElfMetadataArchitecture =
(typeof ELF_METADATA_ARCHITECTURES)[number];
@@ -90,6 +92,9 @@ export interface MipsAbiFlags {
*/
export type RiscvElfMetadata = ElfFamilyMetadata;
/** Raw PowerPC declarations; family identity does not select a processor variant. */
export type PowerPcElfMetadata = ElfFamilyMetadata;
/** Byte range of the selected slice inside its universal Mach-O container. */
export interface UniversalMachOSlice {
readonly offset: number;
@@ -140,6 +145,15 @@ export type BinaryTarget =
readonly executableRole?: never;
readonly managed?: never;
})
| (BinaryTargetIdentity & {
readonly kind: "executable";
readonly format: "elf";
readonly architecture: "powerpc";
readonly availableArchitectures: readonly "powerpc"[];
readonly powerpc: PowerPcElfMetadata;
readonly executableRole?: never;
readonly managed?: never;
})
| (ExecutableTarget & {
readonly format: "mach-o" | "elf" | "dos-mz" | "dos-com";
readonly universalSlice?: never;
+12
View File
@@ -0,0 +1,12 @@
/**
* Apple publisher identity pinned for the official Hopper macOS bundle. The
* team identifier was captured from `Hopper-6.6.0-demo.dmg` at
* `https://www.hopperapp.com:443/downloader/public/Hopper-6.6.0-demo.dmg`
* (34846271 bytes, SHA-1 0be530f038249742590992199c7b042ea20ca955) with
* `codesign -dvvv`, which reported
* `Authority=Developer ID Application: Cryptic Apps (2AMA2753NF)`.
*/
export const HOPPER_PUBLISHER = Object.freeze({
bundleIdentifier: "com.cryptic-apps.hopper-web-4",
teamIdentifier: "2AMA2753NF",
});
@@ -107,8 +107,8 @@ export const comparisonLimitations = (
): string[] => {
const limitations: string[] = [
"Metadata tokens are build-local coordinates; matched pairs are remaps, not persistent identities.",
"Methods pair by exact CIL/signature, then declared type, name, and signature, then structural shape; names alone are never a matching basis.",
"Fields pair by exact signature, then declared type, name, and signature; an undecoded signature is compared only by its raw bytes, and an unmatched member with one stays unknown.",
"Methods pair within one declared type and name by exact CIL/signature, then signature, then structural shape; only then may exact CIL/signature or structural shape alone remap across declared types or names, never taking a member whose own declared type and name remain unpaired on the other side or whose declaring type is unresolved. Names alone are never a matching basis.",
"Fields pair by declared type, name, and signature, then remap by exact decoded signature alone under the same rule; an undecoded signature is compared only by its raw bytes, and an unmatched member with one stays unknown.",
];
if (left.coverage.state !== "complete" || right.coverage.state !== "complete")
limitations.push("At least one managed member observation is partial.");
@@ -5,6 +5,7 @@ import type { ManagedMemberComparisonResult } from "./managedMemberComparison.js
import {
type ManagedFieldMatches,
type ManagedMethodMatches,
identityOf,
sha256,
} from "./managedMemberComparisonMatch.js";
@@ -211,33 +212,36 @@ const absenceLimitations = (
`signature-not-decoded: The member's ${member.signature.parse_status} signature was compared only by exact declared type, name, and raw bytes, so a changed or renamed counterpart could not be sought.`,
]),
...(member.signature.parse_status === "decoded" &&
opposite.oppositeUndecodedNames.has(declaredName(member))
opposite.oppositeUndecodedNames.size > 0 &&
opposite.oppositeUndecodedNames.has(identityOf(member))
? [
`counterpart-signature-not-decoded: The ${opposite.opposite} ${opposite.inventory} inventory has an unpaired member with the same declared type and name whose signature was not decoded, so it may be this member with a changed signature.`,
]
: []),
];
const declaredName = (member: Method | Field): string =>
JSON.stringify([member.declaring_type, member.name]);
/**
* Declared names of unpaired members whose signature was not decoded: one-sided
* members and candidates of ambiguous groups, which also remain unpaired.
*/
interface Unpaired {
readonly item: Method | Field;
readonly identity: string;
}
const undecodedOneSidedNames = (members: {
readonly leftOnly: readonly { readonly item: Method | Field }[];
readonly rightOnly: readonly { readonly item: Method | Field }[];
readonly leftOnly: readonly Unpaired[];
readonly rightOnly: readonly Unpaired[];
readonly ambiguous: readonly {
readonly left: readonly { readonly item: Method | Field }[];
readonly right: readonly { readonly item: Method | Field }[];
readonly left: readonly Unpaired[];
readonly right: readonly Unpaired[];
}[];
}): UndecodedOneSidedNames => {
const names = (side: readonly { readonly item: Method | Field }[]) =>
const names = (side: readonly Unpaired[]) =>
new Set(
side
.filter(({ item }) => item.signature.parse_status !== "decoded")
.map(({ item }) => declaredName(item)),
.map(({ identity }) => identity),
);
return {
left: names([
@@ -300,18 +304,20 @@ const fieldOnlyItem = (
};
/**
* Name the identity key an ambiguous group shares. The exact-signature key
* already includes the declared type and name, so names cannot separate it.
* Name only evidence shared by the whole group. Exact-signature ambiguities
* include both declared-identity collisions and unresolved raw-signature choices.
*/
const ambiguityLimitation = (
members: "methods" | "fields",
basis: MethodItem["match"]["basis"],
): string =>
basis === "exact-signature"
? `Multiple managed ${members} share the same declared type, name, and raw signature; REA did not guess a token remap.`
: members === "methods"
? "Multiple managed methods share the same non-name identity key; REA did not guess a token remap."
: "Multiple managed fields share the same signature; REA did not guess a token remap from names.";
? `Multiple managed ${members} share the same raw signature; REA did not guess declared ownership or a token remap.`
: basis === "exact-il-signature"
? "Multiple managed methods share the same CIL and signature; REA did not guess a token remap."
: members === "methods"
? "Multiple managed methods share the same structural shape; REA did not guess a token remap."
: "Multiple managed fields share the same signature; REA did not guess a token remap.";
export const buildMethodItems = (
matches: ManagedMethodMatches,
@@ -28,8 +28,11 @@ type ConcreteMatchBasis = Exclude<MatchBasis, "none">;
interface Keyed<Item> {
readonly item: Item;
/** Declared type and name; an unresolved type is null, the name remains. */
readonly identity: string;
readonly exactKey: string | null;
readonly signatureKey: string | null;
readonly rawSignatureKey: string;
readonly structuralKey: string | null;
}
@@ -48,6 +51,8 @@ interface Ambiguous<Item> {
const keyMethod = (item: Method): Keyed<Method> => ({
item,
identity: identityOf(item),
rawSignatureKey: item.signature.raw_sha256,
exactKey:
item.body.status === "present" &&
item.body.il_sha256 !== null &&
@@ -99,6 +104,8 @@ const keyMethod = (item: Method): Keyed<Method> => ({
const keyField = (item: Field): Keyed<Field> => ({
item,
identity: identityOf(item),
rawSignatureKey: item.signature.raw_sha256,
exactKey:
item.signature.parse_status === "decoded"
? sha256(["field-exact", item.signature.raw_sha256])
@@ -126,15 +133,34 @@ const matchMethods = (
matchByKeys({
left,
right,
exactBasis: "exact-il-signature",
fallbackBases: [
rounds: [
{
basis: "exact-il-signature",
key: declaredAs(({ exactKey }) => exactKey),
},
{
basis: "exact-signature",
key: ({ signatureKey }) => signatureKey,
competitionKey: ({ rawSignatureKey }) => rawSignatureKey,
},
{
basis: "structural-method-shape",
// An unresolved owner has no declared shape match to prove.
key: declaredAs((keyed) =>
keyed.item.declaring_type === null ? null : keyed.structuralKey,
),
competitionKey: ({ structuralKey }) => structuralKey,
},
{
basis: "exact-il-signature",
key: ({ exactKey }) => exactKey,
remap: true,
},
{
basis: "structural-method-shape",
key: ({ structuralKey }) => structuralKey,
competitionKey: ({ structuralKey }) => structuralKey,
remap: true,
},
],
});
@@ -146,11 +172,20 @@ const matchFields = (
matchByKeys({
left,
right,
exactBasis: "field-signature",
fallbackBases: [
rounds: [
{
basis: "field-signature",
key: declaredAs(({ exactKey }) => exactKey),
},
{
basis: "exact-signature",
key: ({ signatureKey }) => signatureKey,
competitionKey: ({ rawSignatureKey }) => rawSignatureKey,
},
{
basis: "field-signature",
key: ({ exactKey }) => exactKey,
remap: true,
},
],
});
@@ -158,21 +193,47 @@ const matchFields = (
export type ManagedMethodMatches = ReturnType<typeof matchMethods>;
export type ManagedFieldMatches = ReturnType<typeof matchFields>;
interface MatchRound<Item> {
readonly basis: ConcreteMatchBasis;
readonly key: (item: Keyed<Item>) => string | null;
/** Name-free evidence paired with the retained member name for contesting. */
readonly competitionKey?: (item: Keyed<Item>) => string | null;
/**
* The key omits the declared type, so a pair may join different declared
* identities; same-identity pairs were taken by earlier rounds. Contention
* still retains the member name, which unresolved owners keep as evidence.
* A member whose own identity is still unpaired on the other side does not
* take part, and a group with a member of an unresolved type stays ambiguous.
*/
readonly remap?: true;
}
interface MatchByKeysInput<Item> {
readonly left: readonly Keyed<Item>[];
readonly right: readonly Keyed<Item>[];
readonly exactBasis: ConcreteMatchBasis;
readonly fallbackBases: readonly {
readonly basis: ConcreteMatchBasis;
readonly key: (item: Keyed<Item>) => string | null;
}[];
readonly rounds: readonly MatchRound<Item>[];
}
const matchByKeys = <Item>({
export const identityOf = ({
declaring_type,
name,
}: {
readonly declaring_type: string | null;
readonly name: string;
}): string => sha256([declaring_type, name]);
/** Narrow a key to members of one declared type and name. */
const declaredAs =
<Item>(key: (item: Keyed<Item>) => string | null) =>
(item: Keyed<Item>): string | null => {
const value = key(item);
return value === null ? null : sha256([item.identity, value]);
};
const matchByKeys = <Item extends Declared>({
left,
right,
exactBasis,
fallbackBases,
rounds,
}: MatchByKeysInput<Item>): {
readonly pairs: readonly MatchedPair<Item>[];
readonly ambiguous: readonly Ambiguous<Item>[];
@@ -181,33 +242,58 @@ const matchByKeys = <Item>({
} => {
const usedLeft = new Set<Keyed<Item>>();
const usedRight = new Set<Keyed<Item>>();
const pairedLeft = new Set<Keyed<Item>>();
const pairedRight = new Set<Keyed<Item>>();
const pairs: MatchedPair<Item>[] = [];
const ambiguous: Ambiguous<Item>[] = [];
const rounds: readonly {
readonly basis: ConcreteMatchBasis;
readonly key: (item: Keyed<Item>) => string | null;
}[] = [
{ basis: exactBasis, key: ({ exactKey }) => exactKey },
...fallbackBases,
];
for (const round of rounds) {
const competitionKey = round.competitionKey ?? ((item) => item.exactKey);
const unpairedLeft = unpairedOf(left, pairedLeft);
const unpairedRight = unpairedOf(right, pairedRight);
const declarable = ownerlessDeclarable(
unpairedLeft,
unpairedRight,
competitionKey,
);
const openLeft = left.filter((item) => !usedLeft.has(item));
const openRight = right.filter((item) => !usedRight.has(item));
// Only open candidates can justify overriding a held declared identity:
// consumed ambiguous rows remain identity barriers, but cannot join a
// later remap group to establish its ambiguity.
const remapContested =
round.remap === true
? ownerlessContested([openLeft, openRight], competitionKey)
: new Set<string>();
const leftGroups = groupBy(
left.filter((item) => !usedLeft.has(item)),
round.remap === true
? remappable(openLeft, unpairedRight, competitionKey, remapContested)
: openLeft.filter(declarable),
round.key,
);
const rightGroups = groupBy(
right.filter((item) => !usedRight.has(item)),
round.remap === true
? remappable(openRight, unpairedLeft, competitionKey, remapContested)
: openRight.filter(declarable),
round.key,
);
for (const [key, leftItems] of leftGroups) {
const rightItems = rightGroups.get(key);
if (rightItems === undefined) continue;
if (leftItems.length === 1 && rightItems.length === 1) {
// An unresolved owner may be either member's own counterpart, so a
// remap group it joins stays ambiguous.
const refused =
round.remap === true &&
[...leftItems, ...rightItems].some(
({ item }) => item.declaring_type === null,
);
if (!refused && leftItems.length === 1 && rightItems.length === 1) {
const [leftItem] = leftItems;
const [rightItem] = rightItems;
if (leftItem === undefined || rightItem === undefined) continue;
usedLeft.add(leftItem);
usedRight.add(rightItem);
pairedLeft.add(leftItem);
pairedRight.add(rightItem);
pairs.push({
left: leftItem,
right: rightItem,
@@ -230,6 +316,22 @@ const matchByKeys = <Item>({
}
}
}
// Contention can prevent a declared signature pair even when CIL/shape
// evidence cannot form a later remap. Held ambiguities still contest
// ownership; only proven pairs leave that evidence pool. Group only open
// candidates so held members retain their original ambiguity and provenance.
for (const group of unresolvedSignatureAmbiguities(
left.filter((item) => !usedLeft.has(item)),
right.filter((item) => !usedRight.has(item)),
ownerlessContested(
[unpairedOf(left, pairedLeft), unpairedOf(right, pairedRight)],
(member) => member.rawSignatureKey,
),
)) {
for (const item of group.left) usedLeft.add(item);
for (const item of group.right) usedRight.add(item);
ambiguous.push(group);
}
return {
pairs,
ambiguous,
@@ -238,6 +340,132 @@ const matchByKeys = <Item>({
};
};
/** Open signature choices contested by unresolved owners, including held rows. */
const unresolvedSignatureAmbiguities = <Item extends Declared>(
left: readonly Keyed<Item>[],
right: readonly Keyed<Item>[],
contested: ReadonlySet<string>,
): Ambiguous<Item>[] => {
const key = (member: Keyed<Item>): string =>
competitionIdentityKey(member, member.rawSignatureKey) ?? "";
const leftGroups = groupBy(left, key);
const rightGroups = groupBy(right, key);
const groups: Ambiguous<Item>[] = [];
for (const [signature, leftItems] of leftGroups) {
const rightItems = rightGroups.get(signature);
if (!contested.has(signature) || rightItems === undefined) continue;
groups.push({
left: leftItems,
right: rightItems,
basis: "exact-signature",
});
}
return groups;
};
interface Declared {
readonly declaring_type: string | null;
readonly name: string;
}
const unpairedOf = <Item>(
items: readonly Keyed<Item>[],
paired: ReadonlySet<Keyed<Item>>,
): Keyed<Item>[] => items.filter((item) => !paired.has(item));
/**
* Round-specific evidence keys an unresolved owner competes for, paired with
* its retained member name. Another same-name member on the same side can be
* the unresolved row's owner; a different name cannot. Contested candidates
* stay in remap rounds rather than pairing by declared identity.
*/
const competitionIdentityKey = <Item extends Declared>(
member: Keyed<Item>,
evidenceKey: string | null,
): string | null =>
evidenceKey === null ? null : JSON.stringify([member.item.name, evidenceKey]);
const ownerlessContested = <Item extends Declared>(
sides: readonly (readonly Keyed<Item>[])[],
keyOf: (item: Keyed<Item>) => string | null,
): ReadonlySet<string> => {
const contested = new Set<string>();
for (const side of sides) {
const seen = new Map<string, number>();
const ownerless = new Set<string>();
for (const member of side) {
const key = competitionIdentityKey(member, keyOf(member));
if (key === null) continue;
seen.set(key, (seen.get(key) ?? 0) + 1);
if (member.item.declaring_type === null) ownerless.add(key);
}
for (const key of ownerless)
if ((seen.get(key) ?? 0) > 1) contested.add(key);
}
return contested;
};
/**
* Members a remap round may group: an unresolved owner, which keeps its group
* ambiguous; a member whose selected key an unresolved owner competes for, so the
* group that preserved the base matcher's ambiguity can form; or no member of
* the same declared identity left unpaired on the other side, whether
* one-sided or in an ambiguous group. The sets are taken before grouping, so
* no group order decides a remap.
*/
const remappable = <Item extends Declared>(
items: readonly Keyed<Item>[],
others: readonly Keyed<Item>[],
keyOf: (item: Keyed<Item>) => string | null,
contested: ReadonlySet<string>,
): Keyed<Item>[] => {
const kept = new Set(others.map(({ identity }) => identity));
return items.filter((member) => {
const key = keyOf(member);
return (
(key !== null &&
contested.has(competitionIdentityKey(member, key) ?? "")) ||
member.item.declaring_type === null ||
!kept.has(member.identity)
);
});
};
/**
* Whether a member may pair in a declared-identity round. A member with an
* unresolved owner may, only when no other member on either side shares its
* selected evidence key: its owner might be any of them, so a shared key
* leaves it to the remap rounds, which keep such a group ambiguous. The
* resolved members that share such a key stay out too, so they cannot pair and
* consume the candidate the ownerless member might own.
*/
const ownerlessDeclarable = <Item extends Declared>(
left: readonly Keyed<Item>[],
right: readonly Keyed<Item>[],
keyOf: (item: Keyed<Item>) => string | null,
): ((member: Keyed<Item>) => boolean) => {
const count = (side: readonly Keyed<Item>[]) => {
const seen = new Map<string, number>();
for (const member of side) {
const key = keyOf(member);
if (key !== null) seen.set(key, (seen.get(key) ?? 0) + 1);
}
return seen;
};
const leftSeen = count(left);
const rightSeen = count(right);
const contested = ownerlessContested([left, right], keyOf);
return (member) => {
const key = keyOf(member);
return key !== null &&
contested.has(competitionIdentityKey(member, key) ?? "")
? false
: member.item.declaring_type !== null ||
key === null ||
((leftSeen.get(key) ?? 0) <= 1 && (rightSeen.get(key) ?? 0) <= 1);
};
};
const groupBy = <Item>(
items: readonly Item[],
keyOf: (item: Item) => string | null,
@@ -266,7 +494,7 @@ const observedBy =
* the other side's selection stays one-sided, so a type selection never yields
* a remap to a member of a type the other side did not inspect.
*/
const matchObserved = <Item extends { readonly declaring_type: string | null }>(
const matchObserved = <Item extends Declared>(
left: readonly Item[],
right: readonly Item[],
sides: {
+9 -1
View File
@@ -21,6 +21,10 @@ import {
ghidraRiscvLanguageOverride,
ghidraRiscvProfileParameters,
} from "./GhidraRiscvProfile.js";
import {
ghidraPowerPcLanguage,
ghidraPowerPcProfileParameters,
} from "./GhidraPowerPcProfile.js";
import { ghidraRawBinaryLanguage } from "./GhidraRawBinaryLanguage.js";
import {
@@ -44,7 +48,9 @@ import {
export const ghidraPinnedLanguage = (
target: BinaryTarget,
): GhidraLanguageOverride | undefined =>
ghidraMipsLanguageOverride(target) ?? ghidraRiscvLanguageOverride(target);
ghidraMipsLanguageOverride(target) ??
ghidraRiscvLanguageOverride(target) ??
ghidraPowerPcLanguage(target);
/** Resolve version-bound, deterministic semantics before Ghidra imports a target. */
export const resolveGhidraAnalysisProfile = async (
@@ -155,6 +161,7 @@ export const resolveGhidraAnalysisProfile = async (
const pinnedLanguage = ghidraPinnedLanguage(target);
if (
pinnedLanguage !== undefined &&
target.architecture !== "powerpc" &&
languageOverride !== undefined &&
(languageOverride.languageId !== pinnedLanguage.languageId ||
(languageOverride.compilerSpecId !== undefined &&
@@ -277,6 +284,7 @@ export const resolveGhidraAnalysisProfile = async (
...analyzerParameters,
...ghidraMipsProfileParameters(target, languageOverride),
...ghidraRiscvProfileParameters(target),
...ghidraPowerPcProfileParameters(target),
...(pspExtension?.ok === true
? {
mips_support_lane: "psp-elf32-exec-eabi32-allegrex-v2",
+166
View File
@@ -0,0 +1,166 @@
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { parseConfig } from "../config/parseConfig.js";
import type {
BinaryTarget,
PowerPcElfMetadata,
} from "../domain/binaryTargetTypes.js";
import { silentLogger } from "../logger.js";
import { HopperProvider } from "../hopper/HopperProvider.js";
import { hopperLoaderArgsForTarget } from "../hopper/HopperAnalysisProfile.js";
import { IdaProvider } from "../ida/IdaProvider.js";
import {
ghidraPinnedLanguage,
ghidraProfileLanguageOverride,
} from "./GhidraAnalysisProfile.js";
import { GhidraProvider } from "./GhidraProvider.js";
import type { GhidraInstallationHost } from "./GhidraInstallation.js";
// Abbreviated from Ghidra 12.1.4 Processors/PowerPC/data/languages/ppc.ldefs.
const PPC_LDEFS = `<?xml version="1.0" encoding="UTF-8"?>
<language_definitions>
<language processor="PowerPC" endian="big" size="32" variant="default"
id="PowerPC:BE:32:default">
<compiler name="default" spec="ppc_32.cspec" id="default"/>
</language>
<language processor="PowerPC" endian="big" size="32" variant="e500"
id="PowerPC:BE:32:e500">
<compiler name="default" spec="ppc_32_e500_be.cspec" id="default"/>
</language>
</language_definitions>
`;
// Fake installation inspection tests admission/profile contracts, not Ghidra.
const host: GhidraInstallationHost = {
platform: "linux",
architecture: "x64",
readText: () => "application.version=12.1.4\n",
executable: () => true,
probeJava: () => ({
version: "21.0.11",
major: 21,
home: "/jdk-21",
bits: 64,
runtime: "jdk",
}),
};
const configFor = (environment: NodeJS.ProcessEnv) => {
const parsed = parseConfig({ GHIDRA_INSTALL_DIR: "/ghidra", ...environment });
if (!parsed.ok) throw parsed.error;
return parsed.value;
};
const config = configFor({});
const target = (change: Partial<PowerPcElfMetadata> = {}): BinaryTarget => ({
path: "/fixture.elf",
sha256: "a".repeat(64),
kind: "executable",
format: "elf",
architecture: "powerpc",
availableArchitectures: ["powerpc"],
powerpc: { elfClass: 32, byteOrder: "big", type: 2, flags: 0, ...change },
});
describe("bounded Ghidra PowerPC admission", () => {
it("binds the generic language instead of the loader's e500 choice", async () => {
const ghidra = new GhidraProvider(config, silentLogger, {}, host);
expect(ghidra.inspectTargetSupport(target()).status).toBe("supported");
const resolved = await ghidra.resolveAnalysisProfile(target());
if (!resolved.ok) throw resolved.error;
const parameters = resolved.value.profile?.parameters ?? {};
expect(parameters).toMatchObject({
architecture: "powerpc",
language_id: "PowerPC:BE:32:default",
compiler_spec_id: "default",
language_selection: "provider-pinned-v1",
powerpc_elf: { elf_class: 32, byte_order: "big", type: 2, flags: 0 },
powerpc_support_lane: "elf32-be-exec-generic-v1",
});
expect(ghidraPinnedLanguage(target())).toEqual({
languageId: "PowerPC:BE:32:default",
compilerSpecId: "default",
});
expect(ghidraProfileLanguageOverride(parameters)).toEqual(
ghidraPinnedLanguage(target()),
);
});
it("lets a configured language replace the lane default", async () => {
const installDir = await createTestTempDirectory("rea-ghidra-ppc-");
const languages = join(
installDir,
"Ghidra/Processors/PowerPC/data/languages",
);
await mkdir(languages, { recursive: true });
await writeFile(join(languages, "ppc.ldefs"), PPC_LDEFS);
const ghidra = new GhidraProvider(
configFor({
GHIDRA_INSTALL_DIR: installDir,
REA_GHIDRA_LANGUAGE_ID: "PowerPC:BE:32:e500",
}),
silentLogger,
{},
host,
);
const resolved = await ghidra.resolveAnalysisProfile(target());
if (!resolved.ok) throw resolved.error;
const parameters = resolved.value.profile?.parameters ?? {};
expect(parameters).toMatchObject({
language_id: "PowerPC:BE:32:e500",
language_selection: "configured-v1",
});
expect(ghidraProfileLanguageOverride(parameters)).toEqual({
languageId: "PowerPC:BE:32:e500",
});
});
it("commits header flags so a changed interpretation changes the profile", async () => {
const ghidra = new GhidraProvider(config, silentLogger, {}, host);
const left = await ghidra.resolveAnalysisProfile(target());
const right = await ghidra.resolveAnalysisProfile(
target({ flags: 0x00010000 }),
);
if (!left.ok) throw left.error;
if (!right.ok) throw right.error;
expect(left.value.profile?.digest).not.toBe(right.value.profile?.digest);
});
it("does not widen Windows P0", () => {
const windows = new GhidraProvider(
config,
silentLogger,
{},
{ ...host, platform: "win32" },
);
expect(windows.inspectTargetSupport(target()).status).toBe("unsupported");
});
it.each([
[{ elfClass: 64 }, "ELF32"],
[{ byteOrder: "little" }, "big-endian"],
[{ type: 1 }, "ET_EXEC"],
[{ type: 3 }, "ET_EXEC"],
[{ flags: 0x80000000 }, "EF_PPC_EMB"],
] satisfies [Partial<PowerPcElfMetadata>, string][])(
"refuses an unverified PowerPC interpretation %j",
async (change, reason) => {
const ghidra = new GhidraProvider(config, silentLogger, {}, host);
const value = target(change);
expect(ghidra.inspectTargetSupport(value)).toMatchObject({
status: "unsupported",
reason: expect.stringContaining(reason),
});
const resolved = await ghidra.resolveAnalysisProfile(value);
expect(resolved.ok).toBe(false);
},
);
it("keeps PowerPC out of unverified Hopper and IDA adapter lanes", () => {
const hopper = new HopperProvider(config, silentLogger, {}, "linux");
expect(hopper.inspectTargetSupport(target()).status).toBe("unsupported");
expect(hopperLoaderArgsForTarget(target()).ok).toBe(false);
const ida = new IdaProvider(config, {});
expect(ida.inspectTargetSupport(target()).status).toBe("unsupported");
});
});
+67
View File
@@ -0,0 +1,67 @@
import type { GhidraLanguageOverride } from "../config/ghidraLanguageOverride.js";
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
import type { JsonValue } from "../domain/jsonValue.js";
/**
* Generic 32-bit big-endian PowerPC. Ghidra's ELF loader selects the e500
* (Book-E/SPE) variant for a plain EM_PPC header, so the lane binds this
* language instead of accepting the loader's choice.
*/
export const GHIDRA_POWERPC_LANGUAGE: GhidraLanguageOverride = {
languageId: "PowerPC:BE:32:default",
compilerSpecId: "default",
};
/** Keep family recognition separate from the deliberately bounded Ghidra lane. */
export const ghidraPowerPcUnsupportedReason = (
target: BinaryTarget,
): string | null => {
if (target.kind !== "executable" || target.architecture !== "powerpc")
return null;
const metadata = target.powerpc;
if (target.format !== "elf" || metadata === undefined)
return "PowerPC analysis requires ELF header metadata; a family name alone does not select a processor variant.";
if (metadata.elfClass !== 32)
return "This Ghidra PowerPC lane supports ELF32 only; PPC64 remains unsupported.";
if (metadata.byteOrder !== "big")
return "This Ghidra PowerPC lane supports big-endian ELF32 only; little-endian PowerPC needs separate verification.";
if (metadata.type !== 2)
return "This Ghidra PowerPC lane supports ET_EXEC only; relocatable objects and shared images need separate load verification.";
// EF_PPC_EMB (0x80000000) marks embedded variants; other bits only declare
// relocation conventions that do not change instruction decoding.
if ((metadata.flags & 0x80000000) !== 0)
return "EF_PPC_EMB declares an embedded PowerPC variant that needs a separately supported processor profile.";
return null;
};
/**
* The language the lane imports with, unless REA_GHIDRA_LANGUAGE_ID already
* selected one through the committed profile.
*/
export const ghidraPowerPcLanguage = (
target: BinaryTarget,
): GhidraLanguageOverride | undefined =>
target.kind === "executable" && target.architecture === "powerpc"
? GHIDRA_POWERPC_LANGUAGE
: undefined;
/** Commit source ELF interpretation without confusing target ISA with host CPU. */
export const ghidraPowerPcProfileParameters = (
target: BinaryTarget,
): Readonly<Record<string, JsonValue>> => {
if (
target.kind !== "executable" ||
target.architecture !== "powerpc" ||
target.powerpc === undefined
)
return {};
return {
powerpc_elf: {
elf_class: target.powerpc.elfClass,
byte_order: target.powerpc.byteOrder,
type: target.powerpc.type,
flags: target.powerpc.flags,
},
powerpc_support_lane: "elf32-be-exec-generic-v1",
};
};
+1
View File
@@ -68,6 +68,7 @@ const SUPPORTED_ARCHITECTURES = new Set([
"mips",
"6502",
"riscv",
"powerpc",
]);
/** Ghidra candidate backed by an isolated ephemeral headless import. */
+3
View File
@@ -1,5 +1,6 @@
import type { BinaryTarget } from "../domain/binaryTargetTypes.js";
import { ghidraMipsUnsupportedReason } from "./GhidraMipsProfile.js";
import { ghidraPowerPcUnsupportedReason } from "./GhidraPowerPcProfile.js";
import { ghidraRiscvUnsupportedReason } from "./GhidraRiscvProfile.js";
/** Allegrex's ELF machine declaration, not a filename or inferred instruction. */
@@ -14,6 +15,8 @@ export const isGhidraPspTarget = (target: BinaryTarget): boolean =>
export const ghidraProcessorUnsupportedReason = (
target: BinaryTarget,
): string | null => {
if (target.kind === "executable" && target.architecture === "powerpc")
return ghidraPowerPcUnsupportedReason(target);
if (target.kind === "executable" && target.architecture === "riscv")
return ghidraRiscvUnsupportedReason(target);
if (
+14
View File
@@ -33,6 +33,7 @@ import {
} from "./HopperTargetLease.js";
import type { HopperTargetLease } from "./HopperTargetLease.js";
import { HOPPER_SHUTDOWN_TIMEOUT_MS } from "./HopperSessionValues.js";
import { verifyMacHopperNativeHelper } from "./HopperMacPublisher.js";
const execFileAsync = promisify(execFile);
const HOPPER_BACKGROUND_STARTUP_MS = 5_000;
@@ -47,6 +48,8 @@ export interface BridgeSession {
/** Process handle returned by a bridge launcher. */
export type BridgeLaunch = {
/** Verified publisher and sealed native-helper identity; omission is unknown. */
readonly nativeHelperVerified?: true;
/** Only literal native-helper arguments support a pre-handoff denial conclusion. */
readonly authorizationDenialBeforeHandoff?: true;
/** Release application/target exclusivity only after verified document and process cleanup. */
@@ -106,6 +109,7 @@ export type HopperApplicationLauncherOptions =
});
export interface HopperApplicationLauncherDependencies {
readonly verifyNativeHelper?: typeof verifyMacHopperNativeHelper;
readonly platform?: NodeJS.Platform;
readonly acquireTargetLease?: typeof acquireHopperTargetLease;
readonly acquireApplicationLease?: typeof acquireLinuxHopperApplicationLease;
@@ -213,6 +217,13 @@ export class HopperApplicationLauncher implements BridgeLauncher {
if (signal?.aborted === true) return err(new HopperCancelledError());
const display = await this.#privateDisplay(signal);
if (!display.ok) return err(display.error);
const nativeHelperVerified =
(this.dependencies.platform ?? process.platform) === "darwin" &&
this.options.launchMode === "native" &&
(await (
this.dependencies.verifyNativeHelper ?? verifyMacHopperNativeHelper
)(this.options.launcherPath, this.options.environment, signal)) ===
"verified";
const prepared = await prepareHopperApplication(
this.options.launcherPath,
this.options.environment,
@@ -257,6 +268,9 @@ export class HopperApplicationLauncher implements BridgeLauncher {
...ownedLauncher,
providerLifetime: "external-application",
shutdownMode: "bridge-request",
...(nativeHelperVerified
? { nativeHelperVerified: true as const }
: {}),
// The vendor helper interpolates these values into AppleScript
// literals without escaping. Preserve them, but withhold the
// pre-handoff conclusion when they can alter that script.
+1
View File
@@ -110,6 +110,7 @@ const HOPPER_DECLINED_ARCHITECTURE_LABELS: Record<
> = {
mips: "MIPS",
riscv: "RISC-V",
powerpc: "PowerPC",
};
/** Derive Hopper's complete non-interactive CLI loader selection. */
+12
View File
@@ -59,6 +59,7 @@ import {
import { connectHopperSocketOnce } from "./HopperSocketConnection.js";
import {
hopperLauncherFailureDiagnostic,
hopperLauncherReportedApplicationNotFound,
nativeHopperLaunchWasDenied,
} from "./HopperProcessDiagnostic.js";
import { DEFAULT_HOPPER_STARTUP_TIMEOUT_MS } from "../config/providerStartupTimeout.js";
@@ -768,6 +769,17 @@ export class HopperClient {
launcherFailure,
});
}
if (hopperLauncherReportedApplicationNotFound(launcherFailure)) {
// Only the verified native helper's Spotlight exit precedes any target
// handoff. Wrappers can forward the same message after opening a document.
if (this.#resources.launch?.nativeHelperVerified === true)
this.#resources.shutdownConfirmed = true;
return new HopperStartError({
userMessage:
"Hopper's launcher could not find Hopper because it relies on Spotlight. Enable Spotlight indexing for the volume that contains Hopper (for example: sudo mdutil -i on /System/Volumes/Data), wait for indexing to finish, then retry opening the target.",
launcherFailure,
});
}
return new HopperStartError({
userMessage:
"Hopper's launcher helper failed before bridge readiness. Review details.launcher and complete Hopper setup before retrying.",
+36
View File
@@ -0,0 +1,36 @@
import { HOPPER_PUBLISHER } from "../domain/hopperPublisher.js";
import { macDeveloperIdRequirement } from "../process/MacCodeSignature.js";
import { execFileOutput } from "../process/ExecFileOutput.js";
/** Failure or unavailable verification leaves custom/native provenance unknown. */
export const verifyMacHopperNativeHelper = async (
launcherPath: string,
environment: Readonly<NodeJS.ProcessEnv>,
signal?: AbortSignal,
): Promise<"verified" | "unknown"> => {
// codesign treats a bare name as a local file, while spawn searches PATH.
// Without an explicit path they need not identify the same executable.
if (!launcherPath.includes("/")) return "unknown";
try {
await execFileOutput(
"/usr/bin/codesign",
[
"--verify",
"--strict",
`-R${macDeveloperIdRequirement("hopper", HOPPER_PUBLISHER.teamIdentifier)}`,
launcherPath,
],
{
env: environment,
...(signal === undefined ? {} : { signal }),
timeout: 5_000,
stopSignal: "SIGKILL",
},
);
return "verified";
} catch (cause: unknown) {
// Optional provenance: a failed check must never authorize lease release.
void cause;
return "unknown";
}
};
+44 -1
View File
@@ -1,7 +1,10 @@
import { expect, it } from "vitest";
import type { HopperLauncherOutcome } from "../domain/hopperErrors.js";
import { nativeHopperLaunchWasDenied } from "./HopperProcessDiagnostic.js";
import {
hopperLauncherReportedApplicationNotFound,
nativeHopperLaunchWasDenied,
} from "./HopperProcessDiagnostic.js";
// Captured from Hopper 6.1.0-demo and 6.6.0-demo's unmodified helpers with native
// AppleScript error -1743 before open dispatch. The target script never ran.
@@ -23,6 +26,8 @@ it("recognizes the native helper's complete authorization rejection", () => {
});
it.each([
{ stdout: { text: "wrapper output\n", bytes: 15, retained_bytes: 15 } },
{ stdout: { text: "", bytes: 1, retained_bytes: 0 } },
{ output_closed: false },
{ diagnostic_truncated: true },
{ signal: "SIGTERM", exit_code: null },
@@ -52,3 +57,41 @@ it.each([
expect(nativeHopperLaunchWasDenied({ ...denied, ...change })).toBe(false);
},
);
// Captured from Hopper 6's unmodified helper with Spotlight indexing disabled.
const notFound: HopperLauncherOutcome = {
exit_code: 3,
signal: null,
stdout: { text: "", bytes: 0, retained_bytes: 0 },
stderr: {
text: "Hopper has not been found.\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable.\n",
bytes: 121,
retained_bytes: 121,
},
output_closed: true,
diagnostic_truncated: false,
};
it("recognizes the native helper's complete Spotlight not-found report", () => {
expect(hopperLauncherReportedApplicationNotFound(notFound)).toBe(true);
expect(nativeHopperLaunchWasDenied(notFound)).toBe(false);
});
it.each([
{ stdout: { text: "wrapper output\n", bytes: 15, retained_bytes: 15 } },
{ stdout: { text: "", bytes: 1, retained_bytes: 0 } },
{ output_closed: false },
{ diagnostic_truncated: true },
{ signal: "SIGTERM", exit_code: null },
{ exit_code: 2 },
{
stderr: {
...notFound.stderr,
text: `${notFound.stderr.text}another failure\n`,
},
},
])("rejects other not-found diagnostic shapes: %j", (change) => {
expect(
hopperLauncherReportedApplicationNotFound({ ...notFound, ...change }),
).toBe(false);
});
+25 -2
View File
@@ -6,6 +6,14 @@ import type { ProviderProcessDiagnostic } from "../process/ProviderProcess.js";
import { parseLinuxPrivateDisplayDiagnostic } from "./LinuxPrivateDisplayDiagnostic.js";
import type { HopperLauncherOutcome } from "../domain/hopperErrors.js";
/** Recognize launcher stderr signatures only with complete, empty stdout. */
const hasCompleteLauncherOutput = (outcome: HopperLauncherOutcome): boolean =>
outcome.output_closed &&
!outcome.diagnostic_truncated &&
outcome.stdout.text === "" &&
outcome.stdout.bytes === 0 &&
outcome.stdout.retained_bytes === 0;
/**
* Hopper's native helper sends activate before its open-document Apple event.
* Recognize its explicit authorization rejection. The launcher must separately
@@ -18,12 +26,27 @@ export const nativeHopperLaunchWasDenied = (
outcome !== undefined &&
outcome.exit_code === 2 &&
outcome.signal === null &&
outcome.output_closed &&
!outcome.diagnostic_truncated &&
hasCompleteLauncherOutput(outcome) &&
/^Cannot launch Hopper!\r?\nNot authorized to send Apple events to [^\r\n]+\.\r?\n?$/u.test(
outcome.stderr.text,
);
/**
* Recognize the launcher's complete Spotlight diagnostic for actionable setup
* guidance. Custom launchers may forward this report after a handoff, so the
* report does not establish whether a document exists or release ownership.
*/
export const hopperLauncherReportedApplicationNotFound = (
outcome: HopperLauncherOutcome | undefined,
): outcome is HopperLauncherOutcome =>
outcome !== undefined &&
outcome.exit_code === 3 &&
outcome.signal === null &&
hasCompleteLauncherOutput(outcome) &&
/^Hopper has not been found\.\r?\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable\.\r?\n?$/u.test(
outcome.stderr.text,
);
/** Extract only a matching Linux adapter failure record. */
export const hopperLauncherFailureDiagnostic = (
event: Extract<ProviderProcessDiagnostic, { readonly type: "exit" }>,
+5 -1
View File
@@ -126,7 +126,11 @@ export class HopperProvider implements AnalysisProviderCandidate {
status: "unsupported",
code: "architecture_unsupported",
reason: `REA's Hopper adapter does not admit ${
target.architecture === "mips" ? "MIPS" : "RISC-V"
target.architecture === "mips"
? "MIPS"
: target.architecture === "riscv"
? "RISC-V"
: "PowerPC"
} targets.`,
diagnostics,
};
+5 -1
View File
@@ -113,7 +113,11 @@ export class IdaProvider implements AnalysisProviderCandidate {
status: "unsupported",
code: "architecture_unsupported",
reason: `REA's IDA adapter has no verified ${
target.architecture === "mips" ? "MIPS" : "RISC-V"
target.architecture === "mips"
? "MIPS"
: target.architecture === "riscv"
? "RISC-V"
: "PowerPC"
} lane. Select a supported Ghidra profile instead.`,
diagnostics,
};
+7
View File
@@ -0,0 +1,7 @@
/** Require a sealed identifier and Apple Developer ID signing team together. */
export const macDeveloperIdRequirement = (
identifier: string,
teamIdentifier: string,
): string =>
`= identifier "${identifier}" and anchor apple generic` +
` and certificate leaf[subject.OU] = "${teamIdentifier}"`;
@@ -50,7 +50,42 @@ test(
});
const mcp = new Client({ name: "hopper-literal-boundary", version: "1" });
let runtime: string | undefined;
onTestFinished(async () => {
try {
await mcp.connect(transport);
const call = (name: string, arguments_: Record<string, string> = {}) =>
mcp.callTool({ name, arguments: arguments_ });
expect((await call("open_binary", { path: target })).isError).not.toBe(
true,
);
const first = parseMcpToolError(await call("current_document"));
runtime = dirname(await readFile(receipt, "utf8"));
expect(first.error).toMatchObject({
code: "cleanup_incomplete",
details: {
diagnostics: {
primary_error: {
code: "provider_unavailable",
details: {
launcher: {
exit_code: 2,
stderr: {
text: expect.stringContaining(
"Not authorized to send Apple events",
),
},
},
},
},
},
},
});
await access(`${receipt}.document-created`);
await access(runtime);
expect(parseMcpToolError(await call("list_procedures"))).toEqual(first);
expect(parseMcpToolError(await call("close_binary")).error.code).toBe(
"cleanup_incomplete",
);
} finally {
// SDK close can return immediately after SIGKILL. Wait for its actual
// child-close event before recovering this process's retained lease.
let serverExited: Promise<void> | undefined;
@@ -77,41 +112,7 @@ test(
});
if (!acquired.acquired) throw new Error("Fixture lease remained live");
await acquired.lease.release();
});
await mcp.connect(transport);
const call = (name: string, arguments_: Record<string, string> = {}) =>
mcp.callTool({ name, arguments: arguments_ });
expect((await call("open_binary", { path: target })).isError).not.toBe(
true,
);
const first = parseMcpToolError(await call("current_document"));
runtime = dirname(await readFile(receipt, "utf8"));
expect(first.error).toMatchObject({
code: "cleanup_incomplete",
details: {
diagnostics: {
primary_error: {
code: "provider_unavailable",
details: {
launcher: {
exit_code: 2,
stderr: {
text: expect.stringContaining(
"Not authorized to send Apple events",
),
},
},
},
},
},
},
});
await access(`${receipt}.document-created`);
await access(runtime);
expect(parseMcpToolError(await call("list_procedures"))).toEqual(first);
expect(parseMcpToolError(await call("close_binary")).error.code).toBe(
"cleanup_incomplete",
);
}
}
},
);
@@ -201,13 +202,13 @@ test("recovers a denied Hopper launch through the same stdio MCP session and CLI
).toBe(false);
});
test(
"preserves a timed-out handoff and closes only a late bridge with matching run identity",
test.for(["handoff", "not-found"] as const)(
"preserves an unconfirmed %s handoff and closes only a late bridge with matching run identity",
{ timeout: 60_000 },
async ({ workspace }) => {
async (launchMode, { workspace }) => {
const target = workspace.path("target");
await copyFile("/usr/bin/true", target);
const mode = await workspace.write("mode", "handoff");
const mode = await workspace.write("mode", launchMode);
const receipt = workspace.path("bootstrap-path");
const transport = new StdioClientTransport({
command: process.execPath,
@@ -257,17 +258,35 @@ test(
details: {
diagnostics: {
primary_error: {
code: "provider_timeout",
details: { timeout_ms: 2000, launcher: { exit_code: 0 } },
code:
launchMode === "handoff"
? "provider_timeout"
: "provider_unavailable",
details:
launchMode === "handoff"
? { timeout_ms: 2000, launcher: { exit_code: 0 } }
: { stage: "launch", launcher: { exit_code: 3 } },
},
},
},
});
if (launchMode === "not-found") {
await access(`${receipt}.document-created`);
expect(initial.error.details).toMatchObject({
diagnostics: {
primary_error: { message: expect.stringContaining("Spotlight") },
},
});
}
expect(parseMcpToolError(await call("list_procedures"))).toEqual(initial);
const status = z.object({
result: z.object({
provider_operation_health: z.object({
state: z.literal("not_started"),
// A timeout observes incomplete startup; a failed launcher without
// confirmed ownership leaves the provider's state unknown.
state: z.literal(
launchMode === "handoff" ? "not_started" : "unknown",
),
stage: z.literal("launch"),
}),
}),
@@ -289,7 +308,7 @@ test(
const token = setting("REA_TOKEN");
const runId = setting("REA_RUN_ID");
runtime = dirname(socket);
const lateBridge = async (identity: string, credential = token) => {
const lateBridge = (identity: string, credential = token) => {
const child = spawn(
process.execPath,
[
@@ -301,14 +320,14 @@ test(
{ stdio: "ignore" },
);
children.push(child);
await vi.waitFor(() => access(socket));
return child;
};
for (const [identity, credential] of [
["different-run", token],
[runId, "different-token"],
] as const) {
const wrong = await lateBridge(identity, credential);
const wrong = lateBridge(identity, credential);
await vi.waitFor(() => access(socket));
expect(parseMcpToolError(await call("close_binary")).error.code).toBe(
"cleanup_incomplete",
);
@@ -319,7 +338,9 @@ test(
}
const closing = call("close_binary");
await delay(150);
const owned = await lateBridge(runId);
// A verified close may remove the socket before the fixture can poll it.
// The successful close and child exit establish the owned bridge outcome.
const owned = lateBridge(runId);
expect((await closing).isError).not.toBe(true);
await vi.waitFor(() => expect(owned.exitCode).toBe(0));
await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" });
@@ -45,6 +45,7 @@ const fixture = async () => {
for (const path of [
"scripts/run-development-tests.mjs",
"scripts/lib/development-tests.mjs",
"scripts/lib/test-projects.mjs",
])
await copyFile(join(process.cwd(), path), join(root, path));
await symlink(
@@ -1,6 +1,6 @@
import { chmod, copyFile, rm } from "node:fs/promises";
import { describe, expect, onTestFinished } from "vitest";
import { describe, expect } from "vitest";
import { z } from "zod";
import { workspaceCliTest } from "../../support/cli/workspaceCliFixture.js";
@@ -39,7 +39,57 @@ describe("Hopper launcher failure through the one-shot CLI", () => {
const targetPath = workspace.path("probe");
await copyFile("/usr/bin/true", targetPath);
const loaderArgs = ["--probe"];
onTestFinished(async () => {
try {
for (const attempt of ["first", "retry"]) {
const result = await cli.run({
arguments: [
"analyze",
targetPath,
"--provider",
"hopper",
"--json",
],
environment: {
...workspace.environment,
HOPPER_LAUNCHER_PATH: launcherPath,
HOPPER_LOADER_ARGS_JSON: JSON.stringify(loaderArgs),
REA_LOG_LEVEL: "silent",
},
timeoutMs: 45_000,
});
const retained = RetainedRuntime.safeParse(result.json);
if (retained.success)
await rm(
retained.data.details.diagnostics.cleanup_error.details
.diagnostics.runtime_root,
{ recursive: true, force: true },
);
expect(result.exitCode, attempt).toBe(1);
expect(result.json, attempt).toMatchObject({
code: "cleanup_incomplete",
details: {
cleanup: "incomplete",
diagnostics: {
primary_error: {
code: "provider_unavailable",
details: {
stage: "launch",
launcher: {
exit_code: 2,
stderr: {
text: expect.stringContaining(
"Document state is unknown",
),
},
},
},
},
},
},
});
}
} finally {
const recovered = await acquireHopperTargetLease({
targetPath,
targetKind: "executable",
@@ -48,50 +98,6 @@ describe("Hopper launcher failure through the one-shot CLI", () => {
});
expect(recovered.acquired).toBe(true);
if (recovered.acquired) await recovered.lease.release();
});
for (const attempt of ["first", "retry"]) {
const result = await cli.run({
arguments: ["analyze", targetPath, "--provider", "hopper", "--json"],
environment: {
...workspace.environment,
HOPPER_LAUNCHER_PATH: launcherPath,
HOPPER_LOADER_ARGS_JSON: JSON.stringify(loaderArgs),
REA_LOG_LEVEL: "silent",
},
timeoutMs: 45_000,
});
const retained = RetainedRuntime.safeParse(result.json);
if (retained.success)
await rm(
retained.data.details.diagnostics.cleanup_error.details.diagnostics
.runtime_root,
{ recursive: true, force: true },
);
expect(result.exitCode, attempt).toBe(1);
expect(result.json, attempt).toMatchObject({
code: "cleanup_incomplete",
details: {
cleanup: "incomplete",
diagnostics: {
primary_error: {
code: "provider_unavailable",
details: {
stage: "launch",
launcher: {
exit_code: 2,
stderr: {
text: expect.stringContaining(
"Document state is unknown",
),
},
},
},
},
},
},
});
}
},
);
@@ -226,10 +226,6 @@ it("bounds the rolling static snapshot with Turbo age and size eviction", async
.object({
cacheMaxAge: z.string(),
cacheMaxSize: z.string(),
tasks: z.record(
z.string(),
z.object({ inputs: z.array(z.unknown()).optional() }),
),
})
.parse(JSON.parse(await readFile(join(repository, "turbo.json"), "utf8")));
// The static job restores the previous snapshot by prefix and saves the
@@ -248,5 +244,4 @@ it("bounds the rolling static snapshot with Turbo age and size eviction", async
unit === "TB" ? value * 1024 * 1024 : unit === "GB" ? value * 1024 : value;
expect(megabytes).toBeGreaterThan(0);
expect(megabytes).toBeLessThanOrEqual(1024);
expect(turbo.tasks["knip"]?.inputs).toContain("vitest.config.ts");
});
@@ -0,0 +1,435 @@
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs";
import {
readStaticCheckArtifact,
selectStaticCheckArtifact,
} from "../../../scripts/ci/artifacts.mjs";
import {
classifyFailure,
createGitHubApi,
reportCiFailures,
type CiRun,
type CiJob,
type FailureReportApi,
} from "../../../scripts/ci/failure-report.mjs";
const sha = (digit: string) => digit.repeat(40);
const diagnostic = {
kind: "lint",
path: "src/example.ts",
code: "no-debugger",
message: "Unexpected debugger statement.",
severity: "error",
} as const;
const context = (
sourceSha: string,
taskHash = "a".repeat(16),
prHeadSha?: string,
) => ({
version: 1 as const,
sourceSha,
...(prHeadSha ? { prHeadSha } : {}),
sourceClean: true,
nodeVersion: "v24.18.0",
npmVersion: "11.16.0",
turboVersion: "2.11.7",
platform: "linux",
architecture: "x64",
taskHashes: { "lint:built": taskHash },
});
const run = (overrides: Partial<CiRun> = {}) =>
normalizeWorkflowRun({
id: 10,
run_attempt: 1,
status: "completed",
conclusion: "failure",
event: "pull_request",
path: ".github/workflows/ci.yml@main",
head_sha: sha("a"),
head_branch: "feature",
repository: { full_name: "owner/rea" },
pull_requests: [
{ base: { sha: sha("b"), repo: { full_name: "owner/rea" } } },
],
...overrides,
});
const job = (overrides: Partial<CiJob> = {}) => ({
name: "Static checks",
status: "completed",
conclusion: "failure",
...overrides,
});
const comparison = (): Parameters<typeof classifyFailure>[0] => ({
repository: "owner/rea",
defaultBranch: "main",
run: run(),
job: job(),
diagnostics: [diagnostic],
context: context(sha("a")),
mainRun: run({
id: 20,
event: "push",
head_sha: sha("c"),
head_branch: "main",
pull_requests: [],
}),
mainJob: job(),
mainDiagnostics: [diagnostic],
mainContext: context(sha("c")),
executionSourcesVerified: true,
ancestry: { status: "ahead", mergeBaseSha: sha("c") },
blobs: { "src/example.ts": true },
});
it("attributes only an identical structured lint failure with matching task, context, ancestry, and source blob", () => {
const result = classifyFailure(comparison());
expect(result).toEqual({
classification: "matching-main-diagnostics",
reason:
"supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
});
});
it.each([
["task hash", { mainContext: context(sha("c"), "b".repeat(16)) }],
["dirty source", { context: { ...context(sha("a")), sourceClean: false } }],
[
"toolchain",
{ mainContext: { ...context(sha("c")), turboVersion: "other" } },
],
[
"source sha",
{ context: context(sha("d")), executionSourcesVerified: false },
],
[
"diagnostic",
{ mainDiagnostics: [{ ...diagnostic, message: "different message" }] },
],
["content", { blobs: { "src/example.ts": false } }],
["ancestry", { ancestry: { status: "diverged", mergeBaseSha: sha("e") } }],
["canceled job", { job: job({ conclusion: "cancelled" }) }],
["not a static job", { job: job({ name: "Test shard (1)" }) }],
] as const)("does not attribute on mismatched %s", (_label, overrides) => {
const result = classifyFailure({ ...comparison(), ...overrides });
expect(result.classification).toBe("unknown");
});
it.each([
[{}, "diagnostic_source_blob_identity_unavailable"],
[
{ "src/example.ts": false },
"diagnostic_source_content_not_identical_at_main_and_tested_base",
],
] as const)(
"distinguishes unavailable blob identity from verified differences (%s)",
(blobs, reason) => {
expect(classifyFailure({ ...comparison(), blobs })).toEqual({
classification: "unknown",
reason,
});
},
);
it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => {
const directory = await createTestTempDirectory("rea-ci-failure-test-");
const calls: string[] = [];
const manifest = {
repository: "owner/rea",
truncation: [],
coverage: { exhaustive: false },
runs: [
{
id: 10,
runAttempt: 1,
workflow: "CI",
event: "pull_request",
conclusion: "failure",
status: "collected",
},
{
id: 11,
runAttempt: 1,
workflow: "CI",
event: "pull_request",
conclusion: "success",
status: "collected",
},
],
};
const failedRun = run();
const api: FailureReportApi = {
async getRun(_repository, id) {
calls.push(`run:${id}`);
return failedRun;
},
async getJobs(_repository, id) {
calls.push(`jobs:${id}`);
return [{ ...job(), run_id: Number(id), run_attempt: 1 }];
},
async getStaticEvidence() {
calls.push("artifact");
return null;
},
async getAncestry() {
throw new Error("must not compare without evidence");
},
async verifyExecutionSource() {
throw new Error("must not verify source without evidence");
},
async getBlobIdentity() {
throw new Error("must not inspect blobs without evidence");
},
};
const report = await reportCiFailures({
repository: "owner/rea",
defaultBranch: "main",
batchManifest: manifest,
api,
outputDirectory: directory,
});
expect(calls).toEqual(["run:10", "jobs:10", "artifact"]);
expect(report).toMatchObject({
informationalOnly: true,
gateEffect: "none; this report never changes CI conclusions",
counts: { unknown: 1 },
});
const json = JSON.parse(
await readFile(join(directory, "report.json"), "utf8"),
);
const markdown = await readFile(join(directory, "report.md"), "utf8");
expect(json.failures[0]).toMatchObject({
classification: "unknown",
reason: "no_matching_verified_main_failure",
});
expect(markdown).toContain("Informational only");
expect(markdown).toContain("Unknown means evidence was incomplete");
});
it.each([
[22, false],
[6, true],
] as const)(
"finds verified main evidence after %s newer PR runs (extra incorrect main hint: %s)",
async (prCount, incorrectMainHints) => {
const directory = await createTestTempDirectory("rea-ci-main-proof-");
const mainSha = sha("c");
const prHeadSha = sha("a");
const mergeSha = sha("d");
const mainRun = run({
id: 20,
event: "push",
head_sha: mainSha,
head_branch: "main",
pull_requests: [],
});
const prRun = run({
head_sha: prHeadSha,
pull_requests: [
{ base: { sha: mainSha, repo: { full_name: "owner/rea" } } },
],
});
const evidence = (sourceSha: string, prHead?: string) => ({
context: context(sourceSha, undefined, prHead),
diagnostics: [diagnostic],
});
const lookups: number[] = [];
const api: FailureReportApi = {
async getRun(_repository, id) {
lookups.push(Number(id));
return Number(id) === 20 ? mainRun : { ...prRun, id };
},
async getJobs(_repository, id) {
return [{ ...job(), run_id: Number(id), run_attempt: 1 }];
},
async getStaticEvidence(_repository, selectedRun) {
return Number(selectedRun.id) === 20
? evidence(mainSha)
: evidence(mergeSha, prHeadSha);
},
async verifyExecutionSource(_repository, selectedRun, sourceContext) {
return selectedRun.event === "pull_request"
? sourceContext.sourceSha === mergeSha &&
sourceContext.prHeadSha === selectedRun.head_sha
: sourceContext.sourceSha === selectedRun.head_sha;
},
async getAncestry() {
return { status: "ahead", mergeBaseSha: mainSha };
},
async getBlobIdentity() {
return { "src/example.ts": true };
},
};
const result = await reportCiFailures({
repository: "owner/rea",
defaultBranch: "main",
batchManifest: {
repository: "owner/rea",
runs: [
...(incorrectMainHints
? [
{
id: 30,
runAttempt: 1,
workflow: "CI",
event: "push",
headBranch: "main",
conclusion: "failure",
status: "collected",
},
]
: []),
...Array.from({ length: prCount }, (_, index) => ({
id: 100 + index,
runAttempt: 1,
workflow: "CI",
event: "pull_request",
headBranch: "feature",
conclusion: "failure",
status: "collected",
})),
{
id: 20,
runAttempt: 1,
workflow: "CI",
event: "push",
headBranch: "main",
conclusion: "failure",
status: "collected",
},
],
},
api,
outputDirectory: directory,
});
expect(result.failures[0]).toMatchObject({
runId: 100,
classification: "matching-main-diagnostics",
reason:
"supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
});
expect(lookups).toHaveLength(
Math.min(prCount, 20) + 1 + Number(incorrectMainHints),
);
expect(lookups[0]).toBe(incorrectMainHints ? 30 : 20);
expect(new Set(lookups).size).toBe(lookups.length);
expect(result.boundedInput.mainCandidateRuns).toBe(1);
expect(result.failures).toHaveLength(Math.min(prCount, 20));
},
);
it("proves checkout merge parents in base-then-PR-head order and rejects false identities", async () => {
const baseSha = sha("b");
const headSha = sha("a");
const mergeSha = sha("c");
const prRun = run({
head_sha: headSha,
pull_requests: [
{ base: { sha: baseSha, repo: { full_name: "owner/rea" } } },
],
});
let parents = [baseSha, headSha];
const calls: string[] = [];
const api = createGitHubApi({
token: "fixture-token",
apiRequest: async (_method, endpoint) => {
calls.push(endpoint);
return {
sha: mergeSha,
parents: parents.map((parentSha) => ({ sha: parentSha })),
};
},
});
const mergeContext = context(mergeSha, undefined, headSha);
expect(
await api.verifyExecutionSource("owner/rea", prRun, mergeContext),
).toBe(true);
expect(calls).toEqual([`repos/owner/rea/commits/${mergeSha}`]);
parents = [headSha, baseSha];
expect(
await api.verifyExecutionSource("owner/rea", prRun, mergeContext),
).toBe(false);
expect(
await api.verifyExecutionSource("owner/rea", prRun, {
...mergeContext,
prHeadSha: sha("e"),
}),
).toBe(false);
});
it("uses static artifacts only when listing is complete, unique, and tied to this run attempt", () => {
const artifact = {
id: 12,
name: "static-check-reports",
expired: false,
workflow_run: { id: 10, head_sha: sha("a") },
};
const runRecord = { id: 10, head_sha: sha("a") };
expect(
selectStaticCheckArtifact(
{ total_count: 1, artifacts: [artifact] },
runRecord,
1,
),
).toBe(artifact);
expect(
selectStaticCheckArtifact(
{ total_count: 2, artifacts: [artifact] },
runRecord,
1,
),
).toBeNull();
expect(
selectStaticCheckArtifact(
{ total_count: 2, artifacts: [artifact, artifact] },
runRecord,
1,
),
).toBeNull();
expect(
selectStaticCheckArtifact(
{
total_count: 1,
artifacts: [
{ ...artifact, workflow_run: { id: 11, head_sha: sha("a") } },
],
},
runRecord,
1,
),
).toBeNull();
expect(
selectStaticCheckArtifact(
{ total_count: 1, artifacts: [artifact] },
runRecord,
2,
),
).toBeNull();
});
it("rejects artifact redirects that could send the bearer token to a non-HTTPS host", async () => {
const calls: string[] = [];
await expect(
readStaticCheckArtifact({
repository: "owner/rea",
artifactId: 12,
token: "fixture-token",
fetchImpl: (async (url: string | URL | Request) => {
calls.push(String(url));
return new Response(null, {
status: 302,
headers: { location: "http://untrusted.example/artifact.zip" },
});
}) as typeof fetch,
}),
).rejects.toThrow("static_artifact_redirect_invalid");
expect(calls).toEqual([
"https://api.github.com/repos/owner/rea/actions/artifacts/12/zip",
]);
});
@@ -0,0 +1,407 @@
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import {
reportCiFailures,
createGitHubApi,
type CiRun,
type FailureReportApi,
type StaticDiagnostic,
type StaticEvidence,
} from "../../../scripts/ci/failure-report.mjs";
const repository = "owner/rea";
const mainSha = "c".repeat(40);
const prSha = "a".repeat(40);
const diagnostic: StaticDiagnostic = {
kind: "lint",
path: "src/example.ts",
code: "no-debugger",
message: "Unexpected debugger statement.",
severity: "error",
};
const evidence = (
run: CiRun,
diagnostics: readonly StaticDiagnostic[],
): StaticEvidence => ({
context: {
version: 1,
sourceSha: run.head_sha!,
sourceClean: true,
nodeVersion: "v24.18.0",
npmVersion: "11.16.0",
turboVersion: "2.11.7",
platform: "linux",
architecture: "x64",
taskHashes: { "lint:built": "a".repeat(16) },
},
diagnostics,
});
const entry = (id: number, event = "pull_request", runAttempt = 1) => ({
id,
runAttempt,
workflow: "CI",
event,
headBranch: event === "pull_request" ? "feature" : "main",
status: "collected",
conclusion: "failure",
});
function fixture(mainIds: readonly number[], prCount: number) {
const jobs: number[] = [];
const artifacts: number[] = [];
const paths: (readonly string[])[] = [];
const runs: Array<
Omit<ReturnType<typeof entry>, "event"> & { event?: string }
> = [
...mainIds.map((id) => entry(id, "schedule")),
...Array.from({ length: prCount }, (_, index) => entry(100 + index)),
];
const api: FailureReportApi = {
async getRun(_repository, id) {
const selected = runs.find((run) => run.id === Number(id))!;
return {
id,
run_attempt: selected.runAttempt,
status: "completed",
conclusion: "failure",
...(selected.event ? { event: selected.event } : {}),
head_branch: selected.headBranch,
path: ".github/workflows/ci.yml@main",
head_sha: mainIds.includes(Number(id)) ? mainSha : prSha,
repository: { full_name: repository },
pull_requests:
selected.event === "pull_request"
? [{ base: { sha: mainSha, repo: { full_name: repository } } }]
: [],
};
},
async getJobs(_repository, id, attempt) {
jobs.push(Number(id));
return [
{
name: "Static checks",
status: "completed",
conclusion: "failure",
run_id: Number(id),
run_attempt: attempt,
},
];
},
async getStaticEvidence(_repository, run) {
artifacts.push(Number(run.id));
return evidence(run, [diagnostic]);
},
async getAncestry() {
return { status: "identical", mergeBaseSha: mainSha };
},
async getBlobIdentity(_repository, _ancestor, _descendant, requested) {
paths.push(requested);
return Object.fromEntries(requested.map((path) => [path, true]));
},
async verifyExecutionSource() {
return true;
},
};
return { api, runs, jobs, artifacts, paths };
}
async function report(input: ReturnType<typeof fixture>) {
return await reportCiFailures({
repository,
defaultBranch: "main",
batchManifest: { repository, runs: input.runs },
api: input.api,
outputDirectory: await createTestTempDirectory("rea-ci-budget-"),
});
}
it("selects PR failures before the cap even after a day of scheduled main failures", async () => {
const input = fixture(
Array.from({ length: 24 }, (_, index) => 20 + index),
1,
);
input.runs.push({
id: 200,
runAttempt: 1,
workflow: "CI",
headBranch: "feature",
status: "collected",
conclusion: "failure",
});
const result = await report(input);
expect(result.failures).toMatchObject([
{ runId: 100, classification: "matching-main-diagnostics" },
]);
expect(result.boundedInput).toMatchObject({
candidateRuns: 1,
mainCandidateRuns: 5,
manifestTruncated: false,
});
expect(input.jobs).toEqual([100, 20]);
});
it("reuses five main job/artifact observations across twenty completed PR comparisons", async () => {
const input = fixture([20, 21, 22, 23, 24], 20);
const load = input.api.getStaticEvidence;
input.api.getStaticEvidence = async (...args) => {
const result = await load(...args);
return Number(args[1].id) < 24
? evidence(args[1], [{ ...diagnostic, message: "different" }])
: result;
};
const result = await report(input);
expect(result.counts).toEqual({
"matching-main-diagnostics": 20,
unknown: 0,
});
expect(input.paths).toHaveLength(100);
expect(input.jobs).toHaveLength(25);
expect(input.artifacts).toHaveLength(25);
for (const id of [20, 21, 22, 23, 24]) {
expect(input.jobs.filter((item) => item === id)).toHaveLength(1);
expect(input.artifacts.filter((item) => item === id)).toHaveLength(1);
}
});
it.each(["missing", "artifact error", "job error", "wrong job attempt"])(
"reuses unavailable main evidence without retries: %s",
async (failure) => {
const input = fixture([20], 3);
const load = input.api.getStaticEvidence;
const jobs = input.api.getJobs;
input.api.getJobs = async (...args) => {
const result = await jobs(...args);
if (Number(args[1]) !== 20) return result;
if (failure === "job error") throw new Error("job_snapshot_unavailable");
return failure === "wrong job attempt"
? result.map((job) => ({ ...job, run_attempt: 2 }))
: result;
};
input.api.getStaticEvidence = async (...args) => {
const result = await load(...args);
if (Number(args[1].id) !== 20) return result;
if (failure === "artifact error") throw new Error("artifact_unavailable");
return null;
};
const result = await report(input);
expect(result.counts.unknown).toBe(3);
expect(input.jobs.filter((id) => id === 20)).toHaveLength(1);
expect(input.artifacts.filter((id) => id === 20)).toHaveLength(
failure === "job error" || failure === "wrong job attempt" ? 0 : 1,
);
},
);
it.each([1, 2])(
"refreshes unavailable main evidence in the next report (attempt %s)",
async (attempt) => {
const input = fixture([20], 1);
expect((await report(input)).counts["matching-main-diagnostics"]).toBe(1);
input.runs[0]!.runAttempt = attempt;
const load = input.api.getStaticEvidence;
input.api.getStaticEvidence = async (...args) => {
const result = await load(...args);
return Number(args[1].id) === 20 ? null : result;
};
expect((await report(input)).counts.unknown).toBe(1);
expect(input.jobs.filter((id) => id === 20)).toHaveLength(2);
expect(input.artifacts.filter((id) => id === 20)).toHaveLength(2);
},
);
it.each([1, 51])(
"fetches only PR paths with %s PR diagnostics and noisy main evidence",
async (count) => {
const input = fixture([20], 1);
const load = input.api.getStaticEvidence;
const diagnostics = Array.from({ length: count }, (_, index) => ({
...diagnostic,
path: `src/pr-${index}.ts`,
}));
input.api.getStaticEvidence = async (...args) => {
await load(...args);
const noise =
Number(args[1].id) === 20
? Array.from({ length: 60 }, (_, index) => ({
...diagnostic,
path: `src/main-${index}.ts`,
}))
: [];
return evidence(args[1], [...noise, ...diagnostics]);
};
const result = await report(input);
expect(input.paths).toEqual([
diagnostics.slice(0, 50).map((item) => item.path),
]);
expect(result.failures[0]!.classification).toBe(
count === 1 ? "matching-main-diagnostics" : "unknown",
);
if (count > 50)
expect(result.failures[0]!.reason).toBe(
"diagnostic_source_blob_identity_unavailable",
);
},
);
it.each(["jobs", "artifact", "comparison"])(
"continues after an isolated main %s failure and preserves its provenance",
async (failure) => {
const input = fixture([20, 21], 3);
const jobs = input.api.getJobs;
const load = input.api.getStaticEvidence;
input.api.getJobs = async (...args) => {
const result = await jobs(...args);
if (Number(args[1]) === 20 && failure === "jobs")
throw new Error("main_jobs_unavailable");
return result;
};
input.api.getStaticEvidence = async (...args) => {
const result = await load(...args);
if (Number(args[1].id) === 20 && failure === "artifact")
throw new Error("main_artifact_unavailable");
return Number(args[1].id) === 20 && failure === "comparison"
? {
...result!,
context: { ...result!.context, sourceSha: "d".repeat(40) },
}
: result;
};
input.api.verifyExecutionSource = async (_repository, run, context) => {
if (Number(run.id) === 20 && context.sourceSha !== run.head_sha)
throw new Error("main_comparison_unavailable");
return true;
};
const result = await report(input);
expect(result.counts).toEqual({
"matching-main-diagnostics": 3,
unknown: 0,
});
for (const item of result.failures) {
expect(item.candidateFailures).toEqual([
{
runId: 20,
runAttempt: 1,
reason: `main_${failure}_unavailable`,
},
]);
}
expect(input.jobs.filter((id) => id === 20)).toHaveLength(1);
expect(input.jobs.filter((id) => id === 21)).toHaveLength(1);
expect(input.artifacts.filter((id) => id === 20)).toHaveLength(
failure === "jobs" ? 0 : 1,
);
expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1);
},
);
it.each([0, 1, 2])(
"preserves every failed job when artifact analysis fails at position %s",
async (position) => {
const input = fixture([20], 1);
const jobs = [
{ name: "Python producer", conclusion: "failure" },
{ name: "Test shard (1)", conclusion: "timed_out" },
];
jobs.splice(position, 0, { name: "Static checks", conclusion: "failure" });
input.api.getJobs = async (_repository, id, attempt) =>
jobs.map((job) => ({
...job,
status: "completed",
run_id: Number(id),
run_attempt: attempt,
}));
input.api.getStaticEvidence = async (_repository, _run, job) => {
if (job.name === "Static checks")
throw new Error("static_artifact_unavailable");
return null;
};
const result = await report(input);
expect(result.counts.unknown).toBe(3);
expect(result.failures).toEqual(
jobs.map((job) => ({
runId: 100,
runAttempt: 1,
runHead: prSha,
event: "pull_request",
job: job.name,
conclusion: job.conclusion,
classification: "unknown",
reason:
job.name === "Static checks"
? "static_artifact_unavailable"
: "no_matching_verified_main_failure",
})),
);
},
);
it("keeps unavailable job inventories at run scope", async () => {
const input = fixture([], 1);
input.api.getJobs = async () => {
throw new Error("job_inventory_unavailable");
};
expect((await report(input)).failures).toEqual([
{
runId: 100,
runAttempt: 1,
classification: "unknown",
reason: "job_inventory_unavailable",
},
]);
});
it.each([
["request", "github_api_temporarily_unavailable"],
["budget", "github_api_request_budget_exhausted"],
["missing sha", "github_content_blob_identity_unavailable"],
["invalid sha", "github_content_blob_identity_unavailable"],
["directory", "github_content_blob_identity_unavailable"],
] as const)(
"retains unavailable blob evidence as a comparison error: %s",
async (failure, reason) => {
const input = fixture([20], 1);
const contentApi = createGitHubApi({
token: "fixture-token",
apiRequest: async () => {
if (failure === "request") throw new Error(reason);
if (failure === "missing sha") return { type: "file" };
if (failure === "invalid sha")
return { type: "file", sha: "not-a-sha" };
if (failure === "directory") return [];
return { type: "file", sha: "e".repeat(40) };
},
});
if (failure === "budget") {
for (let id = 1; id <= 250; id++) await contentApi.getRun(repository, id);
}
input.api.getBlobIdentity = contentApi.getBlobIdentity;
const result = await report(input);
expect(result.failures).toMatchObject([
{
job: "Static checks",
classification: "unknown",
reason: "no_matching_verified_main_failure",
candidateFailures: [{ runId: 20, runAttempt: 1, reason }],
},
]);
},
);
it.each([true, false])(
"reports blob equality only from verified file SHAs (equal: %s)",
async (equal) => {
const api = createGitHubApi({
token: "fixture-token",
apiRequest: async (_method, endpoint) => ({
type: "file",
sha:
equal || endpoint.includes(`ref=${mainSha}`)
? "e".repeat(40)
: "f".repeat(40),
}),
});
expect(
await api.getBlobIdentity(repository, mainSha, prSha, ["src/example.ts"]),
).toEqual({ "src/example.ts": equal });
},
);
@@ -0,0 +1,313 @@
import { execFile } from "node:child_process";
import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { runInNewContext } from "node:vm";
import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
import {
parseStaticTaskContext,
recordStaticTaskContext,
} from "../../../scripts/ci/static-task-context.mjs";
import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const execute = promisify(execFile);
const scopeFlags = async () =>
z
.object({ CI_FLAGS: z.array(z.string()) })
.parse(
await import(
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
),
).CI_FLAGS;
const workflow = async () =>
z
.object({
jobs: z.record(
z.string(),
z
.object({
needs: z.union([z.string(), z.array(z.string())]).optional(),
if: z.string().optional(),
steps: z
.array(
z
.object({
run: z.string().optional(),
env: z.record(z.string(), z.string()).optional(),
})
.passthrough(),
)
.optional(),
})
.passthrough(),
),
})
.parse(
parse(
await readFile(
new URL("../../../.github/workflows/ci.yml", import.meta.url),
"utf8",
),
),
).jobs;
it("starts source feedback before build and requires both groups without waiving missing jobs", async () => {
const jobs = await workflow();
expect(
jobs.static?.steps?.find(
(step) => step.run === "npm run ci:static-context",
)?.["continue-on-error"],
).toBe(true);
expect(jobs["test-source"]?.needs).toBe("changes");
expect(jobs["test-shard"]?.needs).toEqual(["changes", "build"]);
expect(
jobs["test-source"]?.steps?.some(
(step) =>
step.run === "npm run test:fast -- --reporter=default --reporter=blob",
),
).toBe(true);
const required = fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
);
const flags = await scopeFlags();
for (const tier of ["fast", "full"]) {
for (const result of [
"success",
"failure",
"cancelled",
"skipped",
"missing",
]) {
const needs = Object.fromEntries(
Object.keys(jobs)
.filter((name) => name !== "required")
.map((name) => [name, { result: "success" }]),
);
const outputs = {
...Object.fromEntries(
flags.map((flag) => [flag, tier === "full" ? "true" : "false"]),
),
tier,
code: "true",
};
const context: Record<
string,
{ result: string; outputs?: Record<string, string> }
> = { ...needs, changes: { result: "success", outputs } };
if (result === "missing") delete context["test-source"];
else context["test-source"] = { result };
const run = execute(process.execPath, [required], {
env: {
...process.env,
CI_JOBS: JSON.stringify(context),
CI_QUALIFICATION_DIRECTORY: "",
},
});
if (
result === "success" ||
(tier === "full" && ["skipped", "missing"].includes(result))
)
await expect(run).resolves.toMatchObject({ stderr: "" });
else
await expect(run).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("test-source"),
});
}
}
});
it("records the actual static task graph with explicit source identity and no ambient environment", async () => {
const directory = await createTestTempDirectory("rea-ci-context-");
const context = await recordStaticTaskContext({ outputDirectory: directory });
const head = await execute("git", ["rev-parse", "HEAD"]);
expect(context.sourceSha).toBe(head.stdout.trim());
expect(context.taskHashes["lint:built"]).toMatch(/^[a-f0-9]{16}$/u);
expect(context.taskHashes.jscpd).toMatch(/^[a-f0-9]{16}$/u);
expect(context).not.toHaveProperty("environmentVariables");
expect(context).not.toHaveProperty("env");
expect(
JSON.parse(await readFile(`${directory}/static-task-context.json`, "utf8")),
).toEqual(context);
for (const invalid of [
undefined,
{},
{ ...context, sourceSha: "bad" },
{ ...context, prHeadSha: "bad" },
{ ...context, taskHashes: { jscpd: "stale" } },
])
expect(() => parseStaticTaskContext(invalid)).toThrow();
});
it("rejects mutation methods at the shared GitHub transport before invoking the CLI", async () => {
const request = createGhApiRequest({ env: { PATH: "" } });
await expect(
request("POST", "repos/owner/repo/actions/runs/1/rerun", {
requests: 0,
startedAt: Date.now(),
limitMs: 1000,
}),
).rejects.toThrow("read-only GET");
});
it("collects basic CI telemetry before optional dependency installation can fail", async () => {
const { jobs } = z
.object({
jobs: z.object({
report: z.object({
steps: z.array(
z.object({
id: z.string().optional(),
run: z.string().optional(),
if: z.string().optional(),
"continue-on-error": z.boolean().optional(),
}),
),
}),
}),
})
.parse(
parse(
await readFile(
new URL("../../../.github/workflows/ci-timings.yml", import.meta.url),
"utf8",
),
),
);
const steps = jobs.report.steps;
const installation = steps.findIndex((step) => step.id === "failure-deps");
for (const script of ["pr-ci-health.mjs", "job-timings-batch.mjs"]) {
const collection = steps.findIndex(
(step) => step.run === `node scripts/ci/${script}`,
);
expect(collection).toBeGreaterThanOrEqual(0);
expect(installation).toBeGreaterThan(collection);
}
expect(steps[installation]).toMatchObject({
run: "npm ci",
"continue-on-error": true,
});
expect(
steps.find((step) => step.id === "npm-cache")?.["continue-on-error"],
).toBe(true);
expect(
steps.find((step) => step.run === "node scripts/ci/failure-report.mjs")?.if,
).toContain("steps.failure-deps.outcome == 'success'");
});
it("merges source and built reports exactly once using the workflow's shard matrix", async () => {
const jobs = await workflow();
const matrix = z
.object({
matrix: z.object({ shard: z.string().regex(/^\$\{\{.*\}\}$/u) }),
})
.parse(jobs["test-shard"]?.strategy).matrix.shard;
const shards = z
.array(z.string().regex(/^\d+\/\d+$/u))
.min(1)
.parse(
runInNewContext(
matrix.slice(3, -2),
{
fromJSON: JSON.parse,
needs: { changes: { outputs: { tier: "fast" } } },
},
{ timeout: 1_000 },
),
);
const labels = Object.fromEntries(
["source", "built"].map((group) => [
group,
z
.string()
.min(1)
.parse(
jobs[group === "source" ? "test-source" : "test-shard"]?.steps?.find(
(step) => step.env?.VITEST_BLOB_LABEL,
)?.env?.VITEST_BLOB_LABEL,
),
]),
);
const root = await createTestTempDirectory("rea-ci-group-merge-");
const config = join(root, "vitest.config.mjs");
await writeFile(
config,
`export default { test: { maxWorkers: 1, projects: [${["source", "built"].map((name) => `{ test: { name: "${name}", globals: true, include: ["${name}*.test.js"] } }`).join(",")}] } };`,
);
const files = [
"source.test.js",
...Array.from(
{ length: shards.length },
(_, index) => `built-${index}.test.js`,
),
];
for (const file of files)
await writeFile(
join(root, file),
'it("runs its selected group", () => expect(true).toBe(true));',
);
const vitest = fileURLToPath(
new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url),
);
const blobs = new Map<string, Buffer>();
const runs = [
{ group: "source", shard: [] },
...shards.map((shard) => ({ group: "built", shard: [`--shard=${shard}`] })),
];
for (const { group, shard } of runs) {
await execute(
process.execPath,
[
vitest,
"run",
"--config",
config,
"--project",
group,
"--reporter=blob",
...shard,
],
{
cwd: root,
env: { ...process.env, VITEST_BLOB_LABEL: labels[group]! },
timeout: 20_000,
},
);
for (const filename of await readdir(join(root, ".vitest/blob"))) {
expect(blobs.has(filename)).toBe(false);
blobs.set(filename, await readFile(join(root, ".vitest/blob", filename)));
}
// Separate hosted jobs start with independent report directories.
await rm(join(root, ".vitest/blob"), { recursive: true });
}
expect(blobs.size).toBe(runs.length);
await mkdir(join(root, ".vitest/blob"), { recursive: true });
for (const [filename, bytes] of blobs)
await writeFile(join(root, ".vitest/blob", filename), bytes);
await execute(
process.execPath,
[
vitest,
"--config",
config,
"--merge-reports",
"--reporter=json",
"--outputFile.json=merged.json",
],
{ cwd: root, timeout: 20_000 },
);
const merged = z
.object({
success: z.literal(true),
testResults: z.array(z.object({ name: z.string() })),
})
.parse(JSON.parse(await readFile(join(root, "merged.json"), "utf8")));
expect(
merged.testResults.map(({ name }) => name.split("/").at(-1)).sort(),
).toEqual(files.toSorted());
});
@@ -18,6 +18,8 @@ it("collects the latest attempt for recent completed CI and Release runs into a
workflow_runs: [
{
id: 42,
event: "push",
head_branch: "main",
run_attempt: 3,
status: "completed",
conclusion: "failure",
@@ -43,51 +45,9 @@ it("collects the latest attempt for recent completed CI and Release runs into a
},
],
},
[
{
jobs: [
{
name: "Release lane",
status: "completed",
conclusion: "success",
created_at: recent,
started_at: recent,
completed_at: new Date(Date.parse(recent) + 50_000).toISOString(),
steps: [],
},
],
},
],
[
{
jobs: [
{
name: "Failed lane",
status: "completed",
conclusion: "failure",
created_at: recent,
started_at: recent,
completed_at: new Date(Date.parse(recent) + 55_000).toISOString(),
steps: [],
},
],
},
],
[
{
jobs: [
{
name: "Cancelled lane",
status: "completed",
conclusion: "cancelled",
created_at: recent,
started_at: recent,
completed_at: new Date(Date.parse(recent) + 30_000).toISOString(),
steps: [],
},
],
},
],
completedJobs("Release lane", "success", recent, 50),
completedJobs("Failed lane", "failure", recent, 55),
completedJobs("Cancelled lane", "cancelled", recent, 30),
]);
const result = await runBatch(fixture);
const manifest = JSON.parse(
@@ -116,8 +76,20 @@ it("collects the latest attempt for recent completed CI and Release runs into a
expect(manifest.errors).toEqual([]);
expect(manifest.runs).toMatchObject([
{ id: 43, runAttempt: 2, conclusion: "success" },
{ id: 42, runAttempt: 3, conclusion: "failure" },
{ id: 41, runAttempt: 1, conclusion: "cancelled" },
{
id: 42,
runAttempt: 3,
conclusion: "failure",
event: "push",
headBranch: "main",
},
{
id: 41,
runAttempt: 1,
conclusion: "cancelled",
event: null,
headBranch: null,
},
]);
expect(manifest.runs).toEqual(
expect.arrayContaining([
@@ -234,6 +206,31 @@ it("retains a partial manifest when its collection budget is exhausted", async (
expect(manifest.truncation.join(" ")).toContain("Time budget");
});
function completedJobs(
name: string,
conclusion: string,
startedAt: string,
seconds: number,
) {
return [
{
jobs: [
{
name,
status: "completed",
conclusion,
created_at: startedAt,
started_at: startedAt,
completed_at: new Date(
Date.parse(startedAt) + seconds * 1000,
).toISOString(),
steps: [],
},
],
},
];
}
async function createFixture(responses: unknown[]) {
const directory = await createTestTempDirectory("rea-ci-job-timings-batch-");
const fakeBin = join(directory, "bin");
@@ -0,0 +1,617 @@
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { delimiter, join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import {
classifyPrCi,
collectPrCiHealth,
matchRunToPullRequest,
writePrCiHealthReport,
} from "../../../scripts/ci/pr-ci-health.mjs";
import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs";
import type {
PrCiPullRequest,
PrCiWorkflowRun,
} from "../../../scripts/ci/pr-ci-health.mjs";
import { executeWorkflowFixture } from "../../support/workflowGit.js";
const reporter = fileURLToPath(
new URL("../../../scripts/ci/pr-ci-health.mjs", import.meta.url),
);
const observedAt = Date.parse("2026-10-14T00:00:00Z");
const headSha = "a".repeat(40);
const mergeSha = "b".repeat(40);
it("classifies workflow states without confusing absent, pending, and completed CI", () => {
const pull = validPull();
const cases = [
[
"startup_failure",
run({ status: "completed", conclusion: "startup_failure" }),
],
[
"queued_pending_executing",
run({ status: "in_progress", conclusion: null }),
],
["approval_required", run({ status: "waiting", conclusion: null })],
["completed_failure", run({ status: "completed", conclusion: "failure" })],
[
"completed_canceled",
run({ status: "completed", conclusion: "cancelled" }),
],
["completed_success", run({ status: "completed", conclusion: "success" })],
["completed_other", run({ status: "completed", conclusion: "neutral" })],
[
"completed_other",
run({ status: "completed", conclusion: "constructor" }),
],
] as const;
for (const [category, evidence] of cases) {
for (const runsComplete of [true, false]) {
const result = classifyPrCi({
pull,
runs: [
{
run: { ...evidence, run_attempt: 1 },
basis: "event_pr_number_and_head_sha",
},
],
runsComplete,
pullRequestInventoryComplete: true,
ambiguousRuns: [],
});
expect(result).toMatchObject({
category: runsComplete ? category : "unknown",
...(runsComplete
? {}
: { reason: "workflow_run_inventory_incomplete" }),
runs: [
{
id: 77,
attempt: 1,
status: evidence.status,
conclusion: evidence.conclusion,
},
],
});
}
}
for (const runsComplete of [true, false]) {
expect(
classifyPrCi({
pull: { ...pull, mergeable: false },
runs: [],
runsComplete,
pullRequestInventoryComplete: true,
ambiguousRuns: [],
}).category,
).toBe("merge_conflict");
}
});
it("requires complete inventories, known mergeability, and immutable head identity to report missing", () => {
const pull = validPull({ mergeable: true });
const classify = (overrides = {}) =>
classifyPrCi({
pull,
runs: [],
runsComplete: true,
pullRequestInventoryComplete: true,
ambiguousRuns: [],
...overrides,
});
expect(classify().category).toBe("missing");
expect(classify({ runsComplete: false })).toMatchObject({
category: "deferred",
reason: "lookup_inventory_incomplete",
});
expect(classify({ pullRequestInventoryComplete: false }).category).toBe(
"deferred",
);
expect(
classify({
ambiguousRuns: [{ run: run(), basis: "ambiguous_head_identity" }],
}).category,
).toBe("unknown");
expect(classify({ pull: { ...pull, mergeable: undefined } }).category).toBe(
"unknown",
);
expect(classify({ pull: { ...pull, identityValid: false } }).category).toBe(
"unknown",
);
});
it("accepts event PR/head evidence for merge-SHA runs and refuses ambiguous branch identities", () => {
const pull = validPull();
const associatedMergeRun = run({
head_sha: mergeSha,
pull_requests: [
{
number: pull.number!,
head: { sha: headSha, repo: { id: pull.head.repositoryId } },
},
],
});
expect(
matchRunToPullRequest({
run: associatedMergeRun,
pull,
identityCounts: new Map([["73\u0000feature", 1]]),
}),
).toEqual({ kind: "match", basis: "event_pr_number_and_head_sha" });
expect(
matchRunToPullRequest({
run: run({
pull_requests: [
{ number: pull.number!, head: { sha: "c".repeat(40) } },
],
}),
pull,
identityCounts: new Map([["73\u0000feature", 1]]),
}).kind,
).toBe("none");
const branchRun = run({ pull_requests: [] });
expect(
matchRunToPullRequest({
run: branchRun,
pull,
identityCounts: new Map([["73\u0000feature", 1]]),
inventoryComplete: false,
}).kind,
).toBe("ambiguous");
expect(
matchRunToPullRequest({
run: branchRun,
pull,
identityCounts: new Map([["73\u0000feature", 1]]),
}),
).toEqual({
kind: "match",
basis: "event_unique_head_ref_and_repository_id",
});
expect(
matchRunToPullRequest({
run: run({ head_sha: mergeSha, pull_requests: [] }),
pull,
identityCounts: new Map([["73\u0000feature", 1]]),
}).kind,
).toBe("none");
expect(
matchRunToPullRequest({
run: run({
pull_requests: [{ number: pull.number!, head: { sha: headSha } }],
}),
pull,
identityCounts: new Map([["73\u0000feature", 1]]),
}).kind,
).toBe("ambiguous");
expect(
matchRunToPullRequest({
run: branchRun,
pull,
identityCounts: new Map([["73\u0000feature", 2]]),
}).kind,
).toBe("ambiguous");
expect(
matchRunToPullRequest({
run: run({ event: "workflow_dispatch" }),
pull,
identityCounts: new Map([["73\u0000feature", 1]]),
}).kind,
).toBe("none");
});
it("scans target-branch PRs through public gh GET requests and writes JSON and Markdown", async () => {
const fixture = await createFixture();
const calls: string[] = [];
const apiRequest: NonNullable<
Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
> = async (method, endpoint) => {
calls.push(`${method} ${endpoint}`);
if (method !== "GET") throw new Error("fixture rejected a non-GET request");
if (endpoint.includes("/pulls?"))
return [
rawPull(42, { mergeable: true }),
rawPull(43, { mergeable: false }),
rawPull(44, {
base: { ref: "release", repo: { id: 11 } },
mergeable: true,
}),
rawPull(45, { draft: true, mergeable: true }),
];
if (endpoint.includes("/actions/workflows/ci.yml/runs?"))
return {
total_count: 1,
workflow_runs: [
run({
id: 9001,
event: "pull_request",
status: "completed",
conclusion: "failure",
head_sha: mergeSha,
pull_requests: [
{
number: 42,
head: { sha: headSha, repo: { id: 73 } },
},
],
}),
],
};
throw new Error(`unexpected GET endpoint: ${endpoint}`);
};
const report = await collectPrCiHealth({
repository: "owner/rea",
repositoryId: 11,
defaultBranch: "main",
now: observedAt,
apiRequest,
});
expect(calls).toHaveLength(2);
expect(calls.every((call) => call.startsWith("GET "))).toBe(true);
expect(calls[0]).toContain("state=open");
expect(calls[0]).not.toContain("base=");
expect(calls[1]).toContain("event=pull_request");
expect(report.pullRequests).toHaveLength(2);
expect(report.pullRequests.map(({ number }) => number)).toEqual([42, 43]);
expect(report.pullRequests.map(({ ci }) => ci.category)).toEqual([
"completed_failure",
"merge_conflict",
]);
expect(report.scan).toMatchObject({
status: "complete",
pullRequestInventoryComplete: true,
workflowRunInventoryComplete: true,
apiRequests: 2,
});
const output = join(fixture.directory, ".cache", "ci-pr-health");
const paths = await writePrCiHealthReport(report, output);
const written = JSON.parse(await readFile(paths.jsonPath, "utf8"));
const markdown = await readFile(paths.markdownPath, "utf8");
expect(written.counts.completed_failure).toBe(1);
expect(markdown).toContain("# Pull request CI health snapshot");
expect(markdown).toContain("#42");
expect(markdown).toContain("completed_failure");
});
it("counts older open PRs when deciding whether a branch/ref fallback is unique", async () => {
const olderPull = rawPull(63, {
created_at: "2026-09-01T00:00:00Z",
head: { ...rawPull(63).head, ref: "shared-feature" },
});
const currentPull = rawPull(64, {
mergeable: true,
head: { ...olderPull.head },
});
const apiRequest: NonNullable<
Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
> = async (_method, endpoint) => {
if (endpoint.includes("/pulls?")) return [currentPull, olderPull];
if (endpoint.includes("/actions/workflows/ci.yml/runs?"))
return {
workflow_runs: [
run({
head_branch: "shared-feature",
pull_requests: [],
}),
],
};
throw new Error(`unexpected GET endpoint: ${endpoint}`);
};
const report = await collectPrCiHealth({
repository: "owner/rea",
repositoryId: 11,
defaultBranch: "main",
now: observedAt,
apiRequest,
});
expect(report.scan.pullRequestInventoryComplete).toBe(true);
expect(report.pullRequests.map(({ number }) => number)).toEqual([64]);
expect(report.pullRequests[0]?.ci).toMatchObject({
category: "unknown",
reason: "workflow_run_identity_ambiguous",
});
});
it.each([false, true])(
"keeps page-capped scans uncertain (observed success: %s)",
async (observedSuccess) => {
const pull = rawPull(62, { mergeable: true });
const apiRequest: NonNullable<
Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
> = async (_method, endpoint) => {
if (endpoint.includes("/pulls?")) return [pull];
const page = Number(
new URL(`https://fixture.invalid/${endpoint}`).searchParams.get("page"),
);
return {
workflow_runs: Array.from({ length: 100 }, (_item, index) =>
observedSuccess && page === 1 && index === 0
? run({
run_attempt: 1,
pull_requests: [
{ number: 62, head: { sha: headSha, repo: { id: 73 } } },
],
})
: run({
id: page * 100 + index,
head_branch: "different-branch",
head_repository: { id: 999 },
pull_requests: [],
}),
),
};
};
const report = await collectPrCiHealth({
repository: "owner/rea",
repositoryId: 11,
defaultBranch: "main",
now: observedAt,
apiRequest,
});
expect(report.scan.workflowRunPages).toBe(5);
expect(report.scan.workflowRunInventoryComplete).toBe(false);
const firstPull = report.pullRequests[0];
if (firstPull === undefined) throw new Error("fixture PR was not scanned");
expect(firstPull.ci).toMatchObject({
category: observedSuccess ? "unknown" : "deferred",
reason: observedSuccess
? "workflow_run_inventory_incomplete"
: "lookup_inventory_incomplete",
});
expect(firstPull.ci.runs).toHaveLength(observedSuccess ? 1 : 0);
},
);
it("keeps the result unknown when the head changes between API snapshots", async () => {
const apiRequest: NonNullable<
Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
> = async (_method, endpoint) => {
if (endpoint.includes("/pulls?")) return [rawPull(63)];
if (endpoint.includes("/actions/workflows/ci.yml/runs?"))
return { workflow_runs: [] };
return {
...rawPull(63, { mergeable: true }),
head: {
...rawPull(63).head,
sha: mergeSha,
},
};
};
const report = await collectPrCiHealth({
repository: "owner/rea",
repositoryId: 11,
defaultBranch: "main",
now: observedAt,
apiRequest,
});
expect(report.pullRequests[0]).toMatchObject({
identityChangedDuringScan: true,
ci: { category: "unknown", reason: "pull_request_head_identity_invalid" },
});
expect(report.scan).toMatchObject({
status: "partial",
errors: ["pull_request_identity_changed_during_scan"],
});
});
it("public CLI uses only GET and produces a report from fixture GitHub responses", async () => {
const fixture = await createFixture();
const ghCalls = join(fixture.directory, "gh-calls.jsonl");
const ghResponse = join(fixture.directory, "gh-response.json");
const fakeGh = join(fixture.fakeBin, "gh");
await writeFile(
ghResponse,
JSON.stringify([
[rawPull(51, { mergeable: true })],
{ total_count: 0, workflow_runs: [] },
]),
);
await writeFile(
fakeGh,
`#!/usr/bin/env node\nconst fs=require("node:fs");\nconst args=process.argv.slice(2);\nfs.appendFileSync(process.env.CI_GH_CALLS, JSON.stringify(args)+"\\n");\nif(args[0]!=="api"||args[1]!=="--method"||args[2]!=="GET")process.exit(41);\nconst page=Number(new URL("https://fixture.invalid/"+args.at(-1)).searchParams.get("page"));\nconst responses=JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSE,"utf8"));\nprocess.stdout.write(JSON.stringify(responses[page===1&&args.at(-1).includes("/pulls?")?0:1]));\n`,
);
await chmod(fakeGh, 0o755);
const outputDirectory = join(fixture.directory, ".cache", "ci-pr-health");
await executeWorkflowFixture(process.execPath, [reporter], {
cwd: fixture.directory,
env: {
...process.env,
PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
CI_REPOSITORY: "owner/rea",
CI_REPOSITORY_ID: "11",
CI_DEFAULT_BRANCH: "main",
CI_PR_HEALTH_DIRECTORY: outputDirectory,
CI_GH_CALLS: ghCalls,
CI_GH_RESPONSE: ghResponse,
},
});
const calls = (await readFile(ghCalls, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(calls).toHaveLength(2);
expect(
calls.every(
(args: string[]) => args[1] === "--method" && args[2] === "GET",
),
).toBe(true);
const report = JSON.parse(
await readFile(join(outputDirectory, "report.json"), "utf8"),
);
expect(report.pullRequests[0].ci.category).toBe("missing");
await expect(
readFile(join(outputDirectory, "summary.md"), "utf8"),
).resolves.toContain("missing");
});
it("rejects non-GET methods before launching the GitHub client", async () => {
const fixture = await createFixture();
const request = createGhApiRequest({
env: {
...process.env,
PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
CI_GH_CALLS: join(fixture.directory, "should-not-exist"),
},
});
await expect(
request("POST", "repos/owner/rea/pulls", {
requests: 0,
startedAt: observedAt,
limitMs: 1_000,
}),
).rejects.toThrow("read-only GET");
await expect(
readFile(join(fixture.directory, "should-not-exist")),
).rejects.toMatchObject({ code: "ENOENT" });
});
function validPull(overrides: Partial<PrCiPullRequest> = {}): PrCiPullRequest {
return {
number: 42,
identityValid: true,
mergeable: true,
head: { repositoryId: 73, ref: "feature", sha: headSha },
...overrides,
};
}
it.each([
["newer run", { id: 78, created_at: "2026-10-13T01:00:00Z" }, "unknown"],
[
"older run",
{ id: 78, created_at: "2026-10-12T23:00:00Z" },
"completed_success",
],
["equal timestamps", { id: 78 }, "unknown"],
[
"retry of an older run",
{ id: 78, created_at: "2026-10-12T23:00:00Z", run_attempt: 2 },
"unknown",
],
["unknown timestamp", { id: 78, created_at: "invalid" }, "unknown"],
["newer attempt", { run_attempt: 3 }, "unknown"],
["older attempt", { run_attempt: 1 }, "completed_success"],
] as const)(
"retains ambiguity evidence for a %s without reporting a stale completion",
(_label, overrides, category) => {
const currentAttempt = "id" in overrides ? 1 : 2;
const result = classifyPrCi({
pull: validPull(),
runs: [
{
run: run({ run_attempt: currentAttempt }),
basis: "event_pr_number_and_head_sha",
},
],
ambiguousRuns: [
{
run: run({ run_attempt: currentAttempt, ...overrides }),
basis: "ambiguous_head_identity",
},
],
runsComplete: true,
pullRequestInventoryComplete: true,
});
expect(result.category).toBe(category);
expect(result.runs).toHaveLength(2);
},
);
it("does not let an older proven run hide a newer run with an unverified repository ID", async () => {
const pull = rawPull(42);
const older = run({
id: 70,
created_at: "2026-10-13T00:00:00Z",
pull_requests: [{ number: 42, head: { sha: headSha, repo: { id: 73 } } }],
});
const newer = run({
id: 71,
created_at: "2026-10-13T01:00:00Z",
status: "in_progress",
conclusion: null,
pull_requests: [{ number: 42, head: { sha: headSha } }],
});
const report = await collectPrCiHealth({
repository: "owner/rea",
repositoryId: 11,
defaultBranch: "main",
now: observedAt,
apiRequest: async (_method, endpoint) => {
if (endpoint.includes("/pulls?")) return [pull];
if (endpoint.endsWith("/pulls/42")) return { ...pull, mergeable: true };
return { workflow_runs: [newer, older] };
},
});
expect(report.pullRequests[0]?.ci).toMatchObject({
category: "unknown",
reason: "workflow_run_identity_ambiguous",
});
expect(report.pullRequests[0]?.ci.runs).toHaveLength(2);
});
it("keeps a proven newest run even when two older matched runs have equal timestamps", () => {
const result = classifyPrCi({
pull: validPull(),
runs: [
run({ id: 70, run_attempt: 1 }),
run({ id: 71, run_attempt: 1, conclusion: "failure" }),
run({
id: 72,
run_attempt: 1,
created_at: "2026-10-13T01:00:00Z",
status: "in_progress",
conclusion: null,
}),
].map((run) => ({ run, basis: "event_pr_number_and_head_sha" })),
ambiguousRuns: [],
runsComplete: true,
pullRequestInventoryComplete: true,
});
expect(result.category).toBe("queued_pending_executing");
expect(result.runs).toHaveLength(3);
});
function run(overrides: Partial<PrCiWorkflowRun> = {}): PrCiWorkflowRun {
return {
id: 77,
event: "pull_request",
head_branch: "feature",
head_repository: { id: 73 },
head_sha: headSha,
status: "completed",
conclusion: "success",
created_at: "2026-10-13T00:00:00Z",
pull_requests: [],
...overrides,
};
}
function rawPull(number: number, overrides: Record<string, unknown> = {}) {
return {
number,
draft: false,
created_at: "2026-10-13T00:00:00Z",
html_url: `https://github.com/owner/rea/pull/${String(number)}`,
base: { ref: "main", repo: { id: 11 } },
head: {
ref: `feature-${String(number)}`,
sha: headSha,
repo: { id: 73, full_name: "contributor/rea" },
},
...overrides,
};
}
async function createFixture() {
const directory = await createTestTempDirectory("rea-ci-pr-health-");
const fakeBin = join(directory, "bin");
await mkdir(fakeBin);
return { directory, fakeBin };
}
@@ -44,7 +44,7 @@ function run(overrides: Record<string, unknown> = {}) {
id: 7,
status: "completed",
conclusion: "success",
path: ".github/workflows/ci.yml",
path: ".github/workflows/ci.yml@main",
event: "schedule",
created_at: new Date(now - 120_000).toISOString(),
head_sha: validSha,
@@ -121,7 +121,11 @@ it("exports every canonical scope with read authority for admission and keeps te
);
expect(timings.on.workflow_run).toBeUndefined();
expect(timings.on.schedule).toHaveLength(1);
expect(timings.permissions).toEqual({ contents: "read", actions: "read" });
expect(timings.permissions).toEqual({
contents: "read",
actions: "read",
"pull-requests": "read",
});
const checkout = timings.jobs.report.steps.find((step) =>
step.uses?.startsWith("actions/checkout@"),
);
@@ -154,6 +158,7 @@ it("holds native and provider fan-out behind static validation while portable fe
"static",
"build",
"test-shard",
"test-source",
"test",
"docs",
"required",
@@ -369,7 +374,7 @@ it("keeps fast feedback separate from complete qualification and release proof",
// release their concurrency slot without waiting for another runner.
expect(ci.jobs.required?.if).toBe("${{ !cancelled() }}");
const shards = ci.jobs["test-shard"]?.steps ?? [];
const fast = shards.find((step) => step.run?.includes("npm run test:run"));
const fast = shards.find((step) => step.run?.includes("npm run test:built"));
const full = shards.find((step) => step.run?.includes("test:ci:shard:run"));
expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'");
expect(fast?.run).toContain("--reporter=blob");
@@ -0,0 +1,99 @@
import { execFile } from "node:child_process";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { parseStaticArchive } from "../../../scripts/ci/artifacts.mjs";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const execute = promisify(execFile);
const producer = fileURLToPath(
new URL("../../../scripts/ci/static-check-report.mjs", import.meta.url),
);
it.each([3.5, 100, 0])(
"retains real lint failures when the duplication threshold is %s",
async (threshold) => {
const root = await createTestTempDirectory("rea-ci-static-archive-");
await mkdir(join(root, "src"));
await mkdir(join(root, "tests"));
await writeFile(
join(root, ".oxlintrc.json"),
JSON.stringify({ rules: { "no-debugger": "error" } }),
);
await writeFile(
join(root, ".jscpd.json"),
JSON.stringify({ threshold, minLines: 5, minTokens: 50 }),
);
await writeFile(
join(root, "src/example.ts"),
"debugger;\nexport const example = true;\n",
);
const duplicate =
"export function compute(value: number) {\n" +
Array.from({ length: 20 }, (_, index) => ` value += ${index};\n`).join(
"",
) +
" return value;\n}\n";
for (const name of threshold === 3.5 ? [] : ["first", "second"])
await writeFile(join(root, `src/${name}.ts`), duplicate);
await expect(
execute(process.execPath, [producer, "lint", "src/example.ts"], {
cwd: root,
}),
).rejects.toMatchObject({ code: 1 });
const duplication = execute(process.execPath, [producer, "duplicates"], {
cwd: root,
});
if (threshold === 0)
await expect(duplication).rejects.toMatchObject({ code: 1 });
else await duplication;
const reports = join(root, ".cache/static-checks");
expect(
JSON.parse(
await readFile(join(reports, "jscpd/jscpd-report.json"), "utf8"),
),
).toMatchObject({ reaCheck: { exitCode: threshold === 0 ? 1 : 0 } });
await writeFile(
join(reports, "static-task-context.json"),
JSON.stringify({
version: 1,
sourceSha: "a".repeat(40),
sourceClean: true,
nodeVersion: process.version,
npmVersion: "11.16.0",
turboVersion: "2.11.7",
platform: process.platform,
architecture: process.arch,
taskHashes: { "lint:built": "a".repeat(16), jscpd: "b".repeat(16) },
}),
);
const archive = join(root, "reports.zip");
await execute(
"zip",
[
"-q",
archive,
"oxlint.json",
"jscpd/jscpd-report.json",
"static-task-context.json",
],
{ cwd: reports },
);
const evidence = await parseStaticArchive(archive);
expect(evidence.diagnostics.filter(({ kind }) => kind === "lint")).toEqual([
expect.objectContaining({
kind: "lint",
path: "src/example.ts",
severity: "error",
}),
]);
const duplicates = evidence.diagnostics.filter(
({ kind }) => kind === "jscpd",
);
if (threshold === 0) expect(duplicates.length).toBeGreaterThan(0);
else expect(duplicates).toEqual([]);
},
);
@@ -35,7 +35,7 @@ function run(id: number, overrides: Record<string, unknown> = {}) {
id,
status: "completed",
conclusion: "success",
path: ".github/workflows/ci.yml",
path: ".github/workflows/ci.yml@main",
event: "schedule",
created_at: new Date(now - 120_000).toISOString(),
head_branch: "main",
@@ -0,0 +1,75 @@
import { execFileSync } from "node:child_process";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import {
BUILT_TEST_PROJECTS,
SOURCE_TEST_PROJECTS,
projectForTest,
} from "../../../scripts/lib/test-projects.mjs";
import {
developmentTestPlan,
parseDevelopmentTestRequest,
} from "../../../scripts/lib/development-tests.mjs";
const repository = process.cwd();
describe("canonical test-project ownership", () => {
it("assigns every tracked test file exactly once to source or built", () => {
const files = execFileSync("git", ["ls-files", "*.test.ts"], {
cwd: repository,
encoding: "utf8",
})
.trim()
.split("\n")
.filter(Boolean);
const grouped = new Map<string, string[]>();
for (const file of files) {
const project = projectForTest(file);
grouped.set(project.group, [...(grouped.get(project.group) ?? []), file]);
}
expect(grouped.get("source")).not.toBeUndefined();
expect(grouped.get("built")).not.toBeUndefined();
expect(
(grouped.get("source")?.length ?? 0) +
(grouped.get("built")?.length ?? 0),
).toBe(files.length);
expect(
SOURCE_TEST_PROJECTS.reduce(
(count, project) => count + project.include.length,
0,
),
).toBeGreaterThan(0);
expect(
BUILT_TEST_PROJECTS.every((project) => project.group === "built"),
).toBe(true);
expect(() => projectForTest("tests/unowned/example.test.ts")).toThrow(
"No test project owns",
);
});
it("owns the Hopper real-provider conformance test as built and requires preparation", () => {
const testPath =
"tests/conformance/providers/hopper/realHopperSearch.test.ts";
expect(projectForTest(testPath)).toMatchObject({
name: "boundary",
group: "built",
});
expect(() => parseDevelopmentTestRequest("local", [testPath])).toThrow(
`Use test:focused for built test projects: ${testPath}`,
);
const request = parseDevelopmentTestRequest("focused", [testPath]);
expect(developmentTestPlan(request)).toMatchObject({ needsBuild: true });
});
it("uses canonical selectors in Vitest while retaining independent runner policy", async () => {
const config = await readFile(join(repository, "vitest.config.ts"), "utf8");
expect(config).toContain(
"[...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]",
);
expect(config).toContain("PROJECT_POLICIES");
expect(config).not.toContain('"src/{contracts,domain}/**/*.test.ts"');
});
});
+140 -15
View File
@@ -3,6 +3,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { runInNewContext } from "node:vm";
import { expect, it } from "vitest";
import { parse } from "yaml";
@@ -48,9 +49,7 @@ const workflow = async () =>
it("merges available failed-shard diagnostics and skips unavailable shard reports", async () => {
const steps = (await workflow()).test!.steps;
const download = steps.find(
(step) =>
step.uses?.includes("actions/download-artifact") &&
step.with?.pattern === "vitest-report-*",
(step) => step.with?.pattern === "vitest-report-*",
);
const merge = steps.findIndex((step) => step.run?.includes("test:ci:merge"));
const upload = steps.findIndex((step) => step.with?.name === "test-results");
@@ -59,8 +58,7 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
);
expect(merge).toBeGreaterThan(0);
expect(download?.if).toContain("!cancelled()");
expect(download?.if).toContain("needs.test-shard.result == 'success'");
expect(download?.if).toContain("needs.test-shard.result == 'failure'");
expect(upload).toBeGreaterThan(merge);
expect(gate).toBeGreaterThan(upload);
const merges = steps.filter((step) => step.run?.includes("--reporter=json"));
@@ -70,8 +68,6 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
"--reporter=./scripts/ci/test-timing-reporter.mjs",
);
expect(step.if).toContain("!cancelled()");
expect(step.if).toContain("needs.test-shard.result == 'success'");
expect(step.if).toContain("needs.test-shard.result == 'failure'");
}
expect(steps[upload]?.with?.path).toContain("test-timings.json");
expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true });
@@ -86,12 +82,23 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
);
const command = steps[gate]?.run;
if (command === undefined) throw new Error("Missing shard result gate");
for (const result of ["failure", "cancelled", "skipped", ""]) {
await expect(
execute("bash", ["-c", command], {
env: { ...process.env, CODE_REQUIRED: "true", SHARD_RESULT: result },
}),
).rejects.toMatchObject({ code: 1 });
for (const tier of ["fast", "full"]) {
for (const shard of ["success", "failure", "cancelled", "skipped", ""]) {
for (const source of ["success", "failure", "cancelled", "skipped", ""]) {
const check = execute("bash", ["-c", command], {
env: {
...process.env,
CODE_REQUIRED: "true",
SHARD_RESULT: shard,
SOURCE_REQUIRED: String(tier === "fast"),
SOURCE_RESULT: source,
},
});
if (shard === "success" && (tier !== "fast" || source === "success"))
await expect(check).resolves.toMatchObject({ stderr: "" });
else await expect(check).rejects.toMatchObject({ code: 1 });
}
}
}
await expect(
execute("bash", ["-c", command], {
@@ -100,6 +107,80 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
).resolves.toMatchObject({ stderr: "" });
});
// These workflow conditions use the shared boolean/string expression subset.
// Evaluate the checked-in expressions so failure-state coverage cannot drift
// into a separate test implementation of the collection policy.
function conditionValue(
condition: string | undefined,
needs: Record<string, unknown>,
canceled: boolean,
) {
if (!condition) throw new Error("Missing report collection condition");
return runInNewContext(
condition
.replaceAll("needs.test-shard", 'needs["test-shard"]')
.replaceAll("needs.test-source", 'needs["test-source"]'),
{ needs, cancelled: () => canceled },
{ timeout: 1_000 },
);
}
it("collects either fast test group while keeping full reports tied to full shards", async () => {
const steps = (await workflow()).test.steps;
const download = steps.find(
(step) => step.with?.pattern === "vitest-report-*",
);
const fast = steps.find(
(step) => step.name === "Merge fast-suite JUnit and timing reports",
);
const full = steps.find(
(step) => step.name === "Merge coverage, JUnit, and timing reports",
);
for (const tier of ["fast", "full"]) {
for (const shard of ["success", "failure", "cancelled", "skipped", ""]) {
for (const source of ["success", "failure", "cancelled", "skipped", ""]) {
for (const [code, canceled] of [
["true", false],
["true", true],
["false", false],
["false", true],
] as const) {
const needs = {
changes: { outputs: { code, tier } },
"test-shard": { result: shard },
"test-source": { result: source },
};
const builtAvailable = ["success", "failure"].includes(shard);
const sourceAvailable = ["success", "failure"].includes(source);
const enabled = code === "true" && !canceled;
const available =
builtAvailable || (tier === "fast" && sourceAvailable);
const expected = enabled
? {
download: available,
fast: tier === "fast" && available,
full: tier === "full" && builtAvailable,
}
: { download: false, fast: false, full: false };
expect(conditionValue(download?.if, needs, canceled)).toBe(
expected.download,
);
expect(conditionValue(fast?.if, needs, canceled)).toBe(expected.fast);
expect(conditionValue(full?.if, needs, canceled)).toBe(expected.full);
}
}
}
}
const summary = steps.find((step) =>
step.run?.includes("scripts/ci/test-report.mjs"),
);
expect(summary?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}");
expect(summary?.env?.SOURCE_RESULT).toBe("${{ needs.test-source.result }}");
expect(summary?.env?.SOURCE_REQUIRED).toBe(
"${{ needs.changes.outputs.tier == 'fast' }}",
);
});
it("keeps diagnosis available after merge failure and publishes only complete successful history", async () => {
const steps = (await workflow()).test!.steps;
const diagnosis = steps.find((step) => step.with?.name === "test-results");
@@ -220,7 +301,12 @@ it("exports portable timing history and removes it on a failed, partial, or inva
{ name: join(root, "tests/fast.test.ts"), durationMs: 10 },
],
};
const run = async (value: unknown, shards = "success", merge = "success") => {
const run = async (
value: unknown,
shards = "success",
merge = "success",
source?: string,
) => {
await writeFile(join(root, "test-timings.json"), JSON.stringify(value));
await writeFile(summary, "");
return execute(process.execPath, [fileURLToPath(script)], {
@@ -229,6 +315,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva
...process.env,
GITHUB_STEP_SUMMARY: summary,
SHARD_RESULT: shards,
SOURCE_REQUIRED: String(source !== undefined),
SOURCE_RESULT: source ?? "",
MERGE_RESULT: merge,
},
});
@@ -256,6 +344,24 @@ it("exports portable timing history and removes it on a failed, partial, or inva
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
expect(await readFile(summary, "utf8")).toContain("diagnostic only");
}
await run(report, "success", "success", "success");
expect(await readFile(history, "utf8")).toContain('"success":true');
for (const [shards, source] of [
["skipped", "success"],
["success", "skipped"],
["success", "cancelled"],
["success", "failure"],
["success", ""],
]) {
await run(report);
await run(report, shards, "success", source);
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
const text = await readFile(summary, "utf8");
expect(text).toContain(`Source result: ${source}.`);
expect(text).toContain("diagnostic only");
if (shards === "skipped" || source !== "failure")
expect(text).toContain("Partial test diagnostics");
}
await run(report);
await expect(
run({
@@ -352,6 +458,21 @@ it("recovers complete module costs from real blob reports without executing test
expect(file.durationMs).toBeCloseTo(
Object.values(file.phases).reduce((sum, value) => sum + value, 0),
);
const partial = await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: {
...process.env,
SHARD_RESULT: "skipped",
SOURCE_REQUIRED: "true",
SOURCE_RESULT: "success",
MERGE_RESULT: "success",
},
});
expect(partial.stdout).toContain("Partial test diagnostics");
expect(partial.stdout).toContain("sample.test.js");
await expect(
readFile(join(root, ".cache/ci-test-history/test-results.json")),
).rejects.toMatchObject({ code: "ENOENT" });
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" },
@@ -420,13 +541,15 @@ it("records failed omissions from full reports without treating partial or unhan
state: index === 1 ? "failed" : "passed",
})),
};
const run = async (report: unknown) => {
const run = async (report: unknown, source = "failure") => {
await writeFile(join(root, "test-timings.json"), JSON.stringify(report));
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: {
...process.env,
SHARD_RESULT: "failure",
SOURCE_REQUIRED: "true",
SOURCE_RESULT: source,
MERGE_RESULT: "failure",
CI_VERIFY_TEST_INVENTORY: "true",
CI_TEST_SELECTION: JSON.stringify(
@@ -458,6 +581,8 @@ it("records failed omissions from full reports without treating partial or unhan
{ ...timing, reason: "interrupted" },
])
expect(await run(report)).toMatchObject({ complete: false });
for (const source of ["skipped", "cancelled", ""])
expect(await run(timing, source)).toMatchObject({ complete: false });
await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({
code: 1,
});
@@ -0,0 +1,54 @@
import { expect, it } from "vitest";
import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs";
it.each([
[".github/workflows/ci.yml", ".github/workflows/ci.yml", null],
[".github/workflows/ci.yml@main", ".github/workflows/ci.yml", "main"],
[
".github/workflows/ci.yml@refs/heads/release/v1",
".github/workflows/ci.yml",
"refs/heads/release/v1",
],
[
".github/workflows/ci.yml@feature@v1",
".github/workflows/ci.yml",
"feature@v1",
],
[
".github/workflows/release.yaml@main",
".github/workflows/release.yaml",
"main",
],
])(
"preserves API path %s while separating its file identity and ref",
(path, file, ref) => {
const raw = { id: 10, path, head_sha: "a".repeat(40) };
const parsed = normalizeWorkflowRun(raw);
expect(parsed).toEqual({ ...raw, workflowPath: { file, ref } });
expect(raw).not.toHaveProperty("workflowPath");
},
);
it.each([
undefined,
null,
42,
"",
".github/workflows/ci.yml@",
".github/workflows/ci.yml@main\n",
".github/workflows/ci.yml@main\0",
"../.github/workflows/ci.yml",
".github/workflows/../ci.yml@main",
".github/workflows/ci.yml-extra@main",
"https://example.test/ci.yml@main",
])(
"keeps malformed workflow path %s unknown instead of inventing a file identity",
(path) => {
const parsed = normalizeWorkflowRun({
path,
workflowPath: { file: ".github/workflows/ci.yml", ref: "main" },
});
expect(parsed.path).toBe(path);
expect(parsed.workflowPath).toBeNull();
},
);
@@ -114,18 +114,34 @@ describe("Hermes YAML shape diagnostics", () => {
"!!omap [{rea: {command: rea, args: [mcp]}}]",
"!!timestamp 2001-12-15",
"!!binary cmVh",
].flatMap((value) => [
{
original: `${value}\n`,
problem: "The configuration root must be an object",
},
{
original: `mcp_servers: ${value}\n`,
problem: '"mcp_servers" must be an object',
},
]),
]
.flatMap((value) => [
{
original: `${value}\n`,
problem: "The configuration root must be an object",
},
{
original: `mcp_servers: ${value}\n`,
problem: '"mcp_servers" must be an object',
},
])
.concat(
// An explicit null root is a value, not an empty file.
[
"null\n",
"~\n",
"# Hermes configuration\nnull\n",
"--- ~\n",
"--- !!null\n",
'!!null ""\n',
"--- &empty\n",
].map((original) => ({
original,
problem: "The configuration root must be an object",
})),
),
)(
"blocks setup and uninstall before changes for tagged non-record YAML: $original",
"blocks setup and uninstall before changes for tagged or null non-record YAML: $original",
async ({ original, problem }) => {
const { home, client } = await hermesClient();
const cursor = join(home, ".cursor", "mcp.json");
@@ -281,7 +297,36 @@ mcp_servers:
});
});
it.each(["", "\n"])(
// Comments and a bare document marker parse to a null document, like an
// empty file, and are kept ahead of the new registration.
});
describe("Hermes YAML empty configurations", () => {
it("rejects multiple YAML documents before treating an empty first root as absent", async () => {
const { client } = await hermesClient();
const original =
"---\n---\nmcp_servers:\n rea:\n command: rea\n args: [mcp]\n";
await writeFile(client.configPath, original);
expect(await inspectClientConfiguration(client, {}, command)).toMatchObject(
{ status: "invalid" },
);
expect(await configureClientConfiguration(client, {}, command)).toEqual({
status: "failed",
reason: "readback",
});
expect(await readFile(client.configPath, "utf8")).toBe(original);
await expect(
readFile(`${client.configPath}.rea.backup`, "utf8"),
).rejects.toThrow();
});
it.each([
"",
"\n",
"# Hermes configuration\n",
"---\n",
"---\n# Hermes configuration\n",
])(
"configures an empty config.yaml %j as a new document",
async (original) => {
const { home, client } = await hermesClient();
@@ -289,7 +334,10 @@ mcp_servers:
expect(
(await configureClientConfiguration(client, {}, command)).status,
).toBe("configured");
expect(parseYaml(await readFile(client.configPath, "utf8"))).toEqual({
const text = await readFile(client.configPath, "utf8");
const kept = original.trimEnd();
expect(text.slice(0, kept.length)).toBe(kept);
expect(parseYaml(text)).toEqual({
mcp_servers: {
rea: { command: command[0], args: command.slice(1), enabled: true },
},
@@ -49,7 +49,7 @@ const lintDiagnostics = async (root: string) =>
),
).diagnostics;
it("includes every actually linted file in both cached lint task inputs", () => {
it("covers the lint inventory and root tool configs in resolved cache inputs", () => {
const scan = spawnSync(
process.execPath,
[join(repository, "node_modules/oxlint/bin/oxlint"), "--debug=files"],
@@ -69,6 +69,7 @@ it("includes every actually linted file in both cached lint task inputs", () =>
"run",
"lint",
"lint:built",
"knip",
"--dry=json",
],
{
@@ -99,6 +100,14 @@ it("includes every actually linted file in both cached lint task inputs", () =>
files.filter((file) => !Object.hasOwn(task.inputs, file)),
`${task.task} omits actual checker inputs`,
).toEqual([]);
const knip = tasks.find(({ task }) => task === "knip");
expect(knip).toBeDefined();
expect(
files.filter(
(file) => !file.includes("/") && !Object.hasOwn(knip!.inputs, file),
),
"knip omits root tool configuration inputs",
).toEqual([]);
});
it("summarizes warnings without losing their complete diagnostic records", async () => {
@@ -1,27 +1,16 @@
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { afterEach, expect, it } from "vitest";
import { expect, it } from "vitest";
import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
import type { JavaScriptApplicationAnalysisResult } from "../../domain/javascript/javascriptApplicationAnalysis.js";
import { buildJavaScriptNativeHandoffs } from "../../domain/javascript/javascriptNativeHandoff.js";
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
const fixtures: string[] = [];
afterEach(async () => {
await Promise.all(
fixtures
.splice(0)
.map((path) => rm(path, { recursive: true, force: true })),
);
});
import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js";
import type { JavaScriptApplicationAnalysisResult } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js";
import { buildJavaScriptNativeHandoffs } from "../../../src/domain/javascript/javascriptNativeHandoff.js";
import { analyzeJavaScriptApplication } from "../../../src/application/javascript/JavaScriptApplicationService.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const analyzeFixture = async (source: string) => {
const fixture = await mkdtemp(join(tmpdir(), "rea-native-types-"));
fixtures.push(fixture);
const fixture = await createTestTempDirectory("rea-native-types-");
const scope = new ArtifactResourceScope();
try {
await writeFile(join(fixture, "main.ts"), `${source}\n`);
@@ -215,7 +215,10 @@ process.stdin.on("data", (chunk) => {
);
execFileSync(process.execPath, ["--check", provider]);
await chmod(provider, 0o700);
manager = new RizinDebugSessionManager({ REA_RIZIN_COMMAND: provider });
manager = new RizinDebugSessionManager({
...process.env,
REA_RIZIN_COMMAND: provider,
});
const started = await manager.start({
path: "/tmp/fake-target",
inferior_tty: tty,
@@ -576,7 +579,7 @@ process.stdin.on("data", (chunk) => {
await chmod(provider, 0o700);
let providerStderr = "";
manager = new RizinDebugSessionManager(
{ REA_RIZIN_COMMAND: provider },
{ ...process.env, REA_RIZIN_COMMAND: provider },
{
spawnProcess: async (spawnOptions) => {
const launched = await spawnOwnedProviderProcess(spawnOptions);
@@ -15,6 +15,7 @@ import type {
} from "../../../../src/hopper/BridgeLauncher.js";
import { HopperApplicationLauncher } from "../../../../src/hopper/BridgeLauncher.js";
import { HopperClient } from "../../../../src/hopper/HopperClient.js";
import { acquireHopperTargetLease } from "../../../../src/hopper/HopperTargetLease.js";
import { HopperFixtureLauncher } from "./hopperClient.fixture.js";
// Failure assertions keep the external document unknown. Teardown then publishes
@@ -118,6 +119,86 @@ it("does not equate an owned native helper exit with an external GUI exit", asyn
});
});
it.skipIf(process.platform !== "darwin")(
"releases the runtime and target lease after a provenance-verified Spotlight exit, then retries",
async () => {
const directory = await createTestTempDirectory(
"rea-hopper-native-not-found-",
);
const targetPath = join(directory, "target");
await writeFile(targetPath, "fixture target");
const loaderArgs = [
"-e",
'process.stderr.write("Hopper has not been found.\\nPlease make sure that Spotlight is active in order for this tool to find Hopper\'s executable.\\n", () => process.exit(3))',
"--",
];
// Simulate a verified native helper at the launcher provenance boundary;
// publisher verification itself has real codesign boundary coverage.
const native = new HopperApplicationLauncher(
{
environment: {},
launcherPath: process.execPath,
targetPath,
targetKind: "executable",
loaderArgs,
bridgeScriptPath: "/rea/hopper_bridge.py",
launchMode: "native",
},
{ verifyNativeHelper: async () => "verified" },
);
const retry = new HopperFixtureLauncher();
let session: BridgeSession | undefined;
const client = new HopperClient({
launcher: {
async launch(input, options) {
if (session !== undefined) return retry.launch(input);
session = input;
const launched = await native.launch(input, options);
expect(launched).toMatchObject({
ok: true,
value: { nativeHelperVerified: true },
});
return launched;
},
},
startupTimeoutMs: 10_000,
});
onTestFinished(async () => {
const closed = await client.close();
if (!closed.ok) throw closed.error;
});
const failed = await client.start();
expect(failed).toMatchObject({
ok: false,
error: { _tag: "HopperStartError", cleanupIncomplete: false },
});
if (failed.ok) throw new Error("Expected Spotlight failure");
expect(projectAnalysisError(failed.error)).toMatchObject({
code: "provider_unavailable",
message: expect.stringContaining("Spotlight"),
details: { launcher: { exit_code: 3 } },
});
if (session === undefined) throw new Error("Missing launch session");
await expect(access(session.directory)).rejects.toMatchObject({
code: "ENOENT",
});
const lease = await acquireHopperTargetLease({
targetPath,
targetKind: "executable",
loaderArgs,
runId: "spotlight-retry-proof",
});
if (!lease.acquired)
throw new Error("Spotlight failure retained the target lease");
await lease.lease.release();
expect(await client.start()).toMatchObject({ ok: true });
expect(await client.callTool("echo", { value: "retried" })).toEqual({
ok: true,
value: { value: "retried" },
});
},
);
it.each(["exit", "signal", "exit75"] as const)(
"reports a failed non-owning launcher %s without waiting for bridge timeout or inventing a GUI exit",
async (termination) => {
@@ -47,7 +47,7 @@ describe("managed member comparison", () => {
}),
]);
expect(result.limitations).toContain(
"Methods pair by exact CIL/signature, then declared type, name, and signature, then structural shape; names alone are never a matching basis.",
"Methods pair within one declared type and name by exact CIL/signature, then signature, then structural shape; only then may exact CIL/signature or structural shape alone remap across declared types or names, never taking a member whose own declared type and name remain unpaired on the other side or whose declaring type is unresolved. Names alone are never a matching basis.",
);
expect(
managedMemberComparisonResultSchema.safeParse({
@@ -339,10 +339,12 @@ describe("managed member comparison uncertainty", () => {
const rightField = right.result.fields[0];
expect(rightField).toBeDefined();
if (rightField === undefined) return;
// Both right fields are renamed, so neither left field keeps its declared
// identity and only the shared signature links them.
const duplicatedRight = {
...right.result,
fields: [
rightField,
{ ...rightField, name: "moved" },
{
...rightField,
token: "0x04000002",
@@ -365,6 +367,9 @@ describe("managed member comparison uncertainty", () => {
candidate_left_tokens: ["0x04000001", "0x04000002"],
candidate_right_tokens: ["0x04000001", "0x04000002"],
}),
limitations: [
"Multiple managed fields share the same signature; REA did not guess a token remap.",
],
}),
]);
});
@@ -582,7 +587,7 @@ describe("managed member comparison of ambiguous undecoded signatures", () => {
basis: "exact-il-signature",
}),
limitations: [
"Multiple managed methods share the same non-name identity key; REA did not guess a token remap.",
"Multiple managed methods share the same CIL and signature; REA did not guess a token remap.",
],
}),
expect.objectContaining({
@@ -670,6 +675,891 @@ describe("managed member comparison of selected types", () => {
});
});
describe("managed member comparison of one signature in several types", () => {
it("pairs a file with itself when two types hold identical members", () => {
const inspected = withTwin(
inspect(buildManagedPeFixture(), "/tmp/twin.dll"),
);
const result = compareManagedMembers(inspected, inspected);
expect(result.matching.ambiguous).toBe(0);
expect(result.summary).toMatchObject({
unchanged: 4,
changed: 0,
added: 0,
removed: 0,
unknown: 0,
});
for (const item of [...result.methods, ...result.fields])
expect(item.right?.token).toBe(item.left?.token);
});
it.each([1, 2])(
"keeps a member with its own changed counterpart beside %i identical members of other types",
(twins) => {
const left = inspect(buildManagedPeFixture(), "/tmp/own-before.dll");
const changed = withTwin(
inspect(
buildManagedPeFixture({
ilBody: Buffer.from([
0x32, 0x02, 0x7b, 0x02, 0x00, 0x00, 0x04, 0x28, 0x02, 0x00, 0x00,
0x0a, 0x2a,
]),
}),
"/tmp/own-after.dll",
),
left,
twins,
);
const result = compareManagedMembers(left, changed);
expect(result.methods).toEqual(
expect.arrayContaining([
expect.objectContaining({
status: "changed",
left: expect.objectContaining({
declaring_type: "Fixture.Program",
}),
right: expect.objectContaining({
declaring_type: "Fixture.Program",
}),
match: expect.objectContaining({ basis: "exact-signature" }),
dimensions: expect.arrayContaining(["cil"]),
}),
expect.objectContaining({
status: "added",
right: expect.objectContaining({ declaring_type: "Other.Twin" }),
}),
]),
);
expect(result.fields).toEqual(
expect.arrayContaining([
expect.objectContaining({
status: "unchanged",
left: expect.objectContaining({
declaring_type: "Fixture.Program",
}),
right: expect.objectContaining({
declaring_type: "Fixture.Program",
}),
}),
expect.objectContaining({
status: "added",
right: expect.objectContaining({ declaring_type: "Other.Twin" }),
}),
]),
);
expect(result.summary).toMatchObject({ unknown: 0, removed: 0 });
},
);
it("does not remap a member whose own identity remains on the other side", () => {
const left = inspect(buildManagedPeFixture(), "/tmp/sig-before.dll");
const changed = withTwin(
inspect(
buildManagedPeFixture({
methodSignature: Buffer.from([0x00, 0x00, 0x08]),
fieldSignature: Buffer.from([0x06, 0x0a]),
}),
"/tmp/sig-after.dll",
),
left,
);
// Either side may hold the member whose own identity is left behind.
const forward = compareManagedMembers(left, changed);
const backward = compareManagedMembers(changed, left);
for (const item of [
...forward.methods,
...forward.fields,
...backward.methods,
...backward.fields,
])
expect(item.match.status).not.toBe("matched");
expect(forward.summary).toMatchObject({ removed: 2, added: 4 });
expect(backward.summary).toMatchObject({ removed: 4, added: 2 });
});
it("does not remap by structural shape a method whose own identity remains on the other side", () => {
const left = inspect(buildManagedPeFixture(), "/tmp/shape-before.dll");
const changed = withTwin(
inspect(
buildManagedPeFixture({
methodSignature: Buffer.from([0x00, 0x00, 0x08]),
}),
"/tmp/shape-after.dll",
),
left,
);
const [own, twin] = changed.result.methods;
if (own === undefined || twin === undefined)
throw new Error("Expected two methods");
// Same body shape, different CIL digest: only the structural round links
// the left method to the twin.
const result = compareManagedMembers(left, {
...changed,
result: {
...changed.result,
methods: [
own,
{
...twin,
body: { ...twin.body, normalized_il_sha256: "0".repeat(64) },
},
],
},
});
for (const item of result.methods)
expect(item.match.status).not.toBe("matched");
});
});
describe("managed member comparison remaps around kept identities", () => {
it("does not hide a member whose own identity remains in an ambiguous remap group", () => {
const left = inspect(buildManagedPeFixture(), "/tmp/group-before.dll");
const changed = withTwin(
inspect(
buildManagedPeFixture({
methodSignature: Buffer.from([0x00, 0x00, 0x08]),
fieldSignature: Buffer.from([0x06, 0x0a]),
}),
"/tmp/group-after.dll",
),
left,
2,
);
const result = compareManagedMembers(left, changed);
expect(result.matching.ambiguous).toBe(0);
expect(result.summary).toMatchObject({ removed: 2, added: 6 });
});
it("remaps to the one candidate whose own identity is not kept elsewhere", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/kept-base.dll");
const changedSignature = inspect(
buildManagedPeFixture({
methodSignature: Buffer.from([0x00, 0x00, 0x08]),
}),
"/tmp/kept-signature.dll",
);
// Left: Fixture.Program.Main, plus Other.Twin.Main with another signature.
const left = withTwin(base, changedSignature);
// Right: Other.Twin.Main and Other.Twin3.Main, both identical to the
// left Fixture.Program.Main, which the right no longer has.
const twins = withTwin(base, base, 2);
const right = {
...twins,
result: { ...twins.result, methods: twins.result.methods.slice(1) },
};
const result = compareManagedMembers(left, right);
expect(result.methods).toEqual(
expect.arrayContaining([
expect.objectContaining({
left: expect.objectContaining({ declaring_type: "Fixture.Program" }),
right: expect.objectContaining({ declaring_type: "Other.Twin3" }),
match: expect.objectContaining({
status: "matched",
basis: "exact-il-signature",
}),
}),
]),
);
expect(
result.methods.filter(({ match }) => match.status === "ambiguous"),
).toEqual([]);
});
it("names the shared structural shape of an ambiguous remap group", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/shape-group.dll");
const twins = withTwin(base, base, 2);
// Two right methods share the left method's shape but not its CIL digest,
// and the right no longer declares the left method's identity.
const right = {
...twins,
result: {
...twins.result,
methods: twins.result.methods.slice(1).map((method) => ({
...method,
body: { ...method.body, normalized_il_sha256: "0".repeat(64) },
})),
fields: [],
},
};
const left = { ...base, result: { ...base.result, fields: [] } };
const result = compareManagedMembers(left, right);
expect(result.methods).toEqual([
expect.objectContaining({
match: expect.objectContaining({
status: "ambiguous",
basis: "structural-method-shape",
}),
limitations: [
"Multiple managed methods share the same structural shape; REA did not guess a token remap.",
],
}),
]);
});
});
describe("managed member comparison of unresolved and held identities", () => {
it("does not remap past an unresolved member that may be the counterpart", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/orphan-copy.dll");
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const left = { ...base, result: { ...base.result, fields: [] } };
// The right keeps Main with an unresolved owner, and an identical copy
// in another type; either could be the left Main.
const right = {
...base,
result: {
...base.result,
methods: [
{ ...method, declaring_type: null, declaring_type_token: null },
{
...method,
declaring_type: "Other.Twin",
declaring_type_token: "0x02000003",
name: "Copy",
token: "0x06000002",
},
],
fields: [],
},
};
for (const result of [
compareManagedMembers(left, right),
compareManagedMembers(right, left),
]) {
expect(
result.methods.filter(({ match }) => match.status === "matched"),
).toEqual([]);
expect(result.summary).toMatchObject({ added: 0, removed: 0 });
}
});
});
describe("managed member comparison of raw-signature contention", () => {
it("contests same-signature methods even when their bodies differ", () => {
const base = inspect(
buildManagedPeFixture(),
"/tmp/ownerless-signatures.dll",
);
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const changedBody = (hash: string, stack: number) => ({
...method.body,
normalized_il_sha256: hash.repeat(64),
max_stack: stack,
});
const withOwnerless = {
...base,
result: {
...base.result,
fields: [],
methods: [
{ ...method, body: changedBody("a", 1) },
{
...method,
token: "0x06000002",
declaring_type: null,
declaring_type_token: null,
body: changedBody("b", 2),
},
],
},
};
const resolvedOnly = {
...base,
result: {
...base.result,
fields: [],
methods: [{ ...method, body: changedBody("c", 3) }],
},
};
for (const result of [
compareManagedMembers(withOwnerless, resolvedOnly),
compareManagedMembers(resolvedOnly, withOwnerless),
]) {
expect(result.methods).toHaveLength(1);
const [item] = result.methods;
if (item === undefined)
throw new Error("Expected uncertain signature group");
expect(item).toMatchObject({
status: "unknown",
match: { status: "ambiguous", basis: "exact-signature" },
});
expect(result.summary).toMatchObject({
added: 0,
removed: 0,
unknown: 1,
});
expect(
item.match.candidate_left_tokens.length +
item.match.candidate_right_tokens.length,
).toBe(3);
}
});
it("contests ownerless undecoded fields by raw signature", () => {
const base = inspect(
buildManagedPeFixture({ fieldSignature: Buffer.from([0x06, 0x7f]) }),
"/tmp/ownerless-field-signatures.dll",
);
const [field] = base.result.fields;
if (field === undefined) throw new Error("Expected a fixture field");
expect(field.signature.parse_status).toBe("unsupported");
const withOwnerless = {
...base,
result: {
...base.result,
methods: [],
fields: [
field,
{
...field,
token: "0x04000002",
declaring_type: null,
declaring_type_token: null,
},
],
},
};
const resolvedOnly = {
...base,
result: { ...base.result, methods: [], fields: [field] },
};
for (const result of [
compareManagedMembers(withOwnerless, resolvedOnly),
compareManagedMembers(resolvedOnly, withOwnerless),
])
expect(
result.fields.filter(({ match }) => match.status === "matched"),
).toEqual([]);
});
});
describe("managed member comparison of consumed ownerless ambiguity", () => {
it("keeps a consumed ownerless ambiguity from authorizing a later remap", () => {
const base = inspect(
buildManagedPeFixture(),
"/tmp/consumed-ownerless.dll",
);
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const candidate = (
raw: string,
token: string,
owner: string | null,
name: string,
stack: number,
) => ({
...method,
token,
declaring_type: owner,
declaring_type_token: owner === null ? null : method.declaring_type_token,
name,
signature: { ...method.signature, raw_sha256: raw.repeat(64) },
body: {
...method.body,
normalized_il_sha256: raw.repeat(64),
max_stack: stack,
},
});
const left = {
...base,
result: {
...base.result,
fields: [],
methods: [
candidate("a", "0x06000001", "T", "M", 1),
candidate("b", "0x06000002", null, "M", 1),
candidate("b", "0x06000003", "D", "D", 2),
],
},
};
const right = {
...base,
result: {
...base.result,
fields: [],
methods: [
candidate("c", "0x06000001", "U", "N", 1),
candidate("b", "0x06000002", null, "M", 1),
candidate("d", "0x06000003", "T", "M", 3),
],
},
};
for (const result of [
compareManagedMembers(left, right),
compareManagedMembers(right, left),
]) {
expect(result.matching.ambiguous).toBeGreaterThan(0);
expect(
result.methods.filter(({ match }) => match.status === "matched"),
).toEqual([]);
}
});
});
describe("managed member comparison of retained ownership contention", () => {
it("keeps changed counterparts unknown after an ownerless candidate is held", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/held-ownerless.dll");
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const body = (hash: string, stack: number) => ({
...method.body,
il_sha256: hash.repeat(64),
normalized_il_sha256: hash.repeat(64),
max_stack: stack,
});
const ownerless = {
...method,
token: "0x06000002",
declaring_type: null,
declaring_type_token: null,
body: body("b", 2),
};
const left = {
...base,
result: {
...base.result,
fields: [],
methods: [{ ...method, body: body("a", 1) }, ownerless],
},
};
const right = {
...base,
result: {
...base.result,
fields: [],
methods: [
{ ...method, body: body("c", 3) },
{
...method,
token: "0x06000002",
declaring_type: "Other.Twin",
name: "Copy",
body: ownerless.body,
},
],
},
};
for (const result of [
compareManagedMembers(left, right),
compareManagedMembers(right, left),
]) {
expect(result.summary).toEqual({
unchanged: 0,
changed: 0,
added: 0,
removed: 0,
unknown: 2,
});
expect(result.methods).toEqual([
expect.objectContaining({
status: "unknown",
match: {
status: "ambiguous",
basis: "exact-il-signature",
confidence: "unknown",
candidate_left_tokens: ["0x06000002"],
candidate_right_tokens: ["0x06000002"],
},
}),
expect.objectContaining({
status: "unknown",
match: {
status: "ambiguous",
basis: "exact-signature",
confidence: "unknown",
candidate_left_tokens: ["0x06000001"],
candidate_right_tokens: ["0x06000001"],
},
}),
]);
}
});
});
describe("managed member comparison of ownerless structural competitors", () => {
it("keeps structural competitors of an ownerless method ambiguous in both directions", () => {
const base = inspect(
buildManagedPeFixture(),
"/tmp/ownerless-shape-competitors.dll",
);
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const withSignature = (raw: string, token: string) => ({
...method,
token,
signature: { ...method.signature, raw_sha256: raw.repeat(64) },
});
const withOwnerless = {
...base,
result: {
...base.result,
methods: [
withSignature("a", "0x06000001"),
{
...withSignature("b", "0x06000002"),
declaring_type: null,
declaring_type_token: null,
},
],
fields: [],
},
};
const resolvedOnly = {
...base,
result: {
...base.result,
methods: [withSignature("c", "0x06000001")],
fields: [],
},
};
for (const result of [
compareManagedMembers(withOwnerless, resolvedOnly),
compareManagedMembers(resolvedOnly, withOwnerless),
]) {
expect(result.matching.ambiguous).toBe(1);
expect(
result.methods.filter(({ match }) => match.status === "matched"),
).toEqual([]);
expect(result.summary).toMatchObject({ added: 0, removed: 0 });
}
});
});
describe("managed member comparison of ownerless members", () => {
it("keeps an ownerless member ambiguous beside a resolved copy, as main does", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-copy.dll");
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const ownerless = {
...method,
declaring_type: null,
declaring_type_token: null,
};
const left = {
...base,
result: { ...base.result, methods: [ownerless], fields: [] },
};
// The right keeps the ownerless Foo and a resolved copy with the same
// CIL and signature; the left Foo's owner may be either one.
const right = {
...base,
result: {
...base.result,
methods: [ownerless, { ...method, token: "0x06000002" }],
fields: [],
},
};
for (const result of [
compareManagedMembers(left, right),
compareManagedMembers(right, left),
]) {
expect(result.matching.ambiguous).toBe(1);
expect(
result.methods.filter(({ match }) => match.status === "matched"),
).toEqual([]);
expect(result.summary).toMatchObject({ added: 0, removed: 0 });
}
});
it("still pairs a lone ownerless member with its lone counterpart", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-lone.dll");
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const ownerless = {
...method,
declaring_type: null,
declaring_type_token: null,
};
const side = {
...base,
result: { ...base.result, methods: [ownerless], fields: [] },
};
const result = compareManagedMembers(side, side);
expect(result.matching.ambiguous).toBe(0);
expect(
result.methods.filter(({ match }) => match.status === "matched"),
).toHaveLength(1);
});
it("keeps a resolved copy out of the declared round when an ownerless competitor is on the left, as main does", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-left.dll");
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const ownerless = {
...method,
declaring_type: null,
declaring_type_token: null,
};
// The left holds the resolved Foo and an ownerless Foo with the same CIL
// and signature; the right holds only the resolved Foo. The ownerless
// Foo's owner may be the resolved one, so the three stay ambiguous.
const withOwnerless = {
...base,
result: { ...base.result, methods: [method, ownerless], fields: [] },
};
const resolvedOnly = {
...base,
result: { ...base.result, methods: [method], fields: [] },
};
const result = compareManagedMembers(withOwnerless, resolvedOnly);
expect(result.matching.ambiguous).toBe(1);
expect(
result.methods.filter(({ match }) => match.status === "matched"),
).toEqual([]);
expect(result.summary).toMatchObject({ added: 0, removed: 0 });
});
it("keeps a resolved copy out of the declared round when an ownerless competitor is on the right, as main does", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-right.dll");
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const ownerless = {
...method,
declaring_type: null,
declaring_type_token: null,
};
const withOwnerless = {
...base,
result: { ...base.result, methods: [method, ownerless], fields: [] },
};
const resolvedOnly = {
...base,
result: { ...base.result, methods: [method], fields: [] },
};
const result = compareManagedMembers(resolvedOnly, withOwnerless);
expect(result.matching.ambiguous).toBe(1);
expect(
result.methods.filter(({ match }) => match.status === "matched"),
).toEqual([]);
expect(result.summary).toMatchObject({ added: 0, removed: 0 });
});
it("keeps an ownerless method ambiguous beside a resolved method of the same shape", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/ownerless-shape.dll");
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const ownerless = {
...method,
declaring_type: null,
declaring_type_token: null,
};
// Raw signature digests differ, so only the decoded shape links them.
const reshaped = (raw: string, token: string) => ({
signature: { ...method.signature, raw_sha256: raw.repeat(64) },
token,
});
const left = {
...base,
result: { ...base.result, methods: [ownerless], fields: [] },
};
const right = {
...base,
result: {
...base.result,
methods: [
{ ...ownerless, ...reshaped("e", "0x06000001") },
{ ...method, name: "Bar", ...reshaped("d", "0x06000002") },
],
fields: [],
},
};
const result = compareManagedMembers(left, right);
expect(result.matching.ambiguous).toBe(1);
expect(
result.methods.filter(({ match }) => match.status === "matched"),
).toEqual([]);
});
});
describe("managed member comparison of unresolved owners", () => {
it("does not pair members of unresolved types under different names", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/orphan-names.dll");
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const orphan = { declaring_type: null, declaring_type_token: null };
const side = (name: string) => ({
...base,
result: {
...base.result,
methods: [{ ...method, ...orphan, name }],
fields: [],
},
});
const result = compareManagedMembers(side("Main"), side("Renamed"));
for (const item of result.methods)
expect(item.match.status).not.toBe("matched");
});
it("keeps ownerless members that share a body ambiguous, as main does", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/orphan-self.dll");
const [method] = base.result.methods;
const [field] = base.result.fields;
if (method === undefined || field === undefined)
throw new Error("Expected a fixture method and field");
const orphan = { declaring_type: null, declaring_type_token: null };
const inspected = {
...base,
result: {
...base.result,
methods: [
{ ...method, ...orphan, name: "p" },
{ ...method, ...orphan, name: "q", token: "0x06000002" },
],
fields: [
{ ...field, ...orphan, name: "a" },
{ ...field, ...orphan, name: "b", token: "0x04000002" },
],
},
};
const result = compareManagedMembers(inspected, inspected);
// Their owners are unknown, so `p` may be `q`; names alone cannot tell.
expect(result.matching.ambiguous).toBe(2);
expect(
[...result.methods, ...result.fields].filter(
({ match }) => match.status === "matched",
),
).toEqual([]);
});
it("does not remap a member whose own identity is in an ambiguous group", () => {
const undecoded = { methodSignature: Buffer.from([0x00, 0x00, 0x7f]) };
const decoded = inspect(buildManagedPeFixture(), "/tmp/held-decoded.dll");
const held = inspect(buildManagedPeFixture(undecoded), "/tmp/held.dll");
const [undecodedMethod] = held.result.methods;
const [decodedMethod] = decoded.result.methods;
if (undecodedMethod === undefined || decodedMethod === undefined)
throw new Error("Expected fixture methods");
// The right keeps two identical undecoded Fixture.Program.Main, which
// stay ambiguous, and a copy of the decoded Main in another type.
const left = {
...held,
result: {
...held.result,
methods: [{ ...undecodedMethod, token: "0x06000002" }, decodedMethod],
},
};
const right = {
...held,
result: {
...held.result,
methods: [
undecodedMethod,
{ ...undecodedMethod, token: "0x06000002" },
{
...decodedMethod,
declaring_type: "Other.Twin",
declaring_type_token: "0x02000003",
token: "0x06000003",
},
],
},
};
for (const result of [
compareManagedMembers(left, right),
compareManagedMembers(right, left),
])
expect(
result.methods.filter(({ match }) => match.status === "matched"),
).toEqual([]);
});
it("does not remap a member whose declaring type is unresolved", () => {
const base = inspect(buildManagedPeFixture(), "/tmp/orphan.dll");
const [method] = base.result.methods;
if (method === undefined) throw new Error("Expected a fixture method");
const left = {
...base,
result: {
...base.result,
methods: [
{ ...method, declaring_type: null, declaring_type_token: null },
],
fields: [],
},
};
const right = {
...base,
result: {
...base.result,
methods: [
{ ...method, declaring_type: "Other.Twin", token: "0x06000002" },
],
fields: [],
},
};
for (const result of [
compareManagedMembers(left, right),
compareManagedMembers(right, left),
])
for (const item of result.methods)
expect(item.match.status).not.toBe("matched");
});
});
/**
* Add copies of the fixture's method and field, each pair declared by another
* type, to an inspection. The copies come from `source`, so they can carry
* the bytes of a different build than the inspection they join.
*/
const withTwin = (
inspected: ReturnType<typeof inspect>,
source: ReturnType<typeof inspect> = inspected,
count = 1,
): ReturnType<typeof inspect> => {
const [method] = source.result.methods;
const [field] = source.result.fields;
if (method === undefined || field === undefined)
throw new Error("Expected a fixture method and field");
const rows = Array.from({ length: count }, (_, index) => index + 2);
const twin = (row: number) => ({
declaring_type: row === 2 ? "Other.Twin" : `Other.Twin${String(row)}`,
declaring_type_token: `0x0200000${String(row + 1)}`,
});
return {
...inspected,
result: {
...inspected.result,
methods: [
...inspected.result.methods,
...rows.map((row) => ({
...method,
...twin(row),
token: `0x0600000${String(row)}`,
})),
],
fields: [
...inspected.result.fields,
...rows.map((row) => ({
...field,
...twin(row),
token: `0x0400000${String(row)}`,
})),
],
},
};
};
const inspect = (bytes: Buffer, path: string, typeNames?: string[]) => {
const target = managedPeFixtureTarget(bytes, path);
const result = inspectManagedMembersBytes(
@@ -6,10 +6,11 @@ import { promisify } from "node:util";
import { describe, expect, it } from "vitest";
import {
HOPPER_PUBLISHER,
macHopperInstallDisclosure,
systemMacHopperInstallHost,
} from "../../../src/application/MacHopper.js";
import { verifyMacHopperNativeHelper } from "../../../src/hopper/HopperMacPublisher.js";
import { HOPPER_PUBLISHER } from "../../../src/domain/hopperPublisher.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const run = promisify(execFile);
@@ -94,6 +95,24 @@ describe("macOS Hopper publisher verification", () => {
);
});
itDarwin(
"withholds native-helper provenance from custom signed or modified executables",
async () => {
const root = await createTestTempDirectory(
"rea-hopper-helper-publisher-",
);
const app = await buildBundle(root, "hopper");
const executable = join(app, ...executableSegments);
await expect(
verifyMacHopperNativeHelper(executable, process.env),
).resolves.toBe("unknown");
await appendFile(executable, "modified\n");
await expect(
verifyMacHopperNativeHelper(executable, process.env),
).resolves.toBe("unknown");
},
);
// Opt-in positive control: point this at a mounted official Hopper bundle,
// for example `hdiutil attach -readonly -nobrowse Hopper-6.6.0-demo.dmg`.
// It proves the pinned requirement is satisfiable by the real publisher, so
@@ -103,6 +122,12 @@ describe("macOS Hopper publisher verification", () => {
"accepts the officially signed Hopper application",
async () => {
await expect(host().validBundle(realApp ?? "")).resolves.toBe("verified");
await expect(
verifyMacHopperNativeHelper(
join(realApp ?? "", ...executableSegments),
process.env,
),
).resolves.toBe("verified");
},
);
});
@@ -0,0 +1,93 @@
import { describe, expect, it } from "vitest";
import type { BinaryTarget } from "../../../src/domain/binaryTargetTypes.js";
import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js";
import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js";
import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js";
import {
createAnalysisSnapshotEntry,
parseAnalysisSnapshot,
serializeAnalysisSnapshot,
snapshotBinding,
snapshotTarget,
snapshotMatchesProfile,
} from "../../../src/domain/analysisSnapshot.js";
import { sessionOutputSchemas } from "../../../src/contracts/toolOutputSchemaGroups.js";
const target: BinaryTarget = {
path: "/source-owned.elf",
sha256: "a".repeat(64),
kind: "executable",
format: "elf",
architecture: "powerpc",
availableArchitectures: ["powerpc"],
powerpc: { elfClass: 32, byteOrder: "big", type: 2, flags: 0 },
};
const provider = { id: "ghidra", name: "Ghidra", version: "12.1.4" };
const profile = createAnalysisProfile(provider, {
powerpc_elf: { elf_class: 32, byte_order: "big", type: 2, flags: 0 },
});
describe("PowerPC identity through lifecycle, Evidence and snapshot consumers", () => {
it("round-trips query bindings without relaxing integrity", () => {
const result = [{ address: "0x100101b0", value: "rea_ppc_entry" }];
const parameters = {};
const evidence = createEvidence(target, provider, {
operation: "list_procedures",
parameters,
result,
analysisProfile: profile,
});
const binding = snapshotBinding(profile);
const entry = createAnalysisSnapshotEntry({
target: snapshotTarget(target),
binding,
operation: "list_procedures",
parameters,
execution: {
result,
rawResult: null,
provider,
analysisProfile: profile,
limitations: [],
locations: [],
subject: target,
},
});
const snapshot = {
target: snapshotTarget(target),
binding,
entries: [entry],
workflow_entries: [],
evidence_bundle: createEvidenceBundle([evidence]),
};
const restored = parseAnalysisSnapshot(
JSON.parse(serializeAnalysisSnapshot(snapshot)),
);
expect(restored.target.architecture).toBe("powerpc");
expect(restored.evidence_bundle.records[0]?.subject?.architecture).toBe(
"powerpc",
);
expect(restored.entries[0]?.execution.subject?.architecture).toBe(
"powerpc",
);
expect(
sessionOutputSchemas.open_binary.parse({ result: target }).result
.architecture,
).toBe("powerpc");
expect(() =>
parseEvidence({
...evidence,
subject: { ...evidence.subject, architecture: "x86" },
}),
).toThrow("semantic identifier");
const changed = createAnalysisProfile(provider, {
powerpc_elf: {
elf_class: 32,
byte_order: "big",
type: 2,
flags: 0x00010000,
},
});
expect(snapshotMatchesProfile(binding, changed)).toBe(false);
});
});
+14
View File
@@ -0,0 +1,14 @@
/* Freestanding, source-owned static-analysis fixture. Never executed by REA. */
volatile unsigned int rea_ppc_global = 7;
const char rea_ppc_marker[] = "rea-ppc-source-owned-fixture";
__attribute__((noinline)) unsigned int rea_ppc_leaf(unsigned int value)
{
return value * 3u + rea_ppc_global;
}
__attribute__((noinline)) unsigned int rea_ppc_entry(unsigned int selector)
{
unsigned int adjusted = selector > 10u ? selector : selector + 1u;
return rea_ppc_leaf(adjusted) + (unsigned char)rea_ppc_marker[0];
}
+29
View File
@@ -0,0 +1,29 @@
import type { PowerPcElfMetadata } from "../../../src/domain/binaryTargetTypes.js";
export interface PowerPcReadelfFacts {
powerpc: PowerPcElfMetadata;
symbols: Record<
| "rea_ppc_entry"
| "rea_ppc_leaf"
| "rea_ppc_probe"
| "rea_ppc_global"
| "rea_ppc_marker",
string
>;
}
export function parsePowerPcReadelf(text: string): PowerPcReadelfFacts;
export function inspectPowerPcReadelf(
path: string,
): Promise<
PowerPcReadelfFacts & { command: string; version: string; raw: string }
>;
export function assertPowerPcLoadedImage(
observations: unknown,
sha256: string,
): void;
export function assertPowerPcProbe(
move: unknown,
branch: unknown,
address: string,
): void;
export function assertPowerPcGlobal(bytes: string): void;
+132
View File
@@ -0,0 +1,132 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
const execute = promisify(execFile);
// The lane binds this language; Ghidra's ELF loader alone selects e500.
const LANGUAGE = "PowerPC:BE:32:default";
const wordHex = (value) => {
const bytes = Buffer.alloc(4);
bytes.writeUInt32BE(value);
return bytes.toString("hex");
};
/** Parse only the documented GNU readelf fields needed by our tiny fixture. */
export function parsePowerPcReadelf(text) {
const field = (expression, label) => {
const matches = [...text.matchAll(expression)];
assert.equal(matches.length, 1, `Missing or ambiguous readelf ${label}`);
return matches[0][1].trim();
};
assert.equal(field(/^\s*Class:\s*(.+)$/gmu, "class"), "ELF32");
assert.equal(
field(/^\s*Data:.*?,\s*(little|big) endian\s*$/gmu, "byte order"),
"big",
);
assert.match(field(/^\s*Type:\s*(.+)$/gmu, "type"), /^EXEC\b/u);
assert.equal(field(/^\s*Machine:\s*(.+)$/gmu, "machine"), "PowerPC");
const flags = Number(field(/^\s*Flags:\s*(0x[\da-f]+)\s*$/gmu, "flags"));
assert.equal(flags, 0, "Fixture ELF flags differ from build intent");
const symbols = {};
for (const name of [
"rea_ppc_entry",
"rea_ppc_leaf",
"rea_ppc_probe",
"rea_ppc_global",
"rea_ppc_marker",
]) {
const value = field(
new RegExp(
`^\\s*\\d+:\\s+([\\da-f]+)\\s+\\d+\\s+(?:FUNC|OBJECT)\\s+GLOBAL\\s+\\S+\\s+\\d+\\s+${name}\\s*$`,
"gmu",
),
`symbol ${name}`,
);
symbols[name] = `0x${BigInt(`0x${value}`).toString(16)}`;
}
return {
powerpc: { elfClass: 32, byteOrder: "big", type: 2, flags },
symbols,
};
}
/** Independent reader is a bounded test prerequisite, never an REA provider. */
export async function inspectPowerPcReadelf(path) {
const command = process.env.REA_PPC_READELF ?? "readelf";
const options = {
env: { ...process.env, LC_ALL: "C" },
timeout: 10000,
maxBuffer: 1024 * 1024,
};
const version = await execute(command, ["--version"], options);
assert.match(
version.stdout,
/^GNU readelf\b/u,
"This lane requires GNU readelf",
);
const result = await execute(command, ["-h", "-s", "-W", path], options);
assert.equal(
result.stderr.trim(),
"",
`readelf diagnostics: ${result.stderr}`,
);
return {
...parsePowerPcReadelf(result.stdout),
command,
version: version.stdout.split("\n")[0],
raw: result.stdout,
};
}
/** Check effective Ghidra observations, not the requested analysis profile. */
export function assertPowerPcLoadedImage(observations, sha256) {
assert.equal(observations.language_id, LANGUAGE);
assert.equal(observations.compiler_spec_id, "default");
assert.ok(
observations.source_files.some(
(file) =>
file.original_sha256 === sha256 && file.modified_sha256 === sha256,
),
"Ghidra loaded source identity changed",
);
}
/** Verify the exact constant/conditional destination in ppc-probe.S. */
export function assertPowerPcProbe(move, branch, address) {
const start = BigInt(address);
for (const [instruction, offset, word] of [
[move, 0n, 0x38641234],
[branch, 8n, 0x41820008],
]) {
assert.equal(instruction.status, "decoded");
assert.equal(BigInt(instruction.address), start + offset);
assert.equal(instruction.architecture, LANGUAGE);
assert.equal(instruction.length, 4);
assert.equal(instruction.bytes, wordHex(word));
}
assert.match(move.mnemonic, /^ADDI$/iu);
assert.ok(
move.operands.some((operand) =>
operand.components.some(
(part) =>
part.kind === "immediate" &&
part.value !== null &&
BigInt(part.value) === 0x1234n,
),
),
"ADDI immediate was not recovered",
);
assert.match(branch.mnemonic, /^BEQ$/iu);
assert.equal(branch.flow.kind, "jump");
assert.equal(branch.flow.conditional, true);
assert.equal(branch.flow.computed, false);
assert.deepEqual(
branch.flow.direct_destinations.map((value) => BigInt(value)),
[start + 16n],
);
}
/** Four bytes must encode the known source-owned global in big-endian order. */
export function assertPowerPcGlobal(bytes) {
assert.equal(bytes, wordHex(7));
}
+187
View File
@@ -0,0 +1,187 @@
import { describe, expect, it } from "vitest";
import {
assertPowerPcGlobal,
assertPowerPcLoadedImage,
assertPowerPcProbe,
parsePowerPcReadelf,
} from "./ppc-checks.mjs";
// GNU readelf 2.47 output for the source-owned Clang 23 / LLD fixture.
// This tests the reader boundary, not Ghidra; the real lane rebuilds it.
const readelf = `ELF Header:
Magic: 7f 45 4c 46 01 02 01 00 00 00 00 00 00 00 00 00
Class: ELF32
Data: 2's complement, big endian
Version: 1 (current)
OS/ABI: UNIX - System V
ABI Version: 0
Type: EXEC (Executable file)
Machine: PowerPC
Version: 0x1
Entry point address: 0x100101b0
Start of program headers: 52 (bytes into file)
Start of section headers: 2188 (bytes into file)
Flags: 0x0
Size of this header: 52 (bytes)
Size of program headers: 32 (bytes)
Number of program headers: 6
Size of section headers: 40 (bytes)
Number of section headers: 18
Section header string table index: 16
Symbol table '.symtab' contains 7 entries:
Num: Value Size Type Bind Vis Ndx Name
0: 00000000 0 NOTYPE LOCAL DEFAULT UND
1: 00000000 0 FILE LOCAL DEFAULT ABS ppc.c
2: 10010180 48 FUNC GLOBAL DEFAULT 4 rea_ppc_leaf
3: 1002023c 4 OBJECT GLOBAL DEFAULT 5 rea_ppc_global
4: 100101b0 116 FUNC GLOBAL DEFAULT 4 rea_ppc_entry
5: 100000f4 29 OBJECT GLOBAL DEFAULT 1 rea_ppc_marker
6: 10010224 24 FUNC GLOBAL DEFAULT 4 rea_ppc_probe
`;
const sha256 = "a".repeat(64);
const image = {
language_id: "PowerPC:BE:32:default",
compiler_spec_id: "default",
source_files: [{ original_sha256: sha256, modified_sha256: sha256 }],
};
const move = {
address: "0x10010224",
status: "decoded",
architecture: "PowerPC:BE:32:default",
length: 4,
bytes: "38641234",
mnemonic: "addi",
operands: [
{ index: 2, components: [{ kind: "immediate", value: "0x1234" }] },
],
};
const branch = {
address: "0x1001022c",
status: "decoded",
architecture: "PowerPC:BE:32:default",
length: 4,
bytes: "41820008",
mnemonic: "beq",
flow: {
kind: "jump",
conditional: true,
computed: false,
direct_destinations: ["0x10010234"],
},
};
describe("PowerPC real-verifier independent expectations", () => {
it("reads GNU header and defined symbols without a Ghidra-derived oracle", () => {
const parsed = parsePowerPcReadelf(readelf);
expect(parsed.powerpc).toEqual({
elfClass: 32,
byteOrder: "big",
type: 2,
flags: 0,
});
expect(parsed.symbols.rea_ppc_probe).toBe("0x10010224");
});
it.each([
["wrong class", "ELF32", "ELF64"],
["wrong byte order", "big endian", "little endian"],
[
"wrong machine",
"Machine: PowerPC",
"Machine: PowerPC64",
],
["wrong object type", "EXEC (Executable file)", "DYN (Shared object file)"],
[
"wrong flags",
"Flags: 0x0",
"Flags: 0x80000000",
],
["missing symbol", "rea_ppc_probe", "unrelated_probe"],
])(
"rejects %s instead of returning a guessed fixture interpretation",
(_name, from, to) => {
expect(() => parsePowerPcReadelf(readelf.replace(from, to))).toThrow();
},
);
it("rejects ambiguous producer fields", () => {
expect(() => parsePowerPcReadelf(`${readelf}\nClass: ELF32\n`)).toThrow(
/ambiguous/u,
);
});
it("checks the effective loaded image, including Ghidra's e500 default", () => {
assertPowerPcLoadedImage(image, sha256);
expect(() =>
assertPowerPcLoadedImage(
{ ...image, language_id: "PowerPC:BE:32:e500" },
sha256,
),
).toThrow();
expect(() =>
assertPowerPcLoadedImage(
{ ...image, compiler_spec_id: "macosx" },
sha256,
),
).toThrow();
expect(() =>
assertPowerPcLoadedImage(
{
...image,
source_files: [
{ original_sha256: sha256, modified_sha256: "b".repeat(64) },
],
},
sha256,
),
).toThrow();
});
it("checks exact probe bytes, immediate and conditional destination", () => {
assertPowerPcProbe(move, branch, "0x10010224");
});
it("rejects plausible nonempty but incorrect instruction observations", () => {
expect(() =>
assertPowerPcProbe({ ...move, bytes: "38641235" }, branch, "0x10010224"),
).toThrow();
expect(() =>
assertPowerPcProbe({ ...move, length: 2 }, branch, "0x10010224"),
).toThrow();
expect(() =>
assertPowerPcProbe(
{
...move,
operands: [
{ index: 2, components: [{ kind: "immediate", value: "0x1235" }] },
],
},
branch,
"0x10010224",
),
).toThrow();
expect(() =>
assertPowerPcProbe(
move,
{
...branch,
flow: { ...branch.flow, direct_destinations: ["0x10010230"] },
},
"0x10010224",
),
).toThrow();
expect(() =>
assertPowerPcProbe(
move,
{ ...branch, flow: { ...branch.flow, conditional: false } },
"0x10010224",
),
).toThrow();
});
it("checks the big-endian global word", () => {
assertPowerPcGlobal("00000007");
expect(() => assertPowerPcGlobal("07000000")).toThrow();
});
});
+62
View File
@@ -0,0 +1,62 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
const execute = promisify(execFile);
/** Build a source-owned freestanding ELF; never execute its target code. */
export async function buildPowerPcFixture(directory) {
const compiler = process.env.REA_PPC_CLANG ?? "clang";
const target = "powerpc-unknown-linux-gnu";
const source = fileURLToPath(new URL("../c/ppc.c", import.meta.url));
const probe = fileURLToPath(new URL("./ppc-probe.S", import.meta.url));
const path = join(directory, `${target}.elf`);
const args = [
`--target=${target}`,
// A 750-family core with the classic FPU, unlike the e500 SPE variant.
"-mcpu=750",
"-fno-pic",
"-fuse-ld=lld",
"-nostdlib",
"-static",
"-O0",
"-g",
"-Wl,-e,rea_ppc_entry",
"-Wl,--build-id=none",
source,
probe,
"-o",
path,
];
const version = await execute(compiler, ["--version"], { timeout: 10000 });
await execute(compiler, args, { timeout: 60000 });
const bytes = await readFile(path);
if (
bytes.length < 52 ||
bytes.toString("hex", 0, 4) !== "7f454c46" ||
bytes[4] !== 1 ||
bytes[5] !== 2 ||
bytes.readUInt16BE(16) !== 2 ||
bytes.readUInt16BE(18) !== 20 ||
bytes.readUInt32BE(36) !== 0
)
throw new Error(
"Compiler did not produce the declared ELF32 big-endian PowerPC fixture",
);
return {
path,
flags: bytes.readUInt32BE(36),
sha256: createHash("sha256").update(bytes).digest("hex"),
source_sha256: createHash("sha256")
.update(await readFile(source))
.digest("hex"),
probe_sha256: createHash("sha256")
.update(await readFile(probe))
.digest("hex"),
compiler: version.stdout.trim(),
arguments: args,
};
}
+17
View File
@@ -0,0 +1,17 @@
/* Source-owned 32-bit PowerPC instructions with fixed, reviewable encodings.
* No target execution. The conditional branch at +8 targets +16.
*/
.text
.p2align 2
.globl rea_ppc_probe
.type rea_ppc_probe, @function
rea_ppc_probe:
addi 3, 4, 0x1234
cmpwi 0, 4, 0
beq 0, .Lzero
blr
.Lzero:
li 3, 7
blr
.size rea_ppc_probe, .-rea_ppc_probe
.section .note.GNU-stack,"",@progbits
+7
View File
@@ -19,6 +19,13 @@ if (mode === "denied" || mode === "denied-after-handoff") {
process.exit(2);
},
);
} else if (mode === "not-found") {
// A custom wrapper can forward the native diagnostic after a handoff.
await writeFile(`${receiptPath}.document-created`, "unconfirmed");
process.stderr.write(
"Hopper has not been found.\nPlease make sure that Spotlight is active in order for this tool to find Hopper's executable.\n",
() => process.exit(3),
);
} else if (mode === "handoff") {
process.exit(0);
} else if (mode === "bridge") {
+4 -8
View File
@@ -63,9 +63,7 @@
"!docs/.vitepress/dist/**",
"!website/_site/**",
"!experiments/analysis-efficiency/results/**",
"vitest.reverse-tools.config.ts",
"vitest.reverse-tools-semantics.config.ts",
"vitest.config.ts"
"*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
]
},
"lint:built": {
@@ -86,9 +84,7 @@
"!docs/.vitepress/dist/**",
"!website/_site/**",
"!experiments/analysis-efficiency/results/**",
"vitest.reverse-tools.config.ts",
"vitest.reverse-tools-semantics.config.ts",
"vitest.config.ts"
"*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
]
},
"format:check": {},
@@ -104,8 +100,8 @@
"tests/**/*.ts",
"tsconfig.json",
"tsconfig.build.json",
"vitest.config.ts",
"docs/.vitepress/config.ts"
"docs/.vitepress/config.ts",
"*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
]
},
"jscpd": {
+36 -64
View File
@@ -3,6 +3,10 @@ import { availableParallelism, tmpdir } from "node:os";
import { join } from "node:path";
import { defineConfig } from "vitest/config";
import {
BUILT_TEST_PROJECTS,
SOURCE_TEST_PROJECTS,
} from "./scripts/lib/test-projects.mjs";
import { CiTestSequencer } from "./tests/ciTestSequencer.js";
@@ -28,57 +32,32 @@ const COVERAGE_SHARD = process.argv.some((argument) =>
// Keep local and CI concurrency bounded by the runner rather than command locks.
const MAX_TEST_WORKERS = Math.min(2, availableParallelism());
const TEST_PROJECTS = [
{
name: "domain",
include: ["src/{contracts,domain}/**/*.test.ts"],
const TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS];
const PROJECT_POLICIES = {
domain: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Domain and contract modules are pure and own no process-global state, so
// per-file module isolation adds startup cost without protecting a seam.
isolate: false,
},
{
name: "services",
include: ["src/application/**/*.test.ts"],
services: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Service tests use recording ports and own no process-global state. Share
// their module graph so file startup does not dominate the lane.
isolate: false,
},
{
name: "adapters",
include: [
"src/*.test.ts",
"src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
],
pool: "forks" as const,
maxWorkers: MAX_TEST_WORKERS,
},
{
name: "composition",
include: ["tests/composition/**/*.test.ts"],
adapters: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
composition: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Composition tests use provider-neutral sessions and test-scoped cleanup;
// sharing their immutable module graph avoids per-file startup work.
isolate: false,
},
{
name: "boundary",
include: ["tests/boundary/**/*.test.ts"],
exclude: [
"tests/boundary/cli/**/*.test.ts",
"tests/boundary/mcp/**/*.test.ts",
"tests/boundary/process/**/*.test.ts",
],
pool: "forks" as const,
maxWorkers: MAX_TEST_WORKERS,
},
{
name: "cli-boundary",
include: ["tests/boundary/cli/**/*.test.ts"],
boundary: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
"cli-boundary": {
pool: "forks" as const,
maxWorkers: MAX_TEST_WORKERS,
// CLI tests create fresh command instances or subprocesses and own their
@@ -88,54 +67,36 @@ const TEST_PROJECTS = [
// before their two heaps compete with the real CLI child processes.
execArgv: ["--max-old-space-size=384"],
},
{
// Process-tree observations share host sampling resources; keep this small
// lane serial while the independent projects run together.
name: "process-boundary",
include: ["tests/boundary/process/**/*.test.ts"],
"process-boundary": {
pool: "forks" as const,
maxWorkers: MAX_TEST_WORKERS,
// Process-tree observations share host sampling resources; keep this small
// lane serial while the independent projects run together.
fileParallelism: false,
sequence: { groupOrder: 1 },
},
{
name: "mcp-boundary",
include: ["tests/boundary/mcp/**/*.test.ts"],
"mcp-boundary": {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Each file creates and closes an explicit in-memory MCP session. Reuse the
// immutable server graph so module startup does not outweigh transport work.
isolate: false,
},
{
name: "acceptance",
include: ["tests/acceptance/**/*.test.ts"],
pool: "forks" as const,
maxWorkers: MAX_TEST_WORKERS,
},
{
name: "process-global",
include: ["tests/process-global/**/*.test.ts"],
pool: "forks" as const,
maxWorkers: MAX_TEST_WORKERS,
},
{
name: "conformance",
include: ["tests/conformance/**/*.test.ts"],
acceptance: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
"process-global": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
conformance: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Contract cases use recording providers and own no process-global state.
isolate: false,
},
{
name: "evaluation",
include: ["tests/evaluation/**/*.test.ts"],
evaluation: {
pool: "threads" as const,
maxWorkers: MAX_TEST_WORKERS,
// Evaluator tests inspect deterministic inputs without mutating shared state.
isolate: false,
},
];
};
const ZERO_COVERAGE_THRESHOLDS = {
statements: 0,
@@ -156,10 +117,21 @@ const ZERO_COVERAGE_THRESHOLDS = {
},
};
const projects = TEST_PROJECTS.map((project) => ({
extends: true as const,
test: project,
}));
const projects = TEST_PROJECTS.map((project) => {
const policy =
PROJECT_POLICIES[project.name as keyof typeof PROJECT_POLICIES];
if (!policy)
throw new Error(`Missing test runner policy for ${project.name}`);
return {
extends: true as const,
test: {
name: project.name,
include: [...project.include],
...(project.exclude ? { exclude: [...project.exclude] } : {}),
...policy,
},
};
});
export default defineConfig({
test: {