diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7baf1f64e9..a836c18406 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,15 @@ on: description: Exact qualification source; must match the caller revision. required: true type: string + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted release qualification passes it. + required: false permissions: contents: read @@ -29,6 +38,15 @@ concurrency: env: HUSKY: "0" + # Set Actions secret TURBO_TOKEN and variable TURBO_TEAM. OIDC can mint + # TURBO_TOKEN via vercel/setup-turborepo-remote-cache-action; do not commit a token. + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Release qualification sets remote_cache_write and passes the + # token. Open pull requests get an empty token and the local cache only. + # Remote reads need a credential, and pull requests do not receive one. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" jobs: changes: @@ -151,14 +169,10 @@ jobs: if: needs.changes.outputs.code == 'true' env: CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CI_STATIC_CONTEXT_SCRIPT: check:ci:fast:tasks run: npm run ci:static-context - if: needs.changes.outputs.code == 'true' - run: npm run check:ci - - if: needs.changes.outputs.code == 'true' - run: npm run verify:test-discovery - - name: Check LLDB producer retention and launch contracts - if: needs.changes.outputs.code == 'true' - run: npm run test:python + run: npm run check:ci:fast - if: needs.changes.outputs.code != 'true' run: npm run format:check - name: Save validated static task cache @@ -180,6 +194,72 @@ jobs: if-no-files-found: ignore retention-days: 7 + static-deep: + name: Deep static checks + needs: changes + # Full qualification only. Provider lanes wait on fast static, not this job. + if: needs.changes.outputs.tier == 'full' && needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache + - run: npm ci + - name: Restore deep static task cache + id: static-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: .turbo/cache/ + key: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.sha }} + restore-keys: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}- + - name: Restore Knip analysis cache + id: knip-cache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: node_modules/.cache/knip + # Each trusted commit publishes a new entry. Restore still matches the + # configuration prefix from the previous successful full run. + key: knip-meta-v1-${{ runner.os }}-${{ hashFiles('knip.json', 'tsconfig.json', 'package.json', 'package-lock.json') }}-${{ github.sha }} + restore-keys: knip-meta-v1-${{ runner.os }}-${{ hashFiles('knip.json', 'tsconfig.json', 'package.json', 'package-lock.json') }}- + - name: Record deep static task source and execution identity + continue-on-error: true + env: + CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CI_STATIC_CONTEXT_SCRIPT: check:ci:deep:tasks + run: npm run ci:static-context + - name: Run deep static checks + run: npm run check:ci:deep + - name: Save validated deep static task cache + if: >- + success() && steps.npm-cache.outputs.can-save == 'true' && + steps.static-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: .turbo/cache/ + key: ${{ steps.static-cache.outputs.cache-primary-key }} + - name: Save Knip analysis cache + if: >- + success() && steps.npm-cache.outputs.can-save == 'true' && + steps.knip-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: node_modules/.cache/knip + key: ${{ steps.knip-cache.outputs.cache-primary-key }} + - name: Upload complete deep static diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: static-deep-check-reports + path: .cache/static-checks/ + include-hidden-files: true + if-no-files-found: ignore + retention-days: 7 + test-source: name: Source tests needs: changes @@ -373,8 +453,9 @@ jobs: retention-days: 7 sqlite-runtime: - # Preserve all selected coverage, but admit expensive host/provider work - # only after static validation. Portable tests still provide early feedback. + # Admit expensive host/provider work after fast static validation + # (typecheck, lint, format). Deep static does not gate these lanes. + # Portable tests still provide early feedback. needs: [changes, static, build] if: needs.changes.outputs.sqlite == 'true' uses: ./.github/workflows/sqlite-runtime.yml @@ -858,68 +939,96 @@ jobs: skills/reverse-engineer-anything if-no-files-found: error + # Provider lanes wait for fast static and the portable build, not deep static. browser: needs: [changes, static, build] if: needs.changes.outputs.browser == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} uses: ./.github/workflows/real-browser.yml with: build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} binary_layout: needs: [changes, static, build] if: needs.changes.outputs.binary_layout == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} uses: ./.github/workflows/real-binary-layout.yml with: build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} evm: needs: [changes, static, build] if: needs.changes.outputs.evm == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} uses: ./.github/workflows/real-evm-interface.yml with: build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} javascript_recovery: needs: [changes, static, build] if: needs.changes.outputs.javascript_recovery == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} uses: ./.github/workflows/real-javascript-recovery.yml with: build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} recorded_crash: needs: [changes, static, build] if: needs.changes.outputs.recorded_crash == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} uses: ./.github/workflows/real-recorded-crash.yml with: build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} web_network: needs: [changes, static, build] if: needs.changes.outputs.web_network == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} uses: ./.github/workflows/real-web-network-captures.yml with: build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} web_runtime: needs: [changes, static, build] if: needs.changes.outputs.web_runtime == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} uses: ./.github/workflows/real-web-runtime.yml with: build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} web_source_map: needs: [changes, static, build] if: needs.changes.outputs.web_source_map == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} uses: ./.github/workflows/real-web-source-map.yml with: build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} agentenv: needs: [changes, static, build] if: needs.changes.outputs.agentenv == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} uses: ./.github/workflows/real-agentenv.yml with: build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} nativeaot: needs: [changes, static] @@ -944,6 +1053,7 @@ jobs: [ changes, static, + static-deep, build, reverse-tools-quality, test-shard, diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index a16cd1e01a..07cf1bae04 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -14,6 +14,11 @@ concurrency: env: HUSKY: "0" + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" jobs: mutation: diff --git a/.github/workflows/real-agentenv.yml b/.github/workflows/real-agentenv.yml index 5d9275efd7..a252eb691c 100644 --- a/.github/workflows/real-agentenv.yml +++ b/.github/workflows/real-agentenv.yml @@ -9,12 +9,28 @@ on: required: false type: string default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false permissions: contents: read env: HUSKY: "0" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" NODE_OPTIONS: --max-old-space-size=768 PIP_DISABLE_PIP_VERSION_CHECK: "1" diff --git a/.github/workflows/real-binary-layout.yml b/.github/workflows/real-binary-layout.yml index 06fcc290a6..24e9794beb 100644 --- a/.github/workflows/real-binary-layout.yml +++ b/.github/workflows/real-binary-layout.yml @@ -14,6 +14,15 @@ on: required: false type: string default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false permissions: contents: read @@ -21,6 +30,13 @@ permissions: env: NODE_OPTIONS: --max-old-space-size=768 HUSKY: "0" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" REA_VERIFY_LARGE_ELF_MCP: ${{ inputs.large_mcp && '1' || '0' }} jobs: diff --git a/.github/workflows/real-browser.yml b/.github/workflows/real-browser.yml index aa7e6ebb22..ff3b49ea65 100644 --- a/.github/workflows/real-browser.yml +++ b/.github/workflows/real-browser.yml @@ -8,10 +8,26 @@ on: required: false type: string default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false workflow_dispatch: env: HUSKY: "0" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" permissions: contents: read diff --git a/.github/workflows/real-cutter.yml b/.github/workflows/real-cutter.yml index da76469594..96dad6b917 100644 --- a/.github/workflows/real-cutter.yml +++ b/.github/workflows/real-cutter.yml @@ -8,6 +8,14 @@ on: permissions: contents: read +env: + HUSKY: "0" + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" + jobs: windows-native-artifact: name: Build Windows bridge security boundary diff --git a/.github/workflows/real-evm-interface.yml b/.github/workflows/real-evm-interface.yml index 3a47b8f907..02cb52ebbf 100644 --- a/.github/workflows/real-evm-interface.yml +++ b/.github/workflows/real-evm-interface.yml @@ -9,6 +9,15 @@ on: required: false type: string default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false permissions: contents: read @@ -16,6 +25,13 @@ permissions: env: NODE_OPTIONS: --max-old-space-size=768 HUSKY: "0" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" jobs: real-evm-interface: diff --git a/.github/workflows/real-ghidra-windows.yml b/.github/workflows/real-ghidra-windows.yml index ce14932063..e489c6d555 100644 --- a/.github/workflows/real-ghidra-windows.yml +++ b/.github/workflows/real-ghidra-windows.yml @@ -6,6 +6,11 @@ on: env: HUSKY: "0" + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" permissions: contents: read diff --git a/.github/workflows/real-go-binary.yml b/.github/workflows/real-go-binary.yml index d99622067d..8ffe3fe58d 100644 --- a/.github/workflows/real-go-binary.yml +++ b/.github/workflows/real-go-binary.yml @@ -13,20 +13,8 @@ on: - "src/composition/go.ts" - "src/cli/goCommands.ts" - "src/server/registerGoTools.ts" - - "src/server/createServer.ts" - - "src/cli.ts" - - "src/cliCommandNames.ts" - - "src/contracts/toolContracts.ts" - - "src/contracts/toolEffects.ts" - - "src/artifacts/readStableArtifact*.ts" - - "src/artifacts/ArtifactReader.ts" - - "src/process/PrivateRuntimeRoot.ts" - "scripts/verify-go-binary.mjs" - "scripts/lib/mcp-verifier-results.mjs" - - "scripts/rea.mjs" - - "package.json" - - "package-lock.json" - - ".nvmrc" permissions: contents: read @@ -38,6 +26,11 @@ concurrency: env: NODE_OPTIONS: --max-old-space-size=768 HUSKY: "0" + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" jobs: real-go-binary: diff --git a/.github/workflows/real-hopper-linux.yml b/.github/workflows/real-hopper-linux.yml index 417af2ca5a..d17efa9f41 100644 --- a/.github/workflows/real-hopper-linux.yml +++ b/.github/workflows/real-hopper-linux.yml @@ -5,6 +5,11 @@ on: env: HUSKY: "0" + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" permissions: contents: read diff --git a/.github/workflows/real-hopper.yml b/.github/workflows/real-hopper.yml index da1c60c9f9..9a2444a9fe 100644 --- a/.github/workflows/real-hopper.yml +++ b/.github/workflows/real-hopper.yml @@ -5,6 +5,11 @@ on: env: HUSKY: "0" + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" permissions: contents: read diff --git a/.github/workflows/real-javascript-recovery.yml b/.github/workflows/real-javascript-recovery.yml index 786d4a2d4a..ff684e64e5 100644 --- a/.github/workflows/real-javascript-recovery.yml +++ b/.github/workflows/real-javascript-recovery.yml @@ -8,6 +8,15 @@ on: required: false type: string default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false workflow_dispatch: permissions: @@ -15,6 +24,13 @@ permissions: env: HUSKY: "0" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" NODE_OPTIONS: "--max-old-space-size=768" RAYON_NUM_THREADS: "1" diff --git a/.github/workflows/real-packet-capture.yml b/.github/workflows/real-packet-capture.yml index ad7b62e9e0..ff4defd254 100644 --- a/.github/workflows/real-packet-capture.yml +++ b/.github/workflows/real-packet-capture.yml @@ -20,21 +20,17 @@ on: - "src/composition/captures.ts" - "src/cli/captureCommands.ts" - "src/server/registerCaptureTools.ts" - - "src/server/createServer.ts" - - "src/server/sessionAvailabilityPolicy.ts" - - "src/application/CapabilityInventory.ts" - - "src/contracts/toolContracts.ts" - - "src/contracts/toolEffects.ts" - - "src/cli.ts" - - "src/cliCommandNames.ts" - - "src/process/**" - - "src/artifacts/readStableArtifact.ts" - "scripts/verify-packet-capture.mjs" - ".github/workflows/real-packet-capture.yml" - - "package*.json" - - ".nvmrc" permissions: contents: read +env: + HUSKY: "0" + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" concurrency: group: real-packet-capture-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true diff --git a/.github/workflows/real-recorded-crash.yml b/.github/workflows/real-recorded-crash.yml index e5f16c30e7..d9c41bafb5 100644 --- a/.github/workflows/real-recorded-crash.yml +++ b/.github/workflows/real-recorded-crash.yml @@ -9,6 +9,15 @@ on: required: false type: string default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false permissions: contents: read @@ -16,6 +25,13 @@ permissions: env: NODE_OPTIONS: --max-old-space-size=768 HUSKY: "0" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" UV_CONCURRENT_BUILDS: "1" UV_CONCURRENT_INSTALLS: "1" UV_PYTHON_DOWNLOADS: never diff --git a/.github/workflows/real-signal-recording.yml b/.github/workflows/real-signal-recording.yml index 1ddc2d5e0b..aeb4217b84 100644 --- a/.github/workflows/real-signal-recording.yml +++ b/.github/workflows/real-signal-recording.yml @@ -13,21 +13,17 @@ on: - "src/composition/signals.ts" - "src/cli/signalCommands.ts" - "src/server/registerSignalTools.ts" - - "src/server/createServer.ts" - - "src/server/sessionAvailabilityPolicy.ts" - - "src/application/CapabilityInventory.ts" - - "src/contracts/toolContracts.ts" - - "src/contracts/toolEffects.ts" - - "src/cli.ts" - - "src/cliCommandNames.ts" - - "src/process/**" - - "src/artifacts/readStableArtifact.ts" - "scripts/verify-signal-recording.mjs" - ".github/workflows/real-signal-recording.yml" - - "package*.json" - - ".nvmrc" permissions: contents: read +env: + HUSKY: "0" + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" concurrency: group: real-signal-recording-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true diff --git a/.github/workflows/real-termux.yml b/.github/workflows/real-termux.yml index e6f3890e13..a8d497b0e8 100644 --- a/.github/workflows/real-termux.yml +++ b/.github/workflows/real-termux.yml @@ -12,6 +12,11 @@ permissions: env: HUSKY: "0" + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" jobs: real-termux: diff --git a/.github/workflows/real-web-network-captures.yml b/.github/workflows/real-web-network-captures.yml index 7901114f0b..ebd4f0a340 100644 --- a/.github/workflows/real-web-network-captures.yml +++ b/.github/workflows/real-web-network-captures.yml @@ -9,12 +9,28 @@ on: required: false type: string default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false permissions: contents: read env: HUSKY: "0" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" NODE_OPTIONS: --max-old-space-size=768 jobs: diff --git a/.github/workflows/real-web-runtime.yml b/.github/workflows/real-web-runtime.yml index 0ceb9b0573..b5da11ae15 100644 --- a/.github/workflows/real-web-runtime.yml +++ b/.github/workflows/real-web-runtime.yml @@ -7,11 +7,27 @@ on: required: false type: string default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false workflow_dispatch: permissions: contents: read env: HUSKY: "0" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" jobs: real-web-runtime: runs-on: ubuntu-latest diff --git a/.github/workflows/real-web-source-map.yml b/.github/workflows/real-web-source-map.yml index eb1ecde342..8de1bc66d0 100644 --- a/.github/workflows/real-web-source-map.yml +++ b/.github/workflows/real-web-source-map.yml @@ -7,11 +7,27 @@ on: required: false type: string default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false workflow_dispatch: permissions: contents: read env: HUSKY: "0" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" jobs: real-web-source-map: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a1ae284e64..b2b4828ab5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,12 @@ on: env: HUSKY: "0" + # Remote cache stays local when either value is empty. + # Writes only on the default branch for push, schedule, manual dispatch, + # and workflow_call. Pull requests get an empty token and remote read-only mode. + TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" concurrency: group: release-${{ inputs.release_branch || github.event.pull_request.merge_commit_sha || github.ref_name }} @@ -301,8 +307,11 @@ jobs: contents: read actions: read uses: ./.github/workflows/ci.yml + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} with: source_sha: ${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }} + remote_cache_write: true rehearsal: name: Rehearse qualified release on a clean runner diff --git a/docs/testing.md b/docs/testing.md index a38ec04ce4..1ab5184aad 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -69,8 +69,14 @@ Source projects also contain large capacity regressions; `test:fast` promises no compiled-runtime prerequisite, not a fixed time budget. `npm run lint` prints errors and a warning summary; `npm run jscpd` prints -duplication statistics. Complete reports remain in `.cache/static-checks/` and -CI uploads them as `static-check-reports`, including after a failed static gate. +duplication statistics. Complete reports remain in `.cache/static-checks/`. +The fast gate uploads them as `static-check-reports`. Full qualification +uploads duplication diagnostics from the deep job as +`static-deep-check-reports`, including after that gate fails. +Task-graph recording runs `check:ci:fast:tasks` and `check:ci:deep:tasks` +without the artifacts lock, so it can record hashes while `check:pr` or +`check:test` already holds that lock. The CI jobs still run the locked +`check:ci:fast` and `check:ci:deep` commands. Use `npm run lint:details` or `npm run jscpd:details` for all console diagnostics. Both duplicate commands scan production and test code with the same 3% limit; the summary commands preserve checker failures and restore reports from cached @@ -481,14 +487,16 @@ real-system lanes remain separate jobs so one kind of evidence cannot stand in for another. Workflow syntax and expression validation runs inside the required static-check job when workflow changes select it, avoiding a separate runner. Native, provider, fixture, SQLite compatibility, and website lanes wait for -static checks to succeed before allocating their runners. This avoids spending -those runner slots on revisions that already fail static validation. Portable -test shards and documentation checks remain independent of static results, so -they can still provide diagnostic feedback. The required gate rejects skipped -selected lanes when static checks fail; passing revisions retain all selected -coverage. This ordering reduces peak demand on invalid revisions but can add -static-check waiting to otherwise passing runs; it is not a measured reduction -in end-to-end latency. +the fast static checks (typecheck, built lint, and format) to succeed before +allocating their runners. Knip, duplication, architecture guards, test +discovery, and the Python producer run in the separate full-tier deep static +job and do not gate provider admission. Portable test shards and documentation +checks remain independent of static results, so they can still provide +diagnostic feedback. The required gate rejects skipped selected lanes when +static checks fail; on the full tier it also requires the deep static job. +Passing revisions retain all selected coverage. This ordering reduces peak +demand on invalid revisions but can add static-check waiting to otherwise +passing runs; it is not a measured reduction in end-to-end latency. Core jobs have execution limits of five minutes for classification, ten minutes for static checks, builds, docs, and report merging, and thirty minutes per @@ -1167,13 +1175,16 @@ scripts also select it. Unrelated code edits retain the complete portable suite on the pinned Node version without starting four additional compatibility jobs. Full qualification retains every SQLite version. Shared process/contract/CLI boundaries, dependencies, other packaging inputs, -unknown paths, and changes to the planner select the full baseline. +unknown paths, and changes to the planner select the fast Linux x64 baseline: +portable code checks and the installed-package runtime lane, without provider +lanes or the multi-host package matrix. A shared file with exactly one owner +still selects that owner. Full qualification still selects every lane. CI timing collection, timing history, and test-report helper edits retain static checks and the complete portable suite without selecting unrelated host/provider lanes. Classification, qualification, gate, release receipt, and unknown CI -scripts keep the broad fallback. +scripts keep that fast baseline. Package script-only edits select their owning verifier when its entrypoint is -identifiable; unknown or shared script changes select everything. Android +identifiable; unknown or shared script changes select the fast baseline. Android build-script edits retain the Linux suite and workflow validation; real Android verification stays manual. Renames account for both paths. @@ -1197,7 +1208,8 @@ or provider effects outside its owners. Ordinary main pushes use the same owner scopes as PRs. Their diff starts at a successful ancestor push run of this CI workflow, rather than the preceding push: changes from failed or cancelled intermediate runs remain in scope. -Missing, untrusted, or non-ancestor history selects every lane. The lookup is +Missing, untrusted, or non-ancestor history selects the fast Linux x64 +baseline: portable checks and the Linux runtime lane. The lookup is bounded to the last 100 successful push runs; lookup failures broaden selection. Every code change still selects all six portable shards, without coverage instrumentation in this fast tier. Ordinary PR matrices cancel remaining @@ -1227,8 +1239,11 @@ immediately starts the separate `Full PR qualification` workflow against the label event's merge revision; subsequent PR revisions also select the full tier in normal CI while that label remains. Unrelated labels do not start validation jobs or replace the main CI checks. Manual dispatch also requests full validation. -Fast-tier broad fallbacks retain all owner lanes and the three-host -package matrix; Intel macOS and coverage remain full-tier obligations. Each +Fast-tier broad fallbacks keep the portable suite and the Linux x64 runtime +baseline. They do not start every owner lane or the three-host package +matrix. Intel macOS and coverage remain full-tier obligations. Real provider +lanes stay path-gated, and a shared registration file listed by many lanes +does not start those lanes on a fast PR. Each classification job records its selection and broad-fallback reasons in the run summary. Representative Git-diff cases live in `tests/boundary/filesystem/ciChangeScope.test.ts`. @@ -1308,9 +1323,14 @@ report in each shard's snapshot. The bounded lookup verifies the artifact's producer; newer schedules that reuse qualification without producing timings do not hide the last full run's artifact. CI assigns discovered files by historical duration, with a deterministic -fallback weight for new files, rather than dividing only by file count. Every +fallback weight for new files, rather than dividing only by file count. +`process-boundary` files are weighted at their full duration because that +project runs files serially. Other projects are weighted at half their file +duration, matching the two-worker cap, so a serial file does not look as cheap +as a file that overlaps with another. Every discovered file still belongs to exactly one shard; worker limits and project -ordering remain unchanged. The merge job rejects successful reports with missing, +ordering remain unchanged. Fast qualification keeps five built shards without +coverage. The merge job rejects successful reports with missing, unexpected, or duplicate portable files. Coverage instrumentation and aggregate thresholds remain enforced in full qualification. Missing or unusable history falls back to Vitest's default assignment. Only @@ -1318,8 +1338,24 @@ timing data is reused; live provider results are never cached. Turbo outputs are restored by task hash, with trusted default-branch runs as cache writers and PRs as restore-only consumers. The static job also restores its Turbo snapshot; each task revalidates its own input hash before reusing outputs or diagnostics. -The Node pin and npm install settings are global hash inputs. Knip includes its ignore rules and VitePress +The Node pin and npm install settings are global hash inputs. Knip runs only +in full qualification, with `--cache` and a GitHub cache of +`node_modules/.cache/knip`. The saved key includes the commit SHA. Restore uses +the configuration prefix, so the next trusted full run can publish a refreshed +snapshot. That cache is an AST cache keyed by file mtime and size, so a +fresh checkout often misses it; Turbo still hashes the Knip task inputs. Knip +includes its ignore rules and VitePress entry, so changes to those inputs cannot reuse an older successful check. +Jobs that run Turbo export `TURBO_TEAM`. A called workflow sees the caller +event, so `workflow_call` is not an event name inside CI. `TURBO_TOKEN` is set +when release qualification passes `remote_cache_write`, or when the ref is the +default branch and the event is push, schedule, or workflow_dispatch, and only +when that secret is available to the run. Release qualification passes +`TURBO_TOKEN` and `remote_cache_write` into the called workflow. Labeled +pull-request qualification does not. Pull requests receive an empty token and +`TURBO_CACHE=local:rw`. Remote reads require `TURBO_TOKEN`, and this repository +does not pass a separate read-only credential into pull requests. An empty +token leaves the local Turbo cache in place. The shared npm-cache action caches downloads, not `node_modules`, and every consumer still runs `npm ci`. Keys cover the runner OS, actual Node architecture, diff --git a/package.json b/package.json index e154ff398a..1a752edc22 100644 --- a/package.json +++ b/package.json @@ -119,7 +119,13 @@ "precheck:pr": "npm run deps:check", "check:pr": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos", "precheck:ci": "npm run deps:check", - "check:ci": "node scripts/run-exclusive.mjs artifacts npm run check:ci:tasks && npm run verify:architecture-guards", + "precheck:ci:fast": "npm run deps:check", + "precheck:ci:deep": "npm run deps:check", + "check:ci": "npm run check:ci:fast && npm run check:ci:deep", + "check:ci:fast": "node scripts/run-exclusive.mjs artifacts npm run check:ci:fast:tasks", + "check:ci:fast:tasks": "turbo run typecheck lint:built format:check", + "check:ci:deep": "node scripts/run-exclusive.mjs artifacts npm run check:ci:deep:tasks && npm run verify:architecture-guards && npm run verify:test-discovery && npm run test:python", + "check:ci:deep:tasks": "turbo run knip metadata:check jscpd scan:todos", "check:ci:tasks": "turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos", "ci:static-context": "node scripts/ci/static-task-context.mjs", "build:cached": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills", @@ -131,7 +137,7 @@ "lint:details": "oxlint && npm run verify:module-boundaries", "lint:built": "npm run lint", "lint:fix": "oxlint --fix && npm run verify:module-boundaries", - "knip": "knip", + "knip": "knip --cache", "jscpd": "node scripts/ci/static-check-report.mjs duplicates", "jscpd:details": "jscpd src/ tests/ --no-tips", "scan:todos": "node scripts/scan-todos.mjs", diff --git a/scripts/ci/artifacts.d.mts b/scripts/ci/artifacts.d.mts index 93cf2f7013..648ed60716 100644 --- a/scripts/ci/artifacts.d.mts +++ b/scripts/ci/artifacts.d.mts @@ -49,6 +49,7 @@ export function selectStaticCheckArtifact( page: unknown, run: { id?: string | number; head_sha?: string }, runAttempt: number, + name?: string, ): unknown | null; export function readStaticCheckArtifact(options: { diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjs index 250a15872c..dbd77c5d60 100644 --- a/scripts/ci/artifacts.mjs +++ b/scripts/ci/artifacts.mjs @@ -7,6 +7,11 @@ import { promisify } from "node:util"; import { parseStaticTaskContext } from "./static-task-context.mjs"; const execute = promisify(execFile); +/** Job display name owns the artifact that contains that gate's diagnostics. */ +export const STATIC_EVIDENCE_ARTIFACTS = { + "Static checks": "static-check-reports", + "Deep static checks": "static-deep-check-reports", +}; const MAX_ARCHIVE_BYTES = 32 * 1024 * 1024; const MAX_MEMBER_BYTES = 4 * 1024 * 1024; const MAX_DIAGNOSTICS = 10_000; @@ -43,7 +48,8 @@ export function classifyFailure({ "run_event_or_repository_identity_unverified", ], () => [ - job?.name !== mainJob?.name || job?.name !== "Static checks", + job?.name !== mainJob?.name || + !Object.hasOwn(STATIC_EVIDENCE_ARTIFACTS, job?.name ?? ""), "failed_job_not_the_same_static_gate", ], () => [ @@ -134,7 +140,12 @@ export function safeSha(value) { ? value.toLowerCase() : null; } -export function selectStaticCheckArtifact(page, run, runAttempt) { +export function selectStaticCheckArtifact( + page, + run, + runAttempt, + name = STATIC_EVIDENCE_ARTIFACTS["Static checks"], +) { if (runAttempt !== 1) return null; if ( !Array.isArray(page?.artifacts) || @@ -143,9 +154,7 @@ export function selectStaticCheckArtifact(page, run, runAttempt) { page.artifacts.length >= 100 ) return null; - const matches = page.artifacts.filter( - (artifact) => artifact.name === "static-check-reports", - ); + const matches = page.artifacts.filter((artifact) => artifact.name === name); if (matches.length !== 1) return null; const [artifact] = matches; if ( diff --git a/scripts/ci/failure-report.mjs b/scripts/ci/failure-report.mjs index 12364c59f1..6009db5a44 100644 --- a/scripts/ci/failure-report.mjs +++ b/scripts/ci/failure-report.mjs @@ -11,6 +11,7 @@ import { readStaticCheckArtifact, safeSha, selectStaticCheckArtifact, + STATIC_EVIDENCE_ARTIFACTS, validRepoPath, } from "./artifacts.mjs"; import { createGhApiRequest } from "./github-api.mjs"; @@ -366,12 +367,18 @@ export function createGitHubApi({ return page.jobs; }, async getStaticEvidence(repository, run, job) { - if (job.name !== "Static checks") return null; + const artifactName = STATIC_EVIDENCE_ARTIFACTS[job.name]; + if (!artifactName) return null; if (run.run_attempt !== 1) return null; const page = await request( `repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`, ); - const artifact = selectStaticCheckArtifact(page, run, run.run_attempt); + const artifact = selectStaticCheckArtifact( + page, + run, + run.run_attempt, + artifactName, + ); if (!artifact) return null; return await readStaticCheckArtifact({ repository, diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index a3155feb6d..e7c3de8ddb 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -42,8 +42,18 @@ const matchesAny = (file, patterns) => patterns.some((pattern) => matchesGlob(file, pattern)); const selected = Object.fromEntries(flags.map((flag) => [flag, false])); const reasons = []; +// Shared process, contract, CLI, and CI-script edits are not a specific provider. +// Fast qualification keeps the portable suite and the Linux x64 runtime +// baseline. Full qualification still selects every lane. +const SHARED_BOUNDARY = + /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/; const full = (reason) => { - for (const flag of flags) selected[flag] = true; + if (tier === "full") { + for (const flag of flags) selected[flag] = true; + } else { + selected.code = true; + selected.runtime = true; + } reasons.push(reason); }; const git = (...args) => @@ -196,6 +206,28 @@ if (tier === "reused") { selected.code = true; continue; } + // Timing and report helpers stay on the portable suite. They are not a + // runtime or provider change. + if (PORTABLE_CI_REPORTERS.has(file)) { + selected.code = true; + continue; + } + // A shared boundary is not evidence that every provider or host changed. + // One matching owner is path evidence. Several owners means the same + // registration file is listed by many lanes; fast qualification does not + // fan those out or start the multi-host package matrix. + if (tier !== "full" && SHARED_BOUNDARY.test(file)) { + selected.code = true; + selected.runtime = true; + const owners = Object.entries(SCOPES) + .filter(([, patterns]) => + patterns.some((pattern) => matchesGlob(file, pattern)), + ) + .map(([owner]) => owner); + if (owners.length === 1) selected[owners[0]] = true; + reasons.push(`Shared boundary on fast tier: ${file}`); + continue; + } const owners = Object.entries(SCOPES) .filter(([, patterns]) => patterns.some((pattern) => matchesGlob(file, pattern)), @@ -226,7 +258,6 @@ if (tier === "reused") { } if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true; if (file.startsWith("tests/")) continue; - if (PORTABLE_CI_REPORTERS.has(file)) continue; if ( /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test( file, diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index 1839a11498..a4da755d77 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -29,6 +29,7 @@ if ( if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); const required = reused ? ["changes"] : ["changes", "static", "test"]; +if (outputs.tier === "full") required.push("static-deep"); const lanes = { code: [ "build", diff --git a/scripts/ci/static-task-context.d.mts b/scripts/ci/static-task-context.d.mts index 164756b15d..0410b1773d 100644 --- a/scripts/ci/static-task-context.d.mts +++ b/scripts/ci/static-task-context.d.mts @@ -15,4 +15,5 @@ export function recordStaticTaskContext(options?: { root?: string; npmCli?: string; outputDirectory?: string; + script?: string; }): Promise; diff --git a/scripts/ci/static-task-context.mjs b/scripts/ci/static-task-context.mjs index 879c568d19..718ae1752b 100644 --- a/scripts/ci/static-task-context.mjs +++ b/scripts/ci/static-task-context.mjs @@ -6,6 +6,10 @@ import { promisify } from "node:util"; import { z } from "zod"; const execute = promisify(execFile); +const STATIC_CONTEXT_SCRIPTS = new Set([ + "check:ci:fast:tasks", + "check:ci:deep:tasks", +]); const contextSchema = z.object({ version: z.literal(1), sourceSha: z.string().regex(/^[a-f0-9]{40}$/u), @@ -30,7 +34,10 @@ export async function recordStaticTaskContext({ root = process.cwd(), npmCli = process.env.npm_execpath, outputDirectory = join(root, ".cache/static-checks"), + script = process.env.CI_STATIC_CONTEXT_SCRIPT ?? "check:ci:fast:tasks", } = {}) { + if (!STATIC_CONTEXT_SCRIPTS.has(script)) + throw new Error("Static context script is not a known check"); if (typeof npmCli !== "string" || !npmCli) throw new Error( "Run static context recording through npm run ci:static-context", @@ -44,7 +51,7 @@ export async function recordStaticTaskContext({ const [graph, source, status, npmVersion, turbo] = await Promise.all([ execute( process.execPath, - [npmCli, "run", "--silent", "check:ci:tasks", "--", "--dry=json"], + [npmCli, "run", "--silent", script, "--", "--dry=json"], options, ), execute("git", ["rev-parse", "HEAD"], options), diff --git a/src/ida/IdaOperationFailures.test.ts b/src/ida/IdaOperationFailures.test.ts index 8886f68611..f32b295e14 100644 --- a/src/ida/IdaOperationFailures.test.ts +++ b/src/ida/IdaOperationFailures.test.ts @@ -479,9 +479,10 @@ describe("headless IDA procedure lookup failures", () => { expect(projected.category, "the category is invalid_input").toBe( "invalid_input", ); - expect(projected.retryable, "the caller can correct and retry").toBe( - true, - ); + expect( + projected.retryable, + "an unchanged invalid request is not retryable", + ).toBe(false); expect( projected.details, "the operation and exactly one issue that names the procedure in its message", diff --git a/tests/boundary/filesystem/ciCachePolicy.test.ts b/tests/boundary/filesystem/ciCachePolicy.test.ts index 17bd6c9747..6a8867b669 100644 --- a/tests/boundary/filesystem/ciCachePolicy.test.ts +++ b/tests/boundary/filesystem/ciCachePolicy.test.ts @@ -219,8 +219,45 @@ it("shares static task snapshots only after validation using the npm cache owner key: "${{ steps.static-cache.outputs.cache-primary-key }}", }); expect(steps.indexOf(save!)).toBeGreaterThan( - steps.findIndex((step) => step.run === "npm run test:python"), + steps.findIndex((step) => step.run === "npm run check:ci:fast"), ); + const deep = ci?.workflow.jobs?.["static-deep"]?.steps ?? []; + const deepSave = deep.find( + (step) => step.name === "Save validated deep static task cache", + ); + const knipRestore = deep.find((step) => step.id === "knip-cache"); + const knipSave = deep.find( + (step) => step.name === "Save Knip analysis cache", + ); + expect(deepSave?.uses).toBe( + "actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830", + ); + expect(deepSave?.if).toContain("success()"); + expect(deepSave?.if).toContain("steps.npm-cache.outputs.can-save == 'true'"); + expect(deepSave?.if).toContain( + "steps.static-cache.outputs.cache-hit != 'true'", + ); + expect(deep.indexOf(deepSave!)).toBeGreaterThan( + deep.findIndex((step) => step.run === "npm run check:ci:deep"), + ); + const knipWith = z + .object({ + path: z.string(), + key: z.string(), + "restore-keys": z.string(), + }) + .parse(knipRestore?.with); + expect(knipWith.path).toBe("node_modules/.cache/knip"); + expect(knipWith.key).toContain("${{ github.sha }}"); + expect(knipWith["restore-keys"]).not.toContain("github.sha"); + expect(knipWith["restore-keys"].startsWith("knip-meta-v1-")).toBe(true); + expect(knipSave?.if).toContain( + "steps.knip-cache.outputs.cache-hit != 'true'", + ); + expect(knipSave?.with).toEqual({ + path: "node_modules/.cache/knip", + key: "${{ steps.knip-cache.outputs.cache-primary-key }}", + }); }); it("passes the npm cache owner's selected path to package installation", async () => { @@ -241,6 +278,73 @@ it("passes the npm cache owner's selected path to package installation", async ( ).toBe("${{ steps.npm-cache.outputs.path }}"); }); +it("exports turbo remote-cache credentials wherever a workflow runs turbo", async () => { + const directory = join(repository, ".github/workflows"); + const files = (await readdir(directory)).filter((file) => + /\.ya?ml$/u.test(file), + ); + const turboCommand = + /\bturbo\b|build:cached|check:ci|docs:check|docs:generate|docs:prepare|test:prepare/u; + const workflowSchema = z.object({ + env: z.record(z.string(), z.unknown()).optional(), + jobs: z.record( + z.string(), + z + .object({ + env: z.record(z.string(), z.unknown()).optional(), + steps: z.array(z.object({ run: z.string().optional() })).optional(), + }) + .passthrough(), + ), + }); + for (const file of files) { + const workflow = workflowSchema.parse( + parse(await readFile(join(directory, file), "utf8")), + ); + for (const [name, job] of Object.entries(workflow.jobs)) { + const runsTurbo = job.steps?.some( + (step) => step.run !== undefined && turboCommand.test(step.run), + ); + if (!runsTurbo) continue; + const env = { ...workflow.env, ...job.env }; + const token = z.string().parse(env.TURBO_TOKEN); + const cache = z.string().parse(env.TURBO_CACHE); + expect(token, `${file}:${name}`).toContain("secrets.TURBO_TOKEN"); + expect(token, `${file}:${name}`).toContain("|| ''"); + expect(token, `${file}:${name}`).not.toBe("${{ secrets.TURBO_TOKEN }}"); + const grantsReleaseWrite = token.includes( + "inputs.remote_cache_write == true", + ); + const grantsCalledEvent = token.includes( + "github.event_name == 'workflow_call'", + ); + expect(grantsReleaseWrite || grantsCalledEvent, `${file}:${name}`).toBe( + true, + ); + if (grantsReleaseWrite) + expect(token, `${file}:${name}`).not.toContain( + "github.event_name == 'workflow_call'", + ); + for (const event of ["push", "schedule", "workflow_dispatch"]) + expect(token, `${file}:${name}`).toContain( + `github.event_name == '${event}'`, + ); + expect(token, `${file}:${name}`).not.toContain( + "github.event_name == 'pull_request'", + ); + if (grantsReleaseWrite) { + expect(cache, `${file}:${name}`).toContain( + "inputs.remote_cache_write == true", + ); + expect(cache, `${file}:${name}`).toContain("|| 'local:rw'"); + expect(cache, `${file}:${name}`).not.toContain("local:rw,remote:r'"); + } else expect(cache, `${file}:${name}`).toContain("local:rw,remote:r'"); + expect(cache, `${file}:${name}`).toContain("local:rw,remote:rw"); + expect(env.TURBO_TEAM, `${file}:${name}`).toBe("${{ vars.TURBO_TEAM }}"); + } + } +}); + it("bounds the rolling static snapshot with Turbo age and size eviction", async () => { const turbo = z .object({ diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 395a597c7d..d68af5ab0c 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -222,12 +222,19 @@ it("preserves a provider lane when a portable CI reporter changes with provider }); }); -it("keeps unknown CI scripts on complete qualification", async () => { +it("keeps unknown CI scripts on the fast Linux baseline", async () => { const { directory, base } = await fixture(); const head = await change(directory, ["scripts/ci/new-qualification.mjs"]); const plan = await classify(directory, base, head); const { CI_FLAGS } = await loadScopes(); - for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true); + expect(plan).toMatchObject({ tier: "fast", code: true, runtime: true }); + for (const flag of CI_FLAGS) + if (flag !== "code" && flag !== "runtime") expect(plan[flag]).toBe(false); + expect(plan.matrix).toEqual({ + include: [{ os: "ubuntu-latest", platform: "linux", architecture: "x64" }], + }); + const full = await classify(directory, base, head, "pull_request", true); + for (const flag of CI_FLAGS) expect(full[flag]).toBe(true); }); it("emits an observational owner proposal without changing ordinary or full portable execution", async () => { @@ -399,7 +406,7 @@ it("includes changes from earlier cancelled pushes since the last successful bas }); it.each(["", "not-a-commit"])( - "fails closed to all lanes for a missing or invalid push baseline (%s)", + "fails closed to the fast Linux baseline for a missing or invalid push baseline (%s)", async (base) => { const { directory } = await fixture(); const head = await change(directory, ["README.md"]); @@ -407,13 +414,20 @@ it.each(["", "not-a-commit"])( expect(plan).toMatchObject({ tier: "fast", code: true, - package: true, - evm: true, - browser: true, + runtime: true, + package: false, + evm: false, + browser: false, reasons: expect.arrayContaining([expect.stringContaining("baseline")]), }); const { CI_FLAGS } = await loadScopes(); - for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true); + for (const flag of CI_FLAGS) + if (flag !== "code" && flag !== "runtime") expect(plan[flag]).toBe(false); + expect(plan.matrix).toEqual({ + include: [ + { os: "ubuntu-latest", platform: "linux", architecture: "x64" }, + ], + }); }, ); @@ -518,9 +532,10 @@ it("fails closed when the successful push baseline is not an ancestor", async () expect(await classify(directory, unrelated, head, "push")).toMatchObject({ tier: "fast", code: true, - package: true, - evm: true, - browser: true, + runtime: true, + package: false, + evm: false, + browser: false, }); expect(head).not.toBe(base); }); @@ -631,17 +646,39 @@ it.each([ ], [ "src/process/ProviderProcess.ts", - { code: true, package: true, evm: true, browser: true }, + { + code: true, + runtime: true, + package: false, + evm: false, + browser: false, + sqlite: false, + }, ], [ "package-lock.json", - { code: true, package: true, evm: true, browser: true }, + { + code: true, + runtime: true, + package: false, + evm: false, + browser: false, + }, ], [ ".github/workflows/real-evm-interface.yml", { code: false, evm: true, browser: false, package: false }, ], - ["unknown-input", { code: true, package: true, evm: true, browser: true }], + [ + "unknown-input", + { + code: true, + runtime: true, + package: false, + evm: false, + browser: false, + }, + ], [ "src/application/Example.ts", { @@ -830,11 +867,35 @@ it.each([ "tests/boundary/mcp/serverLifecycleMcp.test.ts", { code: true, sqlite: true }, ], - ["src/process/OwnedCommand.ts", { code: true, sqlite: true, runtime: true }], - ["src/server/createServer.ts", { code: true, sqlite: true, runtime: true }], + [ + "src/process/OwnedCommand.ts", + { + code: true, + sqlite: false, + runtime: true, + package: false, + browser: false, + }, + ], + [ + "src/server/createServer.ts", + { + code: true, + sqlite: false, + runtime: true, + package: false, + browser: false, + }, + ], [ "src/contracts/toolContracts.ts", - { code: true, sqlite: true, runtime: true }, + { + code: true, + sqlite: false, + runtime: true, + package: false, + browser: false, + }, ], [ "src/domain/analysisErrorProjection.ts", @@ -927,8 +988,10 @@ it("scopes package script edits while retaining broad dependency checks", async ); expect(await classify(directory, head, dependency)).toMatchObject({ code: true, - browser: true, - package: true, + runtime: true, + browser: false, + package: false, + evm: false, }); }); @@ -1089,11 +1152,24 @@ it.each([ ], [ ".nvmrc", - { code: true, package: true, browser: true, evm: true, sqlite: true }, + { + code: true, + runtime: true, + package: false, + browser: false, + evm: false, + sqlite: false, + }, ], [ "tsconfig.build.json", - { code: true, package: true, browser: true, evm: true }, + { + code: true, + runtime: true, + package: false, + browser: false, + evm: false, + }, ], ["docs/.vitepress/config.ts", { code: true, docs: true, package: false }], [ @@ -1117,7 +1193,10 @@ it.each([ "src/contracts/evm/tools.ts", { code: true, docs: true, evm: true, package: false }, ], - ["scripts/unknown-website-helper.mjs", { code: true, package: true }], + [ + "scripts/unknown-website-helper.mjs", + { code: true, runtime: true, package: false, browser: false }, + ], [ ".github/workflows/release.yml", { workflows: true, code: true, package: false, website: false }, @@ -1229,9 +1308,11 @@ it("retains a broad fallback when a provider script adds shared shell work", asy }; const head = await change(directory, ["package.json"], JSON.stringify(after)); expect(await classify(directory, base, head)).toMatchObject({ - evm: true, - browser: true, - package: true, + code: true, + runtime: true, + evm: false, + browser: false, + package: false, }); }); diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.ts index c2a282906c..f77ec00ec6 100644 --- a/tests/boundary/filesystem/ciFailureReport.test.ts +++ b/tests/boundary/filesystem/ciFailureReport.test.ts @@ -87,6 +87,30 @@ const comparison = (): Parameters[0] => ({ blobs: { "src/example.ts": true }, }); +it("attributes a deep duplication failure when both gates record the same jscpd task", () => { + const duplicate = { kind: "jscpd" as const, paths: ["src/a.ts", "src/b.ts"] }; + const deepContext = (sourceSha: string) => ({ + ...context(sourceSha), + taskHashes: { jscpd: "c".repeat(16) }, + }); + expect( + classifyFailure({ + ...comparison(), + job: job({ name: "Deep static checks" }), + mainJob: job({ name: "Deep static checks" }), + diagnostics: [duplicate], + mainDiagnostics: [duplicate], + context: deepContext(sha("a")), + mainContext: deepContext(sha("c")), + blobs: { "src/a.ts": true, "src/b.ts": true }, + }), + ).toEqual({ + classification: "matching-main-diagnostics", + reason: + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + }); +}); + it("attributes only an identical structured lint failure with matching task, context, ancestry, and source blob", () => { const result = classifyFailure(comparison()); expect(result).toEqual({ @@ -115,6 +139,10 @@ it.each([ ["ancestry", { ancestry: { status: "diverged", mergeBaseSha: sha("e") } }], ["canceled job", { job: job({ conclusion: "cancelled" }) }], ["not a static job", { job: job({ name: "Test shard (1)" }) }], + [ + "deep gate compared with the fast gate", + { job: job({ name: "Deep static checks" }) }, + ], ] as const)("does not attribute on mismatched %s", (_label, overrides) => { const result = classifyFailure({ ...comparison(), ...overrides }); expect(result.classification).toBe("unknown"); @@ -411,6 +439,76 @@ it("uses static artifacts only when listing is complete, unique, and tied to thi 2, ), ).toBeNull(); + const deep = { ...artifact, id: 13, name: "static-deep-check-reports" }; + expect( + selectStaticCheckArtifact( + { total_count: 2, artifacts: [artifact, deep] }, + runRecord, + 1, + "static-deep-check-reports", + ), + ).toBe(deep); + expect( + selectStaticCheckArtifact( + { total_count: 2, artifacts: [artifact, deep] }, + runRecord, + 1, + ), + ).toBe(artifact); + expect( + selectStaticCheckArtifact( + { total_count: 2, artifacts: [deep, deep] }, + runRecord, + 1, + "static-deep-check-reports", + ), + ).toBeNull(); +}); + +it("loads each static gate from its own artifact", async () => { + const downloads: string[] = []; + const api = createGitHubApi({ + token: "fixture-token", + fetchImpl: async (url) => { + downloads.push(String(url)); + throw new Error("stop-before-download"); + }, + apiRequest: async () => ({ + total_count: 2, + artifacts: [ + { + id: 1, + name: "static-check-reports", + expired: false, + workflow_run: { id: 10, head_sha: sha("a") }, + }, + { + id: 2, + name: "static-deep-check-reports", + expired: false, + workflow_run: { id: 10, head_sha: sha("a") }, + }, + ], + }), + }); + const runRecord = { id: 10, run_attempt: 1, head_sha: sha("a") }; + await expect( + api.getStaticEvidence("owner/rea", runRecord, { + name: "Deep static checks", + }), + ).rejects.toThrow("stop-before-download"); + await expect( + api.getStaticEvidence("owner/rea", runRecord, { name: "Static checks" }), + ).rejects.toThrow("stop-before-download"); + expect( + await api.getStaticEvidence("owner/rea", runRecord, { + name: "Test shard (1)", + }), + ).toBeNull(); + expect(downloads).toEqual([ + "https://api.github.com/repos/owner/rea/actions/artifacts/2/zip", + "https://api.github.com/repos/owner/rea/actions/artifacts/1/zip", + ]); }); it("rejects artifact redirects that could send the bearer token to a non-HTTPS host", async () => { diff --git a/tests/boundary/filesystem/ciFeedback.test.ts b/tests/boundary/filesystem/ciFeedback.test.ts index e813b0d7c9..c82dbe3360 100644 --- a/tests/boundary/filesystem/ciFeedback.test.ts +++ b/tests/boundary/filesystem/ciFeedback.test.ts @@ -201,8 +201,41 @@ it("starts source feedback before build and requires both groups without waiving expect( jobs.static?.steps?.find( (step) => step.run === "npm run ci:static-context", - )?.["continue-on-error"], - ).toBe(true); + ), + ).toMatchObject({ + "continue-on-error": true, + env: { CI_STATIC_CONTEXT_SCRIPT: "check:ci:fast:tasks" }, + }); + expect( + jobs["static-deep"]?.steps?.find( + (step) => step.run === "npm run ci:static-context", + ), + ).toMatchObject({ + "continue-on-error": true, + env: { CI_STATIC_CONTEXT_SCRIPT: "check:ci:deep:tasks" }, + }); + const deepUpload = jobs["static-deep"]?.steps?.find( + (step) => step.with?.name === "static-deep-check-reports", + ); + expect(deepUpload?.if).toBe("always()"); + expect(deepUpload?.with).toMatchObject({ + path: ".cache/static-checks/", + "if-no-files-found": "ignore", + "retention-days": 7, + }); + const deepSteps = jobs["static-deep"]?.steps ?? []; + const contextIndex = deepSteps.findIndex( + (step) => step.run === "npm run ci:static-context", + ); + const checkIndex = deepSteps.findIndex( + (step) => step.run === "npm run check:ci:deep", + ); + const uploadIndex = deepSteps.findIndex( + (step) => step.with?.name === "static-deep-check-reports", + ); + expect(contextIndex).toBeGreaterThanOrEqual(0); + expect(checkIndex).toBeGreaterThan(contextIndex); + expect(uploadIndex).toBeGreaterThan(checkIndex); expect(jobs["test-source"]?.needs).toBe("changes"); expect(jobs["test-shard"]?.needs).toEqual(["changes", "build"]); expect( @@ -268,7 +301,29 @@ it("records the actual static task graph with explicit source identity and no am const head = await execute("git", ["rev-parse", "HEAD"]); expect(context.sourceSha).toBe(head.stdout.trim()); expect(context.taskHashes["lint:built"]).toMatch(/^[a-f0-9]{16}$/u); - expect(context.taskHashes.jscpd).toMatch(/^[a-f0-9]{16}$/u); + expect(context.taskHashes.typecheck).toMatch(/^[a-f0-9]{16}$/u); + expect(context.taskHashes["format:check"]).toMatch(/^[a-f0-9]{16}$/u); + expect(context.taskHashes).not.toHaveProperty("jscpd"); + expect(context.taskHashes).not.toHaveProperty("knip"); + const deep = await recordStaticTaskContext({ + outputDirectory: `${directory}/deep`, + script: "check:ci:deep:tasks", + }); + expect(deep.taskHashes.jscpd).toMatch(/^[a-f0-9]{16}$/u); + expect(deep.taskHashes.knip).toMatch(/^[a-f0-9]{16}$/u); + expect(deep.taskHashes).not.toHaveProperty("lint:built"); + await expect( + recordStaticTaskContext({ + outputDirectory: directory, + script: "check:ci", + }), + ).rejects.toThrow("Static context script is not a known check"); + await expect( + recordStaticTaskContext({ + outputDirectory: directory, + script: "check:ci:fast", + }), + ).rejects.toThrow("Static context script is not a known check"); expect(context).not.toHaveProperty("environmentVariables"); expect(context).not.toHaveProperty("env"); expect( diff --git a/tests/boundary/filesystem/ciPrEvents.test.ts b/tests/boundary/filesystem/ciPrEvents.test.ts index 3562b5df95..b68bb43af0 100644 --- a/tests/boundary/filesystem/ciPrEvents.test.ts +++ b/tests/boundary/filesystem/ciPrEvents.test.ts @@ -12,21 +12,23 @@ async function workflow(name: string) { concurrency: z.object({ group: z.string() }).optional(), jobs: z.record( z.string(), - z.object({ - name: z.string().optional(), - if: z.string().optional(), - uses: z.string().optional(), - with: z.record(z.string(), z.string()).optional(), - steps: z - .array( - z.object({ - run: z.string().optional(), - uses: z.string().optional(), - env: z.record(z.string(), z.string()).optional(), - }), - ) - .optional(), - }), + z + .object({ + name: z.string().optional(), + if: z.string().optional(), + uses: z.string().optional(), + with: z.record(z.string(), z.string()).optional(), + steps: z + .array( + z.object({ + run: z.string().optional(), + uses: z.string().optional(), + env: z.record(z.string(), z.string()).optional(), + }), + ) + .optional(), + }) + .passthrough(), ), }) .parse( @@ -92,6 +94,7 @@ it("qualifies only ci:full label additions at the caller's merge revision", asyn with: { source_sha: "${{ github.sha }}" }, }, ]); + expect(full.jobs.qualification).not.toHaveProperty("secrets"); // Metadata events must not create skipped replacements for the main CI checks. expect(full.jobs.required).toBeUndefined(); expect(full.concurrency).toBeUndefined(); diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 81fb54bc54..7cfd26448d 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -156,6 +156,7 @@ it("holds native and provider fan-out behind static validation while portable fe const earlyJobs = new Set([ "changes", "static", + "static-deep", "build", "test-shard", "test-source", @@ -171,8 +172,112 @@ it("holds native and provider fan-out behind static validation while portable fe expect(job.if).not.toMatch(/always\(|cancelled\(/u); } expect(jobs["test-shard"]?.needs).not.toContain("static"); + const stepRuns = z.array(z.object({ run: z.string().optional() })); + expect(jobs["static-deep"]?.needs).toBe("changes"); + expect(String(jobs["static-deep"]?.if)).toContain("tier == 'full'"); + expect(String(jobs["static-deep"]?.if)).toContain("code == 'true'"); + const staticRuns = stepRuns.parse(jobs.static?.steps); + const deepRuns = stepRuns.parse(jobs["static-deep"]?.steps); + expect(staticRuns.some((step) => step.run === "npm run check:ci:fast")).toBe( + true, + ); + expect( + staticRuns.some( + (step) => + step.run === "npm run check:ci:deep" || + step.run === "npm run test:python", + ), + ).toBe(false); + expect(deepRuns.some((step) => step.run === "npm run check:ci:deep")).toBe( + true, + ); + for (const [name, job] of Object.entries(jobs)) { + if ( + name === "required" || + name === "static-deep" || + job.needs === undefined + ) + continue; + expect(job.needs, `${name} must not wait for deep static`).not.toContain( + "static-deep", + ); + } }); +const trustedRemoteCacheWrite = + "${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }}"; + +async function assertProviderCacheCallers( + jobs: Awaited>["jobs"], + consumers: readonly string[], +) { + for (const name of consumers) { + const caller = jobs[name]; + expect(caller?.needs).toContain("build"); + expect(caller?.secrets).toEqual({ + TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}", + }); + expect(caller?.with).toEqual({ + build_cache_artifact: "rea-build-cache", + remote_cache_write: trustedRemoteCacheWrite, + }); + const path = z.string().parse(caller?.uses); + const called = z + .object({ + on: z.object({ + workflow_call: z.object({ + inputs: z.record(z.string(), z.unknown()), + secrets: z.object({ + TURBO_TOKEN: z.object({ required: z.literal(false) }), + }), + }), + workflow_dispatch: z.unknown(), + }), + jobs: z.record( + z.string(), + z.object({ + steps: z.array( + z.object({ + uses: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + }), + ), + }), + ), + }) + .parse( + parse( + await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"), + ), + ); + expect(called.on.workflow_call.inputs.build_cache_artifact).toMatchObject({ + type: "string", + default: "", + }); + expect(called.on.workflow_call.inputs.remote_cache_write).toMatchObject({ + type: "boolean", + required: false, + default: false, + }); + expect(Object.hasOwn(called.on, "workflow_dispatch")).toBe(true); + for (const job of Object.values(called.jobs)) + expect( + job.steps.find( + (step) => step.uses === "./.github/actions/portable-build-cache", + )?.with, + ).toEqual({ + build_cache_artifact: "${{ inputs.build_cache_artifact }}", + }); + } + for (const name of [ + "sqlite-runtime", + "nativeaot", + "ghidra_bridge", + "website", + ]) + expect(jobs[name]?.secrets, name).toBeUndefined(); +} + it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => { const { jobs } = await workflow(); const buildSteps = z @@ -236,48 +341,7 @@ it("binds provider build reuse to the current run while keeping standalone dispa ), ); expect(jobs["test-shard"]?.if).toBe("needs.changes.outputs.code == 'true'"); - for (const name of consumers) { - const caller = jobs[name]; - expect(caller?.needs).toContain("build"); - expect(caller?.with).toEqual({ build_cache_artifact: "rea-build-cache" }); - const path = z.string().parse(caller?.uses); - const called = z - .object({ - on: z.object({ - workflow_call: z.object({ - inputs: z.record(z.string(), z.unknown()), - }), - workflow_dispatch: z.unknown(), - }), - jobs: z.record( - z.string(), - z.object({ - steps: z.array( - z.object({ - uses: z.string().optional(), - with: z.record(z.string(), z.unknown()).optional(), - }), - ), - }), - ), - }) - .parse( - parse( - await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"), - ), - ); - expect(called.on.workflow_call.inputs.build_cache_artifact).toMatchObject({ - type: "string", - default: "", - }); - expect(Object.hasOwn(called.on, "workflow_dispatch")).toBe(true); - for (const job of Object.values(called.jobs)) - expect( - job.steps.find( - (step) => step.uses === "./.github/actions/portable-build-cache", - )?.with, - ).toEqual({ build_cache_artifact: "${{ inputs.build_cache_artifact }}" }); - } + await assertProviderCacheCallers(jobs, consumers); const action = z .object({ runs: z.object({ @@ -369,6 +433,14 @@ it("keeps fast feedback separate from complete qualification and release proof", required: z.literal(true), type: z.literal("string"), }), + remote_cache_write: z.object({ + required: z.literal(false), + type: z.literal("boolean"), + default: z.literal(false), + }), + }), + secrets: z.object({ + TURBO_TOKEN: z.object({ required: z.literal(false) }), }), }), }), diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 233d7c8eb6..fe65133320 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -61,6 +61,10 @@ const workflow = async () => build: z.object({ steps: stepsSchema }), docs: z.object({ steps: stepsSchema }), static: z.object({ steps: stepsSchema }), + "static-deep": z.object({ + if: z.string(), + steps: stepsSchema, + }), }), }) .parse( @@ -327,10 +331,26 @@ it("runs the existing onboarding and Python producer guards from their selected expect(scripts["docs:check"]).toContain( "tests/conformance/readmeConsistency.test.ts", ); - const producer = jobs.static.steps.find( - (step) => step.run === "npm run test:python", + expect( + jobs.static.steps.some((step) => step.run === "npm run test:python"), + ).toBe(false); + expect( + jobs.static.steps.some((step) => step.run === "npm run check:ci:fast"), + ).toBe(true); + const deep = jobs["static-deep"]; + expect(deep?.if).toBe( + "needs.changes.outputs.tier == 'full' && needs.changes.outputs.code == 'true'", + ); + expect(deep?.steps.some((step) => step.run === "npm run check:ci:deep")).toBe( + true, + ); + expect(scripts["check:ci:deep"]).toContain("npm run test:python"); + expect(scripts["check:ci:fast"]).toBe( + "node scripts/run-exclusive.mjs artifacts npm run check:ci:fast:tasks", + ); + expect(scripts["check:ci:fast:tasks"]).toBe( + "turbo run typecheck lint:built format:check", ); - expect(producer?.if).toBe("needs.changes.outputs.code == 'true'"); expect(scripts["test:python"]).toContain( "bridge/native/test_rea_lldb_tracer.py", ); diff --git a/tests/boundary/filesystem/ciTestSequencer.test.ts b/tests/boundary/filesystem/ciTestSequencer.test.ts index fec32826d7..320bef0b22 100644 --- a/tests/boundary/filesystem/ciTestSequencer.test.ts +++ b/tests/boundary/filesystem/ciTestSequencer.test.ts @@ -8,8 +8,10 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { allocateByDuration, + estimatedShardCost, parseTestDurations, readTestDurations, + weightFileDurations, } from "../../ciTestSequencer.js"; describe("CI test history", () => { @@ -85,6 +87,27 @@ describe("CI test history", () => { }); describe("duration shard allocation", () => { + it("charges serial process-boundary files their full duration and other projects one worker share", () => { + expect(estimatedShardCost("process-boundary", 400)).toBe(400); + expect(estimatedShardCost("domain", 400)).toBe(200); + expect(estimatedShardCost("adapters", 400)).toBe(200); + }); + + it("fills unknown files from the raw median before serial and shared weighting", () => { + expect( + weightFileDurations([ + { project: "domain", duration: 400 }, + { project: "process-boundary", duration: undefined }, + { project: "adapters", duration: undefined }, + ]), + ).toEqual([200, 400, 200]); + expect( + weightFileDurations([ + { project: "process-boundary", duration: undefined }, + ]), + ).toEqual([undefined]); + }); + it.prop([ fc.uniqueArray(fc.integer({ min: 0, max: 10_000 }), { minLength: 1, diff --git a/tests/boundary/filesystem/releaseWorkflow.test.ts b/tests/boundary/filesystem/releaseWorkflow.test.ts index 8dac943392..5edd662e1d 100644 --- a/tests/boundary/filesystem/releaseWorkflow.test.ts +++ b/tests/boundary/filesystem/releaseWorkflow.test.ts @@ -41,7 +41,8 @@ async function readReleaseWorkflow() { if: z.string(), needs: z.array(z.string()), uses: z.string(), - with: z.record(z.string(), z.string()), + secrets: z.record(z.string(), z.string()), + with: z.record(z.string(), z.union([z.string(), z.boolean()])), }), publish: jobSchema, rehearsal: jobSchema.extend({ @@ -288,9 +289,13 @@ it("binds npm and MCP publication to the same immutable release SHA", async () = "needs.release-please.outputs.release_created", ); expect(workflow.jobs.qualification.uses).toBe("./.github/workflows/ci.yml"); + expect(workflow.jobs.qualification.secrets).toEqual({ + TURBO_TOKEN: "${{ secrets.TURBO_TOKEN }}", + }); expect(workflow.jobs.qualification.with).toEqual({ source_sha: "${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }}", + remote_cache_write: true, }); expect(workflow.jobs.publish.needs).toContain("qualification"); expect(workflow.jobs.publish.needs).toContain("rehearsal"); diff --git a/tests/ciTestSequencer.ts b/tests/ciTestSequencer.ts index 996e8ce7a6..49de240f5b 100644 --- a/tests/ciTestSequencer.ts +++ b/tests/ciTestSequencer.ts @@ -62,6 +62,33 @@ export async function readTestDurations( type TimedItem = { item: T; key: string; duration: number | undefined }; +// process-boundary runs its files one at a time. Other projects overlap up to +// the runner worker cap, so equal file-duration sums are not equal wall time. +const PARALLEL_TEST_WORKERS = 2; + +export function estimatedShardCost( + project: string, + durationMs: number, +): number { + if (project === "process-boundary") return durationMs; + return durationMs / PARALLEL_TEST_WORKERS; +} + +/** Weight raw durations, filling gaps from the raw median before worker sharing. */ +export function weightFileDurations( + entries: readonly { project: string; duration: number | undefined }[], +): (number | undefined)[] { + const known = entries + .flatMap(({ duration }) => (duration === undefined ? [] : [duration])) + .sort((left, right) => left - right); + if (!known.length) return entries.map(() => undefined); + const lower = known[Math.floor((known.length - 1) / 2)]!; + const median = lower + (known[Math.floor(known.length / 2)]! - lower) / 2; + return entries.map(({ project, duration }) => + estimatedShardCost(project, duration ?? median), + ); +} + /** Distribute the longest files first, retaining every discovered specification. */ export function allocateByDuration( items: readonly TimedItem[], @@ -105,16 +132,23 @@ export class CiTestSequencer extends BaseSequencer { const durations = await readTestDurations(historyPath, root); if (!durations) return super.shard(files); - const weighted = files.map((item) => { + const timed = files.map((item) => { const path = relative(root, resolve(root, item.moduleId)) .split(sep) .join("/"); return { item, key: path + "\0" + item.project.name + "\0" + item.pool, + project: item.project.name, duration: durations.get(path), }; }); + const costs = weightFileDurations(timed); + const weighted = timed.map((entry, index) => ({ + item: entry.item, + key: entry.key, + duration: costs[index], + })); const shards = allocateByDuration(weighted, shard.count); return shards ? shards[shard.index - 1]! : super.shard(files); }