diff --git a/.github/actions/portable-build-cache/action.yml b/.github/actions/portable-build-cache/action.yml index ad127ce3dc..cee775ec52 100644 --- a/.github/actions/portable-build-cache/action.yml +++ b/.github/actions/portable-build-cache/action.yml @@ -1,14 +1,24 @@ name: Portable build cache -description: Reuse Turbo build outputs without serializing verification jobs. +description: Restore portable Turbo build outputs from the current run or cross-run cache. inputs: save: description: Save newly built outputs after the job succeeds. default: "false" + build_cache_artifact: + description: Reuse portable Turbo outputs uploaded by the current workflow run. + default: "" runs: using: composite steps: + - name: Download portable build outputs from this workflow run + if: inputs.build_cache_artifact != '' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.build_cache_artifact }} + path: .turbo/cache/ - name: Resolve portable build task hashes id: plan + if: inputs.build_cache_artifact == '' shell: bash env: BUILD_CACHE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc', '.nvmrc', 'package-lock.json') }} @@ -28,14 +38,14 @@ runs: console.log(`prefix=${process.env.BUILD_CACHE_PREFIX}-`); ' >> "$GITHUB_OUTPUT" - name: Restore portable build outputs - if: inputs.save != 'true' + if: inputs.build_cache_artifact == '' && inputs.save != 'true' uses: actions/cache/restore@v4 with: path: .turbo/cache/ key: ${{ steps.plan.outputs.key }} restore-keys: ${{ steps.plan.outputs.prefix }} - name: Restore and save portable build outputs - if: inputs.save == 'true' + if: inputs.build_cache_artifact == '' && inputs.save == 'true' uses: actions/cache@v4 with: path: .turbo/cache/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28373f89d2..e0516da850 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,7 +190,18 @@ jobs: build: name: Build package needs: changes - if: needs.changes.outputs.code == 'true' + # Reusable provider workflows consume this run's portable build artifact, + # even when a workflow-only edit selects the lane without portable tests. + if: >- + needs.changes.outputs.code == 'true' || + needs.changes.outputs.browser == 'true' || + needs.changes.outputs.binary_layout == 'true' || + needs.changes.outputs.evm == 'true' || + needs.changes.outputs.javascript_recovery == 'true' || + needs.changes.outputs.recorded_crash == 'true' || + needs.changes.outputs.web_network == 'true' || + needs.changes.outputs.web_runtime == 'true' || + needs.changes.outputs.web_source_map == 'true' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -673,44 +684,60 @@ jobs: if-no-files-found: error browser: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.browser == 'true' uses: ./.github/workflows/real-browser.yml + with: + build_cache_artifact: rea-build-cache binary_layout: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.binary_layout == 'true' uses: ./.github/workflows/real-binary-layout.yml + with: + build_cache_artifact: rea-build-cache evm: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.evm == 'true' uses: ./.github/workflows/real-evm-interface.yml + with: + build_cache_artifact: rea-build-cache javascript_recovery: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.javascript_recovery == 'true' uses: ./.github/workflows/real-javascript-recovery.yml + with: + build_cache_artifact: rea-build-cache recorded_crash: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.recorded_crash == 'true' uses: ./.github/workflows/real-recorded-crash.yml + with: + build_cache_artifact: rea-build-cache web_network: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_network == 'true' uses: ./.github/workflows/real-web-network-captures.yml + with: + build_cache_artifact: rea-build-cache web_runtime: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_runtime == 'true' uses: ./.github/workflows/real-web-runtime.yml + with: + build_cache_artifact: rea-build-cache web_source_map: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_source_map == 'true' uses: ./.github/workflows/real-web-source-map.yml + with: + build_cache_artifact: rea-build-cache nativeaot: needs: [changes, static] diff --git a/.github/workflows/real-binary-layout.yml b/.github/workflows/real-binary-layout.yml index 8dd6f98667..5cec952ce7 100644 --- a/.github/workflows/real-binary-layout.yml +++ b/.github/workflows/real-binary-layout.yml @@ -8,6 +8,12 @@ on: type: boolean default: false workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -29,6 +35,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire exact upstream libraries in an isolated runner environment shell: bash run: | diff --git a/.github/workflows/real-browser.yml b/.github/workflows/real-browser.yml index 11a8b7a539..912fdafb1b 100644 --- a/.github/workflows/real-browser.yml +++ b/.github/workflows/real-browser.yml @@ -2,6 +2,12 @@ name: Real browser verification on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: env: @@ -22,6 +28,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run verify:browser env: REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome diff --git a/.github/workflows/real-evm-interface.yml b/.github/workflows/real-evm-interface.yml index b335f13911..dfdd3970fe 100644 --- a/.github/workflows/real-evm-interface.yml +++ b/.github/workflows/real-evm-interface.yml @@ -3,6 +3,12 @@ name: Real offline EVM interface on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -23,6 +29,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire unchanged fixture compiler in isolated runner storage shell: bash run: | diff --git a/.github/workflows/real-javascript-recovery.yml b/.github/workflows/real-javascript-recovery.yml index d94127d639..817dc4dc06 100644 --- a/.github/workflows/real-javascript-recovery.yml +++ b/.github/workflows/real-javascript-recovery.yml @@ -2,6 +2,12 @@ name: Real JavaScript recovery on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: @@ -24,6 +30,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Fetch pinned Wakaru release into the job workspace shell: bash run: | diff --git a/.github/workflows/real-recorded-crash.yml b/.github/workflows/real-recorded-crash.yml index dc56034c8f..04467a6089 100644 --- a/.github/workflows/real-recorded-crash.yml +++ b/.github/workflows/real-recorded-crash.yml @@ -3,6 +3,12 @@ name: Real recorded crash evidence on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -26,6 +32,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Install GDB on the disposable CI runner shell: bash run: | diff --git a/.github/workflows/real-web-network-captures.yml b/.github/workflows/real-web-network-captures.yml index df340520d4..d41bb37a56 100644 --- a/.github/workflows/real-web-network-captures.yml +++ b/.github/workflows/real-web-network-captures.yml @@ -3,6 +3,12 @@ name: Real historical web network captures on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -25,6 +31,8 @@ jobs: run: npm install --global npm@11.16.0 - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire fixed upstream native profile in runner temporary storage shell: bash run: | diff --git a/.github/workflows/real-web-runtime.yml b/.github/workflows/real-web-runtime.yml index 0ecdb92c65..2575c55f9c 100644 --- a/.github/workflows/real-web-runtime.yml +++ b/.github/workflows/real-web-runtime.yml @@ -1,6 +1,12 @@ name: Real website runtime attribution on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: contents: read @@ -18,6 +24,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run verify:browser:runtime env: REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome diff --git a/.github/workflows/real-web-source-map.yml b/.github/workflows/real-web-source-map.yml index b946621299..8e0869eb78 100644 --- a/.github/workflows/real-web-source-map.yml +++ b/.github/workflows/real-web-source-map.yml @@ -1,6 +1,12 @@ name: Real website source-map verification on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: contents: read @@ -18,6 +24,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Install the pinned compiler into an isolated fixture directory run: npm install --prefix "$RUNNER_TEMP/rea-web-source-map-compiler" --no-audit --no-fund --package-lock=false esbuild@0.25.10 - run: npm run verify:browser:source-maps diff --git a/docs/testing.md b/docs/testing.md index 03121e172e..1288a954a1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -982,6 +982,16 @@ timing data is reused; live provider results are never cached. Turbo outputs are restored by task hash, with trusted default-branch runs as cache writers and PRs as restore-only consumers. npm caches hold downloads, not `node_modules`. +The eight automatic browser, binary-layout, EVM, JavaScript-recovery, +recorded-crash, and web provider lanes download `rea-build-cache` from their own +run after the build and static jobs succeed. Their existing build commands still +validate task hashes and rebuild mismatched inputs; provider observations always +run. Provider-workflow-only edits also admit the build without enabling source +test shards. Direct provider dispatches retain the cross-run restore-only cache. +This removes repeated compilation on fresh source revisions, but adds the build +as a prerequisite: a delayed build can delay provider admission. Compare queue +and execution times before claiming an end-to-end speedup. + These native baseline checks complement the Linux source-test shards and the separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA, browser and managed-tool claims require their corresponding verification lanes. diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index fa4b71ebf2..163219c39d 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -680,7 +680,9 @@ it.each([ async (scope, name) => { const workflow = z .object({ - on: z.object({ workflow_call: z.null() }).passthrough(), + on: z + .object({ workflow_call: z.object({}).passthrough().nullable() }) + .passthrough(), }) .parse( parse( diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index e460bb52b5..b6e02b54af 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -168,6 +168,115 @@ it("holds native and provider fan-out behind static validation while portable fe expect(jobs["test-shard"]?.needs).not.toContain("static"); }); +it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => { + const { jobs } = await workflow(); + const consumers = [ + "browser", + "binary_layout", + "evm", + "javascript_recovery", + "recorded_crash", + "web_network", + "web_runtime", + "web_source_map", + ]; + const buildTerms = z + .string() + .parse(jobs.build?.if) + .split("||") + .map((term) => term.trim()); + // Provider workflow edits select their lane without code=true. The build + // producer must still run, without selecting the portable test shards. + expect(new Set(buildTerms)).toEqual( + new Set( + ["code", ...consumers].map( + (scope) => `needs.changes.outputs.${scope} == 'true'`, + ), + ), + ); + expect(jobs["test-shard"]?.if).toBe("needs.changes.outputs.code == 'true'"); + for (const name of consumers) { + const caller = jobs[name]; + expect(caller?.needs).toContain("build"); + expect(caller?.with).toEqual({ build_cache_artifact: "rea-build-cache" }); + const path = z.string().parse(caller?.uses); + const called = z + .object({ + on: z.object({ + workflow_call: z.object({ + inputs: z.record(z.string(), z.unknown()), + }), + workflow_dispatch: z.unknown(), + }), + jobs: z.record( + z.string(), + z.object({ + steps: z.array( + z.object({ + uses: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + }), + ), + }), + ), + }) + .parse( + parse( + await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"), + ), + ); + expect(called.on.workflow_call.inputs.build_cache_artifact).toMatchObject({ + type: "string", + default: "", + }); + expect(Object.hasOwn(called.on, "workflow_dispatch")).toBe(true); + for (const job of Object.values(called.jobs)) + expect( + job.steps.find( + (step) => step.uses === "./.github/actions/portable-build-cache", + )?.with, + ).toEqual({ build_cache_artifact: "${{ inputs.build_cache_artifact }}" }); + } + const action = z + .object({ + runs: z.object({ + steps: z.array( + z.object({ + id: z.string().optional(), + uses: z.string().optional(), + if: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + "continue-on-error": z.boolean().optional(), + }), + ), + }), + }) + .parse( + parse( + await readFile( + new URL( + "../../../.github/actions/portable-build-cache/action.yml", + import.meta.url, + ), + "utf8", + ), + ), + ); + const download = action.runs.steps.find( + (step) => step.uses === "actions/download-artifact@v4", + ); + expect(download?.with).toEqual({ + name: "${{ inputs.build_cache_artifact }}", + path: ".turbo/cache/", + }); + expect(download?.if).toBe("inputs.build_cache_artifact != ''"); + expect(download?.["continue-on-error"]).not.toBe(true); + for (const step of action.runs.steps.filter( + (step) => step.id === "plan" || step.uses?.startsWith("actions/cache"), + )) + expect(step.if).toContain("inputs.build_cache_artifact == ''"); +}); + it("runs workflow validation inside required static checks when selected", async () => { const { jobs } = await workflow(); const steps = z diff --git a/tests/boundary/filesystem/testTaskCache.test.ts b/tests/boundary/filesystem/testTaskCache.test.ts index 7e06ce1a1a..5d877e8292 100644 --- a/tests/boundary/filesystem/testTaskCache.test.ts +++ b/tests/boundary/filesystem/testTaskCache.test.ts @@ -1,6 +1,6 @@ import { execFile } from "node:child_process"; -import { readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { cp, readFile, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; import { promisify } from "node:util"; import { expect, it } from "vitest"; @@ -78,7 +78,7 @@ it.skipIf(process.platform === "android")( await workflowGit(root, ["init", "--initial-branch=main"]); await workflowGit(root, ["add", "."]); await workflowGit(root, ["commit", "-m", "test: seed cache fixture"]); - const run = () => + const run = (cwd = root) => execute( process.execPath, [ @@ -87,7 +87,7 @@ it.skipIf(process.platform === "android")( "test:run", "--output-logs=errors-only", ], - { cwd: root, env: { ...process.env, TURBO_DAEMON: "false" } }, + { cwd, env: { ...process.env, TURBO_DAEMON: "false" } }, ); await run(); await writeFile(join(root, "README_zh.md"), "wrong command"); @@ -104,5 +104,32 @@ it.skipIf(process.platform === "android")( "prepared\n", ); } + // Provider jobs receive the producer's cache in a fresh checkout, not its + // outputs. Turbo must restore identical inputs and rebuild changed ones. + const consumer = await createTestTempDirectory("rea-cache-consumer-"); + await cp(root, consumer, { + recursive: true, + filter: (path) => + basename(path) !== ".turbo" && + !/^prepared-\d+\.txt$/u.test(basename(path)), + }); + await cp(join(root, ".turbo/cache"), join(consumer, ".turbo/cache"), { + recursive: true, + }); + const restored = await run(consumer); + expect(restored.stdout).toContain(`${dependencies.length} cached`); + for (const index of dependencies.keys()) + expect( + await readFile(join(consumer, `prepared-${index}.txt`), "utf8"), + ).toBe("prepared\n"); + await writeFile( + join(consumer, "artifact-input.txt"), + "changed producer input", + ); + await run(consumer); + for (const index of dependencies.keys()) + expect( + await readFile(join(consumer, `prepared-${index}.txt`), "utf8"), + ).toBe("prepared\nprepared\n"); }, );