Shard playwright tests, cache tracker npm deps and browsers (#5603)

.github/workflows/tracker.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - 'tracker/**'
+      - "tracker/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -14,15 +14,68 @@ jobs:
   test:
     timeout-minutes: 15
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 23.2.0
+          cache: 'npm'
+          cache-dependency-path: tracker/package-lock.json
       - name: Install dependencies
         run: npm --prefix ./tracker ci
-    - name: Install Playwright Browsers
+      - name: Cache Playwright browsers
+        uses: actions/cache@v4
+        id: playwright-cache
+        with:
+          path: |
+            ~/.cache/ms-playwright
+            ~/.cache/ms-playwright-github
+          key: playwright-${{ runner.os }}-${{ hashFiles('tracker/package-lock.json') }}
+          restore-keys: |
+            playwright-${{ runner.os }}-
+      - name: Install Playwright system dependencies
         working-directory: ./tracker
-      run: npx playwright install --with-deps
+        run: npx playwright install-deps
+      - name: Install Playwright Browsers
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
+        working-directory: ./tracker
+        run: npx playwright install
       - name: Run Playwright tests
-      run: npm --prefix ./tracker test
+        run: npm --prefix ./tracker test -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter=blob
+      - name: Upload blob report to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: blob-report-${{ matrix.shardIndex }}
+          path: tracker/blob-report
+          retention-days: 1
+  merge-sharded-test-report:
+    if: ${{ !cancelled() }}
+    needs: [test]
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 23.2.0
+          cache: 'npm'
+          cache-dependency-path: tracker/package-lock.json
+      - name: Install dependencies
+        run: npm --prefix ./tracker ci
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge into list report
+        working-directory: ./tracker
+        run: npx playwright merge-reports --reporter list ../all-blob-reports

tracker/playwright.config.ts

@@ -1,4 +1,3 @@
-// @ts-check
 import { defineConfig, devices } from '@playwright/test'
 
 /**
@@ -9,12 +8,16 @@ export default defineConfig({
   /* Can be overridden in specific tests with test('a longer running test', async () => { test.setTimeout(<longer timeout>); // test content... }) */
   timeout: 10 * 1000,
   fullyParallel: true,
+  /* Optimize workers for CI vs local development */
+  workers: process.env.CI ? 4 : undefined,
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
-  retries: process.env.CI ? 1 : 0,
+  retries: process.env.CI ? 2 : 0,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
   reporter: 'list',
+  /* Limit the number of failures on CI to save resources */
+  maxFailures: process.env.CI ? 10 : undefined,
   /*
   Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions.
   NOTE: We run the installation support tests on Chrome only because the Browserless /function API
@@ -23,24 +26,24 @@ export default defineConfig({
   projects: [
     {
       name: 'chromium',
-      use: { ...devices['Desktop Chrome'] },
+      use: { ...devices['Desktop Chrome'] }
     },
 
     {
       name: 'firefox',
       use: { ...devices['Desktop Firefox'] },
-      testIgnore: 'test/installation_support/**',
+      testIgnore: 'test/installation_support/**'
     },
 
     {
       name: 'webkit',
       use: { ...devices['Desktop Safari'] },
-      testIgnore: 'test/installation_support/**',
+      testIgnore: 'test/installation_support/**'
-    },
+    }
   ],
   webServer: {
     command: 'npm run start',
     port: 3000,
     reuseExistingServer: !process.env.CI
-  },
+  }
-});
+})