mirror of
https://github.com/morluto/rea
synced 2026-10-25 22:08:11 -04:00
ci: keep npm download caches from Turbo snapshot growth (#5655)
* ci: keep npm download caches from Turbo snapshot growth Per-commit static and portable Turbo snapshots had filled the 10 GB Actions cache, which is what makes warm npm ci cheap. Trusted static runs now keep two generations and one download-cache writer per host. Co-authored-by: morluto <morluto@users.noreply.github.com> * fix(ci): declare cache retention types for strict checks Typecheck imports cache-retention.mjs and needs a sibling declaration. The generation count stays module-private so knip does not report it. Co-authored-by: morluto <morluto@users.noreply.github.com> * fix(ci): keep compatible caches and qualification read-only Npm dedupe deletes a record only when another has the same key, version, and ref. Cache deletion moves to a follow-up workflow so reusable qualification can keep its read-only token. Co-authored-by: morluto <morluto@users.noreply.github.com> * fix(ci): retain caches after failed default-branch runs Static and build save Turbo snapshots before later lanes finish. Retention now follows a failed, timed-out, or cancelled trusted run, not only a green one. Co-authored-by: morluto <morluto@users.noreply.github.com> * test(cli): pass application trace input as a file The CLI already reads workflow JSON from a path. Putting the full application Evidence document on argv exceeds the runner argument limit. Co-authored-by: morluto <morluto@users.noreply.github.com> * fix(ci): retain deep and Knip snapshots without deleting a repeated id Fast and deep static checks publish rea-static-deep-v1 and knip-meta-v1 entries per commit. Retention prunes those families with the same two-generation rule. A cache id listed twice is one record; inconsistent repeats abort before any delete. Co-authored-by: morluto <morluto@users.noreply.github.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: morluto <morluto@users.noreply.github.com>
This commit is contained in:
morluto
@@ -1,5 +1,10 @@
|
||||
name: Cache npm package downloads
|
||||
description: Restore npm's download cache and save it only from trusted default-branch events.
|
||||
description: Restore npm's download cache. Trusted default-branch jobs that opt in may save it.
|
||||
inputs:
|
||||
save:
|
||||
description: Save the shared download cache when this event is allowed to write it.
|
||||
required: false
|
||||
default: "false"
|
||||
outputs:
|
||||
can-save:
|
||||
description: Whether this run may update the shared npm download cache.
|
||||
@@ -23,20 +28,25 @@ runs:
|
||||
can_save=true
|
||||
fi
|
||||
cache_key="npm-download-${RUNNER_OS}-${runtime_arch}-${LOCKFILES_HASH}"
|
||||
# Same platform only. A lockfile change can reuse tarballs from the previous key.
|
||||
restore_key="npm-download-${RUNNER_OS}-${runtime_arch}-"
|
||||
{
|
||||
printf 'can-save=%s\n' "${can_save}"
|
||||
printf 'path=%s\n' "${cache_path}"
|
||||
printf 'key=%s\n' "${cache_key}"
|
||||
printf 'restore-key=%s\n' "${restore_key}"
|
||||
} >> "${GITHUB_OUTPUT}"
|
||||
- name: Restore and save npm package downloads
|
||||
if: steps.context.outputs.can-save == 'true'
|
||||
if: steps.context.outputs.can-save == 'true' && inputs.save == 'true'
|
||||
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: ${{ steps.context.outputs.path }}
|
||||
key: ${{ steps.context.outputs.key }}
|
||||
restore-keys: ${{ steps.context.outputs.restore-key }}
|
||||
- name: Restore npm package downloads without saving
|
||||
if: steps.context.outputs.can-save != 'true'
|
||||
if: steps.context.outputs.can-save != 'true' || inputs.save != 'true'
|
||||
uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
|
||||
with:
|
||||
path: ${{ steps.context.outputs.path }}
|
||||
key: ${{ steps.context.outputs.key }}
|
||||
restore-keys: ${{ steps.context.outputs.restore-key }}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
name: Cache retention
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["CI"]
|
||||
types: [completed]
|
||||
|
||||
# Qualification calls ci.yml with actions: read. A nested workflow cannot raise
|
||||
# that token, so cache deletion stays in this workflow.
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write
|
||||
|
||||
concurrency:
|
||||
group: cache-retention-${{ github.event.workflow_run.head_sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
retain:
|
||||
name: Retain Turbo snapshots and npm download caches
|
||||
# Checkout and run only a same-repository default-branch commit. Pull
|
||||
# requests, including forks, do not reach these steps. Static and build
|
||||
# save Turbo snapshots before later lanes finish, so a failed, timed-out,
|
||||
# or cancelled run still retains those snapshots.
|
||||
if: >-
|
||||
(github.event.workflow_run.conclusion == 'success' ||
|
||||
github.event.workflow_run.conclusion == 'failure' ||
|
||||
github.event.workflow_run.conclusion == 'timed_out' ||
|
||||
github.event.workflow_run.conclusion == 'cancelled') &&
|
||||
github.event.workflow_run.head_repository.full_name == github.repository &&
|
||||
github.event.workflow_run.head_branch == github.event.repository.default_branch &&
|
||||
(github.event.workflow_run.event == 'push' ||
|
||||
github.event.workflow_run.event == 'schedule' ||
|
||||
github.event.workflow_run.event == 'workflow_dispatch')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Retain current Turbo snapshots and npm download caches
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CACHE_PREFIX: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-
|
||||
CACHE_KEEP_KEY: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
|
||||
CACHE_DEEP_PREFIX: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-
|
||||
CACHE_DEEP_KEEP_KEY: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
|
||||
CACHE_KNIP_PREFIX: knip-meta-v1-${{ runner.os }}-
|
||||
CACHE_KNIP_KEEP_KEY: knip-meta-v1-${{ runner.os }}-${{ hashFiles('knip.json', 'tsconfig.json', 'package.json', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
|
||||
CACHE_PORTABLE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-
|
||||
DEDUPE_NPM_DOWNLOADS: "true"
|
||||
run: node scripts/ci/cache-retention.mjs
|
||||
@@ -146,6 +146,9 @@ jobs:
|
||||
- name: Cache npm package downloads
|
||||
id: npm-cache
|
||||
uses: ./.github/actions/npm-cache
|
||||
with:
|
||||
# Linux x64 writer. Parallel Linux jobs restore this key.
|
||||
save: "true"
|
||||
- run: npm ci
|
||||
- name: Restore static task cache
|
||||
if: needs.changes.outputs.code == 'true'
|
||||
@@ -658,6 +661,9 @@ jobs:
|
||||
- name: Cache npm package downloads
|
||||
id: npm-cache
|
||||
uses: ./.github/actions/npm-cache
|
||||
with:
|
||||
# ubuntu-latest is the Linux x64 writer above. Other hosts save their own key.
|
||||
save: ${{ matrix.os != 'ubuntu-latest' }}
|
||||
- name: Verify native runtime identity
|
||||
env:
|
||||
EXPECTED_CI_PLATFORM: ${{ matrix.platform }}
|
||||
@@ -794,6 +800,8 @@ jobs:
|
||||
- name: Cache npm package downloads
|
||||
id: npm-cache
|
||||
uses: ./.github/actions/npm-cache
|
||||
with:
|
||||
save: "true"
|
||||
- run: npm ci
|
||||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
|
||||
+32
-9
@@ -579,8 +579,9 @@ metadata, and available manifests into `.cache/ci-build-cache/`.
|
||||
`portable-build-cache.json` records availability per task. Evicted or partially
|
||||
missing entries are omitted as whole entries, and consumers rebuild cache misses,
|
||||
including when every entry is unavailable. Malformed plans, non-file entries,
|
||||
and I/O errors other than disappearance still fail publication. Older cross-run entries stay in the
|
||||
original cache. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack
|
||||
and I/O errors other than disappearance still fail publication. After a trusted
|
||||
static run, superseded cross-run Turbo snapshots for that OS and architecture
|
||||
are deleted down to the two newest generations. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack
|
||||
commands, which restore matching outputs or rebuild on a cache miss. Native
|
||||
artifacts, dependency installs, package archives, and runtime verification remain
|
||||
host-local. Keep platform-dependent outputs out of this shared cache.
|
||||
@@ -1370,13 +1371,35 @@ token leaves the local Turbo cache in place.
|
||||
|
||||
The shared npm-cache action caches downloads, not `node_modules`, and every
|
||||
consumer still runs `npm ci`. Keys cover the runner OS, actual Node architecture,
|
||||
root lockfiles, and `.npmrc`. Only push, schedule, and manual events on the
|
||||
repository's default branch may save shared npm caches; PRs restore only.
|
||||
Static snapshots use the same write authority and are saved only after the
|
||||
static job's validation succeeds. Main warms the new cache namespace after
|
||||
merge; a cold cache or changed lockfile falls back to the normal install.
|
||||
This prevents each PR from saving another branch-scoped copy of the same
|
||||
download cache. It does not remove runner assignment waits.
|
||||
root lockfiles, and `.npmrc`. An exact-key miss restores the newest cache for
|
||||
that OS and architecture, so a lockfile change reuses unchanged tarballs while
|
||||
`npm ci` still verifies the lockfile. Only push, schedule, and manual events on
|
||||
the repository's default branch may save shared npm caches, and only from the
|
||||
opted-in writer for that host: Static checks for Linux x64, the package matrix
|
||||
for every other included host, and the Windows curated lane. Every other job
|
||||
restores without saving, so parallel jobs do not publish duplicate copies.
|
||||
Pull requests restore only. Static snapshots are saved only after the static
|
||||
job's validation succeeds. Cache deletion is a
|
||||
separate workflow, because reusable qualification calls this workflow and
|
||||
cannot raise its read-only token. After a push, schedule, or manual CI run on
|
||||
the default branch finishes, including when a later lane fails, times out, or
|
||||
is cancelled, that workflow runs on `ubuntu-latest` and deletes older
|
||||
`rea-static-v1`, `rea-static-deep-v1`, and `rea-portable-build-v2` entries for
|
||||
that runner's OS and architecture, plus older `knip-meta-v1` entries for that
|
||||
OS. It keeps two generations plus the key that run saved. A fast run that did
|
||||
not publish a deep or Knip snapshot still leaves the two newest entries in
|
||||
those families. It
|
||||
deletes an npm download record only when another record has the same key, cache
|
||||
version, and git ref, and it keeps the most recently accessed copy. A different
|
||||
version or ref is a different restore target and stays. One cache id is one
|
||||
record: a repeated id keeps the later access time when its key, version, ref,
|
||||
size, and creation time agree, and an inconsistent repeat deletes nothing. An
|
||||
incomplete or unexpected cache list deletes nothing.
|
||||
Distinct lockfile generations of the download cache stay, so the restore
|
||||
prefix still has a seed.
|
||||
Main warms the new cache namespace after merge; a cold cache falls back to the
|
||||
normal install. This prevents each PR from saving another branch-scoped copy of
|
||||
the same download cache. It does not remove runner assignment waits.
|
||||
|
||||
Portable owner-based test selection remains observational. Its reports compare
|
||||
proposed omissions with the full run; incomplete or skipped observations do not
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
export interface RetainedCacheEntry {
|
||||
readonly id: number;
|
||||
readonly key: string;
|
||||
readonly createdAt: number;
|
||||
readonly accessedAt: number;
|
||||
readonly size: number;
|
||||
}
|
||||
|
||||
export interface CacheListRecord {
|
||||
readonly id: number;
|
||||
readonly key: string;
|
||||
readonly version: string;
|
||||
readonly ref: string;
|
||||
readonly created_at: string;
|
||||
readonly last_accessed_at?: string;
|
||||
readonly size_in_bytes: number;
|
||||
}
|
||||
|
||||
export interface RetentionFamily {
|
||||
readonly prefix: string;
|
||||
readonly keepKey?: string;
|
||||
}
|
||||
|
||||
export type CacheRequest = (
|
||||
method: string,
|
||||
endpoint: string,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export function assertRetentionPrefix(prefix: string): void;
|
||||
|
||||
export function selectPrefixDeletions(
|
||||
caches: readonly unknown[],
|
||||
family?: RetentionFamily,
|
||||
keepGenerations?: number,
|
||||
): RetainedCacheEntry[];
|
||||
|
||||
export function selectNpmDownloadDuplicateDeletions(
|
||||
caches: readonly unknown[],
|
||||
): RetainedCacheEntry[];
|
||||
|
||||
export function listAllCaches(
|
||||
request: CacheRequest,
|
||||
repository: string,
|
||||
prefix: string,
|
||||
): Promise<readonly CacheListRecord[]>;
|
||||
|
||||
export function retainCaches(options: {
|
||||
repository: string;
|
||||
request: CacheRequest;
|
||||
deleteCache: (id: number) => Promise<void>;
|
||||
families?: readonly RetentionFamily[];
|
||||
dedupeNpmDownloads?: boolean;
|
||||
dryRun?: boolean;
|
||||
}): Promise<{
|
||||
dryRun: boolean;
|
||||
plan: readonly {
|
||||
prefix: string;
|
||||
deletions: readonly RetainedCacheEntry[];
|
||||
}[];
|
||||
npmDeletions: readonly RetainedCacheEntry[];
|
||||
}>;
|
||||
@@ -0,0 +1,358 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { realpathSync } from "node:fs";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const MAX_BUFFER = 8 * 1024 * 1024;
|
||||
const REQUEST_TIMEOUT_MS = 20_000;
|
||||
const MAX_PAGES = 10;
|
||||
const PAGE_SIZE = 100;
|
||||
|
||||
// Keep the newest snapshots, not only the key just saved. A concurrent job may
|
||||
// still be restoring the previous generation; deleting that id fails its restore.
|
||||
// The following trusted run drops anything older than those two.
|
||||
const RETAINED_CACHE_GENERATIONS = 2;
|
||||
|
||||
const PREFIX_PATTERNS = [
|
||||
/^rea-static-v1-[A-Za-z0-9]+-[A-Za-z0-9]+-$/u,
|
||||
/^rea-static-deep-v1-[A-Za-z0-9]+-[A-Za-z0-9]+-$/u,
|
||||
/^rea-portable-build-v2-[A-Za-z0-9]+-[A-Za-z0-9]+-$/u,
|
||||
/^knip-meta-v1-[A-Za-z0-9]+-$/u,
|
||||
];
|
||||
|
||||
/** Reject prefixes that could match npm downloads or another platform. */
|
||||
export function assertRetentionPrefix(prefix) {
|
||||
if (
|
||||
typeof prefix !== "string" ||
|
||||
!PREFIX_PATTERNS.some((pattern) => pattern.test(prefix))
|
||||
)
|
||||
throw new Error(
|
||||
"Cache retention prefix must be one Turbo family for one OS and architecture, or one Knip metadata family for one OS",
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeCache(cache, index) {
|
||||
if (typeof cache !== "object" || cache === null || Array.isArray(cache))
|
||||
throw new Error(`Cache entry ${String(index + 1)} is malformed`);
|
||||
const { id, key, size_in_bytes: size } = cache;
|
||||
if (!Number.isSafeInteger(id) || id <= 0)
|
||||
throw new Error(`Cache entry ${String(index + 1)} has an invalid id`);
|
||||
if (typeof key !== "string" || key.length === 0 || /[\r\n]/u.test(key))
|
||||
throw new Error(`Cache entry ${String(index + 1)} has an invalid key`);
|
||||
const createdAt = Date.parse(cache.created_at);
|
||||
const accessedAt = Date.parse(cache.last_accessed_at ?? cache.created_at);
|
||||
if (!Number.isFinite(createdAt) || !Number.isFinite(accessedAt))
|
||||
throw new Error(
|
||||
`Cache entry ${String(index + 1)} has an invalid timestamp`,
|
||||
);
|
||||
if (!Number.isSafeInteger(size) || size < 0)
|
||||
throw new Error(`Cache entry ${String(index + 1)} has an invalid size`);
|
||||
// actions/cache restores a key only when this metadata hash matches.
|
||||
const version = cache.version;
|
||||
if (typeof version !== "string" || !/^[a-f0-9]{64}$/u.test(version))
|
||||
throw new Error(`Cache entry ${String(index + 1)} has an invalid version`);
|
||||
// The git ref is the cache's access scope. Another branch cannot use it.
|
||||
const ref = cache.ref;
|
||||
if (typeof ref !== "string" || !/^refs\/[^\s]+$/u.test(ref))
|
||||
throw new Error(`Cache entry ${String(index + 1)} has an invalid ref`);
|
||||
return { id, key, version, ref, createdAt, accessedAt, size };
|
||||
}
|
||||
|
||||
function sameStableIdentity(left, right) {
|
||||
return (
|
||||
left.key === right.key &&
|
||||
left.version === right.version &&
|
||||
left.ref === right.ref &&
|
||||
left.size === right.size &&
|
||||
left.createdAt === right.createdAt
|
||||
);
|
||||
}
|
||||
|
||||
// The list API orders by last access. A restore between pages can place one
|
||||
// id on two pages. That is still one cache: collapse it, or abort when the
|
||||
// pages disagree, before any id can be both kept and deleted.
|
||||
function uniqueById(entries) {
|
||||
const byId = new Map();
|
||||
for (const entry of entries) {
|
||||
const existing = byId.get(entry.id);
|
||||
if (existing === undefined) {
|
||||
byId.set(entry.id, entry);
|
||||
continue;
|
||||
}
|
||||
if (!sameStableIdentity(existing, entry))
|
||||
throw new Error(
|
||||
`Cache id ${String(entry.id)} was listed with inconsistent fields`,
|
||||
);
|
||||
if (entry.accessedAt > existing.accessedAt) byId.set(entry.id, entry);
|
||||
}
|
||||
return [...byId.values()];
|
||||
}
|
||||
|
||||
function byNewest(left, right) {
|
||||
if (left.createdAt !== right.createdAt)
|
||||
return right.createdAt - left.createdAt;
|
||||
return right.id - left.id;
|
||||
}
|
||||
|
||||
/**
|
||||
* Choose Turbo snapshot ids to delete. `keepKey` is protected even when it is
|
||||
* older than the retained generations. Keys outside the prefix abort retention.
|
||||
*/
|
||||
export function selectPrefixDeletions(
|
||||
caches,
|
||||
{ prefix, keepKey } = {},
|
||||
keepGenerations = RETAINED_CACHE_GENERATIONS,
|
||||
) {
|
||||
assertRetentionPrefix(prefix);
|
||||
if (!Number.isSafeInteger(keepGenerations) || keepGenerations < 1)
|
||||
throw new Error("Cache retention must keep at least one generation");
|
||||
if (keepKey !== undefined) {
|
||||
if (
|
||||
typeof keepKey !== "string" ||
|
||||
!keepKey.startsWith(prefix) ||
|
||||
keepKey === prefix
|
||||
)
|
||||
throw new Error("Cache retention keep key must extend its prefix");
|
||||
}
|
||||
const entries = uniqueById(caches.map(normalizeCache));
|
||||
for (const entry of entries)
|
||||
if (!entry.key.startsWith(prefix))
|
||||
throw new Error(
|
||||
`Refusing to evaluate a cache outside ${prefix}: ${entry.key}`,
|
||||
);
|
||||
const protectedIds = new Set(
|
||||
[...entries]
|
||||
.sort(byNewest)
|
||||
.slice(0, keepGenerations)
|
||||
.map((entry) => entry.id),
|
||||
);
|
||||
if (keepKey !== undefined)
|
||||
for (const entry of entries)
|
||||
if (entry.key === keepKey) protectedIds.add(entry.id);
|
||||
return entries.filter((entry) => !protectedIds.has(entry.id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop extra copies of one npm restore target. Records that share a user key
|
||||
* but differ by cache version or git ref stay, because a workflow can restore
|
||||
* only a matching version inside its visible scope.
|
||||
*/
|
||||
export function selectNpmDownloadDuplicateDeletions(caches) {
|
||||
const entries = uniqueById(caches.map(normalizeCache));
|
||||
for (const entry of entries)
|
||||
if (!entry.key.startsWith("npm-download-"))
|
||||
throw new Error(
|
||||
`Refusing to evaluate a non-download cache: ${entry.key}`,
|
||||
);
|
||||
const groups = new Map();
|
||||
for (const entry of entries) {
|
||||
const identity = `${entry.version}\n${entry.ref}\n${entry.key}`;
|
||||
const group = groups.get(identity) ?? [];
|
||||
group.push(entry);
|
||||
groups.set(identity, group);
|
||||
}
|
||||
const deletions = [];
|
||||
for (const group of groups.values()) {
|
||||
if (group.length < 2) continue;
|
||||
const ranked = [...group].sort((left, right) => {
|
||||
if (left.accessedAt !== right.accessedAt)
|
||||
return right.accessedAt - left.accessedAt;
|
||||
if (left.createdAt !== right.createdAt)
|
||||
return right.createdAt - left.createdAt;
|
||||
return right.id - left.id;
|
||||
});
|
||||
deletions.push(...ranked.slice(1));
|
||||
}
|
||||
return deletions;
|
||||
}
|
||||
|
||||
export async function listAllCaches(request, repository, prefix) {
|
||||
const caches = [];
|
||||
for (let page = 1; page <= MAX_PAGES; page += 1) {
|
||||
const body = await request(
|
||||
"GET",
|
||||
`repos/${repository}/actions/caches?per_page=${String(PAGE_SIZE)}&page=${String(page)}&key=${encodeURIComponent(prefix)}`,
|
||||
);
|
||||
if (
|
||||
typeof body !== "object" ||
|
||||
body === null ||
|
||||
Array.isArray(body) ||
|
||||
!Array.isArray(body.actions_caches)
|
||||
)
|
||||
throw new Error("GitHub cache list is malformed");
|
||||
caches.push(...body.actions_caches);
|
||||
if (body.actions_caches.length < PAGE_SIZE) return caches;
|
||||
}
|
||||
throw new Error(
|
||||
`Cache list for ${prefix} exceeded ${String(MAX_PAGES)} pages; deleted nothing`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* List every requested family before deleting. A malformed list or an unsafe
|
||||
* prefix leaves the repository caches unchanged.
|
||||
*/
|
||||
export async function retainCaches({
|
||||
repository,
|
||||
request,
|
||||
deleteCache,
|
||||
families = [],
|
||||
dedupeNpmDownloads = false,
|
||||
dryRun = false,
|
||||
}) {
|
||||
if (
|
||||
typeof repository !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
|
||||
)
|
||||
throw new Error(
|
||||
"CI_REPOSITORY must be an owner/name repository identifier",
|
||||
);
|
||||
for (const family of families) {
|
||||
assertRetentionPrefix(family.prefix);
|
||||
if (
|
||||
family.keepKey !== undefined &&
|
||||
(typeof family.keepKey !== "string" ||
|
||||
!family.keepKey.startsWith(family.prefix) ||
|
||||
family.keepKey === family.prefix)
|
||||
)
|
||||
throw new Error("Cache retention keep key must extend its prefix");
|
||||
}
|
||||
const plan = [];
|
||||
for (const family of families) {
|
||||
const caches = await listAllCaches(request, repository, family.prefix);
|
||||
plan.push({
|
||||
prefix: family.prefix,
|
||||
deletions: selectPrefixDeletions(caches, family),
|
||||
});
|
||||
}
|
||||
const npmDeletions = dedupeNpmDownloads
|
||||
? selectNpmDownloadDuplicateDeletions(
|
||||
await listAllCaches(request, repository, "npm-download-"),
|
||||
)
|
||||
: [];
|
||||
if (!dryRun)
|
||||
for (const entry of [
|
||||
...plan.flatMap((family) => family.deletions),
|
||||
...npmDeletions,
|
||||
])
|
||||
await deleteCache(entry.id);
|
||||
return { dryRun, plan, npmDeletions };
|
||||
}
|
||||
|
||||
function repositoryFromEnv(env) {
|
||||
const repository = env.CI_REPOSITORY;
|
||||
if (
|
||||
typeof repository !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
|
||||
)
|
||||
throw new Error(
|
||||
"CI_REPOSITORY must be an owner/name repository identifier",
|
||||
);
|
||||
return repository;
|
||||
}
|
||||
|
||||
function familyWithKeepKey(env, prefixName, keepName) {
|
||||
const prefix = env[prefixName];
|
||||
const keepKey = env[keepName];
|
||||
const hasPrefix = typeof prefix === "string" && prefix.length > 0;
|
||||
const hasKeepKey = typeof keepKey === "string" && keepKey.length > 0;
|
||||
if (!hasPrefix && !hasKeepKey) return undefined;
|
||||
if (!hasPrefix || !hasKeepKey)
|
||||
throw new Error(`${keepName} is required with ${prefixName}`);
|
||||
return { prefix, keepKey };
|
||||
}
|
||||
|
||||
function familiesFromEnv(env) {
|
||||
const families = [
|
||||
familyWithKeepKey(env, "CACHE_PREFIX", "CACHE_KEEP_KEY"),
|
||||
familyWithKeepKey(env, "CACHE_DEEP_PREFIX", "CACHE_DEEP_KEEP_KEY"),
|
||||
familyWithKeepKey(env, "CACHE_KNIP_PREFIX", "CACHE_KNIP_KEEP_KEY"),
|
||||
typeof env.CACHE_PORTABLE_PREFIX === "string" &&
|
||||
env.CACHE_PORTABLE_PREFIX.length > 0
|
||||
? { prefix: env.CACHE_PORTABLE_PREFIX }
|
||||
: undefined,
|
||||
].filter((family) => family !== undefined);
|
||||
if (families.length === 0 && env.DEDUPE_NPM_DOWNLOADS !== "true")
|
||||
throw new Error(
|
||||
"Cache retention requires CACHE_PREFIX or CACHE_PORTABLE_PREFIX or npm dedupe",
|
||||
);
|
||||
return families;
|
||||
}
|
||||
|
||||
function failureText(error) {
|
||||
if (typeof error !== "object" || error === null) return String(error);
|
||||
const stderr =
|
||||
"stderr" in error && error.stderr !== undefined ? String(error.stderr) : "";
|
||||
const message = error instanceof Error ? error.message : "";
|
||||
return `${message}\n${stderr}`;
|
||||
}
|
||||
|
||||
async function githubRequest(method, endpoint, env) {
|
||||
const { stdout } = await execFileAsync(
|
||||
"gh",
|
||||
["api", "--method", method, endpoint],
|
||||
{
|
||||
env,
|
||||
encoding: "utf8",
|
||||
maxBuffer: MAX_BUFFER,
|
||||
timeout: REQUEST_TIMEOUT_MS,
|
||||
},
|
||||
);
|
||||
if (method === "DELETE") return undefined;
|
||||
try {
|
||||
return JSON.parse(stdout);
|
||||
} catch (cause) {
|
||||
throw new Error("GitHub cache API returned malformed JSON", { cause });
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const env = process.env;
|
||||
const dryRun = env.CACHE_RETENTION_DRY_RUN === "true";
|
||||
const report = await retainCaches({
|
||||
repository: repositoryFromEnv(env),
|
||||
request: (method, endpoint) => githubRequest(method, endpoint, env),
|
||||
deleteCache: async (id) => {
|
||||
try {
|
||||
await githubRequest(
|
||||
"DELETE",
|
||||
`repos/${repositoryFromEnv(env)}/actions/caches/${String(id)}`,
|
||||
env,
|
||||
);
|
||||
} catch (error) {
|
||||
if (/404|not found/iu.test(failureText(error))) return;
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
families: familiesFromEnv(env),
|
||||
dedupeNpmDownloads: env.DEDUPE_NPM_DOWNLOADS === "true",
|
||||
dryRun,
|
||||
});
|
||||
const summary = {
|
||||
dryRun: report.dryRun,
|
||||
families: report.plan.map((family) => ({
|
||||
prefix: family.prefix,
|
||||
deleteCount: family.deletions.length,
|
||||
deleteBytes: family.deletions.reduce((sum, entry) => sum + entry.size, 0),
|
||||
})),
|
||||
npmDuplicateDeleteCount: report.npmDeletions.length,
|
||||
npmDuplicateDeleteBytes: report.npmDeletions.reduce(
|
||||
(sum, entry) => sum + entry.size,
|
||||
0,
|
||||
),
|
||||
};
|
||||
process.stdout.write(`${JSON.stringify(summary)}\n`);
|
||||
}
|
||||
|
||||
function isDirectRun() {
|
||||
const entry = process.argv[1];
|
||||
if (!entry) return false;
|
||||
try {
|
||||
return import.meta.url === pathToFileURL(realpathSync(entry)).href;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (isDirectRun()) await main();
|
||||
@@ -104,12 +104,12 @@ cliTest.for(["CLI", "stdio MCP"] as const)(
|
||||
};
|
||||
let traceEvidence: unknown;
|
||||
if (surface === "CLI") {
|
||||
// The application Evidence document exceeds the runner argument limit
|
||||
// when it is placed on argv. The command reads the same JSON from a file.
|
||||
const traceInput = join(root, "trace-input.json");
|
||||
await writeFile(traceInput, JSON.stringify(traceArguments));
|
||||
const response = await cli.run({
|
||||
arguments: [
|
||||
"trace-application-feature",
|
||||
JSON.stringify(traceArguments),
|
||||
"--json",
|
||||
],
|
||||
arguments: ["trace-application-feature", traceInput, "--json"],
|
||||
});
|
||||
expect(response.exitCode, response.stderr).toBe(0);
|
||||
traceEvidence = response.json;
|
||||
|
||||
@@ -13,6 +13,12 @@ const actionPath = join(repository, ".github/actions/npm-cache/action.yml");
|
||||
const readAction = async () =>
|
||||
z
|
||||
.object({
|
||||
inputs: z
|
||||
.record(
|
||||
z.string(),
|
||||
z.object({ default: z.string().optional() }).passthrough(),
|
||||
)
|
||||
.optional(),
|
||||
outputs: z.record(z.string(), z.object({ value: z.string() })),
|
||||
runs: z.object({
|
||||
steps: z.array(
|
||||
@@ -42,9 +48,11 @@ const readWorkflows = async () => {
|
||||
jobs: z
|
||||
.record(
|
||||
z.string(),
|
||||
z.object({
|
||||
steps: z.array(z.record(z.string(), z.unknown())).optional(),
|
||||
}),
|
||||
z
|
||||
.object({
|
||||
steps: z.array(z.record(z.string(), z.unknown())).optional(),
|
||||
})
|
||||
.passthrough(),
|
||||
)
|
||||
.optional(),
|
||||
})
|
||||
@@ -107,6 +115,9 @@ printf '%s\n' 'C:\Users\runner with spaces\AppData\Local\npm-cache'
|
||||
expect(output).toContain(
|
||||
`key=npm-download-Windows-${process.arch}-lock-hash\n`,
|
||||
);
|
||||
expect(output).toContain(
|
||||
`restore-key=npm-download-Windows-${process.arch}-\n`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -120,6 +131,7 @@ printf '%s\n' 'C:\Users\runner with spaces\AppData\Local\npm-cache'
|
||||
step.uses?.startsWith("actions/cache/restore@"),
|
||||
);
|
||||
|
||||
expect(action.inputs?.save?.default).toBe("false");
|
||||
expect(action.outputs["can-save"]?.value).toBe(
|
||||
"${{ steps.context.outputs.can-save }}",
|
||||
);
|
||||
@@ -138,19 +150,27 @@ printf '%s\n' 'C:\Users\runner with spaces\AppData\Local\npm-cache'
|
||||
expect(restoreAndSave?.uses).toBe(
|
||||
"actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830",
|
||||
);
|
||||
expect(restoreAndSave?.if).toBe("steps.context.outputs.can-save == 'true'");
|
||||
expect(restoreAndSave?.if).toBe(
|
||||
"steps.context.outputs.can-save == 'true' && inputs.save == 'true'",
|
||||
);
|
||||
expect(restoreOnly?.uses).toBe(
|
||||
"actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830",
|
||||
);
|
||||
expect(restoreOnly?.if).toBe("steps.context.outputs.can-save != 'true'");
|
||||
expect(restoreOnly?.if).toBe(
|
||||
"steps.context.outputs.can-save != 'true' || inputs.save != 'true'",
|
||||
);
|
||||
expect(restoreAndSave?.with?.path).toBe(
|
||||
"${{ steps.context.outputs.path }}",
|
||||
);
|
||||
expect(restoreOnly?.with?.path).toBe(restoreAndSave?.with?.path);
|
||||
expect(restoreAndSave?.with?.key).toBe("${{ steps.context.outputs.key }}");
|
||||
expect(restoreOnly?.with?.key).toBe(restoreAndSave?.with?.key);
|
||||
expect(restoreAndSave?.with).not.toHaveProperty("restore-keys");
|
||||
expect(restoreOnly?.with).not.toHaveProperty("restore-keys");
|
||||
expect(restoreAndSave?.with?.["restore-keys"]).toBe(
|
||||
"${{ steps.context.outputs.restore-key }}",
|
||||
);
|
||||
expect(restoreOnly?.with?.["restore-keys"]).toBe(
|
||||
restoreAndSave?.with?.["restore-keys"],
|
||||
);
|
||||
});
|
||||
|
||||
it("replaces every inline setup-node npm cache immediately and preserves conditions", async () => {
|
||||
@@ -258,6 +278,153 @@ it("shares static task snapshots only after validation using the npm cache owner
|
||||
path: "node_modules/.cache/knip",
|
||||
key: "${{ steps.knip-cache.outputs.cache-primary-key }}",
|
||||
});
|
||||
expect(
|
||||
steps.find((step) => step.run === "node scripts/ci/cache-retention.mjs"),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("deletes superseded caches only from the trusted follow-up workflow", async () => {
|
||||
const workflow = z
|
||||
.object({
|
||||
on: z.object({
|
||||
workflow_run: z.object({
|
||||
workflows: z.array(z.literal("CI")),
|
||||
types: z.array(z.literal("completed")),
|
||||
}),
|
||||
workflow_call: z.unknown().optional(),
|
||||
pull_request: z.unknown().optional(),
|
||||
}),
|
||||
permissions: z.record(z.string(), z.string()),
|
||||
jobs: z.record(
|
||||
z.string(),
|
||||
z.object({
|
||||
if: z.string(),
|
||||
"runs-on": z.literal("ubuntu-latest"),
|
||||
steps: z.array(z.record(z.string(), z.unknown())),
|
||||
}),
|
||||
),
|
||||
})
|
||||
.parse(
|
||||
parse(
|
||||
await readFile(
|
||||
join(repository, ".github/workflows/cache-retention.yml"),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(workflow.on.workflow_call).toBeUndefined();
|
||||
expect(workflow.on.pull_request).toBeUndefined();
|
||||
expect(workflow.permissions).toEqual({
|
||||
contents: "read",
|
||||
actions: "write",
|
||||
});
|
||||
const job = workflow.jobs.retain;
|
||||
expect(job?.if).toContain(
|
||||
"github.event.workflow_run.conclusion == 'success'",
|
||||
);
|
||||
expect(job?.if).toContain(
|
||||
"github.event.workflow_run.conclusion == 'failure'",
|
||||
);
|
||||
expect(job?.if).toContain(
|
||||
"github.event.workflow_run.conclusion == 'timed_out'",
|
||||
);
|
||||
expect(job?.if).toContain(
|
||||
"github.event.workflow_run.conclusion == 'cancelled'",
|
||||
);
|
||||
expect(job?.if).toContain(
|
||||
"github.event.workflow_run.head_repository.full_name == github.repository",
|
||||
);
|
||||
expect(job?.if).toContain(
|
||||
"github.event.workflow_run.head_branch == github.event.repository.default_branch",
|
||||
);
|
||||
expect(job?.if).toContain("github.event.workflow_run.event == 'push'");
|
||||
expect(job?.if).toContain("github.event.workflow_run.event == 'schedule'");
|
||||
expect(job?.if).toContain(
|
||||
"github.event.workflow_run.event == 'workflow_dispatch'",
|
||||
);
|
||||
expect(job?.if).not.toContain("pull_request");
|
||||
const steps = job?.steps ?? [];
|
||||
expect(steps.some((step) => step.run === "npm ci")).toBe(false);
|
||||
const checkout = steps[0];
|
||||
expect(checkout?.uses).toBe(
|
||||
"actions/checkout@11d5960a326750d5838078e36cf38b85af677262",
|
||||
);
|
||||
expect(checkout?.with).toMatchObject({
|
||||
ref: "${{ github.event.workflow_run.head_sha }}",
|
||||
"persist-credentials": false,
|
||||
});
|
||||
const retention = steps.find(
|
||||
(step) => step.run === "node scripts/ci/cache-retention.mjs",
|
||||
);
|
||||
const retentionEnv = z.record(z.string(), z.string()).parse(retention?.env);
|
||||
expect(retentionEnv.CACHE_PREFIX).toBe(
|
||||
"rea-static-v1-${{ runner.os }}-${{ runner.arch }}-",
|
||||
);
|
||||
expect(retentionEnv.CACHE_KEEP_KEY).toBe(
|
||||
"rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}",
|
||||
);
|
||||
expect(retentionEnv.CACHE_PORTABLE_PREFIX).toBe(
|
||||
"rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-",
|
||||
);
|
||||
expect(retentionEnv.DEDUPE_NPM_DOWNLOADS).toBe("true");
|
||||
expect(retentionEnv.GH_TOKEN).toBe("${{ github.token }}");
|
||||
const ci = (await readWorkflows()).find(({ file }) => file === "ci.yml");
|
||||
expect(JSON.stringify(ci?.workflow)).not.toContain('"actions":"write"');
|
||||
const deepSteps = ci?.workflow.jobs?.["static-deep"]?.steps ?? [];
|
||||
const publishedKey = (id: string) =>
|
||||
z
|
||||
.string()
|
||||
.parse(
|
||||
z
|
||||
.record(z.string(), z.unknown())
|
||||
.parse(deepSteps.find((step) => step.id === id)?.with).key,
|
||||
);
|
||||
const savedOnThisRun = "${{ github.event.workflow_run.head_sha }}";
|
||||
const deepPrefix = "rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-";
|
||||
const knipPrefix = "knip-meta-v1-${{ runner.os }}-";
|
||||
expect(retentionEnv.CACHE_DEEP_PREFIX).toBe(deepPrefix);
|
||||
expect(retentionEnv.CACHE_DEEP_KEEP_KEY).toBe(
|
||||
publishedKey("static-cache").replaceAll(
|
||||
"${{ github.sha }}",
|
||||
savedOnThisRun,
|
||||
),
|
||||
);
|
||||
expect(publishedKey("static-cache").startsWith(deepPrefix)).toBe(true);
|
||||
expect(retentionEnv.CACHE_KNIP_PREFIX).toBe(knipPrefix);
|
||||
expect(retentionEnv.CACHE_KNIP_KEEP_KEY).toBe(
|
||||
publishedKey("knip-cache").replaceAll("${{ github.sha }}", savedOnThisRun),
|
||||
);
|
||||
expect(publishedKey("knip-cache").startsWith(knipPrefix)).toBe(true);
|
||||
});
|
||||
|
||||
it("publishes each hosted npm download cache from one writer", async () => {
|
||||
const workflows = await readWorkflows();
|
||||
const writers: { file: string; job: string; save: string }[] = [];
|
||||
for (const { file, workflow } of workflows)
|
||||
for (const [job, definition] of Object.entries(workflow.jobs ?? {}))
|
||||
for (const step of definition.steps ?? []) {
|
||||
if (step.uses !== "./.github/actions/npm-cache") continue;
|
||||
const save = z
|
||||
.record(z.string(), z.unknown())
|
||||
.parse(step.with ?? {}).save;
|
||||
if (save === undefined) continue;
|
||||
writers.push({ file, job, save: z.string().parse(save) });
|
||||
}
|
||||
expect(writers).toEqual([
|
||||
{ file: "ci.yml", job: "static", save: "true" },
|
||||
{
|
||||
file: "ci.yml",
|
||||
job: "package-e2e",
|
||||
save: "${{ matrix.os != 'ubuntu-latest' }}",
|
||||
},
|
||||
{ file: "ci.yml", job: "windows-curated", save: "true" },
|
||||
]);
|
||||
const ci = workflows.find(({ file }) => file === "ci.yml");
|
||||
expect(ci?.workflow.jobs?.static?.permissions).toBeUndefined();
|
||||
expect(ci?.workflow.jobs?.build?.permissions).toEqual({
|
||||
contents: "read",
|
||||
actions: "read",
|
||||
});
|
||||
});
|
||||
|
||||
it("passes the npm cache owner's selected path to package installation", async () => {
|
||||
|
||||
@@ -0,0 +1,464 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { chmod, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
listAllCaches,
|
||||
retainCaches,
|
||||
selectNpmDownloadDuplicateDeletions,
|
||||
selectPrefixDeletions,
|
||||
} from "../../../scripts/ci/cache-retention.mjs";
|
||||
|
||||
const cache = (
|
||||
id: number,
|
||||
key: string,
|
||||
created: string,
|
||||
accessed = created,
|
||||
size = 100,
|
||||
version = "a".repeat(64),
|
||||
ref = "refs/heads/main",
|
||||
) => ({
|
||||
id,
|
||||
key,
|
||||
version,
|
||||
ref,
|
||||
created_at: created,
|
||||
last_accessed_at: accessed,
|
||||
size_in_bytes: size,
|
||||
});
|
||||
|
||||
describe("Turbo snapshot retention", () => {
|
||||
const prefix = "rea-static-v1-Linux-X64-";
|
||||
|
||||
it("keeps the two newest snapshots and the saved key", () => {
|
||||
const keepKey = `${prefix}saved`;
|
||||
const deletions = selectPrefixDeletions(
|
||||
[
|
||||
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z", undefined, 10),
|
||||
cache(2, `${prefix}older`, "2026-10-20T02:00:00Z", undefined, 20),
|
||||
cache(3, `${prefix}recent`, "2026-10-20T03:00:00Z", undefined, 30),
|
||||
cache(4, keepKey, "2026-10-20T00:00:00Z", undefined, 40),
|
||||
],
|
||||
{ prefix, keepKey },
|
||||
);
|
||||
expect(deletions.map((entry) => entry.id)).toEqual([1]);
|
||||
expect(deletions.reduce((sum, entry) => sum + entry.size, 0)).toBe(10);
|
||||
});
|
||||
|
||||
it("keeps an in-flight generation when the saved key is already newest", () => {
|
||||
const keepKey = `${prefix}newest`;
|
||||
const deletions = selectPrefixDeletions(
|
||||
[
|
||||
cache(1, `${prefix}a`, "2026-10-20T01:00:00Z"),
|
||||
cache(2, `${prefix}b`, "2026-10-20T02:00:00Z"),
|
||||
cache(3, `${prefix}c`, "2026-10-20T03:00:00Z"),
|
||||
cache(4, keepKey, "2026-10-20T04:00:00Z"),
|
||||
],
|
||||
{ prefix, keepKey },
|
||||
);
|
||||
expect(
|
||||
deletions.map((entry) => entry.id).sort((left, right) => left - right),
|
||||
).toEqual([1, 2]);
|
||||
});
|
||||
|
||||
it("keeps two generations of the deep and Knip snapshot families", () => {
|
||||
const deep = "rea-static-deep-v1-Linux-X64-";
|
||||
const knip = "knip-meta-v1-Linux-";
|
||||
expect(
|
||||
selectPrefixDeletions(
|
||||
[
|
||||
cache(1, `${deep}old`, "2026-10-20T01:00:00Z"),
|
||||
cache(2, `${deep}mid`, "2026-10-20T02:00:00Z"),
|
||||
cache(3, `${deep}new`, "2026-10-20T03:00:00Z"),
|
||||
],
|
||||
{ prefix: deep, keepKey: `${deep}new` },
|
||||
).map((entry) => entry.id),
|
||||
).toEqual([1]);
|
||||
expect(
|
||||
selectPrefixDeletions(
|
||||
[
|
||||
cache(4, `${knip}old`, "2026-10-20T01:00:00Z"),
|
||||
cache(5, `${knip}mid`, "2026-10-20T02:00:00Z"),
|
||||
cache(6, `${knip}new`, "2026-10-20T03:00:00Z"),
|
||||
],
|
||||
{ prefix: knip, keepKey: `${knip}new` },
|
||||
).map((entry) => entry.id),
|
||||
).toEqual([4]);
|
||||
});
|
||||
|
||||
it("retains the newest portable snapshots without a caller keep key", () => {
|
||||
const portable = "rea-portable-build-v2-Linux-X64-";
|
||||
const deletions = selectPrefixDeletions(
|
||||
[
|
||||
cache(7, `${portable}one`, "2026-10-20T01:00:00Z"),
|
||||
cache(8, `${portable}two`, "2026-10-20T02:00:00Z"),
|
||||
cache(9, `${portable}three`, "2026-10-20T03:00:00Z"),
|
||||
],
|
||||
{ prefix: portable },
|
||||
);
|
||||
expect(deletions.map((entry) => entry.id)).toEqual([7]);
|
||||
});
|
||||
|
||||
it("refuses prefixes and lists that can reach download caches", () => {
|
||||
expect(() =>
|
||||
selectPrefixDeletions([], { prefix: "npm-download-Linux-x64-" }),
|
||||
).toThrow(/one Turbo family/u);
|
||||
expect(() =>
|
||||
selectPrefixDeletions([], { prefix: "rea-static-v1-" }),
|
||||
).toThrow(/one Turbo family/u);
|
||||
expect(() =>
|
||||
selectPrefixDeletions([], { prefix: "knip-meta-v1-" }),
|
||||
).toThrow(/one Turbo family/u);
|
||||
expect(() =>
|
||||
selectPrefixDeletions(
|
||||
[cache(1, "npm-download-Linux-x64-abc", "2026-10-20T01:00:00Z")],
|
||||
{
|
||||
prefix,
|
||||
keepKey: `${prefix}saved`,
|
||||
},
|
||||
),
|
||||
).toThrow(/outside/u);
|
||||
expect(() =>
|
||||
selectPrefixDeletions([], {
|
||||
prefix,
|
||||
keepKey: "rea-static-v1-Windows-X64-other",
|
||||
}),
|
||||
).toThrow(/keep key/u);
|
||||
});
|
||||
});
|
||||
|
||||
describe("npm download duplicate retention", () => {
|
||||
it("deletes only an older copy of the same key, version, and ref", () => {
|
||||
const key = "npm-download-Linux-x64-lock";
|
||||
const version = "a".repeat(64);
|
||||
const ref = "refs/heads/main";
|
||||
const deletions = selectNpmDownloadDuplicateDeletions([
|
||||
cache(
|
||||
1,
|
||||
key,
|
||||
"2026-10-20T01:00:00Z",
|
||||
"2026-10-20T02:00:00Z",
|
||||
179,
|
||||
version,
|
||||
ref,
|
||||
),
|
||||
cache(
|
||||
2,
|
||||
key,
|
||||
"2026-10-20T01:05:00Z",
|
||||
"2026-10-20T03:00:00Z",
|
||||
199,
|
||||
version,
|
||||
ref,
|
||||
),
|
||||
cache(
|
||||
3,
|
||||
"npm-download-Windows-x64-lock",
|
||||
"2026-10-20T01:00:00Z",
|
||||
undefined,
|
||||
186,
|
||||
),
|
||||
]);
|
||||
expect(deletions.map((entry) => entry.id)).toEqual([1]);
|
||||
});
|
||||
|
||||
it("keeps records that share a key but differ by version or ref", () => {
|
||||
const key = "npm-download-Linux-x64-lock";
|
||||
const version = "a".repeat(64);
|
||||
const otherVersion = "b".repeat(64);
|
||||
const deletions = selectNpmDownloadDuplicateDeletions([
|
||||
cache(
|
||||
1,
|
||||
key,
|
||||
"2026-10-20T01:00:00Z",
|
||||
"2026-10-20T04:00:00Z",
|
||||
199,
|
||||
version,
|
||||
"refs/heads/main",
|
||||
),
|
||||
cache(
|
||||
2,
|
||||
key,
|
||||
"2026-10-20T01:00:00Z",
|
||||
"2026-10-20T03:00:00Z",
|
||||
188,
|
||||
otherVersion,
|
||||
"refs/heads/main",
|
||||
),
|
||||
cache(
|
||||
3,
|
||||
key,
|
||||
"2026-10-20T01:00:00Z",
|
||||
"2026-10-20T02:00:00Z",
|
||||
180,
|
||||
version,
|
||||
"refs/heads/feature",
|
||||
),
|
||||
]);
|
||||
expect(deletions).toEqual([]);
|
||||
});
|
||||
|
||||
it("refuses a download record without a version or ref", () => {
|
||||
const key = "npm-download-Linux-x64-lock";
|
||||
const { version: _version, ...withoutVersion } = cache(
|
||||
1,
|
||||
key,
|
||||
"2026-10-20T01:00:00Z",
|
||||
);
|
||||
expect(() => selectNpmDownloadDuplicateDeletions([withoutVersion])).toThrow(
|
||||
/version/u,
|
||||
);
|
||||
const { ref: _ref, ...withoutRef } = cache(2, key, "2026-10-20T01:00:00Z");
|
||||
expect(() => selectNpmDownloadDuplicateDeletions([withoutRef])).toThrow(
|
||||
/ref/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("does not delete the only copy when pagination repeats its id", () => {
|
||||
const key = "npm-download-Linux-x64-lock";
|
||||
const version = "a".repeat(64);
|
||||
const ref = "refs/heads/main";
|
||||
const deletions = selectNpmDownloadDuplicateDeletions([
|
||||
cache(
|
||||
5,
|
||||
key,
|
||||
"2026-10-20T01:00:00Z",
|
||||
"2026-10-20T02:00:00Z",
|
||||
179,
|
||||
version,
|
||||
ref,
|
||||
),
|
||||
cache(
|
||||
5,
|
||||
key,
|
||||
"2026-10-20T01:00:00Z",
|
||||
"2026-10-20T04:00:00Z",
|
||||
179,
|
||||
version,
|
||||
ref,
|
||||
),
|
||||
]);
|
||||
expect(deletions).toEqual([]);
|
||||
});
|
||||
|
||||
it("aborts when the same id is listed with a different version", () => {
|
||||
const key = "npm-download-Linux-x64-lock";
|
||||
expect(() =>
|
||||
selectNpmDownloadDuplicateDeletions([
|
||||
cache(5, key, "2026-10-20T01:00:00Z", undefined, 179, "a".repeat(64)),
|
||||
cache(5, key, "2026-10-20T01:00:00Z", undefined, 179, "b".repeat(64)),
|
||||
]),
|
||||
).toThrow(/inconsistent/u);
|
||||
});
|
||||
|
||||
it("refuses a Turbo snapshot in the download list", () => {
|
||||
expect(() =>
|
||||
selectNpmDownloadDuplicateDeletions([
|
||||
cache(1, "rea-static-v1-Linux-X64-sha", "2026-10-20T01:00:00Z"),
|
||||
]),
|
||||
).toThrow(/non-download/u);
|
||||
});
|
||||
});
|
||||
|
||||
describe("retention orchestration", () => {
|
||||
const repository = "morluto/rea";
|
||||
const prefix = "rea-static-v1-Linux-X64-";
|
||||
|
||||
it("deletes nothing when a later list is incomplete", async () => {
|
||||
const deleted: number[] = [];
|
||||
const page = Array.from({ length: 100 }, (_, index) =>
|
||||
cache(index + 1, `${prefix}${String(index)}`, "2026-10-20T01:00:00Z"),
|
||||
);
|
||||
await expect(
|
||||
retainCaches({
|
||||
repository,
|
||||
request: async (_method, endpoint) => {
|
||||
if (endpoint.includes("npm-download-"))
|
||||
throw new Error("list failed after the first family");
|
||||
return { actions_caches: page };
|
||||
},
|
||||
deleteCache: async (id) => {
|
||||
deleted.push(id);
|
||||
},
|
||||
families: [{ prefix, keepKey: `${prefix}0` }],
|
||||
dedupeNpmDownloads: true,
|
||||
}),
|
||||
).rejects.toThrow(/list failed|exceeded/u);
|
||||
expect(deleted).toEqual([]);
|
||||
});
|
||||
|
||||
it("stops listing once a short page proves the inventory is complete", async () => {
|
||||
const requests: string[] = [];
|
||||
const caches = await listAllCaches(
|
||||
async (_method, endpoint) => {
|
||||
requests.push(endpoint);
|
||||
const page = new URL(endpoint, "https://example.test").searchParams.get(
|
||||
"page",
|
||||
);
|
||||
return {
|
||||
actions_caches:
|
||||
page === "1"
|
||||
? Array.from({ length: 100 }, (_, index) =>
|
||||
cache(
|
||||
index + 1,
|
||||
`${prefix}${String(index)}`,
|
||||
"2026-10-20T01:00:00Z",
|
||||
),
|
||||
)
|
||||
: [cache(101, `${prefix}last`, "2026-10-20T02:00:00Z")],
|
||||
};
|
||||
},
|
||||
repository,
|
||||
prefix,
|
||||
);
|
||||
expect(caches).toHaveLength(101);
|
||||
expect(requests).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("applies every family only after both lists succeed", async () => {
|
||||
const deleted: number[] = [];
|
||||
const keepKey = `${prefix}saved`;
|
||||
const portable = "rea-portable-build-v2-Linux-X64-";
|
||||
const result = await retainCaches({
|
||||
repository,
|
||||
request: async (_method, endpoint) => {
|
||||
const key = new URL(endpoint, "https://example.test").searchParams.get(
|
||||
"key",
|
||||
);
|
||||
if (key === prefix)
|
||||
return {
|
||||
actions_caches: [
|
||||
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z"),
|
||||
cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z"),
|
||||
cache(3, keepKey, "2026-10-20T03:00:00Z"),
|
||||
],
|
||||
};
|
||||
if (key === portable)
|
||||
return {
|
||||
actions_caches: [
|
||||
cache(4, `${portable}old`, "2026-10-20T01:00:00Z"),
|
||||
cache(5, `${portable}new`, "2026-10-20T02:00:00Z"),
|
||||
],
|
||||
};
|
||||
return {
|
||||
actions_caches: [
|
||||
cache(
|
||||
6,
|
||||
"npm-download-Linux-x64-lock",
|
||||
"2026-10-20T01:00:00Z",
|
||||
"2026-10-20T01:00:00Z",
|
||||
),
|
||||
cache(
|
||||
7,
|
||||
"npm-download-Linux-x64-lock",
|
||||
"2026-10-20T02:00:00Z",
|
||||
"2026-10-20T03:00:00Z",
|
||||
),
|
||||
],
|
||||
};
|
||||
},
|
||||
deleteCache: async (id) => {
|
||||
deleted.push(id);
|
||||
},
|
||||
families: [{ prefix, keepKey }, { prefix: portable }],
|
||||
dedupeNpmDownloads: true,
|
||||
});
|
||||
expect(deleted).toEqual([1, 6]);
|
||||
expect(result.plan[0]?.deletions.map((entry) => entry.id)).toEqual([1]);
|
||||
expect(result.plan[1]?.deletions).toEqual([]);
|
||||
expect(result.npmDeletions.map((entry) => entry.id)).toEqual([6]);
|
||||
});
|
||||
|
||||
it("dry-run reports the same deletions without calling delete", async () => {
|
||||
const result = await retainCaches({
|
||||
repository,
|
||||
request: async () => ({
|
||||
actions_caches: [
|
||||
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z", undefined, 50),
|
||||
cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z", undefined, 60),
|
||||
cache(3, `${prefix}new`, "2026-10-20T03:00:00Z", undefined, 70),
|
||||
],
|
||||
}),
|
||||
deleteCache: async () => {
|
||||
throw new Error("dry-run must not delete");
|
||||
},
|
||||
families: [{ prefix, keepKey: `${prefix}new` }],
|
||||
dryRun: true,
|
||||
});
|
||||
expect(result.dryRun).toBe(true);
|
||||
expect(result.plan[0]?.deletions.map((entry) => entry.id)).toEqual([1]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("repeated cache ids", () => {
|
||||
const repository = "morluto/rea";
|
||||
const prefix = "rea-static-v1-Linux-X64-";
|
||||
|
||||
it("treats one cache id on two pages as a single snapshot", () => {
|
||||
const keepKey = `${prefix}saved`;
|
||||
const deletions = selectPrefixDeletions(
|
||||
[
|
||||
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z"),
|
||||
cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z"),
|
||||
cache(3, keepKey, "2026-10-20T03:00:00Z", "2026-10-20T03:00:00Z"),
|
||||
cache(3, keepKey, "2026-10-20T03:00:00Z", "2026-10-20T04:00:00Z"),
|
||||
],
|
||||
{ prefix, keepKey },
|
||||
);
|
||||
expect(deletions.map((entry) => entry.id)).toEqual([1]);
|
||||
});
|
||||
|
||||
it("does not delete when one cache id is listed with inconsistent fields", async () => {
|
||||
const deleted: number[] = [];
|
||||
const repeated = cache(3, `${prefix}new`, "2026-10-20T03:00:00Z");
|
||||
await expect(
|
||||
retainCaches({
|
||||
repository,
|
||||
request: async () => ({
|
||||
actions_caches: [
|
||||
cache(1, `${prefix}old`, "2026-10-20T01:00:00Z"),
|
||||
cache(2, `${prefix}mid`, "2026-10-20T02:00:00Z"),
|
||||
repeated,
|
||||
{ ...repeated, version: "b".repeat(64) },
|
||||
],
|
||||
}),
|
||||
deleteCache: async (id) => {
|
||||
deleted.push(id);
|
||||
},
|
||||
families: [{ prefix, keepKey: `${prefix}new` }],
|
||||
}),
|
||||
).rejects.toThrow(/inconsistent/u);
|
||||
expect(deleted).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
it("runs retention from the command line without deleting on an unsafe prefix", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-cache-retention-"));
|
||||
const log = join(root, "gh.log");
|
||||
const gh = join(root, "gh");
|
||||
await writeFile(
|
||||
gh,
|
||||
`#!/bin/sh
|
||||
printf '%s\\n' "$@" >> "${log}"
|
||||
exit 1
|
||||
`,
|
||||
);
|
||||
await chmod(gh, 0o755);
|
||||
expect(() =>
|
||||
execFileSync("node", ["scripts/ci/cache-retention.mjs"], {
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${root}${delimiter}${process.env.PATH ?? ""}`,
|
||||
CI_REPOSITORY: "morluto/rea",
|
||||
CACHE_PREFIX: "npm-download-Linux-x64-",
|
||||
CACHE_KEEP_KEY: "npm-download-Linux-x64-lock",
|
||||
},
|
||||
encoding: "utf8",
|
||||
}),
|
||||
).toThrow(/one Turbo family/u);
|
||||
await expect(readFile(log, "utf8")).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
Reference in New Issue
Block a user