mirror of https://github.com/astral-sh/uv
Refactor Docker image builds to use Depot (#13459)
## Summary Simplify the Docker image build process to leverage Depot container builders for faster layer caching and native multi-platform image builds. The combo of the two removes the need to save cache to registries and do complex merge operations across GHA runners to get a multi-platform image. ## Test Plan UV team will need to add a trust relationship in Depot so that the container builds can authenticate and run. This can be done following these docs: https://depot.dev/docs/cli/authentication#adding-a-trust-relationship-for-github-actions. Once that is done, this should just work as before, but without all of the extra work around manifests. We should double that all of the tagging still makes sense for you all, as some bits of that were unclear. Additional context in this draft PR: https://github.com/astral-sh/uv/pull/9156 --------- Co-authored-by: Zanie Blue <contact@zanie.dev>
This commit is contained in:
Kyle Galbraith
GitHub
parent
423cfaabf5
commit
d73d3e8b53
|
|
@ -1,11 +1,19 @@
|
|||
# Build and publish a Docker image.
|
||||
# Build and publish Docker images.
|
||||
#
|
||||
# Assumed to run as a subworkflow of .github/workflows/release.yml; specifically, as a local
|
||||
# artifacts job within `cargo-dist`.
|
||||
# Uses Depot for multi-platform builds. Includes both a `uv` base image, which
|
||||
# is just the binary in a scratch image, and a set of extra, common images with
|
||||
# the uv binary installed.
|
||||
#
|
||||
# TODO(charlie): Ideally, the publish step would happen as a publish job within `cargo-dist`, but
|
||||
# sharing the built image as an artifact between jobs is challenging.
|
||||
name: "Build Docker image"
|
||||
# Images are built on all runs.
|
||||
#
|
||||
# On release, assumed to run as a subworkflow of .github/workflows/release.yml;
|
||||
# specifically, as a local artifacts job within `cargo-dist`. In this case,
|
||||
# images are published based on the `plan`.
|
||||
#
|
||||
# TODO(charlie): Ideally, the publish step would happen as a publish job within
|
||||
# `cargo-dist`, but sharing the built image as an artifact between jobs is
|
||||
# challenging.
|
||||
name: "Docker images"
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
|
|
@ -32,18 +40,19 @@ env:
|
|||
UV_BASE_IMG: ghcr.io/${{ github.repository_owner }}/uv
|
||||
|
||||
jobs:
|
||||
docker-build:
|
||||
docker-publish-base:
|
||||
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }}
|
||||
name: Build Docker image (ghcr.io/astral-sh/uv) for ${{ matrix.platform }}
|
||||
name: uv
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # for Depot OIDC
|
||||
packages: write # for GHCR
|
||||
environment:
|
||||
name: release
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
platform:
|
||||
- linux/amd64
|
||||
- linux/arm64
|
||||
outputs:
|
||||
image-tags: ${{ steps.meta.outputs.tags }}
|
||||
image-digest: ${{ steps.build.outputs.digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
|
|
@ -57,14 +66,14 @@ jobs:
|
|||
username: astralshbot
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN_RO }}
|
||||
|
||||
- uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
|
||||
|
||||
- uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5
|
||||
|
||||
- name: Check tag consistency
|
||||
if: ${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
run: |
|
||||
|
|
@ -87,96 +96,33 @@ jobs:
|
|||
tags: |
|
||||
type=raw,value=dry-run,enable=${{ inputs.plan == '' || fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
type=pep440,pattern={{ version }},value=${{ inputs.plan != '' && fromJson(inputs.plan).announcement_tag || 'dry-run' }},enable=${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
type=pep440,pattern={{ major }}.{{ minor }},value=${{ inputs.plan != '' && fromJson(inputs.plan).announcement_tag || 'dry-run' }},enable=${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
|
||||
- name: Normalize Platform Pair (replace / with -)
|
||||
run: |
|
||||
platform=${{ matrix.platform }}
|
||||
echo "PLATFORM_TUPLE=${platform//\//-}" >> $GITHUB_ENV
|
||||
|
||||
# Adapted from https://docs.docker.com/build/ci/github-actions/multi-platform/
|
||||
- name: Build and push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
|
||||
with:
|
||||
project: 7hd4vdzmw5 # astral-sh/uv
|
||||
context: .
|
||||
platforms: ${{ matrix.platform }}
|
||||
cache-from: type=gha,scope=uv-${{ env.PLATFORM_TUPLE }}
|
||||
cache-to: type=gha,mode=min,scope=uv-${{ env.PLATFORM_TUPLE }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: ${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
outputs: type=image,name=${{ env.UV_BASE_IMG }},push-by-digest=true,name-canonical=true,push=${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
|
||||
- name: Export digests
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Upload digests
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
- name: Generate artifact attestation for base image
|
||||
if: ${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
uses: actions/attest-build-provenance@c074443f1aee8d4aeeae555aebba3282517141b2 # v2.2.3
|
||||
with:
|
||||
name: digests-${{ env.PLATFORM_TUPLE }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
docker-publish:
|
||||
name: Publish Docker image (ghcr.io/astral-sh/uv)
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: release
|
||||
needs:
|
||||
- docker-build
|
||||
if: ${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
steps:
|
||||
# Login to DockerHub first, to avoid rate-limiting
|
||||
- uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
username: astralshbot
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN_RO }}
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
|
||||
- uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
|
||||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
|
||||
with:
|
||||
images: ${{ env.UV_BASE_IMG }}
|
||||
# Order is on purpose such that the label org.opencontainers.image.version has the first pattern with the full version
|
||||
tags: |
|
||||
type=pep440,pattern={{ version }},value=${{ fromJson(inputs.plan).announcement_tag }}
|
||||
type=pep440,pattern={{ major }}.{{ minor }},value=${{ fromJson(inputs.plan).announcement_tag }}
|
||||
|
||||
- uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Adapted from https://docs.docker.com/build/ci/github-actions/multi-platform/
|
||||
- name: Create manifest list and push
|
||||
working-directory: /tmp/digests
|
||||
# The jq command expands the docker/metadata json "tags" array entry to `-t tag1 -t tag2 ...` for each tag in the array
|
||||
# The printf will expand the base image with the `<UV_BASE_IMG>@sha256:<sha256> ...` for each sha256 in the directory
|
||||
# The final command becomes `docker buildx imagetools create -t tag1 -t tag2 ... <UV_BASE_IMG>@sha256:<sha256_1> <UV_BASE_IMG>@sha256:<sha256_2> ...`
|
||||
run: |
|
||||
docker buildx imagetools create \
|
||||
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ env.UV_BASE_IMG }}@sha256:%s ' *)
|
||||
subject-name: ${{ env.UV_BASE_IMG }}
|
||||
subject-digest: ${{ steps.build.outputs.digest }}
|
||||
|
||||
docker-publish-extra:
|
||||
name: Publish additional Docker image based on ${{ matrix.image-mapping }}
|
||||
name: ${{ matrix.image-mapping }}
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: release
|
||||
needs:
|
||||
- docker-publish
|
||||
- docker-publish-base
|
||||
if: ${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
permissions:
|
||||
packages: write
|
||||
|
|
@ -215,18 +161,19 @@ jobs:
|
|||
steps:
|
||||
# Login to DockerHub first, to avoid rate-limiting
|
||||
- uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
if: ${{ github.event.pull_request.head.repo.full_name == 'astral-sh/uv' }}
|
||||
with:
|
||||
username: astralshbot
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN_RO }}
|
||||
|
||||
- uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
|
||||
|
||||
- uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: depot/setup-action@b0b1ea4f69e92ebf5dea3f8713a1b0c37b2126a5
|
||||
|
||||
- name: Generate Dynamic Dockerfile Tags
|
||||
shell: bash
|
||||
run: |
|
||||
|
|
@ -257,9 +204,6 @@ jobs:
|
|||
# Remove the trailing newline from the pattern list
|
||||
TAG_PATTERNS="${TAG_PATTERNS%\\n}"
|
||||
|
||||
# Export image cache name
|
||||
echo "IMAGE_REF=${BASE_IMAGE//:/-}" >> $GITHUB_ENV
|
||||
|
||||
# Export tag patterns using the multiline env var syntax
|
||||
{
|
||||
echo "TAG_PATTERNS<<EOF"
|
||||
|
|
@ -282,13 +226,11 @@ jobs:
|
|||
|
||||
- name: Build and push
|
||||
id: build-and-push
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
uses: depot/build-push-action@636daae76684e38c301daa0c5eca1c095b24e780 # v1.14.0
|
||||
with:
|
||||
context: .
|
||||
project: 7hd4vdzmw5 # astral-sh/uv
|
||||
platforms: linux/amd64,linux/arm64
|
||||
# We do not really need to cache here as the Dockerfile is tiny
|
||||
#cache-from: type=gha,scope=uv-${{ env.IMAGE_REF }}
|
||||
#cache-to: type=gha,mode=min,scope=uv-${{ env.IMAGE_REF }}
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
|
@ -299,93 +241,31 @@ jobs:
|
|||
with:
|
||||
subject-name: ${{ env.UV_BASE_IMG }}
|
||||
subject-digest: ${{ steps.build-and-push.outputs.digest }}
|
||||
# push-to-registry is explicitly not enabled to maintain full control over the top image
|
||||
|
||||
# This is effectively a duplicate of `docker-publish` to make https://github.com/astral-sh/uv/pkgs/container/uv
|
||||
# show the uv base image first since GitHub always shows the last updated image digests
|
||||
# This works by annotating the original digests (previously non-annotated) which triggers an update to ghcr.io
|
||||
docker-republish:
|
||||
name: Annotate Docker image (ghcr.io/astral-sh/uv)
|
||||
# Re-tag the base image, to ensure it's shown as the newest on the registry UI
|
||||
docker-retag-base:
|
||||
name: retag uv
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: release
|
||||
needs:
|
||||
- docker-publish-base
|
||||
- docker-publish-extra
|
||||
if: ${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }}
|
||||
permissions:
|
||||
packages: write
|
||||
attestations: write # needed to push image attestations to the Github attestation store
|
||||
id-token: write # needed for signing the images with GitHub OIDC Token
|
||||
steps:
|
||||
# Login to DockerHub first, to avoid rate-limiting
|
||||
- uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
username: astralshbot
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN_RO }}
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
|
||||
- uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
|
||||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
|
||||
env:
|
||||
DOCKER_METADATA_ANNOTATIONS_LEVELS: index
|
||||
with:
|
||||
images: ${{ env.UV_BASE_IMG }}
|
||||
# Order is on purpose such that the label org.opencontainers.image.version has the first pattern with the full version
|
||||
tags: |
|
||||
type=pep440,pattern={{ version }},value=${{ fromJson(inputs.plan).announcement_tag }}
|
||||
type=pep440,pattern={{ major }}.{{ minor }},value=${{ fromJson(inputs.plan).announcement_tag }}
|
||||
|
||||
- uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Adapted from https://docs.docker.com/build/ci/github-actions/multi-platform/
|
||||
- name: Create manifest list and push
|
||||
working-directory: /tmp/digests
|
||||
# The readarray part is used to make sure the quoting and special characters are preserved on expansion (e.g. spaces)
|
||||
# The jq command expands the docker/metadata json "tags" array entry to `-t tag1 -t tag2 ...` for each tag in the array
|
||||
# The printf will expand the base image with the `<UV_BASE_IMG>@sha256:<sha256> ...` for each sha256 in the directory
|
||||
# The final command becomes `docker buildx imagetools create -t tag1 -t tag2 ... <UV_BASE_IMG>@sha256:<sha256_1> <UV_BASE_IMG>@sha256:<sha256_2> ...`
|
||||
- name: Push tags
|
||||
env:
|
||||
IMAGE: ${{ env.UV_BASE_IMG }}
|
||||
DIGEST: ${{ needs.docker-publish-base.outputs.image-digest }}
|
||||
TAGS: ${{ needs.docker-publish-base.outputs.image-tags }}
|
||||
run: |
|
||||
readarray -t lines <<< "$DOCKER_METADATA_OUTPUT_ANNOTATIONS"; annotations=(); for line in "${lines[@]}"; do annotations+=(--annotation "$line"); done
|
||||
docker buildx imagetools create \
|
||||
"${annotations[@]}" \
|
||||
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ env.UV_BASE_IMG }}@sha256:%s ' *)
|
||||
|
||||
- name: Share manifest digest
|
||||
id: manifest-digest
|
||||
# To sign the manifest, we need it's digest. Unfortunately "docker
|
||||
# buildx imagetools create" does not (yet) have a clean way of sharing
|
||||
# the digest of the manifest it creates (see docker/buildx#2407), so
|
||||
# we use a separate command to retrieve it.
|
||||
# imagetools inspect [TAG] --format '{{json .Manifest}}' gives us
|
||||
# the machine readable JSON description of the manifest, and the
|
||||
# jq command extracts the digest from this. The digest is then
|
||||
# sent to the Github step output file for sharing with other steps.
|
||||
run: |
|
||||
digest="$(
|
||||
docker buildx imagetools inspect \
|
||||
"${UV_BASE_IMG}:${DOCKER_METADATA_OUTPUT_VERSION}" \
|
||||
--format '{{json .Manifest}}' \
|
||||
| jq -r '.digest'
|
||||
)"
|
||||
echo "digest=${digest}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Generate artifact attestation
|
||||
uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2.4.0
|
||||
with:
|
||||
subject-name: ${{ env.UV_BASE_IMG }}
|
||||
subject-digest: ${{ steps.manifest-digest.outputs.digest }}
|
||||
# push-to-registry is explicitly not enabled to maintain full control over the top image
|
||||
docker pull "${IMAGE}@${DIGEST}"
|
||||
for tag in $TAGS; do
|
||||
docker tag "${IMAGE}@${DIGEST}" "${tag}"
|
||||
docker push "${tag}"
|
||||
done
|
||||
|
|
|
|||
Loading…
Reference in New Issue