From 8d7d02193e9baa094595ec8571478241cd5bc77d Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 15 Jul 2025 18:14:32 +0100 Subject: [PATCH] Rework typeshed-sync workflow to also add docstrings for Windows- and MacOS-specific APIs (#19360) --- .github/workflows/sync_typeshed.yaml | 179 ++++++++++++++++++++------- scripts/codemod_docstrings.sh | 27 ++++ 2 files changed, 160 insertions(+), 46 deletions(-) create mode 100755 scripts/codemod_docstrings.sh diff --git a/.github/workflows/sync_typeshed.yaml b/.github/workflows/sync_typeshed.yaml index 2d7dea5cda..f8d2f60dc2 100644 --- a/.github/workflows/sync_typeshed.yaml +++ b/.github/workflows/sync_typeshed.yaml @@ -1,5 +1,25 @@ name: Sync typeshed +# How this works: +# +# 1. A Linux worker: +# a. Checks out Ruff and typeshed +# b. Deletes the vendored typeshed stdlib stubs from Ruff +# c. Copies the latest versions of the stubs from typeshed +# d. Uses docstring-adder to sync all docstrings available on Linux +# e. Creates a new branch on the upstream astral-sh/ruff repository +# f. Commits the changes it's made and pushes them to the new upstream branch +# 2. Once the Linux worker is done, a Windows worker: +# a. Checks out the branch created by the Linux worker +# b. Syncs all docstrings available on Windows that are not available on Linux +# c. Commits the changes and pushes them to the same upstream branch +# 3. Once the Windows worker is done, a MacOS worker: +# a. Checks out the branch created by the Linux worker +# b. Syncs all docstrings available on MacOS that are not available on Linux or Windows +# c. Commits the changes and pushes them to the same upstream branch +# d. Creates a PR against the `main` branch using the branch all three workers have pushed to +# 4. If any of steps 1-3 failed, an issue is created in the `astral-sh/ruff` repository + on: workflow_dispatch: schedule: @@ -10,7 +30,13 @@ env: FORCE_COLOR: 1 GH_TOKEN: ${{ github.token }} + # The name of the upstream branch that the first worker creates, + # and which all three workers push to. + UPSTREAM_BRANCH: typeshedbot/sync-typeshed + jobs: + # Sync typeshed stubs, and sync all docstrings available on Linux. + # Push the changes to a new branch on the upstream repository. sync: name: Sync typeshed runs-on: ubuntu-latest @@ -19,7 +45,6 @@ jobs: if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }} permissions: contents: write - pull-requests: write steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 name: Checkout Ruff @@ -37,67 +62,129 @@ jobs: git config --global user.name typeshedbot git config --global user.email '<>' - uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1 - - name: Sync typeshed - id: sync + - name: Sync typeshed stubs run: | - docstring_adder="git+https://github.com/astral-sh/docstring-adder.git@6de51c5f44aea11fe8c8f2d30f9ee0683682c3d2" + rm -rf ruff/crates/ty_vendored/vendor/typeshed + mkdir ruff/crates/ty_vendored/vendor/typeshed + cp typeshed/README.md ruff/crates/ty_vendored/vendor/typeshed + cp typeshed/LICENSE ruff/crates/ty_vendored/vendor/typeshed - # Run with the full matrix of Python versions supported by typeshed, - # so that we codemod in docstrings that only exist on certain versions. - # - # The codemod will only add docstrings to functions/classes that do not - # already have docstrings. We run with Python 3.14 before running with - # any other Python version so that we get the Python 3.14 version of the - # docstring for a definition that exists on all Python versions: if we - # ran with Python 3.9 first, then the later runs with Python 3.10+ would - # not modify the docstring that had already been added using the old version of Python. - # - # TODO: In order to add docstrings for platform-specific APIs, we would also - # need to run the codemod on Windows. We get the runtime docstrings by inspecting - # the docstrings at runtime, so if an API doesn't exist at runtime (because e.g. - # it's Windows-specific and we're running on Linux), then we won't add a docstring to it. - # - uvx --python=3.14 --force-reinstall --from="${docstring_adder}" add-docstrings --stdlib-path ./typeshed/stdlib - uvx --python=3.13 --force-reinstall --from="${docstring_adder}" add-docstrings --stdlib-path ./typeshed/stdlib - uvx --python=3.12 --force-reinstall --from="${docstring_adder}" add-docstrings --stdlib-path ./typeshed/stdlib - uvx --python=3.11 --force-reinstall --from="${docstring_adder}" add-docstrings --stdlib-path ./typeshed/stdlib - uvx --python=3.10 --force-reinstall --from="${docstring_adder}" add-docstrings --stdlib-path ./typeshed/stdlib - uvx --python=3.9 --force-reinstall --from="${docstring_adder}" add-docstrings --stdlib-path ./typeshed/stdlib + # The pyproject.toml file is needed by a later job for the black configuration. + # It's deleted before creating the PR. + cp typeshed/pyproject.toml ruff/crates/ty_vendored/vendor/typeshed + + cp -r typeshed/stdlib ruff/crates/ty_vendored/vendor/typeshed/stdlib + rm -rf ruff/crates/ty_vendored/vendor/typeshed/stdlib/@tests + git -C typeshed rev-parse HEAD > ruff/crates/ty_vendored/vendor/typeshed/source_commit.txt + cd ruff + git checkout -b typeshedbot/sync-typeshed + git add . + git commit -m "Sync typeshed. Source commit: https://github.com/python/typeshed/commit/$(git -C ../typeshed rev-parse HEAD)" --allow-empty + - name: Sync Linux docstrings + if: ${{ success() }} + run: | + cd ruff + ./scripts/codemod_docstrings.sh + git commit -am "Sync Linux docstrings" --allow-empty + - name: Push the changes + id: commit + if: ${{ success() }} + run: git push --force --set-upstream origin "${UPSTREAM_BRANCH}" + + # Checkout the branch created by the sync job, + # and sync all docstrings available on Windows that are not available on Linux. + # Commit the changes and push them to the same branch. + docstrings-windows: + runs-on: windows-latest + timeout-minutes: 20 + needs: [sync] + + # Don't run the cron job on forks. + # The job will also be skipped if the sync job failed, because it's specified in `needs` above, + # and we haven't used `always()` in the `if` condition here + # (https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-requiring-successful-dependent-jobs) + if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }} + + permissions: + contents: write + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + name: Checkout Ruff + with: + persist-credentials: true + ref: ${{ env.UPSTREAM_BRANCH}} + - uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1 + - name: Setup git + run: | + git config --global user.name typeshedbot + git config --global user.email '<>' + - name: Sync Windows docstrings + id: docstrings + shell: bash + run: ./scripts/codemod_docstrings.sh + - name: Commit the changes + if: ${{ steps.docstrings.outcome == 'success' }} + run: | + git commit -am "Sync Windows docstrings" --allow-empty + git push + + # Checkout the branch created by the sync job, + # and sync all docstrings available on macOS that are not available on Linux or Windows. + # Push the changes to the same branch and create a PR against the `main` branch using that branch. + docstrings-macos-and-pr: + runs-on: macos-latest + timeout-minutes: 20 + needs: [sync, docstrings-windows] + + # Don't run the cron job on forks. + # The job will also be skipped if the sync or docstrings-windows jobs failed, + # because they're specified in `needs` above and we haven't used an `always()` condition in the `if` here + # (https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-requiring-successful-dependent-jobs) + if: ${{ github.repository == 'astral-sh/ruff' || github.event_name != 'schedule' }} + + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + name: Checkout Ruff + with: + persist-credentials: true + ref: ${{ env.UPSTREAM_BRANCH}} + - uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1 + - name: Setup git + run: | + git config --global user.name typeshedbot + git config --global user.email '<>' + - name: Sync macOS docstrings + run: ./scripts/codemod_docstrings.sh + - name: Commit and push the changes + if: ${{ success() }} + run: | + git commit -am "Sync macOS docstrings" --allow-empty # Here we just reformat the codemodded stubs so that they are # consistent with the other typeshed stubs around them. # Typeshed formats code using black in their CI, so we just invoke # black on the stubs the same way that typeshed does. - uvx --directory=typeshed pre-commit run -a black || true + uvx black crates/ty_vendored/vendor/typeshed/stdlib --config crates/ty_vendored/vendor/typeshed/pyproject.toml || true + git commit -am "Format codemodded docstrings" --allow-empty - rm -rf ruff/crates/ty_vendored/vendor/typeshed - mkdir ruff/crates/ty_vendored/vendor/typeshed - cp typeshed/README.md ruff/crates/ty_vendored/vendor/typeshed - cp typeshed/LICENSE ruff/crates/ty_vendored/vendor/typeshed - cp -r typeshed/stdlib ruff/crates/ty_vendored/vendor/typeshed/stdlib - rm -rf ruff/crates/ty_vendored/vendor/typeshed/stdlib/@tests - git -C typeshed rev-parse HEAD > ruff/crates/ty_vendored/vendor/typeshed/source_commit.txt - - name: Commit the changes - id: commit - if: ${{ steps.sync.outcome == 'success' }} - run: | - cd ruff - git checkout -b typeshedbot/sync-typeshed - git add . - git diff --staged --quiet || git commit -m "Sync typeshed. Source commit: https://github.com/python/typeshed/commit/$(git -C ../typeshed rev-parse HEAD)" + rm crates/ty_vendored/vendor/typeshed/pyproject.toml + git commit -am "Remove pyproject.toml file" + + git push - name: Create a PR - if: ${{ steps.sync.outcome == 'success' && steps.commit.outcome == 'success' }} + if: ${{ success() }} run: | - cd ruff - git push --force origin typeshedbot/sync-typeshed gh pr list --repo "$GITHUB_REPOSITORY" --head typeshedbot/sync-typeshed --json id --jq length | grep 1 && exit 0 # exit if there is existing pr gh pr create --title "[ty] Sync vendored typeshed stubs" --body "Close and reopen this PR to trigger CI" --label "ty" create-issue-on-failure: name: Create an issue if the typeshed sync failed runs-on: ubuntu-latest - needs: [sync] - if: ${{ github.repository == 'astral-sh/ruff' && always() && github.event_name == 'schedule' && needs.sync.result == 'failure' }} + needs: [sync, docstrings-windows, docstrings-macos-and-pr] + if: ${{ github.repository == 'astral-sh/ruff' && always() && github.event_name == 'schedule' && (needs.sync.result == 'failure' || needs.docstrings-windows.result == 'failure' || needs.docstrings-macos-and-pr.result == 'failure') }} permissions: issues: write steps: diff --git a/scripts/codemod_docstrings.sh b/scripts/codemod_docstrings.sh new file mode 100755 index 0000000000..4f998b74da --- /dev/null +++ b/scripts/codemod_docstrings.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# This script uses the https://github.com/astral-sh/docstring-adder tool to codemod docstrings into our vendored typeshed stubs. +# +# We run the tool with the full matrix of Python versions supported by typeshed, +# so that we codemod in docstrings that only exist on certain versions. +# +# The codemod will only add docstrings to functions/classes that do not +# already have docstrings. We run with Python 3.14 before running with +# any other Python version so that we get the Python 3.14 version of the +# docstring for a definition that exists on all Python versions: if we +# ran with Python 3.9 first, then the later runs with Python 3.10+ would +# not modify the docstring that had already been added using the old version of Python. +# +# Note that the codemod can only add docstrings if they exist on the Python platform +# the codemod is run with. If you need to add docstrings for a Windows-specific API, +# you'll need to run the codemod on a Windows machine. + +set -eu + +docstring_adder="git+https://github.com/astral-sh/docstring-adder.git@7f350b03ee83dd44ebd8010228ad3dfca34a7887" +stdlib_path="./crates/ty_vendored/vendor/typeshed/stdlib" + +for python_version in 3.14 3.13 3.12 3.11 3.10 3.9 +do + uvx --python="$python_version" --force-reinstall --from="${docstring_adder}" add-docstrings --stdlib-path="${stdlib_path}" +done