diff --git a/.github/actions/fix-bss-and-generate-patch/action.yml b/.github/actions/fix-bss-and-generate-patch/action.yml new file mode 100644 index 0000000000..726f693047 --- /dev/null +++ b/.github/actions/fix-bss-and-generate-patch/action.yml @@ -0,0 +1,20 @@ +inputs: + version: + required: true + +runs: + using: composite + steps: + - name: Fix BSS + shell: sh + run: .venv/bin/python3 tools/fix_bss.py -v ${{ inputs.version }} + + - name: Generate patch + shell: sh + run: git diff > fix_bss_${{ inputs.version }}.patch + + - name: Upload patch + uses: actions/upload-artifact@v7 + with: + name: fix_bss_${{ inputs.version }}.patch + path: fix_bss_${{ inputs.version }}.patch diff --git a/.github/scripts/apply_fix_bss_patches.py b/.github/scripts/apply_fix_bss_patches.py new file mode 100644 index 0000000000..06c79fe521 --- /dev/null +++ b/.github/scripts/apply_fix_bss_patches.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: © 2026 ZeldaRET +# SPDX-License-Identifier: CC0-1.0 + +from pathlib import Path +import re +import subprocess + + +def get_increment_block_numbers(p: Path, version: str): + increment_block_numbers: list[int] = [] + is_in_pragma = False + n_fake_structs = None + for l in p.read_text().splitlines(): + if l.startswith("#pragma increment_block_number"): + is_in_pragma = True + n_fake_structs = 0 + if is_in_pragma: + m = next(re.finditer(rf"{version}:(\d+)", l), None) + if m is not None: + n_fake_structs = int(m.group(1)) + if is_in_pragma and not l.endswith("\\"): + is_in_pragma = False + assert n_fake_structs is not None + increment_block_numbers.append(n_fake_structs) + n_fake_structs = None + return increment_block_numbers + + +# Formats #pragma increment_block_number as a list of lines +def format_pragma(amounts: dict[str, int], max_line_length: int) -> list[str]: + lines = [] + pragma_start = "#pragma increment_block_number " + current_line = pragma_start + '"' + first = True + for version, amount in sorted(amounts.items()): + part = f"{version}:{amount}" + if len(current_line) + len(" ") + len(part) + len('" \\') > max_line_length: + lines.append(current_line + '" ') + current_line = " " * len(pragma_start) + '"' + first = True + if not first: + current_line += " " + current_line += part + first = False + lines.append(current_line + '"\n') + + if len(lines) >= 2: + # add and align vertically all continuation \ characters + n_align = max(map(len, lines[:-1])) + for i in range(len(lines) - 1): + lines[i] = f"{lines[i]:{n_align}}\\\n" + + return lines + + +def set_increment_block_numbers( + p: Path, increment_block_numbers_by_version: dict[str, list[int]] +): + print(p, increment_block_numbers_by_version) + i_pragma = 0 + is_in_pragma = False + pragma_lines = [] + new_lines = [] + for l in p.read_text().splitlines(keepends=True): + if l.startswith("#pragma increment_block_number"): + is_in_pragma = True + if not is_in_pragma: + new_lines.append(l) + if is_in_pragma: + pragma_lines.append(l.removesuffix("\\\n")) + if is_in_pragma and not l.endswith("\\\n"): + is_in_pragma = False + pragma_string = "".join(pragma_lines) + amounts: dict[str, int] = {} + for part in pragma_string.replace('"', "").split()[2:]: + version, amount_str = part.split(":") + amount = int(amount_str) + amounts[version] = amount + for ( + version, + increment_block_numbers, + ) in increment_block_numbers_by_version.items(): + amounts[version] = increment_block_numbers[i_pragma] + i_pragma += 1 + column_limit = 120 # matches .clang-format's ColumnLimit + new_pragma_lines = format_pragma(amounts, column_limit) + new_lines.extend(new_pragma_lines) + p.write_text("".join(new_lines)) + + +increment_block_numbers_by_version_by_file: dict[Path, dict[str, list[int]]] = {} +for p in Path(".").glob("fix_bss_*.patch"): + version = p.name.removeprefix("fix_bss_").removesuffix(".patch") + subprocess.check_call(["git", "apply", str(p)]) + touched_files = subprocess.check_output( + "git diff --name-only".split(), + text=True, + ).splitlines() + for file in touched_files: + file_p = Path(file) + increment_block_numbers = get_increment_block_numbers(file_p, version) + increment_block_numbers_by_version_by_file.setdefault(file_p, {})[ + version + ] = increment_block_numbers + subprocess.check_call("git checkout -- .".split()) + + +for ( + file, + increment_block_numbers_by_version, +) in increment_block_numbers_by_version_by_file.items(): + set_increment_block_numbers(file, increment_block_numbers_by_version) diff --git a/.github/scripts/generate_patch.sh b/.github/scripts/generate_patch.sh new file mode 100755 index 0000000000..04cc431b04 --- /dev/null +++ b/.github/scripts/generate_patch.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +PATCH=$(git diff | base64 -w 0) +if [ -n "$PATCH" ]; then + echo 'Fixes were made for your PR. To apply these changes to your working directory, copy and run the following command:' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "echo -n $PATCH | base64 -d | git apply -" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..cd2132a82d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,89 @@ +name: Build + +# Build on every branch push, tag push, and pull request change: +on: + push: + pull_request: + +jobs: + build_repo: + # This is a *private* build container. + container: ghcr.io/zeldaret/oot-build:main + + name: Build repo (${{ matrix.version }}) + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + version: + - ntsc-1.0 # N64 NTSC 1.0 (Japan/US depending on REGION) + - ntsc-1.1 # N64 NTSC 1.1 (Japan/US depending on REGION) + - pal-1.0 # N64 PAL 1.0 (Europe) + - ntsc-1.2 # N64 NTSC 1.2 (Japan/US depending on REGION) + - pal-1.1 # N64 PAL 1.1 (Europe) + - gc-jp # GameCube Japan + - gc-jp-mq # GameCube Japan Master Quest + - gc-us # GameCube US + - gc-us-mq # GameCube US Master Quest + - gc-eu-dbg-2 # GameCube Europe/PAL Debug (earlier build) + - gc-eu-mq-dbg # GameCube Europe/PAL Master Quest Debug + - gc-eu-dbg # GameCube Europe/PAL Debug + - gc-eu # GameCube Europe/PAL + - gc-eu-mq # GameCube Europe/PAL Master Quest + - gc-jp-ce # GameCube Japan (Collector's Edition disc) + - ique-cn # iQue Player (Simplified Chinese) + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: git config safe.directory + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Install system dependencies + run: | + apt-get install -y git build-essential binutils-mips-linux-gnu curl python3 python3-pip python3-venv libxml2-dev + + - name: Get the dependency + run: ln -s /orig/${{ matrix.version }}/baserom.z64 baseroms/${{ matrix.version }}/baserom.z64 + + - name: Setup + run: make -j $(nproc) VERSION=${{ matrix.version }} setup + + - name: Build ${{ matrix.version }} + id: build + run: make -j $(nproc) VERSION=${{ matrix.version }} + + - name: Fix BSS and generate patch + if: failure() && steps.build.outcome == 'failure' + uses: ./.github/actions/fix-bss-and-generate-patch + with: + version: ${{ matrix.version }} + + - name: Upload map + uses: actions/upload-artifact@v6 + with: + name: oot-${{ matrix.version }}.map + path: build/${{ matrix.version }}/oot-${{ matrix.version }}.map + + merge_bss_fixes: + name: Merge BSS fixes + runs-on: ubuntu-latest + needs: [build_repo] + if: '!cancelled()' # Run even if build_repo fails + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download patches + uses: actions/download-artifact@v8 + with: + pattern: fix_bss_*.patch + merge-multiple: true + + - name: Apply patches + run: python3 .github/scripts/apply_fix_bss_patches.py + + - name: Generate patch + run: .github/scripts/generate_patch.sh diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml new file mode 100644 index 0000000000..de00677a0e --- /dev/null +++ b/.github/workflows/format.yml @@ -0,0 +1,22 @@ +name: Check format + +# Build on every branch push, tag push, and pull request change: +on: + push: + pull_request: + +jobs: + build: + name: Check format + runs-on: ubuntu-latest + + steps: + - name: Checkout reposistory + uses: actions/checkout@v6 + + - name: Install package requirements + run: | + sudo apt-get install -y python3 clang-format-14 clang-tidy-14 + + - name: Check formatting + run: python3 tools/check_format.py diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 3e4cba55cf..0000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,187 +0,0 @@ -pipeline { - agent { - label 'oot' - } - - stages { - stage('Check formatting (full)') { - when { - branch 'main' - } - steps { - echo 'Checking formatting on all files...' - sh 'python3 tools/check_format.py' - } - } - stage('Check formatting (modified)') { - when { - not { - branch 'main' - } - } - steps { - catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - echo 'Checking formatting on modified files...' - sh 'python3 tools/check_format.py --verbose --compare-to origin/main' - } - } - } - // The ROMs are built in an order that maximizes compiler flags coverage in a "fail fast" approach. - // Specifically we start with a retail ROM for BSS ordering, and make sure we cover all of - // N64/GC/NTSC/PAL/MQ/DEBUG as quickly as possible. - stage('Build ntsc-1.0') { - steps { - script { - build('ntsc-1.0', 'oot-ntsc-1.0-us.z64') - } - } - } - stage('Build gc-jp') { - steps { - script { - build('gc-jp', 'oot-gc-jp.z64') - } - } - } - stage('Build gc-eu-mq') { - steps { - script { - build('gc-eu-mq', 'oot-gc-eu-mq.z64') - } - } - } - stage('Build gc-eu-mq-dbg') { - steps { - script { - build('gc-eu-mq-dbg', 'oot-gc-eu-mq-dbg.z64') - } - } - } - stage('Build pal-1.0') { - steps { - script { - build('pal-1.0', 'oot-pal-1.0.z64') - } - } - } - stage('Build ntsc-1.2') { - steps { - script { - build('ntsc-1.2', 'oot-ntsc-1.2-us.z64') - } - } - } - stage('Build gc-us') { - steps { - script { - build('gc-us', 'oot-gc-us.z64') - } - } - } - stage('Build gc-jp-ce') { - steps { - script { - build('gc-jp-ce', 'oot-gc-jp-ce.z64') - } - } - } - stage('Build gc-eu') { - steps { - script { - build('gc-eu', 'oot-gc-eu.z64') - } - } - } - stage('Build gc-jp-mq') { - steps { - script { - build('gc-jp-mq', 'oot-gc-jp-mq.z64') - } - } - } - stage('Build pal-1.1') { - steps { - script { - build('pal-1.1', 'oot-pal-1.1.z64') - } - } - } - stage('Build ntsc-1.1') { - steps { - script { - build('ntsc-1.1', 'oot-ntsc-1.1-us.z64') - } - } - } - stage('Build gc-us-mq') { - steps { - script { - build('gc-us-mq', 'oot-gc-us-mq.z64') - } - } - } - stage('Build ique-cn') { - steps { - script { - build('ique-cn', 'oot-ique-cn.z64') - } - } - } - stage('Build gc-eu-dbg-2') { - steps { - script { - build('gc-eu-dbg-2', 'oot-gc-eu-dbg-proto.z64') - } - } - } - stage('Build gc-eu-dbg') { - steps { - script { - build('gc-eu-dbg', 'oot-gc-eu-dbg.z64') - } - } - } - stage('Generate patch') { - when { - not { - branch 'main' - } - } - steps { - sh 'git diff' - echo 'Generating patch...' - sh 'tools/generate_patch_from_jenkins.sh' - } - } - } - post { - always { - echo "Finished, deleting directory." - deleteDir() - } - cleanup { - echo "Clean up in post." - cleanWs(cleanWhenNotBuilt: false, - deleteDirs: true, - disableDeferredWipeout: true, - notFailBuild: true) - } - } -} - -def build(String version, String rom) { - sh "ln -s /usr/local/etc/roms/${rom} baseroms/${version}/baserom.z64" - sh "make -j\$(nproc) setup VERSION=${version}" - try { - sh "make -j\$(nproc) VERSION=${version}" - } catch (e) { - echo "Build failed, attempting to fix BSS ordering..." - sh ".venv/bin/python3 tools/fix_bss.py -v ${version}" - // If fix_bss.py succeeds, continue the build, but ensure both the build and current stage are marked as failed - catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { - sh 'exit 1' - } - } finally { - sh "make clean assetclean VERSION=${version}" - } -} diff --git a/README.md b/README.md index 582939dd45..3a3a73ea0c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # The Legend of Zelda: Ocarina of Time -[![Build Status][jenkins-badge]][jenkins] [![Decompilation Progress][progress-badge]][progress] [![Contributors][contributors-badge]][contributors] [![Discord Channel][discord-badge]][discord] +[![Build Status][gha-badge]][gha] [![Decompilation Progress][progress-badge]][progress] [![Contributors][contributors-badge]][contributors] [![Discord Channel][discord-badge]][discord] -[jenkins]: https://jenkins.deco.mp/job/OOT/job/main -[jenkins-badge]: https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fjenkins.deco.mp%2Fjob%2FOOT%2Fjob%2Fmain +[gha]: https://github.com/zeldaret/oot/actions/workflows/ci.yml?query=branch%3Amain+event%3Apush +[gha-badge]: https://img.shields.io/github/actions/workflow/status/zeldaret/oot/ci [progress]: https://zelda.deco.mp/games/oot [progress-badge]: https://img.shields.io/endpoint?url=https://zelda.deco.mp/assets/csv/progress-oot-shield.json diff --git a/tools/check_format.py b/tools/check_format.py index 18692cb6d1..7d4c27b99f 100644 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -2,11 +2,9 @@ # SPDX-License-Identifier: CC0-1.0 import subprocess -import argparse import difflib import multiprocessing -import glob -import os.path +import os import sys sys.path.insert(0, os.curdir) @@ -19,39 +17,8 @@ def get_git_status(): return subprocess.check_output("git status --porcelain".split(), text=True) -def get_modified_files_to_format(compare_to): - modified_files_str = subprocess.check_output( - ["git", "diff", "--name-only", compare_to], text=True - ) - modified_files = set(modified_files_str.splitlines()) - - all_src_files, all_extra_files = format.list_files_to_format() - # Split modified_files between source files and extra files (see format.py) - # This also filters out deleted files that no longer exist - modified_src_files_existing = list(modified_files.intersection(all_src_files)) - modified_extra_files_existing = list(modified_files.intersection(all_extra_files)) - - return modified_src_files_existing, modified_extra_files_existing - - def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--verbose", action="store_true") - parser.add_argument("--compare-to", dest="compare_to") - args = parser.parse_args() - - if args.compare_to: - src_files, extra_files = get_modified_files_to_format(args.compare_to) - if args.verbose: - print("Formatting specific files:") - print(len(src_files), src_files) - print(len(extra_files), extra_files) - if not src_files and not extra_files: - if args.verbose: - print("Nothing to format") - exit(0) - else: - src_files, extra_files = format.list_files_to_format() + src_files, extra_files = format.list_files_to_format() nb_jobs = multiprocessing.cpu_count() diff --git a/tools/generate_patch_from_jenkins.sh b/tools/generate_patch_from_jenkins.sh deleted file mode 100755 index 0965015ccb..0000000000 --- a/tools/generate_patch_from_jenkins.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -PATCH=$(git diff | base64 -w 0) -if [ -n "$PATCH" ]; then - echo "Jenkins made some fixes to your PR. To apply these changes to your working directory," - echo "copy and run the following command (triple-click to select the entire line):" - echo - echo "echo -n $PATCH | base64 -d | git apply -" - echo -fi