mirror of
https://github.com/zeldaret/oot
synced 2026-05-22 22:44:26 -04:00
Migrate CI to Github Actions (#2742)
* Migrate CI to Github Actions (#2740) Changes the CI system from Jenkins to Github Actions (GHA) as discussed on discord. I tried to adapt the original jenkinsfile as I could. One major difference from the old system is each version will now be built in parallel instead of sequentially. Also, the mapfiles will be uploaded as an GHA artifact to the Action workflow. I think having them easily available would be nice. This specific approach to handle GHA for decomp projects was adapted from the one used by the GC/Wii community. It is documented here https://github.com/encounter/dtk-template/blob/main/docs/github_actions.md There's a writeup about this adaptation for N64 projects [here](https://github.com/AngheloAlf/drmario64/pull/19). * Rename generate_patch_from_gha -> gha_fix_bss_and_generate_patch * Make check formatting always on the full repo * ruin z_fishing bss for CI testing purposes * fix not passing VERSION to tools/gha_fix_bss_and_generate_patch.sh * debugging gha_fix_bss_and_generate_patch.sh * fix the debugging... * git config safe.directory * rm debugging stuff from gha_fix_bss_and_generate_patch.sh * fix_bss gha-side machinery, attempt 1 * fix1 * fix2 * debug1 * checkout repo in merge_bss_fixes job * fix3 * fix4 * fix5 * some prettify * apply_fix_bss_patches.py complete * fix6 * generate_patch.sh is back * apply fix bss changes from gha! it works! * generate_patch.sh suggestion * ruin bss again for testing * unruin bss * Update .github/workflows/format.yml Co-authored-by: Anghelo Carvajal <angheloalf95@gmail.com> * update matrix version list * finalize apply_fix_bss_patches.py --------- Co-authored-by: Anghelo Carvajal <angheloalf95@gmail.com>
This commit is contained in:
Dragorn421
@@ -0,0 +1,20 @@
|
||||
inputs:
|
||||
version:
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Fix BSS
|
||||
shell: sh
|
||||
run: .venv/bin/python3 tools/fix_bss.py -v ${{ inputs.version }}
|
||||
|
||||
- name: Generate patch
|
||||
shell: sh
|
||||
run: git diff > fix_bss_${{ inputs.version }}.patch
|
||||
|
||||
- name: Upload patch
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fix_bss_${{ inputs.version }}.patch
|
||||
path: fix_bss_${{ inputs.version }}.patch
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-FileCopyrightText: © 2026 ZeldaRET
|
||||
# SPDX-License-Identifier: CC0-1.0
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
|
||||
def get_increment_block_numbers(p: Path, version: str):
|
||||
increment_block_numbers: list[int] = []
|
||||
is_in_pragma = False
|
||||
n_fake_structs = None
|
||||
for l in p.read_text().splitlines():
|
||||
if l.startswith("#pragma increment_block_number"):
|
||||
is_in_pragma = True
|
||||
n_fake_structs = 0
|
||||
if is_in_pragma:
|
||||
m = next(re.finditer(rf"{version}:(\d+)", l), None)
|
||||
if m is not None:
|
||||
n_fake_structs = int(m.group(1))
|
||||
if is_in_pragma and not l.endswith("\\"):
|
||||
is_in_pragma = False
|
||||
assert n_fake_structs is not None
|
||||
increment_block_numbers.append(n_fake_structs)
|
||||
n_fake_structs = None
|
||||
return increment_block_numbers
|
||||
|
||||
|
||||
# Formats #pragma increment_block_number as a list of lines
|
||||
def format_pragma(amounts: dict[str, int], max_line_length: int) -> list[str]:
|
||||
lines = []
|
||||
pragma_start = "#pragma increment_block_number "
|
||||
current_line = pragma_start + '"'
|
||||
first = True
|
||||
for version, amount in sorted(amounts.items()):
|
||||
part = f"{version}:{amount}"
|
||||
if len(current_line) + len(" ") + len(part) + len('" \\') > max_line_length:
|
||||
lines.append(current_line + '" ')
|
||||
current_line = " " * len(pragma_start) + '"'
|
||||
first = True
|
||||
if not first:
|
||||
current_line += " "
|
||||
current_line += part
|
||||
first = False
|
||||
lines.append(current_line + '"\n')
|
||||
|
||||
if len(lines) >= 2:
|
||||
# add and align vertically all continuation \ characters
|
||||
n_align = max(map(len, lines[:-1]))
|
||||
for i in range(len(lines) - 1):
|
||||
lines[i] = f"{lines[i]:{n_align}}\\\n"
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def set_increment_block_numbers(
|
||||
p: Path, increment_block_numbers_by_version: dict[str, list[int]]
|
||||
):
|
||||
print(p, increment_block_numbers_by_version)
|
||||
i_pragma = 0
|
||||
is_in_pragma = False
|
||||
pragma_lines = []
|
||||
new_lines = []
|
||||
for l in p.read_text().splitlines(keepends=True):
|
||||
if l.startswith("#pragma increment_block_number"):
|
||||
is_in_pragma = True
|
||||
if not is_in_pragma:
|
||||
new_lines.append(l)
|
||||
if is_in_pragma:
|
||||
pragma_lines.append(l.removesuffix("\\\n"))
|
||||
if is_in_pragma and not l.endswith("\\\n"):
|
||||
is_in_pragma = False
|
||||
pragma_string = "".join(pragma_lines)
|
||||
amounts: dict[str, int] = {}
|
||||
for part in pragma_string.replace('"', "").split()[2:]:
|
||||
version, amount_str = part.split(":")
|
||||
amount = int(amount_str)
|
||||
amounts[version] = amount
|
||||
for (
|
||||
version,
|
||||
increment_block_numbers,
|
||||
) in increment_block_numbers_by_version.items():
|
||||
amounts[version] = increment_block_numbers[i_pragma]
|
||||
i_pragma += 1
|
||||
column_limit = 120 # matches .clang-format's ColumnLimit
|
||||
new_pragma_lines = format_pragma(amounts, column_limit)
|
||||
new_lines.extend(new_pragma_lines)
|
||||
p.write_text("".join(new_lines))
|
||||
|
||||
|
||||
increment_block_numbers_by_version_by_file: dict[Path, dict[str, list[int]]] = {}
|
||||
for p in Path(".").glob("fix_bss_*.patch"):
|
||||
version = p.name.removeprefix("fix_bss_").removesuffix(".patch")
|
||||
subprocess.check_call(["git", "apply", str(p)])
|
||||
touched_files = subprocess.check_output(
|
||||
"git diff --name-only".split(),
|
||||
text=True,
|
||||
).splitlines()
|
||||
for file in touched_files:
|
||||
file_p = Path(file)
|
||||
increment_block_numbers = get_increment_block_numbers(file_p, version)
|
||||
increment_block_numbers_by_version_by_file.setdefault(file_p, {})[
|
||||
version
|
||||
] = increment_block_numbers
|
||||
subprocess.check_call("git checkout -- .".split())
|
||||
|
||||
|
||||
for (
|
||||
file,
|
||||
increment_block_numbers_by_version,
|
||||
) in increment_block_numbers_by_version_by_file.items():
|
||||
set_increment_block_numbers(file, increment_block_numbers_by_version)
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PATCH=$(git diff | base64 -w 0)
|
||||
if [ -n "$PATCH" ]; then
|
||||
echo 'Fixes were made for your PR. To apply these changes to your working directory, copy and run the following command:' >> $GITHUB_STEP_SUMMARY
|
||||
echo '```' >> $GITHUB_STEP_SUMMARY
|
||||
echo "echo -n $PATCH | base64 -d | git apply -" >> $GITHUB_STEP_SUMMARY
|
||||
echo '```' >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
@@ -0,0 +1,89 @@
|
||||
name: Build
|
||||
|
||||
# Build on every branch push, tag push, and pull request change:
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build_repo:
|
||||
# This is a *private* build container.
|
||||
container: ghcr.io/zeldaret/oot-build:main
|
||||
|
||||
name: Build repo (${{ matrix.version }})
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
version:
|
||||
- ntsc-1.0 # N64 NTSC 1.0 (Japan/US depending on REGION)
|
||||
- ntsc-1.1 # N64 NTSC 1.1 (Japan/US depending on REGION)
|
||||
- pal-1.0 # N64 PAL 1.0 (Europe)
|
||||
- ntsc-1.2 # N64 NTSC 1.2 (Japan/US depending on REGION)
|
||||
- pal-1.1 # N64 PAL 1.1 (Europe)
|
||||
- gc-jp # GameCube Japan
|
||||
- gc-jp-mq # GameCube Japan Master Quest
|
||||
- gc-us # GameCube US
|
||||
- gc-us-mq # GameCube US Master Quest
|
||||
- gc-eu-dbg-2 # GameCube Europe/PAL Debug (earlier build)
|
||||
- gc-eu-mq-dbg # GameCube Europe/PAL Master Quest Debug
|
||||
- gc-eu-dbg # GameCube Europe/PAL Debug
|
||||
- gc-eu # GameCube Europe/PAL
|
||||
- gc-eu-mq # GameCube Europe/PAL Master Quest
|
||||
- gc-jp-ce # GameCube Japan (Collector's Edition disc)
|
||||
- ique-cn # iQue Player (Simplified Chinese)
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: git config safe.directory
|
||||
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
apt-get install -y git build-essential binutils-mips-linux-gnu curl python3 python3-pip python3-venv libxml2-dev
|
||||
|
||||
- name: Get the dependency
|
||||
run: ln -s /orig/${{ matrix.version }}/baserom.z64 baseroms/${{ matrix.version }}/baserom.z64
|
||||
|
||||
- name: Setup
|
||||
run: make -j $(nproc) VERSION=${{ matrix.version }} setup
|
||||
|
||||
- name: Build ${{ matrix.version }}
|
||||
id: build
|
||||
run: make -j $(nproc) VERSION=${{ matrix.version }}
|
||||
|
||||
- name: Fix BSS and generate patch
|
||||
if: failure() && steps.build.outcome == 'failure'
|
||||
uses: ./.github/actions/fix-bss-and-generate-patch
|
||||
with:
|
||||
version: ${{ matrix.version }}
|
||||
|
||||
- name: Upload map
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: oot-${{ matrix.version }}.map
|
||||
path: build/${{ matrix.version }}/oot-${{ matrix.version }}.map
|
||||
|
||||
merge_bss_fixes:
|
||||
name: Merge BSS fixes
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build_repo]
|
||||
if: '!cancelled()' # Run even if build_repo fails
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download patches
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: fix_bss_*.patch
|
||||
merge-multiple: true
|
||||
|
||||
- name: Apply patches
|
||||
run: python3 .github/scripts/apply_fix_bss_patches.py
|
||||
|
||||
- name: Generate patch
|
||||
run: .github/scripts/generate_patch.sh
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Check format
|
||||
|
||||
# Build on every branch push, tag push, and pull request change:
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Check format
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout reposistory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install package requirements
|
||||
run: |
|
||||
sudo apt-get install -y python3 clang-format-14 clang-tidy-14
|
||||
|
||||
- name: Check formatting
|
||||
run: python3 tools/check_format.py
|
||||
Vendored
-187
@@ -1,187 +0,0 @@
|
||||
pipeline {
|
||||
agent {
|
||||
label 'oot'
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Check formatting (full)') {
|
||||
when {
|
||||
branch 'main'
|
||||
}
|
||||
steps {
|
||||
echo 'Checking formatting on all files...'
|
||||
sh 'python3 tools/check_format.py'
|
||||
}
|
||||
}
|
||||
stage('Check formatting (modified)') {
|
||||
when {
|
||||
not {
|
||||
branch 'main'
|
||||
}
|
||||
}
|
||||
steps {
|
||||
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
|
||||
echo 'Checking formatting on modified files...'
|
||||
sh 'python3 tools/check_format.py --verbose --compare-to origin/main'
|
||||
}
|
||||
}
|
||||
}
|
||||
// The ROMs are built in an order that maximizes compiler flags coverage in a "fail fast" approach.
|
||||
// Specifically we start with a retail ROM for BSS ordering, and make sure we cover all of
|
||||
// N64/GC/NTSC/PAL/MQ/DEBUG as quickly as possible.
|
||||
stage('Build ntsc-1.0') {
|
||||
steps {
|
||||
script {
|
||||
build('ntsc-1.0', 'oot-ntsc-1.0-us.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-jp') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-jp', 'oot-gc-jp.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-eu-mq') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-eu-mq', 'oot-gc-eu-mq.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-eu-mq-dbg') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-eu-mq-dbg', 'oot-gc-eu-mq-dbg.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build pal-1.0') {
|
||||
steps {
|
||||
script {
|
||||
build('pal-1.0', 'oot-pal-1.0.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build ntsc-1.2') {
|
||||
steps {
|
||||
script {
|
||||
build('ntsc-1.2', 'oot-ntsc-1.2-us.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-us') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-us', 'oot-gc-us.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-jp-ce') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-jp-ce', 'oot-gc-jp-ce.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-eu') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-eu', 'oot-gc-eu.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-jp-mq') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-jp-mq', 'oot-gc-jp-mq.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build pal-1.1') {
|
||||
steps {
|
||||
script {
|
||||
build('pal-1.1', 'oot-pal-1.1.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build ntsc-1.1') {
|
||||
steps {
|
||||
script {
|
||||
build('ntsc-1.1', 'oot-ntsc-1.1-us.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-us-mq') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-us-mq', 'oot-gc-us-mq.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build ique-cn') {
|
||||
steps {
|
||||
script {
|
||||
build('ique-cn', 'oot-ique-cn.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-eu-dbg-2') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-eu-dbg-2', 'oot-gc-eu-dbg-proto.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Build gc-eu-dbg') {
|
||||
steps {
|
||||
script {
|
||||
build('gc-eu-dbg', 'oot-gc-eu-dbg.z64')
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Generate patch') {
|
||||
when {
|
||||
not {
|
||||
branch 'main'
|
||||
}
|
||||
}
|
||||
steps {
|
||||
sh 'git diff'
|
||||
echo 'Generating patch...'
|
||||
sh 'tools/generate_patch_from_jenkins.sh'
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
echo "Finished, deleting directory."
|
||||
deleteDir()
|
||||
}
|
||||
cleanup {
|
||||
echo "Clean up in post."
|
||||
cleanWs(cleanWhenNotBuilt: false,
|
||||
deleteDirs: true,
|
||||
disableDeferredWipeout: true,
|
||||
notFailBuild: true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def build(String version, String rom) {
|
||||
sh "ln -s /usr/local/etc/roms/${rom} baseroms/${version}/baserom.z64"
|
||||
sh "make -j\$(nproc) setup VERSION=${version}"
|
||||
try {
|
||||
sh "make -j\$(nproc) VERSION=${version}"
|
||||
} catch (e) {
|
||||
echo "Build failed, attempting to fix BSS ordering..."
|
||||
sh ".venv/bin/python3 tools/fix_bss.py -v ${version}"
|
||||
// If fix_bss.py succeeds, continue the build, but ensure both the build and current stage are marked as failed
|
||||
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
|
||||
sh 'exit 1'
|
||||
}
|
||||
} finally {
|
||||
sh "make clean assetclean VERSION=${version}"
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
# The Legend of Zelda: Ocarina of Time
|
||||
|
||||
[![Build Status][jenkins-badge]][jenkins] [![Decompilation Progress][progress-badge]][progress] [![Contributors][contributors-badge]][contributors] [![Discord Channel][discord-badge]][discord]
|
||||
[![Build Status][gha-badge]][gha] [![Decompilation Progress][progress-badge]][progress] [![Contributors][contributors-badge]][contributors] [![Discord Channel][discord-badge]][discord]
|
||||
|
||||
[jenkins]: https://jenkins.deco.mp/job/OOT/job/main
|
||||
[jenkins-badge]: https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fjenkins.deco.mp%2Fjob%2FOOT%2Fjob%2Fmain
|
||||
[gha]: https://github.com/zeldaret/oot/actions/workflows/ci.yml?query=branch%3Amain+event%3Apush
|
||||
[gha-badge]: https://img.shields.io/github/actions/workflow/status/zeldaret/oot/ci
|
||||
|
||||
[progress]: https://zelda.deco.mp/games/oot
|
||||
[progress-badge]: https://img.shields.io/endpoint?url=https://zelda.deco.mp/assets/csv/progress-oot-shield.json
|
||||
|
||||
+2
-35
@@ -2,11 +2,9 @@
|
||||
# SPDX-License-Identifier: CC0-1.0
|
||||
|
||||
import subprocess
|
||||
import argparse
|
||||
import difflib
|
||||
import multiprocessing
|
||||
import glob
|
||||
import os.path
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.curdir)
|
||||
@@ -19,39 +17,8 @@ def get_git_status():
|
||||
return subprocess.check_output("git status --porcelain".split(), text=True)
|
||||
|
||||
|
||||
def get_modified_files_to_format(compare_to):
|
||||
modified_files_str = subprocess.check_output(
|
||||
["git", "diff", "--name-only", compare_to], text=True
|
||||
)
|
||||
modified_files = set(modified_files_str.splitlines())
|
||||
|
||||
all_src_files, all_extra_files = format.list_files_to_format()
|
||||
# Split modified_files between source files and extra files (see format.py)
|
||||
# This also filters out deleted files that no longer exist
|
||||
modified_src_files_existing = list(modified_files.intersection(all_src_files))
|
||||
modified_extra_files_existing = list(modified_files.intersection(all_extra_files))
|
||||
|
||||
return modified_src_files_existing, modified_extra_files_existing
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--verbose", action="store_true")
|
||||
parser.add_argument("--compare-to", dest="compare_to")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.compare_to:
|
||||
src_files, extra_files = get_modified_files_to_format(args.compare_to)
|
||||
if args.verbose:
|
||||
print("Formatting specific files:")
|
||||
print(len(src_files), src_files)
|
||||
print(len(extra_files), extra_files)
|
||||
if not src_files and not extra_files:
|
||||
if args.verbose:
|
||||
print("Nothing to format")
|
||||
exit(0)
|
||||
else:
|
||||
src_files, extra_files = format.list_files_to_format()
|
||||
src_files, extra_files = format.list_files_to_format()
|
||||
|
||||
nb_jobs = multiprocessing.cpu_count()
|
||||
|
||||
|
||||
@@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PATCH=$(git diff | base64 -w 0)
|
||||
if [ -n "$PATCH" ]; then
|
||||
echo "Jenkins made some fixes to your PR. To apply these changes to your working directory,"
|
||||
echo "copy and run the following command (triple-click to select the entire line):"
|
||||
echo
|
||||
echo "echo -n $PATCH | base64 -d | git apply -"
|
||||
echo
|
||||
fi
|
||||
Reference in New Issue
Block a user