mirror of
https://github.com/zeldaret/mm.git
synced 2026-08-26 15:36:54 -04:00
Migrate CI to Github Actions (#1885)
* Migrate CI to Github Actions (#1884) Changes the CI system from Jenkins to Github Actions (GHA) like we did for OoT This is an adaptation of the Jenkinsfile we have and the solution made by @Dragorn421 for OoT Relevant OoT PRs: - https://github.com/zeldaret/oot/pull/2740 - https://github.com/zeldaret/oot/pull/2742 - https://github.com/zeldaret/oot/pull/2754 New features include: - Not needing to selfhost a runner - Required private stuff for building is hosted on a private repo - Building multiple versions in parallel - Our old jenkinsfile wasn't building `n64-jp-1.1`. Even if it doesn't match right now, I think it is important checking a PR doesn't completely break it. - Run bss fixer in CI - Upload mapfiles as build artifacts This specific approach to handle GHA for decomp projects was adapted from the one used by the GC/Wii community. It is documented here https://github.com/encounter/dtk-template/blob/main/docs/github_actions.md There's a writeup about this adaptation for N64 projects [here](https://github.com/AngheloAlf/drmario64/pull/19). * Remove duplicated step id * Wire up compiler_archives machinery * Remove extra quote * Select bash as the default shell * Fix asset extraction for WIP versions * Properly fix the assets step this time * fix assets building for jp * Prevent running CI on forks Incorporates the changes from https://github.com/zeldaret/oot/pull/2788 * Incorporate fix_bss tiebreaking algorithm from https://github.com/zeldaret/oot/pull/2779
This commit is contained in:
Anghelo Carvajal
@@ -0,0 +1,20 @@
|
||||
inputs:
|
||||
version:
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Fix BSS
|
||||
shell: sh
|
||||
run: .venv/bin/python3 tools/fix_bss.py -v ${{ inputs.version }}
|
||||
|
||||
- name: Generate patch
|
||||
shell: sh
|
||||
run: git diff > fix_bss_${{ inputs.version }}.patch
|
||||
|
||||
- name: Upload patch
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fix_bss_${{ inputs.version }}.patch
|
||||
path: fix_bss_${{ inputs.version }}.patch
|
||||
@@ -0,0 +1,112 @@
|
||||
# SPDX-FileCopyrightText: © 2026 ZeldaRET
|
||||
# SPDX-License-Identifier: CC0-1.0
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
|
||||
def get_increment_block_numbers(p: Path, version: str):
|
||||
increment_block_numbers: list[int] = []
|
||||
is_in_pragma = False
|
||||
n_fake_structs = None
|
||||
for l in p.read_text().splitlines():
|
||||
if l.startswith("#pragma increment_block_number"):
|
||||
is_in_pragma = True
|
||||
n_fake_structs = 0
|
||||
if is_in_pragma:
|
||||
m = next(re.finditer(rf"{version}:(\d+)", l), None)
|
||||
if m is not None:
|
||||
n_fake_structs = int(m.group(1))
|
||||
if is_in_pragma and not l.endswith("\\"):
|
||||
is_in_pragma = False
|
||||
assert n_fake_structs is not None
|
||||
increment_block_numbers.append(n_fake_structs)
|
||||
n_fake_structs = None
|
||||
return increment_block_numbers
|
||||
|
||||
|
||||
# Formats #pragma increment_block_number as a list of lines
|
||||
def format_pragma(amounts: dict[str, int], max_line_length: int) -> list[str]:
|
||||
lines = []
|
||||
pragma_start = "#pragma increment_block_number "
|
||||
current_line = pragma_start + '"'
|
||||
first = True
|
||||
for version, amount in sorted(amounts.items()):
|
||||
part = f"{version}:{amount}"
|
||||
if len(current_line) + len(" ") + len(part) + len('" \\') > max_line_length:
|
||||
lines.append(current_line + '" ')
|
||||
current_line = " " * len(pragma_start) + '"'
|
||||
first = True
|
||||
if not first:
|
||||
current_line += " "
|
||||
current_line += part
|
||||
first = False
|
||||
lines.append(current_line + '"\n')
|
||||
|
||||
if len(lines) >= 2:
|
||||
# add and align vertically all continuation \ characters
|
||||
n_align = max(map(len, lines[:-1]))
|
||||
for i in range(len(lines) - 1):
|
||||
lines[i] = f"{lines[i]:{n_align}}\\\n"
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def set_increment_block_numbers(
|
||||
p: Path, increment_block_numbers_by_version: dict[str, list[int]]
|
||||
):
|
||||
print(p, increment_block_numbers_by_version)
|
||||
i_pragma = 0
|
||||
is_in_pragma = False
|
||||
pragma_lines = []
|
||||
new_lines = []
|
||||
for l in p.read_text().splitlines(keepends=True):
|
||||
if l.startswith("#pragma increment_block_number"):
|
||||
is_in_pragma = True
|
||||
if not is_in_pragma:
|
||||
new_lines.append(l)
|
||||
if is_in_pragma:
|
||||
pragma_lines.append(l.removesuffix("\\\n"))
|
||||
if is_in_pragma and not l.endswith("\\\n"):
|
||||
is_in_pragma = False
|
||||
pragma_string = "".join(pragma_lines)
|
||||
amounts: dict[str, int] = {}
|
||||
for part in pragma_string.replace('"', "").split()[2:]:
|
||||
version, amount_str = part.split(":")
|
||||
amount = int(amount_str)
|
||||
amounts[version] = amount
|
||||
for (
|
||||
version,
|
||||
increment_block_numbers,
|
||||
) in increment_block_numbers_by_version.items():
|
||||
amounts[version] = increment_block_numbers[i_pragma]
|
||||
i_pragma += 1
|
||||
column_limit = 120 # matches .clang-format's ColumnLimit
|
||||
new_pragma_lines = format_pragma(amounts, column_limit)
|
||||
new_lines.extend(new_pragma_lines)
|
||||
p.write_text("".join(new_lines))
|
||||
|
||||
|
||||
increment_block_numbers_by_version_by_file: dict[Path, dict[str, list[int]]] = {}
|
||||
for p in Path(".").glob("fix_bss_*.patch"):
|
||||
version = p.name.removeprefix("fix_bss_").removesuffix(".patch")
|
||||
subprocess.check_call(["git", "apply", str(p)])
|
||||
touched_files = subprocess.check_output(
|
||||
"git diff --name-only".split(),
|
||||
text=True,
|
||||
).splitlines()
|
||||
for file in touched_files:
|
||||
file_p = Path(file)
|
||||
increment_block_numbers = get_increment_block_numbers(file_p, version)
|
||||
increment_block_numbers_by_version_by_file.setdefault(file_p, {})[
|
||||
version
|
||||
] = increment_block_numbers
|
||||
subprocess.check_call("git checkout -- .".split())
|
||||
|
||||
|
||||
for (
|
||||
file,
|
||||
increment_block_numbers_by_version,
|
||||
) in increment_block_numbers_by_version_by_file.items():
|
||||
set_increment_block_numbers(file, increment_block_numbers_by_version)
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PATCH=$(git diff | base64 -w 0)
|
||||
if [ -n "$PATCH" ]; then
|
||||
echo 'Fixes were made for your PR. To apply these changes to your working directory, copy and run the following command:' >> $GITHUB_STEP_SUMMARY
|
||||
echo '```' >> $GITHUB_STEP_SUMMARY
|
||||
echo "echo -n $PATCH | base64 -d | git apply -" >> $GITHUB_STEP_SUMMARY
|
||||
echo '```' >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
@@ -0,0 +1,151 @@
|
||||
name: Build
|
||||
|
||||
# Build on every branch push, tag push, and pull request change:
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build_repo:
|
||||
# This is a *private* build container.
|
||||
container: ghcr.io/zeldaret/mm-build:main
|
||||
# Prevent running outside zeldaret/mm as fetching the container would fail anyway.
|
||||
if: ${{ github.repository == 'zeldaret/mm' }}
|
||||
|
||||
name: Build repo (${{ matrix.version }})
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
version:
|
||||
- n64-jp-1.1 # N64 Japan 1.1
|
||||
- n64-us # N64 USA
|
||||
include:
|
||||
- version: n64-jp-1.1
|
||||
non_matching: 1
|
||||
|
||||
# By default no version in the matrix contains a non_matching value,
|
||||
# meaning ${{ matrix.non_matching }} expand to an empty string, unless it
|
||||
# is explicitly listed on the `include` block.
|
||||
# Use the value from the matrix if it exists, or fallback to 0 if it doesn't.
|
||||
env:
|
||||
NON_MATCHING: ${{ matrix.non_matching || 0 }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: git config safe.directory
|
||||
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
apt-get install -y git build-essential binutils-mips-linux-gnu curl python3 python3-pip python3-venv libxml2-dev
|
||||
|
||||
- name: Get the dependency
|
||||
run: ln -s /orig/${{ matrix.version }}/baserom.z64 baseroms/${{ matrix.version }}/baserom.z64
|
||||
|
||||
# The compiler archives are embedded in the runner image, to avoid downloading them from GitHub (during make setup), which occasionally fails.
|
||||
- name: Provide compiler archives
|
||||
run: |
|
||||
mkdir -p tools/compiler_archives/
|
||||
ln -s /compiler_archives tools/compiler_archives/archives
|
||||
|
||||
- name: venv
|
||||
run: make -j $(nproc) VERSION=${{ matrix.version }} venv
|
||||
|
||||
- name: Setup
|
||||
run: make -j $(nproc) VERSION=${{ matrix.version }} setup 2> >(tee tools/warnings_count/warnings_setup_new.txt)
|
||||
|
||||
- name: Check setup warnings
|
||||
run: ./tools/warnings_count/compare_warnings.sh setup
|
||||
|
||||
- name: Assets
|
||||
if: matrix.non_matching != 1
|
||||
run: make -j $(nproc) VERSION=${{ matrix.version }} assets 2> >(tee tools/warnings_count/warnings_assets_new.txt)
|
||||
|
||||
# WIP versions do not have asset extraction properly set up yet.
|
||||
# Instead we rely on the assets from the US version for now.
|
||||
- name: Assets US for WIP versions
|
||||
if: matrix.non_matching == 1
|
||||
run: |
|
||||
ln -s /orig/n64-us/baserom.z64 baseroms/n64-us/baserom.z64
|
||||
make -j $(nproc) VERSION=n64-us setup
|
||||
make -j $(nproc) VERSION=${{ matrix.version }} assets 2> >(tee tools/warnings_count/warnings_assets_new.txt)
|
||||
|
||||
- name: Check assets warnings
|
||||
run: ./tools/warnings_count/compare_warnings.sh assets
|
||||
|
||||
- name: Disasm
|
||||
run: make -j $(nproc) VERSION=${{ matrix.version }} disasm 2> >(tee tools/warnings_count/warnings_disasm_new.txt)
|
||||
|
||||
- name: Check disasm warnings
|
||||
run: ./tools/warnings_count/compare_warnings.sh disasm
|
||||
|
||||
- name: Build ${{ matrix.version }}
|
||||
id: build
|
||||
run: make -j $(nproc) VERSION=${{ matrix.version }} rom 2> >(tee tools/warnings_count/warnings_build_new.txt)
|
||||
|
||||
- name: Check build warnings
|
||||
run: ./tools/warnings_count/compare_warnings.sh build
|
||||
|
||||
- name: Compress ${{ matrix.version }}
|
||||
run: make -j $(nproc) VERSION=${{ matrix.version }} compress 2> >(tee tools/warnings_count/warnings_compress_new.txt)
|
||||
|
||||
- name: Check compress warnings
|
||||
run: ./tools/warnings_count/compare_warnings.sh compress
|
||||
|
||||
- name: Fix BSS and generate patch
|
||||
if: failure() && steps.build.outcome == 'failure'
|
||||
uses: ./.github/actions/fix-bss-and-generate-patch
|
||||
with:
|
||||
version: ${{ matrix.version }}
|
||||
|
||||
- name: Show warnings
|
||||
if: failure() && steps.build.outcome == 'failure'
|
||||
run: cat tools/warnings_count/warnings_setup_new.txt tools/warnings_count/warnings_assets_new.txt tools/warnings_count/warnings_disasm_new.txt tools/warnings_count/warnings_build_new.txt tools/warnings_count/warnings_compress_new.txt
|
||||
|
||||
- name: Upload map
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: mm-${{ matrix.version }}.map
|
||||
path: build/${{ matrix.version }}/mm-${{ matrix.version }}.map
|
||||
|
||||
# This job does not do anything, its purpose is to be used as a status check in GitHub rules.
|
||||
all_versions_built:
|
||||
name: All versions built
|
||||
needs: [build_repo]
|
||||
runs-on: ubuntu-latest
|
||||
# Solution 1 from https://github.com/actions/runner/issues/2566#issuecomment-3053484216
|
||||
if: 'always() && ${{ github.repository == ''zeldaret/mm'' }}'
|
||||
steps:
|
||||
- run: |
|
||||
if [ "${{ needs.build_repo.result }}" != "success" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
merge_bss_fixes:
|
||||
name: Merge BSS fixes
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build_repo]
|
||||
if: '!cancelled() && ${{ github.repository == ''zeldaret/mm'' }}' # Run even if build_repo fails
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Download patches
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: fix_bss_*.patch
|
||||
merge-multiple: true
|
||||
|
||||
- name: Apply patches
|
||||
run: python3 .github/scripts/apply_fix_bss_patches.py
|
||||
|
||||
- name: Generate patch
|
||||
run: .github/scripts/generate_patch.sh
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Check format
|
||||
|
||||
# Build on every branch push, tag push, and pull request change:
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Check format
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout reposistory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Install package requirements
|
||||
run: |
|
||||
sudo apt-get install -y python3 clang-format-14 clang-tidy-14
|
||||
|
||||
- name: Check formatting
|
||||
run: tools/check_format.sh
|
||||
Reference in New Issue
Block a user