From 38290003b08c146353b037a7b015d904de5767a8 Mon Sep 17 00:00:00 2001 From: Trueffel <106771418+Trueffeloot@users.noreply.github.com> Date: Sun, 18 Jun 2023 20:26:12 +0200 Subject: [PATCH 1/3] tp-progress (#1) --- .github/workflows/ci-cd.yml | 46 +++++++++++++++++++++++++ .github/workflows/ok-check.yml | 20 ----------- tools/tp.py | 46 +++++-------------------- tools/upload-progress.py | 62 ++++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/ci-cd.yml delete mode 100644 .github/workflows/ok-check.yml create mode 100755 tools/upload-progress.py diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 0000000000..e9cfd8ac21 --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,46 @@ +name: CI/CD + +on: + push: + branches: + - master + pull_request: + branches: + - master + types: [synchronize] + +jobs: + build: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + container: + image: ghcr.io/pheenoh/zeldaret-tp:latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Copy in dol and compilers + run: cp /tmp/baserom.dol ./baserom.dol && cp -r /tmp/mwcc_compiler/ tools/mwcc_compiler && cp tools/mwcc_compiler/2.7/mwcceppc.exe tools/mwcc_compiler/2.7/mwcceppc_modded.exe && chown root /github/home/ + - name: Run Make (OK) + run: make all rels && ./tp check --rels + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: artifact-${{ github.event.pull_request.number }} + path: ./progress-${{ github.event.pull_request.number }}.json + release: + if: ${{ github.event_name == 'push' }} + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: artifact-${{ github.event.pull_request.number }} + - name: Upload Progress to Frogress + env: + PROGRESS_API_KEY: ${{ secrets.FROGRESS_API_KEY }} + run: ./tools/upload-progress.py -b https://progress.deco.mp/ -p twilightprincess -v gcn_usa \ + progress-${{ github.event.pull_request.number }}.json + \ No newline at end of file diff --git a/.github/workflows/ok-check.yml b/.github/workflows/ok-check.yml deleted file mode 100644 index f2c3446476..0000000000 --- a/.github/workflows/ok-check.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: OK Check - -on: pull_request - -jobs: - build: - - runs-on: ubuntu-latest - container: - image: ghcr.io/pheenoh/zeldaret-tp:latest - - steps: - - name: Checkout - uses: actions/checkout@v1 - with: - token: ${{secrets.MY_REPO_PAT}} - - name: Copy in dol and compilers - run: cp /tmp/baserom.dol ./baserom.dol && cp -r /tmp/mwcc_compiler/ tools/mwcc_compiler && cp tools/mwcc_compiler/2.7/mwcceppc.exe tools/mwcc_compiler/2.7/mwcceppc_modded.exe && chown root /github/home/ - - name: Run Make (OK) - run: make all rels && ./tp check --rels diff --git a/tools/tp.py b/tools/tp.py index 9f8c7dd7ba..4773454153 100644 --- a/tools/tp.py +++ b/tools/tp.py @@ -717,43 +717,15 @@ def calculate_progress(build_path: Path, matching: bool, format: str, print_rels ) ) elif format == "JSON": - matching = {} - non_matching = {} - - matching["main.dol"] = { - "decompiled": dol_progress.decompiled, - "total": dol_progress.size, - "sections": [ - { - name: { - "decompiled": sec.decompiled, - "total": sec.size, - } - } - for name, sec in dol_progress.sections.items() - ], - } - - if rels_progress: - matching["rels"] = { - "decompiled": rel_decompiled, - "total": rel_size, - } - - for rel in rels_progress: - matching[rel.name] = { - "decompiled": rel.decompiled, - "total": rel.size, - } - - print( - json.dumps( - { - "matching": matching, - "non-matchgin": non_matching, - } - ) - ) + data = { + "code": decompiled_size, + "code/total": total_size, + "dol": dol_progress.decompiled, + "dol/total": dol_progress.size, + "rels": rel_decompiled, + "rels/total": rel_size, + } + print(json.dumps(data)) else: print(dol_progress.percentage) print(100 * (rel_decompiled / rel_size)) diff --git a/tools/upload-progress.py b/tools/upload-progress.py new file mode 100755 index 0000000000..d12e7caf1d --- /dev/null +++ b/tools/upload-progress.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import subprocess +from pprint import pprint + +import requests + + +def get_git_commit_timestamp() -> int: + return int(subprocess.check_output(['git', 'show', '-s', '--format=%ct']).decode('ascii').rstrip()) + + +def get_git_commit_sha() -> str: + return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip() + + +def generate_url(args: argparse.Namespace) -> str: + url_components = [args.base_url.rstrip('/'), 'data'] + + for arg in [args.project, args.version.replace('.', '-')]: + if arg != "": + url_components.append(arg) + + return str.join('/', url_components) + '/' + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description="Upload progress information.") + parser.add_argument("-b", "--base_url", help="API base URL", required=True) + parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)") + parser.add_argument("-p", "--project", help="Project slug", required=True) + parser.add_argument("-v", "--version", help="Version slug", required=True) + parser.add_argument("input", help="Progress JSON input") + + args = parser.parse_args() + api_key = args.api_key or os.environ.get("PROGRESS_API_KEY") + if not api_key: + raise "API key required" + url = generate_url(args) + + entries = [] + with open(args.input, "r") as f: + data = json.load(f) + entries.append({ + "timestamp": get_git_commit_timestamp(), + "git_hash": get_git_commit_sha(), + "categories": { + "default": data, + }, + }) + + print("Publishing entries to", url) + pprint(entries) + data = { + "api_key": api_key, + "entries": entries, + } + r = requests.post(url, json=data) + r.raise_for_status() + print("Done!") \ No newline at end of file From 42cd2b01e653816d19030f0fb19eb1ebd2f4a1d7 Mon Sep 17 00:00:00 2001 From: Trueffel <106771418+Trueffeloot@users.noreply.github.com> Date: Mon, 19 Jun 2023 21:22:34 +0200 Subject: [PATCH 2/3] Tp-progress-work-2 (#2) --- tools/tp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/tp.py b/tools/tp.py index 4773454153..edb5b908be 100644 --- a/tools/tp.py +++ b/tools/tp.py @@ -717,6 +717,7 @@ def calculate_progress(build_path: Path, matching: bool, format: str, print_rels ) ) elif format == "JSON": + # TODO: add dol sections instead of total dol. data = { "code": decompiled_size, "code/total": total_size, From 67783acac051eef335ed59757e6c3c1dcd1dbde2 Mon Sep 17 00:00:00 2001 From: Trueffel <106771418+Trueffeloot@users.noreply.github.com> Date: Mon, 19 Jun 2023 21:41:00 +0200 Subject: [PATCH 3/3] seperate (#3) --- .github/workflows/cd.yml | 36 ++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 .github/workflows/cd.yml create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 0000000000..73e1532666 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,36 @@ +name: CD + +on: + push: + branches: + - master + +env: + WORKFLOW: "ci.yml" + +jobs: + download: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Get latest workflow run ID + id: get_run_id + run: | + RUN_ID=$(curl --request GET \ + --url https://api.github.com/repos/${{ github.repository }}/actions/workflows/${{ env.WORKFLOW }}/runs \ + --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + --header 'content-type: application/json' | jq '.workflow_runs[0].id') + echo "run_id=$RUN_ID" >> $GITHUB_OUTPUT + - name: Download artifact + uses: dawidd6/action-download-artifact@v2.27.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ steps.get_run_id.outputs.run_id }} + name: artifact-${{ steps.get_run_id.outputs.run_id }} + workflow: ${{ env.WORKFLOW }} + - name: Upload Progress to Frogress + env: + PROGRESS_API_KEY: ${{ secrets.FROGRESS_API_KEY }} + run: ./tools/upload-progress.py -b https://progress.deco.mp/ -p twilightprincess -v gcn_usa \ + progress-${{ steps.get_run_id.outputs.run_id }}.json \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..09ae82e493 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: CI + +on: + pull_request: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + container: + image: ghcr.io/pheenoh/zeldaret-tp:latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Copy in dol and compilers + run: cp /tmp/baserom.dol ./baserom.dol && cp -r /tmp/mwcc_compiler/ tools/mwcc_compiler && cp tools/mwcc_compiler/2.7/mwcceppc.exe tools/mwcc_compiler/2.7/mwcceppc_modded.exe && chown root /github/home/ + - name: Run Make (OK) + run: make all rels && ./tp check --rels + - name: Create JSON for Progress + run: ./tp progress -f JSON > progress-${{ github.run_id }}.json + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: artifact-${{ github.run_id }} + path: ./progress-${{ github.run_id }}.json + \ No newline at end of file