Generate decomp.dev progress reports (#249)

* Generate decomp.dev progress reports - Wire up CI to generate decomp.dev progress reports (using the objdiff report format) and upload them as GHA artifacts, by using `mapfile_parser`. - This also adds a `decomp.yaml` file, which is used by `mapfile_parser` to generate the objdiff reports. - The `progress.py` script used to locally visualize the current progress has been updated to parse objdiff reports too. - `mapfile_parser` 2.13.0 or above is now required. - Delete the old `upload_frogress.py` script. Once we have this we can create an af entry in [decomp.dev](https://decomp.dev/). I'll talk with Encounter to migrate the old frogress progress into decomp.dev too * Activate venv
2026-09-28 23:00:07 -04:00 · 2026-08-16 13:56:24 -04:00
parent d7ab4030a6
commit 4ddba04604
6 changed files with 190 additions and 148 deletions
@@ -86,6 +86,17 @@ jobs:
        if: failure() && steps.build.outcome == 'failure'
        run: cat tools/warnings_count/warnings_setup_new.txt tools/warnings_count/warnings_assets_new.txt tools/warnings_count/warnings_disasm_new.txt tools/warnings_count/warnings_build_new.txt tools/warnings_count/warnings_compress_new.txt

+      - name: Generate objdiff report
+        run: |
+          . .venv/bin/activate
+          python3 -m mapfile_parser objdiff_report report.json --version ${{ matrix.version }}
+
+      - name: Upload progress report artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: ${{ matrix.version }}_report
+          path: report.json
+
      - name: Upload map
        uses: actions/upload-artifact@v7
        with:
@@ -26,3 +26,5 @@ ctx.c.m2c

 .vscode/*
 !.vscode/c_cpp_properties.json
+
+report.json
@@ -0,0 +1,114 @@
+name: Animal Forest
+repo: https://github.com/zeldaret/af
+platform: n64
+build_system: make
+versions:
+  - name: jp
+    fullname: jp
+    sha1: 3ee1f866eb08c0ce57a30e09f0c8b0a16a760c8f
+    paths:
+      target: "baseroms/jp/baserom-decompressed.z64"
+
+      build_dir: "build"
+      map: "build/animalforest-jp.map"
+      compiled_target: "build/animalforest-jp.map"
+      elf: "build/animalforest-jp.elf"
+
+      expected_dir: "expected/"
+
+      asm: "asm/jp"
+      nonmatchings: "asm/jp/nonmatchings"
+
+      compressed_target: "baseroms/jp/baserom.z64"
+      compressed_compiled_target: "build/animalforest-jp-compressed.z64"
+
+tools:
+  mapfile_parser:
+    progress_report:
+      check_asm_paths: False
+      report_data: True
+      prefixes_to_trim:
+        - build/assets/jp/
+        - build/assets/
+        - build/lib/
+        - build/src/
+        - build/asm/jp/data/
+        - build/asm/jp/
+        - build/asm/data/
+        - build/asm/
+        - build/
+
+      # List of categories. `id`s must be unique, but each path may be
+      # duplicated across categories.
+      categories:
+        - id: boot
+          name: Segment boot
+          paths:
+            - boot/
+        - id: dmadata
+          name: Segment dmadata
+          paths:
+            - dmadata/
+        - id: code
+          name: Segment code
+          paths:
+            - code/
+        - id: overlays
+          name: All Overlays
+          paths:
+            - overlays/
+        - id: objects
+          name: All objects
+          paths:
+            - objects/
+
+        - id: assets
+          name: All assets
+          paths:
+            - assets/
+            - misc/
+            - combi/
+
+        - id: buffers
+          name: Buffers
+          paths:
+            - buffers/
+
+        - id: libraries
+          name: Libraries
+          paths:
+            - boot/libc/
+            - boot/libc64/
+            - boot/libm/
+            - boot/libu64/
+            - libultra/
+
+        - id: overlays_actors
+          name: Overlays actors
+          paths:
+            - overlays/actors/
+
+        - id: overlays_furniture
+          name: Overlays furniture
+          paths:
+            - overlays/furniture/
+
+        - id: overlays_gamestates
+          name: Overlays gamestates
+          paths:
+            - overlays/gamestates/
+
+        - id: overlays_managers
+          name: Overlays managers
+          paths:
+            - overlays/managers/
+
+        - id: overlays_submenu
+          name: Overlays submenu
+          paths:
+            - overlays/submenu/
+
+        - id: overlays_weather
+          name: Overlays weather
+          paths:
+            - overlays/weather/
@@ -1,5 +1,5 @@
 splat64[mips]==0.24.0
-mapfile-parser>=2.4.0,<3.0.0
+mapfile-parser>=2.13.0,<3.0.0
 colorama
 libyaz0

@@ -1,132 +1,87 @@
 #!/usr/bin/env python3

-# SPDX-FileCopyrightText: © 2023 ZeldaRET
+# SPDX-FileCopyrightText: © 2023-2025 ZeldaRET
 # SPDX-License-Identifier: MIT

 from __future__ import annotations

 import argparse
-import mapfile_parser
+import sys
 from pathlib import Path

-
-ASMPATH = Path("asm")
-NONMATCHINGS = "nonmatchings"
+import decomp_settings
+import mapfile_parser


-def getProgressFromMapFile(mapFile: mapfile_parser.MapFile, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]=dict(), pathIndex: int=2) -> tuple[mapfile_parser.ProgressStats, dict[str, mapfile_parser.ProgressStats]]:
-    totalStats = mapfile_parser.ProgressStats()
-    progressPerFolder: dict[str, mapfile_parser.ProgressStats] = dict()
+def doThing(
+    version: str,
+    settings: decomp_settings.Config,
+    units: bool,
+    sort: bool,
+    remaining: bool,
+):
+    selectedVersion = settings.get_version_by_name(version)
+    assert selectedVersion is not None
+    mapPath = Path(selectedVersion.paths.map)
+    outputPath = Path(selectedVersion.paths.build_dir) / "report.json"

-    for segment in mapFile:
-        for file in segment:
-            if len(file) == 0:
-                continue
+    specificSettings = mapfile_parser.frontends.objdiff_report.SpecificSettings.fromDecompConfig(settings)
+    assert specificSettings is not None
+    prefixesToTrim = specificSettings.prefixesToTrim
+    pathIndex = specificSettings.pathIndex
+    assert pathIndex is not None

-            folder = file.filepath.parts[pathIndex]
+    reportCategories = mapfile_parser.ReportCategories()
+    for cat in specificSettings.categories:
+        reportCategories.push(cat.ide, cat.name, cat.paths)

-            if folder in aliases:
-                folder = aliases[folder]
+    if specificSettings.checkAsmPaths:
+        assert selectedVersion.paths.asm is not None
+        asmPath = Path(selectedVersion.paths.asm)
+    else:
+        asmPath = None

-            if folder not in progressPerFolder:
-                progressPerFolder[folder] = mapfile_parser.ProgressStats()
+    summaryTableConfig = mapfile_parser.frontends.objdiff_report.SummaryTableConfig(
+        doUnits=units,
+        sort=sort,
+        remaining=remaining,
+    )

-            originalFilePath = Path(*file.filepath.parts[pathIndex:])
-
-            extensionlessFilePath = originalFilePath
-            while extensionlessFilePath.suffix:
-                extensionlessFilePath = extensionlessFilePath.with_suffix("")
-
-            fullAsmFile = asmPath / extensionlessFilePath.with_suffix(".s")
-
-            handwrittenAsmFiles = [Path("boot/libc64/fp.o"), Path("makerom/entry.o")]
-            
-            if originalFilePath in handwrittenAsmFiles:
-                wholeFileIsUndecomped = False
-            else:
-                wholeFileIsUndecomped = fullAsmFile.exists()
-
-
-            for func in file:
-                funcAsmPath = nonmatchings / extensionlessFilePath / f"{func.name}.s"
-
-                symSize = 0
-                if func.size is not None:
-                    symSize = func.size
-
-                if wholeFileIsUndecomped:
-                    totalStats.undecompedSize += symSize
-                    progressPerFolder[folder].undecompedSize += symSize
-                elif funcAsmPath.exists():
-                    totalStats.undecompedSize += symSize
-                    progressPerFolder[folder].undecompedSize += symSize
-                else:
-                    totalStats.decompedSize += symSize
-                    progressPerFolder[folder].decompedSize += symSize
-
-    return totalStats, progressPerFolder
-
-
-def getProgress(mapPath: Path, version: str) -> tuple[mapfile_parser.ProgressStats, dict[str, mapfile_parser.ProgressStats]]:
-    mapFile = mapfile_parser.MapFile()
-    mapFile.readMapFile(mapPath)
-
-    for segment in mapFile:
-        for file in segment:
-            if len(file) == 0:
-                continue
-
-            filepathParts = list(file.filepath.parts)
-            if version in filepathParts:
-                filepathParts.remove(version)
-            file.filepath = Path(*filepathParts)
-
-    nonMatchingsPath = ASMPATH / version / NONMATCHINGS
-
-    return getProgressFromMapFile(mapFile.filterBySectionType(".text"), ASMPATH / version, nonMatchingsPath, aliases={"ultralib": "libultra"})
-
-def getAssetProgress(mapPath: Path, version: str):
-    totalStats = mapfile_parser.ProgressStats()
-    progressPerFolder: dict[str, mapfile_parser.ProgressStats] = dict()
-    progressPerFolder["objects"] = mapfile_parser.ProgressStats()
-    progressPerFolder["unidentified"] = mapfile_parser.ProgressStats()
-
-    mapFile = mapfile_parser.MapFile()
-    mapFile.readMapFile(mapPath)
-    mapFile = mapFile.filterBySectionType(".data")
-
-    for segment in mapFile:
-        for file in segment:
-
-            if len(file) == 0:
-                continue
-
-            if str(file.filepath).startswith("build/src/objects"):
-                totalStats.decompedSize += file.size
-                progressPerFolder["objects"].decompedSize += file.size
-            elif str(file.filepath).startswith("build/assets/" + version + "/objects"):
-                totalStats.undecompedSize += file.size
-                progressPerFolder["objects"].undecompedSize += file.size
-            elif str(file.filepath).startswith("build/assets/" + version):
-                progressPerFolder["unidentified"].undecompedSize += file.size
-                totalStats.undecompedSize += file.size
-    
-    return totalStats, progressPerFolder
+    exitcode = mapfile_parser.frontends.objdiff_report.doObjdiffReport(
+        mapPath,
+        outputPath,
+        prefixesToTrim,
+        reportCategories,
+        pathIndex=pathIndex,
+        asmPath=asmPath,
+        summaryTableConfig=summaryTableConfig,
+    )
+    sys.exit(exitcode)

 def progressMain():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-v", "--version", help="version to process", default="jp")
+    settings = decomp_settings.scan_for_config()
+    versionsChoices = [x.name for x in settings.versions]
+
+    parser = argparse.ArgumentParser(description="Print the progress for each category in your terminal.")
+    parser.add_argument("-v", "--version", help="version to process", choices=versionsChoices, default=versionsChoices[0])
+    parser.add_argument("-u", "--units", help="Print units instead of categories", action="store_true")
+    parser.add_argument("-s", "--sort", help="Sort by decomped size", action="store_true")
+    parser.add_argument("-r", "--remaining", help="Print an extra column indicating the remaining percentage to match of each entry", action="store_true")

    args = parser.parse_args()
+    version: str = args.version
+    units: bool = args.units
+    sort: bool = args.sort
+    remaining: bool = args.remaining

-    mapPath = Path("build") / f"animalforest-{args.version}.map"
+    doThing(
+        version,
+        settings,
+        units,
+        sort,
+        remaining,
+    )

-    codeTotalStats, codeProgressPerFolder = getProgress(mapPath, args.version)
-    assetTotalStats, assetProgressPerFolder = getAssetProgress(mapPath, args.version)
-    print("code:")
-    mapfile_parser.progress_stats.printStats(codeTotalStats, codeProgressPerFolder)
-    print("\n\nassets:")
-    mapfile_parser.progress_stats.printStats(assetTotalStats, assetProgressPerFolder)

 if __name__ == "__main__":
    progressMain()
@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-
-# SPDX-FileCopyrightText: © 2023 ZeldaRET
-# SPDX-License-Identifier: MIT
-
-from __future__ import annotations
-
-import argparse
-import mapfile_parser
-from pathlib import Path
-
-import progress
-
-
-BASE_URL = "https://progress.deco.mp"
-PROJECT = "animalforest"
-
-def uploadProgressMain():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("version", help="Version slug")
-    parser.add_argument("--apikey", help="API key")
-
-    args = parser.parse_args()
-
-    version: str = args.version
-    apikey: str = args.apikey
-    mapPath = Path("build") / f"animalforest-{args.version}.map"
-
-    codeTotalStats, codeProgressPerFolder = progress.getProgress(mapPath, version)
-    codeEntries: dict[str, int] = mapfile_parser.frontends.upload_frogress.getFrogressEntriesFromStats(codeTotalStats, codeProgressPerFolder, verbose=True)
-
-    assetTotalStats, assetProgressPerFolder = progress.getAssetProgress(mapPath, version)
-    assetEntries: dict[str, int] = mapfile_parser.frontends.upload_frogress.getFrogressEntriesFromStats(assetTotalStats, assetProgressPerFolder, verbose=True)
-
-    url = mapfile_parser.utils.generateFrogressEndpointUrl(BASE_URL, PROJECT, version)
-    mapfile_parser.frontends.upload_frogress.uploadEntriesToFrogress(codeEntries, "code", url, apikey=apikey, verbose=True)
-    mapfile_parser.frontends.upload_frogress.uploadEntriesToFrogress(assetEntries, "assets", url, apikey=apikey, verbose=True)
-
-if __name__ == '__main__':
-    uploadProgressMain()