Merge branch 'master' of github.com:water111/jak-project into w/fcs-debug-vis

.clang-format

@@ -1,4 +1,18 @@
 ---
 BasedOnStyle: Chromium
 ColumnLimit: 100
-SortIncludes: false
+SortIncludes: true
+IncludeBlocks: Regroup
+IncludeCategories:
+  - Regex: '^<[a-zA-Z_\-\.]+>'
+    Priority: 1
+  - Regex: '^"[a-zA-Z_\-\.]+"'
+    Priority: 2
+  - Regex: '^(<|")common.*'
+    Priority: 3
+  - Regex: '^"third-party.*'
+    Priority: 4
+    SortPriority: 5
+  - Regex: '.*'
+    Priority: 5
+    SortPriority: 4

.gitattributes

@@ -3,6 +3,8 @@ third-party/**/* linguist-generated
 test/decompiler/reference/** linguist-vendored
 # suppress from PR diffs - is just reviewing the same thing twice (if properly added to g_src)
 test/decompiler/reference/** linguist-generated
+goal_src/import/** linguist-vendored
+goal_src/import/** linguist-generated
 *.gc linguist-language=lisp
 *.gd linguist-language=lisp
 *.gs linguist-language=Scheme

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,81 @@
+name: "\U0001F41B Bug Report"
+description: Create a bug report.
+labels: ["bug"]
+body:
+  - type: checkboxes
+    attributes:
+      label: Acknowledgements
+      description: Take a couple minutes to help our maintainers work faster.
+      options:
+        - label: I have [searched](https://github.com/open-goal/jak-project/issues?q=is%3Aissue+is%3Aopen+label%3Abug+sort%3Aupdated-desc) for duplicate or closed bug reports
+          required: true
+        - label: I understand that I am supposed to provide my own legitimately obtained copy of the game
+          required: true
+
+  - type: textarea
+    attributes:
+      label: Describe the Bug
+      description: A clear and concise description of what the bug is. You may post screenshots or videos of the bug here.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: How To Reproduce
+      description: Steps to reproduce the behavior. You can also post a video of it here.
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Does this problem occur on original hardware or PCSX2?
+      description: Some things that may seem like bugs are actually exactly how the original game behaved.
+      options:
+        - Yes, it's unique to OpenGOAL
+        - Didn't check
+        - Not needed, bug is obvious
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Expected Behavior
+      description: A clear and concise description of the expected behavior.
+      placeholder: When I do X, Y should happen.
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: Operating System
+      description: Windows version, Linux distribution, etc.
+    validations:
+      required: true
+
+  - type: input
+    attributes:
+      label: OpenGOAL Version
+      description: Either the release version (vX.X.X) or the commit sha if you've built from source
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Game Version
+      options:
+        - NTSC 1.0 (black label)
+        - NTSC Greatest Hits version (red label)
+        - PAL
+        - JP
+    validations:
+      required: true
+
+  - type: dropdown
+    attributes:
+      label: Are you using a high refresh-rate monitor (>60Hz)
+      options:
+        - "No"
+        - "Yes"
+    validations:
+      required: true
+

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: true

.github/ISSUE_TEMPLATE/enhancement-request.yml

@@ -0,0 +1,24 @@
+name: "\U0001F680 Enhancement Request"
+description: Suggest an improvement for an existing feature.
+labels: ["enhancement"]
+body:
+  - type: textarea
+    attributes:
+      label: What feature is your idea related to?
+      description: A clear and concise description of what the problem is.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Describe the solution you'd like.
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Additional context
+      description: Any other context or screenshots about the enhancement request here.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,24 @@
+name: "\U0001F3AE Game Feature Request"
+description: Suggest an idea for the game
+labels: ["game"]
+body:
+  - type: textarea
+    attributes:
+      label: Is your feature request related to a problem?
+      description: A clear and concise description of what the problem is.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Describe the solution you'd like.
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false

.github/scripts/releases/error-code-metadata.json

@@ -0,0 +1,29 @@
+{
+  "4000": {
+    "msg": "Validation Failed: Cannot locate ELF"
+  },
+  "4001": {
+    "msg": "Validation Failed: Game serial missing from valid game database"
+  },
+  "4002": {
+    "msg": "Validation Failed: ELF missing from valid game database"
+  },
+  "4010": {
+    "msg": "Validation Failed: Bad or unexpected ISO contents"
+  },
+  "4011": {
+    "msg": "Validation Failed: Extracted an unexpected amount of files from ISO"
+  },
+  "4020": {
+    "msg": "Validation Failed: ISO Extraction failed, output directory did not contain expected files"
+  },
+  "4030": {
+    "msg": "Decompilation Failed: An error occurred"
+  }
+  "4040": {
+    "msg": "Extraction Failed: Provided game data file was not an ISO file"
+  }
+  "4041": {
+    "msg": "Extraction Failed: Provided game data file was corrupt or has unexpected size"
+  }
+}

.github/scripts/releases/extract_build_linux.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -e
+
+DEST=${1}
+SOURCE=${2}
+
+mkdir -p $DEST
+
+cp $SOURCE/build/game/gk $DEST
+cp $SOURCE/build/goalc/goalc $DEST
+cp $SOURCE/build/decompiler/extractor $DEST
+
+strip $DEST/gk
+strip $DEST/goalc
+strip $DEST/extractor
+
+mkdir -p $DEST/data
+mkdir -p $DEST/data/launcher/
+mkdir -p $DEST/data/decompiler/
+mkdir -p $DEST/data/assets
+mkdir -p $DEST/data/game
+mkdir -p $DEST/data/log
+mkdir -p $DEST/data/game/graphics/opengl_renderer/
+
+cp -r $SOURCE/.github/scripts/releases/error-code-metadata.json $DEST/data/launcher/error-code-metadata.json
+cp -r $SOURCE/decompiler/config $DEST/data/decompiler/
+cp -r $SOURCE/goal_src $DEST/data
+cp -r $SOURCE/game/assets $DEST/data/game/
+cp -r $SOURCE/game/graphics/opengl_renderer/shaders $DEST/data/game/graphics/opengl_renderer
+cp -r $SOURCE/custom_levels $DEST/data

.github/scripts/releases/extract_build_windows.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -e
+
+DEST=${1}
+SOURCE=${2}
+
+mkdir -p $DEST
+
+cp $SOURCE/build/bin/gk.exe $DEST
+cp $SOURCE/build/bin/goalc.exe $DEST
+cp $SOURCE/build/bin/extractor.exe $DEST
+
+mkdir -p $DEST/data
+mkdir -p $DEST/data/launcher/
+mkdir -p $DEST/data/decompiler/
+mkdir -p $DEST/data/assets
+mkdir -p $DEST/data/game
+mkdir -p $DEST/data/log
+mkdir -p $DEST/data/game/graphics/opengl_renderer/
+
+cp -r $SOURCE/.github/scripts/releases/error-code-metadata.json $DEST/data/launcher/error-code-metadata.json
+cp -r $SOURCE/decompiler/config $DEST/data/decompiler/
+cp -r $SOURCE/goal_src $DEST/data
+cp -r $SOURCE/game/assets $DEST/data/game/
+cp -r $SOURCE/game/graphics/opengl_renderer/shaders $DEST/data/game/graphics/opengl_renderer
+cp -r $SOURCE/custom_levels $DEST/data

.github/scripts/releases/upload-release-artifacts/index.js

@@ -111,12 +111,8 @@ const { data: releaseAssetsPost } = await octokit.rest.repos.listReleaseAssets({
 
 // Expected Assets, if we have all of them, we will publish it
 let expectedAssets = {
-  "windows-decompiler": false,
-  "windows-gk": false,
-  "windows-goalc": false,
-  "linux-decompiler": false,
-  "linux-gk": false,
-  "linux-goalc": false,
+  "windows": false,
+  "linux" : false,
 }
 
 for (var i = 0; i < releaseAssetsPost.length; i++) {

.github/workflows/inform-pages-repo.yaml

@@ -20,7 +20,7 @@ jobs:
     steps:
       - name: Send Dispatch
         if: github.repository == 'open-goal/jak-project'
-        uses: peter-evans/repository-dispatch@v1
+        uses: peter-evans/repository-dispatch@v2
         with:
           token: ${{ secrets.BOT_PAT }}
           repository: 'open-goal/open-goal.github.io'

.github/workflows/linter-workflow.yaml

@@ -21,7 +21,9 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Get Package Dependencies
-        run: sudo apt install clang-format clang-tidy
+        run: |
+          sudo apt install clang-format clang-tidy
+          clang-format -version
 
       - name: Check Clang-Formatting
         run: |

.github/workflows/linux-workflow.yaml

@@ -41,7 +41,7 @@ jobs:
         run: git submodule update --init --recursive -j 2
 
       - name: Get Common Package Dependencies
-        run: sudo apt install build-essential cmake clang gcc g++ lcov make nasm libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev
+        run: sudo apt install build-essential cmake clang gcc g++ lcov make nasm libxrandr-dev libxinerama-dev libxcursor-dev libpulse-dev libxi-dev zip
 
       - name: Get Clang
         if: matrix.compiler == 'clang'
@@ -53,7 +53,7 @@ jobs:
           cache_key: ${{ matrix.os }}-${{ matrix.compiler }}
 
       - name: CMake Generation - Clang
-        if: matrix.compiler == 'clang'
+        if: matrix.compiler == 'clang' && !startsWith(github.ref, 'refs/tags/')
         run: |
           export CC=clang
           export CXX=clang++
@@ -62,7 +62,19 @@ jobs:
             -DBUILD_FOR_RELEASE=ON \
             -DCMAKE_C_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache \
             -DCMAKE_CXX_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache \
-            -DCODE_COVERAGE=ON -DASAN_BUILD=ON
+            -DASAN_BUILD=ON
+
+      - name: CMake Generation - Clang - No ASAN
+        if: matrix.compiler == 'clang' && startsWith(github.ref, 'refs/tags/')
+        run: |
+          export CC=clang
+          export CXX=clang++
+          cmake -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DBUILD_FOR_RELEASE=ON \
+            -DCMAKE_C_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER="${{ github.workspace }}"/buildcache/bin/buildcache \
+            -DASAN_BUILD=OFF
 
       - name: CMake Generation - GCC
         if: matrix.compiler == 'gcc'
@@ -88,7 +100,7 @@ jobs:
         run: ./test_code_coverage.sh
 
       - name: Submit Coverage Report to Codacy
-        if: ${{ matrix.compiler }} == 'gcc'
+        if: matrix.compiler == 'gcc'
         uses: codacy/codacy-coverage-reporter-action@v1
         continue-on-error: true
         with:
@@ -100,22 +112,17 @@ jobs:
       - name: Prepare Build Artifacts
         if: github.repository == 'open-goal/jak-project' && startsWith(github.ref, 'refs/tags/') && matrix.compiler == 'clang'
         run: |
-          mkdir -p ./ci-artifacts/
-          cp ./build/decompiler/decompiler ./ci-artifacts/linux-decompiler.bin
-          cp ./build/game/gk ./ci-artifacts/linux-gk.bin
-          cp ./build/goalc/goalc ./ci-artifacts/linux-goalc.bin
-          chmod +x ./ci-artifacts/*.bin
-          strip --strip-debug ./ci-artifacts/linux-decompiler.bin
-          strip --strip-debug ./ci-artifacts/linux-gk.bin
-          strip --strip-debug ./ci-artifacts/linux-goalc.bin
-          ls -l ./ci-artifacts/
+          mkdir -p ./ci-artifacts/out
+          ./.github/scripts/releases/extract_build_linux.sh ./ci-artifacts/out ./
+          cd ci-artifacts/out
+          tar czf ../linux.tar.gz .
 
       - name: Upload Assets and Potential Publish Release
         if: github.repository == 'open-goal/jak-project' && startsWith(github.ref, 'refs/tags/') && matrix.compiler == 'clang'
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_PAT }}
           ASSET_DIR: ${{ github.WORKSPACE }}/ci-artifacts
-          ASSET_EXTENSION: bin
+          ASSET_EXTENSION: gz
           TAG_TO_SEARCH_FOR: ${{ github.REF }}
         run: |
           cd ./.github/scripts/releases/upload-release-artifacts

.github/workflows/update-controller-db.yaml

@@ -0,0 +1,26 @@
+name: Update Controller Database
+
+on:
+  schedule:
+    - cron: "0 16 * * 1" # every monday @ 12pm EST - https://crontab.guru/#0_16_*_*_1
+
+jobs:
+  update-controller-db:
+    if: github.repository == 'open-goal/jak-project'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.BOT_PAT }}
+
+      - name: Get Latest DB
+        run: |
+          wget -O ./game/assets/sdl_controller_db.txt https://raw.githubusercontent.com/gabomdq/SDL_GameControllerDB/master/gamecontrollerdb.txt
+
+      - name: commit version bump
+        uses: EndBug/add-and-commit@v9
+        with:
+          default_author: github_actor
+          author_name: "OpenGOALBot"
+          author_email: "OpenGOALBot@users.noreply.github.com"
+          message: "Updating Controller Database"

.github/workflows/windows-workflow.yaml

@@ -81,19 +81,18 @@ jobs:
       # ---- Release / Tagging related steps ----
       - name: Prepare Build Artifacts
         if: github.repository == 'open-goal/jak-project' && startsWith(github.ref, 'refs/tags/') && matrix.compiler == 'clang'
+        shell: bash
         run: |
-          mkdir -p ./ci-artifacts/
-          cp ./build/bin/decompiler.exe ./ci-artifacts/windows-decompiler.exe
-          cp ./build/bin/gk.exe ./ci-artifacts/windows-gk.exe
-          cp ./build/bin/goalc.exe ./ci-artifacts/windows-goalc.exe
-          ls ./ci-artifacts/
+          mkdir -p ./ci-artifacts/out
+          ./.github/scripts/releases/extract_build_windows.sh ./ci-artifacts/out ./
+          7z a -tzip ./ci-artifacts/windows.zip ./ci-artifacts/out/*
 
       - name: Upload Assets and Potential Publish Release
         if: github.repository == 'open-goal/jak-project' && startsWith(github.ref, 'refs/tags/') && matrix.compiler == 'clang'
         env:
           GITHUB_TOKEN: ${{ secrets.BOT_PAT }}
           ASSET_DIR: ${{ github.WORKSPACE }}/ci-artifacts
-          ASSET_EXTENSION: exe
+          ASSET_EXTENSION: zip
           TAG_TO_SEARCH_FOR: ${{ github.REF }}
         run: |
           cd ./.github/scripts/releases/upload-release-artifacts

.gitignore

@@ -6,6 +6,7 @@ cmake-build-debug/*
 .idea/*
 build/*
 decompiler_out/*
+decompiler_out2/*
 logs/*
 
 # wsl apparently builds to here?
@@ -39,4 +40,8 @@ imgui.ini
 node_modules/
 
 # texture replacements
-texture_replacements/*
+texture_replacements/*
+
+# generated cmake files
+svnrev.h
+ci-artifacts/

.vs/launch.vs.json

@@ -75,7 +75,7 @@
       "project" : "CMakeLists.txt",
       "projectTarget" : "goalc.exe (bin\\goalc.exe)",
       "name" : "Run - REPL - Auto Listen",
-      "args" : [ "-auto-lt" ]
+      "args" : [ "--auto-lt" ]
     },
     {
       "type" : "default",
@@ -84,6 +84,13 @@
       "name" : "Run - Decompiler - Jak 1",
       "args" : [ "${workspaceRoot}/decompiler/config/jak1_ntsc_black_label.jsonc", "${workspaceRoot}/iso_data", "${workspaceRoot}/decompiler_out"]
     },
+    {
+      "type" : "default",
+      "project" : "CMakeLists.txt",
+      "projectTarget" : "decompiler.exe (bin\\decompiler.exe)",
+      "name" : "Run - Decompiler - Jak 1 - Data Only",
+      "args" : [ "${workspaceRoot}/decompiler/config/jak1_ntsc_black_label.jsonc", "${workspaceRoot}/iso_data", "${workspaceRoot}/decompiler_out", "decompile_code=false"]
+    },
     {
       "type" : "default",
       "project" : "CMakeLists.txt",
@@ -125,6 +132,13 @@
       "projectTarget" : "dgo_unpacker.exe (bin\\dgo_unpacker.exe)",
       "name" : "Run - DGO Unpacker (test)",
       "args" : [ "C:\\GameData\\Jak1\\Backup\\DGO-PAL\\GAME", "C:\\GameData\\Jak1\\Backup\\DISC-PAL\\CGO\\GAME.CGO"]
+    },
+    {
+      "type" : "default",
+      "project" : "CMakeLists.txt",
+      "projectTarget" : "extractor.exe (bin\\extractor.exe)",
+      "name" : "Run - Extractor - Extract",
+      "args" : [ "\"C:\\Game\\PS2SMB\\DVD\\SCPS_150.21.Jak & Daxter - TPL (NTSC-J).iso\""]
     }
   ]
 }

CMakeLists.txt

@@ -21,20 +21,24 @@ endif()
 if(MSVC AND (CMAKE_CXX_COMPILER_ID STREQUAL "Clang"))
     message(STATUS "Clang on MSVC detected! Adding compile flags")
     set(CMAKE_CXX_FLAGS
-        "-Xclang -fcxx-exceptions \
+        "${CMAKE_CXX_FLAGS} \
+        -Xclang -fcxx-exceptions \
         -Xclang -fexceptions \
         -Xclang -std=c++17 \
         -Xclang -D_CRT_SECURE_NO_WARNINGS \
-        -mavx \
-        -Wno-c++11-narrowing -Wno-c++98-compat -W3")
-
-    # additional c++ flags for release mode for our projects
-    if(CMAKE_BUILD_TYPE MATCHES "Release")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ob2")
-    endif()
+        /arch:AVX \
+        -Wno-c++11-narrowing -Wno-c++98-compat -Wno-c++20-compat -W3")
 
     # linker flags
-    set(CMAKE_EXE_LINKER_FLAGS "/STACK:16000000,16384")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:16000000,16384")
+
+    # additional c++ and linker flags for release mode for our projects
+    if(CMAKE_BUILD_TYPE MATCHES "Release")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ob2")
+    elseif(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ob2")
+        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
+    endif()
 
 elseif(UNIX)
     message(STATUS "GCC detected! Adding compile flags")
@@ -69,10 +73,18 @@ elseif(MSVC)
       # set(CMAKE_CXX_FLAGS_DEBUG "/ZI")
     endif()
     # c++ flags for all build types
-    set(CMAKE_CXX_FLAGS "/EHsc /utf-8 /arch:AVX")
-    
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /utf-8 /arch:AVX")
+
     # linker flags
-    set(CMAKE_EXE_LINKER_FLAGS "/STACK:16000000,16384")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:16000000,16384")
+    
+    # additional c++ and linker flags for specific build types
+    if(CMAKE_BUILD_TYPE MATCHES "Release")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ob2")
+    elseif(CMAKE_BUILD_TYPE MATCHES "RelWithDebInfo")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /Ob2")
+        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
+    endif()
 endif()
 
 if(WIN32)
@@ -80,6 +92,9 @@ if(WIN32)
     set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
     set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
     set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+    
+    set(CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION 7.1.7600.0.30514) # win7.1, supports xp
+    message("Windows SDK version: ${CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION}")
 endif()
 
 if(ASAN_BUILD)
@@ -112,6 +127,8 @@ add_subdirectory(common)
 # build decompiler
 add_subdirectory(decompiler)
 
+add_subdirectory(third-party/cubeb)
+
 # build glfw library
 add_subdirectory(third-party/glfw)
 add_subdirectory(third-party/zstd)
@@ -145,6 +162,11 @@ add_subdirectory(third-party/lzokay EXCLUDE_FROM_ALL)
 
 # build format library
 add_subdirectory(third-party/fmt EXCLUDE_FROM_ALL)
+add_subdirectory(third-party/stb_image EXCLUDE_FROM_ALL)
+add_subdirectory(third-party/tiny_gltf EXCLUDE_FROM_ALL)
+add_subdirectory(third-party/xdelta3 EXCLUDE_FROM_ALL)
+
+
 # discord rich presence
 include_directories(third-party/discord-rpc/include)
 add_subdirectory(third-party/discord-rpc EXCLUDE_FROM_ALL)

CMakePresets.json

@@ -54,6 +54,45 @@
       },
       "vendor": { "microsoft.com/VisualStudioSettings/CMake/1.0": { "hostOS": [ "Windows" ] } }
     },
+    {
+      "name": "RelWithDebInfo-clang",
+      "displayName": "Windows RelWithDebInfo (clang-cl)",
+      "description": "Target Windows with the Visual Studio development environment.",
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/out/build/Release",
+      "architecture": {
+        "value": "x64",
+        "strategy": "external"
+      },
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
+        "INSTALL_GTEST": "True",
+        "CMAKE_C_COMPILER": "clang-cl",
+        "CMAKE_CXX_COMPILER": "clang-cl"
+      },
+      "vendor": { "microsoft.com/VisualStudioSettings/CMake/1.0": { "hostOS": [ "Windows" ] } }
+    },
+    {
+      "name": "Release-clang-static",
+      "displayName": "Windows Release - Static (clang-cl)",
+      "description": "Target Windows with the Visual Studio development environment.",
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/out/build/Release",
+      "architecture": {
+        "value": "x64",
+        "strategy": "external"
+      },
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
+        "INSTALL_GTEST": "True",
+        "CMAKE_C_COMPILER": "clang-cl",
+        "CMAKE_CXX_COMPILER": "clang-cl",
+        "BUILD_FOR_RELEASE": "true"
+      },
+      "vendor": { "microsoft.com/VisualStudioSettings/CMake/1.0": { "hostOS": [ "Windows" ] } }
+    },
     {
       "name": "Debug-msvc",
       "displayName": "Windows Debug (msvc)",

FAQ.md

@@ -1,73 +1,72 @@
-### How is this project possible? 
+### How finished is the project?
+We estimate it is about 80% done. Most of the renderers and sound are finished, but there are still a few bugs.
+The project is still in development. If you want to give it a try, feel free, but we don't provide detailed support yet.
+
+### Is there a way to try it without compiling it myself?
+Yes, there are release versions available here: https://github.com/open-goal/jak-project/releases
+Download the latest one and extract all the files. Drag an ISO of the game to the "extractor" program to extract the assets. After that finishes, you can run the "gk" program to start playing.
+
+### How is this project possible?
 Naughty Dog created a custom programming language called [GOAL](https://en.wikipedia.org/wiki/Game_Oriented_Assembly_Lisp), along with a compiler for it. GOAL makes up 98% of all the code in Jak & Daxter, and the GOAL compiler does not have the same amount of optimizations as C or C++ compilers have, which means that the compiler often times left specific instructions that denote some specific kind of control flow, which lets us piece back the original structure of the code accurately for about 99% of the functions in the game. Nearly every type in GOAL also has an autogenerated debug `inspect` method, which prints all of a type's fields by default. These `inspect` methods can tell us what fields were in the type, at what offset, and what their type was.
 
-### How finished is the project?
-We estimate it is about 75% done. Most of the renderers are finished and there is no sound. 
-
 ### When will the project be finished?
-Hopefully just a few more months! 
-
-### Why is there no sound?
-We have not started work on sound at the moment. 
+Hopefully just a few more months!
 
 ### Will you also decompile Jak II?
-Likely yes! 
+Yes! But we haven't started yet.
 
 ### Will you also decompile Jak X?
-We don't know yet. 
+We don't know yet.
 
-### How can I run the game?
-The project readme contains instructions on installing and running the game. You need to bring your own copy of the game, we provide none of its assets. 
+### How can I compile the project for myself?
+The project readme contains instructions on installing and running the game. You need to bring your own copy of the game, we provide none of its assets.
 
 ### I am having trouble running the game!
-Keep in mind the project is currently in development and is not adapted for end users at the moment. If you are having trouble, there are some extra troubleshooting steps at the bottom of this page. If the common issues do not include your specific issue, feel free to join the discord server and ask for help in the help channel. 
+Keep in mind the project is currently in development and is not adapted for end users at the moment. If you are having trouble, there are some extra troubleshooting steps at the bottom of this page. If the common issues do not include your specific issue, feel free to join the discord server and ask for help in the help channel.
 
 ### Does the game have differences? I don't remember these options!
-We have added a plethora of options to the game settings (and removed some that didn't make sense) so that you can have a more up to date experience, or a more PS2-like experience if you decide. It is up to you! There are also a bunch of extra goodies and added secrets to find out. We are aiming to keep the core gameplay (controls, physics, behaviors, etc.) identical however, so if you find any issues or differences with this then do not hesitate to tell us about it. 
+We have added a plethora of options to the game settings (and removed some that didn't make sense) so that you can have a more up to date experience, or a more PS2-like experience if you decide. It is up to you! There are also a bunch of extra goodies and added secrets to find out. We are aiming to keep the core gameplay (controls, physics, behaviors, etc.) identical however, so if you find any issues or differences with this then do not hesitate to tell us about it.
 
 ### Where can I find a list of differences?
-[This page](https://github.com/open-goal/jak-project/issues/386) contains a list of ideas and suggestions we have for the game, but it does not cover all the deliberate changes we have made. We will probably compile a list at some point! In the meantime, feel free to ask us about this. 
+[This page](https://github.com/open-goal/jak-project/issues/386) contains a list of ideas and suggestions we have for the game, but it does not cover all the deliberate changes we have made. We will probably compile a list at some point! In the meantime, feel free to ask us about this.
 
 ### Will there be bug fixes to the game?
-The original game had a few issues of varying severity - from progress softlocks to crashing - that we are aiming to patch. The complexity of these issues is varying (if they were simple to reproduce and fix, Naughty Dog likely would have done it), but so far we have fixed 2 issues that could crash the game. 
+The original game had a few issues of varying severity - from progress softlocks to crashing - that we are aiming to patch. The complexity of these issues is varying (if they were simple to reproduce and fix, Naughty Dog likely would have done it), but so far we have fixed 2 issues that could crash the game.
 
 ### Will there be mod support?
-In the future yes! We intend to make it as easy as it can be to make your own code or import graphics for the game. 
+In the future yes! We intend to make it as easy as we can to make your own code or import graphics for the game.
 
 ### What is this menu? Why am I not dying sometimes?
-This might be the result of the game beeing booted in debugging mode. The project is still in development and we rely on the game's built-in debug features (along with some we made ourselves) for testing and development. The final product will have this debug mode disabled. In the meantime, have fun with all the options! Keep in mind some don't work very well. 
+This might be the result of the game being booted in debugging mode. The project is still in development and we rely on the game's built-in debug features (along with some we made ourselves) for testing and development. The final product will have this debug mode disabled. In the meantime, have fun with all the options! Keep in mind some don't work very well.
 
 ### What are the minimum requirements?
 These are not totally set in stone as the project is not complete. These are the minimum architecture requirements for the project:
 - x86-64 CPU with AVX support. Most, if not all, 64-bit CPUs should work. OpenGOAL has a very small CPU footprint, so most modern CPUs will run the game fine.
 - Graphics card that supports OpenGL 4.3. The rendering code is still very much in progress. NVIDIA graphics cards should have no issue running the game, but AMD cards might struggle unless you have a more powerful one.
-- You might experience slower loads with a hard drive compared to an SSD. 
+- You might experience slower loads with a hard drive compared to an SSD.
 
 ### Will there be a Vulkan version of the game?
-Probably yes, in the future! 
+Probably yes, in the future!
 
 ### I'd like to contribute, but don't know where to start!
-Great to hear! Any help is good help. If you haven't already, please read the project readme. It has information on what is done, what isn't done, what is being worked on, the project layout and how it all works. Then just join our development discord server and we can help you further. 
+Great to hear! Any help is good help. If you haven't already, please read the project readme. It has information on what is done, what isn't done, what is being worked on, the project layout and how it all works. Then just join our development Discord server and we can help you further.
 
 ### Will you also decompile other games that aren't Jak?
-No. 
+No.
 
 ### Is this project legal?
-Yes. 
+Yes.
 
-## Troubleshooting 
-
-### None of the tools are working!
-Because of its in-development state, we provide no pre-built binaries at the moment. The project readme has instructions on how to compile the tools. 
+## Troubleshooting
 
 ### I am getting "illegal operation"/"illegal instruction" errors when running the game or tools.
 This shouldn't happen, so please contact us in the support channel of the discord server and we can help, and tell us if your CPU supports AVX, AVX2 or AVX512.
 
 ### The game is really slow.
-Your GPU is probably too weak and our renderers too slow. However, we plan on making significant changes to the renderers over time to improve performance. The merc renderer right now is very slow, if you would like to turn off then head to the debug menu by pressing R3, then go to `Render...` and turn `merc` off. 
+Your GPU is probably too weak and our renderers too slow. We plan on making significant changes to the renderers over time to improve performance. The merc renderer right now is very slow, if you would like to turn it off then head to the debug menu by pressing R3, then go to `Render...` and turn `merc` off.
 
 ### The game crashed!!
-Crap! Contact us in the help channel on the discord server and tell us about it! Post the log files as well. These are located in the `log` folder of the project. 
+Curses! Contact us in the help channel on the discord server and tell us about it! Post the log files as well. These are located in the `log` folder of the project. Make sure you tell us exactly how the crash happened.
 
-### The game crashed when entering the final boss area!
-This one specifically is known. We are investigating this at the moment, so don't worry.
+### I am having some other issue or found a bug with the game/tools.
+First, scroll through our [issues](https://github.com/open-goal/jak-project/issues) page to make sure that your issue hasn't been reported already. If it hasn't, feel free to open up a new issue, or report it to us directly. Make sure you provide reproduction steps! We can't always guess why things go wrong.

README.md

@@ -22,7 +22,6 @@
 - [Getting Started - Linux](#getting-started---linux)
   - [Ubuntu (20.04)](#ubuntu-2004)
   - [Arch](#arch)
-  - [With Nix](#with-nix)
 - [Getting Started - Windows](#getting-started---windows)
   - [Required Software](#required-software)
   - [Setting up and Opening the Project](#setting-up-and-opening-the-project)
@@ -47,24 +46,24 @@ This project is to port Jak 1 (NTSC, "black label" version) to PC. Over 98% of t
 - create tools to repack game assets into a format that our port uses.
 
 Our objectives are:
-- make the port a "native application" on x86-64, with high performance. It shouldn't emulated, interpreted, or transpiled.
+- make the port a "native application" on x86-64, with high performance. It shouldn't be emulated, interpreted, or transpiled.
 - Our GOAL compiler's performance should be around the same as unoptimized C.
 - try to match things from the original game and development as possible. For example, the original GOAL compiler supported live modification of code while the game is running, so we do the same, even though it's not required for just porting the game.
 - support modifications. It should be possible to make edits to the code without everything else breaking.
 
 We support both Linux and Windows on x86-64.
 
-We have a Discord server where we discuss development. https://discord.gg/VZbXMHXzWv
+### **[Please read the FAQ by clicking here if you have any questions.](FAQ.md)**
 
-**[If you have any questions, please read the FAQ here beforehand.](FAQ.md)**
+We have a Discord server where we discuss development: https://discord.gg/VZbXMHXzWv
 
 ## Current Status
 
-So far, we've decompiled around 400,000 lines of GOAL code, out of an estimated 500,000 total lines. We have a working OpenGL renderer which renders most of the game world and foreground. Levels are fully playable, and you can finish the game with 100% completion! There is currently *no* audio.
+So far, we've decompiled around 400,000 lines of GOAL code, out of an estimated 500,000 total lines from the original game. We have a working OpenGL renderer which renders most of the game world and foreground. Levels are fully playable, and you can finish the game with 100% completion!
 
 Here are some screenshots of the renderer:
-![](./docs/img/screenshot_hut_new_small.png)
-![](./docs/img/screenshot_junglenew_custommood_small.png)
+![](./docs/img/promosmall1.png)
+![](./docs/img/promosmall2.png)
 
 YouTube playlist:
 https://www.youtube.com/playlist?list=PLWx9T30aAT50cLnCTY1SAbt2TtWQzKfXX
@@ -75,10 +74,9 @@ We don't save any assets from the game - you must bring your own copy of the gam
 
 ## What's Next
 
-- Continue decompilation of GOAL code. We've made huge progress recently in decompiling gameplay code. We're finishing that up and also working on the some of the rendering code. Here's our decompilation progress over the past year: ![](./docs/img/code_progress.png)
-- Bug testing! The game can be beaten 100%, but it's possible a few things do not work correctly.
-- Improve the decompiler. We are always finding new features and macros in the GOAL language.
-- Investigate more complicated renderers. We have an in-progress port of the "merc" foreground renderer, shown in the screenshots above.
+- Bug testing! The game can be beaten 100% from start to finish, but it's possible a few things do not work correctly.
+- Improve the decompiler and clean up decompilation of GOAL code. We are always finding new features and macros in the GOAL language.
+- Investigate and rewrite the more complicated renderers.
 
 ## Getting Started - Linux
 
@@ -87,7 +85,7 @@ We don't save any assets from the game - you must bring your own copy of the gam
 Install packages and init repository:
 
 ```sh
-sudo apt install gcc make cmake build-essential g++ nasm clang-format libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev python
+sudo apt install gcc make cmake build-essential g++ nasm clang-format libxrandr-dev libxinerama-dev libxcursor-dev libpulse-dev libxi-dev python
 sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
 ```
 
@@ -120,10 +118,12 @@ cmake -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_EXE_LINKER_FLAGS="-fuse
 Install packages and init repository:
 
 ```sh
-sudo pacman -S gcc make cmake base-devel g++ nasm python
-yay -S taskfile-git
+sudo pacman -S cmake libpulse base-devel nasm python
+yay -S go-task
 ```
 
+For Arch only, replace `task` with `go-task` in the rest of the instructions.
+
 Compile:
 
 ```sh
@@ -136,24 +136,26 @@ Run tests:
 ./test.sh
 ```
 
-### With Nix
+### Fedora
 
-If your Nix supports flakes:
+Install packages and init repository:
 
 ```sh
-nix develop # development environment
-nix build # package
-nix develop '.#jak-asan-dev' # development environment with Clang
-nix build '.#jak-asan' # package with Clang ASan build
+sudo dnf install cmake lld clang nasm libX11-devel libXrandr-devel libXinerama-devel libXcursor-devel libXi-devel pulseaudio-libs-devel
+sudo sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin
 ```
 
-Otherwise, with traditional Nix:
+Compile with `clang`:
 
 ```sh
-nix-shell # development environment
-nix-build # package
-nix-shell -A packages.x86_64-linux.jak-asan-dev # development environment with Clang
-nix-build -A packages.x86_64-linux.jak-asan # package with Clang ASan build
+cmake -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -B build
+cmake --build build -j$(nproc)
+```
+
+Run tests:
+
+```sh
+./test.sh
 ```
 
 ## Getting Started - Windows
@@ -221,16 +223,16 @@ task repl
 You will be greeted with a prompt like so:
 
 ```sh
- _____             _____ _____ _____ __    
-|     |___ ___ ___|   __|     |  _  |  |   
-|  |  | . | -_|   |  |  |  |  |     |  |__ 
+ _____             _____ _____ _____ __
+|     |___ ___ ___|   __|     |  _  |  |
+|  |  | . | -_|   |  |  |  |  |     |  |__
 |_____|  _|___|_|_|_____|_____|__|__|_____|
       |_|
 Welcome to OpenGOAL 0.8!
 Run (repl-help) for help with common commands and REPL usage.
 Run (lt) to connect to the local target.
 
-g > 
+g >
 ```
 
 Run the following to build the game:
@@ -301,7 +303,7 @@ gc>
 
 In the graphics window, you can use the period key to bring up the debug menu. Controllers also work, using the same mapping as the original game.
 
-Check out the `pc_debug`, `examples` and `engine/pc/` folders under `goal_src` for some examples of GOAL code we wrote. The debug files that are not loaded automatically by the engine have instructions for how to run them.
+Check out the `pc_debug`, `examples` and `pc` folders under `goal_src` for some examples of GOAL code we wrote. The debug files that are not loaded automatically by the engine have instructions for how to run them.
 
 ## Project Layout
 
@@ -315,14 +317,15 @@ The second component to the project is the decompiler. You must have a copy of t
 
 Then run `decomp.sh` (Linux) or `decomp-jak1.bat` (Windows) to run the decompiler. The decompiler will extract assets to the `assets` folder. These assets will be used by the compiler when building the port, and you may want to turn asset extraction off after running it once. The decompiler will output code and other data intended to be inspected by humans in the `decompiler_out` folder. Stuff in this folder will not be used by the compiler.
 
-The third is the game source code, written in OpenGOAL. This is located in `goal_src`. All GOAL and GOOS code should be in this folder. Right now most of this is placeholders or incomplete, but you can take a look at `kernel/gcommon.gc` or `goal-lib.gc` to see some in-progress source code.
+The third is the game source code, written in OpenGOAL. This is located in `goal_src`. All GOAL and GOOS code should be in this folder.
 
 The final component is the "runtime", located in `game`. This is the part of the game that's written in C++. In the port, that includes:
 - The "C Kernel", which contains the GOAL linker and some low-level GOAL language features. GOAL has a completely custom dynamically linked object file format so in order to load the first GOAL code, you need a linker written in C++. Some low-level functions for memory allocation, communicating with the I/O Processor, symbol table, strings, and the type system are also implemented in C, as these are required for the linker. It also listens for incoming messages from the compiler and passes them to the running game. This also initializes the game, by initializing the PS2 hardware, allocating the GOAL heaps, loading the GOAL kernel off of the DVD, and executing the kernel dispatcher function. This is in the `game/kernel` folder. This should be as close as possible to the game, and all differences should be noted with a comment.
 - Implementation of Sony's standard library. GOAL code can call C library functions, and Naughty Dog used some Sony library functions to access files, memory cards, controllers, and communicate with the separate I/O Processor. The library functions are in `game/sce`. Implementations of library features specific to the PC port are located in `game/system`.
-- The I/O Processor driver, Overlord. The PS2 had a separate CPU called the I/O Processor (IOP) that was directly connected to the DVD drive hardware and the sound hardware. Naughty Dog created a custom driver for the IOP that handled streaming data off of the DVD. It is much more complicated than I first expected. It's located in `game/overlord`. Like the C kernel, we try to keep this as close as possible to the actual game.
-- Sound Code. Naughty Dog used a third party library for sound. We have not started on this yet.
-- PC specific graphics stuff. We have a functional OpenGL renderer and context that can create a game window and display graphics on it. The specific renderers used by the game however are mostly unimplemented. We have a debug, sprite and tfrag renderers, which is enough to render most of the game world, on-screen text and sprites like the progress menu. This is located in `game/graphics`. Many liberties will be taken to make this work, but the end result should closely match the actual game.
+- The I/O Processor driver, OVERLORD. The PS2 had a separate CPU called the I/O Processor (IOP) that was directly connected to the DVD drive hardware and the sound hardware. Naughty Dog created a custom driver for the IOP that handled streaming data off of the DVD. It is much more complicated than I first expected. It's located in `game/overlord`. Like the C kernel, we try to keep this as close as possible to the actual game.
+- Sound code. Naughty Dog used a third party library for sound called `989SND`. Code for the library and an interface for it is located in `game/sound`.
+- PC specific graphics code. We have a functional OpenGL renderer and context that can create a game window and display graphics on it. The specific renderers used by the game however are mostly implemented. Aside from post-processing effects, everything in the game is rendered. This is located in `game/graphics`. While many liberties will be taken to make this work, the end result should very closely match the actual game.
+- Extra assets used by the port in some fashion, located in `game/assets`. These include extra text files, icons, etc.
 
 ## Directory Layout
 
@@ -368,6 +371,7 @@ The final component is the "runtime", located in `game`. This is the part of the
     - `examples`: Example GOAL code we have written.
     - `test`: Test code we have written.
     - `pc_debug`: GOAL code for PC-specific debugging. Also written by us.
+    - `pc`: GOAL code for PC-specific features. Also written by us.
     - `user`: User sandboxes. Written by you! And anyone else that wishes to share their user code.
 - `goalc`: The OpenGOAL compiler.
     - `compiler`: The implementation of the OpenGOAL language.
@@ -377,7 +381,7 @@ The final component is the "runtime", located in `game`. This is the part of the
     - `listener`: The OpenGOAL listener, which connects the compiler to a running GOAL program for the interactive REPL.
     - `make`: The OpenGOAL build system, builds both code and data files.
     - `regalloc`: Register allocator.
-- `iso_data`:
+- `iso_data`: Location of the user-provided DVD contents of the game that the decompiler extracts game assets and code from.
 - `out`: Outputs from the build process. Only the `iso` subfolder should contain assets used by the game.
     - `iso`: Final outputs that are used by the game.
     - `obj`: Object files generated by the compiler.
@@ -386,7 +390,8 @@ The final component is the "runtime", located in `game`. This is the part of the
 - `test`: Unit tests (run on GitHub Actions).
 - `third-party`: Third party libraries.
     - CMake Code Coverage. For code coverage statistics on GitHub builds.
-    - `fmt`. String formatting library.
+    - `discord-rpc`: Discord Rich Presence library.
+    - `fmt`: String formatting library.
     - `googletest`: Test framework.
     - `inja`: templating library used for generating test code for compiler tests.
     - `lzokay`: decompression code for Jak 2 and later DGOs.

Taskfile.yml

@@ -6,43 +6,48 @@ includes:
 tasks:
   # GENERAL
   extract-jak1:
+    desc: "Extracts Jak 1 - NTSC - Black Label assets"
     preconditions:
       - sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/decompiler{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate decompiler executable -- Have you compiled in release mode?"
+        msg: "Couldn't locate decompiler executable in '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler'"
     cmds:
       - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/jak1_ntsc_black_label.jsonc" "./iso_data" "./decompiler_out" "decompile_code=false"'
   boot-game:
+    desc: "Boots the game"
     preconditions:
-      - sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate runtime executable -- Have you compiled in release mode?"
+      - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
+        msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
     cmds:
       - "{{.GK_BIN_RELEASE_DIR}}/gk -boot -fakeiso -debug -v"
   run-game:
+    desc: "Start the game's runtime"
     preconditions:
-      - sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate runtime executable -- Have you compiled in release mode?"
+      - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
+        msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
     cmds:
       - "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -v"
   run-game-quiet:
     preconditions:
-      - sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate runtime executable -- Have you compiled in release mode?"
+      - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
+        msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
     cmds:
       - "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso"
   repl:
+    desc: "Start the REPL"
     env:
       OPENGOAL_DECOMP_DIR: "jak1/"
     preconditions:
-      - sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate compiler executable -- Have you compiled in release mode?"
+      - sh: test -f {{.GOALC_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}}
+        msg: "Couldn't locate compiler executable in '{{.GOALC_BIN_RELEASE_DIR}}/goalc'"
     cmds:
       - "{{.GOALC_BIN_RELEASE_DIR}}/goalc"
   # DEVELOPMENT
   format:
+    desc: "Format code"
     cmds:
       - cmd: python ./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test -i
       # npm install -g prettier
-      - cmd: prettier --write ./decompiler/config/jak1_ntsc_black_label/*.jsonc
+      - cmd: npx prettier --write ./decompiler/config/jak1_ntsc_black_label/*.jsonc
         ignore_error: true
   run-game-headless:
     cmds:
@@ -51,7 +56,7 @@ tasks:
     env:
       OPENGOAL_DECOMP_DIR: "jak1/"
     cmds:
-      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc -auto-lt"
+      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt"
   # DECOMPILING
   decomp:
     cmds:
@@ -125,7 +130,3 @@ tasks:
   cast-repl:
     cmds:
       - cmd: python ./scripts/cast-repl.py
-  # Doesn't Currently Work
-  # clean-all-types:
-  #   cmds:
-  #     - python ./scripts/cleanup-all-types.py

common/CMakeLists.txt

@@ -1,12 +1,16 @@
 add_library(common
         audio/audio_formats.cpp
         cross_os_debug/xdbg.cpp
-        cross_sockets/xsocket.cpp
+        cross_sockets/XSocket.cpp
+        cross_sockets/XSocketServer.cpp
+        cross_sockets/XSocketClient.cpp
+        custom_data/pack_helpers.cpp
         custom_data/TFrag3Data.cpp
+        deserialization/subtitles/subtitles.cpp
         dma/dma.cpp
         dma/dma_copy.cpp
         dma/gs.cpp
-
+        global_profiler/GlobalProfiler.cpp
         goos/Interpreter.cpp
         goos/Object.cpp
         goos/ParseHelpers.cpp
@@ -16,8 +20,11 @@ add_library(common
         goos/Reader.cpp
         goos/TextDB.cpp
         goos/ReplUtils.cpp
-        math/geometry.cpp
         log/log.cpp
+        math/geometry.cpp
+        nrepl/ReplClient.cpp
+        nrepl/ReplServer.cpp
+        serialization/subtitles/subtitles.cpp
         type_system/defenum.cpp
         type_system/deftype.cpp
         type_system/state.cpp
@@ -28,21 +35,20 @@ add_library(common
         util/Assert.cpp
         util/BitUtils.cpp
         util/compress.cpp
+        util/crc32.cpp
         util/dgo_util.cpp
         util/DgoReader.cpp
         util/DgoWriter.cpp
         util/diff.cpp
         util/FileUtil.cpp
         util/json_util.cpp
+        util/read_iso_file.cpp
+        util/SimpleThreadGroup.cpp
         util/Timer.cpp
         util/os.cpp
         util/print_float.cpp
         util/FontUtils.cpp
-        util/FrameLimiter.cpp
-        util/image_loading.cpp
-        goos/Printer.cpp
-        goos/PrettyPrinter2.cpp
-        )
+        util/FrameLimiter.cpp)
 
 target_link_libraries(common fmt lzokay replxx libzstd_static)
 

common/audio/audio_formats.cpp

@@ -1,5 +1,7 @@
 #include "audio_formats.h"
+
 #include "common/util/BinaryWriter.h"
+
 #include "third-party/fmt/core.h"
 
 /*!
@@ -289,8 +291,7 @@ void test_encode_adpcm(const std::vector<s16>& samples,
       for (int i = 0; i < 5; i++) {
         fmt::print(" [{}] {} {}\n", i, filter_errors[i], filter_shifts[i]);
       }
-      fmt::print("prev: {} {}\n", prev_block_samples[0], prev_block_samples[1]);
-      ASSERT(false);
+      ASSERT_MSG(false, fmt::format("prev: {} {}", prev_block_samples[0], prev_block_samples[1]));
     }
 
     prev_block_samples[0] = samples.at(block_idx * 28 + 27);

common/audio/audio_formats.h

@@ -3,8 +3,8 @@
 #include <string>
 #include <vector>
 
-#include "common/util/BinaryReader.h"
 #include "common/common_types.h"
+#include "common/util/BinaryReader.h"
 
 // The header data for a simple wave file
 struct WaveFileHeader {

common/cross_os_debug/xdbg.cpp

@@ -4,27 +4,32 @@
  * Nothing in here should hold state, that should all be managed in Debugger.
  */
 
-#include <cstring>
-#include "common/goal_constants.h"
-#include "common/util/Timer.h"
-#include "third-party/fmt/core.h"
 #include "xdbg.h"
 
+#include <cstring>
+
+#include "common/goal_constants.h"
+#include "common/util/Timer.h"
+
+#include "third-party/fmt/core.h"
+
 #ifdef __linux
+#include <fcntl.h>
 #include <unistd.h>
+
+#include <sys/prctl.h>
+#include <sys/ptrace.h>
 #include <sys/stat.h>
 #include <sys/syscall.h>
-#include <sys/ptrace.h>
-#include <sys/prctl.h>
 #include <sys/types.h>
 #include <sys/user.h>
 #include <sys/wait.h>
-#include <fcntl.h>
 #elif _WIN32
 #define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
-#include <mutex>
 #include <condition_variable>
+#include <mutex>
 #endif
 
 namespace xdbg {
@@ -38,9 +43,7 @@ ThreadID::ThreadID(pid_t _id) : id(_id) {}
 /*!
  * In Linux, the string representation of a ThreadID is just the number printed in base 10
  */
-ThreadID::ThreadID(const std::string& str) {
-  id = std::stoi(str);
-}
+ThreadID::ThreadID(const std::string& str) : id(std::stoi(str)) {}
 
 std::string ThreadID::to_string() const {
   return std::to_string(id);
@@ -523,7 +526,7 @@ bool check_stopped(const ThreadID& tid, SignalInfo* out) {
         out->kind = SignalInfo::DISAPPEARED;
         break;
       default:
-        printf("[Debugger] unhandled debug event %d\n", debugEvent.dwDebugEventCode);
+        printf("[Debugger] unhandled debug event %lu\n", debugEvent.dwDebugEventCode);
         out->kind = SignalInfo::UNKNOWN;
         break;
     }

common/cross_os_debug/xdbg.h

@@ -6,14 +6,16 @@
 
 #pragma once
 
-#include <string>
 #include <cstdint>
+#include <string>
+
 #include "common/common_types.h"
 
 #ifdef __linux
 #include <sys/types.h>
 #elif _WIN32
 #define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #endif
 

common/cross_sockets/XSocket.cpp

@@ -3,9 +3,10 @@
  * Cross platform socket library used for the listener.
  */
 
+// clang-format off
 #ifdef __linux
-#include <sys/socket.h>
 #include <netinet/tcp.h>
+#include <sys/socket.h>
 #include <unistd.h>
 #elif _WIN32
 #define WIN32_LEAN_AND_MEAN
@@ -16,6 +17,9 @@
 #include <stdio.h>
 #include <string.h>
 
+#include "third-party/fmt/core.h"
+// clang-format on
+
 int open_socket(int af, int type, int protocol) {
 #ifdef __linux
   return socket(af, type, protocol);
@@ -32,6 +36,71 @@ int open_socket(int af, int type, int protocol) {
 #endif
 }
 
+int connect_socket(int socket, sockaddr* addr, int nameLen) {
+  int result = connect(socket, addr, nameLen);
+  if (result == -1) {
+    return -1;
+  }
+  return result;
+}
+
+#ifdef __linux
+int accept_socket(int socket, sockaddr* addr, socklen_t* addrLen) {
+  return accept(socket, addr, addrLen);
+}
+int select_and_accept_socket(int socket, sockaddr* addr, socklen_t* addrLen, int microSeconds) {
+  struct timeval timeout;
+  timeout.tv_sec = 0;
+  timeout.tv_usec = microSeconds;
+  // Use select so it can timeout, accept on the returned socket if it is correct
+  fd_set read_sockets;
+  FD_ZERO(&read_sockets);
+  FD_SET(socket, &read_sockets);
+  auto activity = select(socket + 1, &read_sockets, NULL, NULL, &timeout);
+  if (activity > 0) {
+    return accept(socket, addr, addrLen);
+  }
+  return -1;
+}
+#endif
+
+#ifdef _WIN32
+int accept_socket(int socket, sockaddr* addr, int* addrLen) {
+  WSADATA wsaData = {0};
+  int iResult = 0;
+  // Initialize Winsock
+  iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
+  if (iResult != 0) {
+    printf("WSAStartup failed: %d\n", iResult);
+    return 1;
+  }
+  return accept(socket, addr, addrLen);
+}
+
+int select_and_accept_socket(int socket, sockaddr* addr, int* addrLen, int microSeconds) {
+  WSADATA wsaData = {0};
+  int iResult = 0;
+  // Initialize Winsock
+  iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
+  if (iResult != 0) {
+    printf("WSAStartup failed: %d\n", iResult);
+    return 1;
+  }
+  struct timeval timeout;
+  timeout.tv_sec = 0;
+  timeout.tv_usec = microSeconds;
+  // Use select so it can timeout, accept on the returned socket if it is correct
+  fd_set read_sockets;
+  FD_ZERO(&read_sockets);
+  FD_SET(socket, &read_sockets);
+  auto activity = select(socket + 1, &read_sockets, NULL, NULL, &timeout);
+  if (activity > 0) {
+    return accept(socket, addr, addrLen);
+  }
+  return -1;
+}
+#endif
+
 void close_socket(int sock) {
   if (sock < 0) {
     return;
@@ -77,11 +146,16 @@ int set_socket_timeout(int socket, long microSeconds) {
 }
 
 int write_to_socket(int socket, const char* buf, int len) {
+  int bytes_wrote = 0;
 #ifdef __linux
-  return write(socket, buf, len);
+  bytes_wrote = send(socket, buf, len, MSG_NOSIGNAL);
 #elif _WIN32
-  return send(socket, buf, len, 0);
+  bytes_wrote = send(socket, buf, len, 0);
 #endif
+  if (bytes_wrote < 0) {
+    fmt::print(stderr, "[XSocket:{}] Error writing to socket\n", socket);
+  }
+  return bytes_wrote;
 }
 
 int read_from_socket(int socket, char* buf, int len) {
@@ -99,4 +173,4 @@ bool socket_timed_out() {
   auto err = WSAGetLastError();
   return err == WSAETIMEDOUT;
 #endif
-}
+}

common/cross_sockets/XSocket.h

@@ -1,17 +1,24 @@
 #pragma once
 
 /*!
- * @file xsocket.h
+ * @file XSocket.h
  * Cross platform socket library used for the listener.
  */
 
+// clang-format off
 #ifdef __linux
-#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
 #include <netinet/tcp.h>
+#include <sys/socket.h>
 #include <unistd.h>
 #elif _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
 #include <WinSock2.h>
 #endif
+// clang-format on
 
 #ifdef __linux
 const int TCP_SOCKET_LEVEL = SOL_TCP;
@@ -20,9 +27,17 @@ const int TCP_SOCKET_LEVEL = IPPROTO_TCP;
 #endif
 
 int open_socket(int af, int type, int protocol);
+int connect_socket(int socket, sockaddr* addr, int nameLen);
+#ifdef __linux
+int accept_socket(int socket, sockaddr* addr, socklen_t* addrLen);
+int select_and_accept_socket(int socket, sockaddr* addr, socklen_t* addrLen, int microSeconds);
+#elif _WIN32
+int accept_socket(int socket, sockaddr* addr, int* addrLen);
+int select_and_accept_socket(int socket, sockaddr* addr, int* addrLen, int microSeconds);
+#endif
 void close_socket(int sock);
 int set_socket_option(int socket, int level, int optname, const void* optval, int optlen);
 int set_socket_timeout(int socket, long microSeconds);
 int write_to_socket(int socket, const char* buf, int len);
 int read_from_socket(int socket, char* buf, int len);
-bool socket_timed_out();
+bool socket_timed_out();

common/cross_sockets/XSocketClient.cpp

@@ -0,0 +1,56 @@
+#include "XSocketClient.h"
+
+#include <string>
+
+#include "common/cross_sockets/XSocket.h"
+
+// clang-format off
+#ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#include <WinSock2.h>
+#include <WS2tcpip.h>
+#endif
+#include "common/nrepl/ReplServer.h"
+
+#include "third-party/fmt/core.h"
+// clang-format on
+
+XSocketClient::XSocketClient(int _tcp_port) {
+  tcp_port = _tcp_port;
+}
+
+XSocketClient::~XSocketClient() {
+  disconnect();
+  client_socket = -1;
+}
+
+void XSocketClient::disconnect() {
+  close_socket(client_socket);
+  client_socket = -1;
+}
+
+bool XSocketClient::connect() {
+  // Open Socket
+  client_socket = open_socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+  if (client_socket < 0) {
+    // TODO - log
+    disconnect();
+    return false;
+  }
+
+  addr.sin_family = AF_INET;
+  addr.sin_addr.s_addr = inet_addr("127.0.0.1");
+  addr.sin_port = htons(tcp_port);
+
+  // Connect to server
+  int result = connect_socket(client_socket, (sockaddr*)&addr, sizeof(addr));
+  if (result == -1) {
+    // TODO - log and close
+    disconnect();
+    return false;
+  }
+
+  return true;
+}

common/cross_sockets/XSocketClient.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <functional>
+#include <mutex>
+#include <thread>
+
+#include "common/common_types.h"
+#include "common/cross_sockets/XSocket.h"
+
+/// @brief A cross platform generic socket client implementation
+class XSocketClient {
+ public:
+  XSocketClient(int _tcp_port);
+  ~XSocketClient();
+
+  XSocketClient(const XSocketClient&) = delete;
+  XSocketClient& operator=(const XSocketClient&) = delete;
+
+  bool connect();
+  void disconnect();
+
+  bool is_connected() { return client_socket != -1; }
+
+ protected:
+  int tcp_port;
+  struct sockaddr_in addr = {};
+  int client_socket = -1;
+};

common/cross_sockets/XSocketServer.cpp

@@ -0,0 +1,90 @@
+// clang-format off
+#include "XSocketServer.h"
+
+#include "common/cross_sockets/XSocket.h"
+
+#include "third-party/fmt/core.h"
+
+#ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#include <WinSock2.h>
+#include <WS2tcpip.h>
+#endif
+// clang-format on
+
+XSocketServer::XSocketServer(std::function<bool()> shutdown_callback,
+                             int _tcp_port,
+                             int _buffer_size)
+    : want_exit_callback(std::move(shutdown_callback)) {
+  tcp_port = _tcp_port;
+  buffer.resize(_buffer_size);
+}
+
+XSocketServer::~XSocketServer() {
+  if (listening_socket >= 0) {
+    close_server_socket();
+  }
+}
+
+void XSocketServer::shutdown_server() {
+  // Close the listening and accepted socket socket
+  close_server_socket();
+}
+
+bool XSocketServer::init_server() {
+  listening_socket = open_socket(AF_INET, SOCK_STREAM, 0);
+  if (listening_socket < 0) {
+    listening_socket = -1;
+    return false;
+  }
+
+#ifdef __linux
+  int server_socket_opt = SO_REUSEADDR | SO_REUSEPORT;
+#elif _WIN32
+  int server_socket_opt = SO_EXCLUSIVEADDRUSE;
+#endif
+
+  int opt = 1;
+  if (set_socket_option(listening_socket, SOL_SOCKET, server_socket_opt, &opt, sizeof(opt)) < 0) {
+    close_server_socket();
+    return false;
+  };
+
+  if (set_socket_option(listening_socket, TCP_SOCKET_LEVEL, TCP_NODELAY, &opt, sizeof(opt)) < 0) {
+    close_server_socket();
+    return false;
+  }
+
+  if (set_socket_timeout(listening_socket, 100000) < 0) {
+    close_server_socket();
+    return false;
+  }
+
+  addr.sin_family = AF_INET;
+  addr.sin_addr.s_addr = INADDR_ANY;
+  addr.sin_port = htons(tcp_port);
+
+  if (bind(listening_socket, (sockaddr*)&addr, sizeof(addr)) < 0) {
+    fmt::print("[XSocketServer:{}] failed to bind\n", tcp_port);
+    close_server_socket();
+    return false;
+  }
+
+  if (listen(listening_socket, 0) < 0) {
+    fmt::print("[XSocketServer:{}] failed to listen\n", tcp_port);
+    close_server_socket();
+    return false;
+  }
+
+  server_initialized = true;
+  fmt::print("[XSocketServer:{}] initialized\n", tcp_port);
+  post_init();
+  return true;
+}
+
+void XSocketServer::close_server_socket() {
+  close_socket(listening_socket);
+  listening_socket = -1;
+}

common/cross_sockets/XSocketServer.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <functional>
+#include <mutex>
+#include <thread>
+
+#include "common/common_types.h"
+#include "common/cross_sockets/XSocket.h"
+
+/// @brief A cross platform generic socket server implementation
+class XSocketServer {
+ public:
+  static constexpr int DEF_BUFFER_SIZE = 32 * 1024 * 1024;
+  XSocketServer(std::function<bool()> shutdown_callback,
+                int _tcp_port,
+                int _buffer_size = DEF_BUFFER_SIZE);
+  virtual ~XSocketServer();
+
+  XSocketServer(const XSocketServer&) = delete;
+  XSocketServer& operator=(const XSocketServer&) = delete;
+
+  bool init_server();
+  void shutdown_server();
+  void close_server_socket();
+
+  // Abstract methods -- use-case dependent
+  virtual void post_init() = 0;
+
+ protected:
+  int tcp_port;
+  struct sockaddr_in addr = {};
+  int listening_socket = -1;
+  std::vector<char> buffer;
+
+  bool server_initialized = false;
+
+  std::function<bool()> want_exit_callback;
+};

common/custom_data/TFrag3Data.cpp

@@ -1,4 +1,8 @@
 #include "Tfrag3Data.h"
+
+#include <algorithm>
+#include <functional>
+
 #include "common/util/Assert.h"
 
 namespace tfrag3 {
@@ -21,25 +25,17 @@ void StripDraw::serialize(Serializer& ser) {
   ser.from_ptr(&mode);
   ser.from_ptr(&tree_tex_id);
   ser.from_pod_vector(&runs);
+  ser.from_pod_vector(&plain_indices);
   ser.from_pod_vector(&vis_groups);
   ser.from_ptr(&num_triangles);
 }
 
-void StripDraw::unpack() {
-  ASSERT(unpacked.vertex_index_stream.empty());
-  for (auto& r : runs) {
-    for (int i = 0; i < r.length; i++) {
-      unpacked.vertex_index_stream.push_back(r.vertex0 + i);
-    }
-    unpacked.vertex_index_stream.push_back(UINT32_MAX);
-  }
-}
-
 void ShrubDraw::serialize(Serializer& ser) {
   ser.from_ptr(&mode);
   ser.from_ptr(&tree_tex_id);
-  ser.from_pod_vector(&vertex_index_stream);
   ser.from_ptr(&num_triangles);
+  ser.from_ptr(&first_index_index);
+  ser.from_ptr(&num_indices);
 }
 
 void InstancedStripDraw::serialize(Serializer& ser) {
@@ -73,6 +69,7 @@ void TfragTree::serialize(Serializer& ser) {
   ser.from_pod_vector(&packed_vertices.cluster_origins);
   ser.from_pod_vector(&colors);
   bvh.serialize(ser);
+  ser.from_ptr(&use_strips);
 }
 
 void TieTree::unpack() {
@@ -87,7 +84,7 @@ void TieTree::unpack() {
         vtx.x = proto_vtx.x;
         vtx.y = proto_vtx.y;
         vtx.z = proto_vtx.z;
-        vtx.q = 1.f;
+        vtx.q_unused = 1.f;
         vtx.s = proto_vtx.s;
         vtx.t = proto_vtx.t;
         i++;
@@ -102,13 +99,24 @@ void TieTree::unpack() {
         vtx.x = temp.x();
         vtx.y = temp.y();
         vtx.z = temp.z();
-        vtx.q = 1.f;
+        vtx.q_unused = 1.f;
         vtx.s = proto_vtx.s;
         vtx.t = proto_vtx.t;
         i++;
       }
     }
   }
+
+  for (auto& draw : static_draws) {
+    draw.unpacked.idx_of_first_idx_in_full_buffer = unpacked.indices.size();
+    ASSERT(draw.plain_indices.empty());
+    for (auto& run : draw.runs) {
+      for (u32 ri = 0; ri < run.length; ri++) {
+        unpacked.indices.push_back(run.vertex0 + ri);
+      }
+      unpacked.indices.push_back(UINT32_MAX);
+    }
+  }
 }
 
 void ShrubTree::unpack() {
@@ -151,9 +159,23 @@ void TfragTree::unpack() {
     o.z = cz + in.zoff * rescale;
     o.s = in.s / (1024.f);
     o.t = in.t / (1024.f);
-    o.q = 1.f;
+    o.q_unused = 1.f;
     o.color_index = in.color_index;
   }
+
+  for (auto& draw : draws) {
+    draw.unpacked.idx_of_first_idx_in_full_buffer = unpacked.indices.size();
+    for (auto& run : draw.runs) {
+      for (u32 ri = 0; ri < run.length; ri++) {
+        unpacked.indices.push_back(run.vertex0 + ri);
+      }
+      if (use_strips) {
+        unpacked.indices.push_back(UINT32_MAX);
+      }
+    }
+    unpacked.indices.insert(unpacked.indices.end(), draw.plain_indices.begin(),
+                            draw.plain_indices.end());
+  }
 }
 
 void TieTree::serialize(Serializer& ser) {
@@ -191,6 +213,7 @@ void TieTree::serialize(Serializer& ser) {
 
 void ShrubTree::serialize(Serializer& ser) {
   ser.from_pod_vector(&time_of_day_colors);
+  ser.from_pod_vector(&indices);
   packed_vertices.serialize(ser);
   if (ser.is_saving()) {
     ser.save<size_t>(static_draws.size());
@@ -221,12 +244,64 @@ void Texture::serialize(Serializer& ser) {
   ser.from_ptr(&load_to_pool);
 }
 
+void CollisionMesh::serialize(Serializer& ser) {
+  ser.from_pod_vector(&vertices);
+}
+
+void MercDraw::serialize(Serializer& ser) {
+  ser.from_ptr(&mode);
+  ser.from_ptr(&tree_tex_id);
+  ser.from_ptr(&first_index);
+  ser.from_ptr(&index_count);
+  ser.from_ptr(&num_triangles);
+}
+
+void MercEffect::serialize(Serializer& ser) {
+  if (ser.is_saving()) {
+    ser.save<size_t>(draws.size());
+  } else {
+    draws.resize(ser.load<size_t>());
+  }
+  for (auto& draw : draws) {
+    draw.serialize(ser);
+  }
+}
+
+void MercModel::serialize(Serializer& ser) {
+  ser.from_str(&name);
+  if (ser.is_saving()) {
+    ser.save<size_t>(effects.size());
+  } else {
+    effects.resize(ser.load<size_t>());
+  }
+  for (auto& effect : effects) {
+    effect.serialize(ser);
+  }
+  ser.from_ptr(&scale_xyz);
+  ser.from_ptr(&max_draws);
+  ser.from_ptr(&max_bones);
+}
+
+void MercModelGroup::serialize(Serializer& ser) {
+  if (ser.is_saving()) {
+    ser.save<size_t>(models.size());
+  } else {
+    models.resize(ser.load<size_t>());
+  }
+  for (auto& model : models) {
+    model.serialize(ser);
+  }
+
+  ser.from_pod_vector(&indices);
+  ser.from_pod_vector(&vertices);
+}
+
 void Level::serialize(Serializer& ser) {
   ser.from_ptr(&version);
   if (ser.is_loading() && version != TFRAG3_VERSION) {
-    fmt::print("version mismatch when loading tfrag3 data. Got {}, expected {}\n", version,
-               TFRAG3_VERSION);
-    ASSERT(false);
+    ASSERT_MSG(false, fmt::format("version mismatch when loading tfrag3 data. Got {}, expected {}, "
+                                  "did you forget to re-decompile?",
+                                  version, TFRAG3_VERSION));
   }
 
   ser.from_str(&level_name);
@@ -271,11 +346,14 @@ void Level::serialize(Serializer& ser) {
     tree.serialize(ser);
   }
 
+  collision.serialize(ser);
+  merc_data.serialize(ser);
+
   ser.from_ptr(&version2);
   if (ser.is_loading() && version2 != TFRAG3_VERSION) {
-    fmt::print("version mismatch when loading tfrag3 data (at end). Got {}, expected {}\n",
-               version2, TFRAG3_VERSION);
-    ASSERT(false);
+    ASSERT_MSG(false, fmt::format(
+                          "version mismatch when loading tfrag3 data (at end). Got {}, expected {}",
+                          version2, TFRAG3_VERSION));
   }
 }
 
@@ -293,6 +371,7 @@ std::array<int, MemoryUsageCategory::NUM_CATEGORIES> Level::get_memory_usage() c
     for (const auto& tfrag_tree : tfrag_tree_geoms) {
       for (const auto& draw : tfrag_tree.draws) {
         result[TFRAG_INDEX] += draw.runs.size() * sizeof(StripDraw::VertexRun);
+        result[TFRAG_INDEX] += draw.plain_indices.size() * sizeof(u32);
         result[TFRAG_VIS] += draw.vis_groups.size() * sizeof(StripDraw::VisGroup);
       }
       result[TFRAG_VERTS] +=
@@ -338,13 +417,67 @@ std::array<int, MemoryUsageCategory::NUM_CATEGORIES> Level::get_memory_usage() c
         shrub_tree.packed_vertices.vertices.size() * sizeof(PackedShrubVertices::Vertex);
     result[SHRUB_VERT] += shrub_tree.packed_vertices.instance_groups.size() *
                           sizeof(PackedShrubVertices::InstanceGroup);
-
-    for (const auto& draw : shrub_tree.static_draws) {
-      result[SHRUB_IND] += sizeof(u32) * draw.vertex_index_stream.size();
-    }
+    result[SHRUB_IND] += sizeof(u32) * shrub_tree.indices.size();
   }
 
+  // merc
+  result[MERC_INDEX] += merc_data.indices.size() * sizeof(u32);
+  result[MERC_VERT] += merc_data.vertices.size() * sizeof(MercVertex);
+
+  // collision
+  result[COLLISION] += sizeof(CollisionMesh::Vertex) * collision.vertices.size();
+
   return result;
 }
 
+void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size) {
+  int total_accounted = 0;
+  auto memory_use_by_category = lev.get_memory_usage();
+
+  std::vector<std::pair<std::string, int>> known_categories = {
+      {"texture", memory_use_by_category[tfrag3::MemoryUsageCategory::TEXTURE]},
+      {"tie-deinst-vis", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_DEINST_VIS]},
+      {"tie-deinst-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_DEINST_INDEX]},
+      {"tie-inst-vis", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_INST_VIS]},
+      {"tie-inst-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_INST_INDEX]},
+      {"tie-bvh", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_BVH]},
+      {"tie-verts", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_VERTS]},
+      {"tie-colors", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_TIME_OF_DAY]},
+      {"tie-wind-inst-info",
+       memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_WIND_INSTANCE_INFO]},
+      {"tie-cidx", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_CIDX]},
+      {"tie-mats", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_MATRICES]},
+      {"tie-grps", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_GRPS]},
+      {"tfrag-vis", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_VIS]},
+      {"tfrag-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_INDEX]},
+      {"tfrag-vert", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_VERTS]},
+      {"tfrag-colors", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_TIME_OF_DAY]},
+      {"tfrag-cluster", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_CLUSTER]},
+      {"tfrag-bvh", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_BVH]},
+      {"shrub-colors", memory_use_by_category[tfrag3::MemoryUsageCategory::SHRUB_TIME_OF_DAY]},
+      {"shrub-vert", memory_use_by_category[tfrag3::MemoryUsageCategory::SHRUB_VERT]},
+      {"shrub-ind", memory_use_by_category[tfrag3::MemoryUsageCategory::SHRUB_IND]},
+      {"collision", memory_use_by_category[tfrag3::MemoryUsageCategory::COLLISION]},
+      {"merc-vert", memory_use_by_category[tfrag3::MemoryUsageCategory::MERC_VERT]},
+      {"merc-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::MERC_INDEX]}};
+  for (auto& known : known_categories) {
+    total_accounted += known.second;
+  }
+
+  known_categories.push_back({"unknown", uncompressed_data_size - total_accounted});
+
+  std::sort(known_categories.begin(), known_categories.end(),
+            [](const auto& a, const auto& b) { return a.second > b.second; });
+
+  for (const auto& x : known_categories) {
+    fmt::print("{:30s} : {:6d} kB {:3.1f}%\n", x.first, x.second / 1024,
+               100.f * (float)x.second / uncompressed_data_size);
+  }
+}
+
+std::size_t PreloadedVertex::hash::operator()(const PreloadedVertex& v) const {
+  return std::hash<float>()(v.x) ^ std::hash<float>()(v.y) ^ std::hash<float>()(v.z) ^
+         std::hash<float>()(v.s) ^ std::hash<float>()(v.t) ^ std::hash<u16>()(v.color_index);
+}
+
 }  // namespace tfrag3

common/custom_data/Tfrag3Data.h

@@ -5,9 +5,9 @@
 
 #include "common/common_types.h"
 #include "common/dma/gs.h"
-#include "common/util/Serializer.h"
 #include "common/math/Vector.h"
 #include "common/util/Assert.h"
+#include "common/util/Serializer.h"
 
 namespace tfrag3 {
 
@@ -44,20 +44,35 @@ enum MemoryUsageCategory {
   SHRUB_TIME_OF_DAY,
   SHRUB_VERT,
   SHRUB_IND,
+
+  MERC_VERT,
+  MERC_INDEX,
+
+  COLLISION,
+
   NUM_CATEGORIES
 };
 
-constexpr int TFRAG3_VERSION = 13;
+constexpr int TFRAG3_VERSION = 20;
 
 // These vertices should be uploaded to the GPU at load time and don't change
 struct PreloadedVertex {
   // the vertex position
   float x, y, z;
   // texture coordinates
-  float s, t, q;
+  float s, t, q_unused;
   // color table index
   u16 color_index;
   u16 pad[3];
+
+  struct hash {
+    std::size_t operator()(const PreloadedVertex& x) const;
+  };
+
+  bool operator==(const PreloadedVertex& other) const {
+    return x == other.x && y == other.y && z == other.z && s == other.s && t == other.t &&
+           color_index == other.color_index;
+  }
 };
 static_assert(sizeof(PreloadedVertex) == 32, "PreloadedVertex size");
 
@@ -84,7 +99,7 @@ struct PackedTfragVertices {
   struct Vertex {
     u16 xoff, yoff, zoff;
     u16 cluster_idx;
-    u16 s, t;
+    s16 s, t;
     u16 color_index;
   };
 
@@ -135,24 +150,23 @@ struct StripDraw {
   u32 tree_tex_id = 0;  // the texture that should be bound for the draw
 
   struct {
-    // the list of vertices in the draw. This includes the restart code of UINT32_MAX that OpenGL
-    // will use to start a new strip.
-    std::vector<u32> vertex_index_stream;
+    u32 idx_of_first_idx_in_full_buffer = 0;
   } unpacked;
 
-  void unpack();
-
+  // indices can be specified as lists of runs and plain indices.
+  // the runs are still drawn with indexed opengl calls, it just uses less space in the file.
   struct VertexRun {
     u32 vertex0;
     u16 length;
   };
-
   std::vector<VertexRun> runs;
+  std::vector<u32> plain_indices;
 
   // to do culling, the above vertex stream is grouped.
   // by following the visgroups and checking the visibility, you can leave out invisible vertices.
   struct VisGroup {
-    u32 num = 0;                // number of vertex indices in this group
+    u32 num_inds = 0;           // number of vertex indices in this group
+    u32 num_tris = 0;           // number of triangles
     u32 vis_idx_in_pc_bvh = 0;  // the visibility group they belong to (in BVH)
   };
   std::vector<VisGroup> vis_groups;
@@ -166,9 +180,8 @@ struct ShrubDraw {
   DrawMode mode;        // the OpenGL draw settings.
   u32 tree_tex_id = 0;  // the texture that should be bound for the draw
 
-  // the list of vertices in the draw. This includes the restart code of UINT32_MAX that OpenGL
-  // will use to start a new strip.
-  std::vector<u32> vertex_index_stream;
+  u32 first_index_index;
+  u32 num_indices;
 
   // for debug counting.
   u32 num_triangles = 0;
@@ -258,9 +271,11 @@ struct TfragTree {
   PackedTfragVertices packed_vertices;
   std::vector<TimeOfDayColor> colors;  // vertex colors (pre-interpolation)
   BVH bvh;                             // the bvh for frustum culling
+  bool use_strips = true;
 
   struct {
     std::vector<PreloadedVertex> vertices;  // mesh vertices
+    std::vector<u32> indices;
   } unpacked;
   void unpack();
   void serialize(Serializer& ser);
@@ -286,6 +301,7 @@ struct TieTree {
 
   struct {
     std::vector<PreloadedVertex> vertices;  // mesh vertices
+    std::vector<u32> indices;
   } unpacked;
 
   void serialize(Serializer& ser);
@@ -298,6 +314,7 @@ struct ShrubTree {
 
   PackedShrubVertices packed_vertices;
   std::vector<ShrubDraw> static_draws;  // the actual topology and settings
+  std::vector<u32> indices;
 
   struct {
     std::vector<ShrubGpuVertex> vertices;  // mesh vertices
@@ -307,6 +324,73 @@ struct ShrubTree {
   void unpack();
 };
 
+struct CollisionMesh {
+  struct Vertex {
+    float x, y, z;
+    u32 flags;
+    s16 nx, ny, nz;
+    u16 pad;
+    u32 pat;
+    u32 pad2;
+  };
+  static_assert(sizeof(Vertex) == 32);
+  std::vector<Vertex> vertices;
+  void serialize(Serializer& ser);
+};
+
+// MERC
+
+struct MercVertex {
+  float pos[3];
+  float pad0;
+
+  float normal[3];
+  float pad1;
+
+  float weights[3];
+  float pad2;
+
+  float st[2];
+
+  u8 rgba[4];
+  u8 mats[3];
+  u8 pad3;
+};
+static_assert(sizeof(MercVertex) == 64);
+
+struct MercDraw {
+  DrawMode mode;
+  u32 tree_tex_id = 0;  // the texture that should be bound for the draw
+
+  u32 first_index;
+  u32 index_count;
+  u32 num_triangles;
+  void serialize(Serializer& ser);
+};
+
+struct MercEffect {
+  std::vector<MercDraw> draws;
+  void serialize(Serializer& ser);
+};
+
+struct MercModel {
+  std::string name;
+  std::vector<MercEffect> effects;
+  float scale_xyz;
+  u32 max_draws;
+  u32 max_bones;
+  void serialize(Serializer& ser);
+};
+
+struct MercModelGroup {
+  std::vector<MercVertex> vertices;
+  std::vector<u32> indices;
+  std::vector<MercModel> models;
+  void serialize(Serializer& ser);
+};
+
+//
+
 constexpr int TFRAG_GEOS = 3;
 constexpr int TIE_GEOS = 4;
 
@@ -317,10 +401,14 @@ struct Level {
   std::array<std::vector<TfragTree>, TFRAG_GEOS> tfrag_trees;
   std::array<std::vector<TieTree>, TIE_GEOS> tie_trees;
   std::vector<ShrubTree> shrub_trees;
+  CollisionMesh collision;
+  MercModelGroup merc_data;
   u16 version2 = TFRAG3_VERSION;
   void serialize(Serializer& ser);
 
   std::array<int, MemoryUsageCategory::NUM_CATEGORIES> get_memory_usage() const;
 };
 
+void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size);
+
 }  // namespace tfrag3

common/custom_data/pack_helpers.cpp

@@ -0,0 +1,72 @@
+#include "pack_helpers.h"
+
+#include <map>
+
+constexpr float kClusterSize = 4096 * 40;  // 100 in-game meters
+constexpr float kMasterOffset = 12000 * 4096;
+
+std::pair<u64, u16> position_to_cluster_and_offset(float in) {
+  in += kMasterOffset;
+  if (in < 0) {
+    fmt::print("negative: {}\n", in);
+  }
+  ASSERT(in >= 0);
+  int cluster_cell = (in / kClusterSize);
+  float leftover = in - (cluster_cell * kClusterSize);
+  u16 offset = (leftover / kClusterSize) * float(UINT16_MAX);
+
+  float recovered = ((float)cluster_cell + ((float)offset / UINT16_MAX)) * kClusterSize;
+  float diff = std::fabs(recovered - in);
+  ASSERT(diff < 7);
+  ASSERT(cluster_cell >= 0);
+  ASSERT(cluster_cell < UINT16_MAX);
+  return {cluster_cell, offset};
+}
+
+void pack_tfrag_vertices(tfrag3::PackedTfragVertices* result,
+                         const std::vector<tfrag3::PreloadedVertex>& vertices) {
+  u32 next_cluster_idx = 0;
+  std::map<u64, u32> clusters;
+
+  for (auto& vtx : vertices) {
+    auto x = position_to_cluster_and_offset(vtx.x);
+    auto y = position_to_cluster_and_offset(vtx.y);
+    auto z = position_to_cluster_and_offset(vtx.z);
+    u64 cluster_id = 0;
+    cluster_id |= x.first;
+    cluster_id |= (y.first << 16);
+    cluster_id |= (z.first << 32);
+
+    auto cluster_it = clusters.find(cluster_id);
+    u32 my_cluster_idx = 0;
+    if (cluster_it == clusters.end()) {
+      // first in cluster
+      clusters[cluster_id] = next_cluster_idx;
+      my_cluster_idx = next_cluster_idx;
+      next_cluster_idx++;
+    } else {
+      my_cluster_idx = cluster_it->second;
+    }
+
+    tfrag3::PackedTfragVertices::Vertex out_vtx;
+    out_vtx.xoff = x.second;
+    out_vtx.yoff = y.second;
+    out_vtx.zoff = z.second;
+    out_vtx.cluster_idx = my_cluster_idx;
+    // TODO check these
+    out_vtx.s = vtx.s * 1024;
+    out_vtx.t = vtx.t * 1024;
+    out_vtx.color_index = vtx.color_index;
+    result->vertices.push_back(out_vtx);
+  }
+
+  result->cluster_origins.resize(next_cluster_idx);
+  for (auto& cluster : clusters) {
+    auto& res = result->cluster_origins[cluster.second];
+    res.x() = (u16)cluster.first;
+    res.y() = (u16)(cluster.first >> 16);
+    res.z() = (u16)(cluster.first >> 32);
+  }
+
+  ASSERT(next_cluster_idx < UINT16_MAX);
+}

common/custom_data/pack_helpers.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include "common/custom_data/Tfrag3Data.h"
+
+void pack_tfrag_vertices(tfrag3::PackedTfragVertices* result,
+                         const std::vector<tfrag3::PreloadedVertex>& vertices);

common/deserialization/subtitles/subtitles.cpp

@@ -0,0 +1,88 @@
+#include "subtitles.h"
+
+#include <algorithm>
+#include <regex>
+
+#include "common/util/FileUtil.h"
+
+#include "third-party/fmt/core.h"
+#include "third-party/fmt/ranges.h"
+#include "third-party/json.hpp"
+
+bool write_subtitle_db_to_files(const GameSubtitleDB& db) {
+  // Write the subtitles out
+  std::vector<int> completed_banks = {};
+  for (const auto& [id, bank] : db.m_banks) {
+    // If we've done the bank before, skip it
+    auto it = find(completed_banks.begin(), completed_banks.end(), bank->m_lang_id);
+    if (it != completed_banks.end()) {
+      continue;
+    }
+    // Check to see if this bank is shared by any other, if so do it at the same time
+    // and skip it
+    // This is basically just to deal with US/UK english in a not so hacky way
+    std::vector<int> banks = {};
+    for (const auto& [_id, _bank] : db.m_banks) {
+      if (_bank->file_path == bank->file_path) {
+        banks.push_back(_bank->m_lang_id);
+        completed_banks.push_back(_bank->m_lang_id);
+      }
+    }
+
+    std::string file_contents = "";
+    file_contents += fmt::format("(language-id {})\n", fmt::join(banks, " "));
+
+    for (const auto& group_name : db.m_subtitle_groups->m_group_order) {
+      file_contents +=
+          fmt::format("\n;; -----------------\n;; {}\n;; -----------------\n", group_name);
+      for (const auto& [scene_name, scene_info] : bank->m_scenes) {
+        if (scene_info.m_sorting_group != group_name) {
+          continue;
+        }
+        file_contents += fmt::format("\n(\"{}\"", scene_name);
+        if (scene_info.m_kind == SubtitleSceneKind::Hint) {
+          file_contents += " :hint #x0";
+        } else if (scene_info.m_kind == SubtitleSceneKind::HintNamed) {
+          file_contents += fmt::format(" :hint #x{0:x}", scene_info.m_id);
+        }
+        file_contents += "\n";
+        if (scene_info.m_lines.empty()) {
+          file_contents += fmt::format("  ()\n");
+        } else {
+          for (const auto& line : scene_info.m_lines) {
+            // Clear screen entries
+            if (line.line_utf8.empty()) {
+              file_contents += fmt::format("  ({})\n", line.frame);
+            } else {
+              file_contents += fmt::format("  ({}", line.frame);
+              if (line.offscreen && scene_info.m_kind == SubtitleSceneKind::Movie) {
+                file_contents += " :offscreen";
+              }
+              file_contents += fmt::format(" \"{}\"", line.speaker_utf8);
+              // escape quotes
+              std::string temp = line.line_utf8;
+              temp = std::regex_replace(temp, std::regex("\""), "\\\"");
+              file_contents += fmt::format(" \"{}\")\n", temp);
+            }
+          }
+        }
+        file_contents += "  )\n";
+      }
+    }
+
+    // Commit it to the file
+    std::string full_path =
+        (file_util::get_jak_project_dir() / std::filesystem::path(bank->file_path)).string();
+    file_util::write_text_file(full_path, file_contents);
+  }
+
+  // Write the subtitle group info out
+  nlohmann::json json(db.m_subtitle_groups->m_groups);
+  json[db.m_subtitle_groups->group_order_key] = nlohmann::json(db.m_subtitle_groups->m_group_order);
+  std::string file_path = (file_util::get_jak_project_dir() / "game" / "assets" / "jak1" /
+                           "subtitle" / "subtitle-groups.json")
+                              .string();
+  file_util::write_text_file(file_path, json.dump(2));
+
+  return true;
+}

common/deserialization/subtitles/subtitles.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#include "common/serialization/subtitles/subtitles.h"
+
+bool write_subtitle_db_to_files(const GameSubtitleDB& db);

common/dma/dma.cpp

@@ -1,8 +1,9 @@
 #include "dma.h"
 
-#include "third-party/fmt/core.h"
 #include "common/util/Assert.h"
 
+#include "third-party/fmt/core.h"
+
 std::string DmaTag::print() {
   std::string result;
   const char* mode_names[8] = {"refe", "cnt", "next", "ref", "refs", "call", "ret", "end"};
@@ -79,6 +80,9 @@ std::string VifCode::print() {
     case Kind::DIRECTHL:
       result = "DIRECTHL";
       break;
+    case Kind::PC_PORT:
+      result = "PC_PORT";
+      break;
     case Kind::UNPACK_V4_8: {
       VifCodeUnpack up(*this);
       result = fmt::format("UNPACK-V4-8: {} addr: {} us: {} tops: {}", num, up.addr_qw,
@@ -115,10 +119,8 @@ std::string VifCode::print() {
     }
 
     default:
-      fmt::print("Unhandled vif code {}\n", (int)kind);
-
       result = "???";
-      ASSERT(false);
+      ASSERT_MSG(false, fmt::format("Unhandled vif code {}", (int)kind));
       break;
   }
   // TODO: the rest of the VIF code.

common/dma/dma.h

@@ -5,11 +5,14 @@
  * PS2 DMA and VIF types.
  */
 
-#include <string>
 #include <cstring>
+#include <string>
+
 #include "common/common_types.h"
 #include "common/util/Assert.h"
 
+#include "third-party/fmt/core.h"
+
 struct DmaStats {
   double sync_time_ms = 0;
   int num_tags = 0;
@@ -68,6 +71,11 @@ inline void emulate_dma(const void* source_base, void* dest_base, u32 tadr, u32
         dest_offset += (1 + tag.qwc) * 16;
         tadr += 16 + tag.qwc * 16;
         break;
+      case DmaTag::Kind::NEXT:
+        memcpy(dst + dest_offset, src + tadr, (1 + tag.qwc) * 16);
+        dest_offset += (1 + tag.qwc) * 16;
+        tadr = tag.addr;
+        break;
       case DmaTag::Kind::REF: {
         // tte
         memcpy(dst + dest_offset, src + tadr, 16);
@@ -91,8 +99,7 @@ inline void emulate_dma(const void* source_base, void* dest_base, u32 tadr, u32
         // does this transfer anything in TTE???
         return;
       default:
-        printf("bad tag: %d\n", (int)tag.kind);
-        ASSERT(false);
+        ASSERT_MSG(false, fmt::format("bad tag: {}", (int)tag.kind));
     }
   }
 }

common/dma/dma_chain_read.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <cstring>
+
 #include "common/dma/dma.h"
 #include "common/util/Assert.h"
 

common/dma/dma_copy.cpp

@@ -1,10 +1,12 @@
 
-#include "common/goal_constants.h"
-#include "common/dma/dma_chain_read.h"
 #include "dma_copy.h"
-#include "third-party/fmt/core.h"
+
+#include "common/dma/dma_chain_read.h"
+#include "common/goal_constants.h"
 #include "common/util/Timer.h"
 
+#include "third-party/fmt/core.h"
+
 /*!
  * Convert a DMA chain to an array of bytes that can be directly fed to VIF.
  */

common/dma/gs.cpp

@@ -1,8 +1,9 @@
 #include "gs.h"
 
-#include "third-party/fmt/core.h"
 #include "common/util/Assert.h"
 
+#include "third-party/fmt/core.h"
+
 std::string reg_descriptor_name(GifTag::RegisterDescriptor reg) {
   switch (reg) {
     case GifTag::RegisterDescriptor::PRIM:

common/dma/gs.h

@@ -30,7 +30,7 @@ struct GifTag {
   };
 
   u32 nloop() const {
-    return data[0] & 0x7f;  // 15 bits
+    return data[0] & 0x7fff;  // 15 bits
   }
 
   bool eop() const { return data[0] & (1ull << 15); }

common/global_profiler/GlobalProfiler.cpp

@@ -0,0 +1,186 @@
+// clang-format off
+#include "GlobalProfiler.h"
+
+#include <chrono>
+#include <cstring>
+#include <thread>
+
+#include "common/util/Assert.h"
+#include "common/util/FileUtil.h"
+
+#include "third-party/fmt/core.h"
+#include "third-party/json.hpp"
+
+#ifdef __linux__
+u32 get_current_tid() {
+  return (u32)pthread_self();
+}
+#else
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include "Processthreadsapi.h"
+u32 get_current_tid() {
+  return (u32)GetCurrentThreadId();
+}
+#endif
+// clang-format on
+
+u64 get_current_ts() {
+  return std::chrono::steady_clock::now().time_since_epoch().count();
+}
+
+GlobalProfiler::GlobalProfiler() {
+  m_t0 = get_current_ts();
+  set_max_events(65536);
+}
+
+void GlobalProfiler::set_max_events(size_t event_count) {
+  ASSERT(!m_enabled);
+  m_nodes.resize(event_count);
+}
+
+void GlobalProfiler::event(const char* name, ProfNode::Kind kind) {
+  if (!m_enabled) {
+    return;
+  }
+  size_t my_idx = (m_next_idx++ % m_nodes.size());
+  auto& node = m_nodes[my_idx];
+  node.ts = get_current_ts() - m_t0;
+  node.tid = get_current_tid();
+  node.kind = kind;
+  strncpy(node.name, name, sizeof(node.name));
+  node.name[sizeof(node.name) - 1] = '\0';
+}
+
+void GlobalProfiler::instant_event(const char* name) {
+  event(name, ProfNode::INSTANT);
+}
+
+void GlobalProfiler::begin_event(const char* name) {
+  event(name, ProfNode::BEGIN);
+}
+
+void GlobalProfiler::end_event() {
+  if (!m_enabled) {
+    return;
+  }
+  size_t my_idx = (m_next_idx++ % m_nodes.size());
+  auto& node = m_nodes[my_idx];
+  node.ts = get_current_ts() - m_t0;
+  node.tid = get_current_tid();
+  node.kind = ProfNode::END;
+  node.name[0] = '\0';
+}
+
+void GlobalProfiler::clear() {
+  m_next_idx = 0;
+}
+
+void GlobalProfiler::set_enable(bool en) {
+  m_enabled = en;
+}
+
+void GlobalProfiler::dump_to_json(const std::string& path) {
+  ASSERT(!m_enabled);
+
+  nlohmann::json json;
+  auto& trace_events = json["traceEvents"];
+  json["displayTimeUnit"] = "ms";
+
+  u64 lowest_ts = UINT64_MAX;
+  struct ThreadInfo {
+    size_t lowest_at_target = UINT64_MAX;
+    size_t highest_at_target = 0;
+    size_t debug = 0;
+    u32 short_id = 0;
+  };
+  std::unordered_map<u32, ThreadInfo> info_per_thread;
+
+  // first, find all the threads
+  std::string kRootName = "ROOT";
+  for (size_t offset = m_nodes.size(); offset-- > 0;) {
+    size_t idx = (m_next_idx + offset) % m_nodes.size();
+    const auto& event = m_nodes[idx];
+
+    if (event.kind != ProfNode::UNUSED) {
+      lowest_ts = std::min(event.ts, lowest_ts);
+    }
+    if (event.kind == ProfNode::INSTANT && kRootName == event.name) {
+      auto& info = info_per_thread[event.tid];
+      info.lowest_at_target = std::min(info.lowest_at_target, offset);
+      info.highest_at_target = std::max(info.highest_at_target, offset);
+    }
+  }
+
+  u32 i = 0;
+  for (auto& info : info_per_thread) {
+    info.second.short_id = i++;
+  }
+
+  for (size_t event_idx = 0; event_idx < m_nodes.size(); event_idx++) {
+    auto& event = m_nodes[(event_idx + m_next_idx) % m_nodes.size()];
+    if (event.kind == ProfNode::UNUSED) {
+      continue;
+    }
+    auto& info = info_per_thread.at(event.tid);
+    if (event_idx < info.lowest_at_target || event_idx > info.highest_at_target) {
+      continue;
+    }
+
+    auto& json_event = trace_events.emplace_back();
+    // name
+    if (event.kind != ProfNode::END) {
+      json_event["name"] = event.name;
+    }
+
+    // cat
+    // json_event["cat"] = "a";  // ??
+    // ph BEi
+    switch (event.kind) {
+      case ProfNode::END:
+        json_event["ph"] = "E";
+        break;
+      case ProfNode::BEGIN:
+        json_event["ph"] = "B";
+        break;
+      case ProfNode::INSTANT:
+        json_event["ph"] = "i";
+        break;
+      default:
+        ASSERT(false);
+    }
+    // pid
+    json_event["pid"] = 1;
+    // tid
+    json_event["tid"] = info.short_id;
+    // ts
+    json_event["ts"] = (event.ts - lowest_ts) / 1000.f;
+    if (event.ts < info.debug) {
+      fmt::print("out of order: {} {} {} ms\n", event.ts / 1000.f, info.debug / 1000.f,
+                 (info.debug - event.ts) / 1000000.f);
+      fmt::print("  idx: {}, range {} {}\n", event_idx, info.lowest_at_target,
+                 info.highest_at_target);
+      fmt::print("  now: {}\n", m_next_idx);
+    }
+    info.debug = event.ts;
+  }
+
+  for (auto& t : info_per_thread) {
+    fmt::print("thread: {}: {} -> {}\n", t.first, t.second.lowest_at_target,
+               t.second.highest_at_target);
+  }
+
+  file_util::write_text_file(path, json.dump());
+}
+
+GlobalProfiler gprof;
+GlobalProfiler& prof() {
+  return gprof;
+}
+
+ScopedEvent scoped_prof(const char* name) {
+  auto& p = prof();
+  p.begin_event(name);
+  return {&p};
+}

common/global_profiler/GlobalProfiler.h

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <atomic>
+#include <string>
+#include <vector>
+
+#include "common/common_types.h"
+
+struct ProfNode {
+  u64 ts;
+  char name[32];
+  enum Kind : u8 { BEGIN, END, INSTANT, UNUSED } kind = UNUSED;
+  u32 tid;
+};
+
+class GlobalProfiler {
+ public:
+  GlobalProfiler();
+  void set_max_events(size_t event_count);
+  void instant_event(const char* name);
+  void begin_event(const char* name);
+  void event(const char* name, ProfNode::Kind kind);
+  void end_event();
+  void clear();
+  void set_enable(bool en);
+  void dump_to_json(const std::string& path);
+
+ private:
+  std::atomic_bool m_enabled = false;
+  u64 m_t0 = 0;
+  std::atomic_size_t m_next_idx = 0;
+  std::vector<ProfNode> m_nodes;
+};
+
+struct ScopedEvent {
+  ScopedEvent(const ScopedEvent&) = delete;
+  ScopedEvent& operator=(const ScopedEvent&) = delete;
+  GlobalProfiler* prof = nullptr;
+  ~ScopedEvent() {
+    if (prof) {
+      prof->end_event();
+    }
+  }
+};
+
+GlobalProfiler& prof();
+ScopedEvent scoped_prof(const char* name);

common/global_profiler/readme.md

@@ -0,0 +1,32 @@
+# Event Profiler
+The event profiler is a tool to analyze timing of multiple frames.  Unlike sampling-based profilers, this profile captures an exact timeline of what happens of what happens when.
+
+## Capturing a profile
+In the OpenGOAL window, click "Profiler" and check "Record" to start recording. The buffer has a fixed maximum size and it will automatically overwrite old data once it is full.
+
+When something interesting happens, click the "dump to file" button to save the buffer (currently a few seconds) to `prof.json` in `jak-project`.
+
+The idea is that you can leave this running as you play, and then when the game stutters or does something interesting, you can click the dump button and get the result.
+
+## Viewing a profile
+Open Google Chrome and go to `chrome://tracing`. Then click load and open the json file.  Or, just drag and drop the file into chrome.
+
+Press `1` for a box drawing tool. This lets you select a region of the flame chart and get a list of events inside the box.
+
+Press `2` for panning.
+
+Press `3` for zooming.
+
+## Adding an event
+The GOAL kernel automatically adds events for each process. If you want to add another event, you can use `(with-profiler "name-of-event" <body>)`. Do not call `suspend` or do a `return` inside of this. If you need more control over stopping/starting, there are functions in `gcommon.gc` to explicitly start/stop events. But you must match them up correctly yourself!
+
+In C++, the graphics profiler automatically adds a profiler nodes as events. To add an event, you can use 
+
+```auto p = scoped_prof("name-of-event");``` 
+
+The event is active from this call until the destruction of `p`.
+
+## Multiple threads
+The event profiler currently works on both the graphics and EE threads. Adding the events can safely be done from any thread, but enable/disable/dump should be done from a single thread at a time.
+
+Each thread should periodically insert a `ROOT` instant event when there are no active range events. This is required to make the retroactive dump feature work properly as the event buffer does not capture the tree structure fully, and it must be able to find a point in time when no events are active.

common/goal_constants.h

@@ -1,6 +1,7 @@
 #pragma once
 
-#include "common_types.h"
+#include "common/common_types.h"
+#include "common/versions.h"
 
 constexpr s32 BINTEGER_OFFSET = 0;
 constexpr s32 PAIR_OFFSET = 2;
@@ -13,10 +14,12 @@ constexpr int ARRAY_DATA_OFFSET = 12;  // not including type tag
  * Here you can change the size of the symbol table!
  * Make sure to also edit the constant in gcommon.gc
  */
-constexpr s32 GOAL_MAX_SYMBOLS = 8192;  // this MUST be a multiple of 2!!
-constexpr s32 SYM_INFO_OFFSET = 8167 * 8 - 4;
-// constexpr s32 GOAL_MAX_SYMBOLS = 16384;
-// constexpr s32 SYM_INFO_OFFSET = GOAL_MAX_SYMBOLS * 8 - 4;
+// constexpr s32 GOAL_MAX_SYMBOLS = 8192;  // this MUST be a multiple of 2!!
+// constexpr s32 SYM_INFO_OFFSET = 8167 * 8 - 4;
+
+namespace jak1 {
+constexpr s32 GOAL_MAX_SYMBOLS = 16384;
+constexpr s32 SYM_INFO_OFFSET = GOAL_MAX_SYMBOLS * 8 - 4;
 
 constexpr s32 SYM_TABLE_MEM_SIZE = GOAL_MAX_SYMBOLS * 8 * 2;
 
@@ -34,7 +37,26 @@ constexpr int bits_for_sym() {
   }
   return b + 1;
 }
-static_assert(bits_for_sym() != 1, "symbol table invalid length");
+static_assert(bits_for_sym() != -1, "symbol table invalid length");
+}  // namespace jak1
+
+namespace jak2 {
+// for now, we don't have the ability to extend the size of the symbol table
+constexpr s32 GOAL_MAX_SYMBOLS = 0x4000;
+constexpr s32 SYM_TABLE_MEM_SIZE = 0x30000;
+// from the "off-by-one" symbol pointer
+constexpr int SYM_TO_STRING_OFFSET = 0xff37;
+constexpr int SYM_TO_HASH_OFFSET = 0x1fe6f;
+}  // namespace jak2
+
+constexpr s32 max_symbols(GameVersion version) {
+  switch (version) {
+    case GameVersion::Jak1:
+      return jak1::GOAL_MAX_SYMBOLS;
+    case GameVersion::Jak2:
+      return jak2::GOAL_MAX_SYMBOLS;
+  }
+}
 
 enum class RegClass { GPR_64, FLOAT, INT_128, VECTOR_FLOAT, INVALID };
 

common/goos/Interpreter.cpp

@@ -3,17 +3,19 @@
  * The GOOS Interpreter and implementation of special and "built-in forms"
  */
 
-#include <utility>
 #include "Interpreter.h"
+
+#include <utility>
+
 #include "ParseHelpers.h"
+
 #include "common/util/FileUtil.h"
+
 #include "third-party/fmt/core.h"
 
 namespace goos {
 Interpreter::Interpreter(const std::string& username) {
   // Interpreter startup:
-  goal_to_goos.reset();
-
   // create the GOOS global environment
   global_environment = EnvironmentObject::make_new("global");
 
@@ -73,7 +75,6 @@ Interpreter::Interpreter(const std::string& username) {
                    {">=", &Interpreter::eval_geq},
                    {"null?", &Interpreter::eval_null},
                    {"type?", &Interpreter::eval_type},
-                   {"current-method-type", &Interpreter::eval_current_method_type},
                    {"fmt", &Interpreter::eval_format},
                    {"error", &Interpreter::eval_error},
                    {"string-ref", &Interpreter::eval_string_ref},
@@ -82,7 +83,10 @@ Interpreter::Interpreter(const std::string& username) {
                    {"ash", &Interpreter::eval_ash},
                    {"symbol->string", &Interpreter::eval_symbol_to_string},
                    {"string->symbol", &Interpreter::eval_string_to_symbol},
-                   {"get-environment-variable", &Interpreter::eval_get_env}};
+                   {"get-environment-variable", &Interpreter::eval_get_env},
+                   {"make-string-hash-table", &Interpreter::eval_make_string_hash_table},
+                   {"hash-table-set!", &Interpreter::eval_hash_table_set},
+                   {"hash-table-try-ref", &Interpreter::eval_hash_table_try_ref}};
 
   string_to_type = {{"empty-list", ObjectType::EMPTY_LIST},
                     {"integer", ObjectType::INTEGER},
@@ -1348,7 +1352,7 @@ Object Interpreter::eval_numequals(const Object& form,
     } break;
 
     default:
-      throw_eval_error(form, "+ must have a numeric argument");
+      throw_eval_error(form, "= must have a numeric argument");
       return Object::make_empty_list();
   }
 
@@ -1550,14 +1554,6 @@ Object Interpreter::eval_type(const Object& form,
   }
 }
 
-Object Interpreter::eval_current_method_type(const Object& form,
-                                             Arguments& args,
-                                             const std::shared_ptr<EnvironmentObject>& env) {
-  (void)env;
-  vararg_check(form, args, {}, {});
-  return SymbolObject::make_new(reader.symbolTable, goal_to_goos.enclosing_method_type);
-}
-
 Object Interpreter::eval_format(const Object& form,
                                 Arguments& args,
                                 const std::shared_ptr<EnvironmentObject>& env) {
@@ -1697,4 +1693,45 @@ Object Interpreter::eval_get_env(const Object& form,
   }
   return StringObject::make_new(env_p);
 }
+
+/*!
+ * Create a new empty hash table object.
+ */
+Object Interpreter::eval_make_string_hash_table(const Object& form,
+                                                Arguments& args,
+                                                const std::shared_ptr<EnvironmentObject>& /*env*/) {
+  vararg_check(form, args, {}, {});
+  return StringHashTableObject::make_new();
+}
+
+/*!
+ * Set a value in a hash table. Overwrites a previous value or inserts a new one.
+ * Returns empty list always.
+ */
+Object Interpreter::eval_hash_table_set(const Object& form,
+                                        Arguments& args,
+                                        const std::shared_ptr<EnvironmentObject>& /*env*/) {
+  vararg_check(form, args, {ObjectType::STRING_HASH_TABLE, ObjectType::STRING, {}}, {});
+  args.unnamed.at(0).as_string_hash_table()->data[args.unnamed.at(1).as_string()->data] =
+      args.unnamed.at(2);
+  return Object::make_empty_list();
+}
+
+/*!
+ * Try to look up a value by key in a hash table. The result is a pair of (success . value).
+ */
+Object Interpreter::eval_hash_table_try_ref(const Object& form,
+                                            Arguments& args,
+                                            const std::shared_ptr<EnvironmentObject>& /*env*/) {
+  vararg_check(form, args, {ObjectType::STRING_HASH_TABLE, ObjectType::STRING}, {});
+  const auto* table = args.unnamed.at(0).as_string_hash_table();
+  const auto& it = table->data.find(args.unnamed.at(1).as_string()->data);
+  if (it == table->data.end()) {
+    // not in table
+    return PairObject::make_new(SymbolObject::make_new(reader.symbolTable, "#f"),
+                                Object::make_empty_list());
+  } else {
+    return PairObject::make_new(SymbolObject::make_new(reader.symbolTable, "#t"), it->second);
+  }
+}
 }  // namespace goos

common/goos/Interpreter.h

@@ -7,6 +7,7 @@
 
 #include <memory>
 #include <optional>
+
 #include "Object.h"
 #include "Reader.h"
 
@@ -49,13 +50,6 @@ class Interpreter {
   Object global_environment;
   Object goal_env;
 
-  // data passed from GOAL to GOOS available to any evaluation.
-  struct GoalToGoosData {
-    std::string enclosing_method_type;
-
-    void reset() { enclosing_method_type = "#f"; }
-  } goal_to_goos;
-
  private:
   friend class Goal;
   void load_goos_library();
@@ -188,9 +182,6 @@ class Interpreter {
   Object eval_type(const Object& form,
                    Arguments& args,
                    const std::shared_ptr<EnvironmentObject>& env);
-  Object eval_current_method_type(const Object& form,
-                                  Arguments& args,
-                                  const std::shared_ptr<EnvironmentObject>& env);
   Object eval_format(const Object& form,
                      Arguments& args,
                      const std::shared_ptr<EnvironmentObject>& env);
@@ -251,6 +242,16 @@ class Interpreter {
                     const Object& rest,
                     const std::shared_ptr<EnvironmentObject>& env);
 
+  Object eval_make_string_hash_table(const Object& form,
+                                     Arguments& args,
+                                     const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_hash_table_try_ref(const Object& form,
+                                 Arguments& args,
+                                 const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_hash_table_set(const Object& form,
+                             Arguments& args,
+                             const std::shared_ptr<EnvironmentObject>& env);
+
   bool want_exit = false;
   bool disable_printing = false;
 

common/goos/Object.cpp

@@ -38,12 +38,15 @@
  *
  */
 
-#include <cinttypes>
 #include "Object.h"
+
+#include <cinttypes>
+
 #include "common/util/FileUtil.h"
-#include "third-party/fmt/core.h"
 #include "common/util/print_float.h"
 
+#include "third-party/fmt/core.h"
+
 namespace goos {
 
 /*!
@@ -73,6 +76,8 @@ std::string object_type_to_string(ObjectType type) {
       return "[macro]";
     case ObjectType::ENVIRONMENT:
       return "[environment]";
+    case ObjectType::STRING_HASH_TABLE:
+      return "[string-hash-table]";
     default:
       throw std::runtime_error("unknown object type in object_type_to_string");
   }
@@ -243,6 +248,8 @@ bool Object::operator==(const Object& other) const {
       }
       return true;
     }
+    case ObjectType::STRING_HASH_TABLE:
+      return as_string_hash_table()->data == other.as_string_hash_table()->data;
 
     default:
       throw std::runtime_error("equality not implemented for " + print());

common/goos/Object.h

@@ -38,16 +38,20 @@
  * MACRO - a heap allocated object representing a GOOS macro
  * ENVIRONMENT - a heap allocated object representing a GOOS environment
  *
+ * STRING_HASH_TABLE - a heap allocated object representing a key-value table. Currently the keys
+ * can only be strings, like make-str-hash-table in MIT Scheme.
+ *
  */
 
-#include <string>
+#include <map>
 #include <memory>
+#include <stdexcept>
+#include <string>
 #include <unordered_map>
 #include <unordered_set>
 #include <utility>
 #include <vector>
-#include <stdexcept>
-#include <map>
+
 #include "common/common_types.h"
 #include "common/util/Assert.h"
 
@@ -76,6 +80,7 @@ enum class ObjectType : u8 {
   LAMBDA,
   MACRO,
   ENVIRONMENT,
+  STRING_HASH_TABLE,
   INVALID
 };
 
@@ -162,6 +167,7 @@ class StringObject;
 class LambdaObject;
 class MacroObject;
 class ArrayObject;
+class StringHashTableObject;
 
 // Wrapper Object class for all objects
 class Object {
@@ -246,6 +252,7 @@ class Object {
   LambdaObject* as_lambda() const;
   MacroObject* as_macro() const;
   ArrayObject* as_array() const;
+  StringHashTableObject* as_string_hash_table() const;
 
   IntType& as_int() {
     if (type != ObjectType::INTEGER) {
@@ -288,7 +295,9 @@ class Object {
   bool is_empty_list() const { return type == ObjectType::EMPTY_LIST; }
   bool is_list() const { return type == ObjectType::EMPTY_LIST || type == ObjectType::PAIR; }
   bool is_int() const { return type == ObjectType::INTEGER; }
+  bool is_int(int val) const { return type == ObjectType::INTEGER && val == as_int(); }
   bool is_float() const { return type == ObjectType::FLOAT; }
+  bool is_float(float val) const { return type == ObjectType::FLOAT && val == as_float(); }
   bool is_char() const { return type == ObjectType::CHAR; }
   bool is_symbol() const { return type == ObjectType::SYMBOL; }
   bool is_symbol(const std::string& name) const;
@@ -297,6 +306,7 @@ class Object {
   bool is_array() const { return type == ObjectType::ARRAY; }
   bool is_env() const { return type == ObjectType::ENVIRONMENT; }
   bool is_macro() const { return type == ObjectType::MACRO; }
+  bool is_string_hash_table() const { return type == ObjectType::STRING_HASH_TABLE; }
 
   bool operator==(const Object& other) const;
   bool operator!=(const Object& other) const { return !((*this) == other); }
@@ -592,6 +602,39 @@ class ArrayObject : public HeapObject {
   Object& operator[](size_t idx) { return data.at(idx); }
 };
 
+class StringHashTableObject : public HeapObject {
+ public:
+  std::unordered_map<std::string, Object> data;
+  StringHashTableObject() = default;
+  static Object make_new() {
+    Object obj;
+    obj.type = ObjectType::STRING_HASH_TABLE;
+    obj.heap_obj = std::make_shared<StringHashTableObject>();
+    return obj;
+  }
+
+  std::string print() const override {
+    std::string result = "{";
+    for (const auto& kv : data) {
+      result += '(';
+      result += kv.first;
+      result += ' ';
+      result += kv.second.print();
+      result += ')';
+      result += ' ';
+    }
+    if (!data.empty()) {
+      result.pop_back();
+    }
+    result += '}';
+    return result;
+  }
+
+  std::string inspect() const override {
+    return "[string-hash-table] kind: string, data: " + print() + "\n";
+  }
+};
+
 Object build_list(const std::vector<Object>& objects);
 Object build_list(std::vector<Object>&& objects);
 
@@ -653,4 +696,12 @@ inline ArrayObject* Object::as_array() const {
   }
   return static_cast<ArrayObject*>(heap_obj.get());
 }
+
+inline StringHashTableObject* Object::as_string_hash_table() const {
+  if (type != ObjectType::STRING_HASH_TABLE) {
+    throw std::runtime_error("as_string_hash_table called on a " + object_type_to_string(type) +
+                             " " + print());
+  }
+  return static_cast<StringHashTableObject*>(heap_obj.get());
+}
 }  // namespace goos

common/goos/ParseHelpers.cpp

@@ -1,5 +1,7 @@
 #include "ParseHelpers.h"
 
+#include "third-party/fmt/core.h"
+
 namespace goos {
 
 bool get_va(const goos::Object& rest, std::string* err_string, goos::Arguments* result) {
@@ -44,16 +46,16 @@ bool va_check(
     std::string* err_string) {
   ASSERT(args.rest.empty());
   if (unnamed.size() != args.unnamed.size()) {
-    *err_string = "Got " + std::to_string(args.unnamed.size()) + " arguments, but expected " +
-                  std::to_string(unnamed.size());
+    *err_string = fmt::format("Got {} arguments, but expected {}",
+                              std::to_string(args.unnamed.size()), std::to_string(unnamed.size()));
     return false;
   }
 
   for (size_t i = 0; i < unnamed.size(); i++) {
     if (unnamed[i].has_value() && unnamed[i] != args.unnamed[i].type) {
-      *err_string = "Argument " + std::to_string(i) + " has type " +
-                    object_type_to_string(args.unnamed[i].type) + " but " +
-                    object_type_to_string(unnamed[i].value()) + " was expected";
+      *err_string = fmt::format("Argument {} has type {} but {} was expected\nArgument is: {}", i,
+                                object_type_to_string(args.unnamed[i].type),
+                                object_type_to_string(unnamed[i].value()), args.unnamed[i].print());
       return false;
     }
   }

common/goos/ParseHelpers.h

@@ -2,8 +2,9 @@
 
 #include <optional>
 #include <string>
-#include <vector>
 #include <unordered_map>
+#include <vector>
+
 #include "Object.h"
 
 namespace goos {
@@ -18,7 +19,7 @@ template <typename T>
 void for_each_in_list(const goos::Object& list, T f) {
   const goos::Object* iter = &list;
   while (iter->is_pair()) {
-    auto lap = iter->as_pair();
+    const auto& lap = iter->as_pair();
     f(lap->car);
     iter = &lap->cdr;
   }

common/goos/PrettyPrinter.cpp

@@ -4,17 +4,20 @@
  * It is not very good, but significantly better than putting everything on one line
  */
 
+#include "PrettyPrinter.h"
+
+#include <cstring>
 #include <stdexcept>
 #include <utility>
-#include <cstring>
-#include "PrettyPrinter.h"
+
 #include "Reader.h"
-#include "third-party/fmt/core.h"
-#include "common/log/log.h"
 
 #include "common/goos/PrettyPrinter2.h"
+#include "common/log/log.h"
 #include "common/util/Assert.h"
 
+#include "third-party/fmt/core.h"
+
 namespace pretty_print {
 
 /*!

common/goos/PrettyPrinter.h

@@ -8,10 +8,11 @@
 
 #include <string>
 #include <vector>
+
 #include "common/goos/Object.h"
-#include "common/goos/Reader.h"
-#include "common/goos/Printer.h"
 #include "common/goos/PrettyPrinter2.h"
+#include "common/goos/Printer.h"
+#include "common/goos/Reader.h"
 
 namespace pretty_print {
 // main pretty print function

common/goos/PrettyPrinter2.cpp

@@ -1,8 +1,10 @@
 #include "PrettyPrinter2.h"
+
 #include "common/common_types.h"
-#include "third-party/fmt/core.h"
 #include "common/util/Assert.h"
 
+#include "third-party/fmt/core.h"
+
 namespace pretty_print {
 
 namespace v2 {
@@ -17,15 +19,10 @@ namespace v2 {
 struct Node {
   Node() = default;
 
-  Node(const std::string& str) {
-    kind = Kind::ATOM;
-    atom_str = str;
-  }
+  Node(const std::string& str) : kind(Kind::ATOM), atom_str(str) {}
 
-  Node(std::vector<Node>&& list, bool is_list) {
-    kind = is_list ? Kind::LIST : Kind::IMPROPER_LIST;
-    child_nodes = std::move(list);
-  }
+  Node(std::vector<Node>&& list, bool is_list)
+      : kind(is_list ? Kind::LIST : Kind::IMPROPER_LIST), child_nodes(std::move(list)) {}
   enum class Kind : u8 { ATOM, LIST, IMPROPER_LIST, INVALID } kind = Kind::INVALID;
 
   std::vector<Node> child_nodes;
@@ -203,8 +200,9 @@ void break_list(Node* node) {
   node->top_line_count = 1;
 
   const std::unordered_set<std::string> sameline_splitters = {
-      "if", "<",   ">",  "<=",  ">=", "set!",   "=",      "!=",     "+",  "-",  "*",
-      "/",  "the", "->", "and", "or", "logand", "logior", "logxor", "+!", "*!", "logtest?"};
+      "if",     "<",  ">",  "<=",       ">=",  "set!",  "=",       "!=",     "+",
+      "-",      "*",  "/",  "the",      "->",  "and",   "or",      "logand", "logior",
+      "logxor", "+!", "*!", "logtest?", "not", "zero?", "nonzero?"};
 
   if (node->child_nodes.at(0).kind == Node::Kind::LIST) {
     // ((foo
@@ -215,11 +213,23 @@ void break_list(Node* node) {
     if (name == "defun" || name == "defun-debug" || name == "defbehavior" || name == "defstate") {
       // things with three things in the top line: (defun <name> <args>
       node->top_line_count = 3;
+    } else if (name == "defskelgroup") {
+      // things with 5 things in the top line: (defskelgroup <name> <art> jgeo janim
+      node->top_line_count = 5;
+      node->sub_elt_indent += name.size();
+    } else if (name == "process-new") {
+      // things with 3 things in the top line
+      node->top_line_count = 3;
+      node->sub_elt_indent += name.size();
+    } else if (name == "ja" || name == "ja-no-eval") {
+      node->top_line_count = 3;
+      node->sub_elt_indent += name.size();
     } else if (name == "defmethod") {
       // things with 4 things in the top line: (defmethod <method> <type> <args>
       node->top_line_count = 4;
     } else if (name == "until" || name == "while" || name == "dotimes" || name == "countdown" ||
-               name == "when" || name == "behavior" || name == "lambda" || name == "defpart") {
+               name == "when" || name == "behavior" || name == "lambda" || name == "defpart" ||
+               name == "define") {
       node->top_line_count = 2;
     } else if (name == "let" || name == "let*" || name == "rlet") {
       // special case for things like let.
@@ -264,9 +274,9 @@ void break_list(Node* node) {
 
 void insert_required_breaks(const std::vector<Node*>& bfs_order) {
   const std::unordered_set<std::string> always_break = {
-      "when",    "defun-debug", "countdown", "case",     "defun",  "defmethod", "let",
-      "until",   "while",       "if",        "dotimes",  "cond",   "else",      "defbehavior",
-      "with-pp", "rlet",        "defstate",  "behavior", "defpart"};
+      "when",    "defun-debug", "countdown", "case",     "defun",   "defmethod", "let",
+      "until",   "while",       "if",        "dotimes",  "cond",    "else",      "defbehavior",
+      "with-pp", "rlet",        "defstate",  "behavior", "defpart", "loop"};
   for (auto node : bfs_order) {
     if (!node->break_list && node->kind == Node::Kind::LIST &&
         node->child_nodes.at(0).kind == Node::Kind::ATOM) {
@@ -363,6 +373,7 @@ void append_node_to_string(const Node* node,
           str.pop_back();
         }
         str.push_back('\n');
+        bool after_key = false;
         for (; node_idx < node->child_nodes.size(); node_idx++) {
           if (node->kind == Node::Kind::IMPROPER_LIST &&
               &node->child_nodes.at(node_idx) == &node->child_nodes.back()) {
@@ -371,9 +382,17 @@ void append_node_to_string(const Node* node,
             }
             str.append(".\n");
           }
-          append_node_to_string(&node->child_nodes.at(node_idx), str, listing_indent,
-                                listing_indent);
-          str.push_back('\n');
+          append_node_to_string(&node->child_nodes.at(node_idx), str,
+                                after_key ? 0 : listing_indent, listing_indent);
+          if (node->child_nodes.at(node_idx).kind == Node::Kind::ATOM &&
+              node->child_nodes.at(node_idx).atom_str.at(0) == ':' &&
+              node->child_nodes.at(node_idx).atom_str.find(' ') == std::string::npos) {
+            str.push_back(' ');
+            after_key = true;
+          } else {
+            str.push_back('\n');
+            after_key = false;
+          }
         }
         for (int i = 0; i < listing_indent; i++) {
           str.push_back(' ');

common/goos/PrettyPrinter2.h

@@ -2,9 +2,10 @@
 
 #include <string>
 #include <vector>
+
 #include "common/goos/Object.h"
-#include "common/goos/Reader.h"
 #include "common/goos/Printer.h"
+#include "common/goos/Reader.h"
 
 namespace pretty_print {
 // main pretty print function

common/goos/Printer.cpp

@@ -1,4 +1,5 @@
 #include "Printer.h"
+
 #include "third-party/fmt/core.h"
 
 namespace pretty_print {

common/goos/Printer.h

@@ -2,6 +2,7 @@
 
 #include <string>
 #include <vector>
+
 #include "common/goos/Object.h"
 #include "common/goos/Reader.h"
 

common/goos/Reader.cpp

@@ -10,11 +10,15 @@
  */
 
 #include "Reader.h"
+
+#include <filesystem>
+
+#include "ReplUtils.h"
+
 #include "common/util/FileUtil.h"
 #include "common/util/FontUtils.h"
+
 #include "third-party/fmt/core.h"
-#include <filesystem>
-#include "ReplUtils.h"
 
 namespace goos {
 
@@ -835,13 +839,6 @@ void Reader::throw_reader_error(TextStream& here, const std::string& err, int se
                            db.get_info_for(here.text, here.seek + seek_offset));
 }
 
-/*!
- * Get the source directory of the current project.
- */
-std::string Reader::get_source_dir() {
-  return file_util::get_project_path();
-}
-
 /*!
  * Convert any string into one that can be read.
  * Unprintable characters become escape sequences, including tab and newline.

common/goos/Reader.h

@@ -12,15 +12,14 @@
  */
 
 #include <memory>
-#include <utility>
-#include <unordered_map>
 #include <optional>
-
-#include "common/goos/Object.h"
-#include "common/goos/TextDB.h"
+#include <unordered_map>
+#include <utility>
 
 #include "ReplUtils.h"
 
+#include "common/goos/Object.h"
+#include "common/goos/TextDB.h"
 #include "common/util/Assert.h"
 
 namespace goos {
@@ -78,7 +77,6 @@ class Reader {
   std::optional<Object> read_from_stdin(const std::string& prompt, ReplWrapper& repl);
   Object read_from_file(const std::vector<std::string>& file_path, bool check_encoding = false);
   bool check_string_is_valid(const std::string& str) const;
-  std::string get_source_dir();
 
   SymbolTable symbolTable;
   TextDb db;

common/goos/ReplUtils.cpp

@@ -1,10 +1,11 @@
 #include "ReplUtils.h"
 
 #include "common/util/FileUtil.h"
-#include "third-party/replxx/include/replxx.hxx"
 #include "common/versions.h"
+
 #include "third-party/fmt/color.h"
 #include "third-party/fmt/core.h"
+#include "third-party/replxx/include/replxx.hxx"
 
 // TODO - expand a list of hints (ie. a hint for defun to show at a glance how to write a function,
 // or perhaps, show the docstring for the current function being used?)
@@ -16,6 +17,7 @@ void ReplWrapper::clear_screen() {
 }
 
 void ReplWrapper::print_welcome_message() {
+  // TODO - dont print on std-out
   // Welcome message / brief intro for documentation
   std::string ascii;
   ascii += " _____             _____ _____ _____ __    \n";
@@ -35,6 +37,10 @@ void ReplWrapper::print_welcome_message() {
   fmt::print(" to connect to the local target.\n\n");
 }
 
+void ReplWrapper::print_to_repl(const std::string_view& str) {
+  repl.print(str.data());
+}
+
 void ReplWrapper::set_history_max_size(size_t len) {
   repl.set_max_history_size(len);
 }

common/goos/ReplUtils.h

@@ -18,6 +18,7 @@ class ReplWrapper {
 
   // Functionality / Commands
   void clear_screen();
+  void print_to_repl(const std::string_view& str);
   void print_welcome_message();
   void set_history_max_size(size_t len);
   const char* readline(const std::string& prompt);

common/goos/TextDB.cpp

@@ -11,11 +11,12 @@
  *   (+ 1 (+ a b)) ; compute the sum
  */
 
-#include "common/util/FileUtil.h"
-#include "third-party/fmt/core.h"
-
 #include "TextDB.h"
 
+#include "common/util/FileUtil.h"
+
+#include "third-party/fmt/core.h"
+
 namespace goos {
 
 /*!

common/goos/TextDB.h

@@ -13,12 +13,12 @@
  *   (+ 1 (+ a b)) ; compute the sum
  */
 
-#include <string>
-#include <vector>
-#include <stdexcept>
-#include <unordered_map>
 #include <memory>
 #include <optional>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <vector>
 
 #include "common/goos/Object.h"
 

common/log/log.cpp

@@ -1,12 +1,17 @@
+#include "log.h"
+
 #include <cstdio>
 #include <cstdlib>
 #include <mutex>
+
 #include "third-party/fmt/color.h"
-#include "log.h"
 #ifdef _WIN32  // see lg::initialize
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #endif
 #include "common/util/Assert.h"
+#include "common/util/FileUtil.h"
 
 namespace lg {
 struct Logger {
@@ -77,6 +82,7 @@ void log_message(level log_level, LogTime& now, const char* message) {
 
 void set_file(const std::string& filename) {
   ASSERT(!gLogger.fp);
+  file_util::create_dir_if_needed_for_file(filename);
   gLogger.fp = fopen(filename.c_str(), "w");
   ASSERT(gLogger.fp);
 }

common/log/log.h

@@ -6,6 +6,7 @@
 #include <sys/time.h>
 #endif
 #include <string>
+
 #include "third-party/fmt/core.h"
 
 namespace lg {

common/math/Vector.h

@@ -89,6 +89,14 @@ class Vector {
     return result;
   }
 
+  Vector<T, Size> operator+(const T& other) const {
+    Vector<T, Size> result;
+    for (int i = 0; i < Size; i++) {
+      result[i] = m_data[i] + other;
+    }
+    return result;
+  }
+
   Vector<T, Size>& operator+=(const Vector<T, Size>& other) {
     for (int i = 0; i < Size; i++) {
       m_data[i] += other[i];
@@ -96,6 +104,20 @@ class Vector {
     return *this;
   }
 
+  Vector<T, Size>& operator-=(const Vector<T, Size>& other) {
+    for (int i = 0; i < Size; i++) {
+      m_data[i] -= other[i];
+    }
+    return *this;
+  }
+
+  Vector<T, Size>& operator-=(const T& other) {
+    for (int i = 0; i < Size; i++) {
+      m_data[i] -= other;
+    }
+    return *this;
+  }
+
   Vector<T, Size> elementwise_multiply(const Vector<T, Size>& other) const {
     Vector<T, Size> result;
     for (int i = 0; i < Size; i++) {
@@ -173,6 +195,18 @@ class Vector {
 
   void normalize(const T& norm = T(1)) { *this = normalized(norm); }
 
+  void max_in_place(const Vector<T, Size>& other) {
+    for (int i = 0; i < Size; i++) {
+      m_data[i] = std::max(m_data[i], other[i]);
+    }
+  }
+
+  void min_in_place(const Vector<T, Size>& other) {
+    for (int i = 0; i < Size; i++) {
+      m_data[i] = std::min(m_data[i], other[i]);
+    }
+  }
+
   std::string to_string_aligned() const {
     std::string result = "[";
     for (auto x : m_data) {
@@ -222,6 +256,8 @@ class Vector {
     }
   }
 
+  void set_zero() { fill(0); }
+
  private:
   T m_data[Size];
 };
@@ -239,8 +275,24 @@ struct Matrix {
     return result;
   }
 
-  //  const T& operator()(int r, int c) const { return m_data[c + r * Cols]; }
-  //  T& operator()(int r, int c) { return m_data[r + c * Rows]; }
+  static Matrix identity() {
+    Matrix result;
+    for (int c = 0; c < Cols; c++) {
+      for (int r = 0; r < Rows; r++) {
+        result(r, c) = r == c ? T(1) : T(0);
+      }
+    }
+    return result;
+  }
+
+  void set_zero() {
+    for (auto& x : m_data) {
+      x = 0;
+    }
+  }
+
+  T& operator()(int r, int c) { return m_data[r + c * Rows]; }
+  const T& operator()(int r, int c) const { return m_data[r + c * Rows]; }
 
   Vector<T, Rows> col(int c) const {
     Vector<T, Rows> result;
@@ -267,6 +319,31 @@ struct Matrix {
     return result;
   }
 
+  template <int OtherCols>
+  Matrix<T, Rows, OtherCols> operator*(const Matrix<T, Cols, OtherCols>& y) const {
+    Matrix<T, Rows, OtherCols> result;
+    result.set_zero();
+    for (int rx = 0; rx < Rows; rx++) {
+      for (int cx = 0; cx < Cols; cx++) {
+        for (int yi = 0; yi < OtherCols; yi++) {
+          result(rx, yi) += operator()(rx, cx) * y(cx, yi);
+        }
+      }
+    }
+    return result;
+  }
+
+  Vector<T, Rows> operator*(const Vector<T, Cols>& y) const {
+    Vector<T, Rows> result;
+    result.set_zero();
+    for (int rx = 0; rx < Rows; rx++) {
+      for (int cx = 0; cx < Cols; cx++) {
+        result[rx] += operator()(rx, cx) * y[cx];
+      }
+    }
+    return result;
+  }
+
  private:
   T m_data[Rows * Cols];
 };

common/math/geometry.cpp

@@ -0,0 +1,40 @@
+#include "geometry.h"
+
+namespace math {
+
+Vector4f bsphere_of_triangle(const Vector3f* v) {
+  Vector4f bsphere;
+  auto& p1 = v[0];
+  auto& p2 = v[1];
+  auto& p3 = v[2];
+  float A = (p1 - p2).length();
+  float B = (p2 - p3).length();
+  float C = (p3 - p1).length();
+
+  const Vector3f *a = &p3, *b = &p1, *c = &p2;
+  if (B < C)
+    std::swap(B, C), std::swap(b, c);
+  if (A < B)
+    std::swap(A, B), std::swap(a, b);
+
+  float r;
+  math::Vector3f origin;
+  if ((B * B) + (C * C) <= (A * A)) {
+    r = A / 2.f;
+    origin = (*b + *c) / 2.f;
+  } else {
+    float cos_a = (B * B + C * C - A * A) / (B * C * 2);
+    r = A / (sqrt(1 - cos_a * cos_a) * 2.f);
+    Vector3f alpha = *a - *c, beta = *b - *c;
+    origin = (beta * alpha.dot(alpha) - alpha * beta.dot(beta)).cross(alpha.cross(beta)) /
+                 (alpha.cross(beta).dot(alpha.cross(beta)) * 2.f) +
+             *c;
+  }
+  bsphere.x() = origin.x();
+  bsphere.y() = origin.y();
+  bsphere.z() = origin.z();
+  bsphere.w() = r;
+  return bsphere;
+}
+
+}  // namespace math

common/math/geometry.h

@@ -44,4 +44,10 @@ RaySphereResult<T> ray_sphere_intersect(const Vector3<T>& ray_origin,
   result.u[1] = minus_b - sqrt_val;
   return result;
 }
+
+math::Vector4f bsphere_of_triangle(const Vector3f* vertices);
+
+inline bool point_in_bsphere(const Vector4f& sphere, const Vector3f& pt) {
+  return (sphere.xyz() - pt).squared_length() <= (sphere.w() * sphere.w());
+}
 }  // namespace math

common/nrepl/ReplClient.cpp

@@ -0,0 +1,36 @@
+// clang-format off
+#include "ReplClient.h"
+
+#include "common/cross_sockets/XSocket.h"
+#include "common/versions.h"
+
+#include "third-party/fmt/core.h"
+
+#ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#include <WinSock2.h>
+#include <WS2tcpip.h>
+#endif
+// clang-format on
+
+void ReplClient::eval(std::string form) {
+  if (!is_connected()) {
+    return;
+  }
+  // TODO - split this up into two writes
+  u32 dataLength = form.length();
+  ReplServerHeader header = {dataLength, ReplServerMessageType::EVAL};
+
+  auto const ptr = reinterpret_cast<char*>(&header);
+  std::vector<char> buffer(ptr, ptr + sizeof header);
+
+  buffer.insert(buffer.end(), form.begin(), form.end());
+
+  int result = write_to_socket(client_socket, buffer.data(), buffer.size());
+  if (result == -1) {
+    // TODO - log
+    disconnect();
+  }
+}

common/nrepl/ReplClient.h

@@ -0,0 +1,16 @@
+#pragma once
+
+#include "ReplServer.h"
+
+#include "common/cross_sockets/XSocketClient.h"
+
+class ReplClient : public XSocketClient {
+ public:
+  using XSocketClient::XSocketClient;
+  virtual ~ReplClient() = default;
+
+  ReplClient& operator=(const ReplClient&) { return *this; }
+
+  // TODO - just void for now :(
+  void eval(std::string form);
+};

common/nrepl/ReplServer.cpp

@@ -0,0 +1,151 @@
+// clang-format off
+#include "ReplServer.h"
+
+#include "common/cross_sockets/XSocket.h"
+#include <common/versions.h>
+
+#include "third-party/fmt/core.h"
+
+#ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#include <WinSock2.h>
+#include <WS2tcpip.h>
+#endif
+// clang-format on
+
+// TODO - basically REPL to listen and inject commands into a running REPL
+// - we will need a C++ side client as well which will let us communicate with the repl via for
+// example, ImgUI
+//
+// TODO - The server also needs to eventually return the result of the evaluation
+
+ReplServer::~ReplServer() {
+  // Close all our client sockets!
+  for (const int& sock : client_sockets) {
+    close_socket(sock);
+  }
+}
+
+void ReplServer::post_init() {
+  // Add the listening socket to our set of sockets
+  fmt::print("[nREPL:{}:{}] awaiting connections\n", tcp_port, listening_socket);
+}
+
+void ReplServer::ping_response(int socket) {
+  std::string ping = fmt::format("Connected to OpenGOAL v{}.{} nREPL!",
+                                 versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
+  auto resp = write_to_socket(socket, ping.c_str(), ping.size());
+  if (resp == -1) {
+    fmt::print("[nREPL:{}] Client Disconnected: {}\n", tcp_port, inet_ntoa(addr.sin_addr),
+               ntohs(addr.sin_port), socket);
+    close_socket(socket);
+    client_sockets.erase(socket);
+  }
+}
+
+std::optional<std::string> ReplServer::get_msg() {
+  // Clear the sockets we are listening on
+  FD_ZERO(&read_sockets);
+
+  // Add the server's main listening socket (where we accept clients from)
+  FD_SET(listening_socket, &read_sockets);
+
+  int max_sd = listening_socket;
+  for (const int& sock : client_sockets) {
+    if (sock > max_sd) {
+      max_sd = sock;
+    }
+    if (sock > 0) {
+      FD_SET(sock, &read_sockets);
+    }
+  }
+
+  // Wait for activity on _something_, with a timeout so we don't get stuck here on exit.
+  struct timeval timeout;
+  timeout.tv_sec = 0;
+  timeout.tv_usec = 100000;
+  auto activity = select(max_sd + 1, &read_sockets, NULL, NULL, &timeout);
+
+  if (activity < 0) {  // TODO - || error!
+    return std::nullopt;
+  }
+
+  // If something happened on the master socket - it's a new connection
+  if (FD_ISSET(listening_socket, &read_sockets)) {
+    socklen_t addr_len = sizeof(addr);
+    auto new_socket = accept_socket(listening_socket, (sockaddr*)&addr, &addr_len);
+    if (new_socket < 0) {
+      // TODO - handle error
+    } else {
+      fmt::print("[nREPL:{}]: New socket connection: {}:{}:{}\n", tcp_port,
+                 inet_ntoa(addr.sin_addr), ntohs(addr.sin_port), new_socket);
+
+      // Say hello
+      ping_response(new_socket);
+      // Track the new socket
+      if ((int)client_sockets.size() < max_clients) {
+        client_sockets.insert(new_socket);
+      } else {
+        // TODO - Respond with NO
+      }
+    }
+  }
+
+  // otherwise (and no matter what) check all the clients to see if they have sent us anything
+  // else its some IO operation on some other socket
+  //
+  // RACE - the first client wins
+
+  // TODO - there are ways to do this with iterators but, couldn't figure it out!
+  std::vector<int> sockets_to_scan(client_sockets.begin(), client_sockets.end());
+  for (const int& sock : sockets_to_scan) {
+    if (FD_ISSET(sock, &read_sockets)) {
+      // Attempt to read a header
+      // TODO - should this be in a loop?
+      auto req_bytes = read_from_socket(sock, header_buffer.data(), header_buffer.size());
+      if (req_bytes == 0) {
+        // Socket disconnected
+        // TODO - add a queue of messages in the ReplWrapper so we can print _BEFORE_ the prompt is
+        // output
+        fmt::print("[nREPL:{}] Client Disconnected: {}\n", tcp_port, inet_ntoa(addr.sin_addr),
+                   ntohs(addr.sin_port), sock);
+
+        // Cleanup the socket and remove it from our set
+        close_socket(sock);
+        client_sockets.erase(sock);
+      } else {
+        // Otherwise, process the message
+        auto* header = (ReplServerHeader*)(header_buffer.data());
+        // get the body of the message
+        int expected_size = header->length;
+        int got = 0;
+        while (got < expected_size) {
+          if (got + expected_size > (int)buffer.size()) {
+            fmt::print(stderr,
+                       "[nREPL:{}]: Bad message, aborting the read.  Got :{}, Expected: {}, Buffer "
+                       "Size: {}",
+                       tcp_port, got, expected_size, buffer.size());
+            return std::nullopt;
+          }
+          auto x = read_from_socket(sock, buffer.data() + got, expected_size - got);
+          if (want_exit_callback()) {
+            return std::nullopt;
+          }
+          got += x > 0 ? x : 0;
+        }
+
+        switch (header->type) {
+          case ReplServerMessageType::PING:
+            ping_response(sock);
+            return std::nullopt;
+          case ReplServerMessageType::EVAL:
+            std::string msg(buffer.data(), header->length);
+            return std::make_optional(msg);
+        }
+      }
+    }
+  }
+  return std::nullopt;
+}

common/nrepl/ReplServer.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <optional>
+#include <set>
+
+#include "common/cross_sockets/XSocketServer.h"
+
+enum ReplServerMessageType { PING = 0, EVAL = 10, SHUTDOWN = 20 };
+
+struct ReplServerHeader {
+  u32 length;
+  u32 type;
+};
+
+class ReplServer : public XSocketServer {
+ public:
+  using XSocketServer::XSocketServer;
+  virtual ~ReplServer();
+
+  void post_init() override;
+
+  std::optional<std::string> get_msg();
+
+ private:
+  int max_clients = 50;
+  std::vector<char> header_buffer = std::vector<char>((int)sizeof(ReplServerHeader));
+  fd_set read_sockets;
+  std::set<int> client_sockets = {};
+
+  void ping_response(int socket);
+};

common/serialization/subtitles/subtitles.cpp

@@ -0,0 +1,425 @@
+#include "subtitles.h"
+
+#include "common/goos/ParseHelpers.h"
+#include "common/goos/Reader.h"
+#include "common/util/FileUtil.h"
+#include "common/util/json_util.h"
+
+#include "third-party/fmt/core.h"
+
+static const std::unordered_map<std::string, GameTextVersion> s_text_ver_enum_map = {
+    {"jak1-v1", GameTextVersion::JAK1_V1},
+    {"jak1-v2", GameTextVersion::JAK1_V2}};
+
+// TODO - why not just return the inputs instead of passing in an empty one?
+void open_text_project(const std::string& kind,
+                       const std::string& filename,
+                       std::unordered_map<GameTextVersion, std::vector<std::string>>& inputs) {
+  goos::Reader reader;
+  auto& proj = reader.read_from_file({filename}).as_pair()->cdr.as_pair()->car;
+  if (!proj.is_pair() || !proj.as_pair()->car.is_symbol() ||
+      proj.as_pair()->car.as_symbol()->name != kind) {
+    throw std::runtime_error(fmt::format("invalid {} project", kind));
+  }
+
+  goos::for_each_in_list(proj.as_pair()->cdr, [&](const goos::Object& o) {
+    if (!o.is_pair()) {
+      throw std::runtime_error(fmt::format("invalid entry in {} project", kind));
+    }
+
+    auto& ver = o.as_pair()->car.as_symbol()->name;
+    auto& in = o.as_pair()->cdr.as_pair()->car.as_string()->data;
+
+    if (s_text_ver_enum_map.count(ver) == 0) {
+      throw std::runtime_error(fmt::format("unknown text version {}", ver));
+    }
+
+    inputs[s_text_ver_enum_map.at(ver)].push_back(in);
+  });
+}
+
+int64_t get_int(const goos::Object& obj) {
+  if (obj.is_int()) {
+    return obj.integer_obj.value;
+  }
+  throw std::runtime_error(obj.print() + " was supposed to be an integer, but isn't");
+}
+
+const goos::Object& car(const goos::Object& x) {
+  if (!x.is_pair()) {
+    throw std::runtime_error("invalid pair");
+  }
+
+  return x.as_pair()->car;
+}
+
+const goos::Object& cdr(const goos::Object& x) {
+  if (!x.is_pair()) {
+    throw std::runtime_error("invalid pair");
+  }
+
+  return x.as_pair()->cdr;
+}
+
+std::string get_string(const goos::Object& x) {
+  if (x.is_string()) {
+    return x.as_string()->data;
+  }
+  throw std::runtime_error(x.print() + " was supposed to be a string, but isn't");
+}
+
+/*!
+ * Parse a game text file.
+ * Information is added to the game text database.
+ *
+ * The file should begin with (language-id x y z...) with the given language IDs.
+ * Each entry should be (id "line for 1st language" "line for 2nd language" ...)
+ * This adds the text line to each of the specified languages.
+ */
+void parse_text(const goos::Object& data, GameTextVersion text_ver, GameTextDB& db) {
+  auto font = get_font_bank(text_ver);
+  std::vector<std::shared_ptr<GameTextBank>> banks;
+  std::string possible_group_name;
+
+  for_each_in_list(data.as_pair()->cdr, [&](const goos::Object& obj) {
+    if (obj.is_pair()) {
+      auto& head = car(obj);
+      if (head.is_symbol("language-id")) {
+        if (banks.size() != 0) {
+          throw std::runtime_error("Languages have been set multiple times.");
+        }
+
+        if (cdr(obj).is_empty_list()) {
+          throw std::runtime_error("At least one language must be set.");
+        }
+
+        if (possible_group_name.empty()) {
+          throw std::runtime_error("Text group must be set before languages.");
+        }
+
+        for_each_in_list(cdr(obj), [&](const goos::Object& obj) {
+          auto lang = get_int(obj);
+          if (!db.bank_exists(possible_group_name, lang)) {
+            // database has no lang in this group yet
+            banks.push_back(db.add_bank(possible_group_name, std::make_shared<GameTextBank>(lang)));
+          } else {
+            banks.push_back(db.bank_by_id(possible_group_name, lang));
+          }
+        });
+      } else if (head.is_symbol("group-name")) {
+        if (!possible_group_name.empty()) {
+          throw std::runtime_error("group-name has been set multiple times.");
+        }
+
+        possible_group_name = get_string(car(cdr(obj)));
+
+        if (possible_group_name.empty()) {
+          throw std::runtime_error("invalid group-name.");
+        }
+
+        if (!cdr(cdr(obj)).is_empty_list()) {
+          throw std::runtime_error("group-name has too many arguments");
+        }
+      } else if (head.is_symbol("credits")) {
+        // parse a "credits" object. it's a list of lines where the ID automatically increments, and
+        // empty lines are skipped
+        if (banks.size() == 0) {
+          throw std::runtime_error("At least one language must be set before defining entries.");
+        }
+
+        if (cdr(obj).is_empty_list() || cdr(cdr(obj)).is_empty_list() ||
+            !car(cdr(obj)).is_symbol(":begin") || !car(cdr(cdr(obj))).is_int()) {
+          throw std::runtime_error("Invalid credits begin param");
+        }
+
+        const auto& it = cdr(cdr(obj));
+        int begin_id = car(it).as_int();
+        int id = begin_id - 1;
+        for_each_in_list(cdr(it), [&](const goos::Object& entry) {
+          ++id;
+          if (entry.is_string()) {
+            if (entry.as_string()->data.empty()) {
+              // empty string! just advance
+              return;
+            }
+
+            auto line = font->convert_utf8_to_game(entry.as_string()->data);
+            // add to all langs
+            for (auto& bank : banks) {
+              bank->set_line(id, line);
+            }
+          } else if (entry.is_pair()) {
+            int b_i = 0;
+            for_each_in_list(entry, [&](const goos::Object& entry) {
+              if (entry.is_string()) {
+                if (b_i >= int(banks.size())) {
+                  throw std::runtime_error(fmt::format("Too many strings in text id #x{:x}", id));
+                }
+
+                auto line = font->convert_utf8_to_game(entry.as_string()->data);
+                banks[b_i++]->set_line(id, line);
+              } else {
+                throw std::runtime_error(fmt::format("Non-string value in text id #x{:x}", id));
+              }
+            });
+            if (b_i != int(banks.size())) {
+              throw std::runtime_error(
+                  fmt::format("Not enough strings specified in text id #x{:x}", id));
+            }
+          } else {
+            throw std::runtime_error(fmt::format("Non-string value in text id #x{:x}", id));
+          }
+        });
+      }
+
+      else if (head.is_int()) {
+        if (banks.size() == 0) {
+          throw std::runtime_error("At least one language must be set before defining entries.");
+        }
+        int i = 0;
+        int id = head.as_int();
+        for_each_in_list(cdr(obj), [&](const goos::Object& entry) {
+          if (entry.is_string()) {
+            if (i >= int(banks.size())) {
+              throw std::runtime_error(fmt::format("Too many strings in text id #x{:x}", id));
+            }
+
+            auto line = font->convert_utf8_to_game(entry.as_string()->data);
+            banks[i++]->set_line(id, line);
+          } else {
+            throw std::runtime_error(fmt::format("Non-string value in text id #x{:x}", id));
+          }
+        });
+        if (i != int(banks.size())) {
+          throw std::runtime_error(
+              fmt::format("Not enough strings specified in text id #x{:x}", id));
+        }
+      } else {
+        throw std::runtime_error("Invalid game text file entry: " + head.print());
+      }
+    } else {
+      throw std::runtime_error("Invalid game text file");
+    }
+  });
+  if (banks.size() == 0) {
+    throw std::runtime_error("At least one language must be set.");
+  }
+}
+
+/*!
+ * Parse a game subtitle file.
+ * Information is added to the game subtitles database.
+ *
+ * The file should begin with (language-id x y z...) for the given language IDs.
+ * Each scene should be (scene-name <entry 1> <entry 2> ... )
+ * This adds the subtitle to each of the specified languages.
+ */
+void parse_subtitle(const goos::Object& data,
+                    GameTextVersion text_ver,
+                    GameSubtitleDB& db,
+                    const std::string& file_path) {
+  auto font = get_font_bank(text_ver);
+  std::map<int, std::shared_ptr<GameSubtitleBank>> banks;
+
+  for_each_in_list(data.as_pair()->cdr, [&](const goos::Object& obj) {
+    if (obj.is_pair()) {
+      auto& head = car(obj);
+      if (head.is_symbol() && head.as_symbol()->name == "language-id") {
+        if (banks.size() != 0) {
+          throw std::runtime_error("Languages have been set multiple times.");
+        }
+
+        if (cdr(obj).is_empty_list()) {
+          throw std::runtime_error("At least one language must be set.");
+        }
+
+        for_each_in_list(cdr(obj), [&](const goos::Object& obj) {
+          auto lang = get_int(obj);
+          if (!db.bank_exists(lang)) {
+            // database has no lang yet
+            banks[lang] = db.add_bank(std::make_shared<GameSubtitleBank>(lang));
+            banks[lang]->file_path = file_path;
+          } else {
+            banks[lang] = db.bank_by_id(lang);
+            banks[lang]->file_path = file_path;
+          }
+        });
+      }
+
+      else if (head.is_string() || head.is_int()) {
+        if (banks.size() == 0) {
+          throw std::runtime_error("At least one language must be set before defining scenes.");
+        }
+        auto kind = SubtitleSceneKind::Movie;
+        int id = 0;
+        auto entries = cdr(obj);
+        if (head.is_int()) {
+          kind = SubtitleSceneKind::Hint;
+        } else if (car(entries).is_symbol()) {
+          const auto& parm = car(entries).as_symbol()->name;
+          if (parm == ":hint") {
+            entries = cdr(entries);
+            id = car(entries).as_int();
+            kind = SubtitleSceneKind::HintNamed;
+          } else {
+            throw std::runtime_error("Unknown parameter for subtitle scene");
+          }
+          entries = cdr(entries);
+        }
+
+        GameSubtitleSceneInfo scene(kind);
+        if (kind == SubtitleSceneKind::Movie || kind == SubtitleSceneKind::HintNamed) {
+          scene.set_name(head.as_string()->data);
+        } else if (kind == SubtitleSceneKind::Hint) {
+          id = head.as_int();
+        }
+        scene.set_id(id);
+        scene.m_sorting_group = db.m_subtitle_groups->find_group(scene.name());
+        scene.m_sorting_group_idx = db.m_subtitle_groups->find_group_index(scene.m_sorting_group);
+
+        for_each_in_list(entries, [&](const goos::Object& entry) {
+          if (entry.is_pair()) {
+            // expected formats:
+            // (time <args>)
+            // all arguments have default values. the arguments are:
+            // "speaker" "line" - two strings. one for the speaker's name and one for the actual
+            //                    line. speaker can be empty. default is just empty string.
+            // :offscreen - speaker is offscreen. default is not offscreen.
+
+            if (!car(entry).is_int()) {
+              throw std::runtime_error("Each entry must start with a timestamp (number)");
+            }
+
+            auto time = car(entry).as_int();
+            goos::StringObject *speaker = nullptr, *line = nullptr;
+            bool offscreen = false;
+            if (scene.kind() == SubtitleSceneKind::Hint ||
+                scene.kind() == SubtitleSceneKind::HintNamed) {
+              offscreen = true;
+            }
+            for_each_in_list(cdr(entry), [&](const goos::Object& arg) {
+              if (arg.is_string()) {
+                if (!speaker) {
+                  speaker = arg.as_string();
+                } else if (!line) {
+                  line = arg.as_string();
+                } else {
+                  throw std::runtime_error("Invalid string in subtitle entry");
+                }
+              } else if (speaker && !line) {
+                throw std::runtime_error(
+                    "Invalid object in subtitle entry, expecting actual line string after speaker");
+              } else if (arg.is_symbol()) {
+                if (scene.kind() == SubtitleSceneKind::Movie &&
+                    arg.as_symbol()->name == ":offscreen") {
+                  offscreen = true;
+                } else {
+                  throw std::runtime_error(
+                      fmt::format("Unknown parameter {} in subtitle", arg.as_symbol()->name));
+                }
+              }
+            });
+            auto line_utf8 = line ? line->data : "";
+            auto line_str = font->convert_utf8_to_game(line_utf8);
+            auto speaker_utf8 = speaker ? speaker->data : "";
+            auto speaker_str = font->convert_utf8_to_game(speaker_utf8);
+            scene.add_line(time, line_str, line_utf8, speaker_str, speaker_utf8, offscreen);
+          } else {
+            throw std::runtime_error("Each entry must be a list");
+          }
+        });
+        for (auto& [lang, bank] : banks) {
+          if (!bank->scene_exists(scene.name())) {
+            bank->add_scene(scene);
+          } else {
+            auto& old_scene = bank->scene_by_name(scene.name());
+            old_scene.from_other_scene(scene);
+          }
+        }
+      } else {
+        throw std::runtime_error("Invalid game subtitles file entry: " + head.print());
+      }
+    } else {
+      throw std::runtime_error("Invalid game subtitles file");
+    }
+  });
+  if (banks.size() == 0) {
+    throw std::runtime_error("At least one language must be set.");
+  }
+}
+
+void GameSubtitleGroups::hydrate_from_asset_file() {
+  std::string file_path = (file_util::get_jak_project_dir() / "game" / "assets" / "jak1" /
+                           "subtitle" / "subtitle-groups.json")
+                              .string();
+  auto config_str = file_util::read_text_file(file_path);
+  auto group_data = parse_commented_json(config_str, file_path);
+
+  for (const auto& [key, val] : group_data.items()) {
+    try {
+      if (key == group_order_key) {
+        m_group_order = val.get<std::vector<std::string>>();
+      } else {
+        m_groups[key] = val.get<std::vector<std::string>>();
+      }
+    } catch (std::exception& ex) {
+      fmt::print("Bad subtitle group entry - {} - {}", key, ex.what());
+    }
+  }
+}
+
+std::string GameSubtitleGroups::find_group(const std::string& scene_name) {
+  for (auto const& [group, scenes] : m_groups) {
+    for (auto const& name : scenes) {
+      if (name == scene_name) {
+        return group;
+      }
+    }
+  }
+  // Add to the uncategorized group if it wasn't found
+  m_groups[uncategorized_group].push_back(scene_name);
+  return uncategorized_group;
+}
+
+int GameSubtitleGroups::find_group_index(const std::string& group_name) {
+  auto it = find(m_group_order.begin(), m_group_order.end(), group_name);
+  if (it != m_group_order.end()) {
+    return it - m_group_order.begin();
+  } else {
+    return m_group_order.size() - 1;
+  }
+}
+
+void GameSubtitleGroups::remove_scene(const std::string& group_name,
+                                      const std::string& scene_name) {
+  // TODO - validate group_name
+  m_groups[group_name].erase(
+      std::remove(m_groups[group_name].begin(), m_groups[group_name].end(), scene_name),
+      m_groups[group_name].end());
+}
+void GameSubtitleGroups::add_scene(const std::string& group_name, const std::string& scene_name) {
+  // TODO - validate group_name
+  // TODO - don't add duplicates
+  m_groups[group_name].push_back(scene_name);
+}
+
+GameSubtitleDB load_subtitle_project() {
+  // Load the subtitle files
+  GameSubtitleDB db;
+  db.m_subtitle_groups = std::make_unique<GameSubtitleGroups>();
+  db.m_subtitle_groups->hydrate_from_asset_file();
+  goos::Reader reader;
+  std::unordered_map<GameTextVersion, std::vector<std::string>> inputs;
+  std::string subtitle_project =
+      (file_util::get_jak_project_dir() / "game" / "assets" / "game_subtitle.gp").string();
+  open_text_project("subtitle", subtitle_project, inputs);
+  for (auto& [ver, in] : inputs) {
+    for (auto& filename : in) {
+      auto code = reader.read_from_file({filename});
+      parse_subtitle(code, ver, db, filename);
+    }
+  }
+  return db;
+}
+
+// TODO - write a deserializer, the compiler still can do the compiling!

common/serialization/subtitles/subtitles.h

@@ -0,0 +1,207 @@
+#pragma once
+
+#include <algorithm>
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_set>
+
+#include "common/goos/Object.h"
+#include "common/util/Assert.h"
+#include "common/util/FontUtils.h"
+
+/*!
+ * The text bank contains all lines (accessed with an ID) for a language.
+ */
+class GameTextBank {
+ public:
+  GameTextBank(int lang_id) : m_lang_id(lang_id) {}
+
+  int lang() const { return m_lang_id; }
+  const std::map<int, std::string>& lines() const { return m_lines; }
+
+  bool line_exists(int id) const { return m_lines.find(id) != m_lines.end(); }
+  std::string line(int id) { return m_lines.at(id); }
+  void set_line(int id, std::string line) { m_lines[id] = line; }
+
+ private:
+  int m_lang_id;
+  std::map<int, std::string> m_lines;
+};
+
+/*!
+ * The text database contains a text bank for each language for each text group.
+ * Each text bank contains a list of text lines. Very simple.
+ */
+class GameTextDB {
+ public:
+  const std::unordered_map<std::string, std::map<int, std::shared_ptr<GameTextBank>>>& groups()
+      const {
+    return m_banks;
+  }
+  const std::map<int, std::shared_ptr<GameTextBank>>& banks(std::string group) const {
+    return m_banks.at(group);
+  }
+
+  bool bank_exists(std::string group, int id) const {
+    if (m_banks.find(group) == m_banks.end())
+      return false;
+    return m_banks.at(group).find(id) != m_banks.at(group).end();
+  }
+
+  std::shared_ptr<GameTextBank> add_bank(std::string group, std::shared_ptr<GameTextBank> bank) {
+    ASSERT(!bank_exists(group, bank->lang()));
+    m_banks[group][bank->lang()] = bank;
+    return bank;
+  }
+  std::shared_ptr<GameTextBank> bank_by_id(std::string group, int id) {
+    if (!bank_exists(group, id)) {
+      return nullptr;
+    }
+    return m_banks.at(group).at(id);
+  }
+
+ private:
+  std::unordered_map<std::string, std::map<int, std::shared_ptr<GameTextBank>>> m_banks;
+};
+
+/*!
+ * The subtitle scene info (accessed through the scene name) contains all lines and their timestamps
+ * and other settings.
+ */
+enum class SubtitleSceneKind { Invalid = -1, Movie = 0, Hint = 1, HintNamed = 2 };
+class GameSubtitleSceneInfo {
+ public:
+  struct SubtitleLine {
+    SubtitleLine(int frame,
+                 std::string line,
+                 std::string line_utf8,
+                 std::string speaker,
+                 std::string speaker_utf8,
+                 bool offscreen)
+        : frame(frame),
+          line(line),
+          line_utf8(line_utf8),
+          speaker(speaker),
+          speaker_utf8(speaker_utf8),
+          offscreen(offscreen) {}
+
+    int frame;
+    std::string line;
+    std::string line_utf8;
+    std::string speaker;
+    std::string speaker_utf8;
+    bool offscreen;
+
+    bool operator<(const SubtitleLine& other) const { return (frame < other.frame); }
+  };
+
+  GameSubtitleSceneInfo() {}
+  GameSubtitleSceneInfo(SubtitleSceneKind kind) : m_kind(kind) {}
+
+  const std::string& name() const { return m_name; }
+  const std::vector<SubtitleLine>& lines() const { return m_lines; }
+  int id() const { return m_id; }
+  SubtitleSceneKind kind() const { return m_kind; }
+
+  void clear_lines() { m_lines.clear(); }
+  void set_name(const std::string& new_name) { m_name = new_name; }
+  void set_id(int new_id) { m_id = new_id; }
+  void from_other_scene(GameSubtitleSceneInfo& scene) {
+    m_name = scene.name();
+    m_lines = scene.lines();
+    m_kind = scene.kind();
+    m_id = scene.id();
+  }
+
+  void add_line(int frame,
+                std::string line,
+                std::string line_utf8,
+                std::string speaker,
+                std::string speaker_utf8,
+                bool offscreen) {
+    m_lines.emplace_back(SubtitleLine(frame, line, line_utf8, speaker, speaker_utf8, offscreen));
+    std::sort(m_lines.begin(), m_lines.end());
+  }
+
+  std::string m_name;
+  int m_id;
+  std::vector<SubtitleLine> m_lines;
+  SubtitleSceneKind m_kind;
+  std::string m_sorting_group;
+  int m_sorting_group_idx;
+};
+
+/*!
+ * The subtitle bank contains subtitles for all scenes in a language.
+ */
+class GameSubtitleBank {
+ public:
+  GameSubtitleBank(int lang_id) : m_lang_id(lang_id) {}
+
+  int lang() const { return m_lang_id; }
+  const std::map<std::string, GameSubtitleSceneInfo>& scenes() const { return m_scenes; }
+
+  bool scene_exists(const std::string& name) const { return m_scenes.find(name) != m_scenes.end(); }
+  GameSubtitleSceneInfo& scene_by_name(const std::string& name) { return m_scenes.at(name); }
+  void add_scene(GameSubtitleSceneInfo& scene) {
+    ASSERT(!scene_exists(scene.name()));
+    m_scenes[scene.name()] = scene;
+  }
+
+  int m_lang_id;
+  std::string file_path;
+
+  std::map<std::string, GameSubtitleSceneInfo> m_scenes;
+};
+
+class GameSubtitleGroups {
+ public:
+  std::vector<std::string> m_group_order;
+  std::map<std::string, std::vector<std::string>> m_groups;
+
+  void hydrate_from_asset_file();
+  std::string find_group(const std::string& scene_name);
+  int find_group_index(const std::string& group_name);
+  void remove_scene(const std::string& group_name, const std::string& scene_name);
+  void add_scene(const std::string& group_name, const std::string& scene_name);
+
+  std::string group_order_key = "_groups";
+  std::string uncategorized_group = "uncategorized";
+};
+
+/*!
+ * The subtitles database contains a subtitles bank for each language.
+ * Each subtitles bank contains a series of subtitle scene infos.
+ */
+class GameSubtitleDB {
+ public:
+  const std::map<int, std::shared_ptr<GameSubtitleBank>>& banks() const { return m_banks; }
+
+  bool bank_exists(int id) const { return m_banks.find(id) != m_banks.end(); }
+
+  std::shared_ptr<GameSubtitleBank> add_bank(std::shared_ptr<GameSubtitleBank> bank) {
+    ASSERT(!bank_exists(bank->lang()));
+    m_banks[bank->lang()] = bank;
+    return bank;
+  }
+  std::shared_ptr<GameSubtitleBank> bank_by_id(int id) {
+    if (!bank_exists(id)) {
+      return nullptr;
+    }
+    return m_banks.at(id);
+  }
+
+  std::map<int, std::shared_ptr<GameSubtitleBank>> m_banks;
+  std::unique_ptr<GameSubtitleGroups> m_subtitle_groups;
+};
+
+// TODO add docstrings
+
+void parse_text(const goos::Object& data, GameTextVersion text_ver, GameTextDB& db);
+void parse_subtitle(const goos::Object& data,
+                    GameTextVersion text_ver,
+                    GameSubtitleDB& db,
+                    const std::string& file_path);
+
+GameSubtitleDB load_subtitle_project();

common/symbols.h

@@ -1,10 +1,13 @@
 #pragma once
 
+#include "common/versions.h"
+
 /*!
  * @file symbols.h
  * The location of fixed symbols in the GOAL symbol table.
  */
 
+namespace jak1_symbols {
 constexpr int FIX_SYM_EMPTY_CAR = -0xc;
 constexpr int FIX_SYM_EMPTY_PAIR = -0xa;
 constexpr int FIX_SYM_EMPTY_CDR = -0x8;
@@ -76,3 +79,96 @@ constexpr int FIX_SYM_SOUND = 0x1b8;                // ??
 constexpr int FIX_SYM_DGO = 0x1c0;                  // ??
 constexpr int FIX_SYM_TOP_LEVEL = 0x1c8;            // ??
 constexpr int FIX_FIXED_SYM_END_OFFSET = 0x1d0;
+}  // namespace jak1_symbols
+
+namespace jak2_symbols {
+constexpr int FIX_SYM_EMPTY_CAR = -0x8;
+constexpr int FIX_SYM_EMPTY_PAIR = -0x6;
+constexpr int FIX_SYM_EMPTY_CDR = -0x4;
+constexpr int FIX_SYM_FALSE = 0x0;  // GOAL boolean #f (note that this is equal to the $s7 register)
+constexpr int FIX_SYM_TRUE = 0x4;   // GOAL boolean #t
+constexpr int FIX_SYM_FUNCTION_TYPE = 0x8;
+constexpr int FIX_SYM_BASIC = 0xc;
+constexpr int FIX_SYM_STRING_TYPE = 0x10;
+constexpr int FIX_SYM_SYMBOL_TYPE = 0x14;
+constexpr int FIX_SYM_TYPE_TYPE = 0x18;
+constexpr int FIX_SYM_OBJECT_TYPE = 0x1c;
+constexpr int FIX_SYM_LINK_BLOCK = 0x20;
+constexpr int FIX_SYM_INTEGER = 0x24;
+constexpr int FIX_SYM_SINTEGER = 0x28;
+constexpr int FIX_SYM_UINTEGER = 0x2c;
+constexpr int FIX_SYM_BINTEGER = 0x30;
+constexpr int FIX_SYM_INT8 = 0x34;
+constexpr int FIX_SYM_INT16 = 0x38;
+constexpr int FIX_SYM_INT32 = 0x3c;
+constexpr int FIX_SYM_INT64 = 0x40;
+constexpr int FIX_SYM_INT128 = 0x44;
+constexpr int FIX_SYM_UINT8 = 0x48;
+constexpr int FIX_SYM_UINT16 = 0x4c;
+constexpr int FIX_SYM_UINT32 = 0x50;
+constexpr int FIX_SYM_UINT64 = 0x54;
+constexpr int FIX_SYM_UINT128 = 0x58;
+constexpr int FIX_SYM_FLOAT = 0x5c;
+constexpr int FIX_SYM_PROCESS_TREE = 0x60;
+constexpr int FIX_SYM_PROCESS_TYPE = 0x64;
+constexpr int FIX_SYM_THREAD = 0x68;
+constexpr int FIX_SYM_STRUCTURE = 0x6c;
+constexpr int FIX_SYM_PAIR_TYPE = 0x70;
+constexpr int FIX_SYM_POINTER = 0x74;
+constexpr int FIX_SYM_NUMBER = 0x78;
+constexpr int FIX_SYM_ARRAY = 0x7c;
+constexpr int FIX_SYM_VU_FUNCTION = 0x80;
+constexpr int FIX_SYM_CONNECTABLE = 0x84;
+constexpr int FIX_SYM_STACK_FRAME = 0x88;
+constexpr int FIX_SYM_FILE_STREAM = 0x8c;
+
+constexpr int FIX_SYM_HEAP = 0x90;
+constexpr int FIX_SYM_NOTHING_FUNC = 0x94;
+constexpr int FIX_SYM_DELETE_BASIC = 0x98;
+constexpr int FIX_SYM_STATIC = 0x9c;
+constexpr int FIX_SYM_GLOBAL_HEAP = 0xa0;
+constexpr int FIX_SYM_DEBUG = 0xa4;
+constexpr int FIX_SYM_LOADING_LEVEL = 0xa8;
+constexpr int FIX_SYM_LOADING_PACKAGE = 0xac;
+constexpr int FIX_SYM_PROCESS_LEVEL_HEAP = 0xb0;
+constexpr int FIX_SYM_STACK = 0xb4;
+constexpr int FIX_SYM_SCRATCH = 0xb8;
+constexpr int FIX_SYM_SCRATCH_TOP = 0xbc;
+
+constexpr int FIX_SYM_ZERO_FUNC = 0xc0;
+constexpr int FIX_SYM_ASIZE_OF_BASIC_FUNC = 0xc4;
+constexpr int FIX_SYM_COPY_BASIC_FUNC = 0xc8;  // bugged name
+constexpr int FIX_SYM_LEVEL = 0xcc;
+constexpr int FIX_SYM_ART_GROUP = 0xd0;
+constexpr int FIX_SYM_TEXTURE_PAGE_DIR = 0xd4;
+constexpr int FIX_SYM_TEXTURE_PAGE = 0xd8;
+constexpr int FIX_SYM_SOUND = 0xdc;
+constexpr int FIX_SYM_DGO = 0xe0;
+constexpr int FIX_SYM_TOP_LEVEL = 0xe4;
+constexpr int FIX_SYM_QUOTE = 0xe8;
+constexpr int FIX_FIXED_SYM_END_OFFSET = 0xec;
+
+}  // namespace jak2_symbols
+
+constexpr int false_symbol_offset() {
+  return jak1_symbols::FIX_SYM_FALSE;
+}
+
+constexpr int true_symbol_offset(GameVersion version) {
+  switch (version) {
+    case GameVersion::Jak1:
+      return jak1_symbols::FIX_SYM_TRUE;
+    case GameVersion::Jak2:
+      return jak2_symbols::FIX_SYM_TRUE;
+  }
+}
+
+constexpr int empty_pair_offset_from_s7(GameVersion version) {
+  switch (version) {
+    case GameVersion::Jak1:
+      return jak1_symbols::FIX_SYM_EMPTY_PAIR;
+    case GameVersion::Jak2:
+      // minus 1 for the symbol table pointer's offset.
+      return jak2_symbols::FIX_SYM_EMPTY_PAIR - 1;
+  }
+}

common/texture/texture_conversion.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include "common/common_types.h"
+#include "common/util/Assert.h"
+
 /*!
  * Convert from a pixel location in a texture (x, y, texture buffer width) to VRAM address (byte).
  * Uses the PSMCT32 format.
@@ -218,4 +221,4 @@ inline u32 rgba16_to_rgba32(u32 in) {
 // texture format enums
 enum class PSM { PSMCT16 = 0x02, PSMT8 = 0x13, PSMT4 = 0x14 };
 // clut format enums
-enum class CPSM { PSMCT32 = 0x0, PSMCT16 = 0x02 };
+enum class CPSM { PSMCT32 = 0x0, PSMCT16 = 0x02 };

common/type_system/Type.cpp

@@ -3,11 +3,14 @@
  * Representation of a GOAL type in the type system.
  */
 
-#include <stdexcept>
-#include "third-party/fmt/core.h"
 #include "Type.h"
+
+#include <stdexcept>
+
 #include "common/util/Assert.h"
 
+#include "third-party/fmt/core.h"
+
 namespace {
 std::string reg_kind_to_string(RegClass kind) {
   switch (kind) {
@@ -936,12 +939,16 @@ std::string BasicType::diff_impl(const Type& other_) const {
 // Bitfield
 /////////////////
 
-BitField::BitField(TypeSpec type, std::string name, int offset, int size)
-    : m_type(std::move(type)), m_name(std::move(name)), m_offset(offset), m_size(size) {}
+BitField::BitField(TypeSpec type, std::string name, int offset, int size, bool skip_in_decomp)
+    : m_type(std::move(type)),
+      m_name(std::move(name)),
+      m_offset(offset),
+      m_size(size),
+      m_skip_in_static_decomp(skip_in_decomp) {}
 
 bool BitField::operator==(const BitField& other) const {
   return m_type == other.m_type && m_name == other.m_name && m_offset == other.m_offset &&
-         other.m_size == m_size;
+         m_size == other.m_size;
 }
 
 std::string BitField::diff(const BitField& other) const {
@@ -963,6 +970,11 @@ std::string BitField::diff(const BitField& other) const {
     result += fmt::format("size: {} vs. {}\n", m_size, other.m_size);
   }
 
+  if (m_skip_in_static_decomp != other.m_skip_in_static_decomp) {
+    result += fmt::format("skip_in_static_decomp: {} vs. {}\n", m_skip_in_static_decomp,
+                          other.m_skip_in_static_decomp);
+  }
+
   return result;
 }
 

common/type_system/Type.h

@@ -5,11 +5,13 @@
  * Representation of a GOAL type in the type system.
  */
 
-#include <string>
 #include <map>
+#include <string>
 #include <unordered_map>
-#include "common/goal_constants.h"
+
 #include "TypeSpec.h"
+
+#include "common/goal_constants.h"
 #include "common/util/Assert.h"
 
 class TypeSystem;
@@ -282,6 +284,7 @@ class StructureType : public ReferenceType {
   bool is_always_stack_singleton() const { return m_always_stack_singleton; }
   void set_allow_misalign(bool misalign) { m_allow_misalign = misalign; }
   void set_gen_inspect(bool gen_inspect) { m_generate_inspect = gen_inspect; }
+  int size() const { return m_size_in_mem; }
 
  protected:
   friend class TypeSystem;
@@ -325,11 +328,12 @@ class BasicType : public StructureType {
 class BitField {
  public:
   BitField() = default;
-  BitField(TypeSpec type, std::string name, int offset, int size);
+  BitField(TypeSpec type, std::string name, int offset, int size, bool skip_in_decomp);
   const std::string name() const { return m_name; }
   int offset() const { return m_offset; }
   int size() const { return m_size; }
   const TypeSpec& type() const { return m_type; }
+  bool skip_in_decomp() const { return m_skip_in_static_decomp; }
   bool operator==(const BitField& other) const;
   bool operator!=(const BitField& other) const { return !((*this) == other); }
   std::string diff(const BitField& other) const;
@@ -340,6 +344,7 @@ class BitField {
   std::string m_name;
   int m_offset = -1;  // in bits
   int m_size = -1;    // in bits.
+  bool m_skip_in_static_decomp = false;
 };
 
 class BitFieldType : public ValueType {

common/type_system/TypeFieldLookup.cpp

@@ -4,9 +4,11 @@
  */
 
 #include <algorithm>
-#include "third-party/fmt/core.h"
+
 #include "TypeSystem.h"
 
+#include "third-party/fmt/core.h"
+
 namespace {
 // debug prints for the reverse lookup
 bool debug_reverse_lookup = false;

common/type_system/TypeSpec.cpp

@@ -3,8 +3,10 @@
  * A GOAL TypeSpec is a reference to a type or compound type.
  */
 
-#include <stdexcept>
 #include "TypeSpec.h"
+
+#include <stdexcept>
+
 #include "third-party/fmt/core.h"
 
 bool TypeTag::operator==(const TypeTag& other) const {

common/type_system/TypeSpec.h

@@ -5,11 +5,12 @@
  * A GOAL TypeSpec is a reference to a type or compound type.
  */
 
-#include <vector>
-#include <string>
 #include <optional>
-#include "common/util/SmallVector.h"
+#include <string>
+#include <vector>
+
 #include "common/util/Assert.h"
+#include "common/util/SmallVector.h"
 
 /*!
  * A :name value modifier to apply to a type.

common/type_system/TypeSystem.cpp

@@ -5,12 +5,15 @@
  * access types, and reverse type lookups.
  */
 
-#include "third-party/fmt/core.h"
-#include "third-party/fmt/color.h"
-#include <stdexcept>
 #include "TypeSystem.h"
-#include "common/util/math_util.h"
+
+#include <stdexcept>
+
 #include "common/util/Assert.h"
+#include "common/util/math_util.h"
+
+#include "third-party/fmt/color.h"
+#include "third-party/fmt/core.h"
 
 namespace {
 template <typename... Args>
@@ -211,6 +214,34 @@ void TypeSystem::forward_declare_type_method_count(const std::string& name, int
   m_forward_declared_method_counts[name] = num_methods;
 }
 
+/*!
+ * forward declare, but allow the method count to be too large by up to 3 because jak2 stores
+ * method counts in v2/v4's like n*4 + 3.
+ */
+void TypeSystem::forward_declare_type_method_count_multiple_of_4(const std::string& name,
+                                                                 int num_methods) {
+  auto existing_fwd = m_forward_declared_method_counts.find(name);
+  if (existing_fwd != m_forward_declared_method_counts.end() &&
+      existing_fwd->second + 3 < num_methods) {
+    throw_typesystem_error(
+        "Type {} was originally forward declared with {} methods and is now being forward declared "
+        "with {} methods",
+        name, existing_fwd->second, num_methods);
+  }
+
+  auto existing_type = m_types.find(name);
+  if (existing_type != m_types.end()) {
+    int existing_count = get_next_method_id(existing_type->second.get());
+    if (existing_count + 3 < num_methods) {
+      throw_typesystem_error(
+          "Type {} was defined with {} methods and is now being forward declared with {} methods",
+          name, existing_count, num_methods);
+    }
+  }
+
+  m_forward_declared_method_counts[name] = num_methods;
+}
+
 int TypeSystem::get_type_method_count(const std::string& name) const {
   auto result = try_get_type_method_count(name);
   if (result) {
@@ -1588,7 +1619,8 @@ void TypeSystem::add_field_to_bitfield(BitFieldType* type,
                                        const std::string& field_name,
                                        const TypeSpec& field_type,
                                        int offset,
-                                       int field_size) {
+                                       int field_size,
+                                       bool skip_in_decomp) {
   // in bits
   auto load_size = lookup_type(field_type)->get_load_size() * 8;
   if (field_size == -1) {
@@ -1616,7 +1648,7 @@ void TypeSystem::add_field_to_bitfield(BitFieldType* type,
         type->get_name(), field_name, offset, offset + field_size);
   }
 
-  BitField field(field_type, field_name, offset, field_size);
+  BitField field(field_type, field_name, offset, field_size, skip_in_decomp);
   type->m_fields.push_back(field);
 }
 

common/type_system/TypeSystem.h

@@ -7,16 +7,16 @@
  * access types, and reverse type lookups.
  */
 
+#include <memory>
+#include <optional>
+#include <stdexcept>
+#include <string>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
-#include <string>
-#include <memory>
-#include <stdexcept>
-#include <optional>
 
-#include "TypeSpec.h"
 #include "Type.h"
+#include "TypeSpec.h"
 
 struct TypeFlags {
   union {
@@ -128,6 +128,8 @@ class TypeSystem {
   void forward_declare_type_as_type(const std::string& name);
   void forward_declare_type_as(const std::string& new_type, const std::string& parent_type);
   void forward_declare_type_method_count(const std::string& name, int num_methods);
+  void forward_declare_type_method_count_multiple_of_4(const std::string& name, int num_methods);
+
   int get_type_method_count(const std::string& name) const;
   std::optional<int> try_get_type_method_count(const std::string& name) const;
   std::string get_runtime_type(const TypeSpec& ts);
@@ -222,7 +224,8 @@ class TypeSystem {
                              const std::string& field_name,
                              const TypeSpec& field_type,
                              int offset,
-                             int field_size);
+                             int field_size,
+                             bool skip_in_decomp);
 
   bool should_use_virtual_methods(const Type* type, int method_id) const;
   bool should_use_virtual_methods(const TypeSpec& type, int method_id) const;

common/type_system/defenum.cpp

@@ -4,12 +4,15 @@
  * This is used both in the compiler and in the decompiler for the type definition file.
  */
 
-#include "common/goos/ParseHelpers.h"
 #include "defenum.h"
+
 #include "deftype.h"
-#include "third-party/fmt/core.h"
+
+#include "common/goos/ParseHelpers.h"
 #include "common/util/BitUtils.h"
 
+#include "third-party/fmt/core.h"
+
 namespace {
 const goos::Object& car(const goos::Object* x) {
   if (!x->is_pair()) {
@@ -60,7 +63,7 @@ EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts) {
   while (current.is_symbol() && symbol_string(current).at(0) == ':') {
     auto option_name = symbol_string(current);
     iter = cdr(iter);
-    auto option_value = car(iter);
+    auto& option_value = car(iter);
     iter = cdr(iter);
 
     if (option_name == ":type") {
@@ -82,7 +85,7 @@ EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts) {
       }
       for (auto& e : other_info->entries()) {
         if (entries.find(e.first) != entries.end()) {
-          throw std::runtime_error(fmt::format("Entry {} appears multiple times.", e.first));
+          throw std::runtime_error(fmt::format("Entry {} appears multiple times", e.first));
         }
         entries[e.first] = e.second;
       }

common/type_system/defenum.h

@@ -7,6 +7,7 @@
  */
 
 #include "TypeSystem.h"
+
 #include "common/goos/Object.h"
 
 EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts);

common/type_system/deftype.cpp

@@ -4,8 +4,10 @@
  * This is used both in the compiler and in the decompiler for the type definition file.
  */
 
-#include "common/goos/ParseHelpers.h"
 #include "deftype.h"
+
+#include "common/goos/ParseHelpers.h"
+
 #include "third-party/fmt/core.h"
 
 /*!
@@ -163,6 +165,7 @@ void add_bitfield(BitFieldType* bitfield_type, TypeSystem* ts, const goos::Objec
 
   int offset_override = -1;
   int size_override = -1;
+  bool skip_in_decomp = false;
 
   if (!rest->is_empty_list()) {
     while (!rest->is_empty_list()) {
@@ -175,6 +178,8 @@ void add_bitfield(BitFieldType* bitfield_type, TypeSystem* ts, const goos::Objec
       } else if (opt_name == ":size") {
         size_override = get_int(car(rest));
         rest = cdr(rest);
+      } else if (opt_name == ":do-not-decompile") {
+        skip_in_decomp = true;
       } else {
         throw std::runtime_error("Invalid option in field specification: " + opt_name);
       }
@@ -186,7 +191,8 @@ void add_bitfield(BitFieldType* bitfield_type, TypeSystem* ts, const goos::Objec
   }
 
   // it's fine if the size is -1, that means it'll just use the type's size.
-  ts->add_field_to_bitfield(bitfield_type, name, type, offset_override, size_override);
+  ts->add_field_to_bitfield(bitfield_type, name, type, offset_override, size_override,
+                            skip_in_decomp);
 }
 
 void declare_method(Type* type, TypeSystem* type_system, const goos::Object& def) {
@@ -312,6 +318,7 @@ StructureDefResult parse_structure_def(
   int method_count_assert = -1;
   uint64_t flag_assert = 0;
   bool flag_assert_set = false;
+  bool set_heapbase = false;
   while (!rest->is_empty_list()) {
     if (car(rest).is_pair()) {
       auto opt_list = &car(rest);
@@ -362,6 +369,7 @@ StructureDefResult parse_structure_def(
         }
         rest = cdr(rest);
         flags.heap_base = hb;
+        set_heapbase = true;
       } else if (opt_name == ":allow-misaligned") {
         result.allow_misaligned = true;
       } else if (opt_name == ":final") {
@@ -374,6 +382,25 @@ StructureDefResult parse_structure_def(
     }
   }
 
+  if (ts->fully_defined_type_exists(TypeSpec("process")) &&
+      ts->tc(TypeSpec("process"), TypeSpec(type->get_parent()))) {
+    // check heap-base if this is a child of process.
+    auto process_type = ts->get_type_of_type<BasicType>("process");
+    auto auto_hb = (flags.size - process_type->size() + 0xf) & ~0xf;
+    if (!set_heapbase) {
+      // wasnt set manually so set automatically.
+      flags.heap_base = auto_hb;
+    } else if (flags.heap_base < auto_hb) {
+      // was set manually so verify if that's correct.
+      throw std::runtime_error(
+          fmt::format("Process heap underflow in type {}: heap-base is {} vs. auto-detected {}",
+                      type->get_name(), flags.heap_base, auto_hb));
+      //} else if (flags.heap_base != auto_hb) {
+      //  fmt::print("Type {} has manual heap-base ({} vs {}). This is fine. \n", type->get_name(),
+      //             flags.heap_base, auto_hb);
+    }
+  }
+
   if (size_assert != -1 && flags.size != u16(size_assert)) {
     throw std::runtime_error("Type " + type->get_name() + " came out to size " +
                              std::to_string(int(flags.size)) + " but size-assert was set to " +
@@ -394,15 +421,6 @@ StructureDefResult parse_structure_def(
                     flags.flag, flag_assert));
   }
 
-  if (result.flags.heap_base) {
-    int heap_start_1 = 112 + result.flags.heap_base;
-    int heap_start_2 = (result.flags.size + 15) & (~15);
-    if (heap_start_1 != heap_start_2) {
-      throw std::runtime_error(fmt::format("Heap base bad on {}: {} vs {}\n", type->get_name(),
-                                           heap_start_1, heap_start_2));
-    }
-  }
-
   result.flags = flags;
   return result;
 }

common/type_system/deftype.h

@@ -7,6 +7,7 @@
  */
 
 #include "TypeSystem.h"
+
 #include "common/goos/Object.h"
 
 struct DeftypeResult {

common/type_system/state.cpp

@@ -1,4 +1,5 @@
 #include "state.h"
+
 #include "common/type_system/TypeSystem.h"
 
 /*!

common/type_system/state.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <optional>
+
 #include "common/type_system/TypeSpec.h"
 
 /*!

common/util/Assert.cpp

@@ -1,9 +1,38 @@
-#include <cstdio>
-#include <cstdlib>
-
 #include "Assert.h"
 
-void private_assert_failed(const char* expr, const char* file, int line, const char* function) {
-  fprintf(stderr, "%s:%d: Assertion failed: %s\nFunction: %s\n", file, line, expr, function);
+#ifndef NO_ASSERT
+
+#include <cstdio>
+#include <cstdlib>
+#include <string_view>
+
+void private_assert_failed(const char* expr,
+                           const char* file,
+                           int line,
+                           const char* function,
+                           const char* msg) {
+  if (!msg || msg[0] == '\0') {
+    fprintf(stderr, "Assertion failed: '%s'\n\tSource: %s:%d\n\tFunction: %s\n", expr, file, line,
+            function);
+  } else {
+    fprintf(stderr, "Assertion failed: '%s'\n\tMessage: %s\n\tSource: %s:%d\n\tFunction: %s\n",
+            expr, msg, file, line, function);
+  }
+  fflush(stdout);  // ensure any stdout logs are flushed before we terminate
+  fflush(stderr);
   abort();
 }
+
+void private_assert_failed(const char* expr,
+                           const char* file,
+                           int line,
+                           const char* function,
+                           const std::string_view& msg) {
+  if (msg.empty()) {
+    private_assert_failed(expr, file, line, function);
+  } else {
+    private_assert_failed(expr, file, line, function, msg.data());
+  }
+}
+
+#endif

common/util/Assert.h

@@ -5,10 +5,21 @@
 
 #pragma once
 
+#ifndef NO_ASSERT
+
+#include <string_view>
+
 [[noreturn]] void private_assert_failed(const char* expr,
                                         const char* file,
                                         int line,
-                                        const char* function);
+                                        const char* function,
+                                        const char* msg = "");
+
+[[noreturn]] void private_assert_failed(const char* expr,
+                                        const char* file,
+                                        int line,
+                                        const char* function,
+                                        const std::string_view& msg);
 
 #ifdef _WIN32
 #define __PRETTY_FUNCTION__ __FUNCSIG__
@@ -16,3 +27,13 @@
 
 #define ASSERT(EX) \
   (void)((EX) || (private_assert_failed(#EX, __FILE__, __LINE__, __PRETTY_FUNCTION__), 0))
+
+#define ASSERT_MSG(EXPR, STR) \
+  (void)((EXPR) || (private_assert_failed(#EXPR, __FILE__, __LINE__, __PRETTY_FUNCTION__, STR), 0))
+
+#else
+
+#define ASSERT(EX) ((void)0)
+#define ASSERT_MSG(EXPR, STR) ((void)0)
+
+#endif

common/util/BinaryReader.h

@@ -8,6 +8,7 @@
 #include <cstdint>
 #include <cstring>
 #include <vector>
+
 #include "common/common_types.h"
 #include "common/util/Assert.h"
 

common/util/BinaryWriter.h

@@ -5,10 +5,11 @@
  * Write raw data like a stream.
  */
 
-#include <stdexcept>
-#include <vector>
 #include <cstdint>
 #include <cstring>
+#include <stdexcept>
+#include <vector>
+
 #include "common/util/Assert.h"
 
 struct BinaryWriterRef {

common/util/BitUtils.h

@@ -1,9 +1,10 @@
 #pragma once
 
 #include <optional>
-#include "common/util/Range.h"
+
 #include "common/common_types.h"
 #include "common/util/Assert.h"
+#include "common/util/Range.h"
 
 constexpr int BITS_PER_BYTE = 8;
 template <typename T>

common/util/CopyOnWrite.h

@@ -1,4 +1,5 @@
 #include <utility>
+
 #include "common/util/Assert.h"
 
 /*

common/util/DgoReader.cpp

@@ -1,13 +1,17 @@
-#include <cstring>
-#include <utility>
-#include <unordered_set>
 #include "DgoReader.h"
+
+#include <cstring>
+#include <unordered_set>
+#include <utility>
+
 #include "BinaryReader.h"
-#include "common/link_types.h"
-#include "third-party/json.hpp"
 #include "BitUtils.h"
 #include "dgo_util.h"
 
+#include "common/link_types.h"
+
+#include "third-party/json.hpp"
+
 DgoReader::DgoReader(std::string file_name, const std::vector<u8>& data)
     : m_file_name(std::move(file_name)) {
   BinaryReader reader(data);