mirror/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project synced 2026-06-05 03:08:40 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'open-goal/master' into v/translations

Browse Source
This commit is contained in:
Tyler Wilding
2023-02-23 17:13:27 -05:00
parent f257cc42bb 2d8ad39fea
commit 75d48a41dc
2735 changed files with 2068207 additions and 46519 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/update-file-progress.py
-35
View File
@@ -1,35 +0,0 @@
import glob
src_files = glob.glob("./goal_src/**/*.g[cs]", recursive=True)
# Find how many of each have been started
src_files_started = 0
src_files_finished = 0
data_files_started = 0
for f in src_files:
with open(f, "r") as temp_file:
lines = temp_file.readlines()
line_count = len(lines)
if line_count > 7:
# Check to see if there are any TODOs
if any("TODO" in string for string in lines):
src_files_finished = src_files_finished + 1
else:
src_files_started = src_files_started + 1
import json
with open('./docs/gh-pages-proj/src/config/progress.json', 'r+', encoding='utf-8') as f:
data = {
'jak1': {
'fileProgress': {
'src_files_total': len(src_files),
'src_files_finished': src_files_finished,
'src_files_started': src_files_started
}
}
}
f.seek(0)
json.dump(data, f, ensure_ascii=False, indent=2)
f.truncate()
.github/scripts/update-gallery.py
-63
View File
@@ -1,63 +0,0 @@
import glob
import os
from pathlib import Path
import json
galleryLinks = {
'jak1': {
'name': "Jak 1",
'media': [],
},
'jak2': {
'name': "Jak 2",
'media': [],
},
'jak3': {
'name': "Jak 3",
'media': [],
},
'jakx': {
'name': "Jak X",
'media': [],
},
'misc': {
'name': "Miscellaneous",
'media': [],
}
}
def get_links(key, folder_to_search):
if os.path.isdir(folder_to_search):
files = glob.glob(folder_to_search + "/*.png", recursive=True)
files.extend(glob.glob(folder_to_search + "/*.jpg", recursive=True))
files.extend(glob.glob(folder_to_search + "/*.jpeg", recursive=True))
for f in files:
galleryLinks[key]["media"].append({
'fileName': os.path.basename(f),
'timestamp': Path(f).stem.split("_")[1],
'caption': Path(f).stem.split("_")[0].replace("-", " ").title(),
'video': False
})
# get videos potentially
if os.path.exists("{}/videos.json".format(folder_to_search)):
with open("{}/videos.json".format(folder_to_search), 'r') as f:
data = json.load(f)
for video in data:
galleryLinks[key]["media"].append({
'link': video["link"].replace("watch?v=", "embed/"),
'timestamp': video["timestamp"],
'video': True
})
# sort by timestamp
galleryLinks[key]["media"].sort(key=lambda x: x["timestamp"], reverse=True)
get_links('jak1', './docs/gh-pages-proj/src/assets/gallery/jak1')
get_links('jak2', './docs/gh-pages-proj/src/assets/gallery/jak2')
get_links('jak3', './docs/gh-pages-proj/src/assets/gallery/jak3')
get_links('jakx', './docs/gh-pages-proj/src/assets/gallery/jakx')
get_links('misc', './docs/gh-pages-proj/src/assets/gallery/misc')
with open('./docs/gh-pages-proj/src/config/gallery.json', 'r+', encoding='utf-8') as f:
f.seek(0)
json.dump(galleryLinks, f, ensure_ascii=False, indent=2)
f.truncate()
.github/workflows/build-matrix.yaml
+9
View File
@@ -7,6 +7,7 @@ on:
pull_request:
branches:
- master
merge_group: {}
jobs:
# Windows
@@ -42,3 +43,11 @@ jobs:
cmakePreset: "Release-linux-gcc"
cachePrefix: ""
secrets: inherit
# MacOS
build_macos_clang:
name: "🍎 MacOS"
uses: ./.github/workflows/macos-build-clang.yaml
with:
cmakePreset: "Release-macos-clang"
cachePrefix: ""
.github/workflows/draft-new-release.yaml
+1 -1
View File
@@ -23,7 +23,7 @@ jobs:
- name: Bump Version and Push Tag
if: github.repository == 'open-goal/jak-project'
id: tag_version
uses: mathieudutour/github-tag-action@v6.0
uses: mathieudutour/github-tag-action@v6.1
with:
github_token: ${{ secrets.BOT_PAT }}
tag_prefix: v
.github/workflows/inform-pages-repo.yaml
+60 -4
View File
@@ -6,17 +6,73 @@ on:
- master
paths:
- 'goal_src/**'
workflow_dispatch: {}
jobs:
inform:
name: Inform Pages Repo
gen-docs:
name: Generate Documentation
if: github.repository == 'open-goal/jak-project'
runs-on: ubuntu-latest
timeout-minutes: 10
runs-on: ubuntu-20.04
timeout-minutes: 45
env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
BUILDCACHE_COMPRESS_FORMAT: ZSTD
BUILDCACHE_COMPRESS_LEVEL: 19
BUILDCACHE_DIRECT_MODE: true
BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
steps:
- name: Checkout Repository
uses: actions/checkout@v3
- name: Install Package Dependencies
run: >
sudo apt install build-essential cmake
clang gcc g++ lcov make nasm libxrandr-dev
libxinerama-dev libxcursor-dev libpulse-dev
libxi-dev zip ninja-build
- name: Setup Buildcache
uses: mikehardy/buildcache-action@v2.1.0
with:
cache_key: linux-ubuntu-20.04--Release-linux-clang
buildcache_tag: v0.28.1
- name: CMake Generation
env:
CC: clang
CXX: clang++
run: |
cmake -B build --preset=Release-linux-clang \
-DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
-DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
- name: Build Project
run: cmake --build build --parallel $((`nproc`)) --target goalc
- name: Generate Docs For Jak 1
run: |
./build/goalc/goalc --cmd "(begin (make-group \"all-code\") (gen-docs \"${PWD}\"))" --game jak1
- name: Generate Docs For Jak 2
run: |
./build/goalc/goalc --cmd "(begin (make-group \"all-code\") (gen-docs \"${PWD}\"))" --game jak2
- name: Upload Docs Artifact
uses: actions/upload-artifact@v3
with:
name: opengoal-docs
if-no-files-found: error
path: |
./jak1*.json
./jak2*.json
- name: Log Run ID
run: echo ${{ github.run_id }}
- name: Send Dispatch
uses: peter-evans/repository-dispatch@v2
with:
token: ${{ secrets.BOT_PAT }}
repository: 'open-goal/open-goal.github.io'
event-type: updateProgress
client-payload: '{"docGenRunId": "${{ github.run_id }}"}'
.github/workflows/linting.yaml
+10 -1
View File
@@ -7,6 +7,7 @@ on:
pull_request:
branches:
- master
merge_group: {}
jobs:
lint:
@@ -17,12 +18,20 @@ jobs:
- name: Checkout Repository
uses: actions/checkout@v3
# wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
# sudo add-apt-repository 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main'
# sudo apt install clang-format-15
# --clang-format-executable $(which clang-format-15)
- name: Get Package Dependencies
run: |
sudo apt install clang-format clang-tidy
sudo apt update
sudo apt install clang-format
clang-format -version
- name: Check Clang-Formatting
run: |
chmod +x ./third-party/run-clang-format/run-clang-format.py
./third-party/run-clang-format/run-clang-format.py -r common decompiler game goalc test tools lsp --color always
- name: Check for Unresolved Conflicts
run: python ./scripts/gsrc/check-for-conflicts.py
.github/workflows/linux-build-clang.yaml
+3 -2
View File
@@ -35,9 +35,10 @@ jobs:
libxi-dev zip ninja-build
- name: Setup Buildcache
uses: mikehardy/buildcache-action@v1.3.0
uses: mikehardy/buildcache-action@v2.1.0
with:
cache_key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
buildcache_tag: v0.28.1
- name: CMake Generation
env:
@@ -49,7 +50,7 @@ jobs:
-DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
- name: Build Project
run: cmake --build build -j$((`nproc`+1))
run: cmake --build build --parallel $((`nproc`))
- name: Run Tests
env:
.github/workflows/linux-build-gcc.yaml
+3 -2
View File
@@ -35,9 +35,10 @@ jobs:
libxi-dev zip ninja-build
- name: Setup Buildcache
uses: mikehardy/buildcache-action@v1.3.0
uses: mikehardy/buildcache-action@v2.1.0
with:
cache_key: linux-ubuntu-20.04-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
buildcache_tag: v0.28.1
- name: CMake Generation
env:
@@ -50,7 +51,7 @@ jobs:
-DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
- name: Build Project
run: cmake --build build -j$((`nproc`+1)) -- -w dupbuild=warn
run: cmake --build build --parallel $((`nproc`)) -- -w dupbuild=warn
- name: Run Tests - With Coverage
working-directory: ./build
.github/workflows/macos-build-clang.yaml
+76
View File
@@ -0,0 +1,76 @@
name: MacOS Build Clang
on:
workflow_call:
inputs:
cmakePreset:
required: true
type: string
cachePrefix:
required: true
type: string
jobs:
build:
name: Clang
runs-on: macos-12
timeout-minutes: 120
env: # overrides: https://github.com/mbitsnbites/buildcache/blob/master/doc/configuration.md
BUILDCACHE_MAX_CACHE_SIZE: 1000000000 # 1gb
BUILDCACHE_COMPRESS_FORMAT: ZSTD
BUILDCACHE_COMPRESS_LEVEL: 19
BUILDCACHE_DIRECT_MODE: true
BUILDCACHE_LOG_FILE: ${{ github.workspace }}/buildcache.log
steps:
- name: Checkout Repository
uses: actions/checkout@v3
- name: Install Package Dependencies
run: brew install cmake nasm ninja
- name: Setup Buildcache
uses: mikehardy/buildcache-action@v2.1.0
with:
cache_key: macos-12-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
buildcache_tag: v0.28.1
- name: CMake Generation
env:
CC: clang
CXX: clang++
run: |
cmake -B build --preset=${{ inputs.cmakePreset }} \
-DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache \
-DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/buildcache/bin/buildcache
# Disabled for now, not all build targets are valid
# - name: Build Project
# run: cmake --build build --parallel $((`sysctl -n hw.logicalcpu`))
# Temporary, selectively build those that work
- name: Build Working Targets
run: |
cmake --build build --target extractor --parallel $((`sysctl -n hw.logicalcpu`)) && \
cmake --build build --target offline-test --parallel $((`sysctl -n hw.logicalcpu`)) && \
cmake --build build --target decompiler --parallel $((`sysctl -n hw.logicalcpu`)) && \
cmake --build build --target lsp --parallel $((`sysctl -n hw.logicalcpu`)) && \
cmake --build build --target goalc --parallel $((`sysctl -n hw.logicalcpu`))
- name: Run Tests
continue-on-error: true # until macOS is stable
env:
GTEST_OUTPUT: "xml:opengoal-test-report.xml"
run: ./test.sh
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
name: opengoal-macos-${{ inputs.cachePrefix }}
if-no-files-found: error
path: |
./build/goalc/goalc
./build/decompiler/extractor
./build/game/gk
./build/lsp/lsp
.github/workflows/release-pipeline.yaml
+7
View File
@@ -79,3 +79,10 @@ jobs:
run: |
TAG_VAL=$(echo ${{ github.REF }} | awk -F'refs/tags/' '{print $2}')
gh release edit ${TAG_VAL} --draft=false --repo open-goal/jak-project
- name: Consume Release in the Launcher
uses: peter-evans/repository-dispatch@v2
with:
token: ${{ secrets.BOT_PAT }}
repository: 'open-goal/launcher'
event-type: releaseLauncher
.github/workflows/update-controller-db.yaml
+11 -6
View File
@@ -3,6 +3,7 @@ name: Update Controller Database
on:
schedule:
- cron: "0 16 * * 1" # every monday @ 12pm EST - https://crontab.guru/#0_16_*_*_1
workflow_dispatch: {}
jobs:
update-controller-db:
@@ -17,10 +18,14 @@ jobs:
run: |
wget -O ./game/assets/sdl_controller_db.txt https://raw.githubusercontent.com/gabomdq/SDL_GameControllerDB/master/gamecontrollerdb.txt
- name: commit version bump
uses: EndBug/add-and-commit@v9
- name: Create Pull Request
uses: peter-evans/create-pull-request@v4
with:
default_author: github_actor
author_name: 'OpenGOALBot'
author_email: 'OpenGOALBot@users.noreply.github.com'
message: "Updating Controller Database"
token: ${{ secrets.BOT_PAT }}
author: 'OpenGOALBot <OpenGOALBot@users.noreply.github.com>'
committer: 'OpenGOALBot <OpenGOALBot@users.noreply.github.com>'
branch: 'bot/update-controller-db'
base: "master"
body: "Updating Controller Database"
commit-message: "ci: updated controller database"
title: "CI: Periodic Controller Database Update"
.github/workflows/windows-build-clang.yaml
+3 -3
View File
@@ -32,7 +32,7 @@ jobs:
run: choco install ${{ github.workspace }}/third-party/nasm/nasm.2.15.05.nupkg
- name: Setup Buildcache
uses: mikehardy/buildcache-action@v1.3.0
uses: mikehardy/buildcache-action@v2.1.0
with:
cache_key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
@@ -44,13 +44,13 @@ jobs:
- name: Build Project
shell: cmd
run: cmake --build build -j 2
run: cmake --build build --parallel %NUMBER_OF_PROCESSORS%
- name: Run Tests
timeout-minutes: 10
env:
GTEST_OUTPUT: "xml:opengoal-test-report.xml"
run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
run: ./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"
- name: Upload artifact
uses: actions/upload-artifact@v3
.github/workflows/windows-build-msvc.yaml
+3 -3
View File
@@ -31,7 +31,7 @@ jobs:
run: choco install ${{ github.workspace }}/third-party/nasm/nasm.2.15.05.nupkg
- name: Setup Buildcache
uses: mikehardy/buildcache-action@v1.3.0
uses: mikehardy/buildcache-action@v2.1.0
with:
cache_key: windows-2022-${{ inputs.cachePrefix }}-${{ inputs.cmakePreset }}
@@ -45,12 +45,12 @@ jobs:
shell: cmd
run: |
set CL=/MP
cmake --build build -j 2
cmake --build build --parallel %NUMBER_OF_PROCESSORS%
- name: Run Tests
timeout-minutes: 10
env:
GTEST_OUTPUT: "xml:opengoal-test-report.xml"
run: |
./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=1 --gtest_filter="-*MANUAL_TEST*"
./build/bin/goalc-test.exe --gtest_color=yes --gtest_brief=0 --gtest_filter="-*MANUAL_TEST*"
.gitignore
+24 -3
View File
@@ -3,12 +3,17 @@
# for clion
cmake-build-debug/*
cmake-build-debug--o0/*
.idea/*
build/*
decompiler_out/*
decompiler_out2/*
logs/*
# for vscode/clangd
.cache/*
.DS_Store
# wsl apparently builds to here?
linux-default/
@@ -24,13 +29,18 @@ linux-default/
*.log
*.p2s
savestate-out/
savestate_out/
failures/
ee-results.json
ee-results*.json
search-results.json
.env
/search-results.json
# graphics debug
debug_out/*
gfx_dumps/*
debug_out/
glb_out/
gfx_dumps/
screenshots/
# game stuff
game_config/*
@@ -47,3 +57,14 @@ svnrev.h
ci-artifacts/
out/build/
__pycache__/
# sqlite stuff
*.db
*.db-journal
# backup files from OpenMaya
*.bak
# docs
/jak1-*.json
/jak2-*.json
.prettierrc
+1
View File
@@ -0,0 +1 @@
endOfLine: "auto"
.vs/launch.vs.json
+76 -18
View File
@@ -30,6 +30,13 @@
"name": "Tests - TypeConsistency - Verbose",
"args": ["--gtest_brief=0", "--gtest_filter=\"*TypeConsistency*\""]
},
{
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "goalc-test.exe (bin\\goalc-test.exe)",
"name": "Tests - TypeConsistency - Jak 2 - Verbose",
"args": ["--gtest_brief=0", "--gtest_filter=\"*Jak2TypeConsistency*\""]
},
{
"type": "default",
"project": "CMakeLists.txt",
@@ -49,6 +56,46 @@
"jak1"
]
},
{
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "offline-test.exe (bin\\offline-test.exe)",
"name": "Tests - Offline Tests - Jak 2",
"args": [
"--iso_data_path",
"${workspaceRoot}/iso_data/jak2",
"--game",
"jak2"
]
},
{
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "offline-test.exe (bin\\offline-test.exe)",
"name": "Tests - Offline Tests - Jak 1 - Specific File",
"args": [
"--iso_data_path",
"${workspaceRoot}/iso_data/jak1",
"--game",
"jak1",
"--file",
"euler-h"
]
},
{
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "offline-test.exe (bin\\offline-test.exe)",
"name": "Tests - Offline Tests - Jak 2 - Specific File",
"args": [
"--iso_data_path",
"${workspaceRoot}/iso_data/jak2",
"--game",
"jak2",
"--file",
"oracle-texture"
]
},
{
"type": "default",
"project": "CMakeLists.txt",
@@ -77,6 +124,20 @@
"name": "Game - Runtime (boot no debug)",
"args": ["-boot", "-fakeiso", "-v"]
},
{
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "gk.exe (bin\\gk.exe)",
"name": "Game - Jak 2 - Runtime (boot)",
"args": ["-boot", "-fakeiso", "-debug", "-v", "-jak2"]
},
{
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "gk.exe (bin\\gk.exe)",
"name": "Game - Jak 2 - Runtime (no boot)",
"args": ["-fakeiso", "-debug", "-v", "-jak2"]
},
{
"type": "default",
"project": "CMakeLists.txt",
@@ -88,8 +149,8 @@
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "goalc.exe (bin\\goalc.exe)",
"name": "REPL - Auto Listen",
"args": ["--user-auto", "--auto-lt"]
"name": "REPL - Jak 2",
"args": ["--user-auto", "--game", "jak2"]
},
{
"type": "default",
@@ -102,17 +163,6 @@
"${workspaceRoot}/decompiler_out"
]
},
{
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "decompiler.exe (bin\\decompiler.exe)",
"name": "Decompiler - Jak 1 - Data Only",
"args": [
"${workspaceRoot}/decompiler/config/jak1_ntsc_black_label.jsonc",
"${workspaceRoot}/iso_data",
"${workspaceRoot}/decompiler_out"
]
},
{
"type": "default",
"project": "CMakeLists.txt",
@@ -128,9 +178,9 @@
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "decompiler.exe (bin\\decompiler.exe)",
"name": "Disassembler - Jak 1",
"name": "Decompiler - Jak 2",
"args": [
"${workspaceRoot}/decompiler/config/jak1_ntsc_black_label.jsonc",
"${workspaceRoot}/decompiler/config/jak2_ntsc_v1.jsonc",
"${workspaceRoot}/iso_data",
"${workspaceRoot}/decompiler_out"
]
@@ -139,11 +189,12 @@
"type": "default",
"project": "CMakeLists.txt",
"projectTarget": "decompiler.exe (bin\\decompiler.exe)",
"name": "Decompiler - Jak 2",
"name": "Decompiler - Jak 2 - Extract",
"args": [
"${workspaceRoot}/decompiler/config/jak2_ntsc_v1.jsonc",
"${workspaceRoot}/iso_data",
"${workspaceRoot}/decompiler_out"
"${workspaceRoot}/decompiler_out",
"--config-override \"{\\\"decompile_code\\\": false}\""
]
},
{
@@ -181,8 +232,15 @@
"type" : "default",
"project" : "CMakeLists.txt",
"projectTarget" : "lsp.exe (bin\\lsp.exe)",
"name" : "Run - LSP",
"name" : "Tools - LSP",
"args" : []
},
{
"type" : "default",
"project" : "CMakeLists.txt",
"projectTarget" : "type_searcher.exe (bin\\type_searcher.exe)",
"name" : "Tools - Type Searcher",
"args" : ["--game", "jak2", "--output-path", "./search-results.json", "--size", 255, "--fields", "[{\\\"type\\\":\\\"quaternion\\\",\\\"offset\\\":48}]"]
}
]
}
.vscode/extensions.json
Vendored
+9
View File
@@ -0,0 +1,9 @@
{
// See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
// Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
// List of extensions which should be recommended for users of this workspace.
"recommendations": ["opengoal.opengoal", "naumovs.color-highlight"],
// List of extensions recommended by VS Code that should not be recommended for users of this workspace.
"unwantedRecommendations": []
}
.vscode/launch.json
Vendored
+3 -2
View File
@@ -8,9 +8,10 @@
"name": "Python",
"type": "python",
"request": "launch",
"program": "${workspaceFolder}/scripts/gsrc/update-from-decomp.py",
"program": "${workspaceFolder}/scripts/gsrc/lint-gsrc-file.py",
"console": "integratedTerminal",
"cwd": "${workspaceFolder}"
"cwd": "${workspaceFolder}",
"args": ["--game", "jak2", "--file", "vehicle-control"]
},
]
}
.vscode/opengoal.code-snippets
Vendored
+26
View File
@@ -0,0 +1,26 @@
{
"og:ignore-from-loc": {
"scope": "opengoal",
"prefix": ["og:ignore-from-loc"],
"body": [";; og:ignore-from-loc"],
"description": "Marks the file to be excluded from the LoC count"
},
"og:ignore-errors": {
"scope": "opengoal",
"prefix": ["og:ignore-errors"],
"body": [";; og:ignore-errors"],
"description": "Ignore lines beginning with ';; ERROR:' or ';; WARN:' when copying decompiled code into it"
},
"og:ignore-form": {
"scope": "opengoal",
"prefix": ["og:ignore-form"],
"body": [";; og:ignore-form:${1:substr}"],
"description": "If the provided substr is found in the starting line of a form, the entire form will be omitted when copying decompiled code into the file"
},
"og:update-with-merge": {
"scope": "opengoal",
"prefix": ["og:update-with-merge"],
"body": [";; og:update-with-merge:${1:substr}"],
"description": "The file will be updated with a git merge-file instead of naive copy-paste"
},
}
.vscode/settings.json
Vendored
+10 -2
View File
@@ -1,4 +1,12 @@
{
"terminal.integrated.scrollback": 32000,
"python.formatting.provider": "black",
"[opengoal]": {
"editor.quickSuggestions": {
"other": true,
"comments": true,
"strings": true
},
"editor.wordBasedSuggestions": true,
"editor.snippetSuggestions": "top"
},
"python.formatting.provider": "black"
}
CMakeLists.txt
+36
View File
@@ -94,6 +94,35 @@ elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
endif()
set(THIRDPARTY_IGNORED_WARNINGS "-w")
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
message(STATUS "AppleClang detected - Setting Defaults")
set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} \
-march=native \
-Wall \
-Winit-self \
-ggdb \
-Wextra \
-Wno-cast-align \
-Wcast-qual \
-Wdisabled-optimization \
-Wformat \
-Wextra \
-Wmissing-include-dirs \
-Woverloaded-virtual \
-Wredundant-decls \
-Wshadow \
-Wsign-promo \
-O3 \
-fdiagnostics-color=always"
)
# additional c++ flags for release mode for our projects
if(CMAKE_BUILD_TYPE MATCHES "Release")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
endif()
set(THIRDPARTY_IGNORED_WARNINGS "-w")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-stack_size -Wl,0x20000000")
elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
message(STATUS "MSVC detected - Setting Defaults")
@@ -145,6 +174,13 @@ include_directories(SYSTEM third-party/inja)
# build repl library
add_subdirectory(third-party/replxx EXCLUDE_FROM_ALL)
# SQLite - Jak 2/3's built in editor
add_definitions(-DHAVE_USLEEP=1)
add_definitions(-DSQLITE_THREADSAFE=1)
add_definitions(-DSQLITE_ENABLE_RTREE)
include_directories(third-party/SQLiteCpp/include)
add_subdirectory(third-party/SQLiteCpp)
string(REPLACE " ${THIRDPARTY_IGNORED_WARNINGS} " "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
# build common library
CMakePresets.json
+16
View File
@@ -57,6 +57,16 @@
"CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}"
}
},
{
"name": "base-macos-release",
"hidden": true,
"inherits": "base",
"binaryDir": "${sourceDir}/build/Release/bin",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Release",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}"
}
},
{
"name": "base-clang",
"hidden": true,
@@ -127,6 +137,12 @@
"description": "Build with Clang as Release without Debug Symbols",
"inherits": ["base-linux-release", "base-clang"]
},
{
"name": "Release-macos-clang",
"displayName": "MacOS Release (clang)",
"description": "Build with Clang as Release without Debug Symbols",
"inherits": ["base-macos-release", "base-clang"]
},
{
"name": "Release-linux-clang-asan",
"displayName": "Linux Release (clang-asan)",
README.md
+18 -5
View File
@@ -4,7 +4,7 @@
<p align="center">
<a href="https://opengoal.dev/docs/intro" rel="nofollow"><img src="https://img.shields.io/badge/Documentation-Here-informational" alt="Documentation Badge" style="max-width:100%;"></a>
<a target="_blank" rel="noopener noreferrer" href="https://github.com/open-goal/jak-project/workflows/Build/badge.svg"><img src="https://github.com/open-goal/jak-project/workflows/Build/badge.svg" alt="Linux and Windows Build" style="max-width:100%;"></a>
<a target="_blank" rel="noopener noreferrer" href="https://github.com/open-goal/jak-project/actions/workflows/build-matrix.yaml"><img src="https://github.com/open-goal/jak-project/actions/workflows/build-matrix.yaml/badge.svg" alt="Linux and Windows Build" style="max-width:100%;"></a>
<a href="https://www.codacy.com/gh/open-goal/jak-project/dashboard?utm_source=github.com&utm_medium=referral&utm_content=open-goal/jak-project&utm_campaign=Badge_Coverage" rel="nofollow"><img src="https://app.codacy.com/project/badge/Coverage/29316d04a1644aa390c33be07289f3f5" alt="Codacy Badge" style="max-width:100%;"></a>
<a href="https://www.codacy.com/gh/open-goal/jak-project/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=open-goal/jak-project&amp;utm_campaign=Badge_Grade" rel="nofollow"><img src="https://app.codacy.com/project/badge/Grade/29316d04a1644aa390c33be07289f3f5" alt="Codacy Badge" style="max-width:100%;"></a>
<a href="https://discord.gg/VZbXMHXzWv"><img src="https://img.shields.io/discord/756287461377703987" alt="Discord"></a>
@@ -112,13 +112,13 @@ This will create an image with all required dependencies and already built.
docker run -v "$(pwd)"/build:/home/jak/jak-project/build -it jak bash
```
Note: If you the build directory you'll need to rerun the build command. Alteratively you can get the build via `docker cp`
Note: If you change the content of the `build/` directory you'll need to rerun the `build` command. Alternatively you can get the build via `docker cp`.
This will link your build folder to the images so can validate your build or test it on an external device.
This will link your `build/` folder to the images so can validate your build or test it on an external device.
Docker images can be linked into your IDE (e.g. CLion) to help with codesniffing, static analysis, run tests and continuous build.
Unfortently you'll still need task runner on your local machine to run the game or instead, manually run the game via the commands found in `Taskfile.yml`
Unfortunately you'll still need task runner on your local machine to run the game or instead, manually run the game via the commands found in `Taskfile.yml`.
### Linux
@@ -244,13 +244,20 @@ Getting a running game involves 4 steps:
#### Extract Assets
First, setup your settings so the following scripts know which game you are using, and which version. In a terminal, run the following:
First, setup your settings so the following scripts know which game you are using, and which version. For the black label version of the game, run the following in a terminal:
```sh
task set-game-jak1
task set-decomp-ntscv1
```
For other versions of the game, you will need to use a different `-set-decomp-<VERSION>` command. An example for the PAL version:
```sh
task set-game-jak1
task set-decomp-pal
```
> Run `task --list` to see the other available options
> At the time of writing, only Jak 1 is expected to work end-to-end!
@@ -292,6 +299,12 @@ Run the following to build the game:
g > (mi)
```
> IMPORTANT NOTE! If you're not using the black label version, you may hit issues trying to run `(mi)` in this step. An example error might include something like:
>
> `Input file iso_data/jak1/MUS/TWEAKVAL.MUS does not exist.`
>
> This is because other version paths are not currently accounted for in the build. A quick workaround is to rename both your `decompiler_out` and `iso_data` folders to use the black label naming, for example changing `decompiler_out/jak1_pal` to `decompiler_out/jak1` and `iso_data/jak1_pal` to `iso_data/jak1`, then running `(mi)` again.
#### Run the Game
Finally the game can be ran. Open a second terminal from the `jak-project` directory and run the following:
Taskfile.yml
+70 -37
View File
@@ -24,9 +24,13 @@ tasks:
cmds:
- 'python ./scripts/tasks/update-env.py --decomp_config ntscv2'
set-decomp-pal:
- 'python ./scripts/tasks/update-env.py --decomp_config pal'
desc: "PAL region version"
cmds:
- 'python ./scripts/tasks/update-env.py --decomp_config pal'
set-decomp-ntscjp:
- 'python ./scripts/tasks/update-env.py --decomp_config ntscjp'
desc: "NTSC-J region version"
cmds:
- 'python ./scripts/tasks/update-env.py --decomp_config ntscjp'
# GENERAL
extract:
desc: "Extracts the game's assets from './iso_data' with the set decompiler config"
@@ -34,7 +38,7 @@ tasks:
- sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/decompiler{{.EXE_FILE_EXTENSION}}
msg: "Couldn't locate decompiler executable in '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler'"
cmds:
- '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" ./iso_data ./decompiler_out --config-override "{\\\"decompile_code\\\": false}"'
- "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --config-override '{\"decompile_code\": false, \"levels_extract\": true, \"allowed_objects\": []}'"
boot-game:
desc: "Boots the game, it will fail if it's not already compiled!"
preconditions:
@@ -55,7 +59,7 @@ tasks:
- sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
cmds:
- "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -v"
- "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -{{.GAME}} -v"
# DEVELOPMENT
repl:
desc: "Start the REPL"
@@ -63,10 +67,7 @@ tasks:
- sh: test -f {{.GOALC_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}}
msg: "Couldn't locate compiler executable in '{{.GOALC_BIN_RELEASE_DIR}}/goalc'"
cmds:
- "{{.GOALC_BIN_RELEASE_DIR}}/goalc --game {{.GAME}}"
repl-lt:
cmds:
- "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt --game {{.GAME}}"
- "{{.GOALC_BIN_RELEASE_DIR}}/goalc --user-auto --game {{.GAME}}"
format:
desc: "Format code"
cmds:
@@ -74,46 +75,78 @@ tasks:
# npm install -g prettier
- cmd: npx prettier --write ./decompiler/config/jak1_ntsc_black_label/*.jsonc
ignore_error: true
- cmd: npx prettier --write ./decompiler/config/jak2/*.jsonc
ignore_error: true
run-game-headless:
cmds:
- "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -nodisplay"
# DECOMPILING
decomp:
cmds:
- '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" "./iso_data" "./decompiler_out"'
- "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --config-override '{\"levels_extract\": false}'"
decomp-file:
cmds:
- "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler \"./decompiler/config/{{.DECOMP_CONFIG}}\" \"./iso_data\" \"./decompiler_out\" --config-override '{\"levels_extract\": false, \"allowed_objects\": [\"{{.FILE}}\"]}'"
decomp-clean:
cmds:
- rm ./decompiler_out/**/*.asm
- rm ./decompiler_out/**/*disasm.gc
# TOOLS
# analyze-ee-memory:
# cmds:
# - '{{.MEMDUMP_BIN_RELEASE_DIR}}/memory_dump_tool "{{.FILE}}" ./ > ee-analysis.log'
# watch-pcsx2:
# cmds:
# - watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}"
# vars:
# SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}'
# TESTS
offline-tests:
- python ./scripts/tasks/clean-decomp.py --game "{{.GAME}}"
lint-gsrc-file:
cmds:
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}}'
- python ./scripts/gsrc/lint-gsrc-file.py --game {{.GAME}} --file {{.FILE}}
update-gsrc:
cmds:
- python ./scripts/gsrc/update-gsrc-via-refs.py --game "{{.GAME}}" --decompiler "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler" --decompiler_config {{.DECOMP_CONFIG}}
update-gsrc-glob:
cmds:
- python ./scripts/gsrc/update-gsrc-via-refs.py --game "{{.GAME}}" --file_pattern "{{.GLOB}}" --decompiler "{{.DECOMP_BIN_RELEASE_DIR}}/decompiler" --decompiler_config {{.DECOMP_CONFIG}}
update-gsrc-file:
cmds:
- task: decomp-file
- python ./scripts/gsrc/update-from-decomp.py --game "{{.GAME}}" --file {{.FILE}}
- task: lint-gsrc-file
# TOOLS
analyze-ee-memory:
cmds:
- python ./scripts/tasks/extract-zip.py --file "{{.FILE}}" --out ./savestate_out/
- '{{.MEMDUMP_BIN_RELEASE_DIR}}/memory_dump_tool ./savestate_out/eeMemory.bin --output-path ./ --game {{.GAME}} > ee-analysis.log'
watch-pcsx2:
# python -m pip install -U "watchdog[watchmedo]"
cmds:
- watchmedo shell-command --drop --patterns="*.p2s" --recursive --command='task analyze-ee-memory FILE="${watch_src_path}"' "{{.SAVESTATE_DIR}}"
vars:
SAVESTATE_DIR: '{{default "." .SAVESTATE_DIR}}'
type-search:
desc: Just an example to show it running
cmds:
- "{{.TYPESEARCH_BIN_RELEASE_DIR}}/type_searcher --output-path ./search-results.json --game {{.GAME}} --fields '[{\"type\":\"int16\",\"offset\":2},{\"type\":\"int16\",\"offset\":4}]'"
# TESTS
offline-tests: # ran by jenkins
cmds:
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --fail-on-cmp'
offline-test-file:
cmds:
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --file {{.FILE}}'
offline-tests-fast:
cmds:
- '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --pretty-print --num_threads 32 --dump_current_output --fail-on-cmp'
# TODO - amalgamate offline-tests and this task, run twice if the previous step fails
update-ref-tests:
cmds:
- cmd: python ./scripts/tasks/default-file-or-folder.py --path failures
- cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --dump_current_output'
- cmd: python ./scripts/tasks/delete-file-or-folder.py --path failures
- cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --game {{.GAME}} --pretty-print --num_threads 32 --dump_current_output --fail-on-cmp'
ignore_error: true
- python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/
- task: offline-tests
# find-label-types:
# cmds:
# - python ./scripts/next-decomp-file.py --files "{{.FILES}}"
# - task: decomp
# - python ./scripts/find-label-types.py --file "{{.FILES}}"
# check-gsrc-file:
# cmds:
# - python ./scripts/check-gsrc-file.py --files "{{.FILES}}"
- python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/ --game {{.GAME}}
- task: offline-tests-fast
update-ref-file:
cmds:
- cmd: python ./scripts/tasks/delete-file-or-folder.py --path failures
- cmd: '{{.OFFLINETEST_BIN_RELEASE_DIR}}/offline-test --iso_data_path "./iso_data/{{.GAME}}" --file {{.FILE}} --game {{.GAME}} --dump_current_output --fail-on-cmp'
ignore_error: true
- python ./scripts/update_decomp_reference.py ./failures ./test/decompiler/reference/ --game {{.GAME}}
- task: offline-test-file
type-test-jak1:
cmds:
- cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*Jak1TypeConsistency*" --gtest_break_on_failure'
type-test:
cmds:
- cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*MANUAL_TEST_TypeConsistencyWithBuildFirst*"'
ignore_error: true
- cmd: '{{.GOALCTEST_BIN_RELEASE_DIR}}/goalc-test --gtest_brief=0 --gtest_filter="*Jak2TypeConsistency*" --gtest_break_on_failure'
common/CMakeLists.txt
+10 -5
View File
@@ -19,11 +19,12 @@ add_library(common
goos/PrettyPrinter2.cpp
goos/Reader.cpp
goos/TextDB.cpp
goos/ReplUtils.cpp
repl/config.cpp
repl/util.cpp
log/log.cpp
math/geometry.cpp
nrepl/ReplClient.cpp
nrepl/ReplServer.cpp
repl/nrepl/ReplClient.cpp
repl/nrepl/ReplServer.cpp
type_system/defenum.cpp
type_system/deftype.cpp
type_system/state.cpp
@@ -42,20 +43,24 @@ add_library(common
util/DgoWriter.cpp
util/diff.cpp
util/FileUtil.cpp
util/FontUtils.cpp
util/json_util.cpp
util/read_iso_file.cpp
util/SimpleThreadGroup.cpp
util/string_util.cpp
util/Timer.cpp
util/os.cpp
util/print_float.cpp
util/FontUtils.cpp
util/FrameLimiter.cpp
util/unicode_util.cpp)
util/unicode_util.cpp
util/term_util.cpp)
target_link_libraries(common fmt lzokay replxx libzstd_static)
if(WIN32)
target_link_libraries(common wsock32 ws2_32 windowsapp)
elseif(APPLE)
# don't need anything special
else()
target_link_libraries(common stdc++fs)
endif()
common/audio/audio_formats.cpp
+6 -5
View File
@@ -1,5 +1,6 @@
#include "audio_formats.h"
#include "common/log/log.h"
#include "common/util/BinaryWriter.h"
#include "third-party/fmt/core.h"
@@ -254,7 +255,7 @@ void test_encode_adpcm(const std::vector<s16>& samples,
}
if (debug) {
fmt::print("Range: {}\n", max_sample - min_sample);
lg::debug("Range: {}", max_sample - min_sample);
}
// see how many bits we need and pick shift.
@@ -283,11 +284,11 @@ void test_encode_adpcm(const std::vector<s16>& samples,
if (filter_errors[best_filter] || best_filter != filter_debug[block_idx] ||
best_shift != shift_debug[block_idx]) {
fmt::print("Block {} me {}, {} : answer {} {}: ERR {}\n", block_idx, best_filter, best_shift,
filter_debug[block_idx], shift_debug[block_idx], filter_errors[best_filter]);
fmt::print("filter errors:\n");
lg::error("Block {} me {}, {} : answer {} {}: ERR {}", block_idx, best_filter, best_shift,
filter_debug[block_idx], shift_debug[block_idx], filter_errors[best_filter]);
lg::error("filter errors:");
for (int i = 0; i < 5; i++) {
fmt::print(" [{}] {} {}\n", i, filter_errors[i], filter_shifts[i]);
lg::error(" [{}] {} {}", i, filter_errors[i], filter_shifts[i]);
}
ASSERT_MSG(false, fmt::format("prev: {} {}", prev_block_samples[0], prev_block_samples[1]));
}
common/common_types.h
+4
View File
@@ -30,3 +30,7 @@ struct u128 {
};
};
static_assert(sizeof(u128) == 16, "u128");
#if defined __linux || defined __linux__ || defined __APPLE__
#define OS_POSIX
#endif
common/cross_os_debug/xdbg.cpp
+56 -6
View File
@@ -471,12 +471,14 @@ bool check_stopped(const ThreadID& tid, SignalInfo* out) {
{
auto exc = debugEvent.u.Exception.ExceptionRecord.ExceptionCode;
if (is_other) {
if (exc == EXCEPTION_BREAKPOINT) {
out->kind = SignalInfo::BREAK;
} else {
// ignore exceptions outside goal thread
ignore_debug_event();
}
ContinueDebugEvent(debugEvent.dwProcessId, debugEvent.dwThreadId,
DBG_EXCEPTION_NOT_HANDLED);
// if (exc == EXCEPTION_BREAKPOINT) {
// out->kind = SignalInfo::BREAK;
// } else {
// // ignore exceptions outside goal thread
// ignore_debug_event();
// }
} else {
switch (exc) {
case EXCEPTION_BREAKPOINT:
@@ -688,6 +690,54 @@ bool set_regs_now(const ThreadID& tid, const Regs& out) {
// todo, set fprs.
return true;
}
#elif __APPLE__
ThreadID::ThreadID(const std::string& str) {}
std::string ThreadID::to_string() const {
return "invalid";
}
ThreadID get_current_thread_id();
bool attach_and_break(const ThreadID& tid);
void allow_debugging();
bool detach_and_resume(const ThreadID& tid) {
return false;
}
bool get_regs_now(const ThreadID& tid, Regs* out) {
return false;
}
bool set_regs_now(const ThreadID& tid, const Regs& in) {
return false;
}
bool break_now(const ThreadID& tid) {
return false;
}
bool cont_now(const ThreadID& tid) {
return false;
}
bool open_memory(const ThreadID& tid, MemoryHandle* out);
bool close_memory(const ThreadID& tid, MemoryHandle* handle) {
return false;
}
bool read_goal_memory(u8* dest_buffer,
int size,
u32 goal_addr,
const DebugContext& context,
const MemoryHandle& mem) {
return false;
}
bool write_goal_memory(const u8* src_buffer,
int size,
u32 goal_addr,
const DebugContext& context,
const MemoryHandle& mem) {
return false;
}
bool check_stopped(const ThreadID& tid, SignalInfo* out) {
return false;
}
#endif
const char* gpr_names[] = {"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
common/cross_os_debug/xdbg.h
+1 -1
View File
@@ -20,7 +20,7 @@
#endif
namespace xdbg {
#ifdef __linux
#ifdef OS_POSIX
/*!
* Identification for a thread.
common/cross_sockets/XSocket.cpp
+18 -14
View File
@@ -4,7 +4,8 @@
*/
// clang-format off
#ifdef __linux
#include "common/common_types.h"
#ifdef OS_POSIX
#include <netinet/tcp.h>
#include <sys/socket.h>
#include <unistd.h>
@@ -18,10 +19,11 @@
#include <string.h>
#include "third-party/fmt/core.h"
#include "common/log/log.h"
// clang-format on
int open_socket(int af, int type, int protocol) {
#ifdef __linux
#ifdef OS_POSIX
return socket(af, type, protocol);
#elif _WIN32
WSADATA wsaData = {0};
@@ -29,7 +31,7 @@ int open_socket(int af, int type, int protocol) {
// Initialize Winsock
iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (iResult != 0) {
printf("WSAStartup failed: %d\n", iResult);
lg::error("WSAStartup failed: {}", iResult);
return 1;
}
return socket(af, type, protocol);
@@ -44,7 +46,7 @@ int connect_socket(int socket, sockaddr* addr, int nameLen) {
return result;
}
#ifdef __linux
#ifdef OS_POSIX
int accept_socket(int socket, sockaddr* addr, socklen_t* addrLen) {
return accept(socket, addr, addrLen);
}
@@ -71,7 +73,7 @@ int accept_socket(int socket, sockaddr* addr, int* addrLen) {
// Initialize Winsock
iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (iResult != 0) {
printf("WSAStartup failed: %d\n", iResult);
lg::error("WSAStartup failed: {}", iResult);
return 1;
}
return accept(socket, addr, addrLen);
@@ -105,7 +107,7 @@ void close_socket(int sock) {
if (sock < 0) {
return;
}
#ifdef __linux
#ifdef OS_POSIX
close(sock);
#elif _WIN32
closesocket(sock);
@@ -116,20 +118,20 @@ void close_socket(int sock) {
int set_socket_option(int socket, int level, int optname, const void* optval, int optlen) {
int ret = setsockopt(socket, level, optname, (const char*)optval, optlen);
if (ret < 0) {
printf("Failed to setsockopt(%d, %d, %d, _, _) - Error: %s\n", socket, level, optname,
strerror(errno));
lg::error("Failed to setsockopt({},{}, {}, _, _) - Error: {}", socket, level, optname,
strerror(errno));
}
#ifdef _WIN32
if (ret < 0) {
int err = WSAGetLastError();
printf("WSAGetLastError: %d\n", err);
lg::error("WSAGetLastError: {}", err);
}
#endif
return ret;
}
int set_socket_timeout(int socket, long microSeconds) {
#ifdef __linux
#ifdef OS_POSIX
struct timeval timeout = {};
timeout.tv_sec = 0;
timeout.tv_usec = microSeconds;
@@ -141,25 +143,27 @@ int set_socket_timeout(int socket, long microSeconds) {
return ret;
#elif _WIN32
DWORD timeout = microSeconds / 1000; // milliseconds
// TODO - NOTE this might be bad / unreliable if the socket ends up being in a bad state
// select() instead should be used, will do so if ends up being an issue in practice
return set_socket_option(socket, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
#endif
}
int write_to_socket(int socket, const char* buf, int len) {
int bytes_wrote = 0;
#ifdef __linux
#ifdef OS_POSIX
bytes_wrote = send(socket, buf, len, MSG_NOSIGNAL);
#elif _WIN32
bytes_wrote = send(socket, buf, len, 0);
#endif
if (bytes_wrote < 0) {
fmt::print(stderr, "[XSocket:{}] Error writing to socket\n", socket);
lg::error("[XSocket:{}] Error writing to socket", socket);
}
return bytes_wrote;
}
int read_from_socket(int socket, char* buf, int len) {
#ifdef __linux
#ifdef OS_POSIX
return read(socket, buf, len);
#elif _WIN32
return recv(socket, buf, len, 0);
@@ -167,7 +171,7 @@ int read_from_socket(int socket, char* buf, int len) {
}
bool socket_timed_out() {
#ifdef __linux
#ifdef OS_POSIX
return errno == EAGAIN;
#elif _WIN32
auto err = WSAGetLastError();
common/cross_sockets/XSocket.h
+5 -2
View File
@@ -6,7 +6,8 @@
*/
// clang-format off
#ifdef __linux
#include "common/common_types.h"
#ifdef OS_POSIX
#include <arpa/inet.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
@@ -24,11 +25,13 @@
const int TCP_SOCKET_LEVEL = SOL_TCP;
#elif _WIN32
const int TCP_SOCKET_LEVEL = IPPROTO_TCP;
#elif __APPLE__
const int TCP_SOCKET_LEVEL = IPPROTO_TCP;
#endif
int open_socket(int af, int type, int protocol);
int connect_socket(int socket, sockaddr* addr, int nameLen);
#ifdef __linux
#ifdef OS_POSIX
int accept_socket(int socket, sockaddr* addr, socklen_t* addrLen);
int select_and_accept_socket(int socket, sockaddr* addr, socklen_t* addrLen, int microSeconds);
#elif _WIN32
common/cross_sockets/XSocketClient.cpp
+1 -1
View File
@@ -12,7 +12,7 @@
#include <WinSock2.h>
#include <WS2tcpip.h>
#endif
#include "common/nrepl/ReplServer.h"
#include "common/repl/nrepl/ReplServer.h"
#include "third-party/fmt/core.h"
// clang-format on
common/cross_sockets/XSocketServer.cpp
+6 -4
View File
@@ -2,6 +2,7 @@
#include "XSocketServer.h"
#include "common/cross_sockets/XSocket.h"
#include "common/common_types.h"
#include "third-party/fmt/core.h"
@@ -12,6 +13,7 @@
#include <WinSock2.h>
#include <WS2tcpip.h>
#endif
#include "common/log/log.h"
// clang-format on
XSocketServer::XSocketServer(std::function<bool()> shutdown_callback,
@@ -40,7 +42,7 @@ bool XSocketServer::init_server() {
return false;
}
#ifdef __linux
#ifdef OS_POSIX
int server_socket_opt = SO_REUSEADDR | SO_REUSEPORT;
#elif _WIN32
int server_socket_opt = SO_EXCLUSIVEADDRUSE;
@@ -67,19 +69,19 @@ bool XSocketServer::init_server() {
addr.sin_port = htons(tcp_port);
if (bind(listening_socket, (sockaddr*)&addr, sizeof(addr)) < 0) {
fmt::print("[XSocketServer:{}] failed to bind\n", tcp_port);
lg::error("[XSocketServer:{}] failed to bind", tcp_port);
close_server_socket();
return false;
}
if (listen(listening_socket, 0) < 0) {
fmt::print("[XSocketServer:{}] failed to listen\n", tcp_port);
lg::error("[XSocketServer:{}] failed to listen", tcp_port);
close_server_socket();
return false;
}
server_initialized = true;
fmt::print("[XSocketServer:{}] initialized\n", tcp_port);
lg::info("[XSocketServer:{}] initialized", tcp_port);
post_init();
return true;
}
common/cross_sockets/XSocketServer.h
+1
View File
@@ -3,6 +3,7 @@
#include <functional>
#include <mutex>
#include <thread>
#include <vector>
#include "common/common_types.h"
#include "common/cross_sockets/XSocket.h"
common/custom_data/TFrag3Data.cpp
+187 -95
View File
@@ -209,6 +209,9 @@ void TieTree::serialize(Serializer& ser) {
packed_vertices.serialize(ser);
ser.from_pod_vector(&colors);
bvh.serialize(ser);
ser.from_ptr(&has_per_proto_visibility_toggle);
ser.from_string_vector(&proto_names);
}
void ShrubTree::serialize(Serializer& ser) {
@@ -256,15 +259,46 @@ void MercDraw::serialize(Serializer& ser) {
ser.from_ptr(&num_triangles);
}
void MercEffect::serialize(Serializer& ser) {
void MercModifiableDrawGroup::serialize(Serializer& ser) {
if (ser.is_saving()) {
ser.save<size_t>(draws.size());
ser.save<size_t>(mod_draw.size());
} else {
draws.resize(ser.load<size_t>());
mod_draw.resize(ser.load<size_t>());
}
for (auto& draw : draws) {
for (auto& draw : mod_draw) {
draw.serialize(ser);
}
if (ser.is_saving()) {
ser.save<size_t>(fix_draw.size());
} else {
fix_draw.resize(ser.load<size_t>());
}
for (auto& draw : fix_draw) {
draw.serialize(ser);
}
ser.from_pod_vector(&vertices);
ser.from_pod_vector(&vertex_lump4_addr);
ser.from_pod_vector(&fragment_mask);
ser.from_ptr(&expect_vidx_end);
}
void MercEffect::serialize(Serializer& ser) {
if (ser.is_saving()) {
ser.save<size_t>(all_draws.size());
} else {
all_draws.resize(ser.load<size_t>());
}
for (auto& draw : all_draws) {
draw.serialize(ser);
}
mod.serialize(ser);
ser.from_ptr(&envmap_mode);
ser.from_ptr(&envmap_texture);
ser.from_ptr(&has_envmap);
ser.from_ptr(&has_mod_draw);
}
void MercModel::serialize(Serializer& ser) {
@@ -279,6 +313,8 @@ void MercModel::serialize(Serializer& ser) {
}
ser.from_ptr(&max_draws);
ser.from_ptr(&max_bones);
ser.from_ptr(&st_vif_add);
ser.from_ptr(&xyz_scale);
}
void MercModelGroup::serialize(Serializer& ser) {
@@ -356,109 +392,163 @@ void Level::serialize(Serializer& ser) {
}
}
std::array<int, MemoryUsageCategory::NUM_CATEGORIES> Level::get_memory_usage() const {
std::array<int, MemoryUsageCategory::NUM_CATEGORIES> result;
result.fill(0);
void MercModifiableDrawGroup::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::MERC_MOD_VERT, sizeof(MercVertex) * vertices.size());
tracker->add(MemoryUsageCategory::MERC_MOD_DRAW_1, sizeof(MercDraw) * fix_draw.size());
tracker->add(MemoryUsageCategory::MERC_MOD_DRAW_2, sizeof(MercDraw) * mod_draw.size());
tracker->add(MemoryUsageCategory::MERC_MOD_TABLE, sizeof(u16) * vertex_lump4_addr.size());
}
// textures
for (const auto& tex : textures) {
result[TEXTURE] += tex.data.size() * sizeof(u32);
void MercEffect::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::MERC_DRAW, sizeof(MercDraw) * all_draws.size());
mod.memory_usage(tracker);
}
void MercModel::memory_usage(MemoryUsageTracker* tracker) const {
for (auto& effect : effects) {
effect.memory_usage(tracker);
}
}
// tfrag
for (const auto& tfrag_tree_geoms : tfrag_trees) {
for (const auto& tfrag_tree : tfrag_tree_geoms) {
for (const auto& draw : tfrag_tree.draws) {
result[TFRAG_INDEX] += draw.runs.size() * sizeof(StripDraw::VertexRun);
result[TFRAG_INDEX] += draw.plain_indices.size() * sizeof(u32);
result[TFRAG_VIS] += draw.vis_groups.size() * sizeof(StripDraw::VisGroup);
}
result[TFRAG_VERTS] +=
tfrag_tree.packed_vertices.vertices.size() * sizeof(PackedTfragVertices::Vertex);
result[TFRAG_CLUSTER] +=
tfrag_tree.packed_vertices.cluster_origins.size() * sizeof(math::Vector<u16, 3>);
result[TFRAG_TIME_OF_DAY] += tfrag_tree.colors.size() * sizeof(TimeOfDayColor);
result[TFRAG_BVH] += tfrag_tree.bvh.vis_nodes.size() * sizeof(VisNode);
void MercModelGroup::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::MERC_VERT, sizeof(MercVertex) * vertices.size());
tracker->add(MemoryUsageCategory::MERC_INDEX, sizeof(u32) * indices.size());
for (auto& model : models) {
model.memory_usage(tracker);
}
}
void CollisionMesh::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::COLLISION, sizeof(Vertex) * vertices.size());
}
void PackedShrubVertices::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::SHRUB_VERT, 64 * matrices.size());
tracker->add(MemoryUsageCategory::SHRUB_VERT, sizeof(InstanceGroup) * instance_groups.size());
tracker->add(MemoryUsageCategory::SHRUB_VERT, sizeof(Vertex) * vertices.size());
}
void ShrubTree::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::SHRUB_TIME_OF_DAY,
sizeof(TimeOfDayColor) * time_of_day_colors.size());
packed_vertices.memory_usage(tracker);
tracker->add(MemoryUsageCategory::SHRUB_DRAW, sizeof(ShrubDraw) * static_draws.size());
tracker->add(MemoryUsageCategory::SHRUB_IND, sizeof(u32) * indices.size());
}
void InstancedStripDraw::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::TIE_INST_INDEX, sizeof(u32) * vertex_index_stream.size());
tracker->add(MemoryUsageCategory::TIE_INST_VIS, sizeof(InstanceGroup) * instance_groups.size());
}
void PackedTieVertices::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::TIE_CIDX, sizeof(u16) * color_indices.size());
tracker->add(MemoryUsageCategory::TIE_MATRICES, 64 * matrices.size());
tracker->add(MemoryUsageCategory::TIE_GRPS, sizeof(MatrixGroup) * matrix_groups.size());
tracker->add(MemoryUsageCategory::TIE_VERTS, sizeof(Vertex) * vertices.size());
}
void TieTree::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::TIE_BVH, sizeof(VisNode) * bvh.vis_nodes.size());
for (auto& draw : static_draws) {
tracker->add(MemoryUsageCategory::TIE_DEINST_INDEX,
draw.runs.size() * sizeof(StripDraw::VertexRun));
tracker->add(MemoryUsageCategory::TIE_DEINST_INDEX, draw.plain_indices.size() * sizeof(u32));
tracker->add(MemoryUsageCategory::TIE_DEINST_VIS,
draw.vis_groups.size() * sizeof(StripDraw::VisGroup));
}
packed_vertices.memory_usage(tracker);
tracker->add(MemoryUsageCategory::TIE_TIME_OF_DAY, sizeof(TimeOfDayColor) * colors.size());
for (auto& draw : instanced_wind_draws) {
draw.memory_usage(tracker);
}
tracker->add(MemoryUsageCategory::TIE_WIND_INSTANCE_INFO,
sizeof(TieWindInstance) * wind_instance_info.size());
}
void PackedTfragVertices::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::TFRAG_VERTS,
sizeof(PackedTfragVertices::Vertex) * vertices.size());
tracker->add(MemoryUsageCategory::TFRAG_CLUSTER,
sizeof(math::Vector<u16, 3>) * cluster_origins.size());
}
void TfragTree::memory_usage(MemoryUsageTracker* tracker) const {
for (auto& draw : draws) {
tracker->add(MemoryUsageCategory::TFRAG_INDEX, draw.runs.size() * sizeof(StripDraw::VertexRun));
tracker->add(MemoryUsageCategory::TFRAG_INDEX, draw.plain_indices.size() * sizeof(u32));
tracker->add(MemoryUsageCategory::TFRAG_VIS,
draw.vis_groups.size() * sizeof(StripDraw::VisGroup));
}
packed_vertices.memory_usage(tracker);
tracker->add(MemoryUsageCategory::TFRAG_TIME_OF_DAY, sizeof(TimeOfDayColor) * colors.size());
tracker->add(MemoryUsageCategory::TFRAG_BVH, sizeof(VisNode) * bvh.vis_nodes.size());
}
void Texture::memory_usage(MemoryUsageTracker* tracker) const {
tracker->add(MemoryUsageCategory::TEXTURE, data.size() * sizeof(u32));
}
void Level::memory_usage(MemoryUsageTracker* tracker) const {
for (const auto& texture : textures) {
texture.memory_usage(tracker);
}
for (const auto& tftk : tfrag_trees) {
for (const auto& tree : tftk) {
tree.memory_usage(tracker);
}
}
// tie
for (const auto& tie_tree_geoms : tie_trees) {
for (const auto& tie_tree : tie_tree_geoms) {
result[TIE_BVH] += tie_tree.bvh.vis_nodes.size();
for (const auto& draw : tie_tree.static_draws) {
result[TIE_DEINST_INDEX] += draw.runs.size() * sizeof(StripDraw::VertexRun);
result[TIE_DEINST_VIS] += draw.vis_groups.size() * sizeof(StripDraw::VisGroup);
}
result[TIE_VERTS] +=
tie_tree.packed_vertices.vertices.size() * sizeof(PackedTieVertices::Vertex);
result[TIE_CIDX] += tie_tree.packed_vertices.color_indices.size() * sizeof(u16);
result[TIE_MATRICES] += tie_tree.packed_vertices.matrices.size() * 4 * 4 * 4;
result[TIE_GRPS] +=
tie_tree.packed_vertices.matrix_groups.size() * sizeof(PackedTieVertices::MatrixGroup);
result[TIE_TIME_OF_DAY] += tie_tree.colors.size() * sizeof(TimeOfDayColor);
for (const auto& draw : tie_tree.instanced_wind_draws) {
result[TIE_INST_INDEX] += draw.vertex_index_stream.size() * sizeof(u32);
result[TIE_INST_VIS] +=
draw.instance_groups.size() * sizeof(InstancedStripDraw::InstanceGroup);
}
result[TIE_WIND_INSTANCE_INFO] +=
tie_tree.wind_instance_info.size() * sizeof(TieWindInstance);
for (const auto& ttk : tie_trees) {
for (const auto& tree : ttk) {
tree.memory_usage(tracker);
}
}
// shrub
for (const auto& shrub_tree : shrub_trees) {
result[SHRUB_TIME_OF_DAY] += shrub_tree.time_of_day_colors.size() * sizeof(TimeOfDayColor);
result[SHRUB_VERT] += shrub_tree.packed_vertices.matrices.size() * 4 * 4 * 4;
result[SHRUB_VERT] +=
shrub_tree.packed_vertices.vertices.size() * sizeof(PackedShrubVertices::Vertex);
result[SHRUB_VERT] += shrub_tree.packed_vertices.instance_groups.size() *
sizeof(PackedShrubVertices::InstanceGroup);
result[SHRUB_IND] += sizeof(u32) * shrub_tree.indices.size();
for (const auto& tree : shrub_trees) {
tree.memory_usage(tracker);
}
// merc
result[MERC_INDEX] += merc_data.indices.size() * sizeof(u32);
result[MERC_VERT] += merc_data.vertices.size() * sizeof(MercVertex);
// collision
result[COLLISION] += sizeof(CollisionMesh::Vertex) * collision.vertices.size();
return result;
collision.memory_usage(tracker);
merc_data.memory_usage(tracker);
}
void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size) {
int total_accounted = 0;
auto memory_use_by_category = lev.get_memory_usage();
MemoryUsageTracker mem_use;
lev.memory_usage(&mem_use);
std::vector<std::pair<std::string, int>> known_categories = {
{"texture", memory_use_by_category[tfrag3::MemoryUsageCategory::TEXTURE]},
{"tie-deinst-vis", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_DEINST_VIS]},
{"tie-deinst-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_DEINST_INDEX]},
{"tie-inst-vis", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_INST_VIS]},
{"tie-inst-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_INST_INDEX]},
{"tie-bvh", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_BVH]},
{"tie-verts", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_VERTS]},
{"tie-colors", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_TIME_OF_DAY]},
{"tie-wind-inst-info",
memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_WIND_INSTANCE_INFO]},
{"tie-cidx", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_CIDX]},
{"tie-mats", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_MATRICES]},
{"tie-grps", memory_use_by_category[tfrag3::MemoryUsageCategory::TIE_GRPS]},
{"tfrag-vis", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_VIS]},
{"tfrag-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_INDEX]},
{"tfrag-vert", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_VERTS]},
{"tfrag-colors", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_TIME_OF_DAY]},
{"tfrag-cluster", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_CLUSTER]},
{"tfrag-bvh", memory_use_by_category[tfrag3::MemoryUsageCategory::TFRAG_BVH]},
{"shrub-colors", memory_use_by_category[tfrag3::MemoryUsageCategory::SHRUB_TIME_OF_DAY]},
{"shrub-vert", memory_use_by_category[tfrag3::MemoryUsageCategory::SHRUB_VERT]},
{"shrub-ind", memory_use_by_category[tfrag3::MemoryUsageCategory::SHRUB_IND]},
{"collision", memory_use_by_category[tfrag3::MemoryUsageCategory::COLLISION]},
{"merc-vert", memory_use_by_category[tfrag3::MemoryUsageCategory::MERC_VERT]},
{"merc-idx", memory_use_by_category[tfrag3::MemoryUsageCategory::MERC_INDEX]}};
{"texture", mem_use.data[tfrag3::MemoryUsageCategory::TEXTURE]},
{"tie-deinst-vis", mem_use.data[tfrag3::MemoryUsageCategory::TIE_DEINST_VIS]},
{"tie-deinst-idx", mem_use.data[tfrag3::MemoryUsageCategory::TIE_DEINST_INDEX]},
{"tie-inst-vis", mem_use.data[tfrag3::MemoryUsageCategory::TIE_INST_VIS]},
{"tie-inst-idx", mem_use.data[tfrag3::MemoryUsageCategory::TIE_INST_INDEX]},
{"tie-bvh", mem_use.data[tfrag3::MemoryUsageCategory::TIE_BVH]},
{"tie-verts", mem_use.data[tfrag3::MemoryUsageCategory::TIE_VERTS]},
{"tie-colors", mem_use.data[tfrag3::MemoryUsageCategory::TIE_TIME_OF_DAY]},
{"tie-wind-inst-info", mem_use.data[tfrag3::MemoryUsageCategory::TIE_WIND_INSTANCE_INFO]},
{"tie-cidx", mem_use.data[tfrag3::MemoryUsageCategory::TIE_CIDX]},
{"tie-mats", mem_use.data[tfrag3::MemoryUsageCategory::TIE_MATRICES]},
{"tie-grps", mem_use.data[tfrag3::MemoryUsageCategory::TIE_GRPS]},
{"tfrag-vis", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_VIS]},
{"tfrag-idx", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_INDEX]},
{"tfrag-vert", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_VERTS]},
{"tfrag-colors", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_TIME_OF_DAY]},
{"tfrag-cluster", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_CLUSTER]},
{"tfrag-bvh", mem_use.data[tfrag3::MemoryUsageCategory::TFRAG_BVH]},
{"shrub-colors", mem_use.data[tfrag3::MemoryUsageCategory::SHRUB_TIME_OF_DAY]},
{"shrub-vert", mem_use.data[tfrag3::MemoryUsageCategory::SHRUB_VERT]},
{"shrub-ind", mem_use.data[tfrag3::MemoryUsageCategory::SHRUB_IND]},
{"shrub-draw", mem_use.data[tfrag3::MemoryUsageCategory::SHRUB_DRAW]},
{"collision", mem_use.data[tfrag3::MemoryUsageCategory::COLLISION]},
{"merc-vert", mem_use.data[tfrag3::MemoryUsageCategory::MERC_VERT]},
{"merc-idx", mem_use.data[tfrag3::MemoryUsageCategory::MERC_INDEX]},
{"merc-draw", mem_use.data[tfrag3::MemoryUsageCategory::MERC_DRAW]},
{"merc-mod-vert", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_VERT]},
{"merc-mod-ind", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_IND]},
{"merc-mod-table", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_TABLE]},
{"merc-mod-draw-1", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_DRAW_1]},
{"merc-mod-draw-2", mem_use.data[tfrag3::MemoryUsageCategory::MERC_MOD_DRAW_2]},
};
for (auto& known : known_categories) {
total_accounted += known.second;
}
@@ -469,8 +559,10 @@ void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size) {
[](const auto& a, const auto& b) { return a.second > b.second; });
for (const auto& x : known_categories) {
fmt::print("{:30s} : {:6d} kB {:3.1f}%\n", x.first, x.second / 1024,
100.f * (float)x.second / uncompressed_data_size);
if (x.second) {
fmt::print("{:30s} : {:6d} kB {:3.1f}%\n", x.first, x.second / 1024,
100.f * (float)x.second / uncompressed_data_size);
}
}
}
common/custom_data/Tfrag3Data.h
+62 -11
View File
@@ -44,16 +44,36 @@ enum MemoryUsageCategory {
SHRUB_TIME_OF_DAY,
SHRUB_VERT,
SHRUB_IND,
SHRUB_DRAW,
MERC_VERT,
MERC_INDEX,
MERC_DRAW,
MERC_MOD_DRAW_1,
MERC_MOD_DRAW_2,
MERC_MOD_VERT,
MERC_MOD_IND,
MERC_MOD_TABLE,
COLLISION,
NUM_CATEGORIES
};
constexpr int TFRAG3_VERSION = 21;
struct MemoryUsageTracker {
u32 data[MemoryUsageCategory::NUM_CATEGORIES];
MemoryUsageTracker() {
for (auto& x : data) {
x = 0;
}
}
void add(MemoryUsageCategory category, u32 size_bytes) { data[category] += size_bytes; }
};
constexpr int TFRAG3_VERSION = 25;
// These vertices should be uploaded to the GPU at load time and don't change
struct PreloadedVertex {
@@ -93,6 +113,7 @@ struct PackedTieVertices {
std::vector<MatrixGroup> matrix_groups; // todo pack
std::vector<Vertex> vertices;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
};
struct PackedTfragVertices {
@@ -102,7 +123,7 @@ struct PackedTfragVertices {
s16 s, t;
u16 color_index;
};
void memory_usage(MemoryUsageTracker* tracker) const;
std::vector<Vertex> vertices;
std::vector<math::Vector<u16, 3>> cluster_origins;
};
@@ -135,7 +156,7 @@ struct PackedShrubVertices {
std::vector<InstanceGroup> instance_groups; // todo pack
std::vector<Vertex> vertices;
u32 total_vertex_count;
void memory_usage(MemoryUsageTracker* tracker) const;
void serialize(Serializer& ser);
};
@@ -167,7 +188,8 @@ struct StripDraw {
struct VisGroup {
u32 num_inds = 0; // number of vertex indices in this group
u32 num_tris = 0; // number of triangles
u32 vis_idx_in_pc_bvh = 0; // the visibility group they belong to (in BVH)
u16 vis_idx_in_pc_bvh = 0; // the visibility group they belong to (in BVH)
u16 tie_proto_idx = 0; // index of tie proto (tie only)
};
std::vector<VisGroup> vis_groups;
@@ -207,6 +229,7 @@ struct InstancedStripDraw {
// for debug counting.
u32 num_triangles = 0;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
};
// node in the BVH.
@@ -256,13 +279,14 @@ struct Texture {
std::string debug_tpage_name;
bool load_to_pool = false;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
};
// Tfrag trees have several kinds:
enum class TFragmentTreeKind { NORMAL, TRANS, DIRT, ICE, LOWRES, LOWRES_TRANS, INVALID };
enum class TFragmentTreeKind { NORMAL, TRANS, DIRT, ICE, LOWRES, LOWRES_TRANS, WATER, INVALID };
constexpr const char* tfrag_tree_names[] = {"normal", "trans", "dirt", "ice",
"lowres", "lowres-trans", "invalid"};
constexpr const char* tfrag_tree_names[] = {"normal", "trans", "dirt", "ice",
"lowres", "lowres-trans", "water", "invalid"};
// A tfrag model
struct TfragTree {
@@ -279,6 +303,7 @@ struct TfragTree {
} unpacked;
void unpack();
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
};
struct TieWindInstance {
@@ -299,12 +324,17 @@ struct TieTree {
std::vector<InstancedStripDraw> instanced_wind_draws;
std::vector<TieWindInstance> wind_instance_info;
// jak 2 and later can toggle on and off visibility per proto by name
bool has_per_proto_visibility_toggle = false;
std::vector<std::string> proto_names;
struct {
std::vector<PreloadedVertex> vertices; // mesh vertices
std::vector<u32> indices;
} unpacked;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
void unpack();
};
@@ -321,6 +351,7 @@ struct ShrubTree {
} unpacked;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
void unpack();
};
@@ -336,6 +367,7 @@ struct CollisionMesh {
static_assert(sizeof(Vertex) == 32);
std::vector<Vertex> vertices;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
};
// MERC
@@ -368,9 +400,25 @@ struct MercDraw {
void serialize(Serializer& ser);
};
struct MercEffect {
std::vector<MercDraw> draws;
struct MercModifiableDrawGroup {
std::vector<MercVertex> vertices;
std::vector<u16> vertex_lump4_addr;
std::vector<MercDraw> fix_draw, mod_draw;
std::vector<u8> fragment_mask;
u32 expect_vidx_end = 0;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
};
struct MercEffect {
std::vector<MercDraw> all_draws;
MercModifiableDrawGroup mod;
DrawMode envmap_mode;
u32 envmap_texture;
bool has_envmap = false;
bool has_mod_draw = false;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
};
struct MercModel {
@@ -378,7 +426,10 @@ struct MercModel {
std::vector<MercEffect> effects;
u32 max_draws;
u32 max_bones;
u32 st_vif_add;
float xyz_scale;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
};
struct MercModelGroup {
@@ -386,6 +437,7 @@ struct MercModelGroup {
std::vector<u32> indices;
std::vector<MercModel> models;
void serialize(Serializer& ser);
void memory_usage(MemoryUsageTracker* tracker) const;
};
//
@@ -404,8 +456,7 @@ struct Level {
MercModelGroup merc_data;
u16 version2 = TFRAG3_VERSION;
void serialize(Serializer& ser);
std::array<int, MemoryUsageCategory::NUM_CATEGORIES> get_memory_usage() const;
void memory_usage(MemoryUsageTracker* tracker) const;
};
void print_memory_usage(const tfrag3::Level& lev, int uncompressed_data_size);
common/custom_data/pack_helpers.cpp
+4 -2
View File
@@ -2,13 +2,15 @@
#include <map>
#include "common/log/log.h"
constexpr float kClusterSize = 4096 * 40; // 100 in-game meters
constexpr float kMasterOffset = 12000 * 4096;
std::pair<u64, u16> position_to_cluster_and_offset(float in) {
in += kMasterOffset;
if (in < 0) {
fmt::print("negative: {}\n", in);
lg::print("negative: {}\n", in);
}
ASSERT(in >= 0);
int cluster_cell = (in / kClusterSize);
@@ -69,4 +71,4 @@ void pack_tfrag_vertices(tfrag3::PackedTfragVertices* result,
}
ASSERT(next_cluster_idx < UINT16_MAX);
}
}
common/dma/dma.h
+1
View File
@@ -113,6 +113,7 @@ struct VifCode {
ITOP = 0b100,
STMOD = 0b101,
PC_PORT = 0b1000, // not a valid PS2 VIF code, but we use this to signal PC-PORT specific stuff
PC_PORT2 = 0b1001,
MSK3PATH = 0b110,
MARK = 0b111,
FLUSHE = 0b10000,
common/dma/dma_copy.cpp
+12 -11
View File
@@ -3,6 +3,7 @@
#include "common/dma/dma_chain_read.h"
#include "common/goal_constants.h"
#include "common/log/log.h"
#include "common/util/Timer.h"
#include "third-party/fmt/core.h"
@@ -36,11 +37,11 @@ void diff_dma_chains(DmaFollower ref, DmaFollower dma) {
auto ref_tag = ref.current_tag();
auto dma_tag = dma.current_tag();
if (ref_tag.kind != dma_tag.kind) {
fmt::print("Bad dma tag kinds\n");
lg::warn("Bad dma tag kinds");
}
if (ref_tag.qwc != dma_tag.qwc) {
fmt::print("Bad dma tag qwc: {} {}\n", ref_tag.qwc, dma_tag.qwc);
lg::warn("Bad dma tag qwc: {} {}", ref_tag.qwc, dma_tag.qwc);
}
auto ref_result = ref.read_and_advance();
@@ -48,19 +49,19 @@ void diff_dma_chains(DmaFollower ref, DmaFollower dma) {
for (int i = 0; i < (int)ref_result.size_bytes; i++) {
if (ref_result.data[i] != dma_result.data[i]) {
fmt::print("Bad data ({} vs {}) at {} into transfer: {} {}\n", ref_result.data[i],
dma_result.data[i], i, ref_tag.print(), dma_tag.print());
lg::error("Bad data ({} vs {}) at {} into transfer: {} {}", ref_result.data[i],
dma_result.data[i], i, ref_tag.print(), dma_tag.print());
return;
}
}
}
if (!ref.ended()) {
fmt::print("dma ended early\n");
lg::warn("dma ended early");
}
if (!dma.ended()) {
fmt::print("dma had extra data\n");
lg::warn("dma had extra data");
}
}
@@ -175,12 +176,12 @@ const DmaData& FixedChunkDmaCopier::run(const void* memory, u32 offset, bool ver
auto v2 = flatten_dma(DmaFollower(m_result.data.data(), m_result.start_offset));
if (ref != v2) {
fmt::print("Verification has failed.\n");
fmt::print("size diff: {} {}\n", ref.size(), v2.size());
lg::error("Verification has failed.");
lg::error("size diff: {} {}", ref.size(), v2.size());
for (size_t i = 0; i < std::min(ref.size(), v2.size()); i++) {
if (ref[i] != v2[i]) {
fmt::print("first diff at {}\n", i);
lg::error("first diff at {}", i);
break;
}
}
@@ -188,10 +189,10 @@ const DmaData& FixedChunkDmaCopier::run(const void* memory, u32 offset, bool ver
DmaFollower(m_result.data.data(), m_result.start_offset));
ASSERT(false);
} else {
fmt::print("verification ok: {} bytes\n", ref.size());
lg::debug("verification ok: {} bytes", ref.size());
}
}
m_result.stats.sync_time_ms = timer.getMs();
return m_result;
}
}
common/dma/gs.cpp
+6 -3
View File
@@ -3,6 +3,7 @@
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/format.h"
std::string reg_descriptor_name(GifTag::RegisterDescriptor reg) {
switch (reg) {
@@ -352,15 +353,17 @@ std::string GsTexa::print() const {
std::string GsTex0::print() const {
return fmt::format(
"tbp0: {} tbw: {} psm: {} tw: {} th: {} tcc: {} tfx: {} cbp: {} cpsm: {} csm: {}\n", tbp0(),
tbw(), psm(), tw(), th(), tcc(), tfx(), cbp(), cpsm(), csm());
tbw(), fmt::underlying(psm()), tw(), th(), tcc(), fmt::underlying(tfx()), cbp(), cpsm(),
csm());
}
std::string GsPrim::print() const {
return fmt::format("0x{:x}, kind {}\n", data, kind());
return fmt::format("0x{:x}, kind {}\n", data, fmt::underlying(kind()));
}
std::string GsFrame::print() const {
return fmt::format("fbp: {} fbw: {} psm: {} fbmsk: {:x}\n", fbp(), fbw(), psm(), fbmsk());
return fmt::format("fbp: {} fbw: {} psm: {} fbmsk: {:x}\n", fbp(), fbw(), fmt::underlying(psm()),
fbmsk());
}
std::string GsXYOffset::print() const {
common/dma/gs.h
+1 -1
View File
@@ -424,7 +424,7 @@ class DrawMode {
SRC_DST_FIX_DST = 4, // fix = 64
ZERO_SRC_SRC_DST = 5,
SRC_SRC_SRC_SRC = 6,
SRC_0_DST_DST = 7
SRC_0_DST_DST = 7 // Note: requires color_mult tricks
};
enum class AlphaTest {
common/global_profiler/GlobalProfiler.cpp
+13 -12
View File
@@ -7,23 +7,25 @@
#include "common/util/Assert.h"
#include "common/util/FileUtil.h"
#include "common/common_types.h"
#include "third-party/fmt/core.h"
#include "third-party/json.hpp"
#ifdef __linux__
u32 get_current_tid() {
return (u32)pthread_self();
#ifdef OS_POSIX
u64 get_current_tid() {
return (u64)pthread_self();
}
#else
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#include "Processthreadsapi.h"
u32 get_current_tid() {
return (u32)GetCurrentThreadId();
u64 get_current_tid() {
return (u64)GetCurrentThreadId();
}
#endif
#include "common/log/log.h"
// clang-format on
u64 get_current_ts() {
@@ -157,18 +159,17 @@ void GlobalProfiler::dump_to_json(const std::string& path) {
// ts
json_event["ts"] = (event.ts - lowest_ts) / 1000.f;
if (event.ts < info.debug) {
fmt::print("out of order: {} {} {} ms\n", event.ts / 1000.f, info.debug / 1000.f,
(info.debug - event.ts) / 1000000.f);
fmt::print(" idx: {}, range {} {}\n", event_idx, info.lowest_at_target,
info.highest_at_target);
fmt::print(" now: {}\n", m_next_idx);
lg::debug("out of order: {} {} {} ms", event.ts / 1000.f, info.debug / 1000.f,
(info.debug - event.ts) / 1000000.f);
lg::debug(" idx: {}, range {} {}", event_idx, info.lowest_at_target, info.highest_at_target);
lg::debug(" now: {}", m_next_idx);
}
info.debug = event.ts;
}
for (auto& t : info_per_thread) {
fmt::print("thread: {}: {} -> {}\n", t.first, t.second.lowest_at_target,
t.second.highest_at_target);
lg::debug("thread: {}: {} -> {}", t.first, t.second.lowest_at_target,
t.second.highest_at_target);
}
file_util::write_text_file(path, json.dump());
common/global_profiler/GlobalProfiler.h
+2 -2
View File
@@ -8,9 +8,9 @@
struct ProfNode {
u64 ts;
char name[32];
u64 tid;
char name[128];
enum Kind : u8 { BEGIN, END, INSTANT, UNUSED } kind = UNUSED;
u32 tid;
};
class GlobalProfiler {
common/goos/Interpreter.cpp
+65 -2
View File
@@ -9,7 +9,10 @@
#include "ParseHelpers.h"
#include "common/goos/Printer.h"
#include "common/log/log.h"
#include "common/util/FileUtil.h"
#include "common/util/string_util.h"
#include "common/util/unicode_util.h"
#include "third-party/fmt/core.h"
@@ -52,6 +55,7 @@ Interpreter::Interpreter(const std::string& username) {
{"begin", &Interpreter::eval_begin},
{"exit", &Interpreter::eval_exit},
{"read", &Interpreter::eval_read},
{"read-data-file", &Interpreter::eval_read_data_file},
{"read-file", &Interpreter::eval_read_file},
{"print", &Interpreter::eval_print},
{"inspect", &Interpreter::eval_inspect},
@@ -81,6 +85,9 @@ Interpreter::Interpreter(const std::string& username) {
{"string-ref", &Interpreter::eval_string_ref},
{"string-length", &Interpreter::eval_string_length},
{"string-append", &Interpreter::eval_string_append},
{"string-starts-with?", &Interpreter::eval_string_starts_with},
{"string-ends-with?", &Interpreter::eval_string_ends_with},
{"string-split", &Interpreter::eval_string_split},
{"ash", &Interpreter::eval_ash},
{"symbol->string", &Interpreter::eval_symbol_to_string},
{"string->symbol", &Interpreter::eval_string_to_symbol},
@@ -159,7 +166,7 @@ HeapObject* Interpreter::intern_ptr(const std::string& name) {
/*!
* Display the REPL, which will run until the user executes exit.
*/
void Interpreter::execute_repl(ReplWrapper& repl) {
void Interpreter::execute_repl(REPL::Wrapper& repl) {
want_exit = false;
while (!want_exit) {
try {
@@ -1001,6 +1008,23 @@ Object Interpreter::eval_read(const Object& form,
return Object::make_empty_list();
}
/*!
* Reads list data from a file, returns the pair. Not a lot of safety here!
*/
Object Interpreter::eval_read_data_file(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env) {
(void)env;
vararg_check(form, args, {ObjectType::STRING}, {});
try {
return reader.read_from_file({args.unnamed.at(0).as_string()->data}).as_pair()->cdr;
} catch (std::runtime_error& e) {
throw_eval_error(form, std::string("reader error inside of read-file:\n") + e.what());
}
return Object::make_empty_list();
}
/*!
* Open and run the Reader on a text file.
*/
@@ -1590,7 +1614,7 @@ Object Interpreter::eval_format(const Object& form,
fmt::format_args(args2.data(), static_cast<unsigned>(args2.size())));
if (truthy(dest)) {
printf("%s", formatted.c_str());
lg::print(formatted.c_str());
}
return StringObject::make_new(formatted);
@@ -1647,6 +1671,45 @@ Object Interpreter::eval_string_append(const Object& form,
return StringObject::make_new(result);
}
Object Interpreter::eval_string_starts_with(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env) {
(void)env;
vararg_check(form, args, {ObjectType::STRING, ObjectType::STRING}, {});
auto& str = args.unnamed.at(0).as_string()->data;
auto& suffix = args.unnamed.at(1).as_string()->data;
if (str_util::starts_with(str, suffix)) {
return SymbolObject::make_new(reader.symbolTable, "#t");
}
return SymbolObject::make_new(reader.symbolTable, "#f");
}
Object Interpreter::eval_string_ends_with(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env) {
(void)env;
vararg_check(form, args, {ObjectType::STRING, ObjectType::STRING}, {});
auto& str = args.unnamed.at(0).as_string()->data;
auto& suffix = args.unnamed.at(1).as_string()->data;
if (str_util::ends_with(str, suffix)) {
return SymbolObject::make_new(reader.symbolTable, "#t");
}
return SymbolObject::make_new(reader.symbolTable, "#f");
}
Object Interpreter::eval_string_split(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env) {
(void)env;
vararg_check(form, args, {ObjectType::STRING, ObjectType::STRING}, {});
auto& str = args.unnamed.at(0).as_string()->data;
auto& delim = args.unnamed.at(1).as_string()->data;
return pretty_print::build_list(str_util::split(str, delim.at(0)));
}
Object Interpreter::eval_ash(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env) {
common/goos/Interpreter.h
+16 -1
View File
@@ -16,7 +16,7 @@ class Interpreter {
public:
Interpreter(const std::string& user_profile = "#f");
~Interpreter();
void execute_repl(ReplWrapper& repl);
void execute_repl(REPL::Wrapper& repl);
void throw_eval_error(const Object& o, const std::string& err);
Object eval_with_rewind(const Object& obj, const std::shared_ptr<EnvironmentObject>& env);
bool get_global_variable_by_name(const std::string& name, Object* dest);
@@ -62,8 +62,11 @@ class Interpreter {
const std::unordered_map<std::string, std::pair<bool, std::optional<ObjectType>>>& named);
Object eval_pair(const Object& o, const std::shared_ptr<EnvironmentObject>& env);
public:
ArgumentSpec parse_arg_spec(const Object& form, Object& rest);
private:
Object quasiquote_helper(const Object& form, const std::shared_ptr<EnvironmentObject>& env);
IntType number_to_integer(const Object& obj);
@@ -116,6 +119,9 @@ class Interpreter {
Object eval_read(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env);
Object eval_read_data_file(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env);
Object eval_read_file(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env);
@@ -197,6 +203,15 @@ class Interpreter {
Object eval_string_append(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env);
Object eval_string_starts_with(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env);
Object eval_string_ends_with(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env);
Object eval_string_split(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env);
Object eval_ash(const Object& form,
Arguments& args,
const std::shared_ptr<EnvironmentObject>& env);
common/goos/ParseHelpers.cpp
+1
View File
@@ -53,6 +53,7 @@ bool va_check(
for (size_t i = 0; i < unnamed.size(); i++) {
if (unnamed[i].has_value() && unnamed[i] != args.unnamed[i].type) {
// special case -- an empty list is a valid pair
*err_string = fmt::format("Argument {} has type {} but {} was expected\nArgument is: {}", i,
object_type_to_string(args.unnamed[i].type),
object_type_to_string(unnamed[i].value()), args.unnamed[i].print());
common/goos/PrettyPrinter.cpp
-1
View File
@@ -13,7 +13,6 @@
#include "Reader.h"
#include "common/goos/PrettyPrinter2.h"
#include "common/log/log.h"
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
common/goos/PrettyPrinter2.cpp
+52 -14
View File
@@ -4,6 +4,7 @@
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/format.h"
namespace pretty_print {
@@ -29,11 +30,14 @@ struct Node {
std::string atom_str;
// number of quotes this is wrapped in.
u32 quoted = 0;
enum class QuoteKind { QUOTE, UNQUOTE, QUASIQUOTE, UNQUOTE_SPLICING };
std::vector<QuoteKind> quotes;
Node* parent = nullptr;
u32 my_depth = 0;
int get_quote_length() const;
void link(Node* this_parent, std::vector<Node*>* bfs_order, u32 depth) {
parent = this_parent;
my_depth = depth;
@@ -86,6 +90,30 @@ struct Node {
u8 sub_elt_indent = 0;
};
inline const std::string quote_symbol(Node::QuoteKind kind) {
switch (kind) {
case Node::QuoteKind::QUOTE:
return "'";
case Node::QuoteKind::QUASIQUOTE:
return "`";
case Node::QuoteKind::UNQUOTE:
return ",";
case Node::QuoteKind::UNQUOTE_SPLICING:
return ",@";
default:
ASSERT_MSG(false, fmt::format("invalid quote kind {}", fmt::underlying(kind)));
return "[invalid]";
}
}
int Node::get_quote_length() const {
int out = 0;
for (auto& q : quotes) {
out += quote_symbol(q).length();
}
return out;
}
Node to_node(const goos::Object& obj) {
switch (obj.type) {
case goos::ObjectType::EMPTY_LIST:
@@ -100,13 +128,22 @@ Node to_node(const goos::Object& obj) {
return Node(obj.print());
case goos::ObjectType::PAIR: {
// we've got three cases: quoted thing, proper list, improper list.
// we've got four cases: quoted thing, unquoted thing, proper list, improper list.
// there's probably a better way to do this but i am lazy
auto& first = obj.as_pair()->car;
if (first.is_symbol() && first.as_symbol()->name == "quote") {
if (first.is_symbol("quote")) {
auto& second = obj.as_pair()->cdr;
if (second.is_pair() && second.as_pair()->cdr.is_empty_list()) {
Node result = to_node(second.as_pair()->car);
result.quoted++;
result.quotes.push_back(Node::QuoteKind::QUOTE);
return result;
}
} else if (first.is_symbol("unquote")) {
auto& second = obj.as_pair()->cdr;
if (second.is_pair() && second.as_pair()->cdr.is_empty_list()) {
Node result = to_node(second.as_pair()->car);
result.quotes.push_back(Node::QuoteKind::UNQUOTE);
return result;
}
}
@@ -148,13 +185,13 @@ void recompute_lengths(const std::vector<Node*>& bfs_order) {
Node* node = *it;
switch (node->kind) {
case Node::Kind::ATOM:
node->text_len = node->atom_str.length() + node->quoted;
node->text_len = node->atom_str.length() + node->get_quote_length();
break;
case Node::Kind::IMPROPER_LIST:
case Node::Kind::LIST: {
if (node->break_list) {
// special case compute first line length
int first_line_len = 1 + node->quoted; // open paren + quotes
int first_line_len = 1 + node->get_quote_length(); // open paren + quotes
int nodes_on_first_line =
std::min(int(node->child_nodes.size()), int(node->top_line_count));
if (nodes_on_first_line > 0) {
@@ -176,7 +213,7 @@ void recompute_lengths(const std::vector<Node*>& bfs_order) {
node->text_len = max_line_len;
} else {
node->text_len = 1 + node->quoted; // open paren + quotes
node->text_len = 1 + node->get_quote_length(); // open paren + quotes
for (auto& child : node->child_nodes) {
node->text_len += (child.text_len + 1); // space or close paren.
}
@@ -231,7 +268,8 @@ void break_list(Node* node) {
name == "when" || name == "behavior" || name == "lambda" || name == "defpart" ||
name == "define") {
node->top_line_count = 2;
} else if (name == "let" || name == "let*" || name == "rlet") {
} else if (name == "let" || name == "let*" || name == "rlet" ||
name == "with-dma-buffer-add-bucket") {
// special case for things like let.
node->top_line_count = 2; // (let <defs>
if (node->child_nodes.size() > 1 && node->child_nodes[1].child_nodes.size() > 1 &&
@@ -276,7 +314,7 @@ void insert_required_breaks(const std::vector<Node*>& bfs_order) {
const std::unordered_set<std::string> always_break = {
"when", "defun-debug", "countdown", "case", "defun", "defmethod", "let",
"until", "while", "if", "dotimes", "cond", "else", "defbehavior",
"with-pp", "rlet", "defstate", "behavior", "defpart", "loop"};
"with-pp", "rlet", "defstate", "behavior", "defpart", "loop", "let*"};
for (auto node : bfs_order) {
if (!node->break_list && node->kind == Node::Kind::LIST &&
node->child_nodes.at(0).kind == Node::Kind::ATOM) {
@@ -292,7 +330,7 @@ int run_algorithm(const std::vector<Node*>& bfs_order, int line_length) {
// - too long
// - not already split.
// the "magic" of v2 is:
// the "too long" check above igores the sublist.
// the "too long" check above ignores the sublist.
int num_broken = 0;
std::optional<s32> min_depth;
@@ -334,8 +372,8 @@ void append_node_to_string(const Node* node,
for (int i = 0; i < init_indent_level; i++) {
str.push_back(' ');
}
for (u32 i = 0; i < node->quoted; i++) {
str.push_back('\'');
for (auto q : node->quotes) {
str.append(quote_symbol(q));
}
switch (node->kind) {
case Node::Kind::ATOM:
@@ -347,7 +385,7 @@ void append_node_to_string(const Node* node,
str.push_back('(');
size_t node_idx = 0;
int listing_indent = next_indent_level + node->quoted + node->sub_elt_indent;
int listing_indent = next_indent_level + node->get_quote_length() + node->sub_elt_indent;
int extra_indent = 0;
int old_indent = listing_indent;
if (node->top_line_count) {
@@ -401,7 +439,7 @@ void append_node_to_string(const Node* node,
} else {
str.push_back('(');
ASSERT(!node->child_nodes.empty());
int listing_indent = next_indent_level + node->quoted;
int listing_indent = next_indent_level + node->get_quote_length();
int extra_indent = 1;
int c0 = 0;
for (auto& child : node->child_nodes) {
common/goos/Printer.cpp
+18 -10
View File
@@ -1,5 +1,11 @@
#include "Printer.h"
#include <cmath>
#include <mutex>
#include "common/goos/Object.h"
#include "common/util/print_float.h"
#include "third-party/fmt/core.h"
namespace pretty_print {
@@ -19,9 +25,7 @@ const std::unordered_map<u32, std::string> const_floats = {{0x40490fda, "PI"},
goos::Object float_representation(float value) {
u32 int_value;
memcpy(&int_value, &value, 4);
u8 exp = (int_value >> 23) & 0xff;
u32 mant = int_value & 0x7fffff;
if ((exp == 0 && mant != 0) || exp == 0xff) {
if (!proper_float(value)) {
// lg::warn("PS2-incompatible float (0x{:08X}) detected! Writing as the-as cast.", int_value);
return pretty_print::build_list("the-as", "float", fmt::format("#x{:x}", int_value));
} else if (const_floats.find(int_value) != const_floats.end()) {
@@ -34,6 +38,7 @@ goos::Object float_representation(float value) {
}
std::unique_ptr<goos::Reader> pretty_printer_reader;
std::mutex pretty_printer_reader_mutex;
goos::Reader& get_pretty_printer_reader() {
if (!pretty_printer_reader) {
@@ -43,9 +48,14 @@ goos::Reader& get_pretty_printer_reader() {
}
goos::Object to_symbol(const std::string& str) {
std::lock_guard<std::mutex> guard(pretty_printer_reader_mutex);
return goos::SymbolObject::make_new(get_pretty_printer_reader().symbolTable, str);
}
goos::Object new_string(const std::string& str) {
return goos::StringObject::make_new(str);
}
goos::Object build_list(const std::string& str) {
return build_list(to_symbol(str));
}
@@ -65,14 +75,12 @@ goos::Object build_list(const std::vector<goos::Object>& objects) {
// build a list out of an array of forms
goos::Object build_list(const goos::Object* objects, int count) {
ASSERT(count);
auto car = objects[0];
goos::Object cdr;
if (count - 1) {
cdr = build_list(objects + 1, count - 1);
} else {
cdr = goos::Object::make_empty_list();
goos::Object result = goos::Object::make_empty_list();
for (int i = count; i-- > 0;) {
result = goos::PairObject::make_new(objects[i], result);
}
return goos::PairObject::make_new(car, cdr);
return result;
}
// build a list out of a vector of strings that are converted to symbols
common/goos/Printer.h
+2
View File
@@ -10,6 +10,8 @@ namespace pretty_print {
// string -> object (as a symbol)
goos::Object to_symbol(const std::string& str);
goos::Object new_string(const std::string& str);
// list with a single symbol from a string
goos::Object build_list(const std::string& str);
common/goos/Reader.cpp
+17 -19
View File
@@ -11,8 +11,8 @@
#include "Reader.h"
#include "ReplUtils.h"
#include "common/log/log.h"
#include "common/repl/util.h"
#include "common/util/FileUtil.h"
#include "common/util/FontUtils.h"
@@ -194,7 +194,7 @@ bool Reader::is_valid_source_char(char c) const {
/*!
* Prompt the user and read the result.
*/
std::optional<Object> Reader::read_from_stdin(const std::string& prompt, ReplWrapper& repl) {
std::optional<Object> Reader::read_from_stdin(const std::string& prompt, REPL::Wrapper& repl) {
// escape code will make sure that we remove any color
std::string prompt_full = "\033[0m" + prompt;
@@ -238,17 +238,9 @@ Object Reader::read_from_string(const std::string& str,
* Read a file
*/
Object Reader::read_from_file(const std::vector<std::string>& file_path, bool check_encoding) {
std::string joined_name;
std::string joined_path = fmt::format("{}", fmt::join(file_path, "/"));
for (const auto& thing : file_path) {
if (!joined_name.empty()) {
joined_name += '/';
}
joined_name += thing;
}
auto textFrag = std::make_shared<FileText>(file_util::get_file_path(file_path), joined_name);
auto textFrag = std::make_shared<FileText>(file_util::get_file_path(file_path), joined_path);
db.insert(textFrag);
auto result = internal_read(textFrag, check_encoding);
@@ -291,11 +283,16 @@ Object Reader::internal_read(std::shared_ptr<SourceText> text,
ts.seek_past_whitespace_and_comments();
// read list!
auto objs = read_list(ts, false);
if (add_top_level) {
return PairObject::make_new(SymbolObject::make_new(symbolTable, "top-level"), objs);
} else {
return objs;
try {
auto objs = read_list(ts, false);
if (add_top_level) {
return PairObject::make_new(SymbolObject::make_new(symbolTable, "top-level"), objs);
} else {
return objs;
}
} catch (std::exception& e) {
lg::print("{}", e.what());
throw;
}
}
@@ -343,7 +340,8 @@ Token Reader::get_next_token(TextStream& stream) {
// Second - not a special token, so we read until we get a character that ends the token.
while (stream.text_remains()) {
char next = stream.peek();
if (next == ' ' || next == '\n' || next == '\t' || next == ')' || next == ';' || next == '(') {
if (next == ' ' || next == '\n' || next == '\t' || next == '\r' || next == ')' || next == ';' ||
next == '(') {
return t;
} else {
// not the end, so add to token.
common/goos/Reader.h
+2 -3
View File
@@ -16,10 +16,9 @@
#include <unordered_map>
#include <utility>
#include "ReplUtils.h"
#include "common/goos/Object.h"
#include "common/goos/TextDB.h"
#include "common/repl/util.h"
#include "common/util/Assert.h"
namespace goos {
@@ -74,7 +73,7 @@ class Reader {
Object read_from_string(const std::string& str,
bool add_top_level = true,
const std::optional<std::string>& string_name = {});
std::optional<Object> read_from_stdin(const std::string& prompt, ReplWrapper& repl);
std::optional<Object> read_from_stdin(const std::string& prompt, REPL::Wrapper& repl);
Object read_from_file(const std::vector<std::string>& file_path, bool check_encoding = false);
bool check_string_is_valid(const std::string& str) const;
common/goos/ReplUtils.cpp
-135
View File
@@ -1,135 +0,0 @@
#include "ReplUtils.h"
#include "common/util/FileUtil.h"
#include "common/versions.h"
#include "third-party/fmt/color.h"
#include "third-party/fmt/core.h"
#include "third-party/replxx/include/replxx.hxx"
// TODO - expand a list of hints (ie. a hint for defun to show at a glance how to write a function,
// or perhaps, show the docstring for the current function being used?)
using Replxx = replxx::Replxx;
void ReplWrapper::clear_screen() {
repl.clear_screen();
}
void ReplWrapper::print_welcome_message() {
// TODO - dont print on std-out
// Welcome message / brief intro for documentation
std::string ascii;
ascii += " _____ _____ _____ _____ __ \n";
ascii += "| |___ ___ ___| __| | _ | | \n";
ascii += "| | | . | -_| | | | | | | |__ \n";
ascii += "|_____| _|___|_|_|_____|_____|__|__|_____|\n";
ascii += " |_| \n";
fmt::print(fmt::emphasis::bold | fg(fmt::color::orange), ascii);
fmt::print("Welcome to OpenGOAL {}.{}!\n", versions::GOAL_VERSION_MAJOR,
versions::GOAL_VERSION_MINOR);
fmt::print("Run ");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(repl-help)");
fmt::print(" for help with common commands and REPL usage.\n");
fmt::print("Run ");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt)");
fmt::print(" to connect to the local target.\n");
fmt::print("Run ");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(mi)");
fmt::print(" to rebuild the entire game.\n\n");
}
void ReplWrapper::print_to_repl(const std::string_view& str) {
repl.print(str.data());
}
void ReplWrapper::set_history_max_size(size_t len) {
repl.set_max_history_size(len);
}
const char* ReplWrapper::readline(const std::string& prompt) {
return repl.input(prompt);
}
void ReplWrapper::add_to_history(const std::string& line) {
repl.history_add(line);
}
void ReplWrapper::save_history() {
fs::path path = file_util::get_user_config_dir() / ".opengoal.repl.history";
file_util::create_dir_if_needed_for_file(path.string());
repl.history_save(path.string());
}
void ReplWrapper::load_history() {
fs::path path = file_util::get_user_config_dir() / ".opengoal.repl.history";
if (fs::exists(path)) {
repl.history_load(path.string());
} else {
fmt::print("Couldn't locate REPL history file at '{}'\n", path.string());
}
}
std::pair<std::string, bool> ReplWrapper::get_current_repl_token(std::string const& context) {
// Find the current token
std::string token = "";
for (auto c = context.crbegin(); c != context.crend(); c++) {
if (std::isspace(*c)) {
break;
} else {
token = *c + token;
}
}
// If there is a preceeding '(' remove it
if (!token.empty() && token.at(0) == '(') {
token.erase(0, 1);
return {token, true};
}
return {token, false};
}
void ReplWrapper::print_help_message() {
fmt::print(fmt::emphasis::bold, "\nREPL Controls:\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(:clear)\n");
fmt::print(" - Clear the current screen\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(e)\n");
fmt::print(" - Exit the compiler\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt [ip-address] [port-number])\n");
fmt::print(
" - Connect the listener to a running target. The IP address defaults to `127.0.0.1` and the "
"port to `8112`\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(r [ip-address] [port-number])\n");
fmt::print(
" - Attempt to reset the target and reconnect. After this, the target will have nothing "
"loaded.\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(:status)\n");
fmt::print(" - Send a ping-like message to the target. Requires the target to be connected\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(shutdown-target)\n");
fmt::print(" - If the target is connected, make it exit\n");
fmt::print(fmt::emphasis::bold, "\nCompiling & Building:\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(mi)\n");
fmt::print(" - Build entire game\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(mng)\n");
fmt::print(" - Build game engine\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(m \"filename\")\n");
fmt::print(" - Compile an OpenGOAL source file\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(ml \"filename\")\n");
fmt::print(" - Compile and Load (or reload) an OpenGOAL source file\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(build-kernel)\n");
fmt::print(" - Build the GOAL kernel\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(make \"file-name\")\n");
fmt::print(
" - Build a file and any out-of-date dependencies. This file must be a target in the make "
"system.\n");
fmt::print(fmt::emphasis::bold, "\nOther:\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::magenta), "(gs)\n");
fmt::print(" - Enter a GOOS REPL\n");
}
void ReplWrapper::init_default_settings() {
repl.set_word_break_characters(" \t");
}
common/goos/ReplUtils.h
-34
View File
@@ -1,34 +0,0 @@
#pragma once
#include <functional>
#include <string>
#include <vector>
#include "third-party/replxx/include/replxx.hxx"
using Replxx = replxx::Replxx;
class ReplWrapper {
Replxx repl;
public:
ReplWrapper() {}
Replxx& get_repl() { return repl; }
void init_default_settings();
// Functionality / Commands
void clear_screen();
void print_to_repl(const std::string_view& str);
void print_welcome_message();
void set_history_max_size(size_t len);
const char* readline(const std::string& prompt);
void add_to_history(const std::string& line);
void save_history();
void load_history();
void print_help_message();
std::pair<std::string, bool> get_current_repl_token(std::string const& context);
std::vector<std::string> examples{};
using cl = Replxx::Color;
std::vector<std::pair<std::string, cl>> regex_colors{};
};
common/log/log.cpp
+39 -11
View File
@@ -35,21 +35,22 @@ struct Logger {
Logger gLogger;
namespace internal {
const char* log_level_names[] = {"trace", "debug", "info", "warn", "error", "die"};
const fmt::color log_colors[] = {fmt::color::gray, fmt::color::turquoise, fmt::color::light_green,
fmt::color::yellow, fmt::color::red, fmt::color::hot_pink};
const char* log_level_names[] = {"trace", "debug", "info", "warn", "error", "die", "die"};
const fmt::color log_colors[] = {
fmt::color::gray, fmt::color::turquoise, fmt::color::light_green, fmt::color::yellow,
fmt::color::red, fmt::color::hot_pink, fmt::color::hot_pink};
void log_message(level log_level, LogTime& now, const char* message) {
#ifdef __linux__
char date_time_buffer[128];
time_t now_seconds = now.tv.tv_sec;
auto now_milliseconds = now.tv.tv_usec / 1000;
strftime(date_time_buffer, 128, "%Y-%m-%d %H:%M:%S", localtime(&now_seconds));
std::string date_string = fmt::format("[{}:{:03d}]", date_time_buffer, now_milliseconds);
strftime(date_time_buffer, 128, "%M:%S", localtime(&now_seconds));
std::string time_string = fmt::format("[{}:{:03d}]", date_time_buffer, now_milliseconds);
#else
char date_time_buffer[128];
strftime(date_time_buffer, 128, "%Y-%m-%d %H:%M:%S", localtime(&now.tim));
std::string date_string = fmt::format("[{}]", date_time_buffer);
strftime(date_time_buffer, 128, "%M:%S", localtime(&now.tim));
std::string time_string = fmt::format("[{}]", date_time_buffer);
#endif
{
@@ -57,25 +58,52 @@ void log_message(level log_level, LogTime& now, const char* message) {
if (gLogger.fp && log_level >= gLogger.file_log_level) {
// log to file
std::string file_string =
fmt::format("{} [{}] {}\n", date_string, log_level_names[int(log_level)], message);
fmt::format("{} [{}] {}\n", time_string, log_level_names[int(log_level)], message);
fwrite(file_string.c_str(), file_string.length(), 1, gLogger.fp);
if (log_level >= gLogger.flush_level) {
fflush(gLogger.fp);
}
}
if (log_level >= gLogger.stdout_log_level) {
fmt::print("{} [", date_string);
if (log_level >= gLogger.stdout_log_level ||
(log_level == level::die && gLogger.stdout_log_level == level::off_unless_die)) {
fmt::print("{} [", time_string);
fmt::print(fg(log_colors[int(log_level)]), "{}", log_level_names[int(log_level)]);
fmt::print("] {}\n", message);
if (log_level >= gLogger.flush_level) {
fflush(stdout);
fflush(stderr);
}
}
}
if (log_level == level::die) {
exit(-1);
fflush(stdout);
fflush(stderr);
if (gLogger.fp) {
fflush(gLogger.fp);
}
abort();
}
}
void log_print(const char* message) {
{
// We always immediately flush prints because since it has no associated level
// it could be anything from a fatal error to a useless debug log.
std::lock_guard<std::mutex> lock(gLogger.mutex);
if (gLogger.fp) {
// Log to File
std::string msg(message);
fwrite(msg.c_str(), msg.length(), 1, gLogger.fp);
fflush(gLogger.fp);
}
if (gLogger.stdout_log_level < lg::level::off_unless_die) {
fmt::print(message);
fflush(stdout);
fflush(stderr);
}
}
}
} // namespace internal
common/log/log.h
+23 -1
View File
@@ -7,6 +7,7 @@
#endif
#include <string>
#include "third-party/fmt/color.h"
#include "third-party/fmt/core.h"
namespace lg {
@@ -22,11 +23,21 @@ struct LogTime {
#endif
// Logging API
enum class level { trace = 0, debug = 1, info = 2, warn = 3, error = 4, die = 5, off = 6 };
enum class level {
trace = 0,
debug = 1,
info = 2,
warn = 3,
error = 4,
die = 5,
off_unless_die = 6,
off = 7
};
namespace internal {
// log implementation stuff, not to be called by the user
void log_message(level log_level, LogTime& now, const char* message);
void log_print(const char* message);
} // namespace internal
void set_file(const std::string& filename);
@@ -49,6 +60,17 @@ void log(level log_level, const std::string& format, Args&&... args) {
internal::log_message(log_level, now, formatted_message.c_str());
}
template <typename... Args>
void print(const std::string& format, Args&&... args) {
std::string formatted_message = fmt::format(format, std::forward<Args>(args)...);
internal::log_print(formatted_message.c_str());
}
template <typename... Args>
void print(const fmt::text_style& ts, const std::string& format, Args&&... args) {
std::string formatted_message = fmt::format(ts, format, std::forward<Args>(args)...);
internal::log_print(formatted_message.c_str());
}
template <typename... Args>
void trace(const std::string& format, Args&&... args) {
log(level::trace, format, std::forward<Args>(args)...);
common/math/Vector.h
+26
View File
@@ -77,7 +77,17 @@ class Vector {
return true;
}
bool operator==(const T other) const {
for (int i = 0; i < Size; i++) {
if (m_data[i] != other) {
return false;
}
}
return true;
}
bool operator!=(const Vector<T, Size>& other) const { return !((*this) == other); }
bool operator!=(const T other) const { return !((*this) == other); }
const T length() const { return std::sqrt(squared_length()); }
@@ -118,6 +128,13 @@ class Vector {
return *this;
}
Vector<T, Size>& operator+=(const T& other) {
for (int i = 0; i < Size; i++) {
m_data[i] += other;
}
return *this;
}
Vector<T, Size> elementwise_multiply(const Vector<T, Size>& other) const {
Vector<T, Size> result;
for (int i = 0; i < Size; i++) {
@@ -225,6 +242,15 @@ class Vector {
return result + "]";
}
std::string to_string_hex_word() const {
std::string result = "[";
for (auto x : m_data) {
result.append(fmt::format("0x{:08x} ", x));
}
result.pop_back();
return result + "]";
}
T* data() { return m_data; }
const T* data() const { return m_data; }
common/repl/config.cpp
+95
View File
@@ -0,0 +1,95 @@
#include "config.h"
#include "common/versions.h"
#include "third-party/fmt/core.h"
namespace REPL {
void to_json(json& j, const Config& obj) {
j = json{
{"numConnectToTargetAttempts", obj.target_connect_attempts},
{"asmFileSearchDirs", obj.asm_file_search_dirs},
{"keybinds", obj.keybinds},
};
}
void from_json(const json& j, Config& obj) {
if (j.contains("numConnectToTargetAttempts")) {
j.at("numConnectToTargetAttempts").get_to(obj.target_connect_attempts);
}
if (j.contains("asmFileSearchDirs")) {
j.at("asmFileSearchDirs").get_to(obj.asm_file_search_dirs);
}
if (j.contains("appendKeybinds")) {
j.at("appendKeybinds").get_to(obj.append_keybinds);
}
if (j.contains("keybinds")) {
std::vector<KeyBind> keybinds = j.at("keybinds");
if (!obj.append_keybinds) {
obj.keybinds = keybinds;
} else {
// append the keybinds
// - start with the provided ones
// - skip ones from the default set if they have the same key + modifier combination
for (const auto& default_bind : obj.keybinds) {
// check if it's a duplicate bind
bool duplicate = false;
for (const auto& new_bind : keybinds) {
if (new_bind.key == default_bind.key && new_bind.modifier == default_bind.modifier) {
duplicate = true;
break;
}
}
if (!duplicate) {
keybinds.push_back(default_bind);
}
}
obj.keybinds = keybinds;
}
}
// if there is game specific configuration, override any values we just set
if (j.contains(version_to_game_name(obj.game_version))) {
from_json(j.at(version_to_game_name(obj.game_version)), obj);
}
}
std::string KeyBind::string() const {
switch (modifier) {
case KeyBind::Modifier::CTRL:
return fmt::format("CTRL-{}", key);
case KeyBind::Modifier::SHIFT:
return fmt::format("SHIFT-{}", key);
case KeyBind::Modifier::META:
return fmt::format("META-{}", key);
}
}
void to_json(json& j, const KeyBind& obj) {
j = json{{"description", obj.description}, {"command", obj.command}, {"key", obj.key}};
switch (obj.modifier) {
case KeyBind::Modifier::CTRL:
j["modifier"] = "ctrl";
break;
case KeyBind::Modifier::SHIFT:
j["modifier"] = "shift";
break;
case KeyBind::Modifier::META:
j["modifier"] = "meta";
break;
}
}
void from_json(const json& j, KeyBind& obj) {
j.at("description").get_to(obj.description);
j.at("command").get_to(obj.command);
j.at("key").get_to(obj.key);
auto modString = j.at("modifier").get<std::string>();
if (modString == "ctrl") {
obj.modifier = KeyBind::Modifier::CTRL;
} else if (modString == "shift") {
obj.modifier = KeyBind::Modifier::SHIFT;
} else if (modString == "meta") {
obj.modifier = KeyBind::Modifier::META;
}
}
} // namespace REPL
common/repl/config.h
+50
View File
@@ -0,0 +1,50 @@
#pragma once
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>
#include "common/versions.h"
#include "third-party/json.hpp"
using json = nlohmann::json;
namespace REPL {
struct KeyBind {
// NOTE - in my experience, meta doesn't work on windows and shift is probably a bad idea when
// typing text! but I leave it up to the user.
enum class Modifier { CTRL, SHIFT, META };
Modifier modifier;
std::string key;
std::string description;
std::string command;
std::string string() const;
};
void to_json(json& j, const KeyBind& obj);
void from_json(const json& j, KeyBind& obj);
struct Config {
GameVersion game_version;
Config(GameVersion _game_version) : game_version(_game_version){};
// this is the default REPL configuration
int target_connect_attempts = 30;
std::vector<std::string> asm_file_search_dirs = {};
bool append_keybinds = true;
std::vector<KeyBind> keybinds = {
{KeyBind::Modifier::CTRL, "T", "Starts up the game runtime", "(test-play)"},
{KeyBind::Modifier::CTRL, "Q", "Exit the REPL", "(e)"},
{KeyBind::Modifier::CTRL, "L", "Listen to an available game process", "(lt)"},
{KeyBind::Modifier::CTRL, "W",
"Halt the attached process so you can re-launch a crashed game", "(:stop)"},
{KeyBind::Modifier::CTRL, "G", "Attach the debugger to the process", "(dbgc)"},
{KeyBind::Modifier::CTRL, "B", "Displays the most recently caught backtrace", "(:di)"},
{KeyBind::Modifier::CTRL, "N", "Full build of the game", "(mi)"}};
};
void to_json(json& j, const Config& obj);
void from_json(const json& j, Config& obj);
} // namespace REPL
common/nrepl/ReplClient.cpp → common/repl/nrepl/ReplClient.cpp
View File
common/nrepl/ReplClient.h → common/repl/nrepl/ReplClient.h
View File
common/nrepl/ReplServer.cpp → common/repl/nrepl/ReplServer.cpp
+3 -3
View File
@@ -2,7 +2,7 @@
#include "ReplServer.h"
#include "common/cross_sockets/XSocket.h"
#include <common/versions.h>
#include "common/versions.h"
#include "third-party/fmt/core.h"
@@ -107,8 +107,8 @@ std::optional<std::string> ReplServer::get_msg() {
auto req_bytes = read_from_socket(sock, header_buffer.data(), header_buffer.size());
if (req_bytes == 0) {
// Socket disconnected
// TODO - add a queue of messages in the ReplWrapper so we can print _BEFORE_ the prompt is
// output
// TODO - add a queue of messages in the REPL::Wrapper so we can print _BEFORE_ the prompt
// is output
fmt::print("[nREPL:{}] Client Disconnected: {}\n", tcp_port, inet_ntoa(addr.sin_addr),
ntohs(addr.sin_port), sock);
common/nrepl/ReplServer.h → common/repl/nrepl/ReplServer.h
View File
common/repl/util.cpp
+248
View File
@@ -0,0 +1,248 @@
#include "util.h"
#include "common/util/FileUtil.h"
#include "common/util/json_util.h"
#include "common/util/string_util.h"
#include "common/versions.h"
#include "third-party/fmt/color.h"
#include "third-party/fmt/core.h"
#include "third-party/replxx/include/replxx.hxx"
// TODO - expand a list of hints (ie. a hint for defun to show at a glance how to write a function,
// or perhaps, show the docstring for the current function being used?)
namespace REPL {
void Wrapper::clear_screen() {
repl.clear_screen();
}
void Wrapper::print_welcome_message() {
// TODO - dont print on std-out
// Welcome message / brief intro for documentation
std::string ascii;
ascii += " _____ _____ _____ _____ __ \n";
ascii += "| |___ ___ ___| __| | _ | | \n";
ascii += "| | | . | -_| | | | | | | |__ \n";
ascii += "|_____| _|___|_|_|_____|_____|__|__|_____|\n";
ascii += " |_| \n";
fmt::print(fmt::emphasis::bold | fg(fmt::color::orange), ascii);
fmt::print("Welcome to OpenGOAL {}.{}!\n", versions::GOAL_VERSION_MAJOR,
versions::GOAL_VERSION_MINOR);
fmt::print("Run {} or {} for help with common commands and REPL usage.\n",
fmt::styled("(repl-help)", fmt::emphasis::bold | fg(fmt::color::cyan)),
fmt::styled("(repl-keybinds)", fmt::emphasis::bold | fg(fmt::color::cyan)));
fmt::print("Run ");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt)");
fmt::print(" to connect to the local target.\n");
fmt::print("Run ");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(mi)");
fmt::print(" to rebuild the entire game.\n\n");
}
void Wrapper::print_to_repl(const std::string& str) {
repl.print(str.data());
}
void Wrapper::set_history_max_size(size_t len) {
repl.set_max_history_size(len);
}
const char* Wrapper::readline(const std::string& prompt) {
return repl.input(prompt);
}
void Wrapper::add_to_history(const std::string& line) {
repl.history_add(line);
}
void Wrapper::save_history() {
fs::path path = file_util::get_user_config_dir() / ".opengoal.repl.history";
file_util::create_dir_if_needed_for_file(path.string());
repl.history_save(path.string());
}
void Wrapper::load_history() {
fs::path path = file_util::get_user_config_dir() / ".opengoal.repl.history";
if (fs::exists(path)) {
repl.history_load(path.string());
} else {
fmt::print("Couldn't locate REPL history file at '{}'\n", path.string());
}
}
std::pair<std::string, bool> Wrapper::get_current_repl_token(std::string const& context) {
// Find the current token
std::string token = "";
for (auto c = context.crbegin(); c != context.crend(); c++) {
if (std::isspace(*c)) {
break;
} else {
token = *c + token;
}
}
// If there is a preceeding '(' remove it
if (!token.empty() && token.at(0) == '(') {
token.erase(0, 1);
return {token, true};
}
return {token, false};
}
void Wrapper::print_help_message() {
fmt::print(fmt::emphasis::bold, "\nREPL Controls:\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(:clear)\n");
fmt::print(" - Clear the current screen\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(e)\n");
fmt::print(" - Exit the compiler\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(lt [ip-address] [port-number])\n");
fmt::print(
" - Connect the listener to a running target. The IP address defaults to `127.0.0.1` and the "
"port to `8112`\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(r [ip-address] [port-number])\n");
fmt::print(
" - Attempt to reset the target and reconnect. After this, the target will have nothing "
"loaded.\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(:status)\n");
fmt::print(" - Send a ping-like message to the target. Requires the target to be connected\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::cyan), "(shutdown-target)\n");
fmt::print(" - If the target is connected, make it exit\n");
fmt::print(fmt::emphasis::bold, "\nCompiling & Building:\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(mi)\n");
fmt::print(" - Build entire game\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(mng)\n");
fmt::print(" - Build game engine\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(m \"filename\")\n");
fmt::print(" - Compile an OpenGOAL source file\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(ml \"filename\")\n");
fmt::print(" - Compile and Load (or reload) an OpenGOAL source file\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(build-kernel)\n");
fmt::print(" - Build the GOAL kernel\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::lime_green), "(make \"file-name\")\n");
fmt::print(
" - Build a file and any out-of-date dependencies. This file must be a target in the make "
"system.\n");
fmt::print(fmt::emphasis::bold, "\nOther:\n");
fmt::print(fmt::emphasis::bold | fg(fmt::color::magenta), "(gs)\n");
fmt::print(" - Enter a GOOS REPL\n");
}
void Wrapper::print_keybind_help() {
fmt::print(fmt::emphasis::bold, "\nREPL KeyBinds:\n");
for (const auto& bind : repl_config.keybinds) {
fmt::print("{}\n", fmt::styled(bind.string(), fmt::fg(fmt::color::cyan)));
fmt::print("{}\n", fmt::styled(bind.description, fmt::fg(fmt::color::gray)));
}
}
replxx::Replxx::key_press_handler_t Wrapper::commit_text_action(std::string text_to_commit) {
return [this, text_to_commit](char32_t code) {
repl.set_state(
replxx::Replxx::State(text_to_commit.c_str(), static_cast<int>(text_to_commit.size())));
return repl.invoke(replxx::Replxx::ACTION::COMMIT_LINE, code);
};
}
void Wrapper::init_settings() {
// NOTE - a nice popular project that uses replxx
// - https://github.com/ClickHouse/ClickHouse/blob/master/base/base/ReplxxLineReader.cpp#L366
repl.set_word_break_characters(" \t");
// Setup default keybinds
for (const auto& bind : repl_config.keybinds) {
char32_t code;
switch (bind.modifier) {
case KeyBind::Modifier::CTRL:
code = replxx::Replxx::KEY::control(bind.key.at(0));
break;
case KeyBind::Modifier::SHIFT:
code = replxx::Replxx::KEY::shift(bind.key.at(0));
break;
case KeyBind::Modifier::META:
code = replxx::Replxx::KEY::meta(bind.key.at(0));
break;
}
repl.bind_key(code, commit_text_action(bind.command));
}
}
// TODO - command to print out keybinds
void Wrapper::reload_startup_file() {
startup_file = load_user_startup_file(username, repl_config.game_version);
}
std::string find_repl_username() {
// Two options - either:
// 1. look for the `user.txt` file, which should only contain the username
// 2. if this is absent AND there is a single folder inside the "user" folder, use that as the
// username
auto user_dir = file_util::get_jak_project_dir() / "goal_src" / "user";
auto dirs = file_util::find_directories_in_dir(user_dir);
if (dirs.size() == 1) {
return dirs.at(0).filename().string();
}
std::regex allowed_chars("(^[0-9a-zA-Z\\-\\.\\!\\?<>]*$)");
if (file_util::file_exists((user_dir / "user.txt").string())) {
auto text = file_util::read_text_file(user_dir / "user.txt");
text = str_util::trim(text);
if (!text.empty() && std::regex_match(text, allowed_chars)) {
return text;
}
}
return "#f";
}
fs::path get_startup_file_path(const std::string& username, const GameVersion game_version) {
// - first check to see if there is a game version specific startup file to prefer
auto game_specific_path = file_util::get_jak_project_dir() / "goal_src" / "user" / username /
fmt::format("startup-{}.gc", version_to_game_name(game_version));
if (file_util::file_exists(game_specific_path.string())) {
return game_specific_path;
}
return file_util::get_jak_project_dir() / "goal_src" / "user" / username / "startup.gc";
}
StartupFile load_user_startup_file(const std::string& username, const GameVersion game_version) {
// Check for a `startup.gc` file, each line will be executed on the REPL on startup
auto startup_file_path = get_startup_file_path(username, game_version);
StartupFile startup_file;
if (file_util::file_exists(startup_file_path.string())) {
auto data = file_util::read_text_file(startup_file_path);
auto startup_cmds = str_util::split(data);
bool found_run_on_listen_line = false;
for (const auto& cmd : startup_cmds) {
if (found_run_on_listen_line) {
startup_file.run_after_listen.push_back(cmd);
} else {
startup_file.run_before_listen.push_back(cmd);
}
if (str_util::contains(cmd, "og:run-below-on-listen")) {
found_run_on_listen_line = true;
}
}
}
return startup_file;
}
REPL::Config load_repl_config(const std::string& username, const GameVersion game_version) {
auto repl_config_path =
file_util::get_jak_project_dir() / "goal_src" / "user" / username / "repl-config.json";
if (file_util::file_exists(repl_config_path.string())) {
try {
REPL::Config config(game_version);
auto repl_config_data =
parse_commented_json(file_util::read_text_file(repl_config_path), "repl-config.json");
from_json(repl_config_data, config);
return config;
} catch (std::exception& e) {
REPL::Config config(game_version);
}
}
return REPL::Config(game_version);
}
} // namespace REPL
common/repl/util.h
+57
View File
@@ -0,0 +1,57 @@
#pragma once
#include <functional>
#include <optional>
#include <string>
#include <vector>
#include "config.h"
#include "third-party/replxx/include/replxx.hxx"
namespace REPL {
struct StartupFile {
std::vector<std::string> run_before_listen = {};
std::vector<std::string> run_after_listen = {};
};
class Wrapper {
replxx::Replxx repl;
public:
std::string username;
Config repl_config;
StartupFile startup_file;
std::vector<std::string> examples{};
std::vector<std::pair<std::string, replxx::Replxx::Color>> regex_colors{};
Wrapper(GameVersion version) : repl_config(version) {}
Wrapper(const std::string& _username, const Config& config, const StartupFile& startup)
: username(_username), repl_config(config), startup_file(startup) {}
replxx::Replxx& get_repl() { return repl; }
void init_settings();
void reload_startup_file();
// Functionality / Commands
void clear_screen();
void print_to_repl(const std::string& str);
void print_welcome_message();
void set_history_max_size(size_t len);
const char* readline(const std::string& prompt);
void add_to_history(const std::string& line);
void save_history();
void load_history();
void print_help_message();
void print_keybind_help();
std::pair<std::string, bool> get_current_repl_token(std::string const& context);
private:
replxx::Replxx::key_press_handler_t commit_text_action(std::string text_to_commit);
std::vector<REPL::KeyBind> keybindings = {};
};
std::string find_repl_username();
StartupFile load_user_startup_file(const std::string& username, const GameVersion game_version);
REPL::Config load_repl_config(const std::string& username, const GameVersion game_version);
} // namespace REPL
common/serialization/subtitles/subtitles_ser.cpp
+1 -1
View File
@@ -389,7 +389,7 @@ void GameSubtitleGroups::hydrate_from_asset_file() {
m_groups[key] = val.get<std::vector<std::string>>();
}
} catch (std::exception& ex) {
fmt::print("Bad subtitle group entry - {} - {}", key, ex.what());
lg::print("Bad subtitle group entry - {} - {}", key, ex.what());
}
}
}
common/texture/texture_conversion.h
+1 -1
View File
@@ -219,6 +219,6 @@ inline u32 rgba16_to_rgba32(u32 in) {
}
// texture format enums
enum class PSM { PSMCT16 = 0x02, PSMT8 = 0x13, PSMT4 = 0x14 };
enum class PSM { PSMCT32 = 0x0, PSMCT16 = 0x02, PSMT8 = 0x13, PSMT4 = 0x14 };
// clut format enums
enum class CPSM { PSMCT32 = 0x0, PSMCT16 = 0x02 };
common/type_system/Type.cpp
+63 -12
View File
@@ -7,6 +7,7 @@
#include <stdexcept>
#include "common/log/log.h"
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
@@ -36,7 +37,8 @@ std::string reg_kind_to_string(RegClass kind) {
bool MethodInfo::operator==(const MethodInfo& other) const {
return id == other.id && name == other.name && type == other.type &&
defined_in_type == other.defined_in_type && other.no_virtual == no_virtual &&
other.overrides_method_type_of_parent == overrides_method_type_of_parent;
other.overrides_parent == overrides_parent &&
only_overrides_docstring == other.only_overrides_docstring;
}
std::string MethodInfo::diff(const MethodInfo& other) const {
@@ -61,10 +63,16 @@ std::string MethodInfo::diff(const MethodInfo& other) const {
result += fmt::format("no_virtual: {} vs. {}\n", no_virtual, other.no_virtual);
}
if (overrides_method_type_of_parent != other.overrides_method_type_of_parent) {
result += fmt::format("overrides_method_type_of_parent: {} vs. {}\n",
overrides_method_type_of_parent, other.overrides_method_type_of_parent);
if (overrides_parent != other.overrides_parent) {
result +=
fmt::format("overrides_parent: {} vs. {}\n", overrides_parent, other.overrides_parent);
}
if (only_overrides_docstring != other.only_overrides_docstring) {
result += fmt::format("only_overrides_docstring: {} vs. {}\n", only_overrides_docstring,
other.only_overrides_docstring);
}
return result;
}
@@ -196,8 +204,8 @@ std::string Type::get_name() const {
std::string Type::get_runtime_name() const {
if (!m_allow_in_runtime) {
fmt::print("[TypeSystem] Tried to use type {} as a runtime type, which is not allowed.\n",
get_name());
lg::print("[TypeSystem] Tried to use type {} as a runtime type, which is not allowed.\n",
get_name());
throw std::runtime_error("get_runtime_name");
}
return m_runtime_name;
@@ -219,8 +227,36 @@ std::string Type::get_parent() const {
* to check if two types are really identical.
*/
bool Type::common_type_info_equal(const Type& other) const {
// Check if methods only differ because of documentation overrides
bool methods_the_same = true;
for (const auto& method : m_methods) {
if (method.only_overrides_docstring) {
// skip these methods, as we _expect_ to not find them in one or the other!
// this is a all-types vs normal code wrinkle
continue;
}
// For each method, find it's matching id, it should only be allowed to be different
// if its just the docstring
bool found_method = false;
for (const auto& _method : other.m_methods) {
if (method.id == _method.id) {
if (method == _method) {
found_method = true;
break;
} else {
methods_the_same = false;
break;
}
}
}
if (!methods_the_same || !found_method) {
methods_the_same = false;
break;
}
}
// clang-format off
return m_methods == other.m_methods &&
return methods_the_same &&
m_states == other.m_states &&
m_new_method_info == other.m_new_method_info &&
m_new_method_info_defined == other.m_new_method_info_defined &&
@@ -346,7 +382,7 @@ bool Type::get_my_method(int id, MethodInfo* out) const {
*/
bool Type::get_my_last_method(MethodInfo* out) const {
for (auto it = m_methods.rbegin(); it != m_methods.rend(); it++) {
if (!it->overrides_method_type_of_parent) {
if (!it->overrides_parent && !it->only_overrides_docstring) {
*out = *it;
return true;
}
@@ -366,12 +402,27 @@ bool Type::get_my_new_method(MethodInfo* out) const {
return false;
}
/*!
* Get the number of legitimate methods / overridden methods. Ignore that which are inherited just
* for documentation overrides
*/
int Type::get_num_methods() const {
int num = 0;
for (auto it = m_methods.rbegin(); it != m_methods.rend(); it++) {
if (it->only_overrides_docstring) {
continue;
}
num++;
}
return num;
}
/*!
* Add a method defined specifically for this type.
*/
const MethodInfo& Type::add_method(const MethodInfo& info) {
for (auto it = m_methods.rbegin(); it != m_methods.rend(); it++) {
if (!it->overrides_method_type_of_parent) {
if (!it->overrides_parent && !it->only_overrides_docstring) {
ASSERT(it->id + 1 == info.id);
break;
}
@@ -411,7 +462,7 @@ std::string Type::print_method_info() const {
void Type::add_state(const std::string& name, const TypeSpec& type) {
if (!m_states.insert({name, type}).second) {
throw std::runtime_error(fmt::format("State {} is multiply defined", name));
throw std::runtime_error(fmt::format("State {} is already defined in type", name));
}
}
@@ -885,9 +936,9 @@ int BasicType::get_offset() const {
}
int BasicType::get_inline_array_start_alignment() const {
if (m_pack) {
if (m_pack || m_allow_misalign) {
// make elements of inline array the minimum allowable alignment.
int alignment = 8;
int alignment = m_allow_misalign ? 4 : 8;
// TODO - I don't know if GOAL actually did this check, maybe packed inline arrays could
// violate these?
for (const auto& field : m_fields) {
common/type_system/Type.h
+17 -8
View File
@@ -12,17 +12,26 @@
#include "TypeSpec.h"
#include "common/goal_constants.h"
#include "common/goos/TextDB.h"
#include "common/util/Assert.h"
class TypeSystem;
// Various metadata that can be associated with a symbol or form
struct DefinitionMetadata {
std::optional<goos::TextDb::ShortInfo> definition_info;
std::optional<std::string> docstring;
};
struct MethodInfo {
int id = -1;
std::string name;
TypeSpec type;
std::string defined_in_type;
bool no_virtual = false;
bool overrides_method_type_of_parent = false;
bool overrides_parent = false;
bool only_overrides_docstring = false;
std::optional<std::string> docstring;
bool operator==(const MethodInfo& other) const;
bool operator!=(const MethodInfo& other) const { return !((*this) == other); }
@@ -80,6 +89,7 @@ class Type {
bool get_my_method(int id, MethodInfo* out) const;
bool get_my_last_method(MethodInfo* out) const;
bool get_my_new_method(MethodInfo* out) const;
int get_num_methods() const;
const MethodInfo& add_method(const MethodInfo& info);
const MethodInfo& add_new_method(const MethodInfo& info);
std::string print_method_info() const;
@@ -106,6 +116,12 @@ class Type {
bool gen_inspect() const { return m_generate_inspect; }
DefinitionMetadata m_metadata;
std::unordered_map<std::string, std::unordered_map<std::string, DefinitionMetadata>>
m_virtual_state_definition_meta = {};
std::unordered_map<std::string, std::unordered_map<std::string, DefinitionMetadata>>
m_state_definition_meta = {};
protected:
Type(std::string parent, std::string name, bool is_boxed, int heap_base);
virtual std::string diff_impl(const Type& other) const = 0;
@@ -123,13 +139,6 @@ class Type {
std::string m_runtime_name;
bool m_is_boxed = false; // does this have runtime type information?
int m_heap_base = 0;
// definition information
// TODO - LSP - .gc support
/*std::string m_defining_file;
int m_line_number;
int m_line_offset;
void update_definition_meta(const std::string& defining_file, int line_number, int line_offset);*/
};
/*!
common/type_system/TypeFieldLookup.cpp
+11 -9
View File
@@ -7,6 +7,8 @@
#include "TypeSystem.h"
#include "common/log/log.h"
#include "third-party/fmt/core.h"
namespace {
@@ -452,8 +454,8 @@ void try_reverse_lookup(const FieldReverseLookupInput& input,
FieldReverseMultiLookupOutput* output,
int max_count) {
if (debug_reverse_lookup) {
fmt::print(" try_reverse_lookup on {} offset {} deref {} stride {}\n", input.base_type.print(),
input.offset, input.deref.has_value(), input.stride);
lg::debug(" try_reverse_lookup on {} offset {} deref {} stride {}", input.base_type.print(),
input.offset, input.deref.has_value(), input.stride);
}
auto base_input_type = input.base_type.base_type();
@@ -484,15 +486,15 @@ FieldReverseLookupOutput TypeSystem::reverse_field_lookup(
/*
if (multi_result.results.size() > 1) {
fmt::print("Multiple:\n");
lg::print("Multiple:\n");
for (auto& result : multi_result.results) {
fmt::print(" [{}] [{}] ", result.total_score, result.result_type.print());
lg::print(" [{}] [{}] ", result.total_score, result.result_type.print());
for (auto& tok : result.tokens) {
fmt::print("{} ", tok.print());
lg::print("{} ", tok.print());
}
fmt::print("\n");
lg::print("\n");
}
fmt::print("\n\n\n");
lg::print("\n\n\n");
}
*/
@@ -510,8 +512,8 @@ FieldReverseMultiLookupOutput TypeSystem::reverse_field_multi_lookup(
const FieldReverseLookupInput& input,
int max_count) const {
if (debug_reverse_lookup) {
fmt::print("reverse_field_lookup on {} offset {} deref {} stride {}\n", input.base_type.print(),
input.offset, input.deref.has_value(), input.stride);
lg::debug("reverse_field_lookup on {} offset {} deref {} stride {}", input.base_type.print(),
input.offset, input.deref.has_value(), input.stride);
}
FieldReverseMultiLookupOutput result;
common/type_system/TypeSpec.h
+7 -1
View File
@@ -94,7 +94,7 @@ class TypeSpec {
void modify_tag(const std::string& tag_name, const std::string& tag_value);
void add_or_modify_tag(const std::string& tag_name, const std::string& tag_value);
const std::string base_type() const { return m_type; }
const std::string& base_type() const { return m_type; }
bool has_single_arg() const {
if (m_arguments) {
@@ -132,6 +132,12 @@ class TypeSpec {
return m_arguments->back();
}
TypeSpec& last_arg() {
ASSERT(m_arguments);
ASSERT(!m_arguments->empty());
return m_arguments->back();
}
bool empty() const {
if (!m_arguments) {
return true;
common/type_system/TypeSystem.cpp
+253 -50
View File
@@ -7,8 +7,10 @@
#include "TypeSystem.h"
#include <algorithm>
#include <stdexcept>
#include "common/log/log.h"
#include "common/util/Assert.h"
#include "common/util/math_util.h"
@@ -18,11 +20,11 @@
namespace {
template <typename... Args>
[[noreturn]] void throw_typesystem_error(const std::string& str, Args&&... args) {
fmt::print(fg(fmt::color::crimson) | fmt::emphasis::bold, "-- Type Error! --\n");
lg::print(fg(fmt::color::crimson) | fmt::emphasis::bold, "-- Type Error! --\n");
if (!str.empty() && str.back() == '\n') {
fmt::print(fg(fmt::color::yellow), str, std::forward<Args>(args)...);
lg::print(fg(fmt::color::yellow), str, std::forward<Args>(args)...);
} else {
fmt::print(fg(fmt::color::yellow), str + '\n', std::forward<Args>(args)...);
lg::print(fg(fmt::color::yellow), str + '\n', std::forward<Args>(args)...);
}
throw std::runtime_error(
@@ -59,9 +61,12 @@ Type* TypeSystem::add_type(const std::string& name, std::unique_ptr<Type> type)
if (*kv->second != *type) {
// exists, and we are trying to change it!
if (m_allow_redefinition) {
fmt::print("[TypeSystem] Type {} was originally\n{}\nand is redefined as\n{}\n",
kv->second->get_name(), kv->second->print(), type->print());
// Check if the type is allowed to be redefined
if (m_allow_redefinition ||
std::find(m_types_allowed_to_be_redefined.begin(), m_types_allowed_to_be_redefined.end(),
kv->second->get_name()) != m_types_allowed_to_be_redefined.end()) {
lg::print("[TypeSystem] Type {} was originally\n{}\nand is redefined as\n{}\n",
kv->second->get_name(), kv->second->print(), type->print());
// extra dangerous, we have allowed type redefinition!
// keep the unique_ptr around, just in case somebody references this old type pointer.
@@ -365,8 +370,9 @@ bool TypeSystem::partially_defined_type_exists(const std::string& name) const {
return m_forward_declared_types.find(name) != m_forward_declared_types.end();
}
TypeSpec TypeSystem::make_array_typespec(const TypeSpec& element_type) const {
return TypeSpec("array", {element_type});
TypeSpec TypeSystem::make_array_typespec(const std::string& array_type,
const TypeSpec& element_type) const {
return TypeSpec(array_type, {element_type});
}
/*!
@@ -471,7 +477,7 @@ Type* TypeSystem::lookup_type_allow_partial_def(const std::string& name) const {
auto fwd_dec = m_forward_declared_types.find(current_name);
if (fwd_dec == m_forward_declared_types.end()) {
if (current_name == name) {
throw_typesystem_error("The type {} is unknown (2).\n", name);
throw_typesystem_error("The type '{}' is unknown (2).\n", name);
} else {
throw_typesystem_error("When looking up forward defined type {}, could not find a type {}.",
name, current_name);
@@ -509,13 +515,29 @@ int TypeSystem::get_load_size_allow_partial_def(const TypeSpec& ts) const {
return partial_def->get_load_size();
}
MethodInfo TypeSystem::override_method(Type* type,
const std::string& /*type_name*/,
const int method_id,
const std::optional<std::string>& docstring) {
// Lookup the method from the parent type
MethodInfo existing_info;
bool exists = try_lookup_method(type->get_parent(), method_id, &existing_info);
if (!exists) {
throw_typesystem_error("Trying to use override a method that has no parent declaration");
}
// use the existing ID.
return type->add_method({existing_info.id, existing_info.name, existing_info.type,
type->get_name(), existing_info.no_virtual, false, true, docstring});
}
MethodInfo TypeSystem::declare_method(const std::string& type_name,
const std::string& method_name,
const std::optional<std::string>& docstring,
bool no_virtual,
const TypeSpec& ts,
bool override_type) {
return declare_method(lookup_type(make_typespec(type_name)), method_name, no_virtual, ts,
override_type);
return declare_method(lookup_type(make_typespec(type_name)), method_name, docstring, no_virtual,
ts, override_type);
}
/*!
@@ -529,6 +551,7 @@ MethodInfo TypeSystem::declare_method(const std::string& type_name,
*/
MethodInfo TypeSystem::declare_method(Type* type,
const std::string& method_name,
const std::optional<std::string>& docstring,
bool no_virtual,
const TypeSpec& ts,
bool override_type,
@@ -537,7 +560,7 @@ MethodInfo TypeSystem::declare_method(Type* type,
if (override_type) {
throw_typesystem_error("Cannot use :replace option with a new method.");
}
return add_new_method(type, ts);
return add_new_method(type, ts, docstring);
}
// look up the method
@@ -557,7 +580,7 @@ MethodInfo TypeSystem::declare_method(Type* type,
// use the existing ID.
return type->add_method(
{existing_info.id, method_name, ts, type->get_name(), no_virtual, true});
{existing_info.id, method_name, ts, type->get_name(), no_virtual, true, false, docstring});
} else {
if (got_existing) {
// make sure we aren't changing anything.
@@ -587,16 +610,17 @@ MethodInfo TypeSystem::declare_method(Type* type,
return existing_info;
} else {
// add a new method!
return type->add_method(
{get_next_method_id(type), method_name, ts, type->get_name(), no_virtual, false});
return type->add_method({get_next_method_id(type), method_name, ts, type->get_name(),
no_virtual, false, false, docstring});
}
}
}
MethodInfo TypeSystem::define_method(const std::string& type_name,
const std::string& method_name,
const TypeSpec& ts) {
return define_method(lookup_type(make_typespec(type_name)), method_name, ts);
const TypeSpec& ts,
const std::optional<std::string>& docstring) {
return define_method(lookup_type(make_typespec(type_name)), method_name, ts, docstring);
}
/*!
@@ -610,9 +634,10 @@ MethodInfo TypeSystem::define_method(const std::string& type_name,
*/
MethodInfo TypeSystem::define_method(Type* type,
const std::string& method_name,
const TypeSpec& ts) {
const TypeSpec& ts,
const std::optional<std::string>& docstring) {
if (method_name == "new") {
return add_new_method(type, ts);
return add_new_method(type, ts, docstring);
}
// look up the method
@@ -640,7 +665,9 @@ MethodInfo TypeSystem::define_method(Type* type,
* If it turns out that other child methods can specialize arguments (seems like a bad idea), this
* may be generalized.
*/
MethodInfo TypeSystem::add_new_method(Type* type, const TypeSpec& ts) {
MethodInfo TypeSystem::add_new_method(Type* type,
const TypeSpec& ts,
const std::optional<std::string>& docstring) {
MethodInfo existing;
if (type->get_my_new_method(&existing)) {
// it exists!
@@ -653,7 +680,7 @@ MethodInfo TypeSystem::add_new_method(Type* type, const TypeSpec& ts) {
return existing;
} else {
return type->add_new_method({0, "new", ts, type->get_name()});
return type->add_new_method({0, "new", ts, type->get_name(), false, false, false, docstring});
}
}
@@ -943,8 +970,9 @@ int TypeSystem::add_field_to_type(StructureType* type,
int aligned_offset = align(offset, field_alignment);
field.mark_as_user_placed();
if (offset != aligned_offset) {
throw_typesystem_error("Tried to place field {} at {}, but it is not aligned correctly\n",
field_name, offset);
throw_typesystem_error(
"Tried to place field {} at {}, but it is not aligned correctly, requires {}\n",
field_name, offset, field_alignment);
}
}
@@ -1038,26 +1066,27 @@ void TypeSystem::add_builtin_types(GameVersion version) {
forward_declare_type_as("memory-usage-block", "basic");
// OBJECT
declare_method(obj_type, "new", false,
declare_method(obj_type, "new", {}, false,
make_function_typespec({"symbol", "type", "int"}, "_type_"), false);
declare_method(obj_type, "delete", false, make_function_typespec({"_type_"}, "none"), false);
declare_method(obj_type, "print", false, make_function_typespec({"_type_"}, "_type_"), false);
declare_method(obj_type, "inspect", false, make_function_typespec({"_type_"}, "_type_"), false);
declare_method(obj_type, "length", false, make_function_typespec({"_type_"}, "int"),
declare_method(obj_type, "delete", {}, false, make_function_typespec({"_type_"}, "none"), false);
declare_method(obj_type, "print", {}, false, make_function_typespec({"_type_"}, "_type_"), false);
declare_method(obj_type, "inspect", {}, false, make_function_typespec({"_type_"}, "_type_"),
false);
declare_method(obj_type, "length", {}, false, make_function_typespec({"_type_"}, "int"),
false); // todo - this integer type?
declare_method(obj_type, "asize-of", false, make_function_typespec({"_type_"}, "int"), false);
declare_method(obj_type, "copy", false, make_function_typespec({"_type_", "symbol"}, "_type_"),
false);
declare_method(obj_type, "relocate", false, make_function_typespec({"_type_", "int"}, "_type_"),
false);
declare_method(obj_type, "mem-usage", false,
declare_method(obj_type, "asize-of", {}, false, make_function_typespec({"_type_"}, "int"), false);
declare_method(obj_type, "copy", {}, false,
make_function_typespec({"_type_", "symbol"}, "_type_"), false);
declare_method(obj_type, "relocate", {}, false,
make_function_typespec({"_type_", "int"}, "_type_"), false);
declare_method(obj_type, "mem-usage", {}, false,
make_function_typespec({"_type_", "memory-usage-block", "int"}, "_type_"), false);
// STRUCTURE
// structure new doesn't support dynamic sizing, which is kinda weird - it grabs the size from
// the type. Dynamic structures use new-dynamic-structure, which is used exactly once ever.
declare_method(structure_type, "new", false, make_function_typespec({"symbol", "type"}, "_type_"),
false);
declare_method(structure_type, "new", {}, false,
make_function_typespec({"symbol", "type"}, "_type_"), false);
// structure_type is a field-less StructureType, so we have to do this to match the runtime.
// structure_type->override_size_in_memory(4);
@@ -1066,7 +1095,7 @@ void TypeSystem::add_builtin_types(GameVersion version) {
add_field_to_type(basic_type, "type", make_typespec("type"));
// the default new basic doesn't support dynamic sizing. anything dynamic will override this
// and then call (method object new) to do the dynamically-sized allocation.
declare_method(basic_type, "new", false, make_function_typespec({"symbol", "type"}, "_type_"),
declare_method(basic_type, "new", {}, false, make_function_typespec({"symbol", "type"}, "_type_"),
false);
// SYMBOL
@@ -1075,11 +1104,11 @@ void TypeSystem::add_builtin_types(GameVersion version) {
}
add_field_to_type(symbol_type, "value", make_typespec("object"));
// a new method which returns type none means new is illegal.
declare_method(symbol_type, "new", false, make_function_typespec({}, "none"), false);
declare_method(symbol_type, "new", {}, false, make_function_typespec({}, "none"), false);
// TYPE
builtin_structure_inherit(type_type);
declare_method(type_type, "new", false,
declare_method(type_type, "new", {}, false,
make_function_typespec({"symbol", "type", "int"}, "_type_"), false);
add_field_to_type(type_type, "symbol", make_typespec("symbol"));
add_field_to_type(type_type, "parent", make_typespec("type"));
@@ -1096,7 +1125,7 @@ void TypeSystem::add_builtin_types(GameVersion version) {
add_field_to_type(string_type, "data", make_typespec("uint8"), false, true); // todo integer type
// string is never deftype'd for the decompiler, so we need to manually give the constructor
// type here.
declare_method(string_type, "new", false,
declare_method(string_type, "new", {}, false,
make_function_typespec({"symbol", "type", "int", "string"}, "_type_"), false);
// FUNCTION
@@ -1125,7 +1154,7 @@ void TypeSystem::add_builtin_types(GameVersion version) {
// todo
builtin_structure_inherit(array_type);
declare_method(array_type, "new", false,
declare_method(array_type, "new", {}, false,
make_function_typespec({"symbol", "type", "type", "int"}, "_type_"), false);
// array has: number, number, type
add_field_to_type(array_type, "length", make_typespec("int32"));
@@ -1135,7 +1164,7 @@ void TypeSystem::add_builtin_types(GameVersion version) {
// pair
pair_type->override_offset(2);
declare_method(pair_type, "new", false,
declare_method(pair_type, "new", {}, false,
make_function_typespec({"symbol", "type", "object", "object"}, "_type_"), false);
add_field_to_type(pair_type, "car", make_typespec("object"));
add_field_to_type(pair_type, "cdr", make_typespec("object"));
@@ -1153,7 +1182,7 @@ void TypeSystem::add_builtin_types(GameVersion version) {
add_field_to_type(file_stream_type, "mode", make_typespec("symbol"));
add_field_to_type(file_stream_type, "name", make_typespec("string"));
add_field_to_type(file_stream_type, "file", make_typespec("uint32"));
declare_method(file_stream_type, "new", false,
declare_method(file_stream_type, "new", {}, false,
make_function_typespec({"symbol", "type", "string", "symbol"}, "_type_"), false);
}
@@ -1256,6 +1285,7 @@ int TypeSystem::get_size_in_type(const Field& field) const {
"Attempted to use `{}` inline, this probably isn't what you wanted.\n",
field_type->get_name());
}
// TODO - crashes LSP
ASSERT(field_type->is_reference());
return field.array_size() * align(field_type->get_size_in_memory(),
field_type->get_inline_array_stride_alignment());
@@ -1294,6 +1324,163 @@ int TypeSystem::get_size_in_type(const Field& field) const {
}
}
std::vector<std::string> TypeSystem::get_all_type_names() {
std::vector<std::string> results = {};
for (const auto& [type_name, type_info] : m_types) {
results.push_back(type_name);
}
return results;
}
std::vector<std::string> TypeSystem::search_types_by_parent_type(
const std::string& parent_type,
const std::optional<std::vector<std::string>>& existing_matches) {
std::vector<std::string> results = {};
// If we've been given a list of already matched types, narrow it down from there, otherwise
// iterate through the entire map
if (existing_matches) {
for (const auto& type_name : existing_matches.value()) {
if (typecheck_base_types(parent_type, type_name, false)) {
results.push_back(type_name);
}
}
} else {
for (const auto& [type_name, type_info] : m_types) {
// Only NullType's have no parent
if (!type_info->has_parent()) {
continue;
}
if (typecheck_base_types(parent_type, type_name, false)) {
results.push_back(type_name);
}
}
}
return results;
}
std::vector<std::string> TypeSystem::search_types_by_minimum_method_id(
const int minimum_method_id,
const std::optional<std::vector<std::string>>& existing_matches) {
std::vector<std::string> results = {};
// If we've been given a list of already matched types, narrow it down from there, otherwise
// iterate through the entire map
if (existing_matches) {
for (const auto& type_name : existing_matches.value()) {
if (get_type_method_count(type_name) - 1 >= minimum_method_id) {
results.push_back(type_name);
}
}
} else {
for (const auto& [type_name, type_info] : m_types) {
if (get_type_method_count(type_name) - 1 >= minimum_method_id) {
results.push_back(type_name);
}
}
}
return results;
}
std::vector<std::string> TypeSystem::search_types_by_size(
const int min_size,
const std::optional<int> max_size,
const std::optional<std::vector<std::string>>& existing_matches) {
std::vector<std::string> results = {};
// If we've been given a list of already matched types, narrow it down from there, otherwise
// iterate through the entire map
if (existing_matches) {
for (const auto& type_name : existing_matches.value()) {
const auto size_of_type = m_types[type_name]->get_size_in_memory();
if (max_size && size_of_type <= max_size && size_of_type >= min_size) {
results.push_back(type_name);
} else if (!max_size && size_of_type == min_size) {
results.push_back(type_name);
}
}
} else {
for (const auto& [type_name, type_info] : m_types) {
// Only NullType's have no parent
if (!type_info->has_parent()) {
continue;
}
const auto size_of_type = m_types[type_name]->get_size_in_memory();
if (max_size && size_of_type <= max_size && size_of_type >= min_size) {
results.push_back(type_name);
} else if (!max_size && size_of_type == min_size) {
results.push_back(type_name);
}
}
}
return results;
}
std::vector<std::string> TypeSystem::search_types_by_fields(
const std::vector<TypeSearchFieldInput>& search_fields,
const std::optional<std::vector<std::string>>& existing_matches) {
// TODO - maybe support partial matches eventually
std::vector<std::string> results = {};
if (existing_matches) {
for (const auto& type_name : existing_matches.value()) {
// For each type, look at it's fields
if (dynamic_cast<StructureType*>(m_types[type_name].get()) != nullptr) {
bool type_valid = true;
auto struct_type = dynamic_cast<StructureType*>(m_types[type_name].get());
for (const auto& req_field : search_fields) {
bool field_valid = false;
// iterate through the type's fields until one is found with the right offset
// once found, check the underlying type name, if it doesn't match it's invalid
// if we don't find one with that offset, it's also invalid
for (const auto& type_field : struct_type->fields()) {
if (type_field.offset() == req_field.field_offset &&
type_field.type().base_type() == req_field.field_type_name) {
field_valid = true;
break;
}
}
if (!field_valid) {
type_valid = false;
break;
}
}
if (type_valid) {
results.push_back(type_name);
}
}
}
} else {
for (const auto& [type_name, type_info] : m_types) {
// For each type, look at it's fields
if (dynamic_cast<StructureType*>(type_info.get()) != nullptr) {
bool type_valid = true;
auto struct_type = dynamic_cast<StructureType*>(type_info.get());
for (const auto& req_field : search_fields) {
bool field_valid = false;
// iterate through the type's fields until one is found with the right offset
// once found, check the underlying type name, if it doesn't match it's invalid
// if we don't find one with that offset, it's also invalid
for (const auto& type_field : struct_type->fields()) {
if (type_field.offset() == req_field.field_offset &&
type_field.type().base_type() == req_field.field_type_name) {
field_valid = true;
break;
}
}
if (!field_valid) {
type_valid = false;
break;
}
}
if (type_valid) {
results.push_back(type_name);
}
}
}
}
return results;
}
/*!
* Add a simple structure type - don't use this outside of add_builtin_types as it forces you to do
* things in the wrong order.
@@ -1400,11 +1587,11 @@ bool TypeSystem::typecheck_and_throw(const TypeSpec& expected,
if (!success) {
if (print_on_error) {
if (error_source_name.empty()) {
fmt::print("[TypeSystem] Got type \"{}\" when expecting \"{}\"\n", actual.print(),
expected.print());
lg::print("[TypeSystem] Got type \"{}\" when expecting \"{}\"\n", actual.print(),
expected.print());
} else {
fmt::print("[TypeSystem] For {}, got type \"{}\" when expecting \"{}\"\n",
error_source_name, actual.print(), expected.print());
lg::print("[TypeSystem] For {}, got type \"{}\" when expecting \"{}\"\n", error_source_name,
actual.print(), expected.print());
}
}
@@ -1705,6 +1892,8 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const {
}
std::string methods_string;
// New Method
auto new_info = type->get_new_method_defined_for_type();
if (new_info) {
methods_string.append("(new (");
@@ -1725,7 +1914,13 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const {
methods_string.append("0)\n ");
}
// Rest of methods
for (auto& info : type->get_methods_defined_for_type()) {
// check if we only override the docstring
if (info.only_overrides_docstring) {
continue;
}
methods_string.append(fmt::format("({} (", info.name));
for (size_t i = 0; i < info.type.arg_count() - 1; i++) {
methods_string.append(info.type.get_arg(i).print());
@@ -1740,7 +1935,7 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const {
methods_string.append(":no-virtual ");
}
if (info.overrides_method_type_of_parent) {
if (info.overrides_parent) {
methods_string.append(":replace ");
}
@@ -1788,7 +1983,11 @@ std::string TypeSystem::generate_deftype_footer(const Type* type) const {
std::string TypeSystem::generate_deftype_for_structure(const StructureType* st) const {
std::string result;
result += fmt::format("(deftype {} ({})\n (", st->get_name(), st->get_parent());
result += fmt::format("(deftype {} ({})\n", st->get_name(), st->get_parent());
if (st->m_metadata.docstring) {
result += fmt::format(" \"{}\"\n", st->m_metadata.docstring.value());
}
result += " (";
int longest_field_name = 0;
int longest_type_name = 0;
@@ -1879,7 +2078,11 @@ std::string TypeSystem::generate_deftype_for_structure(const StructureType* st)
std::string TypeSystem::generate_deftype_for_bitfield(const BitFieldType* type) const {
std::string result;
result += fmt::format("(deftype {} ({})\n (", type->get_name(), type->get_parent());
result += fmt::format("(deftype {} ({})\n", type->get_name(), type->get_parent());
if (type->m_metadata.docstring) {
result += fmt::format(" \"{}\"\n", type->m_metadata.docstring.value());
}
result += " (";
int longest_field_name = 0;
int longest_type_name = 0;
common/type_system/TypeSystem.h
+44 -4
View File
@@ -143,7 +143,7 @@ class TypeSystem {
bool fully_defined_type_exists(const TypeSpec& type) const;
bool partially_defined_type_exists(const std::string& name) const;
TypeSpec make_typespec(const std::string& name) const;
TypeSpec make_array_typespec(const TypeSpec& element_type) const;
TypeSpec make_array_typespec(const std::string& array_type, const TypeSpec& element_type) const;
TypeSpec make_function_typespec(const std::vector<std::string>& arg_types,
const std::string& return_type) const;
@@ -160,22 +160,34 @@ class TypeSystem {
int get_load_size_allow_partial_def(const TypeSpec& ts) const;
MethodInfo override_method(Type* type,
const std::string& type_name,
const int method_id,
const std::optional<std::string>& docstring);
MethodInfo declare_method(const std::string& type_name,
const std::string& method_name,
const std::optional<std::string>& docstring,
bool no_virtual,
const TypeSpec& ts,
bool override_type);
MethodInfo declare_method(Type* type,
const std::string& method_name,
const std::optional<std::string>& docstring,
bool no_virtual,
const TypeSpec& ts,
bool override_type,
int id = -1);
MethodInfo define_method(const std::string& type_name,
const std::string& method_name,
const TypeSpec& ts);
MethodInfo define_method(Type* type, const std::string& method_name, const TypeSpec& ts);
MethodInfo add_new_method(Type* type, const TypeSpec& ts);
const TypeSpec& ts,
const std::optional<std::string>& docstring);
MethodInfo define_method(Type* type,
const std::string& method_name,
const TypeSpec& ts,
const std::optional<std::string>& docstring);
MethodInfo add_new_method(Type* type,
const TypeSpec& ts,
const std::optional<std::string>& docstring);
MethodInfo lookup_method(const std::string& type_name, const std::string& method_name) const;
MethodInfo lookup_method(const std::string& type_name, int method_id) const;
bool try_lookup_method(const Type* type, const std::string& method_name, MethodInfo* info) const;
@@ -251,6 +263,33 @@ class TypeSystem {
int get_size_in_type(const Field& field) const;
void add_type_to_allowed_redefinition_list(const std::string& type_name) {
m_types_allowed_to_be_redefined.push_back(type_name);
}
std::vector<std::string> get_all_type_names();
std::vector<std::string> search_types_by_parent_type(
const std::string& parent_type,
const std::optional<std::vector<std::string>>& existing_matches = {});
std::vector<std::string> search_types_by_minimum_method_id(
const int minimum_method_id,
const std::optional<std::vector<std::string>>& existing_matches = {});
std::vector<std::string> search_types_by_size(
const int min_size,
const std::optional<int> max_size,
const std::optional<std::vector<std::string>>& existing_matches = {});
struct TypeSearchFieldInput {
std::string field_type_name;
int field_offset;
};
std::vector<std::string> search_types_by_fields(
const std::vector<TypeSearchFieldInput>& search_fields,
const std::optional<std::vector<std::string>>& existing_matches = {});
private:
std::string lca_base(const std::string& a, const std::string& b) const;
bool typecheck_base_types(const std::string& expected,
@@ -276,6 +315,7 @@ class TypeSystem {
std::vector<std::unique_ptr<Type>> m_old_types;
std::vector<std::string> m_types_allowed_to_be_redefined;
bool m_allow_redefinition = false;
};
common/type_system/defenum.cpp
+21 -8
View File
@@ -9,7 +9,9 @@
#include "deftype.h"
#include "common/goos/ParseHelpers.h"
#include "common/log/log.h"
#include "common/util/BitUtils.h"
#include "common/util/string_util.h"
#include "third-party/fmt/core.h"
@@ -43,7 +45,9 @@ std::string symbol_string(const goos::Object& obj) {
} // namespace
EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts) {
EnumType* parse_defenum(const goos::Object& defenum,
TypeSystem* ts,
DefinitionMetadata* symbol_metadata) {
// default enum type will be int32.
TypeSpec base_type = ts->make_typespec("int32");
bool is_bitfield = false;
@@ -53,6 +57,14 @@ EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts) {
auto& enum_name_obj = car(iter);
iter = cdr(iter);
// check for docstring
if (iter->is_pair() && car(iter).is_string()) {
// TODO - docstring - store and use docstring if coming from the compiler
if (symbol_metadata) {
symbol_metadata->docstring = str_util::trim_newline_indents(car(iter).as_string()->data);
}
iter = cdr(iter);
}
if (!enum_name_obj.is_symbol()) {
throw std::runtime_error("defenum must be given a symbol as its name");
@@ -74,8 +86,8 @@ EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts) {
} else if (symbol_string(option_value) == "#f") {
is_bitfield = false;
} else {
fmt::print("Invalid option {} to :bitfield option.\n", option_value.print());
throw std::runtime_error("invalid bitfield option");
throw std::runtime_error(
fmt::format("Invalid option {} to :bitfield option.\n", option_value.print()));
}
} else if (option_name == ":copy-entries") {
auto other_info = ts->try_enum_lookup(parse_typespec(ts, option_value));
@@ -90,8 +102,7 @@ EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts) {
entries[e.first] = e.second;
}
} else {
fmt::print("Unknown option {} for defenum.\n", option_name);
throw std::runtime_error("unknown option for defenum");
throw std::runtime_error(fmt::format("Unknown option {} for defenum.\n", option_name));
}
if (iter->is_pair()) {
@@ -115,12 +126,13 @@ EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts) {
if (!rest->is_empty_list()) {
auto& value = car(rest);
if (!value.is_int()) {
fmt::print("Expected integer for enum value, got {}\n", value.print());
throw std::runtime_error(
fmt::format("Expected integer for enum value, got {}\n", value.print()));
}
auto entry_val = value.integer_obj.value;
if (!integer_fits(entry_val, type->get_load_size(), type->get_load_signed())) {
fmt::print("Integer {} does not fit inside a {}\n", entry_val, type->get_name());
lg::warn("Integer {} does not fit inside a {}", entry_val, type->get_name());
}
if (!entries.size()) {
@@ -130,7 +142,8 @@ EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts) {
rest = cdr(rest);
if (!rest->is_empty_list()) {
fmt::print("Got too many items in defenum {} entry {}\n", name, entry_name);
throw std::runtime_error(
fmt::format("Got too many items in defenum {} entry {}\n", name, entry_name));
}
entries[entry_name] = entry_val;
common/type_system/defenum.h
+3 -1
View File
@@ -10,4 +10,6 @@
#include "common/goos/Object.h"
EnumType* parse_defenum(const goos::Object& defenum, TypeSystem* ts);
EnumType* parse_defenum(const goos::Object& defenum,
TypeSystem* ts,
DefinitionMetadata* symbol_metadata);
common/type_system/deftype.cpp
+206 -62
View File
@@ -6,7 +6,12 @@
#include "deftype.h"
#include <unordered_map>
#include "common/goos/ParseHelpers.h"
#include "common/log/log.h"
#include "common/type_system/state.h"
#include "common/util/string_util.h"
#include "third-party/fmt/core.h"
@@ -195,42 +200,148 @@ void add_bitfield(BitFieldType* bitfield_type, TypeSystem* ts, const goos::Objec
skip_in_decomp);
}
void declare_method(Type* type, TypeSystem* type_system, const goos::Object& def) {
struct StructureDefResult {
TypeFlags flags;
bool generate_runtime_type = true;
bool pack_me = false;
bool allow_misaligned = false;
bool final = false;
bool always_stack_singleton = false;
std::unordered_map<std::string, std::unordered_map<std::string, DefinitionMetadata>>
virtual_state_definitions;
std::unordered_map<std::string, std::unordered_map<std::string, DefinitionMetadata>>
state_definitions;
void append_virtual_state_def(const std::string& state_name,
const StateHandler handler,
DefinitionMetadata data) {
if (virtual_state_definitions.count(state_name) == 0) {
virtual_state_definitions[state_name] = std::unordered_map<std::string, DefinitionMetadata>();
}
virtual_state_definitions[state_name][handler_kind_to_name(handler)] = data;
}
void append_state_def(const std::string& state_name,
const StateHandler handler,
DefinitionMetadata data) {
if (state_definitions.count(state_name) == 0) {
state_definitions[state_name] = std::unordered_map<std::string, DefinitionMetadata>();
}
state_definitions[state_name][handler_kind_to_name(handler)] = data;
}
};
void declare_method(Type* type,
TypeSystem* type_system,
const goos::Object& def,
StructureDefResult& struct_def) {
for_each_in_list(def, [&](const goos::Object& _obj) {
auto obj = &_obj;
// (name args return-type [:no-virtual] [:replace] [:state] [id])
auto method_name = symbol_string(car(obj));
obj = cdr(obj);
auto& args = car(obj);
obj = cdr(obj);
auto& return_type = car(obj);
obj = cdr(obj);
// (name args return-type [:no-virtual] [:replace] [:state] [:behavior] [id])
// or alternatively
// (:override-doc "new-docstring" [id])
// - this effectively does a :replace without having to re-define the name and signature and
// keep that in-sync
std::string method_name;
TypeSpec function_typespec("function");
std::optional<std::string> docstring;
goos::Object args;
goos::Object return_type;
bool no_virtual = false;
bool replace_method = false;
TypeSpec function_typespec("function");
bool overriding_doc = false;
if (!obj->is_empty_list() && car(obj).is_symbol(":no-virtual")) {
if (!obj->is_empty_list() && car(obj).is_symbol(":override-doc")) {
obj = cdr(obj);
no_virtual = true;
if (car(obj).is_string()) {
docstring = str_util::trim_newline_indents(car(obj).as_string()->data);
overriding_doc = true;
obj = cdr(obj);
} else {
throw std::runtime_error("Specified :override-doc with no docstring!");
}
}
if (!obj->is_empty_list() && car(obj).is_symbol(":replace")) {
obj = cdr(obj);
replace_method = true;
}
if (!obj->is_empty_list() && car(obj).is_symbol(":state")) {
obj = cdr(obj);
function_typespec = TypeSpec("state");
}
if (!obj->is_empty_list() && car(obj).is_symbol(":behavior")) {
obj = cdr(obj);
function_typespec.add_new_tag("behavior", symbol_string(obj->as_pair()->car));
if (!overriding_doc) {
// name
method_name = symbol_string(car(obj));
obj = cdr(obj);
// docstring
if (obj->is_pair() && car(obj).is_string()) {
docstring = str_util::trim_newline_indents(car(obj).as_string()->data);
obj = cdr(obj);
}
// args
args = car(obj);
obj = cdr(obj);
// return type
return_type = car(obj);
obj = cdr(obj);
// Iterate through the remainder of the form's supported keywords
// this int is assumed to be the id, and always at the end!
//
// Doing it like this makes the ordering not critical
while (!obj->is_empty_list() && car(obj).is_symbol()) {
const auto& keyword = car(obj).as_symbol()->name;
if (keyword == ":no-virtual") {
no_virtual = true;
} else if (keyword == ":replace") {
replace_method = true;
} else if (keyword == ":state") {
auto behavior_tag = function_typespec.try_get_tag("behavior");
function_typespec = TypeSpec("state");
if (behavior_tag) {
function_typespec.add_new_tag("behavior", behavior_tag.value());
}
// parse state docstrings if available
if (car(cdr(obj)).is_list()) {
obj = cdr(obj);
auto docstring_list = &car(obj);
auto elem = docstring_list;
while (!elem->is_empty_list() && car(elem).is_symbol()) {
const auto& handler = car(elem).as_symbol()->name;
const auto handler_kind = handler_keyword_to_kind(handler);
// Get the docstring
elem = cdr(elem);
if (!car(elem).is_string()) {
throw std::runtime_error("Missing a docstring for a state handler!");
}
DefinitionMetadata def_meta;
// TODO - definition location info
def_meta.docstring = car(elem).as_string()->data;
struct_def.append_virtual_state_def(method_name, handler_kind, def_meta);
elem = cdr(elem);
}
}
} else if (keyword == ":behavior") {
obj = cdr(obj);
if (!car(obj).is_symbol()) {
lg::print(
":behavior tag used without providing the process type name in a method "
"declaration. {}::{}\n",
type->get_name(), method_name.c_str());
throw std::runtime_error("Bad usage of :behavior in a method declaration");
}
function_typespec.add_new_tag("behavior", symbol_string(obj->as_pair()->car));
}
obj = cdr(obj);
}
// fill in args now that we've finalized the function spec
for_each_in_list(args, [&](const goos::Object& o) {
function_typespec.add_arg(parse_typespec(type_system, o));
});
function_typespec.add_arg(parse_typespec(type_system, return_type));
}
// determine the method id, it should be the last in the list
int id = -1;
if (!obj->is_empty_list() && car(obj).is_int()) {
auto& id_obj = car(obj);
@@ -239,35 +350,62 @@ void declare_method(Type* type, TypeSystem* type_system, const goos::Object& def
}
if (!obj->is_empty_list()) {
throw std::runtime_error("too many things in method def: " + def.print());
throw std::runtime_error("found symbols after the `id` in a method defintion: " +
def.print());
}
for_each_in_list(args, [&](const goos::Object& o) {
function_typespec.add_arg(parse_typespec(type_system, o));
});
function_typespec.add_arg(parse_typespec(type_system, return_type));
auto info = type_system->declare_method(type, method_name, no_virtual, function_typespec,
replace_method, id);
MethodInfo info;
if (overriding_doc) {
info = type_system->override_method(type, method_name, id, docstring);
} else {
info = type_system->declare_method(type, method_name, docstring, no_virtual,
function_typespec, replace_method, id);
}
// check the method assert
if (id != -1) {
// method id assert!
if (id != info.id) {
printf("WARNING - ID assert failed on method %s of type %s (wanted %d got %d)\n",
method_name.c_str(), type->get_name().c_str(), id, info.id);
lg::print("WARNING - ID assert failed on method {} of type {} (wanted {} got {})\n",
method_name.c_str(), type->get_name().c_str(), id, info.id);
throw std::runtime_error("Method ID assert failed");
}
}
});
}
void declare_state(Type* type, TypeSystem* type_system, const goos::Object& def) {
void declare_state(Type* type,
TypeSystem* type_system,
const goos::Object& def,
StructureDefResult& struct_def) {
for_each_in_list(def, [&](const goos::Object& _obj) {
auto obj = &_obj;
if (obj->is_list()) {
// (name ,@args)
// (name [(:event "docstring"...)] ,@args)
auto state_name = symbol_string(car(obj));
if (!cdr(obj)->is_empty_list() && car(cdr(obj)).is_list()) {
obj = cdr(obj);
auto docstring_list = &car(obj);
auto elem = docstring_list;
while (!elem->is_empty_list() && car(elem).is_symbol()) {
const auto& handler = car(elem).as_symbol()->name;
const auto handler_kind = handler_keyword_to_kind(handler);
// Get the docstring
elem = cdr(elem);
if (!car(elem).is_string()) {
throw std::runtime_error("Missing a docstring for a state handler!");
}
DefinitionMetadata def_meta;
// TODO - definition location info
def_meta.docstring = car(elem).as_string()->data;
struct_def.append_state_def(state_name, handler_kind, def_meta);
elem = cdr(elem);
}
}
auto args = cdr(obj);
TypeSpec state_typespec("state");
@@ -290,15 +428,6 @@ void declare_state(Type* type, TypeSystem* type_system, const goos::Object& def)
});
}
struct StructureDefResult {
TypeFlags flags;
bool generate_runtime_type = true;
bool pack_me = false;
bool allow_misaligned = false;
bool final = false;
bool always_stack_singleton = false;
};
StructureDefResult parse_structure_def(
StructureType* type,
TypeSystem* ts,
@@ -327,9 +456,9 @@ StructureDefResult parse_structure_def(
auto list_name = symbol_string(first);
if (list_name == ":methods") {
declare_method(type, ts, *opt_list);
declare_method(type, ts, *opt_list, result);
} else if (list_name == ":states") {
declare_state(type, ts, *opt_list);
declare_state(type, ts, *opt_list, result);
} else {
throw std::runtime_error("Invalid option list in field specification: " +
car(rest).print());
@@ -396,15 +525,15 @@ StructureDefResult parse_structure_def(
fmt::format("Process heap underflow in type {}: heap-base is {} vs. auto-detected {}",
type->get_name(), flags.heap_base, auto_hb));
//} else if (flags.heap_base != auto_hb) {
// fmt::print("Type {} has manual heap-base ({} vs {}). This is fine. \n", type->get_name(),
// lg::print("Type {} has manual heap-base ({} vs {}). This is fine. \n", type->get_name(),
// flags.heap_base, auto_hb);
}
}
if (size_assert != -1 && flags.size != u16(size_assert)) {
throw std::runtime_error("Type " + type->get_name() + " came out to size " +
std::to_string(int(flags.size)) + " but size-assert was set to " +
std::to_string(size_assert));
throw std::runtime_error(
fmt::format("Type {} came out to size {}[{:#x}] but size-assert was set to {}",
type->get_name(), int(flags.size), int(flags.size), size_assert));
}
flags.methods = ts->get_next_method_id(type);
@@ -453,7 +582,8 @@ BitFieldTypeDefResult parse_bitfield_type_def(BitFieldType* type,
opt_list = cdr(opt_list);
if (symbol_string(first) == ":methods") {
declare_method(type, ts, *opt_list);
auto dummy = StructureDefResult();
declare_method(type, ts, *opt_list, dummy);
} else {
throw std::runtime_error("Invalid option list in field specification: " +
car(rest).print());
@@ -491,9 +621,9 @@ BitFieldTypeDefResult parse_bitfield_type_def(BitFieldType* type,
}
if (size_assert != -1 && flags.size != u16(size_assert)) {
throw std::runtime_error("Type " + type->get_name() + " came out to size " +
std::to_string(int(flags.size)) + " but size-assert was set to " +
std::to_string(size_assert));
throw std::runtime_error(
fmt::format("Type {} came out to size {}[{:#x}] but size-assert was set to {}",
type->get_name(), int(flags.size), int(flags.size), size_assert));
}
flags.methods = ts->get_next_method_id(type);
@@ -566,6 +696,7 @@ TypeSpec parse_typespec(const TypeSystem* type_system, const goos::Object& src)
DeftypeResult parse_deftype(const goos::Object& deftype,
TypeSystem* ts,
std::unordered_map<goos::HeapObject*, goos::Object>* constants) {
DefinitionMetadata symbol_metadata;
std::unordered_map<goos::HeapObject*, goos::Object> no_consts;
auto& constants_to_use = no_consts;
if (constants != nullptr) {
@@ -578,6 +709,11 @@ DeftypeResult parse_deftype(const goos::Object& deftype,
iter = cdr(iter);
auto& parent_list_obj = car(iter);
iter = cdr(iter);
// check for docstring
if (iter->is_pair() && car(iter).is_string()) {
symbol_metadata.docstring = str_util::trim_newline_indents(car(iter).as_string()->data);
iter = cdr(iter);
}
auto& field_list_obj = car(iter);
iter = cdr(iter);
auto& options_obj = *iter;
@@ -590,9 +726,11 @@ DeftypeResult parse_deftype(const goos::Object& deftype,
auto parent_type_name = deftype_parent_list(parent_list_obj);
auto parent_type = ts->make_typespec(parent_type_name);
DeftypeResult result;
std::optional<StructureDefResult> structure_result;
if (is_type("basic", parent_type, ts)) {
auto new_type = std::make_unique<BasicType>(parent_type_name, name, false, 0);
new_type->m_metadata = symbol_metadata;
auto pto = dynamic_cast<BasicType*>(ts->lookup_type(parent_type));
ASSERT(pto);
if (pto->final()) {
@@ -606,18 +744,15 @@ DeftypeResult parse_deftype(const goos::Object& deftype,
parse_structure_def(new_type.get(), ts, field_list_obj, options_obj, constants_to_use);
result.flags = sr.flags;
result.create_runtime_type = sr.generate_runtime_type;
structure_result = sr;
if (sr.pack_me) {
new_type->set_pack(true);
}
if (sr.allow_misaligned) {
fmt::print(
"[TypeSystem] :allow-misaligned was set on {}, which is a basic and cannot "
"be misaligned\n",
name);
throw std::runtime_error("invalid pack option on basic");
new_type->set_allow_misalign(true);
}
if (sr.always_stack_singleton) {
fmt::print(
lg::print(
"[TypeSystem] :always-stack-singleton was set on {}, which is a basic and cannot "
"be a stack singleton\n",
name);
@@ -630,6 +765,7 @@ DeftypeResult parse_deftype(const goos::Object& deftype,
ts->add_type(name, std::move(new_type));
} else if (is_type("structure", parent_type, ts)) {
auto new_type = std::make_unique<StructureType>(parent_type_name, name, false, false, false, 0);
new_type->m_metadata = symbol_metadata;
auto pto = dynamic_cast<StructureType*>(ts->lookup_type(parent_type));
ASSERT(pto);
new_type->inherit(pto);
@@ -638,6 +774,7 @@ DeftypeResult parse_deftype(const goos::Object& deftype,
parse_structure_def(new_type.get(), ts, field_list_obj, options_obj, constants_to_use);
result.flags = sr.flags;
result.create_runtime_type = sr.generate_runtime_type;
structure_result = sr;
if (sr.pack_me) {
new_type->set_pack(true);
}
@@ -658,6 +795,7 @@ DeftypeResult parse_deftype(const goos::Object& deftype,
ASSERT(pto);
auto new_type = std::make_unique<BitFieldType>(
parent_type_name, name, pto->get_size_in_memory(), pto->get_load_signed());
new_type->m_metadata = symbol_metadata;
auto parent_value = dynamic_cast<ValueType*>(pto);
ASSERT(parent_value);
new_type->inherit(parent_value);
@@ -673,5 +811,11 @@ DeftypeResult parse_deftype(const goos::Object& deftype,
result.type = ts->make_typespec(name);
result.type_info = ts->lookup_type(result.type);
if (structure_result) {
result.type_info->m_state_definition_meta = structure_result->state_definitions;
result.type_info->m_virtual_state_definition_meta = structure_result->virtual_state_definitions;
}
return result;
}
common/type_system/state.cpp
+21
View File
@@ -18,6 +18,11 @@ TypeSpec state_to_go_function(const TypeSpec& state_type, const TypeSpec& return
return result;
}
StateHandler handler_keyword_to_kind(std::string keyword) {
// Remove the first character (should be a :)
return handler_name_to_kind(keyword.erase(0, 1));
}
StateHandler handler_name_to_kind(const std::string& name) {
if (name == "enter") {
return StateHandler::ENTER;
@@ -87,6 +92,22 @@ TypeSpec get_state_handler_type(StateHandler kind, const TypeSpec& state_type) {
return result;
}
std::vector<std::string> get_state_handler_arg_names(StateHandler kind) {
switch (kind) {
case StateHandler::CODE:
// can have args, but are arbitrary
case StateHandler::ENTER:
case StateHandler::TRANS:
case StateHandler::POST:
case StateHandler::EXIT:
return {};
case StateHandler::EVENT:
return {"proc", "arg1", "event-type", "event"};
default:
ASSERT(false);
}
}
namespace {
TypeSpec func_to_state_type(const TypeSpec& func_type, const TypeSpec& proc_type) {
TypeSpec result("state");
common/type_system/state.h
+2
View File
@@ -13,10 +13,12 @@ enum class StateHandler { ENTER, EXIT, CODE, TRANS, POST, EVENT };
class TypeSystem;
TypeSpec state_to_go_function(const TypeSpec& state_type, const TypeSpec& return_type);
StateHandler handler_keyword_to_kind(std::string keyword);
StateHandler handler_name_to_kind(const std::string& name);
std::string handler_kind_to_name(StateHandler kind);
TypeSpec get_state_handler_type(const std::string& handler_name, const TypeSpec& state_type);
TypeSpec get_state_handler_type(StateHandler kind, const TypeSpec& state_type);
std::vector<std::string> get_state_handler_arg_names(StateHandler kind);
std::optional<TypeSpec> get_state_type_from_enter_and_code(const TypeSpec& enter_func_type,
const TypeSpec& code_func_type,
common/util/Assert.cpp
+9 -6
View File
@@ -6,20 +6,23 @@
#include <cstdlib>
#include <string_view>
#include "common/log/log.h"
void private_assert_failed(const char* expr,
const char* file,
int line,
const char* function,
const char* msg) {
if (!msg || msg[0] == '\0') {
fprintf(stderr, "Assertion failed: '%s'\n\tSource: %s:%d\n\tFunction: %s\n", expr, file, line,
function);
std::string log = fmt::format("Assertion failed: '{}'\n\tSource: {}:{}\n\tFunction: {}\n", expr,
file, line, function);
lg::die(log);
} else {
fprintf(stderr, "Assertion failed: '%s'\n\tMessage: %s\n\tSource: %s:%d\n\tFunction: %s\n",
expr, msg, file, line, function);
std::string log =
fmt::format("Assertion failed: '{}'\n\tMessage: {}\n\tSource: {}:{}\n\tFunction: {}\n",
expr, msg, file, line, function);
lg::die(log);
}
fflush(stdout); // ensure any stdout logs are flushed before we terminate
fflush(stderr);
abort();
}
common/util/Assert.h
+5
View File
@@ -28,9 +28,14 @@
#define ASSERT(EX) \
(void)((EX) || (private_assert_failed(#EX, __FILE__, __LINE__, __PRETTY_FUNCTION__), 0))
#define ASSERT_NOT_REACHED() \
(void)((private_assert_failed("not reached", __FILE__, __LINE__, __PRETTY_FUNCTION__), 0))
#define ASSERT_MSG(EXPR, STR) \
(void)((EXPR) || (private_assert_failed(#EXPR, __FILE__, __LINE__, __PRETTY_FUNCTION__, STR), 0))
#define ASSERT_NOT_REACHED_MSG(STR) \
(void)((private_assert_failed("not reached", __FILE__, __LINE__, __PRETTY_FUNCTION__, STR), 0))
#else
#define ASSERT(EX) ((void)0)
common/util/FileUtil.cpp
+58 -6
View File
@@ -5,6 +5,7 @@
#include "FileUtil.h"
#include <algorithm>
#include <cstdio> /* defines FILENAME_MAX */
#include <cstdlib>
#include <fstream>
@@ -76,6 +77,11 @@ fs::path get_user_memcard_dir(GameVersion game_version) {
return get_user_config_dir() / game_version_name / "saves";
}
fs::path get_user_misc_dir(GameVersion game_version) {
auto game_version_name = game_version_names[game_version];
return get_user_config_dir() / game_version_name / "misc";
}
struct {
bool initialized = false;
fs::path path_to_data;
@@ -139,7 +145,7 @@ bool setup_project_path(std::optional<fs::path> project_path_override) {
if (project_path_override) {
gFilePathInfo.path_to_data = *project_path_override;
gFilePathInfo.initialized = true;
fmt::print("Using explicitly set project path: {}\n", project_path_override->string());
lg::info("Using explicitly set project path: {}", project_path_override->string());
return true;
}
@@ -147,7 +153,7 @@ bool setup_project_path(std::optional<fs::path> project_path_override) {
if (data_path) {
gFilePathInfo.path_to_data = *data_path;
gFilePathInfo.initialized = true;
fmt::print("Using data path: {}\n", data_path->string());
lg::info("Using data path: {}", data_path->string());
return true;
}
@@ -155,11 +161,11 @@ bool setup_project_path(std::optional<fs::path> project_path_override) {
if (development_repo_path) {
gFilePathInfo.path_to_data = *development_repo_path;
gFilePathInfo.initialized = true;
fmt::print("Using development repo path: {}\n", *development_repo_path);
lg::info("Using development repo path: {}", *development_repo_path);
return true;
}
fmt::print("Failed to initialize project path.\n");
lg::error("Failed to initialize project path.");
return false;
}
@@ -316,10 +322,46 @@ std::string base_name(const std::string& filename) {
break;
}
}
return filename.substr(pos);
}
std::string base_name_no_ext(const std::string& filename) {
size_t pos = 0;
ASSERT(!filename.empty());
for (size_t i = filename.size() - 1; i-- > 0;) {
if (filename.at(i) == '/' || filename.at(i) == '\\') {
pos = (i + 1);
break;
}
}
std::string file_name = filename.substr(pos);
return file_name.substr(0, file_name.find_last_of('.'));
;
}
std::string split_path_at(const fs::path& path, const std::vector<std::string>& folders) {
std::string split_str = "";
for (const auto& folder : folders) {
#ifdef _WIN32
split_str += folder + "\\";
#else
split_str += folder + "/";
#endif
}
const auto& path_str = path.u8string();
return path_str.substr(path_str.find(split_str) + split_str.length());
}
std::string convert_to_unix_path_separators(const std::string& path) {
#ifdef _WIN32
std::string copy = path;
std::replace(copy.begin(), copy.end(), '\\', '/');
return copy;
#else
return path;
#endif
}
void ISONameFromAnimationName(char* dst, const char* src) {
// The Animation Name is a bunch of words separated by dashes
@@ -524,7 +566,7 @@ std::vector<fs::path> find_files_recursively(const fs::path& base_dir, const std
std::vector<fs::path> files = {};
for (auto& p : fs::recursive_directory_iterator(base_dir)) {
if (p.is_regular_file()) {
if (std::regex_match(fs::path(p.path()).filename().string(), pattern)) {
if (std::regex_match(p.path().filename().string(), pattern)) {
files.push_back(p.path());
}
}
@@ -532,4 +574,14 @@ std::vector<fs::path> find_files_recursively(const fs::path& base_dir, const std
return files;
}
std::vector<fs::path> find_directories_in_dir(const fs::path& base_dir) {
std::vector<fs::path> dirs = {};
for (auto& p : fs::recursive_directory_iterator(base_dir)) {
if (p.is_directory()) {
dirs.push_back(p.path());
}
}
return dirs;
}
} // namespace file_util
common/util/FileUtil.h
+5
View File
@@ -31,6 +31,7 @@ fs::path get_user_home_dir();
fs::path get_user_config_dir();
fs::path get_user_settings_dir(GameVersion game_version);
fs::path get_user_memcard_dir(GameVersion game_version);
fs::path get_user_misc_dir(GameVersion game_version);
fs::path get_jak_project_dir();
bool create_dir_if_needed(const fs::path& path);
@@ -51,6 +52,9 @@ bool is_printable_char(char c);
std::string combine_path(const std::string& parent, const std::string& child);
bool file_exists(const std::string& path);
std::string base_name(const std::string& filename);
std::string base_name_no_ext(const std::string& filename);
std::string split_path_at(const fs::path& path, const std::vector<std::string>& folders);
std::string convert_to_unix_path_separators(const std::string& path);
void MakeISOName(char* dst, const char* src);
void ISONameFromAnimationName(char* dst, const char* src);
void assert_file_exists(const char* path, const char* error_message);
@@ -58,4 +62,5 @@ bool dgo_header_is_compressed(const std::vector<u8>& data);
std::vector<u8> decompress_dgo(const std::vector<u8>& data_in);
FILE* open_file(const fs::path& path, const std::string& mode);
std::vector<fs::path> find_files_recursively(const fs::path& base_dir, const std::regex& pattern);
std::vector<fs::path> find_directories_in_dir(const fs::path& base_dir);
} // namespace file_util
common/util/FontUtils.cpp
+307 -17
View File
@@ -15,6 +15,7 @@
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/format.h"
namespace {
@@ -29,7 +30,8 @@ bool hex_char(char c) {
const std::unordered_map<std::string, GameTextVersion> sTextVerEnumMap = {
{"jak1-v1", GameTextVersion::JAK1_V1},
{"jak1-v2", GameTextVersion::JAK1_V2}};
{"jak1-v2", GameTextVersion::JAK1_V2},
{"jak2", GameTextVersion::JAK2}};
const std::string& get_text_version_name(GameTextVersion version) {
for (auto& [name, ver] : sTextVerEnumMap) {
@@ -37,7 +39,7 @@ const std::string& get_text_version_name(GameTextVersion version) {
return name;
}
}
throw std::runtime_error(fmt::format("invalid text version {}", version));
throw std::runtime_error(fmt::format("invalid text version {}", fmt::underlying(version)));
}
GameTextFontBank::GameTextFontBank(GameTextVersion version,
@@ -119,6 +121,10 @@ std::string GameTextFontBank::replace_to_utf8(std::string& str) const {
}
std::string GameTextFontBank::replace_to_game(std::string& str) const {
for (auto& info : *m_replace_info) {
if (info.to.empty()) {
// Skip empty replacements, else it's an infinite loop
continue;
}
auto pos = str.find(info.to);
while (pos != std::string::npos) {
str.replace(pos, info.to.size(), info.from);
@@ -157,6 +163,19 @@ std::string GameTextFontBank::convert_utf8_to_game(std::string str) const {
return str;
}
bool GameTextFontBank::valid_char_range(const char in) const {
if (m_version == GameTextVersion::JAK1_V1 || m_version == GameTextVersion::JAK1_V2) {
return ((in >= '0' && in <= '9') || (in >= 'A' && in <= 'Z') ||
m_passthrus->find(in) != m_passthrus->end()) &&
in != '\\';
} else if (m_version == GameTextVersion::JAK2) {
return ((in >= '0' && in <= '9') || (in >= 'A' && in <= 'Z') || (in >= 'a' && in <= 'z') ||
m_passthrus->find(in) != m_passthrus->end()) &&
in != '\\';
}
return false;
}
/*!
* Turn a normal readable string into a string readable in the in-game font encoding and converts
* \cXX escape sequences
@@ -219,9 +238,7 @@ std::string GameTextFontBank::convert_game_to_utf8(const char* in) const {
if (remap != nullptr) {
result.append(remap->chars);
in += remap->bytes.size() - 1;
} else if (((*in >= '0' && *in <= '9') || (*in >= 'A' && *in <= 'Z') ||
m_passthrus->find(*in) != m_passthrus->end()) &&
*in != '\\') {
} else if (valid_char_range(*in)) {
result.push_back(*in);
} else if (*in == '\n') {
result += "\\n";
@@ -250,9 +267,9 @@ static std::vector<ReplaceInfo> s_replace_info_null = {};
* - Jak & Daxter: The Precursor Legacy (Black Label)
*/
static std::unordered_set<char> s_passthrus = {'~', ' ', ',', '.', '-', '+', '(', ')',
'!', ':', '?', '=', '%', '*', '/', '#',
';', '<', '>', '@', '[', '_'};
static std::unordered_set<char> s_passthrus_jak1 = {'~', ' ', ',', '.', '-', '+', '(', ')',
'!', ':', '?', '=', '%', '*', '/', '#',
';', '<', '>', '@', '[', '_'};
static std::vector<EncodeInfo> s_encode_info_jak1 = {
// random
@@ -487,6 +504,10 @@ static std::vector<ReplaceInfo> s_replace_info_jak1 = {
{"O~Y~-22H~-4V'~Z", "Ó"},
{"U~Y~-24H~-3V'~Z", "Ú"},
// double acute accents
{"O~Y~-28H~-4V'~-9H'~Z", "Ő"}, // custom
{"U~Y~-27H~-4V'~-12H'~Z", "Ű"}, // custom
// circumflex
{"A~Y~-20H~-4V^~Z", "Â"}, // custom
{"E~Y~-20H~-5V^~Z", "Ê"},
@@ -578,10 +599,10 @@ static std::vector<ReplaceInfo> s_replace_info_jak1 = {
{"~Y~22L<~Z~Y~24L#~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<PAD_SQUARE>"}, // custom
};
GameTextFontBank g_font_bank_jak1(GameTextVersion::JAK1_V1,
&s_encode_info_jak1,
&s_replace_info_jak1,
&s_passthrus);
GameTextFontBank g_font_bank_jak1_v1(GameTextVersion::JAK1_V1,
&s_encode_info_jak1,
&s_replace_info_jak1,
&s_passthrus_jak1);
/*!
* ================================
@@ -815,12 +836,281 @@ static std::vector<EncodeInfo> s_encode_info_jak1_v2 = {
GameTextFontBank g_font_bank_jak1_v2(GameTextVersion::JAK1_V2,
&s_encode_info_jak1_v2,
&s_replace_info_jak1,
&s_passthrus);
&s_passthrus_jak1);
/*!
* ================================
* GAME TEXT FONT BANK - JAK 2
* ================================
* This font is used in:
* - Jak 2 - NTSC - v1
*/
static std::unordered_set<char> s_passthrus_jak2 = {'~', ' ', ',', '.', '-', '+', '(', ')',
'!', ':', '?', '=', '%', '*', '/', '#',
';', '<', '>', '@', '[', '_'};
static std::vector<ReplaceInfo> s_replace_info_jak2 = {
// other
{"A~Y~-21H~-5Vº~Z", "Å"},
{"N~Y~-6Hº~Z~+10H", "Nº"},
{"~+4VÇ~-4V", "ç"},
// tildes
{"N~Y~-22H~-4V<TIL>~Z", "Ñ"},
{"n~Y~-24H~-4V<TIL>~Z", "ñ"},
{"A~Y~-21H~-5V<TIL>~Z", "Ã"},
{"O~Y~-22H~-4V<TIL>~Z", "Õ"},
// acute accents
{"A~Y~-21H~-5V'~Z", "Á"},
{"A~Y~-26H~-8V'~Z", "<Á_V2>"}, // unfortunate...
{"a~Y~-25H~-5V'~Z", "á"},
{"E~Y~-23H~-9V'~Z", "É"},
{"e~Y~-26H~-5V'~Z", "é"},
{"I~Y~-19H~-5V'~Z", "Í"},
{"i~Y~-19H~-8V'~Z", "í"},
{"O~Y~-22H~-4V'~Z", "Ó"},
{"o~Y~-26H~-4V'~Z", "ó"},
{"U~Y~-24H~-3V'~Z", "Ú"},
{"u~Y~-24H~-3V'~Z", "ú"},
// circumflex
{"A~Y~-20H~-4V^~Z", "Â"},
{"a~Y~-24H~-5V^~Z", "â"},
{"E~Y~-20H~-5V^~Z", "Ê"},
{"e~Y~-25H~-4V^~Zt", "ê"},
{"I~Y~-19H~-5V^~Z", "Î"},
{"i~Y~-19H~-8V^~Z", "î"},
{"O~Y~-20H~-4V^~Z", "Ô"},
{"o~Y~-25H~-4V^~Z", "ô"},
{"U~Y~-24H~-3V^~Z", "Û"},
{"u~Y~-23H~-3V^~Z", "û"},
// grave accents
{"A~Y~-26H~-8V`~Z", "À"},
{"a~Y~-25H~-5V`~Z", "à"},
{"E~Y~-23H~-9V`~Z", "È"},
{"e~Y~-26H~-5V`~Z", "è"},
{"I~Y~-19H~-5V`~Z", "Ì"},
{"i~Y~-19H~-8V`~Z", "ì"},
{"O~Y~-22H~-4V`~Z", "Ò"},
{"o~Y~-26H~-4V`~Z", "ò"},
{"U~Y~-24H~-3V`~Z", "Ù"},
{"u~Y~-24H~-3V`~Z", "ù"},
// umlaut
{"A~Y~-26H~-8V¨~Z", "Ä"},
{"a~Y~-25H~-5V¨~Z", "ä"},
{"E~Y~-20H~-5V¨~Z", "Ë"},
{"I~Y~-19H~-5V¨~Z", "Ï"},
{"O~Y~-26H~-8V¨~Z", "Ö"},
{"o~Y~-26H~-4V¨~Z", "ö"},
{"U~Y~-25H~-8V¨~Z", "Ü"},
{"u~Y~-24H~-3V¨~Z", "ü"},
// dakuten katakana
{"~Yウ~Z゛", "ヴ"},
{"~Yカ~Z゛", "ガ"},
{"~Yキ~Z゛", "ギ"},
{"~Yク~Z゛", "グ"},
{"~Yケ~Z゛", "ゲ"},
{"~Yコ~Z゛", "ゴ"},
{"~Yサ~Z゛", "ザ"},
{"~Yシ~Z゛", "ジ"},
{"~Yス~Z゛", "ズ"},
{"~Yセ~Z゛", "ゼ"},
{"~Yソ~Z゛", "ゾ"},
{"~Yタ~Z゛", "ダ"},
{"~Yチ~Z゛", "ヂ"},
{"~Yツ~Z゛", "ヅ"},
{"~Yテ~Z゛", "デ"},
{"~Yト~Z゛", "ド"},
{"~Yハ~Z゛", "バ"},
{"~Yヒ~Z゛", "ビ"},
{"~Yフ~Z゛", "ブ"},
{"~Yヘ~Z゛", "ベ"},
{"~Yホ~Z゛", "ボ"},
// handakuten katakana
{"~Yハ~Z゜", "パ"},
{"~Yヒ~Z゜", "ピ"},
{"~Yフ~Z゜", "プ"},
{"~Yヘ~Z゜", "ペ"},
{"~Yホ~Z゜", "ポ"},
// dakuten hiragana
{"~Yか~Z゛", "が"},
{"~Yき~Z゛", "ぎ"},
{"~Yく~Z゛", "ぐ"},
{"~Yけ~Z゛", "げ"},
{"~Yこ~Z゛", "ご"},
{"~Yさ~Z゛", "ざ"},
{"~Yし~Z゛", "じ"},
{"~Yす~Z゛", "ず"},
{"~Yせ~Z゛", "ぜ"},
{"~Yそ~Z゛", "ぞ"},
{"~Yた~Z゛", "だ"},
{"~Yち~Z゛", "ぢ"},
{"~Yつ~Z゛", "づ"},
{"~Yて~Z゛", "で"},
{"~Yと~Z゛", "ど"},
{"~Yは~Z゛", "ば"},
{"~Yひ~Z゛", "び"},
{"~Yふ~Z゛", "ぶ"},
{"~Yへ~Z゛", "べ"},
{"~Yほ~Z゛", "ぼ"},
// handakuten hiragana
{"~Yは~Z゜", "ぱ"},
{"~Yひ~Z゜", "ぴ"},
{"~Yふ~Z゜", "ぷ"},
{"~Yへ~Z゜", "ぺ"},
{"~Yほ~Z゜", "ぽ"},
// japanese punctuation
{",~+8H", "、"},
{"~+8H ", "　"},
// (hack) special case kanji
{"~~", "世"},
// playstation buttons
// - face
{"~Y~22L<~Z~Y~27L*~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<PAD_X>"},
{"~Y~22L<~Z~Y~26L;~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<PAD_TRIANGLE>"},
{"~Y~22L<~Z~Y~25L@~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<PAD_CIRCLE>"},
{"~Y~22L<~Z~Y~24L#~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<PAD_SQUARE>"},
// - dpad
{"~Y~22L<SYM7>~Z~3L~+17H~-13V<SYM8>~Z~22L~+17H~+14V<SYM9>~Z~22L~+32H<SYM10>~Z~+56H",
"<PAD_DPAD_UP>"},
{"~Y~22L<SYM7>~Z~3L~+17H~-13V<SYM8>~Z~3L~+17H~+14V<SYM9>~Z~22L~+32H<SYM10>~Z~+56H",
"<PAD_DPAD_DOWN>"},
{"~Y~22L<SYM7>~Z~22L~+17H~-13V<SYM8>~Z~22L~+17H~+14V<SYM9>~Z~22L~+32H<SYM10>~Z~+56H",
"<PAD_DPAD_ANY>"},
// - shoulder
{"~Y~22L~-2H~-12V<SYM1><SYM2>~Z~22L~-2H~+17V<SYM3><SYM4>~Z~1L~+4H~+3V<SYM5>~Z~+38H",
"<PAD_L1>"},
{"~Y~22L~-2H~-12V<SYM1><SYM2>~Z~22L~-2H~+17V<SYM3><SYM4>~Z~1L~+6H~+3V<SYM6>~Z~+38H",
"<PAD_R1>"},
{"~Y~22L~-2H~-6V<SYM11><SYM12>~Z~22L~-2H~+16V<SYM15><SYM14>~Z~1L~+5H~-2V<SYM13>~Z~+38H",
"<PAD_R2>"},
{"~Y~22L~-2H~-6V<SYM11><SYM12>~Z~22L~-2H~+16V<SYM15><SYM14>~Z~1L~+5H~-2V<SYM22>~Z~+38H",
"<PAD_L2>"},
// - analog
{"~1L~+8H~Y<SYM16>~Z~6L~-16H<SYM21>~Z~+16h~6L<SYM20>~Z~6L~-15V<SYM24>~Z~+13V~6L<SYM28>~Z~-10H~+"
"9V~"
"6L<SYM17>~Z~+10H~+9V~6L<SYM31>~Z~-10H~-11V~6L<SYM23>~Z~+10H~-11V~6L<SYM29>~Z~+32H",
"<PAD_ANALOG_ANY>"},
{"~Y~1L~+8H<SYM16>~Z~6L~-8H<SYM21>~Z~+24H~6L<SYM20>~Z~+40H", "<PAD_ANALOG_LEFT_RIGHT>"},
{"~Y~1L<SYM16>~Z~6L~-15V<SYM24>~Z~+13V~6L<SYM28>~Z~+26H", "<PAD_ANALOG_UP_DOWN>"},
// icons
{"~Y~6L<~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<ICON_MISSION_COMPLETE>"},
{"~Y~3L<~Z~Y~1L>~Z~Y~23L[~Z~+26H", "<ICON_MISSION_TODO>"},
// flags
{"~Y~6L<SYM18>~Z~+15H~1L<SYM18>~Z~+30H~3L<SYM18>~Z~+45H", "<FLAG_ITALIAN>"},
{"~Y~5L<SYM19>~Z~3L<SYM32>~<SYM26>~-1H~Y~5L<SYM19>~Z~3L<SYM32>~Z~+26H", "<FLAG_SPAIN>"},
{"~Y~39L~~~Z~3L<SYM33>~Z~5L<SYM25>~<SYM26>~-1H~Y~39L~~~Z~3L<SYM33>~Z~5L<SYM25>~Z~+26H",
"<FLAG_GERMAN>"},
{"~Y~7L<SYM18>~Z~+15H~1L<SYM18>~Z~+30H~3L<SYM18>~Z~+47H", "<FLAG_FRANCE>"},
{"~Y~1L<SYM19>~Z~3L<SYM34>~Z~7L<SYM37>~<SYM26>~-1H~Y~1L<SYM19>~Z~3L<SYM30>~Z~7L<SYM42>~Z~+26H",
"<FLAG_USA>"},
{"~Y~1L<SYM19>~Z~3L<SYM35>~Z~7L<SYM38>~<SYM26>~-1H~Y~1L<SYM19>~Z~3L<SYM40>~Z~+26H",
"<FLAG_UK>"},
{"~Y~1L<SYM19>~Z~39L<SYM36>~<SYM26>~-1H~Y~1L<SYM19>~Z~39L<SYM39>~Z~-11H~7L<SYM41>~Z~-11H~3L<"
"SYM43>~Z~+26H",
"<FLAG_JAPAN>"},
{"~Y~1L<SYM19>~<SYM26>~-1H~Y~1L<SYM19>~Z~-11H~3L<SYM27>~Z~+26H", "<FLAG_SOUTH_KOREA>"},
// weird stuff
// - descenders
{"~+7Vp~-7V", "p"},
{"~+7Vy~-7V", "y"},
{"~+7Vg~-7V", "g"},
{"~+7Vq~-7V", "q"},
{"~+1Vj~-1V", "j"},
{"\\\\\\\\", "\\n"},
// - symbols and ligatures
{"~-4H~-3V\\c19~+3V~-4H",
"<SUPERSCRIPT_QUOTE>"}, // used for the 4<__> place in spanish. the 5th uses the same
// character but looks different...?
{"~Y~-6Hº~Z~+10H", "°"},
// Color / Emphasis
{"~[~1L", "<COLOR_WHITE>"},
{"~[~32L", "<COLOR_DEFAULT>"}};
static std::vector<EncodeInfo> s_encode_info_jak2 = {
{"_", {0x03}}, // large space
{"ˇ", {0x10}}, // caron
{"`", {0x11}}, // grave accent
{"'", {0x12}}, // apostrophe
{"^", {0x13}}, // circumflex
{"<TIL>", {0x14}}, // tilde
{"¨", {0x15}}, // umlaut
{"º", {0x16}}, // numero/overring
{"¡", {0x17}}, // inverted exclamation mark
{"¿", {0x18}}, // inverted question mark
{"Ç", {0x1d}}, // c-cedilla
{"ß", {0x1f}}, // eszett
{"œ", {0x5e}}, // ligature o+e
// Re-purposed japanese/korean symbols that are used as part of drawing icons/flags/pad buttons
// TODO - japanese and korean encodings
{"<SYM26>", {0x5d}},
{"<SYM33>", {0x7f}},
{"<SYM25>", {0x80}},
{"<SYM18>", {0x81}},
{"<SYM19>", {0x85}},
{"<SYM27>", {0x86}},
{"<SYM36>", {0x87}},
{"<SYM39>", {0x88}},
{"<SYM43>", {0x89}},
{"<SYM41>", {0x8a}},
{"<SYM32>", {0x8b}},
{"<SYM34>", {0x8c}},
{"<SYM30>", {0x8d}},
{"<SYM37>", {0x8e}},
{"<SYM42>", {0x8f}},
{"<SYM40>", {0x90}},
{"<SYM16>", {0x91}},
{"<SYM6>", {0x94}},
{"<SYM5>", {0x95}},
{"<SYM13>", {0x96}},
{"<SYM22>", {0x97}},
{"<SYM28>", {0x98}},
{"<SYM31>", {0x99}},
{"<SYM35>", {0x9a}},
{"<SYM38>", {0x9b}},
{"<SYM24>", {0x9c}},
{"<SYM23>", {0x9d}},
{"<SYM21>", {0x9e}},
{"<SYM17>", {0x9f}},
{"<SYM9>", {0xa0}},
{"<SYM7>", {0xa1}},
{"<SYM8>", {0xa2}},
{"<SYM10>", {0xa3}},
{"<SYM20>", {0xa4}},
{"<SYM29>", {0xa5}},
{"<SYM1>", {0xa6}},
{"<SYM2>", {0xa7}},
{"<SYM11>", {0xa8}},
{"<SYM12>", {0xa9}},
{"<SYM3>", {0xb0}},
{"<SYM4>", {0xb1}},
{"<SYM14>", {0xb3}},
{"<SYM15>", {0xb2}},
};
GameTextFontBank g_font_bank_jak2(GameTextVersion::JAK2,
&s_encode_info_null,
&s_replace_info_null,
&s_passthrus);
&s_encode_info_jak2,
//&s_replace_info_null,
&s_replace_info_jak2,
&s_passthrus_jak2);
/*!
* ========================
@@ -830,7 +1120,7 @@ GameTextFontBank g_font_bank_jak2(GameTextVersion::JAK2,
*/
std::map<GameTextVersion, GameTextFontBank*> g_font_banks = {
{GameTextVersion::JAK1_V1, &g_font_bank_jak1},
{GameTextVersion::JAK1_V1, &g_font_bank_jak1_v1},
{GameTextVersion::JAK1_V2, &g_font_bank_jak1_v2},
{GameTextVersion::JAK2, &g_font_bank_jak2}};
common/util/FontUtils.h
+4
View File
@@ -75,6 +75,10 @@ class GameTextFontBank {
GameTextVersion version() const { return m_version; }
// TODO - methods would help make this code a lot better for different game versions
// hacking it for now
bool valid_char_range(const char in) const;
std::string convert_utf8_to_game_with_escape(const std::string& str) const;
std::string convert_utf8_to_game(std::string str) const;
std::string convert_game_to_utf8(const char* in) const;
common/util/FrameLimiter.cpp
+3 -1
View File
@@ -2,6 +2,8 @@
#include <thread>
#include "common/common_types.h"
double FrameLimiter::round_to_nearest_60fps(double current) {
double one_frame = 1.f / 60.f;
int frames_missed = (current / one_frame); // rounds down
@@ -11,7 +13,7 @@ double FrameLimiter::round_to_nearest_60fps(double current) {
return (frames_missed + 1) * one_frame;
}
#ifdef __linux__
#ifdef OS_POSIX
FrameLimiter::FrameLimiter() {}
common/util/Serializer.h
+11
View File
@@ -156,6 +156,17 @@ class Serializer {
from_raw_data(vec->data(), sizeof(T) * vec->size());
}
void from_string_vector(std::vector<std::string>* vec) {
if (is_saving()) {
save<size_t>(vec->size());
} else {
vec->resize(load<size_t>());
}
for (auto& str : *vec) {
from_str(&str);
}
}
/*!
* Are we saving?
*/
common/util/Timer.cpp
+4 -2
View File
@@ -1,5 +1,7 @@
#include "Timer.h"
#include "common/common_types.h"
#ifdef _WIN32
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
@@ -37,7 +39,7 @@ int Timer::clock_gettime_monotonic(struct timespec* tv) const {
#endif
void Timer::start() {
#ifdef __linux__
#ifdef OS_POSIX
clock_gettime(CLOCK_MONOTONIC, &_startTime);
#elif _WIN32
clock_gettime_monotonic(&_startTime);
@@ -46,7 +48,7 @@ void Timer::start() {
int64_t Timer::getNs() const {
struct timespec now = {};
#ifdef __linux__
#ifdef OS_POSIX
clock_gettime(CLOCK_MONOTONIC, &now);
#elif _WIN32
clock_gettime_monotonic(&now);
common/util/Trie.h
+10
View File
@@ -35,6 +35,9 @@ class Trie {
// Get all objects starting with the given prefix.
std::vector<T*> lookup_prefix(const std::string& str) const;
// Get all nodes in the tree.
std::vector<T*> get_all_nodes() const;
~Trie();
private:
@@ -178,3 +181,10 @@ template <typename T>
std::vector<T*> Trie<T>::lookup_prefix(const std::string& str) const {
return m_root.lookup_prefix(str.c_str());
}
template <typename T>
std::vector<T*> Trie<T>::get_all_nodes() const {
std::vector<T*> result;
m_root.get_all_children(result);
return result;
}
common/util/crc32.cpp
+22 -1
View File
@@ -1,6 +1,26 @@
#include "crc32.h"
#include <cstring>
#ifdef __arm__
#include <arm_acle.h>
u32 crc32(const u8* data, size_t size) {
u32 result = 0xffffffff;
while (size >= 4) {
u32 x;
memcpy(&x, data, 4);
data += 4;
size -= 4;
result = __crc32w(result, x);
}
while (size) {
result = __crc32b(result, *data);
data++;
size--;
}
return ~result;
}
#else
#include <immintrin.h>
u32 crc32(const u8* data, size_t size) {
@@ -18,4 +38,5 @@ u32 crc32(const u8* data, size_t size) {
size--;
}
return ~result;
}
}
#endif
common/util/diff.cpp
+3
View File
@@ -296,6 +296,8 @@ std::vector<std::string> SplitString(const ::std::string& str, char delimiter =
}
} // namespace
namespace google_diff {
std::string diff_strings(const std::string& lhs, const std::string& rhs) {
if (!lhs.empty() && !rhs.empty()) {
const std::vector<std::string> lhs_lines = SplitString(lhs);
@@ -310,3 +312,4 @@ std::string diff_strings(const std::string& lhs, const std::string& rhs) {
std::vector<std::string> split_string(const ::std::string& str, char delimiter) {
return SplitString(str, delimiter);
}
} // namespace google_diff
common/util/diff.h
+2 -1
View File
@@ -6,6 +6,7 @@
/*!
* Diff two strings. This uses the code from gtest's diff implementation.
*/
namespace google_diff {
std::string diff_strings(const std::string& lhs, const std::string& rhs);
std::vector<std::string> split_string(const ::std::string& str, char delimiter = '\n');
} // namespace google_diff
common/util/os.cpp
+7
View File
@@ -21,6 +21,13 @@ size_t get_peak_rss() {
#ifdef _WIN32
// windows has a __cpuid
#include <intrin.h>
#elif __APPLE__
// for now, just return 0's.
void __cpuidex(int result[4], int eax, int ecx) {
for (int i = 0; i < 4; i++) {
result[i] = 0;
}
}
#else
// using int to be compatible with msvc's intrinsic
void __cpuidex(int result[4], int eax, int ecx) {
common/util/print_float.cpp
+20
View File
@@ -29,6 +29,14 @@ std::string meters_to_string(float value, bool append_trailing_decimal) {
return float_to_string(value / METER_LENGTH, append_trailing_decimal);
}
/*!
* Wrapper around float_to_string, for printing degrees. Unlike float_to_string, it does not append
* decimals by default.
*/
std::string degrees_to_string(float value, bool append_trailing_decimal) {
return float_to_string(value / DEGREES_LENGTH, append_trailing_decimal);
}
/*!
* Convert a fixed point value to a float. Fixed point values usually end up with strange numbers
* that were definitely not what was written when we do a naive conversion. This function
@@ -200,3 +208,15 @@ int float_to_cstr(float value, char* buffer, bool append_trailing_decimal) {
}
return i;
}
bool proper_float(float value) {
u32 int_value;
memcpy(&int_value, &value, 4);
u8 exp = (int_value >> 23) & 0xff;
u32 mant = int_value & 0x7fffff;
if ((exp == 0 && mant != 0) || exp == 0xff || !std::isfinite(value)) {
return false;
} else {
return true;
}
}
common/util/print_float.h
+2
View File
@@ -8,5 +8,7 @@ float fixed_point_to_float(s64 value, s64 scale);
std::string fixed_point_to_string(s64 value, s64 scale, bool append_trailing_decimal = false);
std::string float_to_string(float value, bool append_trailing_decimal = true);
std::string meters_to_string(float value, bool append_trailing_decimal = false);
std::string degrees_to_string(float value, bool append_trailing_decimal = false);
std::string seconds_to_string(s64 value, bool append_trailing_decimal = false);
int float_to_cstr(float value, char* buffer, bool append_trailing_decimal = true);
bool proper_float(float value);
common/util/read_iso_file.cpp
+3 -1
View File
@@ -5,6 +5,8 @@
#include "common/util/Assert.h"
#include "common/util/FileUtil.h"
#include "third-party/zstd/lib/common/xxhash.h"
IsoFile::IsoFile() {
root.is_dir = true;
}
@@ -102,7 +104,7 @@ void unpack_entry(FILE* fp,
file_util::write_binary_file(path_to_entry.string(), buffer.data(), buffer.size());
iso.files_extracted++;
if (iso.shouldHash) {
xxh::hash_t<64> hash = xxh::xxhash<64>(buffer);
auto hash = XXH64(buffer.data(), buffer.size(), 0);
iso.hashes.push_back(hash);
}
}
common/util/read_iso_file.h
+1 -3
View File
@@ -5,8 +5,6 @@
#include "common/util/FileUtil.h"
#include "third-party/xxhash.hpp"
struct IsoFile {
struct Entry {
bool is_dir = false;
@@ -29,7 +27,7 @@ struct IsoFile {
bool shouldHash = false;
// There is no reason to map to the files, as we don't retain mappings of each file's expected
// hash
std::vector<xxh::hash64_t> hashes = {};
std::vector<uint64_t> hashes = {};
IsoFile();
};
common/util/set_util.h
+22
View File
@@ -0,0 +1,22 @@
#pragma once
#include <unordered_set>
namespace set_util {
template <typename T>
std::unordered_set<T> intersection(std::unordered_set<T> set1, std::unordered_set<T> set2) {
if (set2.size() < set1.size()) {
auto temp = set1;
set1 = set2;
set2 = temp;
}
std::unordered_set<T> m(set1.begin(), set1.end());
std::unordered_set<T> res;
for (auto a : set2)
if (m.count(a)) {
res.insert(a);
m.erase(a);
}
return res;
}
} // namespace set_util
common/util/string_util.cpp
+96
View File
@@ -0,0 +1,96 @@
#include "string_util.h"
#include <regex>
#include "common/util/diff.h"
namespace str_util {
const std::string WHITESPACE = " \n\r\t\f\v";
bool contains(const std::string& s, const std::string& substr) {
return s.find(substr) != std::string::npos;
}
bool starts_with(const std::string& s, const std::string& prefix) {
return s.size() >= prefix.size() && 0 == s.compare(0, prefix.size(), prefix);
}
bool ends_with(const std::string& s, const std::string& suffix) {
return s.size() >= suffix.size() &&
0 == s.compare(s.size() - suffix.size(), suffix.size(), suffix);
}
std::string ltrim(const std::string& s) {
size_t start = s.find_first_not_of(WHITESPACE);
return (start == std::string::npos) ? "" : s.substr(start);
}
std::string rtrim(const std::string& s) {
size_t end = s.find_last_not_of(WHITESPACE);
return (end == std::string::npos) ? "" : s.substr(0, end + 1);
}
std::string trim(const std::string& s) {
return rtrim(ltrim(s));
}
std::string trim_newline_indents(const std::string& s) {
auto lines = split(s, '\n');
std::vector<std::string> trimmed_lines;
std::transform(lines.begin(), lines.end(), std::back_inserter(trimmed_lines),
[](const std::string& line) { return ltrim(line); });
return join(trimmed_lines, "\n");
}
std::string join(const std::vector<std::string>& strs, const std::string& join_with) {
std::string out;
for (size_t i = 0; i < strs.size(); i++) {
out += strs.at(i);
if (i < strs.size() - 1) {
out += join_with;
}
}
return out;
}
int line_count(const std::string& str) {
int result = 0;
for (auto& c : str) {
if (c == '\n') {
result++;
}
}
return result;
}
// NOTE - this won't work running within gk.exe!
bool valid_regex(const std::string& regex) {
try {
std::regex re(regex);
} catch (const std::regex_error& e) {
return false;
}
return true;
}
std::string diff(const std::string& lhs, const std::string& rhs) {
return google_diff::diff_strings(lhs, rhs);
}
/// Default splits on \n characters
std::vector<std::string> split(const ::std::string& str, char delimiter) {
return google_diff::split_string(str, delimiter);
}
std::vector<std::string> regex_get_capture_groups(const std::string& str,
const std::string& regex) {
std::vector<std::string> groups;
std::smatch matches;
if (std::regex_search(str, matches, std::regex(regex))) {
for (size_t i = 1; i < matches.size(); i++) {
groups.push_back(matches[i].str());
}
}
return groups;
}
} // namespace str_util
common/util/string_util.h
+23
View File
@@ -0,0 +1,23 @@
#pragma once
#include <string>
#include <vector>
namespace str_util {
bool contains(const std::string& s, const std::string& substr);
bool starts_with(const std::string& s, const std::string& prefix);
bool ends_with(const std::string& s, const std::string& prefix);
std::string ltrim(const std::string& s);
std::string rtrim(const std::string& s);
std::string trim(const std::string& s);
/// Given a string with new-lines, split and trim the leading whitespace from each line
/// then return the string with the new-lines back in place.
std::string trim_newline_indents(const std::string& s);
int line_count(const std::string& str);
bool valid_regex(const std::string& regex);
std::string diff(const std::string& lhs, const std::string& rhs);
/// Default splits on \n characters
std::vector<std::string> split(const ::std::string& str, char delimiter = '\n');
std::string join(const std::vector<std::string>& strs, const std::string& join_with);
std::vector<std::string> regex_get_capture_groups(const std::string& str, const std::string& regex);
} // namespace str_util
common/util/term_util.cpp
+49
View File
@@ -0,0 +1,49 @@
#include "term_util.h"
#if defined _WIN32
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <condition_variable>
#include <mutex>
#elif defined(__LINUX__) || defined(__gnu_linux__) || defined(__linux__) || defined(__APPLE__)
#include <cstdlib>
#include <stdio.h>
#include <unistd.h>
#include <sys/ioctl.h>
#endif
namespace term_util {
void clear() {
#if defined _WIN32
system("cls");
#elif defined(__LINUX__) || defined(__gnu_linux__) || defined(__linux__) || defined(__APPLE__)
system("clear");
#endif
}
int row_count() {
#if defined _WIN32
CONSOLE_SCREEN_BUFFER_INFO csbi;
GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi);
return csbi.srWindow.Bottom - csbi.srWindow.Top + 1;
#elif defined(__LINUX__) || defined(__gnu_linux__) || defined(__linux__) || defined(__APPLE__)
struct winsize w;
ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
return w.ws_row;
#endif
}
int col_count() {
#if defined _WIN32
CONSOLE_SCREEN_BUFFER_INFO csbi;
GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi);
return csbi.srWindow.Right - csbi.srWindow.Left + 1;
#elif defined(__LINUX__) || defined(__gnu_linux__) || defined(__linux__) || defined(__APPLE__)
struct winsize w;
ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
return w.ws_col;
#endif
}
} // namespace term_util

Some files were not shown because too many files have changed in this diff Show More