Merge remote-tracking branch 'upstream/master'

.editorconfig

@@ -1,5 +0,0 @@
-# Rules in this file were initially inferred by Visual Studio IntelliCode from the C:\Users\xtvas\Repositories\jak-project codebase based on best match to current usage at 2020-08-28
-# You can modify the rules from these initially generated values to suit your own policies
-# You can learn more about editorconfig here: https://docs.microsoft.com/en-us/visualstudio/ide/editorconfig-code-style-settings-reference
-[*.cs]
-

.gitattributes

@@ -0,0 +1,4 @@
+third-party/* linguist-vendored
+*.gc linguist-language=lisp
+*.gd linguist-language=lisp
+*.gs linguist-language=Scheme

CMakeLists.txt

@@ -6,12 +6,13 @@ set(CMAKE_CXX_STANDARD 14)
 
 # Set default compile flags for GCC
 # optimization level can be set here. Note that game/ overwrites this for building game C++ code.
-if(CMAKE_COMPILER_IS_GNUCXX)
+if (CMAKE_COMPILER_IS_GNUCXX)
     message(STATUS "GCC detected, adding compile flags")
-    set(CMAKE_CXX_FLAGS 
+    set(CMAKE_CXX_FLAGS
         "${CMAKE_CXX_FLAGS} \
         -Wall \
-				-Winit-self \
+        -Winit-self \
+        -ggdb \
         -Wextra \
         -Wcast-align \
         -Wcast-qual \
@@ -22,16 +23,17 @@ if(CMAKE_COMPILER_IS_GNUCXX)
         -Wredundant-decls \
         -Wshadow \
         -Wsign-promo")
-else()
-		set(CMAKE_CXX_FLAGS "/EHsc")
-endif(CMAKE_COMPILER_IS_GNUCXX)
+else ()
+    set(CMAKE_CXX_FLAGS "/EHsc")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /STACK:10000000")
+endif (CMAKE_COMPILER_IS_GNUCXX)
 
 IF (WIN32)
-  set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
-  set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
-  set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
-  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
-ENDIF()
+    set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
+    set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+    set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
+    set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+ENDIF ()
 
 option(CODE_COVERAGE "Enable Code Coverage Compiler Flags" OFF)
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/modules/)
@@ -50,9 +52,15 @@ include_directories(./)
 # build asset packer/unpacker
 add_subdirectory(asset_tool)
 
+# build goos
+add_subdirectory(common/goos)
+
 # build type_system library for compiler/decompiler
 add_subdirectory(common/type_system)
 
+# build common_util library
+add_subdirectory(common/util)
+
 # build cross platform socket library
 add_subdirectory(common/cross_sockets)
 
@@ -79,5 +87,5 @@ add_subdirectory(third-party/fmt)
 
 # windows memory management lib
 IF (WIN32)
-  add_subdirectory(third-party/mman)
-ENDIF()
+    add_subdirectory(third-party/mman)
+ENDIF ()

README.md

@@ -90,93 +90,6 @@ TODO
 - running tests
 - etc
 
-## Project Layout
-
-- `goalc` is the GOAL compiler
-  - `gs` contains GOOS code for parts of GOOS implemented in GOOS
-  - `gc` contains GOAL code for parts of GOAL implemented in GOAL (must generate no machine code, just defining macros)
-- `decompiler` is the decompiler
-- `decompiler_out` is the decompiler output
-- `data` will contain big assets and the output of the GOAL compiler (not checked in to git)
-- `out` will contain the finished game (not checked into git)
-- `resources` will contain data which is checked into git
-- `game` will contain the game source code
-- `common` will contain all data/type shared between different applications.
-- `doc` will contain documentation (markdown format?)
-- `iso_data` is where the files from the DVD go
-- `third-party` will contain code we didn't write. Google Test is a git submodule in this folder.
-- `tests` will contain all tests
-- `asset_tool` will contain the asset packer/unpacker
-
-## Design
-
-(if anybody has better ideas, feel free to suggest improvements! This is just a rough plan for now)
-
-- All C++ code should build from the top-level `cmake`.
-- All C++ applications (GOAL compiler, asset extractor, asset packer, runtime, test) should have a script in the top level which launches them.
-- All file paths should be relative to the `jak` folder.
-- The planned workflow for building a game:
-  - `git submodule update --init --recursive` : check out gtest
-  - `mkdir build; cd build` : create build folder for C++
-  - `cmake ..; make -j` : build C++ code
-  - `cd ..`
-  - `./test.sh` : run gtests
-  - `./asset_extractor.sh ./iso_data` : extract assets from game
-  - `./build_engine.sh` : run GOAL compiler to build all game code
-  - `./build_game.sh` : run the asset packer to build the game
-  - `./run_game.sh` : run the game
-- Workflow for development:
-  - `./gc.sh` : run the compiler in interactive mode
-  - `./gs.sh` : run a goos interpreter in interactive mode
-  - `./decomp.sh : run the decompiler
-
-## Current State
-
-- GOAL compiler just implements the GOOS Scheme Macro Language. Running `./gc.sh` just loads the GOOS library (`goalc/gs/goos-lib.gs`) and then goes into an interactive mode. Use `(exit)` to exit.
-- `./test.sh` runs tests for some game C++ code, for GOOS, for the reader, for the listener connection, and for some early emitter stuff.
-- The runtime boots in `fakeiso` mode which will load some dummy files. Then the C Kernel (`game/kernel`) will load the `KERNEL.CGO` and `GAME.CGO` files, which are from the "proof of concept" GOAL compiler. If you run `./gk.sh`, you should see it load stuff, then print:
-
-```
-calling play!
-~~ HACK ~~ : fake play has been called
-InitListenerConnect
-InitCheckListener
-kernel: machine started
-
-```
-
-where the `~~ HACK ~~` message is from code in `KERNEL.CGO`.
-
-## Coding Guidelines
-
-- Avoid warnings
-- Use asserts over throwing exceptions in game code (throwing exceptions from C code called by GOAL code is sketchy)
-
-## TODOs
-
-- Build on Windows!
-  - File paths
-  - Timer
-  - Windows calling convention for assembly stuff
-  - performance stats for `SystemThread` (probably just get rid of these performance stats completely)
-  - `mmap`ing executable memory
-  - line input library (appears windows compatible?)
-- Clean up possible duplicate code in compiler/decompiler `util` folder, consider a common utility library
-- Clean up header guard names (or just use `#pragma once`?)
-- Investigate a better config format
-  - The current JSON library seems to have issues with comments, which I really like
-- Clean up use of namespaces
-- Clean up the print message when `gk` starts.
-- Finish commenting runtime stuff
-- Runtime document
-- GOOS document
-- Listener protocol document
-- GOAL Compiler IR
-- GOAL Compiler Skeleton
-- Gtest setup for checking decompiler results against hand-decompiled stuff
-- Clean up decompiler print spam, finish up the CFG stuff
-- Decompiler document
-
 ## Project Description
 
 This project is to port Jak 1 (NTSC, "black label" version) to PC. The strategy is to:
@@ -211,6 +124,67 @@ Some statistics:
 
 The rough timeline is to finish sometime in 2022. If it looks like this is impossible, the project will be abandoned. But I have already spent about 4 months preparing to start this and seems doable. I also have some background in compilers, and familiarity with PS2 (worked on DobieStation PS2 emulator) / MIPS in general (wrote a PS1 emulator). I think the trick will be making good automated tools - the approach taken for SM64 and other N64 decompilations is way too labor-intensive to work.
 
+## Project Layout
+
+- `goalc` is the GOAL compiler
+- `decompiler` is the decompiler
+- `decompiler_out` is the decompiler output
+- `data` will contain big assets and the output of the GOAL compiler (not checked in to git)
+- `out` will contain the finished game (not checked into git)
+- `resources` will contain data which is checked into git
+- `game` will contain the game source code (C/C++)
+- `goal_src` will contain GOAL source code for the game
+- `common` will contain all data/type shared between different applications.
+- `doc` will contain documentation
+- `iso_data` is where the files from the DVD go
+- `third-party` will contain code we didn't write. Google Test is a git submodule in this folder.
+- `tests` will contain all tests
+- `asset_tool` will contain the asset packer/unpacker
+
+## Design
+
+(if anybody has better ideas, feel free to suggest improvements! This is just a rough plan for now)
+
+- All C++ code should build from the top-level `cmake`.
+- All C++ applications (GOAL compiler, asset extractor, asset packer, runtime, test) should have a script in the top level which launches them.
+- All file paths should be relative to the `jak` folder.
+- The planned workflow for building a game:
+  - `git submodule update --init --recursive` : check out gtest
+  - `mkdir build; cd build` : create build folder for C++
+  - `cmake ..; make -j` : build C++ code
+  - `cd ..`
+  - `./test.sh` : run gtests
+  - `./asset_extractor.sh ./iso_data` : extract assets from game
+  - `./build_engine.sh` : run GOAL compiler to build all game code
+  - `./build_game.sh` : run the asset packer to build the game
+  - `./run_game.sh` : run the game
+- Workflow for development:
+  - `./gc.sh` : run the compiler in interactive mode
+  - `./gs.sh` : run a goos interpreter in interactive mode
+  - `./decomp.sh : run the decompiler
+
+
+## Coding Guidelines
+
+- Avoid warnings
+- Use asserts over throwing exceptions in game code (throwing exceptions from C code called by GOAL code is sketchy)
+
+## TODOs
+
+- Clean up header guard names (or just use `#pragma once`?)
+- Investigate a better config format
+  - The current JSON library seems to have issues with comments, which I really like
+- Clean up use of namespaces
+- Clean up the print message when `gk` starts.
+- Finish commenting runtime stuff
+- Runtime document
+- GOOS document
+- Listener protocol document
+- Gtest setup for checking decompiler results against hand-decompiled stuff
+- Clean up decompiler print spam, finish up the CFG stuff
+- Decompiler document
+
+
 ### GOAL Decompiler
 
 The decompiler is in progress, at
@@ -272,80 +246,6 @@ The compiler will reuse a significant amount of code from my existing LISP compi
 
 An important part of the compiler is the test suite. Without tests the compiler will be full of bugs. So every feature should have a good set of tests.
 
-The major components are
-
-- [ ] GOAL-IR, a typed linear intermediate representation for GOAL code
-
-  - [ ] "Environment"
-  - [ ] "Ref"
-  - [ ] Constant propagation of integers/floats
-  - [ ] Constant propagation of memory locations (this is required to make sure bitfields updates know where to write their result)
-  - [ ] Ref `in_gpr`
-
-- [ ] The type system
-
-  - [ ] Type inheritance
-  - [ ] Integer/float/pointer types (value semantics)
-  - [ ] Reference types
-  - [ ] Bitfield types
-  - [ ] 128-bit integer types (EE GPRs are 128-bit when used with MMI instructions, these will become `xmm`'s)
-  - [ ] Floating point vector types
-  - [ ] Structure types
-  - [ ] Inline structures as fields
-  - [ ] Array access / alignment rules
-  - [ ] Pointer dereferencing
-  - [ ] Methods (dynamic and static dispatch)
-  - [ ] Function arguments/returns
-  - [ ] Global symbols
-  - [ ] Type checking
-  - [ ] Lowest common ancestor
-  - [ ] Built-in types in the GOAL runtime/C Kernel
-
-- [x] The GOOS Macro Language
-
-  - [x] S-expression parser (the "Reader")
-  - [x] Reader text db (for error messages that point to a specific line)
-  - [x] Scheme interpreter
-
-- [ ] Front-end (convert s-expressions (a tree structure) to GOAL-IR (a linear representation))
-- [ ] Parsing helpers
-- [ ] Macro expansion
-- [ ] Control flow/block forms
-- [ ] Type definitions
-- [ ] Inline assembly forms
-- [ ] Function/method call
-- [ ] Math forms
-- [ ] Lexical scoping (immediate application of `lambda`)
-- [ ] Function inlining (slightly different scoping rules of immediate `lambda`)
-- [ ] Function/macro definition
-- [ ] Static Objects
-
-- [ ] Regsiter allocation
-- [ ] Analysis
-- [ ] Allocation
-- [ ] Stack spilling
-- [ ] `xmm` and `gpr` promotion/demotions for EE 128-bit register usage
-
-- [ ] Codegen / Emitter (convert GOAL-IR + register allocations to x86 object file format)
-- [ ] Emitter (convert GOAL-IR to instructions)
-- [ ] x86-64 instruction generation (actually generate the machine code)
-- [ ] Linking data
-- [ ] 64-bit GPR
-- [ ] 32-bit float
-- [ ] 128-bit GPR
-- [ ] 32-bit float x4 vector register
-- [ ] function prologue/epilogue
-- [ ] stack spilling
-- [ ] static object and static object links
-
-- [ ] Listener/REPL
-  - [ ] Network connection
-  - [ ] Tracking loaded files
-  - [ ] Sending code from REPL
-  - [ ] GOOS REPL option
-  - [ ] Expand single macro debugging feature
-  - [ ] Interface for running gtests
-
 ### Asset Extraction Tool
 
 Not started yet. The simplest version of this tool is just to use the decompiler logic to turn the level/art-group/texture/TXT files into GOAL source code, and copy all STR/sound/visibility files, as these don't need to be modified.

common/common_types.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file common_types.h
  * Common Integer Types.

common/cross_sockets/xsocket.cpp

@@ -40,7 +40,7 @@ void close_socket(int sock) {
 }
 
 int set_socket_option(int socket, int level, int optname, const void* optval, int optlen) {
-  int ret = setsockopt(socket, level, optname, (char*)&optval, optlen);
+  int ret = setsockopt(socket, level, optname, (const char*)optval, optlen);
   if (ret < 0) {
     printf("Failed to setsockopt(%d, %d, %d, _, _) - Error: %s\n", socket, level, optname,
            strerror(errno));
@@ -66,7 +66,7 @@ int set_socket_timeout(int socket, long microSeconds) {
   }
   return ret;
 #elif _WIN32
-  unsigned long timeout = microSeconds / 1000;  // milliseconds
+  DWORD timeout = microSeconds / 1000;  // milliseconds
   return set_socket_option(socket, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
 #endif
 }
@@ -86,3 +86,12 @@ int read_from_socket(int socket, char* buf, int len) {
   return recv(socket, buf, len, 0);
 #endif
 }
+
+bool socket_timed_out() {
+#ifdef __linux
+  return errno == EAGAIN;
+#elif _WIN32
+  auto err = WSAGetLastError();
+  return err == WSAETIMEDOUT;
+#endif
+}

common/cross_sockets/xsocket.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifdef __linux
 #include <sys/socket.h>
 #include <netinet/tcp.h>
@@ -18,3 +20,4 @@ int set_socket_option(int socket, int level, int optname, const void* optval, in
 int set_socket_timeout(int socket, long microSeconds);
 int write_to_socket(int socket, const char* buf, int len);
 int read_from_socket(int socket, char* buf, int len);
+bool socket_timed_out();

common/goal_constants.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_GOAL_CONSTANTS_H
 #define JAK_GOAL_CONSTANTS_H
 

common/goos/CMakeLists.txt

@@ -1,2 +1,2 @@
 add_library(goos SHARED Object.cpp TextDB.cpp Reader.cpp Interpreter.cpp InterpreterEval.cpp)
-target_link_libraries(goos util)
+target_link_libraries(goos common_util)

common/goos/Interpreter.cpp

@@ -379,8 +379,8 @@ ArgumentSpec Interpreter::parse_arg_spec(const Object& form, Object& rest) {
 void Interpreter::vararg_check(
     const Object& form,
     const Arguments& args,
-    const std::vector<util::MatchParam<ObjectType>>& unnamed,
-    const std::unordered_map<std::string, std::pair<bool, util::MatchParam<ObjectType>>>& named) {
+    const std::vector<MatchParam<ObjectType>>& unnamed,
+    const std::unordered_map<std::string, std::pair<bool, MatchParam<ObjectType>>>& named) {
   assert(args.rest.empty());
   if (unnamed.size() != args.unnamed.size()) {
     throw_eval_error(form, "Got " + std::to_string(args.unnamed.size()) +
@@ -805,11 +805,9 @@ Object Interpreter::eval_quasiquote(const Object& form,
   return quasiquote_helper(rest.as_pair()->car, env);
 }
 
-namespace {
-bool truthy(const Object& o) {
+bool Interpreter::truthy(const Object& o) {
   return !(o.is_symbol() && o.as_symbol()->name == "#f");
 }
-}  // namespace
 
 /*!
  * Scheme "cond" statement - tested by integrated tests only.

common/goos/Interpreter.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file Interpreter.h
  * The GOOS Interpreter
@@ -9,7 +11,7 @@
 #include <memory>
 #include "Object.h"
 #include "Reader.h"
-#include "goalc/util/MatchParam.h"
+#include "common/util/MatchParam.h"
 
 namespace goos {
 class Interpreter {
@@ -23,6 +25,16 @@ class Interpreter {
   Object eval(Object obj, const std::shared_ptr<EnvironmentObject>& env);
   Object intern(const std::string& name);
   void disable_printfs();
+  Object eval_symbol(const Object& sym, const std::shared_ptr<EnvironmentObject>& env);
+  Arguments get_args(const Object& form, const Object& rest, const ArgumentSpec& spec);
+  void set_args_in_env(const Object& form,
+                       const Arguments& args,
+                       const ArgumentSpec& arg_spec,
+                       const std::shared_ptr<EnvironmentObject>& env);
+  Object eval_list_return_last(const Object& form,
+                               Object rest,
+                               const std::shared_ptr<EnvironmentObject>& env);
+  bool truthy(const Object& o);
 
   Reader reader;
   Object global_environment;
@@ -43,18 +55,13 @@ class Interpreter {
   void vararg_check(
       const Object& form,
       const Arguments& args,
-      const std::vector<util::MatchParam<ObjectType>>& unnamed,
-      const std::unordered_map<std::string, std::pair<bool, util::MatchParam<ObjectType>>>& named);
+      const std::vector<MatchParam<ObjectType>>& unnamed,
+      const std::unordered_map<std::string, std::pair<bool, MatchParam<ObjectType>>>& named);
 
   Object eval_pair(const Object& o, const std::shared_ptr<EnvironmentObject>& env);
-  Object eval_symbol(const Object& sym, const std::shared_ptr<EnvironmentObject>& env);
-  Arguments get_args(const Object& form, const Object& rest, const ArgumentSpec& spec);
   void eval_args(Arguments* args, const std::shared_ptr<EnvironmentObject>& env);
   ArgumentSpec parse_arg_spec(const Object& form, Object& rest);
 
-  Object eval_list_return_last(const Object& form,
-                               Object rest,
-                               const std::shared_ptr<EnvironmentObject>& env);
   Object quasiquote_helper(const Object& form, const std::shared_ptr<EnvironmentObject>& env);
 
   IntType number_to_integer(const Object& obj);
@@ -206,11 +213,6 @@ class Interpreter {
                     const Object& rest,
                     const std::shared_ptr<EnvironmentObject>& env);
 
-  void set_args_in_env(const Object& form,
-                       const Arguments& args,
-                       const ArgumentSpec& arg_spec,
-                       const std::shared_ptr<EnvironmentObject>& env);
-
   bool want_exit = false;
   bool disable_printing = false;
 

common/goos/InterpreterEval.cpp

@@ -66,7 +66,7 @@ Object Interpreter::eval_read_file(const Object& form,
   vararg_check(form, args, {ObjectType::STRING}, {});
 
   try {
-    return reader.read_from_file(args.unnamed.at(0).as_string()->data);
+    return reader.read_from_file({args.unnamed.at(0).as_string()->data});
   } catch (std::runtime_error& e) {
     throw_eval_error(form, std::string("reader error inside of read-file:\n") + e.what());
   }
@@ -84,7 +84,7 @@ Object Interpreter::eval_load_file(const Object& form,
 
   Object o;
   try {
-    o = reader.read_from_file(args.unnamed.at(0).as_string()->data);
+    o = reader.read_from_file({args.unnamed.at(0).as_string()->data});
   } catch (std::runtime_error& e) {
     throw_eval_error(form, std::string("reader error inside of load-file:\n") + e.what());
   }

common/goos/Object.cpp

@@ -1,5 +1,5 @@
 #include "Object.h"
-#include "goalc/util/text_util.h"
+#include "common/util/FileUtil.h"
 
 namespace goos {
 
@@ -53,7 +53,7 @@ std::string fixed_to_string(FloatType x) {
 template <>
 std::string fixed_to_string(char x) {
   char buff[256];
-  if (util::is_printable_char(x) && x != ' ') {
+  if (file_util::is_printable_char(x) && x != ' ') {
     // can print directly
     sprintf(buff, "#\\%c", x);
     return {buff};
@@ -233,4 +233,13 @@ ArgumentSpec make_varargs() {
   return as;
 }
 
+bool Arguments::only_contains_named(const std::unordered_set<std::string>& names) {
+  for (auto& kv : named) {
+    if (names.find(kv.first) == names.end()) {
+      return false;
+    }
+  }
+  return true;
+}
+
 }  // namespace goos

common/goos/Object.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file Object.h
  * An "Object" represents a scheme object.
@@ -45,6 +47,7 @@
 #include <cassert>
 #include <memory>
 #include <unordered_map>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 #include <stdexcept>
@@ -128,7 +131,7 @@ class FixedObject {
       return object_type_to_string(ObjectType::INTEGER);
     if (std::is_same<T, char>())
       return object_type_to_string(ObjectType::CHAR);
-    assert(false);
+    throw std::runtime_error("Unsupported FixedObject type");
   }
 };
 
@@ -299,6 +302,7 @@ class Object {
   }
 
   bool is_empty_list() const { return type == ObjectType::EMPTY_LIST; }
+  bool is_list() const { return type == ObjectType::EMPTY_LIST || type == ObjectType::PAIR; }
   bool is_int() const { return type == ObjectType::INTEGER; }
   bool is_float() const { return type == ObjectType::FLOAT; }
   bool is_char() const { return type == ObjectType::CHAR; }
@@ -527,6 +531,7 @@ struct Arguments {
   Object get_named(const std::string& name, const Object& default_value);
   Object get_named(const std::string& name);
   bool has_named(const std::string& name);
+  bool only_contains_named(const std::unordered_set<std::string>& names);
 };
 
 class LambdaObject : public HeapObject {

common/goos/Reader.cpp

@@ -11,8 +11,7 @@
 
 #include "Reader.h"
 #include "third-party/linenoise.h"
-#include "goalc/util/file_io.h"
-#include "goalc/util/text_util.h"
+#include "common/util/FileUtil.h"
 
 namespace goos {
 
@@ -98,15 +97,6 @@ Reader::Reader() {
   for (const char* c = bonus; *c; c++) {
     valid_symbols_chars[(int)*c] = true;
   }
-
-  // find the source directory
-  auto result = std::getenv("NEXT_DIR");
-  if (!result) {
-    throw std::runtime_error(
-        "Environment variable NEXT_DIR is not set.  Please set this to point to next/");
-  }
-
-  source_dir = result;
 }
 
 /*!
@@ -147,8 +137,8 @@ Object Reader::read_from_string(const std::string& str) {
 /*!
  * Read a file
  */
-Object Reader::read_from_file(const std::string& filename) {
-  auto textFrag = std::make_shared<FileText>(util::combine_path(get_source_dir(), filename));
+Object Reader::read_from_file(const std::vector<std::string>& file_path) {
+  auto textFrag = std::make_shared<FileText>(file_util::get_file_path(file_path));
   db.insert(textFrag);
 
   auto result = internal_read(textFrag);
@@ -670,7 +660,7 @@ bool Reader::try_token_as_integer(const Token& tok, Object& obj) {
 
 bool Reader::try_token_as_char(const Token& tok, Object& obj) {
   if (tok.text.size() >= 3 && tok.text[0] == '#' && tok.text[1] == '\\') {
-    if (tok.text.size() == 3 && util::is_printable_char(tok.text[2]) && tok.text[2] != ' ') {
+    if (tok.text.size() == 3 && file_util::is_printable_char(tok.text[2]) && tok.text[2] != ' ') {
       obj = Object::make_char(tok.text[2]);
       return true;
     }
@@ -705,6 +695,6 @@ void Reader::throw_reader_error(TextStream& here, const std::string& err, int se
  * Get the source directory of the current project.
  */
 std::string Reader::get_source_dir() {
-  return source_dir;
+  return file_util::get_project_path();
 }
 }  // namespace goos

common/goos/Reader.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file Reader.h
  *
@@ -17,8 +19,8 @@
 #include <utility>
 #include <unordered_map>
 
-#include "goalc/goos/Object.h"
-#include "goalc/goos/TextDB.h"
+#include "common/goos/Object.h"
+#include "common/goos/TextDB.h"
 
 namespace goos {
 
@@ -70,7 +72,7 @@ class Reader {
   Reader();
   Object read_from_string(const std::string& str);
   Object read_from_stdin(const std::string& prompt_name);
-  Object read_from_file(const std::string& filename);
+  Object read_from_file(const std::vector<std::string>& file_path);
 
   std::string get_source_dir();
 
@@ -97,7 +99,6 @@ class Reader {
 
   char valid_symbols_chars[256];
 
-  std::string source_dir;
   std::unordered_map<std::string, std::string> reader_macros;
 };
 }  // namespace goos

common/goos/TextDB.cpp

@@ -11,7 +11,7 @@
  *   (+ 1 (+ a b)) ; compute the sum
  */
 
-#include "goalc/util/file_io.h"
+#include "common/util/FileUtil.h"
 
 #include "TextDB.h"
 
@@ -60,7 +60,8 @@ int SourceText::get_line_idx(int offset) {
       return line;
     }
   }
-  assert(false);
+  throw std::runtime_error("Unable to get line index for character at position " +
+                           std::to_string(offset));
 }
 
 /*!
@@ -79,7 +80,7 @@ std::pair<int, int> SourceText::get_containing_line(int offset) {
  * Read text from a file.
  */
 FileText::FileText(std::string filename_) : filename(std::move(filename_)) {
-  text = util::read_text_file(filename);
+  text = file_util::read_text_file(filename);
   build_offsets();
 }
 

common/goos/TextDB.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file TextDB.h
  * The Text Database for storing source code text.
@@ -20,7 +22,7 @@
 #include <unordered_map>
 #include <memory>
 
-#include "goalc/goos/Object.h"
+#include "common/goos/Object.h"
 
 namespace goos {
 /*!

common/link_types.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file link_types.h
  * Types used in the linking data, shared between the object file generator and the kernel's linker.

common/listener_common.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file listener_common.h
  * Common types shared between the compiler and the runtime for the listener connection.
@@ -33,7 +35,7 @@ enum class ListenerMessageKind : u16 {
 /*!
  * Type of message sent from compiler
  */
-enum ListenerToTargetMsgKind {
+enum ListenerToTargetMsgKind : u16 {
   LTT_MSG_POKE = 1,           //! "Poke" the game and have it flush buffers
   LTT_MSG_INSEPCT = 5,        //! Inspect an object
   LTT_MSG_PRINT = 6,          //! Print an object
@@ -47,11 +49,15 @@ enum ListenerToTargetMsgKind {
  * TODO - there are other copies of this somewhere
  */
 struct ListenerMessageHeader {
-  Deci2Header deci2_header;      //! The header used for DECI2 communication
-  ListenerMessageKind msg_kind;  //! GOAL Listener message kind
-  u16 u6;                        //! Unknown
-  u32 msg_size;                  //! Size of data after this header
-  u64 u8;                        //! Unknown
+  Deci2Header deci2_header;  //! The header used for DECI2 communication
+  union {
+    ListenerMessageKind msg_kind;  //! GOAL Listener message kind
+    ListenerToTargetMsgKind ltt_msg_kind;
+  };
+
+  u16 u6;        //! Unknown
+  u32 msg_size;  //! Size of data after this header
+  u64 u8;        //! Unknown
 };
 
 constexpr int DECI2_PORT = 8112;  // TODO - is this a good choise?

common/symbols.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file symbols.h
  * The location of fixed symbols in the GOAL symbol table.

common/type_system/CMakeLists.txt

@@ -2,6 +2,7 @@ add_library(type_system
         SHARED
         TypeSystem.cpp
         Type.cpp
-        TypeSpec.cpp)
+        TypeSpec.cpp
+        deftype.cpp)
 
-target_link_libraries(type_system fmt)
+target_link_libraries(type_system fmt goos)

common/type_system/Type.cpp

@@ -16,7 +16,7 @@ std::string reg_kind_to_string(RegKind kind) {
     case RegKind::FLOAT_4X:
       return "float-4x";
     default:
-      assert(false);
+      throw std::runtime_error("Unsupported RegKind");
   }
 }
 
@@ -467,7 +467,7 @@ void StructureType::inherit(StructureType* parent) {
 }
 
 bool StructureType::operator==(const Type& other) const {
-  if (typeid(StructureType) != typeid(other)) {
+  if (typeid(*this) != typeid(other)) {
     return false;
   }
 

common/type_system/Type.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_TYPE_H
 #define JAK_TYPE_H
 
@@ -213,7 +215,7 @@ class StructureType : public ReferenceType {
                 bool pack = false);
   std::string print() const override;
   void inherit(StructureType* parent);
-  const std::vector<Field>& fields() { return m_fields; }
+  const std::vector<Field>& fields() const { return m_fields; }
   bool operator==(const Type& other) const override;
   int get_size_in_memory() const override;
   int get_offset() const override;

common/type_system/TypeSpec.cpp

@@ -43,4 +43,21 @@ TypeSpec TypeSpec::substitute_for_method_call(const std::string& method_type) co
     result.m_arguments.push_back(x.substitute_for_method_call(method_type));
   }
   return result;
+}
+
+bool TypeSpec::is_compatible_child_method(const TypeSpec& implementation,
+                                          const std::string& child_type) const {
+  bool ok = implementation.m_type == m_type ||
+            (m_type == "_type_" && implementation.m_type == child_type);
+  if (!ok || implementation.m_arguments.size() != m_arguments.size()) {
+    return false;
+  }
+
+  for (size_t i = 0; i < m_arguments.size(); i++) {
+    if (!m_arguments[i].is_compatible_child_method(implementation.m_arguments[i], child_type)) {
+      return false;
+    }
+  }
+
+  return true;
 }

common/type_system/TypeSpec.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file TypeSpec.h
  */
@@ -28,6 +30,8 @@ class TypeSpec {
 
   bool operator!=(const TypeSpec& other) const;
   bool operator==(const TypeSpec& other) const;
+  bool is_compatible_child_method(const TypeSpec& implementation,
+                                  const std::string& child_type) const;
   std::string print() const;
 
   void add_arg(const TypeSpec& ts) { m_arguments.push_back(ts); }
@@ -43,6 +47,14 @@ class TypeSpec {
 
   TypeSpec substitute_for_method_call(const std::string& method_type) const;
 
+  size_t arg_count() const { return m_arguments.size(); }
+
+  const TypeSpec& get_arg(int idx) const { return m_arguments.at(idx); }
+  const TypeSpec& last_arg() const {
+    assert(!m_arguments.empty());
+    return m_arguments.back();
+  }
+
  private:
   friend class TypeSystem;
   std::string m_type;

common/type_system/TypeSystem.cpp

@@ -228,6 +228,12 @@ Type* TypeSystem::lookup_type(const TypeSpec& ts) const {
   return lookup_type(ts.base_type());
 }
 
+MethodInfo TypeSystem::add_method(const std::string& type_name,
+                                  const std::string& method_name,
+                                  const TypeSpec& ts) {
+  return add_method(lookup_type(make_typespec(type_name)), method_name, ts);
+}
+
 /*!
  * Add a method, if it doesn't exist. If the method already exists (possibly in a parent), checks to
  * see if this is an identical definition.  If not, it's an error, and if so, nothing happens.
@@ -263,7 +269,7 @@ MethodInfo TypeSystem::add_method(Type* type, const std::string& method_name, co
 
   if (got_existing) {
     // make sure we aren't changing anything.
-    if (ts != existing_info.type) {
+    if (!existing_info.type.is_compatible_child_method(ts, type->get_name())) {
       fmt::print(
           "[TypeSystem] The method {} of type {} was originally defined as {}, but has been "
           "redefined as {}\n",
@@ -591,6 +597,7 @@ void TypeSystem::add_builtin_types() {
 
   // TYPE
   builtin_structure_inherit(type_type);
+  add_method(type_type, "new", make_function_typespec({"symbol", "type", "int"}, "_type_"));
   add_field_to_type(type_type, "symbol", make_typespec("symbol"));
   add_field_to_type(type_type, "parent", make_typespec("type"));
   add_field_to_type(type_type, "allocated-size", make_typespec("uint16"));  // todo, u16 or s16?

common/type_system/TypeSystem.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_TYPESYSTEM_H
 #define JAK_TYPESYSTEM_H
 
@@ -49,6 +51,9 @@ class TypeSystem {
   Type* lookup_type(const TypeSpec& ts) const;
   Type* lookup_type(const std::string& name) const;
 
+  MethodInfo add_method(const std::string& type_name,
+                        const std::string& method_name,
+                        const TypeSpec& ts);
   MethodInfo add_method(Type* type, const std::string& method_name, const TypeSpec& ts);
   MethodInfo add_new_method(Type* type, const TypeSpec& ts);
   MethodInfo lookup_method(const std::string& type_name, const std::string& method_name);
@@ -74,6 +79,7 @@ class TypeSystem {
                  bool print_on_error = true,
                  bool throw_on_error = true) const;
   std::vector<std::string> get_path_up_tree(const std::string& type);
+  int get_next_method_id(Type* type);
 
   /*!
    * Get a type by name and cast to a child class of Type*. Must succeed.
@@ -97,7 +103,6 @@ class TypeSystem {
   int get_size_in_type(const Field& field);
   int get_alignment_in_type(const Field& field);
   Field lookup_field(const std::string& type_name, const std::string& field_name);
-  int get_next_method_id(Type* type);
   StructureType* add_builtin_structure(const std::string& parent,
                                        const std::string& type_name,
                                        bool boxed = false);

common/type_system/deftype.cpp

@@ -0,0 +1,325 @@
+#include "deftype.h"
+#include "third-party/fmt/core.h"
+
+/*!
+ * Missing Features
+ * - Bitfields
+ * - Int128 children
+ * - Refer to yourself (structure/basic only)
+ * - Method List
+ * - Evaluate constants.
+ *
+ */
+
+namespace {
+const goos::Object& car(const goos::Object* x) {
+  if (!x->is_pair()) {
+    throw std::runtime_error("invalid deftype form");
+  }
+
+  return x->as_pair()->car;
+}
+
+const goos::Object* cdr(const goos::Object* x) {
+  if (!x->is_pair()) {
+    throw std::runtime_error("invalid deftype form");
+  }
+
+  return &x->as_pair()->cdr;
+}
+
+std::string deftype_parent_list(const goos::Object& list) {
+  if (!list.is_pair()) {
+    throw std::runtime_error("invalid parent list in deftype: " + list.print());
+  }
+
+  auto parent = list.as_pair()->car;
+  auto rest = list.as_pair()->cdr;
+  if (!rest.is_empty_list()) {
+    throw std::runtime_error("invalid parent list in deftype - can only have one parent");
+  }
+
+  if (!parent.is_symbol()) {
+    throw std::runtime_error("invalid parent in deftype parent list");
+  }
+
+  return parent.as_symbol()->name;
+}
+
+bool is_type(const std::string& expected, const TypeSpec& actual, const TypeSystem* ts) {
+  return ts->typecheck(ts->make_typespec(expected), actual, "", false, false);
+}
+
+template <typename T>
+void for_each_in_list(const goos::Object& list, T f) {
+  const goos::Object* iter = &list;
+  while (iter->is_pair()) {
+    auto lap = iter->as_pair();
+    f(lap->car);
+    iter = &lap->cdr;
+  }
+
+  if (!iter->is_empty_list()) {
+    throw std::runtime_error("invalid list in for_each_in_list: " + list.print());
+  }
+}
+
+std::string symbol_string(const goos::Object& obj) {
+  if (obj.is_symbol()) {
+    return obj.as_symbol()->name;
+  }
+  throw std::runtime_error(obj.print() + " was supposed to be a symbol, but isn't");
+}
+
+int64_t get_int(const goos::Object& obj) {
+  if (obj.is_int()) {
+    return obj.integer_obj.value;
+  }
+  throw std::runtime_error(obj.print() + " was supposed to be an integer, but isn't");
+}
+
+TypeSpec parse_typespec(TypeSystem* type_system, const goos::Object& src) {
+  if (src.is_symbol()) {
+    return type_system->make_typespec(symbol_string(src));
+  } else if (src.is_pair()) {
+    TypeSpec ts = type_system->make_typespec(symbol_string(car(&src)));
+    const auto& rest = *cdr(&src);
+
+    for_each_in_list(rest,
+                     [&](const goos::Object& o) { ts.add_arg(parse_typespec(type_system, o)); });
+
+    return ts;
+  } else {
+    throw std::runtime_error("invalid typespec: " + src.print());
+  }
+  assert(false);
+  return {};
+}
+
+void add_field(StructureType* structure, TypeSystem* ts, const goos::Object& def) {
+  auto rest = &def;
+
+  auto name = symbol_string(car(rest));
+  rest = cdr(rest);
+
+  auto type = parse_typespec(ts, car(rest));
+  rest = cdr(rest);
+
+  int array_size = -1;
+  bool is_inline = false;
+  bool is_dynamic = false;
+  int offset_override = -1;
+  int offset_assert = -1;
+
+  if (!rest->is_empty_list()) {
+    if (car(rest).is_int()) {
+      array_size = car(rest).integer_obj.value;
+      rest = cdr(rest);
+    }
+
+    while (!rest->is_empty_list()) {
+      auto opt_name = symbol_string(car(rest));
+      rest = cdr(rest);
+
+      if (opt_name == ":inline") {
+        is_inline = true;
+      } else if (opt_name == ":dynamic") {
+        is_dynamic = true;
+      } else if (opt_name == ":offset") {
+        offset_override = get_int(car(rest));
+        rest = cdr(rest);
+      } else if (opt_name == ":offset-assert") {
+        offset_assert = get_int(car(rest));
+        if (offset_assert == -1) {
+          throw std::runtime_error("Cannot use -1 as offset-assert");
+        }
+        rest = cdr(rest);
+      } else {
+        throw std::runtime_error("Invalid option in field specification: " + opt_name);
+      }
+    }
+  }
+
+  int actual_offset = ts->add_field_to_type(structure, name, type, is_inline, is_dynamic,
+                                            array_size, offset_override);
+  if (offset_assert != -1 && actual_offset != offset_assert) {
+    throw std::runtime_error("Field " + name + " was placed at " + std::to_string(actual_offset) +
+                             " but offset-assert was set to " + std::to_string(offset_assert));
+  }
+}
+
+void declare_method(StructureType* type, TypeSystem* type_system, const goos::Object& def) {
+  for_each_in_list(def, [&](const goos::Object& _obj) {
+    auto obj = &_obj;
+    // (name args return-type [id])
+    auto method_name = symbol_string(car(obj));
+    obj = cdr(obj);
+    auto& args = car(obj);
+    obj = cdr(obj);
+    auto& return_type = car(obj);
+    obj = cdr(obj);
+    int id = -1;
+    if (!obj->is_empty_list()) {
+      auto& id_obj = car(obj);
+      id = get_int(id_obj);
+      if (!cdr(obj)->is_empty_list()) {
+        throw std::runtime_error("too many things in method def: " + def.print());
+      }
+    }
+
+    std::vector<std::string> arg_types;
+    for_each_in_list(args, [&](const goos::Object& o) {
+      if (o.is_symbol()) {
+        arg_types.emplace_back(symbol_string(o));
+      } else {
+        auto next = cdr(&o);
+        arg_types.emplace_back(symbol_string(car(next)));
+        if (!cdr(next)->is_empty_list()) {
+          throw std::runtime_error("too many things in method def arg type: " + def.print());
+        };
+      }
+    });
+
+    auto info = type_system->add_method(
+        type, method_name,
+        type_system->make_function_typespec(arg_types, symbol_string(return_type)));
+
+    // check the method assert
+    if (id != -1) {
+      // method id assert!
+      if (id != info.id) {
+        printf("WARNING - ID assert failed on method %s of type %s (wanted %d got %d)\n",
+               method_name.c_str(), type->get_name().c_str(), id, info.id);
+        throw std::runtime_error("Method ID assert failed");
+      }
+    }
+  });
+}
+
+TypeFlags parse_structure_def(StructureType* type,
+                              TypeSystem* ts,
+                              const goos::Object& fields,
+                              const goos::Object& options) {
+  (void)options;
+  for_each_in_list(fields, [&](const goos::Object& o) { add_field(type, ts, o); });
+  TypeFlags flags;
+  flags.heap_base = 0;
+
+  flags.size = type->get_size_in_memory();
+  flags.pad = 0;
+
+  auto* rest = &options;
+  int size_assert = -1;
+  int method_count_assert = -1;
+  uint64_t flag_assert = 0;
+  bool flag_assert_set = false;
+  while (!rest->is_empty_list()) {
+    if (car(rest).is_pair()) {
+      auto opt_list = &car(rest);
+      auto& first = car(opt_list);
+      opt_list = cdr(opt_list);
+
+      if (symbol_string(first) == ":methods") {
+        declare_method(type, ts, *opt_list);
+      } else {
+        throw std::runtime_error("Invalid option list in field specification: " +
+                                 car(rest).print());
+      }
+
+      rest = cdr(rest);
+    } else {
+      auto opt_name = symbol_string(car(rest));
+      rest = cdr(rest);
+
+      if (opt_name == ":size-assert") {
+        size_assert = get_int(car(rest));
+        if (size_assert == -1) {
+          throw std::runtime_error("Cannot use -1 as size-assert");
+        }
+        rest = cdr(rest);
+      } else if (opt_name == ":method-count-assert") {
+        method_count_assert = get_int(car(rest));
+        if (method_count_assert == -1) {
+          throw std::runtime_error("Cannot use -1 as method_count_assert");
+        }
+        rest = cdr(rest);
+      } else if (opt_name == ":flag-assert") {
+        flag_assert = get_int(car(rest));
+        flag_assert_set = true;
+        rest = cdr(rest);
+      } else {
+        throw std::runtime_error("Invalid option in field specification: " + opt_name);
+      }
+    }
+  }
+
+  if (size_assert != -1 && flags.size != u16(size_assert)) {
+    throw std::runtime_error("Type " + type->get_name() + " came out to size " +
+                             std::to_string(int(flags.size)) + " but size-assert was set to " +
+                             std::to_string(size_assert));
+  }
+
+  flags.methods = ts->get_next_method_id(type);
+
+  if (method_count_assert != -1 && flags.methods != u16(method_count_assert)) {
+    throw std::runtime_error(
+        "Type " + type->get_name() + " has " + std::to_string(int(flags.methods)) +
+        " methods, but method-count-assert was set to " + std::to_string(method_count_assert));
+  }
+
+  if (flag_assert_set && (flags.flag != flag_assert)) {
+    throw std::runtime_error(
+        fmt::format("Type {} has flag 0x{:x} but flag-assert was set to 0x{:x}", type->get_name(),
+                    flags.flag, flag_assert));
+  }
+
+  return flags;
+}
+
+}  // namespace
+
+DeftypeResult parse_deftype(const goos::Object& deftype, TypeSystem* ts) {
+  auto iter = &deftype;
+
+  auto& type_name_obj = car(iter);
+  iter = cdr(iter);
+  auto& parent_list_obj = car(iter);
+  iter = cdr(iter);
+  auto& field_list_obj = car(iter);
+  iter = cdr(iter);
+  auto& options_obj = *iter;
+
+  if (!type_name_obj.is_symbol()) {
+    throw std::runtime_error("deftype must be given a symbol as the type name");
+  }
+
+  auto name = type_name_obj.as_symbol()->name;
+  auto parent_type_name = deftype_parent_list(parent_list_obj);
+  auto parent_type = ts->make_typespec(parent_type_name);
+  DeftypeResult result;
+
+  if (is_type("basic", parent_type, ts)) {
+    auto new_type = std::make_unique<BasicType>(parent_type_name, name);
+    auto pto = dynamic_cast<BasicType*>(ts->lookup_type(parent_type));
+    assert(pto);
+    new_type->inherit(pto);
+    result.flags = parse_structure_def(new_type.get(), ts, field_list_obj, options_obj);
+    ts->add_type(name, std::move(new_type));
+  } else if (is_type("structure", parent_type, ts)) {
+    auto new_type = std::make_unique<StructureType>(parent_type_name, name);
+    auto pto = dynamic_cast<StructureType*>(ts->lookup_type(parent_type));
+    assert(pto);
+    new_type->inherit(pto);
+    result.flags = parse_structure_def(new_type.get(), ts, field_list_obj, options_obj);
+    ts->add_type(name, std::move(new_type));
+  } else if (is_type("integer", parent_type, ts)) {
+    throw std::runtime_error("Creating a child type of integer is not supported yet.");
+  } else {
+    throw std::runtime_error("Creating a child type from " + parent_type.print() +
+                             " is not allowed or not supported yet.");
+  }
+
+  result.type = ts->make_typespec(name);
+  result.type_info = ts->lookup_type(result.type);
+  return result;
+}

common/type_system/deftype.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "TypeSystem.h"
+#include "common/goos/Object.h"
+
+struct TypeFlags {
+  union {
+    uint64_t flag = 0;
+    struct {
+      uint16_t size;
+      uint16_t heap_base;
+      uint16_t methods;
+      uint16_t pad;
+    };
+  };
+};
+
+struct DeftypeResult {
+  TypeFlags flags;
+  TypeSpec type;
+  Type* type_info = nullptr;
+};
+
+DeftypeResult parse_deftype(const goos::Object& deftype, TypeSystem* ts);

common/type_system/type_util.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_TYPE_UTIL_H
 #define JAK_TYPE_UTIL_H
 

common/util/BinaryReader.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_V2_BINARYREADER_H
 #define JAK_V2_BINARYREADER_H
 

common/util/BinaryWriter.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#ifndef JAK_BINARYWRITER_H
+#define JAK_BINARYWRITER_H
+
+#include <cassert>
+#include <stdexcept>
+#include <vector>
+#include <cstdint>
+#include <cstring>
+
+struct BinaryWriterRef {
+  size_t offset;
+  size_t write_size;
+};
+
+class BinaryWriter {
+ public:
+  BinaryWriter() = default;
+
+  template <typename T>
+  BinaryWriterRef add(const T& obj) {
+    auto orig_size = data.size();
+    data.resize(orig_size + sizeof(T));
+    memcpy(data.data() + orig_size, &obj, sizeof(T));
+    return {orig_size, sizeof(T)};
+  }
+
+  template <typename T>
+  void add_at_ref(const T& obj, const BinaryWriterRef& ref) {
+    assert(ref.write_size == sizeof(T));
+    assert(ref.offset + ref.write_size < get_size());
+    memcpy(data.data() + ref.offset, &obj, sizeof(T));
+  }
+
+  void add_str_len(const std::string& str, size_t len) { add_cstr_len(str.c_str(), len); }
+
+  void add_cstr_len(const char* str, size_t len) {
+    size_t i = 0;
+    while (*str) {
+      data.push_back(*str);
+      str++;
+      i++;
+    }
+
+    while (i < len) {
+      data.push_back(0);
+      i++;
+    }
+  }
+
+  BinaryWriterRef add_data(void* d, size_t len) {
+    auto orig_size = data.size();
+    data.resize(orig_size + len);
+    memcpy(data.data() + orig_size, d, len);
+    return {orig_size, len};
+  }
+
+  size_t get_size() { return data.size(); }
+
+  void* get_data() { return data.data(); }
+
+  void write_to_file(const std::string& filename) {
+    auto fp = fopen(filename.c_str(), "wb");
+    if (!fp)
+      throw std::runtime_error("failed to open " + filename);
+    if (fwrite(get_data(), get_size(), 1, fp) != 1)
+      throw std::runtime_error("failed to write " + filename);
+    fclose(fp);
+  }
+
+ private:
+  std::vector<uint8_t> data;
+};
+
+#endif  // JAK_BINARYWRITER_H

common/util/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library(common_util
+        SHARED
+        FileUtil.cpp
+        DgoWriter.cpp
+        Timer.cpp)

common/util/DgoWriter.cpp

@@ -0,0 +1,28 @@
+#include "BinaryWriter.h"
+#include "FileUtil.h"
+#include "DgoWriter.h"
+
+void build_dgo(const DgoDescription& description) {
+  BinaryWriter writer;
+  // dgo header
+  writer.add<uint32_t>(description.entries.size());
+  writer.add_cstr_len(description.dgo_name.c_str(), 60);
+
+  for (auto& obj : description.entries) {
+    auto obj_data = file_util::read_binary_file(file_util::get_file_path({"data", obj.file_name}));
+    // size
+    writer.add<uint32_t>(obj_data.size());
+    // name
+    writer.add_str_len(obj.name_in_dgo, 60);
+    // data
+    writer.add_data(obj_data.data(), obj_data.size());
+    // pad
+    while (writer.get_size() & 0xf) {
+      writer.add<uint8_t>(0);
+    }
+  }
+
+  printf("DGO: %15s %.3f MB\n", description.dgo_name.c_str(),
+         (float)(writer.get_size()) / (1 << 20));
+  writer.write_to_file(file_util::get_file_path({"out", description.dgo_name}));
+}

common/util/DgoWriter.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#ifndef JAK_DGOWRITER_H
+#define JAK_DGOWRITER_H
+
+#include <vector>
+
+struct DgoDescription {
+  std::string dgo_name;
+  struct DgoEntry {
+    std::string file_name;
+    std::string name_in_dgo;
+  };
+  std::vector<DgoEntry> entries;
+};
+
+void build_dgo(const DgoDescription& description);
+
+#endif  // JAK_DGOWRITER_H

common/util/FileUtil.cpp

@@ -0,0 +1,110 @@
+#include "FileUtil.h"
+#include <iostream>
+#include <stdio.h> /* defines FILENAME_MAX */
+#include <fstream>
+#include <sstream>
+#include <cassert>
+
+#ifdef _WIN32
+#include <Windows.h>
+#else
+#include <unistd.h>
+#include <cstring>
+#endif
+
+std::string file_util::get_project_path() {
+#ifdef _WIN32
+  char buffer[FILENAME_MAX];
+  GetModuleFileNameA(NULL, buffer, FILENAME_MAX);
+  std::string::size_type pos =
+      std::string(buffer).rfind("jak-project");  // Strip file path down to \jak-project\ directory
+  return std::string(buffer).substr(
+      0, pos + 11);  // + 12 to include "\jak-project" in the returned filepath
+#else
+  // do Linux stuff
+  char buffer[FILENAME_MAX + 1];
+  auto len = readlink("/proc/self/exe", buffer,
+                      FILENAME_MAX);  // /proc/self acts like a "virtual folder" containing
+                                      // information about the current process
+  buffer[len] = '\0';
+  std::string::size_type pos =
+      std::string(buffer).rfind("jak-project");  // Strip file path down to /jak-project/ directory
+  return std::string(buffer).substr(
+      0, pos + 11);  // + 12 to include "/jak-project" in the returned filepath
+#endif
+}
+
+std::string file_util::get_file_path(const std::vector<std::string>& input) {
+  std::string currentPath = file_util::get_project_path();
+  char dirSeparator;
+
+#ifdef _WIN32
+  dirSeparator = '\\';
+#else
+  dirSeparator = '/';
+#endif
+
+  std::string filePath = currentPath;
+  for (int i = 0; i < int(input.size()); i++) {
+    filePath = filePath + dirSeparator + input[i];
+  }
+
+  return filePath;
+}
+
+void file_util::write_binary_file(const std::string& name, void* data, size_t size) {
+  FILE* fp = fopen(name.c_str(), "wb");
+  if (!fp) {
+    throw std::runtime_error("couldn't open file " + name);
+  }
+
+  if (fwrite(data, size, 1, fp) != 1) {
+    throw std::runtime_error("couldn't write file " + name);
+  }
+
+  fclose(fp);
+}
+
+void file_util::write_text_file(const std::string& file_name, const std::string& text) {
+  FILE* fp = fopen(file_name.c_str(), "w");
+  if (!fp) {
+    printf("Failed to fopen %s\n", file_name.c_str());
+    throw std::runtime_error("Failed to open file");
+  }
+  fprintf(fp, "%s\n", text.c_str());
+  fclose(fp);
+}
+
+std::vector<uint8_t> file_util::read_binary_file(const std::string& filename) {
+  auto fp = fopen(filename.c_str(), "rb");
+  if (!fp)
+    throw std::runtime_error("File " + filename +
+                             " cannot be opened: " + std::string(strerror(errno)));
+  fseek(fp, 0, SEEK_END);
+  auto len = ftell(fp);
+  rewind(fp);
+
+  std::vector<uint8_t> data;
+  data.resize(len);
+
+  if (fread(data.data(), len, 1, fp) != 1) {
+    throw std::runtime_error("File " + filename + " cannot be read");
+  }
+  fclose(fp);
+
+  return data;
+}
+
+std::string file_util::read_text_file(const std::string& path) {
+  std::ifstream file(path);
+  if (!file.good()) {
+    throw std::runtime_error("couldn't open " + path);
+  }
+  std::stringstream ss;
+  ss << file.rdbuf();
+  return ss.str();
+}
+
+bool file_util::is_printable_char(char c) {
+  return c >= ' ' && c <= '~';
+}

common/util/FileUtil.h

@@ -0,0 +1,13 @@
+#pragma once
+#include <string>
+#include <vector>
+
+namespace file_util {
+std::string get_project_path();
+std::string get_file_path(const std::vector<std::string>& input);
+void write_binary_file(const std::string& name, void* data, size_t size);
+void write_text_file(const std::string& file_name, const std::string& text);
+std::vector<uint8_t> read_binary_file(const std::string& filename);
+std::string read_text_file(const std::string& path);
+bool is_printable_char(char c);
+}  // namespace file_util

common/util/MatchParam.h

@@ -1,7 +1,8 @@
+#pragma once
+
 #ifndef JAK1_MATCHPARAM_H
 #define JAK1_MATCHPARAM_H
 
-namespace util {
 template <typename T>
 struct MatchParam {
   MatchParam() { is_wildcard = true; }
@@ -18,6 +19,5 @@ struct MatchParam {
   bool operator==(const T& other) const { return is_wildcard || (value == other); }
   bool operator!=(const T& other) const { return !(*this == other); }
 };
-}  // namespace util
 
 #endif  // JAK1_MATCHPARAM_H

common/util/Timer.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_V2_TIMER_H
 #define JAK_V2_TIMER_H
 

common/versions.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file versions.h
  * Version numbers for GOAL Language, Kernel, etc...

decompiler/CMakeLists.txt

@@ -12,7 +12,6 @@ add_executable(decompiler
         util/FileIO.cpp
         config.cpp
         util/LispPrint.cpp
-        util/Timer.cpp
         Function/BasicBlocks.cpp
         Disasm/InstructionMatching.cpp
         TypeSystem/GoalType.cpp
@@ -22,4 +21,5 @@ add_executable(decompiler
         TypeSystem/TypeSpec.cpp Function/CfgVtx.cpp Function/CfgVtx.h)
 
 target_link_libraries(decompiler
-        minilzo)
+        minilzo
+        common_util)

decompiler/Disasm/Instruction.cpp

@@ -26,7 +26,7 @@ std::string InstructionAtom::to_string(const LinkedObjectFile& file) const {
     case IMM_SYM:
       return sym;
     default:
-      assert(false);
+      throw std::runtime_error("Unsupported InstructionAtom");
   }
 }
 

decompiler/Disasm/Instruction.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file Instruction.h
  * An EE instruction, represented as an operation, plus a list of source/destination atoms.

decompiler/Disasm/InstructionDecode.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file InstructionDecode.h
  * The Instruction Decoder - converts a LinkedWord into a Instruction.

decompiler/Disasm/InstructionMatching.h

@@ -1,24 +1,10 @@
+#pragma once
+
 #ifndef JAK_DISASSEMBLER_INSTRUCTIONMATCHING_H
 #define JAK_DISASSEMBLER_INSTRUCTIONMATCHING_H
 
 #include "Instruction.h"
-
-template <typename T>
-struct MatchParam {
-  MatchParam() { is_wildcard = true; }
-
-  // intentionally not explicit so you don't have to put MatchParam<whatever>(blah) everywhere
-  MatchParam(T x) {
-    value = x;
-    is_wildcard = false;
-  }
-
-  T value;
-  bool is_wildcard = true;
-
-  bool operator==(const T& other) { return is_wildcard || (value == other); }
-  bool operator!=(const T& other) { return !(*this == other); }
-};
+#include "common/util/MatchParam.h"
 
 bool is_no_link_gpr_store(const Instruction& instr,
                           MatchParam<int> size,

decompiler/Disasm/OpcodeInfo.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file OpcodeInfo.h
  * Decoding info for each opcode.

decompiler/Disasm/Register.cpp

@@ -5,6 +5,7 @@
 
 #include "Register.h"
 #include <cassert>
+#include <stdexcept>
 
 ////////////////////////////
 // Register Name Constants
@@ -126,7 +127,7 @@ const char* Register::to_charp() const {
     case Reg::PCR:
       return pcr_to_charp(get_pcr());
     default:
-      assert(false);
+      throw std::runtime_error("Unsupported Register");
   }
 }
 

decompiler/Disasm/Register.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file Register.h
  * Representation of an EE register.

decompiler/Function/BasicBlocks.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_DISASSEMBLER_BASICBLOCKS_H
 #define JAK_DISASSEMBLER_BASICBLOCKS_H
 

decompiler/Function/CfgVtx.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_DISASSEMBLER_CFGVTX_H
 #define JAK_DISASSEMBLER_CFGVTX_H
 

decompiler/Function/Function.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef NEXT_FUNCTION_H
 #define NEXT_FUNCTION_H
 
@@ -30,7 +32,7 @@ struct FunctionName {
       case FunctionKind::UNIDENTIFIED:
         return "(?)";
       default:
-        assert(false);
+        throw std::runtime_error("Unsupported FunctionKind");
     }
   }
 

decompiler/ObjectFile/LinkedObjectFile.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file LinkedObjectFile.h
  * An object file's data with linking information included.

decompiler/ObjectFile/LinkedObjectFileCreation.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file LinkedObjectFileCreation.h
  * Create a LinkedObjectFile from raw object file data.

decompiler/ObjectFile/LinkedWord.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file LinkedWord.h
  * A word (4 bytes), possibly with some linking info.

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -13,9 +13,10 @@
 #include "LinkedObjectFileCreation.h"
 #include "decompiler/config.h"
 #include "third-party/minilzo/minilzo.h"
-#include "decompiler/util/BinaryReader.h"
+#include "common/util/BinaryReader.h"
 #include "decompiler/util/FileIO.h"
-#include "decompiler/util/Timer.h"
+#include "common/util/Timer.h"
+#include "common/util/FileUtil.h"
 #include "decompiler/Function/BasicBlocks.h"
 
 /*!
@@ -70,7 +71,7 @@ ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos) {
   }
 
   printf("ObjectFileDB Initialized:\n");
-  printf(" total dgos: %ld\n", _dgos.size());
+  printf(" total dgos: %lld\n", _dgos.size());
   printf(" total data: %d bytes\n", stats.total_dgo_bytes);
   printf(" total objs: %d\n", stats.total_obj_files);
   printf(" unique objs: %d\n", stats.unique_obj_files);
@@ -142,7 +143,7 @@ constexpr int MAX_CHUNK_SIZE = 0x8000;
  * Load the objects stored in the given DGO into the ObjectFileDB
  */
 void ObjectFileDB::get_objs_from_dgo(const std::string& filename) {
-  auto dgo_data = read_binary_file(filename);
+  auto dgo_data = file_util::read_binary_file(filename);
   stats.total_dgo_bytes += dgo_data.size();
 
   const char jak2_header[] = "oZlB";
@@ -415,7 +416,7 @@ void ObjectFileDB::write_object_file_words(const std::string& output_dir, bool d
       auto file_text = obj.linked_data.print_words();
       auto file_name = combine_path(output_dir, obj.record.to_unique_name() + ".txt");
       total_bytes += file_text.size();
-      write_text_file(file_name, file_text);
+      file_util::write_text_file(file_name, file_text);
       total_files++;
     }
   });
@@ -442,7 +443,7 @@ void ObjectFileDB::write_disassembly(const std::string& output_dir,
       auto file_text = obj.linked_data.print_disassembly();
       auto file_name = combine_path(output_dir, obj.record.to_unique_name() + ".func");
       total_bytes += file_text.size();
-      write_text_file(file_name, file_text);
+      file_util::write_text_file(file_name, file_text);
       total_files++;
     }
   });
@@ -517,7 +518,7 @@ void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) {
   });
 
   auto file_name = combine_path(output_dir, "all_scripts.lisp");
-  write_text_file(file_name, all_scripts);
+  file_util::write_text_file(file_name, all_scripts);
 
   printf("Found scripts:\n");
   printf(" total %.3f ms\n", timer.getMs());

decompiler/ObjectFile/ObjectFileDB.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file ObjectFileDB.h
  * A "database" of object files found in DGO files.

decompiler/TypeSystem/GoalFunction.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_DISASSEMBLER_GOALFUNCTION_H
 #define JAK_DISASSEMBLER_GOALFUNCTION_H
 

decompiler/TypeSystem/GoalSymbol.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_DISASSEMBLER_GOALSYMBOL_H
 #define JAK_DISASSEMBLER_GOALSYMBOL_H
 

decompiler/TypeSystem/GoalType.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_DISASSEMBLER_GOALTYPE_H
 #define JAK_DISASSEMBLER_GOALTYPE_H
 

decompiler/TypeSystem/TypeInfo.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_DISASSEMBLER_TYPEINFO_H
 #define JAK_DISASSEMBLER_TYPEINFO_H
 

decompiler/TypeSystem/TypeSpec.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK_DISASSEMBLER_TYPESPEC_H
 #define JAK_DISASSEMBLER_TYPESPEC_H
 

decompiler/config.cpp

@@ -1,6 +1,7 @@
 #include "config.h"
 #include "third-party/json.hpp"
 #include "util/FileIO.h"
+#include "common/util/FileUtil.h"
 
 Config gConfig;
 
@@ -9,7 +10,7 @@ Config& get_config() {
 }
 
 void set_config(const std::string& path_to_config_file) {
-  auto config_str = read_text_file(path_to_config_file);
+  auto config_str = file_util::read_text_file(path_to_config_file);
   // to ignore comments in json, which may be useful
   auto cfg = nlohmann::json::parse(config_str, nullptr, true, true);
 

decompiler/config.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK2_DISASSEMBLER_CONFIG_H
 #define JAK2_DISASSEMBLER_CONFIG_H
 

decompiler/main.cpp

@@ -5,6 +5,7 @@
 #include "config.h"
 #include "util/FileIO.h"
 #include "TypeSystem/TypeInfo.h"
+#include "common/util/FileUtil.h"
 
 int main(int argc, char** argv) {
   printf("Jak Disassembler\n");
@@ -26,8 +27,8 @@ int main(int argc, char** argv) {
   }
 
   ObjectFileDB db(dgos);
-  write_text_file(combine_path(out_folder, "dgo.txt"), db.generate_dgo_listing());
-  write_text_file(combine_path(out_folder, "obj.txt"), db.generate_obj_listing());
+  file_util::write_text_file(combine_path(out_folder, "dgo.txt"), db.generate_dgo_listing());
+  file_util::write_text_file(combine_path(out_folder, "obj.txt"), db.generate_obj_listing());
 
   db.process_link_data();
   db.find_code();

decompiler/util/FileIO.cpp

@@ -3,35 +3,10 @@
 #include <sstream>
 #include <cassert>
 
-std::string read_text_file(const std::string& path) {
-  std::ifstream file(path);
-  std::stringstream ss;
-  ss << file.rdbuf();
-  return ss.str();
-}
-
 std::string combine_path(const std::string& parent, const std::string& child) {
   return parent + "/" + child;
 }
 
-std::vector<uint8_t> read_binary_file(const std::string& filename) {
-  auto fp = fopen(filename.c_str(), "rb");
-  if (!fp)
-    throw std::runtime_error("File " + filename + " cannot be opened");
-  fseek(fp, 0, SEEK_END);
-  auto len = ftell(fp);
-  rewind(fp);
-
-  std::vector<uint8_t> data;
-  data.resize(len);
-
-  if (fread(data.data(), len, 1, fp) != 1) {
-    throw std::runtime_error("File " + filename + " cannot be read");
-  }
-
-  return data;
-}
-
 std::string base_name(const std::string& filename) {
   size_t pos = 0;
   assert(!filename.empty());
@@ -70,13 +45,3 @@ uint32_t crc32(const uint8_t* data, size_t size) {
 uint32_t crc32(const std::vector<uint8_t>& data) {
   return crc32(data.data(), data.size());
 }
-
-void write_text_file(const std::string& file_name, const std::string& text) {
-  FILE* fp = fopen(file_name.c_str(), "w");
-  if (!fp) {
-    printf("Failed to fopen %s\n", file_name.c_str());
-    throw std::runtime_error("Failed to open file");
-  }
-  fprintf(fp, "%s\n", text.c_str());
-  fclose(fp);
-}

decompiler/util/FileIO.h

@@ -1,15 +1,13 @@
+#pragma once
+
 #ifndef JAK_V2_FILEIO_H
 #define JAK_V2_FILEIO_H
 
 #include <string>
 #include <vector>
 
-std::string read_text_file(const std::string& path);
 std::string combine_path(const std::string& parent, const std::string& child);
-std::vector<uint8_t> read_binary_file(const std::string& filename);
 std::string base_name(const std::string& filename);
-void write_text_file(const std::string& file_name, const std::string& text);
-
 void init_crc();
 uint32_t crc32(const uint8_t* data, size_t size);
 uint32_t crc32(const std::vector<uint8_t>& data);

decompiler/util/LispPrint.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #ifndef JAK2_DISASSEMBLER_LISPPRINT_H
 #define JAK2_DISASSEMBLER_LISPPRINT_H
 

doc/compiler_example.md

@@ -0,0 +1,440 @@
+# Compiler Example
+This describes how the compiler works on a small piece of code in `example_goal.gc`.
+To run this yourself, start the compiler and runtime, then run:
+
+```
+(lt)
+(asm-file "doc/example_goal.gc" :color :load)
+```
+
+And you should see:
+```
+The value of 10 factorial is 3628800
+```
+
+# Overview
+The code to read from the GOAL REPL is in `Compiler.cpp`, in `Compiler::execute_repl`. Compiling an `asm-file` form will call `Compiler::compile_asm_file` in `CompilerControl.cpp`, which is where we'll start.
+
+I've divided the process into these steps:
+1. __Read__: Convert from text to a representation of Lisp syntax.
+2. __IR-Pass__: Convert the S-Expressions to an intermediate representation (IR).
+3. __Register Allocation__: Map variables in the IR (`IRegister`s) to real hardware registers
+4. __Code Generation__: Generate x86-64 instructions from the IR
+5. __Object File Generation__: Put the instructions and static data in an object file and generate linking data
+6. __Sending__: Send the code to the runtime
+7. __Linking__: The runtime links the code so it can be run, and runs it.
+8. __Result__: The code prints the message which is sent back to the REPL.
+
+# Reader
+The reader converts GOAL/GOOS source into a `goos::Object`. One of the core ideas of lisp is that "code is data", so GOAL code is represented as GOOS data.  This makes it easy for GOOS macros to operate on GOAL code.  A GOOS object can represent a number, string, pair, etc. This strips out comments/whitespace. 
+
+The reader is run with this code:
+```
+auto code = m_goos.reader.read_from_file({filename});
+```
+If you were to `code.print()`, you would get:
+```
+(top-level (defun factorial-iterative ((x integer)) (let ((result 1)) (while (!= x 1) (set! result (* result x)) (set! x (- x 1))) result)) (define-extern _format function) (define format _format) (let ((x 10)) (format #t "The value of ~D factorial is ~D~%" x (factorial-iterative x))))
+```
+
+
+There are a few details worth mentioning about this process:
+- The reader will expand `'my-symbol` to `(quote my-symbol)`
+- The reader will throw errors on synatx errors (mismatched parentheses, bad strings/numbers, etc.)
+- Using `read_from_file` adds information about where each thing came from to a map stored in the reader.  This map is used to determine the source file/line for compiler errors.
+
+# IR Pass
+This pass converts code (represented as a `goos::Object`) into intermediate representation.  This is stored in an `Env*`, a tree structure.  At the top is a `GlobalEnv*`, then an `FileEnv` for each file compiled, then a `FunctionEnv` for each function in the file.  There are environments within `FunctionEnv` that are used for lexical scoping and the other types of GOAL scopes. The Intermediate Representation (IR) is a list per function that's built up in order as the compiler goes through the function. Note that the IR is a list of instructions and doesn't have a tree or more complicated structure.  Here's an example of the IR for the example function:
+
+```
+Function: function-factorial-iterative
+  mov-ic igpr-2, 1 
+  mov igpr-3, igpr-2 
+  goto label-10
+  mov igpr-4, igpr-3
+  imul igpr-4, igpr-0
+  mov igpr-3, igpr-4
+  mov igpr-5, igpr-0
+  mov-ic igpr-6, 1
+  subi igpr-5, igpr-6
+  mov igpr-0, igpr-5
+  mov-ic igpr-7, 1
+  j(igpr-0 != igpr-7) label-3
+  ret igpr-1 igpr-3
+
+Function: function-top-level
+  mov-fa igpr-0, function-factorial-iterative
+  mov 'factorial-iterative, igpr-0
+  mov igpr-1, '_format
+  mov 'format, igpr-1
+  mov-ic igpr-2, 10
+  mov igpr-3, igpr-2
+  mov igpr-4, '#t
+  mov-sva igpr-5, static-string "The value of ~D factorial is ~D~%"
+  mov igpr-6, 'factorial-iterative
+  mov igpr-8, igpr-3
+  call igpr-6 (ret igpr-7) (args igpr-8)
+  mov igpr-9, igpr-7
+  mov igpr-10, 'format
+  mov igpr-12, igpr-4
+  mov igpr-13, igpr-5
+  mov igpr-14, igpr-3
+  mov igpr-15, igpr-9
+  call igpr-10 (ret igpr-11) (args igpr-12 igpr-13 igpr-14 igpr-15)
+  mov igpr-16, igpr-11
+  ret igpr-17 igpr-16
+```
+The `function-top-level` is the "top level" function, which is everything not in a function. In the example, this is just defining the function, defining `format`, and calling `format`. 
+
+You'll notice that there are a ton of `mov`s between `igpr`. The compiler inserts tons of moves. Because this is all done in a single pass, there's a lot of cases where the compiler can't know if a move is needed or not. But the register allocator can figure it out and will remove most unneeded moves.  Adding moves can also prevent stack spills. For example, consider the case where you want to get the return value of function `a`, then call function `b`, then call function `c` with the return value of function `a`. If there are a lot of moves, the register allocator can figure out a way to temporarily stash the value in a saved register instead of spilling to the stack.
+
+Another thing to notice is that GOAL nested function calls suck. Example:
+```
+(format #t "The value of ~D factorial is ~D~%" x (factorial-iterative x))
+```
+requires loading `format`, `#t`, the string, and `x` into registers, then calling `(factorial-iterative x)`, then calling `format`. This has to be done this way, just in case the `factorial-iterative` call modifies the value of `format`, `#t`, or `x`.
+
+## IR Pass Implementation
+An important type in the compiler is `Val`, which is a specification on how to get a value.  A `Val` has an associated GOAL type (`TypeSpec`) and the IR Pass should take care of all type checking.  A `Val` can represent a constant, a memory location (relative to pointer, or static data, etc), a spot in an array, a register etc.  A `Val` representing a register is a `RegVal`, which contains an `IRegister`. An `IRegister` is a register that's not yet mapped to the hardware, and instead has a unique integer to identify itself. The IR assumes there are infinitely many `IRegister`s, and a later stage maps `IRegister`s to real hardware registers.
+
+The general process starts with a `compile` function, which dispatches other `compile_<thing>` functions as needed.  These generally take in `goos::Object` as a code input, emit IR into an `Env`/or modify things in the `Env`, and return a `Val*` describing the result.  
+
+In general, GOAL is very greedy and `compile` functions emit IR to do things, then put the result in a register, and return a `RegVal`. 
+
+However, there is an exception for memory related things.  Consider
+```
+(-> my-object my-field) ;  my_object->my_field in C
+```	
+This shouldn't return a `Val` for a register containing the value of `my_object->my_field`, but should instead return something that represents "the memory location of my_field in my_object". This way you can do 
+```
+(set! (-> my-object my-field) val)
+;; or
+(& (-> my-object my-field)) ;; &my_object->my_field
+```
+and the compiler will have enough information to figure out the memory address. 
+
+If the compiler actually needs the value of something, and wants to be sure its a value in a register, it will use the `to_reg` method of `Val`. This will emit IR into the current function to get the value in a register, then return a `Val` that represents this register.  Example `to_reg` implementation for integer constants:
+```
+RegVal* IntegerConstantVal::to_reg(Env* fe) {
+  auto rv = fe->make_gpr(m_ts);
+  fe->emit(std::make_unique<IR_LoadConstant64>(rv, m_value));
+  return rv;
+}
+```
+
+
+Note that `to_reg` can emit code like reading from memory, where the order of operations really matters, so you have to be very careful.
+
+It's extremely dangerous to let a memory reference `Val` propagate too far. Consider this example:
+
+```
+(let ((x (-> my-object my-field)))
+  (set! (-> my-object my-field) 12)
+  x
+  )
+```
+Where `x` should be the old value of `my-field`. The `Val` for `x` needs to be `to_reg`ed _before_ getting inside the `let`. There's also some potential confusion around the order that you compile and `to_gpr` things.  In a case where you need a bunch of values in gprs, you should do the `to_gpr` immediately after compiling to match the exact behavior of the original GOAL. For example
+
+```
+(+ (-> my-array (* x y)) (some-function))
+;; like c++ my_array[x*y] + some_function()
+```
+When we `compile` the `(-> my-array (* x y))`, it will emit code to calculate the `(*x y)`, but won't actually do the memory access until we call `to_reg` on the result.  This memory access should happen __before__ `some-function` is called.
+
+
+In general, each time you `compile` something, you should immediately `to_gpr` it, _before_ `compile`ing the next thing.  Many places will only accept a `RegVal` as an input to help with this.  Also, the result for almost all compilation functions should be `to_reg`ed. The only exceptions are forms which deal with memory references (address of operator, dereference operator) or math.
+
+Another important thing is that compilation functions should _never_ modify any existing `IRegister`s or `Val`s, unless that function is `set!`, which handles the situation correctly. Instead, create new `IRegister`s and move into those. I am planning to implement a `settable` flag to help reduce errors. 
+
+For example:
+- `RegVal` storing a local variable: is `settable`, you can modify local variables by writing to the register they use.
+- `RegVal` storing the result of a memory dereference: not `settable`, you should set the memory instead.
+- `RegVal` containing the result of converting a `xmm` to `gpr`: not settable, you need to set the original `xmm` instead
+
+The only settable `RegVal` is one corresponding to a local variable.
+
+## Following the Code
+
+This pass runs from here:
+```
+auto obj_file = compile_object_file(obj_file_name, code, !no_code);
+```
+That function sets up a `FileEnv*`, then runs
+```
+  file_env->add_top_level_function(
+      compile_top_level_function("top-level", std::move(code), compilation_env));
+```      
+which compiles the body of the function with:
+```
+auto result = compile_error_guard(code, fe.get());
+```
+
+The `compile_error_guard` function takes in code (as a `goos::Object`) and a `Env*`, and returns a `Val` representing the return value of the code. It calls the `compile` function, but wraps it in a `try catch` block to catch any compilation errors and print an error message.  In the case where there's no error, it just does:
+```
+return compile(code, env);
+```
+
+The `compile` function is pretty simple:
+```
+/*!
+ * Highest level compile function
+ */
+Val* Compiler::compile(const goos::Object& code, Env* env) {
+  switch (code.type) {
+    case goos::ObjectType::PAIR:
+      return compile_pair(code, env);
+    case goos::ObjectType::INTEGER:
+      return compile_integer(code, env);
+    case goos::ObjectType::SYMBOL:
+      return compile_symbol(code, env);
+    case goos::ObjectType::STRING:
+      return compile_string(code, env);
+    case goos::ObjectType::FLOAT:
+      return compile_float(code, env);
+    default:
+      ice("Don't know how to compile " + code.print());
+  }
+  return get_none();
+}
+```
+In our case, the code starts with `(defun..`, which is actually a GOOS macro. It throws away the docstring, creates a lambda, then stores the function in a symbol:
+```
+;; Define a new function
+(defmacro defun (name bindings &rest body)
+  (if (and
+        (> (length body) 1)      ;; more than one thing in function
+        (string? (first body))   ;; first thing is a string
+        )
+    ;; then it's a docstring and we ignore it.
+    `(define ,name (lambda :name ,name ,bindings ,@(cdr body)))
+    ;; otherwise don't ignore it.
+    `(define ,name (lambda :name ,name ,bindings ,@body))
+    )
+  )
+```
+
+The compiler notices this is a macro in `compile_pair`:
+```
+  if (head.is_symbol()) {
+  	// ...
+
+    goos::Object macro_obj;
+    if (try_getting_macro_from_goos(head, &macro_obj)) {
+      return compile_goos_macro(code, macro_obj, rest, env);
+    }
+
+    // ...
+  }
+
+```
+The `compile_goos_macro` function sets up a GOOS environment and interprets the GOOS macro to generate more GOAL code:
+```
+Val* Compiler::compile_goos_macro(const goos::Object& o,
+                                  const goos::Object& macro_obj,
+                                  const goos::Object& rest,
+                                  Env* env) {
+  auto macro = macro_obj.as_macro();
+  Arguments args = m_goos.get_args(o, rest, macro->args);
+  auto mac_env_obj = EnvironmentObject::make_new(); // GOOS environment
+  auto mac_env = mac_env_obj.as_env();
+  mac_env->parent_env = m_goos.global_environment.as_env();
+  m_goos.set_args_in_env(o, args, macro->args, mac_env);
+  auto goos_result = m_goos.eval_list_return_last(macro->body, macro->body, mac_env); // evaluate GOOS macro
+  return compile_error_guard(goos_result, env); // compile resulting GOAL code
+}
+```
+and the last line of that function compiles the result of macro expansion in GOAL. 
+
+As an example, I'm going to look at `compile_add`, which handles the `+` form, and is representative of typical compiler code for this part.  We start by checking that the arguments look valid:
+```
+Val* Compiler::compile_add(const goos::Object& form, const goos::Object& rest, Env* env) {
+  auto args = get_va(form, rest); // get arguments to + in a list
+  if (!args.named.empty() || args.unnamed.empty()) {
+    throw_compile_error(form, "Invalid + form");
+  }
+```
+
+Then we compile the first thing in the `(+ ...` form, get its type, and pick a math mode (int, float):
+```  
+auto first_val = compile_error_guard(args.unnamed.at(0), env);
+auto first_type = first_val->type();
+auto math_type = get_math_mode(first_type);
+```
+In the integer case, we first create a new variable in the IR called an `IRegister` that must be in a GPR (as opposed to an XMM floating point register), and then emit an IR instruction that sets this result register to the first argument.
+```
+auto result = env->make_gpr(first_type);
+env->emit(std::make_unique<IR_RegSet>(result, first_val->to_gpr(env)));
+```
+
+Then, for each of the remaining arguments, we do:
+```
+      for (size_t i = 1; i < args.unnamed.size(); i++) {
+        env->emit(std::make_unique<IR_IntegerMath>(
+            IntegerMathKind::ADD_64, result,
+            to_math_type(compile_error_guard(args.unnamed.at(i), env), math_type, env)
+                ->to_gpr(env)));
+      }
+```
+which emits an IR to add the value to the sum.  The `to_math_type` will emit any code needed to convert this to the correct numeric type (returns either a numeric constant or a `RegVal` containing the value).
+
+An important detail is that we create a new register which will hold the result. This may seem inefficient in cases, but a later compile pass will try to make this new register be the same register as `first_val` if possible, and will eliminate the `IR_RegSet`.
+
+# Register Allocation
+
+This step figures out how to match up `IRegister`s to real hardware registers. In the case where there aren't enough hardware registers, it figures out how to "spill" variables onto the stack. The current implementation is a very greedy one, so it doesn't always succeed at doing things perfectly. The stack spilling is also not handled very efficiently, but the hope is that most functions won't require stack spilling.
+
+This step is run from `compile_asm_function` on the line:
+```
+color_object_file(obj_file);
+
+void Compiler::color_object_file(FileEnv* env) {
+  for (auto& f : env->functions()) {
+    AllocationInput input;
+    for (auto& i : f->code()) {
+      input.instructions.push_back(i->to_rai());
+      input.debug_instruction_names.push_back(i->print());
+    }
+    input.max_vars = f->max_vars();
+    input.constraints = f->constraints();
+
+    // debug prints removed
+
+    f->set_allocations(allocate_registers(input));
+  }
+}
+```
+The actual algorithm is too complicated to describe here, but it figures out a mapping from `IRegister`s to hardware registers. It also figures out how much space on the stack is needed for any stack spills, which saved registers will be used, and deals with aligning the stack.
+
+
+# Code Generation
+This part actually generates the static data and x86 instructions and stores them in an `ObjectGenerator`. See `CodeGenerator::do_function`. It emits the function prologue and epilogue, as well as any extra loads/stores from the stack that the register allocator added.  Each `IR` gets to emit instructions with:
+```
+ir->do_codegen(&m_gen, allocs, i_rec);
+```
+
+Each IR has its own `do_codegen` that emits the right instruction, and also any linking data that's needed. For example, instructions that access the symbol table are patched by the runtime to directly access the correct slot of the hash table, so the `do_codegen` also lets the `ObjectGenerator` know about this link:
+```
+// IR_GetSymbolValue::do_codegen
+
+// look at register allocation result to determine hw register
+auto dst_reg = get_reg(m_dest, allocs, irec);
+
+// add an instruction
+auto instr = gen->add_instr(IGen::load32u_gpr64_gpr64_plus_gpr64_plus_s32(
+                             dst_reg, gRegInfo.get_st_reg(), gRegInfo.get_offset_reg(), 0x0badbeef), irec);
+
+// add link info
+gen->link_instruction_symbol_mem(instr, m_src->name());
+```
+here `0xbadbeef` is used as a placeholder offset - the runtime should patch this to the actual offset of the symbol.
+
+There's a ton of book-keeping to figure out the correct offsets for `rip`-relative addressing, or how to deal with jumps to/from IR which become multiple (or zero!) x86-64 instructions.  It should all be handled by `ObjectFileGenerator`, and not in `do_codegen` or `CodeGenerator`.
+
+# Object File Generation
+Once the `CodeGenerator` is done going through all functions and static data, it runs:
+```
+return m_gen.generate_data_v3().to_vector();
+```
+This actually lays out everything in memory. It takes a few passes because x86 instructions are variable length (may even change based on which reigsters are used!), so it's a little bit tricky to figure out offsets between different instructions or instructions and data. Finally it generates link data tables, which efficiently pack together links to the same symbols into a single entry, to avoid duplicated symbol names.  The link table also contains information about linking references in between different segments, as different parts of the object file may be loaded into different spots in memory, and will need to reference each other.
+
+This is the final result for top-level function (stored in top-level segment)
+```
+;; prologue
+push   rbx
+push   rbp
+push   r10
+push   r11
+push   rbx
+
+;; load address of factorial-iterative function
+lea    rax,[rip+0x0]
+;; convert to GOAL pointer
+sub    rax,r15
+;; store in symbol table
+mov    DWORD PTR [r15+r14*1+0xbadbeef],eax
+
+;; load _format value
+mov    eax,DWORD PTR [r15+r14*1+0xbadbeef]
+;; store in format
+mov    DWORD PTR [r15+r14*1+0xbadbeef],eax
+
+;; load constant 10 (the greedy regalloc does poorly here)
+mov    eax,0xa
+mov    rbx,rax
+
+;; load format from symbol table
+mov    ebp,DWORD PTR [r15+r14*1+0xbadbeef]
+
+;; load #t from symbol table
+mov    r10d,DWORD PTR [r15+r14*1+0xbadbeef]
+
+;; get address of string
+lea    r11,[rip+0x0]
+;; convert to GOAL pointer
+sub    r11,r15
+
+;; get factorial-iterative from symbol table
+mov    ecx,DWORD PTR [r15+r14*1+0xbadbeef]
+;; move 10 into argument register
+mov    rdi,rbx
+;; convert factorial-iterative to a real pointer
+add    rcx,r15
+;; call factorial
+call   rcx
+
+;; move args into argument registers
+mov    rdi,r10
+mov    rsi,r11
+mov    rdx,rbx
+mov    rcx,rax
+
+;; call format
+add    rbp,r15
+call   rbp
+
+;; epilogue
+pop    rbx
+pop    r11
+pop    r10
+pop    rbp
+pop    rbx
+ret
+```
+and the factorial function (stored in main segment)
+```
+mov    eax,0x1
+jmp    0x18
+mul   eax,edi
+movsxd rax,eax
+mov    ecx,0x1
+sub    rdi,rcx
+mov    ecx,0x1
+cmp    rdi,rcx
+jne    0xa
+ret
+```
+
+# Sending and Receiving
+The result of `codegen_object_file` is sent with:
+```
+m_listener.send_code(data);
+```
+which adds a message header then just sends the code over the socket.
+
+The receive process is complicated, so this is just a quick summary
+- `Deci2Server` receives it
+- calls `GoalProtoHandler` in chunks, which stores it in a buffer (`MessBuffArea`)
+- Once fully received, `WaitForMessageAndAck` will return a pointer to the message. The name of this function is totally wrong, it doesn't wait for a message, and it doesn't ack the message.
+- The main loop in `KernelCheckAndDispatch` will see this message and run `ProcessListenerMessage`
+- `ProcessListenerMessage` sees that it has code, copies the message to the debug heap, and links it.
+```
+auto buffer = kmalloc(kdebugheap, MessCount, 0, "listener-link-block");
+memcpy(buffer.c(), msg.c(), MessCount);
+ListenerLinkBlock->value = buffer.offset + 4;
+ListenerFunction->value = link_and_exec(buffer, "*listener*", 0, kdebugheap, LINK_FLAG_FORCE_DEBUG).offset;
+```
+- The `link_and_exec` function doesn't actually exectue anything becuase it doesn't have the `LINK_FLAG_EXECUTE` set, it just links things. It moves the top level function and linking data to the top of the heap (temporary storage for the kernel) and keep both the main segment and debug segment of the code on the debug heap.  It'll move them together and eliminate gaps before linking.  After linking, the `ListenerFunction->value` will contain a pointer to the top level function, which is stored in the top temp area of the heap.  This `ListenerFunction` is the GOAL `*listener-function*` symbol.
+- The next time the GOAL kernel runs, it will notice that `*listener-function*` is set, then call this function, then set it to `#f` to indicate it called the function.
+- This 
+- After this, `ClearPending()` is called, which sends all of the `print` messages with the `Deci2Server` back to the compiler.
+- Because the GOAL kernel changed `ListenerFunction` to `#f`, it does a `SendAck()` to send a special `ACK` message to the compiler, saying "I got the function, ran it, and didn't crash. Now I'm ready for more messages."

doc/example_goal.gc

@@ -0,0 +1,18 @@
+(defun factorial-iterative ((x integer))
+  (let ((result 1))
+    (while (!= x 1)
+	   (set! result (* result x))
+	   (set! x (- x 1))
+	   )
+    result
+    )
+  )
+
+;; until we load KERNEL.CGO automatically, we have to do this to
+;; make format work correctly.
+(define-extern _format function)
+(define format _format)
+
+(let ((x 10))
+  (format #t "The value of ~D factorial is ~D~%" x (factorial-iterative x))
+  )

doc/goal_todo.md

@@ -0,0 +1,19 @@
+Done (has documentation)
+- `e`
+- `:exit`
+- `asm-file`, `m`, `ml`
+- `listen-to-target`, `reset-target`, `:status`, `lt`, `r`
+
+Done (needs documentation)
+- `top-level`
+- `begin`
+- `seval`
+- `#cond`, `#when`, `#unless`
+
+- Macro System
+
+
+
+
+Todo
+- Type System

game/CMakeLists.txt

@@ -1,5 +1,5 @@
 # We define our own compilation flags here.
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 
 # Set default compile flags for GCC
 # optimization level can be set here. Note that game/ overwrites this for building game C++ code.
@@ -78,8 +78,8 @@ add_library(runtime ${RUNTIME_SOURCE})
 
 IF (WIN32)
   # set stuff for windows
-  target_link_libraries(gk cross_sockets mman)
+  target_link_libraries(gk cross_sockets mman common_util)
 ELSE()
   # set stuff for other systems
-  target_link_libraries(gk cross_sockets pthread)
+  target_link_libraries(gk cross_sockets pthread common_util)
 ENDIF()

game/common/dgo_rpc_types.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file dgo_rpc_types.h
  * Types used for the DGO Remote Procedure Call between the EE and the IOP

game/common/loader_rpc_types.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file loader_rpc_types.h
  * Types used for the Loader Remote Procedure Call between the EE and the IOP

game/common/play_rpc_types.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file play_rpc_types.h
  * Types used for the play Remote Procedure Call between the EE and the IOP.

game/common/player_rpc_types.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file player_rpc_types.h
  * Types used for the player Remote Procedure Call between the EE and the IOP.

game/common/ramdisk_rpc_types.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file ramdisk_rpc_types.h
  * Types used for the RamDisk Remote Procedure Call between the EE and the IOP

game/fake_iso.txt

@@ -2,8 +2,8 @@
 ; Each entry should consist of an ISO name, followed by a file name
 ; note that tweakval, vagdir, screen1 have dummy data for now.
 
-KERNEL.CGO resources/KERNEL.CGO
-GAME.CGO resources/GAME.CGO
+KERNEL.CGO out/KERNEL.CGO
+GAME.CGO out/GAME.CGO
 TEST.CGO resources/TEST.CGO
 TWEAKVAL.MUS resources/TWEAKVAL.MUS
 VAGDIR.AYB resources/VAGDIR.AYB

game/kernel/Ptr.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file Ptr.h
  * Representation of a GOAL pointer which can be converted to/from a C pointer.

game/kernel/asm_funcs.asm

@@ -121,9 +121,9 @@ _call_goal_asm_linux:
   ;; set GOAL function pointer
   mov r13, rcx
   ;; offset
-  mov r15, r8
+  mov r14, r8
   ;; symbol table
-  mov r14, r9
+  mov r15, r9
   ;; call GOAL by function pointer
   call r13
 
@@ -165,8 +165,8 @@ _call_goal_asm_win32:
   mov rsi, rdx ;; rsi is GOAL second argument, rdx is windows second argument
   mov rdx, r8  ;; rdx is GOAL third argument, r8 is windows third argument
   mov r13, r9  ;; r13 is GOAL fp, r9 is windows fourth argument
-  mov r15, [rsp + 144] ;; symbol table
-  mov r14, [rsp + 152] ;; offset
+  mov r15, [rsp + 152] ;; symbol table
+  mov r14, [rsp + 144] ;; offset
   
   call r13
   

game/kernel/fileio.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file fileio.h
  * GOAL Low-Level File I/O and String Utilities

game/kernel/kboot.cpp

@@ -15,6 +15,7 @@
 #include "kscheme.h"
 #include "ksocket.h"
 #include "klisten.h"
+#include "kprint.h"
 
 #ifdef _WIN32
 #include "Windows.h"
@@ -136,7 +137,30 @@ void KernelCheckAndDispatch() {
     auto old_listener = ListenerFunction->value;
     // dispatch the kernel
     //(**kernel_dispatcher)();
-    call_goal(Ptr<Function>(kernel_dispatcher->value), 0, 0, 0, s7.offset, g_ee_main_mem);
+
+    // todo remove. this is added while KERNEL.CGO is broken.
+    if (MasterUseKernel) {
+      call_goal(Ptr<Function>(kernel_dispatcher->value), 0, 0, 0, s7.offset, g_ee_main_mem);
+    } else {
+      if (ListenerFunction->value != s7.offset) {
+        fprintf(stderr, "Running Listener Function:\n");
+        auto cptr = Ptr<u8>(ListenerFunction->value).c();
+        for (int i = 0; i < 40; i++) {
+          fprintf(stderr, "%x ", cptr[i]);
+        }
+        fprintf(stderr, "\n");
+        auto result =
+            call_goal(Ptr<Function>(ListenerFunction->value), 0, 0, 0, s7.offset, g_ee_main_mem);
+        fprintf(stderr, "result of listener function: %lld\n", result);
+#ifdef __linux__
+        cprintf("%ld\n", result);
+#else
+        cprintf("%lld\n", result);
+#endif
+        ListenerFunction->value = s7.offset;
+      }
+    }
+
     ClearPending();
 
     // if the listener function changed, it means the kernel ran it, so we should notify compiler.

game/kernel/kboot.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file kboot.h
  * GOAL Boot.  Contains the "main" function to launch GOAL runtime.
@@ -73,4 +75,6 @@ void KernelCheckAndDispatch();
  */
 void KernelShutdown();
 
+constexpr bool MasterUseKernel = false;
+
 #endif  // RUNTIME_KBOOT_H

game/kernel/kdgo.cpp

@@ -138,8 +138,8 @@ u32 InitRPC() {
  */
 void StopIOP() {
   x[2] = 0x14;  // todo - this type and message
-  RpcSync(PLAYER_RPC_CHANNEL);
-  RpcCall(PLAYER_RPC_CHANNEL, 0, false, x, 0x50, nullptr, 0);
+                //  RpcSync(PLAYER_RPC_CHANNEL);
+                //  RpcCall(PLAYER_RPC_CHANNEL, 0, false, x, 0x50, nullptr, 0);
   printf("IOP shut down\n");
   //  sceDmaSync(0x10009000, 0, 0);
   printf("DMA shut down\n");

game/kernel/kdgo.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file kdgo.h
  * Loading DGO Files.  Also has some general SIF RPC stuff used for RPCs other than DGO loading.

game/kernel/kdsnetm.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file kdsnetm.h
  * Low-level DECI2 wrapper for ksocket

game/kernel/klink.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file klink.cpp
  * GOAL Linker for x86-64

game/kernel/klisten.cpp

@@ -133,6 +133,9 @@ void ProcessListenerMessage(Ptr<char> msg) {
       break;
     case LTT_MSG_RESET:
       MasterExit = 1;
+      if (protoBlock.msg_id == UINT64_MAX) {
+        MasterExit = 2;
+      }
       break;
     case LTT_MSG_CODE: {
       auto buffer = kmalloc(kdebugheap, MessCount, 0, "listener-link-block");
@@ -146,6 +149,7 @@ void ProcessListenerMessage(Ptr<char> msg) {
       // this setup allows listener function execution to clean up after itself.
       ListenerFunction->value =
           link_and_exec(buffer, "*listener*", 0, kdebugheap, LINK_FLAG_FORCE_DEBUG).offset;
+      fprintf(stderr, "ListenerFunction is now 0x%x\n", ListenerFunction->value);
       return;  // don't ack yet, this will happen after the function runs.
     } break;
     default:

game/kernel/klisten.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file klisten.h
  * Implementation of the Listener protocol

game/kernel/kmachine.cpp

@@ -595,7 +595,8 @@ void InitMachineScheme() {
     intern_from_c("*kernel-boot-level*")->value = intern_from_c(DebugBootLevel).offset;
   }
 
-  if (DiskBoot) {
+  // todo remove MasterUseKernel
+  if (DiskBoot && MasterUseKernel) {
     *EnableMethodSet = (*EnableMethodSet) + 1;
     load_and_link_dgo_from_c("game", kglobalheap,
                              LINK_FLAG_OUTPUT_LOAD | LINK_FLAG_EXECUTE | LINK_FLAG_PRINT_LOGIN,

game/kernel/kmachine.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file kmachine.h
  * GOAL Machine.  Contains low-level hardware interfaces for GOAL.

game/kernel/kmalloc.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file kmalloc.h
  * GOAL Kernel memory allocator.

game/kernel/kmemcard.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file kmemcard.h
  * Memory card interface. Very messy code.

game/kernel/kprint.cpp

@@ -884,6 +884,8 @@ s32 format_impl(uint64_t* args) {
                 call_method_of_type(in, type, GOAL_PRINT_METHOD);
               }
             } else {
+              // TODO - if we can't throw exceptions, what is the option?
+              // log, break and continue?
               throw std::runtime_error("failed to find symbol in format!");
             }
           }

game/kernel/kprint.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file kprint.h
  * GOAL Print.  Contains GOAL I/O, Print, Format...

game/kernel/kscheme.cpp

@@ -342,7 +342,8 @@ Ptr<Function> make_function_from_c_linux(void* func) {
  * Create a GOAL function from a C function. This doesn't export it as a global function, it just
  * creates a function object on the global heap.
  *
- * The implementation is to create a simple trampoline function which jumps to the C function.
+ * This creates a simple trampoline function which jumps to the C function and reorders the
+ * arguments to be correct for Windows.
  */
 Ptr<Function> make_function_from_c_win32(void* func) {
   // allocate a function object on the global heap
@@ -386,6 +387,38 @@ Ptr<Function> make_function_from_c_win32(void* func) {
   return mem.cast<Function>();
 }
 
+/*!
+ * Create a GOAL function from a C function.  This calls a windows function, but doesn't scramble
+ * the argument order.  It's supposed to be used with _format_win32 which assumes GOAL order.
+ */
+Ptr<Function> make_function_for_format_from_c_win32(void* func) {
+  // allocate a function object on the global heap
+  auto mem = Ptr<u8>(
+      alloc_heap_object(s7.offset + FIX_SYM_GLOBAL_HEAP, *(s7 + FIX_SYM_FUNCTION_TYPE), 0x80));
+  auto f = (uint64_t)func;
+  auto fp = (u8*)&f;
+
+  int i = 0;
+  // we will put the function address in RAX with a movabs rax, imm8
+  mem.c()[i++] = 0x48;
+  mem.c()[i++] = 0xb8;
+  for (int j = 0; j < 8; j++) {
+    mem.c()[i++] = fp[j];
+  }
+
+  /*
+   * sub rsp, 40
+   * call rax
+   * add rsp, 40
+   * ret
+   */
+  for (auto x : {0x48, 0x83, 0xEC, 0x28, 0xFF, 0xD0, 0x48, 0x83, 0xC4, 0x28, 0xC3}) {
+    mem.c()[i++] = x;
+  }
+
+  return mem.cast<Function>();
+}
+
 /*!
  * Create a GOAL function from a C function. This doesn't export it as a global function, it just
  * creates a function object on the global heap.
@@ -440,6 +473,17 @@ Ptr<Function> make_function_symbol_from_c(const char* name, void* f) {
   return func;
 }
 
+/*!
+ * Given a C function and a name, create a GOAL function and store it in the symbol with the given
+ * name. This is designed for _format_win32, which is special because it takes 8 arguments.
+ */
+Ptr<Function> make_format_function_symbol_from_c_win32(const char* name, void* f) {
+  auto sym = intern_from_c(name);
+  auto func = make_function_for_format_from_c_win32(f);
+  sym->value = func.offset;
+  return func;
+}
+
 /*!
  * Set the named symbol to the value. This isn't specific to functions.
  */
@@ -958,7 +1002,9 @@ uint64_t _call_goal_asm_win32(u64 a0, u64 a1, u64 a2, void* fptr, void* st_ptr,
  * Wrapper around _call_goal_asm for calling a GOAL function from C.
  */
 u64 call_goal(Ptr<Function> f, u64 a, u64 b, u64 c, u64 st, void* offset) {
-  auto st_ptr = (void*)((uint8_t*)(offset) + st);
+  // auto st_ptr = (void*)((uint8_t*)(offset) + st); updated for the new compiler!
+  void* st_ptr = (void*)st;
+
   void* fptr = f.c();
 #ifdef __linux__
   return _call_goal_asm_linux(a, b, c, fptr, st_ptr, offset);
@@ -1776,7 +1822,7 @@ s32 InitHeapAndSymbol() {
 #ifdef __linux__
   make_function_symbol_from_c("_format", (void*)_format_linux);
 #elif _WIN32
-  make_function_symbol_from_c("_format", (void*)_format_win32);
+  make_format_function_symbol_from_c_win32("_format", (void*)_format_win32);
 #endif
 
   // allocations
@@ -1828,25 +1874,28 @@ s32 InitHeapAndSymbol() {
   intern_from_c("*boot-video-mode*")->value = 0;
 
   // load the kernel!
-  method_set_symbol->value++;
-  load_and_link_dgo_from_c("kernel", kglobalheap,
-                           LINK_FLAG_OUTPUT_LOAD | LINK_FLAG_EXECUTE | LINK_FLAG_PRINT_LOGIN,
-                           0x400000);
-  method_set_symbol->value--;
+  // todo, remove MasterUseKernel
+  if (MasterUseKernel) {
+    method_set_symbol->value++;
+    load_and_link_dgo_from_c("kernel", kglobalheap,
+                             LINK_FLAG_OUTPUT_LOAD | LINK_FLAG_EXECUTE | LINK_FLAG_PRINT_LOGIN,
+                             0x400000);
+    method_set_symbol->value--;
 
-  // check the kernel version!
-  auto kernel_version = intern_from_c("*kernel-version*")->value;
-  if (!kernel_version || ((kernel_version >> 0x13) != KERNEL_VERSION_MAJOR)) {
-    MsgErr("\n");
-    MsgErr(
-        "dkernel: compiled C kernel version is %d.%d but the goal kernel is %d.%d\n\tfrom the "
-        "goal> prompt (:mch) then mkee your kernel in linux.\n",
-        KERNEL_VERSION_MAJOR, KERNEL_VERSION_MINOR, kernel_version >> 0x13,
-        (kernel_version >> 3) & 0xffff);
-    return -1;
-  } else {
-    printf("Got correct kernel version %d.%d\n", kernel_version >> 0x13,
-           (kernel_version >> 3) & 0xffff);
+    // check the kernel version!
+    auto kernel_version = intern_from_c("*kernel-version*")->value;
+    if (!kernel_version || ((kernel_version >> 0x13) != KERNEL_VERSION_MAJOR)) {
+      MsgErr("\n");
+      MsgErr(
+          "dkernel: compiled C kernel version is %d.%d but the goal kernel is %d.%d\n\tfrom the "
+          "goal> prompt (:mch) then mkee your kernel in linux.\n",
+          KERNEL_VERSION_MAJOR, KERNEL_VERSION_MINOR, kernel_version >> 0x13,
+          (kernel_version >> 3) & 0xffff);
+      return -1;
+    } else {
+      printf("Got correct kernel version %d.%d\n", kernel_version >> 0x13,
+             (kernel_version >> 3) & 0xffff);
+    }
   }
 
   // setup deci2count for message counter.

game/kernel/kscheme.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file kscheme.h
  * Implementation of GOAL runtime.

game/kernel/ksocket.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file ksocket.h
  * GOAL Socket connection to listener using DECI2/DSNET

game/kernel/ksound.h

@@ -1,3 +1,5 @@
+#pragma once
+
 /*!
  * @file ksound.h
  * There's not much here. My guess is this was set up as framework to match the kmachine.cpp format,