goalc: Add TCP server socket in REPL process (#1335)

* goalc: cleanup goalc's main method and add nrepl listener socket

* deps: add standalone ASIO for sockets

* lint: formatting

* common: make a common interface for creating a server socket

* goalc: setup new repl server

* deps: remove asio

* goalc: debug issues, nrepl is working again

* git: rename files

* attempt to fix linux function call

* test

* scripts: make the error message even more obvious....

* goalc: make suggested changes, still can't reconnect properly

* game: pull out single-client logic from XSocketServer

* nrepl: supports multiple clients and disconnection/reconnects

* goalc: some minor fixes for tests

* goalc: save repl history when the compiler reloads

* common: add include for linux networking

* a few small changes to fix tests

* is it the assert?

* change thread start order and add a print to an assert

Co-authored-by: water <awaterford111445@gmail.com>

.vs/launch.vs.json

@@ -75,7 +75,7 @@
       "project" : "CMakeLists.txt",
       "projectTarget" : "goalc.exe (bin\\goalc.exe)",
       "name" : "Run - REPL - Auto Listen",
-      "args" : [ "-auto-lt" ]
+      "args" : [ "--auto-lt" ]
     },
     {
       "type" : "default",

Taskfile.yml

@@ -9,27 +9,27 @@ tasks:
     desc: "Extracts Jak 1 - NTSC - Black Label assets"
     preconditions:
       - sh: test -f {{.DECOMP_BIN_RELEASE_DIR}}/decompiler{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate decompiler executable -- Have you compiled in release mode?"
+        msg: "Couldn't locate decompiler executable in '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler'"
     cmds:
       - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/jak1_ntsc_black_label.jsonc" "./iso_data" "./decompiler_out" "decompile_code=false"'
   boot-game:
     desc: "Boots the game"
     preconditions:
       - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate runtime executable -- Have you compiled in release mode?"
+        msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
     cmds:
       - "{{.GK_BIN_RELEASE_DIR}}/gk -boot -fakeiso -debug -v"
   run-game:
     desc: "Start the game's runtime"
     preconditions:
       - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate runtime executable -- Have you compiled in release mode?"
+        msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
     cmds:
       - "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso -debug -v"
   run-game-quiet:
     preconditions:
       - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate runtime executable -- Have you compiled in release mode?"
+        msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'"
     cmds:
       - "{{.GK_BIN_RELEASE_DIR}}/gk -fakeiso"
   repl:
@@ -38,7 +38,7 @@ tasks:
       OPENGOAL_DECOMP_DIR: "jak1/"
     preconditions:
       - sh: test -f {{.GOALC_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}}
-        msg: "Couldn't locate compiler executable -- Have you compiled in release mode?"
+        msg: "Couldn't locate compiler executable in '{{.GOALC_BIN_RELEASE_DIR}}/goalc'"
     cmds:
       - "{{.GOALC_BIN_RELEASE_DIR}}/goalc"
   # DEVELOPMENT
@@ -56,7 +56,7 @@ tasks:
     env:
       OPENGOAL_DECOMP_DIR: "jak1/"
     cmds:
-      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc -auto-lt"
+      - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt"
   # DECOMPILING
   decomp:
     cmds:
@@ -130,7 +130,3 @@ tasks:
   cast-repl:
     cmds:
       - cmd: python ./scripts/cast-repl.py
-  # Doesn't Currently Work
-  # clean-all-types:
-  #   cmds:
-  #     - python ./scripts/cleanup-all-types.py

common/CMakeLists.txt

@@ -1,7 +1,8 @@
 add_library(common
         audio/audio_formats.cpp
         cross_os_debug/xdbg.cpp
-        cross_sockets/xsocket.cpp
+        cross_sockets/XSocket.cpp
+        cross_sockets/XSocketServer.cpp
         custom_data/TFrag3Data.cpp
         dma/dma.cpp
         dma/dma_copy.cpp
@@ -44,8 +45,7 @@ add_library(common
         util/FrameLimiter.cpp
         util/image_loading.cpp
         goos/Printer.cpp
-        goos/PrettyPrinter2.cpp
-        )
+        goos/PrettyPrinter2.cpp)
 
 target_link_libraries(common fmt lzokay replxx libzstd_static)
 

common/cross_os_debug/xdbg.cpp

@@ -22,6 +22,7 @@
 #include <fcntl.h>
 #elif _WIN32
 #define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #include <mutex>
 #include <condition_variable>

common/cross_os_debug/xdbg.h

@@ -14,6 +14,7 @@
 #include <sys/types.h>
 #elif _WIN32
 #define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #endif
 

common/cross_sockets/XSocket.cpp

@@ -16,6 +16,8 @@
 #include <stdio.h>
 #include <string.h>
 
+#include "third-party/fmt/core.h"
+
 int open_socket(int af, int type, int protocol) {
 #ifdef __linux
   return socket(af, type, protocol);
@@ -32,6 +34,26 @@ int open_socket(int af, int type, int protocol) {
 #endif
 }
 
+#ifdef __linux
+int accept_socket(int socket, sockaddr* addr, socklen_t* addrLen) {
+  return accept(socket, addr, addrLen);
+}
+#endif
+
+#ifdef _WIN32
+int accept_socket(int socket, sockaddr* addr, int* addrLen) {
+  WSADATA wsaData = {0};
+  int iResult = 0;
+  // Initialize Winsock
+  iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
+  if (iResult != 0) {
+    printf("WSAStartup failed: %d\n", iResult);
+    return 1;
+  }
+  return accept(socket, addr, addrLen);
+}
+#endif
+
 void close_socket(int sock) {
   if (sock < 0) {
     return;
@@ -77,11 +99,16 @@ int set_socket_timeout(int socket, long microSeconds) {
 }
 
 int write_to_socket(int socket, const char* buf, int len) {
+  int bytes_wrote = 0;
 #ifdef __linux
-  return write(socket, buf, len);
+  bytes_wrote = write(socket, buf, len);
 #elif _WIN32
-  return send(socket, buf, len, 0);
+  bytes_wrote = send(socket, buf, len, 0);
 #endif
+  if (bytes_wrote < 0) {
+    fmt::print(stderr, "[XSocket:{}] Error writing to socket\n", socket);
+  }
+  return bytes_wrote;
 }
 
 int read_from_socket(int socket, char* buf, int len) {
@@ -99,4 +126,4 @@ bool socket_timed_out() {
   auto err = WSAGetLastError();
   return err == WSAETIMEDOUT;
 #endif
-}
+}

common/cross_sockets/XSocket.h

@@ -1,7 +1,7 @@
 #pragma once
 
 /*!
- * @file xsocket.h
+ * @file XSocket.h
  * Cross platform socket library used for the listener.
  */
 
@@ -9,7 +9,12 @@
 #include <sys/socket.h>
 #include <netinet/tcp.h>
 #include <unistd.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
 #elif _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
 #include <WinSock2.h>
 #endif
 
@@ -20,9 +25,14 @@ const int TCP_SOCKET_LEVEL = IPPROTO_TCP;
 #endif
 
 int open_socket(int af, int type, int protocol);
+#ifdef __linux
+int accept_socket(int socket, sockaddr* addr, socklen_t* addrLen);
+#elif _WIN32
+int accept_socket(int socket, sockaddr* addr, int* addrLen);
+#endif
 void close_socket(int sock);
 int set_socket_option(int socket, int level, int optname, const void* optval, int optlen);
 int set_socket_timeout(int socket, long microSeconds);
 int write_to_socket(int socket, const char* buf, int len);
 int read_from_socket(int socket, char* buf, int len);
-bool socket_timed_out();
+bool socket_timed_out();

common/cross_sockets/XSocketServer.cpp

@@ -0,0 +1,88 @@
+#include "XSocketServer.h"
+
+#include "third-party/fmt/core.h"
+
+#include "common/cross_sockets/XSocket.h"
+
+#ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#include <WinSock2.h>
+#include <WS2tcpip.h>
+#endif
+
+XSocketServer::XSocketServer(std::function<bool()> shutdown_callback,
+                             int _tcp_port,
+                             int _buffer_size)
+    : want_exit_callback(std::move(shutdown_callback)) {
+  tcp_port = _tcp_port;
+  buffer.resize(_buffer_size);
+}
+
+XSocketServer::~XSocketServer() {
+  if (listening_socket >= 0) {
+    close_server_socket();
+  }
+}
+
+void XSocketServer::shutdown_server() {
+  // Close the listening and accepted socket socket
+  close_server_socket();
+}
+
+bool XSocketServer::init_server() {
+  listening_socket = open_socket(AF_INET, SOCK_STREAM, 0);
+  if (listening_socket < 0) {
+    listening_socket = -1;
+    return false;
+  }
+
+#ifdef __linux
+  int server_socket_opt = SO_REUSEADDR | SO_REUSEPORT;
+#elif _WIN32
+  int server_socket_opt = SO_EXCLUSIVEADDRUSE;
+#endif
+
+  int opt = 1;
+  if (set_socket_option(listening_socket, SOL_SOCKET, server_socket_opt, &opt, sizeof(opt)) < 0) {
+    close_server_socket();
+    return false;
+  };
+
+  if (set_socket_option(listening_socket, TCP_SOCKET_LEVEL, TCP_NODELAY, &opt, sizeof(opt)) < 0) {
+    close_server_socket();
+    return false;
+  }
+
+  if (set_socket_timeout(listening_socket, 100000) < 0) {
+    close_server_socket();
+    return false;
+  }
+
+  addr.sin_family = AF_INET;
+  addr.sin_addr.s_addr = INADDR_ANY;
+  addr.sin_port = htons(tcp_port);
+
+  if (bind(listening_socket, (sockaddr*)&addr, sizeof(addr)) < 0) {
+    fmt::print("[XSocketServer:{}] failed to bind\n", tcp_port);
+    close_server_socket();
+    return false;
+  }
+
+  if (listen(listening_socket, 0) < 0) {
+    fmt::print("[XSocketServer:{}] failed to listen\n", tcp_port);
+    close_server_socket();
+    return false;
+  }
+
+  server_initialized = true;
+  fmt::print("[XSocketServer:{}] initialized\n", tcp_port);
+  post_init();
+  return true;
+}
+
+void XSocketServer::close_server_socket() {
+  close_socket(listening_socket);
+  listening_socket = -1;
+}

common/cross_sockets/XSocketServer.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include "common/cross_sockets/XSocket.h"
+
+#include <thread>
+#include "common/common_types.h"
+#include <functional>
+#include <mutex>
+
+/// @brief A cross platform generic socket server implementation
+class XSocketServer {
+ public:
+  static constexpr int DEF_BUFFER_SIZE = 32 * 1024 * 1024;
+  XSocketServer(std::function<bool()> shutdown_callback,
+                int _tcp_port,
+                int _buffer_size = DEF_BUFFER_SIZE);
+  virtual ~XSocketServer();
+
+  XSocketServer(const XSocketServer&) = delete;
+  XSocketServer& operator=(const XSocketServer&) = delete;
+
+  bool init_server();
+  void shutdown_server();
+  void close_server_socket();
+
+  // Abstract methods -- use-case dependent
+  virtual void post_init() = 0;
+
+ protected:
+  int tcp_port;
+  struct sockaddr_in addr = {};
+  int listening_socket = -1;
+  std::vector<char> buffer;
+
+  bool server_initialized = false;
+
+  std::function<bool()> want_exit_callback;
+};

common/global_profiler/GlobalProfiler.cpp

@@ -14,6 +14,7 @@ u32 get_current_tid() {
 }
 #else
 #define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <windows.h>
 #include "Processthreadsapi.h"
 u32 get_current_tid() {

common/goos/ReplUtils.cpp

@@ -16,6 +16,7 @@ void ReplWrapper::clear_screen() {
 }
 
 void ReplWrapper::print_welcome_message() {
+  // TODO - dont print on std-out
   // Welcome message / brief intro for documentation
   std::string ascii;
   ascii += " _____             _____ _____ _____ __    \n";
@@ -35,6 +36,10 @@ void ReplWrapper::print_welcome_message() {
   fmt::print(" to connect to the local target.\n\n");
 }
 
+void ReplWrapper::print_to_repl(const std::string_view& str) {
+  repl.print(str.data());
+}
+
 void ReplWrapper::set_history_max_size(size_t len) {
   repl.set_max_history_size(len);
 }

common/goos/ReplUtils.h

@@ -18,6 +18,7 @@ class ReplWrapper {
 
   // Functionality / Commands
   void clear_screen();
+  void print_to_repl(const std::string_view& str);
   void print_welcome_message();
   void set_history_max_size(size_t len);
   const char* readline(const std::string& prompt);

common/log/log.cpp

@@ -4,6 +4,8 @@
 #include "third-party/fmt/color.h"
 #include "log.h"
 #ifdef _WIN32  // see lg::initialize
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #endif
 #include "common/util/Assert.h"

common/util/FileUtil.cpp

@@ -24,6 +24,8 @@
 #include "third-party/lzokay/lzokay.hpp"
 
 #ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #else
 #include <unistd.h>

common/util/FrameLimiter.cpp

@@ -41,6 +41,7 @@ void FrameLimiter::run(double target_fps,
 
 #else
 
+#define NOMINMAX
 #include <Windows.h>
 
 FrameLimiter::FrameLimiter() {
@@ -74,4 +75,4 @@ void FrameLimiter::run(double target_fps,
   m_timer.start();
 }
 
-#endif
+#endif

common/util/Timer.cpp

@@ -1,6 +1,8 @@
 #include "Timer.h"
 
 #ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #define MS_PER_SEC 1000ULL  // MS = milliseconds
 #define US_PER_MS 1000ULL   // US = microseconds

decompiler/extractor/main.cpp

@@ -337,7 +337,6 @@ int main(int argc, char** argv) {
   app.add_flag("-p,--play", flag_play, "Play the game");
   app.add_flag("-f,--folder", flag_folder, "Extract from folder");
   app.validate_positionals();
-
   CLI11_PARSE(app, argc, argv);
 
   fmt::print("Working Directory - {}\n", std::filesystem::current_path().string());

game/kernel/kboot.cpp

@@ -22,6 +22,8 @@
 #include "kprint.h"
 
 #ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include "Windows.h"
 #include <io.h>
 #elif __linux__

game/runtime.cpp

@@ -9,6 +9,8 @@
 #elif _WIN32
 #include <io.h>
 #include "third-party/mman/mman.h"
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #endif
 
@@ -74,7 +76,7 @@ void deci2_runner(SystemThreadInterface& iface) {
   std::function<bool()> shutdown_callback = [&]() { return iface.get_want_exit(); };
 
   // create and register server
-  Deci2Server server(shutdown_callback);
+  Deci2Server server(shutdown_callback, DECI2_PORT);
   ee::LIBRARY_sceDeci2_register(&server);
 
   // now its ok to continue with initialization
@@ -84,20 +86,20 @@ void deci2_runner(SystemThreadInterface& iface) {
   lg::debug("[DECI2] Waiting for EE to register protos");
   server.wait_for_protos_ready();
   // then allow the server to accept connections
-  if (!server.init()) {
-    ASSERT(false);
+  if (!server.init_server()) {
+    ASSERT_MSG(false, "[DECI2] Server not initialized even if protocols are ready, aborting");
   }
 
   lg::debug("[DECI2] Waiting for listener...");
   bool saw_listener = false;
   while (!iface.get_want_exit()) {
-    if (server.check_for_listener()) {
+    if (server.is_client_connected()) {
       if (!saw_listener) {
         lg::debug("[DECI2] Connected!");
       }
       saw_listener = true;
       // we have a listener, run!
-      server.run();
+      server.read_data();
     } else {
       // no connection yet.  Do a sleep so we don't spam checking the listener.
       std::this_thread::sleep_for(std::chrono::microseconds(50000));
@@ -314,10 +316,8 @@ RuntimeExitStatus exec_runtime(int argc, char** argv) {
 
   // step 3: start the EE!
   iop_thread.start(iop_runner);
-  ee_thread.start(ee_runner);
-
   deci_thread.start(deci2_runner);
-
+  ee_thread.start(ee_runner);
   if (VM::use) {
     vm_dmac_thread.start(dmac_runner);
   }

game/system/Deci2Server.cpp

@@ -4,154 +4,58 @@
  * Works with deci2.cpp (sceDeci2) to implement the networking on target
  */
 
-#include <cstdio>
-#include <utility>
+#include "Deci2Server.h"
 
-// TODO - i think im not including the dependency right..?
-#include "common/cross_sockets/xsocket.h"
+#include "common/cross_sockets/XSocket.h"
 
-#ifdef __linux
-#include <sys/socket.h>
-#include <netinet/tcp.h>
-#include <unistd.h>
-#elif _WIN32
+#include "common/versions.h"
+#include <common/listener_common.h>
+#include <common/util/Assert.h>
+
+#include "third-party/fmt/core.h"
+
+#ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
 #include <Windows.h>
 #include <WinSock2.h>
 #include <WS2tcpip.h>
 #endif
 
-#include "common/listener_common.h"
-#include "common/versions.h"
-#include "Deci2Server.h"
-#include "common/util/Assert.h"
-
-Deci2Server::Deci2Server(std::function<bool()> shutdown_callback)
-    : want_exit(std::move(shutdown_callback)) {
-  buffer = new char[BUFFER_SIZE];
-}
-
 Deci2Server::~Deci2Server() {
-  close_server_socket();
-  close_socket(new_sock);
-
-  // if accept thread is running, kill it
+  // Cleanup the accept thread
   if (accept_thread_running) {
     kill_accept_thread = true;
     accept_thread.join();
     accept_thread_running = false;
   }
 
-  delete[] buffer;
+  close_socket(accepted_socket);
 }
 
-/*!
- * Start waiting for the Listener to connect
- */
-bool Deci2Server::init() {
-  server_socket = open_socket(AF_INET, SOCK_STREAM, 0);
-  if (server_socket < 0) {
-    server_socket = -1;
-    return false;
-  }
-
-#ifdef __linux
-  int server_socket_opt = SO_REUSEADDR | SO_REUSEPORT;
-#elif _WIN32
-  int server_socket_opt = SO_EXCLUSIVEADDRUSE;
-#endif
-
-  int opt = 1;
-  if (set_socket_option(server_socket, SOL_SOCKET, server_socket_opt, &opt, sizeof(opt)) < 0) {
-    close_server_socket();
-    return false;
-  };
-
-  if (set_socket_option(server_socket, TCP_SOCKET_LEVEL, TCP_NODELAY, &opt, sizeof(opt)) < 0) {
-    close_server_socket();
-    return false;
-  }
-
-  if (set_socket_timeout(server_socket, 100000) < 0) {
-    close_server_socket();
-    return false;
-  }
-
-  addr.sin_family = AF_INET;
-  addr.sin_addr.s_addr = INADDR_ANY;
-  addr.sin_port = htons(DECI2_PORT);
-
-  if (bind(server_socket, (sockaddr*)&addr, sizeof(addr)) < 0) {
-    printf("[Deci2Server] Failed to bind\n");
-    close_server_socket();
-    return false;
-  }
-
-  if (listen(server_socket, 0) < 0) {
-    printf("[Deci2Server] Failed to listen\n");
-    close_server_socket();
-    return false;
-  }
-
-  server_initialized = true;
+void Deci2Server::post_init() {
+  fmt::print("[Deci2Server:{}] awaiting connections\n", tcp_port);
   accept_thread_running = true;
   kill_accept_thread = false;
   accept_thread = std::thread(&Deci2Server::accept_thread_func, this);
-  return true;
 }
 
-void Deci2Server::close_server_socket() {
-  close_socket(server_socket);
-  server_socket = -1;
-}
-
-/*!
- * Return true if the listener is connected.
- */
-bool Deci2Server::check_for_listener() {
-  if (server_connected) {
-    if (accept_thread_running) {
-      accept_thread.join();
-      accept_thread_running = false;
-    }
-    return true;
-  } else {
-    return false;
-  }
-}
-
-/*!
- * Send data from buffer. User must provide appropriate headers.
- */
-void Deci2Server::send_data(void* buf, u16 len) {
-  lock();
-  if (!server_connected) {
-    printf("[DECI2] send while not connected, not sending!\n");
-  } else {
-    uint16_t prog = 0;
-    while (prog < len) {
-      int wrote = write_to_socket(new_sock, (char*)(buf) + prog, len - prog);
-      prog += wrote;
-      if (!server_connected || want_exit()) {
-        unlock();
-        return;
-      }
+void Deci2Server::accept_thread_func() {
+  socklen_t addr_len = sizeof(addr);
+  while (!kill_accept_thread) {
+    accepted_socket = accept_socket(listening_socket, (sockaddr*)&addr, &addr_len);
+    if (accepted_socket >= 0) {
+      set_socket_timeout(accepted_socket, 100000);
+      u32 versions[2] = {versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR};
+      write_to_socket(accepted_socket, (char*)&versions, 8);  // todo, check result?
+      client_connected = true;
+      return;
     }
   }
-  unlock();
 }
 
-/*!
- * Lock the DECI mutex. Should be done before modifying protocols.
- */
-void Deci2Server::lock() {
-  deci_mutex.lock();
-}
-
-/*!
- * Unlock the DECI mutex. Should be done after modifying protocols.
- */
-void Deci2Server::unlock() {
-  deci_mutex.unlock();
+bool Deci2Server::is_client_connected() {
+  return client_connected;
 }
 
 /*!
@@ -161,7 +65,7 @@ void Deci2Server::unlock() {
 void Deci2Server::wait_for_protos_ready() {
   if (protocols_ready)
     return;
-  std::unique_lock<std::mutex> lk(deci_mutex);
+  std::unique_lock<std::mutex> lk(server_mutex);
   cv.wait(lk, [&] { return protocols_ready; });
 }
 
@@ -180,20 +84,24 @@ void Deci2Server::send_proto_ready(Deci2Driver* drivers, int* driver_count) {
   cv.notify_all();
 }
 
-void Deci2Server::run() {
+void Deci2Server::read_data() {
+  if (!is_client_connected()) {
+    return;
+  }
+
   int desired_size = (int)sizeof(Deci2Header);
   int got = 0;
 
   while (got < desired_size) {
-    ASSERT(got + desired_size < BUFFER_SIZE);
-    auto x = read_from_socket(new_sock, buffer + got, desired_size - got);
-    if (want_exit()) {
+    ASSERT(got + desired_size < buffer.size());
+    auto x = read_from_socket(accepted_socket, buffer.data() + got, desired_size - got);
+    if (want_exit_callback()) {
       return;
     }
     got += x > 0 ? x : 0;
   }
 
-  auto* hdr = (Deci2Header*)(buffer);
+  auto* hdr = (Deci2Header*)(buffer.data());
   fprintf(stderr, "[DECI2] Got message: %d %d 0x%x %c -> %c\n", hdr->len, hdr->rsvd, hdr->proto,
           hdr->src, hdr->dst);
 
@@ -222,12 +130,12 @@ void Deci2Server::run() {
   auto& driver = d2_drivers[handler];
 
   u32 sent_to_program = 0;
-  while (!want_exit() && (hdr->rsvd < hdr->len || sent_to_program < hdr->rsvd)) {
+  while (!want_exit_callback() && (hdr->rsvd < hdr->len || sent_to_program < hdr->rsvd)) {
     // send what we have to the program
     if (sent_to_program < hdr->rsvd) {
       //      driver.next_recv_size = 0;
       //      driver.next_recv = nullptr;
-      driver.recv_buffer = buffer + sent_to_program;
+      driver.recv_buffer = buffer.data() + sent_to_program;
       driver.available_to_receive = hdr->rsvd - sent_to_program;
       (driver.handler)(DECI2_READ, driver.available_to_receive, driver.opt);
       //      memcpy(driver.next_recv, buffer + sent_to_program, driver.next_recv_size);
@@ -236,8 +144,8 @@ void Deci2Server::run() {
 
     // receive from network
     if (hdr->rsvd < hdr->len) {
-      auto x = read_from_socket(new_sock, buffer + hdr->rsvd, hdr->len - hdr->rsvd);
-      if (want_exit()) {
+      auto x = read_from_socket(accepted_socket, buffer.data() + hdr->rsvd, hdr->len - hdr->rsvd);
+      if (want_exit_callback()) {
         return;
       }
       got += x > 0 ? x : 0;
@@ -249,21 +157,28 @@ void Deci2Server::run() {
   unlock();
 }
 
-/*!
- * Background thread for waiting for the listener.
- */
-void Deci2Server::accept_thread_func() {
-  socklen_t l = sizeof(addr);
-  while (!kill_accept_thread) {
-    // TODO - might want to do a WSAStartUp call here as well, else it won't be balanced on the
-    // close
-    new_sock = accept(server_socket, (sockaddr*)&addr, &l);
-    if (new_sock >= 0) {
-      set_socket_timeout(new_sock, 100000);
-      u32 versions[2] = {versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR};
-      write_to_socket(new_sock, (char*)&versions, 8);  // todo, check result?
-      server_connected = true;
-      return;
+void Deci2Server::send_data(void* buf, u16 len) {
+  lock();
+  if (!client_connected) {
+    printf("[DECI2] send while not connected, not sending!\n");
+  } else {
+    uint16_t prog = 0;
+    while (prog < len) {
+      int wrote = write_to_socket(accepted_socket, (char*)(buf) + prog, len - prog);
+      prog += wrote;
+      if (!client_connected || want_exit_callback()) {
+        unlock();
+        return;
+      }
     }
   }
+  unlock();
+}
+
+void Deci2Server::lock() {
+  server_mutex.lock();
+}
+
+void Deci2Server::unlock() {
+  server_mutex.unlock();
 }

game/system/Deci2Server.h

@@ -1,55 +1,44 @@
 #pragma once
 
-/*!
- * @file Deci2Server.h
- * Basic implementation of a DECI2 server.
- * Works with deci2.cpp (sceDeci2) to implement the networking on target
- */
+#include "common/cross_sockets/XSocketServer.h"
 
-#ifdef __linux
-#include <netinet/in.h>
-#elif _WIN32
-#include <Windows.h>
-#endif
-#include <thread>
-#include <mutex>
+#include "deci_common.h"
 #include <condition_variable>
-#include <functional>
-#include "game/system/deci_common.h"
 
-class Deci2Server {
+/// @brief Basic implementation of a DECI2 server.
+/// Works with deci2.cpp(sceDeci2) to implement the networking on target
+class Deci2Server : public XSocketServer {
  public:
-  static constexpr int BUFFER_SIZE = 32 * 1024 * 1024;
-  Deci2Server(std::function<bool()> shutdown_callback);
-  ~Deci2Server();
-  bool init();
-  bool check_for_listener();
+  using XSocketServer::XSocketServer;
+  virtual ~Deci2Server();
+
+  void post_init() override;
+
+  void read_data();
   void send_data(void* buf, u16 len);
 
-  void lock();
-  void unlock();
+  bool is_client_connected();
   void wait_for_protos_ready();
   void send_proto_ready(Deci2Driver* drivers, int* driver_count);
 
-  void run();
+  void lock();
+  void unlock();
+
+ protected:
+  void accept_thread_func();
 
  private:
-  void close_server_socket();
-  void accept_thread_func();
-  bool kill_accept_thread = false;
-  char* buffer = nullptr;
-  int server_socket = -1;
-  struct sockaddr_in addr = {};
-  int new_sock = -1;
-  bool server_initialized = false;
-  bool accept_thread_running = false;
-  bool server_connected = false;
-  std::function<bool()> want_exit;
-  std::thread accept_thread;
-
-  std::condition_variable cv;
   bool protocols_ready = false;
-  std::mutex deci_mutex;
+  std::condition_variable cv;
   Deci2Driver* d2_drivers = nullptr;
   int* d2_driver_count = nullptr;
+
+  int accepted_socket = -1;
+  bool kill_accept_thread = false;
+  bool accept_thread_running = false;
+
+  std::thread accept_thread;
+  std::mutex server_mutex;
+
+  bool client_connected = false;
 };

goalc/CMakeLists.txt

@@ -26,6 +26,7 @@ add_library(compiler
         compiler/compilation/Type.cpp
         compiler/compilation/State.cpp
         compiler/compilation/Static.cpp
+        compiler/nrepl/ReplServer.cpp
         compiler/Util.cpp
         data_compiler/game_text_common.cpp
         data_compiler/dir_tpages.cpp

goalc/compiler/Compiler.cpp

@@ -1,3 +1,5 @@
+#include "nrepl/ReplServer.h"  // this import has to come first because WinSock sucks
+
 #include "Compiler.h"
 #include <chrono>
 #include <thread>
@@ -44,91 +46,94 @@ Compiler::Compiler(const std::string& user_profile, std::unique_ptr<ReplWrapper>
   // load auto-complete history, only if we are running in the interactive mode.
   if (m_repl) {
     m_repl->load_history();
+    // init repl
+    m_repl->print_welcome_message();
+    auto examples = m_repl->examples;
+    auto regex_colors = m_repl->regex_colors;
+    m_repl->init_default_settings();
+    using namespace std::placeholders;
+    m_repl->get_repl().set_completion_callback(
+        std::bind(&Compiler::find_symbols_by_prefix, this, _1, _2, std::cref(examples)));
+    m_repl->get_repl().set_hint_callback(
+        std::bind(&Compiler::find_hints_by_prefix, this, _1, _2, _3, std::cref(examples)));
+    m_repl->get_repl().set_highlighter_callback(
+        std::bind(&Compiler::repl_coloring, this, _1, _2, std::cref(regex_colors)));
   }
 
   // add GOOS forms that get info from the compiler
   setup_goos_forms();
 }
 
-ReplStatus Compiler::execute_repl(bool auto_listen, bool auto_debug) {
-  // init repl
-  m_repl->print_welcome_message();
-  auto examples = m_repl->examples;
-  auto regex_colors = m_repl->regex_colors;
-  m_repl->init_default_settings();
-  using namespace std::placeholders;
-  m_repl->get_repl().set_completion_callback(
-      std::bind(&Compiler::find_symbols_by_prefix, this, _1, _2, std::cref(examples)));
-  m_repl->get_repl().set_hint_callback(
-      std::bind(&Compiler::find_hints_by_prefix, this, _1, _2, _3, std::cref(examples)));
-  m_repl->get_repl().set_highlighter_callback(
-      std::bind(&Compiler::repl_coloring, this, _1, _2, std::cref(regex_colors)));
-
-  std::string auto_input;
-  if (auto_debug || auto_listen) {
-    auto_input.append("(lt)");
-  }
-  if (auto_debug) {
-    auto_input.append("(dbg) (:cont)");
-  }
-
-  while (!m_want_exit && !m_want_reload) {
-    try {
-      std::optional<goos::Object> code;
-
-      if (auto_input.empty()) {
-        // 1). get a line from the user (READ)
-        std::string prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::cyan), "g > ");
-        if (m_listener.is_connected()) {
-          prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::lime_green), "gc> ");
-        }
-        if (m_debugger.is_halted()) {
-          prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::magenta), "gs> ");
-        } else if (m_debugger.is_attached()) {
-          prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::red), "gr> ");
-        }
-
-        code = m_goos.reader.read_from_stdin(prompt, *m_repl);
-      } else {
-        code = m_goos.reader.read_from_string(auto_input);
-        auto_input.clear();
-      }
-
-      if (!code) {
-        continue;
-      }
-
-      // 2). compile
-      auto obj_file = compile_object_file("repl", *code, m_listener.is_connected());
-      if (m_settings.debug_print_ir) {
-        obj_file->debug_print_tl();
-      }
-
-      if (!obj_file->is_empty()) {
-        // 3). color
-        color_object_file(obj_file);
-
-        // 4). codegen
-        auto data = codegen_object_file(obj_file);
-
-        // 4). send!
-        if (m_listener.is_connected()) {
-          m_listener.send_code(data);
-          if (!m_listener.most_recent_send_was_acked()) {
-            print_compiler_warning("Runtime is not responding. Did it crash?\n");
-          }
-        }
-      }
-
-    } catch (std::exception& e) {
-      print_compiler_warning("REPL Error: {}\n", e.what());
-    }
-  }
-
+Compiler::~Compiler() {
   if (m_listener.is_connected()) {
     m_listener.send_reset(false);  // reset the target
     m_listener.disconnect();
   }
+}
+
+void Compiler::save_repl_history() {
+  m_repl->save_history();
+}
+
+void Compiler::print_to_repl(const std::string_view& str) {
+  m_repl->print_to_repl(str);
+}
+
+std::string Compiler::get_prompt() {
+  std::string prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::cyan), "g > ");
+  if (m_listener.is_connected()) {
+    prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::lime_green), "gc> ");
+  }
+  if (m_debugger.is_halted()) {
+    prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::magenta), "gs> ");
+  } else if (m_debugger.is_attached()) {
+    prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::red), "gr> ");
+  }
+  return "\033[0m" + prompt;
+}
+
+std::string Compiler::get_repl_input() {
+  auto str = m_repl->readline(get_prompt());
+  if (str) {
+    m_repl->add_to_history(str);
+    return str;
+  } else {
+    return "";
+  }
+}
+
+ReplStatus Compiler::handle_repl_string(const std::string& input) {
+  if (input.empty()) {
+    return ReplStatus::OK;
+  }
+
+  try {
+    // 1). read
+    goos::Object code = m_goos.reader.read_from_string(input, true);
+    // 2). compile
+    auto obj_file = compile_object_file("repl", code, m_listener.is_connected());
+    if (m_settings.debug_print_ir) {
+      obj_file->debug_print_tl();
+    }
+
+    if (!obj_file->is_empty()) {
+      // 3). color
+      color_object_file(obj_file);
+
+      // 4). codegen
+      auto data = codegen_object_file(obj_file);
+
+      // 4). send!
+      if (m_listener.is_connected()) {
+        m_listener.send_code(data);
+        if (!m_listener.most_recent_send_was_acked()) {
+          print_compiler_warning("Runtime is not responding. Did it crash?\n");
+        }
+      }
+    }
+  } catch (std::exception& e) {
+    print_compiler_warning("REPL Error: {}\n", e.what());
+  }
 
   if (m_want_exit) {
     return ReplStatus::WANT_EXIT;

goalc/compiler/Compiler.h

@@ -20,6 +20,8 @@
 #include "goalc/make/MakeSystem.h"
 #include "goalc/data_compiler/game_text_common.h"
 
+#include <mutex>
+
 enum MathMode { MATH_INT, MATH_BINT, MATH_FLOAT, MATH_INVALID };
 
 enum class ReplStatus { OK, WANT_EXIT, WANT_RELOAD };
@@ -27,7 +29,12 @@ enum class ReplStatus { OK, WANT_EXIT, WANT_RELOAD };
 class Compiler {
  public:
   Compiler(const std::string& user_profile = "#f", std::unique_ptr<ReplWrapper> repl = nullptr);
-  ReplStatus execute_repl(bool auto_listen = false, bool auto_debug = false);
+  ~Compiler();
+  void save_repl_history();
+  void print_to_repl(const std::string_view& str);
+  std::string get_prompt();
+  std::string get_repl_input();
+  ReplStatus handle_repl_string(const std::string& input);
   goos::Interpreter& get_goos() { return m_goos; }
   FileEnv* compile_object_file(const std::string& name, goos::Object code, bool allow_emit);
   std::unique_ptr<FunctionEnv> compile_top_level_function(const std::string& name,

goalc/compiler/nrepl/ReplServer.cpp

@@ -0,0 +1,143 @@
+#include "ReplServer.h"
+
+#include "common/cross_sockets/XSocket.h"
+
+#include "third-party/fmt/core.h"
+#include <common/versions.h>
+
+#ifdef _WIN32
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#include <WinSock2.h>
+#include <WS2tcpip.h>
+#endif
+
+// TODO - basically REPL to listen and inject commands into a running REPL
+// - we will need a C++ side client as well which will let us communicate with the repl via for
+// example, ImgUI
+//
+// TODO - The server also needs to eventually return the result of the evaluation
+
+ReplServer::~ReplServer() {
+  // Close all our client sockets!
+  for (const int& sock : client_sockets) {
+    close_socket(sock);
+  }
+}
+
+void ReplServer::post_init() {
+  // Add the listening socket to our set of sockets
+  fmt::print("[nREPL:{}:{}] awaiting connections\n", tcp_port, listening_socket);
+}
+
+void ReplServer::ping_response(int socket) {
+  std::string ping = fmt::format("Connected to OpenGOAL v{}.{} nREPL!",
+                                 versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
+  write_to_socket(socket, ping.c_str(), ping.size());
+}
+
+std::optional<std::string> ReplServer::get_msg() {
+  // Clear the sockets we are listening on
+  FD_ZERO(&read_sockets);
+
+  // Add the server's main listening socket (where we accept clients from)
+  FD_SET(listening_socket, &read_sockets);
+
+  int max_sd = listening_socket;
+  for (const int& sock : client_sockets) {
+    if (sock > max_sd) {
+      max_sd = sock;
+    }
+    if (sock > 0) {
+      FD_SET(sock, &read_sockets);
+    }
+  }
+
+  // Wait for activity on _something_, with a timeout so we don't get stuck here on exit.
+  struct timeval timeout;
+  timeout.tv_sec = 0;
+  timeout.tv_usec = 100000;
+  auto activity = select(max_sd + 1, &read_sockets, NULL, NULL, &timeout);
+
+  if (activity < 0) {  // TODO - || error!
+    return std::nullopt;
+  }
+
+  // If something happened on the master socket - it's a new connection
+  if (FD_ISSET(listening_socket, &read_sockets)) {
+    socklen_t addr_len = sizeof(addr);
+    auto new_socket = accept_socket(listening_socket, (sockaddr*)&addr, &addr_len);
+    if (new_socket < 0) {
+      // TODO - handle error
+    } else {
+      fmt::print("[nREPL:{}]: New socket connection: {}:{}:{}\n", tcp_port,
+                 inet_ntoa(addr.sin_addr), ntohs(addr.sin_port), new_socket);
+
+      // Say hello
+      ping_response(new_socket);
+      // Track the new socket
+      if (client_sockets.size() < max_clients) {
+        client_sockets.insert(new_socket);
+      } else {
+        // TODO - Respond with NO
+      }
+    }
+  }
+
+  // otherwise (and no matter what) check all the clients to see if they have sent us anything
+  // else its some IO operation on some other socket
+  //
+  // RACE - the first client wins
+
+  // TODO - there are ways to do this with iterators but, couldn't figure it out!
+  std::vector<int> sockets_to_scan(client_sockets.begin(), client_sockets.end());
+  for (const int& sock : sockets_to_scan) {
+    if (FD_ISSET(sock, &read_sockets)) {
+      // Attempt to read a header
+      // TODO - should this be in a loop?
+      auto req_bytes = read_from_socket(sock, header_buffer.data(), header_buffer.size());
+      if (req_bytes == 0) {
+        // Socket disconnected
+        // TODO - add a queue of messages in the ReplWrapper so we can print _BEFORE_ the prompt is
+        // output
+        fmt::print("[nREPL:{}] Client Disconnected: {}\n", tcp_port, inet_ntoa(addr.sin_addr),
+                   ntohs(addr.sin_port), sock);
+
+        // Cleanup the socket and remove it from our set
+        close_socket(sock);
+        client_sockets.erase(sock);
+      } else {
+        // Otherwise, process the message
+        auto* header = (ReplServerHeader*)(header_buffer.data());
+        // get the body of the message
+        int expected_size = header->length;
+        int got = 0;
+        while (got < expected_size) {
+          if (got + expected_size > buffer.size()) {
+            fmt::print(stderr,
+                       "[nREPL:{}]: Bad message, aborting the read.  Got :{}, Expected: {}, Buffer "
+                       "Size: {}",
+                       tcp_port, got, expected_size, buffer.size());
+            return std::nullopt;
+          }
+          auto x = read_from_socket(sock, buffer.data() + got, expected_size - got);
+          if (want_exit_callback()) {
+            return std::nullopt;
+          }
+          got += x > 0 ? x : 0;
+        }
+
+        switch (header->type) {
+          case ReplServerMessageType::PING:
+            ping_response(sock);
+            return std::nullopt;
+          case ReplServerMessageType::EVAL:
+            std::string msg(buffer.data(), header->length);
+            return std::make_optional(msg);
+        }
+      }
+    }
+  }
+  return std::nullopt;
+}

goalc/compiler/nrepl/ReplServer.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include "common/cross_sockets/XSocketServer.h"
+
+#include "goalc/compiler/Compiler.h"
+
+enum ReplServerMessageType { PING = 0, EVAL = 10, SHUTDOWN = 20 };
+
+struct ReplServerHeader {
+  u32 length;
+  u32 type;
+};
+
+class ReplServer : public XSocketServer {
+ public:
+  using XSocketServer::XSocketServer;
+  virtual ~ReplServer();
+
+  void post_init() override;
+
+  std::optional<std::string> get_msg();
+
+ private:
+  int max_clients = 50;
+  std::vector<char> header_buffer = std::vector<char>((int)sizeof(ReplServerHeader));
+  fd_set read_sockets;
+  std::set<int> client_sockets = {};
+
+  void ping_response(int socket);
+};

goalc/listener/Listener.cpp

@@ -18,7 +18,7 @@
 #undef max
 #endif
 
-#include "common/cross_sockets/xsocket.h"
+#include "common/cross_sockets/XSocket.h"
 
 #include <stdexcept>
 #include <cstring>

goalc/main.cpp

@@ -4,10 +4,13 @@
 #include "common/util/FileUtil.h"
 #include "common/log/log.h"
 
+#include "third-party/CLI11.hpp"
 #include "third-party/fmt/core.h"
 #include "third-party/fmt/color.h"
 
 #include "common/goos/ReplUtils.h"
+#include <regex>
+#include <goalc/compiler/nrepl/ReplServer.h>
 
 void setup_logging(bool verbose) {
   lg::set_file(file_util::get_file_path({"log/compiler.txt"}));
@@ -24,76 +27,127 @@ void setup_logging(bool verbose) {
 }
 
 int main(int argc, char** argv) {
-  (void)argc;
-  (void)argv;
-  if (!file_util::setup_project_path(std::nullopt)) {
-    return 1;
-  }
-  std::string argument;
-  std::string username = "#f";
   bool verbose = false;
   bool auto_listen = false;
   bool auto_debug = false;
-  for (int i = 1; i < argc; i++) {
-    if (std::string("-v") == argv[i]) {
-      verbose = true;
-    } else if (std::string("-cmd") == argv[i] && i + 1 < argc) {
-      argument = argv[++i];
-    } else if (std::string("-auto-lt") == argv[i]) {
-      auto_listen = true;
-    } else if (std::string("-auto-dbg") == argv[i]) {
-      auto_debug = true;
-    } else if (std::string("-user") == argv[i] && i + 1 < argc) {
-      username = argv[++i];
-    } else if (std::string("-user-auto") == argv[i]) {
-      try {
-        auto text = std::make_shared<goos::FileText>(
-            file_util::get_file_path({"goal_src", "user", "user.txt"}), "goal_src/user/user.txt");
-        goos::TextStream ts(text);
-        ts.seek_past_whitespace_and_comments();
-        username.clear();
-        while (ts.text_remains()) {
-          char c = ts.read();
-          if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
-              c == '-' || c == '.' || c == '!' || c == '?' || c == '<' || c == '>') {
-            username.push_back(c);
-          } else {
-            break;
-          }
+  bool auto_find_user = false;
+  std::string cmd = "";
+  std::string username = "#f";
+  int nrepl_port = 8181;
+
+  CLI::App app{"OpenGOAL Compiler / REPL"};
+  app.add_option("-c,--cmd", cmd, "Specify a command to run");
+  app.add_option("-u,--user", username,
+                 "Specify the username to use for your user profile in 'goal_src/user/'");
+  app.add_option("-p,--port", nrepl_port, "Specify the nREPL port.  Defaults to 8181");
+  app.add_flag("-v,--verbose", verbose, "Enable verbose output");
+  app.add_flag("--auto-lt", auto_listen,
+               "Attempt to automatically connect to the listener on startup");
+  app.add_flag("--auto-dbg", auto_debug,
+               "Attempt to automatically connect to the debugger on startup");
+  app.add_flag("--user-auto", auto_find_user,
+               "Attempt to automatically deduce the user, overrides '-user'");
+  app.validate_positionals();
+  CLI11_PARSE(app, argc, argv);
+
+  if (!file_util::setup_project_path(std::nullopt)) {
+    return 1;
+  }
+
+  if (auto_find_user) {
+    username = "#f";
+    std::regex allowed_chars("[0-9a-zA-Z\\-\\.\\!\\?<>]");
+    try {
+      auto text = std::make_shared<goos::FileText>(
+          file_util::get_file_path({"goal_src", "user", "user.txt"}), "goal_src/user/user.txt");
+      goos::TextStream ts(text);
+      ts.seek_past_whitespace_and_comments();
+      std::string found_username;
+      while (ts.text_remains()) {
+        auto character = std::string(1, ts.read());
+        if (!std::regex_match(character, allowed_chars)) {
+          break;
         }
-        if (username.empty()) {
-          username = "#f";
-        }
-      } catch (std::exception& e) {
-        printf("error opening user desc file: %s\n", e.what());
-        username = "#f";
+        found_username.push_back(ts.read());
       }
+      if (!found_username.empty()) {
+        username = found_username;
+      }
+    } catch (std::exception& e) {
+      printf("error opening user desc file: %s\n", e.what());
     }
   }
+
   setup_logging(verbose);
 
   lg::info("OpenGOAL Compiler {}.{}", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR);
 
   // Init REPL
+  ReplStatus status = ReplStatus::WANT_RELOAD;
+  std::function<bool()> shutdown_callback = [&]() { return status == ReplStatus::WANT_EXIT; };
+  ReplServer repl_server(shutdown_callback, nrepl_port);
+  bool repl_server_ok = repl_server.init_server();
+  std::thread nrepl_thread;
   // the compiler may throw an exception if it fails to load its standard library.
   try {
     std::unique_ptr<Compiler> compiler;
-    if (argument.empty()) {
-      ReplStatus status = ReplStatus::WANT_RELOAD;
-      while (status == ReplStatus::WANT_RELOAD) {
-        compiler = std::make_unique<Compiler>(username, std::make_unique<ReplWrapper>());
-        status = compiler->execute_repl(auto_listen, auto_debug);
-        if (status == ReplStatus::WANT_RELOAD) {
-          fmt::print("Reloading compiler...\n");
-        }
-      }
-    } else {
+    std::mutex compiler_mutex;
+    // if a command is provided on the command line, no REPL just run the compiler on it
+    if (!cmd.empty()) {
       compiler = std::make_unique<Compiler>();
-      compiler->run_front_end_on_string(argument);
+      compiler->run_front_end_on_string(cmd);
+      return 0;
+    }
+    // Start nREPL Server
+    if (repl_server_ok) {
+      nrepl_thread = std::thread([&]() {
+        while (!shutdown_callback()) {
+          auto resp = repl_server.get_msg();
+          if (resp) {
+            std::lock_guard<std::mutex> lock(compiler_mutex);
+            status = compiler->handle_repl_string(resp.value());
+            // Print out the prompt, just for better UX
+            compiler->print_to_repl(compiler->get_prompt());
+          }
+          std::this_thread::sleep_for(std::chrono::microseconds(50000));
+        }
+      });
+    }
+    // Run automatic forms if applicable
+    if (auto_debug || auto_listen) {
+      std::lock_guard<std::mutex> lock(compiler_mutex);
+      status = compiler->handle_repl_string("(lt)");
+    }
+    if (auto_debug) {
+      std::lock_guard<std::mutex> lock(compiler_mutex);
+      status = compiler->handle_repl_string("(dbg) (:cont)");
+    }
+    // Poll Terminal
+    while (status != ReplStatus::WANT_EXIT) {
+      if (status == ReplStatus::WANT_RELOAD) {
+        fmt::print("Reloading compiler...\n");
+        std::lock_guard<std::mutex> lock(compiler_mutex);
+        if (compiler) {
+          compiler->save_repl_history();
+        }
+        compiler = std::make_unique<Compiler>(username, std::make_unique<ReplWrapper>());
+        status = ReplStatus::OK;
+      }
+      std::string input_from_stdin = compiler->get_repl_input();
+      if (!input_from_stdin.empty()) {
+        // lock, while we compile
+        std::lock_guard<std::mutex> lock(compiler_mutex);
+        status = compiler->handle_repl_string(input_from_stdin);
+      }
     }
   } catch (std::exception& e) {
-    fmt::print("Compiler Fatal Error: {}\n", e.what());
+    fmt::print(stderr, "Compiler Fatal Error: {}\n", e.what());
   }
 
+  // Cleanup
+  if (repl_server_ok) {
+    repl_server.shutdown_server();
+    nrepl_thread.join();
+  }
   return 0;
 }

scripts/batch/gc-dbg.bat

@@ -1,4 +1,4 @@
 @echo off
 cd ..\..
-out\build\Release\bin\goalc -v -auto-dbg -user-auto
+out\build\Release\bin\goalc -v --auto-dbg --user-auto
 pause

scripts/batch/gc-no-lt.bat

@@ -1,4 +1,4 @@
 @echo off
 cd ..\..
-out\build\Release\bin\goalc -v -user-auto
+out\build\Release\bin\goalc -v --user-auto
 pause

scripts/batch/gc.bat

@@ -1,4 +1,4 @@
 @echo off
 cd ..\..
-out\build\Release\bin\goalc -v -auto-lt -user-auto
+out\build\Release\bin\goalc -v --auto-lt --user-auto
 pause

scripts/nrepl-test.py

@@ -0,0 +1,15 @@
+import socket
+import struct
+clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM);
+clientSocket.connect(("127.0.0.1", 8181))
+print(clientSocket)
+data = clientSocket.recv(1024)
+print(data.decode())
+
+form = "(:status)"
+
+header = struct.pack('<II', len(form), 10)
+
+clientSocket.sendall(header + form.encode())
+
+

scripts/shell/offline_test_git_branch.sh

@@ -48,7 +48,7 @@ echo " ================ Decompiling..."
 ../scripts/shell/decomp.sh
 
 echo " ================ Building project..."
-../scripts/shell/gc.sh -cmd \(make-group\ \"iso\"\)
+../scripts/shell/gc.sh --cmd \(make-group\ \"iso\"\)
 
 echo " ================ Checking assets..."
 ../scripts/shell/check.sh

test/test_listener_deci2.cpp

@@ -15,12 +15,12 @@ TEST(Listener, ListenerCreation) {
 }
 
 TEST(Listener, DeciCreation) {
-  Deci2Server s(always_false);
+  Deci2Server s(always_false, DECI2_PORT);
 }
 
 TEST(Listener, DeciInit) {
-  Deci2Server s(always_false);
-  EXPECT_TRUE(s.init());
+  Deci2Server s(always_false, DECI2_PORT);
+  EXPECT_TRUE(s.init_server());
 }
 
 /*!
@@ -38,61 +38,61 @@ TEST(Listener, ListenToNothing) {
 }
 
 TEST(Listener, DeciCheckNoListener) {
-  Deci2Server s(always_false);
-  EXPECT_TRUE(s.init());
-  EXPECT_FALSE(s.check_for_listener());
-  EXPECT_FALSE(s.check_for_listener());
-  EXPECT_FALSE(s.check_for_listener());
+  Deci2Server s(always_false, DECI2_PORT);
+  EXPECT_TRUE(s.init_server());
+  EXPECT_FALSE(s.is_client_connected());
+  EXPECT_FALSE(s.is_client_connected());
+  EXPECT_FALSE(s.is_client_connected());
 }
 
 TEST(Listener, CheckConnectionStaysAlive) {
-  Deci2Server s(always_false);
-  EXPECT_TRUE(s.init());
-  EXPECT_FALSE(s.check_for_listener());
+  Deci2Server s(always_false, DECI2_PORT);
+  EXPECT_TRUE(s.init_server());
+  EXPECT_FALSE(s.is_client_connected());
   Listener l;
-  EXPECT_FALSE(s.check_for_listener());
+  EXPECT_FALSE(s.is_client_connected());
   bool connected = l.connect_to_target();
   EXPECT_TRUE(connected);
   // TODO - some sort of backoff and retry would be better
-  while (connected && !s.check_for_listener()) {
+  while (connected && !s.is_client_connected()) {
   }
 
-  EXPECT_TRUE(s.check_for_listener());
+  EXPECT_TRUE(s.is_client_connected());
   std::this_thread::sleep_for(std::chrono::milliseconds(500));
-  EXPECT_TRUE(s.check_for_listener());
+  EXPECT_TRUE(s.is_client_connected());
   EXPECT_TRUE(l.is_connected());
 }
 
 TEST(Listener, DeciThenListener) {
   for (int i = 0; i < 3; i++) {
-    Deci2Server s(always_false);
-    EXPECT_TRUE(s.init());
-    EXPECT_FALSE(s.check_for_listener());
-    EXPECT_FALSE(s.check_for_listener());
-    EXPECT_FALSE(s.check_for_listener());
+    Deci2Server s(always_false, DECI2_PORT);
+    EXPECT_TRUE(s.init_server());
+    EXPECT_FALSE(s.is_client_connected());
+    EXPECT_FALSE(s.is_client_connected());
+    EXPECT_FALSE(s.is_client_connected());
     Listener l;
-    EXPECT_FALSE(s.check_for_listener());
-    EXPECT_FALSE(s.check_for_listener());
+    EXPECT_FALSE(s.is_client_connected());
+    EXPECT_FALSE(s.is_client_connected());
     bool connected = l.connect_to_target();
     EXPECT_TRUE(connected);
     // TODO - some sort of backoff and retry would be better
-    while (connected && !s.check_for_listener()) {
+    while (connected && !s.is_client_connected()) {
     }
 
-    EXPECT_TRUE(s.check_for_listener());
+    EXPECT_TRUE(s.is_client_connected());
   }
 }
 
 TEST(Listener, DeciThenListener2) {
   for (int i = 0; i < 3; i++) {
-    Deci2Server s(always_false);
-    EXPECT_TRUE(s.init());
-    EXPECT_FALSE(s.check_for_listener());
-    EXPECT_FALSE(s.check_for_listener());
-    EXPECT_FALSE(s.check_for_listener());
+    Deci2Server s(always_false, DECI2_PORT);
+    EXPECT_TRUE(s.init_server());
+    EXPECT_FALSE(s.is_client_connected());
+    EXPECT_FALSE(s.is_client_connected());
+    EXPECT_FALSE(s.is_client_connected());
     Listener l;
-    EXPECT_FALSE(s.check_for_listener());
-    EXPECT_FALSE(s.check_for_listener());
+    EXPECT_FALSE(s.is_client_connected());
+    EXPECT_FALSE(s.is_client_connected());
     EXPECT_TRUE(l.connect_to_target());
   }
 }
@@ -101,13 +101,13 @@ TEST(Listener, ListenerThenDeci) {
   for (int i = 0; i < 3; i++) {
     Listener l;
     EXPECT_FALSE(l.connect_to_target());
-    Deci2Server s(always_false);
-    EXPECT_TRUE(s.init());
-    EXPECT_FALSE(s.check_for_listener());
+    Deci2Server s(always_false, DECI2_PORT);
+    EXPECT_TRUE(s.init_server());
+    EXPECT_FALSE(s.is_client_connected());
     bool connected = l.connect_to_target();
     EXPECT_TRUE(connected);
     // TODO - some sort of backoff and retry would be better
-    while (connected && !s.check_for_listener()) {
+    while (connected && !s.is_client_connected()) {
     }
   }
 }