From 57d8226e21828413ceedeb6e73fed0ddd5913d94 Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Sun, 24 Apr 2022 00:14:23 -0400 Subject: [PATCH] goalc: cleanup goalc's main method and add nrepl listener socket --- .vs/launch.vs.json | 2 +- CMakeLists.txt | 3 + Taskfile.yml | 2 +- decompiler/extractor/main.cpp | 1 - goalc/CMakeLists.txt | 1 + goalc/compiler/Compiler.cpp | 127 +++++++++++++---------- goalc/compiler/Compiler.h | 9 +- goalc/compiler/nrepl/ReplServer.cpp | 60 +++++++++++ goalc/compiler/nrepl/ReplServer.h | 38 +++++++ goalc/main.cpp | 96 +++++++++-------- scripts/batch/gc-dbg.bat | 2 +- scripts/batch/gc-no-lt.bat | 2 +- scripts/batch/gc.bat | 2 +- scripts/shell/offline_test_git_branch.sh | 2 +- 14 files changed, 241 insertions(+), 106 deletions(-) create mode 100644 goalc/compiler/nrepl/ReplServer.cpp create mode 100644 goalc/compiler/nrepl/ReplServer.h diff --git a/.vs/launch.vs.json b/.vs/launch.vs.json index 8c65df0c74..440f0071d0 100644 --- a/.vs/launch.vs.json +++ b/.vs/launch.vs.json @@ -75,7 +75,7 @@ "project" : "CMakeLists.txt", "projectTarget" : "goalc.exe (bin\\goalc.exe)", "name" : "Run - REPL - Auto Listen", - "args" : [ "-auto-lt" ] + "args" : [ "--auto-lt" ] }, { "type" : "default", diff --git a/CMakeLists.txt b/CMakeLists.txt index 310a515116..2454875280 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,6 +139,9 @@ add_subdirectory(third-party/imgui) # build the game code in C++ add_subdirectory(game) +# build asio library +include_directories(third-party/asio/asio/include) + # build the compiler add_subdirectory(goalc) diff --git a/Taskfile.yml b/Taskfile.yml index 0a9139f7ca..0762f0ff72 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -56,7 +56,7 @@ tasks: env: OPENGOAL_DECOMP_DIR: "jak1/" cmds: - - "{{.GOALC_BIN_RELEASE_DIR}}/goalc -auto-lt" + - "{{.GOALC_BIN_RELEASE_DIR}}/goalc --auto-lt" # DECOMPILING decomp: cmds: diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index 943352becc..275c144e23 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -335,7 +335,6 @@ int main(int argc, char** argv) { app.add_flag("-c,--compile", flag_compile, "Compile the game"); app.add_flag("-p,--play", flag_play, "Play the game"); app.validate_positionals(); - CLI11_PARSE(app, argc, argv); fmt::print("Working Directory - {}\n", std::filesystem::current_path().string()); diff --git a/goalc/CMakeLists.txt b/goalc/CMakeLists.txt index 72dfd374f4..6a7a10c314 100644 --- a/goalc/CMakeLists.txt +++ b/goalc/CMakeLists.txt @@ -26,6 +26,7 @@ add_library(compiler compiler/compilation/Type.cpp compiler/compilation/State.cpp compiler/compilation/Static.cpp + compiler/nrepl/ReplServer.cpp compiler/Util.cpp data_compiler/game_text.cpp data_compiler/dir_tpages.cpp diff --git a/goalc/compiler/Compiler.cpp b/goalc/compiler/Compiler.cpp index 374155ce80..30f61f5c96 100644 --- a/goalc/compiler/Compiler.cpp +++ b/goalc/compiler/Compiler.cpp @@ -1,3 +1,5 @@ +#include "nrepl/ReplServer.h" // this import has to come first because WinSock sucks + #include "Compiler.h" #include #include @@ -12,7 +14,9 @@ using namespace goos; -Compiler::Compiler(const std::string& user_profile, std::unique_ptr repl) +Compiler::Compiler(const int nrepl_port, + const std::string& user_profile, + std::unique_ptr repl) : m_goos(user_profile), m_debugger(&m_listener, &m_goos.reader), m_repl(std::move(repl)) { m_listener.add_debugger(&m_debugger); m_ts.add_builtin_types(); @@ -48,6 +52,18 @@ Compiler::Compiler(const std::string& user_profile, std::unique_ptr // add GOOS forms that get info from the compiler setup_goos_forms(); + + m_nrepl_port = nrepl_port; + fmt::print("[nREPL]: Server Will Listen for a Connection on Port {}!\n\r", m_nrepl_port); + nrepl_thread = std::thread([&]() { + asio::io_context io_context; + ReplServer s(io_context, this); + try { + io_context.run(); + } catch (std::exception& e) { + print_compiler_warning("Could not setup nREPL {}\n", e.what()); + } + }); } ReplStatus Compiler::execute_repl(bool auto_listen, bool auto_debug) { @@ -64,65 +80,15 @@ ReplStatus Compiler::execute_repl(bool auto_listen, bool auto_debug) { m_repl->get_repl().set_highlighter_callback( std::bind(&Compiler::repl_coloring, this, _1, _2, std::cref(regex_colors))); - std::string auto_input; if (auto_debug || auto_listen) { - auto_input.append("(lt)"); + read_eval_print("(lt)"); } if (auto_debug) { - auto_input.append("(dbg) (:cont)"); + read_eval_print("(dbg) (:cont)"); } while (!m_want_exit && !m_want_reload) { - try { - std::optional code; - - if (auto_input.empty()) { - // 1). get a line from the user (READ) - std::string prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::cyan), "g > "); - if (m_listener.is_connected()) { - prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::lime_green), "gc> "); - } - if (m_debugger.is_halted()) { - prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::magenta), "gs> "); - } else if (m_debugger.is_attached()) { - prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::red), "gr> "); - } - - code = m_goos.reader.read_from_stdin(prompt, *m_repl); - } else { - code = m_goos.reader.read_from_string(auto_input); - auto_input.clear(); - } - - if (!code) { - continue; - } - - // 2). compile - auto obj_file = compile_object_file("repl", *code, m_listener.is_connected()); - if (m_settings.debug_print_ir) { - obj_file->debug_print_tl(); - } - - if (!obj_file->is_empty()) { - // 3). color - color_object_file(obj_file); - - // 4). codegen - auto data = codegen_object_file(obj_file); - - // 4). send! - if (m_listener.is_connected()) { - m_listener.send_code(data); - if (!m_listener.most_recent_send_was_acked()) { - print_compiler_warning("Runtime is not responding. Did it crash?\n"); - } - } - } - - } catch (std::exception& e) { - print_compiler_warning("REPL Error: {}\n", e.what()); - } + read_eval_print(); } if (m_listener.is_connected()) { @@ -141,6 +107,59 @@ ReplStatus Compiler::execute_repl(bool auto_listen, bool auto_debug) { return ReplStatus::OK; } +void Compiler::read_eval_print(std::string input) { + try { + std::optional code; + + // Explicitly specified input + if (!input.empty()) { + code = m_goos.reader.read_from_string(input); + } else { + // if this is pulled out into a function....illegal instruction on checking the debugger? strange + std::string prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::cyan), "g > "); + if (m_listener.is_connected()) { + prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::lime_green), "gc> "); + } + if (m_debugger.is_halted()) { + prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::magenta), "gs> "); + } else if (m_debugger.is_attached()) { + prompt = fmt::format(fmt::emphasis::bold | fg(fmt::color::red), "gr> "); + } + // 1). get a line from the user (READ) + code = m_goos.reader.read_from_stdin(prompt, *m_repl); + } + + if (!code) { + return; + } + + // 2). compile + auto obj_file = compile_object_file("repl", *code, m_listener.is_connected()); + if (m_settings.debug_print_ir) { + obj_file->debug_print_tl(); + } + + if (!obj_file->is_empty()) { + // 3). color + color_object_file(obj_file); + + // 4). codegen + auto data = codegen_object_file(obj_file); + + // 4). send! + if (m_listener.is_connected()) { + m_listener.send_code(data); + if (!m_listener.most_recent_send_was_acked()) { + print_compiler_warning("Runtime is not responding. Did it crash?\n"); + } + } + } + + } catch (std::exception& e) { + print_compiler_warning("REPL Error: {}\n", e.what()); + } +} + FileEnv* Compiler::compile_object_file(const std::string& name, goos::Object code, bool allow_emit) { diff --git a/goalc/compiler/Compiler.h b/goalc/compiler/Compiler.h index 081ab1d585..b74a207ba2 100644 --- a/goalc/compiler/Compiler.h +++ b/goalc/compiler/Compiler.h @@ -27,7 +27,10 @@ enum class ReplStatus { OK, WANT_EXIT, WANT_RELOAD }; class Compiler { public: - Compiler(const std::string& user_profile = "#f", std::unique_ptr repl = nullptr); + Compiler(const int nrepl_port = 8181, + const std::string& user_profile = "#f", + std::unique_ptr repl = nullptr); + void read_eval_print(std::string input = ""); ReplStatus execute_repl(bool auto_listen = false, bool auto_debug = false); goos::Interpreter& get_goos() { return m_goos; } FileEnv* compile_object_file(const std::string& name, goos::Object code, bool allow_emit); @@ -70,6 +73,8 @@ class Compiler { bool knows_object_file(const std::string& name); MakeSystem& make_system() { return m_make; } + int m_nrepl_port; + private: TypeSystem m_ts; std::unique_ptr m_global_env = nullptr; @@ -89,6 +94,8 @@ class Compiler { std::unique_ptr m_repl; MakeSystem m_make; + std::thread nrepl_thread; + struct DebugStats { int num_spills = 0; int num_spills_v1 = 0; diff --git a/goalc/compiler/nrepl/ReplServer.cpp b/goalc/compiler/nrepl/ReplServer.cpp new file mode 100644 index 0000000000..338b906fb5 --- /dev/null +++ b/goalc/compiler/nrepl/ReplServer.cpp @@ -0,0 +1,60 @@ +#include "ReplServer.h" + +#include "third-party/fmt/core.h" + +// TODO - basically REPL to listen and inject commands into a running REPL +// - we will need a C++ side client as well which will let us communicate with the repl via for +// example, ImgUI +// +// TODO - The server also needs to eventually return the result of the evaluation + +ReplSession::ReplSession(tcp::socket socket, Compiler* repl) : socket_(std::move(socket)) { + m_repl = repl; +} + +void ReplSession::start() { + fmt::print("[nREPL]: Client Connected!\n\r"); + do_read(); +} + +void ReplSession::do_read() { + auto self(shared_from_this()); + socket_.async_read_some(asio::buffer(data_, max_length), + [this, self](std::error_code ec, std::size_t length) { + if (!ec) { + auto input = std::string(data_, length); + if (!input.empty()) { + m_repl->read_eval_print(input); + } + // TODO - i think this is kinda a hack, but its to keep the server + // cycling + do_write(0); + } + }); +} + +void ReplSession::do_write(std::size_t length) { + auto self(shared_from_this()); + asio::async_write(socket_, asio::buffer(data_, length), + [this, self](std::error_code ec, std::size_t /*length*/) { + if (!ec) { + do_read(); + } + }); +} + +ReplServer::ReplServer(asio::io_context& io_context, Compiler* repl) + : acceptor_(io_context, tcp::endpoint(tcp::v4(), repl->m_nrepl_port)), socket_(io_context) { + m_repl = repl; + m_port = repl->m_nrepl_port; + do_accept(); +} + +void ReplServer::do_accept() { + acceptor_.async_accept(socket_, [this](std::error_code ec) { + if (!ec) { + std::make_shared(std::move(socket_), this->m_repl)->start(); + } + do_accept(); + }); +} diff --git a/goalc/compiler/nrepl/ReplServer.h b/goalc/compiler/nrepl/ReplServer.h new file mode 100644 index 0000000000..5190552fd5 --- /dev/null +++ b/goalc/compiler/nrepl/ReplServer.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +#include "goalc/compiler/Compiler.h" + +using asio::ip::tcp; + +class ReplSession : public std::enable_shared_from_this { + public: + ReplSession(tcp::socket socket, Compiler* repl); + + void start(); + + private: + Compiler* m_repl; + tcp::socket socket_; + enum { max_length = 1024 * 20 }; + char data_[max_length]; + + void do_read(); + void do_write(std::size_t length); +}; + +class ReplServer { + public: + ReplServer(asio::io_context& io_context, Compiler* repl); + + int m_port; + + private: + Compiler* m_repl; + tcp::acceptor acceptor_; + tcp::socket socket_; + + void do_accept(); +}; diff --git a/goalc/main.cpp b/goalc/main.cpp index ff395afb24..f37eaf984f 100644 --- a/goalc/main.cpp +++ b/goalc/main.cpp @@ -4,10 +4,12 @@ #include "common/util/FileUtil.h" #include "common/log/log.h" +#include "third-party/CLI11.hpp" #include "third-party/fmt/core.h" #include "third-party/fmt/color.h" #include "common/goos/ReplUtils.h" +#include void setup_logging(bool verbose) { lg::set_file(file_util::get_file_path({"log/compiler.txt"})); @@ -24,52 +26,58 @@ void setup_logging(bool verbose) { } int main(int argc, char** argv) { - (void)argc; - (void)argv; - if (!file_util::setup_project_path(std::nullopt)) { - return 1; - } - std::string argument; - std::string username = "#f"; bool verbose = false; bool auto_listen = false; bool auto_debug = false; - for (int i = 1; i < argc; i++) { - if (std::string("-v") == argv[i]) { - verbose = true; - } else if (std::string("-cmd") == argv[i] && i + 1 < argc) { - argument = argv[++i]; - } else if (std::string("-auto-lt") == argv[i]) { - auto_listen = true; - } else if (std::string("-auto-dbg") == argv[i]) { - auto_debug = true; - } else if (std::string("-user") == argv[i] && i + 1 < argc) { - username = argv[++i]; - } else if (std::string("-user-auto") == argv[i]) { - try { - auto text = std::make_shared( - file_util::get_file_path({"goal_src", "user", "user.txt"}), "goal_src/user/user.txt"); - goos::TextStream ts(text); - ts.seek_past_whitespace_and_comments(); - username.clear(); - while (ts.text_remains()) { - char c = ts.read(); - if ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || - c == '-' || c == '.' || c == '!' || c == '?' || c == '<' || c == '>') { - username.push_back(c); - } else { - break; - } + bool auto_find_user = false; + std::string cmd = ""; + std::string username = "#f"; + int nrepl_port = 8181; + + CLI::App app{"OpenGOAL Compiler / REPL"}; + app.add_option("-c,--cmd", cmd, "Specify a command to run"); + app.add_option("-u,--user", username, + "Specify the username to use for your user profile in 'goal_src/user/'"); + app.add_option("-p,--port", nrepl_port, "Specify the nREPL port. Defaults to 8181"); + app.add_flag("-v,--verbose", verbose, "Enable verbose output"); + app.add_flag("--auto-lt", auto_listen, + "Attempt to automatically connect to the listener on startup"); + app.add_flag("--auto-dbg", auto_debug, + "Attempt to automatically connect to the debugger on startup"); + app.add_flag("--user-auto", auto_find_user, + "Attempt to automatically deduce the user, overrides '-user'"); + app.validate_positionals(); + CLI11_PARSE(app, argc, argv); + + if (!file_util::setup_project_path(std::nullopt)) { + return 1; + } + + if (auto_find_user) { + username = "#f"; + std::regex allowed_chars("[0-9a-zA-Z\\-\\.\\!\\?<>]"); + try { + auto text = std::make_shared( + file_util::get_file_path({"goal_src", "user", "user.txt"}), "goal_src/user/user.txt"); + goos::TextStream ts(text); + ts.seek_past_whitespace_and_comments(); + std::string found_username; + while (ts.text_remains()) { + auto character = std::string(1, ts.read()); + if (std::regex_match(character, allowed_chars)) { + found_username.push_back(ts.read()); + } else { + break; } - if (username.empty()) { - username = "#f"; - } - } catch (std::exception& e) { - printf("error opening user desc file: %s\n", e.what()); - username = "#f"; } + if (!found_username.empty()) { + username = found_username; + } + } catch (std::exception& e) { + printf("error opening user desc file: %s\n", e.what()); } } + setup_logging(verbose); lg::info("OpenGOAL Compiler {}.{}", versions::GOAL_VERSION_MAJOR, versions::GOAL_VERSION_MINOR); @@ -78,18 +86,18 @@ int main(int argc, char** argv) { // the compiler may throw an exception if it fails to load its standard library. try { std::unique_ptr compiler; - if (argument.empty()) { + if (!cmd.empty()) { + compiler = std::make_unique(); + compiler->run_front_end_on_string(cmd); + } else { ReplStatus status = ReplStatus::WANT_RELOAD; while (status == ReplStatus::WANT_RELOAD) { - compiler = std::make_unique(username, std::make_unique()); + compiler = std::make_unique(nrepl_port, username, std::make_unique()); status = compiler->execute_repl(auto_listen, auto_debug); if (status == ReplStatus::WANT_RELOAD) { fmt::print("Reloading compiler...\n"); } } - } else { - compiler = std::make_unique(); - compiler->run_front_end_on_string(argument); } } catch (std::exception& e) { fmt::print("Compiler Fatal Error: {}\n", e.what()); diff --git a/scripts/batch/gc-dbg.bat b/scripts/batch/gc-dbg.bat index 5c69b5e26b..8754dc8d99 100644 --- a/scripts/batch/gc-dbg.bat +++ b/scripts/batch/gc-dbg.bat @@ -1,4 +1,4 @@ @echo off cd ..\.. -out\build\Release\bin\goalc -v -auto-dbg -user-auto +out\build\Release\bin\goalc -v --auto-dbg --user-auto pause diff --git a/scripts/batch/gc-no-lt.bat b/scripts/batch/gc-no-lt.bat index 82b4e4d087..f094f42f04 100644 --- a/scripts/batch/gc-no-lt.bat +++ b/scripts/batch/gc-no-lt.bat @@ -1,4 +1,4 @@ @echo off cd ..\.. -out\build\Release\bin\goalc -v -user-auto +out\build\Release\bin\goalc -v --user-auto pause diff --git a/scripts/batch/gc.bat b/scripts/batch/gc.bat index 2e72bdb9b7..f0007965e7 100644 --- a/scripts/batch/gc.bat +++ b/scripts/batch/gc.bat @@ -1,4 +1,4 @@ @echo off cd ..\.. -out\build\Release\bin\goalc -v -auto-lt -user-auto +out\build\Release\bin\goalc -v --auto-lt --user-auto pause diff --git a/scripts/shell/offline_test_git_branch.sh b/scripts/shell/offline_test_git_branch.sh index 325f2160bb..0d07d11c9e 100755 --- a/scripts/shell/offline_test_git_branch.sh +++ b/scripts/shell/offline_test_git_branch.sh @@ -48,7 +48,7 @@ echo " ================ Decompiling..." ../scripts/shell/decomp.sh echo " ================ Building project..." -../scripts/shell/gc.sh -cmd \(make-group\ \"iso\"\) +../scripts/shell/gc.sh --cmd \(make-group\ \"iso\"\) echo " ================ Checking assets..." ../scripts/shell/check.sh