formatter: add tree-sitter dependency and commit early draft work on a proper code formatter (#2536)

common/CMakeLists.txt

@@ -22,32 +22,34 @@ endfunction()
 write_revision_h()
 
 add_library(common
-        versions/versions.cpp
         audio/audio_formats.cpp
         cross_os_debug/xdbg.cpp
         cross_sockets/XSocket.cpp
-        cross_sockets/XSocketServer.cpp
         cross_sockets/XSocketClient.cpp
+        cross_sockets/XSocketServer.cpp
         custom_data/pack_helpers.cpp
         custom_data/TFrag3Data.cpp
-        dma/dma.cpp
         dma/dma_copy.cpp
+        dma/dma.cpp
         dma/gs.cpp
+        formatter/formatter.cpp
         global_profiler/GlobalProfiler.cpp
         goos/Interpreter.cpp
         goos/Object.cpp
         goos/ParseHelpers.cpp
-        goos/Printer.cpp
         goos/PrettyPrinter.cpp
         goos/PrettyPrinter2.cpp
+        goos/Printer.cpp
         goos/Reader.cpp
         goos/TextDB.cpp
-        repl/config.cpp
-        repl/util.cpp
         log/log.cpp
         math/geometry.cpp
+        repl/config.cpp
         repl/nrepl/ReplClient.cpp
         repl/nrepl/ReplServer.cpp
+        repl/util.cpp
+        serialization/subtitles/subtitles_deser.cpp
+        serialization/subtitles/subtitles_ser.cpp
         type_system/defenum.cpp
         type_system/deftype.cpp
         type_system/state.cpp
@@ -55,8 +57,6 @@ add_library(common
         type_system/TypeFieldLookup.cpp
         type_system/TypeSpec.cpp
         type_system/TypeSystem.cpp
-        serialization/subtitles/subtitles_ser.cpp
-        serialization/subtitles/subtitles_deser.cpp
         util/Assert.cpp
         util/BitUtils.cpp
         util/compress.cpp
@@ -67,18 +67,20 @@ add_library(common
         util/diff.cpp
         util/FileUtil.cpp
         util/FontUtils.cpp
+        util/FrameLimiter.cpp
         util/json_util.cpp
+        util/os.cpp
+        util/print_float.cpp
         util/read_iso_file.cpp
         util/SimpleThreadGroup.cpp
         util/string_util.cpp
+        util/term_util.cpp
         util/Timer.cpp
-        util/os.cpp
-        util/print_float.cpp
-        util/FrameLimiter.cpp
         util/unicode_util.cpp
-        util/term_util.cpp  )
+        versions/versions.cpp
+        )
 
-target_link_libraries(common fmt lzokay replxx libzstd_static)
+target_link_libraries(common fmt lzokay replxx libzstd_static tree-sitter)
 
 if(WIN32)
     target_link_libraries(common wsock32 ws2_32 windowsapp)

common/formatter/formatter.cpp

@@ -0,0 +1,161 @@
+#include "formatter.h"
+
+#include "common/util/FileUtil.h"
+#include "common/util/string_util.h"
+
+#include "tree_sitter/api.h"
+
+#include "third-party/fmt/core.h"
+
+// Declare the `tree_sitter_opengoal` function, which is
+// implemented by the `tree-sitter-opengoal` library.
+extern "C" {
+extern const TSLanguage* tree_sitter_opengoal();
+}
+
+void walk_tree(TSTreeCursor* cursor, std::string& output, const std::string& source_code) {
+  // an imperative breadth-first-search
+  while (true) {
+    // Process the node
+    const auto curr_node = ts_tree_cursor_current_node(cursor);
+    const std::string curr_node_type = ts_node_type(curr_node);
+    std::string curr_node_field_name;
+    if (ts_tree_cursor_current_field_name(cursor)) {
+      curr_node_field_name = ts_tree_cursor_current_field_name(cursor);
+    }
+    if (curr_node_field_name == "open") {
+      output += "(";
+    } else if (curr_node_field_name == "close") {
+      output.pop_back();
+      output += ") ";
+    }
+    if (curr_node_type == "sym_name" || curr_node_type == "num_lit" ||
+        curr_node_type == "str_lit") {
+      uint32_t start = ts_node_start_byte(curr_node);
+      uint32_t end = ts_node_end_byte(curr_node);
+      const char* type = ts_node_type(curr_node);
+      // TODO - if it's a string literal, take out any newlines and reflow the string to the
+      // line-length
+      const auto contents = source_code.substr(start, end - start);
+      output += contents + " ";
+    }
+
+    if (ts_tree_cursor_goto_first_child(cursor)) {
+      continue;
+    }
+
+    if (ts_tree_cursor_goto_next_sibling(cursor)) {
+      continue;
+    }
+
+    while (true) {
+      if (!ts_tree_cursor_goto_parent(cursor)) {
+        if (output.at(output.length() - 1) == ' ') {
+          output.pop_back();
+        }
+        return;
+      }
+      if (ts_tree_cursor_goto_next_sibling(cursor)) {
+        break;
+      }
+    }
+  }
+}
+
+// TODO - move this to str_util
+std::string repeat(size_t n, const std::string& str) {
+  if (n == 0 || str.empty())
+    return {};
+  if (n == 1)
+    return str;
+  const auto period = str.size();
+  if (period == 1)
+    return std::string(n, str.front());
+
+  std::string ret(str);
+  ret.reserve(period * n);
+  std::size_t m{2};
+  for (; m < n; m *= 2)
+    ret += ret;
+  ret.append(ret.c_str(), (n - (m / 2)) * period);
+  return ret;
+}
+
+// It's possible to walk a tree-sitter tree imperatively with a cursor
+// but the code for that is more verbose and less intuitive and I'm not sure how much
+// of a benefit I'd get out of it since for formatting i basically have to convert every
+// cursor to it's fat node
+//
+// But in any case, do it the easy way first and refactor later
+void format_code(const std::string& source,
+                 TSNode curr_node,
+                 std::string& output,
+                 std::string curr_form_head = "",
+                 int indent = 0) {
+  if (ts_node_child_count(curr_node) == 0) {
+    uint32_t start = ts_node_start_byte(curr_node);
+    uint32_t end = ts_node_end_byte(curr_node);
+    // TODO - if it's a string literal, take out any newlines and reflow the string to the
+    // line-length
+    const auto contents = source.substr(start, end - start);
+    if (contents == ")") {
+      output.pop_back();
+      output += ") ";
+    } else if (contents == "(") {
+      output += "(";
+    } else {
+      output += contents + " ";
+    }
+    return;
+  }
+  const std::string curr_node_type = ts_node_type(curr_node);
+  for (int i = 0; i < ts_node_child_count(curr_node); i++) {
+    auto child_node = ts_node_child(curr_node, i);
+    // If we are opening a list, peek at the first element in the list
+    // this is so we can properly handle indentation based on different forms
+    if (curr_node_type == "list_lit" && i == 1) {
+      uint32_t start = ts_node_start_byte(child_node);
+      uint32_t end = ts_node_end_byte(child_node);
+      // TODO - if it's a string literal, take out any newlines and reflow the string to the
+      // line-length
+      curr_form_head = source.substr(start, end - start);
+    }
+    std::string curr_node_field_name;
+    auto curr_field_name_raw = ts_node_field_name_for_child(
+        curr_node, i);  // TODO - why is this always returning `close` for the opening paren..
+    if (curr_field_name_raw) {
+      curr_node_field_name = curr_field_name_raw;
+    }
+    if (curr_form_head == "defun" && i == 4) {
+      indent += 2;
+      output += "\n" + repeat(indent, " ");
+    } else if (curr_form_head == "defun" && i == 5) {
+      output += "\n" + repeat(indent, " ");
+    }
+    format_code(source, child_node, output, curr_form_head, indent);
+    if (curr_node_type == "source") {
+      output += "\n\n";
+    }
+  }
+}
+
+std::string formatter::format_code(const std::string& source) {
+  // Create a parser.
+  std::shared_ptr<TSParser> parser(ts_parser_new(), TreeSitterParserDeleter());
+
+  // Set the parser's language (JSON in this case).
+  ts_parser_set_language(parser.get(), tree_sitter_opengoal());
+
+  // Build a syntax tree based on source code stored in a string.
+  std::shared_ptr<TSTree> tree(
+      ts_parser_parse_string(parser.get(), NULL, source.c_str(), source.length()),
+      TreeSitterTreeDeleter());
+
+  // Get the root node of the syntax tree.
+  TSNode root_node = ts_tree_root_node(tree.get());
+
+  std::string output = "";
+  format_code(source, root_node, output, "", 0);
+
+  return str_util::trim(output);
+}

common/formatter/formatter.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include <string>
+
+#include "tree_sitter/api.h"
+
+namespace formatter {
+struct TreeSitterParserDeleter {
+  void operator()(TSParser* ptr) const { ts_parser_delete(ptr); }
+};
+
+struct TreeSitterTreeDeleter {
+  void operator()(TSTree* ptr) const { ts_tree_delete(ptr); }
+};
+
+std::string format_code(const std::string& source);
+}  // namespace formatter

common/util/string_util.cpp

@@ -93,4 +93,12 @@ std::vector<std::string> regex_get_capture_groups(const std::string& str,
   }
   return groups;
 }
+
+bool replace(std::string& str, const std::string& from, const std::string& to) {
+  size_t start_pos = str.find(from);
+  if (start_pos == std::string::npos)
+    return false;
+  str.replace(start_pos, from.length(), to);
+  return true;
+}
 }  // namespace str_util

common/util/string_util.h

@@ -20,4 +20,5 @@ std::string diff(const std::string& lhs, const std::string& rhs);
 std::vector<std::string> split(const ::std::string& str, char delimiter = '\n');
 std::string join(const std::vector<std::string>& strs, const std::string& join_with);
 std::vector<std::string> regex_get_capture_groups(const std::string& str, const std::string& regex);
+bool replace(std::string& str, const std::string& from, const std::string& to);
 }  // namespace str_util

common/versions/versions.cpp

@@ -47,11 +47,11 @@ std::vector<std::string> valid_game_version_names() {
 }
 
 std::string build_revision() {
-  if (BUILT_TAG != "") {
-    return BUILT_TAG;
+  if (std::string(BUILT_TAG) != "") {
+    return std::string(BUILT_TAG);
   }
-  if (BUILT_SHA != "") {
-    return BUILT_SHA;
+  if (std::string(BUILT_SHA) != "") {
+    return std::string(BUILT_SHA);
   }
   return "Unknown Revision";
 }