docs: Automatically generate documentation from goal_src code (#2214)

This automatically generates documentation from goal_src docstrings, think doxygen/java-docs/rust docs/etc. It mostly supports everything already, but here are the following things that aren't yet complete: - file descriptions - high-level documentation to go along with this (think pure markdown docs describing overall systems that would be co-located in goal_src for organizational purposes) - enums - states - std-lib functions (all have empty strings right now for docs anyway) The job of the new `gen-docs` function is solely to generate a bunch of JSON data which should give you everything you need to generate some decent documentation (outputting markdown/html/pdf/etc). It is not it's responsibility to do that nice formatting -- this is by design to intentionally delegate that responsibility elsewhere. Side-note, this is about 12-15MB of minified json for jak 2 so far :) In our normal "goal_src has changed" action -- we will generate this data, and the website can download it -- use the information to generate the documentation at build time -- and it will be included in the site. Likewise, if we wanted to include docs along with releases for offline viewing, we could do so in a similar fashion (just write a formatting script to generate said documentation). Lastly this work somewhat paves the way for doing more interesting things in the LSP like: - whats the docstring for this symbol? - autocompleting function arguments - type checking function arguments - where is this symbol defined? - etc Fixes #2215
2026-08-31 17:43:06 -04:00 · 2023-02-20 19:49:37 -05:00
parent 65f2146d73
commit 39658dfd71
23 changed files with 994 additions and 279 deletions
@@ -14,6 +14,8 @@

 #include "goalc/compiler/Compiler.h"
 #include "goalc/compiler/IR.h"
+#include "goalc/compiler/SymbolInfo.h"
+#include "goalc/compiler/docs/DocTypes.h"
 #include "goalc/data_compiler/dir_tpages.h"
 #include "goalc/data_compiler/game_count.h"
 #include "goalc/data_compiler/game_text_common.h"
@@ -355,7 +357,8 @@ std::string Compiler::make_symbol_info_description(const SymbolInfo& info) {
    case SymbolInfo::Kind::LANGUAGE_BUILTIN:
      return fmt::format("[Built-in Form] {}\n", info.name());
    case SymbolInfo::Kind::METHOD:
-      return fmt::format("[Method] Type: {} Method Name: {} Defined: {}", info.type(), info.name(),
+      return fmt::format("[Method] Type: {} Method Name: {} Defined: {}",
+                         info.method_info().defined_in_type, info.name(),
                         m_goos.reader.db.get_info_for(info.src_form()));
    case SymbolInfo::Kind::TYPE:
      return fmt::format("[Type] Name: {} Defined: {}", info.name(),
@@ -550,13 +553,28 @@ Val* Compiler::compile_autocomplete(const goos::Object& form, const goos::Object
  return get_none();
 }

-Val* Compiler::compile_add_macro_to_autocomplete(const goos::Object& form,
-                                                 const goos::Object& rest,
-                                                 Env* env) {
+Val* Compiler::compile_update_macro_metadata(const goos::Object& form,
+                                             const goos::Object& rest,
+                                             Env* env) {
  (void)env;
  auto args = get_va(form, rest);
-  va_check(form, args, {goos::ObjectType::SYMBOL}, {});
-  m_symbol_info.add_macro(args.unnamed.at(0).as_symbol()->name, form);
+  // We have to manually check the args here, as an empty list is considered something distinct from
+  // a pair
+  if (args.unnamed.size() != 3 || args.unnamed.at(0).type != goos::ObjectType::SYMBOL ||
+      args.unnamed.at(1).type != goos::ObjectType::STRING ||
+      (args.unnamed.at(2).type != goos::ObjectType::PAIR &&
+       args.unnamed.at(2).type != goos::ObjectType::EMPTY_LIST)) {
+    throw_compiler_error(form, "Invalid arguments provided to `update-macro-metadata");
+  }
+
+  auto& name = args.unnamed.at(0).as_symbol()->name;
+
+  auto arg_spec = m_goos.parse_arg_spec(form, args.unnamed.at(2));
+  m_macro_specs[name] = arg_spec;
+
+  SymbolInfo::Metadata sym_meta;
+  sym_meta.docstring = args.unnamed.at(1).as_string()->data;
+  m_symbol_info.add_macro(name, form, sym_meta);
  return get_none();
 }

@@ -616,3 +634,185 @@ Val* Compiler::compile_print_debug_compiler_stats(const goos::Object& form,

  return get_none();
 }
+
+Val* Compiler::compile_gen_docs(const goos::Object& form, const goos::Object& rest, Env*) {
+  auto args = get_va(form, rest);
+  va_check(form, args, {goos::ObjectType::STRING}, {});
+
+  const auto& doc_path = fs::path(args.unnamed.at(0).as_string()->data);
+  lg::info("Saving docs to: {}", doc_path.string());
+
+  const auto symbols = m_symbol_info.get_all_symbols();
+
+  std::unordered_map<std::string, Docs::SymbolDocumentation> all_symbols;
+  std::unordered_map<std::string, Docs::FileDocumentation> file_docs;
+
+  lg::info("Processing {} symbols...", symbols.size());
+  int count = 0;
+  for (const auto& sym_info : symbols) {
+    count++;
+    if (count % 100 == 0 || count == symbols.size()) {
+      lg::info("Processing [{}/{}] symbols...", count, symbols.size());
+    }
+    std::optional<Docs::DefinitionLocation> def_loc;
+    const auto& goos_info = m_goos.reader.db.get_short_info_for(sym_info.src_form());
+    if (goos_info) {
+      Docs::DefinitionLocation new_def_loc;
+      new_def_loc.filename = file_util::convert_to_unix_path_separators(file_util::split_path_at(
+          goos_info->filename, {"goal_src", version_to_game_name(m_version)}));
+      new_def_loc.line_idx = goos_info->line_idx_to_display;
+      new_def_loc.char_idx = goos_info->pos_in_line;
+      def_loc = new_def_loc;
+    }
+
+    Docs::SymbolDocumentation sym_doc;
+    sym_doc.name = sym_info.name();
+    sym_doc.description = sym_info.meta().docstring;
+    sym_doc.kind = sym_info.kind();
+    sym_doc.def_location = def_loc;
+
+    if (all_symbols.count(sym_info.name()) > 1) {
+      lg::error("A symbol was defined twice, how did this happen? {}", sym_info.name());
+    } else {
+      all_symbols.emplace(sym_info.name(), sym_doc);
+    }
+
+    Docs::FileDocumentation file_doc;
+    std::string file_doc_key;
+    if (!goos_info) {
+      file_doc_key = "unknown";
+    } else {
+      file_doc_key = file_util::convert_to_unix_path_separators(
+          file_util::split_path_at(goos_info->filename, {"goal_src"}));
+    }
+
+    if (file_docs.count(file_doc_key) != 0) {
+      file_doc = file_docs.at(file_doc_key);
+    } else {
+      file_doc = Docs::FileDocumentation();
+    }
+
+    // TODO - states / enums / built-ins
+    if (sym_info.kind() == SymbolInfo::Kind::GLOBAL_VAR ||
+        sym_info.kind() == SymbolInfo::Kind::CONSTANT) {
+      Docs::VariableDocumentation var;
+      var.name = sym_info.name();
+      var.description = sym_info.meta().docstring;
+      if (sym_info.kind() == SymbolInfo::Kind::CONSTANT) {
+        var.type = "unknown";  // Unfortunately, constants are not properly typed
+      } else {
+        var.type = m_symbol_types.at(var.name).base_type();
+      }
+      var.def_location = def_loc;
+      if (sym_info.kind() == SymbolInfo::Kind::GLOBAL_VAR) {
+        file_doc.global_vars.push_back(var);
+      } else {
+        file_doc.constants.push_back(var);
+      }
+    } else if (sym_info.kind() == SymbolInfo::Kind::FUNCTION) {
+      Docs::FunctionDocumentation func;
+      func.name = sym_info.name();
+      func.description = sym_info.meta().docstring;
+      func.def_location = def_loc;
+      func.args = Docs::get_args_from_docstring(sym_info.args(), func.description);
+      // The last arg in the typespec is the return type
+      const auto& func_type = m_symbol_types.at(func.name);
+      func.return_type = func_type.last_arg().base_type();
+      file_doc.functions.push_back(func);
+    } else if (sym_info.kind() == SymbolInfo::Kind::TYPE) {
+      Docs::TypeDocumentation type;
+      type.name = sym_info.name();
+      type.description = sym_info.meta().docstring;
+      type.def_location = def_loc;
+      const auto& type_info = m_ts.lookup_type(type.name);
+      type.parent_type = type_info->get_parent();
+      type.size = type_info->get_size_in_memory();
+      type.method_count = type_info->get_methods_defined_for_type().size();
+      if (m_ts.typecheck_and_throw(m_ts.make_typespec("structure"), m_ts.make_typespec(type.name),
+                                   "", false, false, false)) {
+        auto struct_info = dynamic_cast<StructureType*>(type_info);
+        for (const auto& field : struct_info->fields()) {
+          Docs::FieldDocumentation field_doc;
+          field_doc.name = field.name();
+          field_doc.description = "";
+          field_doc.type = field.type().base_type();
+          field_doc.is_array = field.is_array();
+          field_doc.is_inline = field.is_inline();
+          field_doc.is_dynamic = field.is_dynamic();
+          type.fields.push_back(field_doc);
+        }
+      }
+      for (const auto& method : type_info->get_methods_defined_for_type()) {
+        // Check to see if it's a state
+        if (m_ts.typecheck_and_throw(m_ts.make_typespec("state"), method.type, "", false, false,
+                                     false)) {
+          Docs::TypeStateDocumentation state_doc;
+          state_doc.id = method.id;
+          state_doc.is_virtual = true;
+          state_doc.name = method.name;
+          type.states.push_back(state_doc);
+        } else {
+          Docs::TypeMethodDocumentation method_doc;
+          method_doc.id = method.id;
+          method_doc.name = method.name;
+          method_doc.is_override = method.overrides_parent;
+          type.methods.push_back(method_doc);
+        }
+      }
+      for (const auto& [state_name, state_info] : type_info->get_states_declared_for_type()) {
+        Docs::TypeStateDocumentation state_doc;
+        state_doc.name = state_name;
+        state_doc.is_virtual = false;
+        type.states.push_back(state_doc);
+      }
+      file_doc.types.push_back(type);
+    } else if (sym_info.kind() == SymbolInfo::Kind::MACRO) {
+      Docs::MacroDocumentation macro_doc;
+      macro_doc.name = sym_info.name();
+      macro_doc.description = sym_info.meta().docstring;
+      macro_doc.def_location = def_loc;
+      const auto& arg_spec = m_macro_specs[macro_doc.name];
+      for (const auto& arg : arg_spec.unnamed) {
+        macro_doc.args.push_back(arg);
+      }
+      for (const auto& arg : arg_spec.named) {
+        std::optional<std::string> def_value;
+        if (arg.second.has_default) {
+          def_value = arg.second.default_value.print();
+        }
+        macro_doc.kwargs.push_back({arg.first, def_value});
+      }
+      if (!arg_spec.rest.empty()) {
+        macro_doc.variadic_arg = arg_spec.rest;
+      }
+      file_doc.macros.push_back(macro_doc);
+    } else if (sym_info.kind() == SymbolInfo::Kind::METHOD) {
+      Docs::MethodDocumentation method_doc;
+      method_doc.name = sym_info.name();
+      method_doc.description = sym_info.meta().docstring;
+      method_doc.def_location = def_loc;
+      const auto& method_info = sym_info.method_info();
+      method_doc.id = method_info.id;
+      method_doc.type = sym_info.method_info().defined_in_type;
+      method_doc.is_override = method_info.overrides_parent;
+      method_doc.args = Docs::get_args_from_docstring(sym_info.args(), method_doc.description);
+      // The last arg in the typespec is the return type
+      const auto& method_type = method_info.type;
+      method_doc.return_type = method_type.last_arg().base_type();
+      method_doc.is_builtin = method_doc.id <= 9;
+      file_doc.methods.push_back(method_doc);
+    }
+    file_docs[file_doc_key] = file_doc;
+  }
+
+  json symbol_map_data(all_symbols);
+  file_util::write_text_file(
+      doc_path / fmt::format("{}-symbol-map.json", version_to_game_name(m_version)),
+      symbol_map_data.dump());
+  json file_docs_data(file_docs);
+  file_util::write_text_file(
+      doc_path / fmt::format("{}-file-docs.json", version_to_game_name(m_version)),
+      file_docs_data.dump());
+
+  return get_none();
+}