improve debugger disasm, :sym-name and fix Windows builds (#959)

* improve debugger disasm, `:sym-name` and fix Windows builds

* >:(

* use this inline constexpr thing??

* fine use strings then

* please.... please work...

* fix windows debugger oopsie

* display rip as goal addr as well

* [debugger] attempt to backtrace even if landed on some garbage memory

* Update CMakePresets.json

goalc/debugger/Debugger.cpp

@@ -215,7 +215,8 @@ std::vector<BacktraceFrame> Debugger::get_backtrace(u64 rip, u64 rsp) {
   }
 
   while (true) {
-    fmt::print("   rsp: 0x{:x} rip: 0x{:x}\n", rsp, rip);
+    fmt::print("   rsp: 0x{:x} (#x{:x}) rip: 0x{:x} (#x{:x})\n", rsp, rsp - m_debug_context.base,
+               rip, rip - m_debug_context.base);
     BacktraceFrame frame;
     frame.rip_info = get_rip_info(rip);
     frame.rsp_at_rip = rsp;
@@ -239,16 +240,25 @@ std::vector<BacktraceFrame> Debugger::get_backtrace(u64 rip, u64 rsp) {
 
     } else {
       if (!frame.rip_info.knows_function) {
-        fmt::print("Unknown Function at 0x{:x}\n", rip);
-        break;
-      }
-      if (!frame.rip_info.func_debug) {
+        fmt::print("Unknown Function at 0x{:x} (#x{:x})\n", rip, rip - m_debug_context.base);
+
+        // attempt to backtrace anyway! if this fails then rip
+        u64 next_rip = 0;
+        if (!read_memory_if_safe<u64>(&next_rip, rsp - m_debug_context.base)) {
+          fmt::print("Invalid return address encountered!\n");
+          break;
+        }
+
+        rip = next_rip;
+        rsp = rsp + 8;  // 8 for the call itself.
+        // break;
+      } else if (!frame.rip_info.func_debug) {
         fmt::print("Function {} has no debug info.\n", frame.rip_info.function_name);
         break;
       } else {
         fmt::print("Function {} with no stack frame data.\n", frame.rip_info.function_name);
+        break;
       }
-      break;
     }
 
     bt.push_back(frame);
@@ -562,15 +572,15 @@ bool Debugger::get_symbol_value(const std::string& sym_name, u32* output) {
 }
 
 /*!
- * Get the value of a symbol by name. Returns if the symbol exists and populates output if it does.
+ * Get the value of a symbol by name. Returns NULL if symbol does not exist.
  */
-const char* Debugger::get_symbol_name_from_offset(s32 ofs) {
+const char* Debugger::get_symbol_name_from_offset(s32 ofs) const {
   assert(is_valid());
   auto kv = m_symbol_offset_to_name_map.find(ofs);
   if (kv != m_symbol_offset_to_name_map.end()) {
     return kv->second.c_str();
   }
-  return "<invalid symbol offset>";
+  return NULL;
 }
 
 /*!
@@ -846,3 +856,72 @@ DebugInfo& Debugger::get_debug_info_for_object(const std::string& object_name) {
 bool Debugger::knows_object(const std::string& object_name) const {
   return m_debug_info.find(object_name) != m_debug_info.end();
 }
+
+/*!
+ * Do x86 disassembly at the specified address and then do some basic string replacement for
+ * symbols. It will attempt to detect symbol dereferences (e.g. *active-pool*), symbol references
+ * (e.g. 'dead), and a special case to detect #f (outputted as '#f for correctness).
+ */
+std::string Debugger::disassemble_x86_with_symbols(int len, u64 base_addr) const {
+  std::vector<u8> mem;
+  mem.resize(len);
+
+  read_memory(mem.data(), len, base_addr);
+
+  auto result = disassemble_x86(mem.data(), mem.size(), get_x86_base_addr() + base_addr);
+
+  // find symbol values!
+  const std::string sym_val_string("[r15+r14*1");
+  size_t pos = 0;
+  while ((pos = result.find(sym_val_string, pos)) != std::string::npos) {
+    size_t read;
+    auto sym_addr = std::stol(result.substr(pos + sym_val_string.length(), 7), &read,
+                              16);  // -0x1234 is 7 characters
+
+    auto sym_name = get_symbol_name_from_offset((s32)sym_addr);
+    if (sym_name) {
+      std::string sym_str(sym_name);
+      result.replace(pos + 1, read + sym_val_string.length() - 1,
+                     sym_str);  // the [ is ignored (result is something like: [identity])
+      pos += sym_str.length() + 1;
+      assert(result.at(pos) == ']');  // maybe?
+    } else {
+      // symbol not found for whatever reason, just use regular disassembly and skip over
+      pos += 1;
+    }
+  }
+
+  // find symbol references!
+  const std::string sym_addr_string("[r14");
+  pos = 0;
+  while ((pos = result.find(sym_addr_string, pos)) != std::string::npos) {
+    size_t read;
+    auto sym_addr = std::stol(result.substr(pos + sym_addr_string.length(), 7), &read,
+                              16);  // -0x1234 is 7 characters
+
+    auto sym_name = get_symbol_name_from_offset((s32)sym_addr);
+    if (sym_name) {
+      std::string sym_str(sym_name);
+      result.replace(pos, read + sym_addr_string.length() + 1, fmt::format("'{}", sym_str));
+      pos += sym_str.length();
+    } else {
+      // symbol not found for whatever reason, just use regular disassembly and skip over
+      pos += 1;
+    }
+  }
+
+  // find #f references!
+  const std::string op_mov_string("] mov ");
+  const std::string sym_false_string(", r14");
+  pos = 0;
+  while ((pos = result.find(op_mov_string, pos)) != std::string::npos) {
+    pos += op_mov_string.length();
+    auto r14_pos = result.find(sym_false_string, pos);
+    if (r14_pos < result.find(op_mov_string, pos)) {
+      result.replace(r14_pos, sym_false_string.length(),
+                     fmt::format(", '{}", get_symbol_name_from_offset(0)));
+    }
+  }
+
+  return result;
+}