add decompiler

decompiler/ObjectFile/LinkedObjectFile.cpp

@@ -0,0 +1,853 @@
+/*!
+ * @file LinkedObjectFile.cpp
+ * An object file's data with linking information included.
+ */
+#include "LinkedObjectFile.h"
+#include <algorithm>
+#include <cassert>
+#include <cstring>
+#include <numeric>
+#include "decompiler/Disasm/InstructionDecode.h"
+#include "decompiler/config.h"
+
+/*!
+ * Set the number of segments in this object file.
+ * This can only be done once, and must be done before adding any words.
+ */
+void LinkedObjectFile::set_segment_count(int n_segs) {
+  assert(segments == 0);
+  segments = n_segs;
+  words_by_seg.resize(n_segs);
+  label_per_seg_by_offset.resize(n_segs);
+  offset_of_data_zone_by_seg.resize(n_segs);
+  functions_by_seg.resize(n_segs);
+}
+
+/*!
+ * Add a single word to the given segment.
+ */
+void LinkedObjectFile::push_back_word_to_segment(uint32_t word, int segment) {
+  words_by_seg.at(segment).emplace_back(word);
+}
+
+/*!
+ * Get a label ID for a label which points to the given offset in the given segment.
+ * Will return an existing label if one exists.
+ */
+int LinkedObjectFile::get_label_id_for(int seg, int offset) {
+  auto kv = label_per_seg_by_offset.at(seg).find(offset);
+  if (kv == label_per_seg_by_offset.at(seg).end()) {
+    // create a new label
+    int id = labels.size();
+    Label label;
+    label.target_segment = seg;
+    label.offset = offset;
+    label.name = "L" + std::to_string(id);
+    label_per_seg_by_offset.at(seg)[offset] = id;
+    labels.push_back(label);
+    return id;
+  } else {
+    // return an existing label
+    auto& label = labels.at(kv->second);
+    assert(label.offset == offset);
+    assert(label.target_segment == seg);
+    return kv->second;
+  }
+}
+
+/*!
+ * Get the ID of the label which points to the given offset in the given segment.
+ * Returns -1 if there is no label.
+ */
+int LinkedObjectFile::get_label_at(int seg, int offset) const {
+  auto kv = label_per_seg_by_offset.at(seg).find(offset);
+  if (kv == label_per_seg_by_offset.at(seg).end()) {
+    return -1;
+  }
+
+  return kv->second;
+}
+
+/*!
+ * Does this label point to code? Can point to the middle of a function, or the start of a function.
+ */
+bool LinkedObjectFile::label_points_to_code(int label_id) const {
+  auto& label = labels.at(label_id);
+  auto data_start = int(offset_of_data_zone_by_seg.at(label.target_segment)) * 4;
+  return label.offset < data_start;
+}
+
+/*!
+ * Get the function starting at this label, or error if there is none.
+ */
+Function& LinkedObjectFile::get_function_at_label(int label_id) {
+  auto& label = labels.at(label_id);
+  for (auto& func : functions_by_seg.at(label.target_segment)) {
+    // + 4 to skip past type tag to the first word, which is were the label points.
+    if (func.start_word * 4 + 4 == label.offset) {
+      return func;
+    }
+  }
+
+  assert(false);
+  return functions_by_seg.front().front(); // to avoid error
+}
+
+/*!
+ * Get the name of the label.
+ */
+std::string LinkedObjectFile::get_label_name(int label_id) const {
+  return labels.at(label_id).name;
+}
+
+/*!
+ * Add link information that a word is a pointer to another word.
+ */
+bool LinkedObjectFile::pointer_link_word(int source_segment,
+                                         int source_offset,
+                                         int dest_segment,
+                                         int dest_offset) {
+  assert((source_offset % 4) == 0);
+
+  auto& word = words_by_seg.at(source_segment).at(source_offset / 4);
+  assert(word.kind == LinkedWord::PLAIN_DATA);
+
+  if (dest_offset / 4 > (int)words_by_seg.at(dest_segment).size()) {
+    //    printf("HACK bad link ignored!\n");
+    return false;
+  }
+  assert(dest_offset / 4 <= (int)words_by_seg.at(dest_segment).size());
+
+  word.kind = LinkedWord::PTR;
+  word.label_id = get_label_id_for(dest_segment, dest_offset);
+  return true;
+}
+
+/*!
+ * Add link information that a word is linked to a symbol/type/empty list.
+ */
+void LinkedObjectFile::symbol_link_word(int source_segment,
+                                        int source_offset,
+                                        const char* name,
+                                        LinkedWord::Kind kind) {
+  assert((source_offset % 4) == 0);
+  auto& word = words_by_seg.at(source_segment).at(source_offset / 4);
+  //  assert(word.kind == LinkedWord::PLAIN_DATA);
+  if (word.kind != LinkedWord::PLAIN_DATA) {
+    printf("bad symbol link word\n");
+  }
+  word.kind = kind;
+  word.symbol_name = name;
+}
+
+/*!
+ * Add link information that a word's lower 16 bits are the offset of the given symbol relative to
+ * the symbol table register.
+ */
+void LinkedObjectFile::symbol_link_offset(int source_segment, int source_offset, const char* name) {
+  assert((source_offset % 4) == 0);
+  auto& word = words_by_seg.at(source_segment).at(source_offset / 4);
+  assert(word.kind == LinkedWord::PLAIN_DATA);
+  word.kind = LinkedWord::SYM_OFFSET;
+  word.symbol_name = name;
+}
+
+/*!
+ * Add link information that a lui/ori pair will load a pointer.
+ */
+void LinkedObjectFile::pointer_link_split_word(int source_segment,
+                                               int source_hi_offset,
+                                               int source_lo_offset,
+                                               int dest_segment,
+                                               int dest_offset) {
+  assert((source_hi_offset % 4) == 0);
+  assert((source_lo_offset % 4) == 0);
+
+  auto& hi_word = words_by_seg.at(source_segment).at(source_hi_offset / 4);
+  auto& lo_word = words_by_seg.at(source_segment).at(source_lo_offset / 4);
+
+  //  assert(dest_offset / 4 <= (int)words_by_seg.at(dest_segment).size());
+  assert(hi_word.kind == LinkedWord::PLAIN_DATA);
+  assert(lo_word.kind == LinkedWord::PLAIN_DATA);
+
+  hi_word.kind = LinkedWord::HI_PTR;
+  hi_word.label_id = get_label_id_for(dest_segment, dest_offset);
+
+  lo_word.kind = LinkedWord::LO_PTR;
+  lo_word.label_id = hi_word.label_id;
+}
+
+/*!
+ * Rename the labels so they are named L1, L2, ..., in the order of the addresses that they refer
+ * to. Will clear any custom label names.
+ */
+uint32_t LinkedObjectFile::set_ordered_label_names() {
+  std::vector<int> indices(labels.size());
+  std::iota(indices.begin(), indices.end(), 0);
+
+  std::sort(indices.begin(), indices.end(), [&](int a, int b) {
+    auto& la = labels.at(a);
+    auto& lb = labels.at(b);
+    if (la.target_segment == lb.target_segment) {
+      return la.offset < lb.offset;
+    }
+    return la.target_segment < lb.target_segment;
+  });
+
+  for (size_t i = 0; i < indices.size(); i++) {
+    auto& label = labels.at(indices[i]);
+    label.name = "L" + std::to_string(i + 1);
+  }
+
+  return labels.size();
+}
+
+static const char* segment_names[] = {"main segment", "debug segment", "top-level segment"};
+
+/*!
+ * Print all the words, with link information and labels.
+ */
+std::string LinkedObjectFile::print_words() {
+  std::string result;
+
+  assert(segments <= 3);
+  for (int seg = segments; seg-- > 0;) {
+    // segment header
+    result += ";------------------------------------------\n;  ";
+    result += segment_names[seg];
+    result += "\n;------------------------------------------\n";
+
+    // print each word in the segment
+    for (size_t i = 0; i < words_by_seg.at(seg).size(); i++) {
+      for (int j = 0; j < 4; j++) {
+        auto label_id = get_label_at(seg, i * 4 + j);
+        if (label_id != -1) {
+          result += labels.at(label_id).name + ":";
+          if (j != 0) {
+            result += " (offset " + std::to_string(j) + ")";
+          }
+          result += "\n";
+        }
+      }
+
+      auto& word = words_by_seg[seg][i];
+      append_word_to_string(result, word);
+    }
+  }
+
+  return result;
+}
+
+/*!
+ * Add a word's printed representation to the end of a string. Internal helper for print_words.
+ */
+void LinkedObjectFile::append_word_to_string(std::string& dest, const LinkedWord& word) const {
+  char buff[128];
+
+  switch (word.kind) {
+    case LinkedWord::PLAIN_DATA:
+      sprintf(buff, "    .word 0x%x\n", word.data);
+      break;
+    case LinkedWord::PTR:
+      sprintf(buff, "    .word %s\n", labels.at(word.label_id).name.c_str());
+      break;
+    case LinkedWord::SYM_PTR:
+      sprintf(buff, "    .symbol %s\n", word.symbol_name.c_str());
+      break;
+    case LinkedWord::TYPE_PTR:
+      sprintf(buff, "    .type %s\n", word.symbol_name.c_str());
+      break;
+    case LinkedWord::EMPTY_PTR:
+      sprintf(buff, "    .empty-list\n");  // ?
+      break;
+    case LinkedWord::HI_PTR:
+      sprintf(buff, "    .ptr-hi 0x%x %s\n", word.data >> 16,
+              labels.at(word.label_id).name.c_str());
+      break;
+    case LinkedWord::LO_PTR:
+      sprintf(buff, "    .ptr-lo 0x%x %s\n", word.data >> 16,
+              labels.at(word.label_id).name.c_str());
+      break;
+    case LinkedWord::SYM_OFFSET:
+      sprintf(buff, "    .sym-off 0x%x %s\n", word.data >> 16, word.symbol_name.c_str());
+      break;
+    default:
+      throw std::runtime_error("nyi");
+  }
+
+  dest += buff;
+}
+
+/*!
+ * For each segment, determine where the data area starts.  Before the data area is the code area.
+ */
+void LinkedObjectFile::find_code() {
+  if (segments == 1) {
+    // single segment object files should never have any code.
+    auto& seg = words_by_seg.front();
+    for (auto& word : seg) {
+      if (!word.symbol_name.empty()) {
+        assert(word.symbol_name != "function");
+      }
+    }
+    offset_of_data_zone_by_seg.at(0) = 0;
+    stats.data_bytes = words_by_seg.front().size() * 4;
+    stats.code_bytes = 0;
+
+  } else if (segments == 3) {
+    // V3 object files will have all the functions, then all the static data.  So to find the
+    // divider, we look for the last "function" tag, then find the last jr $ra instruction after
+    // that (plus one for delay slot) and assume that after that is data.  Additionally, we check to
+    // make sure that there are no "function" type tags in the data section, although this is
+    // redundant.
+    for (int i = 0; i < segments; i++) {
+      // try to find the last reference to "function":
+      bool found_function = false;
+      size_t function_loc = -1;
+      for (size_t j = words_by_seg.at(i).size(); j-- > 0;) {
+        auto& word = words_by_seg.at(i).at(j);
+        if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "function") {
+          function_loc = j;
+          found_function = true;
+          break;
+        }
+      }
+
+      if (found_function) {
+        // look forward until we find "jr ra"
+        const uint32_t jr_ra = 0x3e00008;
+        bool found_jr_ra = false;
+        size_t jr_ra_loc = -1;
+
+        for (size_t j = function_loc; j < words_by_seg.at(i).size(); j++) {
+          auto& word = words_by_seg.at(i).at(j);
+          if (word.kind == LinkedWord::PLAIN_DATA && word.data == jr_ra) {
+            found_jr_ra = true;
+            jr_ra_loc = j;
+          }
+        }
+
+        assert(found_jr_ra);
+        assert(jr_ra_loc + 1 < words_by_seg.at(i).size());
+        offset_of_data_zone_by_seg.at(i) = jr_ra_loc + 2;
+
+      } else {
+        // no functions
+        offset_of_data_zone_by_seg.at(i) = 0;
+      }
+
+      // add label for debug purposes
+      if (offset_of_data_zone_by_seg.at(i) < words_by_seg.at(i).size()) {
+        auto data_label_id = get_label_id_for(i, 4 * (offset_of_data_zone_by_seg.at(i)));
+        labels.at(data_label_id).name = "L-data-start";
+      }
+
+      // verify there are no functions after the data section starts
+      for (size_t j = offset_of_data_zone_by_seg.at(i); j < words_by_seg.at(i).size(); j++) {
+        auto& word = words_by_seg.at(i).at(j);
+        if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "function") {
+          assert(false);
+        }
+      }
+
+      // sizes:
+      stats.data_bytes += 4 * (words_by_seg.at(i).size() - offset_of_data_zone_by_seg.at(i)) * 4;
+      stats.code_bytes += 4 * offset_of_data_zone_by_seg.at(i);
+    }
+  } else {
+    // for files which we couldn't extract link data yet, they will have 0 segments and its ok.
+    assert(segments == 0);
+  }
+}
+
+/*!
+ * Find all the functions in each segment.
+ */
+void LinkedObjectFile::find_functions() {
+  if (segments == 1) {
+    // it's a v2 file, shouldn't have any functions
+    assert(offset_of_data_zone_by_seg.at(0) == 0);
+  } else {
+    // we assume functions don't have any data in between them, so we use the "function" type tag to
+    // mark the end of the previous function and the start of the next.  This means that some
+    // functions will have a few 0x0 words after then for padding (GOAL functions are aligned), but
+    // this is something that the disassembler should handle.
+    for (int seg = 0; seg < segments; seg++) {
+      // start at the end and work backward...
+      int function_end = offset_of_data_zone_by_seg.at(seg);
+      while (function_end > 0) {
+        // back up until we find function type tag
+        int function_tag_loc = function_end;
+        bool found_function_tag_loc = false;
+        for (; function_tag_loc-- > 0;) {
+          auto& word = words_by_seg.at(seg).at(function_tag_loc);
+          if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "function") {
+            found_function_tag_loc = true;
+            break;
+          }
+        }
+
+        // mark this as a function, and try again from the current function start
+        assert(found_function_tag_loc);
+        stats.function_count++;
+        functions_by_seg.at(seg).emplace_back(function_tag_loc, function_end);
+        function_end = function_tag_loc;
+      }
+
+      std::reverse(functions_by_seg.at(seg).begin(), functions_by_seg.at(seg).end());
+    }
+  }
+}
+
+/*!
+ * Run the disassembler on all functions.
+ */
+void LinkedObjectFile::disassemble_functions() {
+  for (int seg = 0; seg < segments; seg++) {
+    for (auto& function : functions_by_seg.at(seg)) {
+      for (auto word = function.start_word; word < function.end_word; word++) {
+        // decode!
+        function.instructions.push_back(
+            decode_instruction(words_by_seg.at(seg).at(word), *this, seg, word));
+        if (function.instructions.back().is_valid()) {
+          stats.decoded_ops++;
+        }
+      }
+    }
+  }
+}
+
+/*!
+ * Analyze disassembly for use of the FP register, and add labels for fp-relative data access
+ */
+void LinkedObjectFile::process_fp_relative_links() {
+  for (int seg = 0; seg < segments; seg++) {
+    for (auto& function : functions_by_seg.at(seg)) {
+      for (size_t instr_idx = 0; instr_idx < function.instructions.size(); instr_idx++) {
+        // we possibly need to look at three instructions
+        auto& instr = function.instructions[instr_idx];
+        auto* prev_instr = (instr_idx > 0) ? &function.instructions[instr_idx - 1] : nullptr;
+        auto* pprev_instr = (instr_idx > 1) ? &function.instructions[instr_idx - 2] : nullptr;
+
+        // ignore storing FP onto the stack
+        if ((instr.kind == InstructionKind::SD || instr.kind == InstructionKind::SQ) &&
+            instr.get_src(0).get_reg() == Register(Reg::GPR, Reg::FP)) {
+          continue;
+        }
+
+        // HACKs
+        if (instr.kind == InstructionKind::PEXTLW) {
+          continue;
+        }
+
+        // search over instruction sources
+        for (int i = 0; i < instr.n_src; i++) {
+          auto& src = instr.src[i];
+          if (src.kind == InstructionAtom::REGISTER     // must be reg
+              && src.get_reg().get_kind() == Reg::GPR   // gpr
+              && src.get_reg().get_gpr() == Reg::FP) {  // fp reg.
+
+            stats.n_fp_reg_use++;
+
+            // offset of fp at this instruction.
+            int current_fp = 4 * (function.start_word + 1);
+            function.uses_fp_register = true;
+
+            switch (instr.kind) {
+              // fp-relative load
+              case InstructionKind::LW:
+              case InstructionKind::LWC1:
+              case InstructionKind::LD:
+              // generate pointer to fp-relative data
+              case InstructionKind::DADDIU: {
+                auto& atom = instr.get_imm_src();
+                atom.set_label(get_label_id_for(seg, current_fp + atom.get_imm()));
+                stats.n_fp_reg_use_resolved++;
+              } break;
+
+              // in the case that addiu doesn't have enough range (+/- 2^15), GOAL has two
+              // strategies: 1). use ori + daddu (ori doesn't sign extend, so this lets us go +2^16,
+              // -0) 2). use lui + ori + daddu (can reach anywhere in the address space) It seems
+              // that addu is used to get pointers to floating point values and daddu is used in
+              // other cases. Also, the position of the fp register is swapped between the two.
+              case InstructionKind::DADDU:
+              case InstructionKind::ADDU: {
+                assert(prev_instr);
+                assert(prev_instr->kind == InstructionKind::ORI);
+                int offset_reg_src_id = instr.kind == InstructionKind::DADDU ? 0 : 1;
+                auto offset_reg = instr.get_src(offset_reg_src_id).get_reg();
+                assert(offset_reg == prev_instr->get_dst(0).get_reg());
+                assert(offset_reg == prev_instr->get_src(0).get_reg());
+                auto& atom = prev_instr->get_imm_src();
+                int additional_offset = 0;
+                if (pprev_instr && pprev_instr->kind == InstructionKind::LUI) {
+                  assert(pprev_instr->get_dst(0).get_reg() == offset_reg);
+                  additional_offset = (1 << 16) * pprev_instr->get_imm_src().get_imm();
+                }
+                atom.set_label(
+                    get_label_id_for(seg, current_fp + atom.get_imm() + additional_offset));
+                stats.n_fp_reg_use_resolved++;
+              } break;
+
+              default:
+                printf("unknown fp using op: %s\n", instr.to_string(*this).c_str());
+                assert(false);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+/*!
+ * Print disassembled functions and data segments.
+ */
+std::string LinkedObjectFile::print_disassembly() {
+  bool write_hex = get_config().write_hex_near_instructions;
+  std::string result;
+
+  assert(segments <= 3);
+  for (int seg = segments; seg-- > 0;) {
+    // segment header
+    result += ";------------------------------------------\n;  ";
+    result += segment_names[seg];
+    result += "\n;------------------------------------------\n\n";
+
+    // functions
+    for (auto& func : functions_by_seg.at(seg)) {
+      result += ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n";
+      result += "; .function " + func.guessed_name.to_string() + "\n";
+      result += ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;\n";
+      result += func.prologue.to_string(2) + "\n";
+      if(!func.warnings.empty()) {
+        result += "Warnings: " + func.warnings + "\n";
+      }
+
+      // print each instruction in the function.
+      bool in_delay_slot = false;
+
+      for (int i = 1; i < func.end_word - func.start_word; i++) {
+        auto label_id = get_label_at(seg, (func.start_word + i) * 4);
+        if (label_id != -1) {
+          result += labels.at(label_id).name + ":\n";
+        }
+
+        for (int j = 1; j < 4; j++) {
+          //          assert(get_label_at(seg, (func.start_word + i)*4 + j) == -1);
+          if (get_label_at(seg, (func.start_word + i) * 4 + j) != -1) {
+            result += "BAD OFFSET LABEL: ";
+            result += labels.at(get_label_at(seg, (func.start_word + i) * 4 + j)).name + "\n";
+            assert(false);
+          }
+        }
+
+        auto& instr = func.instructions.at(i);
+        std::string line = "    " + instr.to_string(*this);
+
+        if (write_hex) {
+          if (line.length() < 60) {
+            line.append(60 - line.length(), ' ');
+          }
+          result += line;
+          result += " ;;";
+          auto& word = words_by_seg[seg].at(func.start_word + i);
+          append_word_to_string(result, word);
+        } else {
+          result += line + "\n";
+        }
+
+        if (in_delay_slot) {
+          result += "\n";
+          in_delay_slot = false;
+        }
+
+        if (gOpcodeInfo[(int)instr.kind].has_delay_slot) {
+          in_delay_slot = true;
+        }
+      }
+      result += "\n";
+      //
+      //      int bid = 0;
+      //      for(auto& bblock : func.basic_blocks) {
+      //        result += "BLOCK " + std::to_string(bid++)+ "\n";
+      //        for(int i = bblock.start_word; i < bblock.end_word; i++) {
+      //          if(i >= 0 && i < func.instructions.size()) {
+      //            result += func.instructions.at(i).to_string(*this) + "\n";
+      //          } else {
+      //            result += "BAD BBLOCK INSTR ID " + std::to_string(i);
+      //          }
+      //        }
+      //      }
+
+      // hack
+      if(func.cfg && !func.cfg->is_fully_resolved()) {
+        result += func.cfg->to_dot();
+        result += "\n";
+      }
+      if(func.cfg) {
+        result += func.cfg->to_form_string() + "\n";
+
+        // To debug block stuff.
+        /*
+        int bid = 0;
+        for(auto& block : func.basic_blocks) {
+          in_delay_slot = false;
+          result += "B" + std::to_string(bid++) + "\n";
+          for(auto i = block.start_word; i < block.end_word; i++) {
+            auto label_id = get_label_at(seg, (func.start_word + i) * 4);
+            if (label_id != -1) {
+              result += labels.at(label_id).name + ":\n";
+            }
+            auto& instr = func.instructions.at(i);
+            result += "    " + instr.to_string(*this) + "\n";
+            if (in_delay_slot) {
+              result += "\n";
+              in_delay_slot = false;
+            }
+
+            if (gOpcodeInfo[(int)instr.kind].has_delay_slot) {
+              in_delay_slot = true;
+            }
+          }
+        }
+         */
+      }
+
+
+      result += "\n\n\n";
+    }
+
+    // print data
+    for (size_t i = offset_of_data_zone_by_seg.at(seg); i < words_by_seg.at(seg).size(); i++) {
+      for (int j = 0; j < 4; j++) {
+        auto label_id = get_label_at(seg, i * 4 + j);
+        if (label_id != -1) {
+          result += labels.at(label_id).name + ":";
+          if (j != 0) {
+            result += " (offset " + std::to_string(j) + ")";
+          }
+          result += "\n";
+        }
+      }
+
+      auto& word = words_by_seg[seg][i];
+      append_word_to_string(result, word);
+
+      if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "string") {
+        result += "; " + get_goal_string(seg, i) + "\n";
+
+      }
+    }
+  }
+
+  return result;
+}
+
+/*!
+ * Hacky way to get a GOAL string object
+ */
+std::string LinkedObjectFile::get_goal_string(int seg, int word_idx) {
+  std::string result = "\"";
+  // next should be the size
+  if (word_idx + 1 >= int(words_by_seg[seg].size())) {
+    return "invalid string!\n";
+  }
+  LinkedWord& size_word = words_by_seg[seg].at(word_idx + 1);
+  if (size_word.kind != LinkedWord::PLAIN_DATA) {
+    // sometimes an array of string pointer triggers this!
+    return "invalid string!\n";
+  }
+
+  //  result += "(size " + std::to_string(size_word.data) + "): ";
+  // now characters...
+  for (size_t i = 0; i < size_word.data; i++) {
+    int word_offset = word_idx + 2 + (i / 4);
+    int byte_offset = i % 4;
+    auto& word = words_by_seg[seg].at(word_offset);
+    if (word.kind != LinkedWord::PLAIN_DATA) {
+      return "invalid string! (check me!)\n";
+    }
+    char cword[4];
+    memcpy(cword, &word.data, 4);
+    result += cword[byte_offset];
+  }
+  return result + "\"";
+}
+
+/*!
+ * Return true if the object file contains any functions at all.
+ */
+bool LinkedObjectFile::has_any_functions() {
+  for (auto& fv : functions_by_seg) {
+    if (!fv.empty())
+      return true;
+  }
+  return false;
+}
+
+/*!
+ * Print all scripts in this file.
+ */
+std::string LinkedObjectFile::print_scripts() {
+  std::string result;
+  for (int seg = 0; seg < segments; seg++) {
+    std::vector<bool> already_printed(words_by_seg[seg].size(), false);
+
+    // the linked list layout algorithm of GOAL puts the first pair first.
+    // so we want to go in forward order to catch the beginning correctly
+    for (size_t word_idx = 0; word_idx < words_by_seg[seg].size(); word_idx++) {
+      // don't print parts of scripts we've already seen
+      // (note that scripts could share contents, which is supported, this is just for starting
+      // off a script print)
+      if (already_printed[word_idx])
+        continue;
+
+      // check for linked list by looking for anything that accesses this as a pair (offset of 2)
+      auto label_id = get_label_at(seg, 4 * word_idx + 2);
+      if (label_id != -1) {
+        auto& label = labels.at(label_id);
+        if ((label.offset & 7) == 2) {
+          result += to_form_script(seg, word_idx, already_printed)->toStringPretty(0, 100) + "\n";
+        }
+      }
+    }
+  }
+  return result;
+}
+
+/*!
+ * Is the object pointed to the empty list?
+ */
+bool LinkedObjectFile::is_empty_list(int seg, int byte_idx) {
+  assert((byte_idx % 4) == 0);
+  auto& word = words_by_seg.at(seg).at(byte_idx / 4);
+  return word.kind == LinkedWord::EMPTY_PTR;
+}
+
+/*!
+ * Convert a linked list to a Form for easy printing.
+ * Note : this takes the address of the car of the pair. which is perhaps a bit confusing
+ * (in GOAL, this would be (&-> obj car))
+ */
+std::shared_ptr<Form> LinkedObjectFile::to_form_script(int seg,
+                                                       int word_idx,
+                                                       std::vector<bool>& seen) {
+  // the object to currently print. to start off, create pair from the car address we've been given.
+  int goal_print_obj = word_idx * 4 + 2;
+
+  // resulting form. we can't have a totally empty list (as an empty list looks like a symbol,
+  // so it wouldn't be flagged), so it's safe to make this a pair.
+  auto result = std::make_shared<Form>();
+  result->kind = FormKind::PAIR;
+
+  // the current pair to fill out.
+  auto fill = result;
+
+  // loop until we run out of things to add
+  for (;;) {
+    // check the thing to print is a a pair.
+    if ((goal_print_obj & 7) == 2) {
+      // first convert the car (again, with (&-> obj car))
+      fill->pair[0] = to_form_script_object(seg, goal_print_obj - 2, seen);
+      seen.at(goal_print_obj / 4) = true;
+
+      auto cdr_addr = goal_print_obj + 2;
+
+      if (is_empty_list(seg, cdr_addr)) {
+        // the list has ended!
+        fill->pair[1] = gSymbolTable.getEmptyPair();
+        return result;
+      } else {
+        // cdr object should be aligned.
+        assert((cdr_addr % 4) == 0);
+        auto& cdr_word = words_by_seg.at(seg).at(cdr_addr / 4);
+        // check for proper list
+        if (cdr_word.kind == LinkedWord::PTR && (labels.at(cdr_word.label_id).offset & 7) == 2) {
+          // yes, proper list. add another pair and link it in to the list.
+          goal_print_obj = labels.at(cdr_word.label_id).offset;
+          fill->pair[1] = std::make_shared<Form>();
+          fill->pair[1]->kind = FormKind::PAIR;
+          fill = fill->pair[1];
+        } else {
+          // improper list, put the last thing in and end
+          fill->pair[1] = to_form_script_object(seg, cdr_addr, seen);
+          return result;
+        }
+      }
+    } else {
+      // improper list, should be impossible to get here because of earlier checks
+      assert(false);
+    }
+  }
+
+  return result;
+}
+
+/*!
+ * Is the thing pointed to a string?
+ */
+bool LinkedObjectFile::is_string(int seg, int byte_idx) {
+  if (byte_idx % 4) {
+    return false;  // must be aligned pointer.
+  }
+  int type_tag_ptr = byte_idx - 4;
+  // must fit in segment
+  if (type_tag_ptr < 0 || size_t(type_tag_ptr) >= words_by_seg.at(seg).size() * 4) {
+    return false;
+  }
+  auto& type_word = words_by_seg.at(seg).at(type_tag_ptr / 4);
+  return type_word.kind == LinkedWord::TYPE_PTR && type_word.symbol_name == "string";
+}
+
+/*!
+ * Convert a (pointer object) to some nice representation.
+ */
+std::shared_ptr<Form> LinkedObjectFile::to_form_script_object(int seg,
+                                                              int byte_idx,
+                                                              std::vector<bool>& seen) {
+  std::shared_ptr<Form> result;
+
+  switch (byte_idx & 7) {
+    case 0:
+    case 4: {
+      auto& word = words_by_seg.at(seg).at(byte_idx / 4);
+      if (word.kind == LinkedWord::SYM_PTR) {
+        // .symbol xxxx
+        result = toForm(word.symbol_name);
+      } else if (word.kind == LinkedWord::PLAIN_DATA) {
+        // .word xxxxx
+        result = toForm(std::to_string(word.data));
+      } else if (word.kind == LinkedWord::PTR) {
+        // might be a sub-list, or some other random pointer
+        auto offset = labels.at(word.label_id).offset;
+        if ((offset & 7) == 2) {
+          // list!
+          result = to_form_script(seg, offset / 4, seen);
+        } else {
+          if (is_string(seg, offset)) {
+            result = toForm(get_goal_string(seg, offset / 4 - 1));
+          } else {
+            // some random pointer, just print the label.
+            result = toForm(labels.at(word.label_id).name);
+          }
+        }
+      } else if (word.kind == LinkedWord::EMPTY_PTR) {
+        result = gSymbolTable.getEmptyPair();
+      } else {
+        std::string debug;
+        append_word_to_string(debug, word);
+        printf("don't know how to print %s\n", debug.c_str());
+        assert(false);
+      }
+    } break;
+
+    case 2:  // bad, a pair snuck through.
+    default:
+      // pointers should be aligned!
+      printf("align %d\n", byte_idx & 7);
+      assert(false);
+  }
+
+  return result;
+}

decompiler/ObjectFile/LinkedObjectFile.h

@@ -0,0 +1,131 @@
+/*!
+ * @file LinkedObjectFile.h
+ * An object file's data with linking information included.
+ */
+
+#ifndef NEXT_LINKEDOBJECTFILE_H
+#define NEXT_LINKEDOBJECTFILE_H
+
+#include <cstdint>
+#include <vector>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include "LinkedWord.h"
+#include "decompiler/Function/Function.h"
+#include "decompiler/util/LispPrint.h"
+
+
+/*!
+ * A label to a location in this object file.
+ * Doesn't have to be word aligned.
+ */
+struct Label {
+  std::string name;
+  int target_segment;
+  int offset; // in bytes
+};
+
+/*!
+ * An object file's data with linking information included.
+ */
+class LinkedObjectFile {
+public:
+  LinkedObjectFile() = default;
+  void set_segment_count(int n_segs);
+  void push_back_word_to_segment(uint32_t word, int segment);
+  int get_label_id_for(int seg, int offset);
+  int get_label_at(int seg, int offset) const;
+  bool label_points_to_code(int label_id) const;
+  bool pointer_link_word(int source_segment, int source_offset, int dest_segment, int dest_offset);
+  void pointer_link_split_word(int source_segment, int source_hi_offset, int source_lo_offset, int dest_segment, int dest_offset);
+  void symbol_link_word(int source_segment, int source_offset, const char* name, LinkedWord::Kind kind);
+  void symbol_link_offset(int source_segment, int source_offset, const char* name);
+  Function& get_function_at_label(int label_id);
+  std::string get_label_name(int label_id) const;
+  uint32_t set_ordered_label_names();
+  void find_code();
+  std::string print_words();
+  void find_functions();
+  void disassemble_functions();
+  void process_fp_relative_links();
+  std::string print_scripts();
+  std::string print_disassembly();
+  bool has_any_functions();
+  void append_word_to_string(std::string& dest, const LinkedWord& word) const;
+
+  struct Stats {
+    uint32_t total_code_bytes = 0;
+    uint32_t total_v2_code_bytes = 0;
+    uint32_t total_v2_pointers = 0;
+    uint32_t total_v2_pointer_seeks = 0;
+    uint32_t total_v2_link_bytes = 0;
+    uint32_t total_v2_symbol_links = 0;
+    uint32_t total_v2_symbol_count = 0;
+
+    uint32_t v3_code_bytes = 0;
+    uint32_t v3_pointers = 0;
+    uint32_t v3_split_pointers = 0;
+    uint32_t v3_word_pointers = 0;
+    uint32_t v3_pointer_seeks = 0;
+    uint32_t v3_link_bytes = 0;
+
+    uint32_t v3_symbol_count = 0;
+    uint32_t v3_symbol_link_offset = 0;
+    uint32_t v3_symbol_link_word = 0;
+
+    uint32_t data_bytes = 0;
+    uint32_t code_bytes = 0;
+
+    uint32_t function_count = 0;
+    uint32_t decoded_ops = 0;
+
+    uint32_t n_fp_reg_use = 0;
+    uint32_t n_fp_reg_use_resolved = 0;
+
+
+    void add(const Stats& other) {
+      total_code_bytes += other.total_code_bytes;
+      total_v2_code_bytes += other.total_v2_code_bytes;
+      total_v2_pointers += other.total_v2_pointers;
+      total_v2_pointer_seeks += other.total_v2_pointer_seeks;
+      total_v2_link_bytes += other.total_v2_link_bytes;
+      total_v2_symbol_links += other.total_v2_symbol_links;
+      total_v2_symbol_count += other.total_v2_symbol_count;
+      v3_code_bytes += other.v3_code_bytes;
+      v3_pointers += other.v3_pointers;
+      v3_pointer_seeks += other.v3_pointer_seeks;
+      v3_link_bytes += other.v3_link_bytes;
+      v3_word_pointers += other.v3_word_pointers;
+      v3_split_pointers += other.v3_split_pointers;
+      v3_symbol_count += other.v3_symbol_count;
+      v3_symbol_link_offset += other.v3_symbol_link_offset;
+      v3_symbol_link_word += other.v3_symbol_link_word;
+      data_bytes += other.data_bytes;
+      code_bytes += other.code_bytes;
+      function_count += other.function_count;
+      decoded_ops += other.decoded_ops;
+      n_fp_reg_use += other.n_fp_reg_use;
+      n_fp_reg_use_resolved += other.n_fp_reg_use_resolved;
+    }
+  } stats;
+
+  int segments = 0;
+  std::vector<std::vector<LinkedWord>> words_by_seg;
+  std::vector<uint32_t> offset_of_data_zone_by_seg;
+  std::vector<std::vector<Function>> functions_by_seg;
+  std::vector<Label> labels;
+
+private:
+  std::shared_ptr<Form> to_form_script(int seg, int word_idx, std::vector<bool>& seen);
+  std::shared_ptr<Form> to_form_script_object(int seg, int byte_idx, std::vector<bool> &seen);
+  bool is_empty_list(int seg, int byte_idx);
+  bool is_string(int seg, int byte_idx);
+  std::string get_goal_string(int seg, int word_idx);
+
+  std::vector<std::unordered_map<int, int>> label_per_seg_by_offset;
+};
+
+
+
+#endif //NEXT_LINKEDOBJECTFILE_H

decompiler/ObjectFile/LinkedObjectFileCreation.cpp

@@ -0,0 +1,797 @@
+/*!
+ * @file LinkedObjectFileCreation.cpp
+ * Create a LinkedObjectFile from raw object file data.
+ * This implements a decoder for the GOAL linking format.
+ */
+
+#include <cassert>
+#include <cstring>
+#include "LinkedObjectFileCreation.h"
+#include "decompiler/config.h"
+#include "decompiler/TypeSystem/TypeInfo.h"
+
+// There are three link versions:
+// V2 - not really in use anymore, but V4 will resue logic from it (and the game didn't rename the
+// functions) V3 - optimized for code and small stuff. Supports segments (main, debug, top-level) V4
+// - optimized for data (never code) and big stuff, special optimization possible for large V4
+// objects at the end of DGO.
+//   internally V4 is really just a V2, but with the link data coming after the object data.
+//   there's a V4 header at the beginning, the object data, and then a V2 header and V2 link data.
+
+// Header for link data used for V2, V3, V4 objects.  For V3/V4, this is found at the beginning of
+// the object data.
+struct LinkHeaderCommon {
+  uint32_t type_tag;  // for the basic offset, is 0 or -1 depending on version
+  uint32_t length;    // different exact meanings, but length of the link data.
+  uint16_t version;   // what version (2, 3, 4)
+};
+
+// Header for link data used for V2 linking data
+struct LinkHeaderV2 {
+  uint32_t type_tag;  // always -1
+  uint32_t length;    // length of link data
+  uint32_t version;   // always 2
+};
+
+// Header for link data used for V4
+struct LinkHeaderV4 {
+  uint32_t type_tag;   // always -1
+  uint32_t length;     // length of V2 link data found after object.
+  uint32_t version;    // always 4
+  uint32_t code_size;  // length of object data before link data starts
+};
+
+// Per-segment info for V3 and V5 link data
+struct SegmentInfo {
+  uint32_t relocs;  // offset of relocation table
+  uint32_t data;    // offset of segment data
+  uint32_t size;    // segment data size (0 if segment doesn't exist)
+  uint32_t magic;   // always 0
+};
+
+struct LinkHeaderV3 {
+  uint32_t type_tag;  // always 0
+  uint32_t length;    // length of link data
+  uint32_t version;   // always 3
+  uint32_t segments;  // always 3
+  char name[64];      // name of object file
+  SegmentInfo segment_info[3];
+};
+
+struct LinkHeaderV5 {
+  uint32_t type_tag;               // 0 always 0?
+  uint32_t length_to_get_to_code;  // 4 length.. of link data?
+  uint16_t version;                // 8
+  uint16_t unknown;                // 10
+  uint32_t pad;                    // 12
+  uint32_t link_length;            // 16
+  uint8_t n_segments;              // 20
+  char name[59];                   // 21 (really??)
+  SegmentInfo segment_info[3];
+};
+
+// The types of symbol links
+enum class SymbolLinkKind {
+  EMPTY_LIST,  // link to the empty list
+  TYPE,        // link to a type
+  SYMBOL       // link to a symbol
+};
+
+/*!
+ * Handle symbol links for a single symbol in a V2/V4 object file.
+ */
+static uint32_t c_symlink2(LinkedObjectFile& f,
+                           const std::vector<uint8_t>& data,
+                           uint32_t code_ptr_offset,
+                           uint32_t link_ptr_offset,
+                           SymbolLinkKind kind,
+                           const char* name,
+                           int seg_id) {
+  get_type_info().inform_symbol_with_no_type_info(name);
+  auto initial_offset = code_ptr_offset;
+  do {
+    auto table_value = data.at(link_ptr_offset);
+    const uint8_t* relocPtr = &data.at(link_ptr_offset);
+
+    // link table has a series of variable-length-encoded integers indicating the seek amount to hit
+    // each reference to the symbol.  It ends when the seek is 0, and all references to this symbol
+    // have been patched.
+    uint32_t seek = table_value;
+    uint32_t next_reloc = link_ptr_offset + 1;
+
+    if (seek & 3) {
+      seek = (relocPtr[1] << 8) | table_value;
+      next_reloc = link_ptr_offset + 2;
+      if (seek & 2) {
+        seek = (relocPtr[2] << 16) | seek;
+        next_reloc = link_ptr_offset + 3;
+        if (seek & 1) {
+          seek = (relocPtr[3] << 24) | seek;
+          next_reloc = link_ptr_offset + 4;
+        }
+      }
+    }
+
+    f.stats.total_v2_symbol_links++;
+    link_ptr_offset = next_reloc;
+
+    code_ptr_offset += (seek & 0xfffffffc);
+
+    // the value of the code gives us more information
+    uint32_t code_value = *(const uint32_t*)(&data.at(code_ptr_offset));
+    if (code_value == 0xffffffff) {
+      // absolute link - replace entire word with a pointer.
+      LinkedWord::Kind word_kind;
+      switch (kind) {
+        case SymbolLinkKind::SYMBOL:
+          word_kind = LinkedWord::SYM_PTR;
+          break;
+        case SymbolLinkKind::EMPTY_LIST:
+          word_kind = LinkedWord::EMPTY_PTR;
+          break;
+        case SymbolLinkKind::TYPE:
+          get_type_info().inform_type(name);
+          word_kind = LinkedWord::TYPE_PTR;
+          break;
+        default:
+          throw std::runtime_error("unhandled SymbolLinkKind");
+      }
+
+      f.symbol_link_word(seg_id, code_ptr_offset - initial_offset, name, word_kind);
+    } else {
+      // offset link - replace lower 16 bits with symbol table offset.
+
+      assert((code_value & 0xffff) == 0 || (code_value & 0xffff) == 0xffff);
+      assert(kind == SymbolLinkKind::SYMBOL);
+      //      assert(false); // this case does not occur in V2/V4.  It does in V3.
+      f.symbol_link_offset(seg_id, code_ptr_offset - initial_offset, name);
+    }
+
+  } while (data.at(link_ptr_offset));
+
+  // seek past terminating 0.
+  return link_ptr_offset + 1;
+}
+
+/*!
+ * Handle symbol links for a single symbol in a V3 object file.
+ */
+static uint32_t c_symlink3(LinkedObjectFile& f,
+                           const std::vector<uint8_t>& data,
+                           uint32_t code_ptr,
+                           uint32_t link_ptr,
+                           SymbolLinkKind kind,
+                           const char* name,
+                           int seg) {
+  get_type_info().inform_symbol_with_no_type_info(name);
+  auto initial_offset = code_ptr;
+  do {
+    // seek, with a variable length encoding that sucks.
+    uint8_t c;
+    do {
+      c = data.at(link_ptr);
+      link_ptr++;
+      code_ptr += c * 4;
+    } while (c == 0xff);
+
+    // identical logic to symlink 2
+    uint32_t code_value = *(const uint32_t*)(&data.at(code_ptr));
+    if (code_value == 0xffffffff) {
+      f.stats.v3_symbol_link_word++;
+      LinkedWord::Kind word_kind;
+      switch (kind) {
+        case SymbolLinkKind::SYMBOL:
+          word_kind = LinkedWord::SYM_PTR;
+          break;
+        case SymbolLinkKind::EMPTY_LIST:
+          word_kind = LinkedWord::EMPTY_PTR;
+          break;
+        case SymbolLinkKind::TYPE:
+          get_type_info().inform_type(name);
+          word_kind = LinkedWord::TYPE_PTR;
+          break;
+        default:
+          throw std::runtime_error("unhandled SymbolLinkKind");
+      }
+
+      f.symbol_link_word(seg, code_ptr - initial_offset, name, word_kind);
+    } else {
+      f.stats.v3_symbol_link_offset++;
+      assert(kind == SymbolLinkKind::SYMBOL);
+      f.symbol_link_offset(seg, code_ptr - initial_offset, name);
+    }
+
+  } while (data.at(link_ptr));
+  return link_ptr + 1;
+}
+
+static uint32_t align64(uint32_t in) {
+  return (in + 63) & (~63);
+}
+
+static uint32_t align16(uint32_t in) {
+  return (in + 15) & (~15);
+}
+
+
+/*!
+ * Process link data for a "V4" object file.
+ * In reality a V4 seems to be just a V2 object, but with the link data after the real data.
+ * There's a V4 header at the very beginning, but another V2 header/link data at the end
+ * -----------------------------------------------
+ * | V4 header | data | V2 header | V2 link data |
+ * -----------------------------------------------
+ */
+static void link_v4(LinkedObjectFile& f,
+                    const std::vector<uint8_t>& data,
+                    const std::string& name) {
+  // read the V4 header to find where the link data really is
+  const auto* header = (const LinkHeaderV4*)&data.at(0);
+  uint32_t link_data_offset = header->code_size + sizeof(LinkHeaderV4);  // no basic offset
+
+  // code starts immediately after the header
+  uint32_t code_offset = sizeof(LinkHeaderV4);
+  uint32_t code_size = header->code_size;
+
+  f.stats.total_code_bytes += code_size;
+  f.stats.total_v2_code_bytes += code_size;
+
+  // add all code
+  const uint8_t* code_start = &data.at(code_offset);
+  const uint8_t* code_end =
+      &data.at(code_offset + code_size);  // safe because link data is after code.
+  assert(((code_end - code_start) % 4) == 0);
+  f.set_segment_count(1);
+  for (auto x = code_start; x < code_end; x += 4) {
+    f.push_back_word_to_segment(*((const uint32_t*)x), 0);
+  }
+
+  // read v2 header after the code
+  const uint8_t* link_data = &data.at(link_data_offset);
+  const auto* link_header_v2 = (const LinkHeaderV2*)(link_data);  // subtract off type tag
+  assert(link_header_v2->type_tag == 0xffffffff);
+  assert(link_header_v2->version == 2);
+  assert(link_header_v2->length == header->length);
+  f.stats.total_v2_link_bytes += link_header_v2->length;
+  uint32_t link_ptr_offset = link_data_offset + sizeof(LinkHeaderV2);
+
+  // first "section" of link data is a list of where all the pointer are.
+  if (data.at(link_ptr_offset) == 0) {
+    // there are no pointers.
+    link_ptr_offset++;
+  } else {
+    // there are pointers.
+    // there are a series of variable-length coded integers, indicating where the pointers are, in
+    // the form: seek_amount, number_of_consecutive_pointers, seek_amount,
+    // number_of_consecutive_pointers, ... , 0
+
+    uint32_t code_ptr_offset = code_offset;
+    bool fixing = false;  // either seeking or fixing
+
+    while (true) {    // loop over entire table
+      while (true) {  // loop over current mode (fixing/seeking)
+        // get count from table
+        auto count = data.at(link_ptr_offset);
+        link_ptr_offset++;
+
+        if (!fixing) {
+          // then we are seeking
+          code_ptr_offset += 4 * count;
+          f.stats.total_v2_pointer_seeks++;
+        } else {
+          // then we are fixing consecutive pointers
+          for (uint8_t i = 0; i < count; i++) {
+            if (!f.pointer_link_word(0, code_ptr_offset - code_offset, 0,
+                                     *((const uint32_t*)(&data.at(code_ptr_offset))))) {
+              printf("WARNING bad link in %s\n", name.c_str());
+            }
+            f.stats.total_v2_pointers++;
+            code_ptr_offset += 4;
+          }
+        }
+
+        // check if we are done with the current integer
+        if (count != 0xff)
+          break;
+
+        // when we "end" an encoded integer on an 0xff, we need an explicit zero byte to change
+        // modes. this handles this special case.
+        if (data.at(link_ptr_offset) == 0) {
+          link_ptr_offset++;
+          fixing = !fixing;
+        }
+      }
+
+      // mode ended, switch
+      fixing = !fixing;
+
+      // we got a zero, that means we're done with pointer fixing.
+      if (data.at(link_ptr_offset) == 0)
+        break;
+    }
+    link_ptr_offset++;
+  }
+
+  // second "section" of link data is a list of symbols to fix up.
+  if (data.at(link_ptr_offset) == 0) {
+    // no symbols
+  } else {
+    while (true) {
+      uint32_t reloc = data.at(link_ptr_offset);
+      link_ptr_offset++;
+
+      const char* s_name;
+      SymbolLinkKind kind;
+
+      if ((reloc & 0x80) == 0) {
+        // it's a symbol
+        if (reloc > 9) {
+          // always happens.
+          link_ptr_offset--;
+        } else {
+          assert(false);
+        }
+
+        s_name = (const char*)(&data.at(link_ptr_offset));
+        kind = SymbolLinkKind::SYMBOL;
+
+      } else {
+        // it's a type
+        kind = SymbolLinkKind::TYPE;
+        uint8_t method_count = reloc & 0x7f;
+        s_name = (const char*)(&data.at(link_ptr_offset));
+        if (method_count == 0) {
+          method_count = 1;
+          // hack which will add 44 methods to _newly created_ types
+          // I assume the thing generating V2 objects didn't know about method counts.
+          // so this was a "safe" backup - if linking a V2 object requires allocating a type.
+          // just be on the safe side.
+          // (see the !symbolValue case in intern_type_from_c)
+        } else {
+          assert(false);
+        }
+      }
+
+      if (std::string("_empty_") == s_name) {
+        assert(kind == SymbolLinkKind::SYMBOL);
+        kind = SymbolLinkKind::EMPTY_LIST;
+      }
+
+      link_ptr_offset += strlen(s_name) + 1;
+      f.stats.total_v2_symbol_count++;
+      link_ptr_offset = c_symlink2(f, data, code_offset, link_ptr_offset, kind, s_name, 0);
+      if (data.at(link_ptr_offset) == 0)
+        break;
+    }
+  }
+
+  // check length
+  assert(link_header_v2->length == align64(link_ptr_offset - link_data_offset + 1));
+  while (link_ptr_offset < data.size()) {
+    assert(data.at(link_ptr_offset) == 0);
+    link_ptr_offset++;
+  }
+}
+
+static void assert_string_empty_after(const char* str, int size) {
+  auto ptr = str;
+  while (*ptr)
+    ptr++;
+  while (ptr - str < size) {
+    assert(!*ptr);
+    ptr++;
+  }
+}
+
+static void link_v5(LinkedObjectFile& f,
+                    const std::vector<uint8_t>& data,
+                    const std::string& name) {
+  auto header = (const LinkHeaderV5*)(&data.at(0));
+  if (header->n_segments == 1) {
+    printf("abandon %s!\n", name.c_str());
+    return;
+  }
+  assert(header->type_tag == 0);
+  assert(name == header->name);
+  assert(header->n_segments == 3);
+  assert(header->pad == 0x50);
+  assert(header->length_to_get_to_code - header->link_length == 0x50);
+
+  f.set_segment_count(3);
+
+  // link v3's data size is data.size() - link_length
+  // link v5's data size is data.size() - new_link_length - 0x50.
+
+  // lbp + 4 points to version?
+  // lbp points to 4 past start of header.
+
+  // lbp[1] = version + unknown 16 bit thing.
+  // lbp[3] = link block length (minus 0x50)
+
+  // todo - check this against the code size we actually got.
+  //  size_t expected_code_size = data.size() - (header->link_length + 0x50);
+
+  uint32_t data_ptr_offset = header->length_to_get_to_code;
+
+  uint32_t segment_data_offsets[3];
+  uint32_t segment_link_offsets[3];
+  uint32_t segment_link_ends[3];
+  for (int i = 0; i < 3; i++) {
+    segment_data_offsets[i] = data_ptr_offset + header->segment_info[i].data;
+    segment_link_offsets[i] = header->segment_info[i].relocs + 0x50;
+    assert(header->segment_info[i].magic == 1);
+  }
+
+  // check that the data region is filled
+  for (int i = 0; i < 2; i++) {
+    assert(align16(segment_data_offsets[i] + header->segment_info[i].size) ==
+           segment_data_offsets[i + 1]);
+  }
+  assert(align16(segment_data_offsets[2] + header->segment_info[2].size) == data.size());
+
+  // loop over segments (reverse order for now)
+  for (int seg_id = 3; seg_id-- > 0;) {
+    // ?? is this right?
+    if (header->segment_info[seg_id].size == 0)
+      continue;
+
+    auto segment_size = header->segment_info[seg_id].size;
+    f.stats.v3_code_bytes += segment_size;
+
+    //    if(gGameVersion == JAK2) {
+    bool adjusted = false;
+    while (segment_size % 4) {
+      segment_size++;
+      adjusted = true;
+    }
+
+    if (adjusted) {
+      printf(
+          "Adjusted the size of segment %d in %s, this is fine, but rare (and may indicate a "
+          "bigger problem if it happens often)\n",
+          seg_id, name.c_str());
+    }
+    //    }
+
+    auto base_ptr = segment_data_offsets[seg_id];
+    auto data_ptr = base_ptr - 4;
+    auto link_ptr = segment_link_offsets[seg_id];
+
+    assert((data_ptr % 4) == 0);
+    assert((segment_size % 4) == 0);
+
+    auto code_start = (const uint32_t*)(&data.at(data_ptr + 4));
+    auto code_end = ((const uint32_t*)(&data.at(data_ptr + segment_size))) + 1;
+    for (auto x = code_start; x < code_end; x++) {
+      f.push_back_word_to_segment(*((const uint32_t*)x), seg_id);
+    }
+    bool fixing = false;
+
+    if (data.at(link_ptr)) {
+      // we have pointers
+      while (true) {
+        while (true) {
+          if (!fixing) {
+            // seeking
+            data_ptr += 4 * data.at(link_ptr);
+            f.stats.v3_pointer_seeks++;
+          } else {
+            // fixing.
+            for (uint32_t i = 0; i < data.at(link_ptr); i++) {
+              f.stats.v3_pointers++;
+              uint32_t old_code = *(const uint32_t*)(&data.at(data_ptr));
+              if ((old_code >> 24) == 0) {
+                f.stats.v3_word_pointers++;
+                if (!f.pointer_link_word(seg_id, data_ptr - base_ptr, seg_id, old_code)) {
+                  printf("WARNING bad pointer_link_word (2) in %s\n", name.c_str());
+                }
+              } else {
+                f.stats.v3_split_pointers++;
+                auto dest_seg = (old_code >> 8) & 0xf;
+                auto lo_hi_offset = (old_code >> 12) & 0xf;
+                assert(lo_hi_offset);
+                assert(dest_seg < 3);
+                auto offset_upper = old_code & 0xff;
+                //                assert(offset_upper == 0);
+                uint32_t low_code = *(const uint32_t*)(&data.at(data_ptr + 4 * lo_hi_offset));
+                uint32_t offset = low_code & 0xffff;
+                if (offset_upper) {
+                  // seems to work fine, no need to warn.
+                  //                  printf("WARNING - offset upper is set in %s\n", name.c_str());
+                  offset += (offset_upper << 16);
+                }
+                f.pointer_link_split_word(seg_id, data_ptr - base_ptr,
+                                          data_ptr + 4 * lo_hi_offset - base_ptr, dest_seg, offset);
+              }
+              data_ptr += 4;
+            }
+          }
+
+          if (data.at(link_ptr) != 0xff)
+            break;
+          link_ptr++;
+          if (data.at(link_ptr) == 0) {
+            link_ptr++;
+            fixing = !fixing;
+          }
+        }
+
+        link_ptr++;
+        fixing = !fixing;
+        if (data.at(link_ptr) == 0)
+          break;
+      }
+    }
+    link_ptr++;
+
+    if (data.at(link_ptr)) {
+      auto sub_link_ptr = link_ptr;
+
+      while (true) {
+        auto reloc = data.at(sub_link_ptr);
+        auto next_link_ptr = sub_link_ptr + 1;
+        link_ptr = next_link_ptr;
+
+        if ((reloc & 0x80) == 0) {
+          link_ptr = sub_link_ptr + 3;  //
+          const char* sname = (const char*)(&data.at(link_ptr));
+          link_ptr += strlen(sname) + 1;
+          // todo segment data offsets...
+
+          if (std::string("_empty_") == sname) {
+            link_ptr = c_symlink2(f, data, segment_data_offsets[seg_id], link_ptr,
+                                  SymbolLinkKind::EMPTY_LIST, sname, seg_id);
+          } else {
+            link_ptr = c_symlink2(f, data, segment_data_offsets[seg_id], link_ptr,
+                                  SymbolLinkKind::SYMBOL, sname, seg_id);
+          }
+        } else if ((reloc & 0x3f) == 0x3f) {
+          assert(false);  // todo, does this ever get hit?
+        } else {
+          int n_methods_base = reloc & 0x3f;
+          int n_methods = n_methods_base * 4;
+          if (n_methods_base) {
+            n_methods += 3;
+          }
+          link_ptr += 2;  // ghidra misses some aliasing here and would have you think this is +1!
+          const char* sname = (const char*)(&data.at(link_ptr));
+          link_ptr += strlen(sname) + 1;
+          link_ptr = c_symlink2(f, data, segment_data_offsets[seg_id], link_ptr,
+                                SymbolLinkKind::TYPE, sname, seg_id);
+        }
+
+        sub_link_ptr = link_ptr;
+        if (!data.at(sub_link_ptr))
+          break;
+      }
+    }
+    segment_link_ends[seg_id] = link_ptr;
+  }
+
+  assert(segment_link_offsets[0] == 128);
+
+  if (header->segment_info[0].size) {
+    assert(segment_link_ends[0] + 1 == segment_link_offsets[1]);
+  } else {
+    assert(segment_link_offsets[0] + 2 == segment_link_offsets[1]);
+  }
+
+  if (header->segment_info[1].size) {
+    assert(segment_link_ends[1] + 1 == segment_link_offsets[2]);
+  } else {
+    assert(segment_link_offsets[1] + 2 == segment_link_offsets[2]);
+  }
+
+  assert(align16(segment_link_ends[2] + 2) == segment_data_offsets[0]);
+}
+
+static void link_v3(LinkedObjectFile& f,
+                    const std::vector<uint8_t>& data,
+                    const std::string& name) {
+  auto header = (const LinkHeaderV3*)(&data.at(0));
+  assert(name == header->name);
+  assert(header->segments == 3);
+
+  f.set_segment_count(3);
+  assert_string_empty_after(header->name, 64);
+
+  for (int i = 0; i < 3; i++) {
+    assert(header->segment_info[i].magic == 0);
+    //    printf(" [%d] %d %d %d %d\n", i, header->segment_info[i].size,
+    //    header->segment_info[i].data, header->segment_info[i].magic,
+    //    header->segment_info[i].relocs);
+  }
+
+  f.stats.v3_link_bytes += header->length;
+  uint32_t data_ptr_offset = header->length;
+
+  uint32_t segment_data_offsets[3];
+  uint32_t segment_link_offsets[3];
+  uint32_t segment_link_ends[3];
+  for (int i = 0; i < 3; i++) {
+    segment_data_offsets[i] = data_ptr_offset + header->segment_info[i].data;
+    segment_link_offsets[i] = header->segment_info[i].relocs;
+  }
+
+  // check that the data region is filled
+  for (int i = 0; i < 2; i++) {
+    assert(align16(segment_data_offsets[i] + header->segment_info[i].size) ==
+           segment_data_offsets[i + 1]);
+  }
+  assert(align16(segment_data_offsets[2] + header->segment_info[2].size) == data.size());
+
+  // todo - check link region is filled.
+
+  // loop over segments (reverse order for now)
+  for (int seg_id = 3; seg_id-- > 0;) {
+    // ?? is this right?
+    if (header->segment_info[seg_id].size == 0)
+      continue;
+
+    auto segment_size = header->segment_info[seg_id].size;
+    f.stats.v3_code_bytes += segment_size;
+
+    // HACK!
+    // why is this a thing?
+    // HACK!
+    if (get_config().game_version == 1 && name == "level-h" && seg_id == 0) {
+      segment_size++;
+    }
+
+    if (get_config().game_version == 2) {
+      bool adjusted = false;
+      while (segment_size % 4) {
+        segment_size++;
+        adjusted = true;
+      }
+
+      if (adjusted) {
+        printf(
+            "Adjusted the size of segment %d in %s, this is fine, but rare (and may indicate a "
+            "bigger problem if it happens often)\n",
+            seg_id, name.c_str());
+      }
+    }
+
+    auto base_ptr = segment_data_offsets[seg_id];
+    auto data_ptr = base_ptr - 4;
+    auto link_ptr = segment_link_offsets[seg_id];
+
+    assert((data_ptr % 4) == 0);
+    assert((segment_size % 4) == 0);
+
+    auto code_start = (const uint32_t*)(&data.at(data_ptr + 4));
+    auto code_end = ((const uint32_t*)(&data.at(data_ptr + segment_size))) + 1;
+    for (auto x = code_start; x < code_end; x++) {
+      f.push_back_word_to_segment(*((const uint32_t*)x), seg_id);
+    }
+    bool fixing = false;
+
+    if (data.at(link_ptr)) {
+      // we have pointers
+      while (true) {
+        while (true) {
+          if (!fixing) {
+            // seeking
+            data_ptr += 4 * data.at(link_ptr);
+            f.stats.v3_pointer_seeks++;
+          } else {
+            // fixing.
+            for (uint32_t i = 0; i < data.at(link_ptr); i++) {
+              f.stats.v3_pointers++;
+              uint32_t old_code = *(const uint32_t*)(&data.at(data_ptr));
+              if ((old_code >> 24) == 0) {
+                f.stats.v3_word_pointers++;
+                if (!f.pointer_link_word(seg_id, data_ptr - base_ptr, seg_id, old_code)) {
+                  printf("WARNING bad pointer_link_word (2) in %s\n", name.c_str());
+                }
+              } else {
+                f.stats.v3_split_pointers++;
+                auto dest_seg = (old_code >> 8) & 0xf;
+                auto lo_hi_offset = (old_code >> 12) & 0xf;
+                assert(lo_hi_offset);
+                assert(dest_seg < 3);
+                auto offset_upper = old_code & 0xff;
+                //                assert(offset_upper == 0);
+                uint32_t low_code = *(const uint32_t*)(&data.at(data_ptr + 4 * lo_hi_offset));
+                uint32_t offset = low_code & 0xffff;
+                if (offset_upper) {
+                  // seems to work fine, no need to warn.
+                  //                  printf("WARNING - offset upper is set in %s\n", name.c_str());
+                  offset += (offset_upper << 16);
+                }
+                f.pointer_link_split_word(seg_id, data_ptr - base_ptr,
+                                          data_ptr + 4 * lo_hi_offset - base_ptr, dest_seg, offset);
+              }
+              data_ptr += 4;
+            }
+          }
+
+          if (data.at(link_ptr) != 0xff)
+            break;
+          link_ptr++;
+          if (data.at(link_ptr) == 0) {
+            link_ptr++;
+            fixing = !fixing;
+          }
+        }
+
+        link_ptr++;
+        fixing = !fixing;
+        if (data.at(link_ptr) == 0)
+          break;
+      }
+    }
+
+    link_ptr++;
+
+    while (data.at(link_ptr)) {
+      auto reloc = data.at(link_ptr);
+      SymbolLinkKind kind;
+      link_ptr++;
+
+      const char* s_name = nullptr;
+      if ((reloc & 0x80) == 0) {
+        // it's a symbol
+        kind = SymbolLinkKind::SYMBOL;
+        link_ptr--;
+        s_name = (const char*)(&data.at(link_ptr));
+      } else {
+        // methods todo
+
+        s_name = (const char*)(&data.at(link_ptr));
+        get_type_info().inform_type_method_count(s_name, reloc & 0x7f);
+        kind = SymbolLinkKind::TYPE;
+      }
+
+      if (std::string("_empty_") == s_name) {
+        assert(kind == SymbolLinkKind::SYMBOL);
+        kind = SymbolLinkKind::EMPTY_LIST;
+      }
+
+      link_ptr += strlen(s_name) + 1;
+      f.stats.v3_symbol_count++;
+      link_ptr = c_symlink3(f, data, base_ptr, link_ptr, kind, s_name, seg_id);
+    }
+    segment_link_ends[seg_id] = link_ptr;
+  }
+
+  assert(segment_link_offsets[0] == 128);
+
+  if (header->segment_info[0].size) {
+    assert(segment_link_ends[0] + 1 == segment_link_offsets[1]);
+  } else {
+    assert(segment_link_offsets[0] + 2 == segment_link_offsets[1]);
+  }
+
+  if (header->segment_info[1].size) {
+    assert(segment_link_ends[1] + 1 == segment_link_offsets[2]);
+  } else {
+    assert(segment_link_offsets[1] + 2 == segment_link_offsets[2]);
+  }
+
+  assert(align16(segment_link_ends[2] + 2) == segment_data_offsets[0]);
+}
+
+/*!
+ * Main function to generate LinkedObjectFiles from raw object data.
+ */
+LinkedObjectFile to_linked_object_file(const std::vector<uint8_t>& data, const std::string& name) {
+  LinkedObjectFile result;
+  const auto* header = (const LinkHeaderCommon*)&data.at(0);
+
+  // use appropriate linker
+  if (header->version == 3) {
+    assert(header->type_tag == 0);
+    link_v3(result, data, name);
+  } else if (header->version == 4) {
+    assert(header->type_tag == 0xffffffff);
+    link_v4(result, data, name);
+  } else if (header->version == 5) {
+    link_v5(result, data, name);
+  } else {
+    assert(false);
+  }
+
+  return result;
+}

decompiler/ObjectFile/LinkedObjectFileCreation.h

@@ -0,0 +1,14 @@
+/*!
+ * @file LinkedObjectFileCreation.h
+ * Create a LinkedObjectFile from raw object file data.
+ * This implements a decoder for the GOAL linking format.
+ */
+
+#ifndef NEXT_LINKEDOBJECTFILECREATION_H
+#define NEXT_LINKEDOBJECTFILECREATION_H
+
+#include "LinkedObjectFile.h"
+
+LinkedObjectFile to_linked_object_file(const std::vector<uint8_t>& data, const std::string& name);
+
+#endif //NEXT_LINKEDOBJECTFILECREATION_H

decompiler/ObjectFile/LinkedWord.h

@@ -0,0 +1,33 @@
+/*!
+ * @file LinkedWord.h
+ * A word (4 bytes), possibly with some linking info.
+ */
+
+#ifndef JAK2_DISASSEMBLER_LINKEDWORD_H
+#define JAK2_DISASSEMBLER_LINKEDWORD_H
+
+#include <cstdint>
+#include <string>
+
+class LinkedWord {
+ public:
+  explicit LinkedWord(uint32_t _data) : data(_data) {}
+
+  enum Kind {
+    PLAIN_DATA,  // just plain data
+    PTR,         // pointer to a location
+    HI_PTR,      // lower 16-bits of this data are the upper 16 bits of a pointer
+    LO_PTR,      // lower 16-bits of this data are the lower 16 bits of a pointer
+    SYM_PTR,     // this is a pointer to a symbol
+    EMPTY_PTR,   // this is a pointer to the empty list
+    SYM_OFFSET,  // this is an offset of a symbol in the symbol table
+    TYPE_PTR     // this is a pointer to a type
+  } kind = PLAIN_DATA;
+
+  uint32_t data = 0;
+
+  int label_id = -1;
+  std::string symbol_name;
+};
+
+#endif  // JAK2_DISASSEMBLER_LINKEDWORD_H

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -0,0 +1,512 @@
+/*!
+ * @file ObjectFileDB.cpp
+ * A "database" of object files found in DGO files.
+ * Eliminates duplicate object files, and also assigns unique names to all object files
+ * (there may be different object files with the same name sometimes)
+ */
+
+#include "ObjectFileDB.h"
+#include <algorithm>
+#include <cstring>
+#include <map>
+#include "LinkedObjectFileCreation.h"
+#include "decompiler/config.h"
+#include "third-party/minilzo/minilzo.h"
+#include "decompiler/util/BinaryReader.h"
+#include "decompiler/util/FileIO.h"
+#include "decompiler/util/Timer.h"
+#include "decompiler/Function/BasicBlocks.h"
+
+/*!
+ * Get a unique name for this object file.
+ */
+std::string ObjectFileRecord::to_unique_name() const {
+  return name + "-v" + std::to_string(version);
+}
+
+/*!
+ * Build an object file DB for the given list of DGOs.
+ */
+ObjectFileDB::ObjectFileDB(const std::vector<std::string>& _dgos) {
+  Timer timer;
+
+  printf("- Initializing ObjectFileDB...\n");
+  for (auto& dgo : _dgos) {
+    get_objs_from_dgo(dgo);
+  }
+
+  printf("ObjectFileDB Initialized:\n");
+  printf(" total dgos: %ld\n", _dgos.size());
+  printf(" total data: %d bytes\n", stats.total_dgo_bytes);
+  printf(" total objs: %d\n", stats.total_obj_files);
+  printf(" unique objs: %d\n", stats.unique_obj_files);
+  printf(" unique data: %d bytes\n", stats.unique_obj_bytes);
+  printf(" total %.1f ms (%.3f MB/sec, %.3f obj/sec)\n", timer.getMs(),
+         stats.total_dgo_bytes / ((1u << 20u) * timer.getSeconds()),
+         stats.total_obj_files / timer.getSeconds());
+  printf("\n");
+}
+
+// Header for a DGO file
+struct DgoHeader {
+  uint32_t size;
+  char name[60];
+};
+
+namespace {
+/*!
+ * Assert false if the char[] has non-null data after the null terminated string.
+ * Used to sanity check the sizes of strings in DGO/object file headers.
+ */
+void assert_string_empty_after(const char* str, int size) {
+  auto ptr = str;
+  while (*ptr)
+    ptr++;
+  while (ptr - str < size) {
+    assert(!*ptr);
+    ptr++;
+  }
+}
+}  // namespace
+
+constexpr int MAX_CHUNK_SIZE = 0x8000;
+/*!
+ * Load the objects stored in the given DGO into the ObjectFileDB
+ */
+void ObjectFileDB::get_objs_from_dgo(const std::string& filename) {
+  auto dgo_data = read_binary_file(filename);
+  stats.total_dgo_bytes += dgo_data.size();
+
+  const char jak2_header[] = "oZlB";
+  bool is_jak2 = true;
+  for (int i = 0; i < 4; i++) {
+    if (jak2_header[i] != dgo_data[i]) {
+      is_jak2 = false;
+    }
+  }
+
+  if (is_jak2) {
+    if (lzo_init() != LZO_E_OK) {
+      assert(false);
+    }
+    BinaryReader compressed_reader(dgo_data);
+    // seek past oZlB
+    compressed_reader.ffwd(4);
+    auto decompressed_size = compressed_reader.read<uint32_t>();
+    std::vector<uint8_t> decompressed_data;
+    decompressed_data.resize(decompressed_size);
+    size_t output_offset = 0;
+    while (true) {
+      // seek past alignment bytes and read the next chunk size
+      uint32_t chunk_size = 0;
+      while (!chunk_size) {
+        chunk_size = compressed_reader.read<uint32_t>();
+      }
+
+      if (chunk_size < MAX_CHUNK_SIZE) {
+        lzo_uint bytes_written;
+        auto lzo_rv =
+            lzo1x_decompress(compressed_reader.here(), chunk_size,
+                             decompressed_data.data() + output_offset, &bytes_written, nullptr);
+        assert(lzo_rv == LZO_E_OK);
+        compressed_reader.ffwd(chunk_size);
+        output_offset += bytes_written;
+      } else {
+        // nope - sometimes chunk_size is bigger than MAX, but we should still use max.
+        //        assert(chunk_size == MAX_CHUNK_SIZE);
+        memcpy(decompressed_data.data() + output_offset, compressed_reader.here(), MAX_CHUNK_SIZE);
+        compressed_reader.ffwd(MAX_CHUNK_SIZE);
+        output_offset += MAX_CHUNK_SIZE;
+      }
+
+      if (output_offset >= decompressed_size)
+        break;
+      while (compressed_reader.get_seek() % 4) {
+        compressed_reader.ffwd(1);
+      }
+    }
+    dgo_data = decompressed_data;
+  }
+
+  BinaryReader reader(dgo_data);
+  auto header = reader.read<DgoHeader>();
+
+  auto dgo_base_name = base_name(filename);
+  assert(header.name == dgo_base_name);
+  assert_string_empty_after(header.name, 60);
+
+  // get all obj files...
+  for (uint32_t i = 0; i < header.size; i++) {
+    auto obj_header = reader.read<DgoHeader>();
+    assert(reader.bytes_left() >= obj_header.size);
+    assert_string_empty_after(obj_header.name, 60);
+
+    add_obj_from_dgo(obj_header.name, reader.here(), obj_header.size, dgo_base_name);
+    reader.ffwd(obj_header.size);
+  }
+
+  // check we're at the end
+  assert(0 == reader.bytes_left());
+}
+
+/*!
+ * Add an object file to the ObjectFileDB
+ */
+void ObjectFileDB::add_obj_from_dgo(const std::string& obj_name,
+                                    uint8_t* obj_data,
+                                    uint32_t obj_size,
+                                    const std::string& dgo_name) {
+  stats.total_obj_files++;
+
+  auto hash = crc32(obj_data, obj_size);
+
+  // first, check to see if we already got it...
+  for (auto& e : obj_files_by_name[obj_name]) {
+    if (e.data.size() == obj_size && e.record.hash == hash) {
+      // already got it!
+      e.reference_count++;
+      auto rec = e.record;
+      obj_files_by_dgo[dgo_name].push_back(rec);
+      return;
+    }
+  }
+
+  // nope, have to add a new one.
+  ObjectFileData data;
+  data.data.resize(obj_size);
+  memcpy(data.data.data(), obj_data, obj_size);
+  data.record.hash = hash;
+  data.record.name = obj_name;
+  if (obj_files_by_name[obj_name].empty()) {
+    // if this is the first time we've seen this object file name, add it in the order.
+    obj_file_order.push_back(obj_name);
+  }
+  data.record.version = obj_files_by_name[obj_name].size();
+  obj_files_by_dgo[dgo_name].push_back(data.record);
+  obj_files_by_name[obj_name].emplace_back(std::move(data));
+  stats.unique_obj_files++;
+  stats.unique_obj_bytes += obj_size;
+}
+
+/*!
+ * Generate a listing of what object files go in which dgos
+ */
+std::string ObjectFileDB::generate_dgo_listing() {
+  std::string result = ";; DGO File Listing\n\n";
+  std::vector<std::string> dgo_names;
+  for (auto& kv : obj_files_by_dgo) {
+    dgo_names.push_back(kv.first);
+  }
+
+  std::sort(dgo_names.begin(), dgo_names.end());
+
+  for (const auto& name : dgo_names) {
+    result += "(\"" + name + "\"\n";
+    for (auto& obj : obj_files_by_dgo[name]) {
+      result += "  " + obj.name + " :version " + std::to_string(obj.version) + "\n";
+    }
+    result += "  )\n\n";
+  }
+
+  return result;
+}
+
+/*!
+ * Process all of the linking data of all objects.
+ */
+void ObjectFileDB::process_link_data() {
+  printf("- Processing Link Data...\n");
+  Timer process_link_timer;
+
+  LinkedObjectFile::Stats combined_stats;
+
+  for_each_obj([&](ObjectFileData& obj) {
+    obj.linked_data = to_linked_object_file(obj.data, obj.record.name);
+    combined_stats.add(obj.linked_data.stats);
+  });
+
+  printf("Processed Link Data:\n");
+  printf(" code %d bytes\n", combined_stats.total_code_bytes);
+  printf(" v2 code %d bytes\n", combined_stats.total_v2_code_bytes);
+  printf(" v2 link data %d bytes\n", combined_stats.total_v2_link_bytes);
+  printf(" v2 pointers %d\n", combined_stats.total_v2_pointers);
+  printf(" v2 pointer seeks %d\n", combined_stats.total_v2_pointer_seeks);
+  printf(" v2 symbols %d\n", combined_stats.total_v2_symbol_count);
+  printf(" v2 symbol links %d\n", combined_stats.total_v2_symbol_links);
+
+  printf(" v3 code %d bytes\n", combined_stats.v3_code_bytes);
+  printf(" v3 link data %d bytes\n", combined_stats.v3_link_bytes);
+  printf(" v3 pointers %d\n", combined_stats.v3_pointers);
+  printf("   split %d\n", combined_stats.v3_split_pointers);
+  printf("   word  %d\n", combined_stats.v3_word_pointers);
+  printf(" v3 pointer seeks %d\n", combined_stats.v3_pointer_seeks);
+  printf(" v3 symbols %d\n", combined_stats.v3_symbol_count);
+  printf(" v3 offset symbol links %d\n", combined_stats.v3_symbol_link_offset);
+  printf(" v3 word symbol links %d\n", combined_stats.v3_symbol_link_word);
+
+  printf(" total %.3f ms\n", process_link_timer.getMs());
+  printf("\n");
+}
+
+/*!
+ * Process all of the labels generated from linking and give them reasonable names.
+ */
+void ObjectFileDB::process_labels() {
+  printf("- Processing Labels...\n");
+  Timer process_label_timer;
+  uint32_t total = 0;
+  for_each_obj([&](ObjectFileData& obj) { total += obj.linked_data.set_ordered_label_names(); });
+
+  printf("Processed Labels:\n");
+  printf(" total %d labels\n", total);
+  printf(" total %.3f ms\n", process_label_timer.getMs());
+  printf("\n");
+}
+
+/*!
+ * Dump object files and their linking data to text files for debugging
+ */
+void ObjectFileDB::write_object_file_words(const std::string& output_dir, bool dump_v3_only) {
+  if (dump_v3_only) {
+    printf("- Writing object file dumps (v3 only)...\n");
+  } else {
+    printf("- Writing object file dumps (all)...\n");
+  }
+
+  Timer timer;
+  uint32_t total_bytes = 0, total_files = 0;
+
+  for_each_obj([&](ObjectFileData& obj) {
+    if (obj.linked_data.segments == 3 || !dump_v3_only) {
+      auto file_text = obj.linked_data.print_words();
+      auto file_name = combine_path(output_dir, obj.record.to_unique_name() + ".txt");
+      total_bytes += file_text.size();
+      write_text_file(file_name, file_text);
+      total_files++;
+    }
+  });
+
+  printf("Wrote object file dumps:\n");
+  printf(" total %d files\n", total_files);
+  printf(" total %.3f MB\n", total_bytes / ((float)(1u << 20u)));
+  printf(" total %.3f ms (%.3f MB/sec)\n", timer.getMs(),
+         total_bytes / ((1u << 20u) * timer.getSeconds()));
+  printf("\n");
+}
+
+/*!
+ * Dump disassembly for object files containing code.  Data zones will also be dumped.
+ */
+void ObjectFileDB::write_disassembly(const std::string& output_dir,
+                                     bool disassemble_objects_without_functions) {
+  printf("- Writing functions...\n");
+  Timer timer;
+  uint32_t total_bytes = 0, total_files = 0;
+
+  for_each_obj([&](ObjectFileData& obj) {
+    if (obj.linked_data.has_any_functions() || disassemble_objects_without_functions) {
+      auto file_text = obj.linked_data.print_disassembly();
+      auto file_name = combine_path(output_dir, obj.record.to_unique_name() + ".func");
+      total_bytes += file_text.size();
+      write_text_file(file_name, file_text);
+      total_files++;
+    }
+  });
+
+  printf("Wrote functions dumps:\n");
+  printf(" total %d files\n", total_files);
+  printf(" total %.3f MB\n", total_bytes / ((float)(1u << 20u)));
+  printf(" total %.3f ms (%.3f MB/sec)\n", timer.getMs(),
+         total_bytes / ((1u << 20u) * timer.getSeconds()));
+  printf("\n");
+}
+
+/*!
+ * Find code/data zones, identify functions, and disassemble
+ */
+void ObjectFileDB::find_code() {
+  printf("- Finding code in object files...\n");
+  LinkedObjectFile::Stats combined_stats;
+  Timer timer;
+
+  for_each_obj([&](ObjectFileData& obj) {
+    //      printf("fc %s\n", obj.record.to_unique_name().c_str());
+    obj.linked_data.find_code();
+    obj.linked_data.find_functions();
+    obj.linked_data.disassemble_functions();
+
+    if (get_config().game_version == 1 || obj.record.to_unique_name() != "effect-control-v0") {
+      obj.linked_data.process_fp_relative_links();
+    } else {
+      printf("skipping process_fp_relative_links in %s\n", obj.record.to_unique_name().c_str());
+    }
+
+    auto& obj_stats = obj.linked_data.stats;
+    if (obj_stats.code_bytes / 4 > obj_stats.decoded_ops) {
+      printf("Failed to decode all in %s (%d / %d)\n", obj.record.to_unique_name().c_str(),
+             obj_stats.decoded_ops, obj_stats.code_bytes / 4);
+    }
+    combined_stats.add(obj.linked_data.stats);
+  });
+
+  printf("Found code:\n");
+  printf(" code %.3f MB\n", combined_stats.code_bytes / (float)(1 << 20));
+  printf(" data %.3f MB\n", combined_stats.data_bytes / (float)(1 << 20));
+  printf(" functions: %d\n", combined_stats.function_count);
+  printf(" fp uses resolved: %d / %d (%.3f %%)\n", combined_stats.n_fp_reg_use_resolved,
+         combined_stats.n_fp_reg_use,
+         100.f * (float)combined_stats.n_fp_reg_use_resolved / combined_stats.n_fp_reg_use);
+  auto total_ops = combined_stats.code_bytes / 4;
+  printf(" decoded %d / %d (%.3f %%)\n", combined_stats.decoded_ops, total_ops,
+         100.f * (float)combined_stats.decoded_ops / total_ops);
+  printf(" total %.3f ms\n", timer.getMs());
+  printf("\n");
+}
+
+/*!
+ * Finds and writes all scripts into a file named all_scripts.lisp.
+ * Doesn't change any state in ObjectFileDB.
+ */
+void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) {
+  printf("- Finding scripts in object files...\n");
+  Timer timer;
+  std::string all_scripts;
+
+  for_each_obj([&](ObjectFileData& obj) {
+    auto scripts = obj.linked_data.print_scripts();
+    if (!scripts.empty()) {
+      all_scripts += ";--------------------------------------\n";
+      all_scripts += "; " + obj.record.to_unique_name() + "\n";
+      all_scripts += ";---------------------------------------\n";
+      all_scripts += scripts;
+    }
+  });
+
+  auto file_name = combine_path(output_dir, "all_scripts.lisp");
+  write_text_file(file_name, all_scripts);
+
+  printf("Found scripts:\n");
+  printf(" total %.3f ms\n", timer.getMs());
+  printf("\n");
+}
+
+void ObjectFileDB::analyze_functions() {
+  printf("- Analyzing Functions...\n");
+  Timer timer;
+
+  int total_functions = 0;
+  int resolved_cfg_functions = 0;
+  const auto& config = get_config();
+
+  {
+    timer.start();
+    for_each_obj([&](ObjectFileData& data) {
+      if (data.linked_data.segments == 3) {
+        // the top level segment should have a single function
+        assert(data.linked_data.functions_by_seg.at(2).size() == 1);
+
+        auto& func = data.linked_data.functions_by_seg.at(2).front();
+        assert(func.guessed_name.empty());
+        func.guessed_name.set_as_top_level();
+        func.find_global_function_defs(data.linked_data);
+        func.find_method_defs(data.linked_data);
+      }
+    });
+
+    // check for function uniqueness.
+    std::unordered_set<std::string> unique_names;
+    std::unordered_map<std::string, std::unordered_set<std::string>> duplicated_functions;
+
+    for_each_function([&](Function& func, int segment_id, ObjectFileData& data) {
+      (void)segment_id;
+      auto name = func.guessed_name.to_string();
+      if (func.guessed_name.expected_unique()) {
+        if(unique_names.find(name) != unique_names.end()) {
+            duplicated_functions[name].insert(data.record.to_unique_name());
+        }
+
+        unique_names.insert(name);
+      }
+
+      if (config.asm_functions_by_name.find(name) != config.asm_functions_by_name.end()) {
+        func.warnings += "flagged as asm by config\n";
+        func.suspected_asm = true;
+      }
+    });
+
+    for_each_function([&](Function& func, int segment_id, ObjectFileData& data) {
+        (void)segment_id;
+        auto name = func.guessed_name.to_string();
+        if(func.guessed_name.expected_unique()) {
+          if(duplicated_functions.find(name) != duplicated_functions.end()) {
+            duplicated_functions[name].insert(data.record.to_unique_name());
+            func.warnings += "this function exists in multiple non-identical object files";
+          }
+        }
+    });
+
+//    for(const auto& kv : duplicated_functions) {
+//      printf("Function %s is found in non-identical object files:\n", kv.first.c_str());
+//      for(const auto& obj : kv.second) {
+//        printf(" %s\n", obj.c_str());
+//      }
+//    }
+  }
+
+  int total_nontrivial_functions = 0;
+  int total_resolved_nontrivial_functions = 0;
+  int total_named_functions = 0;
+
+  std::map<int, std::vector<std::string>> unresolved_by_length;
+  if (get_config().find_basic_blocks) {
+    timer.start();
+    int total_basic_blocks = 0;
+    for_each_function([&](Function& func, int segment_id, ObjectFileData& data) {
+      auto blocks = find_blocks_in_function(data.linked_data, segment_id, func);
+      total_basic_blocks += blocks.size();
+      func.basic_blocks = blocks;
+
+      if(!func.suspected_asm) {
+          func.analyze_prologue(data.linked_data);
+          func.cfg = build_cfg(data.linked_data, segment_id, func);
+          total_functions++;
+          if (func.cfg->is_fully_resolved()) {
+              resolved_cfg_functions++;
+          }
+      } else {
+        resolved_cfg_functions++;
+      }
+
+
+      if(func.basic_blocks.size() > 1 && !func.suspected_asm) {
+        total_nontrivial_functions++;
+        if(func.cfg->is_fully_resolved()) {
+          total_resolved_nontrivial_functions++;
+        } else {
+          if(!func.guessed_name.empty()) {
+            unresolved_by_length[func.end_word - func.start_word].push_back(func.guessed_name.to_string());
+          }
+        }
+      }
+
+      if(!func.guessed_name.empty()) {
+        total_named_functions++;
+      }
+    });
+
+    printf("Found %d functions (%d with nontrivial cfgs)\n", total_functions, total_nontrivial_functions);
+    printf("Named %d/%d functions (%.2f%%)\n", total_named_functions, total_functions, 100.f * float(total_named_functions) / float(total_functions));
+    printf("Found %d basic blocks in %.3f ms\n", total_basic_blocks, timer.getMs());
+    printf(" %d/%d functions passed cfg analysis stage (%.2f%%)\n", resolved_cfg_functions, total_functions,
+           100.f * float(resolved_cfg_functions) / float(total_functions));
+    printf(" %d/%d nontrivial cfg's resolved (%.2f%%)\n", total_resolved_nontrivial_functions, total_nontrivial_functions,
+           100.f * float(total_resolved_nontrivial_functions) / float(total_nontrivial_functions));
+
+    for(auto& kv : unresolved_by_length) {
+      printf("LEN %d\n", kv.first);
+      for(auto& x : kv.second) {
+        printf("  %s\n", x.c_str());
+      }
+    }
+  }
+}

decompiler/ObjectFile/ObjectFileDB.h

@@ -0,0 +1,105 @@
+/*!
+ * @file ObjectFileDB.h
+ * A "database" of object files found in DGO files.
+ * Eliminates duplicate object files, and also assigns unique names to all object files
+ * (there may be different object files with the same name sometimes)
+ */
+
+#ifndef JAK2_DISASSEMBLER_OBJECTFILEDB_H
+#define JAK2_DISASSEMBLER_OBJECTFILEDB_H
+
+#include <cassert>
+#include <string>
+#include <unordered_map>
+#include <vector>
+#include "LinkedObjectFile.h"
+
+/*!
+ * A "record" which can be used to identify an object file.
+ */
+struct ObjectFileRecord {
+  std::string name;
+  int version = -1;
+  uint32_t hash = 0;
+  std::string to_unique_name() const;
+};
+
+/*!
+ * All of the data for a single object file
+ */
+struct ObjectFileData {
+  std::vector<uint8_t> data;     // raw bytes
+  LinkedObjectFile linked_data;  // data including linking annotations
+  ObjectFileRecord record;       // name
+  uint32_t reference_count = 0;  // number of times its used.
+};
+
+class ObjectFileDB {
+ public:
+  ObjectFileDB(const std::vector<std::string>& _dgos);
+  std::string generate_dgo_listing();
+  void process_link_data();
+  void process_labels();
+  void find_code();
+  void find_and_write_scripts(const std::string& output_dir);
+
+  void write_object_file_words(const std::string& output_dir, bool dump_v3_only);
+  void write_disassembly(const std::string& output_dir, bool disassemble_objects_without_functions);
+  void analyze_functions();
+
+ private:
+  void get_objs_from_dgo(const std::string& filename);
+  void add_obj_from_dgo(const std::string& obj_name,
+                        uint8_t* obj_data,
+                        uint32_t obj_size,
+                        const std::string& dgo_name);
+
+  /*!
+   * Apply f to all ObjectFileData's. Does it in the right order.
+   */
+  template <typename Func>
+  void for_each_obj(Func f) {
+    assert(obj_files_by_name.size() == obj_file_order.size());
+    for(const auto& name : obj_file_order) {
+      for(auto& obj : obj_files_by_name.at(name)) {
+        f(obj);
+      }
+    }
+  }
+
+  /*!
+   * Apply f to all functions
+   * takes (Function, segment, linked_data)
+   * Does it in the right order.
+   */
+  template <typename Func>
+  void for_each_function(Func f) {
+    for_each_obj([&](ObjectFileData& data) {
+//      printf("IN %s\n", data.record.to_unique_name().c_str());
+      for (int i = 0; i < int(data.linked_data.segments); i++) {
+//        printf("seg %d\n", i);
+        int fn = 0;
+        for (auto& goal_func : data.linked_data.functions_by_seg.at(i)) {
+//          printf("fn %d\n", fn);
+          f(goal_func, i, data);
+          fn++;
+        }
+      }
+    });
+  }
+
+  // Danger: after adding all object files, we assume that the vector never reallocates.
+  std::unordered_map<std::string, std::vector<ObjectFileData>> obj_files_by_name;
+  std::unordered_map<std::string, std::vector<ObjectFileRecord>> obj_files_by_dgo;
+
+  std::vector<std::string> obj_file_order;
+
+  struct {
+    uint32_t total_dgo_bytes = 0;
+    uint32_t total_obj_files = 0;
+    uint32_t unique_obj_files = 0;
+    uint32_t unique_obj_bytes = 0;
+  } stats;
+};
+
+#endif  // JAK2_DISASSEMBLER_OBJECTFILEDB_H