[graphics] tfrag3 renderer (#978)

* begin work

* work

* working objs

* exporting

* it works

* before some time of day fixes

* add time of day interp and also fix zbuffer

* some small blending fixes

* improve randomess

* clean up extraction and missing blend mode

* culling, time of day, more level fixes

* more cleanup

* cleanup memory usage

* windows fix

common/CMakeLists.txt

@@ -3,6 +3,7 @@ add_library(common
         audio/audio_formats.cpp
         cross_os_debug/xdbg.cpp
         cross_sockets/xsocket.cpp
+        custom_data/TFrag3Data.cpp
         dma/dma.cpp
         dma/dma_copy.cpp
         dma/gs.cpp
@@ -32,6 +33,7 @@ add_library(common
         util/FileUtil.cpp
         util/json_util.cpp
         util/Timer.cpp
+        util/os.cpp
         util/print_float.cpp
         util/FontUtils.cpp
         util/image_loading.cpp)

common/custom_data/TFrag3Data.cpp

@@ -0,0 +1,81 @@
+#include "Tfrag3Data.h"
+#include "common/util/assert.h"
+
+namespace tfrag3 {
+
+void Draw::serialize(Serializer& ser) {
+  ser.from_ptr(&mode);
+  ser.from_ptr(&tree_tex_id);
+  ser.from_pod_vector(&vertex_index_stream);
+  ser.from_pod_vector(&vis_groups);
+  ser.from_ptr(&num_triangles);
+}
+
+void Tree::serialize(Serializer& ser) {
+  ser.from_ptr(&kind);
+
+  if (ser.is_saving()) {
+    ser.save<size_t>(draws.size());
+  } else {
+    draws.resize(ser.load<size_t>());
+  }
+  for (auto& draw : draws) {
+    draw.serialize(ser);
+  }
+
+  ser.from_pod_vector(&vertices);
+  ser.from_pod_vector(&color_indices_per_vertex);
+  ser.from_pod_vector(&vis_nodes);
+  ser.from_pod_vector(&colors);
+  ser.from_ptr(&first_leaf_node);
+  ser.from_ptr(&last_leaf_node);
+  ser.from_ptr(&first_root);
+  ser.from_ptr(&num_roots);
+  ser.from_ptr(&only_children);
+}
+
+void Texture::serialize(Serializer& ser) {
+  ser.from_ptr(&w);
+  ser.from_ptr(&h);
+  ser.from_pod_vector(&data);
+  ser.from_str(&debug_name);
+  ser.from_str(&debug_tpage_name);
+}
+
+void Level::serialize(Serializer& ser) {
+  ser.from_ptr(&version);
+  if (ser.is_loading() && version != TFRAG3_VERSION) {
+    fmt::print("version mismatch when loading tfrag3 data. Got {}, expected {}\n", version,
+               TFRAG3_VERSION);
+    assert(false);
+  }
+
+  ser.from_str(&level_name);
+
+  if (ser.is_saving()) {
+    ser.save<size_t>(textures.size());
+  } else {
+    textures.resize(ser.load<size_t>());
+  }
+  for (auto& tex : textures) {
+    tex.serialize(ser);
+  }
+
+  if (ser.is_saving()) {
+    ser.save<size_t>(trees.size());
+  } else {
+    trees.resize(ser.load<size_t>());
+  }
+  for (auto& tree : trees) {
+    tree.serialize(ser);
+  }
+
+  ser.from_ptr(&version2);
+  if (ser.is_loading() && version2 != TFRAG3_VERSION) {
+    fmt::print("version mismatch when loading tfrag3 data (at end). Got {}, expected {}\n",
+               version2, TFRAG3_VERSION);
+    assert(false);
+  }
+}
+
+}  // namespace tfrag3

common/custom_data/Tfrag3Data.h

@@ -0,0 +1,99 @@
+#pragma once
+
+// Data format for the tfrag3 renderer.
+
+#include "common/common_types.h"
+#include "common/util/assert.h"
+#include "common/dma/gs.h"
+#include "common/util/Serializer.h"
+#include "common/math/Vector.h"
+
+namespace tfrag3 {
+
+constexpr int TFRAG3_VERSION = 6;
+
+// These vertices should be uploaded to the GPU at load time and don't change
+struct PreloadedVertex {
+  // the vertex position
+  float x, y, z;
+  // texture coordinates
+  float s, t, q;
+  // currently unused, color table indices.
+  u16 color_index;
+  u16 pad[3];
+};
+static_assert(sizeof(PreloadedVertex) == 32, "PreloadedVertex size");
+
+// Settings for an OpenGL draw
+struct Draw {
+  DrawMode mode;        // the OpenGL draw settings.
+  u32 tree_tex_id = 0;  // the texture that should be bound for the draw
+
+  // the list of vertices in the draw.
+  std::vector<u32> vertex_index_stream;
+
+  // to do culling, the above vertex stream is grouped.
+  // by following the visgroups and checking the visibility of the tfrag_idx, you can leave out
+  // invisible vertices.
+  struct VisGroup {
+    u32 num = 0;
+    u32 tfrag_idx = 0;
+  };
+  std::vector<VisGroup> vis_groups;
+  u32 num_triangles = 0;
+
+  void serialize(Serializer& ser);
+};
+
+struct VisNode {
+  math::Vector<float, 4> bsphere;  // the bounding sphere, in meters (4096 = 1 game meter). w = rad
+  u16 child_id = 0xffff;           // the ID of our first child.
+  u8 num_kids = 0xff;              // number of children. The children are consecutive in memory
+  u8 flags = 0;                    // flags.  If 1, we have a DrawVisNode child, otherwise a Tfrag.
+};
+
+enum class TFragmentTreeKind { NORMAL, TRANS, DIRT, ICE, LOWRES, LOWRES_TRANS, INVALID };
+
+constexpr const char* tfrag_tree_names[] = {"normal", "trans",        "dirt",   "ice",
+                                            "lowres", "lowres-trans", "invalid"};
+
+struct TimeOfDayColor {
+  math::Vector<u8, 4> rgba[8];
+};
+
+struct Tree {
+  TFragmentTreeKind kind;
+  std::vector<Draw> draws;
+  std::vector<u16> color_indices_per_vertex;
+  std::vector<VisNode> vis_nodes;
+  std::vector<PreloadedVertex> vertices;
+  std::vector<TimeOfDayColor> colors;
+  u16 first_leaf_node = 0;
+  u16 last_leaf_node = 0;
+  u16 first_root = 0;
+  u16 num_roots = 0;
+  bool only_children = false;
+
+  void serialize(Serializer& ser);
+};
+
+struct Texture {
+  u16 w, h;
+  u32 combo_id = 0;
+  std::vector<u32> data;
+  std::string debug_name;
+  std::string debug_tpage_name;
+
+  void serialize(Serializer& ser);
+};
+
+struct Level {
+  u16 version = TFRAG3_VERSION;
+  std::string level_name;
+  std::vector<Texture> textures;
+  std::vector<Tree> trees;
+  u16 version2 = TFRAG3_VERSION;
+  void serialize(Serializer& ser);
+};
+
+}  // namespace tfrag3

common/dma/gs.cpp

@@ -356,4 +356,79 @@ std::string GsTex0::print() const {
 
 std::string GsPrim::print() const {
   return fmt::format("0x{:x}, kind {}\n", data, kind());
+}
+
+std::string DrawMode::to_string() const {
+  std::string result;
+  result += fmt::format(" depth-write: {}\n", get_depth_write_enable());
+  result += fmt::format(" depth-test: ");
+  switch (get_depth_test()) {
+    case GsTest::ZTest::NEVER:
+      result += "never\n";
+      break;
+    case GsTest::ZTest::GEQUAL:
+      result += "gequal\n";
+      break;
+    case GsTest::ZTest::ALWAYS:
+      result += "always\n";
+      break;
+    case GsTest::ZTest::GREATER:
+      result += "greater\n";
+      break;
+    default:
+      assert(false);
+  }
+  result += fmt::format(" alpha: ");
+  switch (get_alpha_blend()) {
+    case AlphaBlend::SRC_0_SRC_DST:
+      result += "src, 0, src, dst\n";
+      break;
+    case AlphaBlend::SRC_DST_SRC_DST:
+      result += "src, dst, src, dst\n";
+      break;
+    case AlphaBlend::DISABLED:
+      result += "disabled\n";
+      break;
+    default:
+      assert(false);
+  }
+  result += fmt::format(" clamp: s {} t {}\n", get_clamp_s_enable(), get_clamp_t_enable());
+  result += fmt::format(" filt: {}\n", get_filt_enable());
+  result += fmt::format(" tcc: {}\n", get_tcc_enable());
+  result += fmt::format(" aref: {}\n", get_aref());
+  result += fmt::format(" ate: {}\n", get_at_enable());
+  result += fmt::format(" atst: ");
+  switch (get_alpha_test()) {
+    case AlphaTest::ALWAYS:
+      result += "always\n";
+      break;
+    case AlphaTest::GEQUAL:
+      result += "gequal\n";
+      break;
+    case AlphaTest::NEVER:
+      result += "never\n";
+      break;
+    default:
+      assert(false);
+  }
+  result += fmt::format(" zte: {}\n", get_zt_enable());
+  result += fmt::format(" abe: {}\n", get_ab_enable());
+  result += fmt::format(" afail: ");
+  switch (get_alpha_fail()) {
+    case GsTest::AlphaFail::KEEP:
+      result += "keep\n";
+      break;
+    case GsTest::AlphaFail::FB_ONLY:
+      result += "fb-only\n";
+      break;
+    case GsTest::AlphaFail::RGB_ONLY:
+      result += "rgb-only\n";
+      break;
+    case GsTest::AlphaFail::ZB_ONLY:
+      result += "zb-only\n";
+      break;
+    default:
+      assert(false);
+  }
+  return result;
 }

common/dma/gs.h

@@ -343,4 +343,140 @@ struct AdGifData {
   u64 clamp_addr;
   u64 alpha_data;
   u64 alpha_addr;
-};
+};
+
+// this represents all of the drawing state, stored as an integer.
+// it can also represent "invalid".
+class DrawMode {
+ public:
+  enum class AlphaBlend { DISABLED = 0, SRC_DST_SRC_DST = 1, SRC_0_SRC_DST = 2, SRC_0_FIX_DST = 3 };
+
+  enum class AlphaTest {
+    NEVER = 0,
+    ALWAYS = 1,
+    GEQUAL = 2,
+  };
+
+  bool get_depth_write_enable() const { return m_val & 0b1; }
+  void enable_depth_write() { m_val = m_val | 0b1; }
+  void disable_depth_write() { m_val = m_val & ~(0b1); }
+
+  GsTest::ZTest get_depth_test() const { return (GsTest::ZTest)((m_val >> 1) & 0b11); }
+  void set_depth_test(GsTest::ZTest dt) { m_val = (m_val & ~(0b110)) | ((u32)(dt) << 1); }
+
+  AlphaBlend get_alpha_blend() const { return (AlphaBlend)((m_val >> 3) & 0b11); }
+  void set_alpha_blend(AlphaBlend ab) { m_val = (m_val & ~(0b11000)) | ((u32)(ab) << 3); }
+
+  u8 get_aref() const { return m_val >> 8; }
+  void set_aref(u8 val) { m_val = (m_val & ~(0xff00)) | (val << 8); }
+
+  AlphaTest get_alpha_test() const { return (AlphaTest)((m_val >> 16) & 0b11); }
+  void set_alpha_test(AlphaTest ab) { m_val = (m_val & ~(0b11 << 16)) | ((u32)(ab) << 16); }
+
+  GsTest::AlphaFail get_alpha_fail() const { return (GsTest::AlphaFail)((m_val >> 21) & 0b11); }
+  void set_alpha_fail(GsTest::AlphaFail ab) { m_val = (m_val & ~(0b11 << 21)) | ((u32)(ab) << 21); }
+
+  bool is_invalid() const { return m_val == UINT32_MAX; }
+  bool is_valid() const { return !is_invalid(); }
+  void set_invalid() { m_val = UINT32_MAX; }
+
+  bool get_clamp_s_enable() const { return m_val & (1 << 5); }
+  void set_clamp_s_enable(bool en) {
+    if (en) {
+      enable_s_clamp();
+    } else {
+      disable_s_clamp();
+    }
+  }
+  void enable_s_clamp() { m_val = m_val | (1 << 5); }
+  void disable_s_clamp() { m_val = m_val & (~(1 << 5)); }
+
+  bool get_filt_enable() const { return m_val & (1 << 6); }
+  void enable_filt() { m_val = m_val | (1 << 6); }
+  void disable_filt() { m_val = m_val & (~(1 << 6)); }
+  void set_filt_enable(bool en) {
+    if (en) {
+      enable_filt();
+    } else {
+      disable_filt();
+    }
+  }
+
+  bool get_tcc_enable() const { return m_val & (1 << 6); }
+  void enable_tcc() { m_val = m_val | (1 << 7); }
+  void disable_tcc() { m_val = m_val & (~(1 << 7)); }
+  void set_tcc(bool en) {
+    if (en) {
+      enable_tcc();
+    } else {
+      disable_tcc();
+    }
+  }
+
+  bool get_at_enable() const { return m_val & (1 << 18); }
+  void enable_at() { m_val = m_val | (1 << 18); }
+  void disable_at() { m_val = m_val & (~(1 << 18)); }
+  void set_at(bool en) {
+    if (en) {
+      enable_at();
+    } else {
+      disable_at();
+    }
+  }
+
+  bool get_zt_enable() const { return m_val & (1 << 19); }
+  void enable_zt() { m_val = m_val | (1 << 19); }
+  void disable_zt() { m_val = m_val & (~(1 << 19)); }
+  void set_zt(bool en) {
+    if (en) {
+      enable_zt();
+    } else {
+      disable_zt();
+    }
+  }
+
+  bool get_ab_enable() const { return m_val & (1 << 20); }
+  void enable_ab() { m_val = m_val | (1 << 20); }
+  void disable_ab() { m_val = m_val & (~(1 << 20)); }
+  void set_ab(bool en) {
+    if (en) {
+      enable_ab();
+    } else {
+      disable_ab();
+    }
+  }
+
+  bool get_clamp_t_enable() const { return m_val & (1 << 23); }
+  void set_clamp_t_enable(bool en) {
+    if (en) {
+      enable_t_clamp();
+    } else {
+      disable_t_clamp();
+    }
+  }
+  void enable_t_clamp() { m_val = m_val | (1 << 23); }
+  void disable_t_clamp() { m_val = m_val & (~(1 << 23)); }
+
+  u32& as_int() { return m_val; }
+
+  bool operator==(const DrawMode& other) const { return m_val == other.m_val; }
+  bool operator!=(const DrawMode& other) const { return m_val != other.m_val; }
+
+  std::string to_string() const;
+
+ private:
+  // 0 - depth write enable
+  // 1, 2 - test: never, always, gequal, greater
+  // 3, 4 - alpha: disable, [src,dst,src,dst], [src,0,src,dst], XX
+  // 5 - clamp enable
+  // 6 - filt enable
+  // 7 - tcc enable
+  // 8,9,10,11,12,14,14,15 - aref
+  // 16, 17 - atest
+  // 18 - ate
+  // 19 - zte
+  // 20 - abe
+  // 21, 22 - afail
+  // 23 t clamp
+  u32 m_val = UINT32_MAX;
+};

common/math/Vector.h

@@ -175,7 +175,7 @@ class Vector {
   const T* data() const { return m_data; }
 
   template <typename U>
-  Vector<U, Size> cast() {
+  Vector<U, Size> cast() const {
     Vector<U, Size> result;
     for (int i = 0; i < Size; i++) {
       result[i] = (U)m_data[i];
@@ -183,6 +183,19 @@ class Vector {
     return result;
   }
 
+  template <int len>
+  Vector<T, len> head() const {
+    static_assert(len < Size);
+    Vector<T, len> result;
+    for (int i = 0; i < len; i++) {
+      result[i] = m_data[i];
+    }
+    return result;
+  }
+
+  Vector<T, 3> xyz() const { return head<3>(); }
+  Vector<T, 3> xy() const { return head<2>(); }
+
  private:
   T m_data[Size];
 };
@@ -211,6 +224,9 @@ struct Matrix {
     return result;
   }
 
+  T* data() { return m_data; }
+  const T* data() const { return m_data; }
+
   std::string to_string_aligned() const {
     std::string result;
     for (int row = 0; row < Rows; row++) {

common/util/Serializer.h

@@ -151,7 +151,7 @@ class Serializer {
     } else {
       vec->resize(load<size_t>());
     }
-    from_raw_data(vec->data(), vec->size());
+    from_raw_data(vec->data(), sizeof(T) * vec->size());
   }
 
   /*!

common/util/os.cpp

@@ -0,0 +1,17 @@
+#include "os.h"
+
+#ifdef __linux__
+
+#include <sys/resource.h>
+
+size_t get_peak_rss() {
+  rusage x;
+  getrusage(RUSAGE_SELF, &x);
+  return x.ru_maxrss * 1024;
+}
+
+#else
+size_t get_peak_rss() {
+  return 0;
+}
+#endif

common/util/os.h

@@ -0,0 +1,6 @@
+#pragma once
+
+#include <cstddef>
+
+// Note: these are not implemented on windows and will return zero.
+size_t get_peak_rss();

decompiler/CMakeLists.txt

@@ -24,6 +24,7 @@ add_library(
         data/game_text.cpp
         data/streamed_audio.cpp
         data/StrFileReader.cpp
+        data/TextureDB.cpp
         data/tpage.cpp
 
         Disasm/Instruction.cpp
@@ -54,6 +55,10 @@ add_library(
         IR2/LabelDB.cpp
         IR2/OpenGoalMapping.cpp
 
+        level_extractor/extract_level.cpp
+        level_extractor/extract_tfrag.cpp
+        level_extractor/BspHeader.cpp
+
         ObjectFile/LinkedObjectFile.cpp
         ObjectFile/LinkedObjectFileCreation.cpp
         ObjectFile/ObjectFileDB.cpp
@@ -63,6 +68,7 @@ add_library(
         util/data_decompile.cpp
         util/DataParser.cpp
         util/DecompilerTypeSystem.cpp
+        util/goal_data_reader.cpp
         util/sparticle_decompile.cpp
         util/TP_Type.cpp
 

decompiler/Disasm/InstructionDecode.cpp

@@ -1135,36 +1135,36 @@ Instruction decode_instruction(LinkedWord& word, LinkedObjectFile& file, int seg
     }
   }
 
-  if (word.kind == LinkedWord::SYM_OFFSET) {
+  if (word.kind() == LinkedWord::SYM_OFFSET) {
     bool fixed = false;
     for (int j = 0; j < i.n_src; j++) {
       if (i.src[j].kind == InstructionAtom::IMM) {
         fixed = true;
-        i.src[j].set_sym(word.symbol_name);
+        i.src[j].set_sym(word.symbol_name());
       }
     }
     assert(fixed);
   }
 
-  if (word.kind == LinkedWord::HI_PTR) {
+  if (word.kind() == LinkedWord::HI_PTR) {
     assert(i.kind == InstructionKind::LUI);
     bool fixed = false;
     for (int j = 0; j < i.n_src; j++) {
       if (i.src[j].kind == InstructionAtom::IMM) {
         fixed = true;
-        i.src[j].set_label(word.label_id);
+        i.src[j].set_label(word.label_id());
       }
     }
     assert(fixed);
   }
 
-  if (word.kind == LinkedWord::LO_PTR) {
+  if (word.kind() == LinkedWord::LO_PTR) {
     assert(i.kind == InstructionKind::ORI);
     bool fixed = false;
     for (int j = 0; j < i.n_src; j++) {
       if (i.src[j].kind == InstructionAtom::IMM) {
         fixed = true;
-        i.src[j].set_label(word.label_id);
+        i.src[j].set_label(word.label_id());
       }
     }
     assert(fixed);

decompiler/IR2/AtomicOpForm.cpp

@@ -499,7 +499,7 @@ FormElement* make_label_load(int label_idx,
       load_size == 4 && hint.result_type == TypeSpec("float")) {
     assert((label.offset % 4) == 0);
     auto word = env.file->words_by_seg.at(label.target_segment).at(label.offset / 4);
-    assert(word.kind == LinkedWord::PLAIN_DATA);
+    assert(word.kind() == LinkedWord::PLAIN_DATA);
     float value;
     memcpy(&value, &word.data, 4);
     return pool.alloc_element<ConstantFloatElement>(value);
@@ -508,8 +508,8 @@ FormElement* make_label_load(int label_idx,
     assert((label.offset % 8) == 0);
     auto word0 = env.file->words_by_seg.at(label.target_segment).at(label.offset / 4);
     auto word1 = env.file->words_by_seg.at(label.target_segment).at(1 + (label.offset / 4));
-    assert(word0.kind == LinkedWord::PLAIN_DATA);
-    assert(word1.kind == LinkedWord::PLAIN_DATA);
+    assert(word0.kind() == LinkedWord::PLAIN_DATA);
+    assert(word1.kind() == LinkedWord::PLAIN_DATA);
     u64 value;
     memcpy(&value, &word0.data, 4);
     memcpy(((u8*)&value) + 4, &word1.data, 4);
@@ -526,8 +526,8 @@ FormElement* make_label_load(int label_idx,
     assert((label.offset % 8) == 0);
     auto word0 = env.file->words_by_seg.at(label.target_segment).at(label.offset / 4);
     auto word1 = env.file->words_by_seg.at(label.target_segment).at(1 + (label.offset / 4));
-    assert(word0.kind == LinkedWord::PLAIN_DATA);
-    assert(word1.kind == LinkedWord::PLAIN_DATA);
+    assert(word0.kind() == LinkedWord::PLAIN_DATA);
+    assert(word1.kind() == LinkedWord::PLAIN_DATA);
     u64 value;
     memcpy(&value, &word0.data, 4);
     memcpy(((u8*)&value) + 4, &word1.data, 4);

decompiler/IR2/Env.cpp

@@ -9,15 +9,22 @@
 #include "common/util/math_util.h"
 
 namespace decompiler {
+
+constexpr const char* reg_names[] = {"a0-0", "a1-0", "a2-0", "a3-0",
+                                     "t0-0", "t1-0", "t2-0", "t3-0"};
+
+const char* get_reg_name(int idx) {
+  if (idx >= 8) {
+    return "INVALID";
+  } else {
+    return reg_names[idx];
+  }
+}
+
 void Env::set_remap_for_function(const TypeSpec& ts) {
   int nargs = ts.arg_count() - 1;
   for (int i = 0; i < nargs; i++) {
-    std::string var_name;
-    var_name.push_back(i >= 4 ? 't' : 'a');
-    var_name.push_back('0' + (i % 4));
-    var_name.push_back('-');
-    var_name.push_back('0');
-    m_var_remap[var_name] = ("arg" + std::to_string(i));
+    m_var_remap[get_reg_name(i)] = ("arg" + std::to_string(i));
   }
   if (ts.try_get_tag("behavior")) {
     m_var_remap["s6-0"] = "self";
@@ -32,12 +39,7 @@ void Env::set_remap_for_new_method(const TypeSpec& ts) {
   m_var_remap["a0-0"] = "allocation";
   m_var_remap["a1-0"] = "type-to-make";
   for (int i = 2; i < nargs; i++) {
-    std::string var_name;
-    var_name.push_back(i >= 4 ? 't' : 'a');
-    var_name.push_back('0' + (i % 4));
-    var_name.push_back('-');
-    var_name.push_back('0');
-    m_var_remap[var_name] = ("arg" + std::to_string(i - 2));
+    m_var_remap[get_reg_name(i)] = ("arg" + std::to_string(i - 2));
   }
   if (ts.try_get_tag("behavior")) {
     m_var_remap["s6-0"] = "self";
@@ -51,12 +53,7 @@ void Env::set_remap_for_method(const TypeSpec& ts) {
   int nargs = ts.arg_count() - 1;
   m_var_remap["a0-0"] = "obj";
   for (int i = 1; i < nargs; i++) {
-    std::string var_name;
-    var_name.push_back(i >= 4 ? 't' : 'a');
-    var_name.push_back('0' + (i % 4));
-    var_name.push_back('-');
-    var_name.push_back('0');
-    m_var_remap[var_name] = ("arg" + std::to_string(i - 1));
+    m_var_remap[get_reg_name(i)] = ("arg" + std::to_string(i - 1));
   }
   if (ts.try_get_tag("behavior")) {
     m_var_remap["s6-0"] = "self";
@@ -69,12 +66,7 @@ void Env::set_remap_for_method(const TypeSpec& ts) {
 void Env::map_args_from_config(const std::vector<std::string>& args_names,
                                const std::unordered_map<std::string, std::string>& var_names) {
   for (size_t i = 0; i < args_names.size(); i++) {
-    std::string var_name;
-    var_name.push_back(i >= 4 ? 't' : 'a');
-    var_name.push_back('0' + (i % 4));
-    var_name.push_back('-');
-    var_name.push_back('0');
-    m_var_remap[var_name] = args_names[i];
+    m_var_remap[get_reg_name(i)] = args_names[i];
   }
 
   for (auto& x : var_names) {
@@ -86,12 +78,7 @@ void Env::map_args_from_config(
     const std::vector<std::string>& args_names,
     const std::unordered_map<std::string, LocalVarOverride>& var_overrides) {
   for (size_t i = 0; i < args_names.size(); i++) {
-    std::string var_name;
-    var_name.push_back(i >= 4 ? 't' : 'a');
-    var_name.push_back('0' + (i % 4));
-    var_name.push_back('-');
-    var_name.push_back('0');
-    m_var_remap[var_name] = args_names[i];
+    m_var_remap[get_reg_name(i)] = args_names[i];
   }
 
   for (auto& x : var_overrides) {

decompiler/ObjectFile/LinkedObjectFile.cpp

@@ -150,7 +150,7 @@ bool LinkedObjectFile::pointer_link_word(int source_segment,
   assert((source_offset % 4) == 0);
 
   auto& word = words_by_seg.at(source_segment).at(source_offset / 4);
-  assert(word.kind == LinkedWord::PLAIN_DATA);
+  assert(word.kind() == LinkedWord::PLAIN_DATA);
 
   if (dest_offset / 4 > (int)words_by_seg.at(dest_segment).size()) {
     //    printf("HACK bad link ignored!\n");
@@ -158,8 +158,7 @@ bool LinkedObjectFile::pointer_link_word(int source_segment,
   }
   assert(dest_offset / 4 <= (int)words_by_seg.at(dest_segment).size());
 
-  word.kind = LinkedWord::PTR;
-  word.label_id = get_label_id_for(dest_segment, dest_offset);
+  word.set_to_pointer(LinkedWord::PTR, get_label_id_for(dest_segment, dest_offset));
   return true;
 }
 
@@ -173,11 +172,10 @@ void LinkedObjectFile::symbol_link_word(int source_segment,
   assert((source_offset % 4) == 0);
   auto& word = words_by_seg.at(source_segment).at(source_offset / 4);
   //  assert(word.kind == LinkedWord::PLAIN_DATA);
-  if (word.kind != LinkedWord::PLAIN_DATA) {
+  if (word.kind() != LinkedWord::PLAIN_DATA) {
     printf("bad symbol link word\n");
   }
-  word.kind = kind;
-  word.symbol_name = name;
+  word.set_to_symbol(kind, name);
 }
 
 /*!
@@ -187,9 +185,8 @@ void LinkedObjectFile::symbol_link_word(int source_segment,
 void LinkedObjectFile::symbol_link_offset(int source_segment, int source_offset, const char* name) {
   assert((source_offset % 4) == 0);
   auto& word = words_by_seg.at(source_segment).at(source_offset / 4);
-  assert(word.kind == LinkedWord::PLAIN_DATA);
-  word.kind = LinkedWord::SYM_OFFSET;
-  word.symbol_name = name;
+  assert(word.kind() == LinkedWord::PLAIN_DATA);
+  word.set_to_symbol(LinkedWord::SYM_OFFSET, name);
 }
 
 /*!
@@ -207,14 +204,11 @@ void LinkedObjectFile::pointer_link_split_word(int source_segment,
   auto& lo_word = words_by_seg.at(source_segment).at(source_lo_offset / 4);
 
   //  assert(dest_offset / 4 <= (int)words_by_seg.at(dest_segment).size());
-  assert(hi_word.kind == LinkedWord::PLAIN_DATA);
-  assert(lo_word.kind == LinkedWord::PLAIN_DATA);
+  assert(hi_word.kind() == LinkedWord::PLAIN_DATA);
+  assert(lo_word.kind() == LinkedWord::PLAIN_DATA);
 
-  hi_word.kind = LinkedWord::HI_PTR;
-  hi_word.label_id = get_label_id_for(dest_segment, dest_offset);
-
-  lo_word.kind = LinkedWord::LO_PTR;
-  lo_word.label_id = hi_word.label_id;
+  hi_word.set_to_pointer(LinkedWord::HI_PTR, get_label_id_for(dest_segment, dest_offset));
+  lo_word.set_to_pointer(LinkedWord::LO_PTR, hi_word.label_id());
 }
 
 /*!
@@ -284,32 +278,32 @@ std::string LinkedObjectFile::print_words() {
 void LinkedObjectFile::append_word_to_string(std::string& dest, const LinkedWord& word) const {
   char buff[128];
 
-  switch (word.kind) {
+  switch (word.kind()) {
     case LinkedWord::PLAIN_DATA:
       sprintf(buff, "    .word 0x%x\n", word.data);
       break;
     case LinkedWord::PTR:
-      sprintf(buff, "    .word %s\n", labels.at(word.label_id).name.c_str());
+      sprintf(buff, "    .word %s\n", labels.at(word.label_id()).name.c_str());
       break;
     case LinkedWord::SYM_PTR:
-      sprintf(buff, "    .symbol %s\n", word.symbol_name.c_str());
+      sprintf(buff, "    .symbol %s\n", word.symbol_name().c_str());
       break;
     case LinkedWord::TYPE_PTR:
-      sprintf(buff, "    .type %s\n", word.symbol_name.c_str());
+      sprintf(buff, "    .type %s\n", word.symbol_name().c_str());
       break;
     case LinkedWord::EMPTY_PTR:
       sprintf(buff, "    .empty-list\n");  // ?
       break;
     case LinkedWord::HI_PTR:
       sprintf(buff, "    .ptr-hi 0x%x %s\n", word.data >> 16,
-              labels.at(word.label_id).name.c_str());
+              labels.at(word.label_id()).name.c_str());
       break;
     case LinkedWord::LO_PTR:
       sprintf(buff, "    .ptr-lo 0x%x %s\n", word.data >> 16,
-              labels.at(word.label_id).name.c_str());
+              labels.at(word.label_id()).name.c_str());
       break;
     case LinkedWord::SYM_OFFSET:
-      sprintf(buff, "    .sym-off 0x%x %s\n", word.data >> 16, word.symbol_name.c_str());
+      sprintf(buff, "    .sym-off 0x%x %s\n", word.data >> 16, word.symbol_name().c_str());
       break;
     default:
       throw std::runtime_error("nyi");
@@ -326,8 +320,8 @@ void LinkedObjectFile::find_code() {
     // single segment object files should never have any code.
     auto& seg = words_by_seg.front();
     for (auto& word : seg) {
-      if (!word.symbol_name.empty()) {
-        assert(word.symbol_name != "function");
+      if (word.kind() == LinkedWord::TYPE_PTR) {
+        assert(word.symbol_name() != "function");
       }
     }
     offset_of_data_zone_by_seg.at(0) = 0;
@@ -346,7 +340,7 @@ void LinkedObjectFile::find_code() {
       size_t function_loc = -1;
       for (size_t j = words_by_seg.at(i).size(); j-- > 0;) {
         auto& word = words_by_seg.at(i).at(j);
-        if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "function") {
+        if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "function") {
           function_loc = j;
           found_function = true;
           break;
@@ -361,7 +355,7 @@ void LinkedObjectFile::find_code() {
 
         for (size_t j = function_loc; j < words_by_seg.at(i).size(); j++) {
           auto& word = words_by_seg.at(i).at(j);
-          if (word.kind == LinkedWord::PLAIN_DATA && word.data == jr_ra) {
+          if (word.kind() == LinkedWord::PLAIN_DATA && word.data == jr_ra) {
             found_jr_ra = true;
             jr_ra_loc = j;
           }
@@ -385,7 +379,7 @@ void LinkedObjectFile::find_code() {
       // verify there are no functions after the data section starts
       for (size_t j = offset_of_data_zone_by_seg.at(i); j < words_by_seg.at(i).size(); j++) {
         auto& word = words_by_seg.at(i).at(j);
-        if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "function") {
+        if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "function") {
           assert(false);
         }
       }
@@ -421,7 +415,7 @@ void LinkedObjectFile::find_functions() {
         bool found_function_tag_loc = false;
         for (; function_tag_loc-- > 0;) {
           auto& word = words_by_seg.at(seg).at(function_tag_loc);
-          if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "function") {
+          if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "function") {
             found_function_tag_loc = true;
             break;
           }
@@ -729,7 +723,7 @@ std::string LinkedObjectFile::print_disassembly(bool write_hex) {
       auto& word = words_by_seg[seg][i];
       append_word_to_string(result, word);
 
-      if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "string") {
+      if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "string") {
         result += "; " + get_goal_string(seg, i) + "\n";
       }
     }
@@ -751,7 +745,7 @@ std::string LinkedObjectFile::get_goal_string(int seg, int word_idx, bool with_q
     return "invalid string!\n";
   }
   const LinkedWord& size_word = words_by_seg[seg].at(word_idx + 1);
-  if (size_word.kind != LinkedWord::PLAIN_DATA) {
+  if (size_word.kind() != LinkedWord::PLAIN_DATA) {
     // sometimes an array of string pointer triggers this!
     return "invalid string!\n";
   }
@@ -762,7 +756,7 @@ std::string LinkedObjectFile::get_goal_string(int seg, int word_idx, bool with_q
     int word_offset = word_idx + 2 + (i / 4);
     int byte_offset = i % 4;
     auto& word = words_by_seg[seg].at(word_offset);
-    if (word.kind != LinkedWord::PLAIN_DATA) {
+    if (word.kind() != LinkedWord::PLAIN_DATA) {
       return "invalid string! (check me!)\n";
     }
     char cword[4];
@@ -827,7 +821,7 @@ std::string LinkedObjectFile::print_scripts() {
 bool LinkedObjectFile::is_empty_list(int seg, int byte_idx) {
   assert((byte_idx % 4) == 0);
   auto& word = words_by_seg.at(seg).at(byte_idx / 4);
-  return word.kind == LinkedWord::EMPTY_PTR;
+  return word.kind() == LinkedWord::EMPTY_PTR;
 }
 
 /*!
@@ -866,9 +860,10 @@ goos::Object LinkedObjectFile::to_form_script(int seg, int word_idx, std::vector
         assert((cdr_addr % 4) == 0);
         auto& cdr_word = words_by_seg.at(seg).at(cdr_addr / 4);
         // check for proper list
-        if (cdr_word.kind == LinkedWord::PTR && (labels.at(cdr_word.label_id).offset & 7) == 2) {
+        if (cdr_word.kind() == LinkedWord::PTR &&
+            (labels.at(cdr_word.label_id()).offset & 7) == 2) {
           // yes, proper list. add another pair and link it in to the list.
-          goal_print_obj = labels.at(cdr_word.label_id).offset;
+          goal_print_obj = labels.at(cdr_word.label_id()).offset;
           fill.as_pair()->cdr = goos::PairObject::make_new(goos::Object::make_empty_list(),
                                                            goos::Object::make_empty_list());
           fill = fill.as_pair()->cdr;
@@ -900,7 +895,7 @@ bool LinkedObjectFile::is_string(int seg, int byte_idx) const {
     return false;
   }
   auto& type_word = words_by_seg.at(seg).at(type_tag_ptr / 4);
-  return type_word.kind == LinkedWord::TYPE_PTR && type_word.symbol_name == "string";
+  return type_word.kind() == LinkedWord::TYPE_PTR && type_word.symbol_name() == "string";
 }
 
 /*!
@@ -915,15 +910,15 @@ goos::Object LinkedObjectFile::to_form_script_object(int seg,
     case 0:
     case 4: {
       auto& word = words_by_seg.at(seg).at(byte_idx / 4);
-      if (word.kind == LinkedWord::SYM_PTR) {
+      if (word.kind() == LinkedWord::SYM_PTR) {
         // .symbol xxxx
-        result = pretty_print::to_symbol(word.symbol_name);
-      } else if (word.kind == LinkedWord::PLAIN_DATA) {
+        result = pretty_print::to_symbol(word.symbol_name());
+      } else if (word.kind() == LinkedWord::PLAIN_DATA) {
         // .word xxxxx
         result = pretty_print::to_symbol(std::to_string(word.data));
-      } else if (word.kind == LinkedWord::PTR) {
+      } else if (word.kind() == LinkedWord::PTR) {
         // might be a sub-list, or some other random pointer
-        auto offset = labels.at(word.label_id).offset;
+        auto offset = labels.at(word.label_id()).offset;
         if ((offset & 7) == 2) {
           // list!
           result = to_form_script(seg, offset / 4, seen);
@@ -932,10 +927,10 @@ goos::Object LinkedObjectFile::to_form_script_object(int seg,
             result = pretty_print::to_symbol(get_goal_string(seg, offset / 4 - 1));
           } else {
             // some random pointer, just print the label.
-            result = pretty_print::to_symbol(labels.at(word.label_id).name);
+            result = pretty_print::to_symbol(labels.at(word.label_id()).name);
           }
         }
-      } else if (word.kind == LinkedWord::EMPTY_PTR) {
+      } else if (word.kind() == LinkedWord::EMPTY_PTR) {
         result = goos::Object::make_empty_list();
       } else {
         std::string debug;
@@ -958,7 +953,7 @@ goos::Object LinkedObjectFile::to_form_script_object(int seg,
 u32 LinkedObjectFile::read_data_word(const DecompilerLabel& label) {
   assert(0 == (label.offset % 4));
   auto& word = words_by_seg.at(label.target_segment).at(label.offset / 4);
-  assert(word.kind == LinkedWord::Kind::PLAIN_DATA);
+  assert(word.kind() == LinkedWord::Kind::PLAIN_DATA);
   return word.data;
 }
 

decompiler/ObjectFile/LinkedWord.h

@@ -7,16 +7,14 @@
 
 #include <cstdint>
 #include <string>
+#include <cstring>
 #include "common/util/assert.h"
-
 #include "common/common_types.h"
 
 namespace decompiler {
 class LinkedWord {
  public:
-  explicit LinkedWord(uint32_t _data) : data(_data) {}
-
-  enum Kind {
+  enum Kind : u8 {
     PLAIN_DATA,  // just plain data
     PTR,         // pointer to a location
     HI_PTR,      // lower 16-bits of this data are the upper 16 bits of a pointer
@@ -25,15 +23,98 @@ class LinkedWord {
     EMPTY_PTR,   // this is a pointer to the empty list
     SYM_OFFSET,  // this is an offset of a symbol in the symbol table
     TYPE_PTR     // this is a pointer to a type
-  } kind = PLAIN_DATA;
+  };
 
-  uint32_t data = 0;
+ private:
+  uintptr_t m_data_ptr = 0;  // 8 bytes
+ public:
+  u32 data;
 
-  int label_id = -1;
-  std::string symbol_name;
+ private:
+  Kind m_kind = PLAIN_DATA;
+
+ public:
+  explicit LinkedWord(uint32_t _data) : data(_data) {}
+
+  bool holds_string() const {
+    return m_kind == SYM_PTR || m_kind == SYM_OFFSET || m_kind == TYPE_PTR;
+  }
+
+  LinkedWord(const LinkedWord& other) {
+    if (&other != this) {
+      data = other.data;
+      m_kind = other.m_kind;
+      if (holds_string()) {
+        const char* other_str = (const char*)(other.m_data_ptr);
+        char* str = new char[strlen(other_str) + 1];
+        strcpy(str, other_str);
+        m_data_ptr = (uintptr_t)str;
+      } else {
+        m_data_ptr = other.m_data_ptr;
+      }
+    }
+  }
+
+  LinkedWord& operator=(const LinkedWord& other) {
+    if (&other != this) {
+      data = other.data;
+      if (holds_string()) {
+        delete[]((char*)m_data_ptr);
+      }
+      m_kind = other.m_kind;
+
+      if (holds_string()) {
+        const char* other_str = (const char*)(other.m_data_ptr);
+        char* str = new char[strlen(other_str) + 1];
+        strcpy(str, other_str);
+        m_data_ptr = (uintptr_t)str;
+      } else {
+        m_data_ptr = other.m_data_ptr;
+      }
+    }
+    return *this;
+  }
+
+  ~LinkedWord() {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+  }
+
+  void set_to_empty_ptr() {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+    m_kind = EMPTY_PTR;
+  }
+
+  void set_to_symbol(Kind kind, const std::string& name) {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+    m_kind = kind;
+    char* str = new char[name.size() + 1];
+    strcpy(str, name.c_str());
+    m_data_ptr = (uintptr_t)str;
+  }
+
+  void set_to_pointer(Kind kind, u32 label_id) {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+    m_data_ptr = label_id;
+    m_kind = kind;
+  }
+
+  void set_to_plain_data() {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+    m_kind = PLAIN_DATA;
+  }
 
   u8 get_byte(int idx) const {
-    assert(kind == PLAIN_DATA);
+    assert(kind() == PLAIN_DATA);
     switch (idx) {
       case 0:
         return data & 0xff;
@@ -48,5 +129,25 @@ class LinkedWord {
         return 0;
     }
   }
+
+  // kind, label_id, symbol_name
+  Kind kind() const { return m_kind; }
+
+  u32 label_id() const {
+    assert(m_kind == PTR || m_kind == LO_PTR || m_kind == HI_PTR);
+    return m_data_ptr;
+  }
+
+  std::string symbol_name() const {
+    assert(holds_string());
+    return (const char*)(m_data_ptr);
+  }
+
+ private:
+  // if plain data, this is empty.
+  // otherwise:
+  //  u8 (kind)
+  //  u32 label id | actual string.
 };
+
 }  // namespace decompiler

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -428,7 +428,7 @@ void ObjectFileDB::process_link_data(const Config& config) {
   });
 
   lg::info("Processed Link Data");
-  lg::info(" Total {} ms\n", process_link_timer.getMs());
+  lg::info(" Total {:.2f} ms\n", process_link_timer.getMs());
   // printf("\n");
 }
 
@@ -443,7 +443,7 @@ void ObjectFileDB::process_labels() {
 
   lg::info("Processed Labels:");
   lg::info(" Total {} labels", total);
-  lg::info(" Total {} ms\n", process_label_timer.getMs());
+  lg::info(" Total {:.2f} ms\n", process_label_timer.getMs());
 }
 
 /*!
@@ -579,7 +579,7 @@ void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) {
   lg::info(" Total {:.3f} ms\n", timer.getMs());
 }
 
-std::string ObjectFileDB::process_tpages() {
+std::string ObjectFileDB::process_tpages(TextureDB& tex_db) {
   lg::info("- Finding textures in tpages...");
   std::string tpage_string = "tpage-";
   int total = 0, success = 0;
@@ -590,7 +590,7 @@ std::string ObjectFileDB::process_tpages() {
   std::string result;
   for_each_obj([&](ObjectFileData& data) {
     if (data.name_in_dgo.substr(0, tpage_string.length()) == tpage_string) {
-      auto statistics = process_tpage(data);
+      auto statistics = process_tpage(data, tex_db);
       total += statistics.total_textures;
       success += statistics.successful_textures;
       total_px += statistics.num_px;
@@ -602,13 +602,13 @@ std::string ObjectFileDB::process_tpages() {
 
   assert(tpage_dir_count <= 1);
 
+  lg::info("Processed {} / {} textures ({} px) {:.2f}% in {:.2f} ms", success, total, total_px,
+           100.f * float(success) / float(total), timer.getMs());
+
   if (tpage_dir_count == 0) {
     lg::warn("Did not find tpage-dir.");
     return {};
   }
-
-  lg::info("Processed {} / {} textures ({} px) {:.2f}% in {:.2f} ms", success, total, total_px,
-           100.f * float(success) / float(total), timer.getMs());
   return result;
 }
 

decompiler/ObjectFile/ObjectFileDB.h

@@ -14,6 +14,7 @@
 #include "LinkedObjectFile.h"
 #include "decompiler/util/DecompilerTypeSystem.h"
 #include "common/common_types.h"
+#include "decompiler/data/TextureDB.h"
 #include "decompiler/analysis/symbol_def_map.h"
 
 namespace decompiler {
@@ -96,7 +97,7 @@ class ObjectFileDB {
   std::string ir2_final_out(ObjectFileData& data,
                             const std::unordered_set<std::string>& skip_functions = {});
 
-  std::string process_tpages();
+  std::string process_tpages(TextureDB& tex_db);
   std::string process_game_count_file();
   std::string process_game_text_files();
 

decompiler/ObjectFile/ObjectFileDB_IR2.cpp

@@ -706,7 +706,7 @@ std::string ObjectFileDB::ir2_to_file(ObjectFileData& data, const Config& config
       auto& word = data.linked_data.words_by_seg[seg][i];
       data.linked_data.append_word_to_string(result, word);
 
-      if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "string") {
+      if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "string") {
         result += "; " + data.linked_data.get_goal_string(seg, i) + "\n";
       }
     }

decompiler/analysis/find_defstates.cpp

@@ -230,24 +230,24 @@ std::string verify_empty_state_and_get_name(DecompiledDataElement* state, const
   auto& words = env.file->words_by_seg.at(lab.target_segment);
 
   auto first_word = words.at(start_word_idx);
-  if (first_word.kind != LinkedWord::TYPE_PTR || first_word.symbol_name != "state") {
+  if (first_word.kind() != LinkedWord::TYPE_PTR || first_word.symbol_name() != "state") {
     env.func->warnings.warn_and_throw("Reference to state bad: invalid type pointer");
   }
 
   auto name_word = words.at(start_word_idx + 1);
-  if (name_word.kind != LinkedWord::SYM_PTR) {
+  if (name_word.kind() != LinkedWord::SYM_PTR) {
     env.func->warnings.warn_and_throw("Reference to state bad: invalid name");
   }
 
   for (int i = 0; i < 7; i++) {
     auto& word = words.at(start_word_idx + 2 + i);
-    if (word.kind != LinkedWord::SYM_PTR || word.symbol_name != "#f") {
+    if (word.kind() != LinkedWord::SYM_PTR || word.symbol_name() != "#f") {
       env.func->warnings.warn_and_throw(
           "Reference to state bad: got a non #f in the initial fields");
     }
   }
 
-  return name_word.symbol_name;
+  return name_word.symbol_name();
 }
 
 FormElement* rewrite_virtual_defstate(

decompiler/analysis/find_skelgroups.cpp

@@ -61,34 +61,34 @@ DefskelgroupElement::StaticInfo inspect_skel_group_data(DecompiledDataElement* s
   auto& words = env.file->words_by_seg.at(lab.target_segment);
 
   auto& type_word = words.at(start_word_idx - 1);
-  if (type_word.kind != LinkedWord::TYPE_PTR || type_word.symbol_name != "skeleton-group") {
+  if (type_word.kind() != LinkedWord::TYPE_PTR || type_word.symbol_name() != "skeleton-group") {
     env.func->warnings.warn_and_throw("Reference to skelgroup bad: invalid type pointer");
   }
   auto& string_word = words.at(start_word_idx);
-  if (string_word.kind != LinkedWord::PTR) {
+  if (string_word.kind() != LinkedWord::PTR) {
     env.func->warnings.warn_and_throw("Reference to skelgroup bad: invalid name label");
   }
   result.art_name = env.file->get_goal_string_by_label(
-      env.file->get_label_by_name(env.file->get_label_name(string_word.label_id)));
+      env.file->get_label_by_name(env.file->get_label_name(string_word.label_id())));
   for (int i = 0; i < 4; i++) {
     auto& word = words.at(start_word_idx + 3 + i);
-    if (word.kind != LinkedWord::PLAIN_DATA) {
+    if (word.kind() != LinkedWord::PLAIN_DATA) {
       env.func->warnings.warn_and_throw("Reference to skelgroup bad: invalid bounds");
     }
     result.bounds[i] = *reinterpret_cast<float*>(&word.data);
   }
   auto& lod_word = words.at(start_word_idx + 9);
-  if (lod_word.kind != LinkedWord::PLAIN_DATA) {
+  if (lod_word.kind() != LinkedWord::PLAIN_DATA) {
     env.func->warnings.warn_and_throw("Reference to skelgroup bad: invalid max-lod");
   }
   result.max_lod = lod_word.data;
   auto& edge_word = words.at(start_word_idx + 14);
-  if (edge_word.kind != LinkedWord::PLAIN_DATA) {
+  if (edge_word.kind() != LinkedWord::PLAIN_DATA) {
     env.func->warnings.warn_and_throw("Reference to skelgroup bad: invalid longest-edge");
   }
   result.longest_edge = *reinterpret_cast<float*>(&edge_word.data);
   auto& other_word = words.at(start_word_idx + 15);
-  if (other_word.kind != LinkedWord::PLAIN_DATA) {
+  if (other_word.kind() != LinkedWord::PLAIN_DATA) {
     env.func->warnings.warn_and_throw("Reference to skelgroup bad: invalid other data");
   }
   result.tex_level = other_word.get_byte(0);

decompiler/analysis/label_types.cpp

@@ -80,8 +80,8 @@ void find_boxed(LabelDB* db, LinkedObjectFile* file) {
       const auto& word = file->words_by_seg.at(lab.target_segment).at((lab.offset - 4) / 4);
       // the snowball-bank is a weird basic with no fields other than the built-in type.
       // so it can actually share a label with something else.
-      if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name != "snowball-bank") {
-        TypeSpec basic_type(word.symbol_name);
+      if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() != "snowball-bank") {
+        TypeSpec basic_type(word.symbol_name());
         const auto& existing = db->lookup(lab.name);
         if (existing.known) {
           if (existing.result_type.base_type() != basic_type.base_type()) {

decompiler/analysis/mips2c.cpp

@@ -516,7 +516,7 @@ Mips2C_Line handle_lwc1(const Instruction& i0,
   if (i0.get_src(0).is_label() && i0.get_src(1).is_reg(Register(Reg::GPR, Reg::FP))) {
     auto& label = file->labels.at(i0.get_src(0).get_label());
     auto& word = file->words_by_seg.at(label.target_segment).at(label.offset / 4);
-    assert(word.kind == LinkedWord::PLAIN_DATA);
+    assert(word.kind() == LinkedWord::PLAIN_DATA);
     float f;
     memcpy(&f, &word.data, 4);
     return {fmt::format("c->fprs[{}] = {};", reg_to_name(i0.get_dst(0)), float_to_string(f)),

decompiler/config.cpp

@@ -192,8 +192,15 @@ Config read_config_file(const std::string& path_to_config_file) {
         max_len;
   }
 
+  for (auto& entry : hacks_json.at("missing_textures")) {
+    int tpage = entry.at(1).get<int>();
+    int idx = entry.at(2).get<int>();
+    config.hacks.missing_textures_by_level[entry.at(0).get<std::string>()].emplace_back(tpage, idx);
+  }
+
   config.bad_format_strings =
       hacks_json.at("bad_format_strings").get<std::unordered_map<std::string, int>>();
+  config.levels_to_extract = cfg.at("levels_to_extract").get<std::vector<std::string>>();
   return config;
 }
 

decompiler/config.h

@@ -76,6 +76,7 @@ struct DecompileHacks {
   std::unordered_map<std::string, std::vector<std::vector<int>>>
       format_ops_with_dynamic_string_by_func_name;
   std::unordered_set<std::string> mips2c_functions_by_name;
+  std::unordered_map<std::string, std::vector<std::pair<int, int>>> missing_textures_by_level;
 };
 
 struct Config {
@@ -125,6 +126,8 @@ struct Config {
 
   std::unordered_map<std::string, int> bad_format_strings;
 
+  std::vector<std::string> levels_to_extract;
+
   DecompileHacks hacks;
 };
 

decompiler/config/all-types.gc

@@ -12933,6 +12933,7 @@
    (pad0              uint8                        :offset 58)
    (pad1              uint8                        :offset 59)
    (generic           generic-tfragment            :offset-assert 60)
+   (generic-u32       uint32                       :offset 60)
    )
   :method-count-assert 18
   :size-assert         #x40

decompiler/config/jak1_ntsc_black_label.jsonc

@@ -15,7 +15,7 @@
   "disassemble_code": false,
 
   // Run the decompiler
-  "decompile_code": true,
+  "decompile_code": false,
 
   ////////////////////////////
   // DATA ANALYSIS OPTIONS
@@ -77,5 +77,36 @@
 
   // optional: a predetermined object file name map from a file.
   // this will make decompilation naming consistent even if you only run on some objects.
-  "obj_file_name_map_file": "goal_src/build/all_objs.json"
+  "obj_file_name_map_file": "goal_src/build/all_objs.json",
+
+  ////////////////////////////
+  // LEVEL EXTRACTION
+  ////////////////////////////
+
+  "levels_to_extract":[
+    "BEA.DGO",
+    "CIT.DGO",
+    "DAR.DGO",
+    "DEM.DGO",
+    "FIN.DGO",
+    "INT.DGO",
+    "JUB.DGO",
+    "JUN.DGO",
+    "FIC.DGO",
+    "LAV.DGO",
+    "MAI.DGO",
+    "MIS.DGO",
+    "OGR.DGO",
+    "ROB.DGO",
+    "ROL.DGO",
+    "SNO.DGO",
+    "SUB.DGO",
+    "SUN.DGO",
+    "SWA.DGO",
+    "TIT.DGO",
+    "TRA.DGO",
+    "VI1.DGO",
+    "VI2.DGO",
+    "VI3.DGO"
+  ]
 }

decompiler/config/jak1_ntsc_black_label/hacks.jsonc

@@ -533,6 +533,12 @@
     "draw-inline-array-tfrag",
     "stats-tfrag-asm",
     "time-of-day-interp-colors-scratch"
+  ],
+
+  // there are some missing textures. I don't know what the game actually does here.
+  // the format for entries is [level, tpage, index]
+  "missing_textures": [
+    ["finalboss", 1419, 3]
   ]
 }
 

decompiler/data/LinkedWordReader.h

@@ -11,9 +11,9 @@ namespace decompiler {
 class LinkedWordReader {
  public:
   explicit LinkedWordReader(const std::vector<LinkedWord>* words) : m_words(words) {}
-  const std::string& get_type_tag() {
-    if (m_words->at(m_offset).kind == LinkedWord::TYPE_PTR) {
-      auto& result = m_words->at(m_offset).symbol_name;
+  std::string get_type_tag() {
+    if (m_words->at(m_offset).kind() == LinkedWord::TYPE_PTR) {
+      auto result = m_words->at(m_offset).symbol_name();
       m_offset++;
       return result;
     } else {
@@ -26,7 +26,7 @@ class LinkedWordReader {
   T get_word() {
     static_assert(sizeof(T) == 4, "size of word in get_word");
     T result;
-    assert(m_words->at(m_offset).kind == LinkedWord::PLAIN_DATA);
+    assert(m_words->at(m_offset).kind() == LinkedWord::PLAIN_DATA);
     memcpy(&result, &m_words->at(m_offset).data, 4);
     m_offset++;
     return result;

decompiler/data/TextureDB.cpp

@@ -0,0 +1,40 @@
+#include "TextureDB.h"
+
+#include "common/util/assert.h"
+#include "third-party/fmt/core.h"
+
+namespace decompiler {
+
+void TextureDB::add_texture(u32 tpage,
+                            u32 texid,
+                            const std::vector<u32>& data,
+                            u16 w,
+                            u16 h,
+                            const std::string& tex_name,
+                            const std::string& tpage_name) {
+  auto existing_tpage_name = tpage_names.find(tpage);
+  if (existing_tpage_name == tpage_names.end()) {
+    tpage_names[tpage] = tpage_name;
+  } else {
+    assert(existing_tpage_name->second == tpage_name);
+  }
+
+  u32 combo_id = (tpage << 16) | texid;
+  auto existing_tex = textures.find(combo_id);
+  if (existing_tex != textures.end()) {
+    assert(existing_tex->second.name == tex_name);
+    assert(existing_tex->second.w == w);
+    assert(existing_tex->second.h == h);
+    assert(existing_tex->second.rgba_bytes == data);
+    assert(existing_tex->second.page == tpage);
+  } else {
+    auto& new_tex = textures[combo_id];
+    new_tex.rgba_bytes = data;
+    new_tex.name = tex_name;
+    new_tex.w = w;
+    new_tex.h = h;
+    new_tex.page = tpage;
+  }
+}
+
+}  // namespace decompiler

decompiler/data/TextureDB.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <vector>
+#include <unordered_map>
+#include <string>
+#include "common/common_types.h"
+
+namespace decompiler {
+struct TextureDB {
+  struct TextureData {
+    u16 w, h;
+    std::string name;
+    u32 page;
+    std::vector<u32> rgba_bytes;
+  };
+
+  std::unordered_map<u32, TextureData> textures;
+  std::unordered_map<u32, std::string> tpage_names;
+
+  void add_texture(u32 tpage,
+                   u32 texid,
+                   const std::vector<u32>& data,
+                   u16 w,
+                   u16 h,
+                   const std::string& tex_name,
+                   const std::string& tpage_name);
+};
+}  // namespace decompiler

decompiler/data/dir_tpages.cpp

@@ -20,27 +20,26 @@ DirTpageResult process_dir_tpages(ObjectFileData& data) {
 
   int word_idx = 0;
   // first is type
-  assert(words.at(word_idx).kind == LinkedWord::TYPE_PTR);
-  assert(words.at(word_idx).symbol_name == "texture-page-dir");
+  assert(words.at(word_idx).kind() == LinkedWord::TYPE_PTR);
+  assert(words.at(word_idx).symbol_name() == "texture-page-dir");
   word_idx++;
   // next is length
-  assert(words.at(word_idx).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(word_idx).kind() == LinkedWord::PLAIN_DATA);
   int dir_length = words.at(word_idx).data;
-  fmt::print("length: {}\n", dir_length);
   word_idx++;
 
   for (int i = 0; i < dir_length; i++) {
-    assert(words.at(word_idx).kind == LinkedWord::PLAIN_DATA);
+    assert(words.at(word_idx).kind() == LinkedWord::PLAIN_DATA);
     u32 entry = words.at(word_idx).data;
     assert((entry & 0xffff7000) == 0);  // 7 checks for sign bit.
     word_idx++;
     result.lengths.push_back(entry & 0xffff);
 
-    assert(words.at(word_idx).kind == LinkedWord::SYM_PTR);
-    assert(words.at(word_idx).symbol_name == "#f");
+    assert(words.at(word_idx).kind() == LinkedWord::SYM_PTR);
+    assert(words.at(word_idx).symbol_name() == "#f");
     word_idx++;
-    assert(words.at(word_idx).kind == LinkedWord::SYM_PTR);
-    assert(words.at(word_idx).symbol_name == "#f");
+    assert(words.at(word_idx).kind() == LinkedWord::SYM_PTR);
+    assert(words.at(word_idx).symbol_name() == "#f");
     word_idx++;
   }
 

decompiler/data/game_text.cpp

@@ -14,15 +14,15 @@ namespace {
 template <typename T>
 T get_word(const LinkedWord& word) {
   T result;
-  assert(word.kind == LinkedWord::PLAIN_DATA);
+  assert(word.kind() == LinkedWord::PLAIN_DATA);
   static_assert(sizeof(T) == 4, "bad get_word size");
   memcpy(&result, &word.data, 4);
   return result;
 }
 
 DecompilerLabel get_label(ObjectFileData& data, const LinkedWord& word) {
-  assert(word.kind == LinkedWord::PTR);
-  return data.linked_data.labels.at(word.label_id);
+  assert(word.kind() == LinkedWord::PTR);
+  return data.linked_data.labels.at(word.label_id());
 }
 
 }  // namespace
@@ -51,8 +51,8 @@ GameTextResult process_game_text(ObjectFileData& data) {
   int offset = 0;
 
   // type tage for game-text-info
-  if (words.at(offset).kind != LinkedWord::TYPE_PTR ||
-      words.front().symbol_name != "game-text-info") {
+  if (words.at(offset).kind() != LinkedWord::TYPE_PTR ||
+      words.front().symbol_name() != "game-text-info") {
     assert(false);
   }
   read_words.at(offset)++;

decompiler/data/tpage.cpp

@@ -223,23 +223,23 @@ int label_to_word_offset(DecompilerLabel l, bool basic) {
 }
 
 std::string get_type_tag(const LinkedWord& word) {
-  assert(word.kind == LinkedWord::TYPE_PTR);
-  return word.symbol_name;
+  assert(word.kind() == LinkedWord::TYPE_PTR);
+  return word.symbol_name();
 }
 
 bool is_type_tag(const LinkedWord& word, const std::string& type) {
-  return word.kind == LinkedWord::TYPE_PTR && word.symbol_name == type;
+  return word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == type;
 }
 
 DecompilerLabel get_label(ObjectFileData& data, const LinkedWord& word) {
-  assert(word.kind == LinkedWord::PTR);
-  return data.linked_data.labels.at(word.label_id);
+  assert(word.kind() == LinkedWord::PTR);
+  return data.linked_data.labels.at(word.label_id());
 }
 
 template <typename T>
 T get_word(const LinkedWord& word) {
   T result;
-  assert(word.kind == LinkedWord::PLAIN_DATA);
+  assert(word.kind() == LinkedWord::PLAIN_DATA);
   static_assert(sizeof(T) == 4, "bad get_word size");
   memcpy(&result, &word.data, 4);
   return result;
@@ -385,8 +385,8 @@ TexturePage read_texture_page(ObjectFileData& data,
   }
 
   for (int i = 0; i < tpage.length; i++) {
-    if (words.at(offset).kind == LinkedWord::SYM_PTR) {
-      if (words.at(offset).symbol_name == "#f") {
+    if (words.at(offset).kind() == LinkedWord::SYM_PTR) {
+      if (words.at(offset).symbol_name() == "#f") {
         tpage.data.emplace_back();
         Texture null_tex;
         null_tex.null_texture = true;
@@ -415,7 +415,7 @@ TexturePage read_texture_page(ObjectFileData& data,
  * Process a texture page.
  * TODO - document
  */
-TPageResultStats process_tpage(ObjectFileData& data) {
+TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db) {
   TPageResultStats stats;
   auto& words = data.linked_data.words_by_seg.at(0);
 
@@ -464,7 +464,9 @@ TPageResultStats process_tpage(ObjectFileData& data) {
   }
 
   // get all textures in the tpage
-  for (auto& tex : texture_page.textures) {
+  for (u32 tex_id = 0; tex_id < texture_page.textures.size(); tex_id++) {
+    auto& tex = texture_page.textures.at(tex_id);
+
     // I think these get inserted for CLUTs, but I'm not sure.
     if (tex.null_texture) {
       continue;
@@ -520,6 +522,8 @@ TPageResultStats process_tpage(ObjectFileData& data) {
                           {"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
                       data.name_in_dgo, tex.name, tex.w, tex.h),
           out.data(), tex.w, tex.h);
+      texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
+                             texture_page.name);
       stats.successful_textures++;
     } else if (tex.psm == int(PSM::PSMT8) && tex.clutpsm == int(CPSM::PSMCT16)) {
       // will store output pixels, rgba (8888)
@@ -566,6 +570,8 @@ TPageResultStats process_tpage(ObjectFileData& data) {
                           {"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
                       data.name_in_dgo, tex.name, tex.w, tex.h),
           out.data(), tex.w, tex.h);
+      texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
+                             texture_page.name);
       stats.successful_textures++;
     } else if (tex.psm == int(PSM::PSMCT16) && tex.clutpsm == 0) {
       // not a clut.
@@ -594,6 +600,8 @@ TPageResultStats process_tpage(ObjectFileData& data) {
                           {"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
                       data.name_in_dgo, tex.name, tex.w, tex.h),
           out.data(), tex.w, tex.h);
+      texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
+                             texture_page.name);
       stats.successful_textures++;
     } else if (tex.psm == int(PSM::PSMT4) && tex.clutpsm == int(CPSM::PSMCT16)) {
       // will store output pixels, rgba (8888)
@@ -638,6 +646,8 @@ TPageResultStats process_tpage(ObjectFileData& data) {
                           {"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
                       data.name_in_dgo, tex.name, tex.w, tex.h),
           out.data(), tex.w, tex.h);
+      texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
+                             texture_page.name);
       stats.successful_textures++;
     } else if (tex.psm == int(PSM::PSMT4) && tex.clutpsm == int(CPSM::PSMCT32)) {
       // will store output pixels, rgba (8888)
@@ -682,6 +692,8 @@ TPageResultStats process_tpage(ObjectFileData& data) {
                           {"assets", "textures", texture_page.name, "{}-{}-{}-{}.png"}),
                       data.name_in_dgo, tex.name, tex.w, tex.h),
           out.data(), tex.w, tex.h);
+      texture_db.add_texture(texture_page.id, tex_id, out, tex.w, tex.h, tex.name,
+                             texture_page.name);
       stats.successful_textures++;
     }
 

decompiler/data/tpage.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "decompiler/data/TextureDB.h"
+
 namespace decompiler {
 struct ObjectFileData;
 
@@ -9,5 +11,5 @@ struct TPageResultStats {
   int num_px = 0;
 };
 
-TPageResultStats process_tpage(ObjectFileData& data);
+TPageResultStats process_tpage(ObjectFileData& data, TextureDB& texture_db);
 }  // namespace decompiler

decompiler/level_extractor/BspHeader.cpp

@@ -2,8 +2,8 @@
 
 #include "decompiler/ObjectFile/LinkedObjectFile.h"
 #include "decompiler/util/DecompilerTypeSystem.h"
-#include "tools/level_tools/goal_data_reader.h"
-#include "tools/level_tools/Error.h"
+#include "decompiler/util/goal_data_reader.h"
+#include "decompiler/util/Error.h"
 #include "common/dma/dma.h"
 
 namespace level_tools {
@@ -24,7 +24,7 @@ void Vector::read_from_file(Ref ref) {
   }
   for (int i = 0; i < 4; i++) {
     const auto& word = ref.data->words_by_seg.at(ref.seg).at((ref.byte_offset / 4) + i);
-    if (word.kind != decompiler::LinkedWord::PLAIN_DATA) {
+    if (word.kind() != decompiler::LinkedWord::PLAIN_DATA) {
       throw Error("vector didn't get plain data.");
     }
     memcpy(data + i, &word.data, 4);
@@ -148,8 +148,8 @@ void TFragmentDebugData::read_from_file(Ref ref,
   auto& words = ref.data->words_by_seg.at(ref.seg);
   for (int i = 0; i < 4; i++) {
     auto& word = words.at((ref.byte_offset / 4) + i);
-    if (word.kind != decompiler::LinkedWord::PLAIN_DATA) {
-      throw Error("debug data word type: {}\n", (int)word.kind);
+    if (word.kind() != decompiler::LinkedWord::PLAIN_DATA) {
+      throw Error("debug data word type: {}\n", (int)word.kind());
     }
     data[i] = word.data;
   }
@@ -164,7 +164,7 @@ void TFragmentDebugData::read_from_file(Ref ref,
   stats->total_tfrag_tris += tris;
 
   auto& debug_word = words.at(4 + (ref.byte_offset / 4));
-  if (debug_word.kind != decompiler::LinkedWord::PLAIN_DATA || debug_word.data != 0) {
+  if (debug_word.kind() != decompiler::LinkedWord::PLAIN_DATA || debug_word.data != 0) {
     throw Error("got debug word.");
   }
 }
@@ -184,8 +184,8 @@ u32 deref_u32(const Ref& ref, int word_offset) {
     throw Error("deref_u32 bad alignment");
   }
   const auto& word = ref.data->words_by_seg.at(ref.seg).at(word_offset + (ref.byte_offset / 4));
-  if (word.kind != decompiler::LinkedWord::PLAIN_DATA) {
-    throw Error("deref_u32 bad kind");
+  if (word.kind() != decompiler::LinkedWord::PLAIN_DATA) {
+    throw Error("deref_u32 bad kind: {}", (int)word.kind());
   }
   return word.data;
 }
@@ -286,6 +286,16 @@ void tfrag_debug_print_unpack(Ref start, int qwc_total) {
   fmt::print("-------------------------------------------\n");
 }
 
+std::vector<u8> read_dma_chain(Ref& start, u32 qwc) {
+  std::vector<u8> result;
+  result.resize(qwc * 16);
+  for (u32 word = 0; word < qwc * 4; word++) {
+    u32 x = deref_u32(start, word);
+    memcpy(result.data() + (word * 4), &x, 4);
+  }
+  return result;
+}
+
 void TFragment::read_from_file(TypedRef ref,
                                const decompiler::DecompilerTypeSystem& dts,
                                DrawStats* stats) {
@@ -298,7 +308,7 @@ void TFragment::read_from_file(TypedRef ref,
 
   auto dma_field = get_field_ref(ref, "dma-qwc", dts);
   auto& dma_word = ref.ref.data->words_by_seg.at(dma_field.seg).at(dma_field.byte_offset / 4);
-  if (dma_word.kind != decompiler::LinkedWord::PLAIN_DATA) {
+  if (dma_word.kind() != decompiler::LinkedWord::PLAIN_DATA) {
     throw Error("bad dma qwc word");
   }
   memcpy(dma_qwc, &dma_word.data, 4);
@@ -313,13 +323,12 @@ void TFragment::read_from_file(TypedRef ref,
   for (int i = 0; i < 3; i++) {
     auto& word = dma_slot.data->words_by_seg.at(dma_slot.seg).at((dma_slot.byte_offset / 4));
     dmas[i].ref = deref_label(dma_slot);
-    dmas[i].label_name = dma_slot.data->labels.at(word.label_id).name;
+    dmas[i].label_name = dma_slot.data->labels.at(word.label_id()).name;
 
     dma_slot.byte_offset += 4;
   }
 
   if (stats->debug_print_dma_data) {
-    fmt::print("qwc's: {} {} {} {}\n", dma_qwc[0], dma_qwc[1], dma_qwc[2], dma_qwc[3]);
     // first, common
     fmt::print("DMA COMMON {}, {} qwc:\n", dmas[0].label_name, dma_qwc[0]);
     tfrag_debug_print_unpack(dmas[0].ref, dma_qwc[0]);
@@ -335,22 +344,50 @@ void TFragment::read_from_file(TypedRef ref,
     // next "level1"
     fmt::print("DMA LEVEL1 {}, {} qwc:\n", dmas[2].label_name, dma_qwc[2]);
     tfrag_debug_print_unpack(dmas[2].ref, dma_qwc[2]);
+
+    fmt::print("qwc's: {} {} {} {}\n", dma_qwc[0], dma_qwc[1], dma_qwc[2], dma_qwc[3]);
   }
 
-  // todo dma
-  // todo dma
-  // todo dma
-
-  // todo shader
-  num_shaders = read_plain_data_field<u8>(ref, "num-shaders", dts);
   num_base_colors = read_plain_data_field<u8>(ref, "num-base-colors", dts);
   num_level0_colors = read_plain_data_field<u8>(ref, "num-level0-colors", dts);
   num_level1_colors = read_plain_data_field<u8>(ref, "num-level1-colors", dts);
+  num_shaders = read_plain_data_field<u8>(ref, "num-shaders", dts);
   color_offset = read_plain_data_field<u8>(ref, "color-offset", dts);
   color_count = read_plain_data_field<u8>(ref, "color-count", dts);
+  dma_base = read_dma_chain(dmas[1].ref, dma_qwc[1]);
+
+  if (num_level0_colors > 0 || num_level1_colors > 0) {
+    // if we're base only, this has junk in it, and it wouldn't be used anyway.
+    dma_common_and_level0 = read_dma_chain(dmas[0].ref, std::max(dma_qwc[3], dma_qwc[0]));
+  }
+
+  dma_level1 = read_dma_chain(dmas[2].ref, dma_qwc[2]);
+
+  // color indices
+  int num_actual_colors = std::max(num_base_colors, std::max(num_level0_colors, num_level1_colors));
+  int num_colors = ((num_actual_colors + 3) / 4) * 4;
+  // each color is a u64 (4x u16 indices)
+  // color_indices.resize(4 * num_colors);
+
+  // 24 = 12 * u32
+  auto color_idx_start = deref_label(get_field_ref(ref, "color-indices", dts));
+  for (int i = 0; i < num_colors / 4; i++) {
+    u32 low = deref_u32(color_idx_start, i * 2);
+    u32 high = deref_u32(color_idx_start, i * 2 + 1);
+    color_indices.push_back(low & 0xffff);
+    color_indices.push_back(low >> 16);
+    color_indices.push_back(high & 0xffff);
+    color_indices.push_back(high >> 16);
+  }
+  assert((int)color_indices.size() == num_colors);
+
+  // todo shader
+
+  assert(num_colors / 4 == color_count);
+  // fmt::print("colors: {} {} {}\n", num_base_colors, num_level0_colors, num_level1_colors);
   assert(read_plain_data_field<u8>(ref, "pad0", dts) == 0);
   assert(read_plain_data_field<u8>(ref, "pad1", dts) == 0);
-  // todo generic
+  assert(read_plain_data_field<u32>(ref, "generic-u32", dts) == 0);
 }
 
 std::string TFragment::print(const PrintSettings& settings, int indent) const {
@@ -687,6 +724,17 @@ void DrawableTreeTfrag::read_from_file(TypedRef ref,
     throw Error("misaligned data array");
   }
 
+  auto palette = deref_label(get_field_ref(ref, "time-of-day-pal", dts));
+  time_of_day.width = deref_u32(palette, 0);
+
+  assert(time_of_day.width == 8);
+  time_of_day.height = deref_u32(palette, 1);
+  time_of_day.pad = deref_u32(palette, 2);
+  assert(time_of_day.pad == 0);
+  for (int i = 0; i < int(8 * time_of_day.height); i++) {
+    time_of_day.colors.push_back(deref_u32(palette, 3 + i));
+  }
+
   for (int idx = 0; idx < length; idx++) {
     Ref array_slot_ref = data_ref;
     array_slot_ref.byte_offset += idx * 4;
@@ -785,7 +833,7 @@ void PrototypeBucketTie::read_from_file(TypedRef ref,
   auto next_slot = get_field_ref(ref, "next", dts);
   for (int i = 0; i < 4; i++) {
     auto& word = ref.ref.data->words_by_seg.at(next_slot.seg).at(i + (next_slot.byte_offset / 4));
-    if (word.kind != decompiler::LinkedWord::PLAIN_DATA) {
+    if (word.kind() != decompiler::LinkedWord::PLAIN_DATA) {
       throw Error("bad word type in PrototypeBucketTie next");
     }
     next[i] = word.data;
@@ -794,7 +842,7 @@ void PrototypeBucketTie::read_from_file(TypedRef ref,
   auto count_slot = get_field_ref(ref, "count", dts);
   for (int i = 0; i < 2; i++) {
     auto& word = ref.ref.data->words_by_seg.at(count_slot.seg).at(i + (count_slot.byte_offset / 4));
-    if (word.kind != decompiler::LinkedWord::PLAIN_DATA) {
+    if (word.kind() != decompiler::LinkedWord::PLAIN_DATA) {
       throw Error("bad word type in PrototypeBucketTie count");
     }
     memcpy(count + 2 * i, &word.data, 4);
@@ -804,7 +852,7 @@ void PrototypeBucketTie::read_from_file(TypedRef ref,
   u8* block_start = (u8*)generic_count;
   for (int i = 0; i < 12; i++) {
     auto& word = ref.ref.data->words_by_seg.at(block_slot.seg).at(i + (block_slot.byte_offset / 4));
-    if (word.kind != decompiler::LinkedWord::PLAIN_DATA) {
+    if (word.kind() != decompiler::LinkedWord::PLAIN_DATA) {
       throw Error("bad word type in PrototypeBucketTie slot");
     }
     memcpy(block_start + 4 * i, &word.data, 4);
@@ -995,6 +1043,24 @@ std::unique_ptr<DrawableTree> make_drawable_tree(TypedRef ref,
     return tree;
   }
 
+  if (ref.type->get_name() == "drawable-tree-lowres-tfrag") {
+    auto tree = std::make_unique<DrawableTreeLowresTfrag>();
+    tree->read_from_file(ref, dts, stats);
+    return tree;
+  }
+
+  if (ref.type->get_name() == "drawable-tree-dirt-tfrag") {
+    auto tree = std::make_unique<DrawableTreeDirtTfrag>();
+    tree->read_from_file(ref, dts, stats);
+    return tree;
+  }
+
+  if (ref.type->get_name() == "drawable-tree-ice-tfrag") {
+    auto tree = std::make_unique<DrawableTreeIceTfrag>();
+    tree->read_from_file(ref, dts, stats);
+    return tree;
+  }
+
   if (ref.type->get_name() == "drawable-tree-instance-tie") {
     auto tree = std::make_unique<DrawableTreeInstanceTie>();
     tree->read_from_file(ref, dts, stats);
@@ -1064,6 +1130,22 @@ void BspHeader::read_from_file(const decompiler::LinkedObjectFile& file,
 
   drawable_tree_array.read_from_file(
       get_and_check_ref_to_basic(ref, "drawable-trees", "drawable-tree-array", dts), dts, stats);
+
+  texture_remap_table.clear();
+
+  s32 tex_remap_len = read_plain_data_field<s32>(ref, "texture-remap-table-len", dts);
+
+  if (tex_remap_len > 0) {
+    auto tex_remap_data = deref_label(get_field_ref(ref, "texture-remap-table", dts));
+    for (int entry = 0; entry < tex_remap_len; entry++) {
+      u64 low = deref_u32(tex_remap_data, 2 * entry);
+      u64 high = deref_u32(tex_remap_data, 2 * entry + 1);
+      TextureRemap remap;
+      remap.original_texid = low;
+      remap.new_texid = high;
+      texture_remap_table.push_back(remap);
+    }
+  }
 }
 
 std::string BspHeader::print(const PrintSettings& settings) const {

decompiler/level_extractor/BspHeader.h

@@ -5,7 +5,7 @@
 #include <memory>
 #include <vector>
 
-#include "tools/level_tools/goal_data_reader.h"
+#include "decompiler/util/goal_data_reader.h"
 
 namespace decompiler {
 class LinkedObjectFile;
@@ -126,8 +126,12 @@ struct TFragment : public Drawable {
   // u32 color_indices;  // 12 - 16 (or colors?)
   Vector bsphere;  // 16 - 32
   // dma common/level0 // 32 - 36
+  std::vector<u8> dma_common_and_level0;
   // dma base // 36 - 40
+  std::vector<u8> dma_base;
   // dma level 1 // 40 - 44
+  std::vector<u8> dma_level1;
+  std::vector<u16> color_indices;
   u8 dma_qwc[4];
   // shader // 48 - 52
   u8 num_shaders;        // 52
@@ -226,6 +230,13 @@ struct DrawableInlineArrayUnknown : public DrawableInlineArray {
 
 struct DrawableTree : public Drawable {};
 
+struct TimeOfDayPalette {
+  u32 width;
+  u32 height;
+  u32 pad;
+  std::vector<u32> colors;
+};
+
 struct DrawableTreeTfrag : public DrawableTree {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
@@ -236,6 +247,7 @@ struct DrawableTreeTfrag : public DrawableTree {
   s16 id;
   s16 length;
   // todo time of day stuff
+  TimeOfDayPalette time_of_day;
   Vector bsphere;
 
   std::vector<std::unique_ptr<DrawableInlineArray>> arrays;
@@ -336,6 +348,18 @@ struct DrawableTreeTransTfrag : public DrawableTreeTfrag {
   std::string my_type() const override { return "drawable-tree-trans-tfrag"; }
 };
 
+struct DrawableTreeLowresTfrag : public DrawableTreeTfrag {
+  std::string my_type() const override { return "drawable-tree-lowres-tfrag"; }
+};
+
+struct DrawableTreeDirtTfrag : public DrawableTreeTfrag {
+  std::string my_type() const override { return "drawable-tree-dirt-tfrag"; }
+};
+
+struct DrawableTreeIceTfrag : public DrawableTreeTfrag {
+  std::string my_type() const override { return "drawable-tree-ice-tfrag"; }
+};
+
 struct DrawableTreeUnknown : public DrawableTree {
   void read_from_file(TypedRef ref,
                       const decompiler::DecompilerTypeSystem& dts,
@@ -356,6 +380,11 @@ struct DrawableTreeArray {
   std::vector<std::unique_ptr<DrawableTree>> trees;
 };
 
+struct TextureRemap {
+  u32 original_texid;
+  u32 new_texid;
+};
+
 struct BspHeader {
   //  (info file-info :offset 4)
   FileInfo file_info;
@@ -377,6 +406,7 @@ struct BspHeader {
   //  ;; some osrt of texture remapping info
   //      (texture-remap-table (pointer uint64) :offset-assert 52)
   //  (texture-remap-table-len int32 :offset-assert 56)
+  std::vector<TextureRemap> texture_remap_table;
   //
   //  (texture-ids (pointer texture-id) :offset-assert 60)
   //  (texture-page-count int32 :offset-assert 64)
@@ -405,7 +435,8 @@ struct BspHeader {
   //  (unk-data-8 uint32 55 :offset-assert 180)
 
   void read_from_file(const decompiler::LinkedObjectFile& file,
-                      const decompiler::DecompilerTypeSystem& dts, DrawStats* stats);
+                      const decompiler::DecompilerTypeSystem& dts,
+                      DrawStats* stats);
 
   std::string print(const PrintSettings& settings) const;
 };

decompiler/level_extractor/extract_level.cpp

@@ -0,0 +1,104 @@
+#include <set>
+
+#include "extract_level.h"
+#include "decompiler/level_extractor/BspHeader.h"
+#include "decompiler/level_extractor/extract_tfrag.h"
+#include "common/util/FileUtil.h"
+
+namespace decompiler {
+
+/*!
+ * Look through files in a DGO and find the bsp-header file (the level)
+ */
+ObjectFileRecord get_bsp_file(const std::vector<ObjectFileRecord>& records) {
+  ObjectFileRecord result;
+  bool found = false;
+  for (auto& file : records) {
+    if (file.name.length() > 4 && file.name.substr(file.name.length() - 4) == "-vis") {
+      assert(!found);
+      found = true;
+      result = file;
+    }
+  }
+  assert(found);
+  return result;
+}
+
+/*!
+ * Make sure a file is a valid bsp-header.
+ */
+bool is_valid_bsp(const decompiler::LinkedObjectFile& file) {
+  if (file.segments != 1) {
+    fmt::print("Got {} segments, but expected 1\n", file.segments);
+    return false;
+  }
+
+  auto& first_word = file.words_by_seg.at(0).at(0);
+  if (first_word.kind() != decompiler::LinkedWord::TYPE_PTR) {
+    fmt::print("Expected the first word to be a type pointer, but it wasn't.\n");
+    return false;
+  }
+
+  if (first_word.symbol_name() != "bsp-header") {
+    fmt::print("Expected to get a bsp-header, but got {} instead.\n", first_word.symbol_name());
+    return false;
+  }
+
+  return true;
+}
+
+void extract_from_level(ObjectFileDB& db,
+                        TextureDB& tex_db,
+                        const std::string& dgo_name,
+                        const DecompileHacks& hacks) {
+  if (db.obj_files_by_dgo.count(dgo_name) == 0) {
+    lg::warn("Skipping extract for {} because the DGO was not part of the input", dgo_name);
+    return;
+  }
+
+  auto bsp_rec = get_bsp_file(db.obj_files_by_dgo.at(dgo_name));
+  std::string level_name = bsp_rec.name.substr(0, bsp_rec.name.length() - 4);
+
+  fmt::print("Processing level {} ({})\n", dgo_name, level_name);
+  auto& bsp_file = db.lookup_record(bsp_rec);
+  bool ok = is_valid_bsp(bsp_file.linked_data);
+  assert(ok);
+
+  level_tools::DrawStats draw_stats;
+  // draw_stats.debug_print_dma_data = true;
+  level_tools::BspHeader bsp_header;
+  bsp_header.read_from_file(bsp_file.linked_data, db.dts, &draw_stats);
+  assert((int)bsp_header.drawable_tree_array.trees.size() == bsp_header.drawable_tree_array.length);
+
+  const std::set<std::string> tfrag_trees = {
+      "drawable-tree-tfrag",     "drawable-tree-trans-tfrag",  "drawable-tree-dirt-tfrag",
+      "drawable-tree-ice-tfrag", "drawable-tree-lowres-tfrag", "drawable-tree-lowres-trans-tfrag"};
+  int i = 0;
+  tfrag3::Level tfrag_level;
+
+  for (auto& draw_tree : bsp_header.drawable_tree_array.trees) {
+    if (tfrag_trees.count(draw_tree->my_type())) {
+      auto as_tfrag_tree = dynamic_cast<level_tools::DrawableTreeTfrag*>(draw_tree.get());
+      fmt::print("  extracting tree {}\n", draw_tree->my_type());
+      assert(as_tfrag_tree);
+      std::vector<std::pair<int, int>> expected_missing_textures;
+      auto it = hacks.missing_textures_by_level.find(level_name);
+      if (it != hacks.missing_textures_by_level.end()) {
+        expected_missing_textures = it->second;
+      }
+      extract_tfrag(as_tfrag_tree, fmt::format("{}-{}", dgo_name, i++),
+                    bsp_header.texture_remap_table, tex_db, expected_missing_textures, tfrag_level);
+    } else {
+      fmt::print("  unsupported tree {}\n", draw_tree->my_type());
+      tfrag_level.trees.emplace_back();
+      tfrag_level.trees.back().kind = tfrag3::TFragmentTreeKind::INVALID;
+    }
+  }
+
+  Serializer ser;
+  tfrag_level.serialize(ser);
+  file_util::write_binary_file(file_util::get_file_path({fmt::format(
+                                   "assets/{}.fr3", dgo_name.substr(0, dgo_name.length() - 4))}),
+                               ser.get_save_result().first, ser.get_save_result().second);
+}
+}  // namespace decompiler

decompiler/level_extractor/extract_level.h

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <vector>
+
+#include "common/math/Vector.h"
+#include "decompiler/ObjectFile/ObjectFileDB.h"
+
+namespace decompiler {
+void extract_from_level(ObjectFileDB& db,
+                        TextureDB& tex_db,
+                        const std::string& dgo_name,
+                        const DecompileHacks& hacks);
+}

decompiler/level_extractor/extract_tfrag.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include "decompiler/level_extractor/BspHeader.h"
+#include "common/math/Vector.h"
+#include "common/custom_data/Tfrag3Data.h"
+#include "decompiler/data/TextureDB.h"
+
+namespace decompiler {
+
+// the different "kinds" of tfrag. The actual renderers are almost identical and the only different
+// is in GS setup (alpha blending) and in how the closest object is used.
+
+// This is the actual tree data, minus the tfrags themselves.
+struct VisNodeTree {
+  std::vector<tfrag3::VisNode> vis_nodes;
+  u16 first_child_node = 0;
+  u16 last_child_node = 0;
+  u16 first_root = 0;
+  u16 num_roots = 0;
+  bool only_children = false;
+};
+
+// The final result
+struct ExtractedTFragmentTree {
+  // TFragmentKind kind = TFragmentKind::INVALID;
+  VisNodeTree vis_nodes;
+
+  u16 num_tfrags = 0;
+  u16 tfrag_base_idx = 0;
+};
+
+// will pool textures with others already in out.
+void extract_tfrag(const level_tools::DrawableTreeTfrag* tree,
+                   const std::string& debug_name,
+                   const std::vector<level_tools::TextureRemap>& map,
+                   const TextureDB& tex_db,
+                   const std::vector<std::pair<int, int>>& expected_missing_textures,
+                   tfrag3::Level& out);
+
+}  // namespace decompiler

decompiler/level_extractor/notes.txt

@@ -0,0 +1,748 @@
+
+
+// The actual "tfrag" type. This only lives on the EE and gets converted to DMA data.
+// some of the DMA data is pointers to chains that live in the static level data.
+// other is colors that are looked up from the palette (on the EE!) then thrown in the
+// double-buffered frame global buffer.
+
+(deftype tfragment (drawable)
+  (
+   (color-index       uint16                       :offset 6)
+   (debug-data        tfragment-debug-data         :offset 8)
+   (color-indices     uint32                       :offset 12)
+   (colors            uint32                       :offset 12)
+   (dma-chain         uint32              3       :offset-assert 32)
+   (dma-common        uint32                       :offset 32)
+   (dma-level-0       uint32                       :offset 32)
+   (dma-base          uint32                       :offset 36)
+   (dma-level-1       uint32                       :offset 40)
+   (dma-qwc           uint8              4       :offset 44)
+   (shader            (inline-array adgif-shader)                       :offset 48)
+   (num-shaders       uint8                        :offset 52)
+   (num-base-colors   uint8                        :offset 53)
+   (num-level0-colors uint8                        :offset 54)
+   (num-level1-colors uint8                        :offset 55)
+   (color-offset      uint8                        :offset 56)
+   (color-count       uint8                        :offset 57)
+   (pad0              uint8                        :offset 58)
+   (pad1              uint8                        :offset 59)
+   (generic           generic-tfragment            :offset-assert 60)
+   (generic-u32       uint32            :offset 60) ;; added
+   )
+  :method-count-assert 18
+  :size-assert         #x40
+  :flag-assert         #x1200000040
+  )
+
+// This is the temp/debug structure used for the EE code
+
+(deftype tfrag-work (structure)
+  ((base-tmpl             dma-packet :inline :offset-assert 0)
+   (level-0-tmpl          dma-packet :inline :offset-assert 16)
+   (common-tmpl           dma-packet :inline :offset-assert 32)
+   (level-1-tmpl          dma-packet :inline :offset-assert 48)
+   (color-tmpl            dma-packet :inline :offset-assert 64)
+   (frag-dists            vector     :inline :offset-assert 80)
+   (max-dist              vector     :inline :offset-assert 96)
+   (min-dist              vector     :inline :offset-assert 112)
+   (color-ptr             vector4w   :inline :offset-assert 128)
+   (tr-stat-tfrag         tr-stat            :offset-assert 144)
+   (tr-stat-tfrag-near    tr-stat            :offset-assert 148)
+   (vu1-enable-tfrag      int32              :offset-assert 152)
+   (vu1-enable-tfrag-near int32              :offset-assert 156)
+   (cur-vis-bits          uint32             :offset-assert 160)
+   (end-vis-bits          uint32             :offset-assert 164)
+   (src-ptr               uint32             :offset-assert 168)
+   (last-call             uint32             :offset-assert 172)
+   (dma-buffer            basic              :offset-assert 176)
+   (test-id               uint32             :offset-assert 180)
+   (wait-from-spr         uint32             :offset-assert 184)
+   (wait-to-spr           uint32             :offset-assert 188)
+   (near-wait-from-spr    uint32             :offset-assert 192)
+   (near-wait-to-spr      uint32             :offset-assert 196)
+   )
+  :method-count-assert 9
+  :size-assert         #xc8
+  :flag-assert         #x9000000c8
+  )
+ */
+
+// base vifs:
+// ??
+// t3
+
+// l0:
+// ??
+// t3
+
+// common:
+// ??
+// t3
+
+// color
+// ??
+// 12 sb color-offset
+// 14 sb num colors, 4 aligned.
+
+/*
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+; .function draw-inline-array-tfrag
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+ ;; there's two double-buffered spad buffers + culling data
+
+   ;; arguments:
+   ;; a0 - occlusion cull list (on spad)
+   ;; a1 - tfrags
+   ;; a2 - num tfrags
+   ;; a3 - dma buf
+
+   ;; constants:
+   ;; t0 = *tfrag-work*
+   ;; t1 = SPR FROM
+   ;; t2 = 0x14000000 ??
+   ;; t4 = SPR TO
+
+   ;; vars:
+   ;; v1 = ptr to dma buffer data
+   ;; t5 = SPR BUFFER 0 (tfrags)
+   ;; a3 = SPR BUFFER 1
+   ;; t3 = ?? (init to 0)
+   ;; t6 = spr buffer 1 use (qwc)
+
+   ;; vf3 = frag-dists
+   ;; vf1 = (1, 1, 1, 1)
+   ;; vf2 = bsphere
+B0:
+L40:
+TFRAG INITIALIZE
+ ;; set up constants
+    daddiu sp, sp, -128
+    sd ra, 0(sp)
+    sq s0, 16(sp)
+    sq s1, 32(sp)
+    sq s2, 48(sp)
+    sq s3, 64(sp)
+    sq s4, 80(sp)
+    sq s5, 96(sp)
+    sq gp, 112(sp)
+    lui t2, 5120     = (0x14000000), a constant (mscal)
+    lw v1, 4(a3)     (-> dma-buf base)
+    lui t3, 4096     = (0x10000000)
+    lui t1, 4096     = (0x10000000)
+    sync.l
+    cache dxwbin v1, 0
+    sync.l
+    cache dxwbin v1, 1
+    sync.l
+    lw t0, *tfrag-work*(s7)
+    ori t4, t3, 54272           = (0x1000D400) SPR TO
+    ori t1, t1, 53248           = (0x1000D000) SPR FROM
+    lui t5, 28672               = (0x70000000)
+    lqc2 vf3, 80(t0)            = (-> *tfrag-work* frag-dists)
+    sw a3, 176(t0)              (set! (-> *tfrag-work* dma-buffer) dma-ptr)
+    ori a3, t5, 2064            setup buffer 1
+    addiu t3, r0, 0             t3 = 0
+    ori t5, t5, 1040            setup buffer 0
+    vmaxw.xyzw vf1, vf0, vf0    vf1 = (1, 1, 1, 1)
+    lh t7, 0(a0)                vis cull load
+    lqc2 vf4, 96(t0)            max-dist
+    addiu a1, a1, -4            remove basic offset
+    addiu t6, r0, 0             t6 = 0
+    or ra, a3, r0               ra = SPAD BUFFER 1
+
+
+SKIP TO FIRST VISIBLE
+ ;; skips ahead until we find some visible tfrags.
+B1:
+L41:
+    bne t7, r0, L42
+    sll r0, r0, 0
+
+B2:
+    addiu a0, a0, 2     ;; + 16 bits in the vis list
+    addiu a1, a1, 1024  ;; 16 * 0x40 = 1024 bytes in tfrag list
+    daddiu a2, a2, -16  ;; num tfrags -= 16
+    lh t7, 0(a0)        ;; next vis
+    blez a2, L69_CLEANUP ;; no visible tfrags, abort!
+    sll r0, r0, 0
+
+B3:
+    beq r0, r0, L41 ;; keep looking
+    sll r0, r0, 0
+
+WAIT_FOR_PREV_DMA
+ ;; waits for any previously running spad dma transfer to end
+B4:
+L42:
+    lw t7, 0(t4)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    andi t7, t7, 256
+    sll r0, r0, 0
+    bne t7, r0, L42
+    sll r0, r0, 0
+
+INIT_FIRST_SPAD_TO
+ ;; initializes the first scratchpad upload of tfrags
+B5:
+    sw a1, 16(t4)       ;; madr = a1
+    xori t7, t5, 1024   ;; t7 = upload addr of tfrags double buffer
+    sw t7, 128(t4)      ;; sadr
+    addiu t7, r0, 64    ;; 64 qw = 16 tfrags
+    sw t7, 32(t4)       ;; qwc
+    addiu t7, r0, 256   ;; go
+    sw t7, 0(t4)        ;; go!
+    sll r0, r0, 0
+
+TFRAG MAIN LOOP TOP
+B6:
+L43:
+    or gp, a0, r0                     ;; gp = temp addr of vis list
+    xori t5, t5, 1024                 ;; toggle to addr of upload tfrags
+    daddiu a0, a0, 2                  ;; advance vis list ptr (16 tfrags)
+    or t9, a0, r0                     ;; t9 = temp addr of next vis list
+    or t8, t5, r0                     ;; t8 = tfrags to use
+
+  ;; next, let's find next block of visible tfrags so we can start it's dma early
+    daddiu t7, a2, -16                ;; t7 = tfrags left after this loop
+    bgtz t7, L45                      ;; if we have them left, jump
+    lh t7, 0(a0)                      ;; and load their vis
+
+B7:
+    beq r0, r0, L48                   ;; none left, skip dma kickoff.
+    sll r0, r0, 0
+
+B8:
+L44:
+    daddiu a2, a2, -16                ;; skip invisible block (dec tfrag counter)
+    addiu a0, a0, 2                   ;; increment vis list
+    blez a2, L48                      ;; did we get to the end of the tfrag list?
+    lh t7, 0(a0)                      ;; check vis again.
+
+B9:
+    sll r0, r0, 0
+    sll r0, r0, 0
+B10:
+L45:
+    beq t7, r0, L44                  ;; we have tfrags left. if 0, they are all hidden, so loop
+    addiu a1, a1, 1024               ;; and advance upload pointer (not done at all yet)
+
+ ;; we reach here if we have tfrags left after this block.
+ ;; so let's upload the next ones to the scratchpad so they are ready by next time.
+B11:
+L46:
+    lw t7, 0(t4)      ;; make sure to-spr is done
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    andi t7, t7, 256
+    sll r0, r0, 0
+    beq t7, r0, L47
+    sll r0, r0, 0
+
+B12:
+    sll r0, r0, 0
+    lw t7, 188(t0)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    daddiu t7, t7, 1 ;; counting how many times we wait
+    sll r0, r0, 0
+    sw t7, 188(t0)
+    beq r0, r0, L46
+    sll r0, r0, 0
+
+B13:
+L47:
+    sw a1, 16(t4)     ;; start the to!
+    xori t7, t5, 1024
+    sw t7, 128(t4)
+    addiu t7, r0, 64
+    sw t7, 32(t4)
+    addiu t7, r0, 256
+    beq r0, r0, L49 ;; skip ahead
+    sw t7, 0(t4)
+
+ ;; only reach here if we dont have any more spr to's
+ ;; still need to sync the to for the block we're about to process
+B14:
+L48:
+    lw t7, 0(t4)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    andi t7, t7, 256
+    sll r0, r0, 0
+    beq t7, r0, L49
+    sll r0, r0, 0
+
+B15:
+    sll r0, r0, 0
+    lw t7, 188(t0)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    daddiu t7, t7, 1
+    sll r0, r0, 0
+    sw t7, 188(t0)
+    beq r0, r0, L48
+    sll r0, r0, 0
+
+ ;; common op start
+ ;; at this point:
+ ;;  t8 is our spad tfrag buffer, with 16 tfrags. at least 1 is visible.
+ ;;  gp is our vis-list pointer
+ ;; we run through this loop 2x, each time doing 8 tfrags.
+
+B16:
+L49:
+    lb t7, 0(gp)      ;; load first 8 frag vis bits
+    addiu gp, gp, 1   ;; inc vis bit ptr.
+    sll r0, r0, 0
+    sw gp, 160(t0)    ;; store cur-vis-bits
+    bne t7, r0, L50   ;; are any visible in the first 8?
+    sw t9, 164(t0)    ;; set end-vis-bits (why?)
+
+B17: ;; none are visible
+    daddiu a2, a2, -8  ;; dec tfrags
+    addiu t8, t8, 512  ;; skip tfrags
+    beq r0, r0, L65    ;; skip ahead!
+    sll r0, r0, 0
+
+B18:
+L50:
+    addiu t9, r0, 128    ;; vis mask init (gets shifted in each run of the 8-loop)
+    lqc2 vf2, 16(t8)     ;; bsphere load
+B19:
+L51:
+    daddiu gp, t6, -124  ;; are we full of stuff in buffer 1?
+    sll r0, r0, 0
+    blez gp, L54
+    sll r0, r0, 0
+
+B20:
+L52:
+    lw ra, 0(t1)       ;; wait for spr-from
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    andi ra, ra, 256
+    sll r0, r0, 0
+    beq ra, r0, L53
+    sll r0, r0, 0
+
+B21:
+    sll r0, r0, 0     ;; count it
+    lw ra, 184(t0)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    daddiu ra, ra, 1
+    sll r0, r0, 0
+    sw ra, 184(t0)
+    beq r0, r0, L52
+    sll r0, r0, 0
+
+B22:
+L53:
+    sw a3, 128(t1)       ;; kick off the next spr-from.
+    xori a3, a3, 6144
+    sw v1, 16(t1)        ;; to the dma-buf
+    sll ra, t6, 4
+    addu v1, v1, ra      ;; add qwc
+    or ra, a3, r0        ;; ra is the spad-side dma buffer to write to
+    sw t6, 32(t1)        ;; qwc
+    addiu t6, r0, 256
+    sw t6, 0(t1)         ;; go!
+    addiu t6, r0, 0      ;; reset use.
+
+ ;; actually building dma.
+B23:
+L54:
+    and gp, t7, t9               ;; vis check
+    vmulax.xyzw acc, vf16, vf2   ;; plane ?
+    beq gp, r0, L64_8loop_reject ;; vis check failed, reject!
+    lwu gp, 36(t8)                      ;; DMA BASE (chain) -------------
+
+B24:
+    vmadday.xyzw acc, vf17, vf2 ;; plane
+    lbu s5, 45(t8)                     ;; DMA QWC1
+    vmaddaz.xyzw acc, vf18, vf2 ;; plane
+    sw gp, 4(t0)                       ;; base tmpl set addr
+    vmsubaw.xyzw acc, vf19, vf0 ;; plane
+    sh s5, 0(t0)                       ;; base tmpl set qwc
+    vmaddw.xyzw vf5, vf1, vf2   ;; plane
+    lwu gp, 32(t8)                     ;; DMA level0 -------------
+    vmulaw.xyzw acc, vf27, vf0  ;; camrot
+    lbu s5, 47(t8)                     ;; DMA QWC3
+    vmaddax.xyzw acc, vf24, vf2 ;; camrot
+    sw gp, 20(t0)                      ;; l0 tmpl set addr
+    vmadday.xyzw acc, vf25, vf2 ;; camrot
+    sh s5, 16(t0)                      ;; l0 tmpl set qwc
+    vmaddaz.xyzw acc, vf26, vf2 ;; camrot
+    lwu gp, 32(t8)                     ;; DMA common --------------
+    qmfc2.i s5, vf5             ;; plane
+    lbu s4, 44(t8)                     ;; DMA QWC0
+    vmaddw.xyzw vf6, vf1, vf2   ;; ??
+    sw gp, 36(t0)                      ;; common tmpl set addr
+    vmsubw.xyzw vf8, vf1, vf2   ;; ??
+    sh s4, 32(t0)                      ;; common tmpl set qwc
+    pcgtw s5, r0, s5            ;; plane check
+    lwu gp, 40(t8)                     ;; DMA level1 -------------
+    ppach s5, r0, s5            ;; plane check
+    lbu s4, 46(t8)                     ;; DMA QWC2
+    vaddz.xyzw vf6, vf3, vf6    ;; dist
+    sw gp, 52(t0)                      ;; l1 tmpl addr
+    vaddz.xyzw vf7, vf3, vf8    ;; dist
+    sw t3, 12(t0)                      ;; !!! set a vif on base, 0 on the first round, at least.
+    bne s5, r0, L63_8loop_reject_tog_vis
+    sh s4, 48(t0)                      ;; l1 tmpl qwc
+
+B25:
+    vmini.xyzw vf4, vf4, vf8 ;; max dist
+    sw t3, 28(t0)            ;; !!! set a vif on l0
+    sll r0, r0, 0
+    lbu s5, 53(t8)           ;; s5 = num-base-colors
+    qmfc2.i gp, vf6          ;; dist
+    sw t3, 44(t0)            ;; !!! set a vif on common
+    qmfc2.i s3, vf7          ;; dist
+    lbu s4, 56(t8)           ;; s4 = color-offset
+    pcgtw s2, r0, gp         ;; dist
+    lw gp, 12(t8)            ;; gp = colors-indices
+    pcgtw s3, r0, s3         ;; dist
+    sb s4, 76(t0)            ;; store color-offset
+    pinteh s4, s2, s3        ;; dist
+    lbu s2, 54(t8)           ;; s2 = num-level0-colors
+    ppacb s3, r0, s4         ;; dist
+    lbu s1, 55(t8)           ;; s1 = num-level1-colors
+    beq s3, r0, L56          ;; jump if dist fails?
+    dsrl32 s4, s3, 8         ;; s4 is the level or something?
+
+B26:
+    beq s2, r0, L56          ;; if we have no level0 colors, use base
+    sll r0, r0, 0
+
+B27:
+    beq s1, r0, L55          ;; if we have no level1 colors, use level0
+    dsrl s5, s3, 16
+
+B28:
+    beq s5, r0, L55          ;; possible l1 skip based on lod
+    dsrl32 s5, s3, 24
+
+B29:
+    bne s5, r0, L64_8loop_reject     ;; possible all skip based on lod.
+    addiu s5, s1, 3                  ;; s5 = num-level1-colors + 3
+
+B30:                  ;; level 1 color setup
+    sra s4, s5, 2   ;; s4  = (num_color + 3) >> 4
+    or s5, s1, r0   ;; s5  = (num_color)
+    sll t3, s4, 2   ;; t3 = num colors, 4 aligned
+    sh s4, 64(t0)   ;; color-tmpl qwc.
+    sll r0, r0, 0
+    sb t3, 78(t0)   ;; vif store
+    daddiu t6, t6, 3 ;; use 3 qw's of global dma.
+    lq s2, 32(t0) ;; load the common-tmpl!
+    sll r0, r0, 0
+    lq s1, 48(t0) ;; load the l1 tmpl!
+    sll r0, r0, 0
+    lq t3, 64(t0) ;; load the color tmpl!
+    sq s2, 0(ra)  ;; store the common!
+    sll r0, r0, 0
+    sq s1, 16(ra) ;; store the l1!
+    dsrl32 s2, s3, 16
+    sq t3, 32(ra) ;; store the color
+    daddiu ra, ra, 48 ;; advance the dma buffer pointer
+    bne s2, r0, L57
+    ori t3, t2, 18    ;; is this.. program 18?
+
+B31:
+    dsrl32 t3, s3, 8
+    sll r0, r0, 0
+    bne t3, r0, L57
+    ori t3, t2, 16
+
+B32:
+    beq r0, r0, L57
+    ori t3, t2, 14
+
+B33:
+L55:
+    bne s4, r0, L64_8loop_reject
+    addiu s5, s2, 3 ;; l0 colors + 3
+
+B34:  l0 color setup
+    sra s4, s5, 2 ;;  >> 2
+    or s5, s2, r0 ;; s5 = qwc
+    sll t3, s4, 2 ;; << 2
+    sh s4, 64(t0) ;; color-tmpl qwc
+    sll r0, r0, 0
+    sb t3, 78(t0)  ;; vif store for unpack??
+    daddiu t6, t6, 2 ;; only 2 qw's
+    lq s2, 16(t0) ;; l0 tmpl
+    sll r0, r0, 0
+    lq t3, 64(t0) ;; color tmp
+    sq s2, 0(ra)
+    dsrl s3, s3, 8
+    sq t3, 16(ra)
+    daddiu ra, ra, 32
+    bne s3, r0, L57
+    ori t3, t2, 10
+
+B35:
+    beq r0, r0, L57
+    ori t3, t2, 8
+
+B36:
+L56:
+    bne s4, r0, L64_8loop_reject
+    addiu s4, s5, 3 ;; base colors + 3
+
+B37:
+    sra s4, s4, 2
+    sll r0, r0, 0
+    sll t3, s4, 2
+    sh s4, 64(t0)
+    sll r0, r0, 0
+    sb t3, 78(t0)
+    ori t3, t2, 6
+    lq s3, 0(t0) ;; base
+    daddiu t6, t6, 2
+    lq s2, 64(t0)  ;; color
+    sq s3, 0(ra)
+    sll r0, r0, 0
+    sq s2, 16(ra)
+    daddiu ra, ra, 32
+
+;; END of the color setup
+B38:
+L57: ;; another opportunity to do some spad swappin
+    addiu s3, r0, 127 ;; s3 = 127
+    daddu s2, t6, s4  ;; s2 = dma-use + color qwc
+    dsubu s3, s3, s2
+    sll r0, r0, 0
+    bgez s3, L60
+    sll r0, r0, 0
+
+B39:
+L58:
+    lw ra, 0(t1)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    andi ra, ra, 256
+    sll r0, r0, 0
+    beq ra, r0, L59
+    sll r0, r0, 0
+
+B40:
+    sll r0, r0, 0
+    lw ra, 184(t0)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    daddiu ra, ra, 1
+    sll r0, r0, 0
+    sw ra, 184(t0)
+    beq r0, r0, L58
+    sll r0, r0, 0
+
+B41:
+L59:
+    sw a3, 128(t1)
+    xori a3, a3, 6144
+    sw v1, 16(t1)
+    sll ra, t6, 4
+    addu v1, v1, ra
+    or ra, a3, r0
+    sw t6, 32(t1)
+    addiu t6, r0, 256
+    sw t6, 0(t1)
+    addiu t6, r0, 0
+B42:
+
+L60:
+    daddu t6, t6, s4      ;; add color imm's to dma buffer length
+    sw t8, 168(t0)        ;; back up tfrag... not enough regs
+    ld s4, 0(gp)          ;; load color-indices (u64 = u16 x 4)
+    daddiu t8, gp, 8      ;; inc colors ptr
+    daddiu gp, s5, -4     ;; gp is color counter. we're using the rounded up to 4 color count.
+    lq s5, 128(t0)        ;; color-ptr x4
+    pextlh s4, r0, s4     ;; expand packed u16's to u32's
+    mfc1 r0, f31          ;; nop
+    paddw s2, s4, s5      ;; add to color pointers
+    mfc1 r0, f31
+    lw s4, 0(s2)          ;; s4 = colors[0]
+    dsra32 s3, s2, 0
+    lw s3, 0(s3)          ;; s5 = colors[1]
+    pcpyud s1, s2, s2
+    lw s2, 0(s1)          ;; s2 = colors[2]
+    dsra32 s1, s1, 0
+    blez gp, L62
+    lw s1, 0(s1)          ;; s1 = colors[3]
+
+B43:
+L61:
+    ld s0, 0(t8)
+    daddiu ra, ra, 16
+    daddiu t8, t8, 8
+    sw s4, -16(ra)
+    daddiu gp, gp, -4
+    sw s3, -12(ra)
+    pextlh s4, r0, s0
+    sw s2, -8(ra)
+    paddw s2, s4, s5
+    sw s1, -4(ra)
+    lw s4, 0(s2)
+    dsra32 s3, s2, 0
+    lw s3, 0(s3)
+    pcpyud s1, s2, s2
+    lw s2, 0(s1)
+    dsra32 s1, s1, 0
+    bgtz gp, L61
+    lw s1, 0(s1)
+
+B44:
+L62:
+    daddiu ra, ra, 16
+    lw t8, 168(t0)
+    sll r0, r0, 0
+    sw s4, -16(ra)
+    sll r0, r0, 0
+    sw s3, -12(ra)
+    sll r0, r0, 0
+    sw s2, -8(ra)
+    sll r0, r0, 0
+    sw s1, -4(ra)
+B45:
+L63_8loop_reject_tog_vis:
+    xor t7, t7, t9 ;; update vis
+    sll r0, r0, 0
+B46:
+L64_8loop_reject:
+    daddiu t8, t8, 64
+    srl t9, t9, 1
+    addiu a2, a2, -1
+    sll r0, r0, 0
+    bne t9, r0, L51
+    lqc2 vf2, 16(t8)
+
+B47:
+L65:
+    sll r0, r0, 0
+    lw gp, 160(t0)
+    sll r0, r0, 0
+    lw t9, 164(t0)
+    bne gp, t9, L49
+    sb t7, -1(gp)
+
+B48:
+    bgtz a2, L43_MAIN_LOOP_TOP
+    sll r0, r0, 0
+
+B49:
+    beq t6, r0, L68
+    sll r0, r0, 0
+
+B50:
+L66:
+    lw a0, 0(t1)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    andi a0, a0, 256
+    sll r0, r0, 0
+    beq a0, r0, L67
+    sll r0, r0, 0
+
+B51:
+    sll r0, r0, 0
+    lw a0, 184(t0)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    daddiu a0, a0, 1
+    sll r0, r0, 0
+    sw a0, 184(t0)
+    beq r0, r0, L66
+    sll r0, r0, 0
+
+B52:
+L67:
+    sw a3, 128(t1)
+    xori a0, a3, 6144
+    sw v1, 16(t1)
+    sll a1, t6, 4
+    addu v1, v1, a1
+    or a0, a0, r0
+    sw t6, 32(t1)
+    addiu a0, r0, 256
+    sw a0, 0(t1)
+    addiu a0, r0, 0
+B53:
+L68:
+    lw a0, 0(t1)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    andi a0, a0, 256
+    sll r0, r0, 0
+    beq a0, r0, L69_CLEANUP
+    sll r0, r0, 0
+
+B54:
+    sll r0, r0, 0
+    lw a0, 184(t0)
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+    daddiu a0, a0, 1
+    sll r0, r0, 0
+    sw a0, 184(t0)
+    beq r0, r0, L68
+    sll r0, r0, 0
+
+B55:
+L69_CLEANUP:
+    lw a0, 176(t0)
+    sll r0, r0, 0
+    sw t3, 172(t0)
+    sll r0, r0, 0
+    sqc2 vf4, 112(t0)
+    sll r0, r0, 0
+    sw v1, 4(a0)
+    sll r0, r0, 0
+    or v0, r0, r0
+    ld ra, 0(sp)
+    lq gp, 112(sp)
+    lq s5, 96(sp)
+    lq s4, 80(sp)
+    lq s3, 64(sp)
+    lq s2, 48(sp)
+    lq s1, 32(sp)
+    lq s0, 16(sp)
+    jr ra
+    daddiu sp, sp, 128
+
+    sll r0, r0, 0
+    sll r0, r0, 0
+    sll r0, r0, 0
+
+
+    Notes on the VU program
+  vi03 is a pointer to an "address book" - a sequence of addresses
+  vi02 contains addresses in this book
+  from these xyw are loaded for vf28 (v3-32, with 2, 1)
+    xy are floats. w is address of next vertex data.
+
+  vi08 is a pointer to adgifs?
+  vi09 is a pointer to some data like [vi12, ?, ?, vi13] ??
+
+  vi12 counter, started negative?
+  vi13 is adgif offset?
+
+  vi04 is a pointer to tri-data:
+    - vertex (w = 128.0?)
+    - ?? (vf20)

decompiler/main.cpp

@@ -7,8 +7,13 @@
 #include "common/util/FileUtil.h"
 #include "common/versions.h"
 #include "decompiler/data/streamed_audio.h"
+#include "decompiler/level_extractor/extract_level.h"
+#include "decompiler/data/TextureDB.h"
+#include "common/util/os.h"
 
 int main(int argc, char** argv) {
+  fmt::print("[Mem] Size of linked word: {}\n", sizeof(decompiler::LinkedWord));
+  fmt::print("[Mem] Top of main: {} MB\n", get_peak_rss() / (1024 * 1024));
   using namespace decompiler;
   lg::set_file(file_util::get_file_path({"log/decompiler.txt"}));
   lg::set_file_level(lg::level::info);
@@ -24,13 +29,14 @@ int main(int argc, char** argv) {
     printf("Usage: decompiler <config_file> <in_folder> <out_folder>\n");
     return 1;
   }
+  fmt::print("[Mem] After init: {} MB\n", get_peak_rss() / (1024 * 1024));
 
   // collect all files to process
   Config config;
   try {
     config = read_config_file(argv[1]);
   } catch (const std::exception& e) {
-    lg::error("Failed to parse config");
+    lg::error("Failed to parse config: {}", e.what());
     return 1;
   }
 
@@ -52,10 +58,14 @@ int main(int argc, char** argv) {
 
   file_util::create_dir_if_needed(out_folder);
 
+  fmt::print("[Mem] After config read: {} MB\n", get_peak_rss() / (1024 * 1024));
+
   // build file database
   lg::info("Setting up object file DB...");
   ObjectFileDB db(dgos, config.obj_file_name_map_file, objs, strs, config);
 
+  fmt::print("[Mem] After DB setup: {} MB\n", get_peak_rss() / (1024 * 1024));
+
   // write out DGO file info
   file_util::write_text_file(file_util::combine_path(out_folder, "dgo.txt"),
                              db.generate_dgo_listing());
@@ -72,8 +82,10 @@ int main(int argc, char** argv) {
 
   // process files (required for all analysis)
   db.process_link_data(config);
+  fmt::print("[Mem] After link data: {} MB\n", get_peak_rss() / (1024 * 1024));
   db.find_code(config);
   db.process_labels();
+  fmt::print("[Mem] After code: {} MB\n", get_peak_rss() / (1024 * 1024));
 
   // print disassembly
   if (config.disassemble_code || config.disassemble_data) {
@@ -93,6 +105,8 @@ int main(int argc, char** argv) {
     db.analyze_functions_ir2(out_folder, config, {});
   }
 
+  fmt::print("[Mem] After decomp: {} MB\n", get_peak_rss() / (1024 * 1024));
+
   // write out all symbols
   file_util::write_text_file(file_util::combine_path(out_folder, "all-syms.gc"),
                              db.dts.dump_symbol_types());
@@ -113,13 +127,18 @@ int main(int argc, char** argv) {
     }
   }
 
+  fmt::print("[Mem] After text: {} MB\n", get_peak_rss() / (1024 * 1024));
+
+  decompiler::TextureDB tex_db;
   if (config.process_tpages) {
-    auto result = db.process_tpages();
+    auto result = db.process_tpages(tex_db);
     if (!result.empty()) {
       file_util::write_text_file(file_util::get_file_path({"assets", "tpage-dir.txt"}), result);
     }
   }
 
+  fmt::print("[Mem] After textures: {} MB\n", get_peak_rss() / (1024 * 1024));
+
   if (config.process_game_count) {
     auto result = db.process_game_count_file();
     if (!result.empty()) {
@@ -127,6 +146,12 @@ int main(int argc, char** argv) {
     }
   }
 
+  for (auto& lev : config.levels_to_extract) {
+    extract_from_level(db, tex_db, lev, config.hacks);
+  }
+
+  fmt::print("[Mem] After extraction: {} MB\n", get_peak_rss() / (1024 * 1024));
+
   if (!config.audio_dir_file_name.empty()) {
     process_streamed_audio(config.audio_dir_file_name, config.streamed_audio_file_names);
   }

decompiler/util/DataParser.cpp

@@ -116,8 +116,7 @@ ParsedData parse_data(const std::string& str) {
     // try as .type
     if (first_thing == ".type") {
       LinkedWord word(0);
-      word.kind = LinkedWord::TYPE_PTR;
-      word.symbol_name = line;
+      word.set_to_symbol(LinkedWord::TYPE_PTR, line);
       result.words.push_back(word);
       byte_offset += 4;
       continue;
@@ -125,8 +124,7 @@ ParsedData parse_data(const std::string& str) {
 
     if (first_thing == ".symbol") {
       LinkedWord word(0);
-      word.kind = LinkedWord::SYM_PTR;
-      word.symbol_name = line;
+      word.set_to_symbol(LinkedWord::SYM_PTR, line);
       result.words.push_back(word);
       byte_offset += 4;
       continue;
@@ -137,7 +135,7 @@ ParsedData parse_data(const std::string& str) {
         throw std::runtime_error("Got something after .empty-list, this is not allowed");
       }
       LinkedWord word(0);
-      word.kind = LinkedWord::EMPTY_PTR;
+      word.set_to_empty_ptr();
       result.words.push_back(word);
       byte_offset += 4;
       continue;
@@ -151,8 +149,7 @@ ParsedData parse_data(const std::string& str) {
           result.labels.emplace_back();
         }
         LinkedWord word(0);
-        word.kind = LinkedWord::PTR;
-        word.label_id = l.idx;
+        word.set_to_pointer(LinkedWord::PTR, l.idx);
         result.words.push_back(word);
         byte_offset += 4;
         continue;
@@ -160,7 +157,7 @@ ParsedData parse_data(const std::string& str) {
         auto val = std::stoull(line, nullptr, 16);
         assert(val <= UINT32_MAX);
         LinkedWord word(val);
-        word.kind = LinkedWord::PLAIN_DATA;
+        word.set_to_plain_data();
         result.words.push_back(word);
         byte_offset += 4;
         continue;
@@ -197,18 +194,18 @@ std::string ParsedData::print() const {
 
     // print word
     auto& word = words.at(idx);
-    switch (word.kind) {
+    switch (word.kind()) {
       case LinkedWord::PLAIN_DATA:
         result += fmt::format("    .word 0x{:x}\n", word.data);
         break;
       case LinkedWord::PTR:
-        result += fmt::format("    .word {}\n", labels.at(word.label_id).name);
+        result += fmt::format("    .word {}\n", labels.at(word.label_id()).name);
         break;
       case LinkedWord::SYM_PTR:
-        result += fmt::format("    .symbol {}\n", word.symbol_name);
+        result += fmt::format("    .symbol {}\n", word.symbol_name());
         break;
       case LinkedWord::TYPE_PTR:
-        result += fmt::format("    .type {}\n", word.symbol_name);
+        result += fmt::format("    .type {}\n", word.symbol_name());
         break;
       case LinkedWord::EMPTY_PTR:
         result += "    .empty-list\n";

decompiler/util/data_decompile.cpp

@@ -116,18 +116,18 @@ std::optional<TypeSpec> get_type_of_label(const DecompilerLabel& label,
   if ((label.offset % 8) == 4) {
     auto type_ptr_word_idx = (label.offset / 4) - 1;
     auto& type_ptr = words.at(label.target_segment).at(type_ptr_word_idx);
-    if (type_ptr.kind != LinkedWord::TYPE_PTR) {
+    if (type_ptr.kind() != LinkedWord::TYPE_PTR) {
       return {};
     }
-    if (type_ptr.symbol_name == "array") {
+    if (type_ptr.symbol_name() == "array") {
       auto content_type_ptr_word_idx = type_ptr_word_idx + 3;
       auto& content_type_ptr = words.at(label.target_segment).at(content_type_ptr_word_idx);
-      if (content_type_ptr.kind != LinkedWord::TYPE_PTR) {
+      if (content_type_ptr.kind() != LinkedWord::TYPE_PTR) {
         return {};
       }
-      return TypeSpec("array", {TypeSpec(content_type_ptr.symbol_name)});
+      return TypeSpec("array", {TypeSpec(content_type_ptr.symbol_name())});
     }
-    return TypeSpec(type_ptr.symbol_name);
+    return TypeSpec(type_ptr.symbol_name());
   } else {
     return {};
   }
@@ -211,14 +211,14 @@ goos::Object decompile_string_at_label(const DecompilerLabel& label,
   assert(label.offset >= 4);
 
   const auto& type_ptr = words.at(label.target_segment).at((label.offset - 4) / 4);
-  if (type_ptr.kind != LinkedWord::TYPE_PTR) {
+  if (type_ptr.kind() != LinkedWord::TYPE_PTR) {
     throw std::runtime_error(fmt::format(
         "Cannot get string at label {}, word before is not a type pointer.", label.name));
   }
 
-  if (type_ptr.symbol_name != "string") {
+  if (type_ptr.symbol_name() != "string") {
     throw std::runtime_error(fmt::format("Cannot get string at label {}, type pointer is for a {}.",
-                                         label.name, type_ptr.symbol_name));
+                                         label.name, type_ptr.symbol_name()));
   }
 
   std::string result;
@@ -230,7 +230,7 @@ goos::Object decompile_string_at_label(const DecompilerLabel& label,
         fmt::format("Cannot get string at label {}, not enough room", label.name));
   }
   const LinkedWord& size_word = words.at(label.target_segment).at(word_idx + 1);
-  if (size_word.kind != LinkedWord::PLAIN_DATA) {
+  if (size_word.kind() != LinkedWord::PLAIN_DATA) {
     // sometimes an array of string pointer triggers this!
     throw std::runtime_error(
         fmt::format("Cannot get string at label {}, size is not plain data.", label.name));
@@ -241,7 +241,7 @@ goos::Object decompile_string_at_label(const DecompilerLabel& label,
     int word_offset = word_idx + 2 + (i / 4);
     int byte_offset = i % 4;
     auto& word = words.at(label.target_segment).at(word_offset);
-    if (word.kind != LinkedWord::PLAIN_DATA) {
+    if (word.kind() != LinkedWord::PLAIN_DATA) {
       throw std::runtime_error(
           fmt::format("Cannot get string at label {}, character is not plain data.", label.name));
     }
@@ -269,7 +269,7 @@ goos::Object decompile_value_array(const TypeSpec& elt_type,
     std::vector<u8> elt_bytes;
     for (int j = start; j < end; j++) {
       auto& word = obj_words.at(j / 4);
-      if (word.kind != LinkedWord::PLAIN_DATA) {
+      if (word.kind() != LinkedWord::PLAIN_DATA) {
         throw std::runtime_error("Got bad word in kind in array of values");
       }
       elt_bytes.push_back(word.get_byte(j % 4));
@@ -343,11 +343,11 @@ goos::Object decomp_ref_to_inline_array_guess_size(
   // we expect that to be a label:
   assert((field_location % 4) == 0);
   auto pointer_to_data = words.at(field_location / 4);
-  assert(pointer_to_data.kind == LinkedWord::PTR);
+  assert(pointer_to_data.kind() == LinkedWord::PTR);
 
   // the data shouldn't have any labels in the middle of it, so we can find the end of the array
   // by searching for the label after the start label.
-  const auto& start_label = labels.at(pointer_to_data.label_id);
+  const auto& start_label = labels.at(pointer_to_data.label_id());
   int end_label_idx =
       index_of_closest_following_label_in_segment(start_label.offset, my_seg, labels);
 
@@ -381,7 +381,7 @@ goos::Object decomp_ref_to_inline_array_guess_size(
   int padding_end = end_offset;
   for (int pad_byte_idx = padding_start; pad_byte_idx < padding_end; pad_byte_idx++) {
     auto& word = all_words.at(my_seg).at(pad_byte_idx / 4);
-    switch (word.kind) {
+    switch (word.kind()) {
       case LinkedWord::PLAIN_DATA:
         assert(word.get_byte(pad_byte_idx) == 0);
         break;
@@ -489,13 +489,13 @@ goos::Object decompile_structure(const TypeSpec& type,
 
   if (is_basic) {
     const auto& word = words.at(label.target_segment).at((offset_location / 4));
-    if (word.kind != LinkedWord::TYPE_PTR) {
+    if (word.kind() != LinkedWord::TYPE_PTR) {
       throw std::runtime_error("Basic does not start with type pointer");
     }
 
-    if (word.symbol_name != actual_type.base_type()) {
+    if (word.symbol_name() != actual_type.base_type()) {
       // we can specify a more specific type.
-      auto got_type = TypeSpec(word.symbol_name);
+      auto got_type = TypeSpec(word.symbol_name());
       if (ts.tc(actual_type, got_type)) {
         actual_type = got_type;
 
@@ -511,7 +511,7 @@ goos::Object decompile_structure(const TypeSpec& type,
       } else {
         throw std::runtime_error(
             fmt::format("Basic has the wrong type pointer, got {} expected {} at label {}:{}",
-                        word.symbol_name, actual_type.base_type(), label.name, label.offset));
+                        word.symbol_name(), actual_type.base_type(), label.name, label.offset));
       }
     }
   }
@@ -552,7 +552,7 @@ goos::Object decompile_structure(const TypeSpec& type,
   std::vector<int> field_status_per_byte;
   for (int i = 0; i < word_count; i++) {
     auto& w = obj_words.at(i);
-    switch (w.kind) {
+    switch (w.kind()) {
       case LinkedWord::TYPE_PTR:
       case LinkedWord::PTR:
       case LinkedWord::SYM_PTR:
@@ -585,11 +585,11 @@ goos::Object decompile_structure(const TypeSpec& type,
     if (is_basic && idx == 0) {
       assert(field.name() == "type" && field.offset() == 0);
       auto& word = obj_words.at(0);
-      if (word.kind != LinkedWord::TYPE_PTR) {
+      if (word.kind() != LinkedWord::TYPE_PTR) {
         throw std::runtime_error("Basic does not start with type pointer");
       }
 
-      if (word.symbol_name != actual_type.base_type()) {
+      if (word.symbol_name() != actual_type.base_type()) {
         // the check above should have caught this.
         assert(false);
       }
@@ -682,20 +682,20 @@ goos::Object decompile_structure(const TypeSpec& type,
                                                         field_start, ts, words, file));
         } else {
           if (field.type().base_type() == "pointer") {
-            if (obj_words.at(field_start / 4).kind != LinkedWord::SYM_PTR) {
+            if (obj_words.at(field_start / 4).kind() != LinkedWord::SYM_PTR) {
               continue;
             }
 
-            if (obj_words.at(field_start / 4).symbol_name != "#f") {
+            if (obj_words.at(field_start / 4).symbol_name() != "#f") {
               lg::warn("Got a weird symbol in a pointer field: {}",
-                       obj_words.at(field_start / 4).symbol_name);
+                       obj_words.at(field_start / 4).symbol_name());
               continue;
             }
 
             field_defs_out.emplace_back(field.name(), pretty_print::to_symbol("#f"));
 
           } else {
-            if (obj_words.at(field_start / 4).kind != LinkedWord::PLAIN_DATA) {
+            if (obj_words.at(field_start / 4).kind() != LinkedWord::PLAIN_DATA) {
               continue;
             }
             std::vector<u8> bytes_out;
@@ -753,7 +753,7 @@ goos::Object decompile_structure(const TypeSpec& type,
         int end_elt = 0;
         for (int elt = len; elt-- > 0;) {
           auto& word = obj_words.at((field_start / 4) + elt);
-          if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) {
+          if (word.kind() == LinkedWord::PLAIN_DATA && word.data == 0) {
             continue;
           }
           end_elt = elt + 1;
@@ -763,25 +763,25 @@ goos::Object decompile_structure(const TypeSpec& type,
         for (int elt = 0; elt < end_elt; elt++) {
           auto& word = obj_words.at((field_start / 4) + elt);
 
-          if (word.kind == LinkedWord::PTR) {
-            array_def.push_back(decompile_at_label(field.type(), labels.at(word.label_id), labels,
+          if (word.kind() == LinkedWord::PTR) {
+            array_def.push_back(decompile_at_label(field.type(), labels.at(word.label_id()), labels,
                                                    words, ts, file));
-          } else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) {
+          } else if (word.kind() == LinkedWord::PLAIN_DATA && word.data == 0) {
             // do nothing, the default is zero?
             array_def.push_back(pretty_print::to_symbol("0"));
-          } else if (word.kind == LinkedWord::SYM_PTR) {
-            if (word.symbol_name == "#f" || word.symbol_name == "#t") {
-              array_def.push_back(pretty_print::to_symbol(fmt::format("{}", word.symbol_name)));
+          } else if (word.kind() == LinkedWord::SYM_PTR) {
+            if (word.symbol_name() == "#f" || word.symbol_name() == "#t") {
+              array_def.push_back(pretty_print::to_symbol(fmt::format("{}", word.symbol_name())));
             } else {
-              array_def.push_back(pretty_print::to_symbol(fmt::format("'{}", word.symbol_name)));
+              array_def.push_back(pretty_print::to_symbol(fmt::format("'{}", word.symbol_name())));
             }
-          } else if (word.kind == LinkedWord::EMPTY_PTR) {
+          } else if (word.kind() == LinkedWord::EMPTY_PTR) {
             array_def.push_back(pretty_print::to_symbol("'()"));
           } else {
             throw std::runtime_error(fmt::format(
                 "Field {} in type {} offset {} did not have a proper reference for "
                 "array element {} k = {}",
-                field.name(), actual_type.print(), field.offset(), elt, (int)word.kind));
+                field.name(), actual_type.print(), field.offset(), elt, (int)word.kind()));
           }
         }
         field_defs_out.emplace_back(field.name(), pretty_print::build_list(array_def));
@@ -795,37 +795,37 @@ goos::Object decompile_structure(const TypeSpec& type,
         assert(field_end - field_start == 4);
         auto& word = obj_words.at(field_start / 4);
 
-        if (word.kind == LinkedWord::PTR) {
+        if (word.kind() == LinkedWord::PTR) {
           if (field.type() == TypeSpec("symbol")) {
             continue;
           }
           field_defs_out.emplace_back(
-              field.name(),
-              decompile_at_label(field.type(), labels.at(word.label_id), labels, words, ts, file));
-        } else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) {
+              field.name(), decompile_at_label(field.type(), labels.at(word.label_id()), labels,
+                                               words, ts, file));
+        } else if (word.kind() == LinkedWord::PLAIN_DATA && word.data == 0) {
           // do nothing, the default is zero?
           field_defs_out.emplace_back(field.name(), pretty_print::to_symbol("0"));
-        } else if (word.kind == LinkedWord::SYM_PTR) {
-          if (word.symbol_name == "#f" || word.symbol_name == "#t") {
+        } else if (word.kind() == LinkedWord::SYM_PTR) {
+          if (word.symbol_name() == "#f" || word.symbol_name() == "#t") {
             field_defs_out.emplace_back(
-                field.name(), pretty_print::to_symbol(fmt::format("{}", word.symbol_name)));
+                field.name(), pretty_print::to_symbol(fmt::format("{}", word.symbol_name())));
           } else {
             field_defs_out.emplace_back(
-                field.name(), pretty_print::to_symbol(fmt::format("'{}", word.symbol_name)));
+                field.name(), pretty_print::to_symbol(fmt::format("'{}", word.symbol_name())));
           }
-        } else if (word.kind == LinkedWord::EMPTY_PTR) {
+        } else if (word.kind() == LinkedWord::EMPTY_PTR) {
           field_defs_out.emplace_back(field.name(), pretty_print::to_symbol("'()"));
-        } else if (word.kind == LinkedWord::TYPE_PTR) {
+        } else if (word.kind() == LinkedWord::TYPE_PTR) {
           if (field.type() != TypeSpec("type")) {
             throw std::runtime_error(
                 fmt::format("Field {} in type {} offset {} had a reference to type {}, but the "
                             "type of the field is not type.",
-                            field.name(), actual_type.print(), field.offset(), word.symbol_name));
+                            field.name(), actual_type.print(), field.offset(), word.symbol_name()));
           }
-          int method_count = ts.get_type_method_count(word.symbol_name);
+          int method_count = ts.get_type_method_count(word.symbol_name());
           field_defs_out.emplace_back(
               field.name(), pretty_print::to_symbol(fmt::format("(type-ref {} :method-count {})",
-                                                                word.symbol_name, method_count)));
+                                                                word.symbol_name(), method_count)));
         } else {
           throw std::runtime_error(
               fmt::format("Field {} in type {} offset {} did not have a proper reference",
@@ -1048,16 +1048,16 @@ goos::Object decompile_boxed_array(const DecompilerLabel& label,
   auto type_ptr_word_idx = (label.offset / 4) - 1;
   if ((label.offset % 8) == 4) {
     auto& type_ptr = words.at(label.target_segment).at(type_ptr_word_idx);
-    if (type_ptr.kind != LinkedWord::TYPE_PTR) {
+    if (type_ptr.kind() != LinkedWord::TYPE_PTR) {
       throw std::runtime_error("Invalid basic in decompile_boxed_array");
     }
-    if (type_ptr.symbol_name == "array") {
+    if (type_ptr.symbol_name() == "array") {
       auto content_type_ptr_word_idx = type_ptr_word_idx + 3;
       auto& content_type_ptr = words.at(label.target_segment).at(content_type_ptr_word_idx);
-      if (content_type_ptr.kind != LinkedWord::TYPE_PTR) {
+      if (content_type_ptr.kind() != LinkedWord::TYPE_PTR) {
         throw std::runtime_error("Invalid content in decompile_boxed_array");
       }
-      content_type = TypeSpec(content_type_ptr.symbol_name);
+      content_type = TypeSpec(content_type_ptr.symbol_name());
     } else {
       throw std::runtime_error("Wrong basic type in decompile_boxed_array");
     }
@@ -1074,7 +1074,8 @@ goos::Object decompile_boxed_array(const DecompilerLabel& label,
   auto& size_word_2 = words.at(label.target_segment).at(type_ptr_word_idx + 2);
   auto first_elt_word_idx = type_ptr_word_idx + 4;
 
-  if (size_word_1.kind != LinkedWord::PLAIN_DATA || size_word_2.kind != LinkedWord::PLAIN_DATA) {
+  if (size_word_1.kind() != LinkedWord::PLAIN_DATA ||
+      size_word_2.kind() != LinkedWord::PLAIN_DATA) {
     throw std::runtime_error("Invalid size in decompile_boxed_array");
   }
 
@@ -1093,20 +1094,20 @@ goos::Object decompile_boxed_array(const DecompilerLabel& label,
 
     for (int elt = 0; elt < array_length; elt++) {
       auto& word = words.at(label.target_segment).at(first_elt_word_idx + elt);
-      if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) {
+      if (word.kind() == LinkedWord::PLAIN_DATA && word.data == 0) {
         result.push_back(pretty_print::to_symbol("0"));
-      } else if (word.kind == LinkedWord::PTR) {
+      } else if (word.kind() == LinkedWord::PTR) {
         if (content_type == TypeSpec("object")) {
           result.push_back(
-              decompile_at_label_guess_type(labels.at(word.label_id), labels, words, ts, file));
+              decompile_at_label_guess_type(labels.at(word.label_id()), labels, words, ts, file));
         } else {
-          result.push_back(
-              decompile_at_label(content_type, labels.at(word.label_id), labels, words, ts, file));
+          result.push_back(decompile_at_label(content_type, labels.at(word.label_id()), labels,
+                                              words, ts, file));
         }
-      } else if (word.kind == LinkedWord::SYM_PTR) {
-        result.push_back(pretty_print::to_symbol(fmt::format("'{}", word.symbol_name)));
+      } else if (word.kind() == LinkedWord::SYM_PTR) {
+        result.push_back(pretty_print::to_symbol(fmt::format("'{}", word.symbol_name())));
       } else {
-        if (content_type == TypeSpec("object") && word.kind == LinkedWord::PLAIN_DATA &&
+        if (content_type == TypeSpec("object") && word.kind() == LinkedWord::PLAIN_DATA &&
             (word.data & 0b111) == 0) {
           s32 val = word.data;
           result.push_back(pretty_print::to_symbol(fmt::format("(the binteger {})", val / 8)));
@@ -1129,7 +1130,7 @@ goos::Object decompile_boxed_array(const DecompilerLabel& label,
 
     for (int elt = 0; elt < array_length; elt++) {
       auto& word = words.at(label.target_segment).at(first_elt_word_idx + elt);
-      auto segment = labels.at(word.label_id).target_segment;
+      auto segment = labels.at(word.label_id()).target_segment;
       result.push_back(decomp_ref_to_inline_array_guess_size(
           words.at(segment), labels, segment, (first_elt_word_idx + elt) * 4, ts, words, file,
           content_type.get_single_arg(), ts.get_deref_info(content_type).stride));
@@ -1152,9 +1153,9 @@ goos::Object decompile_boxed_array(const DecompilerLabel& label,
       std::vector<u8> elt_bytes;
       for (int j = start; j < end; j++) {
         auto& word = words.at(label.target_segment).at(j / 4);
-        if (word.kind != LinkedWord::PLAIN_DATA) {
+        if (word.kind() != LinkedWord::PLAIN_DATA) {
           throw std::runtime_error(
-              fmt::format("Got bad word of kind {} in boxed array of values", word.kind));
+              fmt::format("Got bad word of kind {} in boxed array of values", word.kind()));
         }
         elt_bytes.push_back(word.get_byte(j % 4));
       }
@@ -1171,8 +1172,8 @@ goos::Object decompile_pair_elt(const LinkedWord& word,
                                 const std::vector<std::vector<LinkedWord>>& words,
                                 const TypeSystem& ts,
                                 const LinkedObjectFile* file) {
-  if (word.kind == LinkedWord::PTR) {
-    auto& label = labels.at(word.label_id);
+  if (word.kind() == LinkedWord::PTR) {
+    auto& label = labels.at(word.label_id());
     auto guessed_type = get_type_of_label(label, words);
     if (!guessed_type.has_value()) {
       throw std::runtime_error("Could not guess the type of " + label.name);
@@ -1183,21 +1184,21 @@ goos::Object decompile_pair_elt(const LinkedWord& word,
     }
 
     return decompile_at_label(*guessed_type, label, labels, words, ts, file);
-  } else if (word.kind == LinkedWord::PLAIN_DATA && word.data == 0) {
+  } else if (word.kind() == LinkedWord::PLAIN_DATA && word.data == 0) {
     // do nothing, the default is zero?
     return pretty_print::to_symbol("0");
-  } else if (word.kind == LinkedWord::SYM_PTR) {
+  } else if (word.kind() == LinkedWord::SYM_PTR) {
     // never quote symbols in a list.
-    return pretty_print::to_symbol(fmt::format("{}", word.symbol_name));
-  } else if (word.kind == LinkedWord::EMPTY_PTR) {
+    return pretty_print::to_symbol(fmt::format("{}", word.symbol_name()));
+  } else if (word.kind() == LinkedWord::EMPTY_PTR) {
     return pretty_print::to_symbol("'()");
-  } else if (word.kind == LinkedWord::PLAIN_DATA && (word.data & 0b111) == 0) {
+  } else if (word.kind() == LinkedWord::PLAIN_DATA && (word.data & 0b111) == 0) {
     return pretty_print::to_symbol(fmt::format("(the binteger {})", ((s32)word.data) >> 3));
-  } else if (word.kind == LinkedWord::PLAIN_DATA) {
+  } else if (word.kind() == LinkedWord::PLAIN_DATA) {
     return pretty_print::to_symbol(fmt::format("#x{:x}", word.data));
   } else {
     throw std::runtime_error(fmt::format("Pair elt did not have a good word kind: k {} d {}",
-                                         (int)word.kind, word.data));
+                                         (int)word.kind(), word.data));
   }
 }
 }  // namespace
@@ -1213,7 +1214,7 @@ goos::Object decompile_pair(const DecompilerLabel& label,
       throw std::runtime_error(fmt::format("Invalid alignment for pair {}\n", label.offset % 16));
     } else {
       auto& word = words.at(label.target_segment).at(label.offset / 4);
-      if (word.kind != LinkedWord::EMPTY_PTR) {
+      if (word.kind() != LinkedWord::EMPTY_PTR) {
         throw std::runtime_error(
             fmt::format("Based on alignment, expected to get empty list for pair, but didn't"));
       }
@@ -1238,7 +1239,7 @@ goos::Object decompile_pair(const DecompilerLabel& label,
 
       auto cdr_word = words.at(to_print.target_segment).at((to_print.offset + 2) / 4);
       // if empty
-      if (cdr_word.kind == LinkedWord::EMPTY_PTR) {
+      if (cdr_word.kind() == LinkedWord::EMPTY_PTR) {
         if (add_quote) {
           return pretty_print::build_list("quote", pretty_print::build_list(list_tokens));
         } else {
@@ -1246,8 +1247,8 @@ goos::Object decompile_pair(const DecompilerLabel& label,
         }
       }
       // if pointer
-      if (cdr_word.kind == LinkedWord::PTR) {
-        to_print = labels.at(cdr_word.label_id);
+      if (cdr_word.kind() == LinkedWord::PTR) {
+        to_print = labels.at(cdr_word.label_id());
         continue;
       }
       // invalid.
@@ -1268,7 +1269,7 @@ goos::Object decompile_pair(const DecompilerLabel& label,
             fmt::format("Invalid alignment for pair {}\n", to_print.offset % 16));
       } else {
         auto& word = words.at(to_print.target_segment).at(to_print.offset / 4);
-        if (word.kind != LinkedWord::EMPTY_PTR) {
+        if (word.kind() != LinkedWord::EMPTY_PTR) {
           throw std::runtime_error(
               fmt::format("Based on alignment, expected to get empty list for pair, but didn't"));
         }
@@ -1302,7 +1303,7 @@ goos::Object decompile_bitfield(const TypeSpec& type,
   std::vector<u8> elt_bytes;
   for (int j = start_byte; j < end_byte; j++) {
     auto& word = words.at(label.target_segment).at(j / 4);
-    if (word.kind != LinkedWord::PLAIN_DATA) {
+    if (word.kind() != LinkedWord::PLAIN_DATA) {
       throw std::runtime_error("Got bad word in static bitfield");
     }
     elt_bytes.push_back(word.get_byte(j % 4));

decompiler/util/goal_data_reader.cpp

@@ -3,7 +3,7 @@
 #include "decompiler/util/DecompilerTypeSystem.h"
 #include "decompiler/ObjectFile/LinkedObjectFile.h"
 
-#include "tools/level_tools/Error.h"
+#include "decompiler/util/Error.h"
 
 void read_plain_data_field(const TypedRef& object,
                            const std::string& field_name,
@@ -43,7 +43,7 @@ void read_plain_data_field(const TypedRef& object,
     int byte_in_word = byte_in_words % 4;
 
     const auto& word = words.at(word_idx);
-    if (word.kind != decompiler::LinkedWord::PLAIN_DATA) {
+    if (word.kind() != decompiler::LinkedWord::PLAIN_DATA) {
       throw Error("Error reading byte {} of field {} (in data, byte {}). Didn't get plain data.",
                   byte, field_name, byte_in_words);
     }
@@ -78,11 +78,11 @@ TypedRef get_and_check_ref_to_basic(const TypedRef& object,
 
   const auto& word = object.ref.data->words_by_seg.at(object.ref.seg).at(byte_in_words / 4);
 
-  if (word.kind != decompiler::LinkedWord::PTR) {
+  if (word.kind() != decompiler::LinkedWord::PTR) {
     throw Error("get_and_check_ref_to_basic did not get a label");
   }
 
-  const auto& label = object.ref.data->labels.at(word.label_id);
+  const auto& label = object.ref.data->labels.at(word.label_id());
 
   Ref ref;
   ref.data = object.ref.data;
@@ -94,18 +94,18 @@ TypedRef get_and_check_ref_to_basic(const TypedRef& object,
   }
 
   const auto& type_tag = object.ref.data->words_by_seg.at(ref.seg).at(ref.byte_offset / 4);
-  if (type_tag.kind != decompiler::LinkedWord::TYPE_PTR) {
+  if (type_tag.kind() != decompiler::LinkedWord::TYPE_PTR) {
     throw Error("get_and_check_ref_to_basic did not find a type tag");
   }
 
-  if (type_tag.symbol_name != expected_type) {
+  if (type_tag.symbol_name() != expected_type) {
     throw Error("get_and_check_ref_to_basic found type {} for field {}, expected {}",
-                type_tag.symbol_name, field_name, expected_type);
+                type_tag.symbol_name(), field_name, expected_type);
   }
 
   TypedRef tr;
   tr.ref = ref;
-  tr.type = dts.ts.lookup_type(type_tag.symbol_name);
+  tr.type = dts.ts.lookup_type(type_tag.symbol_name());
   return tr;
 }
 
@@ -132,11 +132,11 @@ std::string read_symbol_field(const TypedRef& object,
 
   const auto& word = object.ref.data->words_by_seg.at(object.ref.seg).at(byte_in_words / 4);
 
-  if (word.kind != decompiler::LinkedWord::SYM_PTR) {
+  if (word.kind() != decompiler::LinkedWord::SYM_PTR) {
     throw Error("read_symbol_field did not get a symbol (offset {} words)", byte_in_words / 4);
   }
 
-  return word.symbol_name;
+  return word.symbol_name();
 }
 
 std::string read_type_field(const TypedRef& object,
@@ -163,11 +163,11 @@ std::string read_type_field(const TypedRef& object,
 
   const auto& word = object.ref.data->words_by_seg.at(object.ref.seg).at(byte_in_words / 4);
 
-  if (word.kind != decompiler::LinkedWord::TYPE_PTR) {
+  if (word.kind() != decompiler::LinkedWord::TYPE_PTR) {
     throw Error("read_type_field did not get a symbol (offset {} words)", byte_in_words / 4);
   }
 
-  return word.symbol_name;
+  return word.symbol_name();
 }
 
 std::string read_string_field(const TypedRef& object,
@@ -193,11 +193,11 @@ std::string read_string_field(const TypedRef& object,
   }
   const auto& word = object.ref.data->words_by_seg.at(object.ref.seg).at(byte_in_words / 4);
 
-  if (word.kind != decompiler::LinkedWord::PTR) {
+  if (word.kind() != decompiler::LinkedWord::PTR) {
     throw Error("read_string_field did not get a label (offset {} words)", byte_in_words / 4);
   }
 
-  const auto& label = object.ref.data->labels.at(word.label_id);
+  const auto& label = object.ref.data->labels.at(word.label_id());
   return object.ref.data->get_goal_string_by_label(label);
 }
 
@@ -218,11 +218,11 @@ std::string get_type_of_basic(const Ref& object) {
 
   const auto& word = object.data->words_by_seg.at(object.seg).at(byte_in_words / 4);
 
-  if (word.kind != decompiler::LinkedWord::TYPE_PTR) {
+  if (word.kind() != decompiler::LinkedWord::TYPE_PTR) {
     throw Error("get_type_of_basic did not get a type tag (offset {} words)", byte_in_words / 4);
   }
 
-  return word.symbol_name;
+  return word.symbol_name();
 }
 
 TypedRef typed_ref_from_basic(const Ref& object, const decompiler::DecompilerTypeSystem& dts) {
@@ -233,13 +233,13 @@ TypedRef typed_ref_from_basic(const Ref& object, const decompiler::DecompilerTyp
 
   const auto& word = object.data->words_by_seg.at(object.seg).at(byte_in_words / 4);
 
-  if (word.kind != decompiler::LinkedWord::TYPE_PTR) {
+  if (word.kind() != decompiler::LinkedWord::TYPE_PTR) {
     throw Error("typed_ref_from_basic did not get a type tag (offset {} words)", byte_in_words / 4);
   }
 
   TypedRef result;
   result.ref = object;
-  result.type = dts.ts.lookup_type(word.symbol_name);
+  result.type = dts.ts.lookup_type(word.symbol_name());
   return result;
 }
 
@@ -251,11 +251,11 @@ Ref deref_label(const Ref& object) {
 
   const auto& word = object.data->words_by_seg.at(object.seg).at(byte_in_words / 4);
 
-  if (word.kind != decompiler::LinkedWord::PTR) {
+  if (word.kind() != decompiler::LinkedWord::PTR) {
     throw Error("deref_label did not get a label (offset {} words)", byte_in_words / 4);
   }
 
-  const auto& lab = object.data->labels.at(word.label_id);
+  const auto& lab = object.data->labels.at(word.label_id());
 
   Ref result;
   result.byte_offset = lab.offset;
@@ -266,15 +266,15 @@ Ref deref_label(const Ref& object) {
 
 std::string inspect_ref(const Ref& ref) {
   auto& word = ref.data->words_by_seg.at(ref.seg).at(ref.byte_offset / 4);
-  switch (word.kind) {
+  switch (word.kind()) {
     case decompiler::LinkedWord::PLAIN_DATA:
       return fmt::format("[0x{:08x} (offset {} bytes)]", word.data, ref.byte_offset % 4);
     case decompiler::LinkedWord::PTR:
-      return fmt::format("[{}]", ref.data->labels.at(word.label_id).name);
+      return fmt::format("[{}]", ref.data->labels.at(word.label_id()).name);
     case decompiler::LinkedWord::SYM_PTR:
-      return fmt::format("['{}]", word.symbol_name);
+      return fmt::format("['{}]", word.symbol_name());
     case decompiler::LinkedWord::TYPE_PTR:
-      return fmt::format("[t'{}]", word.symbol_name);
+      return fmt::format("[t'{}]", word.symbol_name());
     default:
       assert(false);
   }

decompiler/util/goal_data_reader.h

@@ -69,4 +69,4 @@ TypedRef typed_ref_from_basic(const Ref& object, const decompiler::DecompilerTyp
 
 Ref deref_label(const Ref& object);
 
-std::string inspect_ref(const Ref& ref);
+std::string inspect_ref(const Ref& ref);

decompiler/util/sparticle_decompile.cpp

@@ -202,10 +202,10 @@ goos::Object decompile_sparticle_tex_field_init(const std::vector<LinkedWord>& w
                                                 const TypeSystem& ts,
                                                 const std::string& field_name,
                                                 const std::string& flag_name) {
-  assert(words.at(1).kind == LinkedWord::PLAIN_DATA);
-  assert(words.at(2).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(1).kind() == LinkedWord::PLAIN_DATA);
+  assert(words.at(2).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(2).data == 0);
-  assert(words.at(3).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(3).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(3).data == 0);
   assert(flag_name == "plain-v1");
 
@@ -217,38 +217,38 @@ goos::Object decompile_sparticle_tex_field_init(const std::vector<LinkedWord>& w
 }
 
 float word_as_float(const LinkedWord& w) {
-  assert(w.kind == LinkedWord::PLAIN_DATA);
+  assert(w.kind() == LinkedWord::PLAIN_DATA);
   float v;
   memcpy(&v, &w.data, 4);
   return v;
 }
 
 s32 word_as_s32(const LinkedWord& w) {
-  assert(w.kind == LinkedWord::PLAIN_DATA);
+  assert(w.kind() == LinkedWord::PLAIN_DATA);
   return w.data;
 }
 
 goos::Object decompile_sparticle_func(const std::vector<LinkedWord>& words,
                                       const std::string& field_name,
                                       const std::string& flag_name) {
-  assert(words.at(1).kind == LinkedWord::SYM_PTR);
-  assert(words.at(2).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(1).kind() == LinkedWord::SYM_PTR);
+  assert(words.at(2).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(2).data == 0);
-  assert(words.at(3).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(3).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(3).data == 0);
   assert(flag_name == "from-pointer");
   return pretty_print::to_symbol(
-      fmt::format("(sp-func {} '{})", field_name, words.at(1).symbol_name));
+      fmt::format("(sp-func {} '{})", field_name, words.at(1).symbol_name()));
 }
 
 goos::Object decompile_sparticle_end(const std::vector<LinkedWord>& words,
                                      const std::string& field_name,
                                      const std::string& flag_name) {
-  assert(words.at(1).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(1).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(1).data == 0);
-  assert(words.at(2).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(2).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(2).data == 0);
-  assert(words.at(3).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(3).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(3).data == 0);
   assert(flag_name == "plain-v1");
   assert(field_name == "spt-end");
@@ -300,10 +300,10 @@ goos::Object decompile_sparticle_userdata(const std::vector<LinkedWord>& words,
 goos::Object decompile_sparticle_int_init(const std::vector<LinkedWord>& words,
                                           const std::string& field_name,
                                           const std::string& flag_name) {
-  assert(words.at(1).kind == LinkedWord::PLAIN_DATA);
-  assert(words.at(2).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(1).kind() == LinkedWord::PLAIN_DATA);
+  assert(words.at(2).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(2).data == 0);
-  assert(words.at(3).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(3).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(3).data == 1);
   assert(flag_name == "plain-v1");
   return pretty_print::to_symbol(
@@ -338,10 +338,10 @@ goos::Object decompile_sparticle_int_with_rand_init(const std::vector<LinkedWord
 goos::Object decompile_sparticle_launcher_by_id(const std::vector<LinkedWord>& words,
                                                 const std::string& field_name,
                                                 const std::string& flag_name) {
-  assert(words.at(1).kind == LinkedWord::PLAIN_DATA);
-  assert(words.at(2).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(1).kind() == LinkedWord::PLAIN_DATA);
+  assert(words.at(2).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(2).data == 0);
-  assert(words.at(3).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(3).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(3).data == 0);
   assert(flag_name == "part-by-id");
   return pretty_print::to_symbol(
@@ -354,9 +354,9 @@ goos::Object decompile_sparticle_flags(const std::vector<LinkedWord>& words,
                                        const std::string& flag_name) {
   assert(flag_name == "plain-v1");
   assert(field_name == "spt-flags");
-  assert(words.at(3).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(3).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(3).data == 1);
-  assert(words.at(2).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(2).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(2).data == 0);
   auto flag_def =
       decompile_bitfield_enum_from_int(TypeSpec("sp-cpuinfo-flag"), ts, word_as_s32(words.at(1)));
@@ -372,10 +372,10 @@ goos::Object decompile_sparticle_flags(const std::vector<LinkedWord>& words,
 goos::Object decompile_sparticle_from_other(const std::vector<LinkedWord>& words,
                                             const std::string& field_name,
                                             const std::string& flag_name) {
-  assert(words.at(1).kind == LinkedWord::PLAIN_DATA);
-  assert(words.at(2).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(1).kind() == LinkedWord::PLAIN_DATA);
+  assert(words.at(2).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(2).data == 0);
-  assert(words.at(3).kind == LinkedWord::PLAIN_DATA);
+  assert(words.at(3).kind() == LinkedWord::PLAIN_DATA);
   assert(words.at(3).data == 1);
   assert(flag_name == "copy-from-other-field");
   return pretty_print::to_symbol(
@@ -547,7 +547,7 @@ goos::Object decompile_sparticle_field_init(const TypeSpec& type,
                    words.at(label.target_segment).begin() + (offset_location / 4),
                    words.at(label.target_segment).begin() + (offset_location / 4) + word_count);
 
-  assert(obj_words.at(0).kind == LinkedWord::PLAIN_DATA);
+  assert(obj_words.at(0).kind() == LinkedWord::PLAIN_DATA);
   u16 field_id = obj_words.at(0).data & 0xffff;
   u16 flags = obj_words.at(0).data >> 16;
 

game/CMakeLists.txt

@@ -89,6 +89,7 @@ set(RUNTIME_SOURCE
         graphics/opengl_renderer/debug_gui.cpp
         graphics/opengl_renderer/DirectRenderer.cpp
         graphics/opengl_renderer/dma_helpers.cpp
+        graphics/opengl_renderer/Loader.cpp
         graphics/opengl_renderer/OpenGLRenderer.cpp
         graphics/opengl_renderer/Profiler.cpp
         graphics/opengl_renderer/Shader.cpp
@@ -97,6 +98,7 @@ set(RUNTIME_SOURCE
         graphics/opengl_renderer/TextureUploadHandler.cpp
         graphics/opengl_renderer/tfrag/BufferedRenderer.cpp
         graphics/opengl_renderer/tfrag/program6_cpu.cpp
+        graphics/opengl_renderer/tfrag/Tfrag3.cpp
         graphics/opengl_renderer/tfrag/tfrag_unpack.cpp
         graphics/opengl_renderer/tfrag/TFragment.cpp
         graphics/texture/TextureConverter.cpp

game/graphics/opengl_renderer/BucketRenderer.h

@@ -6,6 +6,7 @@
 #include "game/graphics/opengl_renderer/Shader.h"
 #include "game/graphics/texture/TexturePool.h"
 #include "game/graphics/opengl_renderer/Profiler.h"
+#include "game/graphics/opengl_renderer/Loader.h"
 
 /*!
  * Matches the bucket-id enum in GOAL
@@ -47,6 +48,8 @@ struct SharedRenderState {
       : texture_pool(_texture_pool) {}
   ShaderLibrary shaders;
   std::shared_ptr<TexturePool> texture_pool;
+  Loader loader;
+
   u32 buckets_base = 0;  // address of buckets array.
   u32 next_bucket = 0;   // address of next bucket that we haven't started rendering in buckets
   u32 default_regs_buffer = 0;  // address of the default regs chain.

game/graphics/opengl_renderer/Loader.cpp

@@ -0,0 +1,34 @@
+#include "Loader.h"
+#include "common/util/Timer.h"
+#include "common/util/FileUtil.h"
+
+namespace {
+std::string uppercase_string(const std::string& s) {
+  std::string result;
+  for (auto c : s) {
+    result.push_back(toupper(c));
+  }
+  return result;
+}
+}  // namespace
+
+tfrag3::Level* Loader::get_tfrag3_level(const std::string& level_name) {
+  const auto& existing = m_tfrag3_levels.find(level_name);
+  if (existing == m_tfrag3_levels.end()) {
+    fmt::print("Loader needs to load tfrag3 level: {}\n", level_name);
+    Timer disk_timer;
+    auto data = file_util::read_binary_file(
+        file_util::get_file_path({fmt::format("assets/{}.fr3", uppercase_string(level_name))}));
+    double disk_load_time = disk_timer.getSeconds();
+
+    Timer import_timer;
+    auto& result = m_tfrag3_levels[level_name];
+    Serializer ser(data.data(), data.size());
+    result.serialize(ser);
+    double import_time = import_timer.getSeconds();
+    fmt::print("Load from file: {:.3f}s, import {:.3f}s\n", disk_load_time, import_time);
+    return &result;
+  } else {
+    return &existing->second;
+  }
+}

game/graphics/opengl_renderer/Loader.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "common/custom_data/Tfrag3Data.h"
+
+class Loader {
+ public:
+  tfrag3::Level* get_tfrag3_level(const std::string& level_name);
+
+ private:
+  std::unordered_map<std::string, tfrag3::Level> m_tfrag3_levels;
+};

game/graphics/opengl_renderer/OpenGLRenderer.cpp

@@ -59,15 +59,22 @@ OpenGLRenderer::OpenGLRenderer(std::shared_ptr<TexturePool> texture_pool)
  * Construct bucket renderers.  We can specify different renderers for different buckets
  */
 void OpenGLRenderer::init_bucket_renderers() {
-  // temp
+  std::vector<tfrag3::TFragmentTreeKind> normal_tfrags = {tfrag3::TFragmentTreeKind::NORMAL,
+                                                          tfrag3::TFragmentTreeKind::LOWRES};
+  std::vector<tfrag3::TFragmentTreeKind> dirt_tfrags = {tfrag3::TFragmentTreeKind::DIRT};
+  // TODO ice
+  // std::vector<tfrag3::TFragmentTreeKind> ice_tfrags = {tfrag3::TFragmentTreeKind::ICE};
+
+  // std::vector<tfrag3::TFragmentTreeKind> trans_tfrags = {tfrag3::TFragmentTreeKind::TRANS,
+  //                                                       tfrag3::TFragmentTreeKind::LOWRES_TRANS};
 
   init_bucket_renderer<EmptyBucketRenderer>("bucket0", BucketId::BUCKET0);
   init_bucket_renderer<SkyRenderer>("sky", BucketId::SKY_DRAW);
 
   init_bucket_renderer<TextureUploadHandler>("tfrag-tex-0", BucketId::TFRAG_TEX_LEVEL0);
-  init_bucket_renderer<TFragment>("tfrag-0", BucketId::TFRAG_LEVEL0, false);
+  init_bucket_renderer<TFragment>("tfrag-0", BucketId::TFRAG_LEVEL0, normal_tfrags, false);
   init_bucket_renderer<TextureUploadHandler>("tfrag-tex-1", BucketId::TFRAG_TEX_LEVEL1);
-  init_bucket_renderer<TFragment>("tfrag-1", BucketId::TFRAG_LEVEL1, false);
+  init_bucket_renderer<TFragment>("tfrag-1", BucketId::TFRAG_LEVEL1, normal_tfrags, false);
   init_bucket_renderer<TextureUploadHandler>("shrub-tex-0", BucketId::SHRUB_TEX_LEVEL0);
   init_bucket_renderer<TextureUploadHandler>("shrub-tex-1", BucketId::SHRUB_TEX_LEVEL1);
   init_bucket_renderer<TextureUploadHandler>("alpha-tex-0", BucketId::ALPHA_TEX_LEVEL0);
@@ -75,10 +82,10 @@ void OpenGLRenderer::init_bucket_renderers() {
   auto sky_blender = std::make_shared<SkyBlender>();
   init_bucket_renderer<SkyBlendHandler>("sky-blend-and-tfrag-trans-0",
                                         BucketId::TFRAG_TRANS0_AND_SKY_BLEND_LEVEL0, sky_blender);
-  init_bucket_renderer<TFragment>("tfrag-dirt-0", BucketId::TFRAG_DIRT_LEVEL0, false);
+  init_bucket_renderer<TFragment>("tfrag-dirt-0", BucketId::TFRAG_DIRT_LEVEL0, dirt_tfrags, false);
   init_bucket_renderer<SkyBlendHandler>("sky-blend-and-tfrag-trans-1",
                                         BucketId::TFRAG_TRANS1_AND_SKY_BLEND_LEVEL1, sky_blender);
-  init_bucket_renderer<TFragment>("tfrag-dirt-1", BucketId::TFRAG_DIRT_LEVEL1, false);
+  init_bucket_renderer<TFragment>("tfrag-dirt-1", BucketId::TFRAG_DIRT_LEVEL1, dirt_tfrags, false);
   init_bucket_renderer<TextureUploadHandler>("pris-tex-0", BucketId::PRIS_TEX_LEVEL0);
   init_bucket_renderer<TextureUploadHandler>("pris-tex-1", BucketId::PRIS_TEX_LEVEL1);
   init_bucket_renderer<TextureUploadHandler>("water-tex-0", BucketId::WATER_TEX_LEVEL0);

game/graphics/opengl_renderer/Profiler.cpp

@@ -99,7 +99,7 @@ void Profiler::draw_node(ProfilerNode& node, bool expand, int depth, float start
   constexpr int origin_x = 40;
   constexpr int origin_y = 60;
   constexpr int row_height = 15;
-  constexpr int px_per_ms = 200;
+  constexpr int px_per_ms = 75;
 
   if (node.m_stats.duration > 0.00001) {
     color = name_to_color(node.m_name);

game/graphics/opengl_renderer/Shader.cpp

@@ -1,6 +1,7 @@
 #include "Shader.h"
 #include "common/util/assert.h"
 #include "common/util/FileUtil.h"
+#include "common/log/log.h"
 #include "game/graphics/pipelines/opengl.h"
 
 Shader::Shader(const std::string& shader_name) {
@@ -22,7 +23,7 @@ Shader::Shader(const std::string& shader_name) {
   glGetShaderiv(m_vert_shader, GL_COMPILE_STATUS, &compile_ok);
   if (!compile_ok) {
     glGetShaderInfoLog(m_vert_shader, len, nullptr, err);
-    printf("Failed to compile vertex shader %s:\n%s\n", shader_name.c_str(), err);
+    lg::error("Failed to compile vertex shader {}:\n{}\n", shader_name.c_str(), err);
     m_is_okay = false;
     return;
   }
@@ -35,7 +36,7 @@ Shader::Shader(const std::string& shader_name) {
   glGetShaderiv(m_frag_shader, GL_COMPILE_STATUS, &compile_ok);
   if (!compile_ok) {
     glGetShaderInfoLog(m_frag_shader, len, nullptr, err);
-    printf("Failed to compile fragment shader %s:\n%s\n", shader_name.c_str(), err);
+    lg::error("Failed to compile fragment shader {}:\n{}\n", shader_name.c_str(), err);
     m_is_okay = false;
     return;
   }
@@ -48,7 +49,7 @@ Shader::Shader(const std::string& shader_name) {
   glGetProgramiv(m_program, GL_LINK_STATUS, &compile_ok);
   if (!compile_ok) {
     glGetProgramInfoLog(m_program, len, nullptr, err);
-    printf("Failed to link shader %s:\n%s\n", shader_name.c_str(), err);
+    lg::error("Failed to link shader {}:\n{}\n", shader_name.c_str(), err);
     m_is_okay = false;
     return;
   }
@@ -76,4 +77,6 @@ ShaderLibrary::ShaderLibrary() {
   at(ShaderId::DEBUG_BUFFERED) = {"debug_buffered"};
   at(ShaderId::BUFFERED_TCC0) = {"buffered_tcc0"};
   at(ShaderId::BUFFERED_TCC1) = {"buffered_tcc1"};
+  at(ShaderId::TFRAG3) = {"tfrag3"};
+  at(ShaderId::TFRAG3_NO_TEX) = {"tfrag3_no_tex"};
 }

game/graphics/opengl_renderer/Shader.h

@@ -34,6 +34,8 @@ enum class ShaderId {
   DEBUG_BUFFERED = 9,
   BUFFERED_TCC0 = 10,
   BUFFERED_TCC1 = 11,
+  TFRAG3 = 12,
+  TFRAG3_NO_TEX = 13,
   MAX_SHADERS
 };
 

game/graphics/opengl_renderer/SkyRenderer.cpp

@@ -243,7 +243,10 @@ SkyBlendHandler::SkyBlendHandler(const std::string& name,
                                  std::shared_ptr<SkyBlender> shared_blender)
     : BucketRenderer(name, my_id),
       m_shared_blender(shared_blender),
-      m_tfrag_renderer(fmt::format("tfrag-{}", name), my_id, true) {}
+      m_tfrag_renderer(fmt::format("tfrag-{}", name),
+                       my_id,
+                       {tfrag3::TFragmentTreeKind::TRANS, tfrag3::TFragmentTreeKind::LOWRES_TRANS},
+                       true) {}
 
 void SkyBlendHandler::handle_sky_copies(DmaFollower& dma,
                                         SharedRenderState* render_state,
@@ -278,6 +281,12 @@ void SkyBlendHandler::render(DmaFollower& dma,
     return;
   }
 
+  if (dma.current_tag().qwc != 8) {
+    auto tfrag_prof = prof.make_scoped_child("tfrag-trans");
+    m_tfrag_renderer.render(dma, render_state, tfrag_prof);
+    return;
+  }
+
   // first is the set-display-gs-state
   auto set_display = dma.read_and_advance();
   assert(set_display.size_bytes == 8 * 16);

game/graphics/opengl_renderer/debug_gui.cpp

@@ -32,7 +32,7 @@ void FrameTimeRecorder::draw_window(const DmaStats& dma_stats) {
   window_pos_pivot.y = 1.0f;
   ImGui::SetNextWindowPos(window_pos, ImGuiCond_Always, window_pos_pivot);
 
-  ImGui::SetNextWindowBgAlpha(0.35f);  // Transparent background
+  ImGui::SetNextWindowBgAlpha(0.85f);  // Transparent background
   if (ImGui::Begin("Frame Timing", p_open, window_flags)) {
     ImGui::Text("DMA: sync ms %.1f, tc %4d, sz %3d KB, ch %d", dma_stats.sync_time_ms,
                 dma_stats.num_tags, (dma_stats.num_data_bytes) / (1 << 10), dma_stats.num_chunks);

game/graphics/opengl_renderer/shaders/buffered_tcc0.vert

@@ -8,7 +8,7 @@ out vec4 fragment_color;
 out vec3 tex_coord;
 
 void main() {
-    gl_Position = vec4((position_in.x - 0.5) * 16. , -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+    gl_Position = vec4((position_in.x - 0.5) * 16. , -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
     fragment_color = vec4(rgba_in.x, rgba_in.y, rgba_in.z, rgba_in.a * 2);
     tex_coord = tex_coord_in;
 }

game/graphics/opengl_renderer/shaders/buffered_tcc1.vert

@@ -8,7 +8,7 @@ out vec4 fragment_color;
 out vec3 tex_coord;
 
 void main() {
-    gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+    gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
     fragment_color = vec4(rgba_in.x, rgba_in.y, rgba_in.z, rgba_in.w * 2.);
     tex_coord = tex_coord_in;
 }

game/graphics/opengl_renderer/shaders/debug_buffered.vert

@@ -11,7 +11,7 @@ out vec4 fragment_color;
 
 void main() {
     // Note: position.y is multiplied by 32 instead of 16 to undo the half-height for interlacing stuff.
-    gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+    gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
     fragment_color = vec4(rgba_in.x, rgba_in.y, rgba_in.z, rgba_in.w + 0.5);
     //fragment_color = vec4(1.0, 0, 0, 0.7);
 }

game/graphics/opengl_renderer/shaders/debug_red.vert

@@ -7,6 +7,6 @@ layout (location = 0) in vec3 position_in;
 out vec4 fragment_color;
 
 void main() {
-  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
   fragment_color = vec4(1.0, 0, 0, 0.7);
 }

game/graphics/opengl_renderer/shaders/direct_basic.vert

@@ -9,6 +9,6 @@ out vec4 fragment_color;
 
 void main() {
   // Note: position.y is multiplied by 32 instead of 16 to undo the half-height for interlacing stuff.
-  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
   fragment_color = vec4(rgba_in.x, rgba_in.y, rgba_in.z, rgba_in.w + 0.5);
 }

game/graphics/opengl_renderer/shaders/direct_basic_textured.vert

@@ -8,7 +8,7 @@ out vec4 fragment_color;
 out vec3 tex_coord;
 
 void main() {
-  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
   fragment_color = vec4(rgba_in.x, rgba_in.y, rgba_in.z, rgba_in.w * 2.);
   tex_coord = tex_coord_in;
 }

game/graphics/opengl_renderer/shaders/direct_basic_textured_tcc0.vert

@@ -8,7 +8,7 @@ out vec4 fragment_color;
 out vec3 tex_coord;
 
 void main() {
-  gl_Position = vec4((position_in.x - 0.5) * 16. , -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+  gl_Position = vec4((position_in.x - 0.5) * 16. , -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
   fragment_color = vec4(rgba_in.x, rgba_in.y, rgba_in.z, rgba_in.a * 2);
   tex_coord = tex_coord_in;
 }

game/graphics/opengl_renderer/shaders/sky.vert

@@ -8,7 +8,7 @@ out vec4 fragment_color;
 noperspective out vec3 tex_coord;
 
 void main() {
-    gl_Position = vec4((position_in.x - 0.5) * 16. , -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+    gl_Position = vec4((position_in.x - 0.5) * 16. , -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
     fragment_color = vec4(rgba_in.x, rgba_in.y, rgba_in.z, rgba_in.a * 2);
     tex_coord = tex_coord_in;
 }

game/graphics/opengl_renderer/shaders/sprite_cpu.vert

@@ -8,7 +8,7 @@ out vec4 fragment_color;
 out vec2 tex_coord;
 
 void main() {
-  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
   fragment_color = vec4(rgba_in.x, rgba_in.y, rgba_in.z, rgba_in.w * 2.);
   tex_coord = tex_coord_in;
 }

game/graphics/opengl_renderer/shaders/sprite_cpu_afail.vert

@@ -8,7 +8,7 @@ out vec4 fragment_color;
 out vec2 tex_coord;
 
 void main() {
-  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z, 1.0);
+  gl_Position = vec4((position_in.x - 0.5) * 16., -(position_in.y - 0.5) * 32, position_in.z * 2 - 1., 1.0);
   fragment_color = vec4(rgba_in.x, rgba_in.y, rgba_in.z, rgba_in.w * 2.);
   tex_coord = tex_coord_in;
 }

game/graphics/opengl_renderer/shaders/tfrag3.frag

@@ -0,0 +1,24 @@
+#version 330 core
+
+out vec4 color;
+
+in vec4 fragment_color;
+in vec3 tex_coord;
+uniform sampler2D tex_T0;
+
+uniform float alpha_min;
+uniform float alpha_max;
+
+void main() {
+    //vec4 T0 = texture(tex_T0, tex_coord);
+    vec4 T0 = texture(tex_T0, tex_coord.xy / tex_coord.z);
+    color = fragment_color * T0 * 2.0;
+
+    if (color.a <= alpha_min) {
+        discard;
+    }
+
+    if (color.a > alpha_max) {
+        discard;
+    }
+}

game/graphics/opengl_renderer/shaders/tfrag3.vert

@@ -0,0 +1,71 @@
+#version 330 core
+
+layout (location = 0) in vec3 position_in;
+layout (location = 1) in vec3 tex_coord_in;
+layout (location = 2) in float time_of_day_index;
+
+uniform vec4 hvdf_offset;
+uniform mat4 camera;
+uniform float fog_constant;
+uniform sampler1D tex_T1; // note, sampled in the vertex shader on purpose.
+
+out vec4 fragment_color;
+out vec3 tex_coord;
+
+void main() {
+
+
+    // old system:
+    // - load vf12
+    // - itof0 vf12
+    // - multiply with camera matrix (add trans)
+    // - let Q = fogx / vf12.w
+    // - xyz *= Q
+    // - xyzw += hvdf_offset
+    // - clip w.
+    // - ftoi4 vf12
+    // use in gs.
+    // gs is 12.4 fixed point, set up with 2048.0 as the center.
+
+    // the itof0 is done in the preprocessing step.  now we have floats.
+
+    // Step 3, the camera transform
+    vec4 transformed = -camera[3].xyzw;
+    transformed += -camera[0] * position_in.x;
+    transformed += -camera[1] * position_in.y;
+    transformed += -camera[2] * position_in.z;
+
+    // compute Q
+    float Q = fog_constant / transformed[3];
+
+    // perspective divide!
+    transformed.xyz *= Q;
+
+    // offset
+    transformed.xyz += hvdf_offset.xyz;
+
+    // ftoi4
+    //transformed.xyzw *= 16;
+
+    // correct xy offset
+    transformed.xy -= (2048.);
+
+    // correct z scale
+    transformed.z /= (16777216);
+    transformed.z *= 2;
+    transformed.z -= 1;
+
+    // correct xy scale
+    transformed.x /= (256);
+    transformed.y /= -(128);
+
+    // hack
+    transformed.xyz *= transformed.w;
+
+    gl_Position = transformed;
+
+    // time of day lookup
+    fragment_color = texture(tex_T1, time_of_day_index / 8192.f);
+    fragment_color.w *= 2;
+    tex_coord = tex_coord_in;
+}

game/graphics/opengl_renderer/shaders/tfrag3_no_tex.frag

@@ -0,0 +1,14 @@
+#version 330 core
+
+out vec4 color;
+
+in vec4 fragment_color;
+in vec3 tex_coord;
+uniform sampler2D tex_T0;
+
+uniform float alpha_min;
+uniform float alpha_max;
+
+void main() {
+    color = fragment_color;
+}

game/graphics/opengl_renderer/shaders/tfrag3_no_tex.vert

@@ -0,0 +1,50 @@
+#version 330 core
+
+layout (location = 0) in vec3 position_in;
+layout (location = 1) in vec4 rgba_in;
+
+uniform vec4 hvdf_offset;
+uniform mat4 camera;
+uniform float fog_constant;
+
+out vec4 fragment_color;
+
+// this is just for debugging.
+void main() {
+    vec4 transformed = -camera[3].xyzw;
+    transformed += -camera[0] * position_in.x;
+    transformed += -camera[1] * position_in.y;
+    transformed += -camera[2] * position_in.z;
+
+    // compute Q
+    float Q = fog_constant / transformed[3];
+
+    // perspective divide!
+    transformed.xyz *= Q;
+
+    // offset
+    transformed.xyz += hvdf_offset.xyz;
+
+    // ftoi4
+    //transformed.xyzw *= 16;
+
+    // correct xy offset
+    transformed.xy -= (2048.);
+
+    // correct z scale
+    transformed.z /= (16777216);
+    transformed.z *= 2;
+    transformed.z -= 1;
+
+    // correct xy scale
+    transformed.x /= (256);
+    transformed.y /= -(128);
+
+    // hack
+    transformed.xyz *= transformed.w;
+
+    gl_Position = transformed;
+
+    // time of day lookup
+    fragment_color = rgba_in;
+}

game/graphics/opengl_renderer/tfrag/BufferedRenderer.cpp

@@ -5,81 +5,6 @@
 
 namespace BufferedRenderer {
 
-std::string DrawMode::to_string() const {
-  std::string result;
-  result += fmt::format(" depth-write: {}\n", get_depth_write_enable());
-  result += fmt::format(" depth-test: ");
-  switch (get_depth_test()) {
-    case GsTest::ZTest::NEVER:
-      result += "never\n";
-      break;
-    case GsTest::ZTest::GEQUAL:
-      result += "gequal\n";
-      break;
-    case GsTest::ZTest::ALWAYS:
-      result += "always\n";
-      break;
-    case GsTest::ZTest::GREATER:
-      result += "greater\n";
-      break;
-    default:
-      assert(false);
-  }
-  result += fmt::format(" alpha: ");
-  switch (get_alpha_blend()) {
-    case AlphaBlend::SRC_0_SRC_DST:
-      result += "src, 0, src, dst\n";
-      break;
-    case AlphaBlend::SRC_DST_SRC_DST:
-      result += "src, dst, src, dst\n";
-      break;
-    case AlphaBlend::DISABLED:
-      result += "disabled\n";
-      break;
-    default:
-      assert(false);
-  }
-  result += fmt::format(" clamp: {}\n", get_clamp_enable());
-  result += fmt::format(" filt: {}\n", get_filt_enable());
-  result += fmt::format(" tcc: {}\n", get_tcc_enable());
-  result += fmt::format(" aref: {}\n", get_aref());
-  result += fmt::format(" ate: {}\n", get_at_enable());
-  result += fmt::format(" atst: ");
-  switch (get_alpha_test()) {
-    case AlphaTest::ALWAYS:
-      result += "always\n";
-      break;
-    case AlphaTest::GEQUAL:
-      result += "gequal\n";
-      break;
-    case AlphaTest::NEVER:
-      result += "never\n";
-      break;
-    default:
-      assert(false);
-  }
-  result += fmt::format(" zte: {}\n", get_zt_enable());
-  result += fmt::format(" abe: {}\n", get_ab_enable());
-  result += fmt::format(" afail: ");
-  switch (get_alpha_fail()) {
-    case GsTest::AlphaFail::KEEP:
-      result += "keep\n";
-      break;
-    case GsTest::AlphaFail::FB_ONLY:
-      result += "fb-only\n";
-      break;
-    case GsTest::AlphaFail::RGB_ONLY:
-      result += "rgb-only\n";
-      break;
-    case GsTest::AlphaFail::ZB_ONLY:
-      result += "zb-only\n";
-      break;
-    default:
-      assert(false);
-  }
-  return result;
-}
-
 Renderer::Renderer(BucketId my_id) : m_my_id(my_id) {
   glGenBuffers(1, &m_ogl.vertex_buffer);
   glGenBuffers(1, &m_ogl.index_buffer);
@@ -203,13 +128,19 @@ void Renderer::setup_opengl_excluding_textures(SharedRenderState* render_state,
       default:
         assert(false);
     }
+  } else {
+    glDisable(GL_BLEND);
   }
 
-  if (mode.get_clamp_enable()) {
+  if (mode.get_clamp_s_enable()) {
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
   } else {
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
+  }
+
+  if (mode.get_clamp_t_enable()) {
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  } else {
     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
   }
 
@@ -245,6 +176,7 @@ void Renderer::setup_opengl_excluding_textures(SharedRenderState* render_state,
         alpha_reject);
     glUniform1i(glGetUniformLocation(render_state->shaders[ShaderId::BUFFERED_TCC1].id(), "T0"), 0);
   } else {
+    fmt::print("tcc off!\n");
     render_state->shaders[ShaderId::BUFFERED_TCC0].activate();
     glUniform1f(
         glGetUniformLocation(render_state->shaders[ShaderId::BUFFERED_TCC0].id(), "alpha_reject"),
@@ -563,8 +495,8 @@ void Builder::handle_clamp1(u64 val) {
     assert(false);
   }
 
-  // this isn't quite right, but I'm hoping it's enough!
-  m_current_mode.set_clamp_enable(val == 0b101);
+  m_current_mode.set_clamp_s_enable(val & 0b1);
+  m_current_mode.set_clamp_t_enable(val & 0b100);
 }
 
 void Builder::handle_alpha1(u64 val) {

game/graphics/opengl_renderer/tfrag/BufferedRenderer.h

@@ -41,134 +41,6 @@ struct Triangle {
   u32 verts[3];
 };
 
-// this represents all of the drawing state, stored as an integer.
-// it can also represent "invalid".
-class DrawMode {
- public:
-  enum class AlphaBlend {
-    DISABLED = 0,
-    SRC_DST_SRC_DST = 1,
-    SRC_0_SRC_DST = 2,
-  };
-
-  enum class AlphaTest {
-    NEVER = 0,
-    ALWAYS = 1,
-    GEQUAL = 2,
-  };
-
-  bool get_depth_write_enable() const { return m_val & 0b1; }
-  void enable_depth_write() { m_val = m_val | 0b1; }
-  void disable_depth_write() { m_val = m_val & ~(0b1); }
-
-  GsTest::ZTest get_depth_test() const { return (GsTest::ZTest)((m_val >> 1) & 0b11); }
-  void set_depth_test(GsTest::ZTest dt) { m_val = (m_val & ~(0b110)) | ((u32)(dt) << 1); }
-
-  AlphaBlend get_alpha_blend() const { return (AlphaBlend)((m_val >> 3) & 0b11); }
-  void set_alpha_blend(AlphaBlend ab) { m_val = (m_val & ~(0b11000)) | ((u32)(ab) << 3); }
-
-  u8 get_aref() const { return m_val >> 8; }
-  void set_aref(u8 val) { m_val = (m_val & ~(0xff00)) | (val << 8); }
-
-  AlphaTest get_alpha_test() const { return (AlphaTest)((m_val >> 16) & 0b11); }
-  void set_alpha_test(AlphaTest ab) { m_val = (m_val & ~(0b11 << 16)) | ((u32)(ab) << 16); }
-
-  GsTest::AlphaFail get_alpha_fail() const { return (GsTest::AlphaFail)((m_val >> 21) & 0b11); }
-  void set_alpha_fail(GsTest::AlphaFail ab) { m_val = (m_val & ~(0b11 << 21)) | ((u32)(ab) << 21); }
-
-  bool is_invalid() const { return m_val == UINT32_MAX; }
-  bool is_valid() const { return !is_invalid(); }
-  void set_invalid() { m_val = UINT32_MAX; }
-
-  bool get_clamp_enable() const { return m_val & (1 << 5); }
-  void set_clamp_enable(bool en) {
-    if (en) {
-      enable_clamp();
-    } else {
-      disable_clamp();
-    }
-  }
-  void enable_clamp() { m_val = m_val | (1 << 5); }
-  void disable_clamp() { m_val = m_val & (~(1 << 5)); }
-
-  bool get_filt_enable() const { return m_val & (1 << 6); }
-  void enable_filt() { m_val = m_val | (1 << 6); }
-  void disable_filt() { m_val = m_val & (~(1 << 6)); }
-  void set_filt_enable(bool en) {
-    if (en) {
-      enable_filt();
-    } else {
-      disable_filt();
-    }
-  }
-
-  bool get_tcc_enable() const { return m_val & (1 << 6); }
-  void enable_tcc() { m_val = m_val | (1 << 7); }
-  void disable_tcc() { m_val = m_val & (~(1 << 7)); }
-  void set_tcc(bool en) {
-    if (en) {
-      enable_tcc();
-    } else {
-      disable_tcc();
-    }
-  }
-
-  bool get_at_enable() const { return m_val & (1 << 18); }
-  void enable_at() { m_val = m_val | (1 << 18); }
-  void disable_at() { m_val = m_val & (~(1 << 18)); }
-  void set_at(bool en) {
-    if (en) {
-      enable_at();
-    } else {
-      disable_at();
-    }
-  }
-
-  bool get_zt_enable() const { return m_val & (1 << 19); }
-  void enable_zt() { m_val = m_val | (1 << 19); }
-  void disable_zt() { m_val = m_val & (~(1 << 19)); }
-  void set_zt(bool en) {
-    if (en) {
-      enable_zt();
-    } else {
-      disable_zt();
-    }
-  }
-
-  bool get_ab_enable() const { return m_val & (1 << 20); }
-  void enable_ab() { m_val = m_val | (1 << 20); }
-  void disable_ab() { m_val = m_val & (~(1 << 20)); }
-  void set_ab(bool en) {
-    if (en) {
-      enable_ab();
-    } else {
-      disable_ab();
-    }
-  }
-
-  u32& as_int() { return m_val; }
-
-  bool operator==(const DrawMode& other) const { return m_val == other.m_val; }
-  bool operator!=(const DrawMode& other) const { return m_val != other.m_val; }
-
-  std::string to_string() const;
-
- private:
-  // 0 - depth write enable
-  // 1, 2 - test: never, always, gequal, greater
-  // 3, 4 - alpha: disable, [src,dst,src,dst], [src,0,src,dst], XX
-  // 5 - clamp enable
-  // 6 - filt enable
-  // 7 - tcc enable
-  // 8,9,10,11,12,14,14,15 - aref
-  // 16, 17 - atest
-  // 18 - ate
-  // 19 - zte
-  // 20 - abe
-  // 21, 22 - afail
-  u32 m_val = UINT32_MAX;
-};
-
 struct Draw {
   DrawMode mode;
   std::vector<Triangle> triangles;  // just indices

game/graphics/opengl_renderer/tfrag/TFragment.cpp

@@ -15,11 +15,15 @@ bool looks_like_tfrag_init(const DmaFollower& follow) {
 }
 }  // namespace
 
-TFragment::TFragment(const std::string& name, BucketId my_id, bool child_mode)
+TFragment::TFragment(const std::string& name,
+                     BucketId my_id,
+                     const std::vector<tfrag3::TFragmentTreeKind>& trees,
+                     bool child_mode)
     : BucketRenderer(name, my_id),
       m_child_mode(child_mode),
       m_direct_renderer(fmt::format("{}.direct", name), my_id, 1024, DirectRenderer::Mode::NORMAL),
-      m_buffered_renderer(my_id) {
+      m_buffered_renderer(my_id),
+      m_tree_kinds(trees) {
   for (auto& buf : m_buffered_data) {
     for (auto& x : buf.pad) {
       x = 0xff;
@@ -31,6 +35,10 @@ TFragment::TFragment(const std::string& name, BucketId my_id, bool child_mode)
   }
 }
 
+constexpr const char* level_names[] = {"bea", "cit", "dar", "fin", "int", "jub", "jun", "fic",
+                                       "lav", "mai", "mis", "ogr", "rob", "rol", "sno", "sub",
+                                       "sun", "swa", "tit", "tra", "vi1", "vi2", "vi3"};
+
 void TFragment::render(DmaFollower& dma,
                        SharedRenderState* render_state,
                        ScopedProfilerNode& prof) {
@@ -73,32 +81,80 @@ void TFragment::render(DmaFollower& dma,
     ImGui::Begin(fmt::format("{} extra", m_name).c_str());
   }
 
-  while (looks_like_tfrag_init(dma)) {
-    m_debug_string += "------------- START!\n";
-    handle_initialization(dma, render_state, prof);
-    int count = 0;
-    // fmt::print("---------------------------------------START\n");
-
-    while (looks_like_tfragment_dma(dma)) {
-      m_stats.tfrag_dma_packets++;
-      auto frag = dma.read_and_advance();
-      m_stats.tfrag_bytes += frag.size_bytes;
-
-      if (m_extra_debug) {
-        handle_tfrag<true>(frag, render_state, prof);
-      } else {
-        handle_tfrag<false>(frag, render_state, prof);
+  if (m_use_tfrag3) {
+    std::string level_name;
+    while (looks_like_tfrag_init(dma)) {
+      handle_initialization(dma, render_state, prof);
+      if (level_name.empty()) {
+        level_name = m_pc_port_data.level_name;
+      } else if (level_name != m_pc_port_data.level_name) {
+        assert(false);
       }
-      if (m_max_draw >= 0 && count++ > m_max_draw) {
-        break;
+
+      while (looks_like_tfragment_dma(dma)) {
+        dma.read_and_advance();
       }
     }
 
-    if (dma.current_tag().qwc == 3) {
+    while (dma.current_tag_offset() != render_state->next_bucket) {
       dma.read_and_advance();
     }
-    if (dma.current_tag().qwc == 0) {
-      dma.read_and_advance();
+
+    assert(!level_name.empty());
+    m_tfrag3.setup_for_level(level_name, render_state);
+    Tfrag3::RenderSettings settings;
+    settings.hvdf_offset = m_tfrag_data.hvdf_offset;
+    settings.fog_x = m_tfrag_data.fog.x();
+    memcpy(settings.math_camera.data(), &m_buffered_data[0].pad[TFragDataMem::TFragMatrix0 * 16],
+           64);
+    settings.tree_idx = 0;
+
+    for (int i = 0; i < 4; i++) {
+      settings.planes[i] = m_pc_port_data.planes[i];
+    }
+
+    if (m_override_time_of_day) {
+      for (int i = 0; i < 8; i++) {
+        settings.time_of_day_weights[i] = m_time_of_days[i];
+      }
+    } else {
+      for (int i = 0; i < 8; i++) {
+        settings.time_of_day_weights[i] =
+            2 * (0xff & m_pc_port_data.itimes[i / 2].data()[2 * (i % 2)]) / 127.f;
+      }
+    }
+
+    auto t3prof = prof.make_scoped_child("t3");
+    m_tfrag3.render_matching_trees(m_tree_kinds, settings, render_state, t3prof);
+
+  } else {
+    while (looks_like_tfrag_init(dma)) {
+      m_debug_string += "------------- START!\n";
+      handle_initialization(dma, render_state, prof);
+      int count = 0;
+      // fmt::print("---------------------------------------START\n");
+
+      while (looks_like_tfragment_dma(dma)) {
+        m_stats.tfrag_dma_packets++;
+        auto frag = dma.read_and_advance();
+        m_stats.tfrag_bytes += frag.size_bytes;
+
+        if (m_extra_debug) {
+          handle_tfrag<true>(frag, render_state, prof);
+        } else {
+          handle_tfrag<false>(frag, render_state, prof);
+        }
+        if (m_max_draw >= 0 && count++ > m_max_draw) {
+          break;
+        }
+      }
+
+      if (dma.current_tag().qwc == 3) {
+        dma.read_and_advance();
+      }
+      if (dma.current_tag().qwc == 0) {
+        dma.read_and_advance();
+      }
     }
   }
 
@@ -120,6 +176,28 @@ void TFragment::render(DmaFollower& dma,
     m_debug_string +=
         fmt::format("DMA {} {} bytes, {}\n", tag, data.size_bytes, data.vifcode0().print());
   }
+
+  if (m_hack_test_many_levels) {
+    for (int i = 0; i < HackManyLevels::NUM_LEVELS; i++) {
+      if (m_many_level_render.level_enables[i]) {
+        m_many_level_render.level_renderers[i].setup_for_level(level_names[i], render_state);
+        Tfrag3::RenderSettings settings;
+        settings.hvdf_offset = m_tfrag_data.hvdf_offset;
+        settings.fog_x = m_tfrag_data.fog.x();
+        memcpy(settings.math_camera.data(),
+               &m_buffered_data[0].pad[TFragDataMem::TFragMatrix0 * 16], 64);
+        settings.tree_idx = 0;
+        for (int j = 0; j < 8; j++) {
+          settings.time_of_day_weights[j] = m_time_of_days[j];
+        }
+
+        auto t3prof = prof.make_scoped_child(level_names[i]);
+
+        m_many_level_render.level_renderers[i].debug_render_all_trees_nolores(settings,
+                                                                              render_state, t3prof);
+      }
+    }
+  }
 }
 void TFragment::draw_debug_window() {
   ImGui::Separator();
@@ -129,28 +207,47 @@ void TFragment::draw_debug_window() {
   if (ImGui::Button("All")) {
     m_max_draw = -1;
   }
-  ImGui::Checkbox("Skip MSCAL", &m_skip_mscals);
-  ImGui::Checkbox("Skip XGKICK", &m_skip_xgkick);
-  ImGui::Checkbox("Prog8 hack", &m_prog8_with_prog6);
-  ImGui::Checkbox("Prog10 hack", &m_prog10_with_prog6);
-  ImGui::Checkbox("Prog18 hack", &m_prog18_with_prog6);
-  ImGui::Checkbox("Others with prog6", &m_all_with_prog6);
-  ImGui::Checkbox("Use Buffered Renderer", &m_use_buffered_renderer);
-  ImGui::Text("packets: %d", m_stats.tfrag_dma_packets);
-  ImGui::Text("frag bytes: %d", m_stats.tfrag_bytes);
-  ImGui::Text("errors: %d", m_stats.error_packets);
-  for (int prog = 0; prog < 12; prog++) {
-    ImGui::Text("  prog %d: %d calls\n", prog, m_stats.per_program[prog].calls);
+  ImGui::Checkbox("Manual Time of Day", &m_override_time_of_day);
+  if (m_override_time_of_day) {
+    for (int i = 0; i < 8; i++) {
+      ImGui::SliderFloat(fmt::format("{}", i).c_str(), m_time_of_days + i, 0.f, 1.f);
+    }
   }
 
-  if (!m_use_buffered_renderer && ImGui::TreeNode("direct")) {
-    m_direct_renderer.draw_debug_window();
-    ImGui::TreePop();
+  ImGui::Checkbox("Hack Test Many (danger)", &m_hack_test_many_levels);
+  if (m_hack_test_many_levels) {
+    for (int i = 0; i < HackManyLevels::NUM_LEVELS; i++) {
+      ImGui::Checkbox(level_names[i], &m_many_level_render.level_enables[i]);
+    }
   }
 
-  if (m_use_buffered_renderer && ImGui::TreeNode("buffered")) {
-    m_buffered_renderer.draw_debug_window();
-    ImGui::TreePop();
+  ImGui::Checkbox("Use TFRAG3", &m_use_tfrag3);
+  if (!m_use_tfrag3) {
+    ImGui::Checkbox("Use Buffered Renderer", &m_use_buffered_renderer);
+    ImGui::Checkbox("Skip MSCAL", &m_skip_mscals);
+    ImGui::Checkbox("Skip XGKICK", &m_skip_xgkick);
+    ImGui::Checkbox("Prog8 hack", &m_prog8_with_prog6);
+    ImGui::Checkbox("Prog10 hack", &m_prog10_with_prog6);
+    ImGui::Checkbox("Prog18 hack", &m_prog18_with_prog6);
+    ImGui::Checkbox("Others with prog6", &m_all_with_prog6);
+    ImGui::Text("packets: %d", m_stats.tfrag_dma_packets);
+    ImGui::Text("frag bytes: %d", m_stats.tfrag_bytes);
+    ImGui::Text("errors: %d", m_stats.error_packets);
+    for (int prog = 0; prog < 12; prog++) {
+      ImGui::Text("  prog %d: %d calls\n", prog, m_stats.per_program[prog].calls);
+    }
+
+    if (!m_use_buffered_renderer && ImGui::TreeNode("direct")) {
+      m_direct_renderer.draw_debug_window();
+      ImGui::TreePop();
+    }
+
+    if (m_use_buffered_renderer && ImGui::TreeNode("buffered")) {
+      m_buffered_renderer.draw_debug_window();
+      ImGui::TreePop();
+    }
+  } else {
+    m_tfrag3.draw_debug_window();
   }
 
   ImGui::TextUnformatted(m_debug_string.data());
@@ -209,6 +306,24 @@ void TFragment::handle_initialization(DmaFollower& dma,
   // lq.xyzw vf04, 664(vi00)    |  nop
   m_globals.vf04_ambient = m_tfrag_data.ambient;  // TODO get rid?
 
+  auto pc_port_data = dma.read_and_advance();
+  assert(pc_port_data.size_bytes == sizeof(PcPortData));
+  memcpy(&m_pc_port_data, pc_port_data.data, sizeof(PcPortData));
+  m_pc_port_data.level_name[11] = '\0';
+
+  for (int i = 0; i < 4; i++) {
+    m_debug_string += fmt::format("p[{}]: {}\n", i, m_pc_port_data.planes[i].to_string_aligned());
+  }
+
+  for (int i = 0; i < 4; i++) {
+    m_debug_string += fmt::format("t[{}]: {:x} {:x} {:x} {:x}\n", i, m_pc_port_data.itimes[i].x(),
+                                  m_pc_port_data.itimes[i].y(), m_pc_port_data.itimes[i].z(),
+                                  m_pc_port_data.itimes[i].w());
+  }
+
+  m_debug_string +=
+      fmt::format("level: {}, tree: {}\n", m_pc_port_data.level_name, m_pc_port_data.tree_idx);
+
   // setup double buffering.
   auto db_setup = dma.read_and_advance();
   assert(db_setup.size_bytes == 0);

game/graphics/opengl_renderer/tfrag/TFragment.h

@@ -3,6 +3,7 @@
 #include "game/graphics/opengl_renderer/BucketRenderer.h"
 #include "game/graphics/opengl_renderer/DirectRenderer.h"
 #include "game/graphics/opengl_renderer/tfrag/BufferedRenderer.h"
+#include "game/graphics/opengl_renderer/tfrag/Tfrag3.h"
 #include "common/dma/gs.h"
 #include "common/math/Vector.h"
 
@@ -40,7 +41,10 @@ struct TFragKickZone {
 
 class TFragment : public BucketRenderer {
  public:
-  TFragment(const std::string& name, BucketId my_id, bool child_mode);
+  TFragment(const std::string& name,
+            BucketId my_id,
+            const std::vector<tfrag3::TFragmentTreeKind>& trees,
+            bool child_mode);
   void render(DmaFollower& dma, SharedRenderState* render_state, ScopedProfilerNode& prof) override;
   void draw_debug_window() override;
 
@@ -193,6 +197,10 @@ class TFragment : public BucketRenderer {
   bool m_prog18_with_prog6 = true;
   bool m_all_with_prog6 = false;
   bool m_use_buffered_renderer = true;
+  bool m_use_tfrag3 = true;
+  bool m_hack_test_many_levels = false;
+  bool m_override_time_of_day = false;
+  float m_time_of_days[8] = {0};
   std::string m_frag_debug;
 
   // GS setup data
@@ -204,6 +212,13 @@ class TFragment : public BucketRenderer {
   TFragData m_tfrag_data;
   TFragKickZone m_kick_data;
 
+  struct PcPortData {
+    Vector4f planes[4];
+    math::Vector<s32, 4> itimes[4];
+    char level_name[12];
+    u32 tree_idx;
+  } m_pc_port_data;
+
   // buffers
   TFragBufferedData m_buffered_data[2];
   int m_uploading_buffer = 0;
@@ -281,4 +296,12 @@ class TFragment : public BucketRenderer {
 
   DirectRenderer m_direct_renderer;
   BufferedRenderer::Builder m_buffered_renderer;
+  Tfrag3 m_tfrag3;
+  std::vector<tfrag3::TFragmentTreeKind> m_tree_kinds;
+
+  struct HackManyLevels {
+    static constexpr int NUM_LEVELS = 23;
+    Tfrag3 level_renderers[NUM_LEVELS];
+    bool level_enables[NUM_LEVELS] = {0};
+  } m_many_level_render;
 };

game/graphics/opengl_renderer/tfrag/Tfrag3.cpp

@@ -0,0 +1,666 @@
+#include "Tfrag3.h"
+
+#include "third-party/imgui/imgui.h"
+
+Tfrag3::Tfrag3() {
+  glGenVertexArrays(1, &m_debug_vao);
+  glBindVertexArray(m_debug_vao);
+  glGenBuffers(1, &m_debug_verts);
+  glBindBuffer(GL_ARRAY_BUFFER, m_debug_verts);
+  glBufferData(GL_ARRAY_BUFFER, DEBUG_TRI_COUNT * 3 * sizeof(DebugVertex), nullptr,
+               GL_DYNAMIC_DRAW);
+  glEnableVertexAttribArray(0);
+  glEnableVertexAttribArray(1);
+  glVertexAttribPointer(0,                                      // location 0 in the shader
+                        3,                                      // 3 values per vert
+                        GL_FLOAT,                               // floats
+                        GL_FALSE,                               // normalized
+                        sizeof(DebugVertex),                    // stride
+                        (void*)offsetof(DebugVertex, position)  // offset (0)
+  );
+
+  glVertexAttribPointer(1,                                  // location 1 in the shader
+                        4,                                  // 4 values per vert
+                        GL_FLOAT,                           // floats
+                        GL_FALSE,                           // normalized
+                        sizeof(DebugVertex),                // stride
+                        (void*)offsetof(DebugVertex, rgba)  // offset (0)
+  );
+  glBindVertexArray(0);
+}
+
+Tfrag3::~Tfrag3() {
+  discard_tree_cache();
+  glDeleteVertexArrays(1, &m_debug_vao);
+}
+
+void Tfrag3::setup_for_level(const std::string& level, SharedRenderState* render_state) {
+  // make sure we have the level data.
+  auto lev_data = render_state->loader.get_tfrag3_level(level);
+  if (m_level_name != level) {
+    fmt::print("new level for tfrag3: {} -> {}\n", m_level_name, level);
+    fmt::print("discarding old stuff\n");
+    discard_tree_cache();
+    fmt::print("level has {} trees\n", lev_data->trees.size());
+    m_cached_trees.resize(lev_data->trees.size());
+
+    size_t idx_buffer_len = 0;
+    size_t time_of_day_count = 0;
+
+    for (size_t tree_idx = 0; tree_idx < lev_data->trees.size(); tree_idx++) {
+      const auto& tree = lev_data->trees[tree_idx];
+      m_cached_trees[tree_idx].kind = tree.kind;
+      if (tree.kind != tfrag3::TFragmentTreeKind::INVALID) {
+        for (auto& draw : tree.draws) {
+          idx_buffer_len = std::max(idx_buffer_len, draw.vertex_index_stream.size());
+        }
+        time_of_day_count = std::max(tree.colors.size(), time_of_day_count);
+        u32 verts = tree.vertices.size();
+        fmt::print("  tree {} has {} verts ({} kB) and {} draws\n", tree_idx, verts,
+                   verts * sizeof(tfrag3::PreloadedVertex) / 1024.f, tree.draws.size());
+        glGenVertexArrays(1, &m_cached_trees[tree_idx].vao);
+        glBindVertexArray(m_cached_trees[tree_idx].vao);
+        glGenBuffers(1, &m_cached_trees[tree_idx].vertex_buffer);
+        m_cached_trees[tree_idx].vert_count = verts;
+        m_cached_trees[tree_idx].draws = &tree.draws;  // todo - should we just copy this?
+        m_cached_trees[tree_idx].colors = &tree.colors;
+        m_cached_trees[tree_idx].vis = &tree.vis_nodes;
+        // don't bother with vis if we only have children.
+        m_cached_trees[tree_idx].num_vis_tree_roots = tree.only_children ? 0 : tree.num_roots;
+        m_cached_trees[tree_idx].vis_tree_root = tree.first_root;
+        m_cached_trees[tree_idx].vis_temp.resize(tree.vis_nodes.size());
+        m_cached_trees[tree_idx].culled_indices.resize(idx_buffer_len);
+        glBindBuffer(GL_ARRAY_BUFFER, m_cached_trees[tree_idx].vertex_buffer);
+        glBufferData(GL_ARRAY_BUFFER, verts * sizeof(tfrag3::PreloadedVertex), nullptr,
+                     GL_DYNAMIC_DRAW);
+        glEnableVertexAttribArray(0);
+        glEnableVertexAttribArray(1);
+        glEnableVertexAttribArray(2);
+
+        glBufferSubData(GL_ARRAY_BUFFER, 0, verts * sizeof(tfrag3::PreloadedVertex),
+                        tree.vertices.data());
+
+        glVertexAttribPointer(0,                                // location 0 in the shader
+                              3,                                // 3 values per vert
+                              GL_FLOAT,                         // floats
+                              GL_FALSE,                         // normalized
+                              sizeof(tfrag3::PreloadedVertex),  // stride
+                              (void*)offsetof(tfrag3::PreloadedVertex, x)  // offset (0)
+        );
+
+        glVertexAttribPointer(1,                                // location 1 in the shader
+                              3,                                // 3 values per vert
+                              GL_FLOAT,                         // floats
+                              GL_FALSE,                         // normalized
+                              sizeof(tfrag3::PreloadedVertex),  // stride
+                              (void*)offsetof(tfrag3::PreloadedVertex, s)  // offset (0)
+        );
+
+        glVertexAttribPointer(2,                                // location 2 in the shader
+                              1,                                // 1 values per vert
+                              GL_UNSIGNED_SHORT,                // u16
+                              GL_FALSE,                         // don't normalize
+                              sizeof(tfrag3::PreloadedVertex),  // stride
+                              (void*)offsetof(tfrag3::PreloadedVertex, color_index)  // offset (0)
+        );
+        glBindVertexArray(0);
+      }
+    }
+
+    fmt::print("level has {} textures\n", lev_data->textures.size());
+    for (auto& tex : lev_data->textures) {
+      GLuint gl_tex;
+      // fmt::print("  tex: {} x {} {} {}\n", tex.w, tex.h, tex.debug_name, tex.debug_tpage_name);
+      glGenTextures(1, &gl_tex);
+      glBindTexture(GL_TEXTURE_2D, gl_tex);
+      glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, tex.w, tex.h, 0, GL_RGBA, GL_UNSIGNED_INT_8_8_8_8_REV,
+                   tex.data.data());
+      glBindTexture(GL_TEXTURE_2D, 0);
+      glActiveTexture(GL_TEXTURE0);
+      glBindTexture(GL_TEXTURE_2D, gl_tex);
+      glGenerateMipmap(GL_TEXTURE_2D);
+
+      float aniso = 0.0f;
+      glGetFloatv(GL_MAX_TEXTURE_MAX_ANISOTROPY, &aniso);
+      glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY, aniso);
+      m_textures.push_back(gl_tex);
+    }
+
+    fmt::print("level max index stream: {}\n", idx_buffer_len);
+    m_has_index_buffer = true;
+    glGenBuffers(1, &m_index_buffer);
+    glActiveTexture(GL_TEXTURE1);
+    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, m_index_buffer);
+    glBufferData(GL_ELEMENT_ARRAY_BUFFER, idx_buffer_len * sizeof(u32), nullptr, GL_DYNAMIC_DRAW);
+
+    fmt::print("level max time of day: {}\n", time_of_day_count);
+    assert(time_of_day_count <= TIME_OF_DAY_COLOR_COUNT);
+    // regardless of how many we use some fixed max
+    // we won't actually interp or upload to gpu the unused ones, but we need a fixed maximum so
+    // indexing works properly.
+    m_color_result.resize(TIME_OF_DAY_COLOR_COUNT);
+    glGenTextures(1, &m_time_of_day_texture);
+    m_has_time_of_day_texture = true;
+    glBindTexture(GL_TEXTURE_1D, m_time_of_day_texture);
+    // just fill with zeros. this lets use use the faster texsubimage later
+    glTexImage1D(GL_TEXTURE_1D, 0, GL_RGBA, TIME_OF_DAY_COLOR_COUNT, 0, GL_RGBA,
+                 GL_UNSIGNED_INT_8_8_8_8, m_color_result.data());
+    glTexParameteri(GL_TEXTURE_1D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+    glTexParameteri(GL_TEXTURE_1D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+
+    m_level_name = level;
+  }
+}
+
+void Tfrag3::first_draw_setup(const RenderSettings& settings, SharedRenderState* render_state) {
+  render_state->shaders[ShaderId::TFRAG3].activate();
+  glUniform1i(glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "tex_T0"), 0);
+  glUniform1i(glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "tex_T1"), 1);
+  glUniformMatrix4fv(glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "camera"),
+                     1, GL_FALSE, settings.math_camera.data());
+  glUniform4f(glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "hvdf_offset"),
+              settings.hvdf_offset[0], settings.hvdf_offset[1], settings.hvdf_offset[2],
+              settings.hvdf_offset[3]);
+  glUniform1f(glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "fog_constant"),
+              settings.fog_x);
+}
+
+Tfrag3::DoubleDraw Tfrag3::setup_shader(const RenderSettings& /*settings*/,
+                                        SharedRenderState* render_state,
+                                        DrawMode mode) {
+  glActiveTexture(GL_TEXTURE0);
+
+  if (mode.get_zt_enable()) {
+    glEnable(GL_DEPTH_TEST);
+    switch (mode.get_depth_test()) {
+      case GsTest::ZTest::NEVER:
+        glDepthFunc(GL_NEVER);
+        break;
+      case GsTest::ZTest::ALWAYS:
+        glDepthFunc(GL_ALWAYS);
+        break;
+      case GsTest::ZTest::GEQUAL:
+        glDepthFunc(GL_GEQUAL);
+        break;
+      case GsTest::ZTest::GREATER:
+        glDepthFunc(GL_GREATER);
+        break;
+      default:
+        assert(false);
+    }
+  } else {
+    glDisable(GL_DEPTH_TEST);
+  }
+
+  if (mode.get_ab_enable() && mode.get_alpha_blend() != DrawMode::AlphaBlend::DISABLED) {
+    glEnable(GL_BLEND);
+    switch (mode.get_alpha_blend()) {
+      case DrawMode::AlphaBlend::SRC_DST_SRC_DST:
+        glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
+        break;
+      case DrawMode::AlphaBlend::SRC_0_SRC_DST:
+        glBlendFunc(GL_SRC_ALPHA, GL_ONE);
+        break;
+      case DrawMode::AlphaBlend::SRC_0_FIX_DST:
+        glBlendFunc(GL_ONE, GL_ONE);
+        break;
+      default:
+        assert(false);
+    }
+  } else {
+    glDisable(GL_BLEND);
+  }
+
+  if (mode.get_clamp_s_enable()) {
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  } else {
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
+  }
+
+  if (mode.get_clamp_t_enable()) {
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  } else {
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
+  }
+
+  if (mode.get_filt_enable()) {
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+  } else {
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+  }
+
+  // for some reason, they set atest NEVER + FB_ONLY to disable depth writes
+  bool alpha_hack_to_disable_z_write = false;
+  DoubleDraw double_draw;
+
+  float alpha_min = 0.;
+  if (mode.get_at_enable()) {
+    switch (mode.get_alpha_test()) {
+      case DrawMode::AlphaTest::ALWAYS:
+        break;
+      case DrawMode::AlphaTest::GEQUAL:
+        alpha_min = mode.get_aref() / 127.f;
+        switch (mode.get_alpha_fail()) {
+          case GsTest::AlphaFail::KEEP:
+            // ok, no need for double draw
+            break;
+          case GsTest::AlphaFail::FB_ONLY:
+            // darn, we need to draw twice
+            double_draw.kind = DoubleDrawKind::AFAIL_NO_DEPTH_WRITE;
+            double_draw.aref = alpha_min;
+            break;
+          default:
+            assert(false);
+        }
+        break;
+      case DrawMode::AlphaTest::NEVER:
+        if (mode.get_alpha_fail() == GsTest::AlphaFail::FB_ONLY) {
+          alpha_hack_to_disable_z_write = true;
+        } else {
+          assert(false);
+        }
+        break;
+      default:
+        assert(false);
+    }
+  }
+
+  if (mode.get_depth_write_enable()) {
+    glDepthMask(GL_TRUE);
+  } else {
+    glDepthMask(GL_FALSE);
+  }
+
+  glUniform1f(glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "alpha_min"),
+              alpha_min);
+  glUniform1f(glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "alpha_max"),
+              10.f);
+
+  return double_draw;
+}
+
+void Tfrag3::render_tree(const RenderSettings& settings,
+                         SharedRenderState* render_state,
+                         ScopedProfilerNode& prof,
+                         bool use_vis) {
+  auto& tree = m_cached_trees.at(settings.tree_idx);
+  assert(tree.kind != tfrag3::TFragmentTreeKind::INVALID);
+
+  if (m_color_result.size() < tree.colors->size()) {
+    m_color_result.resize(tree.colors->size());
+  }
+  interp_time_of_day_slow(settings.time_of_day_weights, *tree.colors, m_color_result.data());
+  glActiveTexture(GL_TEXTURE1);
+  glBindTexture(GL_TEXTURE_1D, m_time_of_day_texture);
+  glTexSubImage1D(GL_TEXTURE_1D, 0, 0, tree.colors->size(), GL_RGBA, GL_UNSIGNED_INT_8_8_8_8_REV,
+                  m_color_result.data());
+
+  first_draw_setup(settings, render_state);
+
+  glBindVertexArray(tree.vao);
+  glBindBuffer(GL_ARRAY_BUFFER, tree.vertex_buffer);
+  glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, m_index_buffer);
+  glActiveTexture(GL_TEXTURE0);
+  glEnable(GL_PRIMITIVE_RESTART);
+  glPrimitiveRestartIndex(UINT32_MAX);
+
+  for (const auto& draw : *tree.draws) {
+    glBindTexture(GL_TEXTURE_2D, m_textures.at(draw.tree_tex_id));
+    auto double_draw = setup_shader(settings, render_state, draw.mode);
+    tree.tris_this_frame += draw.num_triangles;
+    tree.draws_this_frame++;
+    int draw_size = draw.vertex_index_stream.size();
+    if (use_vis) {
+      int vtx_idx = 0;
+      int out_idx = 0;
+      for (auto& grp : draw.vis_groups) {
+        if (grp.tfrag_idx == 0xffffffff || tree.vis_temp.at(grp.tfrag_idx)) {
+          memcpy(&tree.culled_indices[out_idx], &draw.vertex_index_stream[vtx_idx],
+                 grp.num * sizeof(u32));
+          out_idx += grp.num;
+        }
+
+        vtx_idx += grp.num;
+      }
+
+      draw_size = out_idx;
+      if (draw_size == 0) {
+        continue;
+      }
+
+      prof.add_draw_call();
+      prof.add_tri(draw.num_triangles * (float)out_idx / draw.vertex_index_stream.size());
+
+      glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, 0, out_idx * sizeof(u32),
+                      tree.culled_indices.data());
+    } else {
+      glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, 0, draw.vertex_index_stream.size() * sizeof(u32),
+                      draw.vertex_index_stream.data());
+    }
+
+    glDrawElements(GL_TRIANGLE_STRIP, draw_size, GL_UNSIGNED_INT, (void*)0);
+
+    switch (double_draw.kind) {
+      case DoubleDrawKind::NONE:
+        break;
+      case DoubleDrawKind::AFAIL_NO_DEPTH_WRITE:
+        prof.add_draw_call();
+        prof.add_tri(draw_size);
+        glUniform1f(glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "alpha_min"),
+                    -10.f);
+        glUniform1f(glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3].id(), "alpha_max"),
+                    double_draw.aref);
+        glDepthMask(GL_FALSE);
+        glDrawElements(GL_TRIANGLE_STRIP, draw_size, GL_UNSIGNED_INT, (void*)0);
+        break;
+      default:
+        assert(false);
+    }
+  }
+  glBindVertexArray(0);
+}
+
+bool sphere_in_view_ref(const math::Vector4f& sphere, const math::Vector4f* planes) {
+  /*
+   *(let ((v1-0 *math-camera*))
+    (.lvf vf6 (&-> arg0 quad))
+    (.lvf vf1 (&-> v1-0 plane 0 quad))
+    (.lvf vf2 (&-> v1-0 plane 1 quad))
+    (.lvf vf3 (&-> v1-0 plane 2 quad))
+    (.lvf vf4 (&-> v1-0 plane 3 quad))
+    )
+   (.mul.x.vf acc vf1 vf6)
+   (.add.mul.y.vf acc vf2 vf6 acc)
+   (.add.mul.z.vf acc vf3 vf6 acc)
+   (.sub.mul.w.vf vf5 vf4 vf0 acc)
+   (.add.w.vf vf5 vf5 vf6)
+   (.mov v1-1 vf5)
+   (.pcgtw v1-2 r0-0 v1-1)
+   (.ppach v1-3 r0-0 v1-2)
+   (zero? (the-as int v1-3))
+   */
+
+  math::Vector4f acc =
+      planes[0] * sphere.x() + planes[1] * sphere.y() + planes[2] * sphere.z() - planes[3];
+
+  return acc.x() > -sphere.w() && acc.y() > -sphere.w() && acc.z() > -sphere.w() &&
+         acc.w() > -sphere.w();
+}
+
+void cull_ref_all(const math::Vector4f* planes,
+                  const std::vector<tfrag3::VisNode>& nodes,
+                  u8* out) {
+  for (size_t i = 0; i < nodes.size(); i++) {
+    out[i] = sphere_in_view_ref(nodes[i].bsphere, planes);
+  }
+}
+
+/*!
+ * Render all trees with settings for the given tree.
+ * This is intended to be used only for debugging when we can't easily get commands for all trees
+ * working.
+ */
+void Tfrag3::render_all_trees(const RenderSettings& settings,
+                              SharedRenderState* render_state,
+                              ScopedProfilerNode& prof) {
+  RenderSettings settings_copy = settings;
+  for (size_t i = 0; i < m_cached_trees.size(); i++) {
+    if (m_cached_trees[i].kind != tfrag3::TFragmentTreeKind::INVALID) {
+      settings_copy.tree_idx = i;
+      render_tree(settings_copy, render_state, prof, false);
+    }
+  }
+}
+
+void Tfrag3::render_matching_trees(const std::vector<tfrag3::TFragmentTreeKind>& trees,
+                                   const RenderSettings& settings,
+                                   SharedRenderState* render_state,
+                                   ScopedProfilerNode& prof) {
+  RenderSettings settings_copy = settings;
+  for (size_t i = 0; i < m_cached_trees.size(); i++) {
+    m_cached_trees[i].reset_stats();
+    if (!m_cached_trees[i].allowed) {
+      continue;
+    }
+    if (std::find(trees.begin(), trees.end(), m_cached_trees[i].kind) != trees.end() ||
+        m_cached_trees[i].forced) {
+      m_cached_trees[i].rendered_this_frame = true;
+      settings_copy.tree_idx = i;
+      cull_ref_all(settings.planes, *m_cached_trees[i].vis, m_cached_trees[i].vis_temp.data());
+      render_tree(settings_copy, render_state, prof, true);
+      if (m_cached_trees[i].cull_debug) {
+        render_tree_cull_debug(settings_copy, render_state, prof);
+      }
+    }
+  }
+}
+
+void Tfrag3::debug_render_all_trees_nolores(const RenderSettings& settings,
+                                            SharedRenderState* render_state,
+                                            ScopedProfilerNode& prof) {
+  RenderSettings settings_copy = settings;
+  for (size_t i = 0; i < m_cached_trees.size(); i++) {
+    if (m_cached_trees[i].kind != tfrag3::TFragmentTreeKind::INVALID &&
+        m_cached_trees[i].kind != tfrag3::TFragmentTreeKind::LOWRES_TRANS &&
+        m_cached_trees[i].kind != tfrag3::TFragmentTreeKind::LOWRES) {
+      settings_copy.tree_idx = i;
+      render_tree(settings_copy, render_state, prof, false);
+    }
+  }
+
+  for (size_t i = 0; i < m_cached_trees.size(); i++) {
+    if (m_cached_trees[i].kind != tfrag3::TFragmentTreeKind::INVALID &&
+        m_cached_trees[i].kind != tfrag3::TFragmentTreeKind::LOWRES_TRANS &&
+        m_cached_trees[i].kind != tfrag3::TFragmentTreeKind::LOWRES) {
+      settings_copy.tree_idx = i;
+      render_tree_cull_debug(settings_copy, render_state, prof);
+    }
+  }
+}
+
+void Tfrag3::draw_debug_window() {
+  for (int i = 0; i < (int)m_cached_trees.size(); i++) {
+    auto& tree = m_cached_trees[i];
+    if (tree.kind == tfrag3::TFragmentTreeKind::INVALID) {
+      continue;
+    }
+    ImGui::PushID(i);
+    ImGui::Text("[%d] %10s", i, tfrag3::tfrag_tree_names[(int)m_cached_trees[i].kind]);
+    ImGui::SameLine();
+    ImGui::Checkbox("Allow?", &tree.allowed);
+    ImGui::SameLine();
+    ImGui::Checkbox("Force?", &tree.forced);
+    ImGui::SameLine();
+    ImGui::Checkbox("cull debug (slow)", &tree.cull_debug);
+    ImGui::PopID();
+    if (tree.rendered_this_frame) {
+      ImGui::Text("  tris: %d draws: %d", tree.tris_this_frame, tree.draws_this_frame);
+      int vis = 0;
+      for (auto x : tree.vis_temp) {
+        if (x) {
+          vis++;
+        }
+      }
+      ImGui::Text("  cull: %d vis out of %d", vis, (int)tree.vis_temp.size());
+    }
+    ImGui::Text("root: %d, roots: %d, nodes %d", tree.vis_tree_root, tree.num_vis_tree_roots,
+                (int)tree.vis->size());
+  }
+}
+
+void Tfrag3::discard_tree_cache() {
+  for (auto tex : m_textures) {
+    glBindTexture(GL_TEXTURE_2D, tex);
+    glDeleteTextures(1, &tex);
+  }
+  m_textures.clear();
+
+  for (auto& tree : m_cached_trees) {
+    if (tree.kind != tfrag3::TFragmentTreeKind::INVALID) {
+      glDeleteBuffers(1, &tree.vertex_buffer);
+      glDeleteVertexArrays(1, &tree.vao);
+    }
+  }
+
+  if (m_has_index_buffer) {
+    glDeleteBuffers(1, &m_index_buffer);
+    m_has_index_buffer = false;
+  }
+
+  if (m_has_time_of_day_texture) {
+    glBindTexture(GL_TEXTURE_1D, m_time_of_day_texture);
+    glDeleteTextures(1, &m_time_of_day_texture);
+    m_has_time_of_day_texture = false;
+  }
+
+  // delete textures and stuff.
+  m_cached_trees.clear();
+}
+
+void Tfrag3::interp_time_of_day_slow(const float weights[8],
+                                     const std::vector<tfrag3::TimeOfDayColor>& in,
+                                     math::Vector<u8, 4>* out) {
+  // Timer interp_timer;
+  for (size_t color = 0; color < in.size(); color++) {
+    math::Vector4f result = math::Vector4f::zero();
+    for (int component = 0; component < 8; component++) {
+      result += in[color].rgba[component].cast<float>() * weights[component];
+    }
+    result[0] = std::min(result[0], 255.f);
+    result[1] = std::min(result[1], 255.f);
+    result[2] = std::min(result[2], 255.f);
+    result[3] = std::min(result[3], 128.f);  // note: different for alpha!
+    out[color] = result.cast<u8>();
+  }
+  // about 70 us, not bad.
+  // fmt::print("interp {} colors {:.2f} ms\n", in.size(), interp_timer.getMs());
+}
+
+namespace {
+
+float frac(float in) {
+  return in - (int)in;
+}
+
+void debug_vis_draw(int first_root,
+                    int tree,
+                    int num,
+                    int depth,
+                    const std::vector<tfrag3::VisNode>& nodes,
+                    std::vector<Tfrag3::DebugVertex>& verts_out) {
+  for (int ki = 0; ki < num; ki++) {
+    auto& node = nodes.at(ki + tree - first_root);
+    assert(node.child_id != 0xffff);
+    math::Vector4f rgba{frac(0.4 * depth), frac(0.7 * depth), frac(0.2 * depth), 0.06};
+    math::Vector3f center = node.bsphere.xyz();
+    float rad = node.bsphere.w();
+    math::Vector3f corners[8] = {center, center, center, center};
+    corners[0].x() += rad;
+    corners[1].x() += rad;
+    corners[2].x() -= rad;
+    corners[3].x() -= rad;
+
+    corners[0].y() += rad;
+    corners[1].y() -= rad;
+    corners[2].y() += rad;
+    corners[3].y() -= rad;
+
+    for (int i = 0; i < 4; i++) {
+      corners[i + 4] = corners[i];
+      corners[i].z() += rad;
+      corners[i + 4].z() -= rad;
+    }
+
+    if (true) {
+      for (int i : {0, 4}) {
+        verts_out.push_back({corners[0 + i], rgba});
+        verts_out.push_back({corners[1 + i], rgba});
+        verts_out.push_back({corners[2 + i], rgba});
+
+        verts_out.push_back({corners[1 + i], rgba});  // 0
+        verts_out.push_back({corners[3 + i], rgba});
+        verts_out.push_back({corners[2 + i], rgba});
+      }
+
+      for (int i : {2, 6, 7, 2, 3, 7, 0, 4, 5, 0, 5, 1, 0, 6, 4, 0, 6, 2, 1, 3, 7, 1, 5, 7}) {
+        verts_out.push_back({corners[i], rgba});
+      }
+
+      constexpr int border0[12] = {0, 4, 6, 2, 2, 6, 3, 7, 0, 1, 2, 3};
+      constexpr int border1[12] = {1, 5, 7, 3, 0, 4, 1, 5, 4, 5, 6, 7};
+      rgba.w() = 1.0;
+
+      for (int i = 0; i < 12; i++) {
+        auto p0 = corners[border0[i]];
+        auto p1 = corners[border1[i]];
+        auto diff = (p1 - p0).normalized();
+        math::Vector3f px = diff.z() == 0 ? math::Vector3f{1, 0, 1} : math::Vector3f{0, 1, 1};
+        auto off = diff.cross(px) * 2000;
+
+        verts_out.push_back({p0 + off, rgba});
+        verts_out.push_back({p0 - off, rgba});
+        verts_out.push_back({p1 - off, rgba});
+
+        verts_out.push_back({p0 + off, rgba});
+        verts_out.push_back({p1 + off, rgba});
+        verts_out.push_back({p1 - off, rgba});
+      }
+    }
+
+    if (node.flags) {
+      debug_vis_draw(first_root, node.child_id, node.num_kids, depth + 1, nodes, verts_out);
+    }
+  }
+}
+
+}  // namespace
+
+void Tfrag3::render_tree_cull_debug(const RenderSettings& settings,
+                                    SharedRenderState* render_state,
+                                    ScopedProfilerNode& prof) {
+  // generate debug verts:
+  m_debug_vert_data.clear();
+  auto& tree = m_cached_trees.at(settings.tree_idx);
+
+  debug_vis_draw(tree.vis_tree_root, tree.vis_tree_root, tree.num_vis_tree_roots, 1, *tree.vis,
+                 m_debug_vert_data);
+
+  render_state->shaders[ShaderId::TFRAG3_NO_TEX].activate();
+  glUniformMatrix4fv(
+      glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3_NO_TEX].id(), "camera"), 1,
+      GL_FALSE, settings.math_camera.data());
+  glUniform4f(
+      glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3_NO_TEX].id(), "hvdf_offset"),
+      settings.hvdf_offset[0], settings.hvdf_offset[1], settings.hvdf_offset[2],
+      settings.hvdf_offset[3]);
+  glUniform1f(
+      glGetUniformLocation(render_state->shaders[ShaderId::TFRAG3_NO_TEX].id(), "fog_constant"),
+      settings.fog_x);
+  // glDisable(GL_DEPTH_TEST);
+  glEnable(GL_DEPTH_TEST);
+  glDepthFunc(GL_GEQUAL);
+  glEnable(GL_BLEND);
+  glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);  // ?
+  glDepthMask(GL_FALSE);
+
+  glBindVertexArray(m_debug_vao);
+  glBindBuffer(GL_ARRAY_BUFFER, m_debug_verts);
+
+  int remaining = m_debug_vert_data.size();
+  int start = 0;
+
+  while (remaining > 0) {
+    int to_do = std::min(DEBUG_TRI_COUNT * 3, remaining);
+
+    glBufferSubData(GL_ARRAY_BUFFER, 0, to_do * sizeof(DebugVertex),
+                    m_debug_vert_data.data() + start);
+    glDrawArrays(GL_TRIANGLES, 0, to_do);
+    prof.add_draw_call();
+    prof.add_tri(to_do / 3);
+
+    remaining -= to_do;
+    start += to_do;
+  }
+}

game/graphics/opengl_renderer/tfrag/Tfrag3.h

@@ -0,0 +1,122 @@
+#pragma once
+
+#include "common/custom_data/Tfrag3Data.h"
+#include "common/math/Vector.h"
+#include "game/graphics/opengl_renderer/BucketRenderer.h"
+#include "game/graphics/pipelines/opengl.h"
+
+class Tfrag3 {
+ public:
+  struct RenderSettings {
+    math::Matrix4f math_camera;
+    math::Vector4f hvdf_offset;
+    float fog_x;
+    const u8* rgba_data;
+    int tree_idx;
+    float time_of_day_weights[8] = {0};
+    math::Vector4f planes[4];
+    bool do_culling = false;
+    bool debug_culling = false;
+    // todo culling planes
+    // todo occlusion culling string.
+  };
+
+  Tfrag3();
+  ~Tfrag3();
+
+  void debug_render_all_trees_nolores(const RenderSettings& settings,
+                                      SharedRenderState* render_state,
+                                      ScopedProfilerNode& prof);
+
+  void render_all_trees(const RenderSettings& settings,
+                        SharedRenderState* render_state,
+                        ScopedProfilerNode& prof);
+
+  void render_matching_trees(const std::vector<tfrag3::TFragmentTreeKind>& trees,
+                             const RenderSettings& settings,
+                             SharedRenderState* render_state,
+                             ScopedProfilerNode& prof);
+
+  void render_tree(const RenderSettings& settings,
+                   SharedRenderState* render_state,
+                   ScopedProfilerNode& prof,
+                   bool use_vis);
+
+  void setup_for_level(const std::string& level, SharedRenderState* render_state);
+  void discard_tree_cache();
+
+  void render_tree_cull_debug(const RenderSettings& settings,
+                              SharedRenderState* render_state,
+                              ScopedProfilerNode& prof);
+
+  void draw_debug_window();
+  struct DebugVertex {
+    math::Vector3f position;
+    math::Vector4f rgba;
+  };
+
+ private:
+  void first_draw_setup(const RenderSettings& settings, SharedRenderState* render_state);
+  enum class DoubleDrawKind { NONE, AFAIL_NO_DEPTH_WRITE };
+  struct DoubleDraw {
+    DoubleDrawKind kind = DoubleDrawKind::NONE;
+    float aref = 0.;
+  };
+
+  DoubleDraw setup_shader(const RenderSettings& settings,
+                          SharedRenderState* render_state,
+                          DrawMode mode);
+  void interp_time_of_day_slow(const float weights[8],
+                               const std::vector<tfrag3::TimeOfDayColor>& in,
+                               math::Vector<u8, 4>* out);
+
+  struct TreeCache {
+    tfrag3::TFragmentTreeKind kind;
+    GLuint vertex_buffer = -1;
+    GLuint vao;
+    u32 vert_count = 0;
+    const std::vector<tfrag3::Draw>* draws = nullptr;
+    const std::vector<tfrag3::TimeOfDayColor>* colors = nullptr;
+    const std::vector<tfrag3::VisNode>* vis = nullptr;
+
+    std::vector<u8> vis_temp;
+    std::vector<u32> culled_indices;
+    int num_vis_tree_roots = 0;
+    int vis_tree_root = 0;
+    int first_vis_leaf = 0;
+
+    void reset_stats() {
+      rendered_this_frame = false;
+      tris_this_frame = 0;
+      draws_this_frame = 0;
+    }
+    bool rendered_this_frame = false;
+    int tris_this_frame = 0;
+    int draws_this_frame = 0;
+    bool allowed = true;
+    bool forced = false;
+    bool cull_debug = false;
+  };
+
+  std::string m_level_name;
+
+  std::vector<GLuint> m_textures;
+  std::vector<TreeCache> m_cached_trees;
+  GLuint m_time_of_day_texture = -1;
+  bool m_has_time_of_day_texture = false;
+
+  std::vector<math::Vector<u8, 4>> m_color_result;
+
+  bool m_has_index_buffer = false;
+  GLuint m_index_buffer = -1;
+
+  GLuint m_debug_vao = -1;
+  GLuint m_debug_verts = -1;
+
+  // in theory could be up to 4096, I think, but we don't see that many...
+  // should be easy to increase (will require a shader change too for indexing)
+  static constexpr int TIME_OF_DAY_COLOR_COUNT = 2048;
+
+  static constexpr int DEBUG_TRI_COUNT = 4096;
+  std::vector<DebugVertex> m_debug_vert_data;
+};

game/graphics/opengl_renderer/tfrag/program6_cpu.cpp

@@ -471,40 +471,6 @@ void TFragment::exec_program_6_process_first(const Prog6Inputs& in,
       break;
     }
   }
-  //  while (m_next_block != TFragJumper::END_PROGRAM) {
-  ////    fmt::print("block {}\n", (int)m_next_block);
-  //    switch (m_next_block) {
-  //      case L128_PART0_X:
-  //        exec_jumper_L128<DEBUG>(in, vars);
-  //        break;
-  //      case L129_PART1_X:
-  //        exec_jumper_L129<DEBUG>(in, vars);
-  //        break;
-  //      case L0x6A1_PART0_Y:
-  //        exec_jumper_L6A1<DEBUG>(in, vars);
-  //        break;
-  //      case L130_PART1_Y:
-  //        exec_jumper_L130<DEBUG>(in, vars);
-  //        break;
-  //      case L0x6B0_PART0_Z:
-  //        exec_jumper_L6B0<DEBUG>(in, vars);
-  //        break;
-  //      case L131_PART1_Z:
-  //        exec_jumper_L131<DEBUG>(in, vars);
-  //        break;
-  //      case L0x6BF_PART0_W:
-  //        exec_jumper_L6BF<DEBUG>(in, vars);
-  //        break;
-  //      case L132_PART1_W:
-  //        exec_jumper_L132<DEBUG>(in, vars);
-  //        break;
-  //      case L122_KICK:
-  //        exec_jumper_L122<DEBUG>(in, vars, render_state, prof);
-  //        break;
-  //      default:
-  //        assert(false);
-  //    }
-  //  }
 }
 
 template <bool DEBUG>

game/mips2c/mips2c_table.h

@@ -4,6 +4,7 @@
 #include <vector>
 #include <string>
 #include <cstring>
+#include <random>
 
 #include "game/kernel/Ptr.h"
 #include "common/common_types.h"
@@ -30,6 +31,7 @@ extern LinkedFunctionTable gLinkedFunctionTable;
 struct Rng {
   Rng() { init(); }
   float R = 0.;
+  std::mt19937 extra_random_generator;
 
   u32 R_u32() {
     u32 result;
@@ -60,6 +62,11 @@ struct Rng {
     u32 y = 1 & (r32 >> 22);
     r32 <<= 1;
     r32 = r32 ^ x ^ y;
+    // we add a bit of extra randomness here. They XOR the random number generator with an
+    // uninitialized register, and in our port this corresponds to some random value on the stack.
+    // If we get unlucky this can end up being the current value in r32, and the random generator
+    // will get stuck outputting 1 for a while.
+    r32 ^= extra_random_generator();
     R = from23_bits(r32);
   }
 

goal_src/engine/gfx/background.gc

@@ -255,10 +255,11 @@
                   )
                 ;; set the level.
                 (set! (-> (scratchpad-object terrain-context) bsp lev-index) (-> s1-0 index))
-                )
+                
               (set! (-> *tfrag-work* min-dist z) 4095996000.0)
               ;; draw!
-              (draw-drawable-tree-tfrag s2-0)
+              (draw-drawable-tree-tfrag s2-0 s1-0)
+              )
               ;; remember closest.
               (set! (-> *level* level (-> (scratchpad-object terrain-context) bsp lev-index) closest-object 0)
                     (-> *tfrag-work* min-dist z)
@@ -281,9 +282,10 @@
                     )
                   )
                 (set! (-> (scratchpad-object terrain-context) bsp lev-index) (-> s1-1 index))
-                )
+                
               (set! (-> *tfrag-work* min-dist z) 4095996000.0)
-              (draw-drawable-tree-trans-tfrag s2-1)
+              (draw-drawable-tree-trans-tfrag s2-1 s1-1)
+              )
               (set! (-> *level* level (-> (scratchpad-object terrain-context) bsp lev-index) closest-object 3)
                     (-> *tfrag-work* min-dist z)
                     )
@@ -306,9 +308,10 @@
                     )
                   )
                 (set! (-> (scratchpad-object terrain-context) bsp lev-index) (-> s1-2 index))
-                )
+                
               (set! (-> *tfrag-work* min-dist z) 4095996000.0)
-              (draw-drawable-tree-dirt-tfrag s2-2)
+              (draw-drawable-tree-dirt-tfrag s2-2 s1-2)
+              )
               (set! (-> *level* level (-> (scratchpad-object terrain-context) bsp lev-index) closest-object 3)
                     (fmin (-> *level* level (-> (scratchpad-object terrain-context) bsp lev-index) closest-object 3)
                       (-> *tfrag-work* min-dist z)
@@ -333,9 +336,10 @@
                     )
                   )
                 (set! (-> (scratchpad-object terrain-context) bsp lev-index) (-> s1-3 index))
-                )
+                
               (set! (-> *tfrag-work* min-dist z) 4095996000.0)
-              (draw-drawable-tree-ice-tfrag s2-3)
+              (draw-drawable-tree-ice-tfrag s2-3 s1-3)
+              )
               (set! (-> *level* level (-> (scratchpad-object terrain-context) bsp lev-index) closest-object 3)
                 (fmin (-> *level* level (-> (scratchpad-object terrain-context) bsp lev-index) closest-object 3)
                   (-> *tfrag-work* min-dist z)
@@ -361,9 +365,10 @@
                     )
                   )
                 (set! (-> (scratchpad-object terrain-context) bsp lev-index) (-> s1-4 index))
-                )
+                
               ;;(format 0 "draw ~A~%" s2-4)
-              (draw-drawable-tree-tfrag s2-4)
+              (draw-drawable-tree-tfrag s2-4 s1-4)
+              )
               )
             )
           
@@ -382,9 +387,10 @@
                     )
                   )
                 (set! (-> (scratchpad-object terrain-context) bsp lev-index) (-> s1-5 index))
-                )
+                
               (set! (-> *tfrag-work* min-dist z) 4095996000.0)
-              (draw-drawable-tree-trans-tfrag s2-5)
+              (draw-drawable-tree-trans-tfrag s2-5 s1-5)
+              )
               (set! (-> *level* level (-> (scratchpad-object terrain-context) bsp lev-index) closest-object 3)
                 (fmin (-> *level* level (-> (scratchpad-object terrain-context) bsp lev-index) closest-object 3)
                   (-> *tfrag-work* min-dist z)

goal_src/engine/gfx/tfrag/tfrag-h.gc

@@ -52,7 +52,8 @@
    (color-count       uint8                :offset  57)
    (pad0              uint8                :offset  58)
    (pad1              uint8                :offset  59)
-   (generic           generic-tfragment                :offset-assert 60)
+   (generic           generic-tfragment    :offset-assert 60)
+   (generic-u32       uint32               :offset 60)
    )
   :method-count-assert 18
   :size-assert         #x40
@@ -256,7 +257,8 @@
 
 (define-extern *tfrag-work* tfrag-work)
 
-(define-extern draw-drawable-tree-tfrag (function drawable-tree-tfrag none))
-(define-extern draw-drawable-tree-trans-tfrag (function drawable-tree-trans-tfrag none))
-(define-extern draw-drawable-tree-dirt-tfrag (function drawable-tree-dirt-tfrag none))
-(define-extern draw-drawable-tree-ice-tfrag (function drawable-tree-ice-tfrag none))
+(define-extern draw-drawable-tree-tfrag (function drawable-tree-tfrag level none))
+(define-extern draw-drawable-tree-trans-tfrag (function drawable-tree-trans-tfrag level none))
+(define-extern draw-drawable-tree-dirt-tfrag (function drawable-tree-dirt-tfrag level none))
+(define-extern draw-drawable-tree-ice-tfrag (function drawable-tree-ice-tfrag level none))
+(define-extern tfrag-init-buffer (function dma-buffer gs-test int level none))

goal_src/engine/gfx/tfrag/tfrag-methods.gc

@@ -43,8 +43,8 @@
 ;; functions.
 
 
-(defun draw-drawable-tree-tfrag ((arg0 drawable-tree-tfrag))
-  "Draw the normal tfrag tree!"
+(defun draw-drawable-tree-tfrag ((arg0 drawable-tree-tfrag) (lev level))
+  "Draw the normal tfrag tree! Added the lev argument for time-of-day integer times"
   (local-vars (r0-0 none) (a0-20 int) (a0-22 int) (a0-38 int) (a0-40 int) (sv-16 (pointer uint8)))
   
   
@@ -107,6 +107,7 @@
               s1-0
               (new 'static 'gs-test :ate #x1 :atst (gs-atest greater-equal) :aref #x26 :zte #x1 :ztst (gs-ztest greater-equal))
               0
+              lev
               )
             
             ;; do the draw!
@@ -197,7 +198,7 @@
   )
 
 
-(defun draw-drawable-tree-trans-tfrag ((arg0 drawable-tree-trans-tfrag))
+(defun draw-drawable-tree-trans-tfrag ((arg0 drawable-tree-trans-tfrag) (lev level))
   (local-vars
    (r0-0 none)
    (a0-18 int)
@@ -248,6 +249,7 @@
         :ztst (gs-ztest greater-equal)
         )
        1
+       lev
        )
       (reset! (-> *perf-stats* data 5))
       (draw-inline-array-tfrag sv-16 s5-1 s4-1 s2-0)
@@ -261,7 +263,7 @@
         (set! (-> (the-as dma-packet v1-34) vif1) (new 'static 'vif-tag))
         (set! (-> s2-0 base) (&+ (the-as pointer v1-34) 16))
         )
-       #|
+       
        (dma-bucket-insert-tag
         (-> *display* frames (-> *display* on-screen) frame bucket-group)
         (the-as
@@ -273,7 +275,7 @@
          )
         s3-0
         (the-as (pointer dma-tag) a3-3)
-        )|#
+        )
        )
       )
      #| TODO
@@ -364,7 +366,7 @@
   )
 
 
-(defun draw-drawable-tree-dirt-tfrag ((arg0 drawable-tree-dirt-tfrag))
+(defun draw-drawable-tree-dirt-tfrag ((arg0 drawable-tree-dirt-tfrag) (lev level))
   (local-vars
    (r0-0 none)
    (a0-18 int)
@@ -421,7 +423,7 @@
              )
         (set! (-> *tfrag-work* wait-to-spr) (the-as uint 0))
         (set! (-> *tfrag-work* wait-from-spr) (the-as uint 0))
-        (tfrag-init-buffer s2-0 (new 'static 'gs-test :ate #x1 :afail #x1 :zte #x1 :ztst (gs-ztest greater-equal)) 1)
+        (tfrag-init-buffer s2-0 (new 'static 'gs-test :ate #x1 :afail #x1 :zte #x1 :ztst (gs-ztest greater-equal)) 1 lev)
         (reset! (-> *perf-stats* data 5))
         
         (draw-inline-array-tfrag sv-16 s5-1 s4-1 s2-0)
@@ -539,7 +541,7 @@
   (none)
   )
 
-(defun draw-drawable-tree-ice-tfrag ((arg0 drawable-tree-ice-tfrag))
+(defun draw-drawable-tree-ice-tfrag ((arg0 drawable-tree-ice-tfrag) (lev level))
   (local-vars
    (r0-0 none)
    (a0-18 int)
@@ -591,6 +593,7 @@
         :ztst (gs-ztest greater-equal)
         )
        1
+       lev
        )
       (reset! (-> *perf-stats* data 5))
       (draw-inline-array-tfrag sv-16 s5-1 s4-1 s2-0)

goal_src/engine/gfx/tfrag/tfrag.gc

@@ -319,6 +319,30 @@
   (none)
   )
 
+(defun add-pc-tfrag3-data ((dma-buf dma-buffer) (lev level))
+  "Add PC-port specific tfrag data"
+  (let ((packet (the-as dma-packet (-> dma-buf base))))
+    (set! (-> packet dma) (new 'static 'dma-tag :id (dma-tag-id cnt) :qwc 9))
+    (set! (-> packet vif0) (new 'static 'vif-tag))
+    (set! (-> packet vif1) (new 'static 'vif-tag :cmd (vif-cmd pc-port)))
+    (set! (-> dma-buf base) (the pointer (&+ packet 16)))
+    )
+  
+  ;; first 4 quadwords are planes, then itimes
+  (let ((data-ptr (the-as (pointer uint128) (-> dma-buf base))))
+    (set! (-> data-ptr 0) (-> *math-camera* plane 0 quad))
+    (set! (-> data-ptr 1) (-> *math-camera* plane 1 quad))
+    (set! (-> data-ptr 2) (-> *math-camera* plane 2 quad))
+    (set! (-> data-ptr 3) (-> *math-camera* plane 3 quad))
+    (set! (-> data-ptr 4) (-> lev mood itimes 0 quad))
+    (set! (-> data-ptr 5) (-> lev mood itimes 1 quad))
+    (set! (-> data-ptr 6) (-> lev mood itimes 2 quad))
+    (set! (-> data-ptr 7) (-> lev mood itimes 3 quad))
+    (charp<-string (the (pointer uint8) (&-> data-ptr 8)) (symbol->string (-> lev nickname)))
+    )
+  (&+! (-> dma-buf base) (* 16 9))
+  )
+
 ;;;;;;;;;;;;;;;;;;;;;
 ;; TFRAG Stats
 ;;;;;;;;;;;;;;;;;;;;;
@@ -365,7 +389,7 @@
 ;; buffer
 ;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defun tfrag-init-buffer ((arg0 dma-buffer) (arg1 gs-test) (arg2 int))
+(defun tfrag-init-buffer ((arg0 dma-buffer) (arg1 gs-test) (arg2 int) (lev level))
   (dma-buffer-add-vu-function arg0 tfrag-vu1-block 1)
   (let* ((v1-0 arg0)
          (a0-2 (the-as object (-> v1-0 base)))
@@ -411,6 +435,9 @@
   (add-tfrag-mtx-0 arg0)
   (add-tfrag-mtx-1 arg0)
   (add-tfrag-data arg0 arg2)
+  (#when PC_PORT
+    (add-pc-tfrag3-data arg0 lev)
+    )
   (let ((v1-3 (the-as object (-> arg0 base))))
     (set! (-> (the-as dma-packet v1-3) dma) (new 'static 'dma-tag :id (dma-tag-id cnt)))
     (set! (-> (the-as dma-packet v1-3) vif0) (new 'static 'vif-tag :cmd (vif-cmd base)))

goalc/compiler/Compiler.h

@@ -45,6 +45,9 @@ class Compiler {
   void run_full_compiler_on_string_no_save(const std::string& src);
   void shutdown_target();
   void enable_throw_on_redefines() { m_throw_on_define_extern_redefinition = true; }
+  void add_ignored_define_extern_symbol(const std::string& name) {
+    m_allow_inconsistent_definition_symbols.insert(name);
+  }
   Debugger& get_debugger() { return m_debugger; }
   listener::Listener& listener() { return m_listener; }
   void poke_target() { m_listener.send_poke(); }
@@ -75,6 +78,7 @@ class Compiler {
   std::unordered_map<goos::HeapObject*, LambdaVal*> m_inlineable_functions;
   CompilerSettings m_settings;
   bool m_throw_on_define_extern_redefinition = false;
+  std::unordered_set<std::string> m_allow_inconsistent_definition_symbols;
   SymbolInfoMap m_symbol_info;
   std::unique_ptr<ReplWrapper> m_repl;
   MakeSystem m_make;

goalc/compiler/compilation/Define.cpp

@@ -80,15 +80,19 @@ Val* Compiler::compile_define_extern(const goos::Object& form, const goos::Objec
 
   auto existing_type = m_symbol_types.find(symbol_string(sym));
   if (existing_type != m_symbol_types.end() && existing_type->second != new_type) {
-    if (m_throw_on_define_extern_redefinition) {
-      throw_compiler_error(form,
-                           "define-extern would redefine the type of symbol {} from {} to {}.",
-                           symbol_string(sym), existing_type->second.print(), new_type.print());
-    } else {
-      print_compiler_warning(
-          "[Warning] define-extern has redefined the type of symbol {}\npreviously: {}\nnow: {}\n",
-          symbol_string(sym).c_str(), existing_type->second.print().c_str(),
-          new_type.print().c_str());
+    if (m_allow_inconsistent_definition_symbols.find(symbol_string(sym)) ==
+        m_allow_inconsistent_definition_symbols.end()) {
+      if (m_throw_on_define_extern_redefinition) {
+        throw_compiler_error(form,
+                             "define-extern would redefine the type of symbol {} from {} to {}.",
+                             symbol_string(sym), existing_type->second.print(), new_type.print());
+      } else {
+        print_compiler_warning(
+            "[Warning] define-extern has redefined the type of symbol {}\npreviously: {}\nnow: "
+            "{}\n",
+            symbol_string(sym).c_str(), existing_type->second.print().c_str(),
+            new_type.print().c_str());
+      }
     }
   }
 

test/decompiler/FormRegressionTest.cpp

@@ -38,8 +38,7 @@ void FormRegressionTest::TestData::add_string_at_label(const std::string& label_
 
   // add string type tag:
   LinkedWord type_tag(0);
-  type_tag.kind = LinkedWord::Kind::TYPE_PTR;
-  type_tag.symbol_name = "string";
+  type_tag.set_to_symbol(decompiler::LinkedWord::TYPE_PTR, "string");
   file.words_by_seg.at(1).push_back(type_tag);
   int string_start = 4 * int(file.words_by_seg.at(1).size());
 

test/decompiler/reference/engine/gfx/tfrag/tfrag-h_REF.gc

@@ -75,6 +75,7 @@
    (pad0              uint8                         :offset         58)
    (pad1              uint8                         :offset         59)
    (generic           generic-tfragment             :offset-assert  60)
+   (generic-u32       uint32                        :offset         60)
    )
   :method-count-assert 18
   :size-assert         #x40

test/decompiler/test_VuDisasm.cpp

@@ -16,7 +16,7 @@ std::vector<u32> get_test_data(const std::string& name) {
 
   std::vector<u32> data;
   for (auto& w : parsed.words) {
-    EXPECT_EQ(w.kind, LinkedWord::Kind::PLAIN_DATA);
+    EXPECT_EQ(w.kind(), LinkedWord::Kind::PLAIN_DATA);
     data.push_back(w.data);
   }
   return data;

test/goalc/test_with_game.cpp

@@ -923,9 +923,18 @@ TEST_F(WithGameTests, Mips2C_CallGoal) {
                                           {"1 2 3 4 5 6 7 8\n12\n"});
 }
 
+void add_expected_type_mismatches(Compiler& c) {
+  c.add_ignored_define_extern_symbol("draw-drawable-tree-tfrag");
+  c.add_ignored_define_extern_symbol("draw-drawable-tree-trans-tfrag");
+  c.add_ignored_define_extern_symbol("draw-drawable-tree-dirt-tfrag");
+  c.add_ignored_define_extern_symbol("draw-drawable-tree-ice-tfrag");
+  c.add_ignored_define_extern_symbol("tfrag-init-buffer");
+}
+
 TEST(TypeConsistency, MANUAL_TEST_TypeConsistencyWithBuildFirst) {
   Compiler compiler;
   compiler.enable_throw_on_redefines();
+  add_expected_type_mismatches(compiler);
   compiler.run_test_no_load("test/goalc/source_templates/with_game/test-build-game.gc");
   compiler.run_test_no_load("decompiler/config/all-types.gc");
 }
@@ -933,6 +942,7 @@ TEST(TypeConsistency, MANUAL_TEST_TypeConsistencyWithBuildFirst) {
 TEST(TypeConsistency, TypeConsistency) {
   Compiler compiler;
   compiler.enable_throw_on_redefines();
+  add_expected_type_mismatches(compiler);
   compiler.run_test_no_load("decompiler/config/all-types.gc");
   compiler.run_test_no_load("test/goalc/source_templates/with_game/test-build-game.gc");
 }

tools/level_tools/CMakeLists.txt

@@ -1,7 +1,5 @@
 
 
-add_library(level_tools goal_data_reader.cpp BspHeader.cpp)
-target_link_libraries(level_tools fmt common decomp)
 
 add_executable(level_dump level_dump/main.cpp)
-target_link_libraries(level_dump fmt common decomp level_tools)
+target_link_libraries(level_dump fmt common decomp)

tools/level_tools/level_dump/main.cpp

@@ -4,9 +4,8 @@
 #include "decompiler/ObjectFile/LinkedObjectFileCreation.h"
 #include "common/util/DgoReader.h"
 
-#include "tools/level_tools/goal_data_reader.h"
-#include "tools/level_tools/BspHeader.h"
-
+#include "decompiler/util/goal_data_reader.h"
+#include "decompiler/level_extractor/BspHeader.h"
 
 #include "common/util/assert.h"
 
@@ -39,13 +38,13 @@ bool is_valid_bsp(const decompiler::LinkedObjectFile& file) {
   }
 
   auto& first_word = file.words_by_seg.at(0).at(0);
-  if (first_word.kind != decompiler::LinkedWord::TYPE_PTR) {
+  if (first_word.kind() != decompiler::LinkedWord::TYPE_PTR) {
     fmt::print("Expected the first word to be a type pointer, but it wasn't.\n");
     return false;
   }
 
-  if (first_word.symbol_name != "bsp-header") {
-    fmt::print("Expected to get a bsp-header, but got {} instead.\n", first_word.symbol_name);
+  if (first_word.symbol_name() != "bsp-header") {
+    fmt::print("Expected to get a bsp-header, but got {} instead.\n", first_word.symbol_name());
     return false;
   }