[graphics] tfrag3 renderer (#978)

* begin work

* work

* working objs

* exporting

* it works

* before some time of day fixes

* add time of day interp and also fix zbuffer

* some small blending fixes

* improve randomess

* clean up extraction and missing blend mode

* culling, time of day, more level fixes

* more cleanup

* cleanup memory usage

* windows fix

decompiler/ObjectFile/LinkedObjectFile.cpp

@@ -150,7 +150,7 @@ bool LinkedObjectFile::pointer_link_word(int source_segment,
   assert((source_offset % 4) == 0);
 
   auto& word = words_by_seg.at(source_segment).at(source_offset / 4);
-  assert(word.kind == LinkedWord::PLAIN_DATA);
+  assert(word.kind() == LinkedWord::PLAIN_DATA);
 
   if (dest_offset / 4 > (int)words_by_seg.at(dest_segment).size()) {
     //    printf("HACK bad link ignored!\n");
@@ -158,8 +158,7 @@ bool LinkedObjectFile::pointer_link_word(int source_segment,
   }
   assert(dest_offset / 4 <= (int)words_by_seg.at(dest_segment).size());
 
-  word.kind = LinkedWord::PTR;
-  word.label_id = get_label_id_for(dest_segment, dest_offset);
+  word.set_to_pointer(LinkedWord::PTR, get_label_id_for(dest_segment, dest_offset));
   return true;
 }
 
@@ -173,11 +172,10 @@ void LinkedObjectFile::symbol_link_word(int source_segment,
   assert((source_offset % 4) == 0);
   auto& word = words_by_seg.at(source_segment).at(source_offset / 4);
   //  assert(word.kind == LinkedWord::PLAIN_DATA);
-  if (word.kind != LinkedWord::PLAIN_DATA) {
+  if (word.kind() != LinkedWord::PLAIN_DATA) {
     printf("bad symbol link word\n");
   }
-  word.kind = kind;
-  word.symbol_name = name;
+  word.set_to_symbol(kind, name);
 }
 
 /*!
@@ -187,9 +185,8 @@ void LinkedObjectFile::symbol_link_word(int source_segment,
 void LinkedObjectFile::symbol_link_offset(int source_segment, int source_offset, const char* name) {
   assert((source_offset % 4) == 0);
   auto& word = words_by_seg.at(source_segment).at(source_offset / 4);
-  assert(word.kind == LinkedWord::PLAIN_DATA);
-  word.kind = LinkedWord::SYM_OFFSET;
-  word.symbol_name = name;
+  assert(word.kind() == LinkedWord::PLAIN_DATA);
+  word.set_to_symbol(LinkedWord::SYM_OFFSET, name);
 }
 
 /*!
@@ -207,14 +204,11 @@ void LinkedObjectFile::pointer_link_split_word(int source_segment,
   auto& lo_word = words_by_seg.at(source_segment).at(source_lo_offset / 4);
 
   //  assert(dest_offset / 4 <= (int)words_by_seg.at(dest_segment).size());
-  assert(hi_word.kind == LinkedWord::PLAIN_DATA);
-  assert(lo_word.kind == LinkedWord::PLAIN_DATA);
+  assert(hi_word.kind() == LinkedWord::PLAIN_DATA);
+  assert(lo_word.kind() == LinkedWord::PLAIN_DATA);
 
-  hi_word.kind = LinkedWord::HI_PTR;
-  hi_word.label_id = get_label_id_for(dest_segment, dest_offset);
-
-  lo_word.kind = LinkedWord::LO_PTR;
-  lo_word.label_id = hi_word.label_id;
+  hi_word.set_to_pointer(LinkedWord::HI_PTR, get_label_id_for(dest_segment, dest_offset));
+  lo_word.set_to_pointer(LinkedWord::LO_PTR, hi_word.label_id());
 }
 
 /*!
@@ -284,32 +278,32 @@ std::string LinkedObjectFile::print_words() {
 void LinkedObjectFile::append_word_to_string(std::string& dest, const LinkedWord& word) const {
   char buff[128];
 
-  switch (word.kind) {
+  switch (word.kind()) {
     case LinkedWord::PLAIN_DATA:
       sprintf(buff, "    .word 0x%x\n", word.data);
       break;
     case LinkedWord::PTR:
-      sprintf(buff, "    .word %s\n", labels.at(word.label_id).name.c_str());
+      sprintf(buff, "    .word %s\n", labels.at(word.label_id()).name.c_str());
       break;
     case LinkedWord::SYM_PTR:
-      sprintf(buff, "    .symbol %s\n", word.symbol_name.c_str());
+      sprintf(buff, "    .symbol %s\n", word.symbol_name().c_str());
       break;
     case LinkedWord::TYPE_PTR:
-      sprintf(buff, "    .type %s\n", word.symbol_name.c_str());
+      sprintf(buff, "    .type %s\n", word.symbol_name().c_str());
       break;
     case LinkedWord::EMPTY_PTR:
       sprintf(buff, "    .empty-list\n");  // ?
       break;
     case LinkedWord::HI_PTR:
       sprintf(buff, "    .ptr-hi 0x%x %s\n", word.data >> 16,
-              labels.at(word.label_id).name.c_str());
+              labels.at(word.label_id()).name.c_str());
       break;
     case LinkedWord::LO_PTR:
       sprintf(buff, "    .ptr-lo 0x%x %s\n", word.data >> 16,
-              labels.at(word.label_id).name.c_str());
+              labels.at(word.label_id()).name.c_str());
       break;
     case LinkedWord::SYM_OFFSET:
-      sprintf(buff, "    .sym-off 0x%x %s\n", word.data >> 16, word.symbol_name.c_str());
+      sprintf(buff, "    .sym-off 0x%x %s\n", word.data >> 16, word.symbol_name().c_str());
       break;
     default:
       throw std::runtime_error("nyi");
@@ -326,8 +320,8 @@ void LinkedObjectFile::find_code() {
     // single segment object files should never have any code.
     auto& seg = words_by_seg.front();
     for (auto& word : seg) {
-      if (!word.symbol_name.empty()) {
-        assert(word.symbol_name != "function");
+      if (word.kind() == LinkedWord::TYPE_PTR) {
+        assert(word.symbol_name() != "function");
       }
     }
     offset_of_data_zone_by_seg.at(0) = 0;
@@ -346,7 +340,7 @@ void LinkedObjectFile::find_code() {
       size_t function_loc = -1;
       for (size_t j = words_by_seg.at(i).size(); j-- > 0;) {
         auto& word = words_by_seg.at(i).at(j);
-        if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "function") {
+        if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "function") {
           function_loc = j;
           found_function = true;
           break;
@@ -361,7 +355,7 @@ void LinkedObjectFile::find_code() {
 
         for (size_t j = function_loc; j < words_by_seg.at(i).size(); j++) {
           auto& word = words_by_seg.at(i).at(j);
-          if (word.kind == LinkedWord::PLAIN_DATA && word.data == jr_ra) {
+          if (word.kind() == LinkedWord::PLAIN_DATA && word.data == jr_ra) {
             found_jr_ra = true;
             jr_ra_loc = j;
           }
@@ -385,7 +379,7 @@ void LinkedObjectFile::find_code() {
       // verify there are no functions after the data section starts
       for (size_t j = offset_of_data_zone_by_seg.at(i); j < words_by_seg.at(i).size(); j++) {
         auto& word = words_by_seg.at(i).at(j);
-        if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "function") {
+        if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "function") {
           assert(false);
         }
       }
@@ -421,7 +415,7 @@ void LinkedObjectFile::find_functions() {
         bool found_function_tag_loc = false;
         for (; function_tag_loc-- > 0;) {
           auto& word = words_by_seg.at(seg).at(function_tag_loc);
-          if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "function") {
+          if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "function") {
             found_function_tag_loc = true;
             break;
           }
@@ -729,7 +723,7 @@ std::string LinkedObjectFile::print_disassembly(bool write_hex) {
       auto& word = words_by_seg[seg][i];
       append_word_to_string(result, word);
 
-      if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "string") {
+      if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "string") {
         result += "; " + get_goal_string(seg, i) + "\n";
       }
     }
@@ -751,7 +745,7 @@ std::string LinkedObjectFile::get_goal_string(int seg, int word_idx, bool with_q
     return "invalid string!\n";
   }
   const LinkedWord& size_word = words_by_seg[seg].at(word_idx + 1);
-  if (size_word.kind != LinkedWord::PLAIN_DATA) {
+  if (size_word.kind() != LinkedWord::PLAIN_DATA) {
     // sometimes an array of string pointer triggers this!
     return "invalid string!\n";
   }
@@ -762,7 +756,7 @@ std::string LinkedObjectFile::get_goal_string(int seg, int word_idx, bool with_q
     int word_offset = word_idx + 2 + (i / 4);
     int byte_offset = i % 4;
     auto& word = words_by_seg[seg].at(word_offset);
-    if (word.kind != LinkedWord::PLAIN_DATA) {
+    if (word.kind() != LinkedWord::PLAIN_DATA) {
       return "invalid string! (check me!)\n";
     }
     char cword[4];
@@ -827,7 +821,7 @@ std::string LinkedObjectFile::print_scripts() {
 bool LinkedObjectFile::is_empty_list(int seg, int byte_idx) {
   assert((byte_idx % 4) == 0);
   auto& word = words_by_seg.at(seg).at(byte_idx / 4);
-  return word.kind == LinkedWord::EMPTY_PTR;
+  return word.kind() == LinkedWord::EMPTY_PTR;
 }
 
 /*!
@@ -866,9 +860,10 @@ goos::Object LinkedObjectFile::to_form_script(int seg, int word_idx, std::vector
         assert((cdr_addr % 4) == 0);
         auto& cdr_word = words_by_seg.at(seg).at(cdr_addr / 4);
         // check for proper list
-        if (cdr_word.kind == LinkedWord::PTR && (labels.at(cdr_word.label_id).offset & 7) == 2) {
+        if (cdr_word.kind() == LinkedWord::PTR &&
+            (labels.at(cdr_word.label_id()).offset & 7) == 2) {
           // yes, proper list. add another pair and link it in to the list.
-          goal_print_obj = labels.at(cdr_word.label_id).offset;
+          goal_print_obj = labels.at(cdr_word.label_id()).offset;
           fill.as_pair()->cdr = goos::PairObject::make_new(goos::Object::make_empty_list(),
                                                            goos::Object::make_empty_list());
           fill = fill.as_pair()->cdr;
@@ -900,7 +895,7 @@ bool LinkedObjectFile::is_string(int seg, int byte_idx) const {
     return false;
   }
   auto& type_word = words_by_seg.at(seg).at(type_tag_ptr / 4);
-  return type_word.kind == LinkedWord::TYPE_PTR && type_word.symbol_name == "string";
+  return type_word.kind() == LinkedWord::TYPE_PTR && type_word.symbol_name() == "string";
 }
 
 /*!
@@ -915,15 +910,15 @@ goos::Object LinkedObjectFile::to_form_script_object(int seg,
     case 0:
     case 4: {
       auto& word = words_by_seg.at(seg).at(byte_idx / 4);
-      if (word.kind == LinkedWord::SYM_PTR) {
+      if (word.kind() == LinkedWord::SYM_PTR) {
         // .symbol xxxx
-        result = pretty_print::to_symbol(word.symbol_name);
-      } else if (word.kind == LinkedWord::PLAIN_DATA) {
+        result = pretty_print::to_symbol(word.symbol_name());
+      } else if (word.kind() == LinkedWord::PLAIN_DATA) {
         // .word xxxxx
         result = pretty_print::to_symbol(std::to_string(word.data));
-      } else if (word.kind == LinkedWord::PTR) {
+      } else if (word.kind() == LinkedWord::PTR) {
         // might be a sub-list, or some other random pointer
-        auto offset = labels.at(word.label_id).offset;
+        auto offset = labels.at(word.label_id()).offset;
         if ((offset & 7) == 2) {
           // list!
           result = to_form_script(seg, offset / 4, seen);
@@ -932,10 +927,10 @@ goos::Object LinkedObjectFile::to_form_script_object(int seg,
             result = pretty_print::to_symbol(get_goal_string(seg, offset / 4 - 1));
           } else {
             // some random pointer, just print the label.
-            result = pretty_print::to_symbol(labels.at(word.label_id).name);
+            result = pretty_print::to_symbol(labels.at(word.label_id()).name);
           }
         }
-      } else if (word.kind == LinkedWord::EMPTY_PTR) {
+      } else if (word.kind() == LinkedWord::EMPTY_PTR) {
         result = goos::Object::make_empty_list();
       } else {
         std::string debug;
@@ -958,7 +953,7 @@ goos::Object LinkedObjectFile::to_form_script_object(int seg,
 u32 LinkedObjectFile::read_data_word(const DecompilerLabel& label) {
   assert(0 == (label.offset % 4));
   auto& word = words_by_seg.at(label.target_segment).at(label.offset / 4);
-  assert(word.kind == LinkedWord::Kind::PLAIN_DATA);
+  assert(word.kind() == LinkedWord::Kind::PLAIN_DATA);
   return word.data;
 }
 

decompiler/ObjectFile/LinkedWord.h

@@ -7,16 +7,14 @@
 
 #include <cstdint>
 #include <string>
+#include <cstring>
 #include "common/util/assert.h"
-
 #include "common/common_types.h"
 
 namespace decompiler {
 class LinkedWord {
  public:
-  explicit LinkedWord(uint32_t _data) : data(_data) {}
-
-  enum Kind {
+  enum Kind : u8 {
     PLAIN_DATA,  // just plain data
     PTR,         // pointer to a location
     HI_PTR,      // lower 16-bits of this data are the upper 16 bits of a pointer
@@ -25,15 +23,98 @@ class LinkedWord {
     EMPTY_PTR,   // this is a pointer to the empty list
     SYM_OFFSET,  // this is an offset of a symbol in the symbol table
     TYPE_PTR     // this is a pointer to a type
-  } kind = PLAIN_DATA;
+  };
 
-  uint32_t data = 0;
+ private:
+  uintptr_t m_data_ptr = 0;  // 8 bytes
+ public:
+  u32 data;
 
-  int label_id = -1;
-  std::string symbol_name;
+ private:
+  Kind m_kind = PLAIN_DATA;
+
+ public:
+  explicit LinkedWord(uint32_t _data) : data(_data) {}
+
+  bool holds_string() const {
+    return m_kind == SYM_PTR || m_kind == SYM_OFFSET || m_kind == TYPE_PTR;
+  }
+
+  LinkedWord(const LinkedWord& other) {
+    if (&other != this) {
+      data = other.data;
+      m_kind = other.m_kind;
+      if (holds_string()) {
+        const char* other_str = (const char*)(other.m_data_ptr);
+        char* str = new char[strlen(other_str) + 1];
+        strcpy(str, other_str);
+        m_data_ptr = (uintptr_t)str;
+      } else {
+        m_data_ptr = other.m_data_ptr;
+      }
+    }
+  }
+
+  LinkedWord& operator=(const LinkedWord& other) {
+    if (&other != this) {
+      data = other.data;
+      if (holds_string()) {
+        delete[]((char*)m_data_ptr);
+      }
+      m_kind = other.m_kind;
+
+      if (holds_string()) {
+        const char* other_str = (const char*)(other.m_data_ptr);
+        char* str = new char[strlen(other_str) + 1];
+        strcpy(str, other_str);
+        m_data_ptr = (uintptr_t)str;
+      } else {
+        m_data_ptr = other.m_data_ptr;
+      }
+    }
+    return *this;
+  }
+
+  ~LinkedWord() {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+  }
+
+  void set_to_empty_ptr() {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+    m_kind = EMPTY_PTR;
+  }
+
+  void set_to_symbol(Kind kind, const std::string& name) {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+    m_kind = kind;
+    char* str = new char[name.size() + 1];
+    strcpy(str, name.c_str());
+    m_data_ptr = (uintptr_t)str;
+  }
+
+  void set_to_pointer(Kind kind, u32 label_id) {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+    m_data_ptr = label_id;
+    m_kind = kind;
+  }
+
+  void set_to_plain_data() {
+    if (holds_string()) {
+      delete[]((char*)m_data_ptr);
+    }
+    m_kind = PLAIN_DATA;
+  }
 
   u8 get_byte(int idx) const {
-    assert(kind == PLAIN_DATA);
+    assert(kind() == PLAIN_DATA);
     switch (idx) {
       case 0:
         return data & 0xff;
@@ -48,5 +129,25 @@ class LinkedWord {
         return 0;
     }
   }
+
+  // kind, label_id, symbol_name
+  Kind kind() const { return m_kind; }
+
+  u32 label_id() const {
+    assert(m_kind == PTR || m_kind == LO_PTR || m_kind == HI_PTR);
+    return m_data_ptr;
+  }
+
+  std::string symbol_name() const {
+    assert(holds_string());
+    return (const char*)(m_data_ptr);
+  }
+
+ private:
+  // if plain data, this is empty.
+  // otherwise:
+  //  u8 (kind)
+  //  u32 label id | actual string.
 };
+
 }  // namespace decompiler

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -428,7 +428,7 @@ void ObjectFileDB::process_link_data(const Config& config) {
   });
 
   lg::info("Processed Link Data");
-  lg::info(" Total {} ms\n", process_link_timer.getMs());
+  lg::info(" Total {:.2f} ms\n", process_link_timer.getMs());
   // printf("\n");
 }
 
@@ -443,7 +443,7 @@ void ObjectFileDB::process_labels() {
 
   lg::info("Processed Labels:");
   lg::info(" Total {} labels", total);
-  lg::info(" Total {} ms\n", process_label_timer.getMs());
+  lg::info(" Total {:.2f} ms\n", process_label_timer.getMs());
 }
 
 /*!
@@ -579,7 +579,7 @@ void ObjectFileDB::find_and_write_scripts(const std::string& output_dir) {
   lg::info(" Total {:.3f} ms\n", timer.getMs());
 }
 
-std::string ObjectFileDB::process_tpages() {
+std::string ObjectFileDB::process_tpages(TextureDB& tex_db) {
   lg::info("- Finding textures in tpages...");
   std::string tpage_string = "tpage-";
   int total = 0, success = 0;
@@ -590,7 +590,7 @@ std::string ObjectFileDB::process_tpages() {
   std::string result;
   for_each_obj([&](ObjectFileData& data) {
     if (data.name_in_dgo.substr(0, tpage_string.length()) == tpage_string) {
-      auto statistics = process_tpage(data);
+      auto statistics = process_tpage(data, tex_db);
       total += statistics.total_textures;
       success += statistics.successful_textures;
       total_px += statistics.num_px;
@@ -602,13 +602,13 @@ std::string ObjectFileDB::process_tpages() {
 
   assert(tpage_dir_count <= 1);
 
+  lg::info("Processed {} / {} textures ({} px) {:.2f}% in {:.2f} ms", success, total, total_px,
+           100.f * float(success) / float(total), timer.getMs());
+
   if (tpage_dir_count == 0) {
     lg::warn("Did not find tpage-dir.");
     return {};
   }
-
-  lg::info("Processed {} / {} textures ({} px) {:.2f}% in {:.2f} ms", success, total, total_px,
-           100.f * float(success) / float(total), timer.getMs());
   return result;
 }
 

decompiler/ObjectFile/ObjectFileDB.h

@@ -14,6 +14,7 @@
 #include "LinkedObjectFile.h"
 #include "decompiler/util/DecompilerTypeSystem.h"
 #include "common/common_types.h"
+#include "decompiler/data/TextureDB.h"
 #include "decompiler/analysis/symbol_def_map.h"
 
 namespace decompiler {
@@ -96,7 +97,7 @@ class ObjectFileDB {
   std::string ir2_final_out(ObjectFileData& data,
                             const std::unordered_set<std::string>& skip_functions = {});
 
-  std::string process_tpages();
+  std::string process_tpages(TextureDB& tex_db);
   std::string process_game_count_file();
   std::string process_game_text_files();
 

decompiler/ObjectFile/ObjectFileDB_IR2.cpp

@@ -706,7 +706,7 @@ std::string ObjectFileDB::ir2_to_file(ObjectFileData& data, const Config& config
       auto& word = data.linked_data.words_by_seg[seg][i];
       data.linked_data.append_word_to_string(result, word);
 
-      if (word.kind == LinkedWord::TYPE_PTR && word.symbol_name == "string") {
+      if (word.kind() == LinkedWord::TYPE_PTR && word.symbol_name() == "string") {
         result += "; " + data.linked_data.get_goal_string(seg, i) + "\n";
       }
     }