[tools] start work on FBX file support

common/CMakeLists.txt

@@ -7,6 +7,8 @@ add_library(common
         dma/dma.cpp
         dma/dma_copy.cpp
         dma/gs.cpp
+        fbx/FBX.cpp
+        fbx/FbxBuilder.cpp
         global_profiler/GlobalProfiler.cpp
         goos/Interpreter.cpp
         goos/Object.cpp
@@ -49,8 +51,8 @@ add_library(common
 
 target_link_libraries(common fmt lzokay replxx libzstd_static)
 
-if(WIN32)
+if (WIN32)
     target_link_libraries(common wsock32 ws2_32 windowsapp)
-else()
+else ()
     target_link_libraries(common stdc++fs)
-endif()
+endif ()

common/fbx/FBX.cpp

@@ -0,0 +1,159 @@
+#include "FBX.h"
+
+namespace fbx {
+
+Property::~Property() {
+  free(m_array);
+}
+
+Property::Property(const Property& other) {
+  m_type = other.m_type;
+  m_array_length = other.m_array_length;
+  if (other.m_array) {
+    size_t array_size_bytes = m_array_length * array_element_size(m_type);
+    m_array = malloc(array_size_bytes);
+    memcpy(m_array, other.m_array, array_size_bytes);
+  } else {
+    m_value = other.m_value;
+  }
+}
+
+Property& Property::operator=(const Property& other) {
+  if (this != &other) {
+    m_type = other.m_type;
+    m_array_length = other.m_array_length;
+
+    if (m_array) {
+      free(m_array);
+      m_array = other.m_array;
+    }
+
+    if (other.m_array) {
+      size_t array_size_bytes = m_array_length * array_element_size(m_type);
+      m_array = malloc(array_size_bytes);
+      memcpy(m_array, other.m_array, array_size_bytes);
+    } else {
+      m_value = other.m_value;
+    }
+  }
+
+  return *this;
+}
+
+Property::Property(const void* data, size_t count, PropertyType type) {
+  m_type = type;
+  m_array_length = count;
+  size_t s = count * array_element_size(type);
+  m_array = malloc(s);
+  memcpy(m_array, data, s);
+}
+
+void Property::serialize(Serializer& ser) {
+  ASSERT(ser.is_saving());
+  ser.save(m_type);
+  switch (m_type) {
+    case PropertyType::INT16:
+      ser.save(m_value.int16);
+      break;
+    case PropertyType::BOOL:
+      ser.save(m_value.boolean ? 1 : 0);
+      break;
+    case PropertyType::INT32:
+      ser.save(m_value.int32);
+      break;
+    case PropertyType::FLOAT:
+      ser.save(m_value.f);
+      break;
+    case PropertyType::DOUBLE:
+      ser.save(m_value.d);
+      break;
+    case PropertyType::INT64:
+      ser.save(m_value.int64);
+      break;
+    case PropertyType::STRING:
+    case PropertyType::RAW_BINARY:
+      ser.save<u32>(m_array_length);  // maybe u64?
+      ser.from_raw_data(m_array, m_array_length);
+      break;
+    case PropertyType::FLOAT_ARRAY:
+    case PropertyType::INT64_ARRAY:
+    case PropertyType::INT32_ARRAY:
+      ser.save<u32>(m_array_length);
+      ser.save<u32>(0);
+      ser.save<u32>(m_array_length * array_element_size(m_type));
+      ser.from_raw_data(m_array, m_array_length * array_element_size(m_type));
+      break;
+  }
+}
+
+void serialize_object_record(Serializer& ser,
+                             std::string& name,
+                             std::vector<Property>& properties,
+                             std::vector<Node>& children) {
+  // ENDOFFSET
+  size_t start_offset = ser.current_offset();
+  ser.save<u64>(UINT64_MAX);
+
+  // NUMPROPERTIES
+  ser.save<u64>(properties.size());
+
+  // PROPERTYLISTLEN
+  size_t property_list_len_slot = ser.current_offset();
+  ser.save<u64>(UINT64_MAX);
+
+  // NAMELEN
+  ASSERT(name.size() <= UINT8_MAX);
+  ser.save<u8>(name.size());
+
+  // NAME
+  ser.from_raw_data(name.data(), name.size());
+  size_t start_of_property_list_offset = ser.current_offset();
+
+  for (auto& prop : properties) {
+    prop.serialize(ser);
+  }
+  ser.save_at_offset<u64>(ser.current_offset() - start_of_property_list_offset,
+                          property_list_len_slot);
+
+  for (auto& child : children) {
+    child.serialize(ser);
+  }
+
+  ser.save<u64>(0);
+  ser.save<u64>(0);
+  ser.save<u64>(0);
+  ser.save<u8>(0);
+
+  size_t end_offset = ser.current_offset();
+  ser.from_ptr_at_offset(&end_offset, start_offset);
+}
+
+void Node::serialize(Serializer& ser) {
+  serialize_object_record(ser, node_type, properties, children);
+}
+
+void FbxRoot::serialize(Serializer& ser) {
+  ASSERT(ser.is_saving());
+
+  const char magic_header[] = "Kaydara FBX Binary  ";
+  ser.from_raw_data((void*)magic_header, 21);
+  uint8_t magic_bytes[2] = {0x1a, 0x00};
+  ser.from_raw_data(magic_bytes, 2);
+
+  uint32_t version = 7500;  // a comment from "Bill" on some blog says that 7.5 uses 64-bit offsets
+  ser.from_ptr(&version);
+
+  //  std::vector<Property> empty_properties;
+  //  std::string empty_name;
+  //  serialize_object_record(ser, empty_name, empty_properties, top_level_nodes);
+
+  for (auto& c : top_level_nodes) {
+    c.serialize(ser);
+  }
+  ser.save<u64>(0);
+  ser.save<u64>(0);
+  ser.save<u64>(0);
+  ser.save<u8>(0);
+}
+
+}  // namespace fbx

common/fbx/FBX.h

@@ -0,0 +1,137 @@
+#pragma once
+
+#include <vector>
+#include <cstdlib>
+#include <cstring>
+#include "common/common_types.h"
+#include "common/util/Assert.h"
+#include "common/util/Serializer.h"
+
+// Description of the FBX format:
+// The "Nodes" are the organization structure and are arranged into a tree
+// Nodes can store "Properties" which are the actual data
+// There is a special "FbxRoot" that is the top level node for the file.
+// Note that this tree structure is completely separate from the tree structure described in the
+// connections section.
+
+namespace fbx {
+
+// null should be 25 bytes, use u64's for the first three fields of node record.
+
+enum class PropertyType : char {
+  // single values
+  INT16 = 'Y',
+  BOOL = 'C',
+  INT32 = 'I',
+  FLOAT = 'F',
+  DOUBLE = 'D',
+  INT64 = 'L',
+
+  // array of values
+  FLOAT_ARRAY = 'f',
+  // DOUBLE_ARRAY = 'd',
+  INT64_ARRAY = 'l',
+  INT32_ARRAY = 'i',
+  // BOOL_ARRAY = 'b',
+
+  // specials
+  STRING = 'S',
+  RAW_BINARY = 'R',
+};
+
+inline size_t array_element_size(PropertyType type) {
+  switch (type) {
+    case PropertyType::FLOAT_ARRAY:
+      return sizeof(float);
+    case PropertyType::INT64_ARRAY:
+      return sizeof(s64);
+    case PropertyType::INT32_ARRAY:
+      return sizeof(s32);
+    case PropertyType::STRING:
+    case PropertyType::RAW_BINARY:
+      return sizeof(u8);
+    default:
+      ASSERT(false);
+  }
+}
+
+class Property {
+ public:
+  ~Property();
+  Property(const Property& other);
+  Property& operator=(const Property& other);
+
+  explicit Property(s16 val) {
+    m_type = PropertyType::INT16;
+    m_value.int16 = val;
+  }
+
+  //  explicit Property(bool val) {
+  //    m_type = PropertyType::BOOL;
+  //    m_value.boolean = val;
+  //  }
+
+  explicit Property(s32 val) {
+    m_type = PropertyType::INT32;
+    m_value.int32 = val;
+  }
+
+  explicit Property(float val) {
+    m_type = PropertyType::FLOAT;
+    m_value.f = val;
+  }
+
+  explicit Property(double val) {
+    m_type = PropertyType::DOUBLE;
+    m_value.d = val;
+  }
+
+  explicit Property(s64 val) {
+    m_type = PropertyType::INT64;
+    m_value.int64 = val;
+  }
+
+  Property(const void* data, size_t count, PropertyType type);
+
+  Property(const s32* val, size_t sz) : Property(val, sz, PropertyType::INT32_ARRAY) {}
+  Property(const s64* val, size_t sz) : Property(val, sz, PropertyType::INT64_ARRAY) {}
+  Property(const float* val, size_t sz) : Property(val, sz, PropertyType::FLOAT_ARRAY) {}
+  explicit Property(const std::vector<float>& vals) : Property(vals.data(), vals.size()) {}
+  explicit Property(const std::vector<s32>& vals) : Property(vals.data(), vals.size()) {}
+  explicit Property(const std::string& str)
+      : Property(str.data(), str.size(), PropertyType::STRING) {}
+
+  void serialize(Serializer& ser);
+
+ private:
+  union PropertyValue {
+    s16 int16;
+    s32 int32;
+    s64 int64;
+    bool boolean;
+    double d;
+    float f;
+  };
+  PropertyType m_type;
+  PropertyValue m_value;
+  void* m_array = nullptr;
+  size_t m_array_length = 0;
+};
+
+struct Node {
+  Node() = default;
+  Node(const std::string& str) : node_type(str) {}
+  std::string node_type;
+  std::vector<Property> properties;
+  std::vector<Node> children;
+  Node& add_node(const std::string& name) { return children.emplace_back(name); }
+  void serialize(Serializer& ser);
+};
+
+struct FbxRoot {
+  std::vector<Node> top_level_nodes;
+  void serialize(Serializer& ser);
+  Node& add_node(const std::string& name) { return top_level_nodes.emplace_back(name); }
+};
+
+}  // namespace fbx

common/fbx/FbxBuilder.cpp

@@ -0,0 +1,83 @@
+#include "FbxBuilder.h"
+#include "common/util/FileUtil.h"
+
+namespace fbx {
+
+std::string make_name_class_string(const std::string& name, const std::string& klass) {
+  std::string result = name;
+  result.push_back(0);
+  result.push_back(1);
+  result.append(klass);
+  return result;
+}
+
+FbxBuilder::FbxBuilder() {}
+
+u64 FbxBuilder::add_tri_mesh_geom(const std::string& name,
+                                  std::vector<math::Vector3f>& vtx_positions,
+                                  std::vector<uint32_t>& vtx_indices) {
+  u64 geom_id = m_next_id++;
+  auto& node = m_geometry_nodes.emplace_back("Geometry");
+  node.properties.emplace_back((s64)geom_id);
+  node.properties.emplace_back(make_name_class_string(name, "Geometry"));
+  node.properties.emplace_back("Mesh");
+  {
+    auto& verts = node.add_node("Vertices");
+    verts.properties.emplace_back(&vtx_positions[0].x(), vtx_positions.size() * 3);
+  }
+  std::vector<int32_t> indices_fbx;
+  for (uint32_t i = 0; i < vtx_indices.size(); i += 3) {
+    indices_fbx.push_back(vtx_indices[i]);
+    indices_fbx.push_back(vtx_indices[i + 1]);
+    indices_fbx.push_back(vtx_indices[i + 2] ^ 0xffffffff);
+  }
+  node.add_node("PolygonVertexIndex").properties.emplace_back(indices_fbx);
+
+  return geom_id;
+}
+
+void FbxBuilder::add_instance_of_geom(u64 geom_id) {
+  u64 model_id = m_next_id++;
+  auto& model = m_model_nodes.emplace_back("Model");
+  model.properties.emplace_back((s64)model_id);
+  model.properties.emplace_back(make_name_class_string("unknown", "Model"));
+  model.properties.emplace_back("unknown");
+
+  {
+    auto& c = m_connections_nodes.emplace_back("Connect");
+    c.properties.emplace_back("OO");
+    c.properties.emplace_back((s64)geom_id);
+    c.properties.emplace_back((s64)model_id);
+  }
+
+  {
+    auto& c = m_connections_nodes.emplace_back("Connect");
+    c.properties.emplace_back("OO");
+    c.properties.emplace_back((s64)model_id);
+    c.properties.emplace_back((s64)0);
+  }
+}
+
+void FbxBuilder::write(const std::string& dest) {
+  FbxRoot root;
+  root.add_node("Creator").properties.emplace_back("OpenGOAL");
+  auto& global_settings = root.add_node("GlobalSettings").children.emplace_back("Properties70");
+  auto& objects = root.add_node("Objects");
+  for (auto& k : m_geometry_nodes) {
+    objects.children.push_back(k);
+  }
+  for (auto& k : m_model_nodes) {
+    objects.children.push_back(k);
+  }
+
+  auto& connections = root.add_node("Connections");
+  for (auto& k : m_connections_nodes) {
+    connections.children.push_back(k);
+  }
+
+  Serializer ser;
+  root.serialize(ser);
+  auto result = ser.get_save_result();
+  file_util::write_binary_file(dest, result.first, result.second);
+}
+}  // namespace fbx

common/fbx/FbxBuilder.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include "common/fbx/FBX.h"
+#include "common/math/Vector.h"
+
+namespace fbx {
+
+class FbxBuilder {
+ public:
+  FbxBuilder();
+
+  u64 add_tri_mesh_geom(const std::string& name,
+                        std::vector<math::Vector3f>& vtx_positions,
+                        std::vector<uint32_t>& vtx_indices);
+  void add_instance_of_geom(u64 geom_id);
+
+  void write(const std::string& dest);
+
+ private:
+  u64 m_next_id = 1;  // root = 0
+  std::vector<Node> m_geometry_nodes;
+  std::vector<Node> m_model_nodes;
+  std::vector<Node> m_connections_nodes;
+};
+
+}  // namespace fbx

common/fbx/test_fbx.cpp

@@ -0,0 +1 @@
+

common/util/Serializer.h

@@ -84,6 +84,11 @@ class Serializer {
     read_or_write(ptr, sizeof(T));
   }
 
+  template <typename T>
+  void from_ptr_at_offset(T* ptr, size_t offset) {
+    read_or_write_at_offset(ptr, sizeof(T), offset);
+  }
+
   /*!
    * Save or load size bytes from ptr.
    */
@@ -109,6 +114,12 @@ class Serializer {
     read_or_write(const_cast<T*>(&thing), sizeof(T));
   }
 
+  template <typename T>
+  void save_at_offset(const T& thing, size_t offset) {
+    ASSERT(m_writing);
+    read_or_write_at_offset(const_cast<T*>(&thing), sizeof(T), offset);
+  }
+
   /*!
    * Save or load a string.
    */
@@ -194,6 +205,8 @@ class Serializer {
    */
   size_t data_size() const { return m_size; }
 
+  size_t current_offset() const { return m_offset; }
+
  private:
   /*!
    * Main function to read and write the buffer.
@@ -214,6 +227,15 @@ class Serializer {
     m_offset += size;
   }
 
+  void read_or_write_at_offset(void* data, size_t size, size_t offset) {
+    ASSERT(offset + size <= m_size);
+    if (m_writing) {
+      memcpy(m_data + offset, data, size);
+    } else {
+      memcpy(data, m_data + offset, size);
+    }
+  }
+
   u8* m_data = nullptr;
   size_t m_size = 0;
   size_t m_offset = 0;

decompiler/level_extractor/extract_merc.cpp

@@ -4,6 +4,7 @@
 #include "decompiler/level_extractor/extract_common.h"
 #include "common/util/FileUtil.h"
 #include "common/util/colors.h"
+#include "common/fbx/FbxBuilder.h"
 
 namespace decompiler {
 
@@ -649,6 +650,36 @@ std::string debug_dump_to_ply(const std::vector<MercDraw>& draws,
   return result;
 }
 
+void debug_dump_to_fbx(const std::vector<MercDraw>& draws,
+                       const std::vector<MercUnpackedVtx>& vertices,
+                       const std::string& path) {
+  std::vector<math::Vector3f> verts;
+  std::vector<uint32_t> faces;
+
+  for (auto& draw : draws) {
+    // add verts...
+    for (size_t ii = 2; ii < draw.indices.size(); ii++) {
+      u32 v0 = draw.indices[ii - 2];
+      u32 v1 = draw.indices[ii - 1];
+      u32 v2 = draw.indices[ii - 0];
+      if (v0 != UINT32_MAX && v1 != UINT32_MAX && v2 != UINT32_MAX) {
+        faces.emplace_back(v0);
+        faces.emplace_back(v1);
+        faces.emplace_back(v2);
+      }
+    }
+  }
+
+
+  for (auto& vtx : vertices) {
+    verts.push_back(vtx.pos);
+  }
+
+  fbx::FbxBuilder builder;
+  builder.add_instance_of_geom(builder.add_tri_mesh_geom("test", verts, faces));
+  builder.write(path);
+}
+
 ConvertedMercEffect convert_merc_effect(const MercEffect& input_effect,
                                         const MercCtrlHeader& ctrl_header,
                                         const TextureDB& tdb,
@@ -808,10 +839,13 @@ ConvertedMercEffect convert_merc_effect(const MercEffect& input_effect,
   }
 
   if (dump) {
-    file_util::write_text_file(
-        file_util::get_file_path(
-            {"debug_out/merc", fmt::format("{}_{}.ply", debug_name, effect_idx)}),
-        debug_dump_to_ply(result.draws, result.vertices));
+    debug_dump_to_fbx(result.draws, result.vertices,
+                      file_util::get_file_path(
+                          {"debug_out/merc", fmt::format("{}_{}.fbx", debug_name, effect_idx)}));
+        file_util::write_text_file(
+            file_util::get_file_path(
+                {"debug_out/merc", fmt::format("{}_{}.ply", debug_name, effect_idx)}),
+            debug_dump_to_ply(result.draws, result.vertices));
   }
 
   return result;