move dgo decompression to common utils and support it in the dgo unpacker (#312)

common/CMakeLists.txt

@@ -22,7 +22,7 @@ add_library(common
         util/Timer.cpp
         )
 
-target_link_libraries(common fmt)
+target_link_libraries(common fmt lzokay)
 
 if(WIN32)
     target_link_libraries(common wsock32 ws2_32)

common/util/DgoReader.cpp

@@ -22,7 +22,14 @@ DgoReader::DgoReader(std::string file_name, const std::vector<u8>& data)
 
     DgoDataEntry entry;
     entry.internal_name = obj_header.name;
+
     entry.unique_name = get_object_file_name(entry.internal_name, reader.here(), obj_header.size);
+    if (all_unique_names.find(entry.unique_name) != all_unique_names.end()) {
+      printf("Warning: there are multiple files named %s\n", entry.unique_name.c_str());
+      entry.unique_name += '-';
+      entry.unique_name += std::to_string(obj_header.size);
+    }
+
     all_unique_names.insert(entry.unique_name);
     entry.data.resize(obj_header.size);
 

common/util/FileUtil.cpp

@@ -10,10 +10,12 @@
 #include <fstream>
 #include <sstream>
 #include <cassert>
+#include <cstdlib>
+#include "common/util/BinaryReader.h"
 #include "BinaryWriter.h"
 #include "common/common_types.h"
 #include "third-party/svpng.h"
-#include <stdlib.h>
+#include "third-party/lzokay/lzokay.hpp"
 
 #ifdef _WIN32
 #include <Windows.h>
@@ -327,4 +329,63 @@ void assert_file_exists(const char* path, const char* error_message) {
   }
 }
 
+/*!
+ * Check if the given DGO header (or entire file) is compressed.
+ */
+bool dgo_header_is_compressed(const std::vector<u8>& data) {
+  const char compressed_header[] = "oZlB";
+  bool is_compressed = true;
+  for (int i = 0; i < 4; i++) {
+    if (compressed_header[i] != data.at(i)) {
+      is_compressed = false;
+    }
+  }
+  return is_compressed;
+}
+
+/*!
+ * Decompress a DGO. Resulting data will start at the DGO header.
+ */
+std::vector<u8> decompress_dgo(const std::vector<u8>& data_in) {
+  constexpr int MAX_CHUNK_SIZE = 0x8000;
+  BinaryReader compressed_reader(data_in);
+  // seek past oZlB
+  compressed_reader.ffwd(4);
+  std::size_t decompressed_size = compressed_reader.read<uint32_t>();
+  std::vector<uint8_t> decompressed_data;
+  decompressed_data.resize(decompressed_size);
+  size_t output_offset = 0;
+  while (true) {
+    // seek past alignment bytes and read the next chunk size
+    uint32_t chunk_size = 0;
+    while (!chunk_size) {
+      chunk_size = compressed_reader.read<uint32_t>();
+    }
+
+    if (chunk_size < MAX_CHUNK_SIZE) {
+      std::size_t bytes_written = 0;
+      lzokay::EResult ok = lzokay::decompress(
+          compressed_reader.here(), chunk_size, decompressed_data.data() + output_offset,
+          decompressed_data.size() - output_offset, bytes_written);
+      assert(ok == lzokay::EResult::Success);
+      compressed_reader.ffwd(chunk_size);
+      output_offset += bytes_written;
+    } else {
+      // nope - sometimes chunk_size is bigger than MAX, but we should still use max.
+      //        assert(chunk_size == MAX_CHUNK_SIZE);
+      memcpy(decompressed_data.data() + output_offset, compressed_reader.here(), MAX_CHUNK_SIZE);
+      compressed_reader.ffwd(MAX_CHUNK_SIZE);
+      output_offset += MAX_CHUNK_SIZE;
+    }
+
+    if (output_offset >= decompressed_size)
+      break;
+    while (compressed_reader.get_seek() % 4) {
+      compressed_reader.ffwd(1);
+    }
+  }
+
+  return decompressed_data;
+}
+
 }  // namespace file_util

common/util/FileUtil.h

@@ -8,6 +8,7 @@
 #include <string>
 #include <vector>
 #include <filesystem>
+#include "common/common_types.h"
 
 namespace file_util {
 std::filesystem::path get_user_home_dir();
@@ -28,4 +29,6 @@ uint32_t crc32(const std::vector<uint8_t>& data);
 void MakeISOName(char* dst, const char* src);
 void ISONameFromAnimationName(char* dst, const char* src);
 void assert_file_exists(const char* path, const char* error_message);
+bool dgo_header_is_compressed(const std::vector<u8>& data);
+std::vector<u8> decompress_dgo(const std::vector<u8>& data_in);
 }  // namespace file_util

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -195,52 +195,8 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename) {
   auto dgo_data = file_util::read_binary_file(filename);
   stats.total_dgo_bytes += dgo_data.size();
 
-  const char jak2_header[] = "oZlB";
-  bool is_jak2 = true;
-  for (int i = 0; i < 4; i++) {
-    if (jak2_header[i] != dgo_data[i]) {
-      is_jak2 = false;
-    }
-  }
-
-  if (is_jak2) {
-    BinaryReader compressed_reader(dgo_data);
-    // seek past oZlB
-    compressed_reader.ffwd(4);
-    std::size_t decompressed_size = compressed_reader.read<uint32_t>();
-    std::vector<uint8_t> decompressed_data;
-    decompressed_data.resize(decompressed_size);
-    size_t output_offset = 0;
-    while (true) {
-      // seek past alignment bytes and read the next chunk size
-      uint32_t chunk_size = 0;
-      while (!chunk_size) {
-        chunk_size = compressed_reader.read<uint32_t>();
-      }
-
-      if (chunk_size < MAX_CHUNK_SIZE) {
-        std::size_t bytes_written = 0;
-        lzokay::EResult ok = lzokay::decompress(
-            compressed_reader.here(), chunk_size, decompressed_data.data() + output_offset,
-            decompressed_data.size() - output_offset, bytes_written);
-        assert(ok == lzokay::EResult::Success);
-        compressed_reader.ffwd(chunk_size);
-        output_offset += bytes_written;
-      } else {
-        // nope - sometimes chunk_size is bigger than MAX, but we should still use max.
-        //        assert(chunk_size == MAX_CHUNK_SIZE);
-        memcpy(decompressed_data.data() + output_offset, compressed_reader.here(), MAX_CHUNK_SIZE);
-        compressed_reader.ffwd(MAX_CHUNK_SIZE);
-        output_offset += MAX_CHUNK_SIZE;
-      }
-
-      if (output_offset >= decompressed_size)
-        break;
-      while (compressed_reader.get_seek() % 4) {
-        compressed_reader.ffwd(1);
-      }
-    }
-    dgo_data = decompressed_data;
+  if (file_util::dgo_header_is_compressed(dgo_data)) {
+    dgo_data = file_util::decompress_dgo(dgo_data);
   }
 
   BinaryReader reader(dgo_data);

tools/dgo_unpacker.cpp

@@ -20,6 +20,13 @@ int main(int argc, char** argv) {
     printf("Unpacking %s\n", base.c_str());
     // read the file
     auto data = file_util::read_binary_file(file_name);
+    if (file_util::dgo_header_is_compressed(data)) {
+      printf(" Detected compressed dgo, decompressing...\n");
+      auto original_size = data.size();
+      data = file_util::decompress_dgo(data);
+      printf(" Decompressed from %d to %d bytes (%.2f%% compression)\n", int(original_size),
+             int(data.size()), 100.f * original_size / data.size());
+    }
     // read as a DGO
     auto dgo = DgoReader(base, data);
     // write dgo description