third-party/licensing: Switch miniLZO usage to LZOkay (#232)

* add rough jak 2 config so the decompiler can atleast unpack dgos

* third-party: commit lzokay

* Switch to lzokay for DGO decompression

* third-party: Remove minilzo library

* third-party: Add lzokay as a SHARED lib

decompiler/CMakeLists.txt

@@ -51,7 +51,7 @@ add_library(
 )
 
 target_link_libraries(decomp
-        minilzo
+        lzokay
         common
         fmt
         )
@@ -63,7 +63,7 @@ add_executable(decompiler
 target_link_libraries(decompiler
         decomp
         common
-        minilzo
+        lzokay
         fmt)
 
 install(TARGETS decompiler)

decompiler/ObjectFile/ObjectFileDB.cpp

@@ -18,7 +18,7 @@
 #include "decompiler/data/game_count.h"
 #include "LinkedObjectFileCreation.h"
 #include "decompiler/config.h"
-#include "third-party/minilzo/minilzo.h"
+#include "third-party/lzokay/lzokay.hpp"
 #include "common/util/BinaryReader.h"
 #include "common/util/Timer.h"
 #include "common/util/FileUtil.h"
@@ -204,13 +204,10 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename) {
   }
 
   if (is_jak2) {
-    if (lzo_init() != LZO_E_OK) {
-      assert(false);
-    }
     BinaryReader compressed_reader(dgo_data);
     // seek past oZlB
     compressed_reader.ffwd(4);
-    auto decompressed_size = compressed_reader.read<uint32_t>();
+    std::size_t decompressed_size = compressed_reader.read<uint32_t>();
     std::vector<uint8_t> decompressed_data;
     decompressed_data.resize(decompressed_size);
     size_t output_offset = 0;
@@ -222,11 +219,11 @@ void ObjectFileDB::get_objs_from_dgo(const std::string& filename) {
       }
 
       if (chunk_size < MAX_CHUNK_SIZE) {
-        lzo_uint bytes_written;
-        auto lzo_rv =
-            lzo1x_decompress(compressed_reader.here(), chunk_size,
-                             decompressed_data.data() + output_offset, &bytes_written, nullptr);
-        assert(lzo_rv == LZO_E_OK);
+        std::size_t bytes_written = 0;
+        lzokay::EResult ok = lzokay::decompress(
+            compressed_reader.here(), chunk_size, decompressed_data.data() + output_offset,
+            decompressed_data.size() - output_offset, bytes_written);
+        assert(ok == lzokay::EResult::Success);
         compressed_reader.ffwd(chunk_size);
         output_offset += bytes_written;
       } else {

decompiler/ObjectFile/ObjectFileDB.h

@@ -91,7 +91,7 @@ class ObjectFileDB {
                             const std::string& obj_name,
                             TypeSpec* result);
 
- private:
+ public:
   void load_map_file(const std::string& map_data);
   void get_objs_from_dgo(const std::string& filename);
   void add_obj_from_dgo(const std::string& obj_name,

decompiler/config/jak2/anonymous_function_types.jsonc

@@ -0,0 +1,10 @@
+{
+	"gkernel":[
+		[16, "(function process symbol)"],
+		[22, "(function process symbol)"],
+		[25, "(function process symbol)"],
+		[28, "(function process symbol)"],
+		[30, "(function process symbol)"],
+		[32, "(function process symbol)"]
+	]
+}

decompiler/config/jak2/type_hints.jsonc

@@ -0,0 +1,112 @@
+{
+	"(method 2 array)":[
+		[23, ["gp", "(array int32)"]],
+		[43, ["gp", "(array uint32)"]],
+		[63, ["gp", "(array int64)"]],
+		[83, ["gp", "(array uint64)"]],
+		[102, ["gp", "(array int8)"]],
+		[121, ["gp", "(array uint8)"]],
+		[141, ["gp", "(array int16)"]],
+		[161, ["gp", "(array uint16)"]],
+		[185, ["gp", "(array uint128)"]],
+		[203, ["gp", "(array int32)"]],
+		[222, ["gp", "(array float)"]],
+		[231, ["gp", "(array float)"]],
+		[248, ["gp", "(array basic)"]],
+		[257, ["gp", "(array basic)"]]
+	],
+
+	"(method 3 array)":[
+		[44, ["gp", "(array int32)"]],
+		[62, ["gp", "(array uint32)"]],
+		[80, ["gp", "(array int64)"]],
+		[98, ["gp", "(array uint64)"]],
+		[115, ["gp", "(array int8)"]],
+		[132, ["gp", "(array int8)"]], // bug in game
+		[150, ["gp", "(array int16)"]],
+		[168, ["gp", "(array uint16)"]],
+		[190, ["gp", "(array uint128)"]],
+		[203, ["gp", "(array int32)"]],
+		[225, ["gp", "(array float)"]],
+		[242, ["gp", "(array basic)"]]
+	],
+
+	"(method 2 handle)":[
+		[10, ["a3", "process"]],
+		[11, ["v1", "int"]],
+		[15, ["gp", "int"]]
+	],
+
+	"(method 3 handle)":[
+		[10, ["gp", "int"]]
+	],
+
+	"(method 0 cpu-thread)":[
+		[13, ["v0", "cpu-thread"]]
+	],
+
+	"(method 0 dead-pool-heap)":[
+		[60, ["v0", "int"]], // a lie, actually the 115 is an align16 constant propagated on addr of heap start.
+		[61, ["a0", "pointer"], ["v0", "dead-pool-heap"]]
+	],
+
+	"(method 21 dead-pool-heap)":[
+		[5, ["v1", "pointer"]],
+		[13, ["a0", "pointer"]],
+		[25, ["v1", "pointer"]]
+	],
+
+	"(method 5 dead-pool-heap)":[
+		[3, ["v1", "int"], ["a0", "int"]]
+	],
+
+	"remove-exit":[
+		[0, ["s6", "process"]]
+	],
+
+	"(method 0 process)":[
+		[12, ["a0", "int"]],
+		[13, ["v0", "process"]]
+	],
+
+	"inspect-process-heap":[
+		[4, ["s5", "basic"]],
+		[17, ["s5", "int"]]
+	],
+
+	"return-from-thread-dead":[
+		[0, ["s6", "process"]]
+	],
+
+	"(method 14 dead-pool)":[
+		[23, ["v1", "process"]], // bad visit order with #f?
+		[28, ["s4", "(pointer process-tree)"]] // bug in real game, see gkernel.gc
+	],
+
+	"throw":[
+		[20, ["s4", "protect-frame"]] // type case
+	],
+
+	"(method 0 protect-frame)":[
+		[0, ["a0", "int"]],
+		[1, ["v0", "protect-frame"]]
+	],
+
+	"(method 9 process)":[
+		[43, ["s5", "process"]]
+	],
+
+	"(method 10 process)":[
+		[24, ["s4", "protect-frame"]]
+	],
+
+	"enter-state":[
+		[67, ["s0", "protect-frame"]]
+	],
+
+	"name=":[
+		[24, ["a1", "symbol"]],
+		[39, ["a0", "symbol"]]
+	]
+
+}

decompiler/config/jak2_ntsc_v1.jsonc

@@ -1,43 +1,50 @@
 {
     "game_version":2,
 
-    "dgo_names":["ART.CGO", "ATE.DGO", "ATO.DGO", "CAB.DGO", "CAP.DGO", "CAS.DGO", "CASCITY.DGO", "CASEXT.DGO",
-                 "CFA.DGO", "CFB.DGO", "CGA.DGO", "CGB.DGO", "CGC.DGO", "CIA.DGO", "CIB.DGO", "CMA.DGO",
-                 "CMB.DGO", "COA.DGO", "COB.DGO", "COMMON.CGO", "CPA.DGO", "CPO.DGO", "CTA.DGO", "CTB.DGO",
-                 "CTC.DGO", "CTYASHA.DGO", "CTYKORA.DGO", "CWI.DGO", "D3A.DGO", "D3B.DGO", "DEMO.DGO", "DG1.DGO",
-                 "DMI.DGO", "DRB.DGO", "DRI.DGO", "DRILLMTN.DGO", "ENGINE.CGO", "FDA.DGO", "FDB.DGO", "FEA.DGO",
-                 "FEB.DGO", "FOB.DGO", "FOR.DGO", "FORDUMPC.DGO", "FORDUMPD.DGO", "FRA.DGO", "FRB.DGO", "GAME.CGO",
-                 "GARAGE.DGO", "GGA.DGO", "HALFPIPE.DGO", "HIDEOUT.DGO", "HIPHOG.DGO", "INTROCST.DGO", "KERNEL.CGO", "KIOSK.DGO",
-                 "LASHGRD.DGO", "LASHTHRN.DGO", "LBBUSH.DGO", "LBOMBBOT.DGO", "LBRNERMK.DGO", "LCGUARD.DGO", "LCITYLOW.DGO", "LDJAKBRN.DGO",
-                 "LERBRNGD.DGO", "LERLCHAL.DGO", "LERLTESS.DGO", "LERROL.DGO", "LGARCSTA.DGO", "LGUARD.DGO", "LHELLDOG.DGO", "LHIPOUT.DGO",
-                 "LINTCSTB.DGO", "LJAKDAX.DGO", "LJKDXASH.DGO", "LKEIRIFT.DGO", "LKIDDOGE.DGO", "LMEETBRT.DGO", "LOUTCSTB.DGO", "LPACKAGE.DGO",
-                 "LPORTRUN.DGO", "LPOWER.DGO", "LPROTECT.DGO", "LPRSNCST.DGO", "LPRTRACE.DGO", "LRACEBB.DGO", "LRACEBF.DGO", "LRACECB.DGO",
-                 "LRACECF.DGO", "LRACEDB.DGO", "LRACEDF.DGO", "LRACELIT.DGO", "LSACK.DGO", "LSAMERGD.DGO", "LSHUTTLE.DGO", "LSMYSBRT.DGO",
-                 "LTENTOB.DGO", "LTENTOUT.DGO", "LTESS.DGO", "LTHRNOUT.DGO", "LTRNKRKD.DGO", "LTRNTESS.DGO", "LTRNYSAM.DGO", "LWHACK.DGO",
-                 "LWIDEA.DGO", "LWIDEB.DGO", "LWIDEC.DGO", "LWIDESTA.DGO", "LYSAMSAM.DGO", "LYSKDCD.DGO", "MCN.DGO", "MTN.DGO",
-                 "MTX.DGO", "NEB.DGO", "NES.DGO", "NESTT.DGO", "ONINTENT.DGO", "ORACLE.DGO", "OUTROCST.DGO", "PAC.DGO",
-                 "PAE.DGO", "PALBOSS.DGO", "PALOUT.DGO", "PAR.DGO", "PAS.DGO", "PORTWALL.DGO", "PRI.DGO", "RUI.DGO",
-                 "SAG.DGO", "SEB.DGO", "SEW.DGO", "SKA.DGO", "STA.DGO", "STADBLMP.DGO", "STB.DGO", "STC.DGO",
-                 "STD.DGO", "STR.DGO", "SWB.DGO", "SWE.DGO", "TBO.DGO", "THR.DGO", "TITLE.DGO", "TOA.DGO",
-                 "TOB.DGO", "TOC.DGO", "TOD.DGO", "TOE.DGO", "TOMBEXT.DGO", "UNB.DGO", "UND.DGO", "VI1.DGO",
-                 "VIN.DGO"],
+    "dgo_names":["CGO/COMMON.CGO"],
 
-     // to write out disassembled functions in .func files
-     "write_disassembly":true,
-     "write_hex_near_instructions":false,
-     // if false, skips disassembling object files without functions, as these are usually large and not interesting yet.
-     "disassemble_objects_without_functions":false,
+    "object_file_names":[],
 
-     // to write out data of each object file
-     "write_hexdump":false,
-     // to write out hexdump on the v3 only, to avoid the huge level data files
-     "write_hexdump_on_v3_only":true,
+    "type_hints_file":"decompiler/config/jak2/type_hints.jsonc",
+    "anonymous_function_types_file":"decompiler/config/jak2/anonymous_function_types.jsonc",
 
-     // to write out "scripts", which are currently just all the linked lists found
-     "write_scripts":true,
+    "str_file_names":[],
+
+    "str_file_names_":[],
+    "allowed_objects":[],
+
+    "analyze_functions":true,
+    "analyze_expressions":false,
+    "function_type_prop":true,
+    "write_disassembly":true,
+    "write_hex_near_instructions":false,
+
+    "run_ir2":true,
+
+    // if false, skips printing disassembly of object with functions, as these are usually large (~1 GB) and not interesting yet.
+    "disassemble_objects_without_functions":false,
+
+    "process_tpages":true,
+    "process_game_text":true,
+    "process_game_count":true,
+    "dump_objs":true,
+    "write_func_json":false,
+
+    // to write out data of each object file
+    "write_hexdump":false,
+    // to write out hexdump on the v3 only, to avoid the huge level data files. Only if write_hexdump is true.
+    "write_hexdump_on_v3_only":true,
+
+    // to write out "scripts", which are currently just all the linked lists found. mostly a jak 2/3 thing
+    "write_scripts":false,
 
 
 
     // Experimental Stuff
-    "find_basic_blocks":true
+    "find_basic_blocks":true,
+
+    "types_with_bad_inspect_methods":[],
+    "no_type_analysis_functions_by_name":[],
+    "asm_functions_by_name":[],
+    "pair_functions_by_name":[]
 }