full ntsc-j mode + greatest hits support (#1525)

* remove str files from inputs

* removed unused verbose flag

* allow some kind of conditional game building for JP extras

* make `PrintBankInfo` do nothing

* clang

* Update gltf_mesh_extract.cpp

* fix `*jak1-full-game*` in extractor

* use json library to build json file

* screw it red label support as well

decompiler/config/jak1_ntsc_black_label/inputs.jsonc

@@ -47,7 +47,7 @@
   ],
 
   // some objects are part of STR files (streaming data). In Jak 1 this is just animations
-  "str_file_names": [
+  "str_file_names": [/*
     "STR/BAFCELL.STR",
     "STR/SWTE4.STR",
     "STR/SWTE3.STR",
@@ -245,7 +245,7 @@
     "STR/GRSOPREB.STR",
     "STR/GRSOBBB.STR",
     "STR/SA3INTRO.STR"
-  ],
+    */],
 
   // some objects are directly stored as files on the DVD. This is just text files.
   "object_file_names": [

decompiler/config/jak1_us2.jsonc

@@ -0,0 +1,116 @@
+{
+  "game_version": 1,
+  "text_version": 11,
+  "game_name": "jak1_us2",
+  "expected_elf_name": "SCUS_971.24",
+
+  // if you want to filter to only some object names.
+  // it will make the decompiler much faster.
+  "allowed_objects": [],
+  "banned_objects": [],
+
+  ////////////////////////////
+  // CODE ANALYSIS OPTIONS
+  ////////////////////////////
+
+  // set to true to generate plain .asm files with MIPS disassembly, with no fancy decompilation.
+  // this is fast and should succeed 100% of the time.
+  "disassemble_code": false,
+
+  // Run the decompiler
+  "decompile_code": false,
+
+  // run the first pass of the decompiler
+  "find_functions": true,
+
+  ////////////////////////////
+  // DATA ANALYSIS OPTIONS
+  ////////////////////////////
+
+  // set to true to generate plain .asm files for data files.
+  // this will display most data as hex, but will add labels/references/type pointers/strings
+  // this generates a huge amount of output if you run it on the entire game.
+  "disassemble_data": false,
+
+  // unpack textures to assets folder
+  "process_tpages": true,
+  // unpack game text to assets folder
+  "process_game_text": true,
+  // unpack game count to assets folder
+  "process_game_count": true,
+  // write goal imports for art groups
+  "process_art_groups": true,
+
+  ///////////////////////////
+  // WEIRD OPTIONS
+  ///////////////////////////
+
+  // these options are used rarely and should usually be left at false
+
+  // generate the symbol_map.json file.
+  // this is a guess at where each symbol is first defined/used.
+  "generate_symbol_definition_map": false,
+
+  // debug option for instruction decoder
+  "write_hex_near_instructions": false,
+
+  // experimental tool to extract linked lists used for region scripting in Jak 2 and Jak 3.
+  "write_scripts": false,
+
+  // hex dump of code/data files.
+  "hexdump_code": false,
+  "hexdump_data": false,
+  // dump raw obj files
+  "dump_objs": true,
+  // print control flow graph
+  "print_cfgs": false,
+
+  // set to true for PAL versions. this will forcefully skip files that have some data missing at the end.
+  "is_pal": false,
+
+  ////////////////////////////
+  // CONFIG FILES
+  ////////////////////////////
+
+  "type_casts_file": "decompiler/config/jak1_pal/type_casts.jsonc",
+  "anonymous_function_types_file": "decompiler/config/jak1_ntsc_black_label/anonymous_function_types.jsonc",
+  "var_names_file": "decompiler/config/jak1_ntsc_black_label/var_names.jsonc",
+  "label_types_file": "decompiler/config/jak1_jp/label_types.jsonc",
+  "stack_structures_file": "decompiler/config/jak1_ntsc_black_label/stack_structures.jsonc",
+  "hacks_file": "decompiler/config/jak1_pal/hacks.jsonc",
+  "inputs_file": "decompiler/config/jak1_ntsc_black_label/inputs.jsonc",
+  "art_info_file": "decompiler/config/jak1_ntsc_black_label/art_info.jsonc",
+  "import_deps_file": "decompiler/config/jak1_ntsc_black_label/import_deps.jsonc",
+  "all_types_file": "decompiler/config/all-types.gc",
+
+  // optional: a predetermined object file name map from a file.
+  // this will make decompilation naming consistent even if you only run on some objects.
+  "obj_file_name_map_file": "goal_src/build/all_objs_jak1_jp.json",
+
+  ////////////////////////////
+  // LEVEL EXTRACTION
+  ////////////////////////////
+
+  // turn this on to extract level background graphics data
+  "levels_extract": true,
+  // turn this on if you want extracted levels to be saved out as .obj files
+  "levels_convert_to_obj": false,
+  // should we extract collision meshes?
+  // these can be displayed in game, but makes the .fr3 files slightly larger
+  "extract_collision": true,
+
+  ////////////////////////////
+  // PATCHING OPTIONS
+  ////////////////////////////
+
+  // these are options related to xdelta3 patches on specific objects
+  // this allows us to get a more consistent input
+
+  // set to true to write new patch files
+  "write_patches": false,
+  // set to true to apply patch files
+  "apply_patches": true,
+  // what to patch an object to and the patch file is
+  "object_patches": {
+  }
+}

decompiler/extractor/main.cpp

@@ -1,7 +1,10 @@
+#include <map>
 #include <regex>
+#include <unordered_map>
 
 #include "common/log/log.h"
 #include "common/util/FileUtil.h"
+#include "common/util/json_util.h"
 #include "common/util/read_iso_file.h"
 
 #include "decompiler/Disasm/OpcodeInfo.h"
@@ -23,12 +26,18 @@ enum class ExtractorErrorCode {
   DECOMPILATION_GENERIC_ERROR = 4030
 };
 
+enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) };
+
+static const std::unordered_map<std::string, GameIsoFlags> sGameIsoFlagNames = {
+    {"jak1-black-label", FLAG_JAK1_BLACK_LABEL}};
+
 struct ISOMetadata {
   std::string canonical_name;
   std::string region;
   int num_files;
   xxh::hash64_t contents_hash;
   std::string decomp_config;
+  GameIsoFlags flags = (GameIsoFlags)0;
 };
 
 // TODO - when we support jak2 and beyond, add which game it's for as well
@@ -39,11 +48,12 @@ static const std::map<std::string, std::map<xxh::hash64_t, ISOMetadata>> isoData
     {"SCUS-97124",
      {{7280758013604870207U,
        {"Jak & Daxter™: The Precursor Legacy (Black Label)", "NTSC-U", 337, 11363853835861842434U,
-        "jak1_ntsc_black_label"}}}},
+        "jak1_ntsc_black_label", FLAG_JAK1_BLACK_LABEL}},
+      {744661860962747854,
+       {"Jak & Daxter™: The Precursor Legacy", "NTSC-U", 338, 8538304367812415885U, "jak1_jp"}}}},
     {"SCES-50361",
      {{12150718117852276522U,
-       {"Jak & Daxter™: The Precursor Legacy (PAL)", "PAL", 338, 16850370297611763875U,
-        "jak1_pal"}}}},
+       {"Jak & Daxter™: The Precursor Legacy", "PAL", 338, 16850370297611763875U, "jak1_pal"}}}},
     {"SCPS-15021",
      {{16909372048085114219U,
        {"ジャックＸダクスター　～　旧世界の遺産", "NTSC-J", 338, 1262350561338887717,
@@ -90,11 +100,12 @@ std::pair<std::optional<std::string>, std::optional<xxh::hash64_t>> findElfFile(
   return {serial, elf_hash};
 }
 
-ExtractorErrorCode validate(const IsoFile& iso_file,
-                            const std::filesystem::path& extracted_iso_path) {
+std::pair<ExtractorErrorCode, std::optional<ISOMetadata>> validate(
+    const IsoFile& iso_file,
+    const std::filesystem::path& extracted_iso_path) {
   if (!std::filesystem::exists(extracted_iso_path / "DGO")) {
     fmt::print(stderr, "ERROR: input folder doesn't have a DGO folder. Is this the right input?\n");
-    return ExtractorErrorCode::VALIDATION_BAD_EXTRACTION;
+    return {ExtractorErrorCode::VALIDATION_BAD_EXTRACTION, std::nullopt};
   }
 
   std::optional<ExtractorErrorCode> error_code;
@@ -116,10 +127,11 @@ ExtractorErrorCode validate(const IsoFile& iso_file,
       error_code = std::make_optional(ExtractorErrorCode::VALIDATION_CANT_LOCATE_ELF);
     }
     // No point in continuing here
-    return error_code.value();
+    return {*error_code, std::nullopt};
   }
 
   // Find the game in our tracking database
+  std::optional<ISOMetadata> meta_res = std::nullopt;
   if (auto dbEntry = isoDatabase.find(serial.value()); dbEntry == isoDatabase.end()) {
     fmt::print(stderr, "ERROR: Serial '{}' not found in the validation database\n", serial.value());
     if (!error_code.has_value()) {
@@ -137,7 +149,8 @@ ExtractorErrorCode validate(const IsoFile& iso_file,
         error_code = std::make_optional(ExtractorErrorCode::VALIDATION_ELF_MISSING_FROM_DB);
       }
     } else {
-      auto meta = meta_entry->second;
+      meta_res = std::make_optional<ISOMetadata>(meta_entry->second);
+      const auto& meta = *meta_res;
       // Print out some information
       fmt::print("Detected Game Metadata:\n");
       fmt::print("\tDetected - {}\n", meta.canonical_name);
@@ -189,10 +202,10 @@ ExtractorErrorCode validate(const IsoFile& iso_file,
                  "Validation has failed to match with expected values, see the above errors for "
                  "specifics. This may be an error in the validation database!\n");
     }
-    return error_code.value();
+    return {*error_code, std::nullopt};
   }
 
-  return ExtractorErrorCode::SUCCESS;
+  return {ExtractorErrorCode::SUCCESS, meta_res};
 }
 
 std::optional<ISOMetadata> determineRelease(const std::filesystem::path& jak1_input_files) {
@@ -313,6 +326,17 @@ void compile(std::filesystem::path extracted_iso_path) {
   compiler.make_system().set_constant("*iso-data*", absolute(extracted_iso_path).string());
   compiler.make_system().set_constant("*use-iso-data-path*", true);
 
+  auto buildinfo_path = (extracted_iso_path / "buildinfo.json").string();
+  auto bi = parse_commented_json(file_util::read_text_file(buildinfo_path), buildinfo_path);
+  auto all_flags = bi.at("flags").get<std::vector<std::string>>();
+  int flags = 0;
+  for (const auto& n : all_flags) {
+    if (auto it = sGameIsoFlagNames.find(n); it != sGameIsoFlagNames.end()) {
+      flags |= it->second;
+    }
+  }
+  compiler.make_system().set_constant("*jak1-full-game*", !(flags & FLAG_JAK1_BLACK_LABEL));
+
   compiler.make_system().load_project_file(
       (file_util::get_jak_project_dir() / "goal_src" / "game.gp").string());
   compiler.run_front_end_on_string("(mi)");
@@ -389,15 +413,17 @@ int main(int argc, char** argv) {
     }
     std::filesystem::create_directories(path_to_iso_files);
 
+    int flags = 0;
     if (std::filesystem::is_regular_file(data_dir_path)) {
       // it's a file, treat it as an ISO
       auto iso_file = extract_files(data_dir_path, path_to_iso_files);
       auto validation_res = validate(iso_file, path_to_iso_files);
-      if (validation_res == ExtractorErrorCode::VALIDATION_BAD_EXTRACTION) {
+      flags = validation_res.second->flags;
+      if (validation_res.first == ExtractorErrorCode::VALIDATION_BAD_EXTRACTION) {
         // We fail here regardless of the flag
-        return static_cast<int>(validation_res);
-      } else if (flag_fail_on_validation && validation_res != ExtractorErrorCode::SUCCESS) {
-        return static_cast<int>(validation_res);
+        return static_cast<int>(validation_res.first);
+      } else if (flag_fail_on_validation && validation_res.first != ExtractorErrorCode::SUCCESS) {
+        return static_cast<int>(validation_res.first);
       }
     } else if (std::filesystem::is_directory(data_dir_path)) {
       if (!flag_folder) {
@@ -407,6 +433,19 @@ int main(int argc, char** argv) {
       }
       path_to_iso_files = data_dir_path;
     }
+
+    // write out a json file with some metadata for the game
+    nlohmann::json buildinfo_json;
+    auto flags_json = nlohmann::json::array();
+    for (const auto& [n, f] : sGameIsoFlagNames) {
+      if (flags & f) {
+        flags_json.push_back(n);
+      }
+    }
+    buildinfo_json["flags"] = flags_json;
+    // something tells me a ps2 game is unlikely to have a json file in root
+    file_util::write_text_file((path_to_iso_files / "buildinfo.json").string(),
+                               buildinfo_json.dump(2));
   }
 
   if (flag_decompile) {

game/kernel/kboot.cpp

@@ -126,9 +126,6 @@ s32 goal_main(int argc, const char* const* argv) {
     masterConfig.aspect = SCE_ASPECT_FULL;
   }
 
-  // Added - the territory is originally hardcoded. this allows us to change it whenever.
-  masterConfig.territory = GAME_TERRITORY_SCEA;
-
   // In retail game, disable debugging modes, and force on DiskBoot
   // MasterDebug = 0;
   // DiskBoot = 1;

game/kernel/kboot.h

@@ -19,7 +19,6 @@ struct MasterConfig {
   u16 timeout;           // todo           8
   u16 volume;            // todo           12
 
-  u16 territory;          // added. this is normally burnt onto the disc executable.
   u16 disable_sound = 0;  // added. disables all sound code.
 };
 

game/kernel/kmachine.cpp

@@ -729,7 +729,7 @@ u64 DecodeVolume() {
 // NOTE: this is originally hardcoded, and returns different values depending on the disc region.
 // it returns 0 for NTSC-U, 1 for PAL and 2 for NTSC-J
 u64 DecodeTerritory() {
-  return masterConfig.territory;
+  return GAME_TERRITORY_SCEA;
 }
 
 u64 DecodeTimeout() {

game/overlord/soundcommon.cpp

@@ -17,6 +17,9 @@ void ReadBankSoundInfo(SoundBank* bank, SoundBank* unk, s32 unk2) {
 }
 
 void PrintBankInfo(SoundBank* bank) {
+  // we dont need this and it spams the console too much
+  return;
+
   printf("Bank %s\n\n", bank->name);
   for (u32 i = 0; i < bank->sound_count; i++) {
     // Some characters use the full 16 characters (bonelurker-grunt) and dont have a null terminator

goal_src/engine/game/main.gc

@@ -341,6 +341,17 @@
   (none)
   )
 
+;;;;;;;;;;;;;;;;;;;;;;
+;; opengoal territory override
+;;;;;;;;;;;;;;;;;;;;;;
+
+(defun scf-get-territory ()
+  "this overrides the kernel version."
+  (if (and *jak1-full-game* (= (language-enum japanese) (-> *setting-control* default language)))
+      GAME_TERRITORY_SCEI
+      GAME_TERRITORY_SCEA)
+  )
+
 ;;;;;;;;;;;;;;;;;;;;;;
 ;; Cheat Codes
 ;;;;;;;;;;;;;;;;;;;;;;

goal_src/engine/game/settings.gc

@@ -306,6 +306,17 @@
         )
       ;; send language change to the IOP.
       (when (!= (-> gp-0 language) (-> s5-1 language))
+        ;; NOTE : pc port added this
+        (cond
+          ((= (language-enum japanese) (-> s5-1 language)) ;; old lang wasnt japanese
+           (sound-bank-unload (static-sound-name "common"))
+           (sound-bank-load (static-sound-name "commonj"))
+           )
+          ((= (language-enum japanese) (-> gp-0 language)) ;; old lang was japanese
+           (sound-bank-unload (static-sound-name "commonj"))
+           (sound-bank-load (static-sound-name "common"))
+           )
+          )
         (set! (-> gp-0 language) (-> s5-1 language))
         (set-language (-> gp-0 language))
         )

goal_src/game.gp

@@ -132,6 +132,8 @@
 
 
 (define *game-directory* (get-environment-variable "OPENGOAL_DECOMP_DIR" :default "jak1/"))
+(when (user? dass)
+    (set! *game-directory* "jak1_pal/"))
 
 (defmacro copy-texture (tpage-id)
   "Copy a texture from the game, using the given tpage ID"
@@ -384,7 +386,11 @@
 
 ;; as we find objects that exist in multiple levels, put them here
 
-(copy-sbk-files "COMMON" "EMPTY1" "EMPTY2")
+;; early versions of the game - including the black label release - do not have all files.
+(if *jak1-full-game*
+    (copy-sbk-files "COMMON" "COMMONJ" "EMPTY1" "EMPTY2")
+    (copy-sbk-files "COMMON" "EMPTY1" "EMPTY2")
+    )
 
 (copy-gos
  "sharkey-ag"
@@ -1958,8 +1964,8 @@
  )
 
 
-(fmt #t "found {} spools\n" (count *all-str*))
-(group-list "spools" (reverse *all-str*))
+;;(fmt #t "found {} spools\n" (count *all-str*))
+(group-list "spools" *all-str*)
 
 
 (group-list "text"

goal_src/goal-lib.gc

@@ -1008,7 +1008,7 @@
 
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;;; art stuf
+;;;; art stuff
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defmacro def-art-elt (group name idx)
@@ -1045,6 +1045,13 @@
   )
 
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;;; build system stuff
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; hack to get a goos variable in goal
+(defmacro __get_jak1_full_game () *jak1-full-game*)
+(defconstant *jak1-full-game* (__get_jak1_full_game))
 
 ;; load the default project
 (load-project "goal_src/game.gp")

goal_src/goos-lib.gs

@@ -452,3 +452,9 @@
 (define *art-info* (make-string-hash-table))
 
 
+;;;;;;;;;;;;;;;;;;;;;;;;
+;;  build system      ;;
+;;;;;;;;;;;;;;;;;;;;;;;;
+
+(define *jak1-full-game* (if (user? dass) #t #f))
+

goalc/compiler/Compiler.cpp

@@ -18,7 +18,10 @@
 using namespace goos;
 
 Compiler::Compiler(const std::string& user_profile, std::unique_ptr<ReplWrapper> repl)
-    : m_goos(user_profile), m_debugger(&m_listener, &m_goos.reader), m_repl(std::move(repl)) {
+    : m_goos(user_profile),
+      m_debugger(&m_listener, &m_goos.reader),
+      m_repl(std::move(repl)),
+      m_make(user_profile) {
   m_listener.add_debugger(&m_debugger);
   m_ts.add_builtin_types();
   m_global_env = std::make_unique<GlobalEnv>();
@@ -31,6 +34,7 @@ Compiler::Compiler(const std::string& user_profile, std::unique_ptr<ReplWrapper>
   Object library_code = m_goos.reader.read_from_file({"goal_src", "goal-lib.gc"});
   compile_object_file("goal-lib", library_code, false);
 
+  // user profile stuff
   if (user_profile != "#f") {
     try {
       Object user_code =

goalc/main.cpp

@@ -22,7 +22,6 @@ void setup_logging() {
 }
 
 int main(int argc, char** argv) {
-  bool verbose = false;
   bool auto_listen = false;
   bool auto_debug = false;
   bool auto_find_user = false;

goalc/make/MakeSystem.cpp

@@ -38,7 +38,7 @@ std::string MakeStep::print() const {
   return result;
 }
 
-MakeSystem::MakeSystem() {
+MakeSystem::MakeSystem(const std::string& username) : m_goos(username) {
   m_goos.register_form("defstep", [=](const goos::Object& obj, goos::Arguments& args,
                                       const std::shared_ptr<goos::EnvironmentObject>& env) {
     return handle_defstep(obj, args, env);

goalc/make/MakeSystem.h

@@ -15,7 +15,7 @@ struct MakeStep {
 
 class MakeSystem {
  public:
-  MakeSystem();
+  MakeSystem(const std::string& username = "#f");
   void load_project_file(const std::string& file_path);
 
   goos::Object handle_defstep(const goos::Object& obj,

scripts/batch/decomp-jak1_us2.bat

@@ -0,0 +1,4 @@
+@echo off
+cd ..\..
+out\build\Release\bin\decompiler decompiler\config\jak1_us2.jsonc iso_data decompiler_out\
+pause

scripts/batch/decomp-jak1sced.bat

@@ -1,4 +0,0 @@
-@echo off
-cd ..\..
-out\build\Release\bin\decompiler decompiler\config\jak1_sced.jsonc iso_data\jak1sced decompiler_out\jak1sced
-pause

scripts/batch/gc-dbg.bat

@@ -1,4 +1,4 @@
 @echo off
 cd ..\..
-out\build\Release\bin\goalc -v --auto-dbg --user-auto
+out\build\Release\bin\goalc --auto-dbg --user-auto
 pause

scripts/batch/gc-no-lt.bat

@@ -1,4 +1,4 @@
 @echo off
 cd ..\..
-out\build\Release\bin\goalc -v --user-auto
+out\build\Release\bin\goalc --user-auto
 pause

scripts/batch/gc.bat

@@ -1,4 +1,4 @@
 @echo off
 cd ..\..
-out\build\Release\bin\goalc -v --auto-lt --user-auto
+out\build\Release\bin\goalc --auto-lt --user-auto
 pause