mirror/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project synced 2026-08-21 23:00:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

[jak2] split up big dgos, some minor cleanup (#2066)

Browse Source 
- Split up DGOs between threads in the multithreaded offline test
- fix some random warnings
- make the sig paths decompile a bit nicer to make some files smaller
This commit is contained in:
water111
2022-12-30 13:33:29 -05:00
committed by GitHub
parent 2eca614870
commit e49e6548e0
17 changed files with 763 additions and 22287 deletions
Download Patch File Download Diff File Expand all files Collapse all files
decompiler/Disasm/Register.cpp
+5 -3
View File
@@ -9,7 +9,7 @@
#include "common/util/Assert.h"
#include "third-party/fmt/core.h"
#include "third-party/fmt/format.h"
namespace decompiler {
namespace Reg {
@@ -128,12 +128,14 @@ Register::Register(Reg::RegisterKind kind, uint32_t num) {
case Reg::COP0:
case Reg::VI:
if (num > 32) {
ASSERT_MSG(false, fmt::format("RegisterKind: {}, greater than 32: {}", kind, num));
ASSERT_MSG(false, fmt::format("RegisterKind: {}, greater than 32: {}",
fmt::underlying(kind), num));
}
break;
case Reg::SPECIAL:
if (num > 4) {
ASSERT_MSG(false, fmt::format("Special RegisterKind: {}, greater than 4: {}", kind, num));
ASSERT_MSG(false, fmt::format("Special RegisterKind: {}, greater than 4: {}",
fmt::underlying(kind), num));
}
break;
default:
decompiler/config/jak2/all-types.gc
+2 -2
View File
@@ -38889,7 +38889,7 @@
)
(deftype sig-path-sample (structure)
((bytes uint8 32 :offset-assert 0)
((bytes uint8 32 :offset-assert 0 :do-not-decompile)
(pos vector :inline :offset-assert 32)
(quat quaternion :inline :offset-assert 48)
(flags uint8 :offset 12)
@@ -43092,7 +43092,7 @@
;; credits ;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(define-extern draw-end-credits (function level float symbol)) ;;
(define-extern draw-end-credits (function level float symbol)) ;;
(define-extern start-credits (function process none))
(define-extern check-pop-level-firework-userdata (function sparticle-system sparticle-cpuinfo sparticle-launchinfo none))
(define-extern check-pop-level-firework-red-userdata (function sparticle-system sparticle-cpuinfo sparticle-launchinfo none))
decompiler/util/data_decompile.cpp
+2 -2
View File
@@ -1545,8 +1545,8 @@ goos::Object decompile_boxed_array(const DecompilerLabel& label,
for (int j = start; j < end; j++) {
auto& word = words.at(label.target_segment).at(j / 4);
if (word.kind() != LinkedWord::PLAIN_DATA) {
throw std::runtime_error(
fmt::format("Got bad word of kind {} in boxed array of values", word.kind()));
throw std::runtime_error(fmt::format("Got bad word of kind {} in boxed array of values",
fmt::underlying(word.kind())));
}
elt_bytes.push_back(word.get_byte(j % 4));
}
game/mips2c/jak2_functions/lights.cpp
-4
View File
@@ -8,7 +8,6 @@ namespace add_light_sphere_to_light_group {
u64 execute(void* ctxt) {
auto* c = (ExecutionContext*)ctxt;
bool bc = false;
u32 call_addr = 0;
bool cop1_bc = false;
// nop // sll r0, r0, 0
// nop // sll r0, r0, 0
@@ -418,7 +417,6 @@ struct Cache {
u64 execute(void* ctxt) {
auto* c = (ExecutionContext*)ctxt;
bool bc = false;
u32 call_addr = 0;
c->daddiu(sp, sp, -48); // daddiu sp, sp, -48
c->sd(ra, 0, sp); // sd ra, 0(sp)
c->daddiu(t0, sp, 16); // daddiu t0, sp, 16
@@ -564,7 +562,6 @@ struct Cache {
u64 execute(void* ctxt) {
auto* c = (ExecutionContext*)ctxt;
bool bc = false;
u32 call_addr = 0;
c->daddiu(sp, sp, -48); // daddiu sp, sp, -48
c->daddiu(v1, sp, 16); // daddiu v1, sp, 16
// nop // sll r0, r0, 0
@@ -700,7 +697,6 @@ namespace light_hash_get_bucket_index {
u64 execute(void* ctxt) {
auto* c = (ExecutionContext*)ctxt;
bool bc = false;
u32 call_addr = 0;
c->daddiu(sp, sp, -32); // daddiu sp, sp, -32
c->daddiu(v1, sp, 16); // daddiu v1, sp, 16
// nop // sll r0, r0, 0
game/overlord/sbank.cpp
+1 -1
View File
@@ -114,7 +114,7 @@ s32 LookupSoundIndex(const char* name, SoundBank** bank_out) {
continue;
}
for (int i = 0; i < bank->sound_count; i++) {
for (int i = 0; i < (int)bank->sound_count; i++) {
if (memcmp(bank->sound[i].name, name, 16) == 0) {
*bank_out = bank;
return i;
game/overlord/srpc.cpp
+1 -1
View File
@@ -751,7 +751,7 @@ static void UnLoadMusic(s32* handle) {
*handle = 0;
}
void* RPC_Loader2(unsigned int fno, void* data, int size) {
void* RPC_Loader2(unsigned int /*fno*/, void* data, int size) {
int n_messages = size / SRPC_MESSAGE_SIZE;
SoundRpcCommand* cmd = (SoundRpcCommand*)(data);
if (!gSoundEnable) {
game/sound/989snd/sfxgrain.cpp
+3 -3
View File
@@ -228,7 +228,7 @@ SFXGrain_LoopContinue::SFXGrain_LoopContinue(SFXGrain& grain) : Grain(grain) {}
SFXGrain_LoopContinue::SFXGrain_LoopContinue(SFXGrain2& grain, u8* data) : Grain(grain) {}
s32 SFXGrain_LoopContinue::execute(blocksound_handler& handler) {
bool found = false;
for (int i = handler.m_next_grain + 1; i < handler.m_sfx.grains.size() && !found; i++) {
for (int i = handler.m_next_grain + 1; i < (int)handler.m_sfx.grains.size() && !found; i++) {
if (handler.m_sfx.grains[i]->type() == grain_type::LOOP_END) {
handler.m_next_grain = i;
found = true;
@@ -453,7 +453,7 @@ SFXGrain_GotoMarker::SFXGrain_GotoMarker(SFXGrain2& grain, u8* data) : Grain(gra
}
s32 SFXGrain_GotoMarker::execute(blocksound_handler& handler) {
bool found = false;
for (int i = 0; i < handler.m_sfx.grains.size() && !found; i++) {
for (int i = 0; i < (int)handler.m_sfx.grains.size() && !found; i++) {
if (handler.m_sfx.grains.at(i)->type() == grain_type::MARKER) {
if (static_cast<SFXGrain_Marker*>(handler.m_sfx.grains.at(i).get())->marker() == m_mark) {
handler.m_next_grain = i - 1;
@@ -482,7 +482,7 @@ s32 SFXGrain_GotoRandomMarker::execute(blocksound_handler& handler) {
s32 range = m_upper_bound - m_lower_bound + 1;
s32 mark = (rand() % range) + m_lower_bound;
for (int i = 0; i < handler.m_sfx.grains.size() && !found; i++) {
for (int i = 0; i < (int)handler.m_sfx.grains.size() && !found; i++) {
if (handler.m_sfx.grains.at(i)->type() == grain_type::MARKER) {
if (static_cast<SFXGrain_Marker*>(handler.m_sfx.grains.at(i).get())->marker() == mark) {
handler.m_next_grain = i - 1;
game/sound/989snd/sfxgrain.h
+4 -4
View File
@@ -132,7 +132,7 @@ class Grain {
virtual ~Grain() = default;
virtual s32 execute(blocksound_handler& handler) { return 0; };
virtual s32 execute(blocksound_handler& /*handler*/) { return 0; };
virtual std::string_view inspect() { return magic_enum::enum_name(type()); };
s32 delay() { return m_delay; }
grain_type type() { return m_type; }
@@ -218,13 +218,13 @@ class SFXGrain_Branch : public Grain {
class SFXGrain_ControlNull : public Grain {
public:
SFXGrain_ControlNull(SFXGrain& grain) : Grain(grain){};
SFXGrain_ControlNull(SFXGrain2& grain, u8* data) : Grain(grain){};
SFXGrain_ControlNull(SFXGrain2& grain, u8* /*data*/) : Grain(grain){};
};
class SFXGrain_LoopStart : public Grain {
public:
SFXGrain_LoopStart(SFXGrain& grain) : Grain(grain){};
SFXGrain_LoopStart(SFXGrain2& grain, u8* data) : Grain(grain){};
SFXGrain_LoopStart(SFXGrain2& grain, u8* /*data*/) : Grain(grain){};
};
class SFXGrain_LoopEnd : public Grain {
@@ -358,7 +358,7 @@ class SFXGrain_TestRegister : public Grain {
class SFXGrain_Marker : public Grain {
public:
SFXGrain_Marker(SFXGrain& grain) : Grain(grain), m_mark(grain.GrainParams.control.param[0]) {}
SFXGrain_Marker(SFXGrain2& grain, u8* data) : Grain(grain), m_mark(grain.OpcodeData.arg[0]) {}
SFXGrain_Marker(SFXGrain2& grain, u8* /*data*/) : Grain(grain), m_mark(grain.OpcodeData.arg[0]) {}
int marker() { return m_mark; }
private:
game/sound/989snd/sound_handler.h
+1 -1
View File
@@ -22,7 +22,7 @@ class sound_handler {
virtual void stop() = 0;
virtual void set_vol_pan(s32 vol, s32 pan) = 0;
virtual void set_pmod(s32 mod) = 0;
virtual void set_pbend(s32 mod){};
virtual void set_pbend(s32 /*mod*/){};
virtual void set_register(u8 /*reg*/, u8 /*value*/) {}
};
} // namespace snd
game/sound/989snd/soundbank.h
+2 -2
View File
@@ -60,8 +60,8 @@ class SoundBank {
SndPlayParams& params) = 0;
virtual std::optional<std::string_view> get_name() { return std::nullopt; };
virtual std::optional<u32> get_sound_by_name(const char* name) { return std::nullopt; };
virtual std::optional<const SFXUserData*> get_sound_user_data(u32 sound_id) {
virtual std::optional<u32> get_sound_by_name(const char* /*name*/) { return std::nullopt; };
virtual std::optional<const SFXUserData*> get_sound_user_data(u32 /*sound_id*/) {
return std::nullopt;
};
goal_src/jak2/levels/underport/sig5-cent1-path0.gc
+116 -3944
View File
File diff suppressed because it is too large Load Diff
goal_src/jak2/levels/underport/sig5-cent2-path0.gc
+210 -7140
View File
File diff suppressed because it is too large Load Diff
test/decompiler/reference/jak2/levels/underport/sig5-cent1-path0_REF.gc
Generated Vendored
+116 -3948
View File
File diff suppressed because it is too large Load Diff
test/decompiler/reference/jak2/levels/underport/sig5-cent2-path0_REF.gc
Generated Vendored
+210 -7144
View File
File diff suppressed because it is too large Load Diff
test/offline/framework/execution.cpp
+37 -41
View File
@@ -74,28 +74,26 @@ OfflineTestCompareResult compare(OfflineTestDecompiler& dc,
const OfflineTestConfig& config) {
OfflineTestCompareResult compare_result;
for (const auto& collection : work_group.work_collections) {
for (const auto& file : collection.source_files) {
work_group.status->update_curr_file(file.name_in_dgo);
auto& data = get_data(dc, file.unique_name, file.name_in_dgo);
std::string result = clean_decompilation_code(data.full_output);
std::string ref = clean_decompilation_code(file_util::read_text_file(file.path.string()));
compare_result.total_files++;
compare_result.total_lines += str_util::line_count(result);
if (result != ref) {
compare_result.failing_files.push_back({file.unique_name, diff_strings(ref, result)});
compare_result.total_pass = false;
if (config.dump_mode) {
auto failure_dir = file_util::get_jak_project_dir() / "failures";
file_util::create_dir_if_needed(failure_dir);
file_util::write_text_file(failure_dir / fmt::format("{}_REF.gc", file.unique_name),
clean_decompilation_code(data.full_output, true));
}
} else {
compare_result.ok_files++;
for (const auto& file : work_group.work_collection.source_files) {
work_group.status->update_curr_file(file.name_in_dgo);
auto& data = get_data(dc, file.unique_name, file.name_in_dgo);
std::string result = clean_decompilation_code(data.full_output);
std::string ref = clean_decompilation_code(file_util::read_text_file(file.path.string()));
compare_result.total_files++;
compare_result.total_lines += str_util::line_count(result);
if (result != ref) {
compare_result.failing_files.push_back({file.unique_name, diff_strings(ref, result)});
compare_result.total_pass = false;
if (config.dump_mode) {
auto failure_dir = file_util::get_jak_project_dir() / "failures";
file_util::create_dir_if_needed(failure_dir);
file_util::write_text_file(failure_dir / fmt::format("{}_REF.gc", file.unique_name),
clean_decompilation_code(data.full_output, true));
}
work_group.status->complete_step();
} else {
compare_result.ok_files++;
}
work_group.status->complete_step();
}
return compare_result;
@@ -114,28 +112,26 @@ OfflineTestCompileResult compile(OfflineTestDecompiler& dc,
int total_lines = 0;
for (const auto& coll : work_group.work_collections) {
for (const auto& file : coll.source_files) {
work_group.status->update_curr_file(file.name_in_dgo);
if (config.skip_compile_files.count(file.name_in_dgo)) {
lg::warn("Skipping {}", file.name_in_dgo);
continue;
}
lg::info("Compiling {}...", file.unique_name);
auto& data = get_data(dc, file.unique_name, file.name_in_dgo);
try {
const auto& src = data.output_with_skips;
total_lines += str_util::line_count(src);
compiler.run_full_compiler_on_string_no_save(src, file.name_in_dgo);
} catch (const std::exception& e) {
result.ok = false;
result.failing_files.push_back({file.name_in_dgo, e.what()});
}
work_group.status->complete_step();
for (const auto& file : work_group.work_collection.source_files) {
work_group.status->update_curr_file(file.name_in_dgo);
if (config.skip_compile_files.count(file.name_in_dgo)) {
lg::warn("Skipping {}", file.name_in_dgo);
continue;
}
lg::info("Compiling {}...", file.unique_name);
auto& data = get_data(dc, file.unique_name, file.name_in_dgo);
try {
const auto& src = data.output_with_skips;
total_lines += str_util::line_count(src);
compiler.run_full_compiler_on_string_no_save(src, file.name_in_dgo);
} catch (const std::exception& e) {
result.ok = false;
result.failing_files.push_back({file.name_in_dgo, e.what()});
}
work_group.status->complete_step();
}
result.num_lines = total_lines;
test/offline/framework/orchestration.cpp
+48 -39
View File
@@ -37,13 +37,11 @@ OfflineTestDecompiler setup_decompiler(const OfflineTestWorkGroup& work,
// modify the config
std::unordered_set<std::string> object_files;
for (const auto& coll : work.work_collections) {
for (auto& file : coll.source_files) {
object_files.insert(file.name_in_dgo); // todo, make this work with unique_name
}
for (auto& file : coll.art_files) {
object_files.insert(file.unique_name);
}
for (auto& file : work.work_collection.source_files) {
object_files.insert(file.name_in_dgo); // todo, make this work with unique_name
}
for (auto& file : work.work_collection.art_files) {
object_files.insert(file.unique_name);
}
dc.config->allowed_objects = object_files;
@@ -69,19 +67,17 @@ OfflineTestDecompiler setup_decompiler(const OfflineTestWorkGroup& work,
if (db_files.size() != object_files.size()) {
lg::error("DB file error: has {} entries, but expected {}", db_files.size(),
object_files.size());
for (const auto& coll : work.work_collections) {
for (auto& file : coll.source_files) {
if (!db_files.count(file.unique_name)) {
lg::error(
"didn't find {}, make sure it's part of the DGO inputs and not in the banned objects "
"list\n",
file.unique_name);
}
for (auto& file : work.work_collection.source_files) {
if (!db_files.count(file.unique_name)) {
lg::error(
"didn't find {}, make sure it's part of the DGO inputs and not in the banned objects "
"list\n",
file.unique_name);
}
for (auto& file : coll.art_files) {
if (!db_files.count(file.unique_name)) {
lg::error("didn't find {}\n", file.unique_name);
}
}
for (auto& file : work.work_collection.art_files) {
if (!db_files.count(file.unique_name)) {
lg::error("didn't find {}\n", file.unique_name);
}
}
exit(1);
@@ -112,16 +108,11 @@ std::vector<std::future<OfflineTestThreadResult>> distribute_work(
}
// Now partition by DGO so that threads do not consume unnecessary or duplicate resources
// this is a half-decent approximation of a greedy-knapsack approach (where the knapsack can be
// overstuffed) Repeatedly just add to the thread with the current least amount of work
//
// TODO - if it ends up being that it would be advantageous to split up a massive dgo into
// multiple threads (ie. lots in engine) then this should be improved to accomodate that.
//
// TODO - additionally, if we have more threads than we can actually utilize we should not
// reserve them and dynamically adjust the used thread count
std::vector<OfflineTestWorkGroup> work_groups = {};
for (int i = 0; i < offline_config.num_threads; i++) {
for (int i = 0; i < (int)offline_config.num_threads; i++) {
auto new_group = OfflineTestWorkGroup();
new_group.status = std::make_shared<OfflineTestThreadStatus>(offline_config);
work_groups.push_back(new_group);
@@ -130,21 +121,39 @@ std::vector<std::future<OfflineTestThreadResult>> distribute_work(
g_offline_test_thread_manager.print_current_test_status(offline_config);
// Count the total number of files.
// We'll divide the files evenly between workers. We want to avoid the case where all workers need
// all DGOs, so assign consecutive files (likely to belong to the same dgo) to the same worker.
int total_files = 0;
for (const auto& [dgo, work] : work_colls) {
// Find the smallest group
u32 smallest_group_idx = 0;
for (int i = 0; i < work_groups.size(); i++) {
if (work_groups[i].work_size() < work_groups[smallest_group_idx].work_size()) {
smallest_group_idx = i;
}
total_files += work.source_files.size() + work.art_files.size();
}
int divisor = (total_files + work_groups.size() - 1) / work_groups.size();
// Divide up the work
int file_idx = 0;
for (const auto& [dgo, work] : work_colls) {
// art files
for (auto& art_file : work.art_files) {
auto& wg = work_groups.at(file_idx / divisor);
wg.dgo_set.insert(dgo);
wg.work_collection.art_files.push_back(art_file);
file_idx++;
}
// Add the DGO and the files to it
work_groups[smallest_group_idx].dgos.push_back(dgo);
work_groups[smallest_group_idx].work_collections.push_back(work);
work_groups[smallest_group_idx].status->dgos.push_back(dgo);
work_groups[smallest_group_idx].status->total_steps =
work_groups[smallest_group_idx].work_size() * 3; // decomp, compare, compile
// source files
for (auto& source_file : work.source_files) {
auto& wg = work_groups.at(file_idx / divisor);
wg.dgo_set.insert(dgo);
wg.work_collection.source_files.push_back(source_file);
file_idx++;
}
}
// Create summary of work for pretty printing.
for (auto& wg : work_groups) {
wg.status->dgos = wg.dgo_set;
wg.status->total_steps = wg.work_size() * 3; // decomp, compare, compile
}
// Now we can finally create the futures
@@ -235,7 +244,7 @@ std::tuple<fmt::color, std::string> thread_stage_to_str(OfflineTestThreadStatus:
}
}
std::string thread_dgos_to_str(std::vector<std::string> dgos) {
std::string thread_dgos_to_str(const std::set<std::string>& dgos) {
std::vector<std::string> ones_to_print = {};
for (const auto& dgo : dgos) {
ones_to_print.push_back(dgo);
@@ -256,7 +265,7 @@ std::string thread_progress_bar(u32 curr_step, u32 total_steps) {
const u32 completed_segments = completion / 10;
std::string progress_bar = "";
int added_segments = 0;
for (int i = 0; i < completed_segments; i++) {
for (int i = 0; i < (int)completed_segments; i++) {
progress_bar += "■";
added_segments++;
}
test/offline/framework/orchestration.h
+5 -8
View File
@@ -2,6 +2,7 @@
#include <future>
#include <mutex>
#include <set>
#include <string>
#include <vector>
@@ -85,7 +86,7 @@ class OfflineTestThreadStatus {
Stage stage = Stage::IDLE;
uint32_t total_steps = 0;
uint32_t curr_step = 0;
std::vector<std::string> dgos = {};
std::set<std::string> dgos;
std::string curr_file;
OfflineTestConfig config;
@@ -100,16 +101,12 @@ struct OfflineTestWorkCollection {
};
struct OfflineTestWorkGroup {
std::vector<std::string> dgos;
std::vector<OfflineTestWorkCollection> work_collections;
std::set<std::string> dgo_set;
OfflineTestWorkCollection work_collection;
std::shared_ptr<OfflineTestThreadStatus> status;
int work_size() const {
int i = 0;
for (const auto& coll : work_collections) {
i += coll.source_files.size() + coll.art_files.size();
}
return i;
return work_collection.source_files.size() + work_collection.art_files.size();
}
};