mirror of
https://github.com/open-goal/jak-project
synced 2026-06-27 19:02:59 -04:00
9c631e11fe
This solves two main problems: - the looming threat of running out of memory since every thread would consume duplicate (and probably not needed) resources - though I will point out, jak 2's offline tests seem to hardly use any memory even with 400+ files, duplicated across many threads. Where as jak 1 does indeed use tons more memory. So I think there is something going on besides just the source files - condense the output so it's much easier to see what is happening / how close the test is to completing. - one annoying thing about the multiple thread change was errors were typically buried far in the middle of the output, this fixes that - refactors the offline test code in general to be a lot more modular The pretty printing is not enabled by default, run with `-p` or `--pretty-print` if you want to use it https://user-images.githubusercontent.com/13153231/205513212-a65c20d4-ce36-44f6-826a-cd475505dbf9.mp4
132 lines
3.3 KiB
C++
132 lines
3.3 KiB
C++
#pragma once
|
|
|
|
#include <future>
|
|
#include <mutex>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "file_management.h"
|
|
|
|
#include "common/util/Timer.h"
|
|
|
|
#include "test/offline/config/config.h"
|
|
|
|
struct OfflineTestCompareResult {
|
|
struct Fail {
|
|
std::string filename;
|
|
std::string diff;
|
|
};
|
|
std::vector<Fail> failing_files;
|
|
int total_files = 0;
|
|
int ok_files = 0;
|
|
int total_lines = 0;
|
|
bool total_pass = true;
|
|
|
|
void add(const OfflineTestCompareResult& other) {
|
|
failing_files.insert(failing_files.end(), other.failing_files.begin(),
|
|
other.failing_files.end());
|
|
total_files += other.total_files;
|
|
ok_files += other.ok_files;
|
|
total_lines += other.total_lines;
|
|
if (!other.total_pass) {
|
|
total_pass = false;
|
|
}
|
|
}
|
|
};
|
|
|
|
struct OfflineTestCompileResult {
|
|
bool ok = true;
|
|
struct Fail {
|
|
std::string filename;
|
|
std::string error;
|
|
};
|
|
std::vector<Fail> failing_files;
|
|
int num_lines = 0;
|
|
void add(const OfflineTestCompileResult& other) {
|
|
failing_files.insert(failing_files.end(), other.failing_files.begin(),
|
|
other.failing_files.end());
|
|
num_lines += other.num_lines;
|
|
if (!other.ok) {
|
|
ok = false;
|
|
}
|
|
}
|
|
};
|
|
|
|
/// @brief A simple struct to contain the reason for failure from a thread
|
|
struct OfflineTestThreadResult {
|
|
int exit_code = 0;
|
|
std::string reason;
|
|
|
|
float time_spent_compiling = 0;
|
|
float time_spent_decompiling = 0;
|
|
float total_time = 0;
|
|
|
|
OfflineTestCompareResult compare;
|
|
OfflineTestCompileResult compile;
|
|
|
|
void add(const OfflineTestThreadResult& other) {
|
|
if (other.exit_code) {
|
|
exit_code = other.exit_code;
|
|
}
|
|
time_spent_compiling += other.time_spent_compiling;
|
|
time_spent_decompiling += other.time_spent_decompiling;
|
|
total_time += other.total_time;
|
|
compare.add(other.compare);
|
|
compile.add(other.compile);
|
|
}
|
|
};
|
|
|
|
class OfflineTestThreadStatus {
|
|
public:
|
|
enum class Stage { IDLE, PREPARING, DECOMPILING, COMPARING, COMPILING, FAILED, FINISHED };
|
|
|
|
OfflineTestThreadStatus(const OfflineTestConfig& _config) : config(_config){};
|
|
|
|
Stage stage = Stage::IDLE;
|
|
uint32_t total_steps = 0;
|
|
uint32_t curr_step = 0;
|
|
std::vector<std::string> dgos = {};
|
|
std::string curr_file;
|
|
OfflineTestConfig config;
|
|
|
|
void update_stage(Stage new_stage);
|
|
void update_curr_file(const std::string& _curr_file);
|
|
void complete_step();
|
|
};
|
|
|
|
struct OfflineTestWorkCollection {
|
|
std::vector<OfflineTestSourceFile> source_files;
|
|
std::vector<OfflineTestArtFile> art_files;
|
|
};
|
|
|
|
struct OfflineTestWorkGroup {
|
|
std::vector<std::string> dgos;
|
|
std::vector<OfflineTestWorkCollection> work_collections;
|
|
std::shared_ptr<OfflineTestThreadStatus> status;
|
|
|
|
int work_size() const {
|
|
int i = 0;
|
|
for (const auto& coll : work_collections) {
|
|
i += coll.source_files.size() + coll.art_files.size();
|
|
}
|
|
return i;
|
|
}
|
|
};
|
|
|
|
class OfflineTestThreadManager {
|
|
public:
|
|
void print_current_test_status(const OfflineTestConfig& config);
|
|
|
|
std::vector<std::shared_ptr<OfflineTestThreadStatus>> statuses = {};
|
|
|
|
private:
|
|
std::mutex print_lock;
|
|
};
|
|
|
|
extern OfflineTestThreadManager g_offline_test_thread_manager;
|
|
|
|
std::vector<std::future<OfflineTestThreadResult>> distribute_work(
|
|
const OfflineTestConfig& offline_config,
|
|
const std::vector<OfflineTestSourceFile>& files,
|
|
const std::vector<OfflineTestArtFile>& art_files);
|