Add extractor tool (#1276)

* first attempt

* fix

* zip to tar

* windows

* try again, std::filesystem sucks

* std::filesystem is still garbage

* std::filesystem is terrible

* std::filesystem continues to waste my time

* again

* neadsflaldksal;df
This commit is contained in:
water111
2022-04-03 19:17:03 -04:00
committed by GitHub
parent f14b7aa0cd
commit 5bd0b735a5
20 changed files with 311 additions and 60 deletions
+5 -11
View File
@@ -41,7 +41,7 @@ jobs:
run: git submodule update --init --recursive -j 2
- name: Get Common Package Dependencies
run: sudo apt install build-essential cmake clang gcc g++ lcov make nasm libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev
run: sudo apt install build-essential cmake clang gcc g++ lcov make nasm libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev zip
- name: Get Clang
if: matrix.compiler == 'clang'
@@ -100,22 +100,16 @@ jobs:
- name: Prepare Build Artifacts
if: github.repository == 'open-goal/jak-project' && startsWith(github.ref, 'refs/tags/') && matrix.compiler == 'clang'
run: |
mkdir -p ./ci-artifacts/
cp ./build/decompiler/decompiler ./ci-artifacts/linux-decompiler.bin
cp ./build/game/gk ./ci-artifacts/linux-gk.bin
cp ./build/goalc/goalc ./ci-artifacts/linux-goalc.bin
chmod +x ./ci-artifacts/*.bin
strip --strip-debug ./ci-artifacts/linux-decompiler.bin
strip --strip-debug ./ci-artifacts/linux-gk.bin
strip --strip-debug ./ci-artifacts/linux-goalc.bin
ls -l ./ci-artifacts/
mkdir -p ./ci-artifacts/out
./scripts/shell/extract_build.sh ./ci-artifacts/out ./
tar czf ./ci-artifacts/opengoal.tar.gz ./ci-artifacts/out
- name: Upload Assets and Potential Publish Release
if: github.repository == 'open-goal/jak-project' && startsWith(github.ref, 'refs/tags/') && matrix.compiler == 'clang'
env:
GITHUB_TOKEN: ${{ secrets.BOT_PAT }}
ASSET_DIR: ${{ github.WORKSPACE }}/ci-artifacts
ASSET_EXTENSION: bin
ASSET_EXTENSION: zip
TAG_TO_SEARCH_FOR: ${{ github.REF }}
run: |
cd ./.github/scripts/releases/upload-release-artifacts
-7
View File
@@ -835,13 +835,6 @@ void Reader::throw_reader_error(TextStream& here, const std::string& err, int se
db.get_info_for(here.text, here.seek + seek_offset));
}
/*!
* Get the source directory of the current project.
*/
std::string Reader::get_source_dir() {
return file_util::get_project_path();
}
/*!
* Convert any string into one that can be read.
* Unprintable characters become escape sequences, including tab and newline.
-1
View File
@@ -78,7 +78,6 @@ class Reader {
std::optional<Object> read_from_stdin(const std::string& prompt, ReplWrapper& repl);
Object read_from_file(const std::vector<std::string>& file_path, bool check_encoding = false);
bool check_string_is_valid(const std::string& str) const;
std::string get_source_dir();
SymbolTable symbolTable;
TextDb db;
+71 -22
View File
@@ -56,14 +56,19 @@ std::filesystem::path get_user_memcard_dir() {
return get_user_game_dir() / "jak1" / "saves";
}
std::string get_project_path() {
struct {
bool initialized = false;
std::filesystem::path path_to_data;
} gFilePathInfo;
/*!
* Get the path to the current executable.
*/
std::string get_current_executable_path() {
#ifdef _WIN32
char buffer[FILENAME_MAX];
GetModuleFileNameA(NULL, buffer, FILENAME_MAX);
std::string::size_type pos =
std::string(buffer).rfind("jak-project"); // Strip file path down to \jak-project\ directory
return std::string(buffer).substr(
0, pos + 11); // + 12 to include "\jak-project" in the returned filepath
return std::string(buffer);
#else
// do Linux stuff
char buffer[FILENAME_MAX + 1];
@@ -71,13 +76,66 @@ std::string get_project_path() {
FILENAME_MAX); // /proc/self acts like a "virtual folder" containing
// information about the current process
buffer[len] = '\0';
std::string::size_type pos =
std::string(buffer).rfind("jak-project"); // Strip file path down to /jak-project/ directory
return std::string(buffer).substr(
0, pos + 11); // + 12 to include "/jak-project" in the returned filepath
return std::string(buffer);
#endif
}
/*!
* See if the current executable is somewhere in jak-project/. If so, return the path to jak-project
*/
std::optional<std::string> try_get_jak_project_path() {
std::string my_path = get_current_executable_path();
std::string::size_type pos =
std::string(my_path).rfind("jak-project"); // Strip file path down to /jak-project/ directory
if (pos == std::string::npos) {
return {};
}
return std::string(my_path).substr(
0, pos + 11); // + 12 to include "/jak-project" in the returned filepath
}
std::optional<std::filesystem::path> try_get_data_dir() {
std::filesystem::path my_path = get_current_executable_path();
auto data_dir = my_path.parent_path() / "data";
if (std::filesystem::exists(data_dir) && std::filesystem::is_directory(data_dir)) {
return data_dir;
} else {
return {};
}
}
bool setup_project_path() {
if (gFilePathInfo.initialized) {
return true;
}
auto data_path = try_get_data_dir();
if (data_path) {
gFilePathInfo.path_to_data = *data_path;
gFilePathInfo.initialized = true;
fmt::print("Using data path: {}\n", data_path->string());
return true;
}
auto development_repo_path = try_get_jak_project_path();
if (development_repo_path) {
gFilePathInfo.path_to_data = *development_repo_path;
gFilePathInfo.initialized = true;
fmt::print("Using development repo path: {}\n", *development_repo_path);
return true;
}
fmt::print("Failed to initialize project path.\n");
return false;
}
std::filesystem::path get_jak_project_dir() {
ASSERT(gFilePathInfo.initialized);
return gFilePathInfo.path_to_data;
}
std::string get_file_path(const std::vector<std::string>& input) {
// TODO - clean this behaviour up, it causes unexpected behaviour when working with files
// the project path should be explicitly provided by whatever if needed
@@ -87,21 +145,12 @@ std::string get_file_path(const std::vector<std::string>& input) {
return input.at(0);
}
std::string currentPath = file_util::get_project_path();
char dirSeparator;
#ifdef _WIN32
dirSeparator = '\\';
#else
dirSeparator = '/';
#endif
std::string filePath = currentPath;
for (int i = 0; i < int(input.size()); i++) {
filePath = filePath + dirSeparator + input[i];
auto current_path = file_util::get_jak_project_dir();
for (auto& str : input) {
current_path /= str;
}
return filePath;
return current_path.string();
}
bool create_dir_if_needed(const std::string& path) {
+4 -3
View File
@@ -8,6 +8,7 @@
#include <string>
#include <vector>
#include <filesystem>
#include <optional>
#include "common/common_types.h"
namespace fs = std::filesystem;
@@ -17,12 +18,12 @@ std::filesystem::path get_user_home_dir();
std::filesystem::path get_user_game_dir();
std::filesystem::path get_user_settings_dir();
std::filesystem::path get_user_memcard_dir();
std::string get_project_path();
std::string get_file_path(const std::vector<std::string>& input);
std::filesystem::path get_jak_project_dir();
bool create_dir_if_needed(const std::string& path);
bool create_dir_if_needed_for_file(const std::string& path);
bool setup_project_path();
std::string get_file_path(const std::vector<std::string>& path);
void write_binary_file(const std::string& name, const void* data, size_t size);
void write_rgba_png(const std::string& name, void* data, int w, int h);
void write_text_file(const std::string& file_name, const std::string& text);
+11
View File
@@ -93,3 +93,14 @@ target_link_libraries(decompiler
common
lzokay
fmt)
add_executable(extractor
extractor/main.cpp)
target_link_libraries(extractor
decomp
common
lzokay
fmt
compiler)
+129
View File
@@ -0,0 +1,129 @@
#include "third-party/fmt/core.h"
#include "common/util/FileUtil.h"
#include "decompiler/Disasm/OpcodeInfo.h"
#include "decompiler/ObjectFile/ObjectFileDB.h"
#include "decompiler/level_extractor/extract_level.h"
#include "decompiler/config.h"
#include "goalc/compiler/Compiler.h"
void setup_global_decompiler_stuff() {
file_util::init_crc();
decompiler::init_opcode_info();
file_util::setup_project_path();
}
int main(int argc, char** argv) {
using namespace decompiler;
fmt::print("OpenGOAL Level Extraction Tool\n");
if (argc != 2) {
fmt::print(" usage: extractor <path-to-jak1-files>\n");
return 1;
}
// todo: print revision here.
setup_global_decompiler_stuff();
std::filesystem::path jak1_input_files(argv[1]);
// make sure the input looks right
if (!std::filesystem::exists(jak1_input_files)) {
fmt::print("Error: input folder {} does not exist\n", jak1_input_files.string());
return 1;
}
if (!std::filesystem::is_directory(jak1_input_files)) {
fmt::print("Error: input folder {} is not a folder.\n", jak1_input_files.string());
return 1;
}
if (!std::filesystem::exists(jak1_input_files / "DGO")) {
fmt::print("Error: input folder doesn't have a DGO folder. Is this the right input?\n");
return 1;
}
Config config = read_config_file(
(file_util::get_jak_project_dir() / "decompiler" / "config" / "jak1_ntsc_black_label.jsonc")
.string(),
{});
std::vector<std::string> dgos, objs;
// grab all DGOS we need (level + common)
for (const auto& dgo_name : config.dgo_names) {
std::string common_name = "GAME.CGO";
if (dgo_name.length() > 3 && dgo_name.substr(dgo_name.length() - 3) == "DGO") {
// ends in DGO, it's a level
dgos.push_back((jak1_input_files / dgo_name).string());
} else if (dgo_name.length() >= common_name.length() &&
dgo_name.substr(dgo_name.length() - common_name.length()) == common_name) {
// it's COMMON.CGO, we need that too.
dgos.push_back((jak1_input_files / dgo_name).string());
}
}
// grab all the object files we need (just text)
for (const auto& obj_name : config.object_file_names) {
if (obj_name.length() > 3 && obj_name.substr(obj_name.length() - 3) == "TXT") {
// ends in DGO, it's a level
objs.push_back((jak1_input_files / obj_name).string());
}
}
// set up objects
ObjectFileDB db(dgos, config.obj_file_name_map_file, objs, {}, config);
// save object files
auto out_folder = (file_util::get_jak_project_dir() / "decompiler_out" / "jak1").string();
auto raw_obj_folder = file_util::combine_path(out_folder, "raw_obj");
file_util::create_dir_if_needed(raw_obj_folder);
db.dump_raw_objects(raw_obj_folder);
// analyze object file link data
db.process_link_data(config);
db.find_code(config);
db.process_labels();
// text files
{
auto result = db.process_game_text_files(config);
if (!result.empty()) {
file_util::write_text_file(file_util::get_file_path({"assets", "game_text.txt"}), result);
}
}
// textures
decompiler::TextureDB tex_db;
file_util::write_text_file(file_util::get_file_path({"assets", "tpage-dir.txt"}),
db.process_tpages(tex_db));
// texture replacements
auto replacements_path = file_util::get_file_path({"texture_replacements"});
if (std::filesystem::exists(replacements_path)) {
tex_db.replace_textures(replacements_path);
}
// game count
{
auto result = db.process_game_count_file();
if (!result.empty()) {
file_util::write_text_file(file_util::get_file_path({"assets", "game_count.txt"}), result);
}
}
// levels
{
extract_common(db, tex_db, "GAME.CGO");
for (auto& lev : config.levels_to_extract) {
extract_from_level(db, tex_db, lev, config.hacks, config.rip_levels);
}
}
// Compile!
Compiler compiler;
compiler.make_system().set_constant("*iso-data*", jak1_input_files.string());
compiler.make_system().set_constant("*use-iso-data-path*", true);
compiler.make_system().load_project_file(
(file_util::get_jak_project_dir() / "goal_src" / "game.gp").string());
compiler.run_front_end_on_string("(mi)");
system((file_util::get_jak_project_dir() / "../gk").string().c_str());
}
+3 -1
View File
@@ -13,9 +13,11 @@
#include "common/util/diff.h"
int main(int argc, char** argv) {
fmt::print("[Mem] Size of linked word: {}\n", sizeof(decompiler::LinkedWord));
fmt::print("[Mem] Top of main: {} MB\n", get_peak_rss() / (1024 * 1024));
using namespace decompiler;
if (!file_util::setup_project_path()) {
return 1;
}
lg::set_file(file_util::get_file_path({"log/decompiler.txt"}));
lg::set_file_level(lg::level::info);
lg::set_stdout_level(lg::level::info);
+2 -1
View File
@@ -156,7 +156,8 @@ static std::shared_ptr<GfxDisplay> gl_make_main_display(int width,
return NULL;
}
std::string image_path = fmt::format("{}/game/assets/appicon.png", file_util::get_project_path());
std::string image_path =
(file_util::get_jak_project_dir() / "game" / "assets" / "appicon.png").string();
GLFWimage images[1];
images[0].pixels =
+8
View File
@@ -81,6 +81,14 @@ void kmachine_init_globals() {
* Modified to use std::string, and removed call to fflush.
*/
void InitParms(int argc, const char* const* argv) {
// Modified default settings:
if (argc == 1) {
DiskBoot = 1;
isodrv = fakeiso;
modsrc = 0;
reboot = 0;
}
for (int i = 1; i < argc; i++) {
std::string arg = argv[i];
// DVD Settings
+4
View File
@@ -50,6 +50,10 @@ int main(int argc, char** argv) {
}
}
if (!file_util::setup_project_path()) {
return 1;
}
gStartTime = time(0);
init_discord_rpc();
+1 -1
View File
@@ -147,7 +147,7 @@ FileRecord* FS_FindIN(const char* iso_name) {
static const char* get_file_path(FileRecord* fr) {
ASSERT(fr->location < fake_iso_entry_count);
static char path_buffer[1024];
strcpy(path_buffer, file_util::get_project_path().c_str());
strcpy(path_buffer, file_util::get_jak_project_dir().string().c_str());
strcat(path_buffer, "/");
strcat(path_buffer, fake_iso_entries[fr->location].file_path);
return path_buffer;
+12 -5
View File
@@ -131,8 +131,15 @@
)
)
(defun get-iso-data-path ()
(if *use-iso-data-path*
(string-append *iso-data* "/")
(string-append "iso_data/" *game-directory* "/")
)
)
(defun copy-iso-file (name subdir ext)
(let* ((path (string-append "iso_data/" *game-directory* subdir name ext))
(let* ((path (string-append (get-iso-data-path) subdir name ext))
(out-name (string-append "out/iso/" name ext)))
(defstep :in path
:tool 'copy
@@ -222,22 +229,22 @@
)
;; the TWEAKVAL file
(defstep :in (string-append "iso_data/" *game-directory* "MUS/TWEAKVAL.MUS")
(defstep :in (string-append (get-iso-data-path) "MUS/TWEAKVAL.MUS")
:tool 'copy
:out '("out/iso/TWEAKVAL.MUS"))
;; the VAGDIR file
(defstep :in (string-append "iso_data/" *game-directory* "VAG/VAGDIR.AYB")
(defstep :in (string-append (get-iso-data-path) "VAG/VAGDIR.AYB")
:tool 'copy
:out '("out/iso/VAGDIR.AYB"))
;; the save icon file
(defstep :in (string-append "iso_data/" *game-directory* "DRIVERS/SAVEGAME.ICO")
(defstep :in (string-append (get-iso-data-path) "DRIVERS/SAVEGAME.ICO")
:tool 'copy
:out '("out/iso/SAVEGAME.ICO"))
;; the loading screen file
(defstep :in (string-append "iso_data/" *game-directory* "DRIVERS/SCREEN1.USA")
(defstep :in (string-append (get-iso-data-path) "DRIVERS/SCREEN1.USA")
:tool 'copy
:out '("out/iso/SCREEN1.USA"))
+1
View File
@@ -66,6 +66,7 @@ class Compiler {
Replxx::colors_t& colors,
std::vector<std::pair<std::string, Replxx::Color>> const& user_data);
bool knows_object_file(const std::string& name);
MakeSystem& make_system() { return m_make; }
private:
TypeSystem m_ts;
+3 -1
View File
@@ -26,7 +26,9 @@ void setup_logging(bool verbose) {
int main(int argc, char** argv) {
(void)argc;
(void)argv;
if (!file_util::setup_project_path()) {
return 1;
}
std::string argument;
std::string username = "#f";
bool verbose = false;
+12
View File
@@ -9,6 +9,7 @@
#include "common/util/Timer.h"
#include "goalc/make/Tools.h"
#include "common/util/FileUtil.h"
std::string MakeStep::print() const {
std::string result = fmt::format("Tool {} with inputs", tool);
@@ -55,6 +56,9 @@ MakeSystem::MakeSystem() {
m_goos.set_global_variable_to_symbol("ASSETS", "#t");
set_constant("*iso-data*", file_util::get_file_path({"iso_data"}));
set_constant("*use-iso-data-path*", false);
add_tool<DgoTool>();
add_tool<TpageDirTool>();
add_tool<CopyTool>();
@@ -365,3 +369,11 @@ bool MakeSystem::make(const std::string& target, bool force, bool verbose) {
make_timer.getSeconds());
return true;
}
void MakeSystem::set_constant(const std::string& name, const std::string& value) {
m_goos.set_global_variable_by_name(name, goos::StringObject::make_new(value));
}
void MakeSystem::set_constant(const std::string& name, bool value) {
m_goos.set_global_variable_to_symbol(name, value ? "#t" : "#f");
}
+2
View File
@@ -35,6 +35,8 @@ class MakeSystem {
bool make(const std::string& target, bool force, bool verbose);
void add_tool(std::shared_ptr<Tool> tool);
void set_constant(const std::string& name, const std::string& value);
void set_constant(const std::string& name, bool value);
template <typename T>
void add_tool() {
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env bash
set -e
DEST=${1}
SOURCE=${2}
mkdir -p $DEST
cp $SOURCE/build/game/gk $DEST
cp $SOURCE/build/goalc/goalc $DEST
cp $SOURCE/build/decompiler/extractor $DEST
strip $DEST/gk
strip $DEST/goalc
strip $DEST/extractor
mkdir -p $DEST/data
mkdir -p $DEST/data/decompiler/
mkdir -p $DEST/data/assets
mkdir -p $DEST/data/game
mkdir -p $DEST/data/log
mkdir -p $DEST/data/game/graphics/opengl_renderer/
cp -r $SOURCE/decompiler/config $DEST/data/decompiler/
cp -r $SOURCE/goal_src $DEST/data
cp -r $SOURCE/game/assets $DEST/data/game/
cp -r $SOURCE/game/graphics/opengl_renderer/shaders $DEST/data/game/graphics/opengl_renderer
+14 -7
View File
@@ -65,8 +65,9 @@ struct OfflineTestConfig {
* Read and parse the json config file, config.json, located in test/offline
*/
OfflineTestConfig parse_config() {
auto json_file_path = file_util::get_file_path({"test", "offline", "config.jsonc"});
auto json = parse_commented_json(file_util::read_text_file(json_file_path), json_file_path);
auto json_file_path = file_util::get_jak_project_dir() / "test" / "offline" / "config.jsonc";
auto json = parse_commented_json(file_util::read_text_file(json_file_path.string()),
json_file_path.string());
OfflineTestConfig result;
result.dgos = json["dgos"].get<std::vector<std::string>>();
result.skip_compile_files = json["skip_compile_files"].get<std::unordered_set<std::string>>();
@@ -98,8 +99,8 @@ std::vector<DecompilerFile> find_files(const std::vector<std::string>& dgos) {
std::vector<DecompilerFile> result;
std::unordered_map<std::string, fs::path> files_with_ref;
for (auto& p : fs::recursive_directory_iterator(
file_util::get_file_path({"test", "decompiler", "reference"}))) {
for (auto& p : fs::recursive_directory_iterator(file_util::get_jak_project_dir() / "test" /
"decompiler" / "reference")) {
if (p.is_regular_file()) {
std::string file_name = fs::path(p.path()).replace_extension().filename().string();
if (file_name.find("_REF") == std::string::npos) {
@@ -114,7 +115,8 @@ std::vector<DecompilerFile> find_files(const std::vector<std::string>& dgos) {
// use the all_objs.json file to place them in the correct build order
auto j = parse_commented_json(
file_util::read_text_file(file_util::get_file_path({"goal_src", "build", "all_objs.json"})),
file_util::read_text_file(
(file_util::get_jak_project_dir() / "goal_src" / "build" / "all_objs.json").string()),
"all_objs.json");
std::unordered_set<std::string> matched_files;
@@ -180,7 +182,9 @@ Decompiler setup_decompiler(const std::vector<DecompilerFile>& files,
file_util::init_crc();
decompiler::init_opcode_info();
dc.config = std::make_unique<decompiler::Config>(decompiler::read_config_file(
file_util::get_file_path({"decompiler", "config", "jak1_ntsc_black_label.jsonc"}), {}));
(file_util::get_jak_project_dir() / "decompiler" / "config" / "jak1_ntsc_black_label.jsonc")
.string(),
{}));
// modify the config
std::unordered_set<std::string> object_files;
@@ -195,7 +199,7 @@ Decompiler setup_decompiler(const std::vector<DecompilerFile>& files,
std::vector<std::string> dgo_paths;
if (args.iso_data_path.empty()) {
for (auto& x : offline_config.dgos) {
dgo_paths.push_back(file_util::get_file_path({"iso_data/jak1", x}));
dgo_paths.push_back((file_util::get_jak_project_dir() / "iso_data" / "jak1").string());
}
} else {
for (auto& x : offline_config.dgos) {
@@ -343,6 +347,9 @@ bool compile(Decompiler& dc,
int main(int argc, char* argv[]) {
fmt::print("Offline Decompiler Test 2\n");
lg::initialize();
if (!file_util::setup_project_path()) {
return 1;
}
fmt::print("Reading config...\n");
auto args = parse_args(argc, argv);
+1
View File
@@ -20,6 +20,7 @@
int main(int argc, char** argv) {
// hopefully get a debug print on github actions
setup_cpu_info();
file_util::setup_project_path();
lg::initialize();
::testing::InitGoogleTest(&argc, argv);