Files
jak-project/goalc/make/MakeSystem.cpp
T
Hat Kid 710f3ac117 custom levels: etie and build actor support for jak2/3 (#3851)
Custom levels for Jak 2/3 now support envmapped TIE geometry. The TIE
extract was also changed to ignore materials that have the specular flag
set, but are missing a roughness texture.

Jak 2/3 now also support the `build-actor` tool.

The `build-custom-level` and `build-actor` macros now have a few new
options:

- Both now have a `force-run` option (`#f` by default) that, when set to
`#t`, will always run level/art group generation even if the output
files are up to date.
- `build-custom-level` has a `gen-fr3` option (`#t` by default) that,
when set to `#f`, will skip generating the FR3 file for the custom level
and only generate the GOAL level file to skip the potentially slow
process of finding and adding art groups and textures. Useful for when
you want to temporarily edit only the GOAL side of the level (such as
entity placement, etc.).
- `build-actor` has a `texture-bucket` option (default 0) which will
determine what DMA sink group the model will be placed in, which is
useful to determine the draw order of the model. Previously, this was
omitted, resulting in shadows not drawing over custom actors because the
actors were put in a bucket that is drawn after shadows (this behavior
can be restored with `:texture-bucket #f`).
2025-02-01 18:04:26 +01:00

562 lines
19 KiB
C++

#include "MakeSystem.h"
#include "common/goos/ParseHelpers.h"
#include "common/log/log.h"
#include "common/util/FileUtil.h"
#include "common/util/Timer.h"
#include "common/util/string_util.h"
#include "goalc/make/CompilerReport.h"
#include "goalc/make/Tools.h"
#include "fmt/color.h"
#include "fmt/core.h"
std::string MakeStep::print() const {
std::string result = fmt::format("Tool {} with inputs", tool);
int i = 0;
for (auto& in : input) {
if (i++ > 0) {
result += ", ";
}
result += fmt::format("\"{}\"", in);
}
result += " and deps:\n ";
for (auto& dep : deps) {
result += dep;
result += '\n';
result += ' ';
}
result.pop_back();
result += fmt::format("will produce outputs:\n ");
for (auto& o : outputs) {
result += o;
result += '\n';
result += ' ';
}
result.pop_back();
return result;
}
MakeSystem::MakeSystem(const std::optional<REPL::Config> repl_config, const std::string& username)
: m_goos(username), m_repl_config(repl_config) {
m_goos.register_form("defstep", [=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_defstep(obj, args, env);
});
m_goos.register_form("basename", [=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_basename(obj, args, env);
});
m_goos.register_form("stem", [=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_stem(obj, args, env);
});
m_goos.register_form("get-gsrc-path", [=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_get_gsrc_path(obj, args, env);
});
m_goos.register_form("map-path!", [=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_map_path(obj, args, env);
});
m_goos.register_form("set-output-prefix",
[=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_set_output_prefix(obj, args, env);
});
m_goos.register_form("set-gsrc-folder!",
[=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_set_gsrc_folder(obj, args, env);
});
m_goos.register_form("get-gsrc-folder", [=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_get_gsrc_folder(obj, args, env);
});
m_goos.register_form("get-game-version-folder",
[=](const goos::Object& obj, goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
return handle_get_game_version_folder(obj, args, env);
});
m_goos.set_global_variable_to_symbol("ASSETS", "#t");
if (m_repl_config && !m_repl_config->iso_path.empty()) {
set_constant("*iso-data*",
file_util::get_iso_dir_for_game(m_repl_config->game_version).string());
set_constant("*use-iso-data-path*", true);
} else {
set_constant("*iso-data*", file_util::get_file_path({"iso_data"}));
set_constant("*use-iso-data-path*", false);
}
add_tool<DgoTool>();
add_tool<TpageDirTool>();
add_tool<CopyTool>();
add_tool<GameCntTool>();
add_tool<GroupTool>();
add_tool<TextTool>();
add_tool<SubtitleTool>();
add_tool<SubtitleV2Tool>();
add_tool<BuildLevelTool>();
add_tool<BuildLevel2Tool>();
add_tool<BuildLevel3Tool>();
add_tool<BuildActorTool>();
add_tool<BuildActor2Tool>();
add_tool<BuildActor3Tool>();
}
/*!
* Load a project file, clearing any project info previously loaded.
*/
void MakeSystem::load_project_file(const std::string& file_path) {
Timer timer;
// clear the previous project
clear_project();
// read the file
auto data = m_goos.reader.read_from_file({file_path});
// interpret it, which will call various handlers.
m_goos.eval(data, m_goos.global_environment.as_env_ptr());
lg::debug("Loaded project {} with {} steps in {} ms\n", file_path, m_output_to_step.size(),
(int)timer.getMs());
m_loaded_projects.push_back(file_path);
}
goos::Object MakeSystem::handle_defstep(const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
m_goos.eval_args(&args, env);
va_check(form, args, {},
{{"out", {true, {goos::ObjectType::PAIR}}},
{"tool", {true, {goos::ObjectType::SYMBOL}}},
{"in", {false, {}}},
{"dep", {false, {}}},
{"arg", {false, {}}}});
auto step = std::make_shared<MakeStep>();
goos::for_each_in_list(args.get_named("out"), [&](const goos::Object& obj) {
step->outputs.push_back(m_path_map.apply_remaps(obj.as_string()->data));
});
step->tool = args.get_named("tool").as_symbol().name_ptr;
if (m_tools.find(step->tool) == m_tools.end()) {
throw std::runtime_error(fmt::format("The tool {} is unknown.", step->tool));
}
if (args.has_named("in")) {
const auto& in = args.get_named("in");
if (in.is_pair()) {
step->input.clear();
goos::for_each_in_list(in, [&](const goos::Object& o) {
step->input.push_back(m_path_map.apply_remaps(o.as_string()->data));
});
} else {
step->input = {m_path_map.apply_remaps(in.as_string()->data)};
}
}
if (args.has_named("dep")) {
goos::for_each_in_list(args.get_named("dep"), [&](const goos::Object& obj) {
step->deps.push_back(m_path_map.apply_remaps(obj.as_string()->data));
});
}
if (args.has_named("arg")) {
step->arg = args.get_named("arg");
} else {
step->arg = goos::Object::make_empty_list();
}
for (auto& output : step->outputs) {
auto existing = m_output_to_step.find(output);
if (existing != m_output_to_step.end()) {
throw std::runtime_error(fmt::format("There are multiple ways to make output {}:\n{}\n{}\n",
output, step->print(), existing->second->print()));
}
m_output_to_step.insert({output, step});
}
return goos::Object::make_empty_list();
}
/*!
* Fully clear the state of the project.
*
*/
void MakeSystem::clear_project() {
m_loaded_projects.clear();
m_output_to_step.clear();
}
void MakeSystem::va_check(
const goos::Object& form,
const goos::Arguments& args,
const std::vector<std::optional<goos::ObjectType>>& unnamed,
const std::unordered_map<std::string, std::pair<bool, std::optional<goos::ObjectType>>>&
named) {
std::string err;
if (!goos::va_check(args, unnamed, named, &err)) {
m_goos.throw_eval_error(form, err);
}
}
goos::Object MakeSystem::handle_basename(const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
m_goos.eval_args(&args, env);
va_check(form, args, {goos::ObjectType::STRING}, {});
fs::path input(args.unnamed.at(0).as_string()->data);
return goos::StringObject::make_new(input.filename().string());
}
goos::Object MakeSystem::handle_stem(const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
m_goos.eval_args(&args, env);
va_check(form, args, {goos::ObjectType::STRING}, {});
fs::path input(args.unnamed.at(0).as_string()->data);
return goos::StringObject::make_new(input.stem().string());
}
goos::Object MakeSystem::handle_get_gsrc_path(const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
if (m_gsrc_folder.empty()) {
throw std::runtime_error("`set-gsrc-folder!` was not called before a `get-gsrc-path`");
}
m_goos.eval_args(&args, env);
va_check(form, args, {goos::ObjectType::STRING}, {});
const auto& file_name = args.unnamed.at(0).as_string()->data;
if (m_gsrc_files.count(file_name) != 0) {
return goos::StringObject::make_new(m_gsrc_files.at(file_name));
} else {
return goos::Object::make_symbol(&m_goos.reader.symbolTable, "#f");
}
}
goos::Object MakeSystem::handle_map_path(const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
m_goos.eval_args(&args, env);
va_check(form, args, {goos::ObjectType::STRING, goos::ObjectType::STRING}, {});
auto old_path = args.unnamed.at(0).as_string()->data;
if (old_path.empty() || old_path[0] != '$') {
throw std::runtime_error(fmt::format("Invalid path remap {}, must start with $", old_path));
}
auto new_path = args.unnamed.at(1).as_string()->data;
m_path_map.path_remap[old_path] = new_path;
return goos::Object::make_empty_list();
}
goos::Object MakeSystem::handle_set_output_prefix(
const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
m_goos.eval_args(&args, env);
va_check(form, args, {goos::ObjectType::STRING}, {});
m_path_map.output_prefix = args.unnamed.at(0).as_string()->data;
return goos::Object::make_empty_list();
}
goos::Object MakeSystem::handle_set_gsrc_folder(
const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
m_goos.eval_args(&args, env);
va_check(form, args, {goos::ObjectType::STRING}, {});
const auto& folder = args.unnamed.at(0).as_string()->data;
m_gsrc_folder = str_util::split(folder, '/');
m_gsrc_files.clear();
auto folder_scan = file_util::get_file_path(m_gsrc_folder);
auto src_files = file_util::find_files_recursively(folder_scan, std::regex(".*\\.gc"));
for (const auto& path : src_files) {
auto name = file_util::base_name_no_ext(path.string());
auto gsrc_path =
file_util::convert_to_unix_path_separators(file_util::split_path_at(path, m_gsrc_folder));
// TODO - this is only "safe" because the current OpenGOAL system requires globally unique
// file names
m_gsrc_files.emplace(name, gsrc_path);
}
return args.unnamed.at(0);
}
goos::Object MakeSystem::handle_get_gsrc_folder(
const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
m_goos.eval_args(&args, env);
va_check(form, args, {}, {});
std::string out;
int idx = 0;
for (const auto& part : m_gsrc_folder) {
if (idx++ > 0) {
out += '/';
}
out += part;
}
return goos::StringObject::make_new(out);
}
goos::Object MakeSystem::handle_get_game_version_folder(
const goos::Object& form,
goos::Arguments& args,
const std::shared_ptr<goos::EnvironmentObject>& env) {
m_goos.eval_args(&args, env);
va_check(form, args, {}, {});
if (m_repl_config) {
return goos::StringObject::make_new(m_repl_config->game_version_folder);
} else {
return goos::StringObject::make_new("");
}
}
void MakeSystem::get_dependencies(const std::string& master_target,
const std::string& output,
std::vector<std::string>* result,
std::unordered_set<std::string>* result_set) const {
// fmt::print(output + "\n");
if (result_set->find(output) != result_set->end()) {
return;
}
const auto& rule_it = m_output_to_step.find(output);
if (rule_it == m_output_to_step.end()) {
throw std::runtime_error(
fmt::format("No rule to make {}, required for {}\n", output, master_target));
}
// what deps do we need?
for (auto& dep : rule_it->second->deps) {
get_dependencies(master_target, dep, result, result_set);
}
const auto& rule = rule_it->second;
for (auto& dep : m_tools.at(rule->tool)
->get_additional_dependencies(
{rule->input, rule->deps, rule->outputs, rule->arg}, m_path_map)) {
get_dependencies(master_target, dep, result, result_set);
}
result->push_back(output);
for (auto& op : rule->outputs) {
result_set->insert(op);
}
}
std::vector<std::string> MakeSystem::get_dependencies(const std::string& target) const {
Timer timer;
std::vector<std::string> result;
std::unordered_set<std::string> added_deps;
get_dependencies(target, target, &result, &added_deps);
lg::print("Successfully found all {} dependencies for target in {:.3f}s\n", result.size(),
timer.getSeconds());
return result;
}
void MakeSystem::add_tool(std::shared_ptr<Tool> tool) {
auto& name = tool->name();
ASSERT(m_tools.find(name) == m_tools.end());
m_tools[name] = tool;
}
std::vector<std::string> MakeSystem::filter_dependencies(const std::vector<std::string>& all_deps) {
Timer timer;
std::vector<std::string> result;
std::unordered_set<std::string> stale_deps;
std::unordered_set<std::string> compiler_sym_needed;
for (auto& to_make : all_deps) {
auto& rule = m_output_to_step.at(to_make);
auto& tool = m_tools.at(rule->tool);
const ToolInput task = {rule->input, rule->deps, rule->outputs, rule->arg};
bool added = false;
if (tool->needs_run(task, m_path_map)) {
result.push_back(to_make);
stale_deps.insert(to_make);
added = true;
}
if (!added) {
// check transitive dependencies
for (auto& dep : rule->deps) {
if (stale_deps.find(dep) != stale_deps.end()) {
result.push_back(to_make);
stale_deps.insert(to_make);
added = true;
break;
}
}
}
if (!added) {
// check transitive dependencies
for (auto& dep : tool->get_additional_dependencies(task, m_path_map)) {
if (stale_deps.find(dep) != stale_deps.end()) {
result.push_back(to_make);
stale_deps.insert(to_make);
added = true;
break;
}
}
}
}
lg::print("Found that {} of {} targets do need rebuilding in {:.3f}s\n", result.size(),
all_deps.size(), timer.getSeconds());
return result;
}
namespace {
void print_input(const std::vector<std::string>& in, char end) {
int i = 0;
std::string all_names;
for (auto& name : in) {
if (i++ > 0) {
all_names += ", ";
}
all_names += name;
}
if (all_names.length() > 70) {
lg::print("{}...{}", all_names.substr(0, 70 - 3), end);
} else {
lg::print("{}{}{}", all_names, std::string(70 - all_names.length(), ' '), end);
}
}
} // namespace
bool MakeSystem::make(const std::string& target_in, bool force, bool verbose, bool gen_report) {
std::string target = m_path_map.apply_remaps(target_in);
auto deps = get_dependencies(target);
// lg::print("All deps:\n");
// for (auto& dep : deps) {
// lg::print("{}\n", dep);
// }
if (!force) {
deps = filter_dependencies(deps);
}
// lg::print("Filt deps:\n");
// for (auto& dep : filtered_deps) {
// lg::print("{}\n", dep);
// }
fs::path report_path;
std::string report_output;
std::string report_contents;
if (gen_report) {
report_path = file_util::get_jak_project_dir() / "goalc-report.html";
lg::print("Will save compiler report to - {}", report_path.string());
// Check if a report is already there, if it is, we'll append to it instead of overwriting it
if (file_util::file_exists(report_path.string())) {
report_output = file_util::read_text_file(report_path);
} else {
report_output = compiler_report_base;
}
report_contents += fmt::format("tests.push({{'name': \"Test - {}\",'files': {{",
str_util::current_isotimestamp());
}
Timer make_timer;
lg::print("Building {} targets...\n", deps.size());
int i = 0;
for (auto& to_make : deps) {
Timer step_timer;
auto& rule = m_output_to_step.at(to_make);
auto& tool = m_tools.at(rule->tool);
int percent = (100.0 * (1 + (i++)) / (deps.size())) + 0.5;
if (verbose) {
lg::print("[{:3d}%] [{:8s}] {}{}\n", percent, tool->name(), rule->input.at(0),
rule->input.size() > 1 ? ", ..." : "");
} else {
lg::print("[{:3d}%] [{:8s}] ", percent, tool->name());
print_input(rule->input, '\r');
}
bool success = false;
try {
success = tool->run({rule->input, rule->deps, rule->outputs, rule->arg}, m_path_map);
} catch (std::exception& e) {
lg::print("\n");
lg::print("Error: {}\n", e.what());
}
if (!success) {
lg::print("Build failed on {}{}\n", rule->input.at(0), rule->input.size() > 1 ? ", ..." : "");
throw std::runtime_error("Build failed.");
return false;
}
const auto seconds = step_timer.getSeconds();
if (verbose) {
if (seconds > 0.05) {
lg::print(fg(fmt::color::yellow), " {:.3f}\n", seconds);
} else {
lg::print(" {:.3f}\n", seconds);
}
} else {
if (seconds > 0.05) {
lg::print("[{:3d}%] [{:8s}] ", percent, tool->name());
lg::print(fg(fmt::color::yellow), "{:.3f} ", seconds);
print_input(rule->input, '\n');
} else {
lg::print("[{:3d}%] [{:8s}] {:.3f} ", percent, tool->name(), seconds);
print_input(rule->input, '\n');
}
}
if (gen_report) {
report_contents +=
fmt::format("\"{}\": {}{}", str_util::split_string(rule->input.at(0), "/").back(),
seconds, i == deps.size() ? "" : ",");
}
}
lg::print("\nSuccessfully built all {} targets in {:.3f}s\n", deps.size(),
make_timer.getSeconds());
if (gen_report) {
report_contents += fmt::format("}}, 'total': {}}});", make_timer.getSeconds());
str_util::replace(report_output, "// DATA ENDS\n",
fmt::format("{}\n// DATA ENDS\n", report_contents));
file_util::write_text_file(report_path, report_output);
lg::print("Saved report to: {}\n", report_path.string());
}
return true;
}
void MakeSystem::set_constant(const std::string& name, const std::string& value) {
m_goos.set_global_variable_by_name(name, goos::StringObject::make_new(value));
}
void MakeSystem::set_constant(const std::string& name, bool value) {
m_goos.set_global_variable_to_symbol(name, value ? "#t" : "#f");
}
void MakeSystem::set_constant(const std::string& name, int value) {
m_goos.set_global_variable_to_int(name, value);
}