mirror/jak-project
1
0
Fork 0
mirror of https://github.com/open-goal/jak-project synced 2026-05-28 08:25:56 -04:00
Code Issues Packages Projects Releases Wiki Activity

Loader improvements (#1378)

Browse Source 
* tfrag3 data for merc2

* dma hooks for merc2

* start designing merc2 opengl, seems like the simple approach will be the best here

* before bone packing experiment

* fix up bones.gc

* use uniform buffer

* speedup, fix faces and eyes

* final fixes

* first pass at loader updates, tie is still bad

* temp

* improved loader

* run iop less often
This commit is contained in:
water111
2022-05-28 20:12:33 -04:00
committed by GitHub
parent e56b2e8d56
commit 2e31d82fb2
21 changed files with 1085 additions and 926 deletions
Download Patch File Download Diff File Expand all files Collapse all files
game/CMakeLists.txt
+2 -1
View File
@@ -98,6 +98,8 @@ set(RUNTIME_SOURCE
graphics/opengl_renderer/ocean/OceanNear_PS2.cpp
graphics/opengl_renderer/ocean/OceanTexture.cpp
graphics/opengl_renderer/ocean/OceanTexture_PC.cpp
graphics/opengl_renderer/loader/Loader.cpp
graphics/opengl_renderer/loader/LoaderStages.cpp
graphics/opengl_renderer/BucketRenderer.cpp
graphics/opengl_renderer/CollideMeshRenderer.cpp
graphics/opengl_renderer/debug_gui.cpp
@@ -105,7 +107,6 @@ set(RUNTIME_SOURCE
graphics/opengl_renderer/DirectRenderer2.cpp
graphics/opengl_renderer/EyeRenderer.cpp
graphics/opengl_renderer/dma_helpers.cpp
graphics/opengl_renderer/Loader.cpp
graphics/opengl_renderer/MercProgram.cpp
graphics/opengl_renderer/MercRenderer.cpp
graphics/opengl_renderer/opengl_utils.cpp
game/common/player_rpc_types.h
-5
View File
@@ -6,10 +6,5 @@
* Note that PLAY and PLAYER are different.
*/
#ifndef JAK1_PLAYER_RPC_TYPES_H
#define JAK1_PLAYER_RPC_TYPES_H
constexpr int PLAYER_RPC_ID = 0xdeb1;
constexpr int PLAYER_RPC_CHANNEL = 0;
#endif // JAK1_PLAYER_RPC_TYPES_H
game/graphics/opengl_renderer/BucketRenderer.h
+1 -1
View File
@@ -6,7 +6,7 @@
#include "game/graphics/opengl_renderer/Shader.h"
#include "game/graphics/texture/TexturePool.h"
#include "game/graphics/opengl_renderer/Profiler.h"
#include "game/graphics/opengl_renderer/Loader.h"
#include "game/graphics/opengl_renderer/loader/Loader.h"
#include "game/graphics/opengl_renderer/buckets.h"
struct LevelVis {
game/graphics/opengl_renderer/Loader.cpp
-732
View File
@@ -1,732 +0,0 @@
#include "Loader.h"
#include "common/util/Timer.h"
#include "common/util/FileUtil.h"
#include "common/util/compress.h"
namespace {
std::string uppercase_string(const std::string& s) {
std::string result;
for (auto c : s) {
result.push_back(toupper(c));
}
return result;
}
} // namespace
Loader::Loader() {
m_loader_thread = std::thread(&Loader::loader_thread, this);
}
Loader::~Loader() {
{
std::lock_guard<std::mutex> lk(m_loader_mutex);
m_want_shutdown = true;
m_loader_cv.notify_all();
}
m_loader_thread.join();
}
/*!
* Try to get a loaded level by name. It may fail and return nullptr.
* Getting a level will reset the counter for the level and prevent it from being kicked out
* for a little while.
*
* This is safe to call from the graphics thread
*/
const Loader::LevelData* Loader::get_tfrag3_level(const std::string& level_name) {
std::unique_lock<std::mutex> lk(m_loader_mutex);
const auto& existing = m_loaded_tfrag3_levels.find(level_name);
if (existing == m_loaded_tfrag3_levels.end()) {
return nullptr;
} else {
existing->second->frames_since_last_used = 0;
return existing->second.get();
}
}
/*!
* The game calls this to give the loader a hint on which levels we want.
* If the loader is not busy, it will begin loading the level.
* This should be called on every frame.
*/
void Loader::set_want_levels(const std::vector<std::string>& levels) {
std::unique_lock<std::mutex> lk(m_loader_mutex);
m_desired_levels = levels;
if (!m_level_to_load.empty()) {
// can't do anything, we're loading a level right now
return;
}
if (!m_initializing_tfrag3_levels.empty()) {
// can't do anything, we're initializing a level right now
return;
}
// loader isn't busy, try to load one of the requested levels.
for (auto& lev : levels) {
auto it = m_loaded_tfrag3_levels.find(lev);
if (it == m_loaded_tfrag3_levels.end()) {
// we haven't loaded it yet. Request this level to load and wake up the thread.
m_level_to_load = lev;
lk.unlock();
m_loader_cv.notify_all();
return;
}
}
}
/*!
* Get all levels that are in memory and used very recently.
*/
std::vector<Loader::LevelData*> Loader::get_in_use_levels() {
std::vector<Loader::LevelData*> result;
std::unique_lock<std::mutex> lk(m_loader_mutex);
for (auto& lev : m_loaded_tfrag3_levels) {
if (lev.second->frames_since_last_used < 5) {
result.push_back(lev.second.get());
}
}
return result;
}
/*!
* Loader function that runs in a completely separate thread.
* This is used for file I/O and unpacking.
*/
void Loader::loader_thread() {
while (!m_want_shutdown) {
std::unique_lock<std::mutex> lk(m_loader_mutex);
// this will keep us asleep until we've got a level to load.
m_loader_cv.wait(lk, [&] { return !m_level_to_load.empty() || m_want_shutdown; });
if (m_want_shutdown) {
return;
}
std::string lev = m_level_to_load;
// don't hold the lock while reading the file.
lk.unlock();
// simulate slower hard drive (so that the loader thread can lose to the game loads)
// std::this_thread::sleep_for(std::chrono::milliseconds(1500));
// load the fr3 file
Timer disk_timer;
auto data = file_util::read_binary_file(
file_util::get_file_path({fmt::format("assets/{}.fr3", uppercase_string(lev))}));
double disk_load_time = disk_timer.getSeconds();
// the FR3 files are compressed
Timer decomp_timer;
auto decomp_data = compression::decompress_zstd(data.data(), data.size());
double decomp_time = decomp_timer.getSeconds();
// Read back into the tfrag3::Level structure
Timer import_timer;
auto result = std::make_unique<tfrag3::Level>();
Serializer ser(decomp_data.data(), decomp_data.size());
result->serialize(ser);
double import_time = import_timer.getSeconds();
// and finally "unpack", which creates the vertex data we'll upload to the GPU
Timer unpack_timer;
for (auto& tie_tree : result->tie_trees) {
for (auto& tree : tie_tree) {
tree.unpack();
}
}
for (auto& t_tree : result->tfrag_trees) {
for (auto& tree : t_tree) {
tree.unpack();
}
}
for (auto& shrub_tree : result->shrub_trees) {
shrub_tree.unpack();
}
fmt::print(
"------------> Load from file: {:.3f}s, import {:.3f}s, decomp {:.3f}s unpack {:.3f}s\n",
disk_load_time, import_time, decomp_time, unpack_timer.getSeconds());
// grab the lock again
lk.lock();
// move this level to "initializing" state.
m_initializing_tfrag3_levels[lev] = std::make_unique<LevelData>(); // reset load state
m_initializing_tfrag3_levels[lev]->level = std::move(result);
m_level_to_load = "";
m_file_load_done_cv.notify_all();
}
}
/*!
* Load a "common" FR3 file that has non-level textures.
* This should be called during initialization, before any threaded loading goes on.
*/
void Loader::load_common(TexturePool& tex_pool, const std::string& name) {
auto data =
file_util::read_binary_file(file_util::get_file_path({fmt::format("assets/{}.fr3", name)}));
auto decomp_data = compression::decompress_zstd(data.data(), data.size());
Serializer ser(decomp_data.data(), decomp_data.size());
m_common_level.level = std::make_unique<tfrag3::Level>();
m_common_level.level->serialize(ser);
for (auto& tex : m_common_level.level->textures) {
m_common_level.textures.push_back(add_texture(tex_pool, tex, true));
}
Timer tim;
init_merc(tim, m_common_level);
}
/*!
* Upload a texture to the GPU, and give it to the pool.
*/
u64 Loader::add_texture(TexturePool& pool, const tfrag3::Texture& tex, bool is_common) {
GLuint gl_tex;
glGenTextures(1, &gl_tex);
glBindTexture(GL_TEXTURE_2D, gl_tex);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, tex.w, tex.h, 0, GL_RGBA, GL_UNSIGNED_INT_8_8_8_8_REV,
tex.data.data());
glBindTexture(GL_TEXTURE_2D, 0);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, gl_tex);
glGenerateMipmap(GL_TEXTURE_2D);
float aniso = 0.0f;
glGetFloatv(GL_MAX_TEXTURE_MAX_ANISOTROPY, &aniso);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY, aniso);
if (tex.load_to_pool) {
TextureInput in;
in.debug_page_name = tex.debug_tpage_name;
in.debug_name = tex.debug_name;
in.w = tex.w;
in.h = tex.h;
in.gpu_texture = gl_tex;
in.common = is_common;
in.id = PcTextureId::from_combo_id(tex.combo_id);
in.src_data = (const u8*)tex.data.data();
pool.give_texture(in);
}
return gl_tex;
}
bool Loader::init_tfrag(Timer& timer, LevelData& data) {
if (data.tfrag_load_done) {
return true;
}
if (data.level->tfrag_trees.front().empty()) {
data.tfrag_load_done = true;
return true;
}
if (!data.tfrag_opengl_created) {
for (int geo = 0; geo < tfrag3::TFRAG_GEOS; geo++) {
auto& in_trees = data.level->tfrag_trees[geo];
for (auto& in_tree : in_trees) {
GLuint& tree_out = data.tfrag_vertex_data[geo].emplace_back();
glGenBuffers(1, &tree_out);
glBindBuffer(GL_ARRAY_BUFFER, tree_out);
glBufferData(GL_ARRAY_BUFFER,
in_tree.unpacked.vertices.size() * sizeof(tfrag3::PreloadedVertex), nullptr,
GL_STATIC_DRAW);
}
}
data.tfrag_opengl_created = true;
return false;
}
constexpr u32 CHUNK_SIZE = 32768;
u32 uploaded_bytes = 0;
while (true) {
const auto& tree = data.level->tfrag_trees[data.tfrag_next_geo][data.tfrag_next_tree];
u32 end_vert_in_tree = tree.unpacked.vertices.size();
// the number of vertices we'd need to finish the tree right now
size_t num_verts_left_in_tree = end_vert_in_tree - data.tfrag_next_vert;
size_t start_vert_for_chunk;
size_t end_vert_for_chunk;
bool complete_tree;
if (num_verts_left_in_tree > CHUNK_SIZE) {
complete_tree = false;
// should only do partial
start_vert_for_chunk = data.tfrag_next_vert;
end_vert_for_chunk = start_vert_for_chunk + CHUNK_SIZE;
data.tfrag_next_vert += CHUNK_SIZE;
} else {
// should do all!
start_vert_for_chunk = data.tfrag_next_vert;
end_vert_for_chunk = end_vert_in_tree;
complete_tree = true;
}
// glBindVertexArray(m_trees[m_load_state.vert_geo][m_load_state.vert_tree].vao);
glBindBuffer(GL_ARRAY_BUFFER,
data.tfrag_vertex_data[data.tfrag_next_geo][data.tfrag_next_tree]);
u32 upload_size = (end_vert_for_chunk - start_vert_for_chunk) * sizeof(tfrag3::PreloadedVertex);
glBufferSubData(GL_ARRAY_BUFFER, start_vert_for_chunk * sizeof(tfrag3::PreloadedVertex),
upload_size, tree.unpacked.vertices.data() + start_vert_for_chunk);
uploaded_bytes += upload_size;
if (complete_tree) {
// and move on to next tree
data.tfrag_next_vert = 0;
data.tfrag_next_tree++;
if (data.tfrag_next_tree >= data.level->tfrag_trees[data.tfrag_next_geo].size()) {
data.tfrag_next_tree = 0;
data.tfrag_next_geo++;
if (data.tfrag_next_geo >= tfrag3::TFRAG_GEOS) {
data.tfrag_load_done = true;
data.tfrag_next_tree = 0;
data.tfrag_next_geo = 0;
data.tfrag_next_vert = 0;
return true;
}
}
}
if (timer.getMs() > Loader::TIE_LOAD_BUDGET || (uploaded_bytes / 1024) > 2048) {
return false;
}
}
}
bool Loader::init_shrub(Timer& timer, LevelData& data) {
if (data.shrub_load_done) {
return true;
}
if (data.level->shrub_trees.empty()) {
data.shrub_load_done = true;
return true;
}
if (!data.shrub_opengl_created) {
for (auto& in_tree : data.level->shrub_trees) {
GLuint& tree_out = data.shrub_vertex_data.emplace_back();
glGenBuffers(1, &tree_out);
glBindBuffer(GL_ARRAY_BUFFER, tree_out);
glBufferData(GL_ARRAY_BUFFER,
in_tree.unpacked.vertices.size() * sizeof(tfrag3::ShrubGpuVertex), nullptr,
GL_STATIC_DRAW);
}
data.shrub_opengl_created = true;
return false;
}
constexpr u32 CHUNK_SIZE = 32768;
u32 uploaded_bytes = 0;
while (true) {
const auto& tree = data.level->shrub_trees[data.shrub_next_tree];
u32 end_vert_in_tree = tree.unpacked.vertices.size();
// the number of vertices we'd need to finish the tree right now
size_t num_verts_left_in_tree = end_vert_in_tree - data.shrub_next_vert;
size_t start_vert_for_chunk;
size_t end_vert_for_chunk;
bool complete_tree;
if (num_verts_left_in_tree > CHUNK_SIZE) {
complete_tree = false;
// should only do partial
start_vert_for_chunk = data.shrub_next_vert;
end_vert_for_chunk = start_vert_for_chunk + CHUNK_SIZE;
data.shrub_next_vert += CHUNK_SIZE;
} else {
// should do all!
start_vert_for_chunk = data.shrub_next_vert;
end_vert_for_chunk = end_vert_in_tree;
complete_tree = true;
}
// glBindVertexArray(m_trees[m_load_state.vert_geo][m_load_state.vert_tree].vao);
glBindBuffer(GL_ARRAY_BUFFER, data.shrub_vertex_data[data.shrub_next_tree]);
u32 upload_size = (end_vert_for_chunk - start_vert_for_chunk) * sizeof(tfrag3::ShrubGpuVertex);
glBufferSubData(GL_ARRAY_BUFFER, start_vert_for_chunk * sizeof(tfrag3::ShrubGpuVertex),
upload_size, tree.unpacked.vertices.data() + start_vert_for_chunk);
uploaded_bytes += upload_size;
if (complete_tree) {
// and move on to next tree
data.shrub_next_vert = 0;
data.shrub_next_tree++;
if (data.shrub_next_tree >= data.level->shrub_trees.size()) {
data.shrub_load_done = true;
return true;
}
}
if (timer.getMs() > Loader::TIE_LOAD_BUDGET || (uploaded_bytes / 1024) > 2048) {
return false;
}
}
}
bool Loader::init_tie(Timer& timer, LevelData& data) {
if (data.tie_load_done) {
return true;
}
if (data.level->tie_trees.front().empty()) {
data.tie_load_done = true;
return true;
}
if (!data.tie_opengl_created) {
for (int geo = 0; geo < tfrag3::TIE_GEOS; geo++) {
auto& in_trees = data.level->tie_trees[geo];
for (auto& in_tree : in_trees) {
LevelData::TieOpenGL& tree_out = data.tie_data[geo].emplace_back();
glGenBuffers(1, &tree_out.vertex_buffer);
glBindBuffer(GL_ARRAY_BUFFER, tree_out.vertex_buffer);
glBufferData(GL_ARRAY_BUFFER,
in_tree.unpacked.vertices.size() * sizeof(tfrag3::PreloadedVertex), nullptr,
GL_STATIC_DRAW);
}
}
data.tie_opengl_created = true;
return false;
}
if (!data.tie_verts_done) {
constexpr u32 CHUNK_SIZE = 32768;
u32 uploaded_bytes = 0;
while (true) {
const auto& tree = data.level->tie_trees[data.tie_next_geo][data.tie_next_tree];
u32 end_vert_in_tree = tree.unpacked.vertices.size();
// the number of vertices we'd need to finish the tree right now
size_t num_verts_left_in_tree = end_vert_in_tree - data.tie_next_vert;
size_t start_vert_for_chunk;
size_t end_vert_for_chunk;
bool complete_tree;
if (num_verts_left_in_tree > CHUNK_SIZE) {
complete_tree = false;
// should only do partial
start_vert_for_chunk = data.tie_next_vert;
end_vert_for_chunk = start_vert_for_chunk + CHUNK_SIZE;
data.tie_next_vert += CHUNK_SIZE;
} else {
// should do all!
start_vert_for_chunk = data.tie_next_vert;
end_vert_for_chunk = end_vert_in_tree;
complete_tree = true;
}
// glBindVertexArray(m_trees[m_load_state.vert_geo][m_load_state.vert_tree].vao);
glBindBuffer(GL_ARRAY_BUFFER,
data.tie_data[data.tie_next_geo][data.tie_next_tree].vertex_buffer);
u32 upload_size =
(end_vert_for_chunk - start_vert_for_chunk) * sizeof(tfrag3::PreloadedVertex);
glBufferSubData(GL_ARRAY_BUFFER, start_vert_for_chunk * sizeof(tfrag3::PreloadedVertex),
upload_size, tree.unpacked.vertices.data() + start_vert_for_chunk);
uploaded_bytes += upload_size;
if (complete_tree) {
// and move on to next tree
data.tie_next_vert = 0;
data.tie_next_tree++;
if (data.tie_next_tree >= data.level->tie_trees[data.tie_next_geo].size()) {
data.tie_next_tree = 0;
data.tie_next_geo++;
if (data.tie_next_geo >= tfrag3::TIE_GEOS) {
data.tie_verts_done = true;
data.tie_next_tree = 0;
data.tie_next_geo = 0;
data.tie_next_vert = 0;
return false;
}
}
}
if (timer.getMs() > Loader::TIE_LOAD_BUDGET || (uploaded_bytes / 1024) > 2048) {
return false;
}
}
}
if (!data.tie_wind_indices_done) {
bool abort = false;
for (; data.tie_next_geo < tfrag3::TIE_GEOS; data.tie_next_geo++) {
auto& geo_trees = data.level->tie_trees[data.tie_next_geo];
for (; data.tie_next_tree < geo_trees.size(); data.tie_next_tree++) {
if (abort) {
return false;
}
auto& in_tree = geo_trees[data.tie_next_tree];
auto& out_tree = data.tie_data[data.tie_next_geo][data.tie_next_tree];
size_t wind_idx_buffer_len = 0;
for (auto& draw : in_tree.instanced_wind_draws) {
wind_idx_buffer_len += draw.vertex_index_stream.size();
}
if (wind_idx_buffer_len > 0) {
out_tree.has_wind = true;
glGenBuffers(1, &out_tree.wind_indices);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, out_tree.wind_indices);
std::vector<u32> temp;
temp.resize(wind_idx_buffer_len);
u32 off = 0;
for (auto& draw : in_tree.instanced_wind_draws) {
memcpy(temp.data() + off, draw.vertex_index_stream.data(),
draw.vertex_index_stream.size() * sizeof(u32));
off += draw.vertex_index_stream.size();
}
glBufferData(GL_ELEMENT_ARRAY_BUFFER, wind_idx_buffer_len * sizeof(u32), temp.data(),
GL_STATIC_DRAW);
abort = true;
}
}
data.tie_next_tree = 0;
}
data.tie_wind_indices_done = true;
data.tie_load_done = true;
return true;
}
return false;
}
bool Loader::init_collide(Timer& /*timer*/, LevelData& data) {
Timer t;
glGenBuffers(1, &data.collide_vertices);
glBindBuffer(GL_ARRAY_BUFFER, data.collide_vertices);
glBufferData(GL_ARRAY_BUFFER,
data.level->collision.vertices.size() * sizeof(tfrag3::CollisionMesh::Vertex),
data.level->collision.vertices.data(), GL_STATIC_DRAW);
fmt::print("init_collide took {:.2f} ms\n", t.getMs());
return true;
}
bool Loader::init_merc(Timer& /*timer*/, LevelData& data) {
Timer t;
glGenBuffers(1, &data.merc_indices);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, data.merc_indices);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, data.level->merc_data.indices.size() * sizeof(u32),
data.level->merc_data.indices.data(), GL_STATIC_DRAW);
glGenBuffers(1, &data.merc_vertices);
glBindBuffer(GL_ARRAY_BUFFER, data.merc_vertices);
glBufferData(GL_ARRAY_BUFFER, data.level->merc_data.vertices.size() * sizeof(tfrag3::MercVertex),
data.level->merc_data.vertices.data(), GL_STATIC_DRAW);
for (auto& model : data.level->merc_data.models) {
data.merc_model_lookup[model.name] = &model;
m_all_merc_models[model.name].push_back({&model, data.load_id, &data});
}
fmt::print("init_merc took {:.2f} ms\n", t.getMs());
return true;
}
bool Loader::upload_textures(Timer& timer, LevelData& data, TexturePool& texture_pool) {
// try to move level from initializing to initialized:
constexpr int MAX_TEX_BYTES_PER_FRAME = 1024 * 128;
int bytes_this_run = 0;
int tex_this_run = 0;
if (data.textures.size() < data.level->textures.size()) {
std::unique_lock<std::mutex> tpool_lock(texture_pool.mutex());
while (data.textures.size() < data.level->textures.size()) {
auto& tex = data.level->textures[data.textures.size()];
data.textures.push_back(add_texture(texture_pool, tex, false));
bytes_this_run += tex.w * tex.h * 4;
tex_this_run++;
if (tex_this_run > 20) {
break;
}
if (bytes_this_run > MAX_TEX_BYTES_PER_FRAME || timer.getMs() > SHARED_TEXTURE_LOAD_BUDGET) {
break;
}
}
}
return data.textures.size() == data.level->textures.size();
}
void Loader::update_blocking(TexturePool& tex_pool) {
fmt::print("NOTE: coming out of blackout on next frame, doing all loads now...\n");
bool missing_levels = true;
while (missing_levels) {
bool needs_run = true;
while (needs_run) {
needs_run = false;
{
std::unique_lock<std::mutex> lk(m_loader_mutex);
if (!m_level_to_load.empty()) {
m_file_load_done_cv.wait(lk, [&]() { return m_level_to_load.empty(); });
}
}
}
needs_run = true;
while (needs_run) {
needs_run = false;
{
std::unique_lock<std::mutex> lk(m_loader_mutex);
if (!m_initializing_tfrag3_levels.empty()) {
needs_run = true;
}
}
if (needs_run) {
update(tex_pool);
}
}
{
std::unique_lock<std::mutex> lk(m_loader_mutex);
missing_levels = false;
for (auto& des : m_desired_levels) {
if (m_loaded_tfrag3_levels.find(des) == m_loaded_tfrag3_levels.end()) {
fmt::print("blackout loader doing additional level {}...\n", des);
missing_levels = true;
}
}
}
if (missing_levels) {
set_want_levels(m_desired_levels);
}
}
fmt::print("Blackout loads done. Current status:");
std::unique_lock<std::mutex> lk(m_loader_mutex);
for (auto& ld : m_loaded_tfrag3_levels) {
fmt::print(" {} is loaded.\n", ld.first);
}
}
void Loader::update(TexturePool& texture_pool) {
Timer loader_timer;
// only main thread can touch this.
for (auto& lev : m_loaded_tfrag3_levels) {
lev.second->frames_since_last_used++;
}
bool did_gpu_stuff = false;
// work on moving initializing to initialized.
{
// accessing initializing, should lock
std::unique_lock<std::mutex> lk(m_loader_mutex);
// grab the first initializing level:
const auto& it = m_initializing_tfrag3_levels.begin();
if (it != m_initializing_tfrag3_levels.end()) {
std::string name = it->first;
auto& lev = it->second;
it->second->load_id = m_id++;
// we're the only place that erases, so it's okay to unlock and hold a reference
lk.unlock();
if (!lev->tfrag_load_done) {
did_gpu_stuff = true;
}
if (upload_textures(loader_timer, *lev, texture_pool)) {
if (init_tie(loader_timer, *lev)) {
if (init_tfrag(loader_timer, *lev)) {
if (init_shrub(loader_timer, *lev)) {
if (init_collide(loader_timer, *lev)) {
if (init_merc(loader_timer, *lev)) {
// we're done! lock before removing from loaded.
lk.lock();
m_loaded_tfrag3_levels[name] = std::move(lev);
m_initializing_tfrag3_levels.erase(it);
}
}
}
}
}
}
}
}
if (!did_gpu_stuff) {
// try to remove levels.
if (m_loaded_tfrag3_levels.size() >= 3) {
for (auto& lev : m_loaded_tfrag3_levels) {
if (lev.second->frames_since_last_used > 180) {
std::unique_lock<std::mutex> lk(texture_pool.mutex());
fmt::print("------------------------- PC unloading {}\n", lev.first);
for (size_t i = 0; i < lev.second->level->textures.size(); i++) {
auto& tex = lev.second->level->textures[i];
if (tex.load_to_pool) {
texture_pool.unload_texture(PcTextureId::from_combo_id(tex.combo_id),
lev.second->textures.at(i));
}
}
lk.unlock();
for (auto tex : lev.second->textures) {
if (EXTRA_TEX_DEBUG) {
for (auto& slot : texture_pool.all_textures()) {
if (slot.source) {
ASSERT(slot.gpu_texture != tex);
} else {
ASSERT(slot.gpu_texture != tex);
}
}
}
glBindTexture(GL_TEXTURE_2D, tex);
glDeleteTextures(1, &tex);
}
for (auto& tie_geo : lev.second->tie_data) {
for (auto& tie_tree : tie_geo) {
glDeleteBuffers(1, &tie_tree.vertex_buffer);
if (tie_tree.has_wind) {
glDeleteBuffers(1, &tie_tree.wind_indices);
}
}
}
for (auto& tfrag_geo : lev.second->tfrag_vertex_data) {
for (auto& tfrag_buff : tfrag_geo) {
glDeleteBuffers(1, &tfrag_buff);
}
}
glDeleteBuffers(1, &lev.second->collide_vertices);
glDeleteBuffers(1, &lev.second->merc_vertices);
glDeleteBuffers(1, &lev.second->merc_indices);
for (auto& model : lev.second->level->merc_data.models) {
auto& mercs = m_all_merc_models.at(model.name);
MercRef ref{&model, lev.second->load_id};
auto it = std::find(mercs.begin(), mercs.end(), ref);
ASSERT_MSG(it != mercs.end(), fmt::format("missing merc: {}\n", model.name));
mercs.erase(it);
}
m_loaded_tfrag3_levels.erase(lev.first);
break;
}
}
}
}
if (loader_timer.getMs() > 5) {
fmt::print("Loader::update slow setup: {:.1f}ms\n", loader_timer.getMs());
}
}
std::optional<Loader::MercRef> Loader::get_merc_model(const char* model_name) {
// don't think we need to lock here...
const auto& it = m_all_merc_models.find(model_name);
if (it != m_all_merc_models.end() && !it->second.empty()) {
// it->second.front().parent_level->frames_since_last_used = 0;
return it->second.front();
} else {
return std::nullopt;
}
}
game/graphics/opengl_renderer/Loader.h
-109
View File
@@ -1,109 +0,0 @@
#pragma once
#include <thread>
#include <mutex>
#include <condition_variable>
#include "game/graphics/pipelines/opengl.h"
#include "game/graphics/texture/TexturePool.h"
#include "common/custom_data/Tfrag3Data.h"
#include "common/util/Timer.h"
class Loader {
public:
static constexpr float TIE_LOAD_BUDGET = 1.5f;
static constexpr float SHARED_TEXTURE_LOAD_BUDGET = 3.f;
Loader();
~Loader();
void update(TexturePool& tex_pool);
void update_blocking(TexturePool& tex_pool);
struct LevelData {
std::unique_ptr<tfrag3::Level> level;
std::vector<GLuint> textures;
u64 load_id = 0;
struct TieOpenGL {
GLuint vertex_buffer;
bool has_wind = false;
GLuint wind_indices;
};
std::array<std::vector<TieOpenGL>, tfrag3::TIE_GEOS> tie_data;
std::array<std::vector<GLuint>, tfrag3::TIE_GEOS> tfrag_vertex_data;
std::vector<GLuint> shrub_vertex_data;
GLuint collide_vertices;
GLuint merc_vertices;
GLuint merc_indices;
std::unordered_map<std::string, const tfrag3::MercModel*> merc_model_lookup;
// internal load state
bool tie_opengl_created = false;
bool tie_verts_done = false;
bool tie_wind_indices_done = false;
bool tie_load_done = false;
u32 tie_next_geo = 0;
u32 tie_next_tree = 0;
u32 tie_next_vert = 0;
bool tfrag_opengl_created = false;
bool tfrag_load_done = false;
u32 tfrag_next_geo = 0;
u32 tfrag_next_tree = 0;
u32 tfrag_next_vert = 0;
bool shrub_opengl_created = false;
bool shrub_load_done = false;
u32 shrub_next_tree = 0;
u32 shrub_next_vert = 0;
int frames_since_last_used = 0;
};
struct MercRef {
const tfrag3::MercModel* model = nullptr;
u64 load_id = 0;
const LevelData* level = nullptr;
bool operator==(const MercRef& other) const {
return model == other.model && load_id == other.load_id;
}
};
const LevelData* get_tfrag3_level(const std::string& level_name);
std::optional<MercRef> get_merc_model(const char* model_name);
void load_common(TexturePool& tex_pool, const std::string& name);
void set_want_levels(const std::vector<std::string>& levels);
std::vector<LevelData*> get_in_use_levels();
private:
void loader_thread();
u64 add_texture(TexturePool& pool, const tfrag3::Texture& tex, bool is_common);
bool upload_textures(Timer& timer, LevelData& data, TexturePool& texture_pool);
bool init_tie(Timer& timer, LevelData& data);
bool init_tfrag(Timer& timer, LevelData& data);
bool init_shrub(Timer& timer, LevelData& data);
bool init_collide(Timer& timer, LevelData& data);
bool init_merc(Timer& timer, LevelData& data);
// used by game and loader thread
std::unordered_map<std::string, std::unique_ptr<LevelData>> m_initializing_tfrag3_levels;
LevelData m_common_level;
std::string m_level_to_load;
std::thread m_loader_thread;
std::mutex m_loader_mutex;
std::condition_variable m_loader_cv;
std::condition_variable m_file_load_done_cv;
bool m_want_shutdown = false;
uint64_t m_id = 0;
// used only by game thread
std::unordered_map<std::string, std::unique_ptr<LevelData>> m_loaded_tfrag3_levels;
std::unordered_map<std::string, std::vector<MercRef>> m_all_merc_models;
std::vector<std::string> m_desired_levels;
};
game/graphics/opengl_renderer/background/Shrub.cpp
+1 -1
View File
@@ -61,7 +61,7 @@ void Shrub::render(DmaFollower& dma, SharedRenderState* render_state, ScopedProf
render_all_trees(settings, render_state, prof);
}
void Shrub::update_load(const Loader::LevelData* loader_data) {
void Shrub::update_load(const LevelData* loader_data) {
const tfrag3::Level* lev_data = loader_data->level.get();
// We changed level!
discard_tree_cache();
game/graphics/opengl_renderer/background/Shrub.h
+1 -1
View File
@@ -24,7 +24,7 @@ class Shrub : public BucketRenderer {
void draw_debug_window() override;
private:
void update_load(const Loader::LevelData* loader_data);
void update_load(const LevelData* loader_data);
void discard_tree_cache();
struct Tree {
game/graphics/opengl_renderer/background/Tfrag3.cpp
+1 -1
View File
@@ -39,7 +39,7 @@ Tfrag3::~Tfrag3() {
}
void Tfrag3::update_load(const std::vector<tfrag3::TFragmentTreeKind>& tree_kinds,
const Loader::LevelData* loader_data) {
const LevelData* loader_data) {
const auto* lev_data = loader_data->level.get();
discard_tree_cache();
for (int geom = 0; geom < GEOM_MAX; ++geom) {
game/graphics/opengl_renderer/background/Tfrag3.h
+1 -1
View File
@@ -44,7 +44,7 @@ class Tfrag3 {
};
void update_load(const std::vector<tfrag3::TFragmentTreeKind>& tree_kinds,
const Loader::LevelData* loader_data);
const LevelData* loader_data);
int lod() const { return Gfx::g_global_settings.lod_tfrag; }
game/graphics/opengl_renderer/background/Tie3.cpp
+1 -1
View File
@@ -14,7 +14,7 @@ Tie3::~Tie3() {
discard_tree_cache();
}
void Tie3::update_load(const Loader::LevelData* loader_data) {
void Tie3::update_load(const LevelData* loader_data) {
const tfrag3::Level* lev_data = loader_data->level.get();
m_wind_vectors.clear();
// We changed level!
game/graphics/opengl_renderer/background/Tie3.h
+1 -1
View File
@@ -40,7 +40,7 @@ class Tie3 : public BucketRenderer {
int lod() const { return Gfx::g_global_settings.lod_tie; }
private:
void update_load(const Loader::LevelData* loader_data);
void update_load(const LevelData* loader_data);
void discard_tree_cache();
void render_tree_wind(int idx,
int geom,
game/graphics/opengl_renderer/foreground/Merc2.cpp
+1 -1
View File
@@ -412,7 +412,7 @@ void Merc2::flush_pending_model(SharedRenderState* render_state, ScopedProfilerN
return;
}
const Loader::LevelData* lev = m_current_model->level;
const LevelData* lev = m_current_model->level;
const tfrag3::MercModel* model = m_current_model->model;
int bone_count = (model->max_bones + 31) & (~31); // todo
game/graphics/opengl_renderer/foreground/Merc2.h
+3 -3
View File
@@ -56,7 +56,7 @@ class Merc2 : public BucketRenderer {
u32 alloc_bones(int count, float scale);
std::optional<Loader::MercRef> m_current_model = std::nullopt;
std::optional<MercRef> m_current_model = std::nullopt;
u16 m_current_effect_enable_bits = 0;
u16 m_current_ignore_alpha_bits = 0;
@@ -120,7 +120,7 @@ class Merc2 : public BucketRenderer {
};
struct LevelDrawBucket {
const Loader::LevelData* level = nullptr;
const LevelData* level = nullptr;
std::vector<Draw> draws;
u32 next_free_draw = 0;
@@ -140,4 +140,4 @@ class Merc2 : public BucketRenderer {
u32 m_next_free_bone = 0;
void flush_draw_buckets(SharedRenderState* render_state, ScopedProfilerNode& prof);
};
};
game/graphics/opengl_renderer/loader/Loader.cpp
+410
View File
@@ -0,0 +1,410 @@
#include "Loader.h"
#include "common/util/Timer.h"
#include "common/util/FileUtil.h"
#include "common/util/compress.h"
#include "game/graphics/opengl_renderer/loader/LoaderStages.h"
namespace {
std::string uppercase_string(const std::string& s) {
std::string result;
for (auto c : s) {
result.push_back(toupper(c));
}
return result;
}
} // namespace
Loader::Loader() {
m_loader_thread = std::thread(&Loader::loader_thread, this);
m_loader_stages = make_loader_stages();
}
Loader::~Loader() {
{
std::lock_guard<std::mutex> lk(m_loader_mutex);
m_want_shutdown = true;
m_loader_cv.notify_all();
}
m_loader_thread.join();
}
/*!
* Try to get a loaded level by name. It may fail and return nullptr.
* Getting a level will reset the counter for the level and prevent it from being kicked out
* for a little while.
*
* This is safe to call from the graphics thread
*/
const LevelData* Loader::get_tfrag3_level(const std::string& level_name) {
std::unique_lock<std::mutex> lk(m_loader_mutex);
const auto& existing = m_loaded_tfrag3_levels.find(level_name);
if (existing == m_loaded_tfrag3_levels.end()) {
return nullptr;
} else {
existing->second->frames_since_last_used = 0;
return existing->second.get();
}
}
/*!
* The game calls this to give the loader a hint on which levels we want.
* If the loader is not busy, it will begin loading the level.
* This should be called on every frame.
*/
void Loader::set_want_levels(const std::vector<std::string>& levels) {
std::unique_lock<std::mutex> lk(m_loader_mutex);
m_desired_levels = levels;
if (!m_level_to_load.empty()) {
// can't do anything, we're loading a level right now
return;
}
if (!m_initializing_tfrag3_levels.empty()) {
// can't do anything, we're initializing a level right now
return;
}
// loader isn't busy, try to load one of the requested levels.
for (auto& lev : levels) {
auto it = m_loaded_tfrag3_levels.find(lev);
if (it == m_loaded_tfrag3_levels.end()) {
// we haven't loaded it yet. Request this level to load and wake up the thread.
m_level_to_load = lev;
lk.unlock();
m_loader_cv.notify_all();
return;
}
}
}
/*!
* Get all levels that are in memory and used very recently.
*/
std::vector<LevelData*> Loader::get_in_use_levels() {
std::vector<LevelData*> result;
std::unique_lock<std::mutex> lk(m_loader_mutex);
for (auto& lev : m_loaded_tfrag3_levels) {
if (lev.second->frames_since_last_used < 5) {
result.push_back(lev.second.get());
}
}
return result;
}
/*!
* Loader function that runs in a completely separate thread.
* This is used for file I/O and unpacking.
*/
void Loader::loader_thread() {
while (!m_want_shutdown) {
std::unique_lock<std::mutex> lk(m_loader_mutex);
// this will keep us asleep until we've got a level to load.
m_loader_cv.wait(lk, [&] { return !m_level_to_load.empty() || m_want_shutdown; });
if (m_want_shutdown) {
return;
}
std::string lev = m_level_to_load;
// don't hold the lock while reading the file.
lk.unlock();
// simulate slower hard drive (so that the loader thread can lose to the game loads)
// std::this_thread::sleep_for(std::chrono::milliseconds(1500));
// load the fr3 file
Timer disk_timer;
auto data = file_util::read_binary_file(
file_util::get_file_path({fmt::format("assets/{}.fr3", uppercase_string(lev))}));
double disk_load_time = disk_timer.getSeconds();
// the FR3 files are compressed
Timer decomp_timer;
auto decomp_data = compression::decompress_zstd(data.data(), data.size());
double decomp_time = decomp_timer.getSeconds();
// Read back into the tfrag3::Level structure
Timer import_timer;
auto result = std::make_unique<tfrag3::Level>();
Serializer ser(decomp_data.data(), decomp_data.size());
result->serialize(ser);
double import_time = import_timer.getSeconds();
// and finally "unpack", which creates the vertex data we'll upload to the GPU
Timer unpack_timer;
for (auto& tie_tree : result->tie_trees) {
for (auto& tree : tie_tree) {
tree.unpack();
}
}
for (auto& t_tree : result->tfrag_trees) {
for (auto& tree : t_tree) {
tree.unpack();
}
}
for (auto& shrub_tree : result->shrub_trees) {
shrub_tree.unpack();
}
fmt::print(
"------------> Load from file: {:.3f}s, import {:.3f}s, decomp {:.3f}s unpack {:.3f}s\n",
disk_load_time, import_time, decomp_time, unpack_timer.getSeconds());
// grab the lock again
lk.lock();
// move this level to "initializing" state.
m_initializing_tfrag3_levels[lev] = std::make_unique<LevelData>(); // reset load state
m_initializing_tfrag3_levels[lev]->level = std::move(result);
m_level_to_load = "";
m_file_load_done_cv.notify_all();
}
}
/*!
* Load a "common" FR3 file that has non-level textures.
* This should be called during initialization, before any threaded loading goes on.
*/
void Loader::load_common(TexturePool& tex_pool, const std::string& name) {
auto data =
file_util::read_binary_file(file_util::get_file_path({fmt::format("assets/{}.fr3", name)}));
auto decomp_data = compression::decompress_zstd(data.data(), data.size());
Serializer ser(decomp_data.data(), decomp_data.size());
m_common_level.level = std::make_unique<tfrag3::Level>();
m_common_level.level->serialize(ser);
for (auto& tex : m_common_level.level->textures) {
m_common_level.textures.push_back(add_texture(tex_pool, tex, true));
}
Timer tim;
MercLoaderStage mls;
LoaderInput input;
input.tex_pool = &tex_pool;
input.mercs = &m_all_merc_models;
input.lev_data = &m_common_level;
bool done = false;
while (!done) {
done = mls.run(tim, input);
}
}
bool Loader::upload_textures(Timer& timer, LevelData& data, TexturePool& texture_pool) {
// try to move level from initializing to initialized:
constexpr int MAX_TEX_BYTES_PER_FRAME = 1024 * 128;
int bytes_this_run = 0;
int tex_this_run = 0;
if (data.textures.size() < data.level->textures.size()) {
std::unique_lock<std::mutex> tpool_lock(texture_pool.mutex());
while (data.textures.size() < data.level->textures.size()) {
auto& tex = data.level->textures[data.textures.size()];
data.textures.push_back(add_texture(texture_pool, tex, false));
bytes_this_run += tex.w * tex.h * 4;
tex_this_run++;
if (tex_this_run > 20) {
break;
}
if (bytes_this_run > MAX_TEX_BYTES_PER_FRAME || timer.getMs() > SHARED_TEXTURE_LOAD_BUDGET) {
break;
}
}
}
return data.textures.size() == data.level->textures.size();
}
void Loader::update_blocking(TexturePool& tex_pool) {
fmt::print("NOTE: coming out of blackout on next frame, doing all loads now...\n");
bool missing_levels = true;
while (missing_levels) {
bool needs_run = true;
while (needs_run) {
needs_run = false;
{
std::unique_lock<std::mutex> lk(m_loader_mutex);
if (!m_level_to_load.empty()) {
m_file_load_done_cv.wait(lk, [&]() { return m_level_to_load.empty(); });
}
}
}
needs_run = true;
while (needs_run) {
needs_run = false;
{
std::unique_lock<std::mutex> lk(m_loader_mutex);
if (!m_initializing_tfrag3_levels.empty()) {
needs_run = true;
}
}
if (needs_run) {
update(tex_pool);
}
}
{
std::unique_lock<std::mutex> lk(m_loader_mutex);
missing_levels = false;
for (auto& des : m_desired_levels) {
if (m_loaded_tfrag3_levels.find(des) == m_loaded_tfrag3_levels.end()) {
fmt::print("blackout loader doing additional level {}...\n", des);
missing_levels = true;
}
}
}
if (missing_levels) {
set_want_levels(m_desired_levels);
}
}
fmt::print("Blackout loads done. Current status:");
std::unique_lock<std::mutex> lk(m_loader_mutex);
for (auto& ld : m_loaded_tfrag3_levels) {
fmt::print(" {} is loaded.\n", ld.first);
}
}
void Loader::update(TexturePool& texture_pool) {
Timer loader_timer;
// only main thread can touch this.
for (auto& lev : m_loaded_tfrag3_levels) {
lev.second->frames_since_last_used++;
}
bool did_gpu_stuff = false;
// work on moving initializing to initialized.
{
// accessing initializing, should lock
std::unique_lock<std::mutex> lk(m_loader_mutex);
// grab the first initializing level:
const auto& it = m_initializing_tfrag3_levels.begin();
if (it != m_initializing_tfrag3_levels.end()) {
did_gpu_stuff = true;
std::string name = it->first;
auto& lev = it->second;
if (it->second->load_id == UINT64_MAX) {
it->second->load_id = m_id++;
}
// we're the only place that erases, so it's okay to unlock and hold a reference
lk.unlock();
bool done = true;
LoaderInput loader_input;
loader_input.lev_data = lev.get();
loader_input.mercs = &m_all_merc_models;
loader_input.tex_pool = &texture_pool;
for (auto& stage : m_loader_stages) {
Timer stage_timer;
done = stage->run(loader_timer, loader_input);
if (stage_timer.getMs() > 5.f) {
fmt::print("stage {} took {:.2f} ms\n", stage->name(), stage_timer.getMs());
}
if (!done) {
break;
}
}
if (done) {
lk.lock();
m_loaded_tfrag3_levels[name] = std::move(lev);
m_initializing_tfrag3_levels.erase(it);
for (auto& stage : m_loader_stages) {
stage->reset();
}
}
}
}
if (!did_gpu_stuff) {
// try to remove levels.
Timer unload_timer;
if (m_loaded_tfrag3_levels.size() >= 3) {
for (auto& lev : m_loaded_tfrag3_levels) {
if (lev.second->frames_since_last_used > 180) {
std::unique_lock<std::mutex> lk(texture_pool.mutex());
fmt::print("------------------------- PC unloading {}\n", lev.first);
for (size_t i = 0; i < lev.second->level->textures.size(); i++) {
auto& tex = lev.second->level->textures[i];
if (tex.load_to_pool) {
texture_pool.unload_texture(PcTextureId::from_combo_id(tex.combo_id),
lev.second->textures.at(i));
}
}
lk.unlock();
for (auto tex : lev.second->textures) {
if (EXTRA_TEX_DEBUG) {
for (auto& slot : texture_pool.all_textures()) {
if (slot.source) {
ASSERT(slot.gpu_texture != tex);
} else {
ASSERT(slot.gpu_texture != tex);
}
}
}
glBindTexture(GL_TEXTURE_2D, tex);
glDeleteTextures(1, &tex);
}
for (auto& tie_geo : lev.second->tie_data) {
for (auto& tie_tree : tie_geo) {
glDeleteBuffers(1, &tie_tree.vertex_buffer);
if (tie_tree.has_wind) {
glDeleteBuffers(1, &tie_tree.wind_indices);
}
}
}
for (auto& tfrag_geo : lev.second->tfrag_vertex_data) {
for (auto& tfrag_buff : tfrag_geo) {
glDeleteBuffers(1, &tfrag_buff);
}
}
glDeleteBuffers(1, &lev.second->collide_vertices);
glDeleteBuffers(1, &lev.second->merc_vertices);
glDeleteBuffers(1, &lev.second->merc_indices);
for (auto& model : lev.second->level->merc_data.models) {
auto& mercs = m_all_merc_models.at(model.name);
MercRef ref{&model, lev.second->load_id};
auto it = std::find(mercs.begin(), mercs.end(), ref);
ASSERT_MSG(it != mercs.end(), fmt::format("missing merc: {}\n", model.name));
mercs.erase(it);
}
m_loaded_tfrag3_levels.erase(lev.first);
break;
}
}
}
if (unload_timer.getMs() > 5.f) {
fmt::print("Unload took {:.2f}\n", unload_timer.getMs());
}
}
if (loader_timer.getMs() > 5) {
fmt::print("Loader::update slow setup: {:.1f}ms\n", loader_timer.getMs());
}
}
std::optional<MercRef> Loader::get_merc_model(const char* model_name) {
// don't think we need to lock here...
const auto& it = m_all_merc_models.find(model_name);
if (it != m_all_merc_models.end() && !it->second.empty()) {
// it->second.front().parent_level->frames_since_last_used = 0;
return it->second.front();
} else {
return std::nullopt;
}
}
game/graphics/opengl_renderer/loader/Loader.h
+52
View File
@@ -0,0 +1,52 @@
#pragma once
#include <thread>
#include <mutex>
#include <condition_variable>
#include "game/graphics/pipelines/opengl.h"
#include "game/graphics/texture/TexturePool.h"
#include "common/custom_data/Tfrag3Data.h"
#include "common/util/Timer.h"
#include "game/graphics/opengl_renderer/loader/common.h"
class Loader {
public:
static constexpr float TIE_LOAD_BUDGET = 1.5f;
static constexpr float SHARED_TEXTURE_LOAD_BUDGET = 3.f;
Loader();
~Loader();
void update(TexturePool& tex_pool);
void update_blocking(TexturePool& tex_pool);
const LevelData* get_tfrag3_level(const std::string& level_name);
std::optional<MercRef> get_merc_model(const char* model_name);
void load_common(TexturePool& tex_pool, const std::string& name);
void set_want_levels(const std::vector<std::string>& levels);
std::vector<LevelData*> get_in_use_levels();
private:
void loader_thread();
bool upload_textures(Timer& timer, LevelData& data, TexturePool& texture_pool);
// used by game and loader thread
std::unordered_map<std::string, std::unique_ptr<LevelData>> m_initializing_tfrag3_levels;
LevelData m_common_level;
std::string m_level_to_load;
std::thread m_loader_thread;
std::mutex m_loader_mutex;
std::condition_variable m_loader_cv;
std::condition_variable m_file_load_done_cv;
bool m_want_shutdown = false;
uint64_t m_id = 0;
// used only by game thread
std::unordered_map<std::string, std::unique_ptr<LevelData>> m_loaded_tfrag3_levels;
std::unordered_map<std::string, std::vector<MercRef>> m_all_merc_models;
std::vector<std::string> m_desired_levels;
std::vector<std::unique_ptr<LoaderStage>> m_loader_stages;
};
game/graphics/opengl_renderer/loader/LoaderStages.cpp
+529
View File
@@ -0,0 +1,529 @@
#include "Loader.h"
#include "LoaderStages.h"
constexpr float LOAD_BUDGET = 2.5f;
/*!
* Upload a texture to the GPU, and give it to the pool.
*/
u64 add_texture(TexturePool& pool, const tfrag3::Texture& tex, bool is_common) {
GLuint gl_tex;
glGenTextures(1, &gl_tex);
glBindTexture(GL_TEXTURE_2D, gl_tex);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, tex.w, tex.h, 0, GL_RGBA, GL_UNSIGNED_INT_8_8_8_8_REV,
tex.data.data());
glBindTexture(GL_TEXTURE_2D, 0);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, gl_tex);
glGenerateMipmap(GL_TEXTURE_2D);
float aniso = 0.0f;
glGetFloatv(GL_MAX_TEXTURE_MAX_ANISOTROPY, &aniso);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY, aniso);
if (tex.load_to_pool) {
TextureInput in;
in.debug_page_name = tex.debug_tpage_name;
in.debug_name = tex.debug_name;
in.w = tex.w;
in.h = tex.h;
in.gpu_texture = gl_tex;
in.common = is_common;
in.id = PcTextureId::from_combo_id(tex.combo_id);
in.src_data = (const u8*)tex.data.data();
pool.give_texture(in);
}
return gl_tex;
}
class TextureLoaderStage : public LoaderStage {
public:
TextureLoaderStage() : LoaderStage("texture") {}
bool run(Timer& timer, LoaderInput& data) override {
constexpr int MAX_TEX_BYTES_PER_FRAME = 1024 * 512;
int bytes_this_run = 0;
int tex_this_run = 0;
if (data.lev_data->textures.size() < data.lev_data->level->textures.size()) {
std::unique_lock<std::mutex> tpool_lock(data.tex_pool->mutex());
while (data.lev_data->textures.size() < data.lev_data->level->textures.size()) {
auto& tex = data.lev_data->level->textures[data.lev_data->textures.size()];
data.lev_data->textures.push_back(add_texture(*data.tex_pool, tex, false));
bytes_this_run += tex.w * tex.h * 4;
tex_this_run++;
if (tex_this_run > 20) {
break;
}
if (bytes_this_run > MAX_TEX_BYTES_PER_FRAME || timer.getMs() > LOAD_BUDGET) {
break;
}
}
}
return data.lev_data->textures.size() == data.lev_data->level->textures.size();
}
void reset() override {}
};
class TfragLoadStage : public LoaderStage {
public:
TfragLoadStage() : LoaderStage("tfrag") {}
bool run(Timer& timer, LoaderInput& data) override {
if (m_done) {
return true;
}
if (data.lev_data->level->tfrag_trees.front().empty()) {
m_done = true;
return true;
}
if (!m_opengl_created) {
for (int geo = 0; geo < tfrag3::TFRAG_GEOS; geo++) {
auto& in_trees = data.lev_data->level->tfrag_trees[geo];
for (auto& in_tree : in_trees) {
GLuint& tree_out = data.lev_data->tfrag_vertex_data[geo].emplace_back();
glGenBuffers(1, &tree_out);
glBindBuffer(GL_ARRAY_BUFFER, tree_out);
glBufferData(GL_ARRAY_BUFFER,
in_tree.unpacked.vertices.size() * sizeof(tfrag3::PreloadedVertex), nullptr,
GL_STATIC_DRAW);
}
}
m_opengl_created = true;
return false;
}
constexpr u32 CHUNK_SIZE = 32768;
u32 uploaded_bytes = 0;
u32 unique_buffers = 0;
while (true) {
const auto& tree = data.lev_data->level->tfrag_trees[m_next_geo][m_next_tree];
u32 end_vert_in_tree = tree.unpacked.vertices.size();
// the number of vertices we'd need to finish the tree right now
size_t num_verts_left_in_tree = end_vert_in_tree - m_next_vert;
size_t start_vert_for_chunk;
size_t end_vert_for_chunk;
bool complete_tree;
if (num_verts_left_in_tree > CHUNK_SIZE) {
complete_tree = false;
// should only do partial
start_vert_for_chunk = m_next_vert;
end_vert_for_chunk = start_vert_for_chunk + CHUNK_SIZE;
m_next_vert += CHUNK_SIZE;
} else {
// should do all!
start_vert_for_chunk = m_next_vert;
end_vert_for_chunk = end_vert_in_tree;
complete_tree = true;
}
glBindBuffer(GL_ARRAY_BUFFER, data.lev_data->tfrag_vertex_data[m_next_geo][m_next_tree]);
u32 upload_size =
(end_vert_for_chunk - start_vert_for_chunk) * sizeof(tfrag3::PreloadedVertex);
glBufferSubData(GL_ARRAY_BUFFER, start_vert_for_chunk * sizeof(tfrag3::PreloadedVertex),
upload_size, tree.unpacked.vertices.data() + start_vert_for_chunk);
uploaded_bytes += upload_size;
if (complete_tree) {
unique_buffers++;
// and move on to next tree
m_next_vert = 0;
m_next_tree++;
if (m_next_tree >= data.lev_data->level->tfrag_trees[m_next_geo].size()) {
m_next_tree = 0;
m_next_geo++;
if (m_next_geo >= tfrag3::TFRAG_GEOS) {
m_next_tree = true;
m_next_tree = 0;
m_next_geo = 0;
m_next_vert = 0;
m_done = true;
return true;
}
}
return false;
}
if (timer.getMs() > LOAD_BUDGET || (uploaded_bytes / 1024) > 2048) {
return false;
}
}
}
void reset() override {
m_done = false;
m_opengl_created = false;
m_next_geo = 0;
m_next_tree = 0;
m_next_vert = 0;
}
private:
bool m_done = false;
bool m_opengl_created = false;
u32 m_next_geo = 0;
u32 m_next_tree = 0;
u32 m_next_vert = 0;
};
class ShrubLoadStage : public LoaderStage {
public:
ShrubLoadStage() : LoaderStage("shrub") {}
bool run(Timer& timer, LoaderInput& data) override {
if (m_done) {
return true;
}
if (data.lev_data->level->shrub_trees.empty()) {
m_done = true;
return true;
}
if (!m_opengl_created) {
for (auto& in_tree : data.lev_data->level->shrub_trees) {
GLuint& tree_out = data.lev_data->shrub_vertex_data.emplace_back();
glGenBuffers(1, &tree_out);
glBindBuffer(GL_ARRAY_BUFFER, tree_out);
glBufferData(GL_ARRAY_BUFFER,
in_tree.unpacked.vertices.size() * sizeof(tfrag3::ShrubGpuVertex), nullptr,
GL_STATIC_DRAW);
}
m_opengl_created = true;
return false;
}
constexpr u32 CHUNK_SIZE = 32768;
u32 uploaded_bytes = 0;
while (true) {
const auto& tree = data.lev_data->level->shrub_trees[m_next_tree];
u32 end_vert_in_tree = tree.unpacked.vertices.size();
// the number of vertices we'd need to finish the tree right now
size_t num_verts_left_in_tree = end_vert_in_tree - m_next_vert;
size_t start_vert_for_chunk;
size_t end_vert_for_chunk;
bool complete_tree;
if (num_verts_left_in_tree > CHUNK_SIZE) {
complete_tree = false;
// should only do partial
start_vert_for_chunk = m_next_vert;
end_vert_for_chunk = start_vert_for_chunk + CHUNK_SIZE;
m_next_vert += CHUNK_SIZE;
} else {
// should do all!
start_vert_for_chunk = m_next_vert;
end_vert_for_chunk = end_vert_in_tree;
complete_tree = true;
}
glBindBuffer(GL_ARRAY_BUFFER, data.lev_data->shrub_vertex_data[m_next_tree]);
u32 upload_size =
(end_vert_for_chunk - start_vert_for_chunk) * sizeof(tfrag3::ShrubGpuVertex);
glBufferSubData(GL_ARRAY_BUFFER, start_vert_for_chunk * sizeof(tfrag3::ShrubGpuVertex),
upload_size, tree.unpacked.vertices.data() + start_vert_for_chunk);
uploaded_bytes += upload_size;
if (complete_tree) {
// and move on to next tree
m_next_vert = 0;
m_next_tree++;
if (m_next_tree >= data.lev_data->level->shrub_trees.size()) {
m_done = true;
return true;
}
}
if (timer.getMs() > LOAD_BUDGET || (uploaded_bytes / 128) > 2048) {
return false;
}
}
}
void reset() override {
m_done = false;
m_opengl_created = false;
m_next_tree = 0;
m_next_vert = 0;
}
private:
bool m_done = false;
bool m_opengl_created = false;
u32 m_next_tree = 0;
u32 m_next_vert = 0;
};
class TieLoadStage : public LoaderStage {
public:
TieLoadStage() : LoaderStage("tie") {}
bool run(Timer& timer, LoaderInput& data) override {
if (m_done) {
return true;
}
if (data.lev_data->level->tie_trees.front().empty()) {
m_done = true;
return true;
}
if (!m_opengl_created) {
for (int geo = 0; geo < tfrag3::TIE_GEOS; geo++) {
auto& in_trees = data.lev_data->level->tie_trees[geo];
for (auto& in_tree : in_trees) {
LevelData::TieOpenGL& tree_out = data.lev_data->tie_data[geo].emplace_back();
glGenBuffers(1, &tree_out.vertex_buffer);
glBindBuffer(GL_ARRAY_BUFFER, tree_out.vertex_buffer);
glBufferData(GL_ARRAY_BUFFER,
in_tree.unpacked.vertices.size() * sizeof(tfrag3::PreloadedVertex), nullptr,
GL_STATIC_DRAW);
}
}
m_opengl_created = true;
return false;
}
if (!m_verts_done) {
constexpr u32 CHUNK_SIZE = 32768;
u32 uploaded_bytes = 0;
while (true) {
const auto& tree = data.lev_data->level->tie_trees[m_next_geo][m_next_tree];
u32 end_vert_in_tree = tree.unpacked.vertices.size();
// the number of vertices we'd need to finish the tree right now
size_t num_verts_left_in_tree = end_vert_in_tree - m_next_vert;
size_t start_vert_for_chunk;
size_t end_vert_for_chunk;
bool complete_tree;
if (num_verts_left_in_tree > CHUNK_SIZE) {
complete_tree = false;
// should only do partial
start_vert_for_chunk = m_next_vert;
end_vert_for_chunk = start_vert_for_chunk + CHUNK_SIZE;
m_next_vert += CHUNK_SIZE;
} else {
// should do all!
start_vert_for_chunk = m_next_vert;
end_vert_for_chunk = end_vert_in_tree;
complete_tree = true;
}
glBindBuffer(GL_ARRAY_BUFFER,
data.lev_data->tie_data[m_next_geo][m_next_tree].vertex_buffer);
u32 upload_size =
(end_vert_for_chunk - start_vert_for_chunk) * sizeof(tfrag3::PreloadedVertex);
glBufferSubData(GL_ARRAY_BUFFER, start_vert_for_chunk * sizeof(tfrag3::PreloadedVertex),
upload_size, tree.unpacked.vertices.data() + start_vert_for_chunk);
uploaded_bytes += upload_size;
if (complete_tree) {
// and move on to next tree
m_next_vert = 0;
m_next_tree++;
if (m_next_tree >= data.lev_data->level->tie_trees[m_next_geo].size()) {
m_next_tree = 0;
m_next_geo++;
if (m_next_geo >= tfrag3::TIE_GEOS) {
m_verts_done = true;
m_next_tree = 0;
m_next_geo = 0;
m_next_vert = 0;
return false;
}
}
}
if (timer.getMs() > LOAD_BUDGET || (uploaded_bytes / 1024) > 2048) {
return false;
}
}
}
if (!m_wind_indices_done) {
bool abort = false;
for (; m_next_geo < tfrag3::TIE_GEOS; m_next_geo++) {
auto& geo_trees = data.lev_data->level->tie_trees[m_next_geo];
for (; m_next_tree < geo_trees.size(); m_next_tree++) {
if (abort) {
return false;
}
auto& in_tree = geo_trees[m_next_tree];
auto& out_tree = data.lev_data->tie_data[m_next_geo][m_next_tree];
size_t wind_idx_buffer_len = 0;
for (auto& draw : in_tree.instanced_wind_draws) {
wind_idx_buffer_len += draw.vertex_index_stream.size();
}
if (wind_idx_buffer_len > 0) {
out_tree.has_wind = true;
glGenBuffers(1, &out_tree.wind_indices);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, out_tree.wind_indices);
std::vector<u32> temp;
temp.resize(wind_idx_buffer_len);
u32 off = 0;
for (auto& draw : in_tree.instanced_wind_draws) {
memcpy(temp.data() + off, draw.vertex_index_stream.data(),
draw.vertex_index_stream.size() * sizeof(u32));
off += draw.vertex_index_stream.size();
}
glBufferData(GL_ELEMENT_ARRAY_BUFFER, wind_idx_buffer_len * sizeof(u32), temp.data(),
GL_STATIC_DRAW);
abort = true;
}
}
m_next_tree = 0;
}
m_indices_done = true;
m_done = true;
return true;
}
return false;
}
void reset() override {
m_done = false;
m_opengl_created = false;
m_next_geo = 0;
m_next_tree = 0;
m_next_vert = 0;
m_verts_done = false;
m_indices_done = false;
m_wind_indices_done = false;
}
private:
bool m_done = false;
bool m_opengl_created = false;
bool m_verts_done = false;
bool m_indices_done = false;
bool m_wind_indices_done = false;
u32 m_next_geo = 0;
u32 m_next_tree = 0;
u32 m_next_vert = 0;
};
class CollideLoaderStage : public LoaderStage {
public:
CollideLoaderStage() : LoaderStage("collide") {}
bool run(Timer& /*timer*/, LoaderInput& data) override {
if (m_done) {
return true;
}
if (!m_opengl_created) {
glGenBuffers(1, &data.lev_data->collide_vertices);
glBindBuffer(GL_ARRAY_BUFFER, data.lev_data->collide_vertices);
glBufferData(
GL_ARRAY_BUFFER,
data.lev_data->level->collision.vertices.size() * sizeof(tfrag3::CollisionMesh::Vertex),
nullptr, GL_STATIC_DRAW);
m_opengl_created = true;
return false;
}
u32 start = m_vtx;
u32 end = std::min((u32)data.lev_data->level->collision.vertices.size(), start + 32768);
glBindBuffer(GL_ARRAY_BUFFER, data.lev_data->collide_vertices);
glBufferSubData(GL_ARRAY_BUFFER, start * sizeof(tfrag3::CollisionMesh::Vertex),
(end - start) * sizeof(tfrag3::CollisionMesh::Vertex),
data.lev_data->level->collision.vertices.data() + start);
m_vtx = end;
if (m_vtx == data.lev_data->level->collision.vertices.size()) {
m_done = true;
return true;
} else {
return false;
}
}
void reset() override {
m_opengl_created = false;
m_vtx = 0;
m_done = false;
}
private:
bool m_opengl_created = false;
u32 m_vtx = 0;
bool m_done = false;
};
MercLoaderStage::MercLoaderStage() : LoaderStage("merc") {}
void MercLoaderStage::reset() {
m_done = false;
m_opengl = false;
m_vtx_uploaded = false;
m_idx = 0;
}
bool MercLoaderStage::run(Timer& /*timer*/, LoaderInput& data) {
if (m_done) {
return true;
}
if (!m_opengl) {
glGenBuffers(1, &data.lev_data->merc_indices);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, data.lev_data->merc_indices);
glBufferData(GL_ELEMENT_ARRAY_BUFFER,
data.lev_data->level->merc_data.indices.size() * sizeof(u32), nullptr,
GL_STATIC_DRAW);
glGenBuffers(1, &data.lev_data->merc_vertices);
glBindBuffer(GL_ARRAY_BUFFER, data.lev_data->merc_vertices);
glBufferData(GL_ARRAY_BUFFER,
data.lev_data->level->merc_data.vertices.size() * sizeof(tfrag3::MercVertex),
nullptr, GL_STATIC_DRAW);
m_opengl = true;
}
if (!m_vtx_uploaded) {
u32 start = m_idx;
m_idx = std::min(start + 32768, (u32)data.lev_data->level->merc_data.indices.size());
glBindBuffer(GL_ARRAY_BUFFER, data.lev_data->merc_indices);
glBufferSubData(GL_ARRAY_BUFFER, start * sizeof(u32), (m_idx - start) * sizeof(u32),
data.lev_data->level->merc_data.indices.data() + start);
if (m_idx != data.lev_data->level->merc_data.indices.size()) {
return false;
} else {
m_idx = 0;
m_vtx_uploaded = true;
}
}
u32 start = m_idx;
m_idx = std::min(start + 32768, (u32)data.lev_data->level->merc_data.vertices.size());
glBindBuffer(GL_ARRAY_BUFFER, data.lev_data->merc_vertices);
glBufferSubData(GL_ARRAY_BUFFER, start * sizeof(tfrag3::MercVertex),
(m_idx - start) * sizeof(tfrag3::MercVertex),
data.lev_data->level->merc_data.vertices.data() + start);
if (m_idx != data.lev_data->level->merc_data.vertices.size()) {
return false;
} else {
m_done = true;
for (auto& model : data.lev_data->level->merc_data.models) {
data.lev_data->merc_model_lookup[model.name] = &model;
(*data.mercs)[model.name].push_back({&model, data.lev_data->load_id, data.lev_data});
}
return true;
}
return true;
}
std::vector<std::unique_ptr<LoaderStage>> make_loader_stages() {
std::vector<std::unique_ptr<LoaderStage>> ret;
ret.push_back(std::make_unique<TieLoadStage>());
ret.push_back(std::make_unique<TextureLoaderStage>());
ret.push_back(std::make_unique<TfragLoadStage>());
ret.push_back(std::make_unique<ShrubLoadStage>());
ret.push_back(std::make_unique<CollideLoaderStage>());
ret.push_back(std::make_unique<MercLoaderStage>());
return ret;
}
game/graphics/opengl_renderer/loader/LoaderStages.h
+19
View File
@@ -0,0 +1,19 @@
#pragma once
#include "game/graphics/opengl_renderer/loader/common.h"
std::vector<std::unique_ptr<LoaderStage>> make_loader_stages();
u64 add_texture(TexturePool& pool, const tfrag3::Texture& tex, bool is_common);
class MercLoaderStage : public LoaderStage {
public:
MercLoaderStage();
bool run(Timer& timer, LoaderInput& data) override;
void reset() override;
private:
bool m_done = false;
bool m_opengl = false;
bool m_vtx_uploaded = false;
u32 m_idx = 0;
};
game/graphics/opengl_renderer/loader/common.h
+50
View File
@@ -0,0 +1,50 @@
#pragma once
struct LevelData {
std::unique_ptr<tfrag3::Level> level;
std::vector<GLuint> textures;
u64 load_id = UINT64_MAX;
struct TieOpenGL {
GLuint vertex_buffer;
bool has_wind = false;
GLuint wind_indices;
};
std::array<std::vector<TieOpenGL>, tfrag3::TIE_GEOS> tie_data;
std::array<std::vector<GLuint>, tfrag3::TIE_GEOS> tfrag_vertex_data;
std::vector<GLuint> shrub_vertex_data;
GLuint collide_vertices;
GLuint merc_vertices;
GLuint merc_indices;
std::unordered_map<std::string, const tfrag3::MercModel*> merc_model_lookup;
int frames_since_last_used = 0;
};
struct MercRef {
const tfrag3::MercModel* model = nullptr;
u64 load_id = 0;
const LevelData* level = nullptr;
bool operator==(const MercRef& other) const {
return model == other.model && load_id == other.load_id;
}
};
struct LoaderInput {
LevelData* lev_data;
TexturePool* tex_pool;
std::unordered_map<std::string, std::vector<MercRef>>* mercs;
};
class LoaderStage {
public:
LoaderStage(const std::string& name) : m_name(name) {}
virtual bool run(Timer& timer, LoaderInput& data) = 0;
virtual void reset() = 0;
virtual ~LoaderStage() = default;
const std::string& name() const { return m_name; }
protected:
std::string m_name;
};
game/sce/sif_ee.cpp
+2 -2
View File
@@ -81,12 +81,12 @@ s32 sceSifCallRpc(sceSifClientData* bd,
ASSERT(!end_para);
ASSERT(mode == 1); // async
iop->kernel.sif_rpc(bd->rpcd.id, fno, mode, send, ssize, recv, rsize);
iop->signal_run_iop();
iop->signal_run_iop(false);
return 0;
}
s32 sceSifCheckStatRpc(sceSifRpcData* bd) {
iop->signal_run_iop();
iop->signal_run_iop(false);
return iop->kernel.sif_busy(bd->id);
}
game/system/iop_thread.cpp
+9 -65
View File
@@ -68,76 +68,20 @@ void IOP::wait_run_iop() {
void IOP::kill_from_ee() {
want_exit = true;
signal_run_iop();
signal_run_iop(true);
}
void IOP::signal_run_iop() {
void IOP::signal_run_iop(bool force) {
std::unique_lock<std::mutex> lk(iters_mutex);
iop_iters_des++; // todo, tune this
if (iop_iters_des - iop_iters_act > 500) {
iop_iters_des = iop_iters_act + 500;
if (iop_iters_act == iop_iters_des || force) {
iop_iters_des++; // todo, tune this
if (iop_iters_des - iop_iters_act > 500) {
iop_iters_des = iop_iters_act + 500;
}
iop_run_cv.notify_all();
}
iop_run_cv.notify_all();
}
IOP::~IOP() {
reset_allocator();
}
// void launch_iop(SystemThreadInterface& interface) {
// IOP iop;
//
// printf("\n\n\n[IOP] Restart!\n");
// iop.reset_allocator();
//
//// dma_init_globals();
//// iso_init_globals();
//// fake_iso_init_globals();
//// // iso_api
//// iso_cd_init_globals();
//// iso_queue_init_globals();
//// // isocommon
//// // overlord
//// ramdisk_init_globals();
//// // sbank
//// // soundcommon
//// srpc_init_globals();
// // ssound
// // stream
//
//// SCE_IOP::PS2_RegisterIOP(&iop);
//// PS2_RegisterIOP_EE(&iop);
// interface.initialization_complete();
//
// printf("[IOP] Wait for OVERLORD to be started...\n");
// iop.wait_for_overlord_start_cmd();
// if(iop.status == IOP_OVERLORD_INIT) {
// printf("[IOP] Run!\n");
// } else {
// printf("[IOP] shutdown!\n");
// return;
// }
//
// iop.reset_allocator();
//
// // init
//#ifdef ENABLE_OVERLORD
// start(iop.overlord_argc, iop.overlord_argv);
//#endif
//
// // unblock the EE, the overlord is set up!
// iop.signal_overlord_init_finish();
//
// // IOP Kernel loop
// while(!interface.get_want_exit() && !iop.want_exit) {
// // the IOP kernel just runs at full blast, so we only run the IOP when the EE is waiting on
// the IOP.
// // Each time the EE is waiting on the IOP, it will run an iteration of the IOP kernel.
// iop.wait_run_iop();
// iop.kernel.dispatchAll();
// }
//
// // stop all threads in the iop kernel.
// // if the threads are not stopped nicely, we will deadlock on trying to destroy the kernel's
// condition variables. iop.kernel.shutdown();
//}
}
game/system/iop_thread.h
+1 -1
View File
@@ -18,7 +18,7 @@ class IOP {
void wait_for_overlord_start_cmd();
void wait_for_overlord_init_finish();
void signal_overlord_init_finish();
void signal_run_iop();
void signal_run_iop(bool force);
void wait_run_iop();
void kill_from_ee();