Minor bug fixes (#2128)

- make sure bsp is processed on `l` levels in extraction (caused missing
remaps)
- clean up a few prints in extraction
- handle the <15 byte differences in art group files automatically (no
more errors about file naming)
- fix potential exception thrown by merc2 in a few ways: fixed bad data
in FR3's, check texture index just in case, and handle exceptions a
little bit better (still a crash, but at least you get a print)
- fix mips2 ocean stuff causing ocean far crashes
This commit is contained in:
water111
2023-01-14 16:26:17 -05:00
committed by GitHub
parent 8a82c2225e
commit a0d2bce27b
20 changed files with 181 additions and 88 deletions
+8
View File
@@ -16,6 +16,14 @@ bool starts_with(const std::string& s, const std::string& prefix) {
return s.rfind(prefix) == 0;
}
bool ends_with(const std::string& s, const std::string& suffix) {
if (s.length() >= suffix.length()) {
return !s.compare(s.length() - suffix.length(), suffix.length(), suffix);
} else {
return false;
}
}
std::string ltrim(const std::string& s) {
size_t start = s.find_first_not_of(WHITESPACE);
return (start == std::string::npos) ? "" : s.substr(start);
+1
View File
@@ -6,6 +6,7 @@
namespace str_util {
bool contains(const std::string& s, const std::string& substr);
bool starts_with(const std::string& s, const std::string& prefix);
bool ends_with(const std::string& s, const std::string& prefix);
std::string ltrim(const std::string& s);
std::string rtrim(const std::string& s);
std::string trim(const std::string& s);
@@ -231,6 +231,7 @@ static void link_v2_or_v4(LinkedObjectFile& f,
const std::string& name,
DecompilerTypeSystem& dts,
GameVersion version) {
(void)name;
const auto* header = (const LinkHeaderV4*)&data.at(0);
ASSERT(header->version == 4 || header->version == 2);
@@ -313,7 +314,8 @@ static void link_v2_or_v4(LinkedObjectFile& f,
for (uint8_t i = 0; i < count; i++) {
if (!f.pointer_link_word(0, code_ptr_offset - code_offset, 0,
*((const uint32_t*)(&data.at(code_ptr_offset))))) {
lg::error("Skipping link in {} because it is out of range!", name.c_str());
// was this just a bug in the linker??
// lg::error("Skipping link in {} because it is out of range!", name.c_str());
}
f.stats.total_v2_pointers++;
code_ptr_offset += 4;
@@ -680,13 +682,6 @@ static void link_v3(LinkedObjectFile& f,
segment_size++;
adjusted = true;
}
if (adjusted) {
printf(
"Adjusted the size of segment %d in %s, this is fine, but rare (and may indicate a "
"bigger problem if it happens often)\n",
seg_id, name.c_str());
}
}
auto base_ptr = segment_data_offsets[seg_id];
+50 -4
View File
@@ -312,6 +312,55 @@ void ObjectFileDB::get_objs_from_dgo(const fs::path& filename, const Config& con
ASSERT(0 == reader.bytes_left());
}
/*!
* Are two object files the same?
* Unfortunately they seemed to have a memory bug in their art-group generator, so there's some
* uninitialized padding bytes.
*/
bool are_objects_the_same(const std::string& obj_name,
size_t size_a,
const u8* a,
size_t size_b,
const u8* b) {
if (size_a != size_b) {
return false;
}
// if they are byte-for-byte the same, it's a match
if (!memcmp(a, b, size_a)) {
return true;
}
// if it's an art group...
if (obj_name.size() > 3 && !obj_name.compare(obj_name.length() - 3, 3, "-ag")) {
// count up the number of differing bytes, and the location of the first one.
size_t first_diff = 0;
size_t last_diff = 0;
int num_diffs = 0;
bool found_first_diff = false;
for (size_t i = 0; i < size_a; i++) {
if (a[i] != b[i]) {
num_diffs++;
last_diff = i;
if (!found_first_diff) {
first_diff = i;
found_first_diff = true;
}
}
}
// find the gap between "code" (really data here) and link table. This has up to 15 bytes of
// uninitialized memory.
const auto* header = (const LinkHeaderV4*)a;
int link_data_offset = header->code_size + sizeof(LinkHeaderV4);
int start_off_diff_from_code_end = link_data_offset - (int)first_diff;
if (num_diffs < 16 && start_off_diff_from_code_end < 16 && (int)last_diff < link_data_offset) {
return true;
}
}
return false;
}
/*!
* Add an object file to the ObjectFileDB
*/
@@ -337,10 +386,7 @@ void ObjectFileDB::add_obj_from_dgo(const std::string& obj_name,
bool duplicated = false;
// first, check to see if we already got it...
for (auto& e : obj_files_by_name[obj_name]) {
if (e.data.size() == obj_size && e.record.hash == hash) {
// just to make sure we don't have a hash collision.
ASSERT(!memcmp(obj_data, e.data.data(), obj_size));
if (are_objects_the_same(obj_name, e.data.size(), e.data.data(), obj_size, obj_data)) {
// already got it!
e.reference_count++;
auto& rec = e.record;
+1 -1
View File
@@ -2047,7 +2047,7 @@ FormElement* rewrite_with_dma_buf_add_bucket(LetElement* in, const Env& env, For
last_part = dynamic_cast<LetElement*>(in->body()->at(in->body()->size() - 1));
if (!last_part) {
lg::error("NO LAST PART AHH wtf!!");
// lg::error("NO LAST PART AHH wtf!!");
return nullptr;
}
+4 -1
View File
@@ -42,7 +42,7 @@
// "DGO/DRB.DGO",
"DGO/ATE.DGO",
// "DGO/LERROL.DGO",
// "DGO/LTRNYSAM.DGO",
"DGO/LTRNYSAM.DGO",
// "DGO/LOUTCSTB.DGO",
// "DGO/LASHTHRN.DGO",
"DGO/TOC.DGO",
@@ -193,6 +193,8 @@
"CTB.DGO",
"CTC.DGO",
"CWI.DGO",
"ATE.DGO",
"ATO.DGO",
"D3A.DGO",
"D3B.DGO",
"DG1.DGO",
@@ -210,6 +212,7 @@
"KIOSK.DGO",
"LDJAKBRN.DGO",
"LGUARD.DGO",
"LTRNYSAM.DGO",
"LPROTECT.DGO",
"LSACK.DGO",
"LWIDEA.DGO",
-3
View File
@@ -933,7 +933,6 @@ void PrototypeBucketTie::read_from_file(TypedRef ref,
break;
case GameVersion::Jak2:
flags = read_plain_data_field<u16>(ref, "flags", dts);
lg::print("flag: {}\n", flags);
break;
default:
ASSERT(false);
@@ -954,8 +953,6 @@ void PrototypeBucketTie::read_from_file(TypedRef ref,
collide_frag.read_from_file(typed_ref_from_basic(p, dts), dts, stats, version);
}
}
} else {
lg::warn("Skipping prototype-bucket-tie collision");
}
auto next_slot = get_field_ref(ref, "next", dts);
+1
View File
@@ -451,6 +451,7 @@ struct PrototypeTie : public DrawableInlineArray {
// a prototype bucket is a collection of 4 different prototypes (called geometries), one for each
// level of detail. All geometries share the same time of day palette.
// the bucket also refers to the fact that it collect instances during actual rendering.
// Note: collision extraction is only supported in jak 1.
struct PrototypeBucketTie {
std::string name; // 4 - 8
u32 flags; // 8 - 12
+16 -2
View File
@@ -7,6 +7,7 @@
#include "common/util/FileUtil.h"
#include "common/util/SimpleThreadGroup.h"
#include "common/util/compress.h"
#include "common/util/string_util.h"
#include "decompiler/level_extractor/BspHeader.h"
#include "decompiler/level_extractor/extract_collide_frags.h"
@@ -21,7 +22,8 @@ namespace decompiler {
/*!
* Look through files in a DGO and find the bsp-header file (the level)
*/
std::optional<ObjectFileRecord> get_bsp_file(const std::vector<ObjectFileRecord>& records) {
std::optional<ObjectFileRecord> get_bsp_file(const std::vector<ObjectFileRecord>& records,
const std::string& dgo_name) {
std::optional<ObjectFileRecord> result;
bool found = false;
for (auto& file : records) {
@@ -31,6 +33,18 @@ std::optional<ObjectFileRecord> get_bsp_file(const std::vector<ObjectFileRecord>
result = file;
}
}
if (!result) {
if (str_util::ends_with(dgo_name, ".DGO") || str_util::ends_with(dgo_name, ".CGO")) {
auto expected_name = dgo_name.substr(0, dgo_name.length() - 4);
for (auto& c : expected_name) {
c = tolower(c);
}
if (!records.empty() && expected_name == records.back().name) {
return records.back();
}
}
}
return result;
}
@@ -116,7 +130,7 @@ std::vector<level_tools::TextureRemap> extract_bsp_from_level(const ObjectFileDB
const DecompileHacks& hacks,
bool extract_collision,
tfrag3::Level& level_data) {
auto bsp_rec = get_bsp_file(db.obj_files_by_dgo.at(dgo_name));
auto bsp_rec = get_bsp_file(db.obj_files_by_dgo.at(dgo_name), dgo_name);
if (!bsp_rec) {
lg::warn("Skipping extract for {} because the BSP file was not found", dgo_name);
return {};
+4 -2
View File
@@ -938,8 +938,10 @@ void extract_merc(const ObjectFileData& ag_data,
// not added to level, add it
auto tex_it = tex_db.textures.find(draw.state.merc_draw_mode.pc_combo_tex_id);
if (tex_it == tex_db.textures.end()) {
lg::error("failed to find texture: 0x{:x} for {}\n",
draw.state.merc_draw_mode.pc_combo_tex_id, ctrl.name);
lg::error("merc failed to find texture: 0x{:x} for {}. Should be in tpage {}",
draw.state.merc_draw_mode.pc_combo_tex_id, ctrl.name,
draw.state.merc_draw_mode.pc_combo_tex_id >> 16);
idx_in_level_texture = 0;
} else {
idx_in_level_texture = out.textures.size();
auto& new_tex = out.textures.emplace_back();
+35 -34
View File
@@ -481,7 +481,6 @@ u32 remap_texture(u32 original, const std::vector<level_tools::TextureRemap>& ma
*/
void update_proto_info(std::vector<TieProtoInfo>* out,
const std::vector<level_tools::TextureRemap>& map,
const TextureDB& tdb,
const std::vector<level_tools::PrototypeBucketTie>& protos,
int geo) {
out->resize(std::max(out->size(), protos.size()));
@@ -557,10 +556,7 @@ void update_proto_info(std::vector<TieProtoInfo>* out,
u32 tpage = new_tex >> 20;
u32 tidx = (new_tex >> 8) & 0b1111'1111'1111;
u32 tex_combo = (((u32)tpage) << 16) | tidx;
// look up the texture to make sure it's valid
auto tex = tdb.textures.find(tex_combo);
// ASSERT(tex != tdb.textures.end());
// remember the texture id
// remember the texture id (may be invalid, will be checked later)
adgif.combo_tex = tex_combo;
// and the hidden value in the unused a+d
memcpy(&adgif.second_w, &gif_data.at(16 * (tex_idx * 5 + 1) + 12), 4);
@@ -2056,9 +2052,7 @@ void add_vertices_and_static_draw(tfrag3::TieTree& tree,
for (auto& vert : strip.verts) {
tree.packed_vertices.vertices.push_back(
{vert.pos.x(), vert.pos.y(), vert.pos.z(), vert.tex.x(), vert.tex.y()});
if (vert.tex.z() != 1.0) {
lg::warn("SKIPPING ASSERT in extract_tie! TODO!");
}
// TODO: check if this means anything.
// ASSERT(vert.tex.z() == 1.);
}
int end = tree.packed_vertices.vertices.size();
@@ -2105,33 +2099,40 @@ void add_vertices_and_static_draw(tfrag3::TieTree& tree,
}
if (idx_in_lev_data == UINT32_MAX) {
// didn't find it, have to add a new one texture.
auto tex_it = tdb.textures.find(combo_tex);
if (tex_it == tdb.textures.end()) {
bool ok_to_miss = false; // for TIE, there's no missing textures.
if (ok_to_miss) {
// we're missing a texture, just use the first one.
tex_it = tdb.textures.begin();
} else {
ASSERT_MSG(
false,
fmt::format(
"texture {} wasn't found. make sure it is loaded somehow. You may need to "
"include ART.DGO or GAME.DGO in addition to the level DGOs for shared "
"textures. tpage is {}. id is {} (0x{:x})",
combo_tex, combo_tex >> 16, combo_tex & 0xffff, combo_tex & 0xffff));
if (combo_tex == 0) {
lg::warn("unhandled texture 0 case in extract_tie for {} {}", lev.level_name,
proto.name);
idx_in_lev_data = 0;
} else {
// didn't find it, have to add a new one texture.
auto tex_it = tdb.textures.find(combo_tex);
if (tex_it == tdb.textures.end()) {
bool ok_to_miss = false; // for TIE, there's no missing textures.
if (ok_to_miss) {
// we're missing a texture, just use the first one.
tex_it = tdb.textures.begin();
} else {
ASSERT_MSG(
false,
fmt::format(
"texture {} wasn't found. make sure it is loaded somehow. You may need "
"to "
"include ART.DGO or GAME.DGO in addition to the level DGOs for shared "
"textures. tpage is {}. id is {} (0x{:x})",
combo_tex, combo_tex >> 16, combo_tex & 0xffff, combo_tex & 0xffff));
}
}
// add a new texture to the level data
idx_in_lev_data = lev.textures.size();
lev.textures.emplace_back();
auto& new_tex = lev.textures.back();
new_tex.combo_id = combo_tex;
new_tex.w = tex_it->second.w;
new_tex.h = tex_it->second.h;
new_tex.debug_name = tex_it->second.name;
new_tex.debug_tpage_name = tdb.tpage_names.at(tex_it->second.page);
new_tex.data = tex_it->second.rgba_bytes;
}
// add a new texture to the level data
idx_in_lev_data = lev.textures.size();
lev.textures.emplace_back();
auto& new_tex = lev.textures.back();
new_tex.combo_id = combo_tex;
new_tex.w = tex_it->second.w;
new_tex.h = tex_it->second.h;
new_tex.debug_name = tex_it->second.name;
new_tex.debug_tpage_name = tdb.tpage_names.at(tex_it->second.page);
new_tex.data = tex_it->second.rgba_bytes;
}
// determine the draw mode
@@ -2339,7 +2340,7 @@ void extract_tie(const level_tools::DrawableTreeInstanceTie* tree,
// convert level format data to a nicer format
auto info =
collect_instance_info(as_instance_array, &tree->prototypes.prototype_array_tie.data, geo);
update_proto_info(&info, tex_map, tex_db, tree->prototypes.prototype_array_tie.data, geo);
update_proto_info(&info, tex_map, tree->prototypes.prototype_array_tie.data, geo);
if (version != GameVersion::Jak2) {
check_wind_vectors_zero(info, tree->prototypes.wind_vectors);
}
@@ -196,6 +196,8 @@ void OpenGLRenderer::init_bucket_renderers_jak2() {
init_bucket_renderer<TextureUploadHandler>("tex-l4-pris", BucketCategory::TEX,
BucketId::TEX_L4_PRIS);
init_bucket_renderer<Merc2>("merc-l4-pris", BucketCategory::MERC, BucketId::MERC_L4_PRIS);
init_bucket_renderer<TextureUploadHandler>("tex-l5-pris", BucketCategory::TEX,
BucketId::TEX_L5_PRIS);
// 220
init_bucket_renderer<TextureUploadHandler>("tex-lcom-pris", BucketCategory::TEX,
BucketId::TEX_LCOM_PRIS);
@@ -205,6 +207,11 @@ void OpenGLRenderer::init_bucket_renderers_jak2() {
init_bucket_renderer<TextureUploadHandler>("tex-l1-pris2", BucketCategory::TEX,
BucketId::TEX_L1_PRIS2);
// 230
init_bucket_renderer<TextureUploadHandler>("tex-l2-pris2", BucketCategory::TEX,
BucketId::TEX_L2_PRIS2);
init_bucket_renderer<TextureUploadHandler>("tex-l3-pris2", BucketCategory::TEX,
BucketId::TEX_L3_PRIS2);
// 240
init_bucket_renderer<TextureUploadHandler>("tex-l4-pris2", BucketCategory::TEX,
BucketId::TEX_L4_PRIS2);
+3
View File
@@ -136,10 +136,13 @@ enum class BucketId {
MERC_L3_PRIS = 209,
TEX_L4_PRIS = 212,
MERC_L4_PRIS = 213,
TEX_L5_PRIS = 216,
TEX_LCOM_PRIS = 220,
MERC_LCOM_PRIS = 221,
TEX_L0_PRIS2 = 224,
TEX_L1_PRIS2 = 228,
TEX_L2_PRIS2 = 232,
TEX_L3_PRIS2 = 236,
TEX_L4_PRIS2 = 240,
TEX_L0_WATER = 252,
MERC_L0_WATER = 253,
@@ -640,7 +640,11 @@ void Merc2::flush_draw_buckets(SharedRenderState* /*render_state*/, ScopedProfil
auto& draw = lev_bucket.draws[di];
glUniform1i(m_uniforms.ignore_alpha, draw.ignore_alpha);
if ((int)draw.texture != last_tex) {
glBindTexture(GL_TEXTURE_2D, lev->textures.at(draw.texture));
if (draw.texture < lev->textures.size()) {
glBindTexture(GL_TEXTURE_2D, lev->textures.at(draw.texture));
} else {
fmt::print("Invalid draw.texture is {}, would have crashed.\n", draw.texture);
}
last_tex = draw.texture;
}
+1 -1
View File
@@ -981,7 +981,7 @@ u64 type_typep(Ptr<Type> t1, Ptr<Type> t2) {
u64 method_set(u32 type_, u32 method_id, u32 method) {
Ptr<Type> type(type_);
if (method_id > 127)
if (method_id > 255)
printf("[METHOD SET ERROR] tried to set method %d\n", method_id);
auto existing_method = type->get_method(method_id).offset;
+5 -3
View File
@@ -120,7 +120,8 @@ u64 execute(void* ctxt) {
c->mov64(a3, t4); // or a3, t4, r0
call_addr = c->gprs[t9].du32[0]; // function call:
c->daddiu(t2, a2, 4); // daddiu t2, a2, 4
c->jalr(call_addr); // jalr ra, t9
// c->jalr(call_addr); // jalr ra, t9
clip_polygon_against_positive_hyperplane::execute(ctxt);
bc = c->sgpr64(t0) == 0; // beq t0, r0, L124
c->load_symbol2(t9, cache.clip_polygon_against_negative_hyperplane);// lw t9, clip-polygon-against-negative-hyperplane(s7)
if (bc) {goto block_11;} // branch non-likely
@@ -130,7 +131,7 @@ u64 execute(void* ctxt) {
call_addr = c->gprs[t9].du32[0]; // function call:
c->daddu(t2, a2, r0); // daddu t2, a2, r0
//c->jalr(call_addr); // jalr ra, t9
clip_polygon_against_positive_hyperplane::execute(ctxt);
clip_polygon_against_negative_hyperplane::execute(ctxt);
bc = c->sgpr64(t0) == 0; // beq t0, r0, L124
// nop // sll r0, r0, 0
if (bc) {goto block_11;} // branch non-likely
@@ -139,7 +140,8 @@ u64 execute(void* ctxt) {
c->mov64(a3, t4); // or a3, t4, r0
call_addr = c->gprs[t9].du32[0]; // function call:
c->daddiu(t2, a2, 4); // daddiu t2, a2, 4
c->jalr(call_addr); // jalr ra, t9
// c->jalr(call_addr); // jalr ra, t9
clip_polygon_against_negative_hyperplane::execute(ctxt);
bc = c->sgpr64(t0) == 0; // beq t0, r0, L124
c->lw(a3, 4, a1); // lw a3, 4(a1)
if (bc) {goto block_11;} // branch non-likely
+7 -1
View File
@@ -365,7 +365,13 @@ RuntimeExitStatus exec_runtime(int argc, char** argv) {
// step 4: wait for EE to signal a shutdown. meanwhile, run video loop on main thread.
// TODO relegate this to its own function
if (enable_display) {
Gfx::Loop([]() { return MasterExit == RuntimeExitStatus::RUNNING; });
try {
Gfx::Loop([]() { return MasterExit == RuntimeExitStatus::RUNNING; });
} catch (std::exception& e) {
fmt::print("Exception thrown from graphics loop: {}\n", e.what());
fmt::print("Everything will crash now. good luck\n");
throw;
}
}
// hack to make the IOP die quicker if it's loading/unloading music
+7 -5
View File
@@ -1160,11 +1160,13 @@
(let ((f0-5 1.0)
(f30-0 1.0)
)
;; TODO - need bigmap
;; (when (= (-> *bigmap* bigmap-index) 13)
;; (set! f0-5 0.333)
;; (set! f30-0 0.75)
;; )
;; added check
(when (nonzero? *bigmap*)
(when (= (-> *bigmap* bigmap-index) 13)
(set! f0-5 0.333)
(set! f30-0 0.75)
)
)
(set! (-> obj frame-speed) (* (+ 4.0 (-> *setting-control* user-current rain)) f0-5))
(set! (-> obj frame-speed2) (* (+ 5.0 (-> *setting-control* user-current rain)) f0-5))
(when (not (paused?))
+22 -22
View File
@@ -2846,24 +2846,24 @@
;; "loutcstb"
;; )
;; ;;;;;;;;;;;;;;;;;;;;;
;; ;; LPACKAGE
;; ;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;
;; LPACKAGE
;;;;;;;;;;;;;;;;;;;;;
;; (cgo "LPACKAGE.DGO" "lpackage.gd")
(cgo "LPACKAGE.DGO" "lpackage.gd")
;; (goal-src-sequence
;; ""
;; :deps ("$OUT/obj/los-control.o")
;; "levels/city/misc/delivery/delivery-task.gc"
;; )
(goal-src-sequence
""
:deps ("$OUT/obj/los-control.o")
"levels/city/misc/delivery/delivery-task.gc"
)
;; (copy-textures 2459)
(copy-textures 2459)
;; (copy-gos
;; "krew-package-ag"
;; "lpackage"
;; )
(copy-gos
"krew-package-ag"
"lpackage"
)
;; ;;;;;;;;;;;;;;;;;;;;;
;; ;; LPORTRUN
@@ -3222,16 +3222,16 @@
;; ;; LTRNYSAM
;; ;;;;;;;;;;;;;;;;;;;;;
;; (cgo "LTRNYSAM.DGO" "ltrnysam.gd")
(cgo "LTRNYSAM.DGO" "ltrnysam.gd")
;; (copy-textures 1774 3255)
(copy-textures 1774 3255)
;; (copy-gos
;; "youngsamos-ltrnysam+0-ag"
;; "youngsamos-highres-ag"
;; "torn-highres-ag"
;; "ltrnysam"
;; )
(copy-gos
"youngsamos-ltrnysam+0-ag"
; "youngsamos-highres-ag"
; "torn-highres-ag"
"ltrnysam"
)
;; ;;;;;;;;;;;;;;;;;;;;;
;; ;; LWHACK
@@ -24,6 +24,7 @@
;; WARN: Return type mismatch object vs none.
(defmethod init-from-entity! hide-door-b ((obj hide-door-b) (arg0 entity-actor))
(stack-size-set! (-> obj main-thread) 1024)
(let ((s5-0 (new 'process 'collide-shape obj (collide-list-enum usually-hit-by-player))))
(set! (-> s5-0 penetrated-by) (penetrate))
(let ((s4-0 (new 'process 'collide-shape-prim-group s5-0 (the-as uint 2) 0)))