Mod SDK: Arc Overlays (#2280)

* Mod SDK: Arc Overlays

* Use DVD Functions for Arc overlays

* Re-add Fetchcontent for json.hpp

* Fix build for actions

* Make sure buffer usues aligned length

* Add include so msvc compiles

* Arc Overlays fixes: remove _arc suffix, handle getting overlayed file size from arcs, copy data correctly, etc.

* Reload overlayed data during loading screens

* Rename sync function

* Free overlayed files on load instead of re-loading them

* Rework arc overlays & load overlay files into host memory

---------

Co-authored-by: Luke Street <luke@street.dev>
This commit is contained in:
jdflyer
2026-08-19 23:47:57 -07:00
committed by GitHub
parent 386d2f964b
commit 5a4e6f3254
13 changed files with 474 additions and 29 deletions
@@ -6,11 +6,17 @@
#include "global.h"
#include "helpers/endian.h"
#if TARGET_PC
#include <atomic>
#include <string>
#include <unordered_map>
#endif
class JKRHeap;
/**
* @ingroup jsystem-jkernel
*
*
*/
struct SArcHeader {
/* 0x00 */ BE(u32) signature;
@@ -25,7 +31,7 @@ struct SArcHeader {
/**
* @ingroup jsystem-jkernel
*
*
*/
struct SArcDataInfo {
/* 0x00 */ BE(u32) num_nodes;
@@ -59,7 +65,7 @@ extern u32 sCurrentDirID__10JKRArchive; // JKRArchive::sCurrentDirID
/**
* @ingroup jsystem-jkernel
*
*
*/
class JKRArchive : public JKRFileLoader {
public:
@@ -192,9 +198,7 @@ public:
u32 countFile() const { return mArcInfoBlock->num_file_entries; }
s32 countDirectory() const { return mArcInfoBlock->num_nodes; }
u8 getMountMode() const { return mMountMode; }
bool isFileEntry(u32 param_0) const {
return getFileAttribute(param_0) & 1;
}
bool isFileEntry(u32 param_0) const { return getFileAttribute(param_0) & 1; }
public:
/* 0x00 */ // vtable
@@ -210,7 +214,36 @@ public:
/* 0x54 */ const char* mStringTable;
#if TARGET_PC
u32 getFileSize(SDIFileEntry* entry) const;
void* getOverlayData(SDIFileEntry* entry, u32* outSize);
bool getOverlayFileSize(SDIFileEntry* entry, u32* outSize) const;
static void notifyOverlayFilesChanged();
protected:
void** mFileData;
struct ArcOverlayResource {
u32 entryIndex;
u32 size;
u64 generation;
};
// Resource pointers remain valid until removed through the JKR resource APIs.
// That way, any pointers to old overlay data remain valid even when the overlay changed.
std::unordered_map<void*, ArcOverlayResource> mArcOverlayResources;
mutable std::unordered_map<u32, void*> mActiveArcOverlayResources;
mutable std::string mArcOverlaysPath;
mutable std::unordered_map<u32, std::string> mIdxToPathMap;
mutable bool mArcOverlaysPathResolved = false;
bool buildArcOverlaysPath() const;
void buildIndexToPathMap(u32 dirIndex, const std::string& currentPath) const;
bool getOverlayPath(SDIFileEntry* entry, std::string& path) const;
void* getActiveOverlayData(SDIFileEntry* entry, u32* outSize) const;
bool copyOverlayData(void* buffer, u32 bufferSize, SDIFileEntry* entry, u32* outSize);
bool getOverlayResourceSize(const void* data, u32* outSize) const;
bool removeOverlayResource(void* resource, bool freeResource);
void removeAllOverlayResources();
#endif
protected:
@@ -234,7 +267,7 @@ public:
} else if (attr & JKRARCHIVE_ATTR_YAZ0) {
return COMPRESSION_YAZ0;
} else {
return COMPRESSION_YAY0;
return COMPRESSION_YAY0;
}
}
@@ -243,6 +276,9 @@ public:
protected:
static DUSK_GAME_DATA u32 sCurrentDirID;
#if TARGET_PC
static std::atomic<u64> sArcOverlayGeneration;
#endif
};
inline JKRCompression JKRConvertAttrToCompressionType(int attr) {
@@ -261,8 +297,8 @@ inline bool JKRRemoveResource(void* resource, JKRFileLoader* fileLoader) {
return JKRFileLoader::removeResource(resource, fileLoader);
}
inline JKRArchive* JKRMountArchive(void* ptr, JKRHeap* heap,
JKRArchive::EMountDirection mountDirection) {
inline JKRArchive* JKRMountArchive(
void* ptr, JKRHeap* heap, JKRArchive::EMountDirection mountDirection) {
return JKRArchive::mount(ptr, heap, mountDirection);
}
+8 -1
View File
@@ -17,7 +17,7 @@ size_t JASResArcLoader::getResSize(JKRArchive const* i_archiveP, u16 i_resourceI
return 0;
}
return file->data_size;
return DUSK_IF_ELSE(i_archiveP->getFileSize(file), file->data_size);
}
size_t JASResArcLoader::getResMaxSize(JKRArchive const* i_archiveP) {
@@ -27,9 +27,16 @@ size_t JASResArcLoader::getResMaxSize(JKRArchive const* i_archiveP) {
for (index = 0; index < fileEntries; index++) {
JKRArchive::SDIFileEntry* file = i_archiveP->findIdxResource(index);
if (file) {
#if TARGET_PC
const u32 fileSize = i_archiveP->getFileSize(file);
if (maxSize < fileSize) {
maxSize = fileSize;
}
#else
if (maxSize < file->data_size) {
maxSize = file->data_size;
}
#endif
}
}
@@ -196,6 +196,13 @@ cleanup:
void* JKRAramArchive::fetchResource(SDIFileEntry* pEntry, u32* pOutSize) {
JUT_ASSERT(442, isMounted());
#if TARGET_PC
if (void* data = getOverlayData(pEntry, pOutSize); data != nullptr) {
return data;
}
#endif
u32 outSize;
u8* outBuf;
if (pOutSize == NULL) {
@@ -231,6 +238,13 @@ void* JKRAramArchive::fetchResource(SDIFileEntry* pEntry, u32* pOutSize) {
void* JKRAramArchive::fetchResource(void* buffer, u32 bufferSize, SDIFileEntry* pEntry,
u32* resourceSize) {
JUT_ASSERT(515, isMounted());
#if TARGET_PC
if (copyOverlayData(buffer, bufferSize, pEntry, resourceSize)) {
return buffer;
}
#endif
u32 size = pEntry->data_size;
if (size > bufferSize) {
size = bufferSize;
@@ -337,6 +351,12 @@ u32 JKRAramArchive::getExpandedResSize(const void* ptr) const {
return this->getResSize(ptr);
}
#if TARGET_PC
if (u32 size; getOverlayResourceSize(ptr, &size)) {
return size;
}
#endif
JKRArchive::SDIFileEntry* entry = this->findPtrResource(ptr);
if (entry == NULL) {
return 0xFFFFFFFF;
+268
View File
@@ -7,6 +7,37 @@
#if TARGET_PC
#include <cassert>
#include <cstddef>
#include <cstdlib>
#include <limits>
#include <ranges>
#include <string_view>
#include "JSystem/JKernel/JKRDvdRipper.h"
#if _WIN32
#include <malloc.h>
#endif
std::atomic<u64> JKRArchive::sArcOverlayGeneration{0};
namespace {
void* alloc_overlay_buffer(u32 size) {
#if _WIN32
return _aligned_malloc(size, alignof(std::max_align_t));
#else
return std::malloc(size);
#endif
}
void free_overlay_buffer(void* data) {
#if _WIN32
_aligned_free(data);
#else
std::free(data);
#endif
}
} // namespace
#endif
DUSK_GAME_DATA u32 JKRArchive::sCurrentDirID;
@@ -42,6 +73,7 @@ JKRArchive::JKRArchive(s32 entryNumber, JKRArchive::EMountMode mountMode) {
JKRArchive::~JKRArchive() {
#if TARGET_PC
removeAllOverlayResources();
if (mFileData != nullptr) {
JKRHeap::getSystemHeap()->free(mFileData);
mFileData = nullptr;
@@ -270,4 +302,240 @@ void JKRArchive::initFileDataPointers() {
mFiles[i].index = i;
}
}
void JKRArchive::notifyOverlayFilesChanged() {
sArcOverlayGeneration.fetch_add(1, std::memory_order_release);
}
bool JKRArchive::buildArcOverlaysPath() const {
if (mArcOverlaysPathResolved) {
return !mArcOverlaysPath.empty();
}
mArcOverlaysPathResolved = true;
if (mEntryNum < 0) {
return false;
}
char pathBuffer[1024];
if (!DVDConvertEntrynumToPath(mEntryNum, pathBuffer, sizeof(pathBuffer))) {
return false;
}
std::string path{pathBuffer};
constexpr std::string_view extension{".arc"};
if (path.size() < extension.size() ||
path.compare(path.size() - extension.size(), extension.size(), extension) != 0)
{
return false;
}
path.resize(path.size() - extension.size());
path.push_back('/');
mArcOverlaysPath = std::move(path);
return true;
}
void JKRArchive::buildIndexToPathMap(u32 dirIndex, const std::string& currentPath) const {
const SDIDirEntry& dir = mNodes[dirIndex];
for (int i = 0; i < dir.num_entries; i++) {
const SDIFileEntry& entry = mFiles[dir.first_file_index + i];
std::string entryName{&mStringTable[entry.getNameOffset()]};
if (entryName == "." || entryName == "..") {
continue;
}
if (entry.isDirectory()) {
buildIndexToPathMap(entry.data_offset, currentPath + entryName + "/");
} else {
mIdxToPathMap[entry.index] = currentPath + entryName;
}
}
}
bool JKRArchive::getOverlayPath(SDIFileEntry* entry, std::string& path) const {
if (entry == nullptr || !buildArcOverlaysPath()) {
return false;
}
if (mIdxToPathMap.empty()) {
buildIndexToPathMap(0, std::string{&mStringTable[mNodes[0].name_offset]} + "/");
}
const auto pathIt = mIdxToPathMap.find(entry->index);
if (pathIt == mIdxToPathMap.end()) {
return false;
}
path = mArcOverlaysPath + pathIt->second;
return true;
}
void* JKRArchive::getActiveOverlayData(SDIFileEntry* entry, u32* outSize) const {
const auto activeIt = mActiveArcOverlayResources.find(entry->index);
if (activeIt == mActiveArcOverlayResources.end()) {
return nullptr;
}
const auto resourceIt = mArcOverlayResources.find(activeIt->second);
const u64 generation = sArcOverlayGeneration.load(std::memory_order_acquire);
if (resourceIt == mArcOverlayResources.end() || resourceIt->second.generation != generation) {
// Keep the allocation owned so raw pointers returned by earlier fetches remain valid.
mActiveArcOverlayResources.erase(activeIt);
return nullptr;
}
if (outSize != nullptr) {
*outSize = resourceIt->second.size;
}
return resourceIt->first;
}
void* JKRArchive::getOverlayData(SDIFileEntry* entry, u32* outSize) {
if (entry == nullptr) {
return nullptr;
}
if (void* data = getActiveOverlayData(entry, outSize)) {
return data;
}
std::string path;
if (!getOverlayPath(entry, path)) {
return nullptr;
}
constexpr u32 alignmentMask = 0x1f;
const u64 generation = sArcOverlayGeneration.load(std::memory_order_acquire);
DVDFileInfo fileInfo{};
if (!DVDOpen(path.c_str(), &fileInfo)) {
return nullptr;
}
const u32 logicalSize = fileInfo.length;
if (logicalSize > static_cast<u32>(std::numeric_limits<s32>::max()) - alignmentMask) {
DVDClose(&fileInfo);
return nullptr;
}
const u32 readSize = ALIGN_NEXT(logicalSize, 0x20);
const u32 allocationSize = readSize == 0 ? 1 : readSize;
void* data = alloc_overlay_buffer(allocationSize);
if (data == nullptr) {
DVDClose(&fileInfo);
return nullptr;
}
const s32 status = DVDReadPrio(&fileInfo, data, readSize, 0, 2);
DVDClose(&fileInfo);
if (status < DVD_RESULT_GOOD || static_cast<u32>(status) != logicalSize) {
free_overlay_buffer(data);
return nullptr;
}
mArcOverlayResources.emplace(data, ArcOverlayResource{
.entryIndex = entry->index,
.size = logicalSize,
.generation = generation,
});
mActiveArcOverlayResources[entry->index] = data;
if (outSize != nullptr) {
*outSize = logicalSize;
}
return data;
}
bool JKRArchive::copyOverlayData(void* buffer, u32 bufferSize, SDIFileEntry* entry, u32* outSize) {
u32 overlaySize;
const void* overlayData = getOverlayData(entry, &overlaySize);
if (overlayData == nullptr) {
return false;
}
const u32 copySize = overlaySize < bufferSize ? overlaySize : bufferSize;
if (copySize != 0) {
memcpy(buffer, overlayData, copySize);
}
if (outSize != nullptr) {
*outSize = copySize;
}
return true;
}
bool JKRArchive::getOverlayResourceSize(const void* data, u32* outSize) const {
const auto resourceIt = mArcOverlayResources.find(const_cast<void*>(data));
if (resourceIt == mArcOverlayResources.end()) {
return false;
}
if (outSize != nullptr) {
*outSize = resourceIt->second.size;
}
return true;
}
bool JKRArchive::getOverlayFileSize(SDIFileEntry* entry, u32* outSize) const {
if (entry == nullptr) {
return false;
}
u32 activeSize;
if (getActiveOverlayData(entry, &activeSize) != nullptr) {
if (outSize != nullptr) {
*outSize = activeSize;
}
return true;
}
std::string path;
if (!getOverlayPath(entry, path)) {
return false;
}
DVDFileInfo fileInfo{};
if (!DVDOpen(path.c_str(), &fileInfo)) {
return false;
}
if (outSize != nullptr) {
*outSize = fileInfo.length;
}
DVDClose(&fileInfo);
return true;
}
u32 JKRArchive::getFileSize(SDIFileEntry* entry) const {
u32 size;
if (getOverlayFileSize(entry, &size)) {
return size;
}
return entry != nullptr ? entry->getSize() : 0;
}
bool JKRArchive::removeOverlayResource(void* resource, bool freeResource) {
const auto resourceIt = mArcOverlayResources.find(resource);
if (resourceIt == mArcOverlayResources.end()) {
return false;
}
const auto activeIt = mActiveArcOverlayResources.find(resourceIt->second.entryIndex);
if (activeIt != mActiveArcOverlayResources.end() && activeIt->second == resource) {
mActiveArcOverlayResources.erase(activeIt);
}
if (freeResource) {
free_overlay_buffer(resource);
}
mArcOverlayResources.erase(resourceIt);
return true;
}
void JKRArchive::removeAllOverlayResources() {
mActiveArcOverlayResources.clear();
for (const auto& key : mArcOverlayResources | std::views::keys) {
free_overlay_buffer(key);
}
mArcOverlayResources.clear();
}
#endif
@@ -246,6 +246,7 @@ u32 JKRArchive::readResource(void* buffer, u32 bufferSize, u16 id) {
void JKRArchive::removeResourceAll() {
if (mArcInfoBlock && mMountMode != MOUNT_MEM) {
IF_DUSK(removeAllOverlayResources();)
SDIFileEntry* fileEntry = mFiles;
for (int i = 0; i < mArcInfoBlock->num_file_entries; i++) {
if (JKAR_DATA(fileEntry)) {
@@ -259,6 +260,13 @@ void JKRArchive::removeResourceAll() {
bool JKRArchive::removeResource(void* resource) {
JUT_ASSERT(678, resource != NULL);
#if TARGET_PC
if (removeOverlayResource(resource, true)) {
return true;
}
#endif
SDIFileEntry* fileEntry = findPtrResource(resource);
if (fileEntry == NULL)
return false;
@@ -270,6 +278,13 @@ bool JKRArchive::removeResource(void* resource) {
bool JKRArchive::detachResource(void* resource) {
JUT_ASSERT(707, resource != NULL);
#if TARGET_PC
if (removeOverlayResource(resource, false)) {
return true;
}
#endif
SDIFileEntry* fileEntry = findPtrResource(resource);
if (fileEntry == NULL)
return false;
@@ -280,6 +295,13 @@ bool JKRArchive::detachResource(void* resource) {
u32 JKRArchive::getResSize(const void* resource) const {
JUT_ASSERT(732, resource != NULL);
#if TARGET_PC
if (u32 size; getOverlayResourceSize(resource, &size)) {
return size;
}
#endif
SDIFileEntry* fileEntry = findPtrResource(resource);
if (fileEntry == NULL)
return -1;
@@ -226,6 +226,13 @@ bool JKRCompArchive::open(s32 entryNum) {
void* JKRCompArchive::fetchResource(SDIFileEntry *fileEntry, u32 *pSize) {
JUT_ASSERT(597, isMounted());
#if TARGET_PC
if (void* data = getOverlayData(fileEntry, pSize); data != nullptr) {
return data;
}
#endif
u32 ptrSize;
u32 size = fileEntry->data_size;
int compression = JKRConvertAttrToCompressionType(u8(fileEntry->type_flags_and_name_offset >> 0x18));
@@ -274,6 +281,13 @@ void *JKRCompArchive::fetchResource(void *data, u32 compressedSize, SDIFileEntry
{
u32 size = 0;
JUT_ASSERT(708, isMounted());
#if TARGET_PC
if (copyOverlayData(data, compressedSize, fileEntry, pSize)) {
return data;
}
#endif
u32 fileSize = fileEntry->data_size;
u32 alignedSize = ALIGN_NEXT(fileSize, 32);
u32 fileFlag = fileEntry->type_flags_and_name_offset >> 0x18;
@@ -319,6 +333,7 @@ void *JKRCompArchive::fetchResource(void *data, u32 compressedSize, SDIFileEntry
void JKRCompArchive::removeResourceAll() {
if (mArcInfoBlock != NULL && mMountMode != MOUNT_MEM) {
IF_DUSK(removeAllOverlayResources();)
SDIFileEntry* fileEntry = mFiles;
for (int i = 0; i < mArcInfoBlock->num_file_entries; i++) {
int tmp = fileEntry->type_flags_and_name_offset >> 0x18;
@@ -336,6 +351,12 @@ void JKRCompArchive::removeResourceAll() {
}
bool JKRCompArchive::removeResource(void* resource) {
#if TARGET_PC
if (removeOverlayResource(resource, true)) {
return true;
}
#endif
SDIFileEntry* fileEntry = findPtrResource(resource);
if (!fileEntry)
return false;
@@ -356,6 +377,12 @@ u32 JKRCompArchive::getExpandedResSize(const void *resource) const
return getResSize(resource);
}
#if TARGET_PC
if (u32 size; getOverlayResourceSize(resource, &size)) {
return size;
}
#endif
SDIFileEntry *fileEntry = findPtrResource(resource);
if(!fileEntry) {
return 0xffffffff;
@@ -147,6 +147,13 @@ cleanup:
void* JKRDvdArchive::fetchResource(SDIFileEntry* fileEntry, u32* returnSize) {
JUT_ASSERT(428, isMounted());
#if TARGET_PC
if (void* data = getOverlayData(fileEntry, returnSize); data != nullptr) {
return data;
}
#endif
u32 tempReturnSize;
if (returnSize == NULL) {
returnSize = &tempReturnSize;
@@ -181,6 +188,13 @@ void* JKRDvdArchive::fetchResource(SDIFileEntry* fileEntry, u32* returnSize) {
void* JKRDvdArchive::fetchResource(void* buffer, u32 bufferSize, SDIFileEntry* fileEntry,
u32* returnSize) {
JUT_ASSERT(504, isMounted());
#if TARGET_PC
if (copyOverlayData(buffer, bufferSize, fileEntry, returnSize)) {
return buffer;
}
#endif
u32 size = fileEntry->data_size;
JKRCompression fileCompression = JKRConvertAttrToCompressionType(u8(fileEntry->type_flags_and_name_offset >> 24));
@@ -344,6 +358,12 @@ u32 JKRDvdArchive::getExpandedResSize(const void* resource) const {
return getResSize(resource);
}
#if TARGET_PC
if (u32 size; getOverlayResourceSize(resource, &size)) {
return size;
}
#endif
SDIFileEntry* fileEntry = findPtrResource(resource);
if (!fileEntry) {
return -1;
@@ -135,6 +135,13 @@ bool JKRMemArchive::open(void* buffer, u32 bufferSize, JKRMemBreakFlag flag) {
void* JKRMemArchive::fetchResource(SDIFileEntry* fileEntry, u32* resourceSize) {
JUT_ASSERT(555, isMounted());
#if TARGET_PC
if (void* data = getOverlayData(fileEntry, resourceSize); data != nullptr) {
return data;
}
#endif
if (!JKAR_DATA(fileEntry)) {
JKAR_DATA(fileEntry) = mArchiveData + fileEntry->data_offset;
}
@@ -149,6 +156,13 @@ void* JKRMemArchive::fetchResource(SDIFileEntry* fileEntry, u32* resourceSize) {
void* JKRMemArchive::fetchResource(void* buffer, u32 bufferSize, SDIFileEntry* fileEntry,
u32* resourceSize) {
JUT_ASSERT(595, isMounted());
#if TARGET_PC
if (copyOverlayData(buffer, bufferSize, fileEntry, resourceSize)) {
return buffer;
}
#endif
u32 srcLength = fileEntry->data_size;
if (srcLength > bufferSize) {
srcLength = bufferSize;
@@ -173,6 +187,8 @@ void* JKRMemArchive::fetchResource(void* buffer, u32 bufferSize, SDIFileEntry* f
void JKRMemArchive::removeResourceAll(void) {
JUT_ASSERT(642, isMounted());
IF_DUSK(removeAllOverlayResources();)
if (mArcInfoBlock == NULL)
return;
if (mMountMode == MOUNT_MEM)
@@ -192,6 +208,12 @@ void JKRMemArchive::removeResourceAll(void) {
bool JKRMemArchive::removeResource(void* resource) {
JUT_ASSERT(673, isMounted());
#if TARGET_PC
if (removeOverlayResource(resource, true)) {
return true;
}
#endif
SDIFileEntry* fileEntry = findPtrResource(resource);
if (!fileEntry)
return false;
@@ -231,6 +253,12 @@ u32 JKRMemArchive::fetchResource_subroutine(u8* src, u32 srcLength, u8* dst, u32
}
u32 JKRMemArchive::getExpandedResSize(const void* resource) const {
#if TARGET_PC
if (u32 overlaySize; getOverlayResourceSize(resource, &overlaySize)) {
return overlaySize;
}
#endif
SDIFileEntry* fileEntry = findPtrResource(resource);
if (fileEntry == NULL)
return -1;