Mod file overlay system

Mods can now replace DVD files with contents of their "overlay" folder

(I'll update the docs later when I do a full pass and make non-code mods
more of a first-class citizen)

Fixes https://github.com/TwilitRealm/dusklight/issues/1306

files.cmake

@@ -1524,6 +1524,7 @@ set(DUSK_FILES
         src/dusk/OSMutex.cpp
         src/dusk/hook_system.cpp
         src/dusk/modding/mod_loader.cpp
+        src/dusk/modding/mod_loader_overlay.cpp
         src/dusk/modding/bundle_disk.cpp
         src/dusk/modding/bundle_zip.cpp
         src/dusk/gx_helper.cpp

include/dusk/mod_loader.hpp

@@ -67,6 +67,7 @@ private:
 
     void tryLoadDusk(const std::filesystem::path& modPath, bool fromDir);
     void buildAPI(LoadedMod& mod);
+    void initOverlayFiles();
 };
 
 }  // namespace dusk

src/dusk/modding/bundle_disk.cpp

@@ -9,10 +9,8 @@ namespace dusk::modding {
 ModBundleDisk::ModBundleDisk(fs::path root) : root_path(std::move(root)) {}
 
 std::vector<u8> ModBundleDisk::readFile(const std::string& fileName) {
-    const fs::path filePath = reinterpret_cast<const char8_t*>(fileName.c_str());
-    const auto finalPath = root_path / fileName;
 
-    return io::FileStream::ReadAllBytes(finalPath);
+    return io::FileStream::ReadAllBytes(toRealPath(fileName));
 }
 
 std::vector<std::string> ModBundleDisk::getFileNames() {
@@ -51,4 +49,13 @@ std::vector<std::string> ModBundleDisk::getFileNames() {
     return files;
 }
 
+size_t ModBundleDisk::getFileSize(const std::string& fileName) {
+    return std::filesystem::file_size(toRealPath(fileName));
+}
+
+std::filesystem::path ModBundleDisk::toRealPath(const std::string& fileName) const {
+    const fs::path filePath = reinterpret_cast<const char8_t*>(fileName.c_str());
+    return root_path / fileName;
+}
+
 }  // namespace dusk::modding

src/dusk/modding/bundle_zip.cpp

@@ -51,4 +51,15 @@ std::vector<std::string> ModBundleZip::getFileNames() {
     return results;
 }
 
+size_t ModBundleZip::getFileSize(const std::string& fileName) {
+    const auto idx = mz_zip_reader_locate_file(&res_zip, fileName.c_str(), nullptr, 0);
+    if (idx < 0) {
+        throw std::runtime_error(fmt::format("Unable to locate file in zip: {}", fileName));
+    }
+
+    mz_zip_archive_file_stat stat{};
+    mz_zip_reader_file_stat(&res_zip, idx, &stat);
+    return stat.m_uncomp_size;
+}
+
 }  // namespace dusk::modding

src/dusk/modding/mod_loader.cpp

@@ -14,6 +14,7 @@
 #include <string>
 #include <unordered_map>
 
+#include "aurora/dvd.h"
 #include "dusk/io.hpp"
 #include "miniz.h"
 #include "nlohmann/json.hpp"
@@ -557,6 +558,8 @@ void ModLoader::init() {
         return;
     }
 
+    initOverlayFiles();
+
     DuskLog.info("ModLoader: initializing {} mod(s)...", m_mods.size());
     for (auto& mod : m_mods) {
         buildAPI(mod);

src/dusk/modding/mod_loader.hpp

@@ -11,6 +11,7 @@ public:
 
     virtual std::vector<u8> readFile(const std::string& fileName) = 0;
     virtual std::vector<std::string> getFileNames() = 0;
+    virtual size_t getFileSize(const std::string& fileName) = 0;
 };
 
 class ModBundleZip final : public ModBundle {
@@ -19,6 +20,7 @@ public:
     ~ModBundleZip() override;
     std::vector<u8> readFile(const std::string& fileName) override;
     std::vector<std::string> getFileNames() override;
+    size_t getFileSize(const std::string& fileName) override;
 
 private:
     std::vector<uint8_t> zip_data;
@@ -32,8 +34,10 @@ public:
     ~ModBundleDisk() override = default;
     std::vector<u8> readFile(const std::string& fileName) override;
     std::vector<std::string> getFileNames() override;
+    size_t getFileSize(const std::string& fileName) override;
 
 private:
+    [[nodiscard]] std::filesystem::path toRealPath(const std::string& fileName) const;
     std::filesystem::path root_path;
 };
 

src/dusk/modding/mod_loader_overlay.cpp

@@ -0,0 +1,108 @@
+#include "aurora/dvd.h"
+#include "aurora/lib/logging.hpp"
+#include "dusk/mod_loader.hpp"
+#include "mod_loader.hpp"
+
+#include <cstring>
+
+using namespace std::string_literals;
+
+namespace {
+
+aurora::Module Log("dusk::modLoader::overlay");
+
+struct OverlayFileData {
+    std::string bundlePath;
+    dusk::LoadedMod* mod; // TODO: is using a raw pointer a bad idea here?
+};
+
+std::vector<OverlayFileData> s_overlayFiles;
+
+void findOverlayFiles(std::vector<AuroraOverlayFile>& files, dusk::LoadedMod& mod) {
+    for (const auto& file : mod.bundle->getFileNames()) {
+        if (!file.starts_with("overlay/")) {
+            continue;
+        }
+
+        auto overlayPath = file.substr("overlay/"s.size());
+        assert(!overlayPath.starts_with('/'));
+        overlayPath.insert(0, "/");
+
+        const auto size = mod.bundle->getFileSize(file);
+
+        const auto index = s_overlayFiles.size();
+        s_overlayFiles.emplace_back(file, &mod);
+        files.emplace_back(strdup(overlayPath.c_str()), reinterpret_cast<void*>(index), size);
+    }
+}
+
+struct OpenOverlayFile {
+    std::vector<u8> data;
+    size_t pos;
+};
+
+void* cbOpen(void* userdata) {
+    const auto index = reinterpret_cast<size_t>(userdata);
+    const auto& fileData = s_overlayFiles[index];
+    auto fileContents = fileData.mod->bundle->readFile(fileData.bundlePath);
+
+    return new OpenOverlayFile(std::move(fileContents), 0);
+}
+
+void cbClose(void* handle) {
+    const auto openFile = static_cast<OpenOverlayFile*>(handle);
+    delete openFile;
+}
+
+int64_t cbRead(void* handle, uint8_t *buf, const size_t len) {
+    auto& openFile = *static_cast<OpenOverlayFile*>(handle);
+
+    const auto remainingSpace = openFile.data.size() - openFile.pos;
+    const auto toRead = std::min(remainingSpace, len);
+    std::memcpy(buf, openFile.data.data() + openFile.pos, toRead);
+    openFile.pos += toRead;
+    return static_cast<int64_t>(toRead);
+}
+
+int64_t cbSeek(void* handle, int64_t offset, int32_t whence) {
+    if (whence != 0) {
+        Log.fatal("Invalid seek mode from aurora: {}", whence);
+    }
+
+    auto& openFile = *static_cast<OpenOverlayFile*>(handle);
+    const auto posSigned = std::clamp(offset, static_cast<int64_t>(0), static_cast<int64_t>(openFile.data.size()));
+    openFile.pos = static_cast<size_t>(posSigned);
+    return posSigned;
+}
+
+constexpr AuroraOverlayCallbacks s_overlayCallbacks = {
+    .open = cbOpen,
+    .close = cbClose,
+    .read = cbRead,
+    .seek = cbSeek,
+};
+
+}
+
+namespace dusk {
+
+void ModLoader::initOverlayFiles() {
+    Log.debug("Initializing overlay files...");
+
+    aurora_dvd_overlay_callbacks(&s_overlayCallbacks);
+
+    std::vector<AuroraOverlayFile> files;
+
+    for (auto& mod : m_mods) {
+        findOverlayFiles(files, mod);
+    }
+
+    Log.debug("Found {} overlay files.", files.size());
+    aurora_dvd_overlay_files(files.data(), files.size());
+
+    for (const auto& file : files) {
+        std::free(const_cast<char*>(file.fileName));
+    }
+}
+
+}  // namespace dusk