Merge pull request #667 from RetroPortingToolKit/work/prepared-resource-overlays

Support verified prepared resources as disc overlays
This commit is contained in:
Matt Stanley
2026-10-15 00:51:10 -07:00
committed by GitHub
7 changed files with 234 additions and 26 deletions
+17
View File
@@ -675,6 +675,23 @@ direct indexed lookup rather than scanning every installed mod.
Feature disc overlays require an exact `disc_sha256` on every target entry.
`expected_sha256` can additionally guard the replaced stock range.
Format 10 also supports an overlay sourced from a verified same-feature
`[[resource]]` with `format = "file"`. Declare exactly one of `file` or
`resource`. Resource overlays require `size`, a nonnegative `resource_offset`
(default zero), the slice's `sha256`, and the stock range's `expected_sha256`.
The source must have a fixed nonzero size and SHA-256 and cannot be input-only.
The complete resource is verified once into immutable bytes during resolution;
all its slices and the runtime resource share that snapshot. Slices may span
sectors. A diagnostic launcher preview defers preparer-owned resource overlays,
while execution rejects missing, corrupt, out-of-range or mismatched content.
No generated media needs to be written into a bundled package directory.
Trusted preparers may use `ModPrepareContext::progress(fraction, message)` when
it is provided. A false result requests cancellation: stop between bounded work
units and before publishing outputs. Keep previously verified cache entries and
publish new entries atomically. The manager checks cancellation before applying
derived resource bindings. Older preparers may leave this callback unused.
## State and migration
`mods/state.toml` format 2 stores selected package versions separately from
+9 -1
View File
@@ -196,6 +196,9 @@ struct ModOverlay {
ModPatchTarget target = ModPatchTarget::DiscRaw;
uint64_t location = 0;
std::filesystem::path file;
// Format 10: verified same-feature derived resource instead of a package file.
std::string resource_id;
uint64_t resource_offset = 0;
std::string sha256;
std::string expected_sha256;
uint64_t size = 0;
@@ -315,6 +318,9 @@ struct ModPackage {
std::vector<ModDerivedDisc> derived_discs;
};
// False requests cancellation. Providers report between bounded work units
// and before publishing outputs; no UI code runs on the preparation worker.
using ModPrepareProgress = std::function<bool(float, const std::string&)>;
struct ModPrepareContext {
const ModPackage& package;
std::string feature_id;
@@ -322,6 +328,7 @@ struct ModPrepareContext {
std::filesystem::path cache_root;
std::filesystem::path application_dir;
std::map<std::string, std::filesystem::path> inputs;
ModPrepareProgress progress;
};
using ModMediaPreparer = std::function<bool(const ModPrepareContext&,
std::map<std::string, std::filesystem::path>&, std::string&)>;
@@ -469,7 +476,8 @@ public:
bool prepare_resources(const std::string& game_id,
const std::filesystem::path& disc_path,
const std::filesystem::path& cache_root,
std::string* error = nullptr, bool cached_only = false);
std::string* error = nullptr, bool cached_only = false,
const ModPrepareProgress& progress = {});
bool install_archive(const std::filesystem::path& archive,
std::string* installed_id = nullptr,
+2 -1
View File
@@ -24,7 +24,8 @@ bool mod_runtime_commit(const std::filesystem::path& disc_path = {},
bool save_selection = true);
/* Prepare a preboot launcher selection; the audited runtime retains a ticket. */
bool mod_runtime_prepare_for_launcher(const std::filesystem::path& disc_path,
std::string* error = nullptr);
std::string* error = nullptr,
const ModPrepareProgress& progress = {});
/* Audited preboot only: load and verify a plan using existing receipts.
* A miss never invokes a media converter or computes a disc digest. */
+80 -22
View File
@@ -25,7 +25,7 @@ namespace PSXRecompV4 {
namespace {
constexpr uint32_t kMinFormatVersion = 1;
constexpr uint32_t kMaxFormatVersion = 9;
constexpr uint32_t kMaxFormatVersion = 10;
constexpr uint64_t kMaxArchiveBytes = 256ull * 1024ull * 1024ull;
constexpr uint32_t kMaxArchiveFiles = 4096;
@@ -2504,11 +2504,29 @@ bool ModPackageManager::read_manifest(const fs::path& path, ModPackage& out,
if (offset < 0)
throw std::runtime_error("overlay offset is negative");
overlay.location = (uint64_t)offset;
const std::string relative_file =
toml::find<std::string>(v, "file");
if (!safe_archive_name(relative_file))
throw std::runtime_error("overlay file path is unsafe");
overlay.file = out.root / fs::path(relative_file);
if (v.contains("file") == v.contains("resource"))
throw std::runtime_error("overlay requires exactly one file or resource");
if (v.contains("resource")) {
if (out.format_version < 10)
throw std::runtime_error("resource overlays require format_version 10");
overlay.resource_id = toml::find<std::string>(v, "resource");
const auto* resource = find_resource(out, overlay.feature_id, overlay.resource_id);
if (!resource || resource->input_only || resource->format != "file" ||
resource->size == 0 || !valid_sha256(resource->sha256))
throw std::runtime_error("overlay requires a verified same-feature file resource");
const int64_t source_offset = toml::find_or<int64_t>(v, "resource_offset", 0);
const int64_t size = toml::find<int64_t>(v, "size");
if (source_offset < 0 || size <= 0 || uint64_t(source_offset) > resource->size ||
uint64_t(size) > resource->size - uint64_t(source_offset))
throw std::runtime_error("overlay resource slice is out of bounds");
overlay.resource_offset = uint64_t(source_offset);
overlay.size = uint64_t(size);
} else {
const std::string relative_file = toml::find<std::string>(v, "file");
if (!safe_archive_name(relative_file))
throw std::runtime_error("overlay file path is unsafe");
overlay.file = out.root / fs::path(relative_file);
}
overlay.sha256 = toml::find<std::string>(v, "sha256");
overlay.expected_sha256 = v.contains("expected_sha256")
? toml::find<std::string>(v, "expected_sha256") : "";
@@ -2517,17 +2535,21 @@ bool ModPackageManager::read_manifest(const fs::path& path, ModPackage& out,
!valid_sha256(overlay.expected_sha256)))
throw std::runtime_error(
"overlay hashes must be lowercase SHA-256");
std::string file_error;
std::vector<uint8_t> payload;
if (!read_file(overlay.file, payload, &file_error))
throw std::runtime_error(file_error);
if (payload.empty())
throw std::runtime_error("overlay payload is empty");
const std::string actual = fingerprint_text(std::string(
(const char*)payload.data(), payload.size()));
if (actual != overlay.sha256)
throw std::runtime_error("overlay payload checksum failed");
overlay.size = payload.size();
if (overlay.resource_id.empty()) {
std::string file_error;
std::vector<uint8_t> payload;
if (!read_file(overlay.file, payload, &file_error))
throw std::runtime_error(file_error);
if (payload.empty())
throw std::runtime_error("overlay payload is empty");
const std::string actual = fingerprint_text(std::string(
(const char*)payload.data(), payload.size()));
if (actual != overlay.sha256)
throw std::runtime_error("overlay payload checksum failed");
overlay.size = payload.size();
} else if (overlay.expected_sha256.empty()) {
throw std::runtime_error("resource overlay requires expected_sha256");
}
if (overlay.location >
std::numeric_limits<uint64_t>::max() - overlay.size)
throw std::runtime_error("overlay range overflows");
@@ -3475,7 +3497,8 @@ std::string ModPackageManager::feature_option_value(
}
bool ModPackageManager::prepare_resources(const std::string& game_id,
const fs::path& disc_path, const fs::path& cache_root, std::string* error, bool cached_only) {
const fs::path& disc_path, const fs::path& cache_root, std::string* error, bool cached_only,
const ModPrepareProgress& progress) {
auto effective = effective_selections(nullptr, nullptr, nullptr);
auto pending = selections_;
try {
@@ -3491,7 +3514,9 @@ bool ModPackageManager::prepare_resources(const std::string& game_id,
const auto provider = media_preparers().find(package->prepare);
if (provider == media_preparers().end())
throw std::runtime_error(package->name + ": media preparer is not installed");
ModPrepareContext context{*package, feature.id, disc_path, cache_root, root_.parent_path(), {}};
if (progress && !progress(-1.0f,"Preparing " + feature.name))
throw std::runtime_error("Preparation canceled.");
ModPrepareContext context{*package, feature.id, disc_path, cache_root, root_.parent_path(), {}, progress};
for (const auto& r : package->resources) if (r.feature_id == feature.id && r.input_only) {
auto path = feature_resource_path(id, feature.id, r.id);
if (path.empty() && r.required)
@@ -3520,6 +3545,9 @@ bool ModPackageManager::prepare_resources(const std::string& game_id,
}
}
} catch (const std::exception& ex) { set_error(error, ex.what()); return false; }
if (progress && !progress(-1.0f,"Verifying prepared content")) {
set_error(error,"Preparation canceled."); return false;
}
selections_ = std::move(pending);
return true;
}
@@ -3649,6 +3677,20 @@ ModResolution ModPackageManager::resolve(const std::string& game_id,
const ModSelection blank;
const ModSelection& selected =
selected_it == selections.end() ? blank : selected_it->second;
std::map<std::pair<std::string,std::string>,
std::shared_ptr<const std::vector<uint8_t>>> resource_snapshots;
auto load_resource = [&](const ModResource& resource,
std::shared_ptr<const std::vector<uint8_t>>& bytes,
std::string& reason) {
const auto key=std::make_pair(resource.feature_id,resource.id);
const auto found=resource_snapshots.find(key);
if(found!=resource_snapshots.end()) { bytes=found->second; return true; }
const auto path=effective_resource_path(*package,selected,resource.feature_id,resource.id);
if(!load_mod_media(path,resource.format,resource.size,resource.sha256,bytes,&reason))
return false;
resource_snapshots.emplace(key,bytes);
return true;
};
if (package->resolver == "declarative") {
for (const ModDerivedDisc& derived : package->derived_discs) {
const ModFeature& legacy = package->features.front();
@@ -3874,6 +3916,8 @@ ModResolution ModPackageManager::resolve(const std::string& game_id,
std::vector<const ModOverlay*> overlays;
overlays.reserve(package->overlays.size());
for (const ModOverlay& overlay : package->overlays) {
if (defer_prepared_resources && !package->prepare.empty() && !overlay.resource_id.empty())
continue;
const ModFeature* feature =
find_feature(*package, overlay.feature_id);
if (!feature ||
@@ -3892,7 +3936,22 @@ ModResolution ModPackageManager::resolve(const std::string& game_id,
for (const ModOverlay* overlay : overlays) {
std::vector<uint8_t> payload;
std::string payload_error;
if (!read_file(overlay->file, payload, &payload_error)) {
bool loaded = false;
if (overlay->resource_id.empty()) {
loaded = read_file(overlay->file, payload, &payload_error);
} else {
const auto* resource = find_resource(*package, overlay->feature_id, overlay->resource_id);
std::shared_ptr<const std::vector<uint8_t>> bytes;
if (resource && load_resource(*resource, bytes, payload_error) && bytes &&
overlay->resource_offset <= bytes->size() &&
overlay->size <= bytes->size() - overlay->resource_offset) {
const auto begin = bytes->begin() + size_t(overlay->resource_offset);
payload.assign(begin, begin + size_t(overlay->size));
loaded = true;
}
if (!loaded && payload_error.empty()) payload_error = "overlay resource is unavailable or out of bounds";
}
if (!loaded) {
result.errors.push_back(
package->id + "/" + overlay->feature_id + ": " +
payload_error);
@@ -3980,8 +4039,7 @@ ModResolution ModPackageManager::resolve(const std::string& game_id,
resolved.sha256 = resource.sha256;
if (!resource.sha256.empty()) {
std::string media_error;
if (!load_mod_media(path, resource.format, resource.size,
resource.sha256, resolved.bytes, &media_error)) {
if (!load_resource(resource, resolved.bytes, media_error)) {
result.errors.push_back(package->id + "/" + resource.feature_id +
": " + resource.id + ": " + media_error);
continue;
+39 -2
View File
@@ -87,6 +87,14 @@ static uint64_t g_mod_vblanks = 0;
namespace PSXRecompV4 {
namespace {
thread_local const ModPrepareProgress* active_prepare_progress = nullptr;
bool preparation_continues(float fraction, const char* text, std::string* error) {
if (!active_prepare_progress || !*active_prepare_progress ||
(*active_prepare_progress)(fraction,text)) return true;
if (error) *error = "Preparation canceled.";
return false;
}
struct RuntimeMods {
struct DiscExtent { uint32_t lba, count; const uint8_t* data; };
@@ -1980,7 +1988,8 @@ bool mod_runtime_prepare_resources(const std::filesystem::path& disc_path, std::
RuntimeMods& s = state();
if (!s.initialized) { timing.success(); return true; }
const auto media_cache = media_cache_root();
if (!s.manager.prepare_resources(s.game_id, disc_path, media_cache, &s.error, s.cached_prepare)) {
if (!s.manager.prepare_resources(s.game_id, disc_path, media_cache, &s.error, s.cached_prepare,
active_prepare_progress ? *active_prepare_progress : ModPrepareProgress{})) {
if (error) *error = s.error;
return false;
}
@@ -2073,6 +2082,10 @@ static bool install_plan(RuntimeMods& s, ModResolution plan,
dependency_files(s, inputs->disc_path, after_resolve) &&
inputs->dependencies == after_resolve;
#endif
if (!preparation_continues(1.0f,"Content ready",&s.error)) {
if (error) *error = s.error;
return false;
}
if (save_selection) {
HostLaunchTimingScope save_timing(HOST_LAUNCH_SAVE_STATE);
if (!s.manager.save_state(&s.error)) {
@@ -2112,6 +2125,7 @@ bool mod_runtime_commit(const std::filesystem::path& disc_path, std::string* err
RuntimeMods& s = state();
s.netplay_view_plan = false;
if (!s.initialized) { timing.success(); return true; }
if (!preparation_continues(-1.0f,"Checking selected content",error)) return false;
#if defined(PSX_LAUNCHER_MOD_COMMIT_WORKER_SAFE)
std::map<std::filesystem::path, std::string> catalog;
if (s.catalog_stamped && (!catalog_files(s, catalog) || catalog != s.loaded_catalog)) {
@@ -2174,7 +2188,15 @@ bool mod_runtime_commit(const std::filesystem::path& disc_path, std::string* err
return ok;
}
bool mod_runtime_prepare_for_launcher(const std::filesystem::path& disc_path, std::string* error) {
bool mod_runtime_prepare_for_launcher(const std::filesystem::path& disc_path, std::string* error,
const ModPrepareProgress& progress) {
struct Scope {
const ModPrepareProgress* previous;
explicit Scope(const ModPrepareProgress& current) : previous(active_prepare_progress) {
active_prepare_progress = &current;
}
~Scope() { active_prepare_progress = previous; }
} scope(progress);
ScopedFlag flag(state().provider_preparing);
return mod_runtime_commit(disc_path, error, true);
}
@@ -2411,6 +2433,21 @@ const RecompLauncherCModProvider* mod_runtime_launcher_provider() {
};
provider.preparation_revision = [](void*) -> unsigned long long { return state().preparation_revision; };
#endif
#if defined(RECOMP_LAUNCHER_HAS_MOD_PREPARATION_PROGRESS)
provider.commit_with_progress = [](void*, const char* path,
int (*report)(void*,float,const char*), void* ctx) -> int {
ModPrepareProgress progress;
if (report) progress=[=](float fraction,const std::string& text) {
return report(ctx,fraction,text.c_str())!=0;
};
std::string reason;
if (!mod_runtime_prepare_for_launcher(path ? std::filesystem::path(path) :
std::filesystem::path(), &reason, progress)) {
set_error(reason); return 0;
}
state().error.clear(); return 1;
};
#endif
#endif
return &provider;
}
+48
View File
@@ -2346,11 +2346,59 @@ int main() {
auto prepared = reload.resolve("SLUS-TEST");
check(prepared.ok && prepared.resources.size()==1 && prepared.resources[0].id=="asset", "only derived assets enter runtime plan");
const auto before = reload.feature_resource_path("cars", "content", "asset");
int progress_calls=0;
check(!reload.prepare_resources("SLUS-TEST",source,folder/"cache",&error,false,
[&](float,const std::string&){return ++progress_calls<2;}),"cancel after provider prevents selection publication");
check(reload.feature_resource_path("cars","content","asset")==before,"cancel retains prior derived binding");
fail=true;check(!reload.prepare_resources("SLUS-TEST", source, folder/"cache", &error), "failed preparation refuses launch");
check(reload.feature_resource_path("cars", "content", "asset")==before, "failed preparation preserves prior bindings");
write_bytes(asset, {4,3,2,1});check(!reload.resolve("SLUS-TEST").ok, "corrupt derived output fails engine verification");
}
{
const auto folder=root/"resource-overlays";
const auto asset=folder/"local"/"derived.dat";
const auto manifest_path=folder/"bundled/prepared.overlay/1.0.0/manifest.toml";
const std::vector<uint8_t> bytes={1,2,3,4,5,6};
const std::vector<uint8_t> slice={3,4,5};
const std::string disc_hash(64,'a');
auto make_manifest=[&](const std::string& source_fields,const std::string& feature="active") {
return "format_version = 10\nid = \"prepared.overlay\"\nversion = \"1.0.0\"\nname = \"Prepared overlay\"\nprepare = \"test.overlay\"\n"
"[[target]]\ngame_id = \"SLUS-TEST\"\ndisc_sha256 = \""+disc_hash+"\"\n"
"[[feature]]\nid = \"active\"\nname = \"Active\"\ndefault_enabled = true\n"
"[[feature]]\nid = \"other\"\nname = \"Other\"\n"
"[[resource]]\nfeature = \""+feature+"\"\nid = \"asset\"\nlabel = \"Derived asset\"\nformat = \"file\"\nrequired = true\nhidden = true\nsize = 6\nsha256 = \""+sha256_hex(bytes)+"\"\n"
"[[overlay]]\nfeature = \"active\"\ntarget = \"disc_raw\"\noffset = 2340\n"+source_fields+
"sha256 = \""+sha256_hex(slice)+"\"\nexpected_sha256 = \""+sha256_hex({0,0,0})+"\"\n";
};
const std::string fields="resource = \"asset\"\nresource_offset = 2\nsize = 3\n";
write_text(manifest_path,make_manifest(fields));
ModPackageManager manager(folder);
check(manager.scan(&error) && manager.scan_errors().empty(),"resource overlay scans before generated file exists");
check(manager.resolve("SLUS-TEST",{},disc_hash,true).ok,"preview defers prepared resource overlay");
check(!manager.resolve("SLUS-TEST",{},disc_hash).ok,"runtime refuses missing overlay source");
write_bytes(asset,bytes);
check(manager.set_feature_resource_path("prepared.overlay","active","asset",asset,&error),error.c_str());
const auto plan=manager.resolve("SLUS-TEST",{},disc_hash);
check(plan.ok && plan.overlays.size()==1 && plan.resources.size()==1,"verified resource overlay resolves");
if(plan.overlays.size()==1) {
check(plan.overlays[0].payload==slice && plan.overlays[0].location==2340,"overlay selects exact declared slice and target offset");
}
write_bytes(asset,{1,2,3,4,5,7});
check(!manager.resolve("SLUS-TEST",{},disc_hash).ok,"mutated resource rejected even when selected slice did not change");
write_text(manifest_path,make_manifest("resource = \"asset\"\nresource_offset = 4\nsize = 3\n"));
ModPackageManager bounds(folder);bounds.scan(&error);
check(!bounds.scan_errors().empty(),"resource slice beyond declared extent rejected");
write_text(manifest_path,make_manifest(fields+"file = \"payload.dat\"\n"));
ModPackageManager both(folder);both.scan(&error);
check(!both.scan_errors().empty(),"ambiguous file and resource sources rejected");
write_text(manifest_path,make_manifest(fields,"other"));
ModPackageManager ownership(folder);ownership.scan(&error);
check(!ownership.scan_errors().empty(),"cross-feature overlay source rejected");
write_text(manifest_path,make_manifest("resource = \"asset\"\nresource_offset = -1\nsize = 3\n"));
ModPackageManager negative(folder);negative.scan(&error);
check(!negative.scan_errors().empty(),"negative resource slice rejected");
}
fs::remove_all(root, ec);
if (failures) {
std::cerr << failures << " mod package test(s) failed\n";
+39
View File
@@ -1238,6 +1238,45 @@ int main(int argc, char** argv) {
check(PSXRecompV4::mod_runtime_commit(raw_path,&error),"reader mount raw disc");
check(psx_mod_read_disc_file("S0/LEVEL.NSF",result.data(),(uint32_t)result.size(),&bytes) &&
std::equal(result.begin(),result.end(),iso.begin()+22*2048),"raw and ISO reads identical");
{
const auto folder=root/"prepared-overlay";
const auto pack_path=folder/"local.pack";
std::vector<uint8_t> pack(3016,0x42),payload(3000);
for(size_t i=0;i<payload.size();++i)payload[i]=uint8_t(i*11+5);
std::copy(payload.begin(),payload.end(),pack.begin()+16);
const uint64_t offset=21*2352+2000;
const std::vector<uint8_t> expected(raw_iso.begin()+offset,raw_iso.begin()+offset+payload.size());
check(PSXRecompV4::mod_register_media_preparer("test.overlay.prepare",
[&](const PSXRecompV4::ModPrepareContext& context,std::map<std::string,fs::path>& outputs,std::string&) {
if(context.progress && !context.progress(0.5f,"Building local pack"))return false;
write_bytes(pack_path,pack);outputs["pack"]=pack_path;return true;
}),"register prepared overlay fixture");
write_text(folder/"bundled/prepared.overlay/1.0.0/manifest.toml",
"format_version = 10\nid = \"prepared.overlay\"\nversion = \"1.0.0\"\nname = \"Prepared overlay\"\nprepare = \"test.overlay.prepare\"\n"
"[[target]]\ngame_id = \"READER\"\ndisc_sha256 = \""+sha256_hex(raw_iso)+"\"\n"
"[[feature]]\nid = \"audio\"\nname = \"Audio\"\ndefault_enabled = true\n"
"[[resource]]\nfeature = \"audio\"\nid = \"pack\"\nlabel = \"Pack\"\nformat = \"file\"\nhidden = true\nrequired = true\nsize = 3016\nsha256 = \""+sha256_hex(pack)+"\"\n"
"[[overlay]]\nfeature = \"audio\"\ntarget = \"disc_raw\"\noffset = "+std::to_string(offset)+"\n"
"resource = \"pack\"\nresource_offset = 16\nsize = 3000\nsha256 = \""+sha256_hex(payload)+"\"\nexpected_sha256 = \""+sha256_hex(expected)+"\"\n");
check(PSXRecompV4::mod_runtime_initialize(folder,"READER",0,{},&error),"initialize prepared overlay");
int reports=0;
check(!PSXRecompV4::mod_runtime_prepare_for_launcher(raw_path,&error,
[&](float fraction,const std::string&){++reports;return fraction<1.0f;}),"cancel before installing prepared plan");
check(reports>1 && error=="Preparation canceled.","runtime propagates cooperative cancellation");
check(PSXRecompV4::mod_runtime_prepare_for_launcher(raw_path,&error),"prepare verified resource-backed overlay");
mod_runtime_enable_disc_patches();
fs::remove(pack_path,ec); // Committed bytes must remain immutable and resident.
for(unsigned lba=21;lba<=23;++lba) {
std::array<uint8_t,2352> sector;
std::copy_n(raw_iso.begin()+lba*2352,2352,sector.begin());
auto wanted=sector;
const uint64_t start=std::max<uint64_t>(offset,lba*2352);
const uint64_t end=std::min<uint64_t>(offset+payload.size(),(lba+1)*2352);
std::copy(payload.begin()+start-offset,payload.begin()+end-offset,wanted.begin()+start-lba*2352);
mod_runtime_patch_disc_sector(lba,1,sector.data(),sector.size());
check(sector==wanted,"resource overlay crosses sectors with exact untouched surroundings");
}
}
/* Verified bytes are scoped to the committed feature, and do not depend
* on reopening an owner file after launch. No launcher is needed here. */
const auto media_root = root / "verified-media";