diff --git a/docs/MOD_PACKAGES.md b/docs/MOD_PACKAGES.md index 2c2ea70a..97ebb8f7 100644 --- a/docs/MOD_PACKAGES.md +++ b/docs/MOD_PACKAGES.md @@ -675,6 +675,23 @@ direct indexed lookup rather than scanning every installed mod. Feature disc overlays require an exact `disc_sha256` on every target entry. `expected_sha256` can additionally guard the replaced stock range. +Format 10 also supports an overlay sourced from a verified same-feature +`[[resource]]` with `format = "file"`. Declare exactly one of `file` or +`resource`. Resource overlays require `size`, a nonnegative `resource_offset` +(default zero), the slice's `sha256`, and the stock range's `expected_sha256`. +The source must have a fixed nonzero size and SHA-256 and cannot be input-only. +The complete resource is verified once into immutable bytes during resolution; +all its slices and the runtime resource share that snapshot. Slices may span +sectors. A diagnostic launcher preview defers preparer-owned resource overlays, +while execution rejects missing, corrupt, out-of-range or mismatched content. +No generated media needs to be written into a bundled package directory. + +Trusted preparers may use `ModPrepareContext::progress(fraction, message)` when +it is provided. A false result requests cancellation: stop between bounded work +units and before publishing outputs. Keep previously verified cache entries and +publish new entries atomically. The manager checks cancellation before applying +derived resource bindings. Older preparers may leave this callback unused. + ## State and migration `mods/state.toml` format 2 stores selected package versions separately from diff --git a/runtime/include/mod_packages.h b/runtime/include/mod_packages.h index 7e9117b7..19e4cdbc 100644 --- a/runtime/include/mod_packages.h +++ b/runtime/include/mod_packages.h @@ -196,6 +196,9 @@ struct ModOverlay { ModPatchTarget target = ModPatchTarget::DiscRaw; uint64_t location = 0; std::filesystem::path file; + // Format 10: verified same-feature derived resource instead of a package file. + std::string resource_id; + uint64_t resource_offset = 0; std::string sha256; std::string expected_sha256; uint64_t size = 0; @@ -315,6 +318,9 @@ struct ModPackage { std::vector derived_discs; }; +// False requests cancellation. Providers report between bounded work units +// and before publishing outputs; no UI code runs on the preparation worker. +using ModPrepareProgress = std::function; struct ModPrepareContext { const ModPackage& package; std::string feature_id; @@ -322,6 +328,7 @@ struct ModPrepareContext { std::filesystem::path cache_root; std::filesystem::path application_dir; std::map inputs; + ModPrepareProgress progress; }; using ModMediaPreparer = std::function&, std::string&)>; @@ -469,7 +476,8 @@ public: bool prepare_resources(const std::string& game_id, const std::filesystem::path& disc_path, const std::filesystem::path& cache_root, - std::string* error = nullptr, bool cached_only = false); + std::string* error = nullptr, bool cached_only = false, + const ModPrepareProgress& progress = {}); bool install_archive(const std::filesystem::path& archive, std::string* installed_id = nullptr, diff --git a/runtime/include/mod_runtime.h b/runtime/include/mod_runtime.h index 4f51a2df..d103736b 100644 --- a/runtime/include/mod_runtime.h +++ b/runtime/include/mod_runtime.h @@ -24,7 +24,8 @@ bool mod_runtime_commit(const std::filesystem::path& disc_path = {}, bool save_selection = true); /* Prepare a preboot launcher selection; the audited runtime retains a ticket. */ bool mod_runtime_prepare_for_launcher(const std::filesystem::path& disc_path, - std::string* error = nullptr); + std::string* error = nullptr, + const ModPrepareProgress& progress = {}); /* Audited preboot only: load and verify a plan using existing receipts. * A miss never invokes a media converter or computes a disc digest. */ diff --git a/runtime/src/mod_packages.cpp b/runtime/src/mod_packages.cpp index 7138cc3d..7f8b57c0 100644 --- a/runtime/src/mod_packages.cpp +++ b/runtime/src/mod_packages.cpp @@ -25,7 +25,7 @@ namespace PSXRecompV4 { namespace { constexpr uint32_t kMinFormatVersion = 1; -constexpr uint32_t kMaxFormatVersion = 9; +constexpr uint32_t kMaxFormatVersion = 10; constexpr uint64_t kMaxArchiveBytes = 256ull * 1024ull * 1024ull; constexpr uint32_t kMaxArchiveFiles = 4096; @@ -2504,11 +2504,29 @@ bool ModPackageManager::read_manifest(const fs::path& path, ModPackage& out, if (offset < 0) throw std::runtime_error("overlay offset is negative"); overlay.location = (uint64_t)offset; - const std::string relative_file = - toml::find(v, "file"); - if (!safe_archive_name(relative_file)) - throw std::runtime_error("overlay file path is unsafe"); - overlay.file = out.root / fs::path(relative_file); + if (v.contains("file") == v.contains("resource")) + throw std::runtime_error("overlay requires exactly one file or resource"); + if (v.contains("resource")) { + if (out.format_version < 10) + throw std::runtime_error("resource overlays require format_version 10"); + overlay.resource_id = toml::find(v, "resource"); + const auto* resource = find_resource(out, overlay.feature_id, overlay.resource_id); + if (!resource || resource->input_only || resource->format != "file" || + resource->size == 0 || !valid_sha256(resource->sha256)) + throw std::runtime_error("overlay requires a verified same-feature file resource"); + const int64_t source_offset = toml::find_or(v, "resource_offset", 0); + const int64_t size = toml::find(v, "size"); + if (source_offset < 0 || size <= 0 || uint64_t(source_offset) > resource->size || + uint64_t(size) > resource->size - uint64_t(source_offset)) + throw std::runtime_error("overlay resource slice is out of bounds"); + overlay.resource_offset = uint64_t(source_offset); + overlay.size = uint64_t(size); + } else { + const std::string relative_file = toml::find(v, "file"); + if (!safe_archive_name(relative_file)) + throw std::runtime_error("overlay file path is unsafe"); + overlay.file = out.root / fs::path(relative_file); + } overlay.sha256 = toml::find(v, "sha256"); overlay.expected_sha256 = v.contains("expected_sha256") ? toml::find(v, "expected_sha256") : ""; @@ -2517,17 +2535,21 @@ bool ModPackageManager::read_manifest(const fs::path& path, ModPackage& out, !valid_sha256(overlay.expected_sha256))) throw std::runtime_error( "overlay hashes must be lowercase SHA-256"); - std::string file_error; - std::vector payload; - if (!read_file(overlay.file, payload, &file_error)) - throw std::runtime_error(file_error); - if (payload.empty()) - throw std::runtime_error("overlay payload is empty"); - const std::string actual = fingerprint_text(std::string( - (const char*)payload.data(), payload.size())); - if (actual != overlay.sha256) - throw std::runtime_error("overlay payload checksum failed"); - overlay.size = payload.size(); + if (overlay.resource_id.empty()) { + std::string file_error; + std::vector payload; + if (!read_file(overlay.file, payload, &file_error)) + throw std::runtime_error(file_error); + if (payload.empty()) + throw std::runtime_error("overlay payload is empty"); + const std::string actual = fingerprint_text(std::string( + (const char*)payload.data(), payload.size())); + if (actual != overlay.sha256) + throw std::runtime_error("overlay payload checksum failed"); + overlay.size = payload.size(); + } else if (overlay.expected_sha256.empty()) { + throw std::runtime_error("resource overlay requires expected_sha256"); + } if (overlay.location > std::numeric_limits::max() - overlay.size) throw std::runtime_error("overlay range overflows"); @@ -3475,7 +3497,8 @@ std::string ModPackageManager::feature_option_value( } bool ModPackageManager::prepare_resources(const std::string& game_id, - const fs::path& disc_path, const fs::path& cache_root, std::string* error, bool cached_only) { + const fs::path& disc_path, const fs::path& cache_root, std::string* error, bool cached_only, + const ModPrepareProgress& progress) { auto effective = effective_selections(nullptr, nullptr, nullptr); auto pending = selections_; try { @@ -3491,7 +3514,9 @@ bool ModPackageManager::prepare_resources(const std::string& game_id, const auto provider = media_preparers().find(package->prepare); if (provider == media_preparers().end()) throw std::runtime_error(package->name + ": media preparer is not installed"); - ModPrepareContext context{*package, feature.id, disc_path, cache_root, root_.parent_path(), {}}; + if (progress && !progress(-1.0f,"Preparing " + feature.name)) + throw std::runtime_error("Preparation canceled."); + ModPrepareContext context{*package, feature.id, disc_path, cache_root, root_.parent_path(), {}, progress}; for (const auto& r : package->resources) if (r.feature_id == feature.id && r.input_only) { auto path = feature_resource_path(id, feature.id, r.id); if (path.empty() && r.required) @@ -3520,6 +3545,9 @@ bool ModPackageManager::prepare_resources(const std::string& game_id, } } } catch (const std::exception& ex) { set_error(error, ex.what()); return false; } + if (progress && !progress(-1.0f,"Verifying prepared content")) { + set_error(error,"Preparation canceled."); return false; + } selections_ = std::move(pending); return true; } @@ -3649,6 +3677,20 @@ ModResolution ModPackageManager::resolve(const std::string& game_id, const ModSelection blank; const ModSelection& selected = selected_it == selections.end() ? blank : selected_it->second; + std::map, + std::shared_ptr>> resource_snapshots; + auto load_resource = [&](const ModResource& resource, + std::shared_ptr>& bytes, + std::string& reason) { + const auto key=std::make_pair(resource.feature_id,resource.id); + const auto found=resource_snapshots.find(key); + if(found!=resource_snapshots.end()) { bytes=found->second; return true; } + const auto path=effective_resource_path(*package,selected,resource.feature_id,resource.id); + if(!load_mod_media(path,resource.format,resource.size,resource.sha256,bytes,&reason)) + return false; + resource_snapshots.emplace(key,bytes); + return true; + }; if (package->resolver == "declarative") { for (const ModDerivedDisc& derived : package->derived_discs) { const ModFeature& legacy = package->features.front(); @@ -3874,6 +3916,8 @@ ModResolution ModPackageManager::resolve(const std::string& game_id, std::vector overlays; overlays.reserve(package->overlays.size()); for (const ModOverlay& overlay : package->overlays) { + if (defer_prepared_resources && !package->prepare.empty() && !overlay.resource_id.empty()) + continue; const ModFeature* feature = find_feature(*package, overlay.feature_id); if (!feature || @@ -3892,7 +3936,22 @@ ModResolution ModPackageManager::resolve(const std::string& game_id, for (const ModOverlay* overlay : overlays) { std::vector payload; std::string payload_error; - if (!read_file(overlay->file, payload, &payload_error)) { + bool loaded = false; + if (overlay->resource_id.empty()) { + loaded = read_file(overlay->file, payload, &payload_error); + } else { + const auto* resource = find_resource(*package, overlay->feature_id, overlay->resource_id); + std::shared_ptr> bytes; + if (resource && load_resource(*resource, bytes, payload_error) && bytes && + overlay->resource_offset <= bytes->size() && + overlay->size <= bytes->size() - overlay->resource_offset) { + const auto begin = bytes->begin() + size_t(overlay->resource_offset); + payload.assign(begin, begin + size_t(overlay->size)); + loaded = true; + } + if (!loaded && payload_error.empty()) payload_error = "overlay resource is unavailable or out of bounds"; + } + if (!loaded) { result.errors.push_back( package->id + "/" + overlay->feature_id + ": " + payload_error); @@ -3980,8 +4039,7 @@ ModResolution ModPackageManager::resolve(const std::string& game_id, resolved.sha256 = resource.sha256; if (!resource.sha256.empty()) { std::string media_error; - if (!load_mod_media(path, resource.format, resource.size, - resource.sha256, resolved.bytes, &media_error)) { + if (!load_resource(resource, resolved.bytes, media_error)) { result.errors.push_back(package->id + "/" + resource.feature_id + ": " + resource.id + ": " + media_error); continue; diff --git a/runtime/src/mod_runtime.cpp b/runtime/src/mod_runtime.cpp index 67846edc..d7c0cdb7 100644 --- a/runtime/src/mod_runtime.cpp +++ b/runtime/src/mod_runtime.cpp @@ -87,6 +87,14 @@ static uint64_t g_mod_vblanks = 0; namespace PSXRecompV4 { namespace { +thread_local const ModPrepareProgress* active_prepare_progress = nullptr; + +bool preparation_continues(float fraction, const char* text, std::string* error) { + if (!active_prepare_progress || !*active_prepare_progress || + (*active_prepare_progress)(fraction,text)) return true; + if (error) *error = "Preparation canceled."; + return false; +} struct RuntimeMods { struct DiscExtent { uint32_t lba, count; const uint8_t* data; }; @@ -1980,7 +1988,8 @@ bool mod_runtime_prepare_resources(const std::filesystem::path& disc_path, std:: RuntimeMods& s = state(); if (!s.initialized) { timing.success(); return true; } const auto media_cache = media_cache_root(); - if (!s.manager.prepare_resources(s.game_id, disc_path, media_cache, &s.error, s.cached_prepare)) { + if (!s.manager.prepare_resources(s.game_id, disc_path, media_cache, &s.error, s.cached_prepare, + active_prepare_progress ? *active_prepare_progress : ModPrepareProgress{})) { if (error) *error = s.error; return false; } @@ -2073,6 +2082,10 @@ static bool install_plan(RuntimeMods& s, ModResolution plan, dependency_files(s, inputs->disc_path, after_resolve) && inputs->dependencies == after_resolve; #endif + if (!preparation_continues(1.0f,"Content ready",&s.error)) { + if (error) *error = s.error; + return false; + } if (save_selection) { HostLaunchTimingScope save_timing(HOST_LAUNCH_SAVE_STATE); if (!s.manager.save_state(&s.error)) { @@ -2112,6 +2125,7 @@ bool mod_runtime_commit(const std::filesystem::path& disc_path, std::string* err RuntimeMods& s = state(); s.netplay_view_plan = false; if (!s.initialized) { timing.success(); return true; } + if (!preparation_continues(-1.0f,"Checking selected content",error)) return false; #if defined(PSX_LAUNCHER_MOD_COMMIT_WORKER_SAFE) std::map catalog; if (s.catalog_stamped && (!catalog_files(s, catalog) || catalog != s.loaded_catalog)) { @@ -2174,7 +2188,15 @@ bool mod_runtime_commit(const std::filesystem::path& disc_path, std::string* err return ok; } -bool mod_runtime_prepare_for_launcher(const std::filesystem::path& disc_path, std::string* error) { +bool mod_runtime_prepare_for_launcher(const std::filesystem::path& disc_path, std::string* error, + const ModPrepareProgress& progress) { + struct Scope { + const ModPrepareProgress* previous; + explicit Scope(const ModPrepareProgress& current) : previous(active_prepare_progress) { + active_prepare_progress = ¤t; + } + ~Scope() { active_prepare_progress = previous; } + } scope(progress); ScopedFlag flag(state().provider_preparing); return mod_runtime_commit(disc_path, error, true); } @@ -2411,6 +2433,21 @@ const RecompLauncherCModProvider* mod_runtime_launcher_provider() { }; provider.preparation_revision = [](void*) -> unsigned long long { return state().preparation_revision; }; #endif +#if defined(RECOMP_LAUNCHER_HAS_MOD_PREPARATION_PROGRESS) + provider.commit_with_progress = [](void*, const char* path, + int (*report)(void*,float,const char*), void* ctx) -> int { + ModPrepareProgress progress; + if (report) progress=[=](float fraction,const std::string& text) { + return report(ctx,fraction,text.c_str())!=0; + }; + std::string reason; + if (!mod_runtime_prepare_for_launcher(path ? std::filesystem::path(path) : + std::filesystem::path(), &reason, progress)) { + set_error(reason); return 0; + } + state().error.clear(); return 1; + }; +#endif #endif return &provider; } diff --git a/runtime/tests/test_mod_packages.cpp b/runtime/tests/test_mod_packages.cpp index 06d71860..cd232153 100644 --- a/runtime/tests/test_mod_packages.cpp +++ b/runtime/tests/test_mod_packages.cpp @@ -2346,11 +2346,59 @@ int main() { auto prepared = reload.resolve("SLUS-TEST"); check(prepared.ok && prepared.resources.size()==1 && prepared.resources[0].id=="asset", "only derived assets enter runtime plan"); const auto before = reload.feature_resource_path("cars", "content", "asset"); + int progress_calls=0; + check(!reload.prepare_resources("SLUS-TEST",source,folder/"cache",&error,false, + [&](float,const std::string&){return ++progress_calls<2;}),"cancel after provider prevents selection publication"); + check(reload.feature_resource_path("cars","content","asset")==before,"cancel retains prior derived binding"); fail=true;check(!reload.prepare_resources("SLUS-TEST", source, folder/"cache", &error), "failed preparation refuses launch"); check(reload.feature_resource_path("cars", "content", "asset")==before, "failed preparation preserves prior bindings"); write_bytes(asset, {4,3,2,1});check(!reload.resolve("SLUS-TEST").ok, "corrupt derived output fails engine verification"); } + { + const auto folder=root/"resource-overlays"; + const auto asset=folder/"local"/"derived.dat"; + const auto manifest_path=folder/"bundled/prepared.overlay/1.0.0/manifest.toml"; + const std::vector bytes={1,2,3,4,5,6}; + const std::vector slice={3,4,5}; + const std::string disc_hash(64,'a'); + auto make_manifest=[&](const std::string& source_fields,const std::string& feature="active") { + return "format_version = 10\nid = \"prepared.overlay\"\nversion = \"1.0.0\"\nname = \"Prepared overlay\"\nprepare = \"test.overlay\"\n" + "[[target]]\ngame_id = \"SLUS-TEST\"\ndisc_sha256 = \""+disc_hash+"\"\n" + "[[feature]]\nid = \"active\"\nname = \"Active\"\ndefault_enabled = true\n" + "[[feature]]\nid = \"other\"\nname = \"Other\"\n" + "[[resource]]\nfeature = \""+feature+"\"\nid = \"asset\"\nlabel = \"Derived asset\"\nformat = \"file\"\nrequired = true\nhidden = true\nsize = 6\nsha256 = \""+sha256_hex(bytes)+"\"\n" + "[[overlay]]\nfeature = \"active\"\ntarget = \"disc_raw\"\noffset = 2340\n"+source_fields+ + "sha256 = \""+sha256_hex(slice)+"\"\nexpected_sha256 = \""+sha256_hex({0,0,0})+"\"\n"; + }; + const std::string fields="resource = \"asset\"\nresource_offset = 2\nsize = 3\n"; + write_text(manifest_path,make_manifest(fields)); + ModPackageManager manager(folder); + check(manager.scan(&error) && manager.scan_errors().empty(),"resource overlay scans before generated file exists"); + check(manager.resolve("SLUS-TEST",{},disc_hash,true).ok,"preview defers prepared resource overlay"); + check(!manager.resolve("SLUS-TEST",{},disc_hash).ok,"runtime refuses missing overlay source"); + write_bytes(asset,bytes); + check(manager.set_feature_resource_path("prepared.overlay","active","asset",asset,&error),error.c_str()); + const auto plan=manager.resolve("SLUS-TEST",{},disc_hash); + check(plan.ok && plan.overlays.size()==1 && plan.resources.size()==1,"verified resource overlay resolves"); + if(plan.overlays.size()==1) { + check(plan.overlays[0].payload==slice && plan.overlays[0].location==2340,"overlay selects exact declared slice and target offset"); + } + write_bytes(asset,{1,2,3,4,5,7}); + check(!manager.resolve("SLUS-TEST",{},disc_hash).ok,"mutated resource rejected even when selected slice did not change"); + write_text(manifest_path,make_manifest("resource = \"asset\"\nresource_offset = 4\nsize = 3\n")); + ModPackageManager bounds(folder);bounds.scan(&error); + check(!bounds.scan_errors().empty(),"resource slice beyond declared extent rejected"); + write_text(manifest_path,make_manifest(fields+"file = \"payload.dat\"\n")); + ModPackageManager both(folder);both.scan(&error); + check(!both.scan_errors().empty(),"ambiguous file and resource sources rejected"); + write_text(manifest_path,make_manifest(fields,"other")); + ModPackageManager ownership(folder);ownership.scan(&error); + check(!ownership.scan_errors().empty(),"cross-feature overlay source rejected"); + write_text(manifest_path,make_manifest("resource = \"asset\"\nresource_offset = -1\nsize = 3\n")); + ModPackageManager negative(folder);negative.scan(&error); + check(!negative.scan_errors().empty(),"negative resource slice rejected"); + } fs::remove_all(root, ec); if (failures) { std::cerr << failures << " mod package test(s) failed\n"; diff --git a/runtime/tests/test_mod_runtime.cpp b/runtime/tests/test_mod_runtime.cpp index 08ee4d9f..3d0d179a 100644 --- a/runtime/tests/test_mod_runtime.cpp +++ b/runtime/tests/test_mod_runtime.cpp @@ -1238,6 +1238,45 @@ int main(int argc, char** argv) { check(PSXRecompV4::mod_runtime_commit(raw_path,&error),"reader mount raw disc"); check(psx_mod_read_disc_file("S0/LEVEL.NSF",result.data(),(uint32_t)result.size(),&bytes) && std::equal(result.begin(),result.end(),iso.begin()+22*2048),"raw and ISO reads identical"); + { + const auto folder=root/"prepared-overlay"; + const auto pack_path=folder/"local.pack"; + std::vector pack(3016,0x42),payload(3000); + for(size_t i=0;i expected(raw_iso.begin()+offset,raw_iso.begin()+offset+payload.size()); + check(PSXRecompV4::mod_register_media_preparer("test.overlay.prepare", + [&](const PSXRecompV4::ModPrepareContext& context,std::map& outputs,std::string&) { + if(context.progress && !context.progress(0.5f,"Building local pack"))return false; + write_bytes(pack_path,pack);outputs["pack"]=pack_path;return true; + }),"register prepared overlay fixture"); + write_text(folder/"bundled/prepared.overlay/1.0.0/manifest.toml", + "format_version = 10\nid = \"prepared.overlay\"\nversion = \"1.0.0\"\nname = \"Prepared overlay\"\nprepare = \"test.overlay.prepare\"\n" + "[[target]]\ngame_id = \"READER\"\ndisc_sha256 = \""+sha256_hex(raw_iso)+"\"\n" + "[[feature]]\nid = \"audio\"\nname = \"Audio\"\ndefault_enabled = true\n" + "[[resource]]\nfeature = \"audio\"\nid = \"pack\"\nlabel = \"Pack\"\nformat = \"file\"\nhidden = true\nrequired = true\nsize = 3016\nsha256 = \""+sha256_hex(pack)+"\"\n" + "[[overlay]]\nfeature = \"audio\"\ntarget = \"disc_raw\"\noffset = "+std::to_string(offset)+"\n" + "resource = \"pack\"\nresource_offset = 16\nsize = 3000\nsha256 = \""+sha256_hex(payload)+"\"\nexpected_sha256 = \""+sha256_hex(expected)+"\"\n"); + check(PSXRecompV4::mod_runtime_initialize(folder,"READER",0,{},&error),"initialize prepared overlay"); + int reports=0; + check(!PSXRecompV4::mod_runtime_prepare_for_launcher(raw_path,&error, + [&](float fraction,const std::string&){++reports;return fraction<1.0f;}),"cancel before installing prepared plan"); + check(reports>1 && error=="Preparation canceled.","runtime propagates cooperative cancellation"); + check(PSXRecompV4::mod_runtime_prepare_for_launcher(raw_path,&error),"prepare verified resource-backed overlay"); + mod_runtime_enable_disc_patches(); + fs::remove(pack_path,ec); // Committed bytes must remain immutable and resident. + for(unsigned lba=21;lba<=23;++lba) { + std::array sector; + std::copy_n(raw_iso.begin()+lba*2352,2352,sector.begin()); + auto wanted=sector; + const uint64_t start=std::max(offset,lba*2352); + const uint64_t end=std::min(offset+payload.size(),(lba+1)*2352); + std::copy(payload.begin()+start-offset,payload.begin()+end-offset,wanted.begin()+start-lba*2352); + mod_runtime_patch_disc_sector(lba,1,sector.data(),sector.size()); + check(sector==wanted,"resource overlay crosses sectors with exact untouched surroundings"); + } + } /* Verified bytes are scoped to the committed feature, and do not depend * on reopening an owner file after launch. No launcher is needed here. */ const auto media_root = root / "verified-media";