Only quarantine content when an interrupted write had a commit in flight

The write-in-progress marker was binary. If it is present then it meant quarantine, so a death in the sliver after the last commit, or an antivirus silently blocking the marker's deletion, would quarantine a perfectly consistent save. The marker now records a phase: before any commit rename has run, every file is still its old version, so a stale "writing" marker is cleared and the save kept. Only "committing", the one state where a mixed old/new set is possible, still quarantines. A blocked deletion is retried, then relabeled, loudly.
This commit is contained in:
Danshet
2026-08-17 21:33:39 +02:00
parent 56c4e277b4
commit f163c1bb39
6 changed files with 456 additions and 27 deletions
@@ -15,6 +15,13 @@ namespace rex::filesystem {
class HostPathEntry;
// Phase words carried on the first line of the write-in-progress marker
// file. The phase tells the NEXT mount what a leftover marker proves about
// the container next to it (see set_write_marker_path below).
inline constexpr char kWriteMarkerPhaseWriting[] = "writing";
inline constexpr char kWriteMarkerPhaseCommitting[] = "committing";
inline constexpr char kWriteMarkerPhaseComplete[] = "complete";
class HostPathDevice : public Device {
public:
HostPathDevice(const std::string_view mount_path, const std::filesystem::path& host_path,
@@ -40,18 +47,43 @@ class HostPathDevice : public Device {
// Write-in-progress marker. When a marker path is set (content
// packages do this), the device keeps a marker file on disk while any
// atomic write session is open and removes it when the last one commits.
// A marker still present at the next mount means a write died mid-flight
// and the container may be torn across files - the content manager then
// quarantines it. Empty path (the default) disables the marker.
// The marker's first line is a phase word telling the NEXT mount what a
// leftover marker proves about the container:
// "writing" - sessions were open but no commit rename had started;
// every file is still its previous version: consistent.
// "committing" - at least one commit rename ran while the marker was
// live; a death after that can tear the container
// ACROSS files (file A committed, file B not).
// "complete" - every commit finished but the marker itself could not
// be deleted (typically an antivirus or indexer holding
// the file): consistent.
// The phase is monotonic within one marker lifetime: it starts at
// "writing" when the first session of a burst opens and never returns
// there until the marker is deleted (or recreated by a later burst).
// The content manager reads the phase at mount time and only quarantines
// a container whose marker does not prove consistency. Empty path (the
// default) disables the marker.
void set_write_marker_path(const std::filesystem::path& marker_path) {
write_marker_path_ = marker_path;
}
void OnAtomicWriteBegin();
// Called immediately before the first commit rename of a session; flips
// the marker phase to "committing" once per marker lifetime.
void OnAtomicWriteCommit();
void OnAtomicWriteEnd();
// Reads the phase word (first line, trimmed) from a marker file. Returns
// an empty string when the file cannot be read - callers must treat that
// as "proves nothing".
static std::string ReadWriteMarkerPhase(const std::filesystem::path& marker_path);
private:
void PopulateEntry(HostPathEntry* parent_entry);
void SweepStaleAtomicArtifacts();
// Truncate-writes the marker file with the given phase word on its first
// line. Caller holds write_marker_mutex_. Returns false when the file
// could not be opened for writing.
bool WriteMarkerPhaseLocked(const char* phase);
std::string name_;
std::filesystem::path host_path_;
@@ -61,6 +93,9 @@ class HostPathDevice : public Device {
std::filesystem::path write_marker_path_;
std::mutex write_marker_mutex_;
int active_atomic_writes_ = 0;
// Whether the current marker has already been flipped to "committing".
// Reset when a new marker lifetime begins.
bool marker_committing_ = false;
};
} // namespace rex::filesystem
@@ -40,10 +40,14 @@ constexpr uint32_t kCurrentlyRunningTitleId = 0xFFFFFFFF;
// Write-in-progress marker: "<package folder>.rex-writing" sits
// next to an extracted content package while any write into it is in flight
// and is removed when the last write commits. Found at mount time it means a
// previous session died mid-write - the package may be torn ACROSS files
// (file A committed, file B not) even though each individual file write is
// atomic - and the package is quarantined so the game recreates it cleanly.
// and is removed when the last write commits. Its first line is a phase word
// (see host_path_device.h). Found at mount time, phase "committing" means a
// previous session died after a commit rename had started - the package may
// be torn ACROSS files (file A committed, file B not) even though each
// individual file write is atomic - and the package is quarantined so the
// game recreates it cleanly. Phases "writing" and "complete" prove the
// package consistent: only the stale marker is cleared. Unreadable or
// unrecognized content quarantines (the conservative default).
inline constexpr char kContentWriteMarkerSuffix[] = ".rex-writing";
struct XCONTENT_DATA {
@@ -225,6 +229,24 @@ class ContentManager {
// folder (and Windows' local-DLL search order).
void DiscoverContainers(const std::filesystem::path& dlc_dir);
// Torn-container self-healing: if the package's write-in-progress
// marker survived to this mount, the last session died mid-flight. The
// marker's phase word decides what that proves: "writing" (no commit
// rename ever started) and "complete" (every commit finished, only the
// marker's own deletion failed) prove the package consistent, so it is
// kept and only the stale marker is cleared, with one error-level log
// line. Phase "committing" - and unreadable or unrecognized content -
// means the package may be torn across files: the folder is renamed
// aside into root_path_/quarantine/"<name>.corrupt-<yyyymmdd-hhmmss>" -
// never deleted, it stays available as diagnostic evidence - so the game
// sees a clean slate and recreates it. This retires the game's own
// "please delete this Game Data" dialog, which asks the player for
// host-side surgery they have no in-game way to do. No-op while the
// package is open in this process (the marker is then a live write, not
// a torn one). Public so the unit drill can exercise the mount-time
// decision table directly.
void QuarantineTornPackage(uint64_t xuid, const XCONTENT_AGGREGATE_DATA& data);
private:
std::filesystem::path ResolvePackageRoot(uint64_t xuid, XContentType content_type,
uint32_t title_id = -1);
@@ -239,17 +261,6 @@ class ContentManager {
ContentPackage* DetachPackage(std::unordered_map<string::string_key_case, ContentPackage*,
string::string_key_case::Hash>::iterator it);
// Torn-container self-healing: if the package's write-in-progress
// marker survived to this mount, the last write died mid-flight. The
// package folder is renamed aside into root_path_/quarantine/
// "<name>.corrupt-<yyyymmdd-hhmmss>" - never deleted, it stays available as
// diagnostic evidence - so the game sees a clean slate and recreates it.
// This retires the game's own "please delete this Game Data" dialog, which
// asks the player for host-side surgery they have no in-game way to do.
// No-op while the package is open in this process (the marker is then a
// live write, not a torn one).
void QuarantineTornPackage(uint64_t xuid, const XCONTENT_AGGREGATE_DATA& data);
// Where a package's data actually lives: a discovered container if one
// exists (containers take priority), else the extracted folder / container
// file at the canonical package path, else empty.
@@ -175,26 +175,64 @@ void HostPathDevice::SweepStaleAtomicArtifacts() {
}
}
bool HostPathDevice::WriteMarkerPhaseLocked(const char* phase) {
auto marker = rex::filesystem::OpenFile(write_marker_path_, "wb");
if (!marker) {
return false;
}
// First line = the phase word (what the mount-time check parses); the rest
// is for a human who finds the file.
fprintf(marker,
"%s\n\n"
"Write-in-progress marker for the content folder next to this file.\n"
"If it is still present at the next launch, the phase word on the first\n"
"line decides what happens: 'committing' moves the folder aside into\n"
"quarantine (kept, never deleted); 'writing' or 'complete' proves the\n"
"folder consistent and only this marker is removed.\n",
phase);
fclose(marker);
return true;
}
void HostPathDevice::OnAtomicWriteBegin() {
if (write_marker_path_.empty()) {
return;
}
std::lock_guard<std::mutex> lock(write_marker_mutex_);
if (active_atomic_writes_++ == 0) {
auto marker = rex::filesystem::OpenFile(write_marker_path_, "wb");
if (marker) {
static const char kMarkerText[] =
"Write in progress. If this file is still here on the next launch, the last "
"write died mid-flight and the container next to it will be quarantined.\n";
fwrite(kMarkerText, 1, sizeof(kMarkerText) - 1, marker);
fclose(marker);
} else {
// A new marker lifetime: phase starts at "writing" - no commit rename
// has run, so a death anywhere before OnAtomicWriteCommit() leaves the
// container provably consistent.
marker_committing_ = false;
if (!WriteMarkerPhaseLocked(kWriteMarkerPhaseWriting)) {
REXFS_WARN("Could not create write-in-progress marker '{}'",
rex::path_to_utf8(write_marker_path_));
}
}
}
void HostPathDevice::OnAtomicWriteCommit() {
if (write_marker_path_.empty()) {
return;
}
std::lock_guard<std::mutex> lock(write_marker_mutex_);
if (active_atomic_writes_ > 0 && !marker_committing_) {
// The first commit rename of this marker's lifetime is about to run:
// from here until the marker is deleted, a death can tear the container
// across files, so a leftover marker must quarantine. Monotonic - the
// phase never returns to "writing" while this marker lives.
if (WriteMarkerPhaseLocked(kWriteMarkerPhaseCommitting)) {
marker_committing_ = true;
} else {
// Leave marker_committing_ unset so the next commit retries; a marker
// stuck on "writing" while commits run would defeat torn-container
// detection.
REXFS_ERROR("Could not flip write-in-progress marker '{}' to phase '{}'",
rex::path_to_utf8(write_marker_path_), kWriteMarkerPhaseCommitting);
}
}
}
void HostPathDevice::OnAtomicWriteEnd() {
if (write_marker_path_.empty()) {
return;
@@ -203,7 +241,49 @@ void HostPathDevice::OnAtomicWriteEnd() {
if (active_atomic_writes_ > 0 && --active_atomic_writes_ == 0) {
std::error_code ec;
std::filesystem::remove(write_marker_path_, ec);
if (ec) {
// One retry: transient sharing violations (an antivirus scanning the
// marker it just saw us write) often clear immediately.
ec.clear();
std::filesystem::remove(write_marker_path_, ec);
}
if (ec) {
// Deletion is blocked, but a scanner that holds the file usually still
// permits rewriting an existing one: record phase "complete" so the
// next mount does not quarantine a container whose writes all
// finished. This failure used to be swallowed silently.
if (WriteMarkerPhaseLocked(kWriteMarkerPhaseComplete)) {
REXFS_ERROR(
"Could not delete write-in-progress marker '{}' ({}); rewrote it as phase '{}' so "
"the container is not quarantined at the next launch",
rex::path_to_utf8(write_marker_path_), ec.message(), kWriteMarkerPhaseComplete);
} else {
REXFS_ERROR(
"Could not delete write-in-progress marker '{}' ({}) nor rewrite its phase - the "
"container may be needlessly quarantined (kept, never deleted) at the next launch",
rex::path_to_utf8(write_marker_path_), ec.message());
}
}
}
}
std::string HostPathDevice::ReadWriteMarkerPhase(const std::filesystem::path& marker_path) {
auto file = rex::filesystem::OpenFile(marker_path, "rb");
if (!file) {
return {};
}
char buffer[64] = {};
const size_t read = fread(buffer, 1, sizeof(buffer) - 1, file);
fclose(file);
std::string phase(buffer, read);
const size_t eol = phase.find_first_of("\r\n");
if (eol != std::string::npos) {
phase.resize(eol);
}
while (!phase.empty() && (phase.back() == ' ' || phase.back() == '\t')) {
phase.pop_back();
}
return phase;
}
} // namespace rex::filesystem
@@ -140,6 +140,11 @@ void HostPathEntry::CommitAtomicWrite(const std::filesystem::path& temp_path, bo
// Write handle closed without writing anything: nothing to commit.
std::filesystem::remove(temp_path, ec);
} else {
// The container is about to change: flip the write marker to its
// "committing" phase BEFORE the first rename, so only a death from here
// on is treated as potentially torn at the next mount. The abandon
// paths above never mutate committed files and take no flip.
host_device->OnAtomicWriteCommit();
// Keep exactly one backup generation, then swap the finished temp in.
std::filesystem::path bak_path = host_path_;
bak_path += kAtomicWriteBackupSuffix;
@@ -291,6 +291,38 @@ void ContentManager::QuarantineTornPackage(uint64_t xuid, const XCONTENT_AGGREGA
return;
}
// The marker's phase word says what the dead session proved about the
// container before it died:
// "writing" - no commit rename ever started; every file is still its
// previous version: consistent, keep it.
// "complete" - every commit finished; only the marker's own deletion
// failed (typically an antivirus holding the file):
// consistent, keep it.
// "committing" - a commit rename ran and the session died before the
// burst closed: may be torn across files, quarantine.
// Unreadable or unrecognized content quarantines too - the conservative
// default for a marker that proves nothing.
const std::string phase = rex::filesystem::HostPathDevice::ReadWriteMarkerPhase(marker_path);
if (phase == rex::filesystem::kWriteMarkerPhaseWriting ||
phase == rex::filesystem::kWriteMarkerPhaseComplete) {
std::error_code rm_ec;
std::filesystem::remove(marker_path, rm_ec);
if (rm_ec) {
REXSYS_ERROR(
"Content '{}' has a stale write marker (phase '{}') proving it consistent, but the "
"marker could not be removed ({}); the container is kept and the check will repeat "
"next mount",
rex::path_to_utf8(package_path), phase, rm_ec.message());
} else {
REXSYS_ERROR(
"Content '{}' had a stale write marker (phase '{}' - the last session died without a "
"commit in flight); the container is consistent, so it was kept and only the marker "
"was cleared",
rex::path_to_utf8(package_path), phase);
}
return;
}
if (std::filesystem::is_directory(package_path, ec) && !ec) {
auto quarantine_root = root_path_ / "quarantine";
std::error_code mkdir_ec;
@@ -17,10 +17,18 @@
#include <native/filesystem/devices/host_path_device.h>
#include <native/filesystem/entry.h>
#include <native/filesystem/file.h>
#include <rex/system/xam/content_manager.h>
// White-box seams for the flush-before-commit drill (src-internal headers).
#include "../../../src/native/filesystem/devices/host_path_entry.h"
#include "../../../src/native/filesystem/devices/host_path_file.h"
namespace fs = std::filesystem;
using rex::filesystem::FileAccess;
using rex::filesystem::HostPathDevice;
using rex::system::XContentType;
using rex::system::xam::ContentManager;
using rex::system::xam::XCONTENT_AGGREGATE_DATA;
namespace {
@@ -150,11 +158,269 @@ TEST_CASE("write marker: present exactly while a write session is open", "[files
REQUIRE(entry->Open(FileAccess::kGenericRead | FileAccess::kGenericWrite, &file) == 0);
// A write session is open: the marker must be on disk NOW - this is what
// survives a kill and triggers quarantine at the next mount.
// survives a kill and decides quarantine at the next mount. Before any
// commit rename its phase must be "writing": a kill here leaves every
// committed file untouched.
CHECK(fs::exists(marker));
CHECK(HostPathDevice::ReadWriteMarkerPhase(marker) ==
rex::filesystem::kWriteMarkerPhaseWriting);
file->Destroy();
CHECK(!fs::exists(marker));
fs::remove(marker, ec);
}
TEST_CASE("write marker: phase flips to 'committing' at the first commit rename and never back",
"[filesystem][atomic_write]") {
TempDir dir("marker-phase");
WriteHostFile(dir.path / "a.dat", "OLD-A");
WriteHostFile(dir.path / "b.dat", "OLD-B");
const fs::path marker = dir.path.parent_path() / "marker-phase-test.rex-writing";
std::error_code ec;
fs::remove(marker, ec);
HostPathDevice device("\\Device\\AtomicTest6\\", dir.path, false);
device.set_write_marker_path(marker);
REQUIRE(device.Initialize());
// Two sessions in one burst, so the marker outlives the first commit.
auto* entry_a = device.ResolvePath("a.dat");
auto* entry_b = device.ResolvePath("b.dat");
REQUIRE(entry_a != nullptr);
REQUIRE(entry_b != nullptr);
rex::filesystem::File* file_a = nullptr;
rex::filesystem::File* file_b = nullptr;
REQUIRE(entry_a->Open(FileAccess::kGenericRead | FileAccess::kGenericWrite, &file_a) == 0);
REQUIRE(entry_b->Open(FileAccess::kGenericRead | FileAccess::kGenericWrite, &file_b) == 0);
CHECK(HostPathDevice::ReadWriteMarkerPhase(marker) ==
rex::filesystem::kWriteMarkerPhaseWriting);
const std::string new_a = "NEW-A";
size_t written = 0;
REQUIRE(file_a->WriteSync({reinterpret_cast<const uint8_t*>(new_a.data()), new_a.size()}, 0,
&written) == 0);
// First commit of the burst: the container is now mutating, and the burst
// is still open (b.dat's session) - a kill from here on may tear it.
file_a->Destroy();
CHECK(fs::exists(marker));
CHECK(HostPathDevice::ReadWriteMarkerPhase(marker) ==
rex::filesystem::kWriteMarkerPhaseCommitting);
// The remaining session closes without a commit: monotonic - the phase
// never returns to "writing"; the whole burst ending deletes the marker.
file_b->Destroy();
CHECK(!fs::exists(marker));
fs::remove(marker, ec);
}
#if defined(_WIN32)
TEST_CASE("write marker: a marker that cannot be deleted is rewritten to phase 'complete'",
"[filesystem][atomic_write]") {
TempDir dir("marker-held");
WriteHostFile(dir.path / "save.dat", "OLD");
const fs::path marker = dir.path.parent_path() / "marker-held-test.rex-writing";
std::error_code ec;
fs::remove(marker, ec);
HostPathDevice device("\\Device\\AtomicTest7\\", dir.path, false);
device.set_write_marker_path(marker);
REQUIRE(device.Initialize());
auto* entry = device.ResolvePath("save.dat");
REQUIRE(entry != nullptr);
rex::filesystem::File* file = nullptr;
REQUIRE(entry->Open(FileAccess::kGenericRead | FileAccess::kGenericWrite, &file) == 0);
REQUIRE(fs::exists(marker));
const std::string new_content = "NEW!";
size_t written = 0;
REQUIRE(file->WriteSync({reinterpret_cast<const uint8_t*>(new_content.data()),
new_content.size()},
0, &written) == 0);
{
// Simulate an antivirus holding the marker: an MSVC ifstream shares
// read/write but not delete, so deletion fails while rewriting the
// content stays possible - exactly the scanner behaviour in the field.
std::ifstream holder(marker, std::ios::binary);
REQUIRE(holder.is_open());
file->Destroy();
// The commit itself went through; the undeletable marker was rewritten
// to "complete" so the next mount keeps the container.
CHECK(ReadHostFile(dir.path / "save.dat") == "NEW!");
CHECK(fs::exists(marker));
CHECK(HostPathDevice::ReadWriteMarkerPhase(marker) ==
rex::filesystem::kWriteMarkerPhaseComplete);
}
fs::remove(marker, ec);
}
#endif // defined(_WIN32)
namespace {
// Records that Flush() ran, and whether the real file was still untouched at
// that moment - i.e. the flush landed BEFORE the commit renames.
class FlushProbeHandle : public rex::filesystem::FileHandle {
public:
FlushProbeHandle(const fs::path& temp_path, const fs::path& real_path, bool* flushed,
bool* real_untouched_at_flush)
: FileHandle(temp_path),
real_path_(real_path),
flushed_(flushed),
real_untouched_at_flush_(real_untouched_at_flush) {}
bool Read(size_t, void*, size_t, size_t*) override { return false; }
bool Write(size_t, const void*, size_t, size_t*) override { return false; }
bool SetLength(size_t) override { return false; }
void Flush() override {
*flushed_ = true;
*real_untouched_at_flush_ = (ReadHostFile(real_path_) == "OLD");
}
private:
fs::path real_path_;
bool* flushed_;
bool* real_untouched_at_flush_;
};
} // namespace
TEST_CASE("atomic write: the temp is flushed to stable storage before the commit renames",
"[filesystem][atomic_write]") {
TempDir dir("flush");
WriteHostFile(dir.path / "save.dat", "OLD");
HostPathDevice device("\\Device\\AtomicTest8\\", dir.path, false);
REQUIRE(device.Initialize());
auto* entry = device.ResolvePath("save.dat");
REQUIRE(entry != nullptr);
// The finished temp, as the guest's last write left it (written after
// Initialize - mount sweeps stale temps).
WriteHostFile(dir.path / "save.dat.rex-tmp", "NEW!");
bool flushed = false;
bool real_untouched_at_flush = false;
auto* file = new rex::filesystem::HostPathFile(
FileAccess::kGenericWrite, static_cast<rex::filesystem::HostPathEntry*>(entry),
std::make_unique<FlushProbeHandle>(dir.path / "save.dat.rex-tmp", dir.path / "save.dat",
&flushed, &real_untouched_at_flush),
dir.path / "save.dat.rex-tmp", /*started_dirty=*/true);
// Close = flush + commit. Rename atomicity only covers which name points
// at which file - the flush is what gets the BYTES to stable storage
// before the temp becomes the real file (power loss would otherwise leave
// a correctly-named file of zeros).
file->Destroy();
CHECK(flushed);
CHECK(real_untouched_at_flush);
CHECK(ReadHostFile(dir.path / "save.dat") == "NEW!");
CHECK(ReadHostFile(dir.path / "save.dat.rex-bak") == "OLD");
}
namespace {
// Fixture for the mount-time torn-container decision table. Lays out a
// content root with one populated extracted package and a stale marker whose
// content the drill controls, then runs the same check the content manager
// runs at XamContentCreate/XamContentOpen time.
struct MountFixture {
TempDir root;
fs::path package_dir;
fs::path marker;
ContentManager manager;
XCONTENT_AGGREGATE_DATA data{};
MountFixture()
: root("mount-check"),
// content_root/xuid/title_id/content_type/name - the layout
// ResolvePackagePath produces for xuid 1, an explicit title id, and
// a saved-game package.
package_dir(root.path / "0000000000000001" / "12345678" / "00000001" / "sav_test"),
marker(fs::path(package_dir) += ".rex-writing"),
manager(nullptr, root.path) {
fs::create_directories(package_dir);
WriteHostFile(package_dir / "gamedata.dat", "SAVED");
data.content_type = XContentType::kSavedGame;
data.title_id = 0x12345678;
data.xuid = 0; // resolves against the xuid argument below
data.set_display_name(u"drill");
data.set_file_name("sav_test");
}
void RunMountCheck() { manager.QuarantineTornPackage(1, data); }
bool PackageIntact() const {
return fs::is_directory(package_dir) &&
ReadHostFile(package_dir / "gamedata.dat") == "SAVED";
}
size_t QuarantinedCount() const {
std::error_code ec;
size_t count = 0;
for (fs::directory_iterator it(root.path / "quarantine", ec), end; !ec && it != end; ++it) {
++count;
}
return count;
}
};
} // namespace
TEST_CASE("mount check: a stale 'writing' marker proves consistency - cleared, not quarantined",
"[filesystem][atomic_write]") {
MountFixture fx;
// The dead session never started a commit rename: every file in the
// package is still its previous version.
WriteHostFile(fx.marker, "writing\n");
fx.RunMountCheck();
CHECK(fx.PackageIntact());
CHECK(!fs::exists(fx.marker));
CHECK(fx.QuarantinedCount() == 0);
}
TEST_CASE("mount check: a stale 'complete' marker proves consistency - cleared, not quarantined",
"[filesystem][atomic_write]") {
MountFixture fx;
// The burst finished every commit; only the marker's own deletion failed
// (the blocked-delete rewrite path wrote this phase).
WriteHostFile(fx.marker, "complete\n");
fx.RunMountCheck();
CHECK(fx.PackageIntact());
CHECK(!fs::exists(fx.marker));
CHECK(fx.QuarantinedCount() == 0);
}
TEST_CASE("mount check: 'committing', unreadable or unrecognized markers quarantine",
"[filesystem][atomic_write]") {
MountFixture fx;
SECTION("phase 'committing': a commit rename ran, the container may be torn") {
WriteHostFile(fx.marker, "committing\n");
}
SECTION("unrecognized content proves nothing - conservative default") {
WriteHostFile(fx.marker, "Write in progress. Legacy prose, no phase word.\n");
}
SECTION("empty marker proves nothing - conservative default") {
WriteHostFile(fx.marker, "");
}
fx.RunMountCheck();
// Moved aside - never deleted - and the marker is gone, so the game
// recreates the package cleanly.
CHECK(!fs::exists(fx.package_dir));
CHECK(!fs::exists(fx.marker));
CHECK(fx.QuarantinedCount() == 1);
}