diff --git a/recompiler/tests/test_segment_aware_codegen.py b/recompiler/tests/test_segment_aware_codegen.py index b57ac371..d0bd982b 100644 --- a/recompiler/tests/test_segment_aware_codegen.py +++ b/recompiler/tests/test_segment_aware_codegen.py @@ -70,6 +70,10 @@ REGRESSION_GUARDS = { # PR D: the call contract compares return PCs in full, in # psx_call_contract and in the BIOS dispatch loop alike. "exact-return-contract": "5.5", + # PR E: overlay code captured at KUSEG/KSEG0/KSEG1 compiles one shard per + # segment, each baking its own segment and a KSEG1 one charging every + # fetch as Beetle does. + "overlay-segment-shards": "5.7", } @@ -394,6 +398,86 @@ def memory_c_store_pc_keys(): return keys, gated +def overlay_shard_gaps(recompiler, gen, info, icache, tmp): + """§5.7: the probe's overlay phase reads its own EXE file back into RAM at + OVL_BUF and calls the copy of `ov_run` through KUSEG, KSEG0 and KSEG1. + Build the capture record overlay_capture.c writes for that (schema v3), + split it with compile_overlays.py's capture_segment_views(), and compile + each view the way its fragment pass does (the entry has no callable + boundary: a dispatch_root) with the real recompiler. Each shard must bake + only its segment's PCs and name its entries by their VA, its manifest must + carry S, and ov_run's straight run must cost what Beetle charges: cached + at KUSEG/KSEG0 (line leaders), +4 per fetch at KSEG1. Returns a note, or + None when every property holds.""" + sys.path.insert(0, os.path.join(ROOT, "tools")) + import compile_overlays as co + import base64 + import binascii + L, ov = info["labels"], info["overlay"] + data = gen.build() # the file: 2 KiB header, then the image + buf, entry = ov["buffer"], ov["ov_run"] + copy = lambda label: buf + 0x800 + L[label] - info["load"] + executed = list(range(copy("ov_run"), copy("ov_getpc") + 8, 4)) + cap = { + "schema": "psxrecomp overlay capture v3", + "load_addr": "0x%08X" % (KSEG0 | buf), "size": len(data), "guard_bytes": 0, + "bytes_b64": base64.b64encode(data).decode(), + "executed_pcs": ["0x%08X" % (KSEG0 | pc) for pc in executed], + "dispatch_entry_pcs": ["0x%08X" % (KSEG0 | entry)], + "dispatch_entry_segments": {n: ["0x%08X" % (s | entry)] for n, s in + (("kuseg", KUSEG), ("kseg0", KSEG0), ("kseg1", KSEG1))}, + "function_entry_pcs": [], "seeds": ["0x%08X" % (KSEG0 | entry)], + } + crc = binascii.crc32(data) & 0xFFFFFFFF + straight = list(range(copy("ov_run_straight"), copy("ov_run_end"), 4)) + problems = [] + views = co.capture_segment_views(cap) + if [int(v["load_addr"], 16) & SEG_MASK for v in views] != [KSEG0, KUSEG, KSEG1]: + return "capture views: %s" % [v["load_addr"] for v in views] + for view in views: + load = int(view["load_addr"], 16) + seg = load & SEG_MASK + with co.image_segment(seg): + _seeds, audit = co.classify_overlay_seeds(view, data, load, len(data), crc, {}) + # No callable boundary: the entry is an isolated-fragment demand, in + # the view's segment. + if audit["dispatch_fragment_demands"] != {seg | entry}: + problems.append("%08X: fragment demands %s" % ( + seg, sorted("%08X" % a for a in audit["dispatch_fragment_demands"]))) + out = os.path.join(tmp, "ovl_%08X" % seg) + os.makedirs(out) + exe = os.path.join(out, "ovl.psx") + with open(exe, "wb") as f: + f.write(co.make_psxexe(load, seg | entry, data, guard_bytes=0)) + seeds_path = os.path.join(out, "seeds.txt") + with open(seeds_path, "w") as f: + f.write("dispatch_root 0x%08X\n" % (seg | entry)) + run_recompiler(recompiler, [exe, "--seeds", seeds_path, "--out-dir", out, "--overlay"]) + bodies = parse_functions(out, r".*_full\.c$") + ranges = next(os.path.join(out, n) for n in os.listdir(out) if n.endswith("_full.ranges")) + with co.image_segment(seg): + func_ids = co.parse_overlay_func_ids(ranges, data, load, len(data)) + manifest = co.overlay_ranges_text(func_ids) + want_names = {"func_%08X" % (seg | entry), "func_%08X" % (seg | copy("ov_getpc"))} + if set(bodies) != want_names: + problems.append("%08X: functions %s" % (seg, sorted(bodies))) + foreign = sorted({pc for body in bodies.values() for pattern in PC_SITES.values() + for pc in pc_constants(body, pattern) if pc & SEG_MASK != seg}) + if foreign: + problems.append("%08X: %d PCs baked in another segment, e.g. 0x%08X" + % (seg, len(foreign), foreign[0])) + if (("S %08X\n" % seg in manifest) != (seg != KSEG0) or + "F %08X " % (seg | entry) not in manifest): + problems.append("%08X: manifest S/F records" % seg) + body = bodies.get("func_%08X" % (seg | entry)) + tags = block_fetches(body, seg | copy("ov_run_straight")) if body else None + want = beetle_cycles([seg | pc for pc in straight]) + got = icache.cycles([tags])[0] if tags else None + if got != want: + problems.append("%08X: straight run %s cycles, Beetle %d" % (seg, got, want)) + return "; ".join(problems) or None + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--recompiler", @@ -562,6 +646,12 @@ def main(): gaps.add("alias-fetch-coherence") notes["alias-fetch-coherence"] = "emitted %s, Beetle %d" % (got, want) + # -- 5.7: overlay code gets a shard per segment it was captured in ---- + note = overlay_shard_gaps(args.recompiler, gen, info, icache, tmp) + if note: + gaps.add("overlay-segment-shards") + notes["overlay-segment-shards"] = note + # -- 5: the BIOS emitter charges every KSEG1 (ROM) fetch ------------------ openbios_toml = os.path.join(ROOT, "bios", "OpenBIOS.toml") with tempfile.TemporaryDirectory() as tmp: diff --git a/tools/segment_testrom/gen_segment_exe.py b/tools/segment_testrom/gen_segment_exe.py index 685cac41..f4c71a0c 100644 --- a/tools/segment_testrom/gen_segment_exe.py +++ b/tools/segment_testrom/gen_segment_exe.py @@ -22,6 +22,24 @@ The program exercises each place the segment reaches: R+0x20 T2 ticks around `probe_run` via its KSEG1 alias (uncached, +4/fetch) R+0x24 `getpc` link inside that run (0xA001xxxx) + R+0x28 T2 ticks around overlay `ov_run` entered at KUSEG (cached fetch) + R+0x2C `ov_getpc` link inside that run (0x000Axxxx) + R+0x30 T2 ticks around `ov_run` entered at KSEG0 (cached fetch) + R+0x34 `ov_getpc` link inside that run (0x800Axxxx) + R+0x38 T2 ticks around `ov_run` entered at KSEG1 (uncached) + R+0x3C `ov_getpc` link inside that run (0xA00Axxxx) + R+0x40 the file descriptor open() returned, R+0x44 the bytes read() + +The overlay phase (docs/SEGMENT_AWARE_CODE.md §5.7) loads code the way a game +loads an overlay: after the three `probe_run` sequences, the program reads its +own EXE file back from the disc through the BIOS file calls (B0:32 open, B0:34 +read, B0:36 close; the kernel DMAs the sectors) into RAM at OVL_BUF, and calls +the copy of `ov_run` there through KUSEG, KSEG0 and KSEG1, timing each run like +`probe_run`. `ov_run` is position independent (its call is a `bgezal`), so the +copy runs as it would at its link address. In the runtime that code is dirty +RAM: the interpreter runs it until overlay capture records its entries, with +each entry's segment, and compile_overlays.py builds a shard per segment. + `seeded` is entered through a register. The test seeds it in the link segment and checks that the recompiler accepts the seed. `probe_run` has a straight run of ten instructions after its `getpc` call returns, which is enough to show per-instruction uncached @@ -59,11 +77,14 @@ SEGMENTS = {"kuseg": KUSEG, "kseg0": KSEG0, "kseg1": KSEG1} LOAD = 0x00010000 # link segment: KUSEG RESULTS = 0x00011000 # outside the image; never a code page +OVL_BUF = 0x000A0000 # where the overlay phase reads the EXE file back +EXE_NAME = "SEGT_001.01" # the probe's name on its disc (SYSTEM.CNF BOOT) IO_BASE = 0x1F800000 # T2 count at +0x1120, mode at +0x1124 STACK_TOP = 0x001FFF00 # KUSEG, like the rest of the header -REG = {"zero": 0, "at": 1, "v0": 2, "v1": 3, "a0": 4, "t0": 8, "t1": 9, - "t2": 10, "t3": 11, "s0": 16, "s1": 17, "t9": 25, "sp": 29, "ra": 31} +REG = {"zero": 0, "at": 1, "v0": 2, "v1": 3, "a0": 4, "a1": 5, "a2": 6, + "t0": 8, "t1": 9, "t2": 10, "t3": 11, "s0": 16, "s1": 17, "s2": 18, + "t9": 25, "sp": 29, "ra": 31} def _r(x): @@ -89,6 +110,8 @@ def andi(rt, rs, imm): return I(0x0C, rs, rt, imm) def ori(rt, rs, imm): return I(0x0D, rs, rt, imm) def lui(rt, imm): return I(0x0F, 0, rt, imm) def lw(rt, off, base): return I(0x23, base, rt, off) +def li(rt, imm): return ori(rt, "zero", imm) +def syscall(): return 0x0000000C def sw(rt, off, base): return I(0x2B, base, rt, off) @@ -124,10 +147,10 @@ class Asm: self.fixups.append((len(self.words), "bgezal", (rs, name))) self.words.append(0) - def la(self, rt, name, segment): - """lui/ori of label `name` in `segment` (the probe's alias).""" - self.fixups.append((len(self.words), "la_hi", (rt, name, segment))) - self.fixups.append((len(self.words) + 1, "la_lo", (rt, name, segment))) + def la(self, rt, name, segment, delta=0): + """lui/ori of label `name` (+delta) in `segment` (the probe's alias).""" + self.fixups.append((len(self.words), "la_hi", (rt, name, segment, delta))) + self.fixups.append((len(self.words) + 1, "la_lo", (rt, name, segment, delta))) self.words.extend([0, 0]) def resolve(self): @@ -145,8 +168,8 @@ class Asm: off = (self.labels[name] - (pc + 4)) >> 2 self.words[i] = I(0x01, rs, 0x11, off) else: - rt, name, segment = arg - va = segment | (self.labels[name] & 0x1FFFFFFF) + rt, name, segment, delta = arg + va = segment | ((self.labels[name] + delta) & 0x1FFFFFFF) self.words[i] = (lui(rt, va >> 16) if kind == "la_hi" else ori(rt, rt, va & 0xFFFF)) return b"".join(struct.pack("> 16), ori("a1", "a1", OVL_BUF & 0xFFFF)) + a.emit(li("a2", 0x1000)) + _bios_b0(a, 0x34) + a.emit(sw("v0", 0x44, "s0")) + a.emit(addu("a0", "s2", "zero")) # close(fd) + _bios_b0(a, 0x36) + for n, segment in enumerate((KUSEG, KSEG0, KSEG1)): + a.line_up(8) # each `lw t1` starts a line + # The copy of ov_run: the file holds the 2 KiB header, then the image. + a.la("t0", "ov_run", segment, OVL_BUF + 0x800 - LOAD) + a.emit(lw("t1", 0x1120, "s1"), jalr("t0"), nop(), + lw("t2", 0x1120, "s1"), nop(), subu("t2", "t2", "t1"), + andi("t2", "t2", 0xFFFF), + sw("t2", 0x28 + 8 * n, "s0"), sw("v1", 0x2C + 8 * n, "s0")) + a.label("spin") + a.j("spin"); a.emit(nop()) + + a.line_up() # the copy starts a line too + a.label("ov_run") # position independent + a.emit(addu("t9", "ra", "zero")) + a.bgezal("zero", "ov_getpc"); a.emit(nop()) + a.label("ov_run_straight") # leader: ov_getpc returns here + a.emit(*[addiu("t3", "t3", 1) for _ in range(8)]) + a.emit(jr("t9"), addu("v0", "t9", "zero")) + a.label("ov_run_end") + a.label("ov_getpc") + a.emit(jr("ra"), addu("v1", "ra", "zero")) + a.label("fname") + name = b"cdrom:\\" + EXE_NAME.encode("ascii") + b";1\0" + name += b"\0" * (-len(name) % 4) + a.emit(*struct.unpack("<%dI" % (len(name) // 4), name)) return a +def _bios_b0(a, fn): + """Call B0 function `fn` through the kernel vector 0xB0 (t1 = function).""" + a.emit(li("t2", 0xB0), jalr("t2"), li("t1", fn)) + + def build(): """Return the EXE bytes (2048-byte header + image padded to 2 KiB).""" a = _assemble() body = a.resolve() + # The overlay phase reads the header and the first 0x800 image bytes. + assert a.labels["ov_getpc"] + 8 - LOAD <= 0x800 body += b"\0" * (-len(body) % 0x800) h = bytearray(2048) h[0:8] = b"PS-X EXE" @@ -228,6 +301,11 @@ def probes(): # Straight-line fetch run inside probe_run: runtime PCs, link segment. "straight": list(range(labels["probe_run_straight"], labels["probe_run_end"], 4)), + # The overlay phase's copy of ov_run (physical) and the name it reads. + "overlay": {"buffer": OVL_BUF, + "ov_run": OVL_BUF + 0x800 + labels["ov_run"] - LOAD, + "ov_getpc": OVL_BUF + 0x800 + labels["ov_getpc"] - LOAD, + "file": EXE_NAME}, }