From e087093591c041a0f9281e7331f36340db205fc0 Mon Sep 17 00:00:00 2001 From: Pegasus Date: Wed, 9 Sep 2026 13:52:03 +0300 Subject: [PATCH] Require pinned observer roles and complete ending evidence (cherry picked from commit 3feb1d6fc84ee9bd4cc3481d9b6315d447ecc25a) --- tools/tasreplays/observation_evidence.py | 22 ++++++---- tools/tasreplays/pepsiman.py | 49 +++++++++++++++++------ tools/tasreplays/test_pepsiman.py | 27 ++++++++++++- tools/tasreplays/verify_scph5500_seeds.py | 47 ++++++++++++++++++++++ 4 files changed, 124 insertions(+), 21 deletions(-) create mode 100644 tools/tasreplays/verify_scph5500_seeds.py diff --git a/tools/tasreplays/observation_evidence.py b/tools/tasreplays/observation_evidence.py index 2b514c8c..550e2d07 100644 --- a/tools/tasreplays/observation_evidence.py +++ b/tools/tasreplays/observation_evidence.py @@ -43,7 +43,14 @@ def terminal_consistency(pages, raw_path, expected_frames, full_sha=None): return raw -def compare_stock_observations(stock, observed, endpoint): +def captured_frames(endpoint, input_frames=71806, checkpoint_every=1200, tail_every=60): + frames = {0, endpoint, input_frames} + frames.update(range(checkpoint_every, endpoint+1, checkpoint_every)) + frames.update(frame for frame in range(input_frames+1, endpoint+1) if frame % tail_every == 0) + return frames + + +def compare_stock_observations(stock, observed, endpoint, input_frames=71806, checkpoint_every=1200, tail_every=60): """Recompute passivity from captures, including all declared sparse images.""" a, b = stock/'ram-frames.tsv', observed/'ram-frames.tsv' with a.open() as left, b.open() as right: @@ -53,16 +60,17 @@ def compare_stock_observations(stock, observed, endpoint): raise ValueError(f'stock/observer RAM or lag mismatch at line {count}') if count != endpoint + 2: raise ValueError('stock/observer return coverage differs') - frames = {0, endpoint, 71806} - frames.update(range(1200, endpoint+1, 1200)) - frames.update(frame for frame in range(71807, endpoint+1) if frame % 60 == 0) + frames = captured_frames(endpoint,input_frames,checkpoint_every,tail_every) + required = {f'frame-{frame:06d}.png' for frame in frames} + inventories = [{p.name for p in root.glob('frame-*.png')} for root in [stock,observed]] + if inventories[0] != inventories[1] or not required <= inventories[0]: + raise ValueError('stock/observer screenshot inventory differs or is incomplete') evidence = [a, b] - for frame in sorted(frames): - name = f'frame-{frame:06d}.png' + for name in sorted(inventories[0]): s, n = stock/name, observed/name # Identical encoded PNG bytes also establish identical decoded pixels. if s.read_bytes() != n.read_bytes(): - raise ValueError(f'stock/observer screenshot differs at frame {frame}') + raise ValueError(f'stock/observer screenshot differs: {name}') evidence += [s, n] for name in ['loaded-bios.json', 'effective-sync.json', 'effective-settings.json']: s, n = stock/name, observed/name diff --git a/tools/tasreplays/pepsiman.py b/tools/tasreplays/pepsiman.py index 8c08de1b..4c51b830 100644 --- a/tools/tasreplays/pepsiman.py +++ b/tools/tasreplays/pepsiman.py @@ -19,7 +19,7 @@ import bk2_intake from bk2_to_psxrti import convert import tekken3 from compare_ram_pages import read_pages -from observation_evidence import compare_returns, terminal_consistency, compare_stock_observations +from observation_evidence import compare_returns, terminal_consistency, compare_stock_observations, captured_frames HERE=Path(__file__).resolve().parent ROOT=HERE.parent.parent @@ -30,6 +30,7 @@ BIOS_SHA='9c0421858e217805f4abe18698afea8d5aa36ff0727eb8484944e00eb5e7eadb' EXE_SHA='96960e14406b58dd71c6717bb374537313ff7b64215330671c989ba2926f3718' STOCK_CORE_SHA='749d6dd58430d010e46ae97c628e113adad5f420c05c2ada0ad35e58191781c0' STOCK_COMMIT='a15b31a46bdac27d843d3ebbc5a860012d8452fb' +OBSERVER_CORE_SHA='fce723aa1e2b73ede285863d4b1841445be47e5dd2ccb054b2e170830a7b5167' TRACKS=[ (124164432,'6452e78fa5d65abec2a5084afa28878bf478c9c63d6ba739b6b542cad366eded'), (10936800,'6cd94daf5173b9f84469683fd0270621a8ca2fbd614a6d8af1e75375c36f2531'), @@ -44,7 +45,9 @@ digest=tekken3.digest write=tekken3.write_json command=tekken3.command -def verify_control_identity(path, manifest, tail): +def verify_control_identity(path, manifest, tail, role='stock'): + if role not in {'stock','observer'}: raise ValueError('explicit stock/observer role required') + if manifest.get('role',role)!=role: raise ValueError('source role differs from requested admission') if (manifest.get('schema'),manifest.get('source_tag'),manifest.get('original_inputs'), manifest.get('cutoff'),manifest.get('neutral_tail')) != ('pepsiman-stock-control-v1','2.3',FRAMES,FRAMES,tail): raise ValueError('source manifest does not describe the completed original movie') @@ -76,11 +79,12 @@ def verify_control_identity(path, manifest, tail): if sync != {'EnableLEC':False,'FIOConfig':{'Multitaps':[False,False],'Memcards':[False,False], 'Devices8':[1,0,0,0,0,0,0,0]}}: raise ValueError('source effective controller/card configuration differs') - if (path/'loaded-core.json').exists(): + if role=='observer': loaded=json.loads((path/'loaded-core.json').read_text()) core=names['octoshock.dll'] if loaded['sha256']!=core['sha256'] or Path(loaded['path']).resolve()!=Path(core['path']).resolve(): raise ValueError('observed loaded core differs from its bound binary') + if core['sha256']!=OBSERVER_CORE_SHA: raise ValueError('observer core is not the pinned candidate getter build') build=json.loads((path/'observer-build.json').read_text()) if (build.get('exit_code'),build.get('source_head'),build.get('upstream_commit'),build.get('core_sha256')) != ( 0,manifest.get('source_commit'),STOCK_COMMIT,loaded['sha256']): @@ -89,7 +93,8 @@ def verify_control_identity(path, manifest, tail): raise ValueError('observer source must be the reviewed leaf-getter-only revision') if not {'Observation230.cs','Observation230.dll','observer-build.json'}<=names.keys(): raise ValueError('observer helper source/build closure is incomplete') - elif names['octoshock.dll']['sha256']!=STOCK_CORE_SHA or manifest.get('source_commit')!=STOCK_COMMIT: + elif (names['octoshock.dll']['sha256']!=STOCK_CORE_SHA or manifest.get('source_commit')!=STOCK_COMMIT + or (path/'loaded-core.json').exists()): raise ValueError('stock control must use the pinned unmodified release core') def cue_files(text): @@ -140,7 +145,7 @@ def boot_program(track): if hashlib.sha256(data).hexdigest()!=EXE_SHA: raise ValueError('wrong boot executable') return data -def admit_source_control(path): +def admit_source_control(path,role='stock'): path=path.resolve(strict=True) complete=json.loads((path/'complete.json').read_text()) end=json.loads((path/'exit.json').read_text()) @@ -156,10 +161,12 @@ def admit_source_control(path): raise ValueError('source did not complete cleanly') if semantic.get('completion_observed') is not True or type(semantic.get('frame')) is not int or not FRAMES<=semantic['frame']<=endpoint: raise ValueError('source completion needs semantic review') + if semantic['image']!=f"frame-{semantic['frame']:06d}.png" or semantic['frame'] not in captured_frames(endpoint,FRAMES): + raise ValueError('semantic evidence must name its actual captured frame') evidence=path/semantic['image'] if not evidence.resolve().is_relative_to(path) or digest(evidence)!=semantic['image_sha256']: raise ValueError('source completion image identity changed') - verify_control_identity(path,manifest,tail) + verify_control_identity(path,manifest,tail,role) with (path/'ram-frames.tsv').open() as rows: if rows.readline().strip()!='frame\tlag_count\tpc\tram_sha256': raise ValueError('unrecognized stock RAM capture') @@ -172,7 +179,7 @@ def admit_source_control(path): if count!=endpoint+1: raise ValueError('stock control misses declared input/neutral boundaries') return {name:{'path':str(path/name),'sha256':digest(path/name)} for name in ['complete.json','exit.json','semantic-review.json','manifest.json','ram-frames.tsv', - 'loaded-bios.json','effective-sync.json','effective-settings.json']} + 'loaded-bios.json','effective-sync.json','effective-settings.json',semantic['image']]} def setup(args): if os.name!='nt': raise ValueError('Windows UCRT build required for this candidate') @@ -180,8 +187,12 @@ def setup(args): raise ValueError('commit source changes before building') build_head=subprocess.check_output(['git','-C',str(ROOT),'rev-parse','HEAD'],text=True).strip() if not 1<=args.jobs<=64: raise ValueError('jobs must be in 1..64') - for tool in ['gcc','g++','cmake','ninja']: + for tool in ['gcc','g++','cmake','ninja','git']: if not shutil.which(tool): raise ValueError('missing tool: '+tool) + # The Windows System32 bash launcher is WSL, not the Git/coreutils runtime + # used by this fingerprint script and by the Windows CMake check. + bash=Path(shutil.which('git')).resolve().parents[1]/'bin/bash.exe' + if not bash.is_file(): raise ValueError('Git for Windows bash is required for BIOS fingerprint verification') macros=subprocess.check_output(['gcc','-dM','-E','-include','_mingw.h','-'],input='',text=True) if not re.search(r'^#define _UCRT\b',macros,re.M): raise ValueError('UCRT compiler required') control=admit_source_control(args.source_control.resolve(strict=True)) @@ -223,6 +234,12 @@ def setup(args): command(['ctest','--test-dir',tools,'--output-on-failure','-j',str(args.jobs)],project/'test-tools.log') command([tools/'psxrecomp-toml.exe',exe,'--output',project/'census.toml','--seeds',project/'seeds.txt'],project/'census.log') q=lambda p:json.dumps(p.as_posix()) + bios_profile=project/'bios.toml' + profile=(ROOT/'bios/SCPH5500.toml').read_text() + profile=re.sub(r'^rom\s*=.*$',lambda _: 'rom = '+q(staged_bios),profile,flags=re.M) + profile=re.sub(r'^seeds\s*=.*$',lambda _: 'seeds = '+q(ROOT/'recompiler/seeds/phase2_ghidra_seeds_SCPH5500.json'),profile,flags=re.M) + profile=re.sub(r'^out_dir\s*=.*$',lambda _: 'out_dir = '+q(ROOT/'generated'),profile,flags=re.M) + bios_profile.write_text(profile,encoding='utf8') game=project/'game.toml' game.write_text(f'''[game] name = "Pepsiman TAS" @@ -234,7 +251,7 @@ text_size = "0x85800" stack_base = "0x801FFFF0" [recompiler] seeds = {q(project/'seeds.txt')} -bios_config = {q(ROOT/'bios/SCPH5500.toml')} +bios_config = {q(bios_profile)} strict = true out_dir = {q(project/'generated')} [runtime] @@ -243,21 +260,27 @@ bios_hle = false [video] renderer = "software" ''',encoding='utf8') - command([tools/'psxrecomp-bios.exe','--config',ROOT/'bios/SCPH5500.toml','--rom',staged_bios, + command([tools/'psxrecomp-bios.exe','--config',bios_profile,'--rom',staged_bios, '--out-dir',ROOT/'generated'],project/'generate-bios.log') + fingerprint=subprocess.check_output([str(bash),(ROOT/'tools/bios_emitter_fingerprint.sh').as_posix(), + bios_profile.as_posix()],cwd=ROOT,text=True).strip() + if not re.fullmatch('[0-9a-f]{64}',fingerprint): raise ValueError('invalid generated BIOS fingerprint') + (ROOT/'generated/SCPH5500.emitter.sha').write_text(fingerprint+'\n') command([tools/'psxrecomp-game.exe','--config',game],project/'generate-game.log') native=project/'native' command(['cmake','-S',HERE,'-B',native,*common,'-DTAS_PROJECT_DIR='+str(project), '-DTAS_GAME_STEM=SLPS_017.62','-DTAS_EXE_NAME=Pepsiman-TAS','-DTAS_WINDOW_TITLE=Pepsiman TAS', '-DPSXRECOMP_BIOS_STEMS=SCPH5500','-DPSX_SHELLWIN_INTERP=ON', + '-DPSXRECOMP_BIOS_PROFILE='+str(bios_profile), + '-D_psxrt_bash='+str(bash), '-DPSX_RECOMP_UI=OFF','-DPSX_NETPLAY=OFF','-DPSX_REWIND=OFF','-DPSX_SETUP_WIZARD=OFF', - '-DPSX_DEBUG_TOOLS=ON','-DPSX_ENABLE_VULKAN=OFF','-DPSXRECOMP_SKIP_BIOS_STALE_CHECK=ON', + '-DPSX_DEBUG_TOOLS=ON','-DPSX_ENABLE_VULKAN=OFF', '-DCMAKE_DISABLE_FIND_PACKAGE_SDL3=TRUE','-DCMAKE_DISABLE_FIND_PACKAGE_ZLIB=TRUE'],project/'configure-native.log') command(['cmake','--build',native,'--parallel',str(args.jobs)],project/'build-native.log') if (subprocess.check_output(['git','-C',str(ROOT),'status','--porcelain'],text=True).strip() or subprocess.check_output(['git','-C',str(ROOT),'rev-parse','HEAD'],text=True).strip()!=build_head): raise ValueError('source changed during candidate build') - files=[disc,bios,staged_bios,movie,exe,game,tape,project/'input.psxrti',project/'seeds.txt',*tracks] + files=[disc,bios,staged_bios,bios_profile,movie,exe,game,tape,project/'input.psxrti',project/'seeds.txt',*tracks] build=native/'Pepsiman-TAS.exe' generated={str(f.relative_to(ROOT)):digest(f) for f in (ROOT/'generated').glob('SCPH5500*') if f.is_file()} generated.update({str(f):digest(f) for f in (project/'generated').glob('*') if f.is_file()}) @@ -329,7 +352,7 @@ def reference(args): stock=args.stock.resolve(strict=True) observed=args.observed.resolve(strict=True) stock_identity=admit_source_control(stock) - observed_identity=admit_source_control(observed) + observed_identity=admit_source_control(observed,'observer') if (stock/'loaded-core.json').exists() or not (observed/'loaded-core.json').exists(): raise ValueError('reference requires distinct stock and rebuilt observer roles') endpoint=json.loads((observed/'complete.json').read_text())['frame'] diff --git a/tools/tasreplays/test_pepsiman.py b/tools/tasreplays/test_pepsiman.py index c11b9713..a2eb4eb8 100644 --- a/tools/tasreplays/test_pepsiman.py +++ b/tools/tasreplays/test_pepsiman.py @@ -6,7 +6,8 @@ from pathlib import Path import tempfile from unittest.mock import patch import pepsiman -from observation_evidence import compare_returns, terminal_consistency +from observation_evidence import compare_returns, terminal_consistency, compare_stock_observations, captured_frames +from verify_scph5500_seeds import derive from compare_ram_pages import MAGIC, page_hash def cue(): @@ -17,6 +18,12 @@ def cue(): return '\n'.join(parts)+'\n' class PepsimanAdmission(unittest.TestCase): + def test_seed_window_keeps_only_exact_bytes(self): + original=bytes(256);target=bytearray(original);target[63]=1 + seeds=[{'address':f'0x{0xBFC00000+i:08X}'} for i in [0,64,192,193]] + kept,dropped=derive(seeds,original,target) + self.assertEqual(kept,[seeds[1],seeds[2]]) + self.assertEqual(dropped,[seeds[0],seeds[3]]) def test_input_identity_shape(self): for size,sha in pepsiman.TRACKS: self.assertEqual(size%2352,0) @@ -124,4 +131,22 @@ class Observations(unittest.TestCase): raw.write_bytes(data[:-1]+b'\1') with self.assertRaises(ValueError): terminal_consistency(pages,raw,3) + def test_passivity_compares_complete_image_inventory(self): + stock=self.root/'stock';observed=self.root/'observed';stock.mkdir();observed.mkdir() + for root in [stock,observed]: + (root/'ram-frames.tsv').write_text('header\n'+'same\n'*7) + for name in ['loaded-bios.json','effective-sync.json','effective-settings.json']:(root/name).write_text('{}') + for frame in captured_frames(6,3,2,2):(root/f'frame-{frame:06d}.png').write_bytes(b'pixels') + compare=lambda:compare_stock_observations(stock,observed,6,3,2,2) + compare() + (observed/'frame-000003.png').write_bytes(b'changed') + with self.assertRaisesRegex(ValueError,'screenshot differs'):compare() + (observed/'frame-000003.png').write_bytes(b'pixels') + (observed/'frame-000005.png').write_bytes(b'extra') + with self.assertRaisesRegex(ValueError,'inventory'):compare() + (stock/'frame-000005.png').write_bytes(b'extra') + compare() + (observed/'frame-000002.png').unlink() + with self.assertRaisesRegex(ValueError,'inventory'):compare() + if __name__=='__main__': unittest.main() diff --git a/tools/tasreplays/verify_scph5500_seeds.py b/tools/tasreplays/verify_scph5500_seeds.py new file mode 100644 index 00000000..121ef8b6 --- /dev/null +++ b/tools/tasreplays/verify_scph5500_seeds.py @@ -0,0 +1,47 @@ +"""Verify the target BIOS seed selection against owned ROMs; emit no ROM bytes.""" +import argparse +import hashlib +import json +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[2] +REFERENCE_SHA='71af94d1e47a68c11e8fdb9f8368040601514a42a5a399cda48c7d3bff1e99d3' +TARGET_SHA='9c0421858e217805f4abe18698afea8d5aa36ff0727eb8484944e00eb5e7eadb' + + +def derive(seeds, reference, target): + kept=[];dropped=[] + for seed in seeds: + offset=int(seed['address'],16)-0xBFC00000 + (kept if 0<=offset and offset+64<=len(reference) and + reference[offset:offset+64]==target[offset:offset+64] else dropped).append(seed) + return kept,dropped + + +def verify(reference_path,target_path): + reference,target=[p.read_bytes() for p in [reference_path,target_path]] + if any(len(b)!=0x80000 for b in [reference,target]):raise ValueError('512 KiB BIOS required') + if hashlib.sha256(reference).hexdigest()!=REFERENCE_SHA or hashlib.sha256(target).hexdigest()!=TARGET_SHA: + raise ValueError('wrong owned BIOS identity') + original=json.loads((ROOT/'recompiler/seeds/phase2_ghidra_seeds.json').read_text())['seeds'] + admitted=json.loads((ROOT/'recompiler/seeds/phase2_ghidra_seeds_SCPH5500.json').read_text()) + kept,dropped=derive(original,reference,target) + reset=admitted['seeds'][0] + if int(reset['address'],16)!=0xBFC00000 or admitted['seeds'][1:]!=kept: + raise ValueError('target seeds differ from exact matching windows plus reset vector') + if any(int(s['address'],16)>=0xBFC18000 for s in kept):raise ValueError('shell native admission is not qualified') + if admitted['seed_count']!=len(kept)+1:raise ValueError('seed inventory count differs') + if reference[0x80:0x18000]!=target[0x80:0x18000]:raise ValueError('claimed kernel instruction bytes differ') + return {'matched_window_seeds':len(kept),'dropped_window_seeds':len(dropped),'explicit_reset_vectors':1, + 'total_admitted_seeds':len(admitted['seeds']),'reference_sha256':REFERENCE_SHA,'target_sha256':TARGET_SHA, + 'seed_sha256':hashlib.sha256((ROOT/'recompiler/seeds/phase2_ghidra_seeds_SCPH5500.json').read_bytes()).hexdigest(), + 'scope':'64-byte windows and explicit architectural reset seed; shell interpreted; whole native validation pending'} + + +if __name__=='__main__': + p=argparse.ArgumentParser(description=__doc__) + p.add_argument('--reference',type=Path,required=True);p.add_argument('--target',type=Path,required=True) + p.add_argument('--output',type=Path,required=True);a=p.parse_args() + result=verify(a.reference,a.target) + with a.output.open('x') as stream:json.dump(result,stream,indent=2) + print(json.dumps(result))