diff --git a/.claude/settings.json b/.claude/settings.json index 5fb21be47..fc1e9b088 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -8,7 +8,7 @@ "hooks": [ { "type": "command", - "command": "bash /home/musashi/bfm-decomp/tools/ghidra_mcp_start.sh" + "command": "bash \"$CLAUDE_PROJECT_DIR\"/tools/ghidra_mcp_start.sh" } ] } @@ -18,7 +18,7 @@ "hooks": [ { "type": "command", - "command": "bash /home/musashi/bfm-decomp/tools/ghidra_mcp_stop.sh", + "command": "bash \"$CLAUDE_PROJECT_DIR\"/tools/ghidra_mcp_stop.sh", "timeout": 150 } ] diff --git a/Makefile b/Makefile index c163cd7ae..f2aa18281 100644 --- a/Makefile +++ b/Makefile @@ -288,6 +288,9 @@ tools-health: # The cookbook index is DERIVED (R33) and self-asserts its coverage (R32). Stale = agents can't # find documented idioms and re-derive them at full token cost (measured, P30 wave 1). $(VENV_PY) tools/cookbook_index.py --check + # P33 B5: the Ghidra roster is DERIVED from config/ghidra/*.jsonl (R33); a stale roster misreports + # which programs' RE work is tracked as text. Pure text check, no Ghidra needed. + $(VENV_PY) tools/ghidra_roster.py --check # Behavioural guards (P31 S70): tools-health audits DATA integrity; these assert that a tool # ACTUALLY DID the work it reports. A guard that is not running is not a guard (R54). $(VENV_PY) tools/work_evidence.py --selftest diff --git a/config/ghidra/ROSTER.md b/config/ghidra/ROSTER.md new file mode 100644 index 000000000..95251ddba --- /dev/null +++ b/config/ghidra/ROSTER.md @@ -0,0 +1,21 @@ +# config/ghidra/ROSTER.md — the Ghidra programs whose hand-authored RE work is tracked as text + +> Generated by `tools/ghidra_roster.py` — never edit by hand. One row per `config/ghidra/.jsonl` +> (the delta format of `tools/ghidra_scripts/ExportAnnotations.java`). Rebuild any row's program from the +> disc + this file with `tools/ghidra_rebuild.sh `; prove it with `--proof` (P33 B5). The binary +> Ghidra project embeds the game's bytes and left git at P33; these files are what R20 backs up instead. + +| program | kind | payload | vram | blocks | hand-authored rows | last proof (this machine) | +|---|---|---|---|---|---|---| +| `SLUS_007.26` | PS-X EXE (PSX Executables Loader) | `extracted/retail/SLUS_007.26` | header | .text 0x80010000–0x80062997; .data 0x80062998–0x80072a37; .sdata 0x80072a38–0x8007474f (+27 uninit/hardware) | 38 — comment 22, func 13, label 3 | PASS 2026-09-07T02:29Z config-sha1 f431642b8854 | +| `aug31_USA_DEMO.EXE` | PS-X EXE (PSX Executables Loader) | `extracted/proto/aug31_USA_DEMO.EXE (prototype disc)` | header | CODE 0x80018000–0x8007cfff (+25 uninit/hardware) | 0 — none (container rows only) | PASS 2026-09-07T02:25Z config-sha1 e86d1f652595 | +| `ov_SC01_077` | raw blob (Raw Binary) | `extracted/retail/SC01.CD.dir/FILE_077.dir/0.4.dec` | 0x80128158 | ram 0x80128158–0x801dab2e (+1 uninit/hardware) | 0 — none (container rows only) | PASS 2026-09-07T02:11Z config-sha1 ef490bd1fd49 | +| `ov_SC06_018` | raw blob (Raw Binary) | `extracted/retail/SC06.CD.dir/FILE_018.dir/0.4.dec` | 0x80128158 | ram 0x80128158–0x801d5b9e (+1 uninit/hardware) | 0 — none (container rows only) | PASS 2026-09-07T02:14Z config-sha1 98f191fcdd08 | +| `resident` | raw blob (Raw Binary) | `extracted/retail/MAIN.CD.dir/FILE_010.dir/1.1` | 0x800CEDF8 | ram 0x800cedf8–0x80128153 (+1 uninit/hardware) | 0 — none (container rows only) | PASS 2026-09-07T02:30Z config-sha1 3b4b91b7fdbe | +| `sep8_SLUS_007.26` | PS-X EXE (PSX Executables Loader) | `extracted/proto/sep8_SLUS_007.26 (prototype disc)` | header | .text 0x80010000–0x80062997; .data 0x80062998–0x80072a37; .sdata 0x80072a38–0x8007474f (+27 uninit/hardware) | 0 — none (container rows only) | PASS 2026-09-07T02:21Z config-sha1 78b798d936cd | + +6 programs tracked. The live project's other programs (the location overlays and `md_*` modules, +129 on disk at P33) hold no hand-authored rows and are regenerated on demand by `tools/prefetch_fleet.py`. +Proof verdicts come from `.run/ghidra_rebuild/.proof` (scratch; absent on a fresh clone). + +_Generated 2026-09-06._ diff --git a/config/ghidra/SLUS_007.26.jsonl b/config/ghidra/SLUS_007.26.jsonl new file mode 100644 index 000000000..b040134fc --- /dev/null +++ b/config/ghidra/SLUS_007.26.jsonl @@ -0,0 +1,70 @@ +{"k":"program","name":"SLUS_007.26","lang":"PSX:LE:32:default","cspec":"default","image_base":"0x80000000","format":"PSX Executables Loader"} +{"k":"block","name":"CACHE","start":"0x1f800000","end":"0x1f8003ff","perm":"rwx","init":false} +{"k":"block","name":"UNK1","start":"0x1f800400","end":"0x1f800fff","perm":"rwx","init":false} +{"k":"block","name":"MCTRL1","start":"0x1f801000","end":"0x1f801023","perm":"rw-","init":false} +{"k":"block","name":"IO_PORTS","start":"0x1f801040","end":"0x1f80105f","perm":"rw-","init":false} +{"k":"block","name":"MCTRL2","start":"0x1f801060","end":"0x1f801063","perm":"rw-","init":false} +{"k":"block","name":"INT_CTRL","start":"0x1f801070","end":"0x1f801075","perm":"rw-","init":false} +{"k":"block","name":"DMA_MDEC_IN","start":"0x1f801080","end":"0x1f80108b","perm":"rw-","init":false} +{"k":"block","name":"DMA_MDEC_OUT","start":"0x1f801090","end":"0x1f80109b","perm":"rw-","init":false} +{"k":"block","name":"DMA_GPU","start":"0x1f8010a0","end":"0x1f8010ab","perm":"rw-","init":false} +{"k":"block","name":"DMA_CDROM","start":"0x1f8010b0","end":"0x1f8010bb","perm":"rw-","init":false} +{"k":"block","name":"DMA_SPU","start":"0x1f8010c0","end":"0x1f8010cb","perm":"rw-","init":false} +{"k":"block","name":"DMA_PIO","start":"0x1f8010d0","end":"0x1f8010db","perm":"rw-","init":false} +{"k":"block","name":"DMA_OTC","start":"0x1f8010e0","end":"0x1f8010eb","perm":"rw-","init":false} +{"k":"block","name":"DMA_CTRL_INT","start":"0x1f8010f0","end":"0x1f8010f7","perm":"rw-","init":false} +{"k":"block","name":"TMR_DOTCLOCK","start":"0x1f801100","end":"0x1f80110f","perm":"rw-","init":false} +{"k":"block","name":"TMR_HRETRACE","start":"0x1f801110","end":"0x1f80111f","perm":"rw-","init":false} +{"k":"block","name":"TMR_SYSCLOCK","start":"0x1f801120","end":"0x1f80112f","perm":"rw-","init":false} +{"k":"block","name":"CDROM_REGS","start":"0x1f801800","end":"0x1f801803","perm":"rw-","init":false} +{"k":"block","name":"GPU_REGS","start":"0x1f801810","end":"0x1f801817","perm":"rw-","init":false} +{"k":"block","name":"MDEC_REGS","start":"0x1f801820","end":"0x1f801827","perm":"rw-","init":false} +{"k":"block","name":"SPU_VOICES","start":"0x1f801c00","end":"0x1f801d7f","perm":"rw-","init":false} +{"k":"block","name":"SPU_CTRL_REGS","start":"0x1f801d80","end":"0x1f801dbf","perm":"rw-","init":false} +{"k":"block","name":"GTEMAC","start":"0x20000000","end":"0x20000263","perm":"r-x","init":false} +{"k":"block","name":"RAM","start":"0x80000000","end":"0x8000ffff","perm":"rwx","init":false} +{"k":"block","name":".text","start":"0x80010000","end":"0x80062997","perm":"r-x","init":true} +{"k":"block","name":".data","start":"0x80062998","end":"0x80072a37","perm":"rw-","init":true} +{"k":"block","name":".sdata","start":"0x80072a38","end":"0x8007474f","perm":"rw-","init":true} +{"k":"block","name":".sbss","start":"0x80074750","end":"0x80074777","perm":"rw-","init":false} +{"k":"block","name":".bss","start":"0x80074778","end":"0x800c7f07","perm":"rw-","init":false} +{"k":"block","name":"RAM","start":"0x800c7f08","end":"0x801fffff","perm":"rwx","init":false} +{"k":"archive","name":"psyq400"} +{"k":"func","addr":"0x80010b40","name":"GameModeDispatch","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[{"name":"local_10","type":"/undefined4","storage":"Stack[-0x10]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_4","type":"/undefined4","storage":"Stack[-0x4]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_8","type":"/undefined4","storage":"Stack[-0x8]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_c","type":"/undefined4","storage":"Stack[-0xc]:4","first":0,"src":"DEFAULT","comment":null}],"comment":"GameModeDispatch -- (*gameModeHandlerTable[gameMode])(). 18 handlers [0]-[17] @0x800629F4;\ngameMode @0x800B99DE. Handler[7]=DebugMenuHandler (the TCRF L3 debug menu, entered by forcing\ngameMode=7). Only the SAVE menu flushes the location overlay; other modes keep it. Phase 3 T8."} +{"k":"func","addr":"0x80011144","name":"DebugMenuHandler","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[],"comment":"DebugMenuHandler -- gameMode 7 handler (gameModeHandlerTable[7]). Loads cdFileLocTable[11] ->\n0x800CEDF8 via CdReadRequest, then inits (FUN_80011778, FUN_80015310). = the TCRF L3 debug menu\n(retail: only room-select works). Entered by forcing gameMode=7 (padState 0x80078DC2 is the\ncheat's L3 condition, NOT a game read). Phase 3 T8."} +{"k":"func","addr":"0x80018730","name":"LzssDecodeSector","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[],"comment":"LZSS streaming decompressor (PAC type-4 overlays). Resumable 5-state machine; consumes input in 0x800-byte CD-sector chunks. Ring window = lzss_ringBuffer @ 0x1F800000 (scratchpad, 1024B). State var lzss_state (0x800C7D24) indexes LzssStateTable (0x80072A30). Match: absolute 10-bit ring index, -1 bias; pos==0 terminates. Source: AiroRom wiki MIPS listing, byte-verified vs local US SLUS-00726 EXE (memory-map §2.1, confidence=verified)."} +{"k":"func","addr":"0x8001971c","name":"LoaderInitFileTable","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[{"name":"local_10","type":"/undefined4","storage":"Stack[-0x10]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_14","type":"/undefined4","storage":"Stack[-0x14]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_18","type":"/undefined4","storage":"Stack[-0x18]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_1c","type":"/undefined4","storage":"Stack[-0x1c]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_20","type":"/undefined4","storage":"Stack[-0x20]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_24","type":"/undefined4","storage":"Stack[-0x24]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_28","type":"/undefined4","storage":"Stack[-0x28]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_2c","type":"/undefined4","storage":"Stack[-0x2c]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_30","type":"/undefined4","storage":"Stack[-0x30]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_38","type":"/undefined4","storage":"Stack[-0x38]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_4","type":"/undefined4","storage":"Stack[-0x4]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_8","type":"/undefined4","storage":"Stack[-0x8]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_c","type":"/undefined4","storage":"Stack[-0xc]:4","first":0,"src":"DEFAULT","comment":null}],"comment":"LoaderInitFileTable -- file-loader directory resolver (called once at boot @0x800101fc).\nSearches \\DEBUG.BIN;1 -> debugBinPresent (0x800747d0); resolves 21 CdPathTable[0x80062C24]\nentries (stride 0x30 = {name[0x14]; CdlFILE}) via CdSearchFile; reads LIST.CD (0xE40 B) via\nCdReadRequest -> listCdBuffer (0x80180000); builds cdFileLocTable (0x800AE830: CdlLOC+size per\nsub-file) over the 8 .CD files (MAIN, SC01-SC07). Provenance: static trace, Phase 3 T2 (verified)."} +{"k":"func","addr":"0x80019a24","name":"CdReadRequest","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[{"name":"local_10","type":"/undefined4","storage":"Stack[-0x10]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_14","type":"/undefined4","storage":"Stack[-0x14]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_18","type":"/undefined4","storage":"Stack[-0x18]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_8","type":"/undefined4","storage":"Stack[-0x8]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_c","type":"/undefined4","storage":"Stack[-0xc]:4","first":0,"src":"DEFAULT","comment":null}],"comment":"CdReadRequest(CdlFILE *src, void *dest, int size, int mode) -- read-request dispatcher.\nGated by CdQueueBusy (0=idle, else refuse). Dedups on cdReq_curSector (*src). Stashes the\nrequest into the control block (cdReq_dest/size/cdlFile/mode) and drives CdReadStateMachine.\nReturns cdReq_result. Callers: LoaderInitFileTable + 5 file-load API sites\n(0x80010ca4,0x80010f1c,0x80011100,0x80011178,0x800112f0). Provenance: static trace, Phase 3 T2 (verified)."} +{"k":"func","addr":"0x80019af8","name":"CdReadStateMachine","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[{"name":"local_10","type":"/undefined4","storage":"Stack[-0x10]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_8","type":"/undefined4","storage":"Stack[-0x8]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_c","type":"/undefined4","storage":"Stack[-0xc]:4","first":0,"src":"DEFAULT","comment":null}],"comment":"CdReadStateMachine -- polled async CD reader (switch on cdReq_state). Hand-rolled low-level\nsequence: CdControl(0x0E CdlSetmode=0xA0) -> CdControl(0x15 CdlSeekL)+CdPosToInt ->\nCdControl(0x06 CdlReadN)+CdReadyCallback(CdReadSectorReadyCB @0x8001a338) -> CdControl(0x09\nCdlPause)+CdSync/CdFlush. Sector drain via CdGetSector inside the ready-callback. The game does\nNOT use PsyQ CdRead()/CdReadSync(). Provenance: static trace, Phase 3 T2 (verified)."} +{"k":"func","addr":"0x8001a338","name":"CdReadSectorReadyCB","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[],"comment":"CdReadSectorReadyCB -- CdReadyCallback for CdReadStateMachine (registered in its case 5). Drains CD\nsectors via CdGetSector and dispatches per file type: RAW -> FUN_8002fc64 (memcpy 0x800 B to\ncdReq_sink 0x800AE728); LZSS -> CdGetSector into lzss_sectorStagingBuf (0x80079A70) then\nLzssDecodeSector (0x80018730) inline, per sector. Decrements cdReq_wordsRemaining (0x800AE748).\nThis is the ONLY caller of LzssDecodeSector. Provenance: static trace, Phase 3 T2/T4 (verified)."} +{"k":"func","addr":"0x8001b3c4","name":"ResourceLoadStateMachine","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[{"name":"local_10","type":"/undefined4","storage":"Stack[-0x10]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_8","type":"/undefined4","storage":"Stack[-0x8]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_c","type":"/undefined4","storage":"Stack[-0xc]:4","first":0,"src":"DEFAULT","comment":null}],"comment":"ResourceLoadStateMachine -- loads a resource by id (resLoad_curId) via resourceIdMap (0x80063138).\nLoad-once cache (resLoad_lastId / resLoad_loadedFileIdx). If field0 < 0 (0xFFFF) -> non-CD path\nFUN_80036d58(field1). Else: ResourceGetCdLoc -> cdReq_curSector; FUN_800363cc(field1,\n&cdFileLocTable[idx], 0x10) streams the load; FUN_8002d4c8(field2, 0) post-processes. Polled on\nresLoad_state; done flag resLoad_result. Provenance: static trace, Phase 3 T3 (verified)."} +{"k":"func","addr":"0x8001b788","name":"ResourceGetCdLoc","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[],"comment":"ResourceGetCdLoc(short id) -> CdlLOC (int). Two-level lookup:\nresourceIdMap[id].field0 (= cdFileLocTable index) -> cdFileLocTable[idx].pos.\nCaller: ResourceLoadStateMachine. Provenance: static trace, Phase 3 T3 (verified)."} +{"k":"func","addr":"0x8002b154","name":"SaveLoadRoutine","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[],"comment":"SaveLoadRoutine -- save/load handler referenced by saveHeaderTemplate (0x80072DF0 ptrs\n0x8002B154/0x8002B1AC/0x8002BEA4). Dispatch branches on (selector & 7). Entry point to the\nPS1 memcard save-data format (Q#5); full data-block format still TBD. Phase 3 T8."} +{"k":"func","addr":"0x800363cc","name":"StreamLoadStateMachine","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[{"name":"local_10","type":"/undefined4","storage":"Stack[-0x10]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_14","type":"/undefined4","storage":"Stack[-0x14]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_18","type":"/undefined4","storage":"Stack[-0x18]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_20","type":"/undefined1","storage":"Stack[-0x20]:1","first":0,"src":"DEFAULT","comment":null},{"name":"local_28","type":"/undefined1","storage":"Stack[-0x28]:1","first":0,"src":"DEFAULT","comment":null},{"name":"local_8","type":"/undefined4","storage":"Stack[-0x8]:4","first":0,"src":"DEFAULT","comment":null},{"name":"local_c","type":"/undefined4","storage":"Stack[-0xc]:4","first":0,"src":"DEFAULT","comment":null}],"comment":"StreamLoadStateMachine (FUN_800363cc) -- second CD loader, DISTINCT from CdReadStateMachine. 0x12-state\nmachine (streamLoad_state 0x8006AF00) with its own CdReadyCallback LAB_800377d8\n(streamLoad_savedReadyCB / streamLoad_cbActive). Issues CdControl SetMode(0xA0)/SeekL(0x15)/ReadN(0x06)\nwith retry + CdMode handling. Driven by ResourceLoadStateMachine(field1, &cdFileLocTable[idx], 0x10).\nExact role (overlay loader vs real-time stream) -> T5. Provenance: static trace, Phase 3 T4 (fn id\nverified; full semantics partial)."} +{"k":"func","addr":"0x8003b0fc","name":"SPU_OBJ_B44","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[],"comment":null} +{"k":"func","addr":"0x8005b7e4","name":"SYS_OBJ_25B0","ret":"/undefined","cc":"unknown","varargs":false,"noreturn":false,"inline":false,"custom":false,"sigsrc":"DEFAULT","params":[],"locals":[],"comment":null} +{"k":"comment","addr":"0x80010b40","type":"plate","text":"GameModeDispatch -- (*gameModeHandlerTable[gameMode])(). 18 handlers [0]-[17] @0x800629F4;\ngameMode @0x800B99DE. Handler[7]=DebugMenuHandler (the TCRF L3 debug menu, entered by forcing\ngameMode=7). Only the SAVE menu flushes the location overlay; other modes keep it. Phase 3 T8."} +{"k":"comment","addr":"0x80011144","type":"plate","text":"DebugMenuHandler -- gameMode 7 handler (gameModeHandlerTable[7]). Loads cdFileLocTable[11] ->\n0x800CEDF8 via CdReadRequest, then inits (FUN_80011778, FUN_80015310). = the TCRF L3 debug menu\n(retail: only room-select works). Entered by forcing gameMode=7 (padState 0x80078DC2 is the\ncheat's L3 condition, NOT a game read). Phase 3 T8."} +{"k":"comment","addr":"0x80018730","type":"plate","text":"LZSS streaming decompressor (PAC type-4 overlays). Resumable 5-state machine; consumes input in 0x800-byte CD-sector chunks. Ring window = lzss_ringBuffer @ 0x1F800000 (scratchpad, 1024B). State var lzss_state (0x800C7D24) indexes LzssStateTable (0x80072A30). Match: absolute 10-bit ring index, -1 bias; pos==0 terminates. Source: AiroRom wiki MIPS listing, byte-verified vs local US SLUS-00726 EXE (memory-map §2.1, confidence=verified)."} +{"k":"comment","addr":"0x8001971c","type":"plate","text":"LoaderInitFileTable -- file-loader directory resolver (called once at boot @0x800101fc).\nSearches \\DEBUG.BIN;1 -> debugBinPresent (0x800747d0); resolves 21 CdPathTable[0x80062C24]\nentries (stride 0x30 = {name[0x14]; CdlFILE}) via CdSearchFile; reads LIST.CD (0xE40 B) via\nCdReadRequest -> listCdBuffer (0x80180000); builds cdFileLocTable (0x800AE830: CdlLOC+size per\nsub-file) over the 8 .CD files (MAIN, SC01-SC07). Provenance: static trace, Phase 3 T2 (verified)."} +{"k":"comment","addr":"0x80019a24","type":"plate","text":"CdReadRequest(CdlFILE *src, void *dest, int size, int mode) -- read-request dispatcher.\nGated by CdQueueBusy (0=idle, else refuse). Dedups on cdReq_curSector (*src). Stashes the\nrequest into the control block (cdReq_dest/size/cdlFile/mode) and drives CdReadStateMachine.\nReturns cdReq_result. Callers: LoaderInitFileTable + 5 file-load API sites\n(0x80010ca4,0x80010f1c,0x80011100,0x80011178,0x800112f0). Provenance: static trace, Phase 3 T2 (verified)."} +{"k":"comment","addr":"0x80019af8","type":"plate","text":"CdReadStateMachine -- polled async CD reader (switch on cdReq_state). Hand-rolled low-level\nsequence: CdControl(0x0E CdlSetmode=0xA0) -> CdControl(0x15 CdlSeekL)+CdPosToInt ->\nCdControl(0x06 CdlReadN)+CdReadyCallback(CdReadSectorReadyCB @0x8001a338) -> CdControl(0x09\nCdlPause)+CdSync/CdFlush. Sector drain via CdGetSector inside the ready-callback. The game does\nNOT use PsyQ CdRead()/CdReadSync(). Provenance: static trace, Phase 3 T2 (verified)."} +{"k":"comment","addr":"0x8001a338","type":"plate","text":"CdReadSectorReadyCB -- CdReadyCallback for CdReadStateMachine (registered in its case 5). Drains CD\nsectors via CdGetSector and dispatches per file type: RAW -> FUN_8002fc64 (memcpy 0x800 B to\ncdReq_sink 0x800AE728); LZSS -> CdGetSector into lzss_sectorStagingBuf (0x80079A70) then\nLzssDecodeSector (0x80018730) inline, per sector. Decrements cdReq_wordsRemaining (0x800AE748).\nThis is the ONLY caller of LzssDecodeSector. Provenance: static trace, Phase 3 T2/T4 (verified)."} +{"k":"comment","addr":"0x8001b3c4","type":"plate","text":"ResourceLoadStateMachine -- loads a resource by id (resLoad_curId) via resourceIdMap (0x80063138).\nLoad-once cache (resLoad_lastId / resLoad_loadedFileIdx). If field0 < 0 (0xFFFF) -> non-CD path\nFUN_80036d58(field1). Else: ResourceGetCdLoc -> cdReq_curSector; FUN_800363cc(field1,\n&cdFileLocTable[idx], 0x10) streams the load; FUN_8002d4c8(field2, 0) post-processes. Polled on\nresLoad_state; done flag resLoad_result. Provenance: static trace, Phase 3 T3 (verified)."} +{"k":"comment","addr":"0x8001b788","type":"plate","text":"ResourceGetCdLoc(short id) -> CdlLOC (int). Two-level lookup:\nresourceIdMap[id].field0 (= cdFileLocTable index) -> cdFileLocTable[idx].pos.\nCaller: ResourceLoadStateMachine. Provenance: static trace, Phase 3 T3 (verified)."} +{"k":"comment","addr":"0x8002b154","type":"plate","text":"SaveLoadRoutine -- save/load handler referenced by saveHeaderTemplate (0x80072DF0 ptrs\n0x8002B154/0x8002B1AC/0x8002BEA4). Dispatch branches on (selector & 7). Entry point to the\nPS1 memcard save-data format (Q#5); full data-block format still TBD. Phase 3 T8."} +{"k":"comment","addr":"0x800363cc","type":"plate","text":"StreamLoadStateMachine (FUN_800363cc) -- second CD loader, DISTINCT from CdReadStateMachine. 0x12-state\nmachine (streamLoad_state 0x8006AF00) with its own CdReadyCallback LAB_800377d8\n(streamLoad_savedReadyCB / streamLoad_cbActive). Issues CdControl SetMode(0xA0)/SeekL(0x15)/ReadN(0x06)\nwith retry + CdMode handling. Driven by ResourceLoadStateMachine(field1, &cdFileLocTable[idx], 0x10).\nExact role (overlay loader vs real-time stream) -> T5. Provenance: static trace, Phase 3 T4 (fn id\nverified; full semantics partial)."} +{"k":"comment","addr":"0x80062c24","type":"plate","text":"CdPathTable: CD-path string table, 23 entries x 0x30-byte stride (0x80062C24..~0x8006307x). Order: \\LIST.CD, \\MAIN.CD, \\SC01..\\SC07.CD, STR movies (ST01-06, SHOPS01, OPEN, END, OUT; ST04/ST05 each twice), \\MUSA.ID (0x80063014), \\DEBUG.BIN (0x80063044). NOTE: \\DEBUG.BIN;1 is a GHOST (absent from retail ISO) => surviving debug-loader path references it (xref anchor, T2/T8). LOGOA/LOGOB.STR and .DA files are NOT in this table (referenced elsewhere, TBD). Source: local EXE scan 2026-06-10, confidence=verified."} +{"k":"comment","addr":"0x80063044","type":"eol","text":"GHOST path: \\DEBUG.BIN;1 is absent from the retail ISO; the code that references this string is the surviving debug loader (prime xref anchor for the debug menu / DEBUG.BIN reachability, T8)."} +{"k":"comment","addr":"0x80063138","type":"plate","text":"resourceIdMap -- resource-ID -> file map, stride 6 bytes = three u16:\n +0 field0 = cdFileLocTable index (0xFFFF = special / non-CD resource)\n +2 field1 = loadParam (overlay/dest slot; arg to FUN_800363cc / FUN_80036d58)\n +4 field2 = procParam (post-load; arg to FUN_8002d4c8; 0 = none)\nIndexed by resLoad_curId. field0*8 indexes cdFileLocTable (0x800AE830: {CdlLOC pos; u32 size}).\nProvenance: static trace, Phase 3 T3 (verified)."} +{"k":"comment","addr":"0x80072a30","type":"plate","text":"LzssStateTable: u32 00000000, u32 FFFFFFFF, then 5 state entry pointers [state0 finish=0x80018824, state1 init=0x80018790, state2 checkBit=0x800187a4, state3 match=0x800187f8, state4 nextBit=0x800188b4]. Content-verified vs memory-map ledger. Indexed by lzss_state."} +{"k":"comment","addr":"0x80072c70","type":"plate","text":"loadDestPtrTable -- table of load DESTINATION addresses (5 entries, runtime-indexed; null-terminated):\n [0] 0x800CEDF8 = resident blob base (= JP 0x800CDF58 + 0xEA0; the dest of all 5 boot-file loads via\n CdReadRequest at 0x80010CA4/0F1C/1100/1178/12F0 -- they read entry[0] @ offset 0x2c70)\n [1] 0x80128158 = location script overlay base (JP 0x80128508 analogue; spans AP-world US patch\n sites 0x8013xxxx-0x8018xxxx and high boss-HP 0x8018EE00..0x801F8714)\n [2] 0x800CAE08, [3] 0x800CCB1C, [4] 0x800C7F08 = resident-region blobs\nAddresses are EXE-constant load dests (static-verified as constants); their semantic ROLES + runtime\nlayout are HEURISTIC pending the T6b live RAM-dump proof. Provenance: static trace, Phase 3 T5."} +{"k":"comment","addr":"0x800747d0","type":"eol","text":"WRITE-ONLY (set by LoaderInitFileTable @0x8001979C, never read) -> the \\DEBUG.BIN;1 loader path is DEAD in retail. Resolves Q#9. (T8)"} +{"k":"comment","addr":"0x80075400","type":"eol","text":"REPORTED (confidence=reported; AP-world): zone-loaded + portal-destination control block. JP delta -0xE70. VERIFY."} +{"k":"comment","addr":"0x80078dc2","type":"eol","text":"REPORTED (confidence=reported; TCRF GameShark D0078DC2 0200): pad-state halfword; 0x0200 = L3. Distinct from 0x80078DCA (do NOT conflate). VERIFY."} +{"k":"comment","addr":"0x80079a70","type":"plate","text":"lzss_sectorStagingBuf -- US LZSS input staging buffer (0x800 B = 1 CD sector). CdReadSectorReadyCB\nfills it via CdGetSector(buf, 0x200 words) then calls LzssDecodeSector(0x80079A70) to decode inline.\nJP analogue 0x80078BD0 (+0xEA0 delta); now BYTE-VERIFIED from US code (was heuristic in memory-map\nS3.2). Provenance: static trace, Phase 3 T4 (verified)."} +{"k":"comment","addr":"0x800b99de","type":"eol","text":"REPORTED (confidence=reported; TCRF debug code + AP-world): game-mode / main-menu flag. Writing 7 here while pad==0x0200 (L3) enters the debug menu. JP=0x800B8B3E (US-0xEA0). VERIFY in Ghidra/emulator before relying."} +{"k":"comment","addr":"0x800b9a08","type":"eol","text":"REPORTED (confidence=reported; 3 sources: autosplitter + AP-world + gamehacking): current location/map id (s16). ~0x1010 town, 0x2018 interiors, 0x3000+ dungeons. Central to overlay loading (T5/T6). VERIFY before relying."} +{"k":"label","addr":"0x8002b154","name":"SaveLoadRoutine","primary":true,"src":"USER_DEFINED","func":true} +{"k":"label","addr":"0x8003b0fc","name":"SPU_OBJ_B44","primary":true,"src":"IMPORTED","func":true} +{"k":"label","addr":"0x8005b7e4","name":"SYS_OBJ_25B0","primary":true,"src":"IMPORTED","func":true} diff --git a/config/ghidra/aug31_USA_DEMO.EXE.jsonl b/config/ghidra/aug31_USA_DEMO.EXE.jsonl new file mode 100644 index 000000000..44032c51c --- /dev/null +++ b/config/ghidra/aug31_USA_DEMO.EXE.jsonl @@ -0,0 +1,28 @@ +{"k":"program","name":"aug31_USA_DEMO.EXE","lang":"PSX:LE:32:default","cspec":"default","image_base":"0x80000000","format":"PSX Executables Loader"} +{"k":"block","name":"CACHE","start":"0x1f800000","end":"0x1f8003ff","perm":"rwx","init":false} +{"k":"block","name":"UNK1","start":"0x1f800400","end":"0x1f800fff","perm":"rwx","init":false} +{"k":"block","name":"MCTRL1","start":"0x1f801000","end":"0x1f801023","perm":"rw-","init":false} +{"k":"block","name":"IO_PORTS","start":"0x1f801040","end":"0x1f80105f","perm":"rw-","init":false} +{"k":"block","name":"MCTRL2","start":"0x1f801060","end":"0x1f801063","perm":"rw-","init":false} +{"k":"block","name":"INT_CTRL","start":"0x1f801070","end":"0x1f801075","perm":"rw-","init":false} +{"k":"block","name":"DMA_MDEC_IN","start":"0x1f801080","end":"0x1f80108b","perm":"rw-","init":false} +{"k":"block","name":"DMA_MDEC_OUT","start":"0x1f801090","end":"0x1f80109b","perm":"rw-","init":false} +{"k":"block","name":"DMA_GPU","start":"0x1f8010a0","end":"0x1f8010ab","perm":"rw-","init":false} +{"k":"block","name":"DMA_CDROM","start":"0x1f8010b0","end":"0x1f8010bb","perm":"rw-","init":false} +{"k":"block","name":"DMA_SPU","start":"0x1f8010c0","end":"0x1f8010cb","perm":"rw-","init":false} +{"k":"block","name":"DMA_PIO","start":"0x1f8010d0","end":"0x1f8010db","perm":"rw-","init":false} +{"k":"block","name":"DMA_OTC","start":"0x1f8010e0","end":"0x1f8010eb","perm":"rw-","init":false} +{"k":"block","name":"DMA_CTRL_INT","start":"0x1f8010f0","end":"0x1f8010f7","perm":"rw-","init":false} +{"k":"block","name":"TMR_DOTCLOCK","start":"0x1f801100","end":"0x1f80110f","perm":"rw-","init":false} +{"k":"block","name":"TMR_HRETRACE","start":"0x1f801110","end":"0x1f80111f","perm":"rw-","init":false} +{"k":"block","name":"TMR_SYSCLOCK","start":"0x1f801120","end":"0x1f80112f","perm":"rw-","init":false} +{"k":"block","name":"CDROM_REGS","start":"0x1f801800","end":"0x1f801803","perm":"rw-","init":false} +{"k":"block","name":"GPU_REGS","start":"0x1f801810","end":"0x1f801817","perm":"rw-","init":false} +{"k":"block","name":"MDEC_REGS","start":"0x1f801820","end":"0x1f801827","perm":"rw-","init":false} +{"k":"block","name":"SPU_VOICES","start":"0x1f801c00","end":"0x1f801d7f","perm":"rw-","init":false} +{"k":"block","name":"SPU_CTRL_REGS","start":"0x1f801d80","end":"0x1f801dbf","perm":"rw-","init":false} +{"k":"block","name":"GTEMAC","start":"0x20000000","end":"0x20000263","perm":"r-x","init":false} +{"k":"block","name":"RAM","start":"0x80000000","end":"0x80017fff","perm":"rwx","init":false} +{"k":"block","name":"CODE","start":"0x80018000","end":"0x8007cfff","perm":"r-x","init":true} +{"k":"block","name":"RAM","start":"0x8007d000","end":"0x801fffff","perm":"r-x","init":false} +{"k":"archive","name":"psyq400"} diff --git a/config/ghidra/ov_SC01_077.jsonl b/config/ghidra/ov_SC01_077.jsonl new file mode 100644 index 000000000..f439ed9dd --- /dev/null +++ b/config/ghidra/ov_SC01_077.jsonl @@ -0,0 +1,5 @@ +{"k":"program","name":"ov_SC01_077","lang":"PSX:LE:32:default","cspec":"default","image_base":"0x00000000","format":"Raw Binary"} +{"k":"block","name":"GTEMAC","start":"0x20000000","end":"0x20000263","perm":"r-x","init":false} +{"k":"block","name":"ram","start":"0x80128158","end":"0x801dab2e","perm":"rwx","init":true} +{"k":"archive","name":"psyq400"} +{"k":"archive","name":"psyq470"} diff --git a/config/ghidra/ov_SC06_018.jsonl b/config/ghidra/ov_SC06_018.jsonl new file mode 100644 index 000000000..de763fafc --- /dev/null +++ b/config/ghidra/ov_SC06_018.jsonl @@ -0,0 +1,5 @@ +{"k":"program","name":"ov_SC06_018","lang":"PSX:LE:32:default","cspec":"default","image_base":"0x00000000","format":"Raw Binary"} +{"k":"block","name":"GTEMAC","start":"0x20000000","end":"0x20000263","perm":"r-x","init":false} +{"k":"block","name":"ram","start":"0x80128158","end":"0x801d5b9e","perm":"rwx","init":true} +{"k":"archive","name":"psyq400"} +{"k":"archive","name":"psyq470"} diff --git a/config/ghidra/resident.jsonl b/config/ghidra/resident.jsonl new file mode 100644 index 000000000..dd4a9edcb --- /dev/null +++ b/config/ghidra/resident.jsonl @@ -0,0 +1,5 @@ +{"k":"program","name":"resident","lang":"PSX:LE:32:default","cspec":"default","image_base":"0x00000000","format":"Raw Binary"} +{"k":"block","name":"GTEMAC","start":"0x20000000","end":"0x20000263","perm":"r-x","init":false} +{"k":"block","name":"ram","start":"0x800cedf8","end":"0x80128153","perm":"rwx","init":true} +{"k":"archive","name":"psyq400"} +{"k":"archive","name":"psyq470"} diff --git a/config/ghidra/sep8_SLUS_007.26.jsonl b/config/ghidra/sep8_SLUS_007.26.jsonl new file mode 100644 index 000000000..d49defdc0 --- /dev/null +++ b/config/ghidra/sep8_SLUS_007.26.jsonl @@ -0,0 +1,32 @@ +{"k":"program","name":"sep8_SLUS_007.26","lang":"PSX:LE:32:default","cspec":"default","image_base":"0x80000000","format":"PSX Executables Loader"} +{"k":"block","name":"CACHE","start":"0x1f800000","end":"0x1f8003ff","perm":"rwx","init":false} +{"k":"block","name":"UNK1","start":"0x1f800400","end":"0x1f800fff","perm":"rwx","init":false} +{"k":"block","name":"MCTRL1","start":"0x1f801000","end":"0x1f801023","perm":"rw-","init":false} +{"k":"block","name":"IO_PORTS","start":"0x1f801040","end":"0x1f80105f","perm":"rw-","init":false} +{"k":"block","name":"MCTRL2","start":"0x1f801060","end":"0x1f801063","perm":"rw-","init":false} +{"k":"block","name":"INT_CTRL","start":"0x1f801070","end":"0x1f801075","perm":"rw-","init":false} +{"k":"block","name":"DMA_MDEC_IN","start":"0x1f801080","end":"0x1f80108b","perm":"rw-","init":false} +{"k":"block","name":"DMA_MDEC_OUT","start":"0x1f801090","end":"0x1f80109b","perm":"rw-","init":false} +{"k":"block","name":"DMA_GPU","start":"0x1f8010a0","end":"0x1f8010ab","perm":"rw-","init":false} +{"k":"block","name":"DMA_CDROM","start":"0x1f8010b0","end":"0x1f8010bb","perm":"rw-","init":false} +{"k":"block","name":"DMA_SPU","start":"0x1f8010c0","end":"0x1f8010cb","perm":"rw-","init":false} +{"k":"block","name":"DMA_PIO","start":"0x1f8010d0","end":"0x1f8010db","perm":"rw-","init":false} +{"k":"block","name":"DMA_OTC","start":"0x1f8010e0","end":"0x1f8010eb","perm":"rw-","init":false} +{"k":"block","name":"DMA_CTRL_INT","start":"0x1f8010f0","end":"0x1f8010f7","perm":"rw-","init":false} +{"k":"block","name":"TMR_DOTCLOCK","start":"0x1f801100","end":"0x1f80110f","perm":"rw-","init":false} +{"k":"block","name":"TMR_HRETRACE","start":"0x1f801110","end":"0x1f80111f","perm":"rw-","init":false} +{"k":"block","name":"TMR_SYSCLOCK","start":"0x1f801120","end":"0x1f80112f","perm":"rw-","init":false} +{"k":"block","name":"CDROM_REGS","start":"0x1f801800","end":"0x1f801803","perm":"rw-","init":false} +{"k":"block","name":"GPU_REGS","start":"0x1f801810","end":"0x1f801817","perm":"rw-","init":false} +{"k":"block","name":"MDEC_REGS","start":"0x1f801820","end":"0x1f801827","perm":"rw-","init":false} +{"k":"block","name":"SPU_VOICES","start":"0x1f801c00","end":"0x1f801d7f","perm":"rw-","init":false} +{"k":"block","name":"SPU_CTRL_REGS","start":"0x1f801d80","end":"0x1f801dbf","perm":"rw-","init":false} +{"k":"block","name":"GTEMAC","start":"0x20000000","end":"0x20000263","perm":"r-x","init":false} +{"k":"block","name":"RAM","start":"0x80000000","end":"0x8000ffff","perm":"rwx","init":false} +{"k":"block","name":".text","start":"0x80010000","end":"0x80062997","perm":"r-x","init":true} +{"k":"block","name":".data","start":"0x80062998","end":"0x80072a37","perm":"rw-","init":true} +{"k":"block","name":".sdata","start":"0x80072a38","end":"0x8007474f","perm":"rw-","init":true} +{"k":"block","name":".sbss","start":"0x80074750","end":"0x80074777","perm":"rw-","init":false} +{"k":"block","name":".bss","start":"0x80074778","end":"0x800c7f07","perm":"rw-","init":false} +{"k":"block","name":"RAM","start":"0x800c7f08","end":"0x801fffff","perm":"rwx","init":false} +{"k":"archive","name":"psyq400"} diff --git a/docs/SETUP.md b/docs/SETUP.md index 64ec4e9ef..68a65d958 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -185,8 +185,8 @@ Lifecycle scripts (under `tools/`): **Session hooks (committed `.claude/settings.json`, as of 2026-06-15):** -- `SessionStart` → runs `ghidra_mcp_start.sh` (auto-starts the MCP server when a Claude Code session begins). -- `SessionEnd` → runs `ghidra_mcp_stop.sh` with **`timeout: 150`** s (auto-saves Ghidra on a clean session exit). +- `SessionStart` → runs `bash "$CLAUDE_PROJECT_DIR"/tools/ghidra_mcp_start.sh` (auto-starts the MCP server when a Claude Code session begins; **P33 B5:** repo-relative, and the script is a silent `exit 0` when `$GHIDRA/support/analyzeHeadless` or `ghidra/bfm.rep` is absent — a contributor's clone has neither). +- `SessionEnd` → runs `bash "$CLAUDE_PROJECT_DIR"/tools/ghidra_mcp_stop.sh` with **`timeout: 150`** s (auto-saves Ghidra on a clean session exit; a no-op when nothing serves on :8080). These hooks live in the **committed `settings.json`** — NOT the gitignored `settings.local.json` — specifically so they are backed up to the remote. Consequence to internalize: **closing Claude Code does NOT save unless the `SessionEnd` hook fires**, and it fires only on *clean* exits — a hard crash of the CC process still loses RAM-only writes. This is exactly why mid-RE clean-stop checkpoints matter. @@ -724,6 +724,11 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo | | `DecompileAt.java` | Decompile the function at a given address (scripted scaffold). | | | `DefineFunctions.java` | Disassemble + create functions at splat's validated entry points (`.run/_funcs.txt`) — completes a raw-blob program's function set (Phase 10). | | | `ApplySymbols.java` + `tools/ghidra_apply_symbols.sh` | **(P31 S78) The Ghidra MIRROR of the curated symbol file (R15/G6), headless with a real save.** `tools/ghidra_apply_symbols.sh [PROG] [symbols files…]` (defaults `SLUS_007.26 config/symbols.us.txt`; MCP must be STOPPED first) reads `name = 0xADDR;` rows and sets every function/label to its curated name; a name held by another address is moved to that address's own curated name first (`firstfile`/`firstfile2`), else to `__at_`. Idempotent; prints `BFMAPPLY renamed_funcs=… unchanged=…`; R9-verify with `ghidra_mcp_verify.sh`. **Use this, not MCP `rename_symbol`/`batch_rename`, for renames:** S78 observed 47 MCP renames NOT persisting through the sentinel stop ("Save succeeded", DB grew, names gone — R9 caught it; cause not yet isolated), while the postScript path persisted 73/73 on the first run. | +| | `ExportAnnotations.java` + `tools/ghidra_export_annotations.sh` | **(P33 B5) The read-only TEXT export of a program** — byte-stable JSONL (fixed key order, sorted, `0x%08x`): `program`/`block`/`archive` container rows, LOCAL-archive types, every function signature (params/locals/storage/sources/comment), defined data, the 5 comment kinds, bookmarks, equates, labels not in the symbol files. `tools/ghidra_export_annotations.sh [PROG…]` → `.run/ghidra_export/.jsonl` (no arg = all programs in one `-readOnly` run; 129 in 18.5 s). MCP must be STOPPED. | +| | `ImportAnnotations.java` | **(P33 B5)** Idempotent compare-before-write import of that JSONL (creates a missing function for a `func` row; refuses unknown row kinds, R43); prints `BFMANN types=… funcs=… … failed=0 rows=N`. **OSGi gotcha:** Ghidra compiles `tools/ghidra_scripts/` as ONE bundle — a compile error in ANY `.java` there breaks EVERY script ("Failed to get OSGi bundle containing script"); javac diagnostics are not shown — compile by hand: `javac -nowarn -d .run/javac_check -cp "$(find ~/ghidra_12.1_PUBLIC/Ghidra -name '*.jar' -path '*/lib/*' | tr '\n' ':')" tools/ghidra_scripts/*.java`. | +| | `tools/ghidra_annotations_delta.py` | **(P33 B5)** `live.jsonl baseline.jsonl out.jsonl [--census]` — the HAND-AUTHORED part of a program = live rows not in the fresh rebuild's baseline (container rows always kept), minus three counted analysis-drift classes: `Error`/`Analysis` bookmarks; `func` rows absent from the baseline with a DEFAULT signature, no comment and an auto name (function-set drift); `func` rows differing from the baseline only by an auto name (the DB lagging the curated symbol file, R15). Prints the census + the dropped counts. | +| | `tools/ghidra_rebuild.sh [--proof] [--keep]` | **(P33 B5) Rebuild ONE program FROM TEXT + the disc** in a scratch project (`build/ghidra_rebuild/proj` — Ghidra refuses a path component starting with `.`, so not `.run/`): import (PSX loader for the 3 EXEs, raw blob at `make -s print-VRAM_BASE` otherwise) + analysis + psyq400.gdt → `DefineFunctions` from the built ELF → `ApplySymbols` (the yaml's symbol files) → baseline export → `ImportAnnotations config/ghidra/.jsonl` → export → delta. `--proof`: `cmp` delta vs the committed file → `PROOF PASS`/`FAIL` (+ `.run/ghidra_rebuild/.proof` marker). Without a committed file it writes `.candidate.jsonl` to review. MCP must be STOPPED. ≈65 s resident/overlays, ≈200 s the EXEs. | +| | `tools/ghidra_roster.py [--check]` | **(P33 B5)** `config/ghidra/ROSTER.md` from the committed files: kind/payload/vram (from the build registry), blocks, hand-authored census, last proof marker. | | | `DecompileFunctions.java` | **Batch**-decompile a list of addresses (arg0 = addr-per-line file, arg1 = out-dir) → `.c` each. Headless harvest Ghidra-C pre-pass (Phase 17); no live MCP / `/mcp` needed. Run: stop MCP, `analyzeHeadless ghidra bfm -process -noanalysis -postScript DecompileFunctions.java `. | | | `tools/ghidra_import.sh` | Headless `analyzeHeadless` import/analysis driver (PS-X EXE; auto-detect PSX loader). | | | `tools/ghidra_import_raw.sh` | Headless import of a RAW flat blob — `BinaryLoader` + `--loader-baseAddr ` + `PSX:LE:32:default` (resident blob / Gen2 overlays; no PS-X EXE header). | @@ -1089,6 +1094,43 @@ fills fast). Nothing is leaking — but the host does not get the memory back on `tools/ghidra_*.sh` are repo-relative (`BFM_GHIDRA_PROJ` overrides the project dir; `ghidra_mcp_verify.sh [PROG]`); Makefile `GHIDRA_PROJ := $(or $(BFM_GHIDRA_PROJ),$(CURDIR)/ghidra)`. +### P33 B5 (S86–S87, 2026-09-06) — Ghidra regenerability: the RE work as text, the binary DB rebuilt and PROVEN from it +- **Why.** The Ghidra project embeds the game's bytes (verified under the page XOR mask) and leaves git at C3. What R20 + backed up as `ghidra/` is now **`config/ghidra/.jsonl`** — the hand-authored rows only — plus the proof that the + program regenerates from the disc + the symbol files + that file. Roster: `config/ghidra/ROSTER.md` (6 programs: the + retail EXE, the resident, `ov_SC01_077`, `ov_SC06_018`, the two prototypes); the other 123 live programs carry no + hand-authored rows and regenerate on demand (`tools/prefetch_fleet.py`). +- **The delta model.** `ExportAnnotations` dumps EVERYTHING; the hand-authored part is what a fresh rebuild does NOT + reproduce (baseline subtraction), minus three measured analysis-drift classes (see the `ghidra_annotations_delta.py` row). + Measured (S87): resident / both overlays → container rows only (their names all come from the symbol files); the retail + EXE → **38 hand-authored rows** (13 annotated functions incl. 3 the ELF does not define — `SaveLoadRoutine`, `SPU_OBJ_B44`, + `SYS_OBJ_25B0` — 22 plate/EOL comments, 3 labels) after dropping 29 set-drift functions in the LINKED regions and 1 Error + bookmark; the two prototypes → 0 after dropping 10 + 5 name-lag rows (their DBs were never re-mirrored from the curated + proto symbol files). **The DB held no hand-authored types at all** (`types=0` in every program). +- **Controls (R39, S87, resident):** negative — one `block` row's end address mutated in the committed file → `PROOF FAIL` + (rc 1); positive round-trip — a synthetic EOL comment, `Note` bookmark, user label and retyped signature appended → run 1 + `BFMANN funcs=1 comments=1 bookmarks=1 labels=1 failed=0`, the re-exported delta contains them, `PROOF PASS`; run 2 with + that delta as the file → `PROOF PASS` (idempotent). The delta filter's own controls: the synthetic rows survive it; a + hand-renamed name-only diff is KEPT (only auto names count as lag). +- **Gotchas that cost the S86 session:** (1) the OSGi bundle — one uncompilable script disables the whole directory and + the headless log names every script, never the error; the fix was a hand `javac` (3 errors: `Long`→`int` unboxing ×2, + and no `LocalVariableImpl(String,int,DataType,int,Program)` ctor — use `VariableStorage(program, off, size)`); + (2) Ghidra refuses a project path with a `.`-prefixed component — the scratch project lives under `build/`; + (3) `-process` without a name processes every program in the folder (one `-readOnly` run exports all 129); + (4) `/undefined` is `DataType.DEFAULT`, in NEITHER type manager — main's first proof "passed" the cmp while 13 of 13 + func rows had failed to import (the plate-comment rows had set the same function comments); the resolver maps it + explicitly and `ghidra_rebuild.sh` now dies unless the import printed `failed=0` (R49; fake `failed=2` refused, + the passing population re-proven); (5) the live EXE's RAM tail block (0x800c7f08+) is `init:true` from the Phase-1 + import and `init:false` rebuilt — no hand-authored row lives there (7 auto functions + 1 Error bookmark), the + committed row takes the rebuild's value. +- **Proofs (S87, all `PROOF PASS`, `failed=0`):** resident 65 s · ov_SC01_077 169 s · ov_SC06_018 173 s · SLUS_007.26 + 210 s (`BFMANN funcs=11 comments=11 unchanged=16 rows=70`) · sep8 202 s · aug31 206 s. `tools/ghidra_roster.py --check` + is in `tools-health` (ignores the per-machine proof column; controls: proof marker removed → OK, census edited → STALE). +- **Same change:** `ExportSymbols.java` R15 fix (output-path arg, refuses to overwrite, refuses `config/`); every + `tools/ghidra_*.sh` and `Makefile` `GHIDRA_PROJ` repo-relative (`BFM_GHIDRA_PROJ` override); `DefineFunctions.java` + takes its list path as arg 1; `ImportPsyqGdt.java` finds the gdt under the install dir; `make print-`; the + `.claude/settings.json` hooks `$CLAUDE_PROJECT_DIR`-relative with the silent no-op (§2.8). + ### P33 B4 (S86, 2026-09-06) — `tools/fetch_psyq.sh`: the OPTIONAL PsyQ SDK objects, user-supplied and verified - **What it is for.** Byte-identity never needs Sony's libraries (without them main links its INCLUDE_ASM tiles — the fresh-clone proof in B3 built all 218 that way). `tools/fetch_psyq.sh [--disc ] [--from ] diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index baec515e5..cf90f4f4a 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -54,8 +54,8 @@ one-time snapshot, `CLAUDE.md` gains "never `git clean -x`" (R20 amendment propo - [x] **B2** 34 absolute includes → `../shared/` + portable-include audit + 15-binary re-gate — xHigh — see Log 2026-09-06 B2 - [x] **B3** `tools/bootstrap.sh` / `make bootstrap` + check-env extensions + fresh-clone proof 218/218 — xHigh — see Log 2026-09-06 B3 - [x] **B4** `tools/fetch_psyq.sh` + CHECKSUMS rows (20 lib40 LIBs, psyq-obj-parser) BEFORE `tools/psyq/` leaves git — xHigh — see Log 2026-09-06 B4 -- [ ] **B5** Ghidra regenerability (ExportAnnotations/ImportAnnotations/ghidra_rebuild.sh `--proof` on SLUS_007.26 + - resident; roster; ExportSymbols R15 fix; path hardcodes; hooks) — Max, MCP stopped +- [x] **B5** Ghidra regenerability (ExportAnnotations/ImportAnnotations/ghidra_rebuild.sh `--proof` on SLUS_007.26 + + resident; roster; ExportSymbols R15 fix; path hardcodes; hooks) — Max (finished at medium, Drew's call) — see Log 2026-09-06 B5 - [ ] **B6** `dumps/CHECKSUMS.sha1` + INDEX.md rewrite + memory-map Source-index row — Low/xHigh - [ ] **B7** No-ROM CI (`no-rom.yml`, `audit_public.py`, `compile_only.py`) — xHigh - [ ] **B8** SETUP.md rows/sections (R21) + `docs/verification.md` — xHigh @@ -193,145 +193,62 @@ Mid-phase rules check after every 4 completed tasks (P6). Commit banked artifact structs in the DB at all); the 154/402 "USER_DEFINED" variables per program are the loader's GTEMAC functions (analysis-origin, will subtract); real user labels beyond the symbol files: aug31 382, sep8 15, SLUS_007.26 1, overlays 1–2. -## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1–B4 ✓, B5 IN PROGRESS (exporter proven; importer + rebuild proof pending on one compile error); NEXT = finish B5 (2026-09-06 ~19:35 MDT, written by session bd19e14a "S86" at Drew's 87%-context pause; SUPERSEDES the earlier blocks) +## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1–B5 ✓; NEXT = B6 (2026-09-06 ~21:00 MDT, written by session fa49faf3 "S87" after the B5 commit; SUPERSEDES the earlier blocks) ### 0. How to use this block You are a FRESH SESSION that has read `PROJECT_CONTEXT.md`, `phase-ends/DIGEST.md`, `PhaseEnd_Phase30/31/32.md` and this file, and nothing else (R64). Replay this block verbatim, state phase / done / NEXT / effort, list the rules from the digest -(R1–R73), then WAIT for Drew. **NEXT = finish B5** (Max). The WIP commit `wip(phase-33): B5 …` holds every B5 file (see -the Log's B5 entry for what is proven and what is not). The SessionStart hook will have RESTARTED the headless MCP server -(it serves `SLUS_007.26` and holds the project lock): run `tools/ghidra_mcp_stop.sh` before ANY headless step (R23), and -prompt Drew for `/mcp` only if you actually need the MCP tools (R29) — B5 needs none. §3 step 1 is the exact resume path. +(R1–R73), then WAIT for Drew. **NEXT = B6** (Low/xHigh). The harness task list must be REBUILT (Drew wants to monitor it — +one TaskCreate per plan item A1…G2, 40 items, mark A1–A4 + B1–B5 completed; R28). The SessionStart hook restarts the headless +MCP server when `ghidra/bfm.rep` exists (it did not stay up in S87 — `ss -tln` showed nothing on :8080; harmless): B6/B7/B8 +need no Ghidra; run `tools/ghidra_mcp_stop.sh` before any headless step (R23). ### 1. Where we are -**Phase 33 — 100% verification + the public flip + Gen2 exit.** Gate 1 approved 2026-09-06 (plan mode, Max). R65–R73 ratified at -gate 1 (DIGEST §3). **Done: A1 (`commit:4012`), A2 (`commit:4013` — the corrected main denominator 45,150 is now the published -number; `.run/sig.main.jsonl` is the sig main is weighed by; `tools/main_seed_ends.py` is new), A3 (`commit:4014`, the -`NO_SDK` knob + `make sdk-dual`, proven 28 s: both legs `143dbb89…`), A4 (`commit:4015`), B1 (`commit:4016`), B2 (`commit:4017`), -B3 (`commit:4018`), B4 (`commit:4019`); B5 in progress (the WIP commit).** The approved plan is VERBATIM -at the end of this file — read its Blocks A–G for every task's files, commands and verification; the "Execution order and -why" section is the sequence; the "Honest scale" table is the budget (≈17–21 sessions; the flip precedes the writing so later -items can be cut). Harness tasks: #1 A1 done, #2 A2 done, #3 A3 next … #41 G2. +**Phase 33 — 100% verification + the public flip + Gen2 exit.** Gate 1 approved 2026-09-06 (plan mode, Max). R65–R73 ratified. +**Done: A1 (`commit:4012`), A2 (`commit:4013`), A3 (`commit:4014`), A4 (`commit:4015`), B1 (`commit:4016`), B2 (`commit:4017`), B3 +(`commit:4018`), B4 (`commit:4019`), B5 (WIP `commit:4020` + the S87 close commit `feat(phase-33): B5 …`).** The approved plan is +VERBATIM at the end of this file — Blocks A–G give every task's files, commands and verification; "Execution order and why" +is the sequence. Effort: Drew ran S87 at **medium** by explicit choice (the plan says Max for B5); the plan's annotations +still stand for the tasks ahead — restate them, Drew decides (R7/R27). -### 2. Facts measured this session that every task depends on (do not re-derive; verify if in doubt, R14) -- **History:** `main` = 4,011 commits (HEAD `commit:4011`); `git rev-list --all` = 4,282 because the `S76-pre-scrub-backup` tag - points at a PARALLEL 126-commit lineage (the Phase-31 S76 trailer scrub's pre-image; NOT an ancestor of main; it still - carries 58 `Claude-Session:` URL trailers and WAS pushed to GitHub — it sits there as unreachable objects), plus - `refs/original/refs/heads/main` (a filter-branch leftover) and 3 stashes; the object store holds 4,401 commit objects. - main still carries 60 `Claude-Session:` trailer lines (20 URL-form in `commit:3901..commit:3920`, 40 bare-UUID). One - pre-existing EMPTY commit on main: `commit:2782` (2026-08-25) — must survive (`--prune-empty auto`, never `always`). - Exactly one commit touches only purge paths: `commit:1712` "session archive update" (2026-08-12). -- **Purge set in history:** the EXE at `extracted/SLUS_007.26` (added `commit:0005`, 2026-06-10) AND `extracted/retail/SLUS_007.26` - (`commit:0017`), same blob; `dumps/*.bin` (28, `commit:0016`); `ghidra/` (15 commits since `commit:0057`; 41 unique blobs; 6 programs - tracked: SLUS_007.26, sep8_SLUS_007.26, aug31_USA_DEMO.EXE, resident, ov_SC01_077, ov_SC06_018; 129 programs on disk, 1,015 MB); - `tools/psyq/` (5 commits since `commit:0029`; 190 files at HEAD); `session archive/` (3 parts 101,711,872 / 101,711,872 / - 67,419,506 bytes — the only blobs > 50 MiB); `tools/ghidra-ext/*.zip` (sha256 GhidrAssistMCP_2.8.0.zip - `983e2add88d70552596dc3160c1ece52ca6ba8ac006c6c61446a780696815dc1`, ghidra_psx_ldr_2026.06.04.zip - `dc57cf1ad126c717f69be63f777d747c9cc0af3b2f833af83eeeac644f8e7c11` — NOT recorded anywhere yet); `tools/brave-CUE/brave.exe` - (`fdc9f48627e8294105350b36e0a857fe878c027221d9e7e28fedb7b219f036d9`). Never committed: the disc, asm/, assets/, expected/, build/. -- **Citations:** 711 hex tokens at HEAD resolve to commits (699 on main: 408×9-char, 181×8, 110×7; 12 off-main, 10 of them in - `phase-ends/logs/Phase31.md`), across phase-ends 979 / docs 183 / tools 19 / .run 17 / src 4 occurrences; 160 distinct hashes - inside commit messages. `\b[0-9a-f]{7,40}\b` is validated (0 word-embedded false hits; `func_800D128C`/`0x800d128c` do not - match). Regex+dict measured 125 MB/s (HEAD's 395 MB text in 3.2 s, 1,202 prefix hits in 93 files). -- **Identities:** author == committer on every commit; `50529377+Druthulu@users.noreply.github.com` ×3,816, `50529377+Druthulu@users.noreply.github.com` ×162, noreply ×33; - the two Gmail strings also appear as text in `phase-ends/PhaseEnd_Phase1.md`, `tools/gater_lane.py`, `tools/parallel_gate.py`; - `git config user.email` is the primary Gmail (switch to the noreply before C7's tip commit). -- **Build facts:** `make extract` never calls `tools/bfm_extract/` (the Makefile's only disc reference is a comment at line 214); - `extract.py:379-381` OVERWRITES the manifest on every run; the committed manifest (1,801 rows) includes the 3 `.DA` audio files - from Tracks 2–4 (a Track-1-only dump gives 1,798 rows); `disks/` here holds Tracks 1–4 + cue. The 11 psyq_integrate calls are - `if [ -d … ]`-guarded (Makefile:864-918); the LINKED count is parsed from the Makefile (`tools/progress.py:485-529`). - `tools/progress.py:944-947` silently drops main when `.run/sig.SLUS_007.26.jsonl` (Ghidra-derived, gitignored) is absent; - `tools/progress.py:1032` hardcodes "1 explained pad-tail". 34 absolute `#include "/home/musashi/bfm-decomp/src/shared/…"` - lines in 19 `src/ov_*/…_jr_*.c` files across 15 binaries. 74 TUs contain `INCLUDE_ASM(` (70 LINKED-region stubs + 4 comment - mentions), 47 use `INCLUDE_RODATA(`; `src/apicard1.c` IS built (fallback tiles). The 4 gitignored `include/*.inc` + - `include_asm.h` are splat's generic presets (identical for every binary). `include/` tracks only `common.h` + `psyq/libcd.h`; - `src/` has zero `#include <…>`. PsyQ 4.0 LIBs (`tools/psyq/lib40/`) have NO recorded public URL (they came from the DTL-S2002 - disc); the RTL 4.2 7z (archive.org) + psyq-obj-parser (decomp.me compilers release) are the only recorded downloads. -- **Verification state at open:** `.run/P32/t4e/tools_health.log` EXIT=0 with one `[warn]` (family_hseq.json missing 6 md_ - binaries) and `sig-main-oracle … 0 PAD-TAIL`; `r22_check.log` 218/218 EXIT=0 at 16:13:17 MDT; report 100.00/100.0/100.0; - `docs/backlog.md` 1 stale row (`func_80062144` inside LINKED `apicard5`); `config/wave_exclude.txt` 0; verbatim 5 == 5. -- **Ghidra tooling:** export exists for symbols only (`ExportSymbols.java`, hardcoded output, blind overwrite — violates R15, - wired to nothing); import path (`ApplySymbols.java` + `ghidra_apply_symbols.sh`) is proven; `ghidra_import.sh` / - `ghidra_import_raw.sh` / `DefineFunctions.java` / `ImportPsyqGdt.java` exist; `prefetch_fleet.py` composes raw imports on demand; - `.run/_funcs.txt` exists for 125 programs. Hardcodes: Makefile:165, all six `tools/ghidra_*.sh`, `DefineFunctions.java:21`, - `ImportPsyqGdt.java:20`, `ghidra_mcp_verify.sh:19`, `.claude/settings.json:11,20`. -- **Environment:** the SessionStart hook's headless Ghidra MCP server is RUNNING (holds the project lock) — run - `tools/ghidra_mcp_stop.sh` before any headless Ghidra step (B5); MCP tools are not connected in this session (connection - failure, not absence). `gh` 2.45.0 on PATH but NOT authenticated (Drew's shell has credentials); `git filter-repo` NOT - installed; **disk: 13 GB free on the 75 GB WSL cap (`.run/` = 38 GB, worktree 44 GB, `.git` 925 MB)**. No `/tmp` (R12). - `~/bfm-decomp/.run/public_audit/` holds this session's blob inventory (`blobs.txt`, 22 MB; gitignored). -- **External facts (from the design agents' web research; treat as data, X2):** frogress projects are admin-created (slug + API - key from the maintainers; POST `progress.deco.mp/data/{project}/{version}/`); decomp.dev reads a GitHub Actions artifact - `_report` in the objdiff report format, registered at `decomp.dev/manage/new`; shields endpoint badges = - `{"schemaVersion":1,"label","message","color"}`; decomp.me presets are created in-browser by any logged-in user (POST - `/api/preset/`), compiler id `gcc2.7.2-psx`, maspsx args via `-Wa,--aspsx-version=2.56,--expand-div`; the BFM Archipelago world - is `github.com/AegeusEvander/Brave-Fencer-Musashi-AP-World`; upstream decomp-permuter's `Scorer.__init__` gained - `ign_branch_targets, objdump_command` after our pin and its MIPS symbol wildcard is `"." in field` (`scorer.py:66-67`). +### 2. Facts measured that every task depends on (do not re-derive; verify if in doubt, R14) +- **History / purge set / citations / identities / build facts / external facts:** unchanged from the S86 block — see the + Log entries and the Approved plan (C1–C7 carry the numbers: main 4,011 commits, `--all` 4,282, 711 resolving hex + citations, the purge paths and the two ghidra-ext zip sha256s `983e2add…` / `dc57cf1a…`, `brave.exe` `fdc9f486…`, the + Gmail identities ×3,816/×162, the empty commit `commit:2782`, the purge-only commit `commit:1712`). +- **Ghidra (B5, PROVEN S87):** the RE work is text now — `config/ghidra/{SLUS_007.26,resident,ov_SC01_077,ov_SC06_018, + sep8_SLUS_007.26,aug31_USA_DEMO.EXE}.jsonl` (145 rows; only main carries hand-authored rows: 38) + `ROSTER.md` + (`tools/ghidra_roster.py`, `--check` in tools-health). `tools/ghidra_rebuild.sh --proof` rebuilds from disc + + symbol files + that file and `cmp`s (all six PASS, ≈65 s raw / ≈200 s EXE; needs `~/ghidra_12.1_PUBLIC`, the psyq400.gdt + in the psx_ldr extension, the built ELF, the extracted payload — protos need `extracted/proto/*` from + `tools/bfm_extract/extract_proto_exe.py`). The delta filter's three drift classes are documented in SETUP (P33 B5). The + live `ghidra/` project is untouched and still on disk (leaves git at B9/C3; the archive repo keeps its history). +- **Verification state:** `.run/P32/t4e/r22_check.log` 218/218 (P32 close); B2 re-gated 15 binaries, B3 fresh-clone 218/218, + A3/B4 `sdk-dual` OK; no fleet-wide R22 since P32 — **A5 is that run** (after B8). `make tools-health` has NOT been run + since A4 (the roster check is new in it) — run it in B8 or A5 and read it. +- **Environment:** `gh` not authenticated; `git filter-repo` NOT installed; disk ≈ 13 GB free of 75 (`.run/` 38 GB — + `build/ghidra_rebuild/` and `.run/ghidra_rebuild/` are scratch, ~1 GB, safe to delete); no `/tmp` (R12). + `.run/ghidra_export/` (139 MB, 129 live exports from S86) is the input for any future re-derivation of a candidate. ### 3. NEXT — in order -0. **Preflight:** `git status --short | grep -v ghidra/` (empty) · `git log -1 --format='%h %s'` · `df -h ~` (≈13 GB free) · - `.venv/bin/python -c 'import splat'` · `ls build/us/SLUS_007.26.map` (main is built; `make check BINARY=main` if not). -1. **B5 — RESUME HERE** (Max). (a) `tools/ghidra_mcp_stop.sh`. (b) Find the compile error: `mv tools/ghidra_scripts/ - ImportAnnotations.java /tmp`-style moves are FORBIDDEN (R12) — instead run `~/ghidra_12.1_PUBLIC/support/analyzeHeadless - build/ghidra_rebuild/proj bfm -process resident -noanalysis -scriptPath tools/ghidra_scripts -postScript - ImportAnnotations.java /dev/null 2>&1 | grep -iE 'error|cannot find|symbol' | head` (the scratch project from the last - run still exists) and read the javac messages; fix `ImportAnnotations.java` (known: the stack-local ctor — use - `new LocalVariableImpl(name, dt, off, currentProgram)` (no first-use) or `new LocalVariableImpl(name, first, dt, new - VariableStorage(currentProgram, off, size), currentProgram)` if that ctor exists — check with `javap -cp ~/ghidra_12.1_PUBLIC/ - Ghidra/Framework/SoftwareModeling/lib/SoftwareModeling.jar ghidra.program.model.listing.LocalVariableImpl`); re-run until - the bundle loads (a bundle failure names EVERY script in the dir — one bad file breaks all). (c) `tools/ghidra_rebuild.sh - resident --keep` → candidate delta `.run/ghidra_rebuild/resident.candidate.jsonl` (census printed): inspect what remains - after the baseline subtraction (expect: few rows — the DB holds no hand-authored types; check the comment/bookmark/label - residue is real RE, not analysis noise; if analysis noise remains, it means auto-analysis is NOT deterministic between the - live import (Phase 10) and today's — then filter those row kinds explicitly and record why). (d) `mkdir -p config/ghidra; - cp .run/ghidra_rebuild/resident.candidate.jsonl config/ghidra/resident.jsonl; tools/ghidra_rebuild.sh resident --proof` → - must print `PROOF PASS`. (e) Negative control (R39): copy the config, change one field name in the copy, point a proof at it - (temporarily swap the file) → must FAIL; restore. (f) The same for `SLUS_007.26` (main; import + analysis ≈ 3 min); then - `ov_SC01_077`, `ov_SC06_018`, `sep8_SLUS_007.26`, `aug31_USA_DEMO.EXE` (the protos need `extracted/proto/*`, present). - (g) `config/ghidra/ROSTER.md` (generated by a tiny script or by hand: program, kind, payload, vram, rows kept, proof - date) — the roster = the 6 formerly-tracked programs; the other 123 regenerate on demand via `prefetch_fleet.py`. - (h) `.claude/settings.json` hooks → `"command": "bash \"$CLAUDE_PROJECT_DIR\"/tools/ghidra_mcp_start.sh"` (and stop); - `tools/ghidra_mcp_start.sh` exits 0 silently when `$GHIDRA/support/analyzeHeadless` is absent (a contributor's session - must not try to launch Ghidra). (i) `docs/SETUP.md` P33 B5 section (R21): every new tool + the OSGi-bundle gotcha + the - '.'-path gotcha + the delta model; `docs/second-oracle.md`/§2.8 pointers as needed. (j) Log entry, tick B5, refresh this - block, commit `feat(phase-33): B5 …`. Then **B6**. - Facts gathered (S86): the existing scripts and their conventions are in `tools/ghidra_scripts/` - (`ExportSymbols.java` symbols-only, hardcoded output + blind overwrite — the R15 violation to fix; `ApplySymbols.java` the - proven text→DB mirror with `BFMAPPLY` counts; `DumpFunctionSignatures.java` has the `jsonEsc` helper + JSONL style to reuse; - `DefineFunctions.java` reads `~/bfm-decomp/.run/_funcs.txt` (hardcoded `user.home`); `ImportPsyqGdt.java` hardcodes - `/home/musashi/ghidra_12.1_PUBLIC/.../psyq400.gdt`); wrappers `tools/ghidra_import.sh ` (PSX loader auto-detect + - analysis + gdt + info; `-overwrite`), `tools/ghidra_import_raw.sh ` (BinaryLoader; stages the blob to - `.run/`), `tools/ghidra_apply_symbols.sh [PROG] [files…]`, `tools/ghidra_mcp_{start,stop,verify}.sh` — ALL hardcode - `PROJ_DIR="$HOME/bfm-decomp/ghidra"` and `SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts"`; `tools/prefetch_fleet.py` - (`vram_of(ov)` from the yaml, `import_overlay`, `run_define`, `mcp_stop_if_running`). The project has **129 programs** - (`ghidra/bfm.rep/idata/*/*.prp`, `.prp` XML with `STATE NAME=… VALUE=…`), `.run/_funcs.txt` exists for 125. The MCP - server is SERVING on :8080 (two analyzeHeadless pids) — stop it first. Headless `-process` without a name processes every - program in the project folder (one `-readOnly` run can export all 129). Design per the plan: `ExportAnnotations.java - [symbols…]`, `ImportAnnotations.java `, `tools/ghidra_rebuild.sh [--proof|--into-live]` - (scratch project under `.run/ghidra_rebuild/`), baseline-before-import to filter analysis noise, negative control, roster - `config/ghidra/ROSTER.md`, proof REQUIRED on `SLUS_007.26` + `resident`; the R15 fix + the path hardcodes + the - `.claude/settings.json` hooks. -2. Then B6 → B7 → B8 → A5 → B9/C3 … per the task list. Next P6 rules check after B8 (12 tasks done). After every task: tick the box, - add a Log line, refresh this checkpoint block (the 🛑 block is the ONLY in-phase context the next session inherits), - commit. Next P6 rules check after B4 (8 tasks done). `extracted/proto/` (sep8 + aug31 EXEs) is regenerated from the - prototype discs by `tools/bfm_extract/extract_proto_exe.py` (docstring examples) — B5 needs both files. +0. **Preflight:** `git status --short | grep -v ghidra/` (empty) · `git log -1 --format='%h %s'` · `df -h ~`. +1. **B6 — dumps** (Low; wording xHigh): `sha1sum dumps/ram_*.bin > dumps/CHECKSUMS.sha1` (expect 28 rows; check + `ls dumps/*.bin | wc -l` first, R41) — committed BEFORE the `.bin` leave git (C3); rewrite `dumps/INDEX.md` (local-only + from P33; the hashes; the re-capture recipe `tools/ram_probe.py snapshot ` → `.run/ram/`, with the honest caveat + that a re-capture is a new state snapshot, never byte-identical to the original); `docs/memory-map.md` Source-index row + for the corpus. Verify: `git ls-files dumps` = INDEX + CHECKSUMS (+ the .bin until C3); `sha1sum --check` 28/28. + Log, tick, refresh this block, commit `docs(phase-33): B6 …`. +2. **B7 — No-ROM CI** (xHigh): per the plan's B7 paragraph (`.github/workflows/no-rom.yml`, `tools/audit_public.py`, + `tools/compile_only.py` with a DERIVED skip list + coverage line; time ONE TU first, R37). Then **B8** (SETUP rows + + `docs/verification.md`), then the **P6 rules check** (12 tasks), then **A5** (the recorded run), then **B9/C3**. -### 4. Files this session touched -A1: `phase-ends/DIGEST.md`, `phase-ends/CURRENT_PHASE.md`. A2: `tools/main_seed_ends.py` (new), `Makefile` (`sig-main` -rewritten; `tools-health` + `report` wiring), `tools/progress.py` (sig selection + SystemExit + `main_oracle_line` + the MAIN -header line), `tools/backlog.py` (`linked_closed`), `tools/dup_report.py` (main sig path), `docs/SETUP.md` (§6.8 + P33 A2 -section), regenerated `docs/progress.fleet.md`, `docs/duplicates.md`, `docs/duplicates.cross.md`, `docs/backlog.md`, -`.run/backlog.jsonl`. A3: `Makefile` (`NO_SDK`, `sdk-dual`, tools-health wiring, `.PHONY`), `docs/SETUP.md` (P33 A3 section). -A4: `tools/family_hseq.py` (`binaries` + `open_instances` in the json; `load()` returns a pair), `tools/audit_binaries.py` -(CHECK 4 reads the scanned set), `docs/family-hseq.md` (regenerated, 0 families), `docs/SETUP.md` (P33 A4 section). -B1–B4: see their Log entries. B5 (WIP): `tools/ghidra_scripts/{ExportAnnotations,ImportAnnotations}.java` (new), -`tools/ghidra_export_annotations.sh`, `tools/ghidra_annotations_delta.py`, `tools/ghidra_rebuild.sh` (new), -`tools/ghidra_scripts/{DefineFunctions,ImportPsyqGdt,ExportSymbols}.java`, the six `tools/ghidra_*.sh`, `Makefile` -(`print-%`, `GHIDRA_PROJ`). Scratch: `.run/ghidra_export/` (129 exports), `.run/ghidra_rebuild/`, `build/ghidra_rebuild/`. -Environment at the pause: the MCP server is STOPPED (saved cleanly, "Save succeeded"); disk ≈ 13 GB free; the SDK object dirs -are rebuilt and the tree's main build is in the WITH-SDK state; `extracted/proto/` regenerated. -Scratch: `.run/public_audit/` (the history inventory), `.run/P33/a2_report.log`, `.run/P33/a3_sdk_dual.log`, -`.run/P33/verify/main_{with_sdk,no_sdk}.map` (to be allowlisted by A5). +### 4. Files S87 touched +`tools/ghidra_scripts/ImportAnnotations.java` (3 compile fixes + the `/undefined` resolver), `tools/ghidra_rebuild.sh` +(`.proof` markers; dies unless `failed=0`), `tools/ghidra_annotations_delta.py` (the three drift classes), new +`tools/ghidra_roster.py`, `tools/ghidra_mcp_start.sh` (silent no-op guard), `.claude/settings.json` (relative hooks), +`Makefile` (roster check in tools-health), `docs/SETUP.md` (P33 B5 section, 5 inventory rows, §2.8), `config/ghidra/*` +(new, 7 files), this file. Scratch: `.run/ghidra_rebuild/` (baselines, candidates, `proof_*.log`, `.proof`, the two chain +scripts `b5_controls.sh`/`b5_proofs.sh` + logs), `build/ghidra_rebuild/` (the scratch project, wiped by `make clean`). The plan file: `~/.claude/plans/max-effort-set-plan-twinkling-moonbeam.md` (copied below). --- diff --git a/tools/ghidra_annotations_delta.py b/tools/ghidra_annotations_delta.py index a9f9a489c..34af6c0f1 100644 --- a/tools/ghidra_annotations_delta.py +++ b/tools/ghidra_annotations_delta.py @@ -6,7 +6,8 @@ `live.jsonl` is ExportAnnotations.java's dump of the program as it is; `baseline.jsonl` is the same dump of a FRESH rebuild of that program (import + auto-analysis + DefineFunctions + ApplySymbols, no annotations imported). Every row auto-analysis produces on its own appears in both and subtracts -itself out; what remains — types, retyped/renamed signatures and locals, typed data, comments, +itself out — minus three counted analysis-drift classes (Error/Analysis bookmarks; auto-named DEFAULT functions the +rebuild did not create; auto-named rows that lag the curated symbol file, R15 — see the rules below); what remains — types, retyped/renamed signatures and locals, typed data, comments, bookmarks, equates, labels that no symbol file carries — is the RE work worth committing: config/ghidra/.jsonl. The container rows (`program`, `block`, `archive`) are always kept from `live` so an image-base or memory-map drift shows as a diff rather than a silent skip. @@ -22,6 +23,29 @@ import json import sys KEEP_ALWAYS = ("program", "block", "archive") +# Analysis-origin rows that differ between the live import (Phase 1/10) and today's rebuild — measured on +# SLUS_007.26 (S87): 29 FUN_* functions inside the LINKED PsyQ regions that the live analysis created and the +# rebuild's did not (DEFAULT signature, no comment, auto name), and one "Unable to resolve constructor" Error +# bookmark. They carry no RE; a proof that kept them could never pass on a fresh rebuild. +ANALYSIS_BOOKMARKS = ("Error", "Analysis") +AUTO_NAME = ("FUN_", "func_", "thunk_FUN_") + + +def is_symbol_file_lag(o, b): + """The live DB was never re-mirrored from the symbol file: it differs from the rebuilt (ApplySymbols) row ONLY + by carrying the auto name. Importing it would overwrite a curated name with FUN_* — the symbol file wins (R15). + Measured on the two prototypes (S87): sep8 10 rows, aug31 5 rows, nothing else.""" + if not str(o.get("name", "")).startswith(AUTO_NAME): + return False + return all(o.get(k) == b.get(k) for k in o if k != "name") + + +def is_function_set_drift(o): + if o.get("sigsrc") != "DEFAULT" or o.get("comment") or o.get("custom"): + return False + if not str(o.get("name", "")).startswith(AUTO_NAME): + return False + return all(p.get("src") == "DEFAULT" for p in o.get("params", []) + o.get("locals", [])) def rows(path): @@ -39,14 +63,32 @@ def main(argv): if len(argv) != 4: sys.exit(__doc__) live, base, out = rows(argv[1]), set(rows(argv[2])), argv[3] - kept = [ln for ln in live if kind(ln) in KEEP_ALWAYS or ln not in base] + base_func_rows = {json.loads(ln)["addr"]: json.loads(ln) for ln in base if kind(ln) == "func"} + base_funcs = set(base_func_rows) + dropped = collections.Counter() + kept = [] + for ln in live: + k = kind(ln) + if k in KEEP_ALWAYS or ln not in base: + o = json.loads(ln) + if k == "bookmark" and o.get("type") in ANALYSIS_BOOKMARKS: + dropped["bookmark:analysis"] += 1 # Ghidra's own Error/Analysis bookmarks — never hand-authored + continue + if k == "func" and o["addr"] not in base_funcs and is_function_set_drift(o): + dropped["func:set-drift"] += 1 # a boundary the live analysis found and the rebuild did not + continue + if k == "func" and o["addr"] in base_funcs and is_symbol_file_lag(o, base_func_rows[o["addr"]]): + dropped["func:name-lag"] += 1 # the DB still says FUN_*; the curated symbol file names it (R15) + continue + kept.append(ln) with open(out, "w", encoding="utf-8", newline="\n") as f: for ln in kept: f.write(ln + "\n") c = collections.Counter(kind(ln) for ln in kept) hand = sum(v for k, v in c.items() if k not in KEEP_ALWAYS) - print(f"delta: {len(live)} live - {len(base)} baseline -> {len(kept)} rows ({hand} hand-authored) -> {out}", - file=sys.stderr) + drop = ", ".join(f"{k} {v}" for k, v in sorted(dropped.items())) or "none" + print(f"delta: {len(live)} live - {len(base)} baseline -> {len(kept)} rows ({hand} hand-authored; " + f"analysis drift dropped: {drop}) -> {out}", file=sys.stderr) if census: for k in sorted(c): print(f" {k:9} {c[k]}") diff --git a/tools/ghidra_mcp_start.sh b/tools/ghidra_mcp_start.sh index 7a73f1ec3..92bf96d09 100644 --- a/tools/ghidra_mcp_start.sh +++ b/tools/ghidra_mcp_start.sh @@ -31,6 +31,12 @@ LOG="$RUNDIR/ghidra-mcp.log" SCRIPTS="$REPO/tools/ghidra_scripts" STOPREQ="${BFM_MCP_STOPREQ:-$RUNDIR/mcp-stop.req}" +# 0) Not this machine's job? A contributor's clone has no Ghidra install and no ghidra/ project +# (the DB left git at P33 B5); the SessionStart hook must then be a silent, successful no-op. +if [ ! -x "$GHIDRA/support/analyzeHeadless" ] || [ ! -d "$PROJ_DIR/$PROJ.rep" ]; then + exit 0 +fi + # 1) Already serving? no-op (idempotent across sessions). if ss -tln 2>/dev/null | grep -qE ":$PORT([^0-9]|$)"; then echo "ghidra-mcp: already serving on :$PORT — no-op" diff --git a/tools/ghidra_rebuild.sh b/tools/ghidra_rebuild.sh index 152748a2c..79f0b87fd 100644 --- a/tools/ghidra_rebuild.sh +++ b/tools/ghidra_rebuild.sh @@ -109,17 +109,23 @@ if [ ! -f "$CONF" ]; then fi say "5. import $CONF" -run_headless -process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript ImportAnnotations.java "$CONF" || die "ImportAnnotations failed" +run_headless -process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript ImportAnnotations.java "$CONF" | tee "$SCR/$PROG.import.log" || die "ImportAnnotations failed" +# R49: a per-row failure inside a rc-0 run is still a failure — a proof over a partial import proves nothing +# (S87: main's 13 func rows failed on "/undefined" while the plate comments made the delta match anyway). +grep -q 'BFMANN .*failed=0 ' "$SCR/$PROG.import.log" || die "ImportAnnotations reported failures (or no BFMANN line) — see $SCR/$PROG.import.log" say "6. export after import + delta" run_headless -process "$PROG" -noanalysis -readOnly -scriptPath "$SCRIPTS" -postScript ExportAnnotations.java "$SCR/$PROG.after.jsonl" "${SYMS[@]}" || die "export failed" .venv/bin/python tools/ghidra_annotations_delta.py "$SCR/$PROG.after.jsonl" "$SCR/$PROG.baseline.jsonl" "$SCR/$PROG.delta.jsonl" --census if [ "$PROOF" = 1 ]; then if cmp -s "$SCR/$PROG.delta.jsonl" "$CONF"; then say "PROOF PASS — the rebuilt program's hand-authored delta == $CONF ($(wc -l < "$CONF") rows)" + # the marker config/ghidra/ROSTER.md (tools/ghidra_roster.py) reports; scratch, never committed + printf 'PASS %s config-sha1 %s\n' "$(date -u +%Y-%m-%dT%H:%MZ)" "$(sha1sum < "$CONF" | cut -c1-12)" > "$SCR/$PROG.proof" [ "$KEEP" = 1 ] || rm -rf "$PROJ_DIR" exit 0 fi say "PROOF FAIL — delta differs from $CONF (diff below, first 40 lines; scratch kept in $SCR)" + printf 'FAIL %s config-sha1 %s\n' "$(date -u +%Y-%m-%dT%H:%MZ)" "$(sha1sum < "$CONF" | cut -c1-12)" > "$SCR/$PROG.proof" diff "$CONF" "$SCR/$PROG.delta.jsonl" | head -40 exit 1 fi diff --git a/tools/ghidra_roster.py b/tools/ghidra_roster.py new file mode 100644 index 000000000..533bc8b4c --- /dev/null +++ b/tools/ghidra_roster.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""ghidra_roster.py — config/ghidra/ROSTER.md from the committed Ghidra annotation files (P33 B5). + + tools/ghidra_roster.py # writes config/ghidra/ROSTER.md + tools/ghidra_roster.py --check # exit 1 if ROSTER.md is stale (for tools-health) + +One row per config/ghidra/.jsonl: the program's kind (PS-X EXE via the PSX loader, or a raw +blob at its vram), its payload and vram as the build registers them (`make -s print-EXE / print-VRAM_BASE` +— the same source tools/ghidra_rebuild.sh imports from, R33), the memory blocks the file pins, the +hand-authored row census (everything that is not a `program`/`block`/`archive` container row), and the +last `--proof` verdict recorded by tools/ghidra_rebuild.sh in .run/ghidra_rebuild/.proof +("unproven here" when that scratch file is absent — a fresh clone has not run the proof yet). +The 123 other programs of the live project carry no hand-authored rows and are regenerated on demand +(tools/prefetch_fleet.py); they are not in the roster on purpose. +""" +import collections +import datetime as dt +import json +import pathlib +import subprocess +import sys + +REPO = pathlib.Path(__file__).resolve().parent.parent +CONF = REPO / "config" / "ghidra" +SCR = REPO / ".run" / "ghidra_rebuild" +CONTAINER = {"program", "block", "archive"} +# The PS-X EXE programs (the PSX loader, no vram argument) — mirrors the case table in ghidra_rebuild.sh. +EXES = { + "SLUS_007.26": ("main", "extracted/retail/SLUS_007.26"), + "sep8_SLUS_007.26": ("proto-sep8", "extracted/proto/sep8_SLUS_007.26 (prototype disc)"), + "aug31_USA_DEMO.EXE": ("proto-demo", "extracted/proto/aug31_USA_DEMO.EXE (prototype disc)"), +} + + +def make_print(var, alias): + r = subprocess.run(["make", "-s", f"print-{var}", f"BINARY={alias}"], cwd=REPO, capture_output=True, text=True) + return r.stdout.strip() if r.returncode == 0 else "?" + + +def row_for(path): + prog = path.stem + kinds = collections.Counter() + blocks = [] + image_base = fmt = "?" + for ln in path.read_text(encoding="utf-8").splitlines(): + if not ln.strip(): + continue + o = json.loads(ln) + kinds[o["k"]] += 1 + if o["k"] == "program": + image_base, fmt = o.get("image_base", "?"), o.get("format", "?") + elif o["k"] == "block": + blocks.append(o) + if prog in EXES: + kind, payload = "PS-X EXE", EXES[prog][1] + vram = "header" + else: + kind, payload, vram = "raw blob", make_print("EXE", prog), make_print("VRAM_BASE", prog) + hand = {k: v for k, v in kinds.items() if k not in CONTAINER} + census = ", ".join(f"{k} {v}" for k, v in sorted(hand.items())) or "none (container rows only)" + proof = SCR / f"{prog}.proof" + verdict = proof.read_text(encoding="utf-8").strip() if proof.exists() else "unproven here" + # the loaded (initialized) blocks by name+range; the loader's hardware/uninitialized blocks only as a count + loaded = [f"{b['name']} {b['start']}–{b['end']}" for b in blocks if b.get("init")] + other = len(blocks) - len(loaded) + blk = "; ".join(loaded) + (f" (+{other} uninit/hardware)" if other else "") + return f"| `{prog}` | {kind} ({fmt}) | `{payload}` | {vram} | {blk} | {sum(hand.values())} — {census} | {verdict} |" + + +def render(): + files = sorted(CONF.glob("*.jsonl")) + lines = [ + "# config/ghidra/ROSTER.md — the Ghidra programs whose hand-authored RE work is tracked as text", + "", + "> Generated by `tools/ghidra_roster.py` — never edit by hand. One row per `config/ghidra/.jsonl`", + "> (the delta format of `tools/ghidra_scripts/ExportAnnotations.java`). Rebuild any row's program from the", + "> disc + this file with `tools/ghidra_rebuild.sh `; prove it with `--proof` (P33 B5). The binary", + "> Ghidra project embeds the game's bytes and left git at P33; these files are what R20 backs up instead.", + "", + "| program | kind | payload | vram | blocks | hand-authored rows | last proof (this machine) |", + "|---|---|---|---|---|---|---|", + ] + lines += [row_for(p) for p in files] + lines += [ + "", + f"{len(files)} programs tracked. The live project's other programs (the location overlays and `md_*` modules,", + "129 on disk at P33) hold no hand-authored rows and are regenerated on demand by `tools/prefetch_fleet.py`.", + "Proof verdicts come from `.run/ghidra_rebuild/.proof` (scratch; absent on a fresh clone).", + "", + f"_Generated {dt.date.today().isoformat()}._", + "", + ] + return "\n".join(lines) + + +def main(argv): + out = CONF / "ROSTER.md" + text = render() + if "--check" in argv: + cur = out.read_text(encoding="utf-8") if out.exists() else "" + # the proof column and the date are per-machine; --check compares everything else (a fresh clone + # has no .run/ghidra_rebuild/*.proof and must not read as stale) + def strip(t): + out = [] + for l in t.splitlines(): + if l.startswith("_Generated"): + continue + if l.startswith("| `"): + l = l.rsplit("|", 2)[0] + out.append(l) + return "\n".join(out) + if strip(cur) != strip(text): + print(f"ghidra_roster: {out} is STALE — run tools/ghidra_roster.py", file=sys.stderr) + return 1 + print(f"ghidra_roster: OK ({len(list(CONF.glob('*.jsonl')))} programs)") + return 0 + out.write_text(text, encoding="utf-8") + print(f"ghidra_roster: wrote {out} ({len(list(CONF.glob('*.jsonl')))} programs)") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tools/ghidra_scripts/ImportAnnotations.java b/tools/ghidra_scripts/ImportAnnotations.java index b0bd48dd1..d1a75346b 100644 --- a/tools/ghidra_scripts/ImportAnnotations.java +++ b/tools/ghidra_scripts/ImportAnnotations.java @@ -104,7 +104,9 @@ public class ImportAnnotations extends GhidraScript { } String suffix = ""; if (cut >= 0) { base = spec.substring(0, cut).trim(); suffix = spec.substring(cut); } - DataType dt = dtm.getDataType(base); + // "/undefined" is Ghidra's DefaultDataType (DataType.DEFAULT) — the default return/param type of every + // analysis-created function; it lives in NEITHER data type manager (S87: 13 of main's 13 func rows failed on it). + DataType dt = (base.equals("/undefined") || base.equals("undefined")) ? DataType.DEFAULT : dtm.getDataType(base); if (dt == null) dt = BuiltInDataTypeManager.getDataTypeManager().getDataType(base); if (dt == null && !base.startsWith("/")) dt = dtm.getDataType("/" + base); if (dt == null) throw new IllegalArgumentException("unresolvable type " + spec); @@ -202,7 +204,7 @@ public class ImportAnnotations extends GhidraScript { try { if (spec.startsWith("Stack[")) { int j = spec.indexOf(']'); - int off = (int) Long.decode(spec.substring(6, j)); + int off = Long.decode(spec.substring(6, j)).intValue(); int size = Integer.parseInt(spec.substring(spec.lastIndexOf(':') + 1)); return new VariableStorage(currentProgram, off, size); } @@ -277,8 +279,11 @@ public class ImportAnnotations extends GhidraScript { } Variable nv; if (storage.startsWith("Stack[")) { - int off = (int) Long.decode(storage.substring(6, storage.indexOf(']'))); - nv = new LocalVariableImpl(s(l, "name"), first, lt, off, currentProgram); + // stack local: "Stack[-0x10]:4" -> VariableStorage(program, offset, size); the (String,int,DataType,int,Program) + // ctor does not exist in Ghidra 12.1 (the S86 OSGi-bundle blocker) + VariableStorage vs = storageOf(storage); + if (vs == null) { skippedStorage++; continue; } + nv = new LocalVariableImpl(s(l, "name"), first, lt, vs, currentProgram); } else { String rn = storage.contains(":") ? storage.substring(0, storage.indexOf(':')) : storage; Register r = currentProgram.getRegister(rn);