From cabab00c0e94b9db0bacba128bfdec30a001f539 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:31:34 -0600 Subject: [PATCH] =?UTF-8?q?wip(phase-33):=20B5=20Ghidra=20regenerability?= =?UTF-8?q?=20=E2=80=94=20ExportAnnotations.java=20+=20ghidra=5Fexport=5Fa?= =?UTF-8?q?nnotations.sh=20PROVEN=20(all=20129=20programs=20exported=20to?= =?UTF-8?q?=20byte-stable=20JSONL=20in=2018.5=20s),=20ghidra=5Fannotations?= =?UTF-8?q?=5Fdelta.py=20(live=20=E2=88=92=20baseline),=20ImportAnnotation?= =?UTF-8?q?s.java=20+=20ghidra=5Frebuild.sh=20WRITTEN=20but=20unproven=20(?= =?UTF-8?q?the=20resident=20rebuild=20hit=20an=20OSGi=20script-bundle=20lo?= =?UTF-8?q?ad=20error=20=E2=80=94=20a=20compile=20error=20in=20the=20new?= =?UTF-8?q?=20importer=20breaks=20every=20script=20in=20the=20dir;=20the?= =?UTF-8?q?=20known=20bad=20LocalVariableImpl=20ctor=20is=20named=20in=20C?= =?UTF-8?q?URRENT=5FPHASE);=20Makefile=20print-%=20+=20GHIDRA=5FPROJ=20rep?= =?UTF-8?q?o-relative;=20the=20six=20ghidra=5F*.sh=20repo-relative=20(BFM?= =?UTF-8?q?=5FGHIDRA=5FPROJ);=20DefineFunctions=20arg=20path;=20ImportPsyq?= =?UTF-8?q?Gdt=20install-dir=20default;=20ExportSymbols=20R15=20fix;=20scr?= =?UTF-8?q?atch=20project=20under=20build/=20(Ghidra=20refuses=20'.'-prefi?= =?UTF-8?q?xed=20path=20components);=20SETUP=20P33=20B5=20section;=20check?= =?UTF-8?q?point=20refreshed=20for=20the=20next=20session=20(S86=20paused?= =?UTF-8?q?=20at=2087%=20context)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 8 +- docs/SETUP.md | 20 ++ phase-ends/CURRENT_PHASE.md | 71 +++- tools/ghidra_annotations_delta.py | 57 +++ tools/ghidra_apply_symbols.sh | 9 +- tools/ghidra_export_annotations.sh | 52 +++ tools/ghidra_import.sh | 5 +- tools/ghidra_import_raw.sh | 7 +- tools/ghidra_mcp_start.sh | 7 +- tools/ghidra_mcp_stop.sh | 3 +- tools/ghidra_mcp_verify.sh | 7 +- tools/ghidra_rebuild.sh | 127 +++++++ tools/ghidra_scripts/DefineFunctions.java | 8 +- tools/ghidra_scripts/ExportAnnotations.java | 295 ++++++++++++++++ tools/ghidra_scripts/ExportSymbols.java | 17 +- tools/ghidra_scripts/ImportAnnotations.java | 370 ++++++++++++++++++++ tools/ghidra_scripts/ImportPsyqGdt.java | 4 +- 17 files changed, 1036 insertions(+), 31 deletions(-) create mode 100644 tools/ghidra_annotations_delta.py create mode 100644 tools/ghidra_export_annotations.sh create mode 100644 tools/ghidra_rebuild.sh create mode 100644 tools/ghidra_scripts/ExportAnnotations.java create mode 100644 tools/ghidra_scripts/ImportAnnotations.java diff --git a/Makefile b/Makefile index e14711fd4..c163cd7ae 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ help: # Phase 7 reports: deterministic, committable docs/ digests. progress/difficulty/dup_report # are Ghidra-free; sig-refresh regenerates dup_report's input from the saved Ghidra DB. GHIDRA := $(or $(GHIDRA_INSTALL_DIR),$(HOME)/ghidra_12.1_PUBLIC) -GHIDRA_PROJ := $(HOME)/bfm-decomp/ghidra +GHIDRA_PROJ := $(or $(BFM_GHIDRA_PROJ),$(CURDIR)/ghidra) # P33 B5: repo-relative (override: BFM_GHIDRA_PROJ) # The corpus oracle (Phase 26-A, R32/R33). A SECOND, INDEPENDENT oracle: it cross-checks splat's # function boundaries against sig_image's, which are derived from the ORIGINAL bytes without splat. @@ -519,6 +519,12 @@ sig-modules: echo "sig-modules: signed $$n modules (of $(words $(MODULE_BINARIES)) onboarded)" if [ "$$bad" -ne 0 ]; then echo "[FAIL] sig-modules: $$bad module(s) failed to sign"; exit 1; fi +# print- (P33 B5): echo one Makefile variable so tools derive per-binary facts from the ONE registry +# (R33) instead of re-parsing overlays.mk/modules.mk — e.g. `make -s print-EXE BINARY=ov_SC01_077`, +# `make -s print-VRAM_BASE BINARY=resident`, `make -s print-BINARIES`. +print-%: + @echo '$($*)' + # bootstrap (P33 B3): the fresh-clone setup — apt presence (printed, never run), the venv from # requirements-python.txt, the submodules, the two cc1 tarballs verified + extracted, then check-env. bootstrap: diff --git a/docs/SETUP.md b/docs/SETUP.md index 66de46282..64ec4e9ef 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -1069,6 +1069,26 @@ fills fast). Nothing is leaking — but the host does not get the memory back on * `gate_main` **REFUSES a draft containing its own `INCLUDE_ASM`** (substituting it restores the stub, so the build passes for free and the function counts as banked), and counts banks from the SOURCE. +### P33 B5 (S86, 2026-09-06) — Ghidra regenerability (IN PROGRESS at the S86 pause; see `phase-ends/CURRENT_PHASE.md`) +- **Model:** the Ghidra database embeds the program bytes, so it leaves git; what it holds beyond the splat symbol files is + exported to text and the project is rebuilt from the disc + that text. `tools/ghidra_scripts/ExportAnnotations.java` + (read-only; byte-stable JSONL: container facts, LOCAL types, every function signature, defined data, the 5 comment kinds, + bookmarks, equates, labels the symbol files lack) via `tools/ghidra_export_annotations.sh [PROG…]` (no PROG = every + program; **129 programs in 18.5 s** → `.run/ghidra_export/`). The committed file `config/ghidra/.jsonl` is the + DELTA (`tools/ghidra_annotations_delta.py live baseline out`) between the live export and a FRESH rebuild's export, so + analysis-origin rows subtract themselves out. `tools/ghidra_rebuild.sh [--proof] [--keep]`: import (PSX loader for + the EXEs, BinaryLoader at `make -s print-VRAM_BASE BINARY=` for the rest) → `DefineFunctions.java` from the built + ELF → `ApplySymbols.java` → baseline export → `ImportAnnotations.java config/ghidra/.jsonl` → export → delta → `cmp`. + Scratch project under `build/ghidra_rebuild/proj` (**Ghidra refuses a project path with a component starting with '.'**, + so never `.run/`). `make print- BINARY=` echoes any per-binary Makefile variable (R33). +- **Gotcha (S86):** Ghidra compiles a `-scriptPath` directory as ONE OSGi bundle — a single script that fails to compile + makes EVERY script in the directory fail with `Failed to get OSGi bundle containing script`. Diagnose by invoking the + suspect script directly and reading the javac lines. +- Also in this change: `ExportSymbols.java` R15 fix (output path arg, refuses to overwrite, refuses `config/`); + `DefineFunctions.java` list path = arg 1; `ImportPsyqGdt.java` default gdt from the Ghidra install dir; the six + `tools/ghidra_*.sh` are repo-relative (`BFM_GHIDRA_PROJ` overrides the project dir; `ghidra_mcp_verify.sh + [PROG]`); Makefile `GHIDRA_PROJ := $(or $(BFM_GHIDRA_PROJ),$(CURDIR)/ghidra)`. + ### P33 B4 (S86, 2026-09-06) — `tools/fetch_psyq.sh`: the OPTIONAL PsyQ SDK objects, user-supplied and verified - **What it is for.** Byte-identity never needs Sony's libraries (without them main links its INCLUDE_ASM tiles — the fresh-clone proof in B3 built all 218 that way). `tools/fetch_psyq.sh [--disc ] [--from ] diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index 6a976b990..baec515e5 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -170,24 +170,45 @@ Mid-phase rules check after every 4 completed tasks (P6). Commit banked artifact with one corrupted LIB → `sha256 MISMATCH for tools/psyq/lib40/LIBTAP.LIB`, rc 1, nothing built. **The run:** with `tools/psyq/lib40/` moved aside, `fetch_psyq.sh --disc ""` extracted the 20 LIBs from the disc (`diff -rq` against the tracked copies: identical), verified them, rebuilt `.run/obj40/*`, `.run/obj42/*`, - `tools/psyq/lib40_elf/`, and `sdk-dual: OK` — **36.9 s wall**, EXIT=0 (`.run/P33/b4_fetch.log`). SETUP: P33 B4 (R21). + `tools/psyq/lib40_elf/`, and `sdk-dual: OK` — **36.9 s wall**, EXIT=0 (`.run/P33/b4_fetch.log`). SETUP: P33 B4 (R21). Commit + `commit:4019`. +- **2026-09-06 (S86) — B5 IN PROGRESS (checkpointed at 87% context; WIP commit).** Done and proven: MCP stopped cleanly; + `tools/ghidra_scripts/ExportAnnotations.java` + `tools/ghidra_export_annotations.sh` — byte-stable JSONL export of a + program's container facts, LOCAL types, every function signature (params/locals/storage/sources), defined data, the 5 + comment kinds, bookmarks, equates, and labels the symbol files do not carry; **all 129 programs exported in 18.5 s** → + `.run/ghidra_export/.jsonl` (139 MB; resident 2,573 rows in 5.7 s). `tools/ghidra_annotations_delta.py` (live − + baseline; container rows always kept; `--census`). Makefile `print-%` helper + `GHIDRA_PROJ` repo-relative; the six + `tools/ghidra_*.sh` wrappers repo-relative (`BFM_GHIDRA_PROJ` override; `ghidra_mcp_verify.sh` takes the program as arg 3 + / `BFM_GHIDRA_PROG`); `DefineFunctions.java` takes the list path as arg 1; `ImportPsyqGdt.java` finds the gdt under + `Application.getInstallationDirectory()`; `ExportSymbols.java` R15 fix (output path arg, refuses to overwrite, refuses + `config/`). Written, NOT yet proven: `tools/ghidra_scripts/ImportAnnotations.java`, `tools/ghidra_rebuild.sh` (its import + step works: the scratch project must live under `build/ghidra_rebuild/proj` — Ghidra refuses a path component starting + with '.'; `resident_funcs.txt` from the ELF = 1,304 entries). **Blocker found:** in the resident rebuild the runs after the + import failed with `Failed to get OSGi bundle containing script: …/tools/ghidra_scripts/ApplySymbols.java` (same for + ExportAnnotations) — Ghidra compiles the script DIRECTORY as one bundle, so ONE file that does not compile breaks every + script in it; `ExportAnnotations` worked on 129 programs BEFORE `ImportAnnotations.java` existed ⇒ the new file almost + certainly has a compile error (one known: `new LocalVariableImpl(name, first, dt, stackOffset, program)` — no such ctor; + the stack form is `(String, DataType, int, Program)` without first-use, or `(String, int, DataType, VariableStorage/Address/ + Register, Program)`). Census of the live exports (heuristic, before baselines): `types=0` in EVERY program (no hand-authored + structs in the DB at all); the 154/402 "USER_DEFINED" variables per program are the loader's GTEMAC functions + (analysis-origin, will subtract); real user labels beyond the symbol files: aug31 382, sep8 15, SLUS_007.26 1, overlays 1–2. -## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1–B4 ✓ (8 tasks; P6 rules check done after B4); NEXT = B5 Ghidra regenerability (Max, MCP stopped) (2026-09-06, written by session bd19e14a "S86"; SUPERSEDES the earlier blocks) +## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1–B4 ✓, B5 IN PROGRESS (exporter proven; importer + rebuild proof pending on one compile error); NEXT = finish B5 (2026-09-06 ~19:35 MDT, written by session bd19e14a "S86" at Drew's 87%-context pause; SUPERSEDES the earlier blocks) ### 0. How to use this block You are a FRESH SESSION that has read `PROJECT_CONTEXT.md`, `phase-ends/DIGEST.md`, `PhaseEnd_Phase30/31/32.md` and this file, and nothing else (R64). Replay this block verbatim, state phase / done / NEXT / effort, list the rules from the digest -(R1–R73), then WAIT for Drew. **NEXT = B5** (Max; the Ghidra text export + rebuild proof — the plan's Block B5 and §3 step 1 -below; the headless MCP server holds the project lock: `tools/ghidra_mcp_stop.sh` FIRST, then every headless step; the -SessionStart hook restarts the server next session and R29 applies). If `git log -1 --format=%s` does not start with -`feat(phase-33): B4`, B4's commit did not land: `tools/fetch_psyq.sh --no-build` must print `verified …` for every piece and -exit 0 before committing (`.run/P33/b4_fetch.log` is the full run). +(R1–R73), then WAIT for Drew. **NEXT = finish B5** (Max). The WIP commit `wip(phase-33): B5 …` holds every B5 file (see +the Log's B5 entry for what is proven and what is not). The SessionStart hook will have RESTARTED the headless MCP server +(it serves `SLUS_007.26` and holds the project lock): run `tools/ghidra_mcp_stop.sh` before ANY headless step (R23), and +prompt Drew for `/mcp` only if you actually need the MCP tools (R29) — B5 needs none. §3 step 1 is the exact resume path. ### 1. Where we are **Phase 33 — 100% verification + the public flip + Gen2 exit.** Gate 1 approved 2026-09-06 (plan mode, Max). R65–R73 ratified at gate 1 (DIGEST §3). **Done: A1 (`commit:4012`), A2 (`commit:4013` — the corrected main denominator 45,150 is now the published -number; `.run/sig.main.jsonl` is the sig main is weighed by; `tools/main_seed_ends.py` is new), A3 (the `NO_SDK` knob + -`make sdk-dual`, proven 28 s: both legs `143dbb89…`).** The approved plan is VERBATIM +number; `.run/sig.main.jsonl` is the sig main is weighed by; `tools/main_seed_ends.py` is new), A3 (`commit:4014`, the +`NO_SDK` knob + `make sdk-dual`, proven 28 s: both legs `143dbb89…`), A4 (`commit:4015`), B1 (`commit:4016`), B2 (`commit:4017`), +B3 (`commit:4018`), B4 (`commit:4019`); B5 in progress (the WIP commit).** The approved plan is VERBATIM at the end of this file — read its Blocks A–G for every task's files, commands and verification; the "Execution order and why" section is the sequence; the "Honest scale" table is the budget (≈17–21 sessions; the flip precedes the writing so later items can be cut). Harness tasks: #1 A1 done, #2 A2 done, #3 A3 next … #41 G2. @@ -250,7 +271,31 @@ items can be cut). Harness tasks: #1 A1 done, #2 A2 done, #3 A3 next … #41 G2. ### 3. NEXT — in order 0. **Preflight:** `git status --short | grep -v ghidra/` (empty) · `git log -1 --format='%h %s'` · `df -h ~` (≈13 GB free) · `.venv/bin/python -c 'import splat'` · `ls build/us/SLUS_007.26.map` (main is built; `make check BINARY=main` if not). -1. **B5** (Max) — facts gathered (S86): the existing scripts and their conventions are in `tools/ghidra_scripts/` +1. **B5 — RESUME HERE** (Max). (a) `tools/ghidra_mcp_stop.sh`. (b) Find the compile error: `mv tools/ghidra_scripts/ + ImportAnnotations.java /tmp`-style moves are FORBIDDEN (R12) — instead run `~/ghidra_12.1_PUBLIC/support/analyzeHeadless + build/ghidra_rebuild/proj bfm -process resident -noanalysis -scriptPath tools/ghidra_scripts -postScript + ImportAnnotations.java /dev/null 2>&1 | grep -iE 'error|cannot find|symbol' | head` (the scratch project from the last + run still exists) and read the javac messages; fix `ImportAnnotations.java` (known: the stack-local ctor — use + `new LocalVariableImpl(name, dt, off, currentProgram)` (no first-use) or `new LocalVariableImpl(name, first, dt, new + VariableStorage(currentProgram, off, size), currentProgram)` if that ctor exists — check with `javap -cp ~/ghidra_12.1_PUBLIC/ + Ghidra/Framework/SoftwareModeling/lib/SoftwareModeling.jar ghidra.program.model.listing.LocalVariableImpl`); re-run until + the bundle loads (a bundle failure names EVERY script in the dir — one bad file breaks all). (c) `tools/ghidra_rebuild.sh + resident --keep` → candidate delta `.run/ghidra_rebuild/resident.candidate.jsonl` (census printed): inspect what remains + after the baseline subtraction (expect: few rows — the DB holds no hand-authored types; check the comment/bookmark/label + residue is real RE, not analysis noise; if analysis noise remains, it means auto-analysis is NOT deterministic between the + live import (Phase 10) and today's — then filter those row kinds explicitly and record why). (d) `mkdir -p config/ghidra; + cp .run/ghidra_rebuild/resident.candidate.jsonl config/ghidra/resident.jsonl; tools/ghidra_rebuild.sh resident --proof` → + must print `PROOF PASS`. (e) Negative control (R39): copy the config, change one field name in the copy, point a proof at it + (temporarily swap the file) → must FAIL; restore. (f) The same for `SLUS_007.26` (main; import + analysis ≈ 3 min); then + `ov_SC01_077`, `ov_SC06_018`, `sep8_SLUS_007.26`, `aug31_USA_DEMO.EXE` (the protos need `extracted/proto/*`, present). + (g) `config/ghidra/ROSTER.md` (generated by a tiny script or by hand: program, kind, payload, vram, rows kept, proof + date) — the roster = the 6 formerly-tracked programs; the other 123 regenerate on demand via `prefetch_fleet.py`. + (h) `.claude/settings.json` hooks → `"command": "bash \"$CLAUDE_PROJECT_DIR\"/tools/ghidra_mcp_start.sh"` (and stop); + `tools/ghidra_mcp_start.sh` exits 0 silently when `$GHIDRA/support/analyzeHeadless` is absent (a contributor's session + must not try to launch Ghidra). (i) `docs/SETUP.md` P33 B5 section (R21): every new tool + the OSGi-bundle gotcha + the + '.'-path gotcha + the delta model; `docs/second-oracle.md`/§2.8 pointers as needed. (j) Log entry, tick B5, refresh this + block, commit `feat(phase-33): B5 …`. Then **B6**. + Facts gathered (S86): the existing scripts and their conventions are in `tools/ghidra_scripts/` (`ExportSymbols.java` symbols-only, hardcoded output + blind overwrite — the R15 violation to fix; `ApplySymbols.java` the proven text→DB mirror with `BFMAPPLY` counts; `DumpFunctionSignatures.java` has the `jsonEsc` helper + JSONL style to reuse; `DefineFunctions.java` reads `~/bfm-decomp/.run/_funcs.txt` (hardcoded `user.home`); `ImportPsyqGdt.java` hardcodes @@ -279,6 +324,12 @@ section), regenerated `docs/progress.fleet.md`, `docs/duplicates.md`, `docs/dupl `.run/backlog.jsonl`. A3: `Makefile` (`NO_SDK`, `sdk-dual`, tools-health wiring, `.PHONY`), `docs/SETUP.md` (P33 A3 section). A4: `tools/family_hseq.py` (`binaries` + `open_instances` in the json; `load()` returns a pair), `tools/audit_binaries.py` (CHECK 4 reads the scanned set), `docs/family-hseq.md` (regenerated, 0 families), `docs/SETUP.md` (P33 A4 section). +B1–B4: see their Log entries. B5 (WIP): `tools/ghidra_scripts/{ExportAnnotations,ImportAnnotations}.java` (new), +`tools/ghidra_export_annotations.sh`, `tools/ghidra_annotations_delta.py`, `tools/ghidra_rebuild.sh` (new), +`tools/ghidra_scripts/{DefineFunctions,ImportPsyqGdt,ExportSymbols}.java`, the six `tools/ghidra_*.sh`, `Makefile` +(`print-%`, `GHIDRA_PROJ`). Scratch: `.run/ghidra_export/` (129 exports), `.run/ghidra_rebuild/`, `build/ghidra_rebuild/`. +Environment at the pause: the MCP server is STOPPED (saved cleanly, "Save succeeded"); disk ≈ 13 GB free; the SDK object dirs +are rebuilt and the tree's main build is in the WITH-SDK state; `extracted/proto/` regenerated. Scratch: `.run/public_audit/` (the history inventory), `.run/P33/a2_report.log`, `.run/P33/a3_sdk_dual.log`, `.run/P33/verify/main_{with_sdk,no_sdk}.map` (to be allowlisted by A5). The plan file: `~/.claude/plans/max-effort-set-plan-twinkling-moonbeam.md` (copied below). diff --git a/tools/ghidra_annotations_delta.py b/tools/ghidra_annotations_delta.py new file mode 100644 index 000000000..a9f9a489c --- /dev/null +++ b/tools/ghidra_annotations_delta.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""ghidra_annotations_delta.py — the HAND-AUTHORED part of a Ghidra program, as a byte-stable file (P33 B5). + + ghidra_annotations_delta.py [--census] + +`live.jsonl` is ExportAnnotations.java's dump of the program as it is; `baseline.jsonl` is the same +dump of a FRESH rebuild of that program (import + auto-analysis + DefineFunctions + ApplySymbols, no +annotations imported). Every row auto-analysis produces on its own appears in both and subtracts +itself out; what remains — types, retyped/renamed signatures and locals, typed data, comments, +bookmarks, equates, labels that no symbol file carries — is the RE work worth committing: +config/ghidra/.jsonl. The container rows (`program`, `block`, `archive`) are always kept +from `live` so an image-base or memory-map drift shows as a diff rather than a silent skip. + +The proof (tools/ghidra_rebuild.sh --proof) rebuilds, imports config/ghidra/.jsonl, exports, +takes the delta against the SAME rebuild's pre-import baseline, and `cmp`s it against the committed file. + +`--census` prints per-kind counts of the delta (the roster decision: a program whose delta holds no +hand-authored rows need not be kept regenerable — it regenerates on demand). +""" +import collections +import json +import sys + +KEEP_ALWAYS = ("program", "block", "archive") + + +def rows(path): + with open(path, encoding="utf-8") as f: + return [ln.rstrip("\n") for ln in f if ln.strip()] + + +def kind(line): + return json.loads(line)["k"] + + +def main(argv): + census = "--census" in argv + argv = [a for a in argv if a != "--census"] + if len(argv) != 4: + sys.exit(__doc__) + live, base, out = rows(argv[1]), set(rows(argv[2])), argv[3] + kept = [ln for ln in live if kind(ln) in KEEP_ALWAYS or ln not in base] + with open(out, "w", encoding="utf-8", newline="\n") as f: + for ln in kept: + f.write(ln + "\n") + c = collections.Counter(kind(ln) for ln in kept) + hand = sum(v for k, v in c.items() if k not in KEEP_ALWAYS) + print(f"delta: {len(live)} live - {len(base)} baseline -> {len(kept)} rows ({hand} hand-authored) -> {out}", + file=sys.stderr) + if census: + for k in sorted(c): + print(f" {k:9} {c[k]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tools/ghidra_apply_symbols.sh b/tools/ghidra_apply_symbols.sh index 531ddd6db..c7f656eb5 100644 --- a/tools/ghidra_apply_symbols.sh +++ b/tools/ghidra_apply_symbols.sh @@ -4,15 +4,16 @@ # the write open needs the project lock. Verify afterwards with tools/ghidra_mcp_verify.sh (R9). # tools/ghidra_apply_symbols.sh [PROG] [symbols files...] (defaults: SLUS_007.26 config/symbols.us.txt) set -uo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}" -PROJ_DIR="$HOME/bfm-decomp/ghidra"; PROJ="bfm" +PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}"; PROJ="bfm" PROG="${1:-SLUS_007.26}"; shift || true -SYMS=("$@"); [ ${#SYMS[@]} -eq 0 ] && SYMS=("$HOME/bfm-decomp/config/symbols.us.txt") -SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts" +SYMS=("$@"); [ ${#SYMS[@]} -eq 0 ] && SYMS=("$REPO/config/symbols.us.txt") +SCRIPTS="$REPO/tools/ghidra_scripts" if ss -tln 2>/dev/null | grep -qE ":8080([^0-9]|$)"; then echo "apply-symbols: ERROR — MCP server serving on :8080. Run tools/ghidra_mcp_stop.sh first."; exit 2 fi -cd "$HOME/bfm-decomp" +cd "$REPO" out=$(timeout 600 "$GHIDRA/support/analyzeHeadless" "$PROJ_DIR" "$PROJ" \ -process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript ApplySymbols.java "${SYMS[@]}" 2>&1) rc=$? diff --git a/tools/ghidra_export_annotations.sh b/tools/ghidra_export_annotations.sh new file mode 100644 index 000000000..0f5573d0c --- /dev/null +++ b/tools/ghidra_export_annotations.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# tools/ghidra_export_annotations.sh [--proj DIR] [--out DIR] [PROG ...] +# +# Read-only export of a Ghidra program's annotations (types, signatures, data, comments, bookmarks, +# equates, labels) to byte-stable JSON-Lines via tools/ghidra_scripts/ExportAnnotations.java (P33 B5). +# With no PROG every program in the project is exported (headless `-process` with no name). +# Output: /.jsonl (default .run/ghidra_export/). The splat symbol files are passed so the +# labels they already carry are skipped (ApplySymbols.java restores those on a rebuild). +# +# PRECONDITION: the MCP server is STOPPED (tools/ghidra_mcp_stop.sh) — even a read-only open needs +# the project lock free. Never writes to the project (-readOnly). +set -uo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}" +PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}"; PROJ="bfm" +OUT="$REPO/.run/ghidra_export" +SCRIPTS="$REPO/tools/ghidra_scripts" +while [ $# -gt 0 ]; do + case "$1" in + --proj) PROJ_DIR="$2"; shift 2 ;; + --out) OUT="$2"; shift 2 ;; + *) break ;; + esac +done +if ss -tln 2>/dev/null | grep -qE ":8080([^0-9]|$)"; then + echo "export-annotations: ERROR — MCP server serving on :8080. Run tools/ghidra_mcp_stop.sh first." >&2; exit 2 +fi +[ -f "$PROJ_DIR/$PROJ.gpr" ] || { echo "export-annotations: ERROR — no project at $PROJ_DIR/$PROJ.gpr" >&2; exit 2; } +mkdir -p "$OUT" +cd "$REPO" +SYMS=(config/symbols.us.txt config/symbols.us.ram.txt config/symbols.resident.txt) +for f in config/symbols.ov_*.txt config/symbols.md_*.txt config/symbols.proto-*.txt; do [ -f "$f" ] && SYMS+=("$f"); done +if [ $# -eq 0 ]; then + procs=(-process) # every program in the project folder +else + procs=() + for p in "$@"; do procs+=(-process "$p"); done +fi +rc=0 +if [ $# -eq 0 ]; then + timeout 7200 "$GHIDRA/support/analyzeHeadless" "$PROJ_DIR" "$PROJ" -process -noanalysis -readOnly \ + -scriptPath "$SCRIPTS" -postScript ExportAnnotations.java "$OUT" "${SYMS[@]}" 2>&1 \ + | grep -E 'BFMEXPORT|ERROR|Exception' ; rc=${PIPESTATUS[0]} +else + for p in "$@"; do + timeout 1800 "$GHIDRA/support/analyzeHeadless" "$PROJ_DIR" "$PROJ" -process "$p" -noanalysis -readOnly \ + -scriptPath "$SCRIPTS" -postScript ExportAnnotations.java "$OUT" "${SYMS[@]}" 2>&1 \ + | grep -E 'BFMEXPORT|ERROR|Exception' ; r=${PIPESTATUS[0]}; [ "$r" -eq 0 ] || rc=$r + done +fi +echo "export-annotations: analyzeHeadless exit=$rc -> $OUT/" +exit $rc diff --git a/tools/ghidra_import.sh b/tools/ghidra_import.sh index 46b90820f..4ee561f29 100644 --- a/tools/ghidra_import.sh +++ b/tools/ghidra_import.sh @@ -14,12 +14,13 @@ # Run tools/ghidra_mcp_stop.sh first. Used in Phase 3.5 to add the two prototypes # (sep8_SLUS_007.26, aug31_USA_DEMO.EXE) alongside retail; reusable for any PS-X EXE. set -uo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption EXE="${1:?usage: ghidra_import.sh }" GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}" -PROJ_DIR="$HOME/bfm-decomp/ghidra" +PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}" PROJ="bfm" -SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts" +SCRIPTS="$REPO/tools/ghidra_scripts" GDT="${PSYQ_GDT:-$GHIDRA/Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt}" PORT="8080" diff --git a/tools/ghidra_import_raw.sh b/tools/ghidra_import_raw.sh index f4a9683aa..6617de042 100644 --- a/tools/ghidra_import_raw.sh +++ b/tools/ghidra_import_raw.sh @@ -18,18 +18,19 @@ # PRECONDITION: the MCP server must be STOPPED (exclusive project lock) — run # tools/ghidra_mcp_stop.sh first. analyzeHeadless commits + saves on a clean exit. set -uo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption BLOB="${1:?usage: ghidra_import_raw.sh }" BASE="${2:?usage: ghidra_import_raw.sh }" NAME="${3:?usage: ghidra_import_raw.sh }" GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}" -PROJ_DIR="$HOME/bfm-decomp/ghidra" +PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}" PROJ="bfm" -SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts" +SCRIPTS="$REPO/tools/ghidra_scripts" GDT="${PSYQ_GDT:-$GHIDRA/Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt}" LANG_ID="PSX:LE:32:default" PORT="8080" -STAGE="$HOME/bfm-decomp/.run/$NAME" # import filename -> program name +STAGE="$REPO/.run/$NAME" # import filename -> program name [ -f "$BLOB" ] || { echo "import-raw: ERROR — blob not found: $BLOB" >&2; exit 2; } [ -f "$GDT" ] || { echo "import-raw: ERROR — PsyQ gdt not found: $GDT" >&2; exit 2; } diff --git a/tools/ghidra_mcp_start.sh b/tools/ghidra_mcp_start.sh index aa6ce736f..7a73f1ec3 100644 --- a/tools/ghidra_mcp_start.sh +++ b/tools/ghidra_mcp_start.sh @@ -19,15 +19,16 @@ # open transaction + saves on close). Save+stop with tools/ghidra_mcp_stop.sh; resume by # re-running this script. There is no mid-session save (GhidrAssist holds an open txn). set -uo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}" -PROJ_DIR="$HOME/bfm-decomp/ghidra" +PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}" PROJ="bfm" PROG="${1:-SLUS_007.26}" # program to serve; default retail, pass a proto name to override PORT="8080" -RUNDIR="$HOME/bfm-decomp/.run"; mkdir -p "$RUNDIR" # project-local runtime scratch (never /tmp) +RUNDIR="$REPO/.run"; mkdir -p "$RUNDIR" # project-local runtime scratch (never /tmp) LOG="$RUNDIR/ghidra-mcp.log" -SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts" +SCRIPTS="$REPO/tools/ghidra_scripts" STOPREQ="${BFM_MCP_STOPREQ:-$RUNDIR/mcp-stop.req}" # 1) Already serving? no-op (idempotent across sessions). diff --git a/tools/ghidra_mcp_stop.sh b/tools/ghidra_mcp_stop.sh index bf42b34f8..b822d51d1 100644 --- a/tools/ghidra_mcp_stop.sh +++ b/tools/ghidra_mcp_stop.sh @@ -4,9 +4,10 @@ # analyzeHeadless COMMITS the pending transaction and saves+closes the project (releasing # the .rep lock). The next session resumes from the saved state. Use at task/phase boundaries. set -uo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption PORT="8080" -RUNDIR="$HOME/bfm-decomp/.run" +RUNDIR="$REPO/.run" LOG="$RUNDIR/ghidra-mcp.log" STOPREQ="${BFM_MCP_STOPREQ:-$RUNDIR/mcp-stop.req}" diff --git a/tools/ghidra_mcp_verify.sh b/tools/ghidra_mcp_verify.sh index 43ba81ac7..9082d09a7 100644 --- a/tools/ghidra_mcp_verify.sh +++ b/tools/ghidra_mcp_verify.sh @@ -9,15 +9,16 @@ # Example (after a phase's last edit was renaming 0x80018730 to LzssDecodeSector): # tools/ghidra_mcp_stop.sh && tools/ghidra_mcp_verify.sh 0x80018730 LzssDecodeSector set -uo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption ADDR="${1:?usage: ghidra_mcp_verify.sh }" EXPECT="${2:?usage: ghidra_mcp_verify.sh }" GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}" -PROJ_DIR="$HOME/bfm-decomp/ghidra" +PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}" PROJ="bfm" -PROG="SLUS_007.26" -SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts" +PROG="${3:-${BFM_GHIDRA_PROG:-SLUS_007.26}}" # P33 B5: 3rd arg / env selects the program +SCRIPTS="$REPO/tools/ghidra_scripts" PORT="8080" if ss -tln 2>/dev/null | grep -qE ":$PORT([^0-9]|$)"; then diff --git a/tools/ghidra_rebuild.sh b/tools/ghidra_rebuild.sh new file mode 100644 index 000000000..152748a2c --- /dev/null +++ b/tools/ghidra_rebuild.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# tools/ghidra_rebuild.sh [--proof] [--keep] +# +# Rebuild ONE Ghidra program FROM TEXT + the disc, in a scratch project (.run/ghidra_rebuild/proj), and +# prove it: the committed config/ghidra/.jsonl (ExportAnnotations.java's delta format) is the +# hand-authored RE work; everything else — the bytes, auto-analysis, the function set, the names — comes +# from the extracted payload, Ghidra, the built ELF and the splat symbol files. P33 B5; makes the Ghidra +# database regenerable so it can leave git (it embeds the program bytes). +# +# 1. import PS-X EXEs (SLUS_007.26, the two prototypes) via the PSX loader; every other program is a +# raw blob at its vram (`make -s print-EXE / print-VRAM_BASE BINARY=` — the build's +# own registry, R33), + auto-analysis + ImportPsyqGdt.java (the psyq400 types). +# 2. functions DefineFunctions.java from the built ELF's text symbols (splat's validated boundaries). +# 3. symbols ApplySymbols.java with the binary's splat symbol files (the yaml's symbol_addrs_path). +# 4. baseline ExportAnnotations.java -> .run/ghidra_rebuild/.baseline.jsonl (nothing hand-made yet) +# 5. import ImportAnnotations.java config/ghidra/.jsonl (when the file exists) +# 6. export -> .run/ghidra_rebuild/.after.jsonl; delta(after, baseline) -> .delta.jsonl +# --proof cmp .delta.jsonl config/ghidra/.jsonl -> "PROOF PASS" (exit 0) or FAIL (exit 1). +# Without a committed file, step 5 is skipped and the delta of the LIVE export (.run/ghidra_export/.jsonl, +# from tools/ghidra_export_annotations.sh) against the baseline is written to .candidate.jsonl — the +# file to review and commit as config/ghidra/.jsonl. +# +# PRECONDITION: the MCP server is STOPPED (tools/ghidra_mcp_stop.sh); the extracted payload and the built ELF +# exist (`make check BINARY=`). The live project (ghidra/) is never touched. --keep leaves the scratch +# project for inspection (the next run wipes it). +set -uo pipefail +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO" +PROG="${1:?usage: ghidra_rebuild.sh [--proof] [--keep]}"; shift +PROOF=0; KEEP=0 +for a in "$@"; do case "$a" in --proof) PROOF=1 ;; --keep) KEEP=1 ;; *) echo "rebuild: unknown arg $a" >&2; exit 2 ;; esac; done +GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}" +HEADLESS="$GHIDRA/support/analyzeHeadless" +SCRIPTS="$REPO/tools/ghidra_scripts" +GDT="${PSYQ_GDT:-$GHIDRA/Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt}" +SCR="$REPO/.run/ghidra_rebuild" # exports, deltas, function lists (plain files) +# Ghidra refuses a project path with a component starting with '.', so the scratch PROJECT lives under +# the gitignored build/ tree (wiped by `make clean`, which is the right lifetime for it). +PROJ_DIR="$REPO/build/ghidra_rebuild/proj"; PROJ="bfm"; STAGEDIR="$REPO/build/ghidra_rebuild" +CONF="$REPO/config/ghidra/$PROG.jsonl" +LIVE="$REPO/.run/ghidra_export/$PROG.jsonl" +say() { printf 'rebuild[%s]: %s\n' "$PROG" "$*"; } +die() { printf 'rebuild[%s]: %s\n' "$PROG" "$*" >&2; exit 1; } +[ -x "$HEADLESS" ] || die "no analyzeHeadless at $HEADLESS (GHIDRA_INSTALL_DIR)" +[ -f "$GDT" ] || die "no psyq400.gdt at $GDT (PSYQ_GDT)" +if ss -tln 2>/dev/null | grep -qE ":8080([^0-9]|$)"; then die "MCP server serving on :8080 — run tools/ghidra_mcp_stop.sh first"; fi + +# ---- what is this program? ---- +case "$PROG" in + SLUS_007.26) KIND=exe; ALIAS=main; EXE=extracted/retail/SLUS_007.26 ;; + sep8_SLUS_007.26) KIND=exe; ALIAS=proto-sep8; EXE=extracted/proto/sep8_SLUS_007.26 ;; + aug31_USA_DEMO.EXE) KIND=exe; ALIAS=proto-demo; EXE=extracted/proto/aug31_USA_DEMO.EXE ;; + *) KIND=raw; ALIAS="$PROG"; EXE="$(make -s print-EXE BINARY="$ALIAS")"; VRAM="$(make -s print-VRAM_BASE BINARY="$ALIAS")" ;; +esac +[ -n "$EXE" ] && [ -f "$EXE" ] || die "payload not found for $PROG ($EXE) — run make disc-extract (or extract_proto_exe.py for a prototype)" +case "$ALIAS" in + proto-sep8) SYMS=(config/symbols.proto-sep8.txt); ELF="" ;; + proto-demo) SYMS=(config/symbols.proto-demo.txt); ELF="" ;; + *) ELF="$(make -s print-ELF BINARY="$ALIAS")" + mapfile -t SYMS < <(.venv/bin/python - "$ALIAS" <<'EOF' +import sys, yaml +y = yaml.safe_load(open(f"config/splat.{'us.exe' if sys.argv[1]=='main' else sys.argv[1]}.yaml")) +p = y["options"].get("symbol_addrs_path", []) +print("\n".join(p if isinstance(p, list) else [p])) +EOF +) ;; +esac + +# ---- scratch project ---- +rm -rf "$PROJ_DIR"; mkdir -p "$PROJ_DIR" "$SCR" +run_headless() { # args: the analyzeHeadless tail; prints the BFM* / REPORT / ERROR lines; returns its exit + timeout 3600 "$HEADLESS" "$PROJ_DIR" "$PROJ" "$@" 2>&1 | grep -E 'BFM|REPORT:|ERROR|Exception|Analysis succeeded|Import' | grep -vE 'INFO (Analysis|REPORT: Import)' | head -40 + return "${PIPESTATUS[0]}" +} + +say "1. import ($KIND) $EXE" +if [ "$KIND" = exe ]; then + run_headless -import "$EXE" -overwrite -scriptPath "$SCRIPTS" -postScript ImportPsyqGdt.java "$GDT" -postScript DumpProgramInfo.java || die "import failed" +else + STAGE="$STAGEDIR/$PROG"; cp -f "$EXE" "$STAGE" + run_headless -import "$STAGE" -overwrite -loader BinaryLoader -loader-baseAddr "$VRAM" -processor PSX:LE:32:default \ + -scriptPath "$SCRIPTS" -postScript ImportPsyqGdt.java "$GDT" -postScript DumpProgramInfo.java || die "import failed" + rm -f "$STAGE" +fi + +if [ -n "$ELF" ] && [ -f "$ELF" ]; then + say "2. functions from $ELF" + LO="$(make -s print-TEXT_LO BINARY="$ALIAS")"; HI="$(make -s print-TEXT_HI BINARY="$ALIAS")" + mipsel-linux-gnu-nm "$ELF" | awk -v lo="$LO" -v hi="$HI" ' + $2=="T" || $2=="t" { a=strtonum("0x" $1); if (a%4==0 && a>=strtonum(lo) && a "$SCR/${PROG}_funcs.txt" + run_headless -process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript DefineFunctions.java "$SCR/${PROG}_funcs.txt" || die "DefineFunctions failed" +else + say "2. functions: no built ELF for $ALIAS — keeping the analysis-found set" +fi + +say "3. symbols: ${SYMS[*]}" +run_headless -process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript ApplySymbols.java "${SYMS[@]}" || die "ApplySymbols failed" + +say "4. baseline export" +run_headless -process "$PROG" -noanalysis -readOnly -scriptPath "$SCRIPTS" -postScript ExportAnnotations.java "$SCR/$PROG.baseline.jsonl" "${SYMS[@]}" || die "baseline export failed" + +if [ ! -f "$CONF" ]; then + [ -f "$LIVE" ] || die "no $CONF and no live export $LIVE (run tools/ghidra_export_annotations.sh $PROG first)" + .venv/bin/python tools/ghidra_annotations_delta.py "$LIVE" "$SCR/$PROG.baseline.jsonl" "$SCR/$PROG.candidate.jsonl" --census + say "no committed file — CANDIDATE written: $SCR/$PROG.candidate.jsonl (review, then cp to $CONF)" + [ "$KEEP" = 1 ] || rm -rf "$PROJ_DIR" + exit 0 +fi + +say "5. import $CONF" +run_headless -process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript ImportAnnotations.java "$CONF" || die "ImportAnnotations failed" +say "6. export after import + delta" +run_headless -process "$PROG" -noanalysis -readOnly -scriptPath "$SCRIPTS" -postScript ExportAnnotations.java "$SCR/$PROG.after.jsonl" "${SYMS[@]}" || die "export failed" +.venv/bin/python tools/ghidra_annotations_delta.py "$SCR/$PROG.after.jsonl" "$SCR/$PROG.baseline.jsonl" "$SCR/$PROG.delta.jsonl" --census +if [ "$PROOF" = 1 ]; then + if cmp -s "$SCR/$PROG.delta.jsonl" "$CONF"; then + say "PROOF PASS — the rebuilt program's hand-authored delta == $CONF ($(wc -l < "$CONF") rows)" + [ "$KEEP" = 1 ] || rm -rf "$PROJ_DIR" + exit 0 + fi + say "PROOF FAIL — delta differs from $CONF (diff below, first 40 lines; scratch kept in $SCR)" + diff "$CONF" "$SCR/$PROG.delta.jsonl" | head -40 + exit 1 +fi +[ "$KEEP" = 1 ] || rm -rf "$PROJ_DIR" +say "done (no --proof requested)" diff --git a/tools/ghidra_scripts/DefineFunctions.java b/tools/ghidra_scripts/DefineFunctions.java index 12c004519..f427d224d 100644 --- a/tools/ghidra_scripts/DefineFunctions.java +++ b/tools/ghidra_scripts/DefineFunctions.java @@ -18,8 +18,12 @@ import java.util.List; public class DefineFunctions extends GhidraScript { public void run() throws Exception { String prog = currentProgram.getName(); - Path in = Paths.get(System.getProperty("user.home"), "bfm-decomp", ".run", prog + "_funcs.txt"); - if (!Files.exists(in)) { println("DefineFunctions: no list at " + in); return; } + // P33 B5: the list path is arg 1 (tools/ghidra_rebuild.sh passes .run/ghidra_rebuild/_funcs.txt); + // without an arg the historical location .run/_funcs.txt under the CWD (the repo) is used — + // no more `user.home/bfm-decomp` assumption. + String[] args = getScriptArgs(); + Path in = (args != null && args.length > 0) ? Paths.get(args[0]) : Paths.get(".run", prog + "_funcs.txt"); + if (!Files.exists(in)) { println("DefineFunctions: no list at " + in.toAbsolutePath()); return; } List lines = Files.readAllLines(in); int created = 0, existed = 0, failed = 0; for (String ln : lines) { diff --git a/tools/ghidra_scripts/ExportAnnotations.java b/tools/ghidra_scripts/ExportAnnotations.java new file mode 100644 index 000000000..3cf33055e --- /dev/null +++ b/tools/ghidra_scripts/ExportAnnotations.java @@ -0,0 +1,295 @@ +// ExportAnnotations.java — headless, READ-ONLY: serialize a program's ANNOTATIONS to byte-stable JSON-Lines +// (P33 B5, the Ghidra-regenerability deliverable carried since Phase 9/10). +// +// WHAT: everything a human (or the MCP) can author in a program that is NOT already text in the repo — +// data types (structs/unions/enums/typedefs/function definitions of the program's LOCAL archive; the +// psyq400 archive types are re-imported by ImportPsyqGdt.java), function signatures (return type, +// calling convention, parameters, locals, flags), defined data, the five comment kinds, bookmarks, +// equates, and non-default labels that the splat symbol files do not already carry — plus the +// container facts (language, image base, memory blocks) so an image-base drift shows as a diff. +// +// HOW IT IS USED: tools/ghidra_rebuild.sh exports the LIVE program and a FRESH rebuild (import + +// analysis + DefineFunctions + ApplySymbols) and stores their DELTA (tools/ghidra_annotations_delta.py) +// as config/ghidra/.jsonl — every row analysis produces on its own subtracts itself out, so +// what is committed is the hand-authored RE work. The proof re-imports that delta and re-exports. +// Byte-stability is the whole contract: fixed key order, sorted rows, 0x%08x addresses, one JSON object +// per line, no whitespace. Deterministic across machines given the same program. +// +// analyzeHeadless bfm -process -noanalysis -readOnly \ +// -scriptPath tools/ghidra_scripts -postScript ExportAnnotations.java [symbols.txt ...] +// +// Arg 1: an output FILE, or a DIRECTORY (then /.jsonl — the all-programs form, run with +// `-process` and no program name). Args 2..: splat symbol files whose `name = 0xADDR;` rows are +// skipped from the label export (ApplySymbols.java restores those). +import ghidra.app.script.GhidraScript; +import ghidra.program.model.address.Address; +import ghidra.program.model.address.AddressIterator; +import ghidra.program.model.data.*; +import ghidra.program.model.listing.*; +import ghidra.program.model.mem.MemoryBlock; +import ghidra.program.model.symbol.*; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ExportAnnotations extends GhidraScript { + + // ---- JSON helpers (the DumpFunctionSignatures.java convention: hand-serialized, canonical) ---- + private static String esc(String s) { + if (s == null) return "null"; + StringBuilder sb = new StringBuilder("\""); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '"' || c == '\\') sb.append('\\').append(c); + else if (c == '\n') sb.append("\\n"); + else if (c == '\r') sb.append("\\r"); + else if (c == '\t') sb.append("\\t"); + else if (c < 0x20) sb.append(String.format("\\u%04x", (int) c)); + else sb.append(c); + } + return sb.append('"').toString(); + } + private static String hexAddr(Address a) { return String.format("\"0x%08x\"", a.getOffset()); } + private static String b(boolean v) { return v ? "true" : "false"; } + + private final Map> symfileNames = new HashMap<>(); // addr -> names the symbol files carry + + @Override + protected void run() throws Exception { + String[] args = getScriptArgs(); + if (args == null || args.length < 1) { println("BFMEXPORT ERROR: usage [symbols.txt ...]"); return; } + Path out = Paths.get(args[0]); + if (Files.isDirectory(out)) out = out.resolve(currentProgram.getName() + ".jsonl"); + Pattern sp = Pattern.compile("^\\s*([A-Za-z_]\\w*)\\s*=\\s*0x([0-9A-Fa-f]+)\\s*;"); + for (int i = 1; i < args.length; i++) { + Path f = Paths.get(args[i]); + if (!Files.isRegularFile(f)) { println("BFMEXPORT WARN: symbols file not found: " + f); continue; } + for (String ln : Files.readAllLines(f)) { + Matcher m = sp.matcher(ln); + if (m.find()) symfileNames.computeIfAbsent(Long.parseLong(m.group(2), 16), k -> new HashSet<>()).add(m.group(1)); + } + } + + List rows = new ArrayList<>(); + int nTypes = program(rows); + int nFuncs = functions(rows); + int nData = data(rows); + int nComments = comments(rows); + int nBookmarks = bookmarks(rows); + int nEquates = equates(rows); + int nLabels = labels(rows); + + StringBuilder sb = new StringBuilder(); + for (String r : rows) sb.append(r).append('\n'); + Files.createDirectories(out.toAbsolutePath().getParent()); + Files.write(out, sb.toString().getBytes(StandardCharsets.UTF_8)); + println("BFMEXPORT program=" + currentProgram.getName() + " types=" + nTypes + " funcs=" + nFuncs + + " data=" + nData + " comments=" + nComments + " bookmarks=" + nBookmarks + " equates=" + nEquates + + " labels=" + nLabels + " rows=" + rows.size() + " -> " + out); + } + + // ---- program, blocks, archives, LOCAL data types (sorted by path) ---- + private int program(List rows) { + Program p = currentProgram; + rows.add("{\"k\":\"program\",\"name\":" + esc(p.getName()) + ",\"lang\":" + esc(p.getLanguageID().getIdAsString()) + + ",\"cspec\":" + esc(p.getCompilerSpec().getCompilerSpecID().getIdAsString()) + + ",\"image_base\":" + hexAddr(p.getImageBase()) + ",\"format\":" + esc(p.getExecutableFormat()) + "}"); + List blocks = new ArrayList<>(); + for (MemoryBlock mb : p.getMemory().getBlocks()) { + String perm = (mb.isRead() ? "r" : "-") + (mb.isWrite() ? "w" : "-") + (mb.isExecute() ? "x" : "-"); + blocks.add("{\"k\":\"block\",\"name\":" + esc(mb.getName()) + ",\"start\":" + hexAddr(mb.getStart()) + + ",\"end\":" + hexAddr(mb.getEnd()) + ",\"perm\":\"" + perm + "\",\"init\":" + b(mb.isInitialized()) + "}"); + } + rows.addAll(blocks); // memory order as Ghidra lists them (address order) + DataTypeManager dtm = p.getDataTypeManager(); + SourceArchive local = dtm.getLocalSourceArchive(); + TreeSet archives = new TreeSet<>(); + for (SourceArchive sa : dtm.getSourceArchives()) { + if (sa.getSourceArchiveID().equals(local.getSourceArchiveID())) continue; + if (sa.getArchiveType() == ArchiveType.BUILT_IN) continue; + archives.add("{\"k\":\"archive\",\"name\":" + esc(sa.getName()) + "}"); + } + rows.addAll(archives); + TreeMap types = new TreeMap<>(); + Iterator it = dtm.getAllDataTypes(); + while (it.hasNext()) { + DataType dt = it.next(); + if (dt instanceof Pointer || dt instanceof Array || dt instanceof BuiltInDataType) continue; + SourceArchive sa = dt.getSourceArchive(); + if (sa != null && !sa.getSourceArchiveID().equals(local.getSourceArchiveID())) continue; // psyq400 etc. + String row = typeRow(dt); + if (row != null) types.put(dt.getPathName(), row); + } + rows.addAll(types.values()); + return types.size(); + } + + private String typeRow(DataType dt) { + String path = esc(dt.getPathName()); + if (dt instanceof Structure) { + Structure s = (Structure) dt; + StringBuilder f = new StringBuilder("["); + boolean first = true; + for (DataTypeComponent c : s.getDefinedComponents()) { + if (!first) f.append(','); first = false; + f.append("{\"off\":").append(c.getOffset()).append(",\"len\":").append(c.getLength()) + .append(",\"name\":").append(esc(c.getFieldName())).append(",\"type\":").append(esc(c.getDataType().getPathName())) + .append(",\"comment\":").append(esc(c.getComment())).append('}'); + } + return "{\"k\":\"type\",\"kind\":\"struct\",\"path\":" + path + ",\"size\":" + s.getLength() + + ",\"packed\":" + b(s.isPackingEnabled()) + ",\"fields\":" + f + "]}"; + } + if (dt instanceof Union) { + Union u = (Union) dt; + StringBuilder f = new StringBuilder("["); + boolean first = true; + for (DataTypeComponent c : u.getComponents()) { + if (!first) f.append(','); first = false; + f.append("{\"len\":").append(c.getLength()).append(",\"name\":").append(esc(c.getFieldName())) + .append(",\"type\":").append(esc(c.getDataType().getPathName())).append(",\"comment\":").append(esc(c.getComment())).append('}'); + } + return "{\"k\":\"type\",\"kind\":\"union\",\"path\":" + path + ",\"size\":" + u.getLength() + + ",\"packed\":" + b(u.isPackingEnabled()) + ",\"fields\":" + f + "]}"; + } + if (dt instanceof ghidra.program.model.data.Enum) { + ghidra.program.model.data.Enum e = (ghidra.program.model.data.Enum) dt; + TreeMap vals = new TreeMap<>(); + for (String n : e.getNames()) vals.put(String.format("%020d|%s", e.getValue(n), n), + "{\"name\":" + esc(n) + ",\"value\":" + e.getValue(n) + "}"); + return "{\"k\":\"type\",\"kind\":\"enum\",\"path\":" + path + ",\"size\":" + e.getLength() + + ",\"values\":[" + String.join(",", vals.values()) + "]}"; + } + if (dt instanceof TypeDef) { + return "{\"k\":\"type\",\"kind\":\"typedef\",\"path\":" + path + ",\"base\":" + esc(((TypeDef) dt).getDataType().getPathName()) + "}"; + } + if (dt instanceof FunctionDefinition) { + FunctionDefinition fd = (FunctionDefinition) dt; + StringBuilder ps = new StringBuilder("["); + boolean first = true; + for (ParameterDefinition pd : fd.getArguments()) { + if (!first) ps.append(','); first = false; + ps.append("{\"name\":").append(esc(pd.getName())).append(",\"type\":").append(esc(pd.getDataType().getPathName())) + .append(",\"comment\":").append(esc(pd.getComment())).append('}'); + } + return "{\"k\":\"type\",\"kind\":\"fndef\",\"path\":" + path + ",\"ret\":" + esc(fd.getReturnType().getPathName()) + + ",\"cc\":" + esc(fd.getCallingConventionName()) + ",\"varargs\":" + b(fd.hasVarArgs()) + ",\"params\":" + ps + "]}"; + } + return "{\"k\":\"type\",\"kind\":\"other\",\"path\":" + path + ",\"class\":" + esc(dt.getClass().getSimpleName()) + "}"; + } + + // ---- functions (every non-external function; the delta keeps the user-touched ones) ---- + private int functions(List rows) { + TreeMap m = new TreeMap<>(); + for (Function f : currentProgram.getFunctionManager().getFunctions(true)) { + if (f.isExternal()) continue; + StringBuilder ps = new StringBuilder("["); + boolean first = true; + for (Parameter p : f.getParameters()) { + if (!first) ps.append(','); first = false; + ps.append("{\"name\":").append(esc(p.getName())).append(",\"type\":").append(esc(p.getDataType().getPathName())) + .append(",\"storage\":").append(esc(p.getVariableStorage().toString())).append(",\"src\":").append(esc(p.getSource().name())).append('}'); + } + ps.append(']'); + TreeMap locals = new TreeMap<>(); + for (Variable v : f.getLocalVariables()) { + String key = String.format("%010d|%s|%s", v.getFirstUseOffset(), v.getVariableStorage().toString(), v.getName()); + locals.put(key, "{\"name\":" + esc(v.getName()) + ",\"type\":" + esc(v.getDataType().getPathName()) + + ",\"storage\":" + esc(v.getVariableStorage().toString()) + ",\"first\":" + v.getFirstUseOffset() + + ",\"src\":" + esc(v.getSource().name()) + ",\"comment\":" + esc(v.getComment()) + "}"); + } + String row = "{\"k\":\"func\",\"addr\":" + hexAddr(f.getEntryPoint()) + ",\"name\":" + esc(f.getName()) + + ",\"ret\":" + esc(f.getReturnType().getPathName()) + ",\"cc\":" + esc(f.getCallingConventionName()) + + ",\"varargs\":" + b(f.hasVarArgs()) + ",\"noreturn\":" + b(f.hasNoReturn()) + ",\"inline\":" + b(f.isInline()) + + ",\"custom\":" + b(f.hasCustomVariableStorage()) + ",\"sigsrc\":" + esc(f.getSignatureSource().name()) + + ",\"params\":" + ps + ",\"locals\":[" + String.join(",", locals.values()) + "]" + + ",\"comment\":" + esc(f.getComment()) + "}"; + m.put(f.getEntryPoint().getOffset(), row); + } + rows.addAll(m.values()); + return m.size(); + } + + // ---- defined data (every typed unit; the delta keeps the user-typed ones) ---- + private int data(List rows) { + TreeMap m = new TreeMap<>(); + DataIterator it = currentProgram.getListing().getDefinedData(true); + while (it.hasNext()) { + Data d = it.next(); + DataType dt = d.getDataType(); + if (dt instanceof DefaultDataType || dt instanceof Undefined) continue; + m.put(d.getAddress().getOffset(), "{\"k\":\"data\",\"addr\":" + hexAddr(d.getAddress()) + ",\"type\":" + esc(dt.getPathName()) + + ",\"len\":" + d.getLength() + "}"); + } + rows.addAll(m.values()); + return m.size(); + } + + // ---- comments, five kinds ---- + private int comments(List rows) { + TreeMap m = new TreeMap<>(); + Listing l = currentProgram.getListing(); + CommentType[] kinds = {CommentType.EOL, CommentType.PRE, CommentType.POST, CommentType.PLATE, CommentType.REPEATABLE}; + String[] names = {"eol", "pre", "post", "plate", "repeat"}; + for (int i = 0; i < kinds.length; i++) { + AddressIterator ai = l.getCommentAddressIterator(kinds[i], currentProgram.getMemory(), true); + while (ai.hasNext()) { + Address a = ai.next(); + String c = l.getComment(kinds[i], a); + if (c == null) continue; + m.put(String.format("%016x|%d", a.getOffset(), i), "{\"k\":\"comment\",\"addr\":" + hexAddr(a) + ",\"type\":\"" + names[i] + "\",\"text\":" + esc(c) + "}"); + } + } + rows.addAll(m.values()); + return m.size(); + } + + private int bookmarks(List rows) { + TreeMap m = new TreeMap<>(); + Iterator it = currentProgram.getBookmarkManager().getBookmarksIterator(); + while (it.hasNext()) { + Bookmark bk = it.next(); + String key = String.format("%016x|%s|%s|%s", bk.getAddress().getOffset(), bk.getTypeString(), bk.getCategory(), bk.getComment()); + m.put(key, "{\"k\":\"bookmark\",\"addr\":" + hexAddr(bk.getAddress()) + ",\"type\":" + esc(bk.getTypeString()) + + ",\"cat\":" + esc(bk.getCategory()) + ",\"text\":" + esc(bk.getComment()) + "}"); + } + rows.addAll(m.values()); + return m.size(); + } + + private int equates(List rows) { + TreeMap m = new TreeMap<>(); + Iterator it = currentProgram.getEquateTable().getEquates(); + while (it.hasNext()) { + Equate e = it.next(); + TreeSet refs = new TreeSet<>(); + for (EquateReference r : e.getReferences()) + refs.add("[" + hexAddr(r.getAddress()) + "," + r.getOpIndex() + "]"); + m.put(e.getName(), "{\"k\":\"equate\",\"name\":" + esc(e.getName()) + ",\"value\":" + e.getValue() + + ",\"refs\":[" + String.join(",", refs) + "]}"); + } + rows.addAll(m.values()); + return m.size(); + } + + // ---- labels: every non-DEFAULT, non-external symbol the symbol files do not already carry ---- + private int labels(List rows) { + TreeMap m = new TreeMap<>(); + for (Symbol s : currentProgram.getSymbolTable().getAllSymbols(true)) { + if (s.isExternal() || s.getSource() == SourceType.DEFAULT) continue; + if (!currentProgram.getMemory().contains(s.getAddress())) continue; + Set have = symfileNames.get(s.getAddress().getOffset()); + if (have != null && have.contains(s.getName())) continue; + boolean isFunc = s.getSymbolType() == SymbolType.FUNCTION; + m.put(String.format("%016x|%s", s.getAddress().getOffset(), s.getName()), + "{\"k\":\"label\",\"addr\":" + hexAddr(s.getAddress()) + ",\"name\":" + esc(s.getName()) + ",\"primary\":" + b(s.isPrimary()) + + ",\"src\":" + esc(s.getSource().name()) + ",\"func\":" + b(isFunc) + "}"); + } + rows.addAll(m.values()); + return m.size(); + } +} diff --git a/tools/ghidra_scripts/ExportSymbols.java b/tools/ghidra_scripts/ExportSymbols.java index c55da9f13..294f384dd 100644 --- a/tools/ghidra_scripts/ExportSymbols.java +++ b/tools/ghidra_scripts/ExportSymbols.java @@ -38,7 +38,22 @@ public class ExportSymbols extends GhidraScript { sb.append("// Durable text record of manual RE work (the .rep is gitignored). Phase-5 splat refines this.\n"); sb.append("// count=").append(n).append("\n\n"); for (String v : rows.values()) sb.append(v).append("\n"); - java.nio.file.Path out = Paths.get(System.getProperty("user.home"), "bfm-decomp", "config", "symbols.us.txt"); + // P33 B5 (rule R15): NEVER overwrite the curated config/symbols.us.txt — re-exporting is a reviewed + // re-merge. Arg 1 = output path (default .run/ghidra_export/.symbols.txt under the CWD); + // an existing output is refused unless arg 2 is the literal `overwrite`. + String[] args = getScriptArgs(); + java.nio.file.Path out = (args != null && args.length > 0) ? Paths.get(args[0]) + : Paths.get(".run", "ghidra_export", currentProgram.getName() + ".symbols.txt"); + boolean overwrite = args != null && args.length > 1 && "overwrite".equals(args[1]); + if (Files.exists(out) && !overwrite) { + println("BFMSYM REFUSED: " + out + " exists (pass `overwrite` as the 2nd arg; config/symbols*.txt is merged by hand, R15)"); + return; + } + if (out.toAbsolutePath().normalize().toString().contains("/config/")) { + println("BFMSYM REFUSED: will not write into config/ (R15 — merge the export by hand): " + out); + return; + } + Files.createDirectories(out.toAbsolutePath().getParent()); Files.write(out, sb.toString().getBytes()); println("BFMSYM exported " + n + " USER_DEFINED symbols -> " + out); } diff --git a/tools/ghidra_scripts/ImportAnnotations.java b/tools/ghidra_scripts/ImportAnnotations.java new file mode 100644 index 000000000..b0bd48dd1 --- /dev/null +++ b/tools/ghidra_scripts/ImportAnnotations.java @@ -0,0 +1,370 @@ +// ImportAnnotations.java — headless: apply an annotations JSON-Lines file (ExportAnnotations.java's format, +// normally the committed config/ghidra/.jsonl) to the current program. The inverse of the export; +// idempotent (compare-before-write everywhere); run WITHOUT -readOnly so analyzeHeadless commits + saves +// on exit (the ApplySymbols.java precedent). P33 B5. +// +// analyzeHeadless bfm -process -noanalysis \ +// -scriptPath tools/ghidra_scripts -postScript ImportAnnotations.java +// +// Refuses (R43) a file whose `program` row names another language / image base, and any row kind it does +// not know — nothing is applied in that case. Prints one BFMANN summary line with every counter. +// Types are applied in two passes (shells first, then fields) so self- and mutually-referencing structs +// resolve; type strings are Ghidra path names with pointer/array suffixes (`/BFM/Actor *[4]`). +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import ghidra.app.script.GhidraScript; +import ghidra.program.model.address.Address; +import ghidra.program.model.data.*; +import ghidra.program.model.lang.Register; +import ghidra.program.model.listing.*; +import ghidra.program.model.symbol.*; +import ghidra.program.model.util.CodeUnitInsertionException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.*; + +public class ImportAnnotations extends GhidraScript { + private static final Set KINDS = new HashSet<>(Arrays.asList( + "program", "block", "archive", "type", "func", "data", "comment", "bookmark", "equate", "label")); + private int nTypes, nFuncs, nData, nComments, nBookmarks, nEquates, nLabels, unchanged, failed, skippedStorage; + private DataTypeManager dtm; + + private static String s(JsonObject o, String k) { JsonElement e = o.get(k); return (e == null || e.isJsonNull()) ? null : e.getAsString(); } + private static boolean bo(JsonObject o, String k) { JsonElement e = o.get(k); return e != null && !e.isJsonNull() && e.getAsBoolean(); } + private static int in(JsonObject o, String k) { JsonElement e = o.get(k); return (e == null || e.isJsonNull()) ? 0 : e.getAsInt(); } + private Address addr(String hex) { return currentProgram.getAddressFactory().getDefaultAddressSpace().getAddress(Long.decode(hex)); } + + @Override + protected void run() throws Exception { + String[] args = getScriptArgs(); + if (args == null || args.length < 1) { println("BFMANN ERROR: usage "); return; } + List rows = new ArrayList<>(); + for (String ln : Files.readAllLines(Paths.get(args[0]), StandardCharsets.UTF_8)) { + if (ln.trim().isEmpty()) continue; + JsonObject o = JsonParser.parseString(ln).getAsJsonObject(); + String k = s(o, "k"); + if (k == null || !KINDS.contains(k)) { println("BFMANN ERROR: unknown row kind " + k + " — nothing applied (R43)"); return; } + rows.add(o); + } + for (JsonObject o : rows) { + if (!"program".equals(s(o, "k"))) continue; + String lang = s(o, "lang"); + long base = Long.decode(s(o, "image_base")); + if (!currentProgram.getLanguageID().getIdAsString().equals(lang) || currentProgram.getImageBase().getOffset() != base) { + println("BFMANN ERROR: file is for " + lang + " @" + s(o, "image_base") + " but the program is " + + currentProgram.getLanguageID().getIdAsString() + " @" + currentProgram.getImageBase() + " — nothing applied (R43)"); + return; + } + } + dtm = currentProgram.getDataTypeManager(); + // pass 1: type shells; pass 2: type bodies (so fields can reference any struct in the file) + List types = new ArrayList<>(); + for (JsonObject o : rows) if ("type".equals(s(o, "k"))) types.add(o); + for (JsonObject o : types) typeShell(o); + for (JsonObject o : types) typeBody(o); + for (JsonObject o : rows) { + String k = s(o, "k"); + try { + switch (k) { + case "func": func(o); break; + case "data": data(o); break; + case "comment": comment(o); break; + case "bookmark": bookmark(o); break; + case "equate": equate(o); break; + case "label": label(o); break; + default: break; // program/block/archive/type handled above + } + } catch (Exception e) { + failed++; + println(" fail " + k + " " + (s(o, "addr") != null ? s(o, "addr") : s(o, "name")) + ": " + e); + } + } + println("BFMANN types=" + nTypes + " funcs=" + nFuncs + " data=" + nData + " comments=" + nComments + + " bookmarks=" + nBookmarks + " equates=" + nEquates + " labels=" + nLabels + " unchanged=" + unchanged + + " skipped_storage=" + skippedStorage + " failed=" + failed + " rows=" + rows.size()); + } + + // ---- data types ---- + private static CategoryPath catOf(String path) { + int i = path.lastIndexOf('/'); + return new CategoryPath(i <= 0 ? "/" : path.substring(0, i)); + } + private static String nameOf(String path) { return path.substring(path.lastIndexOf('/') + 1); } + + /** Resolve a Ghidra path name with optional pointer/array suffixes (`/u8 *[4]`) to a DataType. */ + private DataType resolveType(String spec) { + String base = spec; + int cut = -1; + for (int i = 0; i < spec.length(); i++) { + char c = spec.charAt(i); + if (c == '[' || (c == '*' && i > 0 && spec.charAt(i - 1) == ' ')) { cut = i; break; } + } + String suffix = ""; + if (cut >= 0) { base = spec.substring(0, cut).trim(); suffix = spec.substring(cut); } + DataType dt = dtm.getDataType(base); + if (dt == null) dt = BuiltInDataTypeManager.getDataTypeManager().getDataType(base); + if (dt == null && !base.startsWith("/")) dt = dtm.getDataType("/" + base); + if (dt == null) throw new IllegalArgumentException("unresolvable type " + spec); + int i = 0; + while (i < suffix.length()) { + char c = suffix.charAt(i); + if (c == ' ') { i++; continue; } + if (c == '*') { dt = new PointerDataType(dt, dtm); i++; continue; } + if (c == '[') { + int j = suffix.indexOf(']', i); + int n = Integer.parseInt(suffix.substring(i + 1, j).trim()); + dt = new ArrayDataType(dt, n, dt.getLength(), dtm); + i = j + 1; continue; + } + throw new IllegalArgumentException("bad type suffix in " + spec); + } + return dt; + } + + private void typeShell(JsonObject o) { + String kind = s(o, "kind"), path = s(o, "path"); + CategoryPath cat = catOf(path); String name = nameOf(path); + if (dtm.getDataType(path) != null) return; + DataType dt; + switch (kind) { + case "struct": dt = new StructureDataType(cat, name, in(o, "size"), dtm); break; + case "union": dt = new UnionDataType(cat, name, dtm); break; + case "enum": dt = new EnumDataType(cat, name, in(o, "size"), dtm); break; + case "typedef": dt = new TypedefDataType(cat, name, resolveType(s(o, "base")), dtm); break; + case "fndef": dt = new FunctionDefinitionDataType(cat, name, dtm); break; + default: failed++; println(" fail type " + path + ": unsupported kind " + kind); return; + } + dtm.resolve(dt, DataTypeConflictHandler.REPLACE_HANDLER); + nTypes++; + } + + private void typeBody(JsonObject o) { + String kind = s(o, "kind"), path = s(o, "path"); + DataType dt = dtm.getDataType(path); + if (dt == null) return; + try { + if ("struct".equals(kind) && dt instanceof Structure) { + Structure st = (Structure) dt; + boolean packed = bo(o, "packed"); + if (st.isPackingEnabled() != packed) st.setPackingEnabled(packed); + if (!packed && st.getLength() != in(o, "size")) { st.deleteAll(); st.growStructure(in(o, "size")); } + for (JsonElement fe : o.getAsJsonArray("fields")) { + JsonObject f = fe.getAsJsonObject(); + DataType ft = resolveType(s(f, "type")); + DataTypeComponent cur = st.getComponentAt(in(f, "off")); + if (cur != null && cur.getOffset() == in(f, "off") && Objects.equals(cur.getFieldName(), s(f, "name")) + && cur.getDataType().getPathName().equals(ft.getPathName()) && cur.getLength() == in(f, "len") + && Objects.equals(cur.getComment(), s(f, "comment"))) { unchanged++; continue; } + if (packed) st.add(ft, in(f, "len"), s(f, "name"), s(f, "comment")); + else st.replaceAtOffset(in(f, "off"), ft, in(f, "len"), s(f, "name"), s(f, "comment")); + } + } else if ("union".equals(kind) && dt instanceof Union) { + Union u = (Union) dt; + if (u.getNumComponents() == 0) + for (JsonElement fe : o.getAsJsonArray("fields")) { + JsonObject f = fe.getAsJsonObject(); + u.add(resolveType(s(f, "type")), in(f, "len"), s(f, "name"), s(f, "comment")); + } + else unchanged++; + } else if ("enum".equals(kind) && dt instanceof ghidra.program.model.data.Enum) { + ghidra.program.model.data.Enum e = (ghidra.program.model.data.Enum) dt; + for (JsonElement ve : o.getAsJsonArray("values")) { + JsonObject v = ve.getAsJsonObject(); + String n = s(v, "name"); long val = v.get("value").getAsLong(); + if (e.contains(n) && e.getValue(n) == val) { unchanged++; continue; } + if (e.contains(n)) e.remove(n); + e.add(n, val); + } + } else if ("fndef".equals(kind) && dt instanceof FunctionDefinition) { + FunctionDefinition fd = (FunctionDefinition) dt; + fd.setReturnType(resolveType(s(o, "ret"))); + List ps = new ArrayList<>(); + for (JsonElement pe : o.getAsJsonArray("params")) { + JsonObject p = pe.getAsJsonObject(); + ps.add(new ParameterDefinitionImpl(s(p, "name"), resolveType(s(p, "type")), s(p, "comment"))); + } + fd.setArguments(ps.toArray(new ParameterDefinition[0])); + fd.setVarArgs(bo(o, "varargs")); + String cc = s(o, "cc"); + if (cc != null) try { fd.setCallingConvention(cc); } catch (Exception ex) { /* unknown cc: keep default */ } + } + } catch (Exception e) { + failed++; println(" fail type body " + path + ": " + e); + } + } + + // ---- functions ---- + private VariableStorage storageOf(String spec) { + // "Stack[-0x10]:4" | "a0:4" | "s0:4" ... ; anything else -> null (dynamic storage) + try { + if (spec.startsWith("Stack[")) { + int j = spec.indexOf(']'); + int off = (int) Long.decode(spec.substring(6, j)); + int size = Integer.parseInt(spec.substring(spec.lastIndexOf(':') + 1)); + return new VariableStorage(currentProgram, off, size); + } + String rn = spec.contains(":") ? spec.substring(0, spec.indexOf(':')) : spec; + Register r = currentProgram.getRegister(rn); + if (r != null) return new VariableStorage(currentProgram, r); + } catch (Exception e) { /* fall through */ } + return null; + } + + private void func(JsonObject o) throws Exception { + Address a = addr(s(o, "addr")); + FunctionManager fm = currentProgram.getFunctionManager(); + Function f = fm.getFunctionAt(a); + if (f == null) { + if (getInstructionAt(a) == null) disassemble(a); + f = createFunction(a, s(o, "name")); + if (f == null) { failed++; println(" fail func " + s(o, "addr") + ": cannot create"); return; } + } + SourceType src = SourceType.valueOf(s(o, "sigsrc")); + boolean changed = false; + String name = s(o, "name"); + if (name != null && !name.equals(f.getName())) { f.setName(name, src == SourceType.DEFAULT ? SourceType.USER_DEFINED : src); changed = true; } + DataType ret = resolveType(s(o, "ret")); + if (!f.getReturnType().getPathName().equals(ret.getPathName())) { f.setReturnType(ret, src); changed = true; } + String cc = s(o, "cc"); + if (cc != null && !cc.equals(f.getCallingConventionName())) { try { f.setCallingConvention(cc); changed = true; } catch (Exception e) { skippedStorage++; } } + if (f.hasVarArgs() != bo(o, "varargs")) { f.setVarArgs(bo(o, "varargs")); changed = true; } + if (f.hasNoReturn() != bo(o, "noreturn")) { f.setNoReturn(bo(o, "noreturn")); changed = true; } + if (f.isInline() != bo(o, "inline")) { f.setInline(bo(o, "inline")); changed = true; } + // parameters + JsonArray ps = o.getAsJsonArray("params"); + boolean custom = bo(o, "custom"); + List want = new ArrayList<>(); + boolean allStorage = true; + for (JsonElement pe : ps) { + JsonObject p = pe.getAsJsonObject(); + DataType pt = resolveType(s(p, "type")); + VariableStorage vs = custom ? storageOf(s(p, "storage")) : null; + if (custom && vs == null) allStorage = false; + want.add(vs != null ? new ParameterImpl(s(p, "name"), pt, vs, currentProgram) : new ParameterImpl(s(p, "name"), pt, currentProgram)); + } + boolean same = f.getParameterCount() == want.size(); + for (int i = 0; same && i < want.size(); i++) { + Parameter cur = f.getParameter(i); + same = Objects.equals(cur.getName(), want.get(i).getName()) && cur.getDataType().getPathName().equals(want.get(i).getDataType().getPathName()) + && (!custom || cur.getVariableStorage().toString().equals(want.get(i).getVariableStorage().toString())); + } + if (!same) { + Function.FunctionUpdateType ut = (custom && allStorage) ? Function.FunctionUpdateType.CUSTOM_STORAGE : Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS; + if (custom && !allStorage) skippedStorage++; + f.replaceParameters(want, ut, true, src == SourceType.DEFAULT ? SourceType.USER_DEFINED : src); + changed = true; + } + // locals: match by storage + first-use; update name/type/comment, else add + for (JsonElement le : o.getAsJsonArray("locals")) { + JsonObject l = le.getAsJsonObject(); + String storage = s(l, "storage"); int first = in(l, "first"); + DataType lt = resolveType(s(l, "type")); + Variable hit = null; + for (Variable v : f.getLocalVariables()) + if (v.getVariableStorage().toString().equals(storage) && v.getFirstUseOffset() == first) { hit = v; break; } + SourceType lsrc = SourceType.valueOf(s(l, "src")); + if (lsrc == SourceType.DEFAULT) lsrc = SourceType.USER_DEFINED; + if (hit != null) { + boolean lc = false; + if (!Objects.equals(hit.getName(), s(l, "name"))) { hit.setName(s(l, "name"), lsrc); lc = true; } + if (!hit.getDataType().getPathName().equals(lt.getPathName())) { hit.setDataType(lt, lsrc); lc = true; } + if (!Objects.equals(hit.getComment(), s(l, "comment"))) { hit.setComment(s(l, "comment")); lc = true; } + if (lc) changed = true; + continue; + } + Variable nv; + if (storage.startsWith("Stack[")) { + int off = (int) Long.decode(storage.substring(6, storage.indexOf(']'))); + nv = new LocalVariableImpl(s(l, "name"), first, lt, off, currentProgram); + } else { + String rn = storage.contains(":") ? storage.substring(0, storage.indexOf(':')) : storage; + Register r = currentProgram.getRegister(rn); + if (r == null) { skippedStorage++; continue; } + nv = new LocalVariableImpl(s(l, "name"), first, lt, r, currentProgram); + } + Variable added = f.addLocalVariable(nv, lsrc); + if (added != null && s(l, "comment") != null) added.setComment(s(l, "comment")); + changed = true; + } + String fc = s(o, "comment"); + if (!Objects.equals(fc, f.getComment())) { f.setComment(fc); changed = true; } + if (changed) nFuncs++; else unchanged++; + } + + // ---- data, comments, bookmarks, equates, labels ---- + private void data(JsonObject o) throws Exception { + Address a = addr(s(o, "addr")); + DataType dt = resolveType(s(o, "type")); + Data cur = currentProgram.getListing().getDefinedDataAt(a); + if (cur != null && cur.getDataType().getPathName().equals(dt.getPathName()) && cur.getLength() == in(o, "len")) { unchanged++; return; } + try { + DataUtilities.createData(currentProgram, a, dt, in(o, "len"), DataUtilities.ClearDataMode.CLEAR_ALL_CONFLICT_DATA); + nData++; + } catch (CodeUnitInsertionException e) { + failed++; println(" fail data " + s(o, "addr") + ": " + e.getMessage()); + } + } + + private void comment(JsonObject o) { + Address a = addr(s(o, "addr")); + CommentType t; + switch (s(o, "type")) { + case "eol": t = CommentType.EOL; break; + case "pre": t = CommentType.PRE; break; + case "post": t = CommentType.POST; break; + case "plate": t = CommentType.PLATE; break; + default: t = CommentType.REPEATABLE; break; + } + Listing l = currentProgram.getListing(); + if (Objects.equals(l.getComment(t, a), s(o, "text"))) { unchanged++; return; } + l.setComment(a, t, s(o, "text")); + nComments++; + } + + private void bookmark(JsonObject o) { + Address a = addr(s(o, "addr")); + BookmarkManager bm = currentProgram.getBookmarkManager(); + Bookmark cur = bm.getBookmark(a, s(o, "type"), s(o, "cat")); + if (cur != null && Objects.equals(cur.getComment(), s(o, "text"))) { unchanged++; return; } + bm.setBookmark(a, s(o, "type"), s(o, "cat"), s(o, "text")); + nBookmarks++; + } + + private void equate(JsonObject o) throws Exception { + EquateTable et = currentProgram.getEquateTable(); + String name = s(o, "name"); long value = o.get("value").getAsLong(); + Equate e = et.getEquate(name); + boolean changed = false; + if (e == null) { e = et.createEquate(name, value); changed = true; } + Set have = new HashSet<>(); + for (EquateReference r : e.getReferences()) have.add(r.getAddress().getOffset() + "|" + r.getOpIndex()); + for (JsonElement re : o.getAsJsonArray("refs")) { + JsonArray r = re.getAsJsonArray(); + Address a = addr(r.get(0).getAsString()); int op = r.get(1).getAsInt(); + if (have.contains(a.getOffset() + "|" + op)) continue; + e.addReference(a, op); changed = true; + } + if (changed) nEquates++; else unchanged++; + } + + private void label(JsonObject o) throws Exception { + Address a = addr(s(o, "addr")); + String name = s(o, "name"); + SourceType src = SourceType.valueOf(s(o, "src")); + if (src == SourceType.DEFAULT) src = SourceType.USER_DEFINED; + SymbolTable st = currentProgram.getSymbolTable(); + for (Symbol sym : st.getSymbols(a)) + if (sym.getName().equals(name)) { + if (bo(o, "primary") && !sym.isPrimary()) { sym.setPrimary(); nLabels++; } else unchanged++; + return; + } + Function f = currentProgram.getFunctionManager().getFunctionAt(a); + if (bo(o, "func") && f != null) { f.setName(name, src); nLabels++; return; } + Symbol created = st.createLabel(a, name, src); + if (bo(o, "primary")) created.setPrimary(); + nLabels++; + } +} diff --git a/tools/ghidra_scripts/ImportPsyqGdt.java b/tools/ghidra_scripts/ImportPsyqGdt.java index 2a8ed83bd..e8e8bf1e9 100644 --- a/tools/ghidra_scripts/ImportPsyqGdt.java +++ b/tools/ghidra_scripts/ImportPsyqGdt.java @@ -17,7 +17,9 @@ import java.util.Iterator; public class ImportPsyqGdt extends GhidraScript { @Override public void run() throws Exception { - String gdt = "/home/musashi/ghidra_12.1_PUBLIC/Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt"; + // P33 B5: default = THIS Ghidra installation's ghidra_psx_ldr extension (no home-directory assumption) + String gdt = new File(ghidra.framework.Application.getInstallationDirectory().getFile(false), + "Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt").getPath(); String[] a = getScriptArgs(); if (a.length > 0) gdt = a[0]; File f = new File(gdt);