mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-10-08 17:51:46 -04:00
wip(phase-33): B5 Ghidra regenerability — ExportAnnotations.java + ghidra_export_annotations.sh PROVEN (all 129 programs exported to byte-stable JSONL in 18.5 s), ghidra_annotations_delta.py (live − baseline), ImportAnnotations.java + ghidra_rebuild.sh WRITTEN but unproven (the resident rebuild hit an OSGi script-bundle load error — a compile error in the new importer breaks every script in the dir; the known bad LocalVariableImpl ctor is named in CURRENT_PHASE); Makefile print-% + GHIDRA_PROJ repo-relative; the six ghidra_*.sh repo-relative (BFM_GHIDRA_PROJ); DefineFunctions arg path; ImportPsyqGdt install-dir default; ExportSymbols R15 fix; scratch project under build/ (Ghidra refuses '.'-prefixed path components); SETUP P33 B5 section; checkpoint refreshed for the next session (S86 paused at 87% context)
This commit is contained in:
Drew T
@@ -168,7 +168,7 @@ help:
|
||||
# Phase 7 reports: deterministic, committable docs/ digests. progress/difficulty/dup_report
|
||||
# are Ghidra-free; sig-refresh regenerates dup_report's input from the saved Ghidra DB.
|
||||
GHIDRA := $(or $(GHIDRA_INSTALL_DIR),$(HOME)/ghidra_12.1_PUBLIC)
|
||||
GHIDRA_PROJ := $(HOME)/bfm-decomp/ghidra
|
||||
GHIDRA_PROJ := $(or $(BFM_GHIDRA_PROJ),$(CURDIR)/ghidra) # P33 B5: repo-relative (override: BFM_GHIDRA_PROJ)
|
||||
|
||||
# The corpus oracle (Phase 26-A, R32/R33). A SECOND, INDEPENDENT oracle: it cross-checks splat's
|
||||
# function boundaries against sig_image's, which are derived from the ORIGINAL bytes without splat.
|
||||
@@ -519,6 +519,12 @@ sig-modules:
|
||||
echo "sig-modules: signed $$n modules (of $(words $(MODULE_BINARIES)) onboarded)"
|
||||
if [ "$$bad" -ne 0 ]; then echo "[FAIL] sig-modules: $$bad module(s) failed to sign"; exit 1; fi
|
||||
|
||||
# print-<VAR> (P33 B5): echo one Makefile variable so tools derive per-binary facts from the ONE registry
|
||||
# (R33) instead of re-parsing overlays.mk/modules.mk — e.g. `make -s print-EXE BINARY=ov_SC01_077`,
|
||||
# `make -s print-VRAM_BASE BINARY=resident`, `make -s print-BINARIES`.
|
||||
print-%:
|
||||
@echo '$($*)'
|
||||
|
||||
# bootstrap (P33 B3): the fresh-clone setup — apt presence (printed, never run), the venv from
|
||||
# requirements-python.txt, the submodules, the two cc1 tarballs verified + extracted, then check-env.
|
||||
bootstrap:
|
||||
|
||||
@@ -1069,6 +1069,26 @@ fills fast). Nothing is leaking — but the host does not get the memory back on
|
||||
* `gate_main` **REFUSES a draft containing its own `INCLUDE_ASM`** (substituting it restores the stub,
|
||||
so the build passes for free and the function counts as banked), and counts banks from the SOURCE.
|
||||
|
||||
### P33 B5 (S86, 2026-09-06) — Ghidra regenerability (IN PROGRESS at the S86 pause; see `phase-ends/CURRENT_PHASE.md`)
|
||||
- **Model:** the Ghidra database embeds the program bytes, so it leaves git; what it holds beyond the splat symbol files is
|
||||
exported to text and the project is rebuilt from the disc + that text. `tools/ghidra_scripts/ExportAnnotations.java`
|
||||
(read-only; byte-stable JSONL: container facts, LOCAL types, every function signature, defined data, the 5 comment kinds,
|
||||
bookmarks, equates, labels the symbol files lack) via `tools/ghidra_export_annotations.sh [PROG…]` (no PROG = every
|
||||
program; **129 programs in 18.5 s** → `.run/ghidra_export/`). The committed file `config/ghidra/<prog>.jsonl` is the
|
||||
DELTA (`tools/ghidra_annotations_delta.py live baseline out`) between the live export and a FRESH rebuild's export, so
|
||||
analysis-origin rows subtract themselves out. `tools/ghidra_rebuild.sh <prog> [--proof] [--keep]`: import (PSX loader for
|
||||
the EXEs, BinaryLoader at `make -s print-VRAM_BASE BINARY=<alias>` for the rest) → `DefineFunctions.java` from the built
|
||||
ELF → `ApplySymbols.java` → baseline export → `ImportAnnotations.java config/ghidra/<prog>.jsonl` → export → delta → `cmp`.
|
||||
Scratch project under `build/ghidra_rebuild/proj` (**Ghidra refuses a project path with a component starting with '.'**,
|
||||
so never `.run/`). `make print-<VAR> BINARY=<alias>` echoes any per-binary Makefile variable (R33).
|
||||
- **Gotcha (S86):** Ghidra compiles a `-scriptPath` directory as ONE OSGi bundle — a single script that fails to compile
|
||||
makes EVERY script in the directory fail with `Failed to get OSGi bundle containing script`. Diagnose by invoking the
|
||||
suspect script directly and reading the javac lines.
|
||||
- Also in this change: `ExportSymbols.java` R15 fix (output path arg, refuses to overwrite, refuses `config/`);
|
||||
`DefineFunctions.java` list path = arg 1; `ImportPsyqGdt.java` default gdt from the Ghidra install dir; the six
|
||||
`tools/ghidra_*.sh` are repo-relative (`BFM_GHIDRA_PROJ` overrides the project dir; `ghidra_mcp_verify.sh <addr> <name>
|
||||
[PROG]`); Makefile `GHIDRA_PROJ := $(or $(BFM_GHIDRA_PROJ),$(CURDIR)/ghidra)`.
|
||||
|
||||
### P33 B4 (S86, 2026-09-06) — `tools/fetch_psyq.sh`: the OPTIONAL PsyQ SDK objects, user-supplied and verified
|
||||
- **What it is for.** Byte-identity never needs Sony's libraries (without them main links its INCLUDE_ASM tiles — the
|
||||
fresh-clone proof in B3 built all 218 that way). `tools/fetch_psyq.sh [--disc <RTL-4.0 Track 1 .bin>] [--from <dir>]
|
||||
|
||||
+61
-10
@@ -170,24 +170,45 @@ Mid-phase rules check after every 4 completed tasks (P6). Commit banked artifact
|
||||
with one corrupted LIB → `sha256 MISMATCH for tools/psyq/lib40/LIBTAP.LIB`, rc 1, nothing built. **The run:** with
|
||||
`tools/psyq/lib40/` moved aside, `fetch_psyq.sh --disc "<our Track 1 .bin>"` extracted the 20 LIBs from the disc
|
||||
(`diff -rq` against the tracked copies: identical), verified them, rebuilt `.run/obj40/*`, `.run/obj42/*`,
|
||||
`tools/psyq/lib40_elf/`, and `sdk-dual: OK` — **36.9 s wall**, EXIT=0 (`.run/P33/b4_fetch.log`). SETUP: P33 B4 (R21).
|
||||
`tools/psyq/lib40_elf/`, and `sdk-dual: OK` — **36.9 s wall**, EXIT=0 (`.run/P33/b4_fetch.log`). SETUP: P33 B4 (R21). Commit
|
||||
`commit:4019`.
|
||||
- **2026-09-06 (S86) — B5 IN PROGRESS (checkpointed at 87% context; WIP commit).** Done and proven: MCP stopped cleanly;
|
||||
`tools/ghidra_scripts/ExportAnnotations.java` + `tools/ghidra_export_annotations.sh` — byte-stable JSONL export of a
|
||||
program's container facts, LOCAL types, every function signature (params/locals/storage/sources), defined data, the 5
|
||||
comment kinds, bookmarks, equates, and labels the symbol files do not carry; **all 129 programs exported in 18.5 s** →
|
||||
`.run/ghidra_export/<prog>.jsonl` (139 MB; resident 2,573 rows in 5.7 s). `tools/ghidra_annotations_delta.py` (live −
|
||||
baseline; container rows always kept; `--census`). Makefile `print-%` helper + `GHIDRA_PROJ` repo-relative; the six
|
||||
`tools/ghidra_*.sh` wrappers repo-relative (`BFM_GHIDRA_PROJ` override; `ghidra_mcp_verify.sh` takes the program as arg 3
|
||||
/ `BFM_GHIDRA_PROG`); `DefineFunctions.java` takes the list path as arg 1; `ImportPsyqGdt.java` finds the gdt under
|
||||
`Application.getInstallationDirectory()`; `ExportSymbols.java` R15 fix (output path arg, refuses to overwrite, refuses
|
||||
`config/`). Written, NOT yet proven: `tools/ghidra_scripts/ImportAnnotations.java`, `tools/ghidra_rebuild.sh` (its import
|
||||
step works: the scratch project must live under `build/ghidra_rebuild/proj` — Ghidra refuses a path component starting
|
||||
with '.'; `resident_funcs.txt` from the ELF = 1,304 entries). **Blocker found:** in the resident rebuild the runs after the
|
||||
import failed with `Failed to get OSGi bundle containing script: …/tools/ghidra_scripts/ApplySymbols.java` (same for
|
||||
ExportAnnotations) — Ghidra compiles the script DIRECTORY as one bundle, so ONE file that does not compile breaks every
|
||||
script in it; `ExportAnnotations` worked on 129 programs BEFORE `ImportAnnotations.java` existed ⇒ the new file almost
|
||||
certainly has a compile error (one known: `new LocalVariableImpl(name, first, dt, stackOffset, program)` — no such ctor;
|
||||
the stack form is `(String, DataType, int, Program)` without first-use, or `(String, int, DataType, VariableStorage/Address/
|
||||
Register, Program)`). Census of the live exports (heuristic, before baselines): `types=0` in EVERY program (no hand-authored
|
||||
structs in the DB at all); the 154/402 "USER_DEFINED" variables per program are the loader's GTEMAC functions
|
||||
(analysis-origin, will subtract); real user labels beyond the symbol files: aug31 382, sep8 15, SLUS_007.26 1, overlays 1–2.
|
||||
|
||||
## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1–B4 ✓ (8 tasks; P6 rules check done after B4); NEXT = B5 Ghidra regenerability (Max, MCP stopped) (2026-09-06, written by session bd19e14a "S86"; SUPERSEDES the earlier blocks)
|
||||
## 🛑 SESSION CHECKPOINT — A1–A4 ✓, B1–B4 ✓, B5 IN PROGRESS (exporter proven; importer + rebuild proof pending on one compile error); NEXT = finish B5 (2026-09-06 ~19:35 MDT, written by session bd19e14a "S86" at Drew's 87%-context pause; SUPERSEDES the earlier blocks)
|
||||
|
||||
### 0. How to use this block
|
||||
You are a FRESH SESSION that has read `PROJECT_CONTEXT.md`, `phase-ends/DIGEST.md`, `PhaseEnd_Phase30/31/32.md` and this file,
|
||||
and nothing else (R64). Replay this block verbatim, state phase / done / NEXT / effort, list the rules from the digest
|
||||
(R1–R73), then WAIT for Drew. **NEXT = B5** (Max; the Ghidra text export + rebuild proof — the plan's Block B5 and §3 step 1
|
||||
below; the headless MCP server holds the project lock: `tools/ghidra_mcp_stop.sh` FIRST, then every headless step; the
|
||||
SessionStart hook restarts the server next session and R29 applies). If `git log -1 --format=%s` does not start with
|
||||
`feat(phase-33): B4`, B4's commit did not land: `tools/fetch_psyq.sh --no-build` must print `verified …` for every piece and
|
||||
exit 0 before committing (`.run/P33/b4_fetch.log` is the full run).
|
||||
(R1–R73), then WAIT for Drew. **NEXT = finish B5** (Max). The WIP commit `wip(phase-33): B5 …` holds every B5 file (see
|
||||
the Log's B5 entry for what is proven and what is not). The SessionStart hook will have RESTARTED the headless MCP server
|
||||
(it serves `SLUS_007.26` and holds the project lock): run `tools/ghidra_mcp_stop.sh` before ANY headless step (R23), and
|
||||
prompt Drew for `/mcp` only if you actually need the MCP tools (R29) — B5 needs none. §3 step 1 is the exact resume path.
|
||||
|
||||
### 1. Where we are
|
||||
**Phase 33 — 100% verification + the public flip + Gen2 exit.** Gate 1 approved 2026-09-06 (plan mode, Max). R65–R73 ratified at
|
||||
gate 1 (DIGEST §3). **Done: A1 (`commit:4012`), A2 (`commit:4013` — the corrected main denominator 45,150 is now the published
|
||||
number; `.run/sig.main.jsonl` is the sig main is weighed by; `tools/main_seed_ends.py` is new), A3 (the `NO_SDK` knob +
|
||||
`make sdk-dual`, proven 28 s: both legs `143dbb89…`).** The approved plan is VERBATIM
|
||||
number; `.run/sig.main.jsonl` is the sig main is weighed by; `tools/main_seed_ends.py` is new), A3 (`commit:4014`, the
|
||||
`NO_SDK` knob + `make sdk-dual`, proven 28 s: both legs `143dbb89…`), A4 (`commit:4015`), B1 (`commit:4016`), B2 (`commit:4017`),
|
||||
B3 (`commit:4018`), B4 (`commit:4019`); B5 in progress (the WIP commit).** The approved plan is VERBATIM
|
||||
at the end of this file — read its Blocks A–G for every task's files, commands and verification; the "Execution order and
|
||||
why" section is the sequence; the "Honest scale" table is the budget (≈17–21 sessions; the flip precedes the writing so later
|
||||
items can be cut). Harness tasks: #1 A1 done, #2 A2 done, #3 A3 next … #41 G2.
|
||||
@@ -250,7 +271,31 @@ items can be cut). Harness tasks: #1 A1 done, #2 A2 done, #3 A3 next … #41 G2.
|
||||
### 3. NEXT — in order
|
||||
0. **Preflight:** `git status --short | grep -v ghidra/` (empty) · `git log -1 --format='%h %s'` · `df -h ~` (≈13 GB free) ·
|
||||
`.venv/bin/python -c 'import splat'` · `ls build/us/SLUS_007.26.map` (main is built; `make check BINARY=main` if not).
|
||||
1. **B5** (Max) — facts gathered (S86): the existing scripts and their conventions are in `tools/ghidra_scripts/`
|
||||
1. **B5 — RESUME HERE** (Max). (a) `tools/ghidra_mcp_stop.sh`. (b) Find the compile error: `mv tools/ghidra_scripts/
|
||||
ImportAnnotations.java /tmp`-style moves are FORBIDDEN (R12) — instead run `~/ghidra_12.1_PUBLIC/support/analyzeHeadless
|
||||
build/ghidra_rebuild/proj bfm -process resident -noanalysis -scriptPath tools/ghidra_scripts -postScript
|
||||
ImportAnnotations.java /dev/null 2>&1 | grep -iE 'error|cannot find|symbol' | head` (the scratch project from the last
|
||||
run still exists) and read the javac messages; fix `ImportAnnotations.java` (known: the stack-local ctor — use
|
||||
`new LocalVariableImpl(name, dt, off, currentProgram)` (no first-use) or `new LocalVariableImpl(name, first, dt, new
|
||||
VariableStorage(currentProgram, off, size), currentProgram)` if that ctor exists — check with `javap -cp ~/ghidra_12.1_PUBLIC/
|
||||
Ghidra/Framework/SoftwareModeling/lib/SoftwareModeling.jar ghidra.program.model.listing.LocalVariableImpl`); re-run until
|
||||
the bundle loads (a bundle failure names EVERY script in the dir — one bad file breaks all). (c) `tools/ghidra_rebuild.sh
|
||||
resident --keep` → candidate delta `.run/ghidra_rebuild/resident.candidate.jsonl` (census printed): inspect what remains
|
||||
after the baseline subtraction (expect: few rows — the DB holds no hand-authored types; check the comment/bookmark/label
|
||||
residue is real RE, not analysis noise; if analysis noise remains, it means auto-analysis is NOT deterministic between the
|
||||
live import (Phase 10) and today's — then filter those row kinds explicitly and record why). (d) `mkdir -p config/ghidra;
|
||||
cp .run/ghidra_rebuild/resident.candidate.jsonl config/ghidra/resident.jsonl; tools/ghidra_rebuild.sh resident --proof` →
|
||||
must print `PROOF PASS`. (e) Negative control (R39): copy the config, change one field name in the copy, point a proof at it
|
||||
(temporarily swap the file) → must FAIL; restore. (f) The same for `SLUS_007.26` (main; import + analysis ≈ 3 min); then
|
||||
`ov_SC01_077`, `ov_SC06_018`, `sep8_SLUS_007.26`, `aug31_USA_DEMO.EXE` (the protos need `extracted/proto/*`, present).
|
||||
(g) `config/ghidra/ROSTER.md` (generated by a tiny script or by hand: program, kind, payload, vram, rows kept, proof
|
||||
date) — the roster = the 6 formerly-tracked programs; the other 123 regenerate on demand via `prefetch_fleet.py`.
|
||||
(h) `.claude/settings.json` hooks → `"command": "bash \"$CLAUDE_PROJECT_DIR\"/tools/ghidra_mcp_start.sh"` (and stop);
|
||||
`tools/ghidra_mcp_start.sh` exits 0 silently when `$GHIDRA/support/analyzeHeadless` is absent (a contributor's session
|
||||
must not try to launch Ghidra). (i) `docs/SETUP.md` P33 B5 section (R21): every new tool + the OSGi-bundle gotcha + the
|
||||
'.'-path gotcha + the delta model; `docs/second-oracle.md`/§2.8 pointers as needed. (j) Log entry, tick B5, refresh this
|
||||
block, commit `feat(phase-33): B5 …`. Then **B6**.
|
||||
Facts gathered (S86): the existing scripts and their conventions are in `tools/ghidra_scripts/`
|
||||
(`ExportSymbols.java` symbols-only, hardcoded output + blind overwrite — the R15 violation to fix; `ApplySymbols.java` the
|
||||
proven text→DB mirror with `BFMAPPLY` counts; `DumpFunctionSignatures.java` has the `jsonEsc` helper + JSONL style to reuse;
|
||||
`DefineFunctions.java` reads `~/bfm-decomp/.run/<prog>_funcs.txt` (hardcoded `user.home`); `ImportPsyqGdt.java` hardcodes
|
||||
@@ -279,6 +324,12 @@ section), regenerated `docs/progress.fleet.md`, `docs/duplicates.md`, `docs/dupl
|
||||
`.run/backlog.jsonl`. A3: `Makefile` (`NO_SDK`, `sdk-dual`, tools-health wiring, `.PHONY`), `docs/SETUP.md` (P33 A3 section).
|
||||
A4: `tools/family_hseq.py` (`binaries` + `open_instances` in the json; `load()` returns a pair), `tools/audit_binaries.py`
|
||||
(CHECK 4 reads the scanned set), `docs/family-hseq.md` (regenerated, 0 families), `docs/SETUP.md` (P33 A4 section).
|
||||
B1–B4: see their Log entries. B5 (WIP): `tools/ghidra_scripts/{ExportAnnotations,ImportAnnotations}.java` (new),
|
||||
`tools/ghidra_export_annotations.sh`, `tools/ghidra_annotations_delta.py`, `tools/ghidra_rebuild.sh` (new),
|
||||
`tools/ghidra_scripts/{DefineFunctions,ImportPsyqGdt,ExportSymbols}.java`, the six `tools/ghidra_*.sh`, `Makefile`
|
||||
(`print-%`, `GHIDRA_PROJ`). Scratch: `.run/ghidra_export/` (129 exports), `.run/ghidra_rebuild/`, `build/ghidra_rebuild/`.
|
||||
Environment at the pause: the MCP server is STOPPED (saved cleanly, "Save succeeded"); disk ≈ 13 GB free; the SDK object dirs
|
||||
are rebuilt and the tree's main build is in the WITH-SDK state; `extracted/proto/` regenerated.
|
||||
Scratch: `.run/public_audit/` (the history inventory), `.run/P33/a2_report.log`, `.run/P33/a3_sdk_dual.log`,
|
||||
`.run/P33/verify/main_{with_sdk,no_sdk}.map` (to be allowlisted by A5).
|
||||
The plan file: `~/.claude/plans/max-effort-set-plan-twinkling-moonbeam.md` (copied below).
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""ghidra_annotations_delta.py — the HAND-AUTHORED part of a Ghidra program, as a byte-stable file (P33 B5).
|
||||
|
||||
ghidra_annotations_delta.py <live.jsonl> <baseline.jsonl> <out.jsonl> [--census]
|
||||
|
||||
`live.jsonl` is ExportAnnotations.java's dump of the program as it is; `baseline.jsonl` is the same
|
||||
dump of a FRESH rebuild of that program (import + auto-analysis + DefineFunctions + ApplySymbols, no
|
||||
annotations imported). Every row auto-analysis produces on its own appears in both and subtracts
|
||||
itself out; what remains — types, retyped/renamed signatures and locals, typed data, comments,
|
||||
bookmarks, equates, labels that no symbol file carries — is the RE work worth committing:
|
||||
config/ghidra/<program>.jsonl. The container rows (`program`, `block`, `archive`) are always kept
|
||||
from `live` so an image-base or memory-map drift shows as a diff rather than a silent skip.
|
||||
|
||||
The proof (tools/ghidra_rebuild.sh --proof) rebuilds, imports config/ghidra/<program>.jsonl, exports,
|
||||
takes the delta against the SAME rebuild's pre-import baseline, and `cmp`s it against the committed file.
|
||||
|
||||
`--census` prints per-kind counts of the delta (the roster decision: a program whose delta holds no
|
||||
hand-authored rows need not be kept regenerable — it regenerates on demand).
|
||||
"""
|
||||
import collections
|
||||
import json
|
||||
import sys
|
||||
|
||||
KEEP_ALWAYS = ("program", "block", "archive")
|
||||
|
||||
|
||||
def rows(path):
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return [ln.rstrip("\n") for ln in f if ln.strip()]
|
||||
|
||||
|
||||
def kind(line):
|
||||
return json.loads(line)["k"]
|
||||
|
||||
|
||||
def main(argv):
|
||||
census = "--census" in argv
|
||||
argv = [a for a in argv if a != "--census"]
|
||||
if len(argv) != 4:
|
||||
sys.exit(__doc__)
|
||||
live, base, out = rows(argv[1]), set(rows(argv[2])), argv[3]
|
||||
kept = [ln for ln in live if kind(ln) in KEEP_ALWAYS or ln not in base]
|
||||
with open(out, "w", encoding="utf-8", newline="\n") as f:
|
||||
for ln in kept:
|
||||
f.write(ln + "\n")
|
||||
c = collections.Counter(kind(ln) for ln in kept)
|
||||
hand = sum(v for k, v in c.items() if k not in KEEP_ALWAYS)
|
||||
print(f"delta: {len(live)} live - {len(base)} baseline -> {len(kept)} rows ({hand} hand-authored) -> {out}",
|
||||
file=sys.stderr)
|
||||
if census:
|
||||
for k in sorted(c):
|
||||
print(f" {k:9} {c[k]}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -4,15 +4,16 @@
|
||||
# the write open needs the project lock. Verify afterwards with tools/ghidra_mcp_verify.sh <addr> <name> (R9).
|
||||
# tools/ghidra_apply_symbols.sh [PROG] [symbols files...] (defaults: SLUS_007.26 config/symbols.us.txt)
|
||||
set -uo pipefail
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption
|
||||
GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}"
|
||||
PROJ_DIR="$HOME/bfm-decomp/ghidra"; PROJ="bfm"
|
||||
PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}"; PROJ="bfm"
|
||||
PROG="${1:-SLUS_007.26}"; shift || true
|
||||
SYMS=("$@"); [ ${#SYMS[@]} -eq 0 ] && SYMS=("$HOME/bfm-decomp/config/symbols.us.txt")
|
||||
SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts"
|
||||
SYMS=("$@"); [ ${#SYMS[@]} -eq 0 ] && SYMS=("$REPO/config/symbols.us.txt")
|
||||
SCRIPTS="$REPO/tools/ghidra_scripts"
|
||||
if ss -tln 2>/dev/null | grep -qE ":8080([^0-9]|$)"; then
|
||||
echo "apply-symbols: ERROR — MCP server serving on :8080. Run tools/ghidra_mcp_stop.sh first."; exit 2
|
||||
fi
|
||||
cd "$HOME/bfm-decomp"
|
||||
cd "$REPO"
|
||||
out=$(timeout 600 "$GHIDRA/support/analyzeHeadless" "$PROJ_DIR" "$PROJ" \
|
||||
-process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript ApplySymbols.java "${SYMS[@]}" 2>&1)
|
||||
rc=$?
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
# tools/ghidra_export_annotations.sh [--proj DIR] [--out DIR] [PROG ...]
|
||||
#
|
||||
# Read-only export of a Ghidra program's annotations (types, signatures, data, comments, bookmarks,
|
||||
# equates, labels) to byte-stable JSON-Lines via tools/ghidra_scripts/ExportAnnotations.java (P33 B5).
|
||||
# With no PROG every program in the project is exported (headless `-process` with no name).
|
||||
# Output: <out>/<PROG>.jsonl (default .run/ghidra_export/). The splat symbol files are passed so the
|
||||
# labels they already carry are skipped (ApplySymbols.java restores those on a rebuild).
|
||||
#
|
||||
# PRECONDITION: the MCP server is STOPPED (tools/ghidra_mcp_stop.sh) — even a read-only open needs
|
||||
# the project lock free. Never writes to the project (-readOnly).
|
||||
set -uo pipefail
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}"
|
||||
PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}"; PROJ="bfm"
|
||||
OUT="$REPO/.run/ghidra_export"
|
||||
SCRIPTS="$REPO/tools/ghidra_scripts"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--proj) PROJ_DIR="$2"; shift 2 ;;
|
||||
--out) OUT="$2"; shift 2 ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
if ss -tln 2>/dev/null | grep -qE ":8080([^0-9]|$)"; then
|
||||
echo "export-annotations: ERROR — MCP server serving on :8080. Run tools/ghidra_mcp_stop.sh first." >&2; exit 2
|
||||
fi
|
||||
[ -f "$PROJ_DIR/$PROJ.gpr" ] || { echo "export-annotations: ERROR — no project at $PROJ_DIR/$PROJ.gpr" >&2; exit 2; }
|
||||
mkdir -p "$OUT"
|
||||
cd "$REPO"
|
||||
SYMS=(config/symbols.us.txt config/symbols.us.ram.txt config/symbols.resident.txt)
|
||||
for f in config/symbols.ov_*.txt config/symbols.md_*.txt config/symbols.proto-*.txt; do [ -f "$f" ] && SYMS+=("$f"); done
|
||||
if [ $# -eq 0 ]; then
|
||||
procs=(-process) # every program in the project folder
|
||||
else
|
||||
procs=()
|
||||
for p in "$@"; do procs+=(-process "$p"); done
|
||||
fi
|
||||
rc=0
|
||||
if [ $# -eq 0 ]; then
|
||||
timeout 7200 "$GHIDRA/support/analyzeHeadless" "$PROJ_DIR" "$PROJ" -process -noanalysis -readOnly \
|
||||
-scriptPath "$SCRIPTS" -postScript ExportAnnotations.java "$OUT" "${SYMS[@]}" 2>&1 \
|
||||
| grep -E 'BFMEXPORT|ERROR|Exception' ; rc=${PIPESTATUS[0]}
|
||||
else
|
||||
for p in "$@"; do
|
||||
timeout 1800 "$GHIDRA/support/analyzeHeadless" "$PROJ_DIR" "$PROJ" -process "$p" -noanalysis -readOnly \
|
||||
-scriptPath "$SCRIPTS" -postScript ExportAnnotations.java "$OUT" "${SYMS[@]}" 2>&1 \
|
||||
| grep -E 'BFMEXPORT|ERROR|Exception' ; r=${PIPESTATUS[0]}; [ "$r" -eq 0 ] || rc=$r
|
||||
done
|
||||
fi
|
||||
echo "export-annotations: analyzeHeadless exit=$rc -> $OUT/"
|
||||
exit $rc
|
||||
@@ -14,12 +14,13 @@
|
||||
# Run tools/ghidra_mcp_stop.sh first. Used in Phase 3.5 to add the two prototypes
|
||||
# (sep8_SLUS_007.26, aug31_USA_DEMO.EXE) alongside retail; reusable for any PS-X EXE.
|
||||
set -uo pipefail
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption
|
||||
|
||||
EXE="${1:?usage: ghidra_import.sh <path-to-ps-x-exe>}"
|
||||
GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}"
|
||||
PROJ_DIR="$HOME/bfm-decomp/ghidra"
|
||||
PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}"
|
||||
PROJ="bfm"
|
||||
SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts"
|
||||
SCRIPTS="$REPO/tools/ghidra_scripts"
|
||||
GDT="${PSYQ_GDT:-$GHIDRA/Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt}"
|
||||
PORT="8080"
|
||||
|
||||
|
||||
@@ -18,18 +18,19 @@
|
||||
# PRECONDITION: the MCP server must be STOPPED (exclusive project lock) — run
|
||||
# tools/ghidra_mcp_stop.sh first. analyzeHeadless commits + saves on a clean exit.
|
||||
set -uo pipefail
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption
|
||||
|
||||
BLOB="${1:?usage: ghidra_import_raw.sh <blob-path> <base-vram> <program-name>}"
|
||||
BASE="${2:?usage: ghidra_import_raw.sh <blob-path> <base-vram> <program-name>}"
|
||||
NAME="${3:?usage: ghidra_import_raw.sh <blob-path> <base-vram> <program-name>}"
|
||||
GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}"
|
||||
PROJ_DIR="$HOME/bfm-decomp/ghidra"
|
||||
PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}"
|
||||
PROJ="bfm"
|
||||
SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts"
|
||||
SCRIPTS="$REPO/tools/ghidra_scripts"
|
||||
GDT="${PSYQ_GDT:-$GHIDRA/Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt}"
|
||||
LANG_ID="PSX:LE:32:default"
|
||||
PORT="8080"
|
||||
STAGE="$HOME/bfm-decomp/.run/$NAME" # import filename -> program name
|
||||
STAGE="$REPO/.run/$NAME" # import filename -> program name
|
||||
|
||||
[ -f "$BLOB" ] || { echo "import-raw: ERROR — blob not found: $BLOB" >&2; exit 2; }
|
||||
[ -f "$GDT" ] || { echo "import-raw: ERROR — PsyQ gdt not found: $GDT" >&2; exit 2; }
|
||||
|
||||
@@ -19,15 +19,16 @@
|
||||
# open transaction + saves on close). Save+stop with tools/ghidra_mcp_stop.sh; resume by
|
||||
# re-running this script. There is no mid-session save (GhidrAssist holds an open txn).
|
||||
set -uo pipefail
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption
|
||||
|
||||
GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}"
|
||||
PROJ_DIR="$HOME/bfm-decomp/ghidra"
|
||||
PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}"
|
||||
PROJ="bfm"
|
||||
PROG="${1:-SLUS_007.26}" # program to serve; default retail, pass a proto name to override
|
||||
PORT="8080"
|
||||
RUNDIR="$HOME/bfm-decomp/.run"; mkdir -p "$RUNDIR" # project-local runtime scratch (never /tmp)
|
||||
RUNDIR="$REPO/.run"; mkdir -p "$RUNDIR" # project-local runtime scratch (never /tmp)
|
||||
LOG="$RUNDIR/ghidra-mcp.log"
|
||||
SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts"
|
||||
SCRIPTS="$REPO/tools/ghidra_scripts"
|
||||
STOPREQ="${BFM_MCP_STOPREQ:-$RUNDIR/mcp-stop.req}"
|
||||
|
||||
# 1) Already serving? no-op (idempotent across sessions).
|
||||
|
||||
@@ -4,9 +4,10 @@
|
||||
# analyzeHeadless COMMITS the pending transaction and saves+closes the project (releasing
|
||||
# the .rep lock). The next session resumes from the saved state. Use at task/phase boundaries.
|
||||
set -uo pipefail
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption
|
||||
|
||||
PORT="8080"
|
||||
RUNDIR="$HOME/bfm-decomp/.run"
|
||||
RUNDIR="$REPO/.run"
|
||||
LOG="$RUNDIR/ghidra-mcp.log"
|
||||
STOPREQ="${BFM_MCP_STOPREQ:-$RUNDIR/mcp-stop.req}"
|
||||
|
||||
|
||||
@@ -9,15 +9,16 @@
|
||||
# Example (after a phase's last edit was renaming 0x80018730 to LzssDecodeSector):
|
||||
# tools/ghidra_mcp_stop.sh && tools/ghidra_mcp_verify.sh 0x80018730 LzssDecodeSector
|
||||
set -uo pipefail
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # P33 B5: repo-relative, no $HOME/bfm-decomp assumption
|
||||
|
||||
ADDR="${1:?usage: ghidra_mcp_verify.sh <addr e.g. 0x80018730> <expected-name>}"
|
||||
EXPECT="${2:?usage: ghidra_mcp_verify.sh <addr> <expected-name>}"
|
||||
|
||||
GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}"
|
||||
PROJ_DIR="$HOME/bfm-decomp/ghidra"
|
||||
PROJ_DIR="${BFM_GHIDRA_PROJ:-$REPO/ghidra}"
|
||||
PROJ="bfm"
|
||||
PROG="SLUS_007.26"
|
||||
SCRIPTS="$HOME/bfm-decomp/tools/ghidra_scripts"
|
||||
PROG="${3:-${BFM_GHIDRA_PROG:-SLUS_007.26}}" # P33 B5: 3rd arg / env selects the program
|
||||
SCRIPTS="$REPO/tools/ghidra_scripts"
|
||||
PORT="8080"
|
||||
|
||||
if ss -tln 2>/dev/null | grep -qE ":$PORT([^0-9]|$)"; then
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
# tools/ghidra_rebuild.sh <program> [--proof] [--keep]
|
||||
#
|
||||
# Rebuild ONE Ghidra program FROM TEXT + the disc, in a scratch project (.run/ghidra_rebuild/proj), and
|
||||
# prove it: the committed config/ghidra/<program>.jsonl (ExportAnnotations.java's delta format) is the
|
||||
# hand-authored RE work; everything else — the bytes, auto-analysis, the function set, the names — comes
|
||||
# from the extracted payload, Ghidra, the built ELF and the splat symbol files. P33 B5; makes the Ghidra
|
||||
# database regenerable so it can leave git (it embeds the program bytes).
|
||||
#
|
||||
# 1. import PS-X EXEs (SLUS_007.26, the two prototypes) via the PSX loader; every other program is a
|
||||
# raw blob at its vram (`make -s print-EXE / print-VRAM_BASE BINARY=<alias>` — the build's
|
||||
# own registry, R33), + auto-analysis + ImportPsyqGdt.java (the psyq400 types).
|
||||
# 2. functions DefineFunctions.java from the built ELF's text symbols (splat's validated boundaries).
|
||||
# 3. symbols ApplySymbols.java with the binary's splat symbol files (the yaml's symbol_addrs_path).
|
||||
# 4. baseline ExportAnnotations.java -> .run/ghidra_rebuild/<program>.baseline.jsonl (nothing hand-made yet)
|
||||
# 5. import ImportAnnotations.java config/ghidra/<program>.jsonl (when the file exists)
|
||||
# 6. export -> .run/ghidra_rebuild/<program>.after.jsonl; delta(after, baseline) -> <program>.delta.jsonl
|
||||
# --proof cmp <program>.delta.jsonl config/ghidra/<program>.jsonl -> "PROOF PASS" (exit 0) or FAIL (exit 1).
|
||||
# Without a committed file, step 5 is skipped and the delta of the LIVE export (.run/ghidra_export/<program>.jsonl,
|
||||
# from tools/ghidra_export_annotations.sh) against the baseline is written to <program>.candidate.jsonl — the
|
||||
# file to review and commit as config/ghidra/<program>.jsonl.
|
||||
#
|
||||
# PRECONDITION: the MCP server is STOPPED (tools/ghidra_mcp_stop.sh); the extracted payload and the built ELF
|
||||
# exist (`make check BINARY=<alias>`). The live project (ghidra/) is never touched. --keep leaves the scratch
|
||||
# project for inspection (the next run wipes it).
|
||||
set -uo pipefail
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO"
|
||||
PROG="${1:?usage: ghidra_rebuild.sh <program> [--proof] [--keep]}"; shift
|
||||
PROOF=0; KEEP=0
|
||||
for a in "$@"; do case "$a" in --proof) PROOF=1 ;; --keep) KEEP=1 ;; *) echo "rebuild: unknown arg $a" >&2; exit 2 ;; esac; done
|
||||
GHIDRA="${GHIDRA_INSTALL_DIR:-$HOME/ghidra_12.1_PUBLIC}"
|
||||
HEADLESS="$GHIDRA/support/analyzeHeadless"
|
||||
SCRIPTS="$REPO/tools/ghidra_scripts"
|
||||
GDT="${PSYQ_GDT:-$GHIDRA/Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt}"
|
||||
SCR="$REPO/.run/ghidra_rebuild" # exports, deltas, function lists (plain files)
|
||||
# Ghidra refuses a project path with a component starting with '.', so the scratch PROJECT lives under
|
||||
# the gitignored build/ tree (wiped by `make clean`, which is the right lifetime for it).
|
||||
PROJ_DIR="$REPO/build/ghidra_rebuild/proj"; PROJ="bfm"; STAGEDIR="$REPO/build/ghidra_rebuild"
|
||||
CONF="$REPO/config/ghidra/$PROG.jsonl"
|
||||
LIVE="$REPO/.run/ghidra_export/$PROG.jsonl"
|
||||
say() { printf 'rebuild[%s]: %s\n' "$PROG" "$*"; }
|
||||
die() { printf 'rebuild[%s]: %s\n' "$PROG" "$*" >&2; exit 1; }
|
||||
[ -x "$HEADLESS" ] || die "no analyzeHeadless at $HEADLESS (GHIDRA_INSTALL_DIR)"
|
||||
[ -f "$GDT" ] || die "no psyq400.gdt at $GDT (PSYQ_GDT)"
|
||||
if ss -tln 2>/dev/null | grep -qE ":8080([^0-9]|$)"; then die "MCP server serving on :8080 — run tools/ghidra_mcp_stop.sh first"; fi
|
||||
|
||||
# ---- what is this program? ----
|
||||
case "$PROG" in
|
||||
SLUS_007.26) KIND=exe; ALIAS=main; EXE=extracted/retail/SLUS_007.26 ;;
|
||||
sep8_SLUS_007.26) KIND=exe; ALIAS=proto-sep8; EXE=extracted/proto/sep8_SLUS_007.26 ;;
|
||||
aug31_USA_DEMO.EXE) KIND=exe; ALIAS=proto-demo; EXE=extracted/proto/aug31_USA_DEMO.EXE ;;
|
||||
*) KIND=raw; ALIAS="$PROG"; EXE="$(make -s print-EXE BINARY="$ALIAS")"; VRAM="$(make -s print-VRAM_BASE BINARY="$ALIAS")" ;;
|
||||
esac
|
||||
[ -n "$EXE" ] && [ -f "$EXE" ] || die "payload not found for $PROG ($EXE) — run make disc-extract (or extract_proto_exe.py for a prototype)"
|
||||
case "$ALIAS" in
|
||||
proto-sep8) SYMS=(config/symbols.proto-sep8.txt); ELF="" ;;
|
||||
proto-demo) SYMS=(config/symbols.proto-demo.txt); ELF="" ;;
|
||||
*) ELF="$(make -s print-ELF BINARY="$ALIAS")"
|
||||
mapfile -t SYMS < <(.venv/bin/python - "$ALIAS" <<'EOF'
|
||||
import sys, yaml
|
||||
y = yaml.safe_load(open(f"config/splat.{'us.exe' if sys.argv[1]=='main' else sys.argv[1]}.yaml"))
|
||||
p = y["options"].get("symbol_addrs_path", [])
|
||||
print("\n".join(p if isinstance(p, list) else [p]))
|
||||
EOF
|
||||
) ;;
|
||||
esac
|
||||
|
||||
# ---- scratch project ----
|
||||
rm -rf "$PROJ_DIR"; mkdir -p "$PROJ_DIR" "$SCR"
|
||||
run_headless() { # args: the analyzeHeadless tail; prints the BFM* / REPORT / ERROR lines; returns its exit
|
||||
timeout 3600 "$HEADLESS" "$PROJ_DIR" "$PROJ" "$@" 2>&1 | grep -E 'BFM|REPORT:|ERROR|Exception|Analysis succeeded|Import' | grep -vE 'INFO (Analysis|REPORT: Import)' | head -40
|
||||
return "${PIPESTATUS[0]}"
|
||||
}
|
||||
|
||||
say "1. import ($KIND) $EXE"
|
||||
if [ "$KIND" = exe ]; then
|
||||
run_headless -import "$EXE" -overwrite -scriptPath "$SCRIPTS" -postScript ImportPsyqGdt.java "$GDT" -postScript DumpProgramInfo.java || die "import failed"
|
||||
else
|
||||
STAGE="$STAGEDIR/$PROG"; cp -f "$EXE" "$STAGE"
|
||||
run_headless -import "$STAGE" -overwrite -loader BinaryLoader -loader-baseAddr "$VRAM" -processor PSX:LE:32:default \
|
||||
-scriptPath "$SCRIPTS" -postScript ImportPsyqGdt.java "$GDT" -postScript DumpProgramInfo.java || die "import failed"
|
||||
rm -f "$STAGE"
|
||||
fi
|
||||
|
||||
if [ -n "$ELF" ] && [ -f "$ELF" ]; then
|
||||
say "2. functions from $ELF"
|
||||
LO="$(make -s print-TEXT_LO BINARY="$ALIAS")"; HI="$(make -s print-TEXT_HI BINARY="$ALIAS")"
|
||||
mipsel-linux-gnu-nm "$ELF" | awk -v lo="$LO" -v hi="$HI" '
|
||||
$2=="T" || $2=="t" { a=strtonum("0x" $1); if (a%4==0 && a>=strtonum(lo) && a<strtonum(hi)) printf "0x%08X\n", a }' \
|
||||
| sort -u > "$SCR/${PROG}_funcs.txt"
|
||||
run_headless -process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript DefineFunctions.java "$SCR/${PROG}_funcs.txt" || die "DefineFunctions failed"
|
||||
else
|
||||
say "2. functions: no built ELF for $ALIAS — keeping the analysis-found set"
|
||||
fi
|
||||
|
||||
say "3. symbols: ${SYMS[*]}"
|
||||
run_headless -process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript ApplySymbols.java "${SYMS[@]}" || die "ApplySymbols failed"
|
||||
|
||||
say "4. baseline export"
|
||||
run_headless -process "$PROG" -noanalysis -readOnly -scriptPath "$SCRIPTS" -postScript ExportAnnotations.java "$SCR/$PROG.baseline.jsonl" "${SYMS[@]}" || die "baseline export failed"
|
||||
|
||||
if [ ! -f "$CONF" ]; then
|
||||
[ -f "$LIVE" ] || die "no $CONF and no live export $LIVE (run tools/ghidra_export_annotations.sh $PROG first)"
|
||||
.venv/bin/python tools/ghidra_annotations_delta.py "$LIVE" "$SCR/$PROG.baseline.jsonl" "$SCR/$PROG.candidate.jsonl" --census
|
||||
say "no committed file — CANDIDATE written: $SCR/$PROG.candidate.jsonl (review, then cp to $CONF)"
|
||||
[ "$KEEP" = 1 ] || rm -rf "$PROJ_DIR"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
say "5. import $CONF"
|
||||
run_headless -process "$PROG" -noanalysis -scriptPath "$SCRIPTS" -postScript ImportAnnotations.java "$CONF" || die "ImportAnnotations failed"
|
||||
say "6. export after import + delta"
|
||||
run_headless -process "$PROG" -noanalysis -readOnly -scriptPath "$SCRIPTS" -postScript ExportAnnotations.java "$SCR/$PROG.after.jsonl" "${SYMS[@]}" || die "export failed"
|
||||
.venv/bin/python tools/ghidra_annotations_delta.py "$SCR/$PROG.after.jsonl" "$SCR/$PROG.baseline.jsonl" "$SCR/$PROG.delta.jsonl" --census
|
||||
if [ "$PROOF" = 1 ]; then
|
||||
if cmp -s "$SCR/$PROG.delta.jsonl" "$CONF"; then
|
||||
say "PROOF PASS — the rebuilt program's hand-authored delta == $CONF ($(wc -l < "$CONF") rows)"
|
||||
[ "$KEEP" = 1 ] || rm -rf "$PROJ_DIR"
|
||||
exit 0
|
||||
fi
|
||||
say "PROOF FAIL — delta differs from $CONF (diff below, first 40 lines; scratch kept in $SCR)"
|
||||
diff "$CONF" "$SCR/$PROG.delta.jsonl" | head -40
|
||||
exit 1
|
||||
fi
|
||||
[ "$KEEP" = 1 ] || rm -rf "$PROJ_DIR"
|
||||
say "done (no --proof requested)"
|
||||
@@ -18,8 +18,12 @@ import java.util.List;
|
||||
public class DefineFunctions extends GhidraScript {
|
||||
public void run() throws Exception {
|
||||
String prog = currentProgram.getName();
|
||||
Path in = Paths.get(System.getProperty("user.home"), "bfm-decomp", ".run", prog + "_funcs.txt");
|
||||
if (!Files.exists(in)) { println("DefineFunctions: no list at " + in); return; }
|
||||
// P33 B5: the list path is arg 1 (tools/ghidra_rebuild.sh passes .run/ghidra_rebuild/<prog>_funcs.txt);
|
||||
// without an arg the historical location .run/<prog>_funcs.txt under the CWD (the repo) is used —
|
||||
// no more `user.home/bfm-decomp` assumption.
|
||||
String[] args = getScriptArgs();
|
||||
Path in = (args != null && args.length > 0) ? Paths.get(args[0]) : Paths.get(".run", prog + "_funcs.txt");
|
||||
if (!Files.exists(in)) { println("DefineFunctions: no list at " + in.toAbsolutePath()); return; }
|
||||
List<String> lines = Files.readAllLines(in);
|
||||
int created = 0, existed = 0, failed = 0;
|
||||
for (String ln : lines) {
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
// ExportAnnotations.java — headless, READ-ONLY: serialize a program's ANNOTATIONS to byte-stable JSON-Lines
|
||||
// (P33 B5, the Ghidra-regenerability deliverable carried since Phase 9/10).
|
||||
//
|
||||
// WHAT: everything a human (or the MCP) can author in a program that is NOT already text in the repo —
|
||||
// data types (structs/unions/enums/typedefs/function definitions of the program's LOCAL archive; the
|
||||
// psyq400 archive types are re-imported by ImportPsyqGdt.java), function signatures (return type,
|
||||
// calling convention, parameters, locals, flags), defined data, the five comment kinds, bookmarks,
|
||||
// equates, and non-default labels that the splat symbol files do not already carry — plus the
|
||||
// container facts (language, image base, memory blocks) so an image-base drift shows as a diff.
|
||||
//
|
||||
// HOW IT IS USED: tools/ghidra_rebuild.sh exports the LIVE program and a FRESH rebuild (import +
|
||||
// analysis + DefineFunctions + ApplySymbols) and stores their DELTA (tools/ghidra_annotations_delta.py)
|
||||
// as config/ghidra/<program>.jsonl — every row analysis produces on its own subtracts itself out, so
|
||||
// what is committed is the hand-authored RE work. The proof re-imports that delta and re-exports.
|
||||
// Byte-stability is the whole contract: fixed key order, sorted rows, 0x%08x addresses, one JSON object
|
||||
// per line, no whitespace. Deterministic across machines given the same program.
|
||||
//
|
||||
// analyzeHeadless <proj_dir> bfm -process <PROG> -noanalysis -readOnly \
|
||||
// -scriptPath tools/ghidra_scripts -postScript ExportAnnotations.java <out.jsonl|out_dir> [symbols.txt ...]
|
||||
//
|
||||
// Arg 1: an output FILE, or a DIRECTORY (then <dir>/<program>.jsonl — the all-programs form, run with
|
||||
// `-process` and no program name). Args 2..: splat symbol files whose `name = 0xADDR;` rows are
|
||||
// skipped from the label export (ApplySymbols.java restores those).
|
||||
import ghidra.app.script.GhidraScript;
|
||||
import ghidra.program.model.address.Address;
|
||||
import ghidra.program.model.address.AddressIterator;
|
||||
import ghidra.program.model.data.*;
|
||||
import ghidra.program.model.listing.*;
|
||||
import ghidra.program.model.mem.MemoryBlock;
|
||||
import ghidra.program.model.symbol.*;
|
||||
import java.io.File;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class ExportAnnotations extends GhidraScript {
|
||||
|
||||
// ---- JSON helpers (the DumpFunctionSignatures.java convention: hand-serialized, canonical) ----
|
||||
private static String esc(String s) {
|
||||
if (s == null) return "null";
|
||||
StringBuilder sb = new StringBuilder("\"");
|
||||
for (int i = 0; i < s.length(); i++) {
|
||||
char c = s.charAt(i);
|
||||
if (c == '"' || c == '\\') sb.append('\\').append(c);
|
||||
else if (c == '\n') sb.append("\\n");
|
||||
else if (c == '\r') sb.append("\\r");
|
||||
else if (c == '\t') sb.append("\\t");
|
||||
else if (c < 0x20) sb.append(String.format("\\u%04x", (int) c));
|
||||
else sb.append(c);
|
||||
}
|
||||
return sb.append('"').toString();
|
||||
}
|
||||
private static String hexAddr(Address a) { return String.format("\"0x%08x\"", a.getOffset()); }
|
||||
private static String b(boolean v) { return v ? "true" : "false"; }
|
||||
|
||||
private final Map<Long, Set<String>> symfileNames = new HashMap<>(); // addr -> names the symbol files carry
|
||||
|
||||
@Override
|
||||
protected void run() throws Exception {
|
||||
String[] args = getScriptArgs();
|
||||
if (args == null || args.length < 1) { println("BFMEXPORT ERROR: usage <out.jsonl|out_dir> [symbols.txt ...]"); return; }
|
||||
Path out = Paths.get(args[0]);
|
||||
if (Files.isDirectory(out)) out = out.resolve(currentProgram.getName() + ".jsonl");
|
||||
Pattern sp = Pattern.compile("^\\s*([A-Za-z_]\\w*)\\s*=\\s*0x([0-9A-Fa-f]+)\\s*;");
|
||||
for (int i = 1; i < args.length; i++) {
|
||||
Path f = Paths.get(args[i]);
|
||||
if (!Files.isRegularFile(f)) { println("BFMEXPORT WARN: symbols file not found: " + f); continue; }
|
||||
for (String ln : Files.readAllLines(f)) {
|
||||
Matcher m = sp.matcher(ln);
|
||||
if (m.find()) symfileNames.computeIfAbsent(Long.parseLong(m.group(2), 16), k -> new HashSet<>()).add(m.group(1));
|
||||
}
|
||||
}
|
||||
|
||||
List<String> rows = new ArrayList<>();
|
||||
int nTypes = program(rows);
|
||||
int nFuncs = functions(rows);
|
||||
int nData = data(rows);
|
||||
int nComments = comments(rows);
|
||||
int nBookmarks = bookmarks(rows);
|
||||
int nEquates = equates(rows);
|
||||
int nLabels = labels(rows);
|
||||
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (String r : rows) sb.append(r).append('\n');
|
||||
Files.createDirectories(out.toAbsolutePath().getParent());
|
||||
Files.write(out, sb.toString().getBytes(StandardCharsets.UTF_8));
|
||||
println("BFMEXPORT program=" + currentProgram.getName() + " types=" + nTypes + " funcs=" + nFuncs
|
||||
+ " data=" + nData + " comments=" + nComments + " bookmarks=" + nBookmarks + " equates=" + nEquates
|
||||
+ " labels=" + nLabels + " rows=" + rows.size() + " -> " + out);
|
||||
}
|
||||
|
||||
// ---- program, blocks, archives, LOCAL data types (sorted by path) ----
|
||||
private int program(List<String> rows) {
|
||||
Program p = currentProgram;
|
||||
rows.add("{\"k\":\"program\",\"name\":" + esc(p.getName()) + ",\"lang\":" + esc(p.getLanguageID().getIdAsString())
|
||||
+ ",\"cspec\":" + esc(p.getCompilerSpec().getCompilerSpecID().getIdAsString())
|
||||
+ ",\"image_base\":" + hexAddr(p.getImageBase()) + ",\"format\":" + esc(p.getExecutableFormat()) + "}");
|
||||
List<String> blocks = new ArrayList<>();
|
||||
for (MemoryBlock mb : p.getMemory().getBlocks()) {
|
||||
String perm = (mb.isRead() ? "r" : "-") + (mb.isWrite() ? "w" : "-") + (mb.isExecute() ? "x" : "-");
|
||||
blocks.add("{\"k\":\"block\",\"name\":" + esc(mb.getName()) + ",\"start\":" + hexAddr(mb.getStart())
|
||||
+ ",\"end\":" + hexAddr(mb.getEnd()) + ",\"perm\":\"" + perm + "\",\"init\":" + b(mb.isInitialized()) + "}");
|
||||
}
|
||||
rows.addAll(blocks); // memory order as Ghidra lists them (address order)
|
||||
DataTypeManager dtm = p.getDataTypeManager();
|
||||
SourceArchive local = dtm.getLocalSourceArchive();
|
||||
TreeSet<String> archives = new TreeSet<>();
|
||||
for (SourceArchive sa : dtm.getSourceArchives()) {
|
||||
if (sa.getSourceArchiveID().equals(local.getSourceArchiveID())) continue;
|
||||
if (sa.getArchiveType() == ArchiveType.BUILT_IN) continue;
|
||||
archives.add("{\"k\":\"archive\",\"name\":" + esc(sa.getName()) + "}");
|
||||
}
|
||||
rows.addAll(archives);
|
||||
TreeMap<String, String> types = new TreeMap<>();
|
||||
Iterator<DataType> it = dtm.getAllDataTypes();
|
||||
while (it.hasNext()) {
|
||||
DataType dt = it.next();
|
||||
if (dt instanceof Pointer || dt instanceof Array || dt instanceof BuiltInDataType) continue;
|
||||
SourceArchive sa = dt.getSourceArchive();
|
||||
if (sa != null && !sa.getSourceArchiveID().equals(local.getSourceArchiveID())) continue; // psyq400 etc.
|
||||
String row = typeRow(dt);
|
||||
if (row != null) types.put(dt.getPathName(), row);
|
||||
}
|
||||
rows.addAll(types.values());
|
||||
return types.size();
|
||||
}
|
||||
|
||||
private String typeRow(DataType dt) {
|
||||
String path = esc(dt.getPathName());
|
||||
if (dt instanceof Structure) {
|
||||
Structure s = (Structure) dt;
|
||||
StringBuilder f = new StringBuilder("[");
|
||||
boolean first = true;
|
||||
for (DataTypeComponent c : s.getDefinedComponents()) {
|
||||
if (!first) f.append(','); first = false;
|
||||
f.append("{\"off\":").append(c.getOffset()).append(",\"len\":").append(c.getLength())
|
||||
.append(",\"name\":").append(esc(c.getFieldName())).append(",\"type\":").append(esc(c.getDataType().getPathName()))
|
||||
.append(",\"comment\":").append(esc(c.getComment())).append('}');
|
||||
}
|
||||
return "{\"k\":\"type\",\"kind\":\"struct\",\"path\":" + path + ",\"size\":" + s.getLength()
|
||||
+ ",\"packed\":" + b(s.isPackingEnabled()) + ",\"fields\":" + f + "]}";
|
||||
}
|
||||
if (dt instanceof Union) {
|
||||
Union u = (Union) dt;
|
||||
StringBuilder f = new StringBuilder("[");
|
||||
boolean first = true;
|
||||
for (DataTypeComponent c : u.getComponents()) {
|
||||
if (!first) f.append(','); first = false;
|
||||
f.append("{\"len\":").append(c.getLength()).append(",\"name\":").append(esc(c.getFieldName()))
|
||||
.append(",\"type\":").append(esc(c.getDataType().getPathName())).append(",\"comment\":").append(esc(c.getComment())).append('}');
|
||||
}
|
||||
return "{\"k\":\"type\",\"kind\":\"union\",\"path\":" + path + ",\"size\":" + u.getLength()
|
||||
+ ",\"packed\":" + b(u.isPackingEnabled()) + ",\"fields\":" + f + "]}";
|
||||
}
|
||||
if (dt instanceof ghidra.program.model.data.Enum) {
|
||||
ghidra.program.model.data.Enum e = (ghidra.program.model.data.Enum) dt;
|
||||
TreeMap<String, String> vals = new TreeMap<>();
|
||||
for (String n : e.getNames()) vals.put(String.format("%020d|%s", e.getValue(n), n),
|
||||
"{\"name\":" + esc(n) + ",\"value\":" + e.getValue(n) + "}");
|
||||
return "{\"k\":\"type\",\"kind\":\"enum\",\"path\":" + path + ",\"size\":" + e.getLength()
|
||||
+ ",\"values\":[" + String.join(",", vals.values()) + "]}";
|
||||
}
|
||||
if (dt instanceof TypeDef) {
|
||||
return "{\"k\":\"type\",\"kind\":\"typedef\",\"path\":" + path + ",\"base\":" + esc(((TypeDef) dt).getDataType().getPathName()) + "}";
|
||||
}
|
||||
if (dt instanceof FunctionDefinition) {
|
||||
FunctionDefinition fd = (FunctionDefinition) dt;
|
||||
StringBuilder ps = new StringBuilder("[");
|
||||
boolean first = true;
|
||||
for (ParameterDefinition pd : fd.getArguments()) {
|
||||
if (!first) ps.append(','); first = false;
|
||||
ps.append("{\"name\":").append(esc(pd.getName())).append(",\"type\":").append(esc(pd.getDataType().getPathName()))
|
||||
.append(",\"comment\":").append(esc(pd.getComment())).append('}');
|
||||
}
|
||||
return "{\"k\":\"type\",\"kind\":\"fndef\",\"path\":" + path + ",\"ret\":" + esc(fd.getReturnType().getPathName())
|
||||
+ ",\"cc\":" + esc(fd.getCallingConventionName()) + ",\"varargs\":" + b(fd.hasVarArgs()) + ",\"params\":" + ps + "]}";
|
||||
}
|
||||
return "{\"k\":\"type\",\"kind\":\"other\",\"path\":" + path + ",\"class\":" + esc(dt.getClass().getSimpleName()) + "}";
|
||||
}
|
||||
|
||||
// ---- functions (every non-external function; the delta keeps the user-touched ones) ----
|
||||
private int functions(List<String> rows) {
|
||||
TreeMap<Long, String> m = new TreeMap<>();
|
||||
for (Function f : currentProgram.getFunctionManager().getFunctions(true)) {
|
||||
if (f.isExternal()) continue;
|
||||
StringBuilder ps = new StringBuilder("[");
|
||||
boolean first = true;
|
||||
for (Parameter p : f.getParameters()) {
|
||||
if (!first) ps.append(','); first = false;
|
||||
ps.append("{\"name\":").append(esc(p.getName())).append(",\"type\":").append(esc(p.getDataType().getPathName()))
|
||||
.append(",\"storage\":").append(esc(p.getVariableStorage().toString())).append(",\"src\":").append(esc(p.getSource().name())).append('}');
|
||||
}
|
||||
ps.append(']');
|
||||
TreeMap<String, String> locals = new TreeMap<>();
|
||||
for (Variable v : f.getLocalVariables()) {
|
||||
String key = String.format("%010d|%s|%s", v.getFirstUseOffset(), v.getVariableStorage().toString(), v.getName());
|
||||
locals.put(key, "{\"name\":" + esc(v.getName()) + ",\"type\":" + esc(v.getDataType().getPathName())
|
||||
+ ",\"storage\":" + esc(v.getVariableStorage().toString()) + ",\"first\":" + v.getFirstUseOffset()
|
||||
+ ",\"src\":" + esc(v.getSource().name()) + ",\"comment\":" + esc(v.getComment()) + "}");
|
||||
}
|
||||
String row = "{\"k\":\"func\",\"addr\":" + hexAddr(f.getEntryPoint()) + ",\"name\":" + esc(f.getName())
|
||||
+ ",\"ret\":" + esc(f.getReturnType().getPathName()) + ",\"cc\":" + esc(f.getCallingConventionName())
|
||||
+ ",\"varargs\":" + b(f.hasVarArgs()) + ",\"noreturn\":" + b(f.hasNoReturn()) + ",\"inline\":" + b(f.isInline())
|
||||
+ ",\"custom\":" + b(f.hasCustomVariableStorage()) + ",\"sigsrc\":" + esc(f.getSignatureSource().name())
|
||||
+ ",\"params\":" + ps + ",\"locals\":[" + String.join(",", locals.values()) + "]"
|
||||
+ ",\"comment\":" + esc(f.getComment()) + "}";
|
||||
m.put(f.getEntryPoint().getOffset(), row);
|
||||
}
|
||||
rows.addAll(m.values());
|
||||
return m.size();
|
||||
}
|
||||
|
||||
// ---- defined data (every typed unit; the delta keeps the user-typed ones) ----
|
||||
private int data(List<String> rows) {
|
||||
TreeMap<Long, String> m = new TreeMap<>();
|
||||
DataIterator it = currentProgram.getListing().getDefinedData(true);
|
||||
while (it.hasNext()) {
|
||||
Data d = it.next();
|
||||
DataType dt = d.getDataType();
|
||||
if (dt instanceof DefaultDataType || dt instanceof Undefined) continue;
|
||||
m.put(d.getAddress().getOffset(), "{\"k\":\"data\",\"addr\":" + hexAddr(d.getAddress()) + ",\"type\":" + esc(dt.getPathName())
|
||||
+ ",\"len\":" + d.getLength() + "}");
|
||||
}
|
||||
rows.addAll(m.values());
|
||||
return m.size();
|
||||
}
|
||||
|
||||
// ---- comments, five kinds ----
|
||||
private int comments(List<String> rows) {
|
||||
TreeMap<String, String> m = new TreeMap<>();
|
||||
Listing l = currentProgram.getListing();
|
||||
CommentType[] kinds = {CommentType.EOL, CommentType.PRE, CommentType.POST, CommentType.PLATE, CommentType.REPEATABLE};
|
||||
String[] names = {"eol", "pre", "post", "plate", "repeat"};
|
||||
for (int i = 0; i < kinds.length; i++) {
|
||||
AddressIterator ai = l.getCommentAddressIterator(kinds[i], currentProgram.getMemory(), true);
|
||||
while (ai.hasNext()) {
|
||||
Address a = ai.next();
|
||||
String c = l.getComment(kinds[i], a);
|
||||
if (c == null) continue;
|
||||
m.put(String.format("%016x|%d", a.getOffset(), i), "{\"k\":\"comment\",\"addr\":" + hexAddr(a) + ",\"type\":\"" + names[i] + "\",\"text\":" + esc(c) + "}");
|
||||
}
|
||||
}
|
||||
rows.addAll(m.values());
|
||||
return m.size();
|
||||
}
|
||||
|
||||
private int bookmarks(List<String> rows) {
|
||||
TreeMap<String, String> m = new TreeMap<>();
|
||||
Iterator<Bookmark> it = currentProgram.getBookmarkManager().getBookmarksIterator();
|
||||
while (it.hasNext()) {
|
||||
Bookmark bk = it.next();
|
||||
String key = String.format("%016x|%s|%s|%s", bk.getAddress().getOffset(), bk.getTypeString(), bk.getCategory(), bk.getComment());
|
||||
m.put(key, "{\"k\":\"bookmark\",\"addr\":" + hexAddr(bk.getAddress()) + ",\"type\":" + esc(bk.getTypeString())
|
||||
+ ",\"cat\":" + esc(bk.getCategory()) + ",\"text\":" + esc(bk.getComment()) + "}");
|
||||
}
|
||||
rows.addAll(m.values());
|
||||
return m.size();
|
||||
}
|
||||
|
||||
private int equates(List<String> rows) {
|
||||
TreeMap<String, String> m = new TreeMap<>();
|
||||
Iterator<Equate> it = currentProgram.getEquateTable().getEquates();
|
||||
while (it.hasNext()) {
|
||||
Equate e = it.next();
|
||||
TreeSet<String> refs = new TreeSet<>();
|
||||
for (EquateReference r : e.getReferences())
|
||||
refs.add("[" + hexAddr(r.getAddress()) + "," + r.getOpIndex() + "]");
|
||||
m.put(e.getName(), "{\"k\":\"equate\",\"name\":" + esc(e.getName()) + ",\"value\":" + e.getValue()
|
||||
+ ",\"refs\":[" + String.join(",", refs) + "]}");
|
||||
}
|
||||
rows.addAll(m.values());
|
||||
return m.size();
|
||||
}
|
||||
|
||||
// ---- labels: every non-DEFAULT, non-external symbol the symbol files do not already carry ----
|
||||
private int labels(List<String> rows) {
|
||||
TreeMap<String, String> m = new TreeMap<>();
|
||||
for (Symbol s : currentProgram.getSymbolTable().getAllSymbols(true)) {
|
||||
if (s.isExternal() || s.getSource() == SourceType.DEFAULT) continue;
|
||||
if (!currentProgram.getMemory().contains(s.getAddress())) continue;
|
||||
Set<String> have = symfileNames.get(s.getAddress().getOffset());
|
||||
if (have != null && have.contains(s.getName())) continue;
|
||||
boolean isFunc = s.getSymbolType() == SymbolType.FUNCTION;
|
||||
m.put(String.format("%016x|%s", s.getAddress().getOffset(), s.getName()),
|
||||
"{\"k\":\"label\",\"addr\":" + hexAddr(s.getAddress()) + ",\"name\":" + esc(s.getName()) + ",\"primary\":" + b(s.isPrimary())
|
||||
+ ",\"src\":" + esc(s.getSource().name()) + ",\"func\":" + b(isFunc) + "}");
|
||||
}
|
||||
rows.addAll(m.values());
|
||||
return m.size();
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,22 @@ public class ExportSymbols extends GhidraScript {
|
||||
sb.append("// Durable text record of manual RE work (the .rep is gitignored). Phase-5 splat refines this.\n");
|
||||
sb.append("// count=").append(n).append("\n\n");
|
||||
for (String v : rows.values()) sb.append(v).append("\n");
|
||||
java.nio.file.Path out = Paths.get(System.getProperty("user.home"), "bfm-decomp", "config", "symbols.us.txt");
|
||||
// P33 B5 (rule R15): NEVER overwrite the curated config/symbols.us.txt — re-exporting is a reviewed
|
||||
// re-merge. Arg 1 = output path (default .run/ghidra_export/<program>.symbols.txt under the CWD);
|
||||
// an existing output is refused unless arg 2 is the literal `overwrite`.
|
||||
String[] args = getScriptArgs();
|
||||
java.nio.file.Path out = (args != null && args.length > 0) ? Paths.get(args[0])
|
||||
: Paths.get(".run", "ghidra_export", currentProgram.getName() + ".symbols.txt");
|
||||
boolean overwrite = args != null && args.length > 1 && "overwrite".equals(args[1]);
|
||||
if (Files.exists(out) && !overwrite) {
|
||||
println("BFMSYM REFUSED: " + out + " exists (pass `overwrite` as the 2nd arg; config/symbols*.txt is merged by hand, R15)");
|
||||
return;
|
||||
}
|
||||
if (out.toAbsolutePath().normalize().toString().contains("/config/")) {
|
||||
println("BFMSYM REFUSED: will not write into config/ (R15 — merge the export by hand): " + out);
|
||||
return;
|
||||
}
|
||||
Files.createDirectories(out.toAbsolutePath().getParent());
|
||||
Files.write(out, sb.toString().getBytes());
|
||||
println("BFMSYM exported " + n + " USER_DEFINED symbols -> " + out);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
// ImportAnnotations.java — headless: apply an annotations JSON-Lines file (ExportAnnotations.java's format,
|
||||
// normally the committed config/ghidra/<program>.jsonl) to the current program. The inverse of the export;
|
||||
// idempotent (compare-before-write everywhere); run WITHOUT -readOnly so analyzeHeadless commits + saves
|
||||
// on exit (the ApplySymbols.java precedent). P33 B5.
|
||||
//
|
||||
// analyzeHeadless <proj_dir> bfm -process <PROG> -noanalysis \
|
||||
// -scriptPath tools/ghidra_scripts -postScript ImportAnnotations.java <in.jsonl>
|
||||
//
|
||||
// Refuses (R43) a file whose `program` row names another language / image base, and any row kind it does
|
||||
// not know — nothing is applied in that case. Prints one BFMANN summary line with every counter.
|
||||
// Types are applied in two passes (shells first, then fields) so self- and mutually-referencing structs
|
||||
// resolve; type strings are Ghidra path names with pointer/array suffixes (`/BFM/Actor *[4]`).
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import ghidra.app.script.GhidraScript;
|
||||
import ghidra.program.model.address.Address;
|
||||
import ghidra.program.model.data.*;
|
||||
import ghidra.program.model.lang.Register;
|
||||
import ghidra.program.model.listing.*;
|
||||
import ghidra.program.model.symbol.*;
|
||||
import ghidra.program.model.util.CodeUnitInsertionException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
|
||||
public class ImportAnnotations extends GhidraScript {
|
||||
private static final Set<String> KINDS = new HashSet<>(Arrays.asList(
|
||||
"program", "block", "archive", "type", "func", "data", "comment", "bookmark", "equate", "label"));
|
||||
private int nTypes, nFuncs, nData, nComments, nBookmarks, nEquates, nLabels, unchanged, failed, skippedStorage;
|
||||
private DataTypeManager dtm;
|
||||
|
||||
private static String s(JsonObject o, String k) { JsonElement e = o.get(k); return (e == null || e.isJsonNull()) ? null : e.getAsString(); }
|
||||
private static boolean bo(JsonObject o, String k) { JsonElement e = o.get(k); return e != null && !e.isJsonNull() && e.getAsBoolean(); }
|
||||
private static int in(JsonObject o, String k) { JsonElement e = o.get(k); return (e == null || e.isJsonNull()) ? 0 : e.getAsInt(); }
|
||||
private Address addr(String hex) { return currentProgram.getAddressFactory().getDefaultAddressSpace().getAddress(Long.decode(hex)); }
|
||||
|
||||
@Override
|
||||
protected void run() throws Exception {
|
||||
String[] args = getScriptArgs();
|
||||
if (args == null || args.length < 1) { println("BFMANN ERROR: usage <in.jsonl>"); return; }
|
||||
List<JsonObject> rows = new ArrayList<>();
|
||||
for (String ln : Files.readAllLines(Paths.get(args[0]), StandardCharsets.UTF_8)) {
|
||||
if (ln.trim().isEmpty()) continue;
|
||||
JsonObject o = JsonParser.parseString(ln).getAsJsonObject();
|
||||
String k = s(o, "k");
|
||||
if (k == null || !KINDS.contains(k)) { println("BFMANN ERROR: unknown row kind " + k + " — nothing applied (R43)"); return; }
|
||||
rows.add(o);
|
||||
}
|
||||
for (JsonObject o : rows) {
|
||||
if (!"program".equals(s(o, "k"))) continue;
|
||||
String lang = s(o, "lang");
|
||||
long base = Long.decode(s(o, "image_base"));
|
||||
if (!currentProgram.getLanguageID().getIdAsString().equals(lang) || currentProgram.getImageBase().getOffset() != base) {
|
||||
println("BFMANN ERROR: file is for " + lang + " @" + s(o, "image_base") + " but the program is "
|
||||
+ currentProgram.getLanguageID().getIdAsString() + " @" + currentProgram.getImageBase() + " — nothing applied (R43)");
|
||||
return;
|
||||
}
|
||||
}
|
||||
dtm = currentProgram.getDataTypeManager();
|
||||
// pass 1: type shells; pass 2: type bodies (so fields can reference any struct in the file)
|
||||
List<JsonObject> types = new ArrayList<>();
|
||||
for (JsonObject o : rows) if ("type".equals(s(o, "k"))) types.add(o);
|
||||
for (JsonObject o : types) typeShell(o);
|
||||
for (JsonObject o : types) typeBody(o);
|
||||
for (JsonObject o : rows) {
|
||||
String k = s(o, "k");
|
||||
try {
|
||||
switch (k) {
|
||||
case "func": func(o); break;
|
||||
case "data": data(o); break;
|
||||
case "comment": comment(o); break;
|
||||
case "bookmark": bookmark(o); break;
|
||||
case "equate": equate(o); break;
|
||||
case "label": label(o); break;
|
||||
default: break; // program/block/archive/type handled above
|
||||
}
|
||||
} catch (Exception e) {
|
||||
failed++;
|
||||
println(" fail " + k + " " + (s(o, "addr") != null ? s(o, "addr") : s(o, "name")) + ": " + e);
|
||||
}
|
||||
}
|
||||
println("BFMANN types=" + nTypes + " funcs=" + nFuncs + " data=" + nData + " comments=" + nComments
|
||||
+ " bookmarks=" + nBookmarks + " equates=" + nEquates + " labels=" + nLabels + " unchanged=" + unchanged
|
||||
+ " skipped_storage=" + skippedStorage + " failed=" + failed + " rows=" + rows.size());
|
||||
}
|
||||
|
||||
// ---- data types ----
|
||||
private static CategoryPath catOf(String path) {
|
||||
int i = path.lastIndexOf('/');
|
||||
return new CategoryPath(i <= 0 ? "/" : path.substring(0, i));
|
||||
}
|
||||
private static String nameOf(String path) { return path.substring(path.lastIndexOf('/') + 1); }
|
||||
|
||||
/** Resolve a Ghidra path name with optional pointer/array suffixes (`/u8 *[4]`) to a DataType. */
|
||||
private DataType resolveType(String spec) {
|
||||
String base = spec;
|
||||
int cut = -1;
|
||||
for (int i = 0; i < spec.length(); i++) {
|
||||
char c = spec.charAt(i);
|
||||
if (c == '[' || (c == '*' && i > 0 && spec.charAt(i - 1) == ' ')) { cut = i; break; }
|
||||
}
|
||||
String suffix = "";
|
||||
if (cut >= 0) { base = spec.substring(0, cut).trim(); suffix = spec.substring(cut); }
|
||||
DataType dt = dtm.getDataType(base);
|
||||
if (dt == null) dt = BuiltInDataTypeManager.getDataTypeManager().getDataType(base);
|
||||
if (dt == null && !base.startsWith("/")) dt = dtm.getDataType("/" + base);
|
||||
if (dt == null) throw new IllegalArgumentException("unresolvable type " + spec);
|
||||
int i = 0;
|
||||
while (i < suffix.length()) {
|
||||
char c = suffix.charAt(i);
|
||||
if (c == ' ') { i++; continue; }
|
||||
if (c == '*') { dt = new PointerDataType(dt, dtm); i++; continue; }
|
||||
if (c == '[') {
|
||||
int j = suffix.indexOf(']', i);
|
||||
int n = Integer.parseInt(suffix.substring(i + 1, j).trim());
|
||||
dt = new ArrayDataType(dt, n, dt.getLength(), dtm);
|
||||
i = j + 1; continue;
|
||||
}
|
||||
throw new IllegalArgumentException("bad type suffix in " + spec);
|
||||
}
|
||||
return dt;
|
||||
}
|
||||
|
||||
private void typeShell(JsonObject o) {
|
||||
String kind = s(o, "kind"), path = s(o, "path");
|
||||
CategoryPath cat = catOf(path); String name = nameOf(path);
|
||||
if (dtm.getDataType(path) != null) return;
|
||||
DataType dt;
|
||||
switch (kind) {
|
||||
case "struct": dt = new StructureDataType(cat, name, in(o, "size"), dtm); break;
|
||||
case "union": dt = new UnionDataType(cat, name, dtm); break;
|
||||
case "enum": dt = new EnumDataType(cat, name, in(o, "size"), dtm); break;
|
||||
case "typedef": dt = new TypedefDataType(cat, name, resolveType(s(o, "base")), dtm); break;
|
||||
case "fndef": dt = new FunctionDefinitionDataType(cat, name, dtm); break;
|
||||
default: failed++; println(" fail type " + path + ": unsupported kind " + kind); return;
|
||||
}
|
||||
dtm.resolve(dt, DataTypeConflictHandler.REPLACE_HANDLER);
|
||||
nTypes++;
|
||||
}
|
||||
|
||||
private void typeBody(JsonObject o) {
|
||||
String kind = s(o, "kind"), path = s(o, "path");
|
||||
DataType dt = dtm.getDataType(path);
|
||||
if (dt == null) return;
|
||||
try {
|
||||
if ("struct".equals(kind) && dt instanceof Structure) {
|
||||
Structure st = (Structure) dt;
|
||||
boolean packed = bo(o, "packed");
|
||||
if (st.isPackingEnabled() != packed) st.setPackingEnabled(packed);
|
||||
if (!packed && st.getLength() != in(o, "size")) { st.deleteAll(); st.growStructure(in(o, "size")); }
|
||||
for (JsonElement fe : o.getAsJsonArray("fields")) {
|
||||
JsonObject f = fe.getAsJsonObject();
|
||||
DataType ft = resolveType(s(f, "type"));
|
||||
DataTypeComponent cur = st.getComponentAt(in(f, "off"));
|
||||
if (cur != null && cur.getOffset() == in(f, "off") && Objects.equals(cur.getFieldName(), s(f, "name"))
|
||||
&& cur.getDataType().getPathName().equals(ft.getPathName()) && cur.getLength() == in(f, "len")
|
||||
&& Objects.equals(cur.getComment(), s(f, "comment"))) { unchanged++; continue; }
|
||||
if (packed) st.add(ft, in(f, "len"), s(f, "name"), s(f, "comment"));
|
||||
else st.replaceAtOffset(in(f, "off"), ft, in(f, "len"), s(f, "name"), s(f, "comment"));
|
||||
}
|
||||
} else if ("union".equals(kind) && dt instanceof Union) {
|
||||
Union u = (Union) dt;
|
||||
if (u.getNumComponents() == 0)
|
||||
for (JsonElement fe : o.getAsJsonArray("fields")) {
|
||||
JsonObject f = fe.getAsJsonObject();
|
||||
u.add(resolveType(s(f, "type")), in(f, "len"), s(f, "name"), s(f, "comment"));
|
||||
}
|
||||
else unchanged++;
|
||||
} else if ("enum".equals(kind) && dt instanceof ghidra.program.model.data.Enum) {
|
||||
ghidra.program.model.data.Enum e = (ghidra.program.model.data.Enum) dt;
|
||||
for (JsonElement ve : o.getAsJsonArray("values")) {
|
||||
JsonObject v = ve.getAsJsonObject();
|
||||
String n = s(v, "name"); long val = v.get("value").getAsLong();
|
||||
if (e.contains(n) && e.getValue(n) == val) { unchanged++; continue; }
|
||||
if (e.contains(n)) e.remove(n);
|
||||
e.add(n, val);
|
||||
}
|
||||
} else if ("fndef".equals(kind) && dt instanceof FunctionDefinition) {
|
||||
FunctionDefinition fd = (FunctionDefinition) dt;
|
||||
fd.setReturnType(resolveType(s(o, "ret")));
|
||||
List<ParameterDefinition> ps = new ArrayList<>();
|
||||
for (JsonElement pe : o.getAsJsonArray("params")) {
|
||||
JsonObject p = pe.getAsJsonObject();
|
||||
ps.add(new ParameterDefinitionImpl(s(p, "name"), resolveType(s(p, "type")), s(p, "comment")));
|
||||
}
|
||||
fd.setArguments(ps.toArray(new ParameterDefinition[0]));
|
||||
fd.setVarArgs(bo(o, "varargs"));
|
||||
String cc = s(o, "cc");
|
||||
if (cc != null) try { fd.setCallingConvention(cc); } catch (Exception ex) { /* unknown cc: keep default */ }
|
||||
}
|
||||
} catch (Exception e) {
|
||||
failed++; println(" fail type body " + path + ": " + e);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- functions ----
|
||||
private VariableStorage storageOf(String spec) {
|
||||
// "Stack[-0x10]:4" | "a0:4" | "s0:4" ... ; anything else -> null (dynamic storage)
|
||||
try {
|
||||
if (spec.startsWith("Stack[")) {
|
||||
int j = spec.indexOf(']');
|
||||
int off = (int) Long.decode(spec.substring(6, j));
|
||||
int size = Integer.parseInt(spec.substring(spec.lastIndexOf(':') + 1));
|
||||
return new VariableStorage(currentProgram, off, size);
|
||||
}
|
||||
String rn = spec.contains(":") ? spec.substring(0, spec.indexOf(':')) : spec;
|
||||
Register r = currentProgram.getRegister(rn);
|
||||
if (r != null) return new VariableStorage(currentProgram, r);
|
||||
} catch (Exception e) { /* fall through */ }
|
||||
return null;
|
||||
}
|
||||
|
||||
private void func(JsonObject o) throws Exception {
|
||||
Address a = addr(s(o, "addr"));
|
||||
FunctionManager fm = currentProgram.getFunctionManager();
|
||||
Function f = fm.getFunctionAt(a);
|
||||
if (f == null) {
|
||||
if (getInstructionAt(a) == null) disassemble(a);
|
||||
f = createFunction(a, s(o, "name"));
|
||||
if (f == null) { failed++; println(" fail func " + s(o, "addr") + ": cannot create"); return; }
|
||||
}
|
||||
SourceType src = SourceType.valueOf(s(o, "sigsrc"));
|
||||
boolean changed = false;
|
||||
String name = s(o, "name");
|
||||
if (name != null && !name.equals(f.getName())) { f.setName(name, src == SourceType.DEFAULT ? SourceType.USER_DEFINED : src); changed = true; }
|
||||
DataType ret = resolveType(s(o, "ret"));
|
||||
if (!f.getReturnType().getPathName().equals(ret.getPathName())) { f.setReturnType(ret, src); changed = true; }
|
||||
String cc = s(o, "cc");
|
||||
if (cc != null && !cc.equals(f.getCallingConventionName())) { try { f.setCallingConvention(cc); changed = true; } catch (Exception e) { skippedStorage++; } }
|
||||
if (f.hasVarArgs() != bo(o, "varargs")) { f.setVarArgs(bo(o, "varargs")); changed = true; }
|
||||
if (f.hasNoReturn() != bo(o, "noreturn")) { f.setNoReturn(bo(o, "noreturn")); changed = true; }
|
||||
if (f.isInline() != bo(o, "inline")) { f.setInline(bo(o, "inline")); changed = true; }
|
||||
// parameters
|
||||
JsonArray ps = o.getAsJsonArray("params");
|
||||
boolean custom = bo(o, "custom");
|
||||
List<Variable> want = new ArrayList<>();
|
||||
boolean allStorage = true;
|
||||
for (JsonElement pe : ps) {
|
||||
JsonObject p = pe.getAsJsonObject();
|
||||
DataType pt = resolveType(s(p, "type"));
|
||||
VariableStorage vs = custom ? storageOf(s(p, "storage")) : null;
|
||||
if (custom && vs == null) allStorage = false;
|
||||
want.add(vs != null ? new ParameterImpl(s(p, "name"), pt, vs, currentProgram) : new ParameterImpl(s(p, "name"), pt, currentProgram));
|
||||
}
|
||||
boolean same = f.getParameterCount() == want.size();
|
||||
for (int i = 0; same && i < want.size(); i++) {
|
||||
Parameter cur = f.getParameter(i);
|
||||
same = Objects.equals(cur.getName(), want.get(i).getName()) && cur.getDataType().getPathName().equals(want.get(i).getDataType().getPathName())
|
||||
&& (!custom || cur.getVariableStorage().toString().equals(want.get(i).getVariableStorage().toString()));
|
||||
}
|
||||
if (!same) {
|
||||
Function.FunctionUpdateType ut = (custom && allStorage) ? Function.FunctionUpdateType.CUSTOM_STORAGE : Function.FunctionUpdateType.DYNAMIC_STORAGE_ALL_PARAMS;
|
||||
if (custom && !allStorage) skippedStorage++;
|
||||
f.replaceParameters(want, ut, true, src == SourceType.DEFAULT ? SourceType.USER_DEFINED : src);
|
||||
changed = true;
|
||||
}
|
||||
// locals: match by storage + first-use; update name/type/comment, else add
|
||||
for (JsonElement le : o.getAsJsonArray("locals")) {
|
||||
JsonObject l = le.getAsJsonObject();
|
||||
String storage = s(l, "storage"); int first = in(l, "first");
|
||||
DataType lt = resolveType(s(l, "type"));
|
||||
Variable hit = null;
|
||||
for (Variable v : f.getLocalVariables())
|
||||
if (v.getVariableStorage().toString().equals(storage) && v.getFirstUseOffset() == first) { hit = v; break; }
|
||||
SourceType lsrc = SourceType.valueOf(s(l, "src"));
|
||||
if (lsrc == SourceType.DEFAULT) lsrc = SourceType.USER_DEFINED;
|
||||
if (hit != null) {
|
||||
boolean lc = false;
|
||||
if (!Objects.equals(hit.getName(), s(l, "name"))) { hit.setName(s(l, "name"), lsrc); lc = true; }
|
||||
if (!hit.getDataType().getPathName().equals(lt.getPathName())) { hit.setDataType(lt, lsrc); lc = true; }
|
||||
if (!Objects.equals(hit.getComment(), s(l, "comment"))) { hit.setComment(s(l, "comment")); lc = true; }
|
||||
if (lc) changed = true;
|
||||
continue;
|
||||
}
|
||||
Variable nv;
|
||||
if (storage.startsWith("Stack[")) {
|
||||
int off = (int) Long.decode(storage.substring(6, storage.indexOf(']')));
|
||||
nv = new LocalVariableImpl(s(l, "name"), first, lt, off, currentProgram);
|
||||
} else {
|
||||
String rn = storage.contains(":") ? storage.substring(0, storage.indexOf(':')) : storage;
|
||||
Register r = currentProgram.getRegister(rn);
|
||||
if (r == null) { skippedStorage++; continue; }
|
||||
nv = new LocalVariableImpl(s(l, "name"), first, lt, r, currentProgram);
|
||||
}
|
||||
Variable added = f.addLocalVariable(nv, lsrc);
|
||||
if (added != null && s(l, "comment") != null) added.setComment(s(l, "comment"));
|
||||
changed = true;
|
||||
}
|
||||
String fc = s(o, "comment");
|
||||
if (!Objects.equals(fc, f.getComment())) { f.setComment(fc); changed = true; }
|
||||
if (changed) nFuncs++; else unchanged++;
|
||||
}
|
||||
|
||||
// ---- data, comments, bookmarks, equates, labels ----
|
||||
private void data(JsonObject o) throws Exception {
|
||||
Address a = addr(s(o, "addr"));
|
||||
DataType dt = resolveType(s(o, "type"));
|
||||
Data cur = currentProgram.getListing().getDefinedDataAt(a);
|
||||
if (cur != null && cur.getDataType().getPathName().equals(dt.getPathName()) && cur.getLength() == in(o, "len")) { unchanged++; return; }
|
||||
try {
|
||||
DataUtilities.createData(currentProgram, a, dt, in(o, "len"), DataUtilities.ClearDataMode.CLEAR_ALL_CONFLICT_DATA);
|
||||
nData++;
|
||||
} catch (CodeUnitInsertionException e) {
|
||||
failed++; println(" fail data " + s(o, "addr") + ": " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void comment(JsonObject o) {
|
||||
Address a = addr(s(o, "addr"));
|
||||
CommentType t;
|
||||
switch (s(o, "type")) {
|
||||
case "eol": t = CommentType.EOL; break;
|
||||
case "pre": t = CommentType.PRE; break;
|
||||
case "post": t = CommentType.POST; break;
|
||||
case "plate": t = CommentType.PLATE; break;
|
||||
default: t = CommentType.REPEATABLE; break;
|
||||
}
|
||||
Listing l = currentProgram.getListing();
|
||||
if (Objects.equals(l.getComment(t, a), s(o, "text"))) { unchanged++; return; }
|
||||
l.setComment(a, t, s(o, "text"));
|
||||
nComments++;
|
||||
}
|
||||
|
||||
private void bookmark(JsonObject o) {
|
||||
Address a = addr(s(o, "addr"));
|
||||
BookmarkManager bm = currentProgram.getBookmarkManager();
|
||||
Bookmark cur = bm.getBookmark(a, s(o, "type"), s(o, "cat"));
|
||||
if (cur != null && Objects.equals(cur.getComment(), s(o, "text"))) { unchanged++; return; }
|
||||
bm.setBookmark(a, s(o, "type"), s(o, "cat"), s(o, "text"));
|
||||
nBookmarks++;
|
||||
}
|
||||
|
||||
private void equate(JsonObject o) throws Exception {
|
||||
EquateTable et = currentProgram.getEquateTable();
|
||||
String name = s(o, "name"); long value = o.get("value").getAsLong();
|
||||
Equate e = et.getEquate(name);
|
||||
boolean changed = false;
|
||||
if (e == null) { e = et.createEquate(name, value); changed = true; }
|
||||
Set<String> have = new HashSet<>();
|
||||
for (EquateReference r : e.getReferences()) have.add(r.getAddress().getOffset() + "|" + r.getOpIndex());
|
||||
for (JsonElement re : o.getAsJsonArray("refs")) {
|
||||
JsonArray r = re.getAsJsonArray();
|
||||
Address a = addr(r.get(0).getAsString()); int op = r.get(1).getAsInt();
|
||||
if (have.contains(a.getOffset() + "|" + op)) continue;
|
||||
e.addReference(a, op); changed = true;
|
||||
}
|
||||
if (changed) nEquates++; else unchanged++;
|
||||
}
|
||||
|
||||
private void label(JsonObject o) throws Exception {
|
||||
Address a = addr(s(o, "addr"));
|
||||
String name = s(o, "name");
|
||||
SourceType src = SourceType.valueOf(s(o, "src"));
|
||||
if (src == SourceType.DEFAULT) src = SourceType.USER_DEFINED;
|
||||
SymbolTable st = currentProgram.getSymbolTable();
|
||||
for (Symbol sym : st.getSymbols(a))
|
||||
if (sym.getName().equals(name)) {
|
||||
if (bo(o, "primary") && !sym.isPrimary()) { sym.setPrimary(); nLabels++; } else unchanged++;
|
||||
return;
|
||||
}
|
||||
Function f = currentProgram.getFunctionManager().getFunctionAt(a);
|
||||
if (bo(o, "func") && f != null) { f.setName(name, src); nLabels++; return; }
|
||||
Symbol created = st.createLabel(a, name, src);
|
||||
if (bo(o, "primary")) created.setPrimary();
|
||||
nLabels++;
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,9 @@ import java.util.Iterator;
|
||||
public class ImportPsyqGdt extends GhidraScript {
|
||||
@Override
|
||||
public void run() throws Exception {
|
||||
String gdt = "/home/musashi/ghidra_12.1_PUBLIC/Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt";
|
||||
// P33 B5: default = THIS Ghidra installation's ghidra_psx_ldr extension (no home-directory assumption)
|
||||
String gdt = new File(ghidra.framework.Application.getInstallationDirectory().getFile(false),
|
||||
"Ghidra/Extensions/ghidra_psx_ldr/data/psyq400.gdt").getPath();
|
||||
String[] a = getScriptArgs();
|
||||
if (a.length > 0) gdt = a[0];
|
||||
File f = new File(gdt);
|
||||
|
||||
Reference in New Issue
Block a user