diff --git a/.gitignore b/.gitignore index 88d8bd746b..790dcb497d 100644 --- a/.gitignore +++ b/.gitignore @@ -486,6 +486,7 @@ unsloth_compiled_cache/ !/.run/P36/delever/*.json /.run/P36/delever/recipes.json /.run/P36/delever/inflight.json +/.run/P36/delever/includers_cache.json !/.run/P36/delever/*.jsonl !/.run/P36/delever/*.txt !/.run/P36/permuter/ diff --git a/decomp-architect/corpus/tools/P10/delever.py b/decomp-architect/corpus/tools/P10/delever.py index 97f6cff51a..51e9762654 100644 --- a/decomp-architect/corpus/tools/P10/delever.py +++ b/decomp-architect/corpus/tools/P10/delever.py @@ -533,18 +533,41 @@ def bodies_from_sites(sites, with_file_scope=False): return bodies -def includers(): - """header rel -> [TU rel] (transitive: a header included by a header is included by that header's includers).""" +INCLUDERS_CACHE = RUN / "includers_cache.json" + + +def includers(use_cache=True): + """header rel -> [TU rel] (transitive: a header included by a header is included by that header's includers). The per-file + include lists are cached on (mtime, size) — masking every file's text costs ~29 s per batch, the cache ~2 s.""" + cache = {} + if use_cache and INCLUDERS_CACHE.exists(): + try: + cache = json.loads(INCLUDERS_CACHE.read_text()) + except Exception: # noqa: BLE001 — a torn cache file is rebuilt + cache = {} + new_cache = {} direct = collections.defaultdict(set) hdr_inc = collections.defaultdict(set) for p in sorted((REPO / "src").rglob("*")): if p.suffix not in (".c", ".h") or p.name.startswith("."): continue rel = p.relative_to(REPO).as_posix() - text = p.read_text(errors="surrogateescape") - for mm in sc.INCLUDE_LINE.finditer(sc.mask_text(text)): - key = os.path.normpath(os.path.join(os.path.dirname(rel), mm.group(1))) - (direct if rel.endswith(".c") else hdr_inc)[key].add(rel) + st = p.stat() + key = f"{st.st_mtime_ns}|{st.st_size}" + ent = cache.get(rel) + if ent and ent[0] == key: + incs = ent[1] + else: + text = p.read_text(errors="surrogateescape") + incs = [os.path.normpath(os.path.join(os.path.dirname(rel), mm.group(1))) for mm in sc.INCLUDE_LINE.finditer(sc.mask_text(text))] + new_cache[rel] = [key, incs] + for k2 in incs: + (direct if rel.endswith(".c") else hdr_inc)[k2].add(rel) + if use_cache: + RUN.mkdir(parents=True, exist_ok=True) + tmp = INCLUDERS_CACHE.with_suffix(".tmp") + tmp.write_text(json.dumps(new_cache)) + os.replace(tmp, INCLUDERS_CACHE) out = {} def resolve(h, seen=()): if h in out: @@ -620,12 +643,15 @@ def ledger_append(rows): def ledger_index(rows): - """(done_after: set of nhash_after, exemplars: nhash_before -> the latest judged row) — the latest row per body wins (R70: - the tie-break is part of the instrument: rows are appended in time order, the last one is the current verdict).""" + """(done: set of (tu, fn, nhash_after), exemplars: nhash_before -> the latest judged row) — the latest row per body wins (R70: + the tie-break is part of the instrument: rows are appended in time order, the last one is the current verdict). "Done" is PER BODY: + another copy of the same text elsewhere is not judged (nor marked) until its own row exists — keying done by text alone let every + fleet-wide copy of an all-NEEDED exemplar (after-hash == before-hash) pass as done, unmarked (the T4 preflight found 2,204 drawable + files where ~2,583 were expected). The text hash is the REPLAY key only.""" done, ex = set(), {} for r in rows: if r.get("verdict") in DONE_VERDICTS and r.get("nhash_after"): - done.add(r["nhash_after"]) + done.add((r["tu"], r["fn"], r["nhash_after"])) if r.get("verdict") in ("LEVER-FREE", "RESIDUE") and r.get("nhash_before") and not r.get("replay_disagreed"): ex[r["nhash_before"]] = r return done, ex @@ -766,7 +792,7 @@ def make_plan(bodies, ledger_rows, headers, batch, only, rejudge=False, redraw=( continue if not b["sites"] and not b["deferred"]: continue - if b["nhash"] in done and not rejudge and latest.get((tu, fn)) not in redraw: + if (tu, fn, b["nhash"]) in done and not rejudge and latest.get((tu, fn)) not in redraw: continue if only and not any(o in b["aliases"] or o == tu or tu.endswith("/" + o) or (b["nhash"] or "").startswith(o) or o == fn for o in only): continue @@ -1433,10 +1459,16 @@ def selftest(): except Refuse: pass # the ledger index: after-hash done, before-hash replays, the latest row wins - rows = [dict(verdict="RESIDUE", nhash_before="h1", nhash_after="h2", sites=[]), dict(verdict="LEVER-FREE", nhash_before="h1", nhash_after="h3", sites=[{"ord": 0}])] + rows = [dict(tu="a.c", fn="f", verdict="RESIDUE", nhash_before="h1", nhash_after="h2", sites=[]), + dict(tu="b.c", fn="g", verdict="LEVER-FREE", nhash_before="h1", nhash_after="h3", sites=[{"ord": 0}])] done, ex = ledger_index(rows) - if done != {"h2", "h3"} or ex["h1"]["nhash_after"] != "h3": - fail("ledger index") + if done != {("a.c", "f", "h2"), ("b.c", "g", "h3")} or ex["h1"]["nhash_after"] != "h3": + fail("ledger index: done is per body, the exemplar per text") + # a copy elsewhere with the exemplar's after-hash is NOT done (it has no row of its own) + fake_bodies = {("c.c", "h"): dict(tu="c.c", fn="h", fn_line=1, fn_end=3, nhash="h2", aliases=["x"], header=False, includers=0, sites=[{"line": 1, "col": 1, "kind": "pin", "cls": "A", "detail": "$1"}], frozen=[], deferred=[])} + plan_, total_, _ = make_plan(fake_bodies, rows, False, 10, None) + if total_ != 1: + fail("a copy with a judged text but no row of its own must still be drawn") # the oracle's crash classification on its real message forms (R103) if not oracle.SIGNAL_LINE.search("bash: line 1: 3845091 Done mipsel-linux-gnu-cpp ...\n 3845092 Aborted (core dumped) | tools/bin/gcc-2.7.2-psx/cc1 -quiet\n"): fail("SIGNAL_LINE must match bash's job-status block") diff --git a/decomp-architect/corpus/tools/P10/delever_cycle.sh b/decomp-architect/corpus/tools/P10/delever_cycle.sh index 32ff55c675..cb573e139b 100644 --- a/decomp-architect/corpus/tools/P10/delever_cycle.sh +++ b/decomp-architect/corpus/tools/P10/delever_cycle.sh @@ -63,6 +63,13 @@ for i, l in enumerate(lines): p.write_text("\n".join(lines)) print("log:", label, cline) EOF + # the index lock: wait out a transient holder (a harness hook's git status), clear a STALE one (no git process alive) — batch tus2 + # lost its commit to a lock created during its own fleet run with no holder left (S98) + for i in 1 2 3 4 5 6; do + [ -e .git/index.lock ] || break + if [ -z "$(ps -eo cmd | grep -E '^git |/git ' | grep -v grep)" ]; then echo "cycle: stale .git/index.lock (no git process) — removed"; rm -f .git/index.lock; break; fi + sleep 10 + done git add -A src && git add .run/P36/delever .run/P36/census .run/P36/baseline phase-ends/CURRENT_PHASE.md || exit 1 git commit -q -m "src(phase-36): ${TASK:-T4} batch $label — $vline | R22 $line | census: $cline (delever_cycle)" || exit 1 echo "cycle: batch $label committed $(git log --oneline -1 | cut -c1-9) — $vline | $line" diff --git a/docs/SETUP.md b/docs/SETUP.md index e90ff39207..5e6c7371c5 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -784,7 +784,7 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo | | `tools/share_census.py` | **(P35 T1)** The fleet-wide census of byte-identical function classes and the S1 "one source per unique function" checker (`--check`, `--selftest`, `--scope`, `--strict-macros`, `--strict-text`); its ledger is `config/dedup_exceptions.tsv`; details in the P35 T1 section below. | | | `tools/lever_census.py` | **(P36 T1)** The census of every compiler-forcing construct ("lever") in the fleet's C — register pins, asm statements by kind (barrier / launder / keepalive / instruction / GTE / verbatim-body), volatile levers, bare `register`, plus the deferred asm-label aliases, builtins and attributes — derived from `share_census`'s scanner with a per-token coverage assertion against the raw text, four known-true controls, the `// !FAKE:` marker split, `--check` (0 unmarked pins/asm AND 0 orphan markers — a `// !FAKE:` line with no pin/asm site on it nor on the line below) and `--check --strict` (0 pins, 0 asm outside the GTE header) gates, the JSON's `head` + stat-based `src_stamp` (delever refuses a census that does not describe the tree) and a per-file walk cache keyed on the walker's own hash (a tool change invalidates it, R35), `--sites` (every site to `.run/P36/census/lever_sites.jsonl`, the delever ledger's input), `--selftest`. Evidence: `.run/P36/census/lever_census.{json,txt}`; `progress.py` publishes the `levers` block and a README sentence from it. | | | `tools/delever_oracle.py` | **(P36 T2)** The fast byte oracle for a single-translation-unit edit: `--recipes` captures every object's exact build command through `make -n -W BINARY=` (the Makefile's own pipeline — the jump-table pad stage, the per-object `-O0` overrides, the twin rule — into `.run/P36/delever/recipes.json`, regenerable in ~15 s); a candidate is compiled IN PLACE with `-o`/`-MF` redirected to scratch and its bytes compared with `build/`'s object from the fleet run (build/ is never written); `--calibrate ` proves 100 % equality untouched + twin == primary + a positive control (a nop injected at the end of a real body → DIFFERS) and writes `calibration.json` (HEAD, config stamp, per-object seconds); `--status`. Measured at T2: main 0.08–0.77 s per object, overlays ~0.14 s. | -| | `tools/delever.py` | **(P36 T2/T3)** The de-lever engine AND the campaign tool. Rewrites per lever class on the raw text at the census's positions (a token mismatch REFUSES, never guesses): pin → plain declaration (type/qualifiers/initializer kept; the `$0` zero-register variable's uses → 0, refused if it is ever assigned), barrier / keep-alive → deleted, launder → deleted when it launders a value into itself, **an ASSIGNMENT `out = in;` when its output and input differ** (deleting those made cc1 2.7.2 abort — the T2 probe counted them NEEDED), a hand-placed instruction → its C (addu-$zero/move/la/lh/lw/addiu/sll/srl/and/andi/lui/li and the lui+addiu / lui+ori pairs), a macro-carried site → deleted for a pure launder statement macro (`SHB`), its value `((T)(p))` for a launder statement-expression, REFUSED for a compound macro (XFER/DRAW/RTP_SND: the lever is in the `#define`, T5), volatile / register → dropped. Per body: replay (a ledger exemplar with the same normalized text: 1 compile) → rung A strip-all → rung B greedy, through `delever_oracle` (verdicts IDENTICAL / DIFFERS / COMPILE-ERROR / COMPILE-CRASH). **The file is the write unit and its final compile the proof**: every body's accepted edits + the file-scope volatile edits + the `// !FAKE:` markers spliced once, compiled through every recipe of the file (a twin's object; every includer of a header, in parallel) — IDENTICAL or the file-scope edits are dropped, or the file is restored and REFUSED (COMBINATION-FAILED). `--plan` / `--apply --batch N --label L [--headers] [--only …] [-j 12]` (TUs in parallel, exemplar files before their copies; headers serial; the tree must be clean, the calibration current, the census's src stamp the tree's — it reruns the census itself), `--redraw REFUSED NOTHING-USABLE` (draw again the bodies whose latest ledger verdict is one of these — after a tool fix), `--restore` (from `inflight.json`, the only restore — R102), `--status`, `--scrub [--only …]` (remove the census's ORPHAN markers — a `// !FAKE:` whose site is gone — each file judged through every recipe; a removed or rewritten site consumes its own trailing marker, so orphans arise only from older tool versions), `--apply-body TU FN FILE --label L [--rung E]` (T6/T7: a reshaped body judged on the bytes, refused if a class A/B lever remains), `--selftest` (the fixture through the census's own parser: every rewrite, the ladder against a stub oracle with a crashing site, the markers, a replay and a disagreeing replay, the ledger index, the oracle's crash forms), `--probe` (T2). The ledger `.run/P36/delever/ledger.jsonl`: one row per (tu, fn, addr, aliases) per judgement with every site's verdict REMOVED / REWRITTEN / NEEDED(oracle word) / REFUSED(why) / DEFERRED, the rung, the calibration id, the body's nhash BEFORE and AFTER — "done" = the current nhash is some row's after-hash; a body with a row's before-hash replays it. Class C/D survivors are never marked (decision 3); class A/B survivors carry `// !FAKE: — (P36