mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-10-04 00:20:10 -04:00
phase-36: T4 (tool fixes between batches) — 'done' is per body (tu, fn, after-hash): keying it by text alone let every fleet-wide copy of an all-NEEDED exemplar pass as done unmarked (2204 → 2483 drawable files); the includer map cached per file (29 s → 1.3 s per batch, equal to the uncached map); the cycle waits out a transient .git/index.lock and clears a stale one (batch tus2's commit was lost to one and committed by hand, 85905840c); kit corpus; the log entry with tus1/tus2 — the number 46,424 sites in 14,758 bodies · marked 9,393 · unmarked 37,031
This commit is contained in:
Drew T
@@ -486,6 +486,7 @@ unsloth_compiled_cache/
|
||||
!/.run/P36/delever/*.json
|
||||
/.run/P36/delever/recipes.json
|
||||
/.run/P36/delever/inflight.json
|
||||
/.run/P36/delever/includers_cache.json
|
||||
!/.run/P36/delever/*.jsonl
|
||||
!/.run/P36/delever/*.txt
|
||||
!/.run/P36/permuter/
|
||||
|
||||
@@ -533,18 +533,41 @@ def bodies_from_sites(sites, with_file_scope=False):
|
||||
return bodies
|
||||
|
||||
|
||||
def includers():
|
||||
"""header rel -> [TU rel] (transitive: a header included by a header is included by that header's includers)."""
|
||||
INCLUDERS_CACHE = RUN / "includers_cache.json"
|
||||
|
||||
|
||||
def includers(use_cache=True):
|
||||
"""header rel -> [TU rel] (transitive: a header included by a header is included by that header's includers). The per-file
|
||||
include lists are cached on (mtime, size) — masking every file's text costs ~29 s per batch, the cache ~2 s."""
|
||||
cache = {}
|
||||
if use_cache and INCLUDERS_CACHE.exists():
|
||||
try:
|
||||
cache = json.loads(INCLUDERS_CACHE.read_text())
|
||||
except Exception: # noqa: BLE001 — a torn cache file is rebuilt
|
||||
cache = {}
|
||||
new_cache = {}
|
||||
direct = collections.defaultdict(set)
|
||||
hdr_inc = collections.defaultdict(set)
|
||||
for p in sorted((REPO / "src").rglob("*")):
|
||||
if p.suffix not in (".c", ".h") or p.name.startswith("."):
|
||||
continue
|
||||
rel = p.relative_to(REPO).as_posix()
|
||||
text = p.read_text(errors="surrogateescape")
|
||||
for mm in sc.INCLUDE_LINE.finditer(sc.mask_text(text)):
|
||||
key = os.path.normpath(os.path.join(os.path.dirname(rel), mm.group(1)))
|
||||
(direct if rel.endswith(".c") else hdr_inc)[key].add(rel)
|
||||
st = p.stat()
|
||||
key = f"{st.st_mtime_ns}|{st.st_size}"
|
||||
ent = cache.get(rel)
|
||||
if ent and ent[0] == key:
|
||||
incs = ent[1]
|
||||
else:
|
||||
text = p.read_text(errors="surrogateescape")
|
||||
incs = [os.path.normpath(os.path.join(os.path.dirname(rel), mm.group(1))) for mm in sc.INCLUDE_LINE.finditer(sc.mask_text(text))]
|
||||
new_cache[rel] = [key, incs]
|
||||
for k2 in incs:
|
||||
(direct if rel.endswith(".c") else hdr_inc)[k2].add(rel)
|
||||
if use_cache:
|
||||
RUN.mkdir(parents=True, exist_ok=True)
|
||||
tmp = INCLUDERS_CACHE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(new_cache))
|
||||
os.replace(tmp, INCLUDERS_CACHE)
|
||||
out = {}
|
||||
def resolve(h, seen=()):
|
||||
if h in out:
|
||||
@@ -620,12 +643,15 @@ def ledger_append(rows):
|
||||
|
||||
|
||||
def ledger_index(rows):
|
||||
"""(done_after: set of nhash_after, exemplars: nhash_before -> the latest judged row) — the latest row per body wins (R70:
|
||||
the tie-break is part of the instrument: rows are appended in time order, the last one is the current verdict)."""
|
||||
"""(done: set of (tu, fn, nhash_after), exemplars: nhash_before -> the latest judged row) — the latest row per body wins (R70:
|
||||
the tie-break is part of the instrument: rows are appended in time order, the last one is the current verdict). "Done" is PER BODY:
|
||||
another copy of the same text elsewhere is not judged (nor marked) until its own row exists — keying done by text alone let every
|
||||
fleet-wide copy of an all-NEEDED exemplar (after-hash == before-hash) pass as done, unmarked (the T4 preflight found 2,204 drawable
|
||||
files where ~2,583 were expected). The text hash is the REPLAY key only."""
|
||||
done, ex = set(), {}
|
||||
for r in rows:
|
||||
if r.get("verdict") in DONE_VERDICTS and r.get("nhash_after"):
|
||||
done.add(r["nhash_after"])
|
||||
done.add((r["tu"], r["fn"], r["nhash_after"]))
|
||||
if r.get("verdict") in ("LEVER-FREE", "RESIDUE") and r.get("nhash_before") and not r.get("replay_disagreed"):
|
||||
ex[r["nhash_before"]] = r
|
||||
return done, ex
|
||||
@@ -766,7 +792,7 @@ def make_plan(bodies, ledger_rows, headers, batch, only, rejudge=False, redraw=(
|
||||
continue
|
||||
if not b["sites"] and not b["deferred"]:
|
||||
continue
|
||||
if b["nhash"] in done and not rejudge and latest.get((tu, fn)) not in redraw:
|
||||
if (tu, fn, b["nhash"]) in done and not rejudge and latest.get((tu, fn)) not in redraw:
|
||||
continue
|
||||
if only and not any(o in b["aliases"] or o == tu or tu.endswith("/" + o) or (b["nhash"] or "").startswith(o) or o == fn for o in only):
|
||||
continue
|
||||
@@ -1433,10 +1459,16 @@ def selftest():
|
||||
except Refuse:
|
||||
pass
|
||||
# the ledger index: after-hash done, before-hash replays, the latest row wins
|
||||
rows = [dict(verdict="RESIDUE", nhash_before="h1", nhash_after="h2", sites=[]), dict(verdict="LEVER-FREE", nhash_before="h1", nhash_after="h3", sites=[{"ord": 0}])]
|
||||
rows = [dict(tu="a.c", fn="f", verdict="RESIDUE", nhash_before="h1", nhash_after="h2", sites=[]),
|
||||
dict(tu="b.c", fn="g", verdict="LEVER-FREE", nhash_before="h1", nhash_after="h3", sites=[{"ord": 0}])]
|
||||
done, ex = ledger_index(rows)
|
||||
if done != {"h2", "h3"} or ex["h1"]["nhash_after"] != "h3":
|
||||
fail("ledger index")
|
||||
if done != {("a.c", "f", "h2"), ("b.c", "g", "h3")} or ex["h1"]["nhash_after"] != "h3":
|
||||
fail("ledger index: done is per body, the exemplar per text")
|
||||
# a copy elsewhere with the exemplar's after-hash is NOT done (it has no row of its own)
|
||||
fake_bodies = {("c.c", "h"): dict(tu="c.c", fn="h", fn_line=1, fn_end=3, nhash="h2", aliases=["x"], header=False, includers=0, sites=[{"line": 1, "col": 1, "kind": "pin", "cls": "A", "detail": "$1"}], frozen=[], deferred=[])}
|
||||
plan_, total_, _ = make_plan(fake_bodies, rows, False, 10, None)
|
||||
if total_ != 1:
|
||||
fail("a copy with a judged text but no row of its own must still be drawn")
|
||||
# the oracle's crash classification on its real message forms (R103)
|
||||
if not oracle.SIGNAL_LINE.search("bash: line 1: 3845091 Done mipsel-linux-gnu-cpp ...\n 3845092 Aborted (core dumped) | tools/bin/gcc-2.7.2-psx/cc1 -quiet\n"):
|
||||
fail("SIGNAL_LINE must match bash's job-status block")
|
||||
|
||||
@@ -63,6 +63,13 @@ for i, l in enumerate(lines):
|
||||
p.write_text("\n".join(lines))
|
||||
print("log:", label, cline)
|
||||
EOF
|
||||
# the index lock: wait out a transient holder (a harness hook's git status), clear a STALE one (no git process alive) — batch tus2
|
||||
# lost its commit to a lock created during its own fleet run with no holder left (S98)
|
||||
for i in 1 2 3 4 5 6; do
|
||||
[ -e .git/index.lock ] || break
|
||||
if [ -z "$(ps -eo cmd | grep -E '^git |/git ' | grep -v grep)" ]; then echo "cycle: stale .git/index.lock (no git process) — removed"; rm -f .git/index.lock; break; fi
|
||||
sleep 10
|
||||
done
|
||||
git add -A src && git add .run/P36/delever .run/P36/census .run/P36/baseline phase-ends/CURRENT_PHASE.md || exit 1
|
||||
git commit -q -m "src(phase-36): ${TASK:-T4} batch $label — $vline | R22 $line | census: $cline (delever_cycle)" || exit 1
|
||||
echo "cycle: batch $label committed $(git log --oneline -1 | cut -c1-9) — $vline | $line"
|
||||
|
||||
+1
-1
@@ -784,7 +784,7 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo
|
||||
| | `tools/share_census.py` | **(P35 T1)** The fleet-wide census of byte-identical function classes and the S1 "one source per unique function" checker (`--check`, `--selftest`, `--scope`, `--strict-macros`, `--strict-text`); its ledger is `config/dedup_exceptions.tsv`; details in the P35 T1 section below. |
|
||||
| | `tools/lever_census.py` | **(P36 T1)** The census of every compiler-forcing construct ("lever") in the fleet's C — register pins, asm statements by kind (barrier / launder / keepalive / instruction / GTE / verbatim-body), volatile levers, bare `register`, plus the deferred asm-label aliases, builtins and attributes — derived from `share_census`'s scanner with a per-token coverage assertion against the raw text, four known-true controls, the `// !FAKE:` marker split, `--check` (0 unmarked pins/asm AND 0 orphan markers — a `// !FAKE:` line with no pin/asm site on it nor on the line below) and `--check --strict` (0 pins, 0 asm outside the GTE header) gates, the JSON's `head` + stat-based `src_stamp` (delever refuses a census that does not describe the tree) and a per-file walk cache keyed on the walker's own hash (a tool change invalidates it, R35), `--sites` (every site to `.run/P36/census/lever_sites.jsonl`, the delever ledger's input), `--selftest`. Evidence: `.run/P36/census/lever_census.{json,txt}`; `progress.py` publishes the `levers` block and a README sentence from it. |
|
||||
| | `tools/delever_oracle.py` | **(P36 T2)** The fast byte oracle for a single-translation-unit edit: `--recipes` captures every object's exact build command through `make -n -W <src> <obj> BINARY=<alias>` (the Makefile's own pipeline — the jump-table pad stage, the per-object `-O0` overrides, the twin rule — into `.run/P36/delever/recipes.json`, regenerable in ~15 s); a candidate is compiled IN PLACE with `-o`/`-MF` redirected to scratch and its bytes compared with `build/`'s object from the fleet run (build/ is never written); `--calibrate <aliases>` proves 100 % equality untouched + twin == primary + a positive control (a nop injected at the end of a real body → DIFFERS) and writes `calibration.json` (HEAD, config stamp, per-object seconds); `--status`. Measured at T2: main 0.08–0.77 s per object, overlays ~0.14 s. |
|
||||
| | `tools/delever.py` | **(P36 T2/T3)** The de-lever engine AND the campaign tool. Rewrites per lever class on the raw text at the census's positions (a token mismatch REFUSES, never guesses): pin → plain declaration (type/qualifiers/initializer kept; the `$0` zero-register variable's uses → 0, refused if it is ever assigned), barrier / keep-alive → deleted, launder → deleted when it launders a value into itself, **an ASSIGNMENT `out = in;` when its output and input differ** (deleting those made cc1 2.7.2 abort — the T2 probe counted them NEEDED), a hand-placed instruction → its C (addu-$zero/move/la/lh/lw/addiu/sll/srl/and/andi/lui/li and the lui+addiu / lui+ori pairs), a macro-carried site → deleted for a pure launder statement macro (`SHB`), its value `((T)(p))` for a launder statement-expression, REFUSED for a compound macro (XFER/DRAW/RTP_SND: the lever is in the `#define`, T5), volatile / register → dropped. Per body: replay (a ledger exemplar with the same normalized text: 1 compile) → rung A strip-all → rung B greedy, through `delever_oracle` (verdicts IDENTICAL / DIFFERS / COMPILE-ERROR / COMPILE-CRASH). **The file is the write unit and its final compile the proof**: every body's accepted edits + the file-scope volatile edits + the `// !FAKE:` markers spliced once, compiled through every recipe of the file (a twin's object; every includer of a header, in parallel) — IDENTICAL or the file-scope edits are dropped, or the file is restored and REFUSED (COMBINATION-FAILED). `--plan` / `--apply --batch N --label L [--headers] [--only …] [-j 12]` (TUs in parallel, exemplar files before their copies; headers serial; the tree must be clean, the calibration current, the census's src stamp the tree's — it reruns the census itself), `--redraw REFUSED NOTHING-USABLE` (draw again the bodies whose latest ledger verdict is one of these — after a tool fix), `--restore` (from `inflight.json`, the only restore — R102), `--status`, `--scrub [--only …]` (remove the census's ORPHAN markers — a `// !FAKE:` whose site is gone — each file judged through every recipe; a removed or rewritten site consumes its own trailing marker, so orphans arise only from older tool versions), `--apply-body TU FN FILE --label L [--rung E]` (T6/T7: a reshaped body judged on the bytes, refused if a class A/B lever remains), `--selftest` (the fixture through the census's own parser: every rewrite, the ladder against a stub oracle with a crashing site, the markers, a replay and a disagreeing replay, the ledger index, the oracle's crash forms), `--probe` (T2). The ledger `.run/P36/delever/ledger.jsonl`: one row per (tu, fn, addr, aliases) per judgement with every site's verdict REMOVED / REWRITTEN / NEEDED(oracle word) / REFUSED(why) / DEFERRED, the rung, the calibration id, the body's nhash BEFORE and AFTER — "done" = the current nhash is some row's after-hash; a body with a row's before-hash replays it. Class C/D survivors are never marked (decision 3); class A/B survivors carry `// !FAKE: <kind> <detail> — <verdict> (P36 <rung> <label>)` at the end of their line. Evidence: `apply_<label>.log`, `batch_<label>.json`. |
|
||||
| | `tools/delever.py` | **(P36 T2/T3)** The de-lever engine AND the campaign tool. Rewrites per lever class on the raw text at the census's positions (a token mismatch REFUSES, never guesses): pin → plain declaration (type/qualifiers/initializer kept; the `$0` zero-register variable's uses → 0, refused if it is ever assigned), barrier / keep-alive → deleted, launder → deleted when it launders a value into itself, **an ASSIGNMENT `out = in;` when its output and input differ** (deleting those made cc1 2.7.2 abort — the T2 probe counted them NEEDED), a hand-placed instruction → its C (addu-$zero/move/la/lh/lw/addiu/sll/srl/and/andi/lui/li and the lui+addiu / lui+ori pairs), a macro-carried site → deleted for a pure launder statement macro (`SHB`), its value `((T)(p))` for a launder statement-expression, REFUSED for a compound macro (XFER/DRAW/RTP_SND: the lever is in the `#define`, T5), volatile / register → dropped. Per body: replay (a ledger exemplar with the same normalized text: 1 compile) → rung A strip-all → rung B greedy, through `delever_oracle` (verdicts IDENTICAL / DIFFERS / COMPILE-ERROR / COMPILE-CRASH). **The file is the write unit and its final compile the proof**: every body's accepted edits + the file-scope volatile edits + the `// !FAKE:` markers spliced once, compiled through every recipe of the file (a twin's object; every includer of a header, in parallel) — IDENTICAL or the file-scope edits are dropped, or the file is restored and REFUSED (COMBINATION-FAILED). `--plan` / `--apply --batch N --label L [--headers] [--only …] [-j 12]` (TUs in parallel, exemplar files before their copies; headers serial; the tree must be clean, the calibration current, the census's src stamp the tree's — it reruns the census itself), `--redraw REFUSED NOTHING-USABLE` (draw again the bodies whose latest ledger verdict is one of these — after a tool fix), `--restore` (from `inflight.json`, the only restore — R102), `--status`, `--scrub [--only …]` (remove the census's ORPHAN markers — a `// !FAKE:` whose site is gone — each file judged through every recipe; a removed or rewritten site consumes its own trailing marker, so orphans arise only from older tool versions), `--apply-body TU FN FILE --label L [--rung E]` (T6/T7: a reshaped body judged on the bytes, refused if a class A/B lever remains), `--selftest` (the fixture through the census's own parser: every rewrite, the ladder against a stub oracle with a crashing site, the markers, a replay and a disagreeing replay, the ledger index, the oracle's crash forms), `--probe` (T2). The ledger `.run/P36/delever/ledger.jsonl`: one row per (tu, fn, addr, aliases) per judgement with every site's verdict REMOVED / REWRITTEN / NEEDED(oracle word) / REFUSED(why) / DEFERRED, the rung, the calibration id, the body's nhash BEFORE and AFTER — "done" = the current nhash is some row's after-hash; a body with a row's before-hash replays it. Class C/D survivors are never marked (decision 3); class A/B survivors carry `// !FAKE: <kind> <detail> — <verdict> (P36 <rung> <label>)` at the end of their line. Evidence: `apply_<label>.log`, `batch_<label>.json`. Per-batch setup: the per-file include lists are cached in the ignored `includers_cache.json` on (mtime, size) — 29 s → 1.3 s, proven equal to the uncached map. |
|
||||
| | `tools/delever_cycle.sh` | **(P36 T3)** The unattended batch cycle: `[LABEL_PREFIX=…] [TASK=T4] [REDRAW="REFUSED NOTHING-USABLE"] tools/delever_cycle.sh START END [BATCH=300] [tus|headers] [ONLY="alias …"]` — per batch: the tree must be clean (a dotfile under `src/` refuses), the oracle recalibrated when stale (2–3 s; every commit stales it), `delever --apply` (exit code + its `final X/X identical` — an empty verify line is a failure, R32), the CLEAN fleet run as the outer gate (`218 passed, 0 failed of 218`, R22 — it also refreshes `build/`, the oracle's baselines), the census rerun on the new tree, the phase-log entry + the 🛑 headline (R101), one commit of `src/` + the ledger/batch record/logs + the census + the log (R42). Stops on the first red with the batch's files in place (inspect, then `delever --restore`); stops cleanly when nothing is drawable. Size a call so a batch stays under ~10 min. |
|
||||
| | `tools/kit_coverage.py` | **(P33.5 task 14.5)** The kit's DISTILLATION coverage: derives the rule population (every `- **R<n>` of DIGEST §3, asserted contiguous) and the hindsight population (every `## ` heading of `docs/accelerators.md` at numbered-item granularity — 58 at S92) and asserts each is cited by a `provenance:` line of the registry seed / the kernels OR dispositioned in `config/kit_coverage_map.tsv` (`G<id>` / `DK-<n>` / `FOLDED:G<id>` / `ENV` / `PA` / `SEED:<file>` / `KIT:<path>` / `RECORD` / `COOKBOOK` / `NOT-PORTABLE`; unknown ids refused, R43); counts with denominators (R41); rc 1 on any gap. In `tools-health` after `tool_census --check`. Its first run found 26 uncited rules and 21 uncited entries → three new kernels (DK-66–DK-68) and 41 authored dispositions. |
|
||||
| | `decomp-architect/` (the day-one decomp kit) | **(P33.5 tasks 9–14)** The package a new matching-decomp project installs as Phase 0.5 on ProjectArchitect 2.0: `README.md` (the three steps), `intake.decomp.md` (ProjectArchitect's twelve items pre-answered + the phase ladder + the six readability inversions), `SETUP.md` (the installer, Step 0 contract … Step 10 verify + hard stop; `answers: <path>` for unattended runs), `decomp-architect.md` (the methodology), `templates/` (the firewall pack — `gitignore.decomp`, `firewall.txt`, `audit_public.template.py`, the planted fixture, `no-rom.template.yml` —, the READMEs, `pa-overlays.md`, `registry-E.decomp.md` G1–G67, the skeletons, `PLACEHOLDERS.md`, `layout-contract.md`), `corpus/decomp-kernels.md` (DK-1 … DK-80), the three dictionaries `corpus/tools/<phase>/` + `corpus/cookbook/` + `corpus/record/` (generated by `make kit-corpus`), `memory-seed/` (18), `tools/MANIFEST.md` (generated). **How it is checked:** `tools/kit_lint.py` (de-specialisation, placeholders, syntax, the gitignore-template diff) + `tools/tool_census.py --check` (the corpora equal their sources) + `tools/gitignore_template_check.py`, all in `tools-health`; the dry-run harness under `.run/P33.5/kit-dryrun/` (`answers.md`, `expected-manifest.txt`, `judge.py`, the install logs and verdicts of runs 1–5 — a kit change is re-verified by a resume on the last throwaway `repo/`, a fresh full run only when SETUP's steps change; the judge compares the real tree's dirty PATH SETS before/after). Wiki page: `docs/wiki/Start-a-new-decomp-project.md`. Split into its own repository after the flip. |
|
||||
|
||||
@@ -328,6 +328,21 @@ accumulate here as the phase produces them.**
|
||||
|
||||
- **2026-09-09 — T4 batch `tus2`** (`.run/P36/delever/run_tus2.log`, `batch_tus2.json`): `delever: batch tus2 — 300 files (TUs; 2483 drawable) · 2752 bodies: 495 lever-free, 2257 residue, 0 refused · sites removed 3505 / needed 5262 / refused 0 / deferred 0 · replays 1350 (0 disagreed) · compiles 5616 in 251 s wall · final 300/300 identical · written 297 files` → R22 (`.run/P36/baseline/r22_tus2.log`) **`check-all: 218 passed, 0 failed of 218`** (`wall=148.11 s`) → census `THE PHASE'S NUMBER (pins + asm statements, GTE excluded): 46,424 sites in 14,758 bodies (2,258 distinct) · marked !FAKE 9,393 · UNMARKED 37,031`.
|
||||
|
||||
- **S98 — T4 in progress: the campaign launched; three tool fixes between batches (each picked up by the next batch — the cycle re-reads
|
||||
the tool per batch; committed once the cycle was idle, R59).** (1) **"Done" is per body:** the T4 preflight drew `2204 of 2204` TU files
|
||||
where ~2,583 were expected — the ledger's done set was keyed by normalized text alone, so every fleet-wide copy of an all-NEEDED exemplar
|
||||
(after-hash == before-hash, e.g. the 133 copies of a residue body) passed as done, never drawn, never marked; now done = (tu, fn,
|
||||
after-hash), the text hash is the REPLAY key only (selftest: a copy with a judged text but no row of its own is drawn) — batch tus2 drew
|
||||
`2483 drawable`. (2) **The includer map is cached** per file on (mtime, size) in the ignored `includers_cache.json`: 29 s → 1.3 s per
|
||||
batch, proven equal to the uncached map (R39). (3) **A stale `.git/index.lock`** (created 03:24:19 during batch tus2's fleet run, no
|
||||
holder — the Makefile runs no git; a harness hook is the likely source) cost tus2 its commit: the batch was complete and gated (apply
|
||||
`final 300/300 identical`, R22 `218 passed`, the census, the log entry) and was committed by hand from its own lines (`85905840c`); the
|
||||
cycle now waits out a transient lock (6 × 10 s) and clears a stale one (no git process alive), logging it. **Batches so far:** `tus1`
|
||||
(`a605e1dba`): 300 files · 1,392 bodies: 299 lever-free, 1,093 residue · sites removed 3,342 / needed 4,177 · replays 1,364 (0 disagreed) ·
|
||||
1,794 compiles in 59 s · R22 218/218 → 49,787 sites; `tus2` (`85905840c`): 300 files · 2,752 bodies: 495 lever-free, 2,257 residue · removed
|
||||
3,505 / needed 5,262 · replays 1,350 (0 disagreed) · 5,616 compiles in 251 s · R22 218/218 → **46,424 sites in 14,758 bodies (2,258
|
||||
distinct) · marked 9,393 · UNMARKED 37,031**. The replay discount is real (≈ half the bodies, 1 compile each, 0 disagreements so far).
|
||||
|
||||
## 🛑 SESSION CHECKPOINT — S98 (2026-09-09): T0 ☑ T1 ☑ T1b ☑ T2 ☑ T3 ☑ — all committed (T3 = `39e3e1851` tools, `cd24727dd` + `7ee458dd7` the ov_SC04_011 batches, this close); NEXT = T4 the mechanical campaign: `tools/delever_cycle.sh` over the whole population (TUs then headers), unattended, one commit per batch, R22 every batch | the number: 53,033 sites in 15,638 bodies (2,246 distinct), 310 marked, 0 orphans | last batch `tus2` on a605e1dba: THE PHASE'S NUMBER (pins + asm statements, GTE excluded): 46,424 sites in 14,758 bodies (2,258 distinct) · marked !FAKE 9,393 · UNMARKED 37,031
|
||||
|
||||
### 0. How to use this block
|
||||
|
||||
+45
-13
@@ -533,18 +533,41 @@ def bodies_from_sites(sites, with_file_scope=False):
|
||||
return bodies
|
||||
|
||||
|
||||
def includers():
|
||||
"""header rel -> [TU rel] (transitive: a header included by a header is included by that header's includers)."""
|
||||
INCLUDERS_CACHE = RUN / "includers_cache.json"
|
||||
|
||||
|
||||
def includers(use_cache=True):
|
||||
"""header rel -> [TU rel] (transitive: a header included by a header is included by that header's includers). The per-file
|
||||
include lists are cached on (mtime, size) — masking every file's text costs ~29 s per batch, the cache ~2 s."""
|
||||
cache = {}
|
||||
if use_cache and INCLUDERS_CACHE.exists():
|
||||
try:
|
||||
cache = json.loads(INCLUDERS_CACHE.read_text())
|
||||
except Exception: # noqa: BLE001 — a torn cache file is rebuilt
|
||||
cache = {}
|
||||
new_cache = {}
|
||||
direct = collections.defaultdict(set)
|
||||
hdr_inc = collections.defaultdict(set)
|
||||
for p in sorted((REPO / "src").rglob("*")):
|
||||
if p.suffix not in (".c", ".h") or p.name.startswith("."):
|
||||
continue
|
||||
rel = p.relative_to(REPO).as_posix()
|
||||
text = p.read_text(errors="surrogateescape")
|
||||
for mm in sc.INCLUDE_LINE.finditer(sc.mask_text(text)):
|
||||
key = os.path.normpath(os.path.join(os.path.dirname(rel), mm.group(1)))
|
||||
(direct if rel.endswith(".c") else hdr_inc)[key].add(rel)
|
||||
st = p.stat()
|
||||
key = f"{st.st_mtime_ns}|{st.st_size}"
|
||||
ent = cache.get(rel)
|
||||
if ent and ent[0] == key:
|
||||
incs = ent[1]
|
||||
else:
|
||||
text = p.read_text(errors="surrogateescape")
|
||||
incs = [os.path.normpath(os.path.join(os.path.dirname(rel), mm.group(1))) for mm in sc.INCLUDE_LINE.finditer(sc.mask_text(text))]
|
||||
new_cache[rel] = [key, incs]
|
||||
for k2 in incs:
|
||||
(direct if rel.endswith(".c") else hdr_inc)[k2].add(rel)
|
||||
if use_cache:
|
||||
RUN.mkdir(parents=True, exist_ok=True)
|
||||
tmp = INCLUDERS_CACHE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(new_cache))
|
||||
os.replace(tmp, INCLUDERS_CACHE)
|
||||
out = {}
|
||||
def resolve(h, seen=()):
|
||||
if h in out:
|
||||
@@ -620,12 +643,15 @@ def ledger_append(rows):
|
||||
|
||||
|
||||
def ledger_index(rows):
|
||||
"""(done_after: set of nhash_after, exemplars: nhash_before -> the latest judged row) — the latest row per body wins (R70:
|
||||
the tie-break is part of the instrument: rows are appended in time order, the last one is the current verdict)."""
|
||||
"""(done: set of (tu, fn, nhash_after), exemplars: nhash_before -> the latest judged row) — the latest row per body wins (R70:
|
||||
the tie-break is part of the instrument: rows are appended in time order, the last one is the current verdict). "Done" is PER BODY:
|
||||
another copy of the same text elsewhere is not judged (nor marked) until its own row exists — keying done by text alone let every
|
||||
fleet-wide copy of an all-NEEDED exemplar (after-hash == before-hash) pass as done, unmarked (the T4 preflight found 2,204 drawable
|
||||
files where ~2,583 were expected). The text hash is the REPLAY key only."""
|
||||
done, ex = set(), {}
|
||||
for r in rows:
|
||||
if r.get("verdict") in DONE_VERDICTS and r.get("nhash_after"):
|
||||
done.add(r["nhash_after"])
|
||||
done.add((r["tu"], r["fn"], r["nhash_after"]))
|
||||
if r.get("verdict") in ("LEVER-FREE", "RESIDUE") and r.get("nhash_before") and not r.get("replay_disagreed"):
|
||||
ex[r["nhash_before"]] = r
|
||||
return done, ex
|
||||
@@ -766,7 +792,7 @@ def make_plan(bodies, ledger_rows, headers, batch, only, rejudge=False, redraw=(
|
||||
continue
|
||||
if not b["sites"] and not b["deferred"]:
|
||||
continue
|
||||
if b["nhash"] in done and not rejudge and latest.get((tu, fn)) not in redraw:
|
||||
if (tu, fn, b["nhash"]) in done and not rejudge and latest.get((tu, fn)) not in redraw:
|
||||
continue
|
||||
if only and not any(o in b["aliases"] or o == tu or tu.endswith("/" + o) or (b["nhash"] or "").startswith(o) or o == fn for o in only):
|
||||
continue
|
||||
@@ -1433,10 +1459,16 @@ def selftest():
|
||||
except Refuse:
|
||||
pass
|
||||
# the ledger index: after-hash done, before-hash replays, the latest row wins
|
||||
rows = [dict(verdict="RESIDUE", nhash_before="h1", nhash_after="h2", sites=[]), dict(verdict="LEVER-FREE", nhash_before="h1", nhash_after="h3", sites=[{"ord": 0}])]
|
||||
rows = [dict(tu="a.c", fn="f", verdict="RESIDUE", nhash_before="h1", nhash_after="h2", sites=[]),
|
||||
dict(tu="b.c", fn="g", verdict="LEVER-FREE", nhash_before="h1", nhash_after="h3", sites=[{"ord": 0}])]
|
||||
done, ex = ledger_index(rows)
|
||||
if done != {"h2", "h3"} or ex["h1"]["nhash_after"] != "h3":
|
||||
fail("ledger index")
|
||||
if done != {("a.c", "f", "h2"), ("b.c", "g", "h3")} or ex["h1"]["nhash_after"] != "h3":
|
||||
fail("ledger index: done is per body, the exemplar per text")
|
||||
# a copy elsewhere with the exemplar's after-hash is NOT done (it has no row of its own)
|
||||
fake_bodies = {("c.c", "h"): dict(tu="c.c", fn="h", fn_line=1, fn_end=3, nhash="h2", aliases=["x"], header=False, includers=0, sites=[{"line": 1, "col": 1, "kind": "pin", "cls": "A", "detail": "$1"}], frozen=[], deferred=[])}
|
||||
plan_, total_, _ = make_plan(fake_bodies, rows, False, 10, None)
|
||||
if total_ != 1:
|
||||
fail("a copy with a judged text but no row of its own must still be drawn")
|
||||
# the oracle's crash classification on its real message forms (R103)
|
||||
if not oracle.SIGNAL_LINE.search("bash: line 1: 3845091 Done mipsel-linux-gnu-cpp ...\n 3845092 Aborted (core dumped) | tools/bin/gcc-2.7.2-psx/cc1 -quiet\n"):
|
||||
fail("SIGNAL_LINE must match bash's job-status block")
|
||||
|
||||
@@ -63,6 +63,13 @@ for i, l in enumerate(lines):
|
||||
p.write_text("\n".join(lines))
|
||||
print("log:", label, cline)
|
||||
EOF
|
||||
# the index lock: wait out a transient holder (a harness hook's git status), clear a STALE one (no git process alive) — batch tus2
|
||||
# lost its commit to a lock created during its own fleet run with no holder left (S98)
|
||||
for i in 1 2 3 4 5 6; do
|
||||
[ -e .git/index.lock ] || break
|
||||
if [ -z "$(ps -eo cmd | grep -E '^git |/git ' | grep -v grep)" ]; then echo "cycle: stale .git/index.lock (no git process) — removed"; rm -f .git/index.lock; break; fi
|
||||
sleep 10
|
||||
done
|
||||
git add -A src && git add .run/P36/delever .run/P36/census .run/P36/baseline phase-ends/CURRENT_PHASE.md || exit 1
|
||||
git commit -q -m "src(phase-36): ${TASK:-T4} batch $label — $vline | R22 $line | census: $cline (delever_cycle)" || exit 1
|
||||
echo "cycle: batch $label committed $(git log --oneline -1 | cut -c1-9) — $vline | $line"
|
||||
|
||||
Reference in New Issue
Block a user