From 5a9f455e60a857edf70102685050f48e4130e2f7 Mon Sep 17 00:00:00 2001 From: Drew T <50529377+Druthulu@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:32:21 -0600 Subject: [PATCH] =?UTF-8?q?tools(phase-35):=20T6=20part=201=20=E2=80=94=20?= =?UTF-8?q?14=20macro-era=20tools=20FROZEN=20(tools/frozen.py:=20one=20ref?= =?UTF-8?q?usal=20from=20main(),=20imports=20never=20exit;=20the=20plan's?= =?UTF-8?q?=207=20+=207=20the=20guard=20census=20found,=20family=5Fremap's?= =?UTF-8?q?=20CLI=20among=20them),=204=20retired=20to=20tools/sunset/=20(d?= =?UTF-8?q?edup=5Fpropagate=20+=20dedup=5Fextend=20->=20share=5Fbody.py=20?= =?UTF-8?q?with=20add=5Fmembers=5Fsurgical=20and=20the=20load=5Fsig=20libr?= =?UTF-8?q?ary=20moved=20verbatim;=20macro=5Fdraft;=20test=5Freconcile=5Fl?= =?UTF-8?q?edger),=20six=20lane=20callers=20repointed=20to=20share=5Fbody?= =?UTF-8?q?=20--apply=20--bucket=20new,=20validate=5Ftargets=20imports=20s?= =?UTF-8?q?hare=5Fbody,=20tool=5Fcensus=20accepts=20FROZEN;=20dictionary?= =?UTF-8?q?=20+=20sunset=20README=20rows;=20kit=20corpus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .run/P35/baseline/kit_corpus_t6a.log | 5 + config/tool_dictionary.tsv | 37 +- decomp-architect/corpus/tools/INDEX.md | 11 +- decomp-architect/corpus/tools/P10/frozen.py | 16 + .../corpus/tools/P10/share_body.py | 96 +- .../corpus/tools/P5/audit_binaries.py | 2 +- .../corpus/tools/P5/blocker_probe.py | 1 + .../corpus/tools/P5/p16_improve.py | 1 + .../corpus/tools/P5/tool_census.py | 4 +- .../corpus/tools/P6/aprop_autodraft.py | 1 + .../corpus/tools/P6/canon_sig_reconcile.py | 1 + .../corpus/tools/P6/conform_decls.py | 1 + .../tools/P6/dedup_extend.py.superseded.md | 3 + .../tools/P6/dedup_propagate.py.superseded.md | 3 + .../corpus/tools/P6/demacroize.py | 1 + .../corpus/tools/P6/family_remap.py | 1 + .../corpus/tools/P6/family_sweep.py | 1 + .../corpus/tools/P6/inject_capped_externs.py | 3 +- .../corpus/tools/P6/normalize_self_decls.py | 1 + .../corpus/tools/P6/o0_subsplit.py | 1 + .../corpus/tools/P6/restore_dropped_decls.py | 1 + decomp-architect/corpus/tools/P7/grinder.py | 4 +- .../corpus/tools/P8/auto_driver.py | 2 +- .../corpus/tools/P8/bulk_harvest.py | 2 +- decomp-architect/corpus/tools/P8/gate_lane.py | 2 +- .../corpus/tools/P8/gate_stage.py | 5 +- .../corpus/tools/P8/gen_harvest_targets.py | 1 + .../corpus/tools/P8/lora_grind.py | 4 +- .../corpus/tools/P8/recover_giant.py | 1 + .../corpus/tools/P8/validate_targets.py | 2 +- decomp-architect/tools/MANIFEST.md | 11 +- docs/tool-index.md | 43 +- phase-ends/CURRENT_PHASE.md | 16 + tools/aprop_autodraft.py | 1 + tools/audit_binaries.py | 2 +- tools/auto_driver.py | 2 +- tools/blocker_probe.py | 1 + tools/bulk_harvest.py | 2 +- tools/canon_sig_reconcile.py | 1 + tools/conform_decls.py | 1 + tools/dedup_extend.py | 298 ----- tools/dedup_propagate.py | 1119 ----------------- tools/demacroize.py | 1 + tools/family_remap.py | 1 + tools/family_sweep.py | 1 + tools/frozen.py | 16 + tools/gate_lane.py | 2 +- tools/gate_stage.py | 5 +- tools/gen_harvest_targets.py | 1 + tools/grinder.py | 4 +- tools/inject_capped_externs.py | 3 +- tools/lora_grind.py | 4 +- tools/macro_draft.py | 80 -- tools/normalize_self_decls.py | 1 + tools/o0_subsplit.py | 1 + tools/p16_improve.py | 1 + tools/recover_giant.py | 1 + tools/restore_dropped_decls.py | 1 + tools/share_body.py | 96 +- tools/sunset/README.md | 4 + .../tools/P6 => tools/sunset}/dedup_extend.py | 0 .../P6 => tools/sunset}/dedup_propagate.py | 0 .../tools/P6 => tools/sunset}/macro_draft.py | 0 .../sunset}/test_reconcile_ledger.py | 0 tools/test_reconcile_ledger.py | 61 - tools/tool_census.py | 4 +- tools/validate_targets.py | 2 +- 67 files changed, 356 insertions(+), 1645 deletions(-) create mode 100644 .run/P35/baseline/kit_corpus_t6a.log create mode 100644 decomp-architect/corpus/tools/P10/frozen.py create mode 100644 decomp-architect/corpus/tools/P6/dedup_extend.py.superseded.md create mode 100644 decomp-architect/corpus/tools/P6/dedup_propagate.py.superseded.md delete mode 100644 tools/dedup_extend.py delete mode 100644 tools/dedup_propagate.py create mode 100644 tools/frozen.py delete mode 100644 tools/macro_draft.py rename {decomp-architect/corpus/tools/P6 => tools/sunset}/dedup_extend.py (100%) rename {decomp-architect/corpus/tools/P6 => tools/sunset}/dedup_propagate.py (100%) rename {decomp-architect/corpus/tools/P6 => tools/sunset}/macro_draft.py (100%) rename {decomp-architect/corpus/tools/P5 => tools/sunset}/test_reconcile_ledger.py (100%) delete mode 100644 tools/test_reconcile_ledger.py diff --git a/.run/P35/baseline/kit_corpus_t6a.log b/.run/P35/baseline/kit_corpus_t6a.log new file mode 100644 index 0000000000..3ccf705b17 --- /dev/null +++ b/.run/P35/baseline/kit_corpus_t6a.log @@ -0,0 +1,5 @@ +.venv/bin/python tools/tool_census.py --all +tool_census: 295 tool files (find == git ls-files) + 38 retired; classes {'LIVE': 236, 'ORPHAN': 30, 'REFERENCED': 29, 'RETIRED': 38}; dictionary rows 333 +tool_census: wrote docs/tool-index.md +tool_census: wrote decomp-architect/tools/MANIFEST.md +tool_census: corpus materialised — 361 verbatim copies, 30 pointers, INDEX.md diff --git a/config/tool_dictionary.tsv b/config/tool_dictionary.tsv index bba206ed6a..2d954767c5 100644 --- a/config/tool_dictionary.tsv +++ b/config/tool_dictionary.tsv @@ -7,7 +7,7 @@ tools/alloc_table.py P7 PORTABLE list each pseudo-register's refs, live length a tools/api_agent.py P8 ADAPT give an external chat model the same multi-turn tool harness an agent gets Gives an external chat model the same multi-turn tool harness an agent gets provider endpoint, repo doc paths LIVE tools/api_draft.py P8 ADAPT draft one function against any chat-completions endpoint, provider-agnostic Provider-agnostic single-shot drafting worker against any chat completions endpoint provider endpoint, repo doc paths LIVE tools/api_rate.py P8 ADAPT measure a drafting provider's real request rate and rate-limit attribution Reports measured request rate and rate-limit attribution from append-only telemetry repo telemetry path STILL-NEEDED measure a drafting provider's true request rate and rate-limit attribution -tools/aprop_autodraft.py P6 ADAPT draft a family member mechanically from a seed body and symbol rebase Mechanically drafts a family member from the seed body plus a positional symbol rebase repo scratch paths LIVE +tools/aprop_autodraft.py P6 ADAPT draft a family member mechanically from a seed body and symbol rebase Mechanically drafts a family member from the seed body plus a positional symbol rebase repo scratch paths FROZEN tools/share_body.py tools/aprop_symfix.py P6 ADAPT rebase a remapped draft's stale seed symbols onto the target's own Rebases stale seed symbols in a mechanically adapted draft onto the target's own symbols repo symbol/config sources LIVE tools/asm_verbatim.py P10 ADAPT emit a function's target assembly as an inline-assembly body Emits the file-scope inline-assembly body form from a disassembly file repo asm layout LIVE tools/atlas.py P5 ADAPT partition every open function into exactly one lever-labelled work group Partitions all open functions into exactly one lever-labelled work group repo scratch paths LIVE @@ -38,7 +38,7 @@ tools/bfm_extract/pac.py PROJECT-ONLY PROJECT-ONLY parse a bespoke archive-chain tools/bfm_extract/test_lzss.py PROJECT-ONLY PROJECT-ONLY unit-test a bespoke compression decoder against known cases Unit tests for that game-specific compression variant project-only LIVE tools/bisect_slate.py P8 ADAPT isolate the byte-wrong drafts in a batch, with a null control first Isolates the byte-wrong drafts in a batch, with a null control first repo build paths SUPERSEDED tools/gate_main_parallel.py tools/blast_radius.py P8 ADAPT measure a change's write set and name the verification it actually requires Measures a change's write set and names the verification it actually requires repo tree layout LIVE -tools/blocker_probe.py P5 ADAPT explain with two oracles why a byte-correct draft fails the whole-binary gate Read-only two-oracle explanation of why a byte-correct draft fails the whole-binary gate repo build paths LIVE +tools/blocker_probe.py P5 ADAPT explain with two oracles why a byte-correct draft fails the whole-binary gate Read-only two-oracle explanation of why a byte-correct draft fails the whole-binary gate repo build paths FROZEN tools/share_body.py tools/bootstrap.sh P9 ADAPT set up a fresh clone idempotently: packages, environments, pinned tools Idempotent fresh-clone setup: check packages, create environments, fetch pinned tools repo layout, package names LIVE tools/build_engine_types.py P10 ADAPT extract inline-defined types and typedefs from a source file into a shared header Extracts inline-defined named types and typedefs from a source file into a shared header repo shared-header path LIVE tools/build_fuel_manifest.py P8 ADAPT unify live stubs into one ranked, class-tagged, cache-annotated target pool Unifies live stubs into one ranked, class-tagged, cache-annotated target pool canonical source binary, repo scratch paths LIVE @@ -50,7 +50,7 @@ tools/burndown.py P5 ADAPT track per-session yield and velocity so a diminishing tools/campaign_status.py P8 ADAPT report every concurrent lane with its own metrics, not only the loud one One status view covering every lane, not only the loud one lane names, repo scratch paths STILL-NEEDED report every concurrent lane with its own metrics, not only the loud one tools/canon_draft_decls.py P6 ADAPT canonicalize a draft's extern and data declarations to the banked-consistent set Canonicalizes a draft's extern and data declarations to the banked-consistent set repo shared-header paths SUPERSEDED tools/sig_unify.py tools/canon_resident_calls.py P6 ADAPT rewrite address-named calls in a draft to their curated symbol names Rewrites address-named calls in a draft to the curated symbol name when one exists repo symbol file path LIVE -tools/canon_sig_reconcile.py P6 ADAPT reconcile a definition's typed signature with what its destination unit declares — part of banking against a canonical type layer from Phase 6; again at readability (Phase 10) Reconciles a definition's typed signature with what its destination unit already declares repo src layout LIVE +tools/canon_sig_reconcile.py P6 ADAPT reconcile a definition's typed signature with what its destination unit declares — part of banking against a canonical type layer from Phase 6; again at readability (Phase 10) Reconciles a definition's typed signature with what its destination unit already declares repo src layout FROZEN the types phase (Phase 37) tools/cast_call_sites.py P10 ADAPT add per-site function-pointer casts so a draft can call a differently typed callee Adds per-site function-pointer casts so a draft can call a differently typed callee repo src layout LIVE tools/cast_self_callers.py P6 ADAPT let a unit keep calling, through per-site casts, the function it now defines Lets a unit keep calling, through a per-site cast, the function it is about to define repo src layout LIVE tools/cc1_dumps.sh P4 ADAPT dump every compiler pass file for a self-contained draft Dumps every compiler pass file for a self-contained draft into a private directory compiler triple, repo scratch paths LIVE @@ -59,17 +59,17 @@ tools/cdecl.py P6 PORTABLE parse C declarators once, for every declaration tool tools/cdtrace.py PROJECT-ONLY PROJECT-ONLY observe a live loader's requests to learn which payload loads where Reads one game's loader request structures at fixed RAM addresses project-only STILL-NEEDED observe a live loader's requests to learn which payload is read to which address tools/claude_wave_packs.py P8 ADAPT build the per-function prompt packs a drafting agent wave works from Builds the per-function prompt packs an agent wave drafts from repo scratch paths, agent harness LIVE tools/compile_only.py P9 ADAPT compile every eligible translation unit with the pinned toolchain, using no derived bytes Compiles every eligible translation unit with the pinned toolchain, without any game bytes compiler triple, repo makefile parsing LIVE -tools/conform_decls.py P6 ADAPT conform every declaration of a function fleet-wide to its byte-true definition — width and signedness proven by the bytes at bank time (Phase 6) Conforms every declaration of a function fleet-wide to its byte-true definition repo src/shared-header layout LIVE +tools/conform_decls.py P6 ADAPT conform every declaration of a function fleet-wide to its byte-true definition — width and signedness proven by the bytes at bank time (Phase 6) Conforms every declaration of a function fleet-wide to its byte-true definition repo src/shared-header layout FROZEN the types phase (Phase 37) tools/cookbook_index.py P9 ADAPT generate a symptom-keyed index of a large technique document Generates a symptom-keyed index of the large technique document repo doc paths LIVE tools/corpus.py P5 ADAPT answer from one derived model whether each function is open, matched or shared The single derived model of the source tree: open, matched or shared, per function repo src/config layout LIVE tools/decl_from_use.py P6 ADAPT infer a minimal extern for a data symbol the destination unit does not declare Infers a minimal extern for a data symbol a draft uses but its destination does not declare repo src layout LIVE tools/decl_prior.py P6 ADAPT compute the fleet's consensus declaration for every symbol, as card fuel — the seed of a canonical type layer (Phase 6) Computes the fleet's consensus declaration for every symbol, as card fuel repo src layout LIVE tools/decompile.py P4 ADAPT turn one function's disassembly into a C scaffold to start a match Wrapper locating a function's disassembly and running the C-scaffold generator on it repo asm paths, decompiler target name STILL-NEEDED turn one function's disassembly into a C scaffold to start a match tools/decompme_replica.sh P4 ADAPT reproduce an external reference toolchain locally and compare its words to target Runs a function through an external reference toolchain build and compares words against the target pinned toolchain versions, network fetch STILL-NEEDED kept: a public preset request needs the replica of the community toolchain to prove equivalence -tools/dedup_extend.py P6 ADAPT extend an existing code-share registry to newly onboarded binaries Extends the existing share registry to newly onboarded binaries repo config/src layout LIVE +tools/dedup_extend.py P6 ADAPT extend an existing code-share registry to newly onboarded binaries Extends the existing share registry to newly onboarded binaries repo config/src layout SUPERSEDED tools/share_body.py tools/dedup_integrate.py P6 ADAPT validate a code-share registry byte-honestly, failing closed on signature drift Byte-honesty validator for the code-share registry; fails closed on signature drift repo config registry path LIVE -tools/dedup_propagate.py P6 ADAPT lift one matched body into a shared macro and instantiate it everywhere Lifts one matched body into a shared macro and instantiates it at every duplicate site repo shared-header/src layout LIVE -tools/demacroize.py P6 ADAPT escape a shared-header declaration conflict locally by unmacroizing one body Per-binary local escape from a shared-header declaration conflict by unmacroizing one body repo shared-header layout LIVE +tools/dedup_propagate.py P6 ADAPT lift one matched body into a shared macro and instantiate it everywhere Lifts one matched body into a shared macro and instantiates it at every duplicate site repo shared-header/src layout SUPERSEDED tools/share_body.py +tools/demacroize.py P6 ADAPT escape a shared-header declaration conflict locally by unmacroizing one body Per-binary local escape from a shared-header declaration conflict by unmacroizing one body repo shared-header layout FROZEN tools/share_body.py tools/diff_autopsy.sh P7 ADAPT find which words diverge when the whole-binary gate says different but not where Reproduces exactly what the gate saw, then decodes the diverging words and restores the tree repo src/build paths STILL-NEEDED find WHICH words diverge when the whole-binary gate says DIFF but not where tools/diff_regions.py P7 ADAPT classify where a remapped sibling's compiled bytes diverge from its target Classifies where a remapped member's compiled bytes diverge from its target repo build paths STILL-NEEDED classify where a mechanically remapped sibling's compiled bytes diverge from its target tools/difficulty.py P5 ADAPT rank unmatched functions easiest-first by size, control flow and call count Ranks unmatched functions easiest-first by size, control flow, table presence and call count repo asm layout LIVE @@ -87,8 +87,8 @@ tools/family_align.py P6 ADAPT classify drifted structural siblings by aligned w tools/family_cousins.py P6 ADAPT cluster the open frontier one tier looser than exact skeleton hashing Similarity clustering one tier looser than exact skeleton hashing over the open frontier repo signature files LIVE tools/family_hseq.py P6 ADAPT rank clusters of the unmatched frontier by mnemonic-skeleton hash Ranked clustering of the unmatched frontier by mnemonic-skeleton hash repo signature files LIVE tools/family_manifest.py P6 ADAPT name the one exemplar to draft per structural family, ranked Ranked structural-family target manifest naming the one exemplar to draft per family repo signature files LIVE -tools/family_remap.py P6 ADAPT remap a matched exemplar's source onto a structural sibling by positional symbol pairing Mechanically remaps a matched exemplar's C onto a structural sibling by positional symbol pairing repo src layout LIVE -tools/family_sweep.py P6 ADAPT sweep a matched exemplar across every same-structure sibling, gate-arbitrated Sweeps a matched exemplar across every same-structure sibling, gate-arbitrated repo src/config layout LIVE +tools/family_remap.py P6 ADAPT remap a matched exemplar's source onto a structural sibling by positional symbol pairing Mechanically remaps a matched exemplar's C onto a structural sibling by positional symbol pairing repo src layout FROZEN tools/share_body.py +tools/family_sweep.py P6 ADAPT sweep a matched exemplar across every same-structure sibling, gate-arbitrated Sweeps a matched exemplar across every same-structure sibling, gate-arbitrated repo src/config layout FROZEN tools/share_body.py tools/fetch_psyq.sh P9 ADAPT acquire and checksum-verify vendor SDK pieces for a linked build Optional acquisition and checksum verification of vendor SDK pieces for the linked build PS1 SDK sources, repo paths LIVE tools/find_addr_refs.py P2 PORTABLE find code that materializes an absolute address, tracking registers rather than pairing windows Register-tracked scan for code that materializes an absolute address; never window-paired MIPS encodings, repo binary registry LIVE tools/fix_arity_callers.py P6 ADAPT repair a shared caller's argument-count conflict for the no-prototype failure class Repairs the shared-caller argument-count conflict for the no-prototype failure class repo shared-header layout LIVE @@ -109,7 +109,7 @@ tools/gate_wave.py P8 ADAPT gate a whole wave by splitting table-bearing drafts tools/gater_lane.py P8 ADAPT drain finished drafts into the gate continuously while drafting still streams Continuous gater draining a wave's finished drafts into the parallel gate, grouped by binary repo scratch paths STILL-NEEDED drain finished drafts into the gate continuously while drafting still streams tools/gccmap_cites.py P7 ADAPT tag every compiler-source citation with the source tree it refers to Tags every compiler-source citation in the codegen-map docs with the tree it refers to repo doc/reference paths LIVE tools/gen_engine_decls.py P6 ADAPT declare every shared function and data symbol once, canonically, in a generated header Declares every shared function and data symbol once, canonically, in a generated header repo shared-header path SUPERSEDED tools/conform_decls.py -tools/gen_harvest_targets.py P8 ADAPT generate a callee-signature-aware target manifest with per-function fleet reach Generates a callee-signature-aware target manifest with per-function fleet reach canonical source binary, repo paths LIVE +tools/gen_harvest_targets.py P8 ADAPT generate a callee-signature-aware target manifest with per-function fleet reach Generates a callee-signature-aware target manifest with per-function fleet reach canonical source binary, repo paths FROZEN tools/share_census.py tools/gen_lib_subsegs.py P3 ADAPT generate segment lines and a stub list for a multi-block vendor library region Generates segment lines and a stub list for a multi-block vendor library region repo config layout, PS1 SDK libs LIVE tools/ghidra_annotations_delta.py P2 ADAPT derive the hand-authored annotations by subtracting a fresh analysis baseline Derives the hand-authored annotation rows by subtracting a fresh rebuild baseline repo paths LIVE tools/ghidra_apply_symbols.sh P10 ADAPT mirror the curated symbol file into the analysis database with a real save Mirrors the curated symbol file into the analysis program headlessly, with a real save repo symbol path, project name STILL-NEEDED mirror the curated symbol file into the analysis database headlessly with a real save @@ -144,7 +144,7 @@ tools/idiom_hunt.py P8 ADAPT discover new compiler idioms from grouped near-miss tools/idiom_loop.py P8 ADAPT drain the easy fuel, learn the next idiom, hand off, repeat Never-ending meta-loop: drain the easy fuel, learn the next idiom, hand off repo scratch paths SUPERSEDED tools/idiom_serial.py tools/idiom_serial.py P8 ADAPT learn idioms serially so each target inherits what the previous one taught The serial idiom-learning lane where each step inherits what the previous learned repo scratch paths LIVE tools/idxtab_map.py PROJECT-ONLY PROJECT-ONLY decode an on-disc index table into a payload-to-binary naming map Decodes one game's on-disc index table and its payload-to-binary naming convention — the load map itself is a Phase-2 task project-only LIVE -tools/inject_capped_externs.py P6 ADAPT free duplicate matches a share propagation skips as not self-contained Frees high-reach inline matches skipped as not self-contained by injecting their file-scope externs repo shared-header layout STILL-NEEDED free duplicate matches a share propagation skips as not self-contained +tools/inject_capped_externs.py P6 ADAPT free duplicate matches a share propagation skips as not self-contained Frees high-reach inline matches skipped as not self-contained by injecting their file-scope externs repo shared-header layout FROZEN tools/share_body.py tools/integration_resolver.py P8 ADAPT re-judge already byte-correct drafts and bank them for zero model tokens Zero-token lane re-judging drafts that are already byte-correct and banking them repo scratch paths LIVE tools/interleave_check.py P6 ADAPT assert a read-only-data carve's interleave order equals the segment sequence Asserts the carve interleave order equals the segment sequence, position by position repo config layout STILL-NEEDED assert a read-only-data carve's interleave order equals the segment sequence tools/journal_notes.py P8 ADAPT mine each function's own past-attempt history out of agent journals Mines per-function past-attempt notes out of the agent journals into card fuel agent transcript paths LIVE @@ -184,7 +184,7 @@ tools/lenmiss_route.py P7 ADAPT route a length-drift near-miss pile through the tools/lift_types.py P6 ADAPT lift a named list of types fleet-wide into the shared type header — the canonical type layer from the first bank (Phase 6, the multiplier); again at readability (Phase 10) Lifts a named list of types fleet-wide into the shared type header repo shared-header path LIVE tools/lint_symbol_refs.py P10 ADAPT flag address-named references whose address now has a curated name Flags address-named references in committed sources whose address now has a curated name repo symbol/src paths LIVE tools/lora_grind.py P8 ADAPT rotate binaries drafting small stubs with a served model, gating and propagating Mass-run driver rotating binaries, drafting small stubs with a served model, gating, propagating repo scratch paths, model endpoint LIVE -tools/macro_draft.py P6 ADAPT materialize a shared macro body back into a compilable standalone draft Materializes a shared macro body back into a compilable standalone draft repo shared-header layout STILL-NEEDED materialize a shared macro body back into a compilable standalone draft +tools/macro_draft.py P6 ADAPT materialize a shared macro body back into a compilable standalone draft Materializes a shared macro body back into a compilable standalone draft repo shared-header layout ONE-OFF the per-function headers under src/shared/ (Phase 35 T4; the macro form is gone) tools/main_diff_locate.py P7 ADAPT turn a red whole-binary result into a named list of divergent symbols Turns a red whole-binary result into a named list of divergent symbols repo build/config layout LIVE tools/main_lane.py P8 ADAPT run the executable's own draft, gate and commit cadence beside other lanes The executable's own draft, gate and commit cadence beside the other lanes repo build/make targets LIVE tools/main_queue_rebuild.py P8 ADAPT rebuild the executable lane's queue from the tree, never from a stale list Rebuilds the executable lane's queue from the tree, never from a stale list repo src layout LIVE @@ -201,17 +201,17 @@ tools/mk_write.py P3 ADAPT write the generated binary-registry makefile safely, tools/neighbor_ref.py P8 ADAPT rank already-matched functions worth reading as worked examples for an open stub For an open stub, ranks already-matched functions worth reading as worked examples repo signature files LIVE tools/new_binary.sh P3 ADAPT onboard any flat blob in one command: config, registry entry, first build One-command onboarding of any flat blob: config from template, registry entry, first build repo config/template paths, the source game's payload classes LIVE tools/new_overlay.sh PROJECT-ONLY PROJECT-ONLY onboard one payload by its container and file-index naming Takes one game's container and file-index naming as its command-line contract project-only LIVE -tools/normalize_self_decls.py P6 ADAPT normalize a function's own declaration when a templated definition lands in a new unit Normalizes a function's own declaration when a templated definition lands in a new unit repo src layout LIVE +tools/normalize_self_decls.py P6 ADAPT normalize a function's own declaration when a templated definition lands in a new unit Normalizes a function's own declaration when a templated definition lands in a new unit repo src layout FROZEN tools/share_body.py tools/o0_boundary.py P6 ADAPT find and bank unoptimized functions stranded at an optimized region's boundary Finds and banks unoptimized functions stranded at the end of an unoptimized region repo config layout STILL-NEEDED find and bank unoptimized functions stranded at an optimized region's boundary tools/o0_detect.py P6 ADAPT answer from a prologue whether a function was built unoptimized The single place that answers whether a function was built unoptimized, from its prologue compiler prologue shape LIVE -tools/o0_subsplit.py P6 ADAPT route an unoptimized address range sitting inside an otherwise optimized object Routes an unoptimized address range that sits inside an otherwise optimized object repo config layout LIVE +tools/o0_subsplit.py P6 ADAPT route an unoptimized address range sitting inside an otherwise optimized object Routes an unoptimized address range that sits inside an otherwise optimized object repo config layout FROZEN the Makefile -O0 globs (tools/rollout_o0.py) tools/objdiff_report.py P9 PORTABLE convert progress metrics into an external progress-report schema Converts the progress JSON into the external progress-report schema external report schema LIVE tools/oracle/reg_renumber_swap.sh P7 ADAPT decide whether a residual reorder is scheduling or a register-grant consequence Mechanized oracle separating a register-allocation swap from a scheduling reorder compiler triple, repo scratch paths STILL-NEEDED decide whether a residual reorder is a scheduling choice or a register-grant consequence tools/oracle_reorder.py P5 ADAPT decide whether a near-miss residual is a source defect or an assembler artifact Decides whether a near-miss residual is a source defect or an assembler artifact pinned assembler flags LIVE tools/orchestrator.py P8 ADAPT pick the pool, emit the batch and record the cycle, deterministically The deterministic half of the unattended loop: pick the pool, emit the batch, record the cycle repo scratch paths LIVE tools/overlay_src_split.py P6 ADAPT partition a source file at object boundaries, aware of comment state Comment-state-aware partition of a source file, the oracle for the split chain repo src layout LIVE tools/ox_campaign.py P8 ADAPT loop draw, draft, pre-filter, gate, commit and ledger unattended The unattended wave loop: draw, draft, pre-filter, gate, commit, ledger, repeat repo scratch paths, provider config LIVE -tools/p16_improve.py P5 ADAPT revert known answers to stubs and categorize every harness failure it exposes Known-answer loop: reverts matched functions to stubs and measures the real pipeline repo src/build paths LIVE +tools/p16_improve.py P5 ADAPT revert known answers to stubs and categorize every harness failure it exposes Known-answer loop: reverts matched functions to stubs and measures the real pipeline repo src/build paths FROZEN tools/share_body.py tools/p16_known_answer.py P5 ADAPT validate a scaffold pipeline against known answers across difficulty bands Graduated known-answer validation of the scaffold pipeline across difficulty bands repo paths SUPERSEDED tools/p16_improve.py tools/p16_permute.py P7 ADAPT drive the permuter per function, building base, target and settings Per-function permuter driver building base, target and settings, then reporting closeness repo scratch paths, permuter layout LIVE tools/pads_audit.py P6 ADAPT derive an object's inter-table padding spec from the bytes instead of searching Derives each object's padding spec from the bytes instead of searching for it repo build/config layout STILL-NEEDED derive an object's inter-table padding spec from the bytes instead of searching @@ -252,7 +252,7 @@ tools/ram_probe.py P2 ADAPT capture and diff live console memory across scenario tools/reconcile_slate.py P6 ADAPT drive a whole batch to declaration-compatible before any rebuild is spent Drives a whole batch to declaration-compatible before any rebuild is spent repo src layout SUPERSEDED tools/pregate_check.py tools/reconcile_tu.py P6 ADAPT conform a draft's data declarations to what its destination unit can see Conforms a draft's data declarations to what the destination translation unit can actually see repo src layout LIVE tools/recover_drafts.py P8 ADAPT recover deleted agent draft files from a run's transcripts Recovers agent-written draft files from a run's transcripts after they were deleted agent transcript paths SUPERSEDED tools/agent_drafts_restore.py -tools/recover_giant.py P8 ADAPT recover a large draft rejected only on declaration plumbing, with canonical externs Canonical-extern recovery for a large draft rejected only on declaration plumbing repo shared-header layout LIVE +tools/recover_giant.py P8 ADAPT recover a large draft rejected only on declaration plumbing, with canonical externs Canonical-extern recovery for a large draft rejected only on declaration plumbing repo shared-header layout FROZEN tools/share_body.py tools/recover_integration.py P8 ADAPT drive batch integration recovery over a directory of correct-but-rejected drafts Batch integration recovery driver over a wave directory of correct-but-rejected drafts repo scratch/src layout LIVE tools/recover_rejects.py P8 ADAPT turn pre-gate rejects back into bankable drafts for zero model tokens Turns pre-gate rejects back into bankable drafts for zero model tokens repo scratch paths LIVE tools/recover_route.py P8 ADAPT name the one recovery tool that actually applies to a drop verdict Given a drop verdict, names the one recovery tool that actually applies repo scratch paths LIVE @@ -263,7 +263,7 @@ tools/residual_class.py P7 PORTABLE map a residual to a named codegen class by d tools/residual_rules.py P7 ADAPT turn a measured residual into the specific documented lever that addresses it Turns a measured residual into the specific documented lever that addresses it repo doc paths SUPERSEDED tools/residual_rules_b.py tools/residual_rules_b.py P7 ADAPT classify a residual's shape to its documented lever, as an independent implementation Independent second implementation of residual-shape to rule classification, for head-to-head repo doc paths LIVE tools/restage_matching.py P8 ADAPT re-stage only drafts that match in their real unit so one failure is isolated Re-stages only the drafts that compile and match in their real unit, so one failure is isolated repo src layout STILL-NEEDED re-stage only drafts that match in their real unit so one failure is isolated -tools/restore_dropped_decls.py P6 ADAPT restore file-scope declarations a share propagation deleted with the body Puts back file-scope declarations a share propagation deleted with the body repo src layout STILL-NEEDED restore file-scope declarations a code-share propagation deleted with the body +tools/restore_dropped_decls.py P6 ADAPT restore file-scope declarations a share propagation deleted with the body Puts back file-scope declarations a share propagation deleted with the body repo src layout FROZEN tools/share_body.py tools/rollout_o0.py P6 ADAPT roll an unoptimized definition atomically across every binary carrying it Generalized two-file atomic rollout of an unoptimized definition across every binary carrying it repo src/config layout LIVE tools/rtu_match.py P4 ADAPT splice a candidate into a copy of its real translation unit and compare bytes Splices a candidate into a copy of the real translation unit and checks the same bytes repo src layout, compiler triple LIVE tools/rtu_second_chance.py P4 ADAPT re-judge standalone compile failures against their real translation unit before dropping Re-judges standalone compile-failures against their real translation unit before dropping them repo scratch paths LIVE @@ -292,7 +292,7 @@ tools/t5_targets.py P8 ADAPT draw one routed slate from the classified frontier tools/t7_bank.py P8 ADAPT bank a batch, reconciling declarations at bank time before the whole-binary gate Batch banking driver applying reconcile-at-bank-time plus the whole-binary gate repo src/config layout SUPERSEDED tools/gate_stage.py tools/test_jtbl_parse_config.py P5 ADAPT regression-test the table-carve configuration parser with a negative control Regression plus negative control for the table-carve configuration parser repo config fixtures LIVE tools/test_o0_detect.py P5 ADAPT negative-control the detector that decides a function's optimization level Negative control for the matcher's optimization-level auto-detection repo paths STILL-NEEDED negative-control the detector that decides a function's optimization level -tools/test_reconcile_ledger.py P5 ADAPT exercise a guarded repair path the full negative control never reaches Targeted proof of the propagation ledger guard the full control never exercised repo paths STILL-NEEDED exercise a guarded repair path the full negative control never reaches +tools/test_reconcile_ledger.py P5 ADAPT exercise a guarded repair path the full negative control never reaches Targeted proof of the propagation ledger guard the full control never exercised repo paths ONE-OFF the S45p7 reconcile-ledger proof, retired with its subject dedup_propagate tools/test_residual_class.py P5 PORTABLE unit-test a residual classifier on hand-encoded words, not its own decoder Synthetic hand-encoded unit tests for the deterministic residual classifier MIPS encodings STILL-NEEDED unit-test a residual classifier on hand-encoded words, not on its own decoder tools/timeline.py P9 ADAPT build a progress timeline from the repository's own committed digests Builds a progress timeline from the repository's own committed digests, with a self-check repo doc paths LIVE tools/tool_census.py P5 ADAPT derive the tool census, the need-keyed index, the kit manifest and the verbatim corpora from one dictionary Derives the tool census, the need-keyed tool index, the kit manifest and the verbatim corpora from one dictionary, coverage-asserted both ways repo layout (tools/, docs/SETUP.md rows, the kit paths) LIVE @@ -331,3 +331,4 @@ tools/share_census.py P10 ADAPT measure duplicate function bodies across the fle tools/macro_to_header.py P10 PROJECT-ONLY convert a shared-body macro header into per-function plain-C headers included at each site The Phase-35 converter: every DEFINE_ macro body becomes a plain-C header under src/shared// included at its site, the legacy name-parameterized headers converted, the registry text-edited, the macro header deleted; --plan / --apply / --finalize / --verify the macro form is this project's; a kit-born project shares headers from its first bank LIVE tools/share_body.py P10 ADAPT share one byte-identical function class across its binaries through an include-at-site header, gated per binary The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new repo paths, the registry and signature schemas LIVE tools/share_body_cycle.sh P10 ADAPT run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new, the gated N/N line read, the phase log entry appended, the bank committed (R42), the clean fleet run every N batches (R22) — stops on the first red repo paths, the phase-log format, the fleet count from config/check.*.sha LIVE +tools/frozen.py P10 PORTABLE refuse, from one place, the command line of a tool the project has frozen The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) nothing LIVE diff --git a/decomp-architect/corpus/tools/INDEX.md b/decomp-architect/corpus/tools/INDEX.md index 027fe596bb..465ded5a59 100644 --- a/decomp-architect/corpus/tools/INDEX.md +++ b/decomp-architect/corpus/tools/INDEX.md @@ -8,8 +8,8 @@ > its platform SDK need the marked adaptation. The last table lists the tools that are project-only in code (their *shape* is a task; > their code does not transfer). *TODO(platform): the MIPS and PlayStation SDK hard-codes are the ones another platform replaces first.* > -> **Coverage:** 298 tool files in scope (submodules, vendored and downloaded code excluded), of which 298 live rows -> below; per phase: P1 2 · P2 26 · P3 17 · P4 9 · P5 27 · P6 54 · P7 20 · P8 86 · P9 27 · P10 14 · PROJECT-ONLY 16. Superseded tools appear only as pointers to their successor (28 pointer rows); one-offs are omitted. Table rows in all: 326 (the installer checks its copy against this figure). +> **Coverage:** 295 tool files in scope (submodules, vendored and downloaded code excluded), of which 295 live rows +> below; per phase: P1 2 · P2 26 · P3 17 · P4 9 · P5 26 · P6 51 · P7 20 · P8 86 · P9 27 · P10 15 · PROJECT-ONLY 16. Superseded tools appear only as pointers to their successor (30 pointer rows); one-offs are omitted. Table rows in all: 325 (the installer checks its copy against this figure). ## P1 — extraction + manifest @@ -98,7 +98,6 @@ | `oracle_reorder.py` | decide whether a near-miss residual is a source defect or an assembler artifact | Decides whether a near-miss residual is a source defect or an assembler artifact | pinned assembler flags | | `tool_census.py` | derive the tool census, the need-keyed index, the kit manifest and the verbatim corpora from one dictionary | Derives the tool census, the need-keyed tool index, the kit manifest and the verbatim corpora from one dictionary, coverage-asserted both ways | repo layout (tools/, docs/SETUP.md rows, the kit paths) | | `wall_sweep.py` | enumerate fleet-wide every instance of one known assembler-level blocker class | Enumerates fleet-wide instances of one known assembler-level blocker class | repo asm layout, toolchain quirk | -| `test_reconcile_ledger.py` | exercise a guarded repair path the full negative control never reaches | Targeted proof of the propagation ledger guard the full control never exercised | repo paths | | `blocker_probe.py` | explain with two oracles why a byte-correct draft fails the whole-binary gate | Read-only two-oracle explanation of why a byte-correct draft fails the whole-binary gate | repo build paths | | `atlas_features.py` | extract one deterministic feature record per function across every registered binary | Extracts one deterministic feature record per function across all registered binaries | repo signature files | | `worklist.py` | join the target pool and near-miss ledger into one byte-weighted queue | Joins the target pool and the near-miss ledger into one byte-weighted ranked queue | repo scratch paths | @@ -138,7 +137,6 @@ | `pads_audit.py` | derive an object's inter-table padding spec from the bytes instead of searching | Derives each object's padding spec from the bytes instead of searching for it | repo build/config layout | | `aprop_autodraft.py` | draft a family member mechanically from a seed body and symbol rebase | Mechanically drafts a family member from the seed body plus a positional symbol rebase | repo scratch paths | | `demacroize.py` | escape a shared-header declaration conflict locally by unmacroizing one body | Per-binary local escape from a shared-header declaration conflict by unmacroizing one body | repo shared-header layout | -| `dedup_extend.py` | extend an existing code-share registry to newly onboarded binaries | Extends the existing share registry to newly onboarded binaries | repo config/src layout | | `o0_boundary.py` | find and bank unoptimized functions stranded at an optimized region's boundary | Finds and banks unoptimized functions stranded at the end of an unoptimized region | repo config layout | | `inject_capped_externs.py` | free duplicate matches a share propagation skips as not self-contained | Frees high-reach inline matches skipped as not self-contained by injecting their file-scope externs | repo shared-header layout | | `decl_from_use.py` | infer a minimal extern for a data symbol the destination unit does not declare | Infers a minimal extern for a data symbol a draft uses but its destination does not declare | repo src layout | @@ -146,8 +144,6 @@ | `match_protos.py` | join two related builds' signature dumps into a function correspondence | Joins per-function signature dumps of two related builds into a function correspondence | repo signature files | | `cast_self_callers.py` | let a unit keep calling, through per-site casts, the function it now defines | Lets a unit keep calling, through a per-site cast, the function it is about to define | repo src layout | | `lift_types.py` | lift a named list of types fleet-wide into the shared type header — the canonical type layer from the first bank (Phase 6, the multiplier); again at readability (Phase 10) | Lifts a named list of types fleet-wide into the shared type header | repo shared-header path | -| `dedup_propagate.py` | lift one matched body into a shared macro and instantiate it everywhere | Lifts one matched body into a shared macro and instantiates it at every duplicate site | repo shared-header/src layout | -| `macro_draft.py` | materialize a shared macro body back into a compilable standalone draft | Materializes a shared macro body back into a compilable standalone draft | repo shared-header layout | | `scope_tu_externs.py` | move a unit's file-scope data externs down into their consumers | Moves a unit's own file-scope data externs down into their consumers to legalize a block-scope type | repo src layout | | `family_manifest.py` | name the one exemplar to draft per structural family, ranked | Ranked structural-family target manifest naming the one exemplar to draft per family | repo signature files | | `normalize_self_decls.py` | normalize a function's own declaration when a templated definition lands in a new unit | Normalizes a function's own declaration when a templated definition lands in a new unit | repo src layout | @@ -338,6 +334,7 @@ | `verbatim_check.py` | guard that every inline-assembly body still reproduces its target bytes | Regression guard that every inline-assembly body still reproduces its target bytes | repo src layout | | `share_census.py` | measure duplicate function bodies across the fleet and assert one source per unique function | The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test | repo paths, the registry and signature schemas | | `ghidra_apply_symbols.sh` | mirror the curated symbol file into the analysis database with a real save | Mirrors the curated symbol file into the analysis program headlessly, with a real save | repo symbol path, project name | +| `frozen.py` | refuse, from one place, the command line of a tool the project has frozen | The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) | nothing | | `verbatim_target_s.py` | regenerate a splitter-format target disassembly for a function no longer stubbed | Regenerates a splitter-format target disassembly for a function that is no longer a stub; --gas emits the assemblable gas-syntax form ($-registers, .L labels, noreorder) that a web diff service accepts as a pasted target | repo build/asm layout | | `share_body_cycle.sh` | run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check | The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new, the gated N/N line read, the phase log entry appended, the bank committed (R42), the clean fleet run every N batches (R22) — stops on the first red | repo paths, the phase-log format, the fleet count from config/check.*.sha | | `share_body.py` | share one byte-identical function class across its binaries through an include-at-site header, gated per binary | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | repo paths, the registry and signature schemas | @@ -374,6 +371,8 @@ | `build_wave.py` | `draw_waves.py` | Builds the next wave from a card pool, ranked by fleet leverage | | `build_wave_args.py` | `wave_args.py` | Emits workflow arguments for a wave from the target manifest, resolving each path | | `canon_draft_decls.py` | `sig_unify.py` | Canonicalizes a draft's extern and data declarations to the banked-consistent set | +| `dedup_extend.py` | `share_body.py` | Extends the existing share registry to newly onboarded binaries | +| `dedup_propagate.py` | `share_body.py` | Lifts one matched body into a shared macro and instantiates it at every duplicate site | | `exemplar_miner.py` | `atlas.py` | Routes every residual stub to its lever and ranks the highest-reach work | | `fix_header_decl.py` | `conform_decls.py` | Rewrites a shared-header caller declaration to a draft's byte-true signature | | `gen_engine_decls.py` | `conform_decls.py` | Declares every shared function and data symbol once, canonically, in a generated header | diff --git a/decomp-architect/corpus/tools/P10/frozen.py b/decomp-architect/corpus/tools/P10/frozen.py new file mode 100644 index 0000000000..3fcfe06501 --- /dev/null +++ b/decomp-architect/corpus/tools/P10/frozen.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""frozen.py — the ONE refusal every FROZEN tool prints from its main() (Phase 35 T6, 2026-09-08). + +A frozen tool is a matching-era tool that reads or writes the `DEFINE_func_` macro form Phase 35 retired (the shared bodies are +plain-C headers under src/shared/ now, one source per unique function — the S1 invariant). Its code stays in tools/ for the record +and for the libraries other tools import from it; only its command line refuses, so an import never exits (cdecl imports +gen_harvest_targets; eighteen tools import family_remap). The dictionary row carries status FROZEN and the successor. + + import frozen; frozen.refuse("family_sweep.py", "tools/share_body.py", "it rewrites the macro form Phase 35 retired") +""" +import sys + + +def refuse(tool, successor, why): + sys.exit(f"{tool}: FROZEN since Phase 35 (2026-09-08) — {why}. Successor: {successor}. The code is kept for the record and its " + f"library functions still import; the command line does nothing (R43). See docs/SETUP.md and config/tool_dictionary.tsv.") diff --git a/decomp-architect/corpus/tools/P10/share_body.py b/decomp-architect/corpus/tools/P10/share_body.py index e4e8291e82..6ee2803e0d 100644 --- a/decomp-architect/corpus/tools/P10/share_body.py +++ b/decomp-architect/corpus/tools/P10/share_body.py @@ -11,7 +11,7 @@ WHAT ONE SHARE IS. A class = one h_exact (the raw instruction bytes) at one vram are still PRIVATE COPIES (a definition in that binary's own TU). Sharing it: the class's ONE body goes to (or already is) a plain-C header under src/shared// (macro_to_header's naming: func_[__h8].h, keyed by the class — R48), every private copy becomes `#include "../shared//
"` at the SAME position (the copy's definition lines, nothing else — the TU keeps its own -declarations), and the registry gains the group (shorthand form, appended by text) or the members (dedup_extend.add_members_surgical). +declarations), and the registry gains the group (shorthand form, appended by text) or the members (add_members_surgical, inherited from the retired dedup_extend). THE EXEMPLAR (for an unregistered class) — printed with every share: the copy whose normalized text the MOST copies share; a tie goes to a pin-free copy; a further tie to the shortest. A body defined under an asm-label alias (aF) gets its own binding @@ -47,12 +47,102 @@ REPO = pathlib.Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO / "tools")) import share_census as sc # noqa: E402 import macro_to_header as m2h # noqa: E402 — Oracles (naming, spaces, sigs), bind_alias_header, include_line, banner -import dedup_extend as de # noqa: E402 — add_members_surgical (the registry's surgical text edit) LEDGER = REPO / "config/dedup_exceptions.tsv" RUN = REPO / ".run/P35/share" +# ---------------------------------------------------------------------------------------------------------------------------- +# The library surface inherited from the retired propagate/extend tools in tools/sunset (Phase 35 T6; verbatim, so their importers +# change one line: `import share_body as dp`). load_sig/sig_path read the per-binary sig files; registered_addrs the registry. +def sig_path(ov): + return REPO / f".run/sig.{ov}.jsonl" + + +def onboarded_overlays(): + """Overlay aliases + (S44) module aliases — every non-main/resident binary a shared body can + propagate into. Modules carry engine functions too; excluding them re-creates the SC07 + invisible-work bug one class over.""" + out = [] + for f, var in (("config/overlays.mk", "OVERLAY_BINARIES"), ("config/modules.mk", "MODULE_BINARIES")): + fp = REPO / f + if fp.exists(): + m = re.search(rf"^{var}\s*:=\s*(.*)$", fp.read_text(), re.M) + if m: + out += m.group(1).split() + return out +def load_sig(ov): + """addr(int) -> {h_exact, h_norm, nins, ...}; {} if absent.""" + p = sig_path(ov) + if not p.exists(): + return {} + out = {} + for ln in p.read_text().splitlines(): + ln = ln.strip() + if ln: + r = json.loads(ln) + out[int(r["addr"], 16)] = r + return out +def registered_addrs(): + """Vaddrs already in config/dedup.us.yaml (shared via ANY mechanism — engine_core, ov_setters, + clearTbl40). --auto-from skips these so the bulk is purely additive and never collides with an + existing share (e.g. a SETTER-matched function) — which would trip the structural check.""" + p = REPO / "config/dedup.us.yaml" + if not p.exists(): + return set() + try: + import yaml + sys.path.insert(0, str(REPO / "tools")) + from dedup_integrate import group_members + data = yaml.safe_load(p.read_text()) or {} + return {v for g in (data.get("groups") or []) for (_b, v, _n) in group_members(g)} + except Exception: + return set() +def sym(addr): + return f"func_{addr:08X}" # splat convention: uppercase 8-hex + + +def add_members_surgical(additions): + """Append binaries to each group's `binaries: [...]` list by TEXT EDIT, in place. + + NEVER `yaml.safe_dump` this file. The first cut of this tool round-tripped it through + safe_dump and silently destroyed BOTH of the things a human needs from it (H5 — "never + silently drop comments on a rewrite"): + * all 47 comment lines — including the curated Phase-11 header explaining WHY the share is + source-level (the linker cannot excise bytes interior to an object) — dumped to nothing; + * every `vram: 0x80162FF4` re-serialized as `vram: 2148937716` (PyYAML parses YAML-1.1 hex + to int, and dumps int as decimal), making 1,832 entries unreadable. + It was invisible to every gate: dedup-check passed 1840/0 and check-all stayed 140/140, + because `_addr()` accepts both forms — the data was fine and the DOCUMENT was ruined. A + formatting-destructive write that all your oracles call green is exactly the class this + project keeps re-learning: the gate measures bytes, not intent. + + `additions` = {group_id: [binary, ...]}. Idempotent: a binary already listed is skipped. + """ + p = str(REPO / "config/dedup.us.yaml") + lines = open(p).read().splitlines(keepends=True) + cur, n = None, 0 + seen = set() + for i, ln in enumerate(lines): + m = re.match(r"^\s*-?\s*id:\s*(\S+)\s*$", ln) + if m: + cur = m.group(1) + continue + if cur and cur in additions and re.match(r"^\s*members:\s*$", ln): + # a VERBOSE group has no `binaries:` line — silently skipping it left five 141-member groups listed at 16 (P35 S94/S96) + raise SystemExit(f"add_members_surgical: {cur} is in the verbose `members:` form — convert it to shorthand first " + f"(share_body.py --repair-registry does), never skip it (R43)") + if cur and cur in additions and re.match(r"^\s*binaries:\s*\[", ln): + seen.add(cur) + add = [b for b in additions[cur] if re.search(rf"\b{re.escape(b)}\b", ln) is None] + if add: + lines[i] = ln.rstrip("\n").rstrip()[:-1].rstrip() + ", " + ", ".join(add) + "]\n" + n += len(add) + cur = None + open(p, "w").write("".join(lines)) + return n + + def log(msg): print(msg, flush=True) @@ -397,7 +487,7 @@ def run_batch(orc, cen, classes, label): bins = [x for x in b.extend.get(gid, []) if x not in rejected_bins.get(c["h"], set())] if bins: ext[gid] = bins - n_ext = de.add_members_surgical(ext) if ext else 0 + n_ext = add_members_surgical(ext) if ext else 0 ok_n = sum(1 for a in gated if results[a][0]) log(f" [{label}] gated {ok_n}/{len(gated)} binaries green · registered {len(reg_entries)} groups · extended {n_ext} members · " f"rejected classes {len(failed_classes)}") diff --git a/decomp-architect/corpus/tools/P5/audit_binaries.py b/decomp-architect/corpus/tools/P5/audit_binaries.py index 93c81ca6e9..9a188e9094 100644 --- a/decomp-architect/corpus/tools/P5/audit_binaries.py +++ b/decomp-architect/corpus/tools/P5/audit_binaries.py @@ -176,7 +176,7 @@ def main(): zero = sorted(b for b in onb if b.startswith("ov_") and memb.get(b, 0) == 0) if zero: warns.append(f"{len(zero)} overlay(s) in 0 dedup groups (onboarded but un-harvested — " - f"candidate for tools/dedup_extend.py): {zero if verbose else zero[:6]}") + f"candidate for tools/share_body.py --plan): {zero if verbose else zero[:6]}") # --- report n_ov = len([b for b in onb if b.startswith("ov_")]) diff --git a/decomp-architect/corpus/tools/P5/blocker_probe.py b/decomp-architect/corpus/tools/P5/blocker_probe.py index af8e363562..e22277be89 100644 --- a/decomp-architect/corpus/tools/P5/blocker_probe.py +++ b/decomp-architect/corpus/tools/P5/blocker_probe.py @@ -223,6 +223,7 @@ def cc1_verdict(fn, stub, draft_path, work): # --------------------------------------------------------------------------------------------- def main(): + import frozen; frozen.refuse("blocker_probe.py", "tools/share_body.py (its per-binary gate names the blocker)", "it probes macro-scope blockers in engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--binary', required=True) diff --git a/decomp-architect/corpus/tools/P5/p16_improve.py b/decomp-architect/corpus/tools/P5/p16_improve.py index 1962f6470f..8aad96e853 100644 --- a/decomp-architect/corpus/tools/P5/p16_improve.py +++ b/decomp-architect/corpus/tools/P5/p16_improve.py @@ -66,6 +66,7 @@ def classify_fail(fn, cpath): def main(): + import frozen; frozen.refuse("p16_improve.py", "tools/share_body.py", "it rewrites macro sites in ov_SC01_077 against engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser() ap.add_argument("--band", choices=list(BANDS), default="easy") ap.add_argument("--n", type=int, default=40) diff --git a/decomp-architect/corpus/tools/P5/tool_census.py b/decomp-architect/corpus/tools/P5/tool_census.py index 8f51336bf5..53fe1fabe3 100644 --- a/decomp-architect/corpus/tools/P5/tool_census.py +++ b/decomp-architect/corpus/tools/P5/tool_census.py @@ -92,7 +92,7 @@ PHASE_TITLES = {"P1": "extraction + manifest", "P2": "the oracles + the load map "P7": "the codegen map, the dumps, the permuter", "P8": "the campaign: cards, lanes, gates, recovery, harvest", "P9": "publish", "P10": "readability", "PROJECT-ONLY": "project-only in code (the shape is a task; the code does not transfer)"} COLS = ["path", "phase", "portability", "need", "what", "adapts", "status", "successor_or_product"] -STATUSES = {"LIVE", "STILL-NEEDED", "SUPERSEDED", "ONE-OFF"} +STATUSES = {"LIVE", "STILL-NEEDED", "FROZEN", "SUPERSEDED", "ONE-OFF"} # FROZEN (P35 T6): in the tree, command line refuses, libraries import def sh(cmd): @@ -191,7 +191,7 @@ def census(): files, retired = enumerate_tools() rows = read_dict() by_path = {r["path"]: r for r in rows} - live_rows = {p for p, r in by_path.items() if r["status"] in ("LIVE", "STILL-NEEDED")} + live_rows = {p for p, r in by_path.items() if r["status"] in ("LIVE", "STILL-NEEDED", "FROZEN")} retired_rows = {p for p, r in by_path.items() if r["status"] in ("SUPERSEDED", "ONE-OFF")} gaps = [] for f in files: diff --git a/decomp-architect/corpus/tools/P6/aprop_autodraft.py b/decomp-architect/corpus/tools/P6/aprop_autodraft.py index 5eb6c6aa4b..e604827b9c 100644 --- a/decomp-architect/corpus/tools/P6/aprop_autodraft.py +++ b/decomp-architect/corpus/tools/P6/aprop_autodraft.py @@ -574,6 +574,7 @@ def _local_verdict(fn, text, sub, o0): def main(): + import frozen; frozen.refuse("aprop_autodraft.py", "tools/share_body.py", "it seeds drafts from the macro bodies in engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser() ap.add_argument("--cards", default=CARDS) ap.add_argument("--outdir", default=".run/aprop_autodraft") diff --git a/decomp-architect/corpus/tools/P6/canon_sig_reconcile.py b/decomp-architect/corpus/tools/P6/canon_sig_reconcile.py index 1374f7a5a1..f6004783cb 100644 --- a/decomp-architect/corpus/tools/P6/canon_sig_reconcile.py +++ b/decomp-architect/corpus/tools/P6/canon_sig_reconcile.py @@ -518,6 +518,7 @@ def reconcile(fn, draft, canon_sig=None, tu_path=None): def main(): + import frozen; frozen.refuse("canon_sig_reconcile.py", "the types phase (Phase 37): one canonical declaration per symbol", "it reconciles declarations against engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser() ap.add_argument('--fn', required=True) ap.add_argument('--draft', required=True) diff --git a/decomp-architect/corpus/tools/P6/conform_decls.py b/decomp-architect/corpus/tools/P6/conform_decls.py index f25f570b78..75adec8e90 100644 --- a/decomp-architect/corpus/tools/P6/conform_decls.py +++ b/decomp-architect/corpus/tools/P6/conform_decls.py @@ -207,6 +207,7 @@ def consumers(fn): def main(): + import frozen; frozen.refuse("conform_decls.py", "the types phase (Phase 37)", "it conforms the fleet's declarations to engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--fn", required=True) diff --git a/decomp-architect/corpus/tools/P6/dedup_extend.py.superseded.md b/decomp-architect/corpus/tools/P6/dedup_extend.py.superseded.md new file mode 100644 index 0000000000..6a4f27d679 --- /dev/null +++ b/decomp-architect/corpus/tools/P6/dedup_extend.py.superseded.md @@ -0,0 +1,3 @@ +# dedup_extend.py — superseded + +Superseded by `share_body.py` (see its row in the index). What it did: Extends the existing share registry to newly onboarded binaries diff --git a/decomp-architect/corpus/tools/P6/dedup_propagate.py.superseded.md b/decomp-architect/corpus/tools/P6/dedup_propagate.py.superseded.md new file mode 100644 index 0000000000..9a09555bc2 --- /dev/null +++ b/decomp-architect/corpus/tools/P6/dedup_propagate.py.superseded.md @@ -0,0 +1,3 @@ +# dedup_propagate.py — superseded + +Superseded by `share_body.py` (see its row in the index). What it did: Lifts one matched body into a shared macro and instantiates it at every duplicate site diff --git a/decomp-architect/corpus/tools/P6/demacroize.py b/decomp-architect/corpus/tools/P6/demacroize.py index 6f82a681a7..f95b8a3a97 100644 --- a/decomp-architect/corpus/tools/P6/demacroize.py +++ b/decomp-architect/corpus/tools/P6/demacroize.py @@ -122,6 +122,7 @@ def plan(binary, fn, draft_text): def main(): + import frozen; frozen.refuse("demacroize.py", "tools/share_body.py", "it turns a macro site back into a private copy — the opposite of the S1 invariant") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--binary', required=True) diff --git a/decomp-architect/corpus/tools/P6/family_remap.py b/decomp-architect/corpus/tools/P6/family_remap.py index 7289f36050..9e38fd8cce 100644 --- a/decomp-architect/corpus/tools/P6/family_remap.py +++ b/decomp-architect/corpus/tools/P6/family_remap.py @@ -1322,6 +1322,7 @@ def remap(addr, from_ov, to_ov, to_addr=None, imm_map=None): def main(): + import frozen; frozen.refuse("family_remap.py", "tools/share_body.py --reexemplar / --apply", "its command line writes a DEFINE_func_ macro head; the library (stream_words, nins_of, …) stays") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--addr", required=True) ap.add_argument("--from", dest="frm", required=True) diff --git a/decomp-architect/corpus/tools/P6/family_sweep.py b/decomp-architect/corpus/tools/P6/family_sweep.py index d62eb55591..8442af77a2 100644 --- a/decomp-architect/corpus/tools/P6/family_sweep.py +++ b/decomp-architect/corpus/tools/P6/family_sweep.py @@ -876,6 +876,7 @@ def hseq_sweep(a): def main(): + import frozen; frozen.refuse("family_sweep.py", "tools/share_body.py", "it remaps and rewrites the DEFINE_func_ macro bodies Phase 35 retired") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--source", default="ov_SC01_077") ap.add_argument("--limit", type=int, default=0, help="cap #exemplars (0 = all)") diff --git a/decomp-architect/corpus/tools/P6/inject_capped_externs.py b/decomp-architect/corpus/tools/P6/inject_capped_externs.py index 7714694d81..c28ea8ebea 100644 --- a/decomp-architect/corpus/tools/P6/inject_capped_externs.py +++ b/decomp-architect/corpus/tools/P6/inject_capped_externs.py @@ -30,7 +30,7 @@ import argparse, re, sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -import dedup_propagate as dp +dp = None # the propagate library is retired to tools/sunset (P35 T6); this tool is FROZEN — main() refuses SYM_RE = re.compile(r'\b(func_[0-9A-Fa-f]{6,8}|D_[0-9A-Fa-f]{6,8})\b') @@ -81,6 +81,7 @@ def inject(defblock, name, ext_map): def main(): + import frozen; frozen.refuse("inject_capped_externs.py", "tools/share_body.py", "it fixed drafts a macro-era propagation skipped, on dedup_propagate (retired)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--overlay", default="ov_SC01_077") diff --git a/decomp-architect/corpus/tools/P6/normalize_self_decls.py b/decomp-architect/corpus/tools/P6/normalize_self_decls.py index bb339fecad..c4891ffd11 100644 --- a/decomp-architect/corpus/tools/P6/normalize_self_decls.py +++ b/decomp-architect/corpus/tools/P6/normalize_self_decls.py @@ -191,6 +191,7 @@ def fix(tu_text, fn, ref_decl=None): def main(): + import frozen; frozen.refuse("normalize_self_decls.py", "tools/share_body.py", "it derives canonical declarations from engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--tu', required=True, help='the TU file to normalize (edited in place unless --out)') ap.add_argument('--fn', required=True, help='func_XXXXXXXX to normalize decls of') diff --git a/decomp-architect/corpus/tools/P6/o0_subsplit.py b/decomp-architect/corpus/tools/P6/o0_subsplit.py index 5b68e152c9..5adbe4a326 100644 --- a/decomp-architect/corpus/tools/P6/o0_subsplit.py +++ b/decomp-architect/corpus/tools/P6/o0_subsplit.py @@ -62,6 +62,7 @@ def free_letters(ov, n): def main(): + import frozen; frozen.refuse("o0_subsplit.py", "the Makefile -O0 globs (rollout_o0 for the record)", "it splits -O0 macro sites out of a TU against the macro form") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("ov") diff --git a/decomp-architect/corpus/tools/P6/restore_dropped_decls.py b/decomp-architect/corpus/tools/P6/restore_dropped_decls.py index 316e43ef31..87c3d30508 100644 --- a/decomp-architect/corpus/tools/P6/restore_dropped_decls.py +++ b/decomp-architect/corpus/tools/P6/restore_dropped_decls.py @@ -71,6 +71,7 @@ def insert_at_file_scope(path, decls): def main(): + import frozen; frozen.refuse("restore_dropped_decls.py", "tools/share_body.py", "it repairs declarations a macro-era propagation deleted; share_body edits definition lines only") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--binary", required=True) ap.add_argument("--ref", required=True, help="git ref holding the pre-deletion sources") diff --git a/decomp-architect/corpus/tools/P7/grinder.py b/decomp-architect/corpus/tools/P7/grinder.py index 04da6a55d1..6a1663d6df 100644 --- a/decomp-architect/corpus/tools/P7/grinder.py +++ b/decomp-architect/corpus/tools/P7/grinder.py @@ -348,8 +348,8 @@ def main(): f"total {banked}; fleet {fp}%") if s.get("verified"): log(" -> propagate as its own batch: " - + "; ".join("tools/dedup_propagate.py --addr 0x%s --recover" % f.split('_')[-1] - for f in s["verified"][:4])) + + "tools/share_body.py --apply --bucket new --batches 1 (P35 T6: shares every new same-address class; " + + "verified: " + " ".join(s["verified"][:4]) + ")") # A permuter win the whole-binary gate STILL rejects is plumbing-bound (not regalloc/sched) — # re-permuting can never bank it. Blacklist so the grinder stops churning it (the §20 trap). rejected = [f for f, _b in won if f not in verified] diff --git a/decomp-architect/corpus/tools/P8/auto_driver.py b/decomp-architect/corpus/tools/P8/auto_driver.py index 6d7688e6d9..6d62f12708 100644 --- a/decomp-architect/corpus/tools/P8/auto_driver.py +++ b/decomp-architect/corpus/tools/P8/auto_driver.py @@ -137,7 +137,7 @@ def byte_gate(draft_dir): def propagate(): - sh([PY, "tools/dedup_propagate.py", "--auto-from", OV, "--min-reach", "2"], timeout=3600) + sh([PY, "tools/share_body.py", "--apply", "--bucket", "new", "--batches", "1"], timeout=3600) # P35 T6: the include-at-site share def main(): diff --git a/decomp-architect/corpus/tools/P8/bulk_harvest.py b/decomp-architect/corpus/tools/P8/bulk_harvest.py index 2e9f16faf7..18e23a1b40 100644 --- a/decomp-architect/corpus/tools/P8/bulk_harvest.py +++ b/decomp-architect/corpus/tools/P8/bulk_harvest.py @@ -264,7 +264,7 @@ def run_cycle(a, api_base, model, tried): for b, fns in banked_by_bin.items(): if any((lg.reach_of(b, fn) or 1) >= 2 for fn in fns): before = gate_stage._dedup_group_count() - sh([PY, "tools/dedup_propagate.py", "--auto-from", b, "--min-reach", "2"], timeout=3600) + sh([PY, "tools/share_body.py", "--apply", "--bucket", "new", "--batches", "1"], timeout=3600) # P35 T6 propagated += max(0, gate_stage._dedup_group_count() - before) merge_backlogs() fp = fleet_pct() diff --git a/decomp-architect/corpus/tools/P8/gate_lane.py b/decomp-architect/corpus/tools/P8/gate_lane.py index 2cd4fadb40..19a3587929 100644 --- a/decomp-architect/corpus/tools/P8/gate_lane.py +++ b/decomp-architect/corpus/tools/P8/gate_lane.py @@ -83,7 +83,7 @@ json.dump(banked,open(outp.replace('.json','_banked.json'),'w')) # propagate per function for fn in banked: addr="0x"+fn.split("_")[1].lower() - r=subprocess.run([".venv/bin/python","tools/dedup_propagate.py","--addr",addr], + r=subprocess.run([".venv/bin/python","tools/share_body.py","--apply","--bucket","new","--batches","1"], # P35 T6 capture_output=True,text=True) if dirty(): tag=os.environ.get("GATE_PHASE","decomp") # P31 T1: was hardcoded "phase-30 S49" diff --git a/decomp-architect/corpus/tools/P8/gate_stage.py b/decomp-architect/corpus/tools/P8/gate_stage.py index e8f181c24d..21c8a7d67a 100644 --- a/decomp-architect/corpus/tools/P8/gate_stage.py +++ b/decomp-architect/corpus/tools/P8/gate_stage.py @@ -519,10 +519,9 @@ def _run_gate_locked(drafts, binary, src, asm, out, good_sha, propagate, source_ # cover write scope, and a killed process performs no undo at all). _budget = min(6 * 3600, 1800 + 1800 * len(verified)) try: - pr = sh([PY, "tools/dedup_propagate.py", "--auto-from", binary, "--min-reach", "2", - "--recover"], timeout=_budget) + pr = sh([PY, "tools/share_body.py", "--apply", "--bucket", "new", "--batches", "1"], timeout=_budget) # P35 T6: the include-at-site share (dedup_propagate retired) except subprocess.TimeoutExpired: - print(f"[gate] FATAL: dedup_propagate exceeded {_budget}s with {len(verified)} banks — " + print(f"[gate] FATAL: share_body exceeded {_budget}s with {len(verified)} banks — " f"the fleet is HALF-PROPAGATED and the tree is DIRTY. Revert (`git checkout -- src/`), " f"then re-gate with --no-propagate and propagate separately.", file=sys.stderr) return {"drafts": len(draft_fns), "banked": len(verified), "propagated": 0, diff --git a/decomp-architect/corpus/tools/P8/gen_harvest_targets.py b/decomp-architect/corpus/tools/P8/gen_harvest_targets.py index 550e261e92..b67e246b04 100644 --- a/decomp-architect/corpus/tools/P8/gen_harvest_targets.py +++ b/decomp-architect/corpus/tools/P8/gen_harvest_targets.py @@ -152,6 +152,7 @@ def collect_stubs(c_path): def main(): + import frozen; frozen.refuse("gen_harvest_targets.py", "tools/share_census.py", "it collects harvest targets from the macro header engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser() ap.add_argument('--source', default='ov_SC01_077') ap.add_argument('--min-reach', type=int, default=2) diff --git a/decomp-architect/corpus/tools/P8/lora_grind.py b/decomp-architect/corpus/tools/P8/lora_grind.py index 9a6aa5661c..f27268fef5 100644 --- a/decomp-architect/corpus/tools/P8/lora_grind.py +++ b/decomp-architect/corpus/tools/P8/lora_grind.py @@ -227,8 +227,8 @@ def main(): # periodic fleet propagate sweep (the multiplier) — every N batches if batch_i % a.propagate_every == 0 and propagated_since: for pb in sorted(propagated_since): - subprocess.run([".venv/bin/python", "tools/dedup_propagate.py", "--auto-from", pb, - "--min-reach", "2"], cwd=REPO, capture_output=True, timeout=3600) + subprocess.run([".venv/bin/python", "tools/share_body.py", "--apply", "--bucket", "new", + "--batches", "1"], cwd=REPO, capture_output=True, timeout=3600) # P35 T6 propagated_since.clear() log("propagate sweep done") # heartbeat + flywheel stats. progress.py --fleet is ~14s (a full 136-binary scan), so diff --git a/decomp-architect/corpus/tools/P8/recover_giant.py b/decomp-architect/corpus/tools/P8/recover_giant.py index 7482bddf80..5ed4799017 100644 --- a/decomp-architect/corpus/tools/P8/recover_giant.py +++ b/decomp-architect/corpus/tools/P8/recover_giant.py @@ -69,6 +69,7 @@ def recover(fn, src, ec): def main(): + import frozen; frozen.refuse("recover_giant.py", "tools/share_body.py", "it recovers a giant into the macro header engine_core.h (gone)") # Phase 35 T6: FROZEN if len(sys.argv) != 4: raise SystemExit(__doc__) fn, inp, outp = sys.argv[1], sys.argv[2], sys.argv[3] diff --git a/decomp-architect/corpus/tools/P8/validate_targets.py b/decomp-architect/corpus/tools/P8/validate_targets.py index 5eb64238b9..f3f377dd91 100644 --- a/decomp-architect/corpus/tools/P8/validate_targets.py +++ b/decomp-architect/corpus/tools/P8/validate_targets.py @@ -41,7 +41,7 @@ import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import corpus -import dedup_propagate as DP +import share_body as DP # load_sig lives in share_body since dedup_propagate retired (P35 T6) REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) diff --git a/decomp-architect/tools/MANIFEST.md b/decomp-architect/tools/MANIFEST.md index 69bc237584..8c3b118310 100644 --- a/decomp-architect/tools/MANIFEST.md +++ b/decomp-architect/tools/MANIFEST.md @@ -8,8 +8,8 @@ > its platform SDK need the marked adaptation. The last table lists the tools that are project-only in code (their *shape* is a task; > their code does not transfer). *TODO(platform): the MIPS and PlayStation SDK hard-codes are the ones another platform replaces first.* > -> **Coverage:** 298 tool files in scope (submodules, vendored and downloaded code excluded), of which 298 live rows -> below; per phase: P1 2 · P2 26 · P3 17 · P4 9 · P5 27 · P6 54 · P7 20 · P8 86 · P9 27 · P10 14 · PROJECT-ONLY 16. Superseded tools appear only as pointers to their successor (28 pointer rows); one-offs are omitted. Table rows in all: 326 (the installer checks its copy against this figure). +> **Coverage:** 295 tool files in scope (submodules, vendored and downloaded code excluded), of which 295 live rows +> below; per phase: P1 2 · P2 26 · P3 17 · P4 9 · P5 26 · P6 51 · P7 20 · P8 86 · P9 27 · P10 15 · PROJECT-ONLY 16. Superseded tools appear only as pointers to their successor (30 pointer rows); one-offs are omitted. Table rows in all: 325 (the installer checks its copy against this figure). ## P1 — extraction + manifest @@ -98,7 +98,6 @@ | `oracle_reorder.py` | decide whether a near-miss residual is a source defect or an assembler artifact | Decides whether a near-miss residual is a source defect or an assembler artifact | pinned assembler flags | | `tool_census.py` | derive the tool census, the need-keyed index, the kit manifest and the verbatim corpora from one dictionary | Derives the tool census, the need-keyed tool index, the kit manifest and the verbatim corpora from one dictionary, coverage-asserted both ways | repo layout (tools/, docs/SETUP.md rows, the kit paths) | | `wall_sweep.py` | enumerate fleet-wide every instance of one known assembler-level blocker class | Enumerates fleet-wide instances of one known assembler-level blocker class | repo asm layout, toolchain quirk | -| `test_reconcile_ledger.py` | exercise a guarded repair path the full negative control never reaches | Targeted proof of the propagation ledger guard the full control never exercised | repo paths | | `blocker_probe.py` | explain with two oracles why a byte-correct draft fails the whole-binary gate | Read-only two-oracle explanation of why a byte-correct draft fails the whole-binary gate | repo build paths | | `atlas_features.py` | extract one deterministic feature record per function across every registered binary | Extracts one deterministic feature record per function across all registered binaries | repo signature files | | `worklist.py` | join the target pool and near-miss ledger into one byte-weighted queue | Joins the target pool and the near-miss ledger into one byte-weighted ranked queue | repo scratch paths | @@ -138,7 +137,6 @@ | `pads_audit.py` | derive an object's inter-table padding spec from the bytes instead of searching | Derives each object's padding spec from the bytes instead of searching for it | repo build/config layout | | `aprop_autodraft.py` | draft a family member mechanically from a seed body and symbol rebase | Mechanically drafts a family member from the seed body plus a positional symbol rebase | repo scratch paths | | `demacroize.py` | escape a shared-header declaration conflict locally by unmacroizing one body | Per-binary local escape from a shared-header declaration conflict by unmacroizing one body | repo shared-header layout | -| `dedup_extend.py` | extend an existing code-share registry to newly onboarded binaries | Extends the existing share registry to newly onboarded binaries | repo config/src layout | | `o0_boundary.py` | find and bank unoptimized functions stranded at an optimized region's boundary | Finds and banks unoptimized functions stranded at the end of an unoptimized region | repo config layout | | `inject_capped_externs.py` | free duplicate matches a share propagation skips as not self-contained | Frees high-reach inline matches skipped as not self-contained by injecting their file-scope externs | repo shared-header layout | | `decl_from_use.py` | infer a minimal extern for a data symbol the destination unit does not declare | Infers a minimal extern for a data symbol a draft uses but its destination does not declare | repo src layout | @@ -146,8 +144,6 @@ | `match_protos.py` | join two related builds' signature dumps into a function correspondence | Joins per-function signature dumps of two related builds into a function correspondence | repo signature files | | `cast_self_callers.py` | let a unit keep calling, through per-site casts, the function it now defines | Lets a unit keep calling, through a per-site cast, the function it is about to define | repo src layout | | `lift_types.py` | lift a named list of types fleet-wide into the shared type header — the canonical type layer from the first bank (Phase 6, the multiplier); again at readability (Phase 10) | Lifts a named list of types fleet-wide into the shared type header | repo shared-header path | -| `dedup_propagate.py` | lift one matched body into a shared macro and instantiate it everywhere | Lifts one matched body into a shared macro and instantiates it at every duplicate site | repo shared-header/src layout | -| `macro_draft.py` | materialize a shared macro body back into a compilable standalone draft | Materializes a shared macro body back into a compilable standalone draft | repo shared-header layout | | `scope_tu_externs.py` | move a unit's file-scope data externs down into their consumers | Moves a unit's own file-scope data externs down into their consumers to legalize a block-scope type | repo src layout | | `family_manifest.py` | name the one exemplar to draft per structural family, ranked | Ranked structural-family target manifest naming the one exemplar to draft per family | repo signature files | | `normalize_self_decls.py` | normalize a function's own declaration when a templated definition lands in a new unit | Normalizes a function's own declaration when a templated definition lands in a new unit | repo src layout | @@ -338,6 +334,7 @@ | `verbatim_check.py` | guard that every inline-assembly body still reproduces its target bytes | Regression guard that every inline-assembly body still reproduces its target bytes | repo src layout | | `share_census.py` | measure duplicate function bodies across the fleet and assert one source per unique function | The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test | repo paths, the registry and signature schemas | | `ghidra_apply_symbols.sh` | mirror the curated symbol file into the analysis database with a real save | Mirrors the curated symbol file into the analysis program headlessly, with a real save | repo symbol path, project name | +| `frozen.py` | refuse, from one place, the command line of a tool the project has frozen | The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) | nothing | | `verbatim_target_s.py` | regenerate a splitter-format target disassembly for a function no longer stubbed | Regenerates a splitter-format target disassembly for a function that is no longer a stub; --gas emits the assemblable gas-syntax form ($-registers, .L labels, noreorder) that a web diff service accepts as a pasted target | repo build/asm layout | | `share_body_cycle.sh` | run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check | The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new, the gated N/N line read, the phase log entry appended, the bank committed (R42), the clean fleet run every N batches (R22) — stops on the first red | repo paths, the phase-log format, the fleet count from config/check.*.sha | | `share_body.py` | share one byte-identical function class across its binaries through an include-at-site header, gated per binary | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | repo paths, the registry and signature schemas | @@ -374,6 +371,8 @@ | `build_wave.py` | `draw_waves.py` | Builds the next wave from a card pool, ranked by fleet leverage | | `build_wave_args.py` | `wave_args.py` | Emits workflow arguments for a wave from the target manifest, resolving each path | | `canon_draft_decls.py` | `sig_unify.py` | Canonicalizes a draft's extern and data declarations to the banked-consistent set | +| `dedup_extend.py` | `share_body.py` | Extends the existing share registry to newly onboarded binaries | +| `dedup_propagate.py` | `share_body.py` | Lifts one matched body into a shared macro and instantiates it at every duplicate site | | `exemplar_miner.py` | `atlas.py` | Routes every residual stub to its lever and ranks the highest-reach work | | `fix_header_decl.py` | `conform_decls.py` | Rewrites a shared-header caller declaration to a draft's byte-true signature | | `gen_engine_decls.py` | `conform_decls.py` | Declares every shared function and data symbol once, canonically, in a generated header | diff --git a/docs/tool-index.md b/docs/tool-index.md index 98f9a5edd2..3f1684e25e 100644 --- a/docs/tool-index.md +++ b/docs/tool-index.md @@ -5,7 +5,7 @@ freshness). The derived columns come from the tree on every run; the authored on dictionary, whose coverage is asserted both ways. Read it by NEED: find the phrase that matches what you are trying to do, then the tool, then what proved it. The same data generates the day-one kit's manifest and its verbatim tool corpus.* -**Coverage:** 298 tool files in scope (submodules, vendored and downloaded code excluded; `find` and `git ls-files` agree) + 34 retired under `tools/sunset/`. Classes: LIVE 237 (a runtime consumer), REFERENCED 29 (a SETUP row only), ORPHAN 32 (neither) — of 298. Portability: PORTABLE 22, ADAPT 259, PROJECT-ONLY 17. +**Coverage:** 295 tool files in scope (submodules, vendored and downloaded code excluded; `find` and `git ls-files` agree) + 38 retired under `tools/sunset/`. Classes: LIVE 236 (a runtime consumer), REFERENCED 29 (a SETUP row only), ORPHAN 30 (neither) — of 295. Portability: PORTABLE 23, ADAPT 255, PROJECT-ONLY 17. ## P1 — extraction + manifest @@ -85,7 +85,7 @@ then what proved it. The same data generates the day-one kit's manifest and its | When you need to… | Tool | What it does | Proven by | Adapt | Class | |---|---|---|---|---|---| -| answer from one derived model whether each function is open, matched or shared | `corpus.py` | The single derived model of the source tree: open, matched or shared, per function | Makefile, api_agent.py, aprop_autodraft.py, aprop_symfix.py (+67) | repo src/config layout | LIVE | +| answer from one derived model whether each function is open, matched or shared | `corpus.py` | The single derived model of the source tree: open, matched or shared, per function | Makefile, api_agent.py, aprop_autodraft.py, aprop_symfix.py (+66) | repo src/config layout | LIVE | | assert every consumer knows about each newly onboarded binary | `audit_binaries.py` | Gate asserting every consumer knows about each newly onboarded binary | Makefile | repo config/registry paths | LIVE | | assert every tracked source is plain text so text searches never skip it | `audit_text_sources.py` | Every tracked C source must be plain text, or text searches silently skip it | .github/workflows/no-rom.yml, Makefile | repo src paths | LIVE | | check that the independent what-remains views agree with the corpus oracle | `audit_frontier.py` | Checks that the independent "what remains" views agree with the corpus oracle | Makefile | repo paths | LIVE | @@ -94,7 +94,6 @@ then what proved it. The same data generates the day-one kit's manifest and its | decide whether a near-miss residual is a source defect or an assembler artifact | `oracle_reorder.py` | Decides whether a near-miss residual is a source defect or an assembler artifact | Makefile, docs/wave-playbook.md, match_one.py, reorder_passthrough.py (+1) | pinned assembler flags | LIVE | | derive the tool census, the need-keyed index, the kit manifest and the verbatim corpora from one dictionary | `tool_census.py` | Derives the tool census, the need-keyed tool index, the kit manifest and the verbatim corpora from one dictionary, coverage-asserted both ways | Makefile | repo layout (tools/, docs/SETUP.md rows, the kit paths) | LIVE | | enumerate fleet-wide every instance of one known assembler-level blocker class | `wall_sweep.py` | Enumerates fleet-wide instances of one known assembler-level blocker class | docs/wave-playbook.md, residual_rules_b.py, triage_ladder.py | repo asm layout, toolchain quirk | LIVE | -| exercise a guarded repair path the full negative control never reaches | `test_reconcile_ledger.py` | Targeted proof of the propagation ledger guard the full control never exercised | — | repo paths | ORPHAN | | explain with two oracles why a byte-correct draft fails the whole-binary gate | `blocker_probe.py` | Read-only two-oracle explanation of why a byte-correct draft fails the whole-binary gate | recover_integration.py | repo build paths | LIVE | | extract one deterministic feature record per function across every registered binary | `atlas_features.py` | Extracts one deterministic feature record per function across all registered binaries | Makefile, atlas.py, len_tells.py | repo signature files | LIVE | | join the target pool and near-miss ledger into one byte-weighted queue | `worklist.py` | Joins the target pool and the near-miss ledger into one byte-weighted ranked queue | build_fuel_manifest.py | repo scratch paths | LIVE | @@ -120,7 +119,7 @@ then what proved it. The same data generates the day-one kit's manifest and its | answer from a prologue whether a function was built unoptimized | `o0_detect.py` | The single place that answers whether a function was built unoptimized, from its prologue | match_one.py, neighbor_ref.py, o0_boundary.py, residual_rules_b.py (+2) | compiler prologue shape | LIVE | | apply one proven codegen lever everywhere the bytes say it belongs | `weave_sweep.py` | Applies one known prologue-scheduling lever everywhere the bytes say it belongs | — | repo src layout, compiler behaviour | REFERENCED | | assert a read-only-data carve's interleave order equals the segment sequence | `interleave_check.py` | Asserts the carve interleave order equals the segment sequence, position by position | — | repo config layout | REFERENCED | -| bank a matched jump-table exemplar across its siblings, reverting on failure | `jtbl_family_bank.py` | Banks a matched jump-table exemplar across its structural siblings, revert-on-fail | dedup_extend.py, family_sweep.py, reloc_verify.py | repo config/src layout | LIVE | +| bank a matched jump-table exemplar across its siblings, reverting on failure | `jtbl_family_bank.py` | Banks a matched jump-table exemplar across its structural siblings, revert-on-fail | family_sweep.py, reloc_verify.py | repo config/src layout | LIVE | | bank a refused draft by copying the destination unit's own declarations into it — the declaration-conflict class a canonical type layer prevents (Phase 6) | `sync_tu_decls.py` | Banks a refused draft by copying the destination unit's own declarations into it | docs/wave-playbook.md, recover_route.py | repo src layout | LIVE | | bank every open stub that already has a banked structural twin | `twin_sweep.py` | Banks every open stub that has an already-banked structural twin, for near-zero tokens | docs/wave-playbook.md | repo signature files | LIVE | | bind a void definition to a symbol the destination declares as returning a value | `fix_decl_mirror.py` | Binds a void definition to its symbol when the destination declares a value return | — | repo src layout | ORPHAN | @@ -129,12 +128,11 @@ then what proved it. The same data generates the day-one kit's manifest and its | cluster the open frontier one tier looser than exact skeleton hashing | `family_cousins.py` | Similarity clustering one tier looser than exact skeleton hashing over the open frontier | Makefile, aprop_autodraft.py, atlas.py, atlas_features.py (+3) | repo signature files | LIVE | | compute the fleet's consensus declaration for every symbol, as card fuel — the seed of a canonical type layer (Phase 6) | `decl_prior.py` | Computes the fleet's consensus declaration for every symbol, as card fuel | build_wave_atlas.py, family_remap.py, t5_cards.py | repo src layout | LIVE | | conform a draft's data declarations to what its destination unit can see | `reconcile_tu.py` | Conforms a draft's data declarations to what the destination translation unit can actually see | gate_stage.py, harvest_verify.py, integration_resolver.py, jtbl_family_bank.py | repo src layout | LIVE | -| conform every declaration of a function fleet-wide to its byte-true definition — width and signedness proven by the bytes at bank time (Phase 6) | `conform_decls.py` | Conforms every declaration of a function fleet-wide to its byte-true definition | family_sweep.py, macro_draft.py | repo src/shared-header layout | LIVE | +| conform every declaration of a function fleet-wide to its byte-true definition — width and signedness proven by the bytes at bank time (Phase 6) | `conform_decls.py` | Conforms every declaration of a function fleet-wide to its byte-true definition | family_sweep.py | repo src/shared-header layout | LIVE | | demote declaration scope as one rung of the recovery ladder | `scope_demote_drafts.py` | The scope-demote step, wired as a rung of the recovery ladder | gate_stage.py, recover_route.py | repo src layout | LIVE | | derive an object's inter-table padding spec from the bytes instead of searching | `pads_audit.py` | Derives each object's padding spec from the bytes instead of searching for it | — | repo build/config layout | REFERENCED | | draft a family member mechanically from a seed body and symbol rebase | `aprop_autodraft.py` | Mechanically drafts a family member from the seed body plus a positional symbol rebase | decl_from_use.py, maintenance.sh, sibling_lane.sh | repo scratch paths | LIVE | | escape a shared-header declaration conflict locally by unmacroizing one body | `demacroize.py` | Per-binary local escape from a shared-header declaration conflict by unmacroizing one body | recover_integration.py | repo shared-header layout | LIVE | -| extend an existing code-share registry to newly onboarded binaries | `dedup_extend.py` | Extends the existing share registry to newly onboarded binaries | audit_binaries.py, share_body.py | repo config/src layout | LIVE | | find and bank unoptimized functions stranded at an optimized region's boundary | `o0_boundary.py` | Finds and banks unoptimized functions stranded at the end of an unoptimized region | — | repo config layout | REFERENCED | | free duplicate matches a share propagation skips as not self-contained | `inject_capped_externs.py` | Frees high-reach inline matches skipped as not self-contained by injecting their file-scope externs | — | repo shared-header layout | REFERENCED | | infer a minimal extern for a data symbol the destination unit does not declare | `decl_from_use.py` | Infers a minimal extern for a data symbol a draft uses but its destination does not declare | aprop_autodraft.py, integration_resolver.py, residual_rules_b.py | repo src layout | LIVE | @@ -142,21 +140,19 @@ then what proved it. The same data generates the day-one kit's manifest and its | join two related builds' signature dumps into a function correspondence | `match_protos.py` | Joins per-function signature dumps of two related builds into a function correspondence | dup_report.py | repo signature files | LIVE | | let a unit keep calling, through per-site casts, the function it now defines | `cast_self_callers.py` | Lets a unit keep calling, through a per-site cast, the function it is about to define | docs/wave-playbook.md, recover_integration.py, recover_route.py | repo src layout | LIVE | | lift a named list of types fleet-wide into the shared type header — the canonical type layer from the first bank (Phase 6, the multiplier); again at readability (Phase 10) | `lift_types.py` | Lifts a named list of types fleet-wide into the shared type header | build_engine_types.py, uniquify_type.py | repo shared-header path | LIVE | -| lift one matched body into a shared macro and instantiate it everywhere | `dedup_propagate.py` | Lifts one matched body into a shared macro and instantiates it at every duplicate site | auto_driver.py, bulk_harvest.py, gate_lane.py, gate_stage.py (+5) | repo shared-header/src layout | LIVE | -| materialize a shared macro body back into a compilable standalone draft | `macro_draft.py` | Materializes a shared macro body back into a compilable standalone draft | — | repo shared-header layout | ORPHAN | | move a unit's file-scope data externs down into their consumers | `scope_tu_externs.py` | Moves a unit's own file-scope data externs down into their consumers to legalize a block-scope type | family_sweep.py, jtbl_family_bank.py, recover_integration.py | repo src layout | LIVE | | name the one exemplar to draft per structural family, ranked | `family_manifest.py` | Ranked structural-family target manifest naming the one exemplar to draft per family | family_hseq.py | repo signature files | LIVE | | normalize a function's own declaration when a templated definition lands in a new unit | `normalize_self_decls.py` | Normalizes a function's own declaration when a templated definition lands in a new unit | family_sweep.py | repo src layout | LIVE | -| parse C declarators once, for every declaration tool to share | `cdecl.py` | The single recursive-descent C declarator parser every declaration tool shares | Makefile, blocker_probe.py, canon_sig_reconcile.py, cast_call_sites.py (+21) | C89 grammar only | LIVE | +| parse C declarators once, for every declaration tool to share | `cdecl.py` | The single recursive-descent C declarator parser every declaration tool shares | Makefile, blocker_probe.py, canon_sig_reconcile.py, cast_call_sites.py (+20) | C89 grammar only | LIVE | | partition a source file at object boundaries, aware of comment state | `overlay_src_split.py` | Comment-state-aware partition of a source file, the oracle for the split chain | jr_isolate_all.py, neighbor_ref.py, o0_subsplit.py, share_census.py (+1) | repo src layout | LIVE | | place a templated body's data externs at a scope the destination accepts | `scope_data_externs.py` | Places a templated body's data externs at the scope the destination unit can accept | family_sweep.py, jtbl_family_bank.py, scope_demote_drafts.py, scope_tu_externs.py | repo src layout | LIVE | | property-test the signature tool on committed fixtures, needing no compiler | `xsig/tests/test_xsig.py` | Property tests for the signature tool on committed fixtures, needing no compiler | .github/workflows/no-rom.yml, Makefile | fixture paths | LIVE | -| rank clusters of the unmatched frontier by mnemonic-skeleton hash | `family_hseq.py` | Ranked clustering of the unmatched frontier by mnemonic-skeleton hash | Makefile, audit_binaries.py, audit_frontier.py, build_wave_atlas.py (+6) | repo signature files | LIVE | +| rank clusters of the unmatched frontier by mnemonic-skeleton hash | `family_hseq.py` | Ranked clustering of the unmatched frontier by mnemonic-skeleton hash | Makefile, audit_binaries.py, audit_frontier.py, build_wave_atlas.py (+5) | repo signature files | LIVE | | rebase a remapped draft's stale seed symbols onto the target's own | `aprop_symfix.py` | Rebases stale seed symbols in a mechanically adapted draft onto the target's own symbols | aprop_autodraft.py, family_cousins.py, integration_resolver.py, recover_rejects.py | repo symbol/config sources | LIVE | | reconcile a definition's typed signature with what its destination unit declares — part of banking against a canonical type layer from Phase 6; again at readability (Phase 10) | `canon_sig_reconcile.py` | Reconciles a definition's typed signature with what its destination unit already declares | family_sweep.py, jtbl_family_bank.py | repo src layout | LIVE | | regenerate the signature tool's fixtures with the pinned toolchain | `xsig/tests/make_fixtures.sh` | Regenerates the signature-tool fixtures with the pinned toolchain | test_xsig.py | compiler triple, repo tool paths | LIVE | -| remap a matched exemplar's source onto a structural sibling by positional symbol pairing | `family_remap.py` | Mechanically remaps a matched exemplar's C onto a structural sibling by positional symbol pairing | aprop_autodraft.py, atlas.py, atlas_features.py, dedup_extend.py (+17) | repo src layout | LIVE | -| repair a shared caller's argument-count conflict for the no-prototype failure class | `fix_arity_callers.py` | Repairs the shared-caller argument-count conflict for the no-prototype failure class | docs/wave-playbook.md, cdecl.py, dedup_propagate.py, gate_stage.py (+2) | repo shared-header layout | LIVE | +| remap a matched exemplar's source onto a structural sibling by positional symbol pairing | `family_remap.py` | Mechanically remaps a matched exemplar's C onto a structural sibling by positional symbol pairing | aprop_autodraft.py, atlas.py, atlas_features.py, diff_regions.py (+16) | repo src layout | LIVE | +| repair a shared caller's argument-count conflict for the no-prototype failure class | `fix_arity_callers.py` | Repairs the shared-caller argument-count conflict for the no-prototype failure class | docs/wave-playbook.md, cdecl.py, gate_stage.py, recover_integration.py (+1) | repo shared-header layout | LIVE | | repair a stale padding spec by search plus byte proof, never by guessing | `jtbl_pads_fix.py` | Repairs a stale padding spec by search plus byte proof, never by guessing | maintenance.sh | repo config layout | LIVE | | report which open stubs became mechanical remaps the moment one was banked | `twin_rescan.py` | After a bank, reports which open stubs just became mechanical remaps instead of drafts | — | repo signature files | REFERENCED | | reproduce the original inter-table padding in a multi-table carve | `jtbl_rodata_pads.py` | Post-assembler filter reproducing the original inter-table padding in a multi-table carve | Makefile, compile_only.py, gate_main.py, jtbl_carve.py | compiler/assembler behaviour, repo build paths | LIVE | @@ -166,11 +162,11 @@ then what proved it. The same data generates the day-one kit's manifest and its | roll an unoptimized definition atomically across every binary carrying it | `rollout_o0.py` | Generalized two-file atomic rollout of an unoptimized definition across every binary carrying it | Makefile, match_one.py | repo src/config layout | LIVE | | route an unoptimized address range sitting inside an otherwise optimized object | `o0_subsplit.py` | Routes an unoptimized address range that sits inside an otherwise optimized object | match_one.py | repo config layout | LIVE | | set up the read-only-data carve for a binary's matched jump-table functions | `jtbl_carve.py` | Sets up the read-only-data carve for a binary's matched jump-table functions | Makefile, config/overlays.mk, build_wave_atlas.py, harvest_verify.py (+7) | repo config/build layout | LIVE | -| sign every function of a flat image at a known base, without a disassembler | `sig_image.py` | Disassembler-free per-function signer for a flat image at a known base, field-identical to the dumper | Makefile, atlas_features.py, corpus.py, dedup_extend.py (+6) | MIPS encodings, repo scratch paths | LIVE | +| sign every function of a flat image at a known base, without a disassembler | `sig_image.py` | Disassembler-free per-function signer for a flat image at a known base, field-identical to the dumper | Makefile, atlas_features.py, corpus.py, disc_audit.py (+5) | MIPS encodings, repo scratch paths | LIVE | | sign functions with relocations masked for cross-project code identification | `xsig/xsig.py` | Relocation-masked per-function signatures for cross-project code identification | .github/workflows/no-rom.yml, Makefile, test_xsig.py | MIPS/relocation model only | LIVE | -| sweep a matched exemplar across every same-structure sibling, gate-arbitrated | `family_sweep.py` | Sweeps a matched exemplar across every same-structure sibling, gate-arbitrated | recover_integration.py, twin_sweep.py | repo src/config layout | LIVE | +| sweep a matched exemplar across every same-structure sibling, gate-arbitrated | `family_sweep.py` | Sweeps a matched exemplar across every same-structure sibling, gate-arbitrated | frozen.py, recover_integration.py, twin_sweep.py | repo src/config layout | LIVE | | unify a draft's callee externs and its own signature to the banked-canonical set | `sig_unify.py` | Unifies a draft's callee externs and its own definition signature to the banked-canonical set | auto_driver.py, cast_call_sites.py, cdecl.py, gate_stage.py (+1) | repo shared-header paths | LIVE | -| validate a code-share registry byte-honestly, failing closed on signature drift | `dedup_integrate.py` | Byte-honesty validator for the code-share registry; fails closed on signature drift | Makefile, dedup_propagate.py, macro_to_header.py, progress.py (+1) | repo config registry path | LIVE | +| validate a code-share registry byte-honestly, failing closed on signature drift | `dedup_integrate.py` | Byte-honesty validator for the code-share registry; fails closed on signature drift | Makefile, macro_to_header.py, progress.py, share_body.py (+1) | repo config registry path | LIVE | ## P7 — the codegen map, the dumps, the permuter @@ -234,7 +230,7 @@ then what proved it. The same data generates the day-one kit's manifest and its | generate a callee-signature-aware target manifest with per-function fleet reach | `gen_harvest_targets.py` | Generates a callee-signature-aware target manifest with per-function fleet reach | cast_call_sites.py, cdecl.py, sig_unify.py | canonical source binary, repo paths | LIVE | | give a wave its own immutable copy of the disassembly its agents read | `wave_snapshot.py` | Gives a wave its own immutable copy of the disassembly files its agents read | build_wave_atlas.py | repo asm paths | LIVE | | give an external chat model the same multi-turn tool harness an agent gets | `api_agent.py` | Gives an external chat model the same multi-turn tool harness an agent gets | claude_wave_packs.py, idiom_serial.py, jtbl_lane.py, ox_campaign.py | provider endpoint, repo doc paths | LIVE | -| hold one reader/writer lock over the shared source state for concurrent lanes | `shared_lock.py` | One reader/writer lock over the shared source state for concurrent lanes | bulk_harvest.py, dedup_propagate.py, fix_arity_callers.py, gate_stage.py | repo lock paths | LIVE | +| hold one reader/writer lock over the shared source state for concurrent lanes | `shared_lock.py` | One reader/writer lock over the shared source state for concurrent lanes | bulk_harvest.py, fix_arity_callers.py, gate_stage.py | repo lock paths | LIVE | | keep an unattended campaign loop alive across run caps and crashes | `lanes/forever.sh` | Keeps the wave campaign running indefinitely across caps and crashes | — | repo scratch paths | ORPHAN | | keep an unattended driver alive across crashes until a clean exit | `auto_supervisor.sh` | Keeps an unattended driver alive across crashes until a clean or requested exit | grinder.py, lora_grind.py | repo scratch paths | LIVE | | keep drafting running continuously, touching only scratch state | `lanes/drafter.sh` | The drafting lane that must never stop; touches only scratch state | relaunch_drafter_shell.sh | repo scratch paths, provider config | LIVE | @@ -274,14 +270,14 @@ then what proved it. The same data generates the day-one kit's manifest and its | run the executable's own draft, gate and commit cadence beside other lanes | `main_lane.py` | The executable's own draft, gate and commit cadence beside the other lanes | main.sh | repo build/make targets | LIVE | | run the executable's own lane loop on its clean-rebuild cadence | `lanes/main.sh` | The executable's own lane loop, on its clean-rebuild cadence | gate_stage.py, restart_main_lane_when_idle.sh | repo make targets | LIVE | | run the free recovery and re-judgement passes whenever the gate is idle | `lanes/maintenance.sh` | The maintenance lane running the free recovery and re-judgement passes | — | repo scratch paths | ORPHAN | -| run the one shared recovery, byte-gate and log ladder every producer calls | `gate_stage.py` | The shared deterministic recovery, gate and log ladder every producer calls | docs/wave-playbook.md, api_agent.py, bulk_harvest.py, dedup_extend.py (+12) | repo src/config layout | LIVE | +| run the one shared recovery, byte-gate and log ladder every producer calls | `gate_stage.py` | The shared deterministic recovery, gate and log ladder every producer calls | docs/wave-playbook.md, api_agent.py, bulk_harvest.py, fix_tu_ret_decls.py (+11) | repo src/config layout | LIVE | | run the token-free permuter grinder beside a saturated gate | `lanes/grinder_lane.sh` | Runs the model-free permuter grinder beside a saturated gater | — | repo scratch paths | ORPHAN | | run two drafting pools against one serial gate | `lanes/campaign.sh` | Dual-pool campaign lane: two drafting pools, one serial gate | forever.sh | provider config, repo paths | LIVE | | save each subagent's full final report to one file per target | `agent_reports.py` | Saves each subagent's full final prose report to one file per target | docs/wave-playbook.md, transcript_dump.py | agent transcript paths | LIVE | | select a worker wave's target batch from the manifest with cache filters | `wave_targets.py` | Selects a worker-wave target batch from the target manifest with cache filters | build_fuel_manifest.py, gate_stage.py, orchestrator.py | repo scratch paths | LIVE | | serve a fine-tuned model locally behind a chat-completions endpoint | `serve_local.py` | Serves the fine-tuned model locally behind a chat-completions endpoint | bulk_harvest.py | training venv, GPU assumptions | LIVE | | spend spare provider capacity on other work when the gate is the constraint | `lanes/elastic.sh` | Spends spare provider capacity on extra work when the main lane cannot | — | provider config, repo paths | ORPHAN | -| substitute, build, keep only if byte-identical, else revert | `harvest_verify.py` | The whole-binary byte gate: substitute, build, keep only if identical, else revert | auto_driver.py, dedup_extend.py, family_sweep.py, gate_stage.py (+5) | repo src/build layout | LIVE | +| substitute, build, keep only if byte-identical, else revert | `harvest_verify.py` | The whole-binary byte gate: substitute, build, keep only if identical, else revert | auto_driver.py, family_sweep.py, gate_stage.py, gater_lane.py (+4) | repo src/build layout | LIVE | | turn mined pairs into a chat-template instruction dataset | `format_finetune.py` | Turns the mined pairs into a chat-template instruction dataset | api_draft.py, train_lora.py | dataset paths, model chat template | LIVE | | turn pre-gate rejects back into bankable drafts for zero model tokens | `recover_rejects.py` | Turns pre-gate rejects back into bankable drafts for zero model tokens | maintenance.sh | repo scratch paths | LIVE | | unify live stubs into one ranked, class-tagged, cache-annotated target pool | `build_fuel_manifest.py` | Unifies live stubs into one ranked, class-tagged, cache-annotated target pool | orchestrator.py, worklist.py | canonical source binary, repo scratch paths | LIVE | @@ -332,11 +328,12 @@ then what proved it. The same data generates the day-one kit's manifest and its | flag address-named references whose address now has a curated name | `lint_symbol_refs.py` | Flags address-named references in committed sources whose address now has a curated name | .github/workflows/no-rom.yml, Makefile | repo symbol/src paths | LIVE | | give each conflicting camp of a same-named type its own name | `uniquify_type.py` | Gives each conflicting camp of a same-named type its own name so every camp becomes liftable | — | repo src layout | REFERENCED | | guard that every inline-assembly body still reproduces its target bytes | `verbatim_check.py` | Regression guard that every inline-assembly body still reproduces its target bytes | .github/workflows/no-rom.yml, verbatim_target_s.py, verbatim_to_stub.py | repo src layout | LIVE | -| measure duplicate function bodies across the fleet and assert one source per unique function | `share_census.py` | The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test | audit_binaries.py, dedup_integrate.py, macro_to_header.py, overlay_src_split.py (+2) | repo paths, the registry and signature schemas | LIVE | +| measure duplicate function bodies across the fleet and assert one source per unique function | `share_census.py` | The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test | audit_binaries.py, dedup_integrate.py, gen_harvest_targets.py, macro_to_header.py (+3) | repo paths, the registry and signature schemas | LIVE | | mirror the curated symbol file into the analysis database with a real save | `ghidra_apply_symbols.sh` | Mirrors the curated symbol file into the analysis program headlessly, with a real save | — | repo symbol path, project name | REFERENCED | +| refuse, from one place, the command line of a tool the project has frozen | `frozen.py` | The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) | aprop_autodraft.py, blocker_probe.py, canon_sig_reconcile.py, conform_decls.py (+10) | nothing | LIVE | | regenerate a splitter-format target disassembly for a function no longer stubbed | `verbatim_target_s.py` | Regenerates a splitter-format target disassembly for a function that is no longer a stub; --gas emits the assemblable gas-syntax form ($-registers, .L labels, noreorder) that a web diff service accepts as a pasted target | decompme_replica.sh | repo build/asm layout | LIVE | | run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check | `share_body_cycle.sh` | The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new, the gated N/N line read, the phase log entry appended, the bank committed (R42), the clean fleet run every N batches (R22) — stops on the first red | — | repo paths, the phase-log format, the fleet count from config/check.*.sha | ORPHAN | -| share one byte-identical function class across its binaries through an include-at-site header, gated per binary | `share_body.py` | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | dedup_extend.py, share_body_cycle.sh, share_census.py | repo paths, the registry and signature schemas | LIVE | +| share one byte-identical function class across its binaries through an include-at-site header, gated per binary | `share_body.py` | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | aprop_autodraft.py, audit_binaries.py, auto_driver.py, blocker_probe.py (+17) | repo paths, the registry and signature schemas | LIVE | | turn an inline-assembly body back into a stub the toolchain can reach | `verbatim_to_stub.py` | Turns an inline-assembly body back into an include-assembly stub | — | repo src/asm layout | REFERENCED | ## PROJECT-ONLY — project-only in code (the shape is a task; the code does not transfer) @@ -354,7 +351,7 @@ then what proved it. The same data generates the day-one kit's manifest and its | list a sound library's address window and per-address exclusions | `make_snd_used.py` | Hard-codes one executable's sound-library address window and per-address exclusions | Makefile, fetch_psyq.sh | project-only | LIVE | | list the interface library's objects and address window the executable actually links | `make_apicard_used.py` | Hard-codes one executable's interface-library address window and object roster | Makefile, fetch_psyq.sh | project-only | LIVE | | observe a live loader's requests to learn which payload loads where | `cdtrace.py` | Reads one game's loader request structures at fixed RAM addresses | — | project-only | ORPHAN | -| onboard one payload by its container and file-index naming | `new_overlay.sh` | Takes one game's container and file-index naming as its command-line contract | Makefile, config/overlays.mk, dedup_extend.py, dup_report.py (+3) | project-only | LIVE | +| onboard one payload by its container and file-index naming | `new_overlay.sh` | Takes one game's container and file-index naming as its command-line contract | Makefile, config/overlays.mk, dup_report.py, family_remap.py (+2) | project-only | LIVE | | parse a bespoke archive-chain header format | `bfm_extract/pac.py` | Parser for one game's bespoke archive-chain header format | crosscheck.py, extract.py | project-only | LIVE | | read a bespoke container's table of contents to locate its files | `bfm_extract/cd_archive.py` | Reader for one game's bespoke container table-of-contents format | crosscheck.py, extract.py | project-only | LIVE | | unit-test a bespoke compression decoder against known cases | `bfm_extract/test_lzss.py` | Unit tests for that game-specific compression variant | .github/workflows/no-rom.yml | project-only | LIVE | @@ -370,6 +367,8 @@ then what proved it. The same data generates the day-one kit's manifest and its | `build_wave.py` | SUPERSEDED | tools/draw_waves.py | Builds the next wave from a card pool, ranked by fleet leverage | | `build_wave_args.py` | SUPERSEDED | tools/wave_args.py | Emits workflow arguments for a wave from the target manifest, resolving each path | | `canon_draft_decls.py` | SUPERSEDED | tools/sig_unify.py | Canonicalizes a draft's extern and data declarations to the banked-consistent set | +| `dedup_extend.py` | SUPERSEDED | tools/share_body.py | Extends the existing share registry to newly onboarded binaries | +| `dedup_propagate.py` | SUPERSEDED | tools/share_body.py | Lifts one matched body into a shared macro and instantiates it at every duplicate site | | `exemplar_miner.py` | SUPERSEDED | tools/atlas.py | Routes every residual stub to its lever and ranks the highest-reach work | | `fix_header_decl.py` | SUPERSEDED | tools/conform_decls.py | Rewrites a shared-header caller declaration to a draft's byte-true signature | | `gen_engine_decls.py` | SUPERSEDED | tools/conform_decls.py | Declares every shared function and data symbol once, canonically, in a generated header | @@ -381,6 +380,7 @@ then what proved it. The same data generates the day-one kit's manifest and its | `lanes/launch_ox.sh` | ONE-OFF | the sharded overnight campaign run it launched (its key file, endpoint and model are that campaign's) | Launcher for a sharded campaign run with a tag and shard count | | `lanes/toolwork.sh` | ONE-OFF | the tooling-design briefs it ran against an external model (its key, endpoint and model are that era's) | Runs tooling-design briefs against an external model with build config as context | | `lenmiss_route.py` | ONE-OFF | the routed outcome of one length-drift near-miss pile (it consumes that audit's classified ledger) | Routes the length-drift near-miss pile through the documented lenses, re-verified against the tree | +| `macro_draft.py` | ONE-OFF | the per-function headers under src/shared/ (Phase 35 T4; the macro form is gone) | Materializes a shared macro body back into a compilable standalone draft | | `p16_known_answer.py` | SUPERSEDED | tools/p16_improve.py | Graduated known-answer validation of the scaffold pipeline across difficulty bands | | `plumbing_groups.py` | SUPERSEDED | tools/gate_triage.py | Groups recorded declaration-conflict failures into sweepable work groups from the ledgers | | `reconcile_slate.py` | SUPERSEDED | tools/pregate_check.py | Drives a whole batch to declaration-compatible before any rebuild is spent | @@ -394,6 +394,7 @@ then what proved it. The same data generates the day-one kit's manifest and its | `t5_distill_args.py` | SUPERSEDED | tools/distill_scan.py | Builds the post-wave distillation slate from a wave's directories | | `t5_targets.py` | SUPERSEDED | tools/draw_waves.py | Draws one routed slate from the frontier classes | | `t7_bank.py` | SUPERSEDED | tools/gate_stage.py | Batch banking driver applying reconcile-at-bank-time plus the whole-binary gate | +| `test_reconcile_ledger.py` | ONE-OFF | the S45p7 reconcile-ledger proof, retired with its subject dedup_propagate | Targeted proof of the propagation ledger guard the full control never exercised | | `treelock.sh` | SUPERSEDED | tools/shared_lock.py | The mutex wrapper for tree-writing campaigns, with status and stale detection | | `verify_map_findings.py` | ONE-OFF | the machine-checked verdicts of one codegen-map audit (docs/gcc-2.7.2-map/regalloc.md) | Machine-checks a codegen-map audit's findings against the actual compiler sources | | `wall_taxonomy.py` | SUPERSEDED | tools/frontier_classify.py | Census-classifies every unmatched residual by blocker class to size each avenue | diff --git a/phase-ends/CURRENT_PHASE.md b/phase-ends/CURRENT_PHASE.md index e3f6255151..d464063b03 100644 --- a/phase-ends/CURRENT_PHASE.md +++ b/phase-ends/CURRENT_PHASE.md @@ -458,6 +458,22 @@ 3,135; shared headers 2,215 → 3,137; same-vram backlog 1,099 classes / 4,755 private copies / 3,658 collapsible → 51 / 160 / 109 (all ledgered); verdict A 1,712 → 2,760; fleet clean-run wall 145 s → 84 s (CPU 2,230 s → 1,335 s); 23 commits in T5. **T5 ☑.** +- **S96 — T6 part 1: freeze + retire.** `tools/frozen.py` (the one refusal, printed from main() — never at import: cdecl imports + gen_harvest_targets, 18 tools import family_remap). **FROZEN (14, status FROZEN + successor in the dictionary):** the plan's 7 + (family_sweep, gen_harvest_targets, p16_improve, recover_giant, restore_dropped_decls, normalize_self_decls, o0_subsplit) + 7 the + guard census found reading or writing the macro form (inject_capped_externs, aprop_autodraft, canon_sig_reconcile, conform_decls, + blocker_probe, demacroize, and family_remap's command line — it writes a `#define DEFINE_func_` head; its library stays). Deviation + from the plan's "7": the S95 ast census measured 30 hits, not 22; twin_sweep/recover_integration (which exec frozen CLIs) stay LIVE + and would print the refusal; family_cousins stays LIVE (`make atlas` runs it) and loses its dead branch in part 2. **RETIRED (4, git mv + to tools/sunset/, README rows):** dedup_propagate + dedup_extend → share_body.py (add_members_surgical and the library surface + onboarded_overlays / load_sig / registered_addrs / sym moved verbatim; validate_targets imports share_body; inject_capped_externs + frozen), macro_draft (product: the headers), test_reconcile_ledger (retired with its subject). Six lane callers repointed to + `share_body.py --apply --bucket new --batches 1` (gate_stage, gate_lane, grinder's message, lora_grind, auto_driver, bulk_harvest); + audit_binaries' hint reworded. `tool_census` accepts FROZEN. Verify: every touched file compiles; `family_sweep.py --source x` → + `FROZEN since Phase 35 … Successor: tools/share_body.py` rc 1; `import share_body, family_remap, gen_harvest_targets, + validate_targets` OK; no live tool names a retired file; `tool_census --check: OK` after `make kit-corpus` (the first chain checked + BEFORE regenerating and a `;` let a partial commit through — R97 again; amended into one commit). + ## Approved plan (verbatim, gate 1 — 2026-09-08) # Phase 35 — Gen3 opens: the dedup phase, "one source per unique function" (v2.0.0 → v2.1.0) diff --git a/tools/aprop_autodraft.py b/tools/aprop_autodraft.py index 5eb6c6aa4b..e604827b9c 100644 --- a/tools/aprop_autodraft.py +++ b/tools/aprop_autodraft.py @@ -574,6 +574,7 @@ def _local_verdict(fn, text, sub, o0): def main(): + import frozen; frozen.refuse("aprop_autodraft.py", "tools/share_body.py", "it seeds drafts from the macro bodies in engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser() ap.add_argument("--cards", default=CARDS) ap.add_argument("--outdir", default=".run/aprop_autodraft") diff --git a/tools/audit_binaries.py b/tools/audit_binaries.py index 93c81ca6e9..9a188e9094 100644 --- a/tools/audit_binaries.py +++ b/tools/audit_binaries.py @@ -176,7 +176,7 @@ def main(): zero = sorted(b for b in onb if b.startswith("ov_") and memb.get(b, 0) == 0) if zero: warns.append(f"{len(zero)} overlay(s) in 0 dedup groups (onboarded but un-harvested — " - f"candidate for tools/dedup_extend.py): {zero if verbose else zero[:6]}") + f"candidate for tools/share_body.py --plan): {zero if verbose else zero[:6]}") # --- report n_ov = len([b for b in onb if b.startswith("ov_")]) diff --git a/tools/auto_driver.py b/tools/auto_driver.py index 6d7688e6d9..6d62f12708 100644 --- a/tools/auto_driver.py +++ b/tools/auto_driver.py @@ -137,7 +137,7 @@ def byte_gate(draft_dir): def propagate(): - sh([PY, "tools/dedup_propagate.py", "--auto-from", OV, "--min-reach", "2"], timeout=3600) + sh([PY, "tools/share_body.py", "--apply", "--bucket", "new", "--batches", "1"], timeout=3600) # P35 T6: the include-at-site share def main(): diff --git a/tools/blocker_probe.py b/tools/blocker_probe.py index af8e363562..e22277be89 100644 --- a/tools/blocker_probe.py +++ b/tools/blocker_probe.py @@ -223,6 +223,7 @@ def cc1_verdict(fn, stub, draft_path, work): # --------------------------------------------------------------------------------------------- def main(): + import frozen; frozen.refuse("blocker_probe.py", "tools/share_body.py (its per-binary gate names the blocker)", "it probes macro-scope blockers in engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--binary', required=True) diff --git a/tools/bulk_harvest.py b/tools/bulk_harvest.py index 2e9f16faf7..18e23a1b40 100644 --- a/tools/bulk_harvest.py +++ b/tools/bulk_harvest.py @@ -264,7 +264,7 @@ def run_cycle(a, api_base, model, tried): for b, fns in banked_by_bin.items(): if any((lg.reach_of(b, fn) or 1) >= 2 for fn in fns): before = gate_stage._dedup_group_count() - sh([PY, "tools/dedup_propagate.py", "--auto-from", b, "--min-reach", "2"], timeout=3600) + sh([PY, "tools/share_body.py", "--apply", "--bucket", "new", "--batches", "1"], timeout=3600) # P35 T6 propagated += max(0, gate_stage._dedup_group_count() - before) merge_backlogs() fp = fleet_pct() diff --git a/tools/canon_sig_reconcile.py b/tools/canon_sig_reconcile.py index 1374f7a5a1..f6004783cb 100644 --- a/tools/canon_sig_reconcile.py +++ b/tools/canon_sig_reconcile.py @@ -518,6 +518,7 @@ def reconcile(fn, draft, canon_sig=None, tu_path=None): def main(): + import frozen; frozen.refuse("canon_sig_reconcile.py", "the types phase (Phase 37): one canonical declaration per symbol", "it reconciles declarations against engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser() ap.add_argument('--fn', required=True) ap.add_argument('--draft', required=True) diff --git a/tools/conform_decls.py b/tools/conform_decls.py index f25f570b78..75adec8e90 100644 --- a/tools/conform_decls.py +++ b/tools/conform_decls.py @@ -207,6 +207,7 @@ def consumers(fn): def main(): + import frozen; frozen.refuse("conform_decls.py", "the types phase (Phase 37)", "it conforms the fleet's declarations to engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--fn", required=True) diff --git a/tools/dedup_extend.py b/tools/dedup_extend.py deleted file mode 100644 index 7d98b49401..0000000000 --- a/tools/dedup_extend.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python3 -"""Phase-28 T4: EXTEND the existing dedup registry to newly-onboarded binaries. - -THE GAP THIS FILLS (byte-measured, P28 T3-A). `tools/new_overlay.sh` produces a byte-clean binary -that is NOT a citizen of the shared-body ecosystem: its `.c` includes only `common.h` (never -`../shared/engine_core.h`), so not one shared engine body can reach it, and it appears in ZERO of -`config/dedup.us.yaml`'s groups. The 4 SC07 overlays the Phase-27 disc audit onboarded therefore sat -at ~80 matched / ~2,400 stubs while their siblings were ~2,150 matched — and 1,689 registry groups -still read "134 binaries", never 138. - -WHY THE EXISTING TOOL CANNOT DO IT. `dedup_propagate` is built for CRACK -> AUTHOR MACRO -> -INSTANTIATE: `--auto-from` scans a source overlay's INLINE DEFS (it planned only 11 fns here, -skipping the ~1,600 whose bodies are ALREADY `DEFINE_func_*` macros in engine_core.h), and `--addr` -dies with "no source overlay has it matched" because no overlay holds an inline def — every one of -the 134 uses the macro. Extending an existing MACRO-BACKED group to a new binary is a different -operation, and nothing implemented it. - -WHY IT IS SAFE (the correctness argument — this is a byte-gate feeder, so it must be explicit). -`h_exact` is the SHA1 of RAW INSTRUCTION BYTES (tools/sig_image.py). Two instances sharing an -h_exact are therefore identical *including* their `jal`/`lui`/`%lo` reloc immediates — same callee -addresses, same data addresses, same symbols. So the body that compiles byte-identically at one -member compiles byte-identically at the other, with no remap at all. (This is exactly why -dup_report calls h_exact "guaranteed byte-match" and h_norm "candidate-only".) The whole-binary -byte-gate remains the sole arbiter (G3/P9): a wrong instantiation changes the bytes and fails SHA1. -A bug here can make this tool FAIL TO BANK; it cannot make it falsely bank. - -REUSE, DON'T REBUILD (R33). This owns only the SET COMPUTATION and the REGISTRY EDIT. The splice and -the gate are `tools/harvest_verify.py` verbatim — which already derives each stub's home TU from the -corpus oracle, chunks + bisects, and reverts on failure. h_exact members are byte-identical by -construction, so the happy path is ONE build per binary (not one per function). - - tools/dedup_extend.py --binaries ov_SC07_006,ov_SC07_007,ov_SC07_010,ov_SC07_011 [--check-only] - [--chunk N] [--limit N] -""" -import argparse -import glob -import json -import os -import re -import subprocess -import sys - -import yaml - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import corpus -import family_hseq # has_mid_jr — the ONE jr oracle (R33) -import family_remap as FR # stream_words/nins_of - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -PY = sys.executable -REGISTRY = "config/dedup.us.yaml" -SHARED_INCLUDE = '#include "../shared/engine_core.h"' -EXTEND_DIR = ".run/extend" - - -def sig_hashes(binary): - """{vram_int: h_exact} from the binary's sig — the ORIGINAL bytes, independent of splat.""" - out = {} - p = os.path.join(REPO, f".run/sig.{binary}.jsonl") - for line in open(p): - d = json.loads(line) - out[int(d["addr"], 16)] = d["h_exact"] - return out - - -def load_groups(): - doc = yaml.safe_load(open(os.path.join(REPO, REGISTRY))) - groups = doc if isinstance(doc, list) else doc.get("groups", doc) - if not isinstance(groups, list): - sys.exit(f"dedup_extend: unexpected {REGISTRY} shape: {type(groups).__name__}") - return doc, groups - - -def add_members_surgical(additions): - """Append binaries to each group's `binaries: [...]` list by TEXT EDIT, in place. - - NEVER `yaml.safe_dump` this file. The first cut of this tool round-tripped it through - safe_dump and silently destroyed BOTH of the things a human needs from it (H5 — "never - silently drop comments on a rewrite"): - * all 47 comment lines — including the curated Phase-11 header explaining WHY the share is - source-level (the linker cannot excise bytes interior to an object) — dumped to nothing; - * every `vram: 0x80162FF4` re-serialized as `vram: 2148937716` (PyYAML parses YAML-1.1 hex - to int, and dumps int as decimal), making 1,832 entries unreadable. - It was invisible to every gate: dedup-check passed 1840/0 and check-all stayed 140/140, - because `_addr()` accepts both forms — the data was fine and the DOCUMENT was ruined. A - formatting-destructive write that all your oracles call green is exactly the class this - project keeps re-learning: the gate measures bytes, not intent. - - `additions` = {group_id: [binary, ...]}. Idempotent: a binary already listed is skipped. - """ - p = os.path.join(REPO, REGISTRY) - lines = open(p).read().splitlines(keepends=True) - cur, n = None, 0 - seen = set() - for i, ln in enumerate(lines): - m = re.match(r"^\s*-?\s*id:\s*(\S+)\s*$", ln) - if m: - cur = m.group(1) - continue - if cur and cur in additions and re.match(r"^\s*members:\s*$", ln): - # a VERBOSE group has no `binaries:` line — silently skipping it left five 141-member groups listed at 16 (P35 S94/S96) - raise SystemExit(f"add_members_surgical: {cur} is in the verbose `members:` form — convert it to shorthand first " - f"(share_body.py --repair-registry does), never skip it (R43)") - if cur and cur in additions and re.match(r"^\s*binaries:\s*\[", ln): - seen.add(cur) - add = [b for b in additions[cur] if re.search(rf"\b{re.escape(b)}\b", ln) is None] - if add: - lines[i] = ln.rstrip("\n").rstrip()[:-1].rstrip() + ", " + ", ".join(add) + "]\n" - n += len(add) - cur = None - open(p, "w").write("".join(lines)) - return n - - -def _addr(v): - return int(v, 16) if isinstance(v, str) else int(v) - - -def plan_for(binary, groups): - """[(group, vram, macro)] for every h_exact group this binary could join, plus the skip tally. - - A group is extendable into `binary` iff: it is h_exact tier, the binary is not already a member, - the binary has a LIVE STUB at the group's vram, that stub's original bytes hash to the group's - recorded h_exact (the C1 equivalence the registry itself asserts), and it is NOT a jr/switch - function (see below). - - THE jr GUARD (cookbook §53). A function with a mid-body `jr` on a non-$ra register dispatches - through a compiler-generated jump table the LINKER must place at the sibling's exact address — a - per-sibling `jtbl_carve` + interleave (`jtbl_family_bank.py`). This tool has no carve step and a - newly-onboarded overlay has no `_jr_*` split at all, so a jr body instantiated here would leave - its table unplaced and the gate would (correctly) reject it. The guard is PREVENTIVE: it skips - and REPORTS rather than feeding the gate drafts that cannot pass (R32 — a skip is only honest if - it is counted and named). On the SC07 set it currently skips ZERO: no jr function is in the - extendable set. - - ⚠️ R14 — WHAT THIS GUARD IS *NOT*. It does NOT explain the 12 DIFFs the first run produced - (`func_80162FF4/801630C4/80163194/8016325C`, the same 4 in every overlay, 12 of 6457 = 0.19%). - I first assumed those were the jr class because ov_SC01_077 hosts them in `_jr_8017A4AC.c` / - `_jr_80182268.c` — a natural read, and WRONG: `has_mid_jr` is **False** for all four (33-52 ins, - no jump table). They merely LIVE in a carved jr-REGION split (the carve region for - `func_8017A4AC` sweeps in every function in its address range), which says nothing about them. - Their DIFF cause is UNDIAGNOSED and logged for follow-up; they are correctly left as stubs by the - gate. Hosting file != function class — do not infer one from the other. - - Detector reused verbatim from `family_hseq.has_mid_jr` (R33 — one oracle, not two). - """ - sig = sig_hashes(binary) - stubs = corpus.stubs(binary) - out, skips = [], {} - for g in groups: - if g.get("tier") != "h_exact": - continue - if "vram" not in g or "binaries" not in g: - skips["verbose-form"] = skips.get("verbose-form", 0) + 1 - continue # not position-locked - if binary in g["binaries"]: - continue - vram = _addr(g["vram"]) - if vram not in stubs: - continue # not a live stub here (matched already, or absent) - if sig.get(vram) != g["hash"]: - continue # DIFFERENT CODE at the same vram — the point of the check - words = FR.stream_words(binary, vram, FR.nins_of(binary, vram)) - if words and family_hseq.has_mid_jr(words): - skips["jr-needs-carve"] = skips.get("jr-needs-carve", 0) + 1 - continue # §53: route via jtbl_family_bank, not here - out.append((g, vram, g["func"])) - return out, skips - - -def ensure_include(binary, apply=True): - """Add the shared-header include if absent. Byte-neutral: engine_core.h is only `#define`s, so - including it without instantiating emits no code (proven: ov_SC07_006 built 7ca772be with the - include + one macro). Returns True if the file was changed.""" - p = os.path.join(REPO, f"src/{binary}/{binary}.c") - t = open(p).read() - if SHARED_INCLUDE in t: - return False - if apply: - t = t.replace('#include "common.h"', '#include "common.h"\n' + SHARED_INCLUDE, 1) - open(p, "w").write(t) - return True - - -def write_drafts(binary, plan): - d = os.path.join(REPO, EXTEND_DIR, binary) - subprocess.run(["rm", "-rf", d], check=False) - os.makedirs(d, exist_ok=True) - for g, vram, macro in plan: - fn = f"func_{vram:08X}" - body = f"{macro}() /* dedup: shared engine-core @0x{vram:08x} (src/shared) */\n" - open(os.path.join(d, fn + ".c"), "w").write(body) - return d - - -def gate(binary, drafts_dir, chunk, ladder=True): - """The whole-binary byte-gate is the sole arbiter (G3/P9). Returns the banked fn set. - - LADDER (S46): this called `harvest_verify` VERBATIM, which is the bare gate with no recovery. - Measured cost of that: 0 of 142 extendable groups banked, and the classified reasons were 118 - PLUMBING / 21 CC1-FAIL / 3 DIFF — i.e. ~1 in 50 was a real byte divergence and everything else - was a declaration conflict in the TARGET TU (`conflicting types for memcpy / ApplyMatrixSV / - D_800AE620 / func_8014C568`). `gate_stage` wraps the same gate in exactly the reconcile ladder - those need (canon_resident_calls -> cast_call_sites -> sig_unify -> harvest_verify), and it - carries the §156 reconcile LEDGER so a failed candidate cannot leave a fleet-shared edit behind. - Measured on the same failure class in the S46 wave residue: 6 of 19 PLUMBING recovered (~32%). - - GATE_NO_ARITY=1 is set for the child: gate_stage's arity pre-pass writes the fleet-shared - engine_core.h BEFORE the gate, and on a failing draft that edit can survive — the F1 defect that - broke 141 of 213 binaries in S45 and made every later gate report `near`. The ladder's other - rungs are draft-local and safe. - """ - good = open(os.path.join(REPO, f"config/check.{binary}.sha")).read().split()[0] - vout = f".run/extend_verified.{binary}.txt" - fout = f".run/extend_failed.{binary}.txt" - if ladder: - cmd = [PY, "tools/gate_stage.py", "--binary", binary, - "--drafts", os.path.relpath(drafts_dir, REPO), "--no-propagate", - "--verified-out", vout, "--failed-out", fout] - else: - cmd = [PY, "tools/harvest_verify.py", "--binary", binary, - "--out", f"build/{binary}/{binary}", "--good-sha", good, - "--drafts", os.path.relpath(drafts_dir, REPO), "--chunk", str(chunk), - "--verified-out", vout, "--failed-out", fout] - env = dict(os.environ, GATE_NO_ARITY="1") - subprocess.run(cmd, cwd=REPO, timeout=7200, env=env) - p = os.path.join(REPO, vout) - return set(open(p).read().split()) if os.path.exists(p) else set() - - -def main(): - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--binaries", required=True, help="comma list of newly-onboarded binaries to wire in") - ap.add_argument("--chunk", type=int, default=512, - help="drafts per build (h_exact is byte-identical by construction, so the happy " - "path is ONE build; harvest_verify bisects on failure). default 512") - ap.add_argument("--limit", type=int, default=0, help="cap #groups per binary (0 = all)") - ap.add_argument("--check-only", action="store_true", help="print the plan, touch nothing") - a = ap.parse_args() - os.chdir(REPO) - - doc, groups = load_groups() - targets = [b.strip() for b in a.binaries.split(",") if b.strip()] - - dirty = subprocess.run("git status --porcelain -- config/ src/", shell=True, - capture_output=True, text=True).stdout.strip() - if dirty and not a.check_only: - sys.exit("dedup_extend: config/ or src/ is dirty — commit first so a failed gate reverts " - "cleanly.\n" + dirty[:400]) - - total_banked, total_planned = 0, 0 - for b in targets: - plan, skips = plan_for(b, groups) - if a.limit: - plan = plan[:a.limit] - total_planned += len(plan) - print(f"[{b}] {len(plan)} extendable h_exact group(s); " - f"include {'ABSENT -> add' if ensure_include(b, apply=False) else 'present'}" - + (f"; skipped {skips}" if skips else "")) - if a.check_only or not plan: - continue - - added_include = ensure_include(b) # True IFF this run inserted the line - d = write_drafts(b, plan) - banked = gate(b, d, a.chunk) - print(f"[{b}] BANKED {len(banked)} / {len(plan)}") - total_banked += len(banked) - - if not banked: - # ONLY undo what THIS run did (§61/§63). The revert is an INVERSE TRANSFORM, not a - # snapshot restore, so firing it on a binary that ALREADY had the include strips a - # load-bearing line: every `DEFINE_func_*()` in that overlay stops resolving. It was - # unconditional until Phase 29 SESSION-19, where a 0-banked run over 135 already-wired - # binaries removed the include from all 135 at once. Invisible to every byte-gate (R34) - # because the damage lands AFTER the last gate has run. - if added_include: - ensure_include_revert(b) - continue - adds = {g["id"]: [b] for g, vram, _ in plan # registry: only what the GATE accepted (P9) - if f"func_{vram:08X}" in banked} - n = add_members_surgical(adds) # TEXT edit — never safe_dump (see the docstring) - print(f"[{b}] registry: +{n} membership(s)") - print(f"\n=== dedup_extend: banked {total_banked} / {total_planned} planned " - f"across {len(targets)} binaries ===") - if a.check_only: - print("(--check-only: no files touched)") - - -def ensure_include_revert(binary): - p = os.path.join(REPO, f"src/{binary}/{binary}.c") - t = open(p).read().replace('#include "common.h"\n' + SHARED_INCLUDE, '#include "common.h"', 1) - open(p, "w").write(t) - - -if __name__ == "__main__": - main() diff --git a/tools/dedup_propagate.py b/tools/dedup_propagate.py deleted file mode 100644 index 5e46478dba..0000000000 --- a/tools/dedup_propagate.py +++ /dev/null @@ -1,1119 +0,0 @@ -#!/usr/bin/env python3 -"""dedup_propagate.py — match once, share many across the overlay fleet (Phase 15, cookbook §14). - -The overlay fleet is position-locked at vram 0x80128158, so a shared engine function has the -SAME vaddr (hence the same `func_` symbol) and BYTE-IDENTICAL body in every overlay that -contains it. This tool takes a function already matched in ONE overlay and propagates that single -matched C body to every onboarded overlay whose signature shows the same `h_exact` at that addr: - - 1. extract the matched body from the source overlay's .c (externs + the function def) - 2. author it ONCE as a `DEFINE_func_()` macro in src/shared/
.h (idempotent) - 3. at each member overlay, replace that function's INCLUDE_ASM stub (or, in the source overlay, - its inline def) in place with `DEFINE_func_()` — address order preserved - 4. BYTE-GATE every touched overlay (`make build BINARY=` == its check.sha); on ANY miss, - restore every file from an in-memory snapshot and abort (fail-closed; nothing wrong can land) - 5. register the group in config/dedup.us.yaml (the byte-honesty registry, validated by - tools/dedup_integrate.py --check) - -Everything is keyed by the integer address: the sig uses lowercase hex ("0x80144b9c"), splat emits -the uppercase symbol ("func_80144B9C") — never compare the strings, always the int. - -Lead with h_exact (guaranteed byte-identity). The per-overlay `make check` is the sole arbiter -(G3/P9): a wrong propagation cannot pass it. - -Usage: - tools/dedup_propagate.py --addr 0x8013xxxx[,0x...] [--source-overlay ov_SC01_077] - tools/dedup_propagate.py --auto-from ov_SC01_077 [--min-reach 2] [--limit N] - tools/dedup_propagate.py ... --check-only # dry run: print the plan, touch nothing -Options: - --header src/shared/engine_core.h target cluster header (created/appended) - --tier h_exact|h_norm default h_exact (h_norm requires the byte-gate to pass on ALL) - --binaries a,b,c restrict members to these onboarded overlays (default: all that share) - --no-gate skip the per-overlay build gate (CI / batch re-gate later) -""" -import argparse, collections, functools, json, pathlib, re, subprocess, sys -import os as _os -import threading as _threading -sys.path.insert(0, _os.path.dirname(_os.path.abspath(__file__))) -import cdecl # the comment/string masking oracle (Phase 26-A) — see _skippable below -import shared_lock # Stage 1: fleet-shared RW lock — propagation is an EXCLUSIVE writer - -ROOT = pathlib.Path(__file__).resolve().parent.parent - - -# ---------------------------------------------------------------- fleet / sig helpers -def onboarded_overlays(): - """Overlay aliases + (S44) module aliases — every non-main/resident binary a shared body can - propagate into. Modules carry engine functions too; excluding them re-creates the SC07 - invisible-work bug one class over.""" - out = [] - for f, var in (("config/overlays.mk", "OVERLAY_BINARIES"), ("config/modules.mk", "MODULE_BINARIES")): - fp = ROOT / f - if fp.exists(): - m = re.search(rf"^{var}\s*:=\s*(.*)$", fp.read_text(), re.M) - if m: - out += m.group(1).split() - return out - - -def sig_path(ov): - return ROOT / f".run/sig.{ov}.jsonl" - - -def load_sig(ov): - """addr(int) -> {h_exact, h_norm, nins, ...}; {} if absent.""" - p = sig_path(ov) - if not p.exists(): - return {} - out = {} - for ln in p.read_text().splitlines(): - ln = ln.strip() - if ln: - r = json.loads(ln) - out[int(r["addr"], 16)] = r - return out - - -def registered_addrs(): - """Vaddrs already in config/dedup.us.yaml (shared via ANY mechanism — engine_core, ov_setters, - clearTbl40). --auto-from skips these so the bulk is purely additive and never collides with an - existing share (e.g. a SETTER-matched function) — which would trip the structural check.""" - p = ROOT / "config/dedup.us.yaml" - if not p.exists(): - return set() - try: - import yaml - sys.path.insert(0, str(ROOT / "tools")) - from dedup_integrate import group_members - data = yaml.safe_load(p.read_text()) or {} - return {v for g in (data.get("groups") or []) for (_b, v, _n) in group_members(g)} - except Exception: - return set() - - -def sym(addr): - return f"func_{addr:08X}" # splat convention: uppercase 8-hex - - -def c_path(ov): - return ROOT / f"src/{ov}/{ov}.c" - - -def overlay_files(ov): - """[(path, asm_subdir)] for ov's source file(s): the main .c plus any Phase-19 split files - (ov_SC01_077_a.c / _o0.c). Single-file overlays return just the main .c — default behaviour - preserved. Lets a fn matched in a split file propagate ×reach (its stub/def lives in _a/_o0).""" - # DERIVED from the tree — a GLOB, never an allowlist (Phase 26-A audit, HIGH). - # - # This was a hardcoded suffix list ("_a","_o0","_o0b","_after") that predated the Phase-26 jr - # carves, so it returned 404 of the fleet's 811 overlay .c. The 407-file gap held **36,135 - # INCLUDE_ASM stubs and ~32,000 inline defs — half the corpus** — and overlay_files gates ALL of - # dedup_propagate (source_text / find_site / apply_plan / struct_check / reconcile_caller_extern). - # 435 of ov_SC01_077's 689 inline defs were invisible to --auto-from. - # - # It is also WHY the 4 functions in .run/audit/a1_harvest_fuel.json (untracked since P33.5) were never propagated: three - # of them are defined in ov_SC01_077_jr_8012ACE0.c, which this list could not see. The dedup group - # was registered anyway and dedup_integrate greenlit the lie (A1). Two silent-skip bugs compounding. - # - # Do NOT "fix" this by adding _jr_* to the tuple: the NEXT split family would re-open the hole. - # The asm_subdir is always the file stem — an invariant the four old entries already satisfied. - out = [(c_path(ov), ov)] - for p in sorted(ROOT.glob(f"src/{ov}/{ov}_*.c")): - out.append((p, p.stem)) - return out - - -def source_text(ov): - """Concatenation of all of ov's split files — for SOURCE-side def finding + body extraction - only (find_site uses the body lines, so cross-file concat is safe; never used to EDIT).""" - return "\n".join(p.read_text() for p, _ in overlay_files(ov)) - - -def stub_line(ov, addr): - return f'INCLUDE_ASM("asm/{ov}/nonmatchings/{ov}", {sym(addr)});' - - -# ---------------------------------------------------------------- body extraction -@functools.lru_cache(maxsize=8) -def _split_masked(text): - """(lines, mlines) for one source text — MEMOIZED, because it does not depend on the address. - - THE WHOLE COST OF THIS TOOL WAS HERE (P31 S75, measured). `find_site` re-ran `cdecl._mask` - (four regex passes, one with `re.S`) plus two `splitlines()` over the ENTIRE concatenated - source on EVERY call, and the caller loops over every function in the binary. On - `ov_SC01_005` (2.50 MB of source, 2,503 functions in the sig): - - splitlines(text) 6.5 ms - cdecl._mask(text) 38.9 ms <- 4 regex passes, one re.S - splitlines(mask) 4.6 ms - ------------------------------- 54 ms PER CALL, before a single line is searched - - and `find_site` is called per-address in the `--auto-from` enumeration, again per-target while - building the plan, and again during propagation. Observed: **29m38s of CPU on ONE core** - (`nlwp=1` — the `ThreadPoolExecutor` below covers only the cpp/cc1 probe, not this), ~92% CPU, - no subprocesses, no writes. The mask is a pure function of the text, so every call after the - first recomputed an identical result: the shape was O(functions x source-bytes) where it should - be O(source-bytes + functions). - - `lru_cache` is the right primitive here rather than a hand-rolled dict: callers already hold one - `txt_cache`/`ctext` string OBJECT per binary and pass it repeatedly, and CPython caches a str's - hash on the object, so a repeat lookup is a pointer compare. maxsize=8 covers source + members - in flight without pinning many megabytes. - - NB this is the SECOND half of this phase's cost to be fixed; the comment on the cc probe below - records the first ("~5 minutes pegged at one core on a 32-core box"). Fixing the subprocess half - left the regex half as the dominant term.""" - lines = text.splitlines() - # Comments blanked ONCE by the project's single masking oracle (R33). Length-preserving, so - # index i into mlines is index i into lines. This is what makes _skippable MULTI-LINE aware. - mlines = cdecl._mask(text).splitlines() - if len(mlines) != len(lines): # R32: the invariant this rests on, asserted not assumed - mlines = lines - return lines, mlines - - -def find_site(text, ov, addr): - """Locate this function in ov's .c. Returns (kind, start, end, body_lines): - kind 'stub' -> the INCLUDE_ASM line (start==end, body_lines None) - kind 'def' -> an inline definition block (preceding contiguous externs .. closing brace) - kind 'macro' -> already a DEFINE_func_() instantiation (already propagated) - None -> not present / matched in some other form.""" - lines, mlines = _split_masked(text) - s = sym(addr) - stub = stub_line(ov, addr) - for i, l in enumerate(lines): - if l.strip() == stub: - return ("stub", i, i, None) - for i, l in enumerate(lines): - if l.strip() == f"DEFINE_{s}()" or l.strip().startswith(f"DEFINE_{s}()"): - return ("macro", i, i, None) - # inline definition: " func_XXXX(...)" at column 0 OR indented (recover_integration/harvest - # write the def block indented -> a column-0-only match silently dropped every indented def from - # propagation, T6 blocker 1), brace on the SAME or the NEXT line (drafts vary the brace placement). - # SAFE against indented CALL-exprs: the pattern requires a TYPE prefix ([A-Za-z_][\w \*]*, which - # admits neither '(' nor '=') before the name AND the whole line to BE the signature (ends ')' or - # '){'), so `if (func_X(...)) {`, `x = func_X(...);`, and a bare `func_X(a);` call never match. - # The signature HEAD. Anchored on a type prefix ([A-Za-z_][\w \*]*, which admits neither '(' nor - # '=') followed by the name and an open paren — so `if (func_X(...))`, `x = func_X(...);` and a - # bare `func_X(a);` call can never match. Deliberately NOT anchored on the line ENDING in ')': - # that anchor silently dropped every MULTI-LINE signature (Phase 26-A audit). - # S33: a function defined under the §37/§73 ASM-LABEL ALIAS is named `aF` in C and only - # BINDS the real symbol via `__asm__("func_")`. A head regex anchored on the literal - # `func_` is structurally blind to it, so find_site returned None and every caller read - # "not matched" — func_801466F0 (137 members / 3,288 ins) sat unreachable behind exactly this. - # family_remap._alias_decl_for already resolves the form (and, since S33, its WRAPPED variant); - # reuse it rather than write a second matcher (R33 — one oracle). - names = [s] - try: - import family_remap as _FR - _alias, _ = _FR._alias_decl_for(lines, addr) - if _alias and _alias.lower() != s.lower(): - names.append(_alias) - except Exception: - pass - defhead = re.compile(r"^\s*[A-Za-z_][\w \*]*\b(?:" + "|".join(re.escape(n) for n in names) + r")\s*\(") - # A K&R parameter declaration, e.g. `s32 arg0;` / `struct S *p[4];` - kr_param = re.compile(r"^\s*[A-Za-z_][\w \t\*]*\b\w+\s*(\[[^\]]*\])?\s*;\s*$") - for i, l in enumerate(lines): - m = defhead.match(l) - if not m: - continue - # Walk from the '(' that opens the parameter list to ITS matching ')', character by character - # and across lines. Do NOT use line.count('(')-count(')') or split(')')[-1]: a single-line body - # containing a call (`void f(int a) { g(a); }`) has balanced parens of its own, so both - # shortcuts land on the WRONG paren and then misread the body's `;` as a prototype terminator. - depth, li, ci = 0, i, m.end() - 1 - while li < len(lines): - line = lines[li] - while ci < len(line): - if line[ci] == "(": - depth += 1 - elif line[ci] == ")": - depth -= 1 - if depth == 0: - break - ci += 1 - if depth == 0 and ci < len(lines[li]): - break - li += 1 - ci = 0 - if li >= len(lines): - continue - close = li # the line on which the parameter list closes - rest = lines[close][ci + 1:] # everything AFTER the signature's own ')' - if rest.lstrip().startswith(";"): - continue # `... );` -> a PROTOTYPE, not a definition - tail = rest - # locate the body's opening brace. It is on the closing line, OR on a following line — and in - # a K&R definition the PARAMETER DECLARATIONS sit between ')' and '{'. The old code demanded - # the next non-blank line start with '{', so it silently dropped EVERY K&R definition — which - # is the project's house style for exactly the biggest, highest-reach functions (func_8015AE2C - # 562 ins, func_80166994, func_80133CD4, func_8015A3C8). Those live in the _jr_* files this - # function could not even open until the overlay_files glob above; fixing one without the other - # would have exposed the files and still dropped their biggest prizes. - if "{" in tail: - bstart = close # single-line body / brace on the signature line - else: - k = close + 1 - while k < len(lines) and (lines[k].strip() == "" or kr_param.match(lines[k])): - k += 1 # skip blanks AND K&R parameter declarations - if k >= len(lines) or "{" not in lines[k]: - continue # no body -> a prototype - bstart = k - if True: - # brace-match forward to the closing '}' - depth = 0 - end = None - for j in range(bstart, len(lines)): - depth += lines[j].count("{") - lines[j].count("}") - if depth <= 0: - end = j - break - if end is None: - return None - # collect contiguous preceding extern declarations (skip blank lines). A trailing - # `/* comment */` after the ; is allowed (a banked extern block often annotates a decl, - # e.g. `extern u8 D_x[]; /* canonical TU type */` — a comment-blind `;\s*$` stopped the - # scan there and dropped every EARLIER extern, failing compiles_standalone on the now- - # undeclared callees/data; T6.4 fix for func_8014E048's pin/asm body). - # SESSION-18: the walk skipped BLANK lines but not STANDALONE COMMENT lines, so a - # full-line `/* ---- */` between two extern groups halted it and silently dropped every - # extern ABOVE the comment. That is the whole "CARRY-FIXABLE" class: the body then fails - # compiles_standalone on now-undeclared data/callees, and the caller filed it under - # "overlay-local TYPE (the real cap)" — a mislabel that wrote the class off for ~4 phases. - # (T6.4 had already fixed the TRAILING-comment case, `extern u8 D_x[]; /* note */`; this - # is the standalone-LINE case it did not reach.) Skip comment-only lines exactly like - # blanks, and drop them from the emitted body so make_macro never sees a `//`. - # S11: the SESSION-18 fix handled BLANK, `//`, and SINGLE-LINE `/* … */` lines, but a - # MULTI-LINE block comment still halted the walk — its middle lines start with `*` and - # its last line ends `*/` without starting `/*`. That is the §134 multi-line-blindness - # class (S6b fixed the same shape three times in family_remap). Deciding on the MASK - # instead of on line syntax subsumes every comment form in one oracle and cannot be - # fooled by a `/*` inside a string. Byte-measured: this is the whole CARRY-FIXABLE - # bucket for func_8012A598 (3,288 templatable ins that were being written off). - def _skippable(idx): - return mlines[idx].strip() == "" - start = i - k = i - 1 - while k >= 0 and _skippable(k): - k -= 1 - while k >= 0 and re.match(r"^\s*extern\b.*;\s*(/\*.*\*/\s*)?$", lines[k]): - start = k - k -= 1 - while k >= 0 and _skippable(k): - k -= 1 - body = [lines[j] for j in range(start, end + 1) if not _skippable(j)] - return ("def", start, end, body) - return None - - -def make_macro(addr, body_lines): - """Turn an extracted body (externs + def) into a `#define DEFINE_func_() \\`-continued macro. - Fails loud on constructs that don't survive line-continuation (// comments, trailing backslash).""" - s = sym(addr) - for ln in body_lines: - if "//" in ln: - raise SystemExit(f"[refuse] {s}: body has a // comment — not macro-safe (handle manually)") - if ln.rstrip().endswith("\\"): - raise SystemExit(f"[refuse] {s}: body line ends with backslash — not macro-safe") - out = [f"#define DEFINE_{s}() \\"] - for idx, ln in enumerate(body_lines): - cont = " \\" if idx < len(body_lines) - 1 else "" - out.append(" " + ln.rstrip() + cont) - return "\n".join(out) + "\n" - - -# ---------------------------------------------------------------- file edits (in memory) -INCLUDE_RE = re.compile(r'^#include\s+"\.\./shared/') - - -def ensure_include(text, header): - rel = f'#include "../shared/{pathlib.Path(header).name}"' - if rel in text: - return text - lines = text.splitlines(keepends=True) - for i, l in enumerate(lines): - if l.strip() == '#include "common.h"': - lines.insert(i + 1, rel + "\n") - return "".join(lines) - return rel + "\n" + text # fallback: prepend - - -def replace_site(text, ov, addr, kind, start, end): - lines = text.splitlines(keepends=True) - repl = f"DEFINE_{sym(addr)}() /* dedup: shared engine-core @0x{addr:08X} (src/shared) */\n" - lines[start:end + 1] = [repl] - return "".join(lines) - - -# ---------------------------------------------------------------- dedup.us.yaml registration -def append_groups(groups): - """Append new groups textually (preserve the file's header comments). Idempotent by id.""" - path = ROOT / "config/dedup.us.yaml" - text = path.read_text() - blocks = [] - for g in groups: - if re.search(rf"^\s*-\s*id:\s*{re.escape(g['id'])}\s*$", text, re.M): - continue # already present - # position-locked share -> compact shorthand (vram + binaries list), ~20x smaller than verbose - # members for fleet-wide groups; group_members() in dedup_integrate expands it. - bins = ", ".join(m["binary"] for m in g["members"]) - blocks.append( - f" - id: {g['id']}\n" - f" tier: {g['tier']}\n" - f" hash: {g['hash']}\n" - f" source: {g['source']}\n" - f" func: DEFINE_{sym(g['addr'])}\n" - f" vram: 0x{g['addr']:08X}\n" - f" binaries: [{bins}]") - if not blocks: - return 0 - if not text.endswith("\n"): - text += "\n" - path.write_text(text + "\n".join(blocks) + "\n") - return len(blocks) - - -_BJOBS = int(_os.environ.get('BFM_BUILD_JOBS') or (_os.cpu_count() or 8)) - - -# ---------------------------------------------------------------- byte gate -def byte_gate(ov): - # -j — see harvest_verify's note: ~6x on a per-binary build, and this gate runs once per - # propagation candidate, which is why a wide propagation dominated a 33-minute gate in P31 S67. - r = subprocess.run(["make", "-j%d" % _BJOBS, "build", f"BINARY={ov}"], cwd=ROOT, - capture_output=True, text=True) - return r.returncode == 0, r.stdout + r.stderr - - -def _jobs(): - """Parallel overlay gates. Env JOBS wins; otherwise use EVERY core. - - Deliberately NOT capped at the Makefile's conservative `JOBS ?= 16` default: each worker is a - `make build` that spends nearly all its wall-clock in cc1/as/ld subprocesses, so the box is the - limit, not Python. Measured 2026-08-07: the serial version ran a propagation for 95 minutes at - load 1.6 on a 32-core machine — ~5% utilisation. Saturate it (Drew, same session). - """ - try: - j = int(_os.environ.get("JOBS", "0")) - except ValueError: - j = 0 - return max(1, j or (_os.cpu_count() or 4)) - - -_SRC_BYTES = {} - - -def _src_bytes(ov): - """Total bytes of an overlay's C sources — the build-cost proxy for longest-first scheduling. - - Cheap and good enough: `make build BINARY=` spends nearly all its time in cc1 on those - files, so source size ranks the giants (ov_SC01_077 & co) to the front of the queue.""" - if ov not in _SRC_BYTES: - try: - _SRC_BYTES[ov] = sum(p.stat().st_size for p, _ in overlay_files(ov) if p.exists()) - except OSError: - _SRC_BYTES[ov] = 0 - return _SRC_BYTES[ov] - - -def gate_all(changed, label=""): - """Byte-gate every touched overlay in parallel; return the FIRST failure (compat wrapper).""" - f = gate_failures(changed, label) - return f[0] if f else None - - -def gate_failures(changed, label=""): - """Byte-gate EVERY touched overlay in parallel; return ALL failures in `changed` ORDER. - - The sweep already builds every overlay and already knows every verdict — returning only the - first threw ~26 of 27 answers away, and the recovery loop then paid a FULL sweep to rediscover - each one (S45: the same two functions were excluded from 27 overlays, one sweep apiece). Same - builds, same determinism, all the information. - - WHY (measured 2026-08-07): this loop used to be serial — one `make build BINARY=` at a - time, over up to 141 members, for each of ~30 functions. A propagation ran 95 minutes at load - 1.6 on a 32-core box: ~5% utilisation. The Makefile has parallelised `extract-all`/`check-all` - since Phase 26 (`xargs -P$(JOBS)`), but dedup_propagate predates that and drives the - SINGLE-binary `build` target from Python, so it never saw any of it. - - SAFE for the same reason `check-all` is: byte_gate only runs `make build BINARY=`, which - writes solely to the per-binary-disjoint `build//**`; it mutates NO source. The splice has - already happened before this is called, and the restore happens after — only the *verification* - is parallel. Threads, not processes: subprocess.run releases the GIL while the build runs. - - DETERMINISM: ThreadPoolExecutor.map preserves input order, so the reported first failure is the - first in `changed` order — identical to the serial loop's verdict, not whichever build finished - first. The serial version short-circuited on the first failure and so did fewer builds; this - does them all, but in parallel, and the all-pass case (the common one) is a straight win. - """ - order = list(changed) - if not order: - return [] - j = min(_jobs(), len(order)) - if j <= 1: - return [ov for ov in order if not byte_gate(ov)[0]] - from concurrent.futures import ThreadPoolExecutor - print(f"[gate] byte-gating {len(order)} overlay(s) with {j} parallel builds{label}", flush=True) - # LONGEST-FIRST (LPT). ThreadPoolExecutor.map starts work in list order, and the giant overlays - # (ov_SC01_077 & co — 15k lines at -O2) sit late in it, so every sweep ended with 31 cores - # watching ONE build: measured 31 s saturated then ~25 s of a single cc1. Starting the big ones - # first overlaps that tail with the crowd. Execution order only — results are re-sorted into - # `changed` order below, so the reported verdict is bit-for-bit the same as the serial loop's. - sched = sorted(order, key=lambda ov: -_src_bytes(ov)) - with ThreadPoolExecutor(max_workers=j) as ex: - results = dict(ex.map(lambda o: (o, byte_gate(o)[0]), sched)) - return [ov for ov in order if not results[ov]] - - -# ---------------------------------------------------------------- straggler caller-extern reconcile (--recover) -def reconcile_caller_extern(ov, addr): - """no-proto every conflicting `extern func_();` caller decl in ov's src files - (main + _a/_o0 splits), keeping the return type, dropping the params. A member overlay that - forward-declares the banked fn with a prototype INCOMPATIBLE with the def (`extern void - func_X(s32,s32,s32);` vs the def's (s32,s32,u32)) makes its TU fail `conflicting types` once the - macro def lands. No-proto'ing that decl is byte-NEUTRAL (the call `func_X(a,b,c)` emits identical - code; a K&R decl is compatible with any promotion-safe def) and lets the def compile — the same - lever as tools/fix_arity_callers.py --any-proto, applied surgically to the failing overlay. The - whole-binary byte-gate stays the sole arbiter (G3/P9). Returns (snapshot, n_edits); the snapshot - restores the files VERBATIM (fix_arity_callers --revert is LOSSY for non-(void) forms).""" - rx = re.compile(rf'(extern\s+[A-Za-z_][\w \t\*]*?\b{sym(addr)}\s*\()\s*[^;)]+?\s*(\)\s*;)', re.I) - snap, n = {}, 0 - for cp, _ in overlay_files(ov): - txt = cp.read_text() - snap[cp] = txt - new, k = rx.subn(r"\1\2", txt) - if k: - cp.write_text(new) - n += k - return snap, n - - -def restore_snapshot(snap): - for cp, txt in snap.items(): - cp.write_text(txt) - - -CC1 = ROOT / "tools/bin/gcc-2.7.2-psx/cc1" - -def compiles_standalone(body_lines): - """True iff the lifted body compiles with ONLY common.h (cpp -> cc1). A body that uses overlay- - LOCAL struct types (named in the SOURCE overlay's .c but not common.h) compiles in the source yet - FAILS in every other overlay — it cannot be mechanically lifted. Pre-filtering on this avoids an - all-or-nothing byte-gate revert and lets us report the non-liftable count honestly (P9).""" - # include the shared engine types so struct-USING bodies (that reference header types like - # `struct Vec`) resolve; a body using an overlay-local typedef not in the header still fails -> skip. - src = ('#include "common.h"\n#include "engine_types.h"\n' - + "\n".join(body_lines) + "\n") - d = ROOT / ".run/dpcc"; d.mkdir(parents=True, exist_ok=True) - # UNIQUE per call: this used to be a fixed `t.c`, so two concurrent probes would compile each - # other's body — the same fake-isolation class as match_one's shared --work dir (P28 T5). The - # plan phase now runs these in parallel, so the shared path is a correctness bug, not a style one. - f = d / f"t.{_os.getpid()}.{_threading.get_ident()}.c"; f.write_text(src) - cpp = subprocess.run(["mipsel-linux-gnu-cpp", "-lang-c", f"-I{ROOT}/include", - f"-I{ROOT}/src/shared", "-undef", - "-fno-builtin", "-Dmips", "-D__GNUC__=2", "-D__OPTIMIZE__", "-Dpsx", - "-D_PSYQ", "-D_MIPSEL", "-D_LANGUAGE_C", str(f)], capture_output=True, text=True) - if cpp.returncode != 0: - return False, (cpp.stderr or "cpp failed") - cc1 = subprocess.run([str(CC1), "-quiet", "-O2", "-G0", "-mips1", "-mcpu=3000", "-mgas", - "-msoft-float", "-fgnu-linker", "-o", "/dev/null"], - input=cpp.stdout, capture_output=True, text=True) - return cc1.returncode == 0, (cc1.stderr or "") - - - -MACRO_RE = re.compile(r'^\s*DEFINE_func_([0-9A-Fa-f]+)\(\)') - - -def place_in_overlay(ov, subplan, header_rel, edit): - """Instantiate each subplan fn's DEFINE_ macro at its site in ONE overlay, via the `edit` - callback (path, newtext). Returns (placed_anything, [unresolved addrs]). - - Module-level ON PURPOSE: the per-overlay search runs in a PROCESS pool (the work is regex over - 15k-line files, which threads cannot parallelise — measured: 138 "parallel" thread searches kept - 0-4 builds alive because they all serialised on the GIL). A process pool needs a picklable - top-level entry point, and both the in-process apply and the workers must use the SAME placement - logic or they will drift (R33).""" - remaining = {p["addr"]: p for p in subplan} - ovc = False - for cp, asm_sub in overlay_files(ov): # main + Phase-19 split files (_a/_o0) - if not remaining: - break - lines = ensure_include(cp.read_text(), header_rel).splitlines(keepends=True) - sp = re.compile(rf'^\s*INCLUDE_ASM\("asm/{re.escape(ov)}/nonmatchings/{re.escape(asm_sub)}",\s*func_([0-9A-Fa-f]+)\);\s*$') - stub_idx, macro_set = {}, set() - for i, l in enumerate(lines): - ms = sp.match(l) - if ms: stub_idx[int(ms.group(1), 16)] = i; continue - mm = MACRO_RE.match(l) - if mm: macro_set.add(int(mm.group(1), 16)) - joined = "".join(lines) - line_repls, def_ranges = {}, [] - for ad in list(remaining): - if ad in macro_set: - del remaining[ad]; continue # already instantiated in this file (idempotent) - repl = f"DEFINE_{sym(ad)}() /* dedup: shared engine-core @0x{ad:08X} (src/shared) */\n" - if ad in stub_idx: - line_repls[stub_idx[ad]] = repl; del remaining[ad]; ovc = True - else: - site = find_site(joined, ov, ad) # inline def in THIS file? (else the next split file) - if site and site[0] == "def": - def_ranges.append((site[1], site[2], repl)); del remaining[ad]; ovc = True - elif site and site[0] == "stub": - # A stub whose INCLUDE_ASM asm-subdir != this file's stem: the `sp` regex anchors - # on the stem and is structurally blind to it, and acting only on 'def' silently - # skipped the site. find_site's stub match is an EXACT stub_line(ov, addr) compare - # against THIS file's text, so placing here cannot cross files or TUs. - line_repls[site[1]] = repl; del remaining[ad]; ovc = True - elif site and site[0] == "macro": - del remaining[ad] # already instantiated (path-mismatch variant) - if line_repls or def_ranges: - for idx, repl in line_repls.items(): - lines[idx] = repl - for start, end, repl in sorted(def_ranges, key=lambda x: -x[0]): - lines[start:end + 1] = [repl] - edit(cp, "".join(lines)) - return ovc, sorted(remaining) - - -def _probe_overlay(ov, plan, header_rel, excl): - """Apply plan-minus-`excl` to ONE overlay, byte-gate it, restore. True iff byte-identical.""" - journal = {} - - def edit(path, newtext): - if path not in journal: - journal[path] = path.read_text() if path.exists() else None - path.write_text(newtext) - - placed, _rem = place_in_overlay( - ov, [p for p in plan if p["addr"] not in excl and ov in p["members"]], header_rel, edit) - try: - return byte_gate(ov)[0] if placed else True - finally: - for path, orig in journal.items(): - if orig is None: - path.unlink(missing_ok=True) - else: - path.write_text(orig) - - -def search_worker(job): - """The minimal exclusion set that makes ONE overlay byte-identical. Runs in its own PROCESS: - an overlay owns its .c files and its build// dir, and the shared header is written once by - the parent and never touched here — so these are genuinely independent.""" - ov, plan, header_rel, suspects = job - cands = [p for p in plan if ov in p["members"]] - sus_here = [p for p in cands if p["addr"] in suspects] - - def probe(excl): - return _probe_overlay(ov, plan, header_rel, excl) - - # FAST PATH: the same 1-2 functions break every overlay, so try the known culprits first. When it - # hits (the common case) this overlay costs 1 + |suspects| builds instead of a full bisection. - excl = None - if sus_here and probe({p["addr"] for p in sus_here}): - excl = sus_here - if excl is None: - order = sus_here + [p for p in cands if p["addr"] not in suspects] - if not probe({p["addr"] for p in order}): - return ov, None # diverges even with everything excluded - lo, hi = 0, len(order) - while lo < hi: # shortest sufficient PREFIX - mid = (lo + hi) // 2 - if probe({p["addr"] for p in order[:mid]}): - hi = mid - else: - lo = mid + 1 - excl = order[:lo] - # SHRINK: drop anything this overlay does not actually need, so the batch never over-excludes - # and silently costs an overlay a member it could have kept. - i = 0 - while i < len(excl): - trial = excl[:i] + excl[i + 1:] - if probe({p["addr"] for p in trial}): - excl = trial - else: - i += 1 - return ov, sorted(p["addr"] for p in excl) - - -# ---------------------------------------------------------------- main -def main(): - ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--addr", help="comma-separated target vaddrs (hex)") - ap.add_argument("--auto-from", help="propagate every matched+shared function of this overlay") - ap.add_argument("--source-overlay", help="overlay to extract the matched body from (default: auto)") - ap.add_argument("--min-reach", type=int, default=2, help="min #overlays sharing (auto mode; default 2)") - ap.add_argument("--limit", type=int, default=0, help="cap #functions (auto mode; 0 = no cap)") - ap.add_argument("--header", default="src/shared/engine_core.h") - ap.add_argument("--tier", default="h_exact", choices=("h_exact", "h_norm")) - ap.add_argument("--binaries", help="restrict members to these onboarded overlays (comma list)") - ap.add_argument("--check-only", action="store_true", help="dry run: print the plan, touch nothing") - ap.add_argument("--no-gate", action="store_true") - ap.add_argument("--recover", action="store_true", - help="on a straggler byte-gate failure, FIRST no-proto that overlay's conflicting " - "caller extern for the fn and re-gate (Part B reconcile — byte-neutral, same " - "lever as fix_arity_callers --any-proto); if still failing, EXCLUDE only that " - "overlay from the fn's members (Part A, ×N-1) rather than dropping the fn from " - "ALL overlays (the historical all-or-nothing). Recovers T6 caller-decl stragglers.") - a = ap.parse_args() - - onb = onboarded_overlays() - restrict = set(a.binaries.split(",")) if a.binaries else None - pool = [ov for ov in onb if (restrict is None or ov in restrict)] - sigs = {ov: load_sig(ov) for ov in pool} - - # ---- choose target addresses + source overlay per target - targets = [] # list of (addr, source_overlay) - if a.auto_from: - src = a.auto_from - ssig = load_sig(src) - ctext = source_text(src) # scan main + split files for matched defs - reg = registered_addrs() # skip functions already shared (additive + resumable) - # -O0 split-file functions are OVERLAY-LOCAL (§18/§20, cont.4): -O0 codegen embeds per-overlay - # %lo data addresses, so their bytes diverge per overlay even though the relocation-MASKED - # h_exact falsely reports reach-134. Cross-overlay-propagating one fails the byte-gate and, - # under the all-or-nothing batch revert, poisons every CLEAN match in the batch (cont.4: - # func_8013C360 reverted 10 good ×134 matches). Never auto-propagate them — detect by def-site - # in the -O0 split file (*_o0.c). (--addr still forces them, for an explicit override.) - o0_skip = set() - for _p, _tag in overlay_files(src): - if _p.name.endswith("_o0.c"): - _o0txt = _p.read_text() - for _ad in ssig: - _s = find_site(_o0txt, src, _ad) - if _s and _s[0] == "def": - o0_skip.add(_ad) - for addr in sorted(ssig): - if addr in reg or addr in o0_skip: - continue - site = find_site(ctext, src, addr) - if not site or site[0] != "def": # only functions matched (inline def) in the source - continue - h = ssig[addr].get(a.tier) - reach = sum(1 for ov in pool if sigs[ov].get(addr, {}).get(a.tier) == h) - if reach >= a.min_reach: - targets.append((addr, src)) - targets.sort(key=lambda t: ssig[t[0]]["nins"]) - if a.limit: - targets = targets[:a.limit] - elif a.addr: - for tok in a.addr.split(","): - addr = int(tok, 16) - src = a.source_overlay - if not src: # auto: an onboarded overlay where it is an inline def - for ov in pool: - site = find_site(source_text(ov), ov, addr) - if site and site[0] == "def": - src = ov - break - if not src: - sys.exit(f"[error] 0x{addr:08X}: no source overlay has it matched — give --source-overlay") - targets.append((addr, src)) - else: - sys.exit("give --addr or --auto-from") - - # ---- build the per-target plan (members + body + hash). Cache the source sig/.c (one src in - # auto-from). Filter out bodies that aren't self-contained (overlay-local types -> not liftable). - plan = [] - cc_queue = [] # candidates awaiting the (parallelised) self-containment probe - sig_cache, txt_cache = {}, {} - n_nondef = n_local = n_lowreach = 0 - # R32: a skip that only increments a counter is invisible work — and `n_local` aggregates THREE - # unrelated causes (macro-unsafe body / inline type def / genuinely overlay-local type), which - # mislabels the queue: a body skipped for containing a `//` comment is a 1-line fix, not the - # per-camp type reconcile. Track and NAME each class separately. - skipped = collections.defaultdict(list) - for addr, src in targets: - ssig = sig_cache.setdefault(src, load_sig(src)) - if addr not in ssig: - continue - h = ssig[addr].get(a.tier) - ctext = txt_cache.setdefault(src, source_text(src)) - site = find_site(ctext, src, addr) - if not site or site[0] != "def": - n_nondef += 1; skipped["not-inline-def"].append(addr); continue - body = site[3] - members = [ov for ov in pool if sigs[ov].get(addr, {}).get(a.tier) == h] - if len(members) < 2: - n_lowreach += 1; skipped["reach<2"].append(addr); continue - if any("//" in l or l.rstrip().endswith("\\") for l in body): - n_local += 1; skipped["macro-unsafe (// or \\)"].append(addr); continue - # A body that inline-DEFINES a named struct/union or a typedef can't be lifted as a macro - # (two macros defining the same type redefine it when both instantiate in one overlay). But - # USING a type that lives in the shared src/shared/engine_types.h header (§14c struct follow-up) - # is fine — the header is included via engine_core.h in every overlay. Anonymous local structs - # (`struct {...} v;`, no name, no typedef) are unique per instantiation -> also fine. So skip - # ONLY inline named-struct defs + typedefs; compiles_standalone (which includes engine_types.h) - # then rejects any body using an overlay-local type NOT yet promoted to the header. - if re.search(r'(\b(struct|union)\s+\w+\s*\{)|(\btypedef\b)', "\n".join(body)): - n_local += 1; skipped["inline type def in body"].append(addr); continue - cc_queue.append(dict(addr=addr, src=src, hash=h, body=body, members=members)) - - # The cpp+cc1 self-containment probe is one INDEPENDENT subprocess pair per candidate and was - # the whole plan phase's cost: ~5 minutes pegged at one core on a 32-core box while every other - # core idled. Run them concurrently (subprocess.run drops the GIL) and keep the plan in the - # original order so the printed plan and every downstream index are unchanged. - if cc_queue: - from concurrent.futures import ThreadPoolExecutor - with ThreadPoolExecutor(max_workers=_jobs()) as ex: - verdicts = list(ex.map(lambda c: compiles_standalone(c["body"]), cc_queue)) - for c, (ok, why) in zip(cc_queue, verdicts): - if not ok: - # SESSION-18: this bucket asserted "overlay-local TYPE" for EVERY failure, which is a - # mislabel — the dominant real cause is that the body references file-scope `extern` - # decls that live OUTSIDE the extracted block (func_80174CB0: 22 of them), exactly the - # gap Phase-27's family_remap._carry_macros closed for file-scope #defines. Name the - # causes apart so the queue can be sized honestly (R32/R33). - undecl = sorted(set(re.findall(r"`([A-Za-z_]\w*)' undeclared", why))) - if undecl: - n_local += 1 - skipped["missing file-scope extern (CARRY-FIXABLE): " + ",".join(undecl[:4])].append(c["addr"]) - else: - n_local += 1; skipped["overlay-local TYPE (the real cap)"].append(c["addr"]) - continue - plan.append(c) - - if n_local or n_nondef or n_lowreach: - print(f"[skip] {n_local} not self-contained (local types), {n_nondef} not inline-def, " - f"{n_lowreach} reach<{a.min_reach}") - for why, addrs in sorted(skipped.items()): - print(f" [{why}] {len(addrs)}: {' '.join('0x%08x' % x for x in sorted(addrs))}") - if not plan: - sys.exit("[error] nothing to propagate") - - print(f"== plan: {len(plan)} function(s), tier={a.tier}, header={a.header} ==") - for p in plan: - print(f" 0x{p['addr']:08X} body={len(p['body'])}L members={len(p['members'])} " - f"[{','.join(m.replace('ov_','') for m in p['members'][:6])}{'…' if len(p['members'])>6 else ''}]") - if a.check_only: - print("(--check-only: no files touched)") - return - - # ---- apply machinery (extracted so the drop-straggler retry can re-apply a sub-plan). - header_path = ROOT / a.header - DEFAULT_H = ("/* src/shared/engine_core.h — Phase 15 shared engine-core bodies (cross-overlay dedup, §14).\n" - " * Each DEFINE_func_XXXX() expands to the WHOLE matched body once; instantiated in place at the\n" - " * func_XXXX site in every overlay that shares it (address order preserved). Registry +\n" - " * byte-honesty: config/dedup.us.yaml + tools/dedup_integrate.py. Tool-generated; do not hand-edit. */\n" - "#ifndef SHARED_ENGINE_CORE_H\n#define SHARED_ENGINE_CORE_H\n#include \"common.h\"\n\n#endif\n") - - def apply_plan(subplan, restrict=None, gaps=None, journal=None, header=True): - """Author each fn's macro into engine_core.h + instantiate it at its site in every member - overlay (optionally restricted to a set — the per-fn straggler trial uses one overlay). Edit - each member overlay ONCE (group by overlay; stub lines replace 1:1, inline defs splice in - REVERSE order). Returns (touched, changed); caller byte-gates `changed` then restore(touched). - - `gaps` (optional dict) collects {overlay: [addrs]} for every claimed member site this could - NOT place. That set used to be dropped on the floor (R32 silent skip): the address simply - stayed in `remaining`, the overlay still landed in `changed` because some OTHER fn placed, - and the only symptom was struct_check's terse "not instantiated" — 92 minutes into a run, - naming no mechanism. The caller now fails on `gaps` FIRST, with a per-site diagnosis.""" - touched = {} if journal is None else journal - _tlock = _threading.Lock() - - def _edit(path, newtext): - with _tlock: # overlays never share a path; the lock guards the dict - if path not in touched: - touched[path] = path.read_text() if path.exists() else None - path.write_text(newtext) - if header: - htext = header_path.read_text() if header_path.exists() else DEFAULT_H - for p in subplan: - if f"DEFINE_{sym(p['addr'])}()" not in htext: - htext = htext.replace("\n#endif\n", - "\n" + make_macro(p["addr"], p["body"]) + "\n#endif\n") - _edit(header_path, htext) - by_ov = {} - for p in subplan: - for ov in p["members"]: - if restrict and ov not in restrict: - continue - by_ov.setdefault(ov, []).append(p) - changed = [] - - def _place(ov): - """Instantiate every planned macro in ONE overlay. Delegates to the module-level - place_in_overlay so the in-process apply and the process-pool searches can never drift.""" - ovc, rem = place_in_overlay(ov, by_ov[ov], a.header, _edit) - return ov, ovc, rem - - ovs = sorted(by_ov) - if len(ovs) > 4: - from concurrent.futures import ThreadPoolExecutor - with ThreadPoolExecutor(max_workers=_jobs()) as ex: - placed = list(ex.map(_place, ovs)) # map preserves order -> `changed` is stable - else: - placed = [_place(ov) for ov in ovs] - for ov, ovc, rem in placed: - if rem and gaps is not None: - gaps[ov] = rem # R32: assert coverage, never skip silently - if ovc: - changed.append(ov) - return touched, changed - - def restore(touched): - for path, orig in touched.items(): - if orig is None: - path.unlink(missing_ok=True) - else: - path.write_text(orig) - - def _dirty_set(): - r = subprocess.run(["git", "status", "--porcelain", "--", "src", "config"], - cwd=ROOT, capture_output=True, text=True) - return {l[3:].strip() for l in r.stdout.splitlines() if l.strip()} - - dirty0 = _dirty_set() # the tree's PRE-EXISTING modifications; residue is measured against this - - def _abort(msg, touched=None): - """Fail CLOSED, and PROVE it (R32/R35). Every abort path must leave the tree exactly as it - found it — but struct_check's exits restored only `touched`, so any Part-B reconcile kept on - disk (and anything else outstanding) survived the "REVERTED" message. That is the §156 class - on a different path, and it is the expensive one: a tree that is dirty in a way nobody knows - about makes EVERY later byte-gate report `near`, so its verdicts are void and get misread as - draft failures (S45p7 lost two whole batches to exactly this). - - So: undo this call's edits AND every kept reconcile, then diff the worktree against the - baseline and SAY which files (if any) survived. A silent leak becomes a loud one.""" - if touched: - restore(touched) - n = _undo_reconciles({ad for ad, _ in kept_reconciles}) - residue = sorted(_dirty_set() - dirty0) - if residue: - print(f"[BUG] abort did NOT fully restore — {len(residue)} file(s) still dirty. " - f"Run `git checkout -- src/ config/` before trusting ANY later byte-gate (R35):") - for f in residue[:20]: - print(f" {f}") - if len(residue) > 20: - print(f" … and {len(residue) - 20} more") - else: - print(f"[revert] tree restored to baseline" - + (f" (undid {n} kept reconcile(s))" if n else "") + "; no residue") - sys.exit(msg) - - def struct_check(subplan, changed, touched): - # the byte-gate CANNOT catch a leftover stub (it is itself byte-identical): every claimed - # member must now instantiate the macro and have no stub. One read per changed overlay. - incasm = lambda ad: re.compile(rf'INCLUDE_ASM\("[^"]+",\s*{sym(ad)}\)') - for ov in changed: - t = source_text(ov) # all split files (post-edit, from disk) - for p in (pp for pp in subplan if ov in pp["members"]): - if f"DEFINE_{sym(p['addr'])}()" not in t: - _abort(f"[FAIL] {ov}: 0x{p['addr']:08X} not instantiated — REVERTED", touched) - if incasm(p["addr"]).search(t): - _abort(f"[FAIL] {ov}: 0x{p['addr']:08X} stub still present — REVERTED", touched) - - # ---- apply with DROP-STRAGGLER retry. A fn whose shared C body, in some OTHER overlay's TU, - # byte-mismatches (-O0 per-overlay %lo data) or compile-errors (cross-overlay loose-typed callee - # decls — cont.4 wave-2) would, under an all-or-nothing batch revert, poison every CLEAN match. - # So on a byte-gate failure: isolate the culprit(s) for the failing overlay (per-fn trial), drop - # them (they stay matched ×1 in the source), and retry the batch with the survivors. The byte-gate - # stays the sole arbiter (G3/P9) — a dropped fn is never banked anywhere it isn't byte-identical. - # P30 S45p7 — RECONCILE LEDGER (the 141/213 breakage, root-caused 2026-08-07). - # Part B below deliberately LEAVES its caller-extern reconcile on disk when it buys the match - # ("keep the reconcile on disk"). That is correct only while the fn ultimately survives. A fn can - # still be dropped by a LATER iteration (a different fail_ov), and when `plan` finally empties the - # `sys.exit` at the bottom used to leave every kept reconcile orphaned — a no-proto'd caller extern - # for a function that was never propagated. Measured cost: dedup_propagate exited 1 leaving - # ov_SC07_* rewritten, and 141 of 213 binaries failed check-all (the wave-2 propagation, this - # session). The byte-gate never mis-banked — it fails closed — but every SUBSEQUENT gate then - # reports `near` against the broken tree, so its verdicts are void (R35), which is how two whole - # batches (4/4 and 20/20) were mis-read as draft failures. - # Fix: ledger every kept reconcile against its fn, and undo it the moment that fn leaves `plan`. - kept_reconciles = [] # [(addr, snapshot)] in apply order - suspects = [] # culprit addrs found so far, tried first - - def _undo_reconciles(addrs): - """Restore reconciles for addrs that did not survive. Reverse order: each snapshot is the - file text captured BEFORE its own edit, so replaying newest->oldest ends on the original.""" - drop = [r for r in kept_reconciles if r[0] in addrs] - for _a, _s in reversed(drop): - restore_snapshot(_s) - if drop: - kept_reconciles[:] = [r for r in kept_reconciles if r[0] not in addrs] - return len(drop) - - while plan: - gaps = {} - touched, changed = apply_plan(plan, gaps=gaps) - if gaps: - # A claimed member whose site this pass could not place. Diagnose it HERE, where the - # per-site evidence still exists, instead of letting struct_check report it as a bare - # "not instantiated" with no mechanism (S45p9: that message cost a whole re-run to - # even locate). Print, per site, what the whole-overlay oracle says the site IS. - print(f"[GAP] {sum(len(v) for v in gaps.values())} instantiation(s) unplaced in " - f"{len(gaps)} overlay(s):") - for ov in sorted(gaps): - t = source_text(ov) - for ad in gaps[ov]: - v = find_site(t, ov, ad) - print(f" {ov} 0x{ad:08X}: whole-overlay find_site=" - f"{v[0] if v else None}, in-sig={ad in load_sig(ov)}, " - f"files={[p.name for p, _ in overlay_files(ov)]}") - _abort("[FAIL] unplaced instantiation(s) — REVERTED (see [GAP] above)", touched) - struct_check(plan, changed, touched) - if a.no_gate: - break - fails = gate_failures(changed, f" ({len(plan)} fn(s) in plan)") - if not fails: - print(f"[ OK ] {len(changed)} overlays byte-identical after propagation") - break - restore(touched) - print(f"[gate] {len(fails)} overlay(s) diverge: " - + ", ".join(fails[:8]) + ("…" if len(fails) > 8 else "")) - - # ---- BATCHED culprit resolution. - # The old loop took the FIRST failure, probed each of the ~30 plan fns against it one build - # at a time, then paid a whole fresh 141-overlay sweep to rediscover the NEXT failure — so - # S45's two divergent functions cost 27 sweeps ×(30 serial builds + a full sweep). But the - # sweep already tells us EVERY failing overlay (gate_failures), and the culprits are almost - # always the SAME functions in all of them. So: find a sufficient exclusion set once, on one - # pivot overlay (binary search, ~log2 builds), then decide per-overlay necessity for the - # whole failing set with ONE PARALLEL SWEEP PER CANDIDATE — 32 builds at a time instead of 1. - def diverging(excl_addrs, ovs, label=""): - """Apply the plan MINUS `excl_addrs` to `ovs`; return the subset still byte-diverging. - This is the overlays' FINAL intended state (every other plan fn applied), so a pass here - is direct evidence for the bank — stronger than the old one-function-in-isolation probe.""" - sub = [p for p in plan if p["addr"] not in excl_addrs] - t, c = apply_plan(sub, restrict=set(ovs)) - inplay = [o for o in c if o in set(ovs)] - bad = set(gate_failures(inplay, label)) - restore(t) - return bad # an overlay nothing was applied to never changed -> counts as passing - - # PER-OVERLAY INDEPENDENT SEARCH. Each overlay's answer depends only on its own .c files and - # its own build// dir — the shared header carries EVERY plan macro regardless of which - # sites get instantiated, so writing it once up front makes the searches disjoint. That means - # 138 searches can run at once instead of |E| fleet-wide sweeps in lock-step, and it cuts - # BUILDS (not just overlap): a sweep pass rebuilds all 138 to answer one question, while a - # search asks each overlay only the questions that overlay's own answer needs. - hdr_journal = {} - # Header-only: author every plan macro once, edit NO overlay. `restrict` is a whitelist, so a - # sentinel alias that matches nothing gives exactly that while reusing apply_plan's macro - # authoring (an empty set would be FALSY and disable the filter entirely — editing all 141). - apply_plan(plan, restrict={"\0no-overlay"}, journal=hdr_journal) - try: - # PROCESSES, not threads. The first cut used a ThreadPoolExecutor and measured 0-4 builds - # alive across 138 "parallel" searches: the work is regex over 15k-line files, so every - # thread queued on the GIL and the box sat at load 3 with 32 cores. Each search only - # touches its own overlay's .c and its own build// dir, and the shared header is - # written once above and never touched by a worker — so separate processes are safe. - order_fails = sorted(fails, key=lambda o: -_src_bytes(o)) # longest-first - # Seed the suspect list with ONE overlay first: the same 1-2 fns break every overlay, and - # a pool submitted all at once would give every worker an EMPTY suspect list and make all - # 138 of them pay a full bisection instead of the 1+|suspects| fast path. - seed_ov, seed_excl = search_worker((order_fails[0], plan, a.header, list(suspects))) - if seed_excl: - for ad in seed_excl: - if ad not in suspects: - suspects.append(ad) - rest = order_fails[1:] - print(f"[search] {len(fails)} independent per-overlay searches " - f"({_jobs()} processes; seeded with {len(suspects)} suspect(s) from {seed_ov})", - flush=True) - found = [(seed_ov, seed_excl)] - if rest: - from concurrent.futures import ProcessPoolExecutor - jobs = [(ov, plan, a.header, list(suspects)) for ov in rest] - with ProcessPoolExecutor(max_workers=_jobs()) as ex: - found += list(ex.map(search_worker, jobs)) - finally: - restore(hdr_journal) - broken = [ov for ov, e in found if e is None] - if broken: - # Diverges with NOTHING of ours applied => already broken before this run, so every - # verdict in this sweep is void (R35). Stop rather than "recover" from a bad baseline. - _abort(f"[FAIL] {', '.join(broken[:5])} diverge with the whole plan excluded — ALREADY " - f"broken at baseline; this run's gate verdicts are not evidence (R35).") - need = {ov: set(e) for ov, e in found} - byfn0 = collections.Counter(ad for e in need.values() for ad in e) - print("[culprit] " + ", ".join(f"0x{ad:08X}x{n}" for ad, n in byfn0.most_common(8))) - - recovered, excluded = [], [] - # Part B (batched) — try the caller-extern reconcile for every (overlay, fn) pair at once, - # then ONE sweep with the full plan: whoever passes keeps its reconcile and its membership. - if a.recover and any(need.values()): - snaps = collections.defaultdict(list) # ov -> [(addr, snapshot)] in apply order - for ov in sorted(need): - for ad in sorted(need[ov]): - snap, n = reconcile_caller_extern(ov, ad) - if n: - snaps[ov].append((ad, snap)) - if snaps: - bad = diverging(set(), sorted(snaps), " (post-reconcile re-gate)") - for ov, lst in snaps.items(): - if ov in bad: - for _ad, snap in reversed(lst): # newest->oldest ends on the original text - restore_snapshot(snap) - else: - for ad, snap in lst: - need[ov].discard(ad) - kept_reconciles.append((ad, snap)) # LEDGERED — undone if the fn drops - recovered.append((ov, ad)) - # Part A — exclude only the overlays that still need it (the fn stays for everyone else). - byaddr = {p["addr"]: p for p in plan} - for ov in sorted(need): - for ad in sorted(need[ov]): - p = byaddr.get(ad) - if not p: - continue - if not a.recover: # historical all-or-nothing: drop the fn outright - p["members"] = [] - else: - p["members"] = [m for m in p["members"] if m != ov] - excluded.append((ov, ad)) - survivors = [p for p in plan if len(p["members"]) >= 2] - dropped = [p for p in plan if len(p["members"]) < 2] - - if recovered: - print(f"[recover] reconciled the conflicting caller extern for {len(recovered)} " - f"(overlay, fn) pair(s) across {len({o for o, _ in recovered})} overlay(s)") - if excluded: - byfn = collections.Counter(ad for _ov, ad in excluded) - print(f"[exclude] {len(excluded)} (overlay, fn) pair(s): " - + ", ".join(f"0x{ad:08X}×{n}" for ad, n in byfn.most_common(6))) - if dropped: - print(f"[drop] {len(dropped)} fn(s) below reach 2 after exclusion: " - + ", ".join(f"0x{p['addr']:08X}" for p in dropped)) - if not (recovered or excluded or dropped): - # Every search came back with an EMPTY exclusion set even though the sweep said these - # overlays diverge -> a multi-fn interaction no per-fn exclusion explains. Conservatively - # drop every fn targeting them (rare; each stays matched ×1) so the rest can proceed, and - # so the loop cannot spin forever making no progress. - stuck = set(fails) - tofail = [p for p in plan if stuck & set(p["members"])] - print(f"[drop] {len(stuck)} overlay(s) fail with no single-fn culprit (interaction) — " - f"dropping their {len(tofail)} fn(s), kept ×1") - survivors = [p for p in plan if not (stuck & set(p["members"]))] - # Any fn that just left `plan` must give back its kept reconcile — otherwise a no-proto'd - # caller extern survives for a function that was never propagated (see the ledger note above). - _gone = {p["addr"] for p in plan} - {p["addr"] for p in survivors} - _n = _undo_reconciles(_gone) - if _n: - print(f"[restore] undid {_n} kept caller-extern reconcile(s) for dropped fn(s)") - plan = survivors - if not plan: - # Nothing survived ⇒ NOTHING may remain edited. _abort undoes every outstanding reconcile - # and PROVES the tree is back at baseline, so a failed propagation cannot leave a poisoned - # tree behind for the next gate to misread (R35). - _abort("[error] all candidates dropped — no cleanly-shareable function") - - # ---- register groups (compact shorthand: position-locked -> vram + binaries list) - groups = [dict(id=f"E_{sym(p['addr'])}", tier=a.tier, hash=p["hash"], - source=a.header, addr=p["addr"], - members=[dict(binary=ov, vram=p["addr"], name=sym(p["addr"])) for ov in p["members"]]) - for p in plan] - n = append_groups(groups) - print(f"== propagated {len(plan)} function(s); {len(changed)} overlays rebuilt byte-identical; " - f"registered {n} new group(s) in config/dedup.us.yaml ==") - - -if __name__ == "__main__": - # Stage 1 (docs/concurrency-design.md): propagation is THE fleet-shared writer — - # it rewrites src/shared/engine_core.h, config/dedup.us.yaml and every member - # overlay .c. It may never overlap a gate reading that state (verdict decay, - # design §2.2b). No-op when gate_stage already holds the lock (BFM_SHARED_LOCK_HELD). - with shared_lock.hold(exclusive=True, announce="dedup_propagate"): - main() diff --git a/tools/demacroize.py b/tools/demacroize.py index 6f82a681a7..f95b8a3a97 100644 --- a/tools/demacroize.py +++ b/tools/demacroize.py @@ -122,6 +122,7 @@ def plan(binary, fn, draft_text): def main(): + import frozen; frozen.refuse("demacroize.py", "tools/share_body.py", "it turns a macro site back into a private copy — the opposite of the S1 invariant") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--binary', required=True) diff --git a/tools/family_remap.py b/tools/family_remap.py index 7289f36050..9e38fd8cce 100644 --- a/tools/family_remap.py +++ b/tools/family_remap.py @@ -1322,6 +1322,7 @@ def remap(addr, from_ov, to_ov, to_addr=None, imm_map=None): def main(): + import frozen; frozen.refuse("family_remap.py", "tools/share_body.py --reexemplar / --apply", "its command line writes a DEFINE_func_ macro head; the library (stream_words, nins_of, …) stays") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--addr", required=True) ap.add_argument("--from", dest="frm", required=True) diff --git a/tools/family_sweep.py b/tools/family_sweep.py index d62eb55591..8442af77a2 100644 --- a/tools/family_sweep.py +++ b/tools/family_sweep.py @@ -876,6 +876,7 @@ def hseq_sweep(a): def main(): + import frozen; frozen.refuse("family_sweep.py", "tools/share_body.py", "it remaps and rewrites the DEFINE_func_ macro bodies Phase 35 retired") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--source", default="ov_SC01_077") ap.add_argument("--limit", type=int, default=0, help="cap #exemplars (0 = all)") diff --git a/tools/frozen.py b/tools/frozen.py new file mode 100644 index 0000000000..3fcfe06501 --- /dev/null +++ b/tools/frozen.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""frozen.py — the ONE refusal every FROZEN tool prints from its main() (Phase 35 T6, 2026-09-08). + +A frozen tool is a matching-era tool that reads or writes the `DEFINE_func_` macro form Phase 35 retired (the shared bodies are +plain-C headers under src/shared/ now, one source per unique function — the S1 invariant). Its code stays in tools/ for the record +and for the libraries other tools import from it; only its command line refuses, so an import never exits (cdecl imports +gen_harvest_targets; eighteen tools import family_remap). The dictionary row carries status FROZEN and the successor. + + import frozen; frozen.refuse("family_sweep.py", "tools/share_body.py", "it rewrites the macro form Phase 35 retired") +""" +import sys + + +def refuse(tool, successor, why): + sys.exit(f"{tool}: FROZEN since Phase 35 (2026-09-08) — {why}. Successor: {successor}. The code is kept for the record and its " + f"library functions still import; the command line does nothing (R43). See docs/SETUP.md and config/tool_dictionary.tsv.") diff --git a/tools/gate_lane.py b/tools/gate_lane.py index 2cd4fadb40..19a3587929 100644 --- a/tools/gate_lane.py +++ b/tools/gate_lane.py @@ -83,7 +83,7 @@ json.dump(banked,open(outp.replace('.json','_banked.json'),'w')) # propagate per function for fn in banked: addr="0x"+fn.split("_")[1].lower() - r=subprocess.run([".venv/bin/python","tools/dedup_propagate.py","--addr",addr], + r=subprocess.run([".venv/bin/python","tools/share_body.py","--apply","--bucket","new","--batches","1"], # P35 T6 capture_output=True,text=True) if dirty(): tag=os.environ.get("GATE_PHASE","decomp") # P31 T1: was hardcoded "phase-30 S49" diff --git a/tools/gate_stage.py b/tools/gate_stage.py index e8f181c24d..21c8a7d67a 100644 --- a/tools/gate_stage.py +++ b/tools/gate_stage.py @@ -519,10 +519,9 @@ def _run_gate_locked(drafts, binary, src, asm, out, good_sha, propagate, source_ # cover write scope, and a killed process performs no undo at all). _budget = min(6 * 3600, 1800 + 1800 * len(verified)) try: - pr = sh([PY, "tools/dedup_propagate.py", "--auto-from", binary, "--min-reach", "2", - "--recover"], timeout=_budget) + pr = sh([PY, "tools/share_body.py", "--apply", "--bucket", "new", "--batches", "1"], timeout=_budget) # P35 T6: the include-at-site share (dedup_propagate retired) except subprocess.TimeoutExpired: - print(f"[gate] FATAL: dedup_propagate exceeded {_budget}s with {len(verified)} banks — " + print(f"[gate] FATAL: share_body exceeded {_budget}s with {len(verified)} banks — " f"the fleet is HALF-PROPAGATED and the tree is DIRTY. Revert (`git checkout -- src/`), " f"then re-gate with --no-propagate and propagate separately.", file=sys.stderr) return {"drafts": len(draft_fns), "banked": len(verified), "propagated": 0, diff --git a/tools/gen_harvest_targets.py b/tools/gen_harvest_targets.py index 550e261e92..b67e246b04 100644 --- a/tools/gen_harvest_targets.py +++ b/tools/gen_harvest_targets.py @@ -152,6 +152,7 @@ def collect_stubs(c_path): def main(): + import frozen; frozen.refuse("gen_harvest_targets.py", "tools/share_census.py", "it collects harvest targets from the macro header engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser() ap.add_argument('--source', default='ov_SC01_077') ap.add_argument('--min-reach', type=int, default=2) diff --git a/tools/grinder.py b/tools/grinder.py index 04da6a55d1..6a1663d6df 100644 --- a/tools/grinder.py +++ b/tools/grinder.py @@ -348,8 +348,8 @@ def main(): f"total {banked}; fleet {fp}%") if s.get("verified"): log(" -> propagate as its own batch: " - + "; ".join("tools/dedup_propagate.py --addr 0x%s --recover" % f.split('_')[-1] - for f in s["verified"][:4])) + + "tools/share_body.py --apply --bucket new --batches 1 (P35 T6: shares every new same-address class; " + + "verified: " + " ".join(s["verified"][:4]) + ")") # A permuter win the whole-binary gate STILL rejects is plumbing-bound (not regalloc/sched) — # re-permuting can never bank it. Blacklist so the grinder stops churning it (the §20 trap). rejected = [f for f, _b in won if f not in verified] diff --git a/tools/inject_capped_externs.py b/tools/inject_capped_externs.py index 7714694d81..c28ea8ebea 100644 --- a/tools/inject_capped_externs.py +++ b/tools/inject_capped_externs.py @@ -30,7 +30,7 @@ import argparse, re, sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -import dedup_propagate as dp +dp = None # the propagate library is retired to tools/sunset (P35 T6); this tool is FROZEN — main() refuses SYM_RE = re.compile(r'\b(func_[0-9A-Fa-f]{6,8}|D_[0-9A-Fa-f]{6,8})\b') @@ -81,6 +81,7 @@ def inject(defblock, name, ext_map): def main(): + import frozen; frozen.refuse("inject_capped_externs.py", "tools/share_body.py", "it fixed drafts a macro-era propagation skipped, on dedup_propagate (retired)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--overlay", default="ov_SC01_077") diff --git a/tools/lora_grind.py b/tools/lora_grind.py index 9a6aa5661c..f27268fef5 100644 --- a/tools/lora_grind.py +++ b/tools/lora_grind.py @@ -227,8 +227,8 @@ def main(): # periodic fleet propagate sweep (the multiplier) — every N batches if batch_i % a.propagate_every == 0 and propagated_since: for pb in sorted(propagated_since): - subprocess.run([".venv/bin/python", "tools/dedup_propagate.py", "--auto-from", pb, - "--min-reach", "2"], cwd=REPO, capture_output=True, timeout=3600) + subprocess.run([".venv/bin/python", "tools/share_body.py", "--apply", "--bucket", "new", + "--batches", "1"], cwd=REPO, capture_output=True, timeout=3600) # P35 T6 propagated_since.clear() log("propagate sweep done") # heartbeat + flywheel stats. progress.py --fleet is ~14s (a full 136-binary scan), so diff --git a/tools/macro_draft.py b/tools/macro_draft.py deleted file mode 100644 index 6dfa3c3f1a..0000000000 --- a/tools/macro_draft.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -"""macro_draft.py — materialize a `DEFINE_func_XXXX()` macro body as a compilable draft .c - -WHY THIS EXISTS (Phase 30 S47) -============================== -`conform_decls.py` needs a **byte-true DEFINITION** to conform the fleet's declarations to, and it -refuses (correctly) when handed a file that holds only a declaration. But for a DEDUPED function the -definition does not live in any .c file at all — it lives inside a `#define DEFINE_func_X() \\ ...` -macro in `src/shared/engine_core.h`, where two things defeat every definition parser: - - * every line ends in a backslash continuation, and - * the definition line is INDENTED, so `^([A-Za-z_]...)` never matches. - -So the single largest class of `conflicting types for func_X` — a deduped callee whose macro-local -`extern` disagrees with the fleet's — was unreachable by the tool built to fix exactly that class. -This bridges the two: it emits the macro body verbatim (dedented, continuations stripped) so the -existing conformer can read the signature it already knows how to read. - -VERBATIM IS THE POINT. The draft is evidence, not a rewrite: nothing is reformatted, no types are -"cleaned up". If this file ever starts editing the body it stops being byte-truth (R33/R35). - -Usage: - tools/macro_draft.py --fn func_80128ED8 [--out .run/draft.c] # default: .run/macro_draft_.c -""" -import argparse, os, re, sys, textwrap - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -HDR = os.path.join(REPO, "src", "shared", "engine_core.h") - - -def extract(fn, header=HDR): - """(body_text, first_line, last_line) of DEFINE_()'s macro body, 1-indexed; None if absent.""" - lines = open(header, errors="replace").read().split("\n") - for i, l in enumerate(lines): - if re.match(rf"\s*#define\s+DEFINE_{re.escape(fn)}\(\)", l): - j = i - while j < len(lines) and lines[j].rstrip().endswith("\\"): - j += 1 - raw = lines[i:j + 1] - # strip the trailing backslash continuation from every line, drop the #define line - body = [x.rstrip()[:-1].rstrip() if x.rstrip().endswith("\\") else x for x in raw][1:] - return textwrap.dedent("\n".join(body)), i + 1, j + 1 - return None - - -def main(): - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--fn", required=True) - ap.add_argument("--out") - ap.add_argument("--header", default=HDR) - a = ap.parse_args() - - got = extract(a.fn, a.header) - if not got: - print(f"macro_draft: no DEFINE_{a.fn}() in {a.header}", file=sys.stderr) - return 2 - body, lo, hi = got - - # R32: prove the emitted text actually contains a DEFINITION of the requested function, at - # column 0, where a definition parser can see it. A dedent that leaves it indented is a silent - # no-op that would surface later as "no DEFINITION" from the conformer. - if not re.search(rf"^[A-Za-z_][\w \t\*]*\b{re.escape(a.fn)}\s*\(", body, re.M): - print(f"macro_draft: DEFINE_{a.fn}() body carries no column-0 definition of {a.fn} " - f"(header lines {lo}-{hi}) — refusing to emit a draft that cannot be parsed", - file=sys.stderr) - return 2 - - out = a.out or os.path.join(REPO, ".run", f"macro_draft_{a.fn}.c") - os.makedirs(os.path.dirname(out), exist_ok=True) - with open(out, "w") as f: - f.write(f'/* verbatim body of DEFINE_{a.fn}() — {a.header} lines {lo}-{hi} */\n') - f.write('#include "engine_types.h"\n') - f.write(body.rstrip() + "\n") - print(out) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/normalize_self_decls.py b/tools/normalize_self_decls.py index bb339fecad..c4891ffd11 100644 --- a/tools/normalize_self_decls.py +++ b/tools/normalize_self_decls.py @@ -191,6 +191,7 @@ def fix(tu_text, fn, ref_decl=None): def main(): + import frozen; frozen.refuse("normalize_self_decls.py", "tools/share_body.py", "it derives canonical declarations from engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--tu', required=True, help='the TU file to normalize (edited in place unless --out)') ap.add_argument('--fn', required=True, help='func_XXXXXXXX to normalize decls of') diff --git a/tools/o0_subsplit.py b/tools/o0_subsplit.py index 5b68e152c9..5adbe4a326 100644 --- a/tools/o0_subsplit.py +++ b/tools/o0_subsplit.py @@ -62,6 +62,7 @@ def free_letters(ov, n): def main(): + import frozen; frozen.refuse("o0_subsplit.py", "the Makefile -O0 globs (rollout_o0 for the record)", "it splits -O0 macro sites out of a TU against the macro form") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("ov") diff --git a/tools/p16_improve.py b/tools/p16_improve.py index 1962f6470f..8aad96e853 100644 --- a/tools/p16_improve.py +++ b/tools/p16_improve.py @@ -66,6 +66,7 @@ def classify_fail(fn, cpath): def main(): + import frozen; frozen.refuse("p16_improve.py", "tools/share_body.py", "it rewrites macro sites in ov_SC01_077 against engine_core.h (gone)") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser() ap.add_argument("--band", choices=list(BANDS), default="easy") ap.add_argument("--n", type=int, default=40) diff --git a/tools/recover_giant.py b/tools/recover_giant.py index 7482bddf80..5ed4799017 100644 --- a/tools/recover_giant.py +++ b/tools/recover_giant.py @@ -69,6 +69,7 @@ def recover(fn, src, ec): def main(): + import frozen; frozen.refuse("recover_giant.py", "tools/share_body.py", "it recovers a giant into the macro header engine_core.h (gone)") # Phase 35 T6: FROZEN if len(sys.argv) != 4: raise SystemExit(__doc__) fn, inp, outp = sys.argv[1], sys.argv[2], sys.argv[3] diff --git a/tools/restore_dropped_decls.py b/tools/restore_dropped_decls.py index 316e43ef31..87c3d30508 100644 --- a/tools/restore_dropped_decls.py +++ b/tools/restore_dropped_decls.py @@ -71,6 +71,7 @@ def insert_at_file_scope(path, decls): def main(): + import frozen; frozen.refuse("restore_dropped_decls.py", "tools/share_body.py", "it repairs declarations a macro-era propagation deleted; share_body edits definition lines only") # Phase 35 T6: FROZEN ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--binary", required=True) ap.add_argument("--ref", required=True, help="git ref holding the pre-deletion sources") diff --git a/tools/share_body.py b/tools/share_body.py index e4e8291e82..6ee2803e0d 100644 --- a/tools/share_body.py +++ b/tools/share_body.py @@ -11,7 +11,7 @@ WHAT ONE SHARE IS. A class = one h_exact (the raw instruction bytes) at one vram are still PRIVATE COPIES (a definition in that binary's own TU). Sharing it: the class's ONE body goes to (or already is) a plain-C header under src/shared// (macro_to_header's naming: func_[__h8].h, keyed by the class — R48), every private copy becomes `#include "../shared//
"` at the SAME position (the copy's definition lines, nothing else — the TU keeps its own -declarations), and the registry gains the group (shorthand form, appended by text) or the members (dedup_extend.add_members_surgical). +declarations), and the registry gains the group (shorthand form, appended by text) or the members (add_members_surgical, inherited from the retired dedup_extend). THE EXEMPLAR (for an unregistered class) — printed with every share: the copy whose normalized text the MOST copies share; a tie goes to a pin-free copy; a further tie to the shortest. A body defined under an asm-label alias (aF) gets its own binding @@ -47,12 +47,102 @@ REPO = pathlib.Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO / "tools")) import share_census as sc # noqa: E402 import macro_to_header as m2h # noqa: E402 — Oracles (naming, spaces, sigs), bind_alias_header, include_line, banner -import dedup_extend as de # noqa: E402 — add_members_surgical (the registry's surgical text edit) LEDGER = REPO / "config/dedup_exceptions.tsv" RUN = REPO / ".run/P35/share" +# ---------------------------------------------------------------------------------------------------------------------------- +# The library surface inherited from the retired propagate/extend tools in tools/sunset (Phase 35 T6; verbatim, so their importers +# change one line: `import share_body as dp`). load_sig/sig_path read the per-binary sig files; registered_addrs the registry. +def sig_path(ov): + return REPO / f".run/sig.{ov}.jsonl" + + +def onboarded_overlays(): + """Overlay aliases + (S44) module aliases — every non-main/resident binary a shared body can + propagate into. Modules carry engine functions too; excluding them re-creates the SC07 + invisible-work bug one class over.""" + out = [] + for f, var in (("config/overlays.mk", "OVERLAY_BINARIES"), ("config/modules.mk", "MODULE_BINARIES")): + fp = REPO / f + if fp.exists(): + m = re.search(rf"^{var}\s*:=\s*(.*)$", fp.read_text(), re.M) + if m: + out += m.group(1).split() + return out +def load_sig(ov): + """addr(int) -> {h_exact, h_norm, nins, ...}; {} if absent.""" + p = sig_path(ov) + if not p.exists(): + return {} + out = {} + for ln in p.read_text().splitlines(): + ln = ln.strip() + if ln: + r = json.loads(ln) + out[int(r["addr"], 16)] = r + return out +def registered_addrs(): + """Vaddrs already in config/dedup.us.yaml (shared via ANY mechanism — engine_core, ov_setters, + clearTbl40). --auto-from skips these so the bulk is purely additive and never collides with an + existing share (e.g. a SETTER-matched function) — which would trip the structural check.""" + p = REPO / "config/dedup.us.yaml" + if not p.exists(): + return set() + try: + import yaml + sys.path.insert(0, str(REPO / "tools")) + from dedup_integrate import group_members + data = yaml.safe_load(p.read_text()) or {} + return {v for g in (data.get("groups") or []) for (_b, v, _n) in group_members(g)} + except Exception: + return set() +def sym(addr): + return f"func_{addr:08X}" # splat convention: uppercase 8-hex + + +def add_members_surgical(additions): + """Append binaries to each group's `binaries: [...]` list by TEXT EDIT, in place. + + NEVER `yaml.safe_dump` this file. The first cut of this tool round-tripped it through + safe_dump and silently destroyed BOTH of the things a human needs from it (H5 — "never + silently drop comments on a rewrite"): + * all 47 comment lines — including the curated Phase-11 header explaining WHY the share is + source-level (the linker cannot excise bytes interior to an object) — dumped to nothing; + * every `vram: 0x80162FF4` re-serialized as `vram: 2148937716` (PyYAML parses YAML-1.1 hex + to int, and dumps int as decimal), making 1,832 entries unreadable. + It was invisible to every gate: dedup-check passed 1840/0 and check-all stayed 140/140, + because `_addr()` accepts both forms — the data was fine and the DOCUMENT was ruined. A + formatting-destructive write that all your oracles call green is exactly the class this + project keeps re-learning: the gate measures bytes, not intent. + + `additions` = {group_id: [binary, ...]}. Idempotent: a binary already listed is skipped. + """ + p = str(REPO / "config/dedup.us.yaml") + lines = open(p).read().splitlines(keepends=True) + cur, n = None, 0 + seen = set() + for i, ln in enumerate(lines): + m = re.match(r"^\s*-?\s*id:\s*(\S+)\s*$", ln) + if m: + cur = m.group(1) + continue + if cur and cur in additions and re.match(r"^\s*members:\s*$", ln): + # a VERBOSE group has no `binaries:` line — silently skipping it left five 141-member groups listed at 16 (P35 S94/S96) + raise SystemExit(f"add_members_surgical: {cur} is in the verbose `members:` form — convert it to shorthand first " + f"(share_body.py --repair-registry does), never skip it (R43)") + if cur and cur in additions and re.match(r"^\s*binaries:\s*\[", ln): + seen.add(cur) + add = [b for b in additions[cur] if re.search(rf"\b{re.escape(b)}\b", ln) is None] + if add: + lines[i] = ln.rstrip("\n").rstrip()[:-1].rstrip() + ", " + ", ".join(add) + "]\n" + n += len(add) + cur = None + open(p, "w").write("".join(lines)) + return n + + def log(msg): print(msg, flush=True) @@ -397,7 +487,7 @@ def run_batch(orc, cen, classes, label): bins = [x for x in b.extend.get(gid, []) if x not in rejected_bins.get(c["h"], set())] if bins: ext[gid] = bins - n_ext = de.add_members_surgical(ext) if ext else 0 + n_ext = add_members_surgical(ext) if ext else 0 ok_n = sum(1 for a in gated if results[a][0]) log(f" [{label}] gated {ok_n}/{len(gated)} binaries green · registered {len(reg_entries)} groups · extended {n_ext} members · " f"rejected classes {len(failed_classes)}") diff --git a/tools/sunset/README.md b/tools/sunset/README.md index ee8fa18c56..56458778dd 100644 --- a/tools/sunset/README.md +++ b/tools/sunset/README.md @@ -41,3 +41,7 @@ successor named), or a one-off (retired, its product named). The dictionary rows | `verify_map_findings.py` (was `tools/verify_map_findings.py`) | ONE-OFF | the machine-checked verdicts of one codegen-map audit (docs/gcc-2.7.2-map/regalloc.md) | ☐ | | `wall_taxonomy.py` (was `tools/wall_taxonomy.py`) | SUPERSEDED | tools/frontier_classify.py | ☐ | | `warmstart.py` (was `tools/warmstart.py`) | SUPERSEDED | tools/permuter_sweep.py | ☐ | +| `dedup_propagate.py` (was `tools/dedup_propagate.py`) | SUPERSEDED | tools/share_body.py (P35 T6: the include-at-site share; --reexemplar, --repair-registry) | ☐ | +| `dedup_extend.py` (was `tools/dedup_extend.py`) | SUPERSEDED | tools/share_body.py (add_members_surgical moved verbatim; the extend bucket) | ☐ | +| `macro_draft.py` (was `tools/macro_draft.py`) | ONE-OFF | the per-function headers under src/shared/ (P35 T4) — no macro body to materialize | ☐ | +| `test_reconcile_ledger.py` (was `tools/test_reconcile_ledger.py`) | ONE-OFF | the S45p7 reconcile-ledger proof, retired with dedup_propagate | ☐ | diff --git a/decomp-architect/corpus/tools/P6/dedup_extend.py b/tools/sunset/dedup_extend.py similarity index 100% rename from decomp-architect/corpus/tools/P6/dedup_extend.py rename to tools/sunset/dedup_extend.py diff --git a/decomp-architect/corpus/tools/P6/dedup_propagate.py b/tools/sunset/dedup_propagate.py similarity index 100% rename from decomp-architect/corpus/tools/P6/dedup_propagate.py rename to tools/sunset/dedup_propagate.py diff --git a/decomp-architect/corpus/tools/P6/macro_draft.py b/tools/sunset/macro_draft.py similarity index 100% rename from decomp-architect/corpus/tools/P6/macro_draft.py rename to tools/sunset/macro_draft.py diff --git a/decomp-architect/corpus/tools/P5/test_reconcile_ledger.py b/tools/sunset/test_reconcile_ledger.py similarity index 100% rename from decomp-architect/corpus/tools/P5/test_reconcile_ledger.py rename to tools/sunset/test_reconcile_ledger.py diff --git a/tools/test_reconcile_ledger.py b/tools/test_reconcile_ledger.py deleted file mode 100644 index 791f038287..0000000000 --- a/tools/test_reconcile_ledger.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -"""Targeted proof of the S45p7 reconcile-ledger fix in dedup_propagate. - -The full-propagation negative control did not fire (that run SUCCEEDED), so the guarded -path was never executed. This exercises the mechanism directly: - - 1. take real overlay files, snapshot them - 2. apply a REAL reconcile via dedup_propagate.reconcile_caller_extern (the same call the - --recover Part B path makes) -> files are now edited on disk - 3. drive the ledger's undo the way the fixed code does when the fn leaves the plan - 4. assert every file is byte-identical to its original - -Read-only w.r.t. git: it restores what it edits, and asserts it did. -""" -import sys, os, hashlib -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools')) -sys.path.insert(0, 'tools') -import dedup_propagate as dp - - -def sha(p): - return hashlib.sha1(open(p, 'rb').read()).hexdigest() - - -def main(): - OV, ADDR = 'ov_SC07_010', 0x80146A6C # the exact overlay+fn from tonight's breakage - files = [cp for cp, _ in dp.overlay_files(OV)] - before = {str(p): sha(p) for p in files} - print(f"[setup] {OV}: {len(files)} source files snapshotted") - - # --- 1. apply a real reconcile (this WRITES) --- - snap, n = dp.reconcile_caller_extern(OV, ADDR) - after_edit = {str(p): sha(p) for p in files} - changed = [k for k in before if before[k] != after_edit.get(k)] - print(f"[apply] reconcile_caller_extern -> {n} edit(s); {len(changed)} file(s) changed on disk") - if n == 0 or not changed: - print("SKIP: this fn/overlay pair produced no reconcile edit — cannot exercise the path.") - return 3 - - # --- 2. drive the ledger undo exactly as the fixed code does --- - kept = [(ADDR, snap)] - drop = [r for r in kept if r[0] in {ADDR}] - for _a, _s in reversed(drop): - dp.restore_snapshot(_s) - print(f"[undo] ledger restored {len(drop)} kept reconcile(s)") - - # --- 3. the assertion --- - after = {str(p): sha(p) for p in files} - bad = [k for k in before if before[k] != after.get(k)] - if bad: - print("FAIL — files NOT restored byte-identically:") - for k in bad[:5]: - print(" ", k) - return 1 - print(f"PASS — all {len(files)} file(s) byte-identical after the ledger undo " - f"(the orphan that broke 141/213 cannot survive this path)") - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tools/tool_census.py b/tools/tool_census.py index 8f51336bf5..53fe1fabe3 100644 --- a/tools/tool_census.py +++ b/tools/tool_census.py @@ -92,7 +92,7 @@ PHASE_TITLES = {"P1": "extraction + manifest", "P2": "the oracles + the load map "P7": "the codegen map, the dumps, the permuter", "P8": "the campaign: cards, lanes, gates, recovery, harvest", "P9": "publish", "P10": "readability", "PROJECT-ONLY": "project-only in code (the shape is a task; the code does not transfer)"} COLS = ["path", "phase", "portability", "need", "what", "adapts", "status", "successor_or_product"] -STATUSES = {"LIVE", "STILL-NEEDED", "SUPERSEDED", "ONE-OFF"} +STATUSES = {"LIVE", "STILL-NEEDED", "FROZEN", "SUPERSEDED", "ONE-OFF"} # FROZEN (P35 T6): in the tree, command line refuses, libraries import def sh(cmd): @@ -191,7 +191,7 @@ def census(): files, retired = enumerate_tools() rows = read_dict() by_path = {r["path"]: r for r in rows} - live_rows = {p for p, r in by_path.items() if r["status"] in ("LIVE", "STILL-NEEDED")} + live_rows = {p for p, r in by_path.items() if r["status"] in ("LIVE", "STILL-NEEDED", "FROZEN")} retired_rows = {p for p, r in by_path.items() if r["status"] in ("SUPERSEDED", "ONE-OFF")} gaps = [] for f in files: diff --git a/tools/validate_targets.py b/tools/validate_targets.py index 5eb64238b9..f3f377dd91 100644 --- a/tools/validate_targets.py +++ b/tools/validate_targets.py @@ -41,7 +41,7 @@ import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import corpus -import dedup_propagate as DP +import share_body as DP # load_sig lives in share_body since dedup_propagate retired (P35 T6) REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))