mirror/BFM-decomp
1
0
Fork 0
mirror of https://github.com/Druthulu/BFM-decomp synced 2026-10-05 08:57:59 -04:00
Code Issues Packages Projects Releases Wiki Activity

phase-36: T3 (tools) — tools/delever.py the campaign tool: --plan/--apply (the file is the write unit, its final compile through every recipe the proof; 'done' = a ledger fact keyed by the body's normalized text before/after; replay from an exemplar in one compile; !FAKE markers on class A/B survivors; --restore from inflight.json; --apply-body for T6/T7; a selftest with a stub oracle), the launder-ASSIGNMENT rewrite (989 of 4,793 direct launders — deleting them made cc1 2.7.2 abort: the probe's 'Done' error, reproduced 3/3), macro shapes (SHB deleted, a launder expression valued, compound macros refused for T5), the oracle's COMPILE-CRASH verdict + judge_all + the mtime restore, the census's src_stamp/head, tools/delever_cycle.sh (apply → R22 → census → log + headline → commit); selftest OK; census 53,234 unchanged; calibration 177/177; plan ov_SC04_011 = 18 files / 176 bodies / 540 sites; SETUP + dictionary rows; kit corpus; T7 waves need Drew's direct approval (decisions)

Browse Source
This commit is contained in:
Drew T
2026-09-09 02:45:58 -06:00
parent 89676f096a
commit 39e3e1851f
18 changed files with 2688 additions and 494 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -485,6 +485,7 @@ unsloth_compiled_cache/
!/.run/P36/delever/*.log !/.run/P36/delever/*.log
!/.run/P36/delever/*.json !/.run/P36/delever/*.json
/.run/P36/delever/recipes.json /.run/P36/delever/recipes.json
/.run/P36/delever/inflight.json
!/.run/P36/delever/*.jsonl !/.run/P36/delever/*.jsonl
!/.run/P36/delever/*.txt !/.run/P36/delever/*.txt
!/.run/P36/permuter/ !/.run/P36/permuter/
.run/P36/census/lever_census.json
+3 -1
View File
@@ -229,8 +229,9 @@
} }
}, },
"coverage_ok": true, "coverage_ok": true,
"elapsed_s": 40.0, "elapsed_s": 36.0,
"generated": "2026-09-09", "generated": "2026-09-09",
"head": "89676f096",
"headers": 3181, "headers": 3181,
"levers_AB": { "levers_AB": {
"asm": 15514, "asm": 15514,
@@ -434,6 +435,7 @@
"volatile_qualified": 136, "volatile_qualified": 136,
"zero": 1337 "zero": 1337
}, },
"src_stamp": "8be8c2a58e0ebac7",
"tus": 4121, "tus": 4121,
"unclassified": 0, "unclassified": 0,
"union_AD": { "union_AD": {
.run/P36/census/lever_census.txt
+2 -2
View File
@@ -12,7 +12,7 @@ lever_census: 218 binaries · 4,121 TUs + 3,181 headers · coverage OK · unclas
E asm-label 7428 1005 6423 737 117 2080 0 7428 {'asm-label': 7428} E asm-label 7428 1005 6423 737 117 2080 0 7428 {'asm-label': 7428}
F builtin 445 445 0 428 27 302 0 445 {'builtin': 445} F builtin 445 445 0 428 27 302 0 445 {'builtin': 445}
G attribute 76 1 75 1 1 40 0 76 {'attribute': 76} G attribute 76 1 75 1 1 40 0 76 {'attribute': 76}
UNION A–D: 19,156 bodies · 2,537 distinct (addresses normalized) · 488 multi-copy classes holding 17,107 bodies · by kind {'ov': 18330, 'shared': 393, 'md': 213, 'main': 204, 'resident': 16} UNION A–D: 19,156 bodies · 2,537 distinct (addresses normalized) · 488 multi-copy classes holding 17,107 bodies · by kind {'ov': 18330, 'md': 213, 'shared': 393, 'main': 204, 'resident': 16}
THE PHASE'S NUMBER (pins + asm statements, GTE excluded): 53,234 sites in 15,679 bodies (2,227 distinct) · marked !FAKE 0 · UNMARKED 53,234 THE PHASE'S NUMBER (pins + asm statements, GTE excluded): 53,234 sites in 15,679 bodies (2,227 distinct) · marked !FAKE 0 · UNMARKED 53,234
pins: 37,720 · $0 1,337 · $sp 13 · with initializer 5,140 · volatile-qualified 136 · bare-name 76 · spellings {'__asm__': 37510, 'asm': 210} pins: 37,720 · $0 1,337 · $sp 13 · with initializer 5,140 · volatile-qualified 136 · bare-name 76 · spellings {'__asm__': 37510, 'asm': 210}
whole-body asm routines in C shells, manifest PERMANENT (hand asm, NOT levers): 22 routines · 2,682 sites (2,660 private copies + 22 shared headers); asm-bodies NOT permanent (levers): 13 site(s) ['func_8001E378:DECOMPILE-NOW', 'func_80020F34:DECOMPILE-NOW', 'func_800249F0:DECOMPILE-NOW', 'func_80025CBC:DECOMPILE-NOW', 'func_80026514:UNCERTAIN', 'func_800268D0:UNCERTAIN', 'func_80027058:DECOMPILE-NOW', 'func_80027200:DECOMPILE-NOW', 'func_800CBA44:DECOMPILE-NOW', 'func_8017D810:DECOMPILE-NOW', 'func_8017E26C:UNCERTAIN', 'func_80184440:DECOMPILE-NOW', 'func_801A3BCC:DECOMPILE-NOW'] whole-body asm routines in C shells, manifest PERMANENT (hand asm, NOT levers): 22 routines · 2,682 sites (2,660 private copies + 22 shared headers); asm-bodies NOT permanent (levers): 13 site(s) ['func_8001E378:DECOMPILE-NOW', 'func_80020F34:DECOMPILE-NOW', 'func_800249F0:DECOMPILE-NOW', 'func_80025CBC:DECOMPILE-NOW', 'func_80026514:UNCERTAIN', 'func_800268D0:UNCERTAIN', 'func_80027058:DECOMPILE-NOW', 'func_80027200:DECOMPILE-NOW', 'func_800CBA44:DECOMPILE-NOW', 'func_8017D810:DECOMPILE-NOW', 'func_8017E26C:UNCERTAIN', 'func_80184440:DECOMPILE-NOW', 'func_801A3BCC:DECOMPILE-NOW']
@@ -25,4 +25,4 @@ lever_census: 218 binaries · 4,121 TUs + 3,181 headers · coverage OK · unclas
src/shared/ov/func_80178004.h pins got 26 expected 26 OK src/shared/ov/func_80178004.h pins got 26 expected 26 OK
ov_SC03_006 func_80184034 bare-name pins got 3 expected 3 OK ov_SC03_006 func_80184034 bare-name pins got 3 expected 3 OK
engine_prelude.h asm sites (a macro definition only) got 0 expected 0 OK engine_prelude.h asm sites (a macro definition only) got 0 expected 0 OK
elapsed 40.0 s elapsed 36.0 s
.run/P36/delever/calibration.json
+176 -176
View File
@@ -1,7 +1,7 @@
{ {
"head": "a1407bdfc", "head": "89676f096",
"stamp": "b44c5a8a3423", "stamp": "b44c5a8a3423",
"generated": "2026-09-09 01:50", "generated": "2026-09-09 02:38",
"aliases": [ "aliases": [
"main", "main",
"ov_SC03_014", "ov_SC03_014",
@@ -21,207 +21,207 @@
"main": { "main": {
"objects": 85, "objects": 85,
"identical": 85, "identical": 85,
"seconds": 6.707000000000002, "seconds": 7.229000000000001,
"mean_s": 0.079 "mean_s": 0.085
}, },
"ov_SC03_014": { "ov_SC03_014": {
"objects": 32, "objects": 32,
"identical": 32, "identical": 32,
"seconds": 4.815, "seconds": 5.186000000000002,
"mean_s": 0.15 "mean_s": 0.162
}, },
"ov_SC03_015": { "ov_SC03_015": {
"objects": 32, "objects": 32,
"identical": 32, "identical": 32,
"seconds": 4.541, "seconds": 4.574999999999999,
"mean_s": 0.142 "mean_s": 0.143
}, },
"ov_SC04_011": { "ov_SC04_011": {
"objects": 28, "objects": 28,
"identical": 28, "identical": 28,
"seconds": 3.943, "seconds": 3.9430000000000005,
"mean_s": 0.141 "mean_s": 0.141
} }
}, },
"per_object_seconds": { "per_object_seconds": {
"build/src/800.o": 0.752, "build/src/800.o": 0.785,
"build/src/800_b.o": 0.07, "build/src/800_b.o": 0.072,
"build/src/800_b_2.o": 0.329, "build/src/800_b_2.o": 0.317,
"build/src/800_b_o0a.o": 0.074, "build/src/800_b_o0a.o": 0.077,
"build/src/800_c.o": 0.195, "build/src/800_c.o": 0.206,
"build/src/800b2.o": 0.078, "build/src/800b2.o": 0.075,
"build/src/apicard1.o": 0.061, "build/src/apicard1.o": 0.087,
"build/src/apicard2.o": 0.069, "build/src/apicard2.o": 0.074,
"build/src/apicard3.o": 0.076, "build/src/apicard3.o": 0.092,
"build/src/apicard4.o": 0.066, "build/src/apicard4.o": 0.075,
"build/src/apicard5.o": 0.086, "build/src/apicard5.o": 0.096,
"build/src/apicard6.o": 0.072, "build/src/apicard6.o": 0.071,
"build/src/apicard7.o": 0.081, "build/src/apicard7.o": 0.083,
"build/src/boot.o": 0.08, "build/src/boot.o": 0.113,
"build/src/gap.o": 0.071, "build/src/gap.o": 0.093,
"build/src/libapi1.o": 0.07, "build/src/libapi1.o": 0.092,
"build/src/libapi2.o": 0.063, "build/src/libapi2.o": 0.064,
"build/src/libc2_1.o": 0.064, "build/src/libc2_1.o": 0.071,
"build/src/libc2_2.o": 0.059, "build/src/libc2_2.o": 0.063,
"build/src/libcd1.o": 0.069, "build/src/libcd1.o": 0.081,
"build/src/libcd2.o": 0.065, "build/src/libcd2.o": 0.07,
"build/src/libetc.o": 0.062, "build/src/libetc.o": 0.073,
"build/src/libgpu.o": 0.059, "build/src/libgpu.o": 0.076,
"build/src/libgpu2.o": 0.067, "build/src/libgpu2.o": 0.088,
"build/src/libgs1.o": 0.057, "build/src/libgs1.o": 0.066,
"build/src/libgs2.o": 0.064, "build/src/libgs2.o": 0.066,
"build/src/libgs3.o": 0.057, "build/src/libgs3.o": 0.063,
"build/src/libgs4.o": 0.064, "build/src/libgs4.o": 0.066,
"build/src/libgs5.o": 0.057, "build/src/libgs5.o": 0.063,
"build/src/libgs6.o": 0.077, "build/src/libgs6.o": 0.088,
"build/src/libgs7.o": 0.061, "build/src/libgs7.o": 0.072,
"build/src/libgs8.o": 0.059, "build/src/libgs8.o": 0.063,
"build/src/libgte1.o": 0.055, "build/src/libgte1.o": 0.061,
"build/src/libgte10.o": 0.065, "build/src/libgte10.o": 0.06,
"build/src/libgte11.o": 0.063, "build/src/libgte11.o": 0.067,
"build/src/libgte12.o": 0.061, "build/src/libgte12.o": 0.062,
"build/src/libgte13.o": 0.056, "build/src/libgte13.o": 0.066,
"build/src/libgte14.o": 0.058, "build/src/libgte14.o": 0.068,
"build/src/libgte15.o": 0.071, "build/src/libgte15.o": 0.061,
"build/src/libgte16.o": 0.063, "build/src/libgte16.o": 0.064,
"build/src/libgte17.o": 0.063, "build/src/libgte17.o": 0.064,
"build/src/libgte18.o": 0.061, "build/src/libgte18.o": 0.065,
"build/src/libgte19.o": 0.063, "build/src/libgte19.o": 0.065,
"build/src/libgte2.o": 0.061, "build/src/libgte2.o": 0.063,
"build/src/libgte20.o": 0.066, "build/src/libgte20.o": 0.063,
"build/src/libgte21.o": 0.058, "build/src/libgte21.o": 0.071,
"build/src/libgte22.o": 0.063, "build/src/libgte22.o": 0.068,
"build/src/libgte23.o": 0.064, "build/src/libgte23.o": 0.057,
"build/src/libgte24.o": 0.057, "build/src/libgte24.o": 0.069,
"build/src/libgte25.o": 0.061, "build/src/libgte25.o": 0.062,
"build/src/libgte26.o": 0.068, "build/src/libgte26.o": 0.071,
"build/src/libgte27.o": 0.062, "build/src/libgte27.o": 0.068,
"build/src/libgte28.o": 0.06, "build/src/libgte28.o": 0.071,
"build/src/libgte29.o": 0.068, "build/src/libgte29.o": 0.068,
"build/src/libgte3.o": 0.069, "build/src/libgte3.o": 0.067,
"build/src/libgte30.o": 0.069, "build/src/libgte30.o": 0.081,
"build/src/libgte4.o": 0.065, "build/src/libgte4.o": 0.07,
"build/src/libgte5.o": 0.065, "build/src/libgte5.o": 0.07,
"build/src/libgte6.o": 0.061, "build/src/libgte6.o": 0.07,
"build/src/libgte7.o": 0.058, "build/src/libgte7.o": 0.064,
"build/src/libgte8.o": 0.059, "build/src/libgte8.o": 0.062,
"build/src/libgte9.o": 0.068, "build/src/libgte9.o": 0.074,
"build/src/libmcrd1.o": 0.082, "build/src/libmcrd1.o": 0.088,
"build/src/libmcrd2.o": 0.064, "build/src/libmcrd2.o": 0.064,
"build/src/libpad1.o": 0.068, "build/src/libpad1.o": 0.076,
"build/src/libpad2.o": 0.066, "build/src/libpad2.o": 0.078,
"build/src/sgap.o": 0.066, "build/src/sgap.o": 0.07,
"build/src/sgap_2.o": 0.067, "build/src/sgap_2.o": 0.067,
"build/src/sgap_3.o": 0.073, "build/src/sgap_3.o": 0.065,
"build/src/sgap_4.o": 0.078, "build/src/sgap_4.o": 0.076,
"build/src/sgap_5.o": 0.065, "build/src/sgap_5.o": 0.07,
"build/src/sgap_6.o": 0.064, "build/src/sgap_6.o": 0.071,
"build/src/sgap_8.o": 0.076, "build/src/sgap_8.o": 0.081,
"build/src/snd1.o": 0.073, "build/src/snd1.o": 0.079,
"build/src/snd10.o": 0.07, "build/src/snd10.o": 0.076,
"build/src/snd11.o": 0.071, "build/src/snd11.o": 0.076,
"build/src/snd12.o": 0.063, "build/src/snd12.o": 0.071,
"build/src/snd2.o": 0.072, "build/src/snd2.o": 0.074,
"build/src/snd3.o": 0.068, "build/src/snd3.o": 0.071,
"build/src/snd4.o": 0.073, "build/src/snd4.o": 0.073,
"build/src/snd5.o": 0.067, "build/src/snd5.o": 0.071,
"build/src/snd6.o": 0.069, "build/src/snd6.o": 0.083,
"build/src/snd7.o": 0.069, "build/src/snd7.o": 0.072,
"build/src/snd8.o": 0.063, "build/src/snd8.o": 0.067,
"build/src/snd9.o": 0.065, "build/src/snd9.o": 0.077,
"build/src/ov_SC03_014/ov_SC03_014.o": 0.141, "build/src/ov_SC03_014/ov_SC03_014.o": 0.157,
"build/src/ov_SC03_014/ov_SC03_014_after.o": 0.509, "build/src/ov_SC03_014/ov_SC03_014_after.o": 0.569,
"build/src/ov_SC03_014/ov_SC03_014_jr_8012ACE0.o": 0.394, "build/src/ov_SC03_014/ov_SC03_014_jr_8012ACE0.o": 0.432,
"build/src/ov_SC03_014/ov_SC03_014_jr_80135888.o": 0.082, "build/src/ov_SC03_014/ov_SC03_014_jr_80135888.o": 0.086,
"build/src/ov_SC03_014/ov_SC03_014_jr_80135A4C.o": 0.08, "build/src/ov_SC03_014/ov_SC03_014_jr_80135A4C.o": 0.068,
"build/src/ov_SC03_014/ov_SC03_014_jr_80135D20.o": 0.126, "build/src/ov_SC03_014/ov_SC03_014_jr_80135D20.o": 0.131,
"build/src/ov_SC03_014/ov_SC03_014_jr_801380E0.o": 0.169, "build/src/ov_SC03_014/ov_SC03_014_jr_801380E0.o": 0.188,
"build/src/ov_SC03_014/ov_SC03_014_jr_8013C98C.o": 0.129, "build/src/ov_SC03_014/ov_SC03_014_jr_8013C98C.o": 0.133,
"build/src/ov_SC03_014/ov_SC03_014_jr_8013F350.o": 0.097, "build/src/ov_SC03_014/ov_SC03_014_jr_8013F350.o": 0.102,
"build/src/ov_SC03_014/ov_SC03_014_jr_8013FFD8.o": 0.074, "build/src/ov_SC03_014/ov_SC03_014_jr_8013FFD8.o": 0.085,
"build/src/ov_SC03_014/ov_SC03_014_jr_80140608.o": 0.182, "build/src/ov_SC03_014/ov_SC03_014_jr_80140608.o": 0.213,
"build/src/ov_SC03_014/ov_SC03_014_jr_8015444C.o": 0.086, "build/src/ov_SC03_014/ov_SC03_014_jr_8015444C.o": 0.098,
"build/src/ov_SC03_014/ov_SC03_014_jr_80154C24.o": 0.174, "build/src/ov_SC03_014/ov_SC03_014_jr_80154C24.o": 0.197,
"build/src/ov_SC03_014/ov_SC03_014_jr_801588CC.o": 0.102, "build/src/ov_SC03_014/ov_SC03_014_jr_801588CC.o": 0.112,
"build/src/ov_SC03_014/ov_SC03_014_jr_80159C84.o": 0.079, "build/src/ov_SC03_014/ov_SC03_014_jr_80159C84.o": 0.105,
"build/src/ov_SC03_014/ov_SC03_014_jr_8015A3C8.o": 0.075, "build/src/ov_SC03_014/ov_SC03_014_jr_8015A3C8.o": 0.079,
"build/src/ov_SC03_014/ov_SC03_014_jr_8015AE2C.o": 0.093, "build/src/ov_SC03_014/ov_SC03_014_jr_8015AE2C.o": 0.121,
"build/src/ov_SC03_014/ov_SC03_014_jr_8015C32C.o": 0.487, "build/src/ov_SC03_014/ov_SC03_014_jr_8015C32C.o": 0.474,
"build/src/ov_SC03_014/ov_SC03_014_jr_8016AB6C.o": 0.267, "build/src/ov_SC03_014/ov_SC03_014_jr_8016AB6C.o": 0.28,
"build/src/ov_SC03_014/ov_SC03_014_jr_80171B4C.o": 0.102, "build/src/ov_SC03_014/ov_SC03_014_jr_80171B4C.o": 0.119,
"build/src/ov_SC03_014/ov_SC03_014_jr_801734BC.o": 0.205, "build/src/ov_SC03_014/ov_SC03_014_jr_801734BC.o": 0.208,
"build/src/ov_SC03_014/ov_SC03_014_jr_801789AC.o": 0.064, "build/src/ov_SC03_014/ov_SC03_014_jr_801789AC.o": 0.075,
"build/src/ov_SC03_014/ov_SC03_014_jr_80178D40.o": 0.099, "build/src/ov_SC03_014/ov_SC03_014_jr_80178D40.o": 0.107,
"build/src/ov_SC03_014/ov_SC03_014_jr_8017A4AC.o": 0.066, "build/src/ov_SC03_014/ov_SC03_014_jr_8017A4AC.o": 0.082,
"build/src/ov_SC03_014/ov_SC03_014_jr_8017AE2C.o": 0.168, "build/src/ov_SC03_014/ov_SC03_014_jr_8017AE2C.o": 0.179,
"build/src/ov_SC03_014/ov_SC03_014_jr_8017EB7C.o": 0.193, "build/src/ov_SC03_014/ov_SC03_014_jr_8017EB7C.o": 0.199,
"build/src/ov_SC03_014/ov_SC03_014_jr_80184440.o": 0.054, "build/src/ov_SC03_014/ov_SC03_014_jr_80184440.o": 0.057,
"build/src/ov_SC03_014/ov_SC03_014_jr_801848E4.o": 0.281, "build/src/ov_SC03_014/ov_SC03_014_jr_801848E4.o": 0.28,
"build/src/ov_SC03_014/ov_SC03_014_o0b.o": 0.059, "build/src/ov_SC03_014/ov_SC03_014_o0b.o": 0.064,
"build/src/ov_SC03_014/ov_SC03_014_o0c.o": 0.06, "build/src/ov_SC03_014/ov_SC03_014_o0c.o": 0.061,
"build/src/ov_SC03_014/ov_SC03_014_o0d.o": 0.053, "build/src/ov_SC03_014/ov_SC03_014_o0d.o": 0.056,
"build/src/ov_SC03_014/ov_SC03_014_o0e.o": 0.065, "build/src/ov_SC03_014/ov_SC03_014_o0e.o": 0.069,
"build/src/ov_SC03_015/ov_SC03_015.o": 0.12, "build/src/ov_SC03_015/ov_SC03_015.o": 0.127,
"build/src/ov_SC03_015/ov_SC03_015_after.o": 0.526, "build/src/ov_SC03_015/ov_SC03_015_after.o": 0.516,
"build/src/ov_SC03_015/ov_SC03_015_jr_8012ACE0.o": 0.382, "build/src/ov_SC03_015/ov_SC03_015_jr_8012ACE0.o": 0.388,
"build/src/ov_SC03_015/ov_SC03_015_jr_80135888.o": 0.051, "build/src/ov_SC03_015/ov_SC03_015_jr_80135888.o": 0.051,
"build/src/ov_SC03_015/ov_SC03_015_jr_80135A4C.o": 0.051, "build/src/ov_SC03_015/ov_SC03_015_jr_80135A4C.o": 0.055,
"build/src/ov_SC03_015/ov_SC03_015_jr_80135D20.o": 0.116, "build/src/ov_SC03_015/ov_SC03_015_jr_80135D20.o": 0.117,
"build/src/ov_SC03_015/ov_SC03_015_jr_801380E0.o": 0.148, "build/src/ov_SC03_015/ov_SC03_015_jr_801380E0.o": 0.153,
"build/src/ov_SC03_015/ov_SC03_015_jr_8013C98C.o": 0.113, "build/src/ov_SC03_015/ov_SC03_015_jr_8013C98C.o": 0.118,
"build/src/ov_SC03_015/ov_SC03_015_jr_8013F350.o": 0.064, "build/src/ov_SC03_015/ov_SC03_015_jr_8013F350.o": 0.074,
"build/src/ov_SC03_015/ov_SC03_015_jr_8013FFD8.o": 0.06, "build/src/ov_SC03_015/ov_SC03_015_jr_8013FFD8.o": 0.063,
"build/src/ov_SC03_015/ov_SC03_015_jr_80140608.o": 0.18, "build/src/ov_SC03_015/ov_SC03_015_jr_80140608.o": 0.183,
"build/src/ov_SC03_015/ov_SC03_015_jr_8015444C.o": 0.07, "build/src/ov_SC03_015/ov_SC03_015_jr_8015444C.o": 0.071,
"build/src/ov_SC03_015/ov_SC03_015_jr_80154C24.o": 0.169, "build/src/ov_SC03_015/ov_SC03_015_jr_80154C24.o": 0.158,
"build/src/ov_SC03_015/ov_SC03_015_jr_801588CC.o": 0.085, "build/src/ov_SC03_015/ov_SC03_015_jr_801588CC.o": 0.085,
"build/src/ov_SC03_015/ov_SC03_015_jr_80159C84.o": 0.066, "build/src/ov_SC03_015/ov_SC03_015_jr_80159C84.o": 0.061,
"build/src/ov_SC03_015/ov_SC03_015_jr_8015A3C8.o": 0.068, "build/src/ov_SC03_015/ov_SC03_015_jr_8015A3C8.o": 0.071,
"build/src/ov_SC03_015/ov_SC03_015_jr_8015AE2C.o": 0.092, "build/src/ov_SC03_015/ov_SC03_015_jr_8015AE2C.o": 0.092,
"build/src/ov_SC03_015/ov_SC03_015_jr_8015C32C.o": 0.472, "build/src/ov_SC03_015/ov_SC03_015_jr_8015C32C.o": 0.446,
"build/src/ov_SC03_015/ov_SC03_015_jr_8016AB6C.o": 0.259, "build/src/ov_SC03_015/ov_SC03_015_jr_8016AB6C.o": 0.269,
"build/src/ov_SC03_015/ov_SC03_015_jr_80171B4C.o": 0.101, "build/src/ov_SC03_015/ov_SC03_015_jr_80171B4C.o": 0.096,
"build/src/ov_SC03_015/ov_SC03_015_jr_801734BC.o": 0.201, "build/src/ov_SC03_015/ov_SC03_015_jr_801734BC.o": 0.21,
"build/src/ov_SC03_015/ov_SC03_015_jr_801789AC.o": 0.058, "build/src/ov_SC03_015/ov_SC03_015_jr_801789AC.o": 0.056,
"build/src/ov_SC03_015/ov_SC03_015_jr_80178D40.o": 0.1, "build/src/ov_SC03_015/ov_SC03_015_jr_80178D40.o": 0.097,
"build/src/ov_SC03_015/ov_SC03_015_jr_8017A4AC.o": 0.064, "build/src/ov_SC03_015/ov_SC03_015_jr_8017A4AC.o": 0.067,
"build/src/ov_SC03_015/ov_SC03_015_jr_8017AE2C.o": 0.164, "build/src/ov_SC03_015/ov_SC03_015_jr_8017AE2C.o": 0.165,
"build/src/ov_SC03_015/ov_SC03_015_jr_8017EB7C.o": 0.189, "build/src/ov_SC03_015/ov_SC03_015_jr_8017EB7C.o": 0.21,
"build/src/ov_SC03_015/ov_SC03_015_jr_80184440.o": 0.05, "build/src/ov_SC03_015/ov_SC03_015_jr_80184440.o": 0.052,
"build/src/ov_SC03_015/ov_SC03_015_jr_801848E4.o": 0.278, "build/src/ov_SC03_015/ov_SC03_015_jr_801848E4.o": 0.284,
"build/src/ov_SC03_015/ov_SC03_015_o0b.o": 0.06, "build/src/ov_SC03_015/ov_SC03_015_o0b.o": 0.06,
"build/src/ov_SC03_015/ov_SC03_015_o0c.o": 0.061, "build/src/ov_SC03_015/ov_SC03_015_o0c.o": 0.06,
"build/src/ov_SC03_015/ov_SC03_015_o0d.o": 0.054, "build/src/ov_SC03_015/ov_SC03_015_o0d.o": 0.05,
"build/src/ov_SC03_015/ov_SC03_015_o0e.o": 0.069, "build/src/ov_SC03_015/ov_SC03_015_o0e.o": 0.07,
"build/src/ov_SC04_011/ov_SC04_011.o": 0.121, "build/src/ov_SC04_011/ov_SC04_011.o": 0.12,
"build/src/ov_SC04_011/ov_SC04_011_after.o": 0.423, "build/src/ov_SC04_011/ov_SC04_011_after.o": 0.428,
"build/src/ov_SC04_011/ov_SC04_011_jr_8012ACE0.o": 0.353, "build/src/ov_SC04_011/ov_SC04_011_jr_8012ACE0.o": 0.348,
"build/src/ov_SC04_011/ov_SC04_011_jr_80135888.o": 0.054, "build/src/ov_SC04_011/ov_SC04_011_jr_80135888.o": 0.054,
"build/src/ov_SC04_011/ov_SC04_011_jr_80135A4C.o": 0.053, "build/src/ov_SC04_011/ov_SC04_011_jr_80135A4C.o": 0.056,
"build/src/ov_SC04_011/ov_SC04_011_jr_80135D20.o": 0.12, "build/src/ov_SC04_011/ov_SC04_011_jr_80135D20.o": 0.115,
"build/src/ov_SC04_011/ov_SC04_011_jr_801380E0.o": 0.152, "build/src/ov_SC04_011/ov_SC04_011_jr_801380E0.o": 0.153,
"build/src/ov_SC04_011/ov_SC04_011_jr_8013C98C.o": 0.109, "build/src/ov_SC04_011/ov_SC04_011_jr_8013C98C.o": 0.112,
"build/src/ov_SC04_011/ov_SC04_011_jr_8013F350.o": 0.076, "build/src/ov_SC04_011/ov_SC04_011_jr_8013F350.o": 0.072,
"build/src/ov_SC04_011/ov_SC04_011_jr_8013FFD8.o": 0.065, "build/src/ov_SC04_011/ov_SC04_011_jr_8013FFD8.o": 0.059,
"build/src/ov_SC04_011/ov_SC04_011_jr_80140608.o": 0.172, "build/src/ov_SC04_011/ov_SC04_011_jr_80140608.o": 0.18,
"build/src/ov_SC04_011/ov_SC04_011_jr_8015444C.o": 0.072, "build/src/ov_SC04_011/ov_SC04_011_jr_8015444C.o": 0.073,
"build/src/ov_SC04_011/ov_SC04_011_jr_80154C24.o": 0.169, "build/src/ov_SC04_011/ov_SC04_011_jr_80154C24.o": 0.162,
"build/src/ov_SC04_011/ov_SC04_011_jr_801588CC.o": 0.092, "build/src/ov_SC04_011/ov_SC04_011_jr_801588CC.o": 0.087,
"build/src/ov_SC04_011/ov_SC04_011_jr_80159C84.o": 0.066, "build/src/ov_SC04_011/ov_SC04_011_jr_80159C84.o": 0.066,
"build/src/ov_SC04_011/ov_SC04_011_jr_8015A3C8.o": 0.073, "build/src/ov_SC04_011/ov_SC04_011_jr_8015A3C8.o": 0.075,
"build/src/ov_SC04_011/ov_SC04_011_jr_8015AE2C.o": 0.093, "build/src/ov_SC04_011/ov_SC04_011_jr_8015AE2C.o": 0.096,
"build/src/ov_SC04_011/ov_SC04_011_jr_8015C32C.o": 0.353, "build/src/ov_SC04_011/ov_SC04_011_jr_8015C32C.o": 0.357,
"build/src/ov_SC04_011/ov_SC04_011_jr_8016AB6C.o": 0.217, "build/src/ov_SC04_011/ov_SC04_011_jr_8016AB6C.o": 0.222,
"build/src/ov_SC04_011/ov_SC04_011_jr_80171B4C.o": 0.099, "build/src/ov_SC04_011/ov_SC04_011_jr_80171B4C.o": 0.097,
"build/src/ov_SC04_011/ov_SC04_011_jr_801734BC.o": 0.164, "build/src/ov_SC04_011/ov_SC04_011_jr_801734BC.o": 0.168,
"build/src/ov_SC04_011/ov_SC04_011_jr_801789AC.o": 0.054, "build/src/ov_SC04_011/ov_SC04_011_jr_801789AC.o": 0.057,
"build/src/ov_SC04_011/ov_SC04_011_jr_80178D40.o": 0.085, "build/src/ov_SC04_011/ov_SC04_011_jr_80178D40.o": 0.088,
"build/src/ov_SC04_011/ov_SC04_011_jr_8017A4AC.o": 0.063, "build/src/ov_SC04_011/ov_SC04_011_jr_8017A4AC.o": 0.06,
"build/src/ov_SC04_011/ov_SC04_011_jr_8017AE2C.o": 0.093, "build/src/ov_SC04_011/ov_SC04_011_jr_8017AE2C.o": 0.095,
"build/src/ov_SC04_011/ov_SC04_011_jr_8017D494.o": 0.444, "build/src/ov_SC04_011/ov_SC04_011_jr_8017D494.o": 0.444,
"build/src/ov_SC04_011/ov_SC04_011_o0b.o": 0.048, "build/src/ov_SC04_011/ov_SC04_011_o0b.o": 0.046,
"build/src/ov_SC04_011/ov_SC04_011_o0c.o": 0.06 "build/src/ov_SC04_011/ov_SC04_011_o0c.o": 0.053
}, },
"ok": true, "ok": true,
"seconds": 2.3 "seconds": 2.4
} }
config/tool_dictionary.tsv
+2 -1
View File
@@ -69,7 +69,7 @@ tools/decompme_replica.sh P4 ADAPT reproduce an external reference toolchain loc
tools/dedup_extend.py P6 ADAPT extend an existing code-share registry to newly onboarded binaries Extends the existing share registry to newly onboarded binaries repo config/src layout SUPERSEDED tools/share_body.py tools/dedup_extend.py P6 ADAPT extend an existing code-share registry to newly onboarded binaries Extends the existing share registry to newly onboarded binaries repo config/src layout SUPERSEDED tools/share_body.py
tools/dedup_integrate.py P6 ADAPT validate a code-share registry byte-honestly, failing closed on signature drift Byte-honesty validator for the code-share registry; fails closed on signature drift repo config registry path LIVE tools/dedup_integrate.py P6 ADAPT validate a code-share registry byte-honestly, failing closed on signature drift Byte-honesty validator for the code-share registry; fails closed on signature drift repo config registry path LIVE
tools/dedup_propagate.py P6 ADAPT lift one matched body into a shared macro and instantiate it everywhere Lifts one matched body into a shared macro and instantiates it at every duplicate site repo shared-header/src layout SUPERSEDED tools/share_body.py tools/dedup_propagate.py P6 ADAPT lift one matched body into a shared macro and instantiate it everywhere Lifts one matched body into a shared macro and instantiates it at every duplicate site repo shared-header/src layout SUPERSEDED tools/share_body.py
tools/delever.py P10 ADAPT take register pins, asm statements, volatile and register levers out of matched C, byte-gated per site The Phase-36 de-lever engine: positional rewrites per lever class (pin -> plain declaration, barrier/launder/keep-alive deleted, a hand-placed instruction -> its C, the zero-register variable -> 0, volatile/register dropped), rung A strip-all and rung B greedy per body through delever_oracle, headers serialized after the TU phase; --probe runs a stratified sample of distinct bodies and prices the campaign with denominators the census site records, the oracle LIVE tools/delever.py P10 ADAPT take register pins, asm statements, volatile and register levers out of matched C, byte-gated per site The Phase-36 de-lever engine and campaign tool: positional rewrites per lever class (pin -> plain declaration, barrier/keep-alive deleted, a launder deleted or turned into the assignment it is, a hand-placed instruction -> its C, the zero-register variable -> 0, a macro-carried site deleted/valued/refused by its macro's shape, volatile/register dropped); per body replay -> rung A strip-all -> rung B greedy through delever_oracle; the file is the write unit and its final compile through every recipe the proof; --plan/--apply batches (TUs parallel, exemplar files first; headers serial, includers parallel), the ledger keyed by tu+fn+addr with the body's nhash before/after, !FAKE markers on class A/B survivors, --restore from inflight.json, --apply-body for T6/T7, --selftest with a stub oracle, --probe (T2) the census site records, the oracle LIVE
tools/delever_oracle.py P10 ADAPT judge one translation-unit edit by the bytes of its object in under a second The Phase-36 fast byte oracle: every object's exact build command captured once via make -n -W (the Makefile's own recipe, pad stage and -O0 overrides included), a candidate compiled in place with -o/-MF redirected to scratch and compared with the fleet run's object; --calibrate proves 100 % equality untouched, twin == primary and a positive control before any verdict is trusted the Makefile's object rules, the twin rule LIVE tools/delever_oracle.py P10 ADAPT judge one translation-unit edit by the bytes of its object in under a second The Phase-36 fast byte oracle: every object's exact build command captured once via make -n -W (the Makefile's own recipe, pad stage and -O0 overrides included), a candidate compiled in place with -o/-MF redirected to scratch and compared with the fleet run's object; --calibrate proves 100 % equality untouched, twin == primary and a positive control before any verdict is trusted the Makefile's object rules, the twin rule LIVE
tools/demacroize.py P6 ADAPT escape a shared-header declaration conflict locally by unmacroizing one body Per-binary local escape from a shared-header declaration conflict by unmacroizing one body repo shared-header layout FROZEN tools/share_body.py tools/demacroize.py P6 ADAPT escape a shared-header declaration conflict locally by unmacroizing one body Per-binary local escape from a shared-header declaration conflict by unmacroizing one body repo shared-header layout FROZEN tools/share_body.py
tools/diff_autopsy.sh P7 ADAPT find which words diverge when the whole-binary gate says different but not where Reproduces exactly what the gate saw, then decodes the diverging words and restores the tree repo src/build paths STILL-NEEDED find WHICH words diverge when the whole-binary gate says DIFF but not where tools/diff_autopsy.sh P7 ADAPT find which words diverge when the whole-binary gate says different but not where Reproduces exactly what the gate saw, then decodes the diverging words and restores the tree repo src/build paths STILL-NEEDED find WHICH words diverge when the whole-binary gate says DIFF but not where
@@ -281,6 +281,7 @@ tools/seed_ref.py P8 ADAPT find an already-banked body that matches an open stub
tools/serve_local.py P8 ADAPT serve a fine-tuned model locally behind a chat-completions endpoint Serves the fine-tuned model locally behind a chat-completions endpoint training venv, GPU assumptions LIVE tools/serve_local.py P8 ADAPT serve a fine-tuned model locally behind a chat-completions endpoint Serves the fine-tuned model locally behind a chat-completions endpoint training venv, GPU assumptions LIVE
tools/share_body.py P10 ADAPT share one byte-identical function class across its binaries through an include-at-site header, gated per binary The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new repo paths, the registry and signature schemas LIVE tools/share_body.py P10 ADAPT share one byte-identical function class across its binaries through an include-at-site header, gated per binary The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new repo paths, the registry and signature schemas LIVE
tools/share_body_cycle.sh P10 ADAPT run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new<k>, the gated N/N line read, the phase log entry appended, the bank committed the moment it is green, the clean fleet run every N batches — stops on the first red repo paths, the phase-log format, the fleet count from config/check.*.sha LIVE tools/share_body_cycle.sh P10 ADAPT run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new<k>, the gated N/N line read, the phase log entry appended, the bank committed the moment it is green, the clean fleet run every N batches — stops on the first red repo paths, the phase-log format, the fleet count from config/check.*.sha LIVE
tools/delever_cycle.sh P10 ADAPT run the de-lever batch cycle unattended: batch, verify by exit code, clean fleet gate, census, log, commit The Phase-36 T3/T4 driver: per batch a clean-tree check, the oracle recalibrated when stale, delever --apply read by exit code and its final X/X line, the clean fleet run as the outer gate (218/218), the census rerun, the phase-log entry and the checkpoint headline, one commit per batch; stops on the first red with the files in place for --restore the batch size, the aliases, the fleet count LIVE
tools/share_census.py P10 ADAPT measure duplicate function bodies across the fleet and assert one source per unique function The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test repo paths, the registry and signature schemas LIVE tools/share_census.py P10 ADAPT measure duplicate function bodies across the fleet and assert one source per unique function The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test repo paths, the registry and signature schemas LIVE
tools/shared_lock.py P8 ADAPT hold one reader/writer lock over the shared source state for concurrent lanes One reader/writer lock over the shared source state for concurrent lanes repo lock paths LIVE tools/shared_lock.py P8 ADAPT hold one reader/writer lock over the shared source state for concurrent lanes One reader/writer lock over the shared source state for concurrent lanes repo lock paths LIVE
tools/sig_image.py P6 PORTABLE sign every function of a flat image at a known base, without a disassembler Disassembler-free per-function signer for a flat image at a known base, field-identical to the dumper MIPS encodings, repo scratch paths LIVE tools/sig_image.py P6 PORTABLE sign every function of a flat image at a known base, without a disassembler Disassembler-free per-function signer for a flat image at a known base, field-identical to the dumper MIPS encodings, repo scratch paths LIVE
1 path phase portability need what adapts status successor_or_product
69 tools/dedup_extend.py P6 ADAPT extend an existing code-share registry to newly onboarded binaries Extends the existing share registry to newly onboarded binaries repo config/src layout SUPERSEDED tools/share_body.py
70 tools/dedup_integrate.py P6 ADAPT validate a code-share registry byte-honestly, failing closed on signature drift Byte-honesty validator for the code-share registry; fails closed on signature drift repo config registry path LIVE
71 tools/dedup_propagate.py P6 ADAPT lift one matched body into a shared macro and instantiate it everywhere Lifts one matched body into a shared macro and instantiates it at every duplicate site repo shared-header/src layout SUPERSEDED tools/share_body.py
72 tools/delever.py P10 ADAPT take register pins, asm statements, volatile and register levers out of matched C, byte-gated per site The Phase-36 de-lever engine: positional rewrites per lever class (pin -> plain declaration, barrier/launder/keep-alive deleted, a hand-placed instruction -> its C, the zero-register variable -> 0, volatile/register dropped), rung A strip-all and rung B greedy per body through delever_oracle, headers serialized after the TU phase; --probe runs a stratified sample of distinct bodies and prices the campaign with denominators The Phase-36 de-lever engine and campaign tool: positional rewrites per lever class (pin -> plain declaration, barrier/keep-alive deleted, a launder deleted or turned into the assignment it is, a hand-placed instruction -> its C, the zero-register variable -> 0, a macro-carried site deleted/valued/refused by its macro's shape, volatile/register dropped); per body replay -> rung A strip-all -> rung B greedy through delever_oracle; the file is the write unit and its final compile through every recipe the proof; --plan/--apply batches (TUs parallel, exemplar files first; headers serial, includers parallel), the ledger keyed by tu+fn+addr with the body's nhash before/after, !FAKE markers on class A/B survivors, --restore from inflight.json, --apply-body for T6/T7, --selftest with a stub oracle, --probe (T2) the census site records, the oracle LIVE
73 tools/delever_oracle.py P10 ADAPT judge one translation-unit edit by the bytes of its object in under a second The Phase-36 fast byte oracle: every object's exact build command captured once via make -n -W (the Makefile's own recipe, pad stage and -O0 overrides included), a candidate compiled in place with -o/-MF redirected to scratch and compared with the fleet run's object; --calibrate proves 100 % equality untouched, twin == primary and a positive control before any verdict is trusted the Makefile's object rules, the twin rule LIVE
74 tools/demacroize.py P6 ADAPT escape a shared-header declaration conflict locally by unmacroizing one body Per-binary local escape from a shared-header declaration conflict by unmacroizing one body repo shared-header layout FROZEN tools/share_body.py
75 tools/diff_autopsy.sh P7 ADAPT find which words diverge when the whole-binary gate says different but not where Reproduces exactly what the gate saw, then decodes the diverging words and restores the tree repo src/build paths STILL-NEEDED find WHICH words diverge when the whole-binary gate says DIFF but not where
281 tools/serve_local.py P8 ADAPT serve a fine-tuned model locally behind a chat-completions endpoint Serves the fine-tuned model locally behind a chat-completions endpoint training venv, GPU assumptions LIVE
282 tools/share_body.py P10 ADAPT share one byte-identical function class across its binaries through an include-at-site header, gated per binary The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new repo paths, the registry and signature schemas LIVE
283 tools/share_body_cycle.sh P10 ADAPT run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new<k>, the gated N/N line read, the phase log entry appended, the bank committed the moment it is green, the clean fleet run every N batches — stops on the first red repo paths, the phase-log format, the fleet count from config/check.*.sha LIVE
284 tools/delever_cycle.sh P10 ADAPT run the de-lever batch cycle unattended: batch, verify by exit code, clean fleet gate, census, log, commit The Phase-36 T3/T4 driver: per batch a clean-tree check, the oracle recalibrated when stale, delever --apply read by exit code and its final X/X line, the clean fleet run as the outer gate (218/218), the census rerun, the phase-log entry and the checkpoint headline, one commit per batch; stops on the first red with the files in place for --restore the batch size, the aliases, the fleet count LIVE
285 tools/share_census.py P10 ADAPT measure duplicate function bodies across the fleet and assert one source per unique function The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test repo paths, the registry and signature schemas LIVE
286 tools/shared_lock.py P8 ADAPT hold one reader/writer lock over the shared source state for concurrent lanes One reader/writer lock over the shared source state for concurrent lanes repo lock paths LIVE
287 tools/sig_image.py P6 PORTABLE sign every function of a flat image at a known base, without a disassembler Disassembler-free per-function signer for a flat image at a known base, field-identical to the dumper MIPS encodings, repo scratch paths LIVE
decomp-architect/corpus/tools/INDEX.md
+4 -3
View File
@@ -8,8 +8,8 @@
> its platform SDK need the marked adaptation. The last table lists the tools that are project-only in code (their *shape* is a task; > its platform SDK need the marked adaptation. The last table lists the tools that are project-only in code (their *shape* is a task;
> their code does not transfer). *TODO(platform): the MIPS and PlayStation SDK hard-codes are the ones another platform replaces first.* > their code does not transfer). *TODO(platform): the MIPS and PlayStation SDK hard-codes are the ones another platform replaces first.*
> >
> **Coverage:** 298 tool files in scope (submodules, vendored and downloaded code excluded), of which 298 live rows > **Coverage:** 299 tool files in scope (submodules, vendored and downloaded code excluded), of which 299 live rows
> below; per phase: P1 2 · P2 26 · P3 17 · P4 9 · P5 26 · P6 51 · P7 20 · P8 86 · P9 27 · P10 18 · PROJECT-ONLY 16. Superseded tools appear only as pointers to their successor (30 pointer rows); one-offs are omitted. Table rows in all: 328 (the installer checks its copy against this figure). > below; per phase: P1 2 · P2 26 · P3 17 · P4 9 · P5 26 · P6 51 · P7 20 · P8 86 · P9 27 · P10 19 · PROJECT-ONLY 16. Superseded tools appear only as pointers to their successor (30 pointer rows); one-offs are omitted. Table rows in all: 329 (the installer checks its copy against this figure).
## P1 — extraction + manifest ## P1 — extraction + manifest
@@ -338,9 +338,10 @@
| `ghidra_apply_symbols.sh` | mirror the curated symbol file into the analysis database with a real save | Mirrors the curated symbol file into the analysis program headlessly, with a real save | repo symbol path, project name | | `ghidra_apply_symbols.sh` | mirror the curated symbol file into the analysis database with a real save | Mirrors the curated symbol file into the analysis program headlessly, with a real save | repo symbol path, project name |
| `frozen.py` | refuse, from one place, the command line of a tool the project has frozen | The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) | nothing | | `frozen.py` | refuse, from one place, the command line of a tool the project has frozen | The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) | nothing |
| `verbatim_target_s.py` | regenerate a splitter-format target disassembly for a function no longer stubbed | Regenerates a splitter-format target disassembly for a function that is no longer a stub; --gas emits the assemblable gas-syntax form ($-registers, .L labels, noreorder) that a web diff service accepts as a pasted target | repo build/asm layout | | `verbatim_target_s.py` | regenerate a splitter-format target disassembly for a function no longer stubbed | Regenerates a splitter-format target disassembly for a function that is no longer a stub; --gas emits the assemblable gas-syntax form ($-registers, .L labels, noreorder) that a web diff service accepts as a pasted target | repo build/asm layout |
| `delever_cycle.sh` | run the de-lever batch cycle unattended: batch, verify by exit code, clean fleet gate, census, log, commit | The Phase-36 T3/T4 driver: per batch a clean-tree check, the oracle recalibrated when stale, delever --apply read by exit code and its final X/X line, the clean fleet run as the outer gate (218/218), the census rerun, the phase-log entry and the checkpoint headline, one commit per batch; stops on the first red with the files in place for --restore | the batch size, the aliases, the fleet count |
| `share_body_cycle.sh` | run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check | The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new<k>, the gated N/N line read, the phase log entry appended, the bank committed the moment it is green, the clean fleet run every N batches — stops on the first red | repo paths, the phase-log format, the fleet count from config/check.*.sha | | `share_body_cycle.sh` | run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check | The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new<k>, the gated N/N line read, the phase log entry appended, the bank committed the moment it is green, the clean fleet run every N batches — stops on the first red | repo paths, the phase-log format, the fleet count from config/check.*.sha |
| `share_body.py` | share one byte-identical function class across its binaries through an include-at-site header, gated per binary | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | repo paths, the registry and signature schemas | | `share_body.py` | share one byte-identical function class across its binaries through an include-at-site header, gated per binary | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | repo paths, the registry and signature schemas |
| `delever.py` | take register pins, asm statements, volatile and register levers out of matched C, byte-gated per site | The Phase-36 de-lever engine: positional rewrites per lever class (pin -> plain declaration, barrier/launder/keep-alive deleted, a hand-placed instruction -> its C, the zero-register variable -> 0, volatile/register dropped), rung A strip-all and rung B greedy per body through delever_oracle, headers serialized after the TU phase; --probe runs a stratified sample of distinct bodies and prices the campaign with denominators | the census site records, the oracle | | `delever.py` | take register pins, asm statements, volatile and register levers out of matched C, byte-gated per site | The Phase-36 de-lever engine and campaign tool: positional rewrites per lever class (pin -> plain declaration, barrier/keep-alive deleted, a launder deleted or turned into the assignment it is, a hand-placed instruction -> its C, the zero-register variable -> 0, a macro-carried site deleted/valued/refused by its macro's shape, volatile/register dropped); per body replay -> rung A strip-all -> rung B greedy through delever_oracle; the file is the write unit and its final compile through every recipe the proof; --plan/--apply batches (TUs parallel, exemplar files first; headers serial, includers parallel), the ledger keyed by tu+fn+addr with the body's nhash before/after, !FAKE markers on class A/B survivors, --restore from inflight.json, --apply-body for T6/T7, --selftest with a stub oracle, --probe (T2) | the census site records, the oracle |
| `verbatim_to_stub.py` | turn an inline-assembly body back into a stub the toolchain can reach | Turns an inline-assembly body back into an include-assembly stub | repo src/asm layout | | `verbatim_to_stub.py` | turn an inline-assembly body back into a stub the toolchain can reach | Turns an inline-assembly body back into an include-assembly stub | repo src/asm layout |
## PROJECT-ONLY — project-only in code (the shape is a task; the code does not transfer) ## PROJECT-ONLY — project-only in code (the shape is a task; the code does not transfer)
decomp-architect/corpus/tools/P10/delever.py
+1102 -149
View File
File diff suppressed because it is too large Load Diff
decomp-architect/corpus/tools/P10/delever_cycle.sh
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
# delever_cycle.sh — the Phase-36 batch cycle, mechanised (T3/T4). For each batch k in [START..END]:
# 1. the tree must be clean (src/ + config/); the oracle's calibration is refreshed (2–3 s: it is keyed to HEAD, every commit stales it);
# 2. `delever.py --apply --batch N --label <mode>k` judges every drawable body of the next N files and leaves the winning texts in place
# (the census is regenerated by the tool when its src stamp is stale); the tool's exit code and its `final X/X identical` are read (R97/R66);
# 3. the outer gate: the CLEAN fleet run (make clean && extract-all && check-all -> `218 passed, 0 failed of 218`, R22) — it also refreshes
# build/, the oracle's baseline objects;
# 4. the census is rerun on the new tree (the commit line quotes it), the phase log gets the batch's entry, the 🛑 headline is refreshed (R101),
# and src/ + the ledger/batch record/log + the census + the phase log are committed (R42).
# Stops on the first red (a non-zero tool exit, a final N/M with N != M, a fleet run that is not 218/218) — the batch's files are then
# still in place: inspect, then `tools/delever.py --restore` (never git checkout, R102). Stops cleanly when nothing is drawable.
# Usage: [LABEL_PREFIX=t3_] [TASK=T4] tools/delever_cycle.sh START END [BATCH=300] [MODE=tus|headers] [ONLY="alias1 alias2 …"]
# size a call so one batch stays < 10 min: TUs ≈ 8 s each single-threaded / 12 workers; a header ≈ 4 s (serial, includers in parallel).
set -o pipefail
cd "$(dirname "$0")/.." || exit 2
START=$1; END=$2; BATCH=${3:-300}; MODE=${4:-tus}; ONLY=${5:-}
[ -n "$START" ] && [ -n "$END" ] || { echo "usage: $0 START END [BATCH] [tus|headers] [ONLY]"; exit 2; }
case "$MODE" in tus) HFLAG="";; headers) HFLAG="--headers";; *) echo "MODE must be tus or headers"; exit 2;; esac
ONLYFLAG=""; [ -n "$ONLY" ] && ONLYFLAG="--only $ONLY"
FLEET=$(ls config/check.*.sha | wc -l)
PY=.venv/bin/python
MARKER='## Approved plan (verbatim, gate 1 — 2026-09-09)'
for k in $(seq "$START" "$END"); do
label="${LABEL_PREFIX:-}${MODE}${k}"
[ -z "$(git status --short -- src config)" ] || { echo "cycle: src/ or config/ is dirty — commit, or tools/delever.py --restore (R42/R102)"; exit 3; }
[ -z "$(git status --short --porcelain src | grep '^??')" ] || { echo "cycle: untracked files under src/ (a dotfile probe?) — sweep them first"; exit 3; }
$PY tools/delever_oracle.py --status >/dev/null 2>&1 || $PY tools/delever_oracle.py --calibrate ov_SC04_011 ov_SC03_015 ov_SC03_014 main -j 16 > .run/P36/delever/calibrate_${label}.log 2>&1 \
|| { echo "cycle: calibration FAILED"; tail -5 .run/P36/delever/calibrate_${label}.log; exit 1; }
log=.run/P36/delever/run_${label}.log
{ /usr/bin/time -f "$label wall=%e s" $PY tools/delever.py --apply --batch "$BATCH" --label "$label" $HFLAG $ONLYFLAG -j 12; echo "exit=$?"; } > "$log" 2>&1
grep -q '^exit=0$' "$log" || { echo "cycle: batch $label — delever exited non-zero"; tail -4 "$log"; exit 1; }
if grep -q 'nothing to do (no drawable file)' "$log"; then echo "cycle: nothing drawable at $label — done"; exit 0; fi
vline=$(grep -oE '^delever: batch .*' "$log" | tail -1)
final=$(echo "$vline" | grep -oE 'final [0-9]+/[0-9]+' | grep -oE '[0-9]+/[0-9]+'); [ "${final%/*}" = "${final#*/}" ] || { echo "cycle: batch $label — $vline"; exit 1; }
if [ "$(echo "$vline" | grep -oE 'written [0-9]+' | grep -oE '[0-9]+')" = "0" ]; then echo "cycle: batch $label wrote no file — $vline"; fi
r22=.run/P36/baseline/r22_${label}.log
{ /usr/bin/time -f "wall=%e s user=%U s sys=%S s" bash -c 'set -o pipefail; make clean && make extract-all JOBS=16 && make check-all JOBS=16'; echo "exit=$?"; } > "$r22" 2>&1
line=$(grep -oE "check-all: [0-9]+ passed, [0-9]+ failed of [0-9]+" "$r22")
if ! grep -q '^exit=0$' "$r22" || [ "$line" != "check-all: $FLEET passed, 0 failed of $FLEET" ]; then echo "cycle: R22 RED after batch $label — $line (the batch's files are in place: inspect, then tools/delever.py --restore)"; exit 1; fi
$PY tools/lever_census.py --sites -j 16 > .run/P36/census/census_${label}.log 2>&1 || { echo "cycle: the census FAILED after batch $label"; tail -3 .run/P36/census/census_${label}.log; exit 1; }
cline=$(grep -oE 'levers A\+B.*' .run/P36/census/lever_census.txt | head -1)
[ -n "$cline" ] || cline=$(grep -oE '^lever_census:.*' .run/P36/census/census_${label}.log | head -1)
TASK=${TASK:-T4} $PY - "$label" "$vline" "$line" "$cline" "$(grep -oE 'wall=[0-9.]+ s' "$r22" | head -1)" "$MARKER" <<'EOF' || exit 1
import os, pathlib, re, sys, time, subprocess
label, vline, r22line, cline, wall, marker = sys.argv[1:7]
task = os.environ.get("TASK", "T4")
p = pathlib.Path("phase-ends/CURRENT_PHASE.md"); t = p.read_text()
head = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()
entry = (f"\n- **{time.strftime('%Y-%m-%d')} — {task} batch `{label}`** (`.run/P36/delever/run_{label}.log`, `batch_{label}.json`): `{vline}` → R22 "
f"(`.run/P36/baseline/r22_{label}.log`) **`{r22line}`** (`{wall}`) → census `{cline}`.")
assert ("\n" + marker) in t, "phase log marker not found"
t = t.replace("\n" + marker, entry + "\n\n" + marker, 1)
# the 🛑 headline (R101): every task-advancing commit refreshes it
lines = t.split("\n")
for i, l in enumerate(lines):
if l.startswith("## 🛑 SESSION CHECKPOINT"):
base = re.sub(r"\s*\|\s*last batch.*$", "", l)
lines[i] = f"{base} | last batch `{label}` on {head}: {cline}"
break
p.write_text("\n".join(lines))
print("log:", label, cline)
EOF
git add -A src && git add .run/P36/delever .run/P36/census .run/P36/baseline phase-ends/CURRENT_PHASE.md || exit 1
git commit -q -m "src(phase-36): ${TASK:-T4} batch $label — $vline | R22 $line | census: $cline (delever_cycle)" || exit 1
echo "cycle: batch $label committed $(git log --oneline -1 | cut -c1-9) — $vline | $line"
done
echo "cycle: batches $START..$END done"
decomp-architect/corpus/tools/P10/delever_oracle.py
+34 -2
View File
@@ -42,6 +42,8 @@ RECIPES = RUN / "recipes.json"
CALIB = RUN / "calibration.json" CALIB = RUN / "calibration.json"
SCRATCH = RUN / "obj" SCRATCH = RUN / "obj"
RECIPE_RE = re.compile(r"^set -o pipefail; (.*) -o (build/\S+\.o)$") RECIPE_RE = re.compile(r"^set -o pipefail; (.*) -o (build/\S+\.o)$")
SIGNAL_LINE = re.compile(r"^\s*\d+\s+(Aborted|Segmentation fault|Illegal instruction|Floating point exception|Bus error|Killed)[^\n]*", re.M)
JOB_STATUS_LINE = re.compile(r"^(bash: line \d+: )?\s*\d+\s+(Done|Exit \d+)\b")
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
@@ -156,11 +158,17 @@ def compile_obj(recipe, text=None, tag="x", write_path=None):
r = subprocess.run(["bash", "-c", cmd], cwd=REPO, capture_output=True, text=True) r = subprocess.run(["bash", "-c", cmd], cwd=REPO, capture_output=True, text=True)
dt = time.time() - t0 dt = time.time() - t0
if r.returncode != 0 or not out.exists(): if r.returncode != 0 or not out.exists():
# a pipeline member killed by a signal (cc1 2.7.2 aborts on some candidates — e.g. a variable whose only definition was a
# launder that got deleted): bash prints the job-status block ("Done" / "Aborted (core dumped)" / "Segmentation fault")
# and the pipeline's rc is >= 128 — that is a CRASH verdict, not a diagnostic (R103: the real message forms)
sig = SIGNAL_LINE.search(r.stderr)
if sig or r.returncode >= 128:
return None, dt, "CRASH: " + (sig.group(0).strip() if sig else f"rc={r.returncode}")
# gcc 2.7.2 prints errors without the word "error" and the assembler floods stderr with `$at` warnings: keep every # gcc 2.7.2 prints errors without the word "error" and the assembler floods stderr with `$at` warnings: keep every
# non-warning line (R103), then the tail # non-warning line (R103), then the tail
errs = [ln for ln in r.stderr.splitlines() errs = [ln for ln in r.stderr.splitlines()
if ln.strip() and not re.search(r"\bwarning:", ln, re.I) and not ln.startswith(("In file included", " " * 16)) if ln.strip() and not re.search(r"\bwarning:", ln, re.I) and not ln.startswith(("In file included", " " * 16))
and ": In function" not in ln] and ": In function" not in ln and not JOB_STATUS_LINE.match(ln)]
return None, dt, ("\n".join(errs[:8]) or r.stderr[-400:]) return None, dt, ("\n".join(errs[:8]) or r.stderr[-400:])
data = out.read_bytes() data = out.read_bytes()
try: try:
@@ -183,10 +191,32 @@ def judge(recipe, text, tag="x", write_path=None):
return "NO-BASELINE", 0.0, "" return "NO-BASELINE", 0.0, ""
data, dt, err = compile_obj(recipe, text, tag, write_path) data, dt, err = compile_obj(recipe, text, tag, write_path)
if data is None: if data is None:
return "COMPILE-ERROR", dt, err return ("COMPILE-CRASH" if err.startswith("CRASH:") else "COMPILE-ERROR"), dt, err
return ("IDENTICAL" if data == base else "DIFFERS"), dt, "" return ("IDENTICAL" if data == base else "DIFFERS"), dt, ""
def recipes_by_src(recipes):
"""src rel -> [recipe] (a twin binary compiles the primary's source into its own object: two recipes, one source)."""
out = {}
for r in recipes.values():
out.setdefault(r["src"], []).append(r)
for v in out.values():
v.sort(key=lambda r: r["obj"])
return out
def judge_all(recipes, text, tag="x", write_path=None):
"""Every recipe of one written file judged in turn (the file is written once, by the first compile; the CALLER restores it).
Returns (verdict, seconds, err): IDENTICAL only if every object is identical; otherwise the first non-identical verdict."""
total, first = 0.0, None
for i, r in enumerate(recipes):
v, dt, err = judge(r, text if i == 0 else None, tag=tag, write_path=write_path)
total += dt
if v != "IDENTICAL" and first is None:
first = (v, err)
return (first[0] if first else "IDENTICAL"), total, (first[1] if first else "")
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
# calibration # calibration
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
@@ -222,6 +252,7 @@ def calibrate(aliases, jobs, recipes):
import share_census as sc import share_census as sc
ctl = recs[0] ctl = recs[0]
src = REPO / ctl["src"] src = REPO / ctl["src"]
st0 = src.stat()
orig = src.read_text(errors="surrogateescape") orig = src.read_text(errors="surrogateescape")
pos_verdict = "SKIPPED" pos_verdict = "SKIPPED"
defs = [r for r in sc.scan_text(orig, ctl["src"], shared_defs=None) if r["form"] == "def" and r.get("nlines", 0) >= 3 and not r.get("empty")] defs = [r for r in sc.scan_text(orig, ctl["src"], shared_defs=None) if r["form"] == "def" and r.get("nlines", 0) >= 3 and not r.get("empty")]
@@ -239,6 +270,7 @@ def calibrate(aliases, jobs, recipes):
pos_verdict = pos_verdict if pos_verdict != "COMPILE-ERROR" else f"COMPILE-ERROR ({err[-100:]})" pos_verdict = pos_verdict if pos_verdict != "COMPILE-ERROR" else f"COMPILE-ERROR ({err[-100:]})"
finally: finally:
src.write_text(orig, errors="surrogateescape") src.write_text(orig, errors="surrogateescape")
os.utime(src, ns=(st0.st_atime_ns, st0.st_mtime_ns)) # the census's src stamp is stat-based: a restore is not a change
by_alias = {} by_alias = {}
for x in results: for x in results:
d = by_alias.setdefault(x["alias"], dict(objects=0, identical=0, seconds=0.0)) d = by_alias.setdefault(x["alias"], dict(objects=0, identical=0, seconds=0.0))
decomp-architect/corpus/tools/P10/lever_census.py
+17
View File
@@ -598,6 +598,22 @@ def _walk_worker(args):
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
# the fleet # the fleet
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
def src_stamp():
"""sha1 over (path, size, mtime_ns) of every .c/.h under src/: the tree the census describes. delever refuses a census whose
stamp is not the working tree's (R43) — a restore-write that changes only an mtime just costs a census rerun (35 s)."""
h = hashlib.sha1()
for p in sorted((REPO / "src").rglob("*")):
if p.suffix in (".c", ".h") and not p.name.startswith("."):
st = p.stat()
h.update(f"{p.relative_to(REPO).as_posix()}|{st.st_size}|{st.st_mtime_ns}\n".encode())
return h.hexdigest()[:16]
def git_head():
import subprocess
return subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=REPO, capture_output=True, text=True).stdout.strip()
def enumerate_files(aliases, dirs): def enumerate_files(aliases, dirs):
"""TU -> [aliases] (a twin's TUs are its primary's, listed for both), plus every header under src/ (aliases = its includers').""" """TU -> [aliases] (a twin's TUs are its primary's, listed for both), plus every header under src/ (aliases = its includers')."""
import compile_only import compile_only
@@ -756,6 +772,7 @@ def run_census(jobs, use_cache=True, out_dir=OUT_DIR_DEFAULT, want_sites=False,
macro_texts[m["name"]].add(m["text_hash"]) macro_texts[m["name"]].add(m["text_hash"])
summary = dict( summary = dict(
generated=time.strftime("%Y-%m-%d"), binaries=len(aliases), tus=len(tu_aliases), headers=len(headers), generated=time.strftime("%Y-%m-%d"), binaries=len(aliases), tus=len(tu_aliases), headers=len(headers),
head=git_head(), src_stamp=src_stamp(),
coverage=coverage, coverage_ok=cov_ok, unclassified=len(unclassified), coverage=coverage, coverage_ok=cov_ok, unclassified=len(unclassified),
verbatim_excluded=dict(sites=verbatim_sites, functions=len(verbatim_fns), manifest_rows=len(verb)), verbatim_excluded=dict(sites=verbatim_sites, functions=len(verbatim_fns), manifest_rows=len(verb)),
classes=classes, classes=classes,
decomp-architect/tools/MANIFEST.md
+4 -3
View File
@@ -8,8 +8,8 @@
> its platform SDK need the marked adaptation. The last table lists the tools that are project-only in code (their *shape* is a task; > its platform SDK need the marked adaptation. The last table lists the tools that are project-only in code (their *shape* is a task;
> their code does not transfer). *TODO(platform): the MIPS and PlayStation SDK hard-codes are the ones another platform replaces first.* > their code does not transfer). *TODO(platform): the MIPS and PlayStation SDK hard-codes are the ones another platform replaces first.*
> >
> **Coverage:** 298 tool files in scope (submodules, vendored and downloaded code excluded), of which 298 live rows > **Coverage:** 299 tool files in scope (submodules, vendored and downloaded code excluded), of which 299 live rows
> below; per phase: P1 2 · P2 26 · P3 17 · P4 9 · P5 26 · P6 51 · P7 20 · P8 86 · P9 27 · P10 18 · PROJECT-ONLY 16. Superseded tools appear only as pointers to their successor (30 pointer rows); one-offs are omitted. Table rows in all: 328 (the installer checks its copy against this figure). > below; per phase: P1 2 · P2 26 · P3 17 · P4 9 · P5 26 · P6 51 · P7 20 · P8 86 · P9 27 · P10 19 · PROJECT-ONLY 16. Superseded tools appear only as pointers to their successor (30 pointer rows); one-offs are omitted. Table rows in all: 329 (the installer checks its copy against this figure).
## P1 — extraction + manifest ## P1 — extraction + manifest
@@ -338,9 +338,10 @@
| `ghidra_apply_symbols.sh` | mirror the curated symbol file into the analysis database with a real save | Mirrors the curated symbol file into the analysis program headlessly, with a real save | repo symbol path, project name | | `ghidra_apply_symbols.sh` | mirror the curated symbol file into the analysis database with a real save | Mirrors the curated symbol file into the analysis program headlessly, with a real save | repo symbol path, project name |
| `frozen.py` | refuse, from one place, the command line of a tool the project has frozen | The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) | nothing | | `frozen.py` | refuse, from one place, the command line of a tool the project has frozen | The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) | nothing |
| `verbatim_target_s.py` | regenerate a splitter-format target disassembly for a function no longer stubbed | Regenerates a splitter-format target disassembly for a function that is no longer a stub; --gas emits the assemblable gas-syntax form ($-registers, .L labels, noreorder) that a web diff service accepts as a pasted target | repo build/asm layout | | `verbatim_target_s.py` | regenerate a splitter-format target disassembly for a function no longer stubbed | Regenerates a splitter-format target disassembly for a function that is no longer a stub; --gas emits the assemblable gas-syntax form ($-registers, .L labels, noreorder) that a web diff service accepts as a pasted target | repo build/asm layout |
| `delever_cycle.sh` | run the de-lever batch cycle unattended: batch, verify by exit code, clean fleet gate, census, log, commit | The Phase-36 T3/T4 driver: per batch a clean-tree check, the oracle recalibrated when stale, delever --apply read by exit code and its final X/X line, the clean fleet run as the outer gate (218/218), the census rerun, the phase-log entry and the checkpoint headline, one commit per batch; stops on the first red with the files in place for --restore | the batch size, the aliases, the fleet count |
| `share_body_cycle.sh` | run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check | The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new<k>, the gated N/N line read, the phase log entry appended, the bank committed the moment it is green, the clean fleet run every N batches — stops on the first red | repo paths, the phase-log format, the fleet count from config/check.*.sha | | `share_body_cycle.sh` | run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check | The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new<k>, the gated N/N line read, the phase log entry appended, the bank committed the moment it is green, the clean fleet run every N batches — stops on the first red | repo paths, the phase-log format, the fleet count from config/check.*.sha |
| `share_body.py` | share one byte-identical function class across its binaries through an include-at-site header, gated per binary | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | repo paths, the registry and signature schemas | | `share_body.py` | share one byte-identical function class across its binaries through an include-at-site header, gated per binary | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | repo paths, the registry and signature schemas |
| `delever.py` | take register pins, asm statements, volatile and register levers out of matched C, byte-gated per site | The Phase-36 de-lever engine: positional rewrites per lever class (pin -> plain declaration, barrier/launder/keep-alive deleted, a hand-placed instruction -> its C, the zero-register variable -> 0, volatile/register dropped), rung A strip-all and rung B greedy per body through delever_oracle, headers serialized after the TU phase; --probe runs a stratified sample of distinct bodies and prices the campaign with denominators | the census site records, the oracle | | `delever.py` | take register pins, asm statements, volatile and register levers out of matched C, byte-gated per site | The Phase-36 de-lever engine and campaign tool: positional rewrites per lever class (pin -> plain declaration, barrier/keep-alive deleted, a launder deleted or turned into the assignment it is, a hand-placed instruction -> its C, the zero-register variable -> 0, a macro-carried site deleted/valued/refused by its macro's shape, volatile/register dropped); per body replay -> rung A strip-all -> rung B greedy through delever_oracle; the file is the write unit and its final compile through every recipe the proof; --plan/--apply batches (TUs parallel, exemplar files first; headers serial, includers parallel), the ledger keyed by tu+fn+addr with the body's nhash before/after, !FAKE markers on class A/B survivors, --restore from inflight.json, --apply-body for T6/T7, --selftest with a stub oracle, --probe (T2) | the census site records, the oracle |
| `verbatim_to_stub.py` | turn an inline-assembly body back into a stub the toolchain can reach | Turns an inline-assembly body back into an include-assembly stub | repo src/asm layout | | `verbatim_to_stub.py` | turn an inline-assembly body back into a stub the toolchain can reach | Turns an inline-assembly body back into an include-assembly stub | repo src/asm layout |
## PROJECT-ONLY — project-only in code (the shape is a task; the code does not transfer) ## PROJECT-ONLY — project-only in code (the shape is a task; the code does not transfer)
docs/SETUP.md
+2 -1
View File
@@ -784,7 +784,8 @@ Every script under `tools/` (plus the two report make-targets), grouped by purpo
| | `tools/share_census.py` | **(P35 T1)** The fleet-wide census of byte-identical function classes and the S1 "one source per unique function" checker (`--check`, `--selftest`, `--scope`, `--strict-macros`, `--strict-text`); its ledger is `config/dedup_exceptions.tsv`; details in the P35 T1 section below. | | | `tools/share_census.py` | **(P35 T1)** The fleet-wide census of byte-identical function classes and the S1 "one source per unique function" checker (`--check`, `--selftest`, `--scope`, `--strict-macros`, `--strict-text`); its ledger is `config/dedup_exceptions.tsv`; details in the P35 T1 section below. |
| | `tools/lever_census.py` | **(P36 T1)** The census of every compiler-forcing construct ("lever") in the fleet's C — register pins, asm statements by kind (barrier / launder / keepalive / instruction / GTE / verbatim-body), volatile levers, bare `register`, plus the deferred asm-label aliases, builtins and attributes — derived from `share_census`'s scanner with a per-token coverage assertion against the raw text, four known-true controls, the `// !FAKE:` marker split, `--check` (0 unmarked pins/asm) and `--check --strict` (0 pins, 0 asm outside the GTE header) gates, `--sites` (every site to `.run/P36/census/lever_sites.jsonl`, the delever ledger's input), `--selftest`. Evidence: `.run/P36/census/lever_census.{json,txt}`; `progress.py` publishes the `levers` block and a README sentence from it. | | | `tools/lever_census.py` | **(P36 T1)** The census of every compiler-forcing construct ("lever") in the fleet's C — register pins, asm statements by kind (barrier / launder / keepalive / instruction / GTE / verbatim-body), volatile levers, bare `register`, plus the deferred asm-label aliases, builtins and attributes — derived from `share_census`'s scanner with a per-token coverage assertion against the raw text, four known-true controls, the `// !FAKE:` marker split, `--check` (0 unmarked pins/asm) and `--check --strict` (0 pins, 0 asm outside the GTE header) gates, `--sites` (every site to `.run/P36/census/lever_sites.jsonl`, the delever ledger's input), `--selftest`. Evidence: `.run/P36/census/lever_census.{json,txt}`; `progress.py` publishes the `levers` block and a README sentence from it. |
| | `tools/delever_oracle.py` | **(P36 T2)** The fast byte oracle for a single-translation-unit edit: `--recipes` captures every object's exact build command through `make -n -W <src> <obj> BINARY=<alias>` (the Makefile's own pipeline — the jump-table pad stage, the per-object `-O0` overrides, the twin rule — into `.run/P36/delever/recipes.json`, regenerable in ~15 s); a candidate is compiled IN PLACE with `-o`/`-MF` redirected to scratch and its bytes compared with `build/`'s object from the fleet run (build/ is never written); `--calibrate <aliases>` proves 100 % equality untouched + twin == primary + a positive control (a nop injected at the end of a real body → DIFFERS) and writes `calibration.json` (HEAD, config stamp, per-object seconds); `--status`. Measured at T2: main 0.08–0.77 s per object, overlays ~0.14 s. | | | `tools/delever_oracle.py` | **(P36 T2)** The fast byte oracle for a single-translation-unit edit: `--recipes` captures every object's exact build command through `make -n -W <src> <obj> BINARY=<alias>` (the Makefile's own pipeline — the jump-table pad stage, the per-object `-O0` overrides, the twin rule — into `.run/P36/delever/recipes.json`, regenerable in ~15 s); a candidate is compiled IN PLACE with `-o`/`-MF` redirected to scratch and its bytes compared with `build/`'s object from the fleet run (build/ is never written); `--calibrate <aliases>` proves 100 % equality untouched + twin == primary + a positive control (a nop injected at the end of a real body → DIFFERS) and writes `calibration.json` (HEAD, config stamp, per-object seconds); `--status`. Measured at T2: main 0.08–0.77 s per object, overlays ~0.14 s. |
| | `tools/delever.py` | **(P36 T2/T3)** The de-lever engine. Rewrites per lever class on the raw text at the census's positions (a token mismatch REFUSES, never guesses): pin → plain declaration (type/qualifiers/initializer kept; the `$0` zero-register variable's uses → 0), barrier / launder / keep-alive → deleted (via-macro `SHB(x)` too), a hand-placed instruction → its C (addu-$zero/move/la/lh/addiu/sll/srl/and/andi/lui/li/lw and the lui+addiu / lui+ori pairs), volatile / register → dropped; rung A strip-all then rung B greedy through `delever_oracle`, the file restored from the in-memory snapshot after every compile; TUs in parallel (one worker per file), shared headers SERIAL after the TU phase (a header edit reaches every includer). `--probe --sample N --seed S` runs the ladder on a stratified sample of DISTINCT bodies → `.run/P36/probe/{probe_results.jsonl,probe_summary.json,probe_table.md}` with the pricing (R41). | | | `tools/delever.py` | **(P36 T2/T3)** The de-lever engine AND the campaign tool. Rewrites per lever class on the raw text at the census's positions (a token mismatch REFUSES, never guesses): pin → plain declaration (type/qualifiers/initializer kept; the `$0` zero-register variable's uses → 0, refused if it is ever assigned), barrier / keep-alive → deleted, launder → deleted when it launders a value into itself, **an ASSIGNMENT `out = in;` when its output and input differ** (deleting those made cc1 2.7.2 abort — the T2 probe counted them NEEDED), a hand-placed instruction → its C (addu-$zero/move/la/lh/lw/addiu/sll/srl/and/andi/lui/li and the lui+addiu / lui+ori pairs), a macro-carried site → deleted for a pure launder statement macro (`SHB`), its value `((T)(p))` for a launder statement-expression, REFUSED for a compound macro (XFER/DRAW/RTP_SND: the lever is in the `#define`, T5), volatile / register → dropped. Per body: replay (a ledger exemplar with the same normalized text: 1 compile) → rung A strip-all → rung B greedy, through `delever_oracle` (verdicts IDENTICAL / DIFFERS / COMPILE-ERROR / COMPILE-CRASH). **The file is the write unit and its final compile the proof**: every body's accepted edits + the file-scope volatile edits + the `// !FAKE:` markers spliced once, compiled through every recipe of the file (a twin's object; every includer of a header, in parallel) — IDENTICAL or the file-scope edits are dropped, or the file is restored and REFUSED (COMBINATION-FAILED). `--plan` / `--apply --batch N --label L [--headers] [--only …] [-j 12]` (TUs in parallel, exemplar files before their copies; headers serial; the tree must be clean, the calibration current, the census's src stamp the tree's — it reruns the census itself), `--restore` (from `inflight.json`, the only restore — R102), `--status`, `--apply-body TU FN FILE --label L [--rung E]` (T6/T7: a reshaped body judged on the bytes, refused if a class A/B lever remains), `--selftest` (the fixture through the census's own parser: every rewrite, the ladder against a stub oracle with a crashing site, the markers, a replay and a disagreeing replay, the ledger index, the oracle's crash forms), `--probe` (T2). The ledger `.run/P36/delever/ledger.jsonl`: one row per (tu, fn, addr, aliases) per judgement with every site's verdict REMOVED / REWRITTEN / NEEDED(oracle word) / REFUSED(why) / DEFERRED, the rung, the calibration id, the body's nhash BEFORE and AFTER — "done" = the current nhash is some row's after-hash; a body with a row's before-hash replays it. Class C/D survivors are never marked (decision 3); class A/B survivors carry `// !FAKE: <kind> <detail> — <verdict> (P36 <rung> <label>)` at the end of their line. Evidence: `apply_<label>.log`, `batch_<label>.json`. |
| | `tools/delever_cycle.sh` | **(P36 T3)** The unattended batch cycle: `[LABEL_PREFIX=…] [TASK=T4] tools/delever_cycle.sh START END [BATCH=300] [tus|headers] [ONLY="alias …"]` — per batch: the tree must be clean (a dotfile under `src/` refuses), the oracle recalibrated when stale (2–3 s; every commit stales it), `delever --apply` (exit code + its `final X/X identical`, R97/R66), the CLEAN fleet run as the outer gate (`218 passed, 0 failed of 218`, R22 — it also refreshes `build/`, the oracle's baselines), the census rerun on the new tree, the phase-log entry + the 🛑 headline (R101), one commit of `src/` + the ledger/batch record/logs + the census + the log (R42). Stops on the first red with the batch's files in place (inspect, then `delever --restore`); stops cleanly when nothing is drawable. Size a call so a batch stays under ~10 min. |
| | `tools/kit_coverage.py` | **(P33.5 task 14.5)** The kit's DISTILLATION coverage: derives the rule population (every `- **R<n>` of DIGEST §3, asserted contiguous) and the hindsight population (every `## ` heading of `docs/accelerators.md` at numbered-item granularity — 58 at S92) and asserts each is cited by a `provenance:` line of the registry seed / the kernels OR dispositioned in `config/kit_coverage_map.tsv` (`G<id>` / `DK-<n>` / `FOLDED:G<id>` / `ENV` / `PA` / `SEED:<file>` / `KIT:<path>` / `RECORD` / `COOKBOOK` / `NOT-PORTABLE`; unknown ids refused, R43); counts with denominators (R41); rc 1 on any gap. In `tools-health` after `tool_census --check`. Its first run found 26 uncited rules and 21 uncited entries → three new kernels (DK-66–DK-68) and 41 authored dispositions. | | | `tools/kit_coverage.py` | **(P33.5 task 14.5)** The kit's DISTILLATION coverage: derives the rule population (every `- **R<n>` of DIGEST §3, asserted contiguous) and the hindsight population (every `## ` heading of `docs/accelerators.md` at numbered-item granularity — 58 at S92) and asserts each is cited by a `provenance:` line of the registry seed / the kernels OR dispositioned in `config/kit_coverage_map.tsv` (`G<id>` / `DK-<n>` / `FOLDED:G<id>` / `ENV` / `PA` / `SEED:<file>` / `KIT:<path>` / `RECORD` / `COOKBOOK` / `NOT-PORTABLE`; unknown ids refused, R43); counts with denominators (R41); rc 1 on any gap. In `tools-health` after `tool_census --check`. Its first run found 26 uncited rules and 21 uncited entries → three new kernels (DK-66–DK-68) and 41 authored dispositions. |
| | `decomp-architect/` (the day-one decomp kit) | **(P33.5 tasks 9–14)** The package a new matching-decomp project installs as Phase 0.5 on ProjectArchitect 2.0: `README.md` (the three steps), `intake.decomp.md` (ProjectArchitect's twelve items pre-answered + the phase ladder + the six readability inversions), `SETUP.md` (the installer, Step 0 contract … Step 10 verify + hard stop; `answers: <path>` for unattended runs), `decomp-architect.md` (the methodology), `templates/` (the firewall pack — `gitignore.decomp`, `firewall.txt`, `audit_public.template.py`, the planted fixture, `no-rom.template.yml` —, the READMEs, `pa-overlays.md`, `registry-E.decomp.md` G1–G67, the skeletons, `PLACEHOLDERS.md`, `layout-contract.md`), `corpus/decomp-kernels.md` (DK-1 … DK-80), the three dictionaries `corpus/tools/<phase>/` + `corpus/cookbook/` + `corpus/record/` (generated by `make kit-corpus`), `memory-seed/` (18), `tools/MANIFEST.md` (generated). **How it is checked:** `tools/kit_lint.py` (de-specialisation, placeholders, syntax, the gitignore-template diff) + `tools/tool_census.py --check` (the corpora equal their sources) + `tools/gitignore_template_check.py`, all in `tools-health`; the dry-run harness under `.run/P33.5/kit-dryrun/` (`answers.md`, `expected-manifest.txt`, `judge.py`, the install logs and verdicts of runs 1–5 — a kit change is re-verified by a resume on the last throwaway `repo/`, a fresh full run only when SETUP's steps change; the judge compares the real tree's dirty PATH SETS before/after). Wiki page: `docs/wiki/Start-a-new-decomp-project.md`. Split into its own repository after the flip. | | | `decomp-architect/` (the day-one decomp kit) | **(P33.5 tasks 9–14)** The package a new matching-decomp project installs as Phase 0.5 on ProjectArchitect 2.0: `README.md` (the three steps), `intake.decomp.md` (ProjectArchitect's twelve items pre-answered + the phase ladder + the six readability inversions), `SETUP.md` (the installer, Step 0 contract … Step 10 verify + hard stop; `answers: <path>` for unattended runs), `decomp-architect.md` (the methodology), `templates/` (the firewall pack — `gitignore.decomp`, `firewall.txt`, `audit_public.template.py`, the planted fixture, `no-rom.template.yml` —, the READMEs, `pa-overlays.md`, `registry-E.decomp.md` G1–G67, the skeletons, `PLACEHOLDERS.md`, `layout-contract.md`), `corpus/decomp-kernels.md` (DK-1 … DK-80), the three dictionaries `corpus/tools/<phase>/` + `corpus/cookbook/` + `corpus/record/` (generated by `make kit-corpus`), `memory-seed/` (18), `tools/MANIFEST.md` (generated). **How it is checked:** `tools/kit_lint.py` (de-specialisation, placeholders, syntax, the gitignore-template diff) + `tools/tool_census.py --check` (the corpora equal their sources) + `tools/gitignore_template_check.py`, all in `tools-health`; the dry-run harness under `.run/P33.5/kit-dryrun/` (`answers.md`, `expected-manifest.txt`, `judge.py`, the install logs and verdicts of runs 1–5 — a kit change is re-verified by a resume on the last throwaway `repo/`, a fresh full run only when SETUP's steps change; the judge compares the real tree's dirty PATH SETS before/after). Wiki page: `docs/wiki/Start-a-new-decomp-project.md`. Split into its own repository after the flip. |
| **Verification** | `tools/verify_contract.sh` | **(P33 A5/C8) THE recorded contract run**: 00 tree · 01 check-env · 02 family_hseq · 03 `make clean && extract-all && check-all` · 04 sdk-dual (or a recorded SKIP) · 05 tools-health (zero `[warn]`) · 06 audit-frontier · 07 audit-disc · 08 report; one log per step ending `EXIT=<rc>`, abort on the first red (R53), every step asserted by its contract line (R49), `SUMMARY.md` generated → `.run/P33/verify/` (tracked evidence, quoted by `docs/verification.md` §2); step 00 ignores its own output dir. ≈14 min on 32 CPUs. | | **Verification** | `tools/verify_contract.sh` | **(P33 A5/C8) THE recorded contract run**: 00 tree · 01 check-env · 02 family_hseq · 03 `make clean && extract-all && check-all` · 04 sdk-dual (or a recorded SKIP) · 05 tools-health (zero `[warn]`) · 06 audit-frontier · 07 audit-disc · 08 report; one log per step ending `EXIT=<rc>`, abort on the first red (R53), every step asserted by its contract line (R49), `SUMMARY.md` generated → `.run/P33/verify/` (tracked evidence, quoted by `docs/verification.md` §2); step 00 ignores its own output dir. ≈14 min on 32 CPUs. |
docs/tool-index.md
+5 -4
View File
@@ -5,7 +5,7 @@ freshness). The derived columns come from the tree on every run; the authored on
dictionary, whose coverage is asserted both ways. Read it by NEED: find the phrase that matches what you are trying to do, then the tool, dictionary, whose coverage is asserted both ways. Read it by NEED: find the phrase that matches what you are trying to do, then the tool,
then what proved it. The same data generates the day-one kit's manifest and its verbatim tool corpus.* then what proved it. The same data generates the day-one kit's manifest and its verbatim tool corpus.*
**Coverage:** 298 tool files in scope (submodules, vendored and downloaded code excluded; `find` and `git ls-files` agree) + 38 retired under `tools/sunset/`. Classes: LIVE 238 (a runtime consumer), REFERENCED 30 (a SETUP row only), ORPHAN 30 (neither) — of 298. Portability: PORTABLE 23, ADAPT 258, PROJECT-ONLY 17. **Coverage:** 299 tool files in scope (submodules, vendored and downloaded code excluded; `find` and `git ls-files` agree) + 38 retired under `tools/sunset/`. Classes: LIVE 240 (a runtime consumer), REFERENCED 29 (a SETUP row only), ORPHAN 30 (neither) — of 299. Portability: PORTABLE 23, ADAPT 259, PROJECT-ONLY 17.
## P1 — extraction + manifest ## P1 — extraction + manifest
@@ -322,21 +322,22 @@ then what proved it. The same data generates the day-one kit's manifest and its
|---|---|---|---|---|---| |---|---|---|---|---|---|
| add per-site function-pointer casts so a draft can call a differently typed callee | `cast_call_sites.py` | Adds per-site function-pointer casts so a draft can call a differently typed callee | canon_sig_reconcile.py, cast_self_callers.py, family_sweep.py, gate_stage.py (+4) | repo src layout | LIVE | | add per-site function-pointer casts so a draft can call a differently typed callee | `cast_call_sites.py` | Adds per-site function-pointer casts so a draft can call a differently typed callee | canon_sig_reconcile.py, cast_self_callers.py, family_sweep.py, gate_stage.py (+4) | repo src layout | LIVE |
| apply curated names and signatures inside the analysis tool and save | `ghidra_scripts/ApplySymbols.java` | The in-tool half of that mirror: apply curated names and signatures, save on exit | ghidra_apply_symbols.sh, ghidra_export_annotations.sh, ghidra_rebuild.sh, ExportAnnotations.java (+1) | none | LIVE | | apply curated names and signatures inside the analysis tool and save | `ghidra_scripts/ApplySymbols.java` | The in-tool half of that mirror: apply curated names and signatures, save on exit | ghidra_apply_symbols.sh, ghidra_export_annotations.sh, ghidra_rebuild.sh, ExportAnnotations.java (+1) | none | LIVE |
| census every compiler-forcing construct in the C and gate the levers-off phase | `lever_census.py` | The Phase-36 census of lever sites (register pins, asm statements by kind, volatile levers, bare register, asm-label aliases, builtins, attributes; the whole-body hand-asm routines and the GTE ops set apart) derived from the shared-body scanner with a per-token coverage assertion against the raw text, four known-true controls, the !FAKE marker split, --check (0 unmarked pins/asm) and --strict (0 pins, 0 asm) gates, --sites for the delever ledger, a fixture self-test | delever.py, progress.py, verbatim_check.py | repo paths, the shared-body scanner | LIVE | | census every compiler-forcing construct in the C and gate the levers-off phase | `lever_census.py` | The Phase-36 census of lever sites (register pins, asm statements by kind, volatile levers, bare register, asm-label aliases, builtins, attributes; the whole-body hand-asm routines and the GTE ops set apart) derived from the shared-body scanner with a per-token coverage assertion against the raw text, four known-true controls, the !FAKE marker split, --check (0 unmarked pins/asm) and --strict (0 pins, 0 asm) gates, --sites for the delever ledger, a fixture self-test | delever.py, delever_cycle.sh, progress.py, verbatim_check.py | repo paths, the shared-body scanner | LIVE |
| convert a shared-body macro header into per-function plain-C headers included at each site | `macro_to_header.py` | The Phase-35 converter: every DEFINE_ macro body becomes a plain-C header under src/shared/<space>/ included at its site, the legacy name-parameterized headers converted, the registry text-edited, the macro header deleted; --plan / --apply / --finalize / --verify | share_body.py, tool_census.py | the macro form is this project's; a kit-born project shares headers from its first bank | LIVE | | convert a shared-body macro header into per-function plain-C headers included at each site | `macro_to_header.py` | The Phase-35 converter: every DEFINE_ macro body becomes a plain-C header under src/shared/<space>/ included at its site, the legacy name-parameterized headers converted, the registry text-edited, the macro header deleted; --plan / --apply / --finalize / --verify | share_body.py, tool_census.py | the macro form is this project's; a kit-born project shares headers from its first bank | LIVE |
| emit a function's target assembly as an inline-assembly body | `asm_verbatim.py` | Emits the file-scope inline-assembly body form from a disassembly file | docs/wave-playbook.md, draft_prechecks.py, gate_main.py, recover_route.py | repo asm layout | LIVE | | emit a function's target assembly as an inline-assembly body | `asm_verbatim.py` | Emits the file-scope inline-assembly body form from a disassembly file | docs/wave-playbook.md, draft_prechecks.py, gate_main.py, recover_route.py | repo asm layout | LIVE |
| extract inline-defined types and typedefs from a source file into a shared header | `build_engine_types.py` | Extracts inline-defined named types and typedefs from a source file into a shared header | inject_capped_externs.py, lift_types.py, uniquify_type.py | repo shared-header path | LIVE | | extract inline-defined types and typedefs from a source file into a shared header | `build_engine_types.py` | Extracts inline-defined named types and typedefs from a source file into a shared header | inject_capped_externs.py, lift_types.py, uniquify_type.py | repo shared-header path | LIVE |
| flag address-named references whose address now has a curated name | `lint_symbol_refs.py` | Flags address-named references in committed sources whose address now has a curated name | .github/workflows/no-rom.yml, Makefile | repo symbol/src paths | LIVE | | flag address-named references whose address now has a curated name | `lint_symbol_refs.py` | Flags address-named references in committed sources whose address now has a curated name | .github/workflows/no-rom.yml, Makefile | repo symbol/src paths | LIVE |
| give each conflicting camp of a same-named type its own name | `uniquify_type.py` | Gives each conflicting camp of a same-named type its own name so every camp becomes liftable | — | repo src layout | REFERENCED | | give each conflicting camp of a same-named type its own name | `uniquify_type.py` | Gives each conflicting camp of a same-named type its own name so every camp becomes liftable | — | repo src layout | REFERENCED |
| guard that every inline-assembly body still reproduces its target bytes | `verbatim_check.py` | Regression guard that every inline-assembly body still reproduces its target bytes | .github/workflows/no-rom.yml, lever_census.py, progress.py, verbatim_target_s.py (+1) | repo src layout | LIVE | | guard that every inline-assembly body still reproduces its target bytes | `verbatim_check.py` | Regression guard that every inline-assembly body still reproduces its target bytes | .github/workflows/no-rom.yml, lever_census.py, progress.py, verbatim_target_s.py (+1) | repo src layout | LIVE |
| judge one translation-unit edit by the bytes of its object in under a second | `delever_oracle.py` | The Phase-36 fast byte oracle: every object's exact build command captured once via make -n -W (the Makefile's own recipe, pad stage and -O0 overrides included), a candidate compiled in place with -o/-MF redirected to scratch and compared with the fleet run's object; --calibrate proves 100 % equality untouched, twin == primary and a positive control before any verdict is trusted | delever.py | the Makefile's object rules, the twin rule | LIVE | | judge one translation-unit edit by the bytes of its object in under a second | `delever_oracle.py` | The Phase-36 fast byte oracle: every object's exact build command captured once via make -n -W (the Makefile's own recipe, pad stage and -O0 overrides included), a candidate compiled in place with -o/-MF redirected to scratch and compared with the fleet run's object; --calibrate proves 100 % equality untouched, twin == primary and a positive control before any verdict is trusted | delever.py, delever_cycle.sh | the Makefile's object rules, the twin rule | LIVE |
| measure duplicate function bodies across the fleet and assert one source per unique function | `share_census.py` | The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test | Makefile, audit_binaries.py, dedup_integrate.py, delever.py (+8) | repo paths, the registry and signature schemas | LIVE | | measure duplicate function bodies across the fleet and assert one source per unique function | `share_census.py` | The census of byte-identical function classes across every binary with their source forms and verdicts, plus the S1 invariant check with its exception ledger and a fixture self-test | Makefile, audit_binaries.py, dedup_integrate.py, delever.py (+8) | repo paths, the registry and signature schemas | LIVE |
| mirror the curated symbol file into the analysis database with a real save | `ghidra_apply_symbols.sh` | Mirrors the curated symbol file into the analysis program headlessly, with a real save | — | repo symbol path, project name | REFERENCED | | mirror the curated symbol file into the analysis database with a real save | `ghidra_apply_symbols.sh` | Mirrors the curated symbol file into the analysis program headlessly, with a real save | — | repo symbol path, project name | REFERENCED |
| refuse, from one place, the command line of a tool the project has frozen | `frozen.py` | The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) | aprop_autodraft.py, blocker_probe.py, canon_sig_reconcile.py, conform_decls.py (+10) | nothing | LIVE | | refuse, from one place, the command line of a tool the project has frozen | `frozen.py` | The one refusal a FROZEN tool prints from its main(): the tool, the successor and why; imports never exit (libraries stay usable) | aprop_autodraft.py, blocker_probe.py, canon_sig_reconcile.py, conform_decls.py (+10) | nothing | LIVE |
| regenerate a splitter-format target disassembly for a function no longer stubbed | `verbatim_target_s.py` | Regenerates a splitter-format target disassembly for a function that is no longer a stub; --gas emits the assemblable gas-syntax form ($-registers, .L labels, noreorder) that a web diff service accepts as a pasted target | decompme_replica.sh | repo build/asm layout | LIVE | | regenerate a splitter-format target disassembly for a function no longer stubbed | `verbatim_target_s.py` | Regenerates a splitter-format target disassembly for a function that is no longer a stub; --gas emits the assemblable gas-syntax form ($-registers, .L labels, noreorder) that a web diff service accepts as a pasted target | decompme_replica.sh | repo build/asm layout | LIVE |
| run the de-lever batch cycle unattended: batch, verify by exit code, clean fleet gate, census, log, commit | `delever_cycle.sh` | The Phase-36 T3/T4 driver: per batch a clean-tree check, the oracle recalibrated when stale, delever --apply read by exit code and its final X/X line, the clean fleet run as the outer gate (218/218), the census rerun, the phase-log entry and the checkpoint headline, one commit per batch; stops on the first red with the files in place for --restore | delever.py | the batch size, the aliases, the fleet count | LIVE |
| run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check | `share_body_cycle.sh` | The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new<k>, the gated N/N line read, the phase log entry appended, the bank committed the moment it is green, the clean fleet run every N batches — stops on the first red | — | repo paths, the phase-log format, the fleet count from config/check.*.sha | ORPHAN | | run the share-body batch cycle unattended: batch, verify by exit code, log, commit, periodic clean fleet check | `share_body_cycle.sh` | The Phase-35 T5 driver for bucket new: per batch share_body --apply --bucket new --batches 1 --label new<k>, the gated N/N line read, the phase log entry appended, the bank committed the moment it is green, the clean fleet run every N batches — stops on the first red | — | repo paths, the phase-log format, the fleet count from config/check.*.sha | ORPHAN |
| share one byte-identical function class across its binaries through an include-at-site header, gated per binary | `share_body.py` | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | aprop_autodraft.py, audit_binaries.py, auto_driver.py, blocker_probe.py (+18) | repo paths, the registry and signature schemas | LIVE | | share one byte-identical function class across its binaries through an include-at-site header, gated per binary | `share_body.py` | The Phase-35 successor of dedup_propagate + dedup_extend for the include-at-site form: exemplar by majority text, the header written once, every private copy replaced by the include at its position, per-binary byte gate with object comparison, bisect on red, the registry appended or extended by text, the exception ledger on refusal; --plan / --apply --bucket extend|new | aprop_autodraft.py, audit_binaries.py, auto_driver.py, blocker_probe.py (+18) | repo paths, the registry and signature schemas | LIVE |
| take register pins, asm statements, volatile and register levers out of matched C, byte-gated per site | `delever.py` | The Phase-36 de-lever engine: positional rewrites per lever class (pin -> plain declaration, barrier/launder/keep-alive deleted, a hand-placed instruction -> its C, the zero-register variable -> 0, volatile/register dropped), rung A strip-all and rung B greedy per body through delever_oracle, headers serialized after the TU phase; --probe runs a stratified sample of distinct bodies and prices the campaign with denominators | — | the census site records, the oracle | REFERENCED | | take register pins, asm statements, volatile and register levers out of matched C, byte-gated per site | `delever.py` | The Phase-36 de-lever engine and campaign tool: positional rewrites per lever class (pin -> plain declaration, barrier/keep-alive deleted, a launder deleted or turned into the assignment it is, a hand-placed instruction -> its C, the zero-register variable -> 0, a macro-carried site deleted/valued/refused by its macro's shape, volatile/register dropped); per body replay -> rung A strip-all -> rung B greedy through delever_oracle; the file is the write unit and its final compile through every recipe the proof; --plan/--apply batches (TUs parallel, exemplar files first; headers serial, includers parallel), the ledger keyed by tu+fn+addr with the body's nhash before/after, !FAKE markers on class A/B survivors, --restore from inflight.json, --apply-body for T6/T7, --selftest with a stub oracle, --probe (T2) | delever_cycle.sh | the census site records, the oracle | LIVE |
| turn an inline-assembly body back into a stub the toolchain can reach | `verbatim_to_stub.py` | Turns an inline-assembly body back into an include-assembly stub | — | repo src/asm layout | REFERENCED | | turn an inline-assembly body back into a stub the toolchain can reach | `verbatim_to_stub.py` | Turns an inline-assembly body back into an include-assembly stub | — | repo src/asm layout | REFERENCED |
## PROJECT-ONLY — project-only in code (the shape is a task; the code does not transfer) ## PROJECT-ONLY — project-only in code (the shape is a task; the code does not transfer)
phase-ends/CURRENT_PHASE.md
+49 -1
View File
@@ -96,6 +96,8 @@
residue counted as a lever). The 20 per-overlay trampoline families (133 private copies each, bytes varying with the wrapped callee) residue counted as a lever). The 20 per-overlay trampoline families (133 private copies each, bytes varying with the wrapped callee)
are the names phase's parameterized-form inheritance. are the names phase's parameterized-form inheritance.
- **S98 (2026-09-09), Drew, mid-T3:** *"dont start ultra code wave for reshaping without my direct approval."* → **T7's reshaping waves (Ultracode) start ONLY on Drew's direct approval in the session that would run them** — the R27 toggle is necessary, not sufficient; the T6 recipe/permuter rungs and the mechanical campaign are not waves and proceed under P3.
## Rules at gate 1 (P10) ## Rules at gate 1 (P10)
**R100–R106 ratified by the plan approval (2026-09-09; the PhaseEnd_Phase35 candidates (a)–(g), operated through P35):** **R100–R106 ratified by the plan approval (2026-09-09; the PhaseEnd_Phase35 candidates (a)–(g), operated through P35):**
@@ -245,7 +247,53 @@ accumulate here as the phase produces them.**
volatile declarations, `$0` variables, instructions. **Design consequences for T3:** per-body compiles at 0.2 s make group testing volatile declarations, `$0` variables, instructions. **Design consequences for T3:** per-body compiles at 0.2 s make group testing
unnecessary (simplicity wins); run T4 first and publish the fall; rung R (T6) targets the three big classes in that order. unnecessary (simplicity wins); run T4 first and publish the fall; rung R (T6) targets the three big classes in that order.
## 🛑 SESSION CHECKPOINT — S97 close (2026-09-09, at 69 % context): T0 ☑ T1 ☑ T1b ☑ T2 ☑ — all committed (T2 `617f29e30`, the checkpoint `772052dff`, this refresh); NEXT = T3 (`tools/delever.py` the campaign tool: the ledger, --plan/--apply batches with the `!FAKE` markers, exemplar/replay, refusals, `delever_cycle.sh`, selftest) in a FRESH session at Max for the design - **S98 2026-09-09 — T3 the campaign tool (design at Max; the tools built, the end-to-end batch follows).** Session start at low effort; Drew:
`/effort max`, "build task list and resume current phase"; the harness task list rebuilt (R28, 12 tasks; T0–T2 completed). Reconnaissance on
T2's engine and probe before any design (R37/R38): three findings that CHANGE the numbers — (1) **989 of the 4,793 direct launders are
ASSIGNMENTS** (`__asm__("" : "=r"(p) : "0"(&D_x))` — output ≠ input); the probe deleted them as statements, which left the variable undefined and
**cc1 2.7.2 ABORTED** (the probe's one "Done" COMPILE-ERROR, `ov_SC03_124 func_80188C68` launder at 3449, reproduced 3/3: rc 134,
`Aborted (core dumped) | tools/bin/gcc-2.7.2-psx/cc1`; the error filter had kept bash's job-status line and dropped the signal line) — the
probe's "launder 77 % needed" is partly this; (2) the **macro-carried launders are mostly compound macros** (`XFER`/`DRAW`/`RTP_SND`/`COPY_TO_FAAC`,
~60 uses in 8 definitions: real code around the asm — the probe's rewrite deleted the whole USE statement, which is a correctness hazard; only
`SHB`/`ENGINE_SHB` (416 defs / 2 texts, 399 uses) are pure launder statements and `LAUNDER_8018A180` a launder statement-expression);
(3) **46 lever bodies live in `-O0` translation units** (boot.c 31 bare `register` + 6 instructions; `md_MAIN_003_o0*` / `md_MAIN_011`) — the
plan's "none have pins" was about pins; they are judged by their own recipes, not refused (the ledger records `o0`). Also measured: 5 lever
bodies not named `func_` (addresses from `config/symbols.us.txt`), 1 class whose members disagree on the site sequence (the replay checks it),
369 header bodies (203 with < 20 includers, 165 with 120–139), 517 file-scope `volatile` declarations in 256 TUs, 2 `volatile` parameters,
0 multi-declarator pins, nested includes only `engine_prelude.h` → `include/` (the includer scan is transitive anyway).
**Design (X1, the non-obvious choices):** the verdict unit is the body, the write unit the file, the proof unit the file's FINAL compile
(every body judged against the original text; all accepted edits + the file-scope volatile edits + the markers spliced once and compiled
through every recipe — a twin's object, every includer of a header in parallel; DIFFERS → the file-scope edits dropped and retried → the file
restored and COMBINATION-FAILED, loud); **"done" is a ledger fact keyed by normalized text** (every row carries the body's `nhash` BEFORE and
AFTER; current nhash ∈ after-hashes → done, ∈ before-hashes → replay in one compile, a disagreement falls through to the full ladder and is
recorded; markers are comments and do not move the hash; a reshaped body gets a new hash and is drawn again — so T6/T7 need no new bookkeeping);
the launder-assignment and macro-shape rewrites above; the oracle's **COMPILE-CRASH** verdict (a signal line / rc ≥ 128, R103) distinct from
COMPILE-ERROR; class C/D survivors judged, ledgered, never marked (decision 3), class A/B survivors marked `// !FAKE: <kind> <detail> — <verdict>
(P36 <rung> <label>)` at the end of their line (one marker per line, an old one replaced); an asm-body DEFERRED (T7) and marked; the census
gains `head` + a stat-based `src_stamp` and `--apply` refuses a census that does not describe the tree (it reruns it); the outer gate of EVERY
batch is the clean fleet run (90 s — cheaper than per-binary checks past ~15 touched binaries, and it refreshes `build/`, the oracle's
baselines); the calibration is rerun after every commit (2.4 s; its positive control now restores the source's mtime so the census stamp
survives it); exemplar files run before their copies (a class cover, phase 1) so the replay discount is real; `inflight.json` (every batch
file's original text, written before the first write) is the only restore (R102). Refusals: a token mismatch, no statement end, a compound
macro, an instruction with no C spelling, an assigned zero-register variable, two `$0` pins in one body, a no-op edit, an empty includer set,
a dirty `src/`, a stale calibration or census, an untracked dotfile under `src/` (the cycle).
**Built:** `tools/delever.py` (1,548 lines: `--plan`, `--apply`, `--restore`, `--status`, `--apply-body`, `--selftest`, `--probe`),
`tools/delever_oracle.py` (COMPILE-CRASH, `recipes_by_src`, `judge_all`, the mtime restore), `tools/lever_census.py` (`src_stamp`, `head`),
`tools/delever_cycle.sh`, `.gitignore` (`inflight.json` ignored; the ledger/batch records/logs already allowlisted), SETUP + dictionary rows,
kit corpus (`tool_census --check: OK`).
**Verify so far:** `delever --selftest: OK — 15 sites in the exemplar, 1 refused rewrite(s), ladder RESIDUE (15 compiles), replay replay
(1 compile), markers 5` (the fixture through the census's own parser: every rewrite incl. the assignment launder, the `$0` uses → 0, the
expression-launder macro → `((s32)(&buf))`, the compound macro REFUSED; the ladder against a stub oracle with two needed sites and one whose
removal crashes → NEEDED COMPILE-CRASH; 5 marker lines, none on a class C/D site; the replay on a copy in another address environment = 1
compile with the same verdicts; a disagreeing replay → rung B; the ledger index; the oracle's signal-line regex on the real bash form).
`lever_census --sites -j 16 --no-cache`: **53,234 / 15,679 / 2,227 unchanged**, `head 89676f096`, `src_stamp` written (32 s).
`delever_oracle --calibrate main ov_SC03_014 ov_SC03_015 ov_SC04_011: 177/177 objects byte-identical untouched; twin checks 32 (0 mismatch);
positive control DIFFERS on build/src/800.o; 2.4 s — OK`. `delever --plan --only ov_SC04_011: 18 of 18 drawable files · 176 bodies (109 in
multi-copy classes) · 540 removable sites`; the campaign order's head: the `*_jr_80140608.c` copies (11 bodies / 30 sites each, classes of
134); headers: 371 drawable files. **Next in this task:** the end-to-end batch `LABEL_PREFIX=t3_ TASK=T3 tools/delever_cycle.sh 1 1 300 tus
ov_SC04_011` (apply → R22 → census → log → commit), then the checkpoint refresh.
## 🛑 SESSION CHECKPOINT — S98 in progress (2026-09-09): T0 ☑ T1 ☑ T1b ☑ T2 ☑ — T3 tools BUILT + selftest OK + calibration OK (this commit); NEXT = the T3 end-to-end batch on ov_SC04_011 via `tools/delever_cycle.sh` (then the checkpoint refresh, then T4). The §0–§4 below are the S97-close text and stay valid except: `tools/delever.py` now has the campaign commands (see the T3 log entry) and T7 needs Drew's direct approval (decisions).
### 0. How to use this block ### 0. How to use this block
A fresh session (S98) reads CLAUDE.md's load order, replays this block verbatim, asks Drew for `/effort max` (T3's design is Max; its A fresh session (S98) reads CLAUDE.md's load order, replays this block verbatim, asks Drew for `/effort max` (T3's design is Max; its
tools/delever.py
+1102 -149
View File
File diff suppressed because it is too large Load Diff
tools/delever_cycle.sh
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
# delever_cycle.sh — the Phase-36 batch cycle, mechanised (T3/T4). For each batch k in [START..END]:
# 1. the tree must be clean (src/ + config/); the oracle's calibration is refreshed (2–3 s: it is keyed to HEAD, every commit stales it);
# 2. `delever.py --apply --batch N --label <mode>k` judges every drawable body of the next N files and leaves the winning texts in place
# (the census is regenerated by the tool when its src stamp is stale); the tool's exit code and its `final X/X identical` are read (R97/R66);
# 3. the outer gate: the CLEAN fleet run (make clean && extract-all && check-all -> `218 passed, 0 failed of 218`, R22) — it also refreshes
# build/, the oracle's baseline objects;
# 4. the census is rerun on the new tree (the commit line quotes it), the phase log gets the batch's entry, the 🛑 headline is refreshed (R101),
# and src/ + the ledger/batch record/log + the census + the phase log are committed (R42).
# Stops on the first red (a non-zero tool exit, a final N/M with N != M, a fleet run that is not 218/218) — the batch's files are then
# still in place: inspect, then `tools/delever.py --restore` (never git checkout, R102). Stops cleanly when nothing is drawable.
# Usage: [LABEL_PREFIX=t3_] [TASK=T4] tools/delever_cycle.sh START END [BATCH=300] [MODE=tus|headers] [ONLY="alias1 alias2 …"]
# size a call so one batch stays < 10 min: TUs ≈ 8 s each single-threaded / 12 workers; a header ≈ 4 s (serial, includers in parallel).
set -o pipefail
cd "$(dirname "$0")/.." || exit 2
START=$1; END=$2; BATCH=${3:-300}; MODE=${4:-tus}; ONLY=${5:-}
[ -n "$START" ] && [ -n "$END" ] || { echo "usage: $0 START END [BATCH] [tus|headers] [ONLY]"; exit 2; }
case "$MODE" in tus) HFLAG="";; headers) HFLAG="--headers";; *) echo "MODE must be tus or headers"; exit 2;; esac
ONLYFLAG=""; [ -n "$ONLY" ] && ONLYFLAG="--only $ONLY"
FLEET=$(ls config/check.*.sha | wc -l)
PY=.venv/bin/python
MARKER='## Approved plan (verbatim, gate 1 — 2026-09-09)'
for k in $(seq "$START" "$END"); do
label="${LABEL_PREFIX:-}${MODE}${k}"
[ -z "$(git status --short -- src config)" ] || { echo "cycle: src/ or config/ is dirty — commit, or tools/delever.py --restore (R42/R102)"; exit 3; }
[ -z "$(git status --short --porcelain src | grep '^??')" ] || { echo "cycle: untracked files under src/ (a dotfile probe?) — sweep them first"; exit 3; }
$PY tools/delever_oracle.py --status >/dev/null 2>&1 || $PY tools/delever_oracle.py --calibrate ov_SC04_011 ov_SC03_015 ov_SC03_014 main -j 16 > .run/P36/delever/calibrate_${label}.log 2>&1 \
|| { echo "cycle: calibration FAILED"; tail -5 .run/P36/delever/calibrate_${label}.log; exit 1; }
log=.run/P36/delever/run_${label}.log
{ /usr/bin/time -f "$label wall=%e s" $PY tools/delever.py --apply --batch "$BATCH" --label "$label" $HFLAG $ONLYFLAG -j 12; echo "exit=$?"; } > "$log" 2>&1
grep -q '^exit=0$' "$log" || { echo "cycle: batch $label — delever exited non-zero"; tail -4 "$log"; exit 1; }
if grep -q 'nothing to do (no drawable file)' "$log"; then echo "cycle: nothing drawable at $label — done"; exit 0; fi
vline=$(grep -oE '^delever: batch .*' "$log" | tail -1)
final=$(echo "$vline" | grep -oE 'final [0-9]+/[0-9]+' | grep -oE '[0-9]+/[0-9]+'); [ "${final%/*}" = "${final#*/}" ] || { echo "cycle: batch $label — $vline"; exit 1; }
if [ "$(echo "$vline" | grep -oE 'written [0-9]+' | grep -oE '[0-9]+')" = "0" ]; then echo "cycle: batch $label wrote no file — $vline"; fi
r22=.run/P36/baseline/r22_${label}.log
{ /usr/bin/time -f "wall=%e s user=%U s sys=%S s" bash -c 'set -o pipefail; make clean && make extract-all JOBS=16 && make check-all JOBS=16'; echo "exit=$?"; } > "$r22" 2>&1
line=$(grep -oE "check-all: [0-9]+ passed, [0-9]+ failed of [0-9]+" "$r22")
if ! grep -q '^exit=0$' "$r22" || [ "$line" != "check-all: $FLEET passed, 0 failed of $FLEET" ]; then echo "cycle: R22 RED after batch $label — $line (the batch's files are in place: inspect, then tools/delever.py --restore)"; exit 1; fi
$PY tools/lever_census.py --sites -j 16 > .run/P36/census/census_${label}.log 2>&1 || { echo "cycle: the census FAILED after batch $label"; tail -3 .run/P36/census/census_${label}.log; exit 1; }
cline=$(grep -oE 'levers A\+B.*' .run/P36/census/lever_census.txt | head -1)
[ -n "$cline" ] || cline=$(grep -oE '^lever_census:.*' .run/P36/census/census_${label}.log | head -1)
TASK=${TASK:-T4} $PY - "$label" "$vline" "$line" "$cline" "$(grep -oE 'wall=[0-9.]+ s' "$r22" | head -1)" "$MARKER" <<'EOF' || exit 1
import os, pathlib, re, sys, time, subprocess
label, vline, r22line, cline, wall, marker = sys.argv[1:7]
task = os.environ.get("TASK", "T4")
p = pathlib.Path("phase-ends/CURRENT_PHASE.md"); t = p.read_text()
head = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True).stdout.strip()
entry = (f"\n- **{time.strftime('%Y-%m-%d')} — {task} batch `{label}`** (`.run/P36/delever/run_{label}.log`, `batch_{label}.json`): `{vline}` → R22 "
f"(`.run/P36/baseline/r22_{label}.log`) **`{r22line}`** (`{wall}`) → census `{cline}`.")
assert ("\n" + marker) in t, "phase log marker not found"
t = t.replace("\n" + marker, entry + "\n\n" + marker, 1)
# the 🛑 headline (R101): every task-advancing commit refreshes it
lines = t.split("\n")
for i, l in enumerate(lines):
if l.startswith("## 🛑 SESSION CHECKPOINT"):
base = re.sub(r"\s*\|\s*last batch.*$", "", l)
lines[i] = f"{base} | last batch `{label}` on {head}: {cline}"
break
p.write_text("\n".join(lines))
print("log:", label, cline)
EOF
git add -A src && git add .run/P36/delever .run/P36/census .run/P36/baseline phase-ends/CURRENT_PHASE.md || exit 1
git commit -q -m "src(phase-36): ${TASK:-T4} batch $label — $vline | R22 $line | census: $cline (delever_cycle)" || exit 1
echo "cycle: batch $label committed $(git log --oneline -1 | cut -c1-9) — $vline | $line"
done
echo "cycle: batches $START..$END done"
tools/delever_oracle.py
+34 -2
View File
@@ -42,6 +42,8 @@ RECIPES = RUN / "recipes.json"
CALIB = RUN / "calibration.json" CALIB = RUN / "calibration.json"
SCRATCH = RUN / "obj" SCRATCH = RUN / "obj"
RECIPE_RE = re.compile(r"^set -o pipefail; (.*) -o (build/\S+\.o)$") RECIPE_RE = re.compile(r"^set -o pipefail; (.*) -o (build/\S+\.o)$")
SIGNAL_LINE = re.compile(r"^\s*\d+\s+(Aborted|Segmentation fault|Illegal instruction|Floating point exception|Bus error|Killed)[^\n]*", re.M)
JOB_STATUS_LINE = re.compile(r"^(bash: line \d+: )?\s*\d+\s+(Done|Exit \d+)\b")
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
@@ -156,11 +158,17 @@ def compile_obj(recipe, text=None, tag="x", write_path=None):
r = subprocess.run(["bash", "-c", cmd], cwd=REPO, capture_output=True, text=True) r = subprocess.run(["bash", "-c", cmd], cwd=REPO, capture_output=True, text=True)
dt = time.time() - t0 dt = time.time() - t0
if r.returncode != 0 or not out.exists(): if r.returncode != 0 or not out.exists():
# a pipeline member killed by a signal (cc1 2.7.2 aborts on some candidates — e.g. a variable whose only definition was a
# launder that got deleted): bash prints the job-status block ("Done" / "Aborted (core dumped)" / "Segmentation fault")
# and the pipeline's rc is >= 128 — that is a CRASH verdict, not a diagnostic (R103: the real message forms)
sig = SIGNAL_LINE.search(r.stderr)
if sig or r.returncode >= 128:
return None, dt, "CRASH: " + (sig.group(0).strip() if sig else f"rc={r.returncode}")
# gcc 2.7.2 prints errors without the word "error" and the assembler floods stderr with `$at` warnings: keep every # gcc 2.7.2 prints errors without the word "error" and the assembler floods stderr with `$at` warnings: keep every
# non-warning line (R103), then the tail # non-warning line (R103), then the tail
errs = [ln for ln in r.stderr.splitlines() errs = [ln for ln in r.stderr.splitlines()
if ln.strip() and not re.search(r"\bwarning:", ln, re.I) and not ln.startswith(("In file included", " " * 16)) if ln.strip() and not re.search(r"\bwarning:", ln, re.I) and not ln.startswith(("In file included", " " * 16))
and ": In function" not in ln] and ": In function" not in ln and not JOB_STATUS_LINE.match(ln)]
return None, dt, ("\n".join(errs[:8]) or r.stderr[-400:]) return None, dt, ("\n".join(errs[:8]) or r.stderr[-400:])
data = out.read_bytes() data = out.read_bytes()
try: try:
@@ -183,10 +191,32 @@ def judge(recipe, text, tag="x", write_path=None):
return "NO-BASELINE", 0.0, "" return "NO-BASELINE", 0.0, ""
data, dt, err = compile_obj(recipe, text, tag, write_path) data, dt, err = compile_obj(recipe, text, tag, write_path)
if data is None: if data is None:
return "COMPILE-ERROR", dt, err return ("COMPILE-CRASH" if err.startswith("CRASH:") else "COMPILE-ERROR"), dt, err
return ("IDENTICAL" if data == base else "DIFFERS"), dt, "" return ("IDENTICAL" if data == base else "DIFFERS"), dt, ""
def recipes_by_src(recipes):
"""src rel -> [recipe] (a twin binary compiles the primary's source into its own object: two recipes, one source)."""
out = {}
for r in recipes.values():
out.setdefault(r["src"], []).append(r)
for v in out.values():
v.sort(key=lambda r: r["obj"])
return out
def judge_all(recipes, text, tag="x", write_path=None):
"""Every recipe of one written file judged in turn (the file is written once, by the first compile; the CALLER restores it).
Returns (verdict, seconds, err): IDENTICAL only if every object is identical; otherwise the first non-identical verdict."""
total, first = 0.0, None
for i, r in enumerate(recipes):
v, dt, err = judge(r, text if i == 0 else None, tag=tag, write_path=write_path)
total += dt
if v != "IDENTICAL" and first is None:
first = (v, err)
return (first[0] if first else "IDENTICAL"), total, (first[1] if first else "")
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
# calibration # calibration
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
@@ -222,6 +252,7 @@ def calibrate(aliases, jobs, recipes):
import share_census as sc import share_census as sc
ctl = recs[0] ctl = recs[0]
src = REPO / ctl["src"] src = REPO / ctl["src"]
st0 = src.stat()
orig = src.read_text(errors="surrogateescape") orig = src.read_text(errors="surrogateescape")
pos_verdict = "SKIPPED" pos_verdict = "SKIPPED"
defs = [r for r in sc.scan_text(orig, ctl["src"], shared_defs=None) if r["form"] == "def" and r.get("nlines", 0) >= 3 and not r.get("empty")] defs = [r for r in sc.scan_text(orig, ctl["src"], shared_defs=None) if r["form"] == "def" and r.get("nlines", 0) >= 3 and not r.get("empty")]
@@ -239,6 +270,7 @@ def calibrate(aliases, jobs, recipes):
pos_verdict = pos_verdict if pos_verdict != "COMPILE-ERROR" else f"COMPILE-ERROR ({err[-100:]})" pos_verdict = pos_verdict if pos_verdict != "COMPILE-ERROR" else f"COMPILE-ERROR ({err[-100:]})"
finally: finally:
src.write_text(orig, errors="surrogateescape") src.write_text(orig, errors="surrogateescape")
os.utime(src, ns=(st0.st_atime_ns, st0.st_mtime_ns)) # the census's src stamp is stat-based: a restore is not a change
by_alias = {} by_alias = {}
for x in results: for x in results:
d = by_alias.setdefault(x["alias"], dict(objects=0, identical=0, seconds=0.0)) d = by_alias.setdefault(x["alias"], dict(objects=0, identical=0, seconds=0.0))
tools/lever_census.py
+17
View File
@@ -598,6 +598,22 @@ def _walk_worker(args):
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
# the fleet # the fleet
# ---------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------
def src_stamp():
"""sha1 over (path, size, mtime_ns) of every .c/.h under src/: the tree the census describes. delever refuses a census whose
stamp is not the working tree's (R43) — a restore-write that changes only an mtime just costs a census rerun (35 s)."""
h = hashlib.sha1()
for p in sorted((REPO / "src").rglob("*")):
if p.suffix in (".c", ".h") and not p.name.startswith("."):
st = p.stat()
h.update(f"{p.relative_to(REPO).as_posix()}|{st.st_size}|{st.st_mtime_ns}\n".encode())
return h.hexdigest()[:16]
def git_head():
import subprocess
return subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=REPO, capture_output=True, text=True).stdout.strip()
def enumerate_files(aliases, dirs): def enumerate_files(aliases, dirs):
"""TU -> [aliases] (a twin's TUs are its primary's, listed for both), plus every header under src/ (aliases = its includers').""" """TU -> [aliases] (a twin's TUs are its primary's, listed for both), plus every header under src/ (aliases = its includers')."""
import compile_only import compile_only
@@ -756,6 +772,7 @@ def run_census(jobs, use_cache=True, out_dir=OUT_DIR_DEFAULT, want_sites=False,
macro_texts[m["name"]].add(m["text_hash"]) macro_texts[m["name"]].add(m["text_hash"])
summary = dict( summary = dict(
generated=time.strftime("%Y-%m-%d"), binaries=len(aliases), tus=len(tu_aliases), headers=len(headers), generated=time.strftime("%Y-%m-%d"), binaries=len(aliases), tus=len(tu_aliases), headers=len(headers),
head=git_head(), src_stamp=src_stamp(),
coverage=coverage, coverage_ok=cov_ok, unclassified=len(unclassified), coverage=coverage, coverage_ok=cov_ok, unclassified=len(unclassified),
verbatim_excluded=dict(sites=verbatim_sites, functions=len(verbatim_fns), manifest_rows=len(verb)), verbatim_excluded=dict(sites=verbatim_sites, functions=len(verbatim_fns), manifest_rows=len(verb)),
classes=classes, classes=classes,