mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 13:33:34 -04:00
Drew T
e76a55abdf
A marker is a CLAIM on work, not a record of it. axbm.json sat in .run/distill_ready for 10.5 hours AFTER its waves were distilled into cookbook §269 — the reviewer landed the sections, updated the mined state, and never removed the marker — and distill_scan's one-pending-marker-at-a-time rule (correct, it stopped eight overlapping batches) then refused to raise anything while 18 waves / 315 novel candidates accumulated behind it. The marker's own waves are checkable against the mined state, so check them: a marker whose every wave is already mined clears itself and says so. Negative control (R39): a marker naming any still-unmined wave survives untouched. Same family as R47 — a stage that consumes work must also consume the token that represents it.
92 lines
4.1 KiB
Python
92 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""distill_scan.py — one pass of the distill lane: which harvested waves still need mining?
|
|
|
|
STATE IS {tag: novel-count AT THE TIME IT WAS MINED}, never a bare done-list. A wave that is
|
|
RE-GATED rewrites its candidate file with new rows under the SAME tag, so "have I seen this tag"
|
|
answers the wrong question: wave `at` was re-gated hours after its first harvest and an mtime-keyed
|
|
seed mis-classified it, and a hand-edited done-list nearly buried `ax`+`bm` — 82 candidates that had
|
|
been gated minutes earlier — because "queued for review" was folded into "reviewed". Compare COUNTS,
|
|
and a tag re-opens the moment its file grows.
|
|
|
|
Writes a READY marker when the pending rows are worth a reviewer's turn. Writes nothing else.
|
|
|
|
The reviewer is an **Opus or Sonnet** subagent, never Fable (Drew, 2026-08-24): distilling is
|
|
judgement over an existing corpus, not a new wall class.
|
|
|
|
distill_scan.py <state.json> <ready_dir> <min_novel> <min_waves>
|
|
"""
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
NOVEL = re.compile(r"\*\*(\d+) novel-idiom candidates\*\*")
|
|
|
|
|
|
def main():
|
|
state_p, ready_dir, min_novel, min_waves = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
|
|
try:
|
|
st = json.load(open(state_p))
|
|
except Exception:
|
|
st = {"mined": {}}
|
|
mined = st.setdefault("mined", {})
|
|
|
|
pend = []
|
|
for p in sorted(glob.glob(".run/idiom_candidates.*.md")):
|
|
tag = os.path.basename(p).split(".")[1]
|
|
if tag == "ALL":
|
|
continue
|
|
m = NOVEL.search(open(p, errors="replace").read(4000))
|
|
n = int(m.group(1)) if m else 0
|
|
if n > mined.get(tag, -1):
|
|
if n:
|
|
pend.append((tag, n, p))
|
|
else:
|
|
mined[tag] = 0 # nothing to mine; close it out at zero
|
|
|
|
# ONE PENDING MARKER AT A TIME. Each pass re-offers everything not yet mined, so without this
|
|
# the lane wrote a fresh, overlapping marker every five minutes — eight batches queued
|
|
# (`bobq`, `bebobq`, `bebjbobq`, …), each a superset of the last, and a reviewer cannot tell
|
|
# which one is the work. The next pass will re-offer whatever is still unmined anyway.
|
|
# A MARKER IS A CLAIM ON WORK, NOT A RECORD OF IT (P31 S60). A reviewer who distils a batch and
|
|
# forgets to remove its marker blocks the lane forever: `axbm.json` sat for 10.5 h AFTER its
|
|
# waves were mined into cookbook §269, while 18 waves / 315 candidates piled up behind it. The
|
|
# marker's own waves are checkable against `mined`, so check them — a stale claim clears itself.
|
|
for mk in sorted(glob.glob(os.path.join(ready_dir, "*.json"))):
|
|
try:
|
|
waves = json.load(open(mk)).get("waves", [])
|
|
except Exception:
|
|
continue
|
|
if waves and all(t in mined for t in waves) and not any(t == p_t for p_t, _n, _p in pend for t in waves):
|
|
os.remove(mk)
|
|
print(" cleared a stale marker (%s): every wave in it is mined"
|
|
% os.path.basename(mk))
|
|
|
|
already = glob.glob(os.path.join(ready_dir, "*.json"))
|
|
if already:
|
|
print(" a batch is already pending review (%s) — not raising another"
|
|
% ", ".join(os.path.basename(x) for x in already[:3]))
|
|
json.dump(st, open(state_p, "w"), indent=1)
|
|
return
|
|
|
|
tot = sum(n for _t, n, _p in pend)
|
|
if pend and (tot >= min_novel or len(pend) >= min_waves):
|
|
batch = "".join(t for t, _n, _p in pend)[:24]
|
|
out = os.path.join(ready_dir, batch + ".json")
|
|
json.dump({"waves": [t for t, _n, _p in pend],
|
|
"novel": tot,
|
|
"counts": {t: n for t, n, _p in pend},
|
|
"files": [p for _t, _n, p in pend],
|
|
"t": time.time()}, open(out, "w"), indent=1)
|
|
print(" BATCH READY: %d wave(s) / %d novel candidates -> %s" % (len(pend), tot, out))
|
|
elif pend:
|
|
print(" accumulating: %d wave(s) / %d novel candidates (need %d or %d waves)"
|
|
% (len(pend), tot, min_novel, min_waves))
|
|
json.dump(st, open(state_p, "w"), indent=1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|