diff --git a/tools/lanes/drafter.sh b/tools/lanes/drafter.sh index 02dfe7770..01ed9befe 100644 --- a/tools/lanes/drafter.sh +++ b/tools/lanes/drafter.sh @@ -79,6 +79,16 @@ # larger output budget costs latency and nothing else. set -u cd /home/musashi/bfm-decomp +# HTTP_TIMEOUT 700 (P31 S59) — MUST be raised WITH MAXTOK; they are one setting, not two. +# Measured on ox-alpha with a real MIPS derivation: 30 tok/s, and an UNCAPPED hard prompt ran +# 265 s for 8,067 completion tokens — i.e. it wanted to finish exactly where the old 8k cap cut it +# off, which is the nudge storm we were seeing. At 30 tok/s a full 16k generation needs ~530 s, so +# leaving the socket at 420 s would have killed the very turns the bigger budget exists to allow — +# and a timeout wastes the whole turn, where truncation at least leaves a partial. +# The ordering that must hold: generation < HTTP_TIMEOUT (700) < stallguard's wedged-agent kill +# (1200 s). 420 was itself a deliberate choice after 1800 parked a hung agent for THIRTY minutes; +# this keeps that concern (a hang costs <12 min) without strangling legitimate deep reasoning. +export HTTP_TIMEOUT=700 export MAX_429=10 while [ ! -e .run/ox_campaign.stop ]; do .venv/bin/python tools/ox_campaign.py --drafter \ diff --git a/tools/lanes/main.sh b/tools/lanes/main.sh index e5da9d67b..fedd00427 100644 --- a/tools/lanes/main.sh +++ b/tools/lanes/main.sh @@ -13,6 +13,16 @@ # excluded — already drafted, never gated, free), then draws and drafts its own main cards. set -u cd /home/musashi/bfm-decomp +# HTTP_TIMEOUT 700 (P31 S59) — MUST be raised WITH MAXTOK; they are one setting, not two. +# Measured on ox-alpha with a real MIPS derivation: 30 tok/s, and an UNCAPPED hard prompt ran +# 265 s for 8,067 completion tokens — i.e. it wanted to finish exactly where the old 8k cap cut it +# off, which is the nudge storm we were seeing. At 30 tok/s a full 16k generation needs ~530 s, so +# leaving the socket at 420 s would have killed the very turns the bigger budget exists to allow — +# and a timeout wastes the whole turn, where truncation at least leaves a partial. +# The ordering that must hold: generation < HTTP_TIMEOUT (700) < stallguard's wedged-agent kill +# (1200 s). 420 was itself a deliberate choice after 1800 parked a hung agent for THIRTY minutes; +# this keeps that concern (a hang costs <12 min) without strangling legitimate deep reasoning. +export HTTP_TIMEOUT=700 while [ ! -e .run/ox_campaign.stop ]; do .venv/bin/python tools/main_lane.py --workers 150 --batch 40 --cards 200 --max-ins 200 --maxtok 16000 2>&1 echo "[$(date +%H:%M:%S)] [main-lane] exited; restarting in 20s"