#!/usr/bin/env python3
"""Find unmatched worklist rows that are FAMILY MEMBERS of already-matched rows.

Phase 11 finding 125 (worker D): three members of one family were found by three different
means -- the size ranker, adjacency, and the redundancy rank -- and all three closed the same
way. *"The finder varies, the price does not."* Once a lever is in hand, a sibling row is a
one-attempt row wherever it sits.

Worker D's operational conclusion is what this tool implements: **families should be SEARCHED
FOR explicitly rather than waited for.** Three rankers found three siblings of one family by
accident; a direct search finds all of them at once.

SIMILARITY is measured on the instruction stream with registers erased, because (finding 120)
the allocator makes copies non-identical, so opcode repetition survives where word repetition
does not. Two signals are combined:

  * opcode-histogram cosine  -- "does this body do the same KINDS of things, in the same
    proportions?"
  * size ratio               -- "is it the same scale?"

Both must be high. A body that merely shares an instruction multiset with a much larger one is
not a sibling, it is a coincidence.

**CALIBRATED THRESHOLD: 0.99, and the useful band is 1.000.** Worker A checked the two
0.97-scoring entries this tool first produced and **neither shares its sibling's body at all** --
one is a table-allocation routine, the other a summing loop. Below ~0.99 the histogram is matching
common *idioms*, not bodies. That is the same false-positive mode as the redundancy ranker
(finding 110): a shared instruction multiset is not a shared body. **Scores of 1.000 mean the
opcode histogram AND the size match exactly, and those transferred 7 for 7 with 5 first-attempt
matches.**

Usage:
    sf3_family [--top N] [--min-score F] [--min-size B] [--max-size B]
"""
from __future__ import annotations

import math
import struct
import sys
from collections import Counter
from pathlib import Path

REPO = Path(__file__).resolve().parent.parent
EXE = REPO / "extracted/SCUS_946.40;1"
REGIONS = REPO / "config/regions.tsv"
WORKLIST = REPO / "config/match_worklist.tsv"
LOAD_ADDRESS = 0x80010000
HEADER_SIZE = 0x800


def body_words(payload: bytes, start: int, end: int) -> list[int]:
    offset = HEADER_SIZE + (start - LOAD_ADDRESS)
    count = (end - start) // 4
    return list(struct.unpack_from(f"<{count}I", payload, offset))


def histogram(words: list[int]) -> Counter[int]:
    """Opcode histogram with registers erased and nops dropped."""
    return Counter(word >> 26 for word in words if word)


def cosine(left: Counter[int], right: Counter[int]) -> float:
    if not left or not right:
        return 0.0
    common = set(left) & set(right)
    dot = sum(left[k] * right[k] for k in common)
    norm = math.sqrt(sum(v * v for v in left.values())) * math.sqrt(sum(v * v for v in right.values()))
    return dot / norm if norm else 0.0


def read_regions(path: Path) -> list[tuple[int, int, str]]:
    """config/regions.tsv: start, end, source[, overrides]."""
    rows = []
    for line in path.read_text().splitlines():
        if line.startswith("#") or not line.strip():
            continue
        fields = line.split("\t")
        rows.append((int(fields[0], 16), int(fields[1], 16), fields[2]))
    return rows


def read_worklist(path: Path) -> list[tuple[int, int]]:
    """config/match_worklist.tsv: rank, address, end, size, ...

    NOTE the column offset -- column 0 is the RANK, not the address. Reading this file with
    the regions layout silently produces garbage (rank numbers as addresses) and the search
    returns nothing at all, which is how this bug was found.
    """
    rows = []
    for line in path.read_text().splitlines():
        if line.startswith("#") or not line.strip():
            continue
        fields = line.split("\t")
        rows.append((int(fields[1], 16), int(fields[2], 16)))
    return rows


def main(argv: list[str]) -> int:
    # Worker A calibrated this: it checked the two 0.97-scoring entries and NEITHER shares its
    # sibling's body at all (0x80050BF4 is a table-allocation routine, 0x80016660 a summing loop).
    # Below ~0.99 the histogram is matching common IDIOMS, not bodies -- the same false-positive
    # mode worker C found at the top of the ranker (finding 110). **1.000 is the useful band.**
    top, min_score, min_size, max_size = 40, 0.99, 0, 1 << 30
    index = 0
    while index < len(argv):
        if argv[index] == "--top":
            index += 1
            top = int(argv[index])
        elif argv[index] == "--min-score":
            index += 1
            min_score = float(argv[index])
        elif argv[index] == "--min-size":
            index += 1
            min_size = int(argv[index])
        elif argv[index] == "--max-size":
            index += 1
            max_size = int(argv[index])
        index += 1

    payload = EXE.read_bytes()

    claimed = read_regions(REGIONS)
    claimed_ranges = [(s, e) for s, e, _ in claimed]
    claimed_starts = {s for s, _, _ in claimed}

    def is_claimed(addr: int) -> bool:
        return addr in claimed_starts or any(s < addr < e for s, e in claimed_ranges)

    matched = []
    for start, end, source in claimed:
        words = body_words(payload, start, end)
        matched.append((start, end, source, histogram(words), end - start))

    results = []
    for start, end in read_worklist(WORKLIST):
        size = end - start
        if size < min_size or size > max_size:
            continue
        if is_claimed(start):
            continue  # already merged -- reporting it is noise (it matched itself)
        mine = histogram(body_words(payload, start, end))
        best = None
        for m_start, m_end, m_source, m_hist, m_size in matched:
            ratio = min(size, m_size) / max(size, m_size)
            score = cosine(mine, m_hist) * ratio
            if best is None or score > best[0]:
                best = (score, m_start, m_end, m_source, m_size, ratio)
        if best and best[0] >= min_score:
            results.append((best[0], start, size, best))

    results.sort(key=lambda row: (-row[0], row[2]))
    print(f"{'score':>6} {'ratio':>6} {'size':>6}  candidate  -> matched sibling")
    for score, start, size, best in results[:top]:
        _, m_start, m_end, m_source, m_size, ratio = best
        print(f"{score:6.3f} {ratio:6.2f} {size:6d}  {hex(start)} -> {hex(m_start)} "
              f"({m_size} B, {m_source})")
    print(f"\ncandidates={len(results)}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main(sys.argv[1:]))
