BFM-decomp/tools/bfm_extract/manifest.py

"""Deterministic SHA1 manifest of the extracted disc (Validation Stage 1).

Walks the extraction output tree and records one JSON object per file --
``{"path", "size", "sha1"}`` -- sorted by path with pinned JSON serialization, so
the manifest is byte-identical across machines and re-runs given the same dump.
``manifest.sha1`` holds the SHA1 of ``manifest.jsonl`` itself: the single value a
contributor diffs to prove their own extraction matches.

See PROJECT_CONTEXT.md "Testing & Validation" stage 1. Pure stdlib.
"""

from __future__ import annotations

import hashlib
import json
from pathlib import Path

MANIFEST_NAME = "manifest.jsonl"
MANIFEST_SHA1_NAME = "manifest.sha1"
_HASH_CHUNK = 1 << 20  # 1 MiB streaming reads
# The manifest's own files are never recorded inside it.
_SELF = {MANIFEST_NAME, MANIFEST_SHA1_NAME}


def _sha1_file(path: Path) -> tuple[int, str]:
    h = hashlib.sha1()
    size = 0
    with open(path, "rb") as fh:
        while True:
            chunk = fh.read(_HASH_CHUNK)
            if not chunk:
                break
            h.update(chunk)
            size += len(chunk)
    return size, h.hexdigest()


def _list_files(out_root: Path) -> list[str]:
    """Sorted POSIX-relative paths of every file under ``out_root`` (excluding
    the manifest itself)."""
    return sorted(
        p.relative_to(out_root).as_posix()
        for p in out_root.rglob("*")
        if p.is_file() and p.name not in _SELF
    )


def build(out_root: Path) -> list[dict]:
    """Deterministic record list for every file under ``out_root`` (sorted)."""
    records = []
    for rel in _list_files(out_root):
        size, sha1 = _sha1_file(out_root / rel)
        records.append({"path": rel, "size": size, "sha1": sha1})
    return records


def render(records: list[dict]) -> str:
    """Pinned, deterministic JSON-Lines serialization (locale-independent)."""
    return "".join(
        json.dumps(r, separators=(",", ":"), ensure_ascii=True, sort_keys=True) + "\n"
        for r in records
    )


def write(out_root: Path, records: list[dict]) -> str:
    """Write manifest.jsonl + manifest.sha1; return the manifest's own SHA1."""
    text = render(records)
    (out_root / MANIFEST_NAME).write_text(text, encoding="ascii", newline="\n")
    digest = hashlib.sha1(text.encode("ascii")).hexdigest()
    (out_root / MANIFEST_SHA1_NAME).write_text(digest + "\n", encoding="ascii", newline="\n")
    return digest


def verify(out_root: Path) -> tuple[bool, list[str]]:
    """Re-hash every file named in manifest.jsonl and compare to the recorded
    size/sha1; also flag any file on disk that the manifest does not list.
    Returns ``(ok, problems)``.
    """
    mpath = out_root / MANIFEST_NAME
    if not mpath.is_file():
        return False, [f"missing {mpath}"]
    problems: list[str] = []
    seen: set[str] = set()
    for line in mpath.read_text(encoding="ascii").splitlines():
        if not line.strip():
            continue
        rec = json.loads(line)
        rel = rec["path"]
        seen.add(rel)
        fp = out_root / rel
        if not fp.is_file():
            problems.append(f"missing file: {rel}")
            continue
        size, sha1 = _sha1_file(fp)
        if size != rec["size"] or sha1 != rec["sha1"]:
            problems.append(f"mismatch: {rel}")
    for rel in sorted(set(_list_files(out_root)) - seen):
        problems.append(f"unlisted on disk: {rel}")
    return (not problems), problems