mirror of
https://github.com/Druthulu/BFM-decomp
synced 2026-09-26 13:33:34 -04:00
Drew T
1353 lines
64 KiB
Python
1353 lines
64 KiB
Python
#!/usr/bin/env python3
|
|
"""tools/cdecl.py — THE C-declaration oracle. One parser; fifteen deleted models.
|
|
|
|
WHY THIS EXISTS (Phase 26-A tooling audit; R33 before R32)
|
|
===========================================================
|
|
Fifteen tools in this repo carry their own regex model of what a C declaration is. They disagree.
|
|
Two tools in ONE pipeline (reconcile_decls and canon_sig_reconcile) already disagree about whether
|
|
`extern s32 D_a, D_b;` is a declaration at all. The audit measured the cost of that:
|
|
|
|
* 6,384 of the fleet's 39,440 data symbols get NO canonical decl, ever.
|
|
* 3,717 symbols get an ACTIVELY WRONG one — a fleet plurality vote won by a minority spelling,
|
|
because every correctly-shaped decl was invisible to the regex that counted the votes.
|
|
* One unparsed `[4]` (engine_core.h:2114, `extern s32 D_80127530[4];`) has been single-handedly
|
|
blocking func_801387B8 — still INCLUDE_ASM in 134 TUs.
|
|
|
|
THE ROOT CAUSE IS ONE CHARACTER CLASS, COPIED FIFTEEN TIMES:
|
|
|
|
extern\\s+([A-Za-z_][\\w\\s\\*]*?\\bD_[0-9A-Fa-f]+\\s*(?:\\[\\s*\\])?)\\s*;
|
|
^^^^^^^^^ ^^^^^^^^
|
|
cannot hold '(' or ',' EMPTY brackets only
|
|
|
|
so three whole declarator shapes are invisible to every one of them:
|
|
|
|
extern void (*D_8018E858[])(void); <- a jump table. the '(' is not in [\\w\\s\\*]
|
|
extern s32 D_80127530[4]; <- a sized array. the '4' is not in \\[\\s*\\]
|
|
extern s32 D_a, D_b, D_c; <- 3 symbols. the ',' is not in [\\w\\s\\*]
|
|
(the WHOLE line is dropped, not just declarator 2..N)
|
|
|
|
Patching fifteen regexes is fifteen chances to diverge again. C's declarator grammar, by contrast,
|
|
is small, closed, and TOTAL — it describes those three shapes and every other one without being
|
|
told about them. So this module parses the grammar, and the fifteen models get deleted.
|
|
|
|
R33: the best outcome is a DELETED SCANNER, not a fixed regex.
|
|
|
|
WHAT IT DERIVES FROM, AND WHY THERE ARE TWO STATEMENT PATHS
|
|
===========================================================
|
|
1. A TU's file scope is decided by cpp, NOT by reading the .c.
|
|
`src/shared/engine_core.h` is 23,546 backslash-continued lines inside 1,801 `#define
|
|
DEFINE_func_*` macro bodies. A declaration in a macro BODY is not a declaration of anything —
|
|
it becomes one only where the macro is INVOKED, above the invocation point (the §8c law that
|
|
cost sessions 5 and 6). So a raw text scan of a TU is wrong in both directions: skip the
|
|
`#define`s and you miss all 1,801; read them and you invent ambient decls the TU never had.
|
|
`cpp` answers the question exactly, in 54 ms on the largest TU. That is why
|
|
`canon_sig_reconcile._file_scope_statements` is the ONE scanner the audit measured CLEAN, and
|
|
why `reconcile_tu._macro_externs()` — a hand-rolled model of macro expansion, `.rstrip('\\\\')`
|
|
and all — is a defect waiting to be deleted. -> `tu_statements()`
|
|
|
|
2. A DRAFT is rewritten in place, so it needs SPANS, and it has no macro bodies.
|
|
-> `split_statements()`, span-preserving, comment/string-masked, brace-depth-aware.
|
|
|
|
Both feed ONE parser.
|
|
|
|
THE COVERAGE ASSERTION IS ITSELF DERIVED (R32, and R33 applied to R32)
|
|
=====================================================================
|
|
Every other tool here hand-maintains an over-approximating candidate regex to measure its own
|
|
coverage. It does not need one. **At file scope, C admits nothing but declarations** — there are no
|
|
executable statements at depth 0. So the candidate set is *every depth-0 statement*, taken from the
|
|
C grammar itself, and the assertion is the strongest one available:
|
|
|
|
a depth-0 statement this parser cannot read is a DEFECT, not a shrug.
|
|
|
|
The only exclusions are NAMED and explicit (unexpanded macro invocations in raw draft text) — never
|
|
"whatever the regex happened not to match", which is the failure mode that wrote the fifteen bugs.
|
|
|
|
A SECOND ORACLE THAT CAN DISAGREE WITH ME (R34)
|
|
===============================================
|
|
A parser I merely believe is correct is worth exactly nothing — the audit exists because six people
|
|
believed their regexes were correct. So `--audit` hands every distinct declaration SHAPE in the
|
|
corpus to the real cross-gcc as
|
|
|
|
<the original declaration>
|
|
<this parser's reconstruction of it>
|
|
|
|
in one TU. If the parse got the type wrong in any way, gcc says `conflicting types` and names me.
|
|
That is not a self-check: it is the same C front end that arbitrates the build, and it cannot be
|
|
fooled by a plausible-looking regex.
|
|
|
|
Usage:
|
|
from cdecl import parse, scope, tu_statements, split_statements
|
|
tools/cdecl.py --audit [--gcc] [--limit N] # coverage + gcc + differential oracles
|
|
tools/cdecl.py --tu src/ov_SC01_077/ov_SC01_077.c [--above func_8012ACE0]
|
|
tools/cdecl.py --parse 'extern void (*D_8018E858[])(void);'
|
|
"""
|
|
import argparse
|
|
import functools
|
|
import glob as _glob
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
# The build's own preprocessor, with the build's own flags (docs/SETUP.md §5.4). Deriving the TU's
|
|
# file scope from anything else would be re-deriving what the build already guarantees.
|
|
CPP = ['mipsel-linux-gnu-cpp', '-lang-c', '-Iinclude', '-undef', '-Wall', '-fno-builtin',
|
|
'-Dmips', '-D__GNUC__=2', '-D__OPTIMIZE__', '-Dpsx', '-D_PSYQ', '-D_MIPSEL', '-D_LANGUAGE_C']
|
|
CC = 'mipsel-linux-gnu-gcc'
|
|
|
|
STORAGE = {'extern', 'static', 'register', 'auto', 'typedef', 'inline'}
|
|
QUALS = {'const', 'volatile'}
|
|
TYPEKW = {'void', 'char', 'short', 'int', 'long', 'float', 'double', 'signed', 'unsigned', '_Bool'}
|
|
TAGKW = {'struct', 'union', 'enum'}
|
|
GNU = {'__asm__', 'asm', '__attribute__', '__extension__', '__inline__'}
|
|
|
|
# Macro invocations that are NOT declarations in RAW draft/source text (they become declarations
|
|
# only after cpp). An EXPLICIT, NAMED exclusion list — the audit's rule: never let the exclusion set
|
|
# be "whatever the parser happened to choke on".
|
|
MACRO_STMT = re.compile(r'^\s*(INCLUDE_ASM)\b') # P35 T6: the dedup macro forms are gone; INCLUDE_ASM is the one macro statement left
|
|
|
|
|
|
class CDeclError(Exception):
|
|
"""A declaration this parser could not fully read. ALWAYS raised, never swallowed."""
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# lexing
|
|
# ---------------------------------------------------------------------------------------------
|
|
_TOKEN = re.compile(r"""
|
|
(?P<ws>\s+)
|
|
| (?P<id>[A-Za-z_]\w*)
|
|
| (?P<num>\.?\d[\w.]*(?:[eEpP][+-]\w*)?)
|
|
| (?P<str>"(?:\\.|[^"\\])*")
|
|
| (?P<chr>'(?:\\.|[^'\\])*')
|
|
| (?P<punct>\.\.\.|[-+*/%&|^!~<>=]=|<<=?|>>=?|&&|\|\||[][(){}.,;:*&+\-/%!~<>=?|^])
|
|
| (?P<other>.)
|
|
""", re.X)
|
|
|
|
|
|
class Tok:
|
|
__slots__ = ('kind', 'text', 'pos')
|
|
|
|
def __init__(self, kind, text, pos):
|
|
self.kind, self.text, self.pos = kind, text, pos
|
|
|
|
def __repr__(self):
|
|
return f'{self.kind}:{self.text!r}'
|
|
|
|
|
|
def tokenize(text):
|
|
"""-> [Tok] with source offsets. Comments are assumed already masked (see _mask).
|
|
|
|
The `other` catch-all means the lexer can NEVER die on an unexpected character (a `#directive`
|
|
inside a function body, say — bodies are consumed verbatim and never parsed). A surprising
|
|
character therefore becomes a token the PARSER rejects in context, naming the statement, rather
|
|
than a lexer crash naming a byte offset."""
|
|
out, i, n = [], 0, len(text)
|
|
while i < n:
|
|
m = _TOKEN.match(text, i)
|
|
i = m.end()
|
|
if m.lastgroup == 'ws':
|
|
continue
|
|
out.append(Tok(m.lastgroup, m.group(), m.start()))
|
|
return out
|
|
|
|
|
|
def _mask(text):
|
|
"""Length-preserving blank-out of comments and string/char literals, so brace/paren counting
|
|
and token scans cannot be fooled by a '}' in a comment or a '"' in a string. Offsets into the
|
|
mask are valid offsets into the original — that is what makes span-preserving rewrites safe."""
|
|
def blank(m):
|
|
return re.sub(r'[^\n]', ' ', m.group(0))
|
|
text = re.sub(r'/\*.*?\*/', blank, text, flags=re.S)
|
|
text = re.sub(r'//[^\n]*', blank, text)
|
|
text = re.sub(r'"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'', blank, text)
|
|
return text
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# statements — two paths, because the two inputs are semantically different
|
|
# ---------------------------------------------------------------------------------------------
|
|
class Stmt:
|
|
"""A depth-0 statement. `start`/`end` index the ORIGINAL text (spans survive masking)."""
|
|
__slots__ = ('text', 'start', 'end')
|
|
|
|
def __init__(self, text, start, end):
|
|
self.text, self.start, self.end = text, start, end
|
|
|
|
def __repr__(self):
|
|
return f'Stmt({self.text[:60]!r} @{self.start}:{self.end})'
|
|
|
|
|
|
def _depth0_spans(text):
|
|
"""Spans of every brace-depth-0 statement. A function DEFINITION flushes at its closing '}'
|
|
(the '{' followed a ')' — ANSI — or a ';' — a K&R declaration-list); everything else flushes at
|
|
its ';'. Preprocessor lines are skipped.
|
|
|
|
A col-0 LINE test is not a file-scope test, and this is why: m2c emits goto labels
|
|
(`done:`, `block_13:`) at column 0 INSIDE function bodies. Every tool that equates "starts at
|
|
column 0" with "is at file scope" is relying on a heuristic the corpus violates.
|
|
"""
|
|
m = _mask(text)
|
|
spans, depth, start, last = [], 0, None, ''
|
|
i, n = 0, len(m)
|
|
while i < n:
|
|
ch = m[i]
|
|
if ch == '#' and (i == 0 or m.rfind('\n', 0, i) == i - 1 - (len(m[:i]) - len(m[:i].rstrip(' \t')))):
|
|
pass # fall through; handled by the line skip below
|
|
if depth == 0 and start is None and not ch.isspace():
|
|
# skip a whole preprocessor line
|
|
line_start = m.rfind('\n', 0, i) + 1
|
|
if m[line_start:i].strip() == '' and ch == '#':
|
|
# Skip the whole LOGICAL preprocessor line, consuming \-continuations: a multi-line
|
|
# #define macro body is NOT a sequence of file-scope statements, so a raw-draft scan
|
|
# that stops at the first '\n' leaks the continuation lines as bogus declarations
|
|
# ('declaration with no declared name'). cpp-derived src TUs never hit this (the macro
|
|
# is already expanded); only the raw .run/drafts* scan does. (R33 — Phase-29.)
|
|
i = m.find('\n', i)
|
|
while i > 0 and m[i - 1] == '\\':
|
|
i = m.find('\n', i + 1)
|
|
if i < 0:
|
|
break
|
|
continue
|
|
start = i
|
|
if ch == '{':
|
|
if depth == 0:
|
|
# What precedes this brace decides what it is. Scan back over ALL preceding text,
|
|
# not just the current statement — because a K&R definition
|
|
# s32 f(arg0) s32 arg0; { ... }
|
|
# already flushed at the declaration-list's ';', leaving the body orphaned. (That
|
|
# is the same K&R shape that made progress.py read 670 banked definitions as
|
|
# forward declarations and count them as nothing.)
|
|
back = next((c for c in reversed(m[:i]) if not c.isspace()), '')
|
|
kr_body = start == i and spans and back == ';' # (start was set to i just above)
|
|
in_body = kr_body or (start != i and back in (')', ';'))
|
|
if not in_body:
|
|
depth += 1 # a struct body or an initializer: keep reading
|
|
i += 1
|
|
continue
|
|
d, i = 1, i + 1 # consume the body to its matching close
|
|
while i < n and d:
|
|
if m[i] == '{':
|
|
d += 1
|
|
elif m[i] == '}':
|
|
d -= 1
|
|
i += 1
|
|
if kr_body:
|
|
# A K&R declaration-list is a SEQUENCE of ';'-terminated declarations, so the
|
|
# prologue may have flushed as SEVERAL spans:
|
|
# void f(a0, arr) s32 a0; | s32 *arr; | { body }
|
|
# Merge backwards until the merged text parses to a FUNCTION declarator — the
|
|
# grammar's own answer to "where does this definition begin". Getting this
|
|
# wrong does not just misplace the body: it leaks the K&R parameter names
|
|
# (a0, arr) into the file-scope namespace as if they were globals.
|
|
j = len(spans) - 1
|
|
while j >= 0 and len(spans) - j <= 12:
|
|
try:
|
|
ds = parse(m[spans[j][0]:spans[-1][1]])
|
|
if ds and ds[0].kind == 'func':
|
|
break
|
|
except CDeclError:
|
|
pass
|
|
j -= 1
|
|
j = max(j, 0)
|
|
spans[j:] = [(spans[j][0], i)]
|
|
else:
|
|
spans.append((start, i))
|
|
start = None
|
|
continue
|
|
depth += 1
|
|
elif ch == '}':
|
|
depth -= 1
|
|
if depth < 0:
|
|
depth = 0
|
|
elif ch == ';' and depth == 0:
|
|
spans.append((start, i + 1))
|
|
start = None
|
|
i += 1
|
|
if start is not None and m[start:].strip():
|
|
spans.append((start, n))
|
|
return spans
|
|
|
|
|
|
def split_statements(text):
|
|
"""RAW, span-preserving depth-0 split. For DRAFTS (which get rewritten, so spans are mandatory,
|
|
and which contain no #define macro bodies)."""
|
|
return [Stmt(text[a:b], a, b) for a, b in _depth0_spans(text)]
|
|
|
|
|
|
_TU_CACHE = {}
|
|
|
|
|
|
def tu_statements(tu_path, above=None):
|
|
"""The file-scope statements the COMPILER sees in `tu_path` — cpp-derived (the §8c law).
|
|
|
|
`above`: a function name; truncate the TU at that function's INCLUDE_ASM stub, so the result is
|
|
exactly what is in scope at the splice point (declarations BELOW it are not visible to a body
|
|
spliced in above them).
|
|
|
|
This is the derived oracle that replaces reconcile_tu._macro_externs()+tu_visible() (a
|
|
hand-rolled model of what cpp does) and canon_sig_reconcile._file_scope_statements (correct, but
|
|
a second copy).
|
|
"""
|
|
key = (os.path.abspath(tu_path), above)
|
|
if key in _TU_CACHE:
|
|
return _TU_CACHE[key]
|
|
|
|
src = tu_path
|
|
tmp = None
|
|
if above:
|
|
text = open(tu_path).read()
|
|
m = re.search(r'^INCLUDE_ASM\("[^"]*",\s*' + re.escape(above) + r'\);\s*$', text, re.M)
|
|
if m:
|
|
# The temp MUST live beside the TU: cpp resolves #include "..." relative to the
|
|
# including file's own directory. Dot-prefixed so no src glob can ever see it.
|
|
tmp = os.path.join(os.path.dirname(tu_path), f'.cdecl_{above}.c')
|
|
open(tmp, 'w').write(text[:m.start()])
|
|
src = tmp
|
|
try:
|
|
p = subprocess.run(CPP + [src], capture_output=True, text=True, cwd=REPO)
|
|
if p.returncode:
|
|
raise CDeclError(f'cpp failed on {tu_path}: {p.stderr[-300:]}')
|
|
out = [s.text for s in split_statements(
|
|
'\n'.join(l for l in p.stdout.split('\n') if not l.startswith('#')))]
|
|
finally:
|
|
if tmp and os.path.exists(tmp):
|
|
os.remove(tmp)
|
|
_TU_CACHE[key] = out
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# the parser — C's declarator grammar, which is total over the shapes fifteen regexes enumerate
|
|
# ---------------------------------------------------------------------------------------------
|
|
PTR, ARR, FUN = 'PTR', 'ARR', 'FUN'
|
|
|
|
|
|
class Declarator:
|
|
"""One declared name, with its type read the way C reads it: OUTWARD from the name.
|
|
|
|
`chain` is the derivation list, name-first — exactly the C rule that regexes cannot express:
|
|
void (*D_x[])(void) -> chain [ARR, PTR, FUN] "D_x is an ARRAY of POINTERs to
|
|
FUNCTIONs returning void"
|
|
s32 D_x[4] -> chain [ARR(4)]
|
|
void func_X(s32) -> chain [FUN]
|
|
u8 *D_x -> chain [PTR]
|
|
"""
|
|
__slots__ = ('name', 'chain', 'base', 'storage', 'quals', 'params', 'pnames', 'is_proto',
|
|
'is_definition', 'text', 'span', '_fun')
|
|
|
|
def __init__(self, name, chain, base, storage, quals):
|
|
self.name, self.chain, self.base = name, chain, base
|
|
self.storage, self.quals = storage, quals
|
|
self.params, self.pnames, self.is_proto, self.is_definition = None, None, None, False
|
|
self.text, self.span, self._fun = None, None, None
|
|
for op in chain:
|
|
if op[0] == FUN:
|
|
self._fun = op
|
|
self.params, self.pnames = op[1], op[2]
|
|
self.is_proto = not op[4] # provisional; parse() resolves the K&R ambiguity
|
|
break
|
|
|
|
# -- the four questions every consumer in this repo actually asks -------------------------
|
|
@property
|
|
def kind(self):
|
|
"""'func' | 'fnptr' | 'fnptr_array' | 'array' | 'ptr' | 'scalar'. Derived from the chain,
|
|
so a fn-ptr can never be mistaken for a scalar — the mistake that elected 3,717 wrong
|
|
canonical decls."""
|
|
ops = [o[0] for o in self.chain]
|
|
if not ops:
|
|
return 'scalar'
|
|
if ops[0] == FUN:
|
|
return 'func'
|
|
if FUN in ops and PTR in ops:
|
|
return 'fnptr_array' if ops[0] == ARR else 'fnptr'
|
|
if ops[0] == ARR:
|
|
return 'array'
|
|
if ops[0] == PTR:
|
|
return 'ptr'
|
|
return 'scalar'
|
|
|
|
@property
|
|
def is_array(self):
|
|
return self.chain and self.chain[0][0] == ARR
|
|
|
|
@property
|
|
def type(self):
|
|
"""The abstract type: the declaration with the name removed. 'void (*[])(void)', 's32 [4]',
|
|
'u8 *'. This is what gcc compares, so this is what a canonical decl must preserve."""
|
|
return _render(self.base, self.chain, '')
|
|
|
|
def declaration(self, name=None, storage=None):
|
|
"""Reconstruct a valid C declaration. The gcc oracle (--audit --gcc) checks that this is
|
|
the SAME declaration as the source it was parsed from."""
|
|
st = self.storage if storage is None else storage
|
|
core = _render(self.base, self.chain, self.name if name is None else name)
|
|
return f'{st + " " if st else ""}{" ".join(self.quals) + " " if self.quals else ""}{core};'
|
|
|
|
def __repr__(self):
|
|
return f'Declarator({self.name}: {self.kind} = {self.type})'
|
|
|
|
|
|
def _ws(s):
|
|
"""Canonical spacing, so two declarations that MEAN the same thing RENDER the same. Consumers
|
|
compare types by string, so the renderer must be deterministic — always parse both sides and
|
|
compare renderings; never compare a rendering against raw source spacing."""
|
|
s = re.sub(r'\s+', ' ', s).strip()
|
|
return re.sub(r'(\w)\s*\(\s*\*', r'\1 (*', s) # `void(*)(void)` -> `void (*)(void)`
|
|
|
|
|
|
def _render(base, chain, name):
|
|
"""Rebuild declarator text from the chain, applying C's inside-out rule: a suffix ([] or ())
|
|
binds tighter than a prefix '*', so a '*' must be parenthesised whenever a suffix follows it."""
|
|
s = name
|
|
for i, op in enumerate(chain):
|
|
nxt = chain[i + 1][0] if i + 1 < len(chain) else None
|
|
if op[0] == PTR:
|
|
s = '*' + s
|
|
if nxt in (ARR, FUN): # *x[] means "array of ptr"; (*x)[] means "ptr to array"
|
|
s = '(' + s + ')'
|
|
elif op[0] == ARR:
|
|
s = f'{s}[{op[1] or ""}]'
|
|
elif op[0] == FUN:
|
|
params = ', '.join(op[1]) if op[1] else ('' if op[4] else 'void')
|
|
s = f'{s}({params})'
|
|
return _ws(f'{base} {s}'.strip() if s else base)
|
|
|
|
|
|
class _P:
|
|
"""Recursive-descent over the declarator grammar. Every failure raises."""
|
|
|
|
def __init__(self, toks, src):
|
|
self.t, self.i, self.src = toks, 0, src
|
|
|
|
def peek(self, k=0):
|
|
return self.t[self.i + k] if self.i + k < len(self.t) else Tok('eof', '', -1)
|
|
|
|
def take(self):
|
|
t = self.peek()
|
|
if t.kind == 'eof':
|
|
raise CDeclError(f'unexpected end of declaration: {self.src[:90]!r}')
|
|
self.i += 1
|
|
return t
|
|
|
|
def expect(self, s):
|
|
t = self.take()
|
|
if t.text != s:
|
|
raise CDeclError(f'expected {s!r}, got {t.text!r} in {self.src[:90]!r}')
|
|
return t
|
|
|
|
def skip_balanced(self, open_t='(', close_t=')'):
|
|
self.expect(open_t)
|
|
d, out = 1, []
|
|
while d:
|
|
t = self.take()
|
|
if t.text == open_t:
|
|
d += 1
|
|
elif t.text == close_t:
|
|
d -= 1
|
|
if not d:
|
|
break
|
|
out.append(t.text)
|
|
return ' '.join(out)
|
|
|
|
def skip_gnu(self):
|
|
"""__attribute__((...)) / __asm__("...") — GNU suffixes that decorate a declarator but
|
|
declare nothing. 1,109 and 27,334 occurrences respectively; a parser that dies on them is
|
|
useless here."""
|
|
while self.peek().text in GNU:
|
|
self.take()
|
|
if self.peek().text == '(':
|
|
self.skip_balanced()
|
|
|
|
# -- decl-specifiers ---------------------------------------------------------------------
|
|
def specifiers(self):
|
|
storage, quals, base, have_type = None, [], [], False
|
|
while True:
|
|
t = self.peek()
|
|
if t.text in STORAGE:
|
|
storage = self.take().text
|
|
continue
|
|
if t.text in QUALS:
|
|
quals.append(self.take().text)
|
|
continue
|
|
if t.text in TYPEKW:
|
|
base.append(self.take().text)
|
|
have_type = True
|
|
continue
|
|
if t.text in TAGKW:
|
|
base.append(self.take().text)
|
|
if self.peek().kind == 'id':
|
|
base.append(self.take().text)
|
|
if self.peek().text == '{':
|
|
base.append('{' + self.skip_balanced('{', '}') + '}')
|
|
have_type = True
|
|
continue
|
|
if t.text in GNU:
|
|
self.skip_gnu()
|
|
continue
|
|
if t.kind == 'id' and not have_type:
|
|
# A typedef-name, or the declared name of an implicit-int declaration?
|
|
# Without a symbol table C itself cannot tell — so use the only local evidence:
|
|
# `M2C_UNK func_X(...)` next is an ID -> this ID is the TYPE
|
|
# `Ent *D_x` next is '*' -> this ID is the TYPE
|
|
# `s32 (*D_801274D0)(s32)` next is '(' + '*' -> a GROUPED declarator follows,
|
|
# so this ID is the TYPE
|
|
# `func_X(s32)` next is '(' + type -> a param list: implicit int,
|
|
# so this ID is the NAME
|
|
nxt, nxt2 = self.peek(1), self.peek(2)
|
|
if nxt.kind == 'id' or nxt.text == '*' or \
|
|
(nxt.text == '(' and nxt2.text in ('*', '(')):
|
|
base.append(self.take().text)
|
|
have_type = True
|
|
continue
|
|
break
|
|
return storage, quals, ' '.join(base)
|
|
|
|
# -- declarators -------------------------------------------------------------------------
|
|
def suffixes(self):
|
|
out = []
|
|
while True:
|
|
t = self.peek()
|
|
if t.text == '[':
|
|
self.take()
|
|
dim = []
|
|
d = 1
|
|
while True:
|
|
t = self.take()
|
|
if t.text == '[':
|
|
d += 1
|
|
elif t.text == ']':
|
|
d -= 1
|
|
if not d:
|
|
break
|
|
dim.append(t.text)
|
|
out.append((ARR, ' '.join(dim) or None))
|
|
continue
|
|
if t.text == '(':
|
|
types, names, all_bare, empty = self.params()
|
|
out.append((FUN, types, names, all_bare, empty))
|
|
continue
|
|
break
|
|
return out
|
|
|
|
def params(self):
|
|
"""-> (types, names, all_bare, empty).
|
|
|
|
`empty` (`f()`) is the NO-PROTOTYPE form — gcc-2.7.2's K&R declaration, which accepts any
|
|
argument list — as distinct from `f(void)`, a prototype taking nothing. That distinction is
|
|
the entire subject matter of fix_arity_callers.py, which spends three regexes expressing
|
|
what the grammar states in one production.
|
|
|
|
`all_bare` (`f(a0, a1)` — every parameter a lone identifier) is AMBIGUOUS in C and cannot be
|
|
resolved here: it is a K&R identifier-list in a DEFINITION, but a prototype with typedef'd
|
|
parameter types in a DECLARATION (`M2C_UNK f(s32, s32);` — `s32` is a typedef, not a
|
|
keyword, so it is lexically identical to a K&R parameter name). The caller resolves it with
|
|
the one piece of evidence that settles it: whether a body follows.
|
|
"""
|
|
self.expect('(')
|
|
if self.peek().text == ')':
|
|
self.take()
|
|
return [], [], False, True # f() — no prototype
|
|
if self.peek().text == 'void' and self.peek(1).text == ')':
|
|
self.take(), self.take()
|
|
return [], [], False, False # f(void) — a prototype, no params
|
|
parts, cur, d = [], [], 0
|
|
while True:
|
|
t = self.take()
|
|
if t.text in '([':
|
|
d += 1
|
|
elif t.text in ')]':
|
|
if t.text == ')' and d == 0:
|
|
break
|
|
d -= 1
|
|
if t.text == ',' and d == 0:
|
|
parts.append(cur)
|
|
cur = []
|
|
continue
|
|
cur.append(t)
|
|
parts.append(cur)
|
|
types, names, all_bare = [], [], True
|
|
for p in parts:
|
|
if not (len(p) == 1 and p[0].kind == 'id' and p[0].text not in TYPEKW):
|
|
all_bare = False
|
|
types.append(_param_type(p, _spaced(p)))
|
|
names.append(_param_name(p))
|
|
return types, names, all_bare, False
|
|
|
|
|
|
def _spaced(toks):
|
|
out = ''
|
|
for t in toks:
|
|
if out and (t.kind in ('id', 'num') and out[-1] not in '*([ ,'):
|
|
out += ' '
|
|
out += t.text
|
|
return out
|
|
|
|
|
|
def _param_name(toks):
|
|
"""The parameter's NAME, if it has one. canon_sig_reconcile keeps the DRAFT's parameter names
|
|
while adopting the TU's canonical parameter TYPES — so both halves must be recoverable."""
|
|
if len(toks) >= 2 and toks[-1].kind == 'id' and toks[-1].text not in TYPEKW \
|
|
and toks[-2].text not in ('struct', 'union', 'enum'):
|
|
return toks[-1].text
|
|
m = re.match(r'^.*\(\s*\*\s*([A-Za-z_]\w*)\s*\).*$', _spaced(toks), re.S) # `void (*a0)(void)`
|
|
return m.group(1) if m else None
|
|
|
|
|
|
def _param_type(toks, txt):
|
|
"""A parameter's TYPE (the name, if any, removed): `s32 a0` -> `s32`, `void (*a0)(void)` ->
|
|
`void (*)(void)`, `s32 *a1` -> `s32 *`."""
|
|
if len(toks) >= 2 and toks[-1].kind == 'id' and toks[-2].text not in ('struct', 'union', 'enum'):
|
|
if toks[-1].text not in TYPEKW and toks[-2].kind != ',':
|
|
return _ws(_spaced(toks[:-1]))
|
|
m = re.match(r'^(.*\(\s*\*\s*)[A-Za-z_]\w*(\s*\).*)$', txt, re.S) # fn-ptr param with a name
|
|
if m:
|
|
return _ws(m.group(1) + m.group(2))
|
|
return _ws(txt)
|
|
|
|
|
|
def parse(stmt, span=None):
|
|
"""Parse ONE declaration -> [Declarator]. Raises CDeclError on anything it cannot fully read.
|
|
|
|
A statement that declares no object (`struct S { ... };`, a bare `;`) returns []."""
|
|
text = _mask(stmt)
|
|
p = _P(tokenize(text), stmt)
|
|
storage, quals, base = p.specifiers()
|
|
out = []
|
|
if p.peek().text in (';', ''):
|
|
return out # a tag definition; declares no name
|
|
while True:
|
|
stars = 0
|
|
while p.peek().text == '*':
|
|
p.take()
|
|
stars += 1
|
|
while p.peek().text in QUALS:
|
|
p.take()
|
|
name, chain = _direct(p)
|
|
chain = chain + [(PTR,)] * stars
|
|
p.skip_gnu()
|
|
if name is None:
|
|
raise CDeclError(f'declaration with no declared name: {stmt[:90]!r}')
|
|
d = Declarator(name, chain, base, storage, quals)
|
|
d.span = span
|
|
d.text = stmt.strip()
|
|
out.append(d)
|
|
t = p.peek()
|
|
if t.text == '=': # initializer — skip, balanced
|
|
p.take()
|
|
d_ = 0
|
|
while p.peek().kind != 'eof':
|
|
tt = p.peek().text
|
|
if tt in '([{':
|
|
d_ += 1
|
|
elif tt in ')]}':
|
|
d_ -= 1
|
|
elif (tt == ',' or tt == ';') and d_ <= 0:
|
|
break
|
|
p.take()
|
|
t = p.peek()
|
|
if t.text == ',':
|
|
p.take()
|
|
continue # `extern s32 D_a, D_b, D_c;`
|
|
if t.text == '{' or (t.kind == 'id' or t.text in ('*',)):
|
|
# a body, or a K&R declaration-list and then a body: this is a DEFINITION.
|
|
for dd in out:
|
|
dd.is_definition = True
|
|
return _resolve_kr(out)
|
|
if t.text == ';' or t.kind == 'eof':
|
|
return _resolve_kr(out)
|
|
raise CDeclError(f'unexpected {t.text!r} after declarator in {stmt[:90]!r}')
|
|
|
|
|
|
def _resolve_kr(decls):
|
|
"""Settle the one ambiguity C cannot settle locally: `f(a0, a1)` is a K&R identifier-list in a
|
|
DEFINITION, but a prototype with typedef'd parameter types in a DECLARATION. Now that we know
|
|
whether a body followed, decide — and for a K&R list, drop the "types" (they are parameter
|
|
NAMES; handing them to a consumer as types would be a lie)."""
|
|
for d in decls:
|
|
op = d._fun
|
|
if op is None:
|
|
continue
|
|
types, names, all_bare, empty = op[1], op[2], op[3], op[4]
|
|
if empty:
|
|
d.is_proto = False # `f()` — no prototype
|
|
elif all_bare and d.is_definition:
|
|
d.is_proto = False # K&R identifier-list
|
|
d.pnames, d.params = list(types), [] # those "types" are the names
|
|
else:
|
|
d.is_proto = True
|
|
return decls
|
|
|
|
|
|
def _direct(p):
|
|
"""direct-declarator := ID suffixes | '(' declarator ')' suffixes"""
|
|
t = p.peek()
|
|
if t.text == '(' and p.peek(1).text in ('*', '('):
|
|
# a GROUPED declarator — `(*D_x[])(void)`. (A '(' that is NOT followed by '*' or '(' is a
|
|
# parameter list on an abstract declarator, which `suffixes()` handles.)
|
|
p.take()
|
|
stars = 0
|
|
while p.peek().text == '*':
|
|
p.take()
|
|
stars += 1
|
|
while p.peek().text in QUALS:
|
|
p.take()
|
|
name, inner = _direct(p)
|
|
p.expect(')')
|
|
inner = inner + [(PTR,)] * stars
|
|
return name, inner + p.suffixes() # the group's suffixes apply OUTSIDE it
|
|
if t.kind == 'id':
|
|
p.take()
|
|
return t.text, p.suffixes()
|
|
return None, p.suffixes() # abstract declarator (a parameter)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# what consumers actually want
|
|
# ---------------------------------------------------------------------------------------------
|
|
def declarations(statements, path=''):
|
|
"""Parse a statement list -> [Declarator]. COVERAGE-ASSERTING (R32): at file scope C admits
|
|
nothing but declarations, so every statement MUST parse. A silent skip is a defect."""
|
|
out, bad = [], []
|
|
for st in statements:
|
|
text = st.text if isinstance(st, Stmt) else st
|
|
span = (st.start, st.end) if isinstance(st, Stmt) else None
|
|
if not text.strip().strip(';'):
|
|
continue
|
|
if MACRO_STMT.match(text): # NAMED, explicit exclusion — never a shrug
|
|
continue
|
|
try:
|
|
out.extend(parse(text, span))
|
|
except CDeclError as e:
|
|
bad.append((text[:100], str(e)))
|
|
if bad:
|
|
raise CDeclError(f'{path}: {len(bad)} file-scope statement(s) did not parse — a depth-0 '
|
|
f'statement C accepts and this parser cannot read is a DEFECT:\n' +
|
|
'\n'.join(f' {t!r}\n -> {e}' for t, e in bad[:5]))
|
|
return out
|
|
|
|
|
|
def scope(statements, path=''):
|
|
"""{name: Declarator} — the TU's file-scope namespace. FIRST declaration of a name wins (they
|
|
must already be mutually compatible: the TU compiles today)."""
|
|
ns = {}
|
|
for d in declarations(statements, path):
|
|
ns.setdefault(d.name, d)
|
|
return ns
|
|
|
|
|
|
def tu_scope(tu_path, above=None):
|
|
"""{name: Declarator} the compiler sees in `tu_path` (optionally above a stub). THE oracle that
|
|
every 'what does this TU declare' question in this repo should be asking.
|
|
|
|
⚠️ `above` answers *"can I USE this symbol without declaring it"* — the visibility question, which
|
|
drives the block-scope-demotion branch. It is the WRONG question for *"will my declaration
|
|
CONFLICT"*: C requires every declaration of a name in a TU to be compatible **regardless of
|
|
order**, so a declaration BELOW the splice point conflicts just as hard as one above. Use the
|
|
full scope (`above=None`) for conflict detection. (`canon_sig_reconcile` drives both decisions
|
|
off one `visible` set — which is why `func_801387B8` hits `conflicting types` against a decl
|
|
1,000 lines below its stub and the recovery pass reports nothing to fix.)"""
|
|
return scope(tu_statements(tu_path, above), tu_path)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# DRAFT TYPEDEF STRIP — one primitive, six copied regexes (Phase-27 T4)
|
|
# ---------------------------------------------------------------------------------------------
|
|
# A drafted matching-C body is written "self-contained": it carries its own `typedef unsigned char
|
|
# u8;` … so it compiles standalone. Spliced into a TU that already provides those names (via
|
|
# common.h / engine_types.h), C89/gcc-2.7.2 REJECTS the redefinition — a plumbing error, not a byte
|
|
# mismatch. Six tools each carried a scalar-name regex to strip the dups; they had COMPLEMENTARY
|
|
# holes (harvest_verify._TD lacked M2C_UNK; masked_diff.SCALAR_TYPEDEF_RE's `[^;]*` body could not
|
|
# cross a `;`, so a multi-typedef LINE stripped none; none handled a struct typedef the target
|
|
# provides), and each silently recorded the resulting compile failure as "not a match". These two
|
|
# functions replace all of it with the parser that already knows what a typedef is.
|
|
|
|
_TYPEDEF_HEAD = re.compile(r'\s*typedef\b')
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def typedef_names(tu_path, above=None):
|
|
"""The set of names the TU declares as typedefs (what a self-contained draft must NOT redefine).
|
|
|
|
Built on tu_statements (cpp-derived, NOT coverage-asserting) rather than tu_scope on purpose:
|
|
this feeds the byte-gate, and a cdecl coverage gap on some unrelated file-scope statement must
|
|
not be able to crash a matching run — only typedef statements are parsed, everything else is
|
|
ignored. PER-TU by design: engine_types.h reaches ~1470 of 1683 TUs, so a global header parse
|
|
would strip a `Blk16` the 213 non-including TUs still need. Cached via tu_statements' own cache."""
|
|
names = set()
|
|
for st in tu_statements(tu_path, above):
|
|
if not _TYPEDEF_HEAD.match(st):
|
|
continue
|
|
try:
|
|
for d in parse(st):
|
|
if d.storage == 'typedef':
|
|
names.add(d.name)
|
|
except CDeclError:
|
|
pass # a typedef this parser can't read -> don't strip on a guess
|
|
return names
|
|
|
|
|
|
def strip_provided_typedefs(draft_text, provided):
|
|
"""Remove from a DRAFT every depth-0 typedef statement whose declared name(s) are ALL in
|
|
`provided` (the target scope already supplies them). Splits multi-typedef lines correctly
|
|
(`split_statements` is depth-aware: `typedef ...;typedef ...;` -> two, `typedef struct{int
|
|
a;}F;` -> one), and covers scalar AND struct typedefs uniformly. KEEPS a draft-local typedef
|
|
whose name `provided` lacks, and KEEPS a mixed statement (some names provided, some not) so a
|
|
needed local type is never dropped — the byte-gate then arbitrates. `provided` empty -> no-op."""
|
|
if not provided:
|
|
return draft_text
|
|
drop = []
|
|
for st in split_statements(draft_text):
|
|
if not _TYPEDEF_HEAD.match(st.text):
|
|
continue
|
|
try:
|
|
ds = parse(st.text)
|
|
except CDeclError:
|
|
continue
|
|
if ds and all(d.storage == 'typedef' and d.name in provided for d in ds):
|
|
drop.append((st.start, st.end))
|
|
if not drop:
|
|
return draft_text
|
|
out, last = [], 0
|
|
for s, e in sorted(drop):
|
|
out.append(draft_text[last:s])
|
|
last = e
|
|
out.append(draft_text[last:])
|
|
return ''.join(out)
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# C TYPE COMPATIBILITY — the predicate four tools each half-implement, and get wrong
|
|
# ---------------------------------------------------------------------------------------------
|
|
# The question every recovery pass actually asks is "will gcc accept the draft's declaration
|
|
# alongside the TU's?" — and that is C's *compatible type* relation, which is NOT string equality:
|
|
#
|
|
# * `extern s32 D_x[];` IS compatible with `extern s32 D_x[4];` (incomplete vs complete array)
|
|
# * `extern s32 D_x;` is NOT compatible with `extern u32 D_x;` (signedness — gcc REJECTS it)
|
|
#
|
|
# Both existing implementations get this exactly backwards. `reconcile_decls._norm_type` collapses
|
|
# `s32|u32|int|unsigned|long` to ONE token, so it declares a signedness change "already compatible"
|
|
# and refuses to repair it — the compile then fails anyway, on a decl the tool looked at and
|
|
# approved. It is right about CODEGEN (same width, same load) and wrong about the C FRONT END, which
|
|
# rejects the redeclaration before codegen is ever reached.
|
|
#
|
|
# Validated against the real cross-gcc over every (TU, draft) declaration pair in the corpus:
|
|
# `tools/cdecl.py --audit --compat`.
|
|
|
|
_PRIM = {'char': 'char', 'short': 'short', 'int': 'int', 'long': 'long', 'float': 'float',
|
|
'double': 'double', 'void': 'void', 'signed': 'int', 'unsigned': 'unsigned int'}
|
|
|
|
|
|
@functools.lru_cache(1)
|
|
def _aliases():
|
|
"""typedef name -> underlying primitive, resolved transitively from the project's own prelude.
|
|
Derived by parsing common.h + engine_types.h with THIS parser — never a hand-kept table."""
|
|
al = {}
|
|
for h in ('include/common.h', 'src/shared/engine_types.h'):
|
|
p = os.path.join(REPO, h)
|
|
if not os.path.exists(p):
|
|
continue
|
|
for st in split_statements(open(p).read()):
|
|
try:
|
|
for d in parse(st.text):
|
|
if d.storage == 'typedef' and not d.chain:
|
|
al[d.name] = d.base
|
|
except CDeclError:
|
|
pass
|
|
for _ in range(4): # resolve chains (u32 -> unsigned int -> …)
|
|
al = {k: al.get(v, v) for k, v in al.items()}
|
|
return al
|
|
|
|
|
|
def _prim(base):
|
|
"""Resolve a base type to its underlying spelling, canonically ordered."""
|
|
al = _aliases()
|
|
words = [al.get(w, w) for w in (base or '').split()]
|
|
words = ' '.join(words).split()
|
|
if any(w in TAGKW for w in words):
|
|
return ' '.join(words) # struct/union/enum: identity by tag
|
|
order = {'unsigned': 0, 'signed': 0, 'long': 1, 'short': 1, 'char': 2, 'int': 3, 'float': 2}
|
|
core = sorted((w for w in words if w not in QUALS), key=lambda w: order.get(w, 4))
|
|
s = ' '.join(core)
|
|
return {'int unsigned': 'unsigned int', 'unsigned': 'unsigned int', 'signed': 'int',
|
|
'signed int': 'int', 'long int': 'long', 'unsigned long int': 'unsigned long'}.get(s, s)
|
|
|
|
|
|
def compatible(a, b):
|
|
"""CAN THESE TWO DECLARATIONS OF ONE NAME COEXIST IN ONE TU? — i.e. **will cc1 accept them.**
|
|
|
|
Not "are these the same type", and NOT what the C standard says. This models **gcc-2.7.2's
|
|
actual behaviour**, because gcc-2.7.2 is what compiles this project, and it is measurably laxer
|
|
than both the standard and modern gcc. Every rule below was either confirmed or REFUTED by
|
|
running the real `cc1` over the live corpus (`tools/cdecl.py --compat`, 1,485 real pairs):
|
|
|
|
* A TYPEDEF may not be redeclared at all — not even identically (`redefinition of 'X'`). Two
|
|
typedef declarations of one name can NEVER coexist, however equal their types. (Modern gcc
|
|
ALLOWS this — C11 relaxed it — which is exactly why the adjudicator must be cc1. This is
|
|
also why `_uniquify_draft_types` must strip-or-rename rather than compare.)
|
|
* QUALIFIERS DO NOT CONFLICT: cc1 accepts `extern u16 X;` beside `extern volatile u16 X;`
|
|
(modern gcc rejects it). REFUTED by the oracle; the rule was removed.
|
|
* THE NO-PROTOTYPE RULE IS **ORDER-DEPENDENT**, and that is the headline. Measured on cc1:
|
|
void X(s16); then void X(); -> ACCEPTS
|
|
void X(); then void X(s16); -> REJECTS (`conflicting types`)
|
|
void X(); then void X(s32)/X(void*) -> ACCEPTS (no default promotion)
|
|
i.e. a later PROTOTYPE must be compatible with the composite type the earlier `()` already
|
|
fixed (whose args are default-promoted) — but an earlier prototype simply wins.
|
|
⚠️ **This is the wall Phase 15 wrote up as "the `()` no-prototype escape can never work"** —
|
|
the basis of the "159 arity/narrow-param conflicts — no clean deterministic fix" dead-end.
|
|
It is only true in ONE DIRECTION. Put the narrow-param prototype FIRST and cc1 accepts it.
|
|
Whether the resulting CODEGEN matches is a separate question the byte-gate answers — but the
|
|
wall's stated cause does not hold. **Re-test target for A10.**
|
|
|
|
So `compatible(first, second)` takes them IN TU ORDER. Callers get the order from
|
|
`tu_scope(above=fn)`: a TU declaration above the splice point precedes the draft's; one below
|
|
follows it.
|
|
|
|
Being permissive is also the SAFE direction: a decl cc1 accepts needs no rewrite, so the draft
|
|
keeps the types it intended (its `volatile`, its narrow params — all load-bearing for codegen),
|
|
and the whole-binary byte-gate remains the sole arbiter of the bytes (G3/P9). Every needless
|
|
rewrite is a perturbation that can only lose a match.
|
|
"""
|
|
if a is None or b is None:
|
|
return True # nothing to conflict with
|
|
if a.storage == 'typedef' or b.storage == 'typedef':
|
|
return False # C89/2.7.2 forbids redeclaring a typedef
|
|
ca, cb = [o[0] for o in a.chain], [o[0] for o in b.chain]
|
|
if ca != cb:
|
|
return False # array vs ptr vs fn: different types
|
|
if _prim(a.base) != _prim(b.base):
|
|
return False # includes the signedness case cc1 rejects
|
|
for x, y in zip(a.chain, b.chain):
|
|
if x[0] == ARR and x[1] and y[1] and x[1] != y[1]:
|
|
return False # T[4] vs T[8]; T[] vs T[4] IS compatible
|
|
if x[0] == FUN:
|
|
xt, yt, xe, ye = x[1], y[1], x[4], y[4] # `empty` == a no-prototype declaration
|
|
if xe and not ye:
|
|
# `()` FIRST, prototype SECOND: the prototype must be compatible with the composite
|
|
# type the `()` already fixed — so no parameter may be altered by default promotion.
|
|
return not any(_prim(p.rstrip('* ')) in _PROMOTES and '*' not in p for p in yt)
|
|
if ye:
|
|
continue # prototype first, `()` second: cc1 accepts
|
|
if len(xt) != len(yt):
|
|
return False # arity
|
|
if any(_prim(p.rstrip('* ')) != _prim(q.rstrip('* ')) or ('*' in p) != ('*' in q)
|
|
for p, q in zip(xt, yt)):
|
|
return False
|
|
return True
|
|
|
|
|
|
# The types C's default argument promotions alter. A later prototype naming one of these is
|
|
# incompatible with an earlier `()` — the one direction in which the "no-prototype escape" really
|
|
# does fail (measured on cc1, not assumed from the standard).
|
|
_PROMOTES = {'char', 'short', 'unsigned char', 'unsigned short', 'signed char', 'float'}
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------
|
|
# THE THREE ORACLES. The deliverable of this module is a MEASUREMENT, not a belief.
|
|
# ---------------------------------------------------------------------------------------------
|
|
def _tus():
|
|
return (sorted(_glob.glob(os.path.join(REPO, 'src/ov_*/*.c'))) +
|
|
sorted(_glob.glob(os.path.join(REPO, 'src/*.c'))) +
|
|
sorted(_glob.glob(os.path.join(REPO, 'src/resident/*.c'))))
|
|
|
|
|
|
def _drafts():
|
|
return sorted(_glob.glob(os.path.join(REPO, '.run/drafts*/*.c')))
|
|
|
|
|
|
def audit_coverage(limit=None, verbose=False):
|
|
"""ORACLE 1 (R32) — COVERAGE, with a candidate set taken from the C grammar rather than from a
|
|
hand-maintained regex: at file scope C admits NOTHING BUT DECLARATIONS, so every depth-0
|
|
statement is a candidate and any one this parser cannot read is a DEFECT."""
|
|
tus, drafts = _tus(), _drafts()
|
|
if limit:
|
|
tus, drafts = tus[:limit], drafts[:limit * 10]
|
|
stmts = decls = 0
|
|
bad = []
|
|
|
|
for p in tus: # TUs: cpp-derived (the §8c law)
|
|
try:
|
|
sts = tu_statements(p)
|
|
except CDeclError as e:
|
|
bad.append((p, str(e)))
|
|
continue
|
|
stmts += len(sts)
|
|
for st in sts:
|
|
if not st.strip().strip(';') or MACRO_STMT.match(st):
|
|
continue
|
|
try:
|
|
decls += len(parse(st))
|
|
except CDeclError as e:
|
|
bad.append((p, f'{st[:90]!r} -> {e}'))
|
|
|
|
for p in drafts: # drafts: raw, span-preserving
|
|
try:
|
|
sts = split_statements(open(p, errors='replace').read())
|
|
except Exception as e:
|
|
bad.append((p, str(e)))
|
|
continue
|
|
stmts += len(sts)
|
|
for st in sts:
|
|
if not st.text.strip().strip(';') or MACRO_STMT.match(st.text):
|
|
continue
|
|
try:
|
|
decls += len(parse(st.text))
|
|
except CDeclError as e:
|
|
bad.append((p, f'{st.text[:90]!r} -> {e}'))
|
|
|
|
# ADJUDICATE MY OWN GAP WITH GCC, rather than deciding for myself which failures "don't count".
|
|
# A statement gcc ALSO rejects is not C, so rejecting it is correct and the INPUT is corrupt; a
|
|
# statement gcc ACCEPTS and this parser does not is MY defect. That verdict belongs to the C
|
|
# front end, not to me — the whole audit exists because six people graded their own homework.
|
|
defects, notc = [], []
|
|
for p, e in bad:
|
|
stmt = e.split(' -> ')[0].strip("'\"")
|
|
# The scratch dir is created here (P34 S93): it lived under .run/audit/, which a scratch prune removed, and the
|
|
# write raised — the bare `except` below then counted every unparsed line as a parser DEFECT (the last close had
|
|
# read the same four lines as NOT-C). An adjudication that cannot run is a REFUSAL, never a verdict (R43/R57).
|
|
src = os.path.join(REPO, '.run/audit/cdecl/adj.c')
|
|
os.makedirs(os.path.dirname(src), exist_ok=True)
|
|
try:
|
|
open(src, 'w').write(_PROBE_HDR + stmt.encode().decode('unicode_escape') + '\n')
|
|
r = subprocess.run([CC, '-fsyntax-only', '-w', '-I' + os.path.join(REPO, 'include'),
|
|
'-I' + os.path.join(REPO, 'src'), src],
|
|
capture_output=True, text=True, cwd=REPO)
|
|
except Exception as ex:
|
|
sys.exit(f'cdecl --audit: cannot adjudicate an unparsed statement with gcc ({ex}); refusing to grade it either way (R43)')
|
|
(notc if r.returncode else defects).append((p, e))
|
|
|
|
print(f'[coverage] {len(tus)} TUs (cpp-derived) + {len(drafts)} drafts (raw)')
|
|
print(f'[coverage] depth-0 statements: {stmts} declarators parsed: {decls}')
|
|
print(f'[coverage] unparsed: {len(bad)} -> PARSER DEFECTS (gcc accepts, cdecl does not): '
|
|
f'{len(defects)} NOT-C (gcc rejects it too — corrupt input): {len(notc)}')
|
|
for p, e in defects[:20 if verbose else 8]:
|
|
print(f' DEFECT {os.path.relpath(p, REPO)}: {e}')
|
|
if notc:
|
|
dirs = sorted({os.path.dirname(os.path.relpath(p, REPO)) for p, _ in notc})
|
|
print(f' NOT-C in {len(dirs)} dir(s) (all scratch, none in src/): {dirs[:6]}')
|
|
print(f' e.g. {notc[0][1][:100]}')
|
|
return not defects
|
|
|
|
|
|
_PROBE_HDR = '#include "common.h"\n#include "shared/engine_types.h"\n'
|
|
|
|
|
|
@functools.lru_cache(1)
|
|
def _known_types():
|
|
"""Every type name common.h + engine_types.h already define — parsed with THIS parser (the
|
|
probe environment must not redefine them)."""
|
|
names = set(TYPEKW) | set(QUALS) | set(TAGKW)
|
|
for h in ('include/common.h', 'src/shared/engine_types.h'):
|
|
p = os.path.join(REPO, h)
|
|
if not os.path.exists(p):
|
|
continue
|
|
for st in split_statements(open(p).read()):
|
|
try:
|
|
for d in parse(st.text):
|
|
if d.storage == 'typedef':
|
|
names.add(d.name)
|
|
except CDeclError:
|
|
pass
|
|
return names
|
|
|
|
|
|
def _synth(decls):
|
|
"""Synthesize a typedef for every TU-LOCAL type a statement names (Vec3s, CdlFILE, E_3B7AC…),
|
|
so gcc can adjudicate a declaration lifted out of its own TU. Safe: BOTH the original and the
|
|
reconstruction see the same synthetic type, so a mangled type still shows up as a mismatch —
|
|
what this cannot do is hide a defect."""
|
|
known, out = _known_types(), set()
|
|
for d in decls:
|
|
for tok in re.findall(r'[A-Za-z_]\w*', (d.base or '') + ' ' + ' '.join(d.params or [])):
|
|
if tok not in known and not re.match(r'^(void|struct|union|enum)$', tok):
|
|
out.add(tok)
|
|
return ''.join(f'typedef int {t};\n' for t in sorted(out))
|
|
|
|
|
|
def _gcc_probe(items, tag, with_recon):
|
|
"""Compile the originals (and optionally this parser's reconstructions) -> gcc's stderr."""
|
|
d = os.path.join(REPO, '.run/audit/cdecl')
|
|
os.makedirs(d, exist_ok=True)
|
|
# UNIQUE PER CALL (P31 S70). This was `probe_{tag}.c` — one fixed filename per tag — which is
|
|
# correct only while _sift is serial. Parallel probes sharing a tag would overwrite each other's
|
|
# source between write and compile and return a verdict about ANOTHER chunk's declarations: a
|
|
# plausible wrong answer, the defect class this session kept finding. The name is per-call now,
|
|
# so the fan-out below cannot alias.
|
|
src = os.path.join(d, f'probe_{tag}_{next(_PROBE_SEQ)}.c')
|
|
with open(src, 'w') as f:
|
|
f.write(_PROBE_HDR)
|
|
f.write(''.join(sorted({s for _, _, syn in items for s in syn.splitlines(True)})))
|
|
for i, (orig, recons, _syn) in enumerate(items):
|
|
f.write(f'/* {i} */ {orig}\n')
|
|
if with_recon:
|
|
for r in recons:
|
|
f.write(f'{r}\n')
|
|
p = subprocess.run([CC, '-fsyntax-only', '-w', '-I' + os.path.join(REPO, 'include'),
|
|
'-I' + os.path.join(REPO, 'src'), src],
|
|
capture_output=True, text=True, cwd=REPO)
|
|
return p.stderr
|
|
|
|
|
|
def _sift(items, with_recon, tag):
|
|
"""-> (accepted, rejected). Batch, bisect only on failure, and run the batches CONCURRENTLY.
|
|
|
|
PARALLEL (P31 S70). Each chunk is an independent gcc invocation, and gcc is the whole cost here
|
|
(`--gcc` documents itself as "slow, decisive"): `make audit-cdecl` was >9 min of tools-health's
|
|
~30, all of it one cc1 at a time on a 32-core box. Threads are right despite the GIL because the
|
|
work happens in `subprocess.run`, which releases it. Results are written back BY INDEX so the
|
|
accepted/rejected lists stay in input order and the run remains deterministic."""
|
|
B = 200
|
|
chunks = [items[b:b + B] for b in range(0, len(items), B)]
|
|
if not chunks:
|
|
return [], []
|
|
nw = max(1, min(32, (os.cpu_count() or 4)))
|
|
per = [None] * len(chunks)
|
|
|
|
def one_chunk(i, chunk):
|
|
if not _gcc_probe(chunk, tag, with_recon):
|
|
return i, list(chunk), []
|
|
# the chunk failed: bisect it, and the individual probes are independent too
|
|
cok, cbad = [None] * len(chunk), [None] * len(chunk)
|
|
|
|
def one_item(j, it):
|
|
e = _gcc_probe([it], tag + '1', with_recon)
|
|
if e:
|
|
cbad[j] = (it, e.strip().split('\n')[0])
|
|
else:
|
|
cok[j] = it
|
|
with cf.ThreadPoolExecutor(max_workers=nw) as ex2:
|
|
list(ex2.map(lambda a: one_item(*a), list(enumerate(chunk))))
|
|
return i, [x for x in cok if x is not None], [x for x in cbad if x is not None]
|
|
|
|
with cf.ThreadPoolExecutor(max_workers=nw) as ex:
|
|
for i, cok, cbad in ex.map(lambda a: one_chunk(*a), list(enumerate(chunks))):
|
|
per[i] = (cok, cbad)
|
|
ok, bad = [], []
|
|
for cok, cbad in per:
|
|
ok.extend(cok)
|
|
bad.extend(cbad)
|
|
return ok, bad
|
|
|
|
|
|
def _tu_statements_list(p):
|
|
"""Picklable worker for the parallel collection pass (ProcessPoolExecutor needs a module-level fn)."""
|
|
try:
|
|
return list(tu_statements(p))
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def audit_gcc(limit=None):
|
|
"""ORACLE 2 (R34) — THE C FRONT END, WHICH CAN DISAGREE WITH ME.
|
|
|
|
For every distinct declaration in the corpus, hand gcc the ORIGINAL and this parser's
|
|
RECONSTRUCTION of it, declaring the same name in one TU. If the parse got the type wrong in any
|
|
way, gcc answers `conflicting types` and names the line. This is not a self-check — it is the
|
|
same front end that arbitrates the build, and it cannot be fooled by a plausible regex.
|
|
|
|
(Definitions are excluded: their bodies cannot compile out of context. Declarations are what the
|
|
fifteen scanners operate on.)"""
|
|
seen, pairs = set(), []
|
|
# NOT PARALLELISED, and the measurement is the point (P31 S70). `tu_statements` over 4,168 TUs is
|
|
# ~787s single-core and is the WHOLE of audit-cdecl's >9min: inside a 10-minute run the
|
|
# `[gcc] N distinct declarations` line never printed, so not one cc1 call had happened yet.
|
|
# A ProcessPoolExecutor over TUs was tried and REVERTED: 12 TUs yield 32,352 statements, so the
|
|
# full pass ships ~11M strings back through IPC and the pickling costs more than the parse it
|
|
# saves. The fix is to dedupe/filter INSIDE the worker (return each TU's distinct candidate
|
|
# declarations, not its raw statement list) — most of those 11M are the same extern lines
|
|
# repeated across TUs — or to memoise per-TU results by content hash. Left measured, not guessed.
|
|
for p in _tus()[:limit] if limit else _tus():
|
|
for st in tu_statements(p):
|
|
if not st.strip().strip(';') or MACRO_STMT.match(st) or st in seen:
|
|
continue
|
|
seen.add(st)
|
|
try:
|
|
ds = parse(st)
|
|
except CDeclError:
|
|
continue # oracle 1 owns unparsed statements
|
|
if not ds or any(d.is_definition for d in ds) or any(d.storage == 'typedef' for d in ds):
|
|
continue
|
|
i = len(pairs)
|
|
orig, recons = st, []
|
|
for d in ds: # rename so 200 can share one probe TU
|
|
new = f'{d.name}_p{i}'
|
|
orig = re.sub(r'\b' + re.escape(d.name) + r'\b', new, orig)
|
|
recons.append(d.declaration(name=new))
|
|
pairs.append((orig, recons, _synth(ds)))
|
|
|
|
print(f'[gcc] {len(pairs)} distinct declarations -> the real cross-compiler')
|
|
|
|
# A: can gcc compile the ORIGINAL at all? If not, the probe environment lacks that TU's local
|
|
# types and CANNOT adjudicate — an honest, COUNTED skip, never a silent one.
|
|
judged, env = _sift(pairs, False, 'a')
|
|
# B: now add MY reconstruction. A declaration that compiled alone and does NOT compile beside
|
|
# my reconstruction of it means the two are DIFFERENT TYPES — i.e. I parsed it wrong.
|
|
good, bad = _sift(judged, True, 'b')
|
|
|
|
print(f'[gcc] adjudicated: {len(judged)} env-skipped (gcc cannot compile the original out of '
|
|
f'its TU): {len(env)}')
|
|
print(f'[gcc] accepted: {len(good)} REJECTED (my reconstruction is a DIFFERENT TYPE): {len(bad)}')
|
|
for (orig, recons, _s), e in bad[:10]:
|
|
print(f' DISAGREES orig: {orig.strip()[:88]}')
|
|
print(f' mine: {recons}')
|
|
print(f' gcc : {e}')
|
|
return not bad
|
|
|
|
|
|
import concurrent.futures as cf
|
|
import itertools
|
|
_PROBE_SEQ = itertools.count()
|
|
|
|
CC1 = os.path.join(REPO, 'tools/bin/gcc-2.7.2-psx/cc1')
|
|
CC1FLAGS = ['-quiet', '-O2', '-G0', '-mips1', '-mcpu=3000', '-mgas', '-msoft-float', '-fgnu-linker']
|
|
_ERR = re.compile(r'error|redefinition|conflicting|incompatible|redeclar|parse error', re.I)
|
|
|
|
|
|
def _cc1_accepts(body):
|
|
"""Does THE REAL BUILD FRONT END (gcc-2.7.2 `cc1`) accept this? — not modern gcc.
|
|
|
|
They DISAGREE, and it matters: C11 permits redefining a typedef to the same type, C89/2.7.2 does
|
|
NOT (`redefinition of 'X'`). Validating a compatibility rule against a compiler that is not the
|
|
one compiling the code is *exactly* the failure this module exists to prevent — a recovery pass
|
|
that "approves" a declaration the real front end then refuses."""
|
|
d = os.path.join(REPO, '.run/audit/cdecl')
|
|
os.makedirs(d, exist_ok=True)
|
|
c, i = os.path.join(d, 'cc1probe.c'), os.path.join(d, 'cc1probe.i')
|
|
open(c, 'w').write(body)
|
|
p = subprocess.run(CPP + ['-Isrc', c], capture_output=True, text=True, cwd=REPO)
|
|
if p.returncode:
|
|
return None # cannot adjudicate
|
|
open(i, 'w').write(p.stdout)
|
|
r = subprocess.run([CC1] + CC1FLAGS + [i, '-o', '/dev/null'],
|
|
capture_output=True, text=True, cwd=REPO)
|
|
return not (r.returncode or _ERR.search(r.stderr or ''))
|
|
|
|
|
|
def _synth_distinct(decls):
|
|
"""Synthesize each unknown type as a DISTINCT struct. Emitting `typedef int Vec8;` would collapse
|
|
`extern Vec8 X;` and `extern int X;` into the same type and the probe would call them compatible
|
|
— the oracle destroying the very distinction it is meant to test."""
|
|
known, out = _known_types(), set()
|
|
for d in decls:
|
|
for tok in re.findall(r'[A-Za-z_]\w*', (d.base or '') + ' ' + ' '.join(d.params or [])):
|
|
if tok not in known and tok not in ('void', 'struct', 'union', 'enum'):
|
|
out.add(tok)
|
|
return ''.join(f'typedef struct {{ int _{t}; }} {t};\n' for t in sorted(out))
|
|
|
|
|
|
def audit_compat(limit=2500):
|
|
"""ORACLE 4 — `compatible()` vs the REAL cc1, on every (TU-decl, draft-decl) pair the corpus
|
|
actually contains. My predicate says accept/reject; gcc-2.7.2 says accept/reject. Any
|
|
disagreement is MY bug, and it is the kind that silently caps the whole recovery pipeline."""
|
|
tus = sorted(_glob.glob(os.path.join(REPO, 'src/ov_SC01_077/*.c')))
|
|
scopes = {t: tu_scope(t) for t in tus}
|
|
pairs = {}
|
|
import random
|
|
for p in random.Random(11).sample(_drafts(), min(limit, len(_drafts()))):
|
|
try:
|
|
txt = open(p, errors='replace').read()
|
|
except Exception:
|
|
continue
|
|
for st in split_statements(txt):
|
|
try:
|
|
ds = parse(st.text)
|
|
except CDeclError:
|
|
continue
|
|
for d in ds:
|
|
if d.is_definition:
|
|
continue
|
|
for ns in scopes.values():
|
|
tu = ns.get(d.name)
|
|
if tu is None:
|
|
continue
|
|
key = (tu.declaration(name='X'), d.declaration(name='X'))
|
|
pairs.setdefault(key, (compatible(tu, d), (tu, d)))
|
|
|
|
print(f'[compat] {len(pairs)} distinct (TU-decl, draft-decl) pairs from the real corpus')
|
|
print(f'[compat] adjudicator: gcc-2.7.2 cc1 (THE BUILD FRONT END), not modern gcc')
|
|
bad, skip = [], 0
|
|
for (tu_d, dr_d), (mine, (tu, d)) in pairs.items():
|
|
body = _PROBE_HDR + _synth_distinct([tu, d]) + tu_d + '\n' + dr_d + '\n'
|
|
got = _cc1_accepts(body)
|
|
if got is None:
|
|
skip += 1
|
|
elif got != mine:
|
|
bad.append(((tu_d, dr_d), mine, got))
|
|
print(f'[compat] agree: {len(pairs) - len(bad) - skip} DISAGREE: {len(bad)} unadjudicable: {skip}')
|
|
for (k, mine, got) in bad[:10]:
|
|
print(f' TU : {k[0]}\n draft: {k[1]}')
|
|
print(f' cdecl says {"compatible" if mine else "CONFLICT"}, cc1 says '
|
|
f'{"compatible" if got else "CONFLICT"}\n')
|
|
# An UNADJUDICABLE pair is a silent skip, and a gate that skips its whole corpus and prints GREEN
|
|
# is the exact bug this module was written to hunt. (It did that here, once: a missing `-Isrc`
|
|
# made cpp fail on all 1,485 probes, `bad` stayed empty, and the audit reported success. R32 is
|
|
# not "fail loud" — it is COUNT WHAT YOU SKIPPED.)
|
|
if skip:
|
|
print(f' !! {skip} pair(s) could not be adjudicated — a skipped check is NOT a passed one')
|
|
return not bad and not skip
|
|
|
|
|
|
def audit_differential():
|
|
"""ORACLE 3 — vs THE FIFTEEN INCUMBENTS. This parser must find a strict SUPERSET of every
|
|
scanner it replaces: any symbol an incumbent sees and this one does not is a defect in THIS
|
|
tool. The reverse (symbols only this one sees) is the measured size of the hole."""
|
|
sys.path.insert(0, os.path.join(REPO, 'tools'))
|
|
# reconcile_decls RETIRED (S45, R33): its DATA_DECL_LINE_RE row proved the superset for
|
|
# 26-A..S44; the incumbent is deleted, so the differential now compares the survivors only.
|
|
# P35 T6: the macro header is gone for good; the differential's only honest result is the statement below.
|
|
if True:
|
|
# Phase 35: the macro header is gone (per-function headers under src/shared/<space>/ hold plain C that tu_scope reads
|
|
# like any TU). The fifteen-incumbent differential was a comparison OVER MACRO TEXT; without that text it has nothing
|
|
# to compare against, and saying so is the honest result (R43 — never fabricate a synthetic TU from the new form).
|
|
print('[differential] the macro-era shared header is gone (Phase 35): the macro-era differential no longer applies — OK')
|
|
return True
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument('--audit', action='store_true', help='oracle 1 (coverage) + oracle 3 (differential)')
|
|
ap.add_argument('--gcc', action='store_true', help='oracle 2: the C front end (slow, decisive)')
|
|
ap.add_argument('--compat', action='store_true',
|
|
help='oracle 4: compatible() vs the REAL gcc-2.7.2 cc1 on live corpus pairs')
|
|
ap.add_argument('--limit', type=int, help='sample N TUs (a fast smoke run)')
|
|
ap.add_argument('--tu', help='print the file-scope scope of a TU')
|
|
ap.add_argument('--above', help='...truncated above this function\'s INCLUDE_ASM stub')
|
|
ap.add_argument('--parse', help='parse one declaration and print it')
|
|
ap.add_argument('-v', '--verbose', action='store_true')
|
|
a = ap.parse_args()
|
|
|
|
if a.parse:
|
|
for d in parse(a.parse):
|
|
print(f'{d.name}: kind={d.kind} type={d.type!r} proto={d.is_proto} '
|
|
f'def={d.is_definition} params={d.params} names={d.pnames}')
|
|
print(f' reconstruction: {d.declaration()}')
|
|
return
|
|
if a.tu:
|
|
ns = tu_scope(a.tu, a.above)
|
|
for n, d in sorted(ns.items()):
|
|
print(f' {n:<24} {d.kind:<12} {d.type}')
|
|
print(f'{len(ns)} file-scope names visible' + (f' above {a.above}' if a.above else ''))
|
|
return
|
|
if a.audit or a.gcc or a.compat:
|
|
ok = True
|
|
if a.audit:
|
|
ok &= audit_coverage(a.limit, a.verbose)
|
|
print()
|
|
ok &= audit_differential()
|
|
if a.gcc:
|
|
print()
|
|
ok &= audit_gcc(a.limit)
|
|
if a.compat:
|
|
print()
|
|
ok &= audit_compat()
|
|
print('\n' + ('cdecl: ALL ORACLES GREEN' if ok else 'cdecl: DEFECTS FOUND (see above)'))
|
|
sys.exit(0 if ok else 1)
|
|
ap.print_help()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|