subrepo asm-processor (#1212)

* yeet

* git subrepo clone git@github.com:simonlindholm/asm-processor.git tools/asm-processor

subrepo:
  subdir:   "tools/asm-processor"
  merged:   "bbd86ea1f"
upstream:
  origin:   "git@github.com:simonlindholm/asm-processor.git"
  branch:   "main"
  commit:   "bbd86ea1f"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"
This commit is contained in:
Anghelo Carvajal
2023-03-17 01:16:30 -03:00
committed by GitHub
parent a17949e730
commit 58022571ba
55 changed files with 3284 additions and 157 deletions
+296 -128
View File
@@ -85,6 +85,18 @@ MIPS_DEBUG_ST_STATIC = 2
MIPS_DEBUG_ST_STATIC_PROC = 14
class ElfFormat:
def __init__(self, is_big_endian):
self.is_big_endian = is_big_endian
self.struct_char = ">" if is_big_endian else "<"
def pack(self, fmt, *args):
return struct.pack(self.struct_char + fmt, *args)
def unpack(self, fmt, data):
return struct.unpack(self.struct_char + fmt, data)
class ElfHeader:
"""
typedef struct {
@@ -107,9 +119,9 @@ class ElfHeader:
def __init__(self, data):
self.e_ident = data[:EI_NIDENT]
self.e_type, self.e_machine, self.e_version, self.e_entry, self.e_phoff, self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize, self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx = struct.unpack('>HHIIIIIHHHHHH', data[EI_NIDENT:])
assert self.e_ident[EI_CLASS] == 1 # 32-bit
assert self.e_ident[EI_DATA] == 2 # big-endian
self.fmt = ElfFormat(is_big_endian=(self.e_ident[EI_DATA] == 2))
self.e_type, self.e_machine, self.e_version, self.e_entry, self.e_phoff, self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize, self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx = self.fmt.unpack('HHIIIIIHHHHHH', data[EI_NIDENT:])
assert self.e_type == 1 # relocatable
assert self.e_machine == 8 # MIPS I Architecture
assert self.e_phoff == 0 # no program header
@@ -117,7 +129,7 @@ class ElfHeader:
assert self.e_shstrndx != SHN_UNDEF
def to_bin(self):
return self.e_ident + struct.pack('>HHIIIIIHHHHHH', self.e_type,
return self.e_ident + self.fmt.pack('HHIIIIIHHHHHH', self.e_type,
self.e_machine, self.e_version, self.e_entry, self.e_phoff,
self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize,
self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx)
@@ -135,8 +147,9 @@ class Symbol:
} Elf32_Sym;
"""
def __init__(self, data, strtab, name=None):
self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx = struct.unpack('>IIIBBH', data)
def __init__(self, fmt, data, strtab, name=None):
self.fmt = fmt
self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx = fmt.unpack('IIIBBH', data)
assert self.st_shndx != SHN_XINDEX, "too many sections (SHN_XINDEX not supported)"
self.bind = st_info >> 4
self.type = st_info & 15
@@ -144,31 +157,32 @@ class Symbol:
self.visibility = self.st_other & 3
@staticmethod
def from_parts(st_name, st_value, st_size, st_info, st_other, st_shndx, strtab, name):
header = struct.pack('>IIIBBH', st_name, st_value, st_size, st_info, st_other, st_shndx)
return Symbol(header, strtab, name)
def from_parts(fmt, st_name, st_value, st_size, st_info, st_other, st_shndx, strtab, name):
header = fmt.pack('IIIBBH', st_name, st_value, st_size, st_info, st_other, st_shndx)
return Symbol(fmt, header, strtab, name)
def to_bin(self):
st_info = (self.bind << 4) | self.type
return struct.pack('>IIIBBH', self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx)
return self.fmt.pack('IIIBBH', self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx)
class Relocation:
def __init__(self, data, sh_type):
def __init__(self, fmt, data, sh_type):
self.fmt = fmt
self.sh_type = sh_type
if sh_type == SHT_REL:
self.r_offset, self.r_info = struct.unpack('>II', data)
self.r_offset, self.r_info = fmt.unpack('II', data)
else:
self.r_offset, self.r_info, self.r_addend = struct.unpack('>III', data)
self.r_offset, self.r_info, self.r_addend = fmt.unpack('III', data)
self.sym_index = self.r_info >> 8
self.rel_type = self.r_info & 0xff
def to_bin(self):
self.r_info = (self.sym_index << 8) | self.rel_type
if self.sh_type == SHT_REL:
return struct.pack('>II', self.r_offset, self.r_info)
return self.fmt.pack('II', self.r_offset, self.r_info)
else:
return struct.pack('>III', self.r_offset, self.r_info, self.r_addend)
return self.fmt.pack('III', self.r_offset, self.r_info, self.r_addend)
class Section:
@@ -187,8 +201,9 @@ class Section:
} Elf32_Shdr;
"""
def __init__(self, header, data, index):
self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize = struct.unpack('>IIIIIIIIII', header)
def __init__(self, fmt, header, data, index):
self.fmt = fmt
self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize = fmt.unpack('IIIIIIIIII', header)
assert not self.sh_flags & SHF_LINK_ORDER
if self.sh_entsize != 0:
assert self.sh_size % self.sh_entsize == 0
@@ -200,9 +215,9 @@ class Section:
self.relocated_by = []
@staticmethod
def from_parts(sh_name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data, index):
header = struct.pack('>IIIIIIIIII', sh_name, sh_type, sh_flags, 0, 0, len(data), sh_link, sh_info, sh_addralign, sh_entsize)
return Section(header, data, index)
def from_parts(fmt, sh_name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data, index):
header = fmt.pack('IIIIIIIIII', sh_name, sh_type, sh_flags, 0, 0, len(data), sh_link, sh_info, sh_addralign, sh_entsize)
return Section(fmt, header, data, index)
def lookup_str(self, index):
assert self.sh_type == SHT_STRTAB
@@ -222,7 +237,7 @@ class Section:
def header_to_bin(self):
if self.sh_type != SHT_NOBITS:
self.sh_size = len(self.data)
return struct.pack('>IIIIIIIIII', self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize)
return self.fmt.pack('IIIIIIIIII', self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize)
def late_init(self, sections):
if self.sh_type == SHT_SYMTAB:
@@ -251,14 +266,14 @@ class Section:
self.strtab = sections[self.sh_link]
entries = []
for i in range(0, self.sh_size, self.sh_entsize):
entries.append(Symbol(self.data[i:i+self.sh_entsize], self.strtab))
entries.append(Symbol(self.fmt, self.data[i:i+self.sh_entsize], self.strtab))
self.symbol_entries = entries
def init_relocs(self):
assert self.is_rel()
entries = []
for i in range(0, self.sh_size, self.sh_entsize):
entries.append(Relocation(self.data[i:i+self.sh_entsize], self.sh_type))
entries.append(Relocation(self.fmt, self.data[i:i+self.sh_entsize], self.sh_type))
self.relocations = entries
def local_symbols(self):
@@ -281,9 +296,9 @@ class Section:
hdrr_cbOptOffset, hdrr_iauxMax, hdrr_cbAuxOffset, hdrr_issMax, \
hdrr_cbSsOffset, hdrr_issExtMax, hdrr_cbSsExtOffset, hdrr_ifdMax, \
hdrr_cbFdOffset, hdrr_crfd, hdrr_cbRfdOffset, hdrr_iextMax, \
hdrr_cbExtOffset = struct.unpack(">HHIIIIIIIIIIIIIIIIIIIIIII", self.data[0:0x60])
hdrr_cbExtOffset = self.fmt.unpack("HHIIIIIIIIIIIIIIIIIIIIIII", self.data[0:0x60])
assert hdrr_magic == 0x7009 , "Invalid magic value for .mdebug symbolic header"
assert hdrr_magic == 0x7009, "Invalid magic value for .mdebug symbolic header"
hdrr_cbLineOffset += shift_by
hdrr_cbDnOffset += shift_by
@@ -297,7 +312,7 @@ class Section:
hdrr_cbRfdOffset += shift_by
hdrr_cbExtOffset += shift_by
new_data[0:0x60] = struct.pack(">HHIIIIIIIIIIIIIIIIIIIIIII", hdrr_magic, hdrr_vstamp, hdrr_ilineMax, hdrr_cbLine, \
new_data[0:0x60] = self.fmt.pack("HHIIIIIIIIIIIIIIIIIIIIIII", hdrr_magic, hdrr_vstamp, hdrr_ilineMax, hdrr_cbLine, \
hdrr_cbLineOffset, hdrr_idnMax, hdrr_cbDnOffset, hdrr_ipdMax, \
hdrr_cbPdOffset, hdrr_isymMax, hdrr_cbSymOffset, hdrr_ioptMax, \
hdrr_cbOptOffset, hdrr_iauxMax, hdrr_cbAuxOffset, hdrr_issMax, \
@@ -313,15 +328,16 @@ class ElfFile:
assert data[:4] == b'\x7fELF', "not an ELF file"
self.elf_header = ElfHeader(data[0:52])
self.fmt = self.elf_header.fmt
offset, size = self.elf_header.e_shoff, self.elf_header.e_shentsize
null_section = Section(data[offset:offset + size], data, 0)
null_section = Section(self.fmt, data[offset:offset + size], data, 0)
num_sections = self.elf_header.e_shnum or null_section.sh_size
self.sections = [null_section]
for i in range(1, num_sections):
ind = offset + i * size
self.sections.append(Section(data[ind:ind + size], data, i))
self.sections.append(Section(self.fmt, data[ind:ind + size], data, i))
symtab = None
for s in self.sections:
@@ -345,7 +361,7 @@ class ElfFile:
def add_section(self, name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data):
shstr = self.sections[self.elf_header.e_shstrndx]
sh_name = shstr.add_str(name)
s = Section.from_parts(sh_name=sh_name, sh_type=sh_type,
s = Section.from_parts(self.fmt, sh_name=sh_name, sh_type=sh_type,
sh_flags=sh_flags, sh_link=sh_link, sh_info=sh_info,
sh_addralign=sh_addralign, sh_entsize=sh_entsize, data=data,
index=len(self.sections))
@@ -421,15 +437,18 @@ class Failure(Exception):
class GlobalState:
def __init__(self, min_instr_count, skip_instr_count, use_jtbl_for_rodata, mips1):
def __init__(self, min_instr_count, skip_instr_count, use_jtbl_for_rodata, prelude_if_late_rodata, mips1, pascal):
# A value that hopefully never appears as a 32-bit rodata constant (or we
# miscompile late rodata). Increases by 1 in each step.
self.late_rodata_hex = 0xE0123456
self.valuectr = 0
self.namectr = 0
self.min_instr_count = min_instr_count
self.skip_instr_count = skip_instr_count
self.use_jtbl_for_rodata = use_jtbl_for_rodata
self.prelude_if_late_rodata = prelude_if_late_rodata
self.mips1 = mips1
self.pascal = pascal
def next_late_rodata_hex(self):
dummy_bytes = struct.pack('>I', self.late_rodata_hex)
@@ -443,6 +462,36 @@ class GlobalState:
self.namectr += 1
return '_asmpp_{}{}'.format(cat, self.namectr)
def func_prologue(self, name):
if self.pascal:
return " ".join([
"procedure {}();".format(name),
"type",
" pi = ^integer;",
" pf = ^single;",
" pd = ^double;",
"var",
" vi: pi;",
" vf: pf;",
" vd: pd;",
"begin",
" vi := vi;",
" vf := vf;",
" vd := vd;",
])
else:
return 'void {}(void) {{'.format(name)
def func_epilogue(self):
if self.pascal:
return "end;"
else:
return "}"
def pascal_assignment(self, tp, val):
self.valuectr += 1
address = (8 * self.valuectr) & 0x7FFF
return 'v{} := p{}({}); v{}^ := {};'.format(tp, tp, address, tp, val)
Function = namedtuple('Function', ['text_glabels', 'asm_conts', 'late_rodata_dummy_bytes', 'jtbl_rodata_size', 'late_rodata_asm_conts', 'fn_desc', 'data'])
@@ -476,6 +525,7 @@ class GlobalAsmBlock:
def count_quoted_size(self, line, z, real_line, output_enc):
line = line.encode(output_enc).decode('latin1')
in_quote = False
has_comma = True
num_parts = 0
ret = 0
i = 0
@@ -486,10 +536,15 @@ class GlobalAsmBlock:
if not in_quote:
if c == '"':
in_quote = True
if z and not has_comma:
self.fail(".asciiz with glued strings is not supported due to GNU as version diffs")
num_parts += 1
elif c == ',':
has_comma = True
else:
if c == '"':
in_quote = False
has_comma = False
continue
ret += 1
if c != '\\':
@@ -554,7 +609,7 @@ class GlobalAsmBlock:
self.text_glabels.append(line.split()[1])
if not line:
pass # empty line
elif line.startswith('glabel ') or (' ' not in line and line.endswith(':')):
elif line.startswith('glabel ') or line.startswith('dlabel ') or line.startswith('endlabel ') or (' ' not in line and line.endswith(':')):
pass # label
elif line.startswith('.section') or line in ['.text', '.data', '.rdata', '.rodata', '.bss', '.late_rodata']:
# section change
@@ -574,7 +629,7 @@ class GlobalAsmBlock:
changed_section = True
elif line.startswith('.incbin'):
self.add_sized(int(line.split(',')[-1].strip(), 0), real_line)
elif line.startswith('.word') or line.startswith('.float'):
elif line.startswith('.word') or line.startswith('.gpword') or line.startswith('.float'):
self.align4()
self.add_sized(4 * len(line.split(',')), real_line)
elif line.startswith('.double'):
@@ -651,7 +706,12 @@ class GlobalAsmBlock:
skip_next = False
needs_double = (self.late_rodata_alignment != 0)
extra_mips1_nop = False
jtbl_size = 11 if state.mips1 else 9
if state.pascal:
jtbl_size = 9 if state.mips1 else 8
jtbl_min_rodata_size = 2
else:
jtbl_size = 11 if state.mips1 else 9
jtbl_min_rodata_size = 5
for i in range(size):
if skip_next:
skip_next = False
@@ -668,9 +728,15 @@ class GlobalAsmBlock:
# - we have at least 10 more instructions to go in this function (otherwise our
# function size computation will be wrong since the delay slot goes unused)
if (not needs_double and state.use_jtbl_for_rodata and i >= 1 and
size - i >= 5 and num_instr - len(late_rodata_fn_output) >= jtbl_size + 1):
cases = " ".join("case {}:".format(case) for case in range(size - i))
late_rodata_fn_output.append("switch (*(volatile int*)0) { " + cases + " ; }")
size - i >= jtbl_min_rodata_size and
num_instr - len(late_rodata_fn_output) >= jtbl_size + 1):
if state.pascal:
cases = " ".join("{}: ;".format(case) for case in range(size - i))
line = "case 0 of " + cases + " otherwise end;"
else:
cases = " ".join("case {}:".format(case) for case in range(size - i))
line = "switch (*(volatile int*)0) { " + cases + " ; }"
late_rodata_fn_output.append(line)
late_rodata_fn_output.extend([""] * (jtbl_size - 1))
jtbl_rodata_size = (size - i) * 4
extra_mips1_nop = i != 2
@@ -681,7 +747,11 @@ class GlobalAsmBlock:
dummy_bytes2 = state.next_late_rodata_hex()
late_rodata_dummy_bytes.append(dummy_bytes2)
fval, = struct.unpack('>d', dummy_bytes + dummy_bytes2)
late_rodata_fn_output.append('*(volatile double*)0 = {};'.format(fval))
if state.pascal:
line = state.pascal_assignment('d', fval)
else:
line = '*(volatile double*)0 = {};'.format(fval)
late_rodata_fn_output.append(line)
skip_next = True
needs_double = False
if state.mips1:
@@ -691,7 +761,11 @@ class GlobalAsmBlock:
extra_mips1_nop = False
else:
fval, = struct.unpack('>f', dummy_bytes)
late_rodata_fn_output.append('*(volatile float*)0 = {}f;'.format(fval))
if state.pascal:
line = state.pascal_assignment('f', fval)
else:
line = '*(volatile float*)0 = {}f;'.format(fval)
late_rodata_fn_output.append(line)
extra_mips1_nop = True
late_rodata_fn_output.append('')
late_rodata_fn_output.append('')
@@ -701,8 +775,8 @@ class GlobalAsmBlock:
text_name = None
if self.fn_section_sizes['.text'] > 0 or late_rodata_fn_output:
text_name = state.make_name('func')
src[0] = 'void {}(void) {{'.format(text_name)
src[self.num_lines] = '}'
src[0] = state.func_prologue(text_name)
src[self.num_lines] = state.func_epilogue()
instr_count = self.fn_section_sizes['.text'] // 4
if instr_count < state.min_instr_count:
self.fail("too short .text block")
@@ -710,6 +784,7 @@ class GlobalAsmBlock:
tot_skipped = 0
fn_emitted = 0
fn_skipped = 0
skipping = True
rodata_stack = late_rodata_fn_output[::-1]
for (line, count) in self.fn_ins_inds:
for _ in range(count):
@@ -718,16 +793,28 @@ class GlobalAsmBlock:
# Don't let functions become too large. When a function reaches 284
# instructions, and -O2 -framepointer flags are passed, the IRIX
# compiler decides it is a great idea to start optimizing more.
# Also, Pascal cannot handle too large functions before it runs out
# of unique statements to write.
fn_emitted = 0
fn_skipped = 0
src[line] += ' }} void {}(void) {{ '.format(state.make_name('large_func'))
if fn_skipped < state.skip_instr_count:
skipping = True
src[line] += (' ' + state.func_epilogue() + ' ' +
state.func_prologue(state.make_name('large_func')) + ' ')
if (
skipping and
fn_skipped < state.skip_instr_count +
(state.prelude_if_late_rodata if rodata_stack else 0)
):
fn_skipped += 1
tot_skipped += 1
elif rodata_stack:
src[line] += rodata_stack.pop()
else:
src[line] += '*(volatile int*)0 = 0;'
skipping = False
if rodata_stack:
src[line] += rodata_stack.pop()
elif state.pascal:
src[line] += state.pascal_assignment('i', '0')
else:
src[line] += '*(volatile int*)0 = 0;'
tot_emitted += 1
fn_emitted += 1
if rodata_stack:
@@ -741,16 +828,24 @@ class GlobalAsmBlock:
rodata_name = None
if self.fn_section_sizes['.rodata'] > 0:
if state.pascal:
self.fail(".rodata isn't supported with Pascal for now")
rodata_name = state.make_name('rodata')
src[self.num_lines] += ' const char {}[{}] = {{1}};'.format(rodata_name, self.fn_section_sizes['.rodata'])
data_name = None
if self.fn_section_sizes['.data'] > 0:
data_name = state.make_name('data')
src[self.num_lines] += ' char {}[{}] = {{1}};'.format(data_name, self.fn_section_sizes['.data'])
if state.pascal:
line = ' var {}: packed array[1..{}] of char := [otherwise: 0];'.format(data_name, self.fn_section_sizes['.data'])
else:
line = ' char {}[{}] = {{1}};'.format(data_name, self.fn_section_sizes['.data'])
src[self.num_lines] += line
bss_name = None
if self.fn_section_sizes['.bss'] > 0:
if state.pascal:
self.fail(".bss isn't supported with Pascal")
bss_name = state.make_name('bss')
src[self.num_lines] += ' char {}[{}];'.format(bss_name, self.fn_section_sizes['.bss'])
@@ -775,43 +870,55 @@ float_regexpr = re.compile(r"[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?f")
def repl_float_hex(m):
return str(struct.unpack(">I", struct.pack(">f", float(m.group(0).strip().rstrip("f"))))[0])
def parse_source(f, opt, framepointer, mips1, input_enc, output_enc, out_dependencies, print_source=None):
if opt in ['O2', 'O1']:
if framepointer:
Opts = namedtuple('Opts', ['opt', 'framepointer', 'mips1', 'kpic', 'pascal', 'input_enc', 'output_enc'])
def parse_source(f, opts, out_dependencies, print_source=None):
if opts.opt in ['O1', 'O2']:
if opts.framepointer:
min_instr_count = 6
skip_instr_count = 5
else:
min_instr_count = 2
skip_instr_count = 1
elif opt == 'O0':
if framepointer:
elif opts.opt == 'O0':
if opts.framepointer:
min_instr_count = 8
skip_instr_count = 8
else:
min_instr_count = 4
skip_instr_count = 4
elif opt == 'g':
if framepointer:
elif opts.opt == 'g':
if opts.framepointer:
min_instr_count = 7
skip_instr_count = 7
else:
min_instr_count = 4
skip_instr_count = 4
else:
if opt != 'g3':
raise Failure("must pass one of -g, -O0, -O1, -O2, -O2 -g3")
if framepointer:
elif opts.opt == 'g3':
if opts.framepointer:
min_instr_count = 4
skip_instr_count = 4
else:
min_instr_count = 2
skip_instr_count = 2
else:
raise Failure("must pass one of -g, -O0, -O1, -O2, -O2 -g3")
prelude_if_late_rodata = 0
if opts.kpic:
# Without optimizations, the PIC prelude always takes up 3 instructions.
# With optimizations, the prelude is optimized out if there's no late rodata.
if opts.opt in ('g3', 'O2'):
prelude_if_late_rodata = 3
else:
min_instr_count += 3
skip_instr_count += 3
use_jtbl_for_rodata = False
if opt in ['O2', 'g3'] and not framepointer:
if opts.opt in ['O2', 'g3'] and not opts.framepointer and not opts.kpic:
use_jtbl_for_rodata = True
state = GlobalState(min_instr_count, skip_instr_count, use_jtbl_for_rodata, mips1)
state = GlobalState(min_instr_count, skip_instr_count, use_jtbl_for_rodata, prelude_if_late_rodata, opts.mips1, opts.pascal)
output_enc = opts.output_enc
global_asm = None
asm_functions = []
@@ -848,7 +955,7 @@ def parse_source(f, opt, framepointer, mips1, input_enc, output_enc, out_depende
fname = line[line.index('(') + 2 : -2]
out_dependencies.append(fname)
global_asm = GlobalAsmBlock(fname)
with open(fname, encoding=input_enc) as f:
with open(fname, encoding=opts.input_enc) as f:
for line2 in f:
global_asm.process_line(line2.rstrip(), output_enc)
src, fn = global_asm.finish(state)
@@ -870,8 +977,8 @@ def parse_source(f, opt, framepointer, mips1, input_enc, output_enc, out_depende
fname = os.path.join(fpath, line[line.index(' ') + 2 : -1])
out_dependencies.append(fname)
include_src = StringIO()
with open(fname, encoding=input_enc) as include_file:
parse_source(include_file, opt, framepointer, mips1, input_enc, output_enc, out_dependencies, include_src)
with open(fname, encoding=opts.input_enc) as include_file:
parse_source(include_file, opts, out_dependencies, include_src)
include_src.write('#line ' + str(line_no + 1) + ' "' + f.name + '"')
output_lines[-1] = include_src.getvalue()
include_src.close()
@@ -891,6 +998,7 @@ def parse_source(f, opt, framepointer, mips1, input_enc, output_enc, out_depende
for line in output_lines:
print_source.write(line + '\n')
else:
newline_encoded = "\n".encode(output_enc)
for line in output_lines:
try:
line_encoded = line.encode(output_enc)
@@ -899,18 +1007,18 @@ def parse_source(f, opt, framepointer, mips1, input_enc, output_enc, out_depende
print("The line:", line)
print("The line, utf-8-encoded:", line.encode("utf-8"))
raise
print_source.write(line_encoded + b'\n')
print_source.write(line_encoded)
print_source.write(newline_encoded)
print_source.flush()
if print_source != sys.stdout.buffer:
print_source.close()
return asm_functions
def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, drop_mdebug_gptab):
def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, drop_mdebug_gptab, convert_statics):
SECTIONS = ['.data', '.text', '.rodata', '.bss']
with open(objfile_name, 'rb') as f:
objfile = ElfFile(f.read())
fmt = objfile.fmt
prev_locs = {
'.text': 0,
@@ -949,6 +1057,11 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
loc = loc[1]
prev_loc = prev_locs[sectype]
if loc < prev_loc:
# If the dummy C generates too little asm, and we have two
# consecutive GLOBAL_ASM blocks, we detect that error here.
# On the other hand, if it generates too much, we don't have
# a good way of discovering that error: it's indistinguishable
# from a static symbol occurring after the GLOBAL_ASM block.
raise Failure("Wrongly computed size for section {} (diff {}). This is an asm-processor bug!".format(sectype, prev_loc- loc))
if loc != prev_loc:
asm.append('.section ' + sectype)
@@ -958,7 +1071,7 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
else:
asm.append('.space {}'.format(loc - prev_loc))
to_copy[sectype].append((loc, size, temp_name, function.fn_desc))
if function.text_glabels:
if function.text_glabels and sectype == '.text':
func_sizes[function.text_glabels[0]] = size
prev_locs[sectype] = loc + size
if not ifdefed:
@@ -980,7 +1093,10 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
if any(late_rodata_asm):
late_rodata_source_name_start = '_asmpp_late_rodata_start'
late_rodata_source_name_end = '_asmpp_late_rodata_end'
asm.append('.rdata')
asm.append('.section .late_rodata')
# Put some padding at the start to avoid conflating symbols with
# references to the whole section.
asm.append('.word 0, 0')
asm.append('glabel {}'.format(late_rodata_source_name_start))
for conts in late_rodata_asm:
asm.extend(conts)
@@ -1011,11 +1127,12 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
# Unify reginfo sections
target_reginfo = objfile.find_section('.reginfo')
source_reginfo_data = list(asm_objfile.find_section('.reginfo').data)
data = list(target_reginfo.data)
for i in range(20):
data[i] |= source_reginfo_data[i]
target_reginfo.data = bytes(data)
if target_reginfo is not None:
source_reginfo_data = list(asm_objfile.find_section('.reginfo').data)
data = list(target_reginfo.data)
for i in range(20):
data[i] |= source_reginfo_data[i]
target_reginfo.data = bytes(data)
# Move over section contents
modified_text_positions = set()
@@ -1052,7 +1169,7 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
# of another way of doing it.
moved_late_rodata = {}
if any(all_late_rodata_dummy_bytes) or any(all_jtbl_rodata_size):
source = asm_objfile.find_section('.rodata')
source = asm_objfile.find_section('.late_rodata')
target = objfile.find_section('.rodata')
source_pos = asm_objfile.symtab.find_symbol_in_section(late_rodata_source_name_start, source)
source_end = asm_objfile.symtab.find_symbol_in_section(late_rodata_source_name_end, source)
@@ -1061,6 +1178,8 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
new_data = list(target.data)
for dummy_bytes_list, jtbl_rodata_size in zip(all_late_rodata_dummy_bytes, all_jtbl_rodata_size):
for index, dummy_bytes in enumerate(dummy_bytes_list):
if not fmt.is_big_endian:
dummy_bytes = dummy_bytes[::-1]
pos = target.data.index(dummy_bytes, last_rodata_pos)
# This check is nice, but makes time complexity worse for large files:
if SLOW_CHECKS and target.data.find(dummy_bytes, pos + 4) != -1:
@@ -1096,7 +1215,7 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
# Find relocated symbols
relocated_symbols = set()
for sectype in SECTIONS:
for sectype in SECTIONS + ['.late_rodata']:
for obj in [asm_objfile, objfile]:
sec = obj.find_section(sectype)
if sec is None:
@@ -1106,50 +1225,60 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
relocated_symbols.add(obj.symtab.symbol_entries[rel.sym_index])
# Move over symbols, deleting the temporary function labels.
# Sometimes this naive procedure results in duplicate symbols, or UNDEF
# symbols that are also defined the same .o file. Hopefully that's fine.
# Skip over local symbols that aren't relocated against, to avoid
# conflicts.
new_local_syms = [s for s in objfile.symtab.local_symbols() if not is_temp_name(s.name)]
new_global_syms = [s for s in objfile.symtab.global_symbols() if not is_temp_name(s.name)]
# Skip over new local symbols that aren't relocated against, to
# avoid conflicts.
empty_symbol = objfile.symtab.symbol_entries[0]
new_syms = [s for s in objfile.symtab.symbol_entries[1:] if not is_temp_name(s.name)]
for i, s in enumerate(asm_objfile.symtab.symbol_entries):
is_local = (i < asm_objfile.symtab.sh_info)
if is_local and s not in relocated_symbols:
continue
if is_temp_name(s.name):
assert s not in relocated_symbols
continue
if s.st_shndx not in [SHN_UNDEF, SHN_ABS]:
section_name = asm_objfile.sections[s.st_shndx].name
if section_name not in SECTIONS:
raise Failure("generated assembly .o must only have symbols for .text, .data, .rodata, ABS and UNDEF, but found " + section_name)
s.st_shndx = objfile.find_section(section_name).index
target_section_name = section_name
if section_name == ".late_rodata":
target_section_name = ".rodata"
elif section_name not in SECTIONS:
raise Failure("generated assembly .o must only have symbols for .text, .data, .rodata, .late_rodata, ABS and UNDEF, but found " + section_name)
objfile_section = objfile.find_section(target_section_name)
if objfile_section is None:
raise Failure("generated assembly .o has section that real objfile lacks: " + target_section_name)
s.st_shndx = objfile_section.index
# glabel's aren't marked as functions, making objdump output confusing. Fix that.
if s.name in all_text_glabels:
s.type = STT_FUNC
if s.name in func_sizes:
s.st_size = func_sizes[s.name]
if objfile.sections[s.st_shndx].name == '.rodata' and s.st_value in moved_late_rodata:
if section_name == '.late_rodata':
if s.st_value == 0:
# This must be a symbol corresponding to the whole .late_rodata
# section, being referred to from a relocation.
# Moving local symbols is tricky, because it requires fixing up
# lo16/hi16 relocation references to .late_rodata+<offset>.
# Just disallow it for now.
raise Failure("local symbols in .late_rodata are not allowed")
s.st_value = moved_late_rodata[s.st_value]
s.st_name += strtab_adj
if is_local:
new_local_syms.append(s)
else:
new_global_syms.append(s)
new_syms.append(s)
make_statics_global = convert_statics in ("global", "global-with-filename")
# Add static symbols from .mdebug, so they can be referred to from GLOBAL_ASM
local_sym_replacements = {}
if mdebug_section:
if mdebug_section and convert_statics != "no":
strtab_index = len(objfile.symtab.strtab.data)
new_strtab_data = []
ifd_max, cb_fd_offset = struct.unpack('>II', mdebug_section.data[18*4 : 20*4])
cb_sym_offset, = struct.unpack('>I', mdebug_section.data[9*4 : 10*4])
cb_ss_offset, = struct.unpack('>I', mdebug_section.data[15*4 : 16*4])
ifd_max, cb_fd_offset = fmt.unpack('II', mdebug_section.data[18*4 : 20*4])
cb_sym_offset, = fmt.unpack('I', mdebug_section.data[9*4 : 10*4])
cb_ss_offset, = fmt.unpack('I', mdebug_section.data[15*4 : 16*4])
for i in range(ifd_max):
offset = cb_fd_offset + 18*4*i
iss_base, _, isym_base, csym = struct.unpack('>IIII', objfile.data[offset + 2*4 : offset + 6*4])
iss_base, _, isym_base, csym = fmt.unpack('IIII', objfile.data[offset + 2*4 : offset + 6*4])
for j in range(csym):
offset2 = cb_sym_offset + 12 * (isym_base + j)
iss, value, st_sc_index = struct.unpack('>III', objfile.data[offset2 : offset2 + 12])
iss, value, st_sc_index = fmt.unpack('III', objfile.data[offset2 : offset2 + 12])
st = (st_sc_index >> 26)
sc = (st_sc_index >> 21) & 0x1f
if st in [MIPS_DEBUG_ST_STATIC, MIPS_DEBUG_ST_STATIC_PROC]:
@@ -1157,44 +1286,72 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
symbol_name_offset_end = objfile.data.find(b'\0', symbol_name_offset)
assert symbol_name_offset_end != -1
symbol_name = objfile.data[symbol_name_offset : symbol_name_offset_end + 1]
symbol_name_str = symbol_name[:-1].decode('latin1')
emitted_symbol_name = symbol_name
if convert_statics == "global-with-filename":
# Change the emitted symbol name to include the filename,
# but don't let that affect deduplication logic.
emitted_symbol_name = objfile_name.encode("utf-8") + b":" + symbol_name
section_name = {1: '.text', 2: '.data', 3: '.bss', 15: '.rodata'}[sc]
section = objfile.find_section(section_name)
symtype = STT_FUNC if sc == 1 else STT_OBJECT
binding = STB_GLOBAL if make_statics_global else STB_LOCAL
sym = Symbol.from_parts(
fmt,
st_name=strtab_index,
st_value=value,
st_size=0,
st_info=(STB_LOCAL << 4 | symtype),
st_info=(binding << 4 | symtype),
st_other=STV_DEFAULT,
st_shndx=section.index,
strtab=objfile.symtab.strtab,
name=symbol_name_str)
local_sym_replacements[symbol_name_str] = len(new_local_syms)
strtab_index += len(symbol_name)
new_strtab_data.append(symbol_name)
new_local_syms.append(sym)
name=symbol_name[:-1].decode('latin1'))
strtab_index += len(emitted_symbol_name)
new_strtab_data.append(emitted_symbol_name)
new_syms.append(sym)
objfile.symtab.strtab.data += b''.join(new_strtab_data)
# To get the linker to use the local symbols, we have to get rid of UNDEF
# global ones.
newer_global_syms = []
for s in new_global_syms:
if s.st_shndx == SHN_UNDEF and s.name in local_sym_replacements:
s.new_index = local_sym_replacements[s.name]
# Get rid of duplicate symbols, favoring ones that are not UNDEF.
# Skip this for unnamed local symbols though.
new_syms.sort(key=lambda s: 0 if s.st_shndx != SHN_UNDEF else 1)
old_syms = []
newer_syms = []
name_to_sym = {}
for s in new_syms:
if s.name == "_gp_disp":
s.type = STT_OBJECT
if s.bind == STB_LOCAL and s.st_shndx == SHN_UNDEF:
raise Failure("local symbol \"" + s.name + "\" is undefined")
if not s.name:
if s.bind != STB_LOCAL:
raise Failure("global symbol with no name")
newer_syms.append(s)
else:
newer_global_syms.append(s)
new_global_syms = newer_global_syms
existing = name_to_sym.get(s.name)
if not existing:
name_to_sym[s.name] = s
newer_syms.append(s)
elif s.st_shndx != SHN_UNDEF:
raise Failure("symbol \"" + s.name + "\" defined twice")
else:
s.replace_by = existing
old_syms.append(s)
new_syms = newer_syms
# Put local symbols in front, with the initial dummy entry first, and
# _gp_disp at the end if it exists.
new_syms.insert(0, empty_symbol)
new_syms.sort(key=lambda s: (s.bind != STB_LOCAL, s.name == "_gp_disp"))
num_local_syms = sum(1 for s in new_syms if s.bind == STB_LOCAL)
new_syms = new_local_syms + new_global_syms
for i, s in enumerate(new_syms):
s.new_index = i
for s in old_syms:
s.new_index = s.replace_by.new_index
objfile.symtab.data = b''.join(s.to_bin() for s in new_syms)
objfile.symtab.sh_info = len(new_local_syms)
objfile.symtab.sh_info = num_local_syms
# Move over relocations
# Fix up relocation symbol references
for sectype in SECTIONS:
source = asm_objfile.find_section(sectype)
target = objfile.find_section(sectype)
if target is not None:
@@ -1206,34 +1363,38 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
sectype == '.rodata' and rel.r_offset in jtbl_rodata_positions):
# don't include relocations for late_rodata dummy code
continue
# hopefully we don't have relocations for local or
# temporary symbols, so new_index exists
rel.sym_index = objfile.symtab.symbol_entries[rel.sym_index].new_index
nrels.append(rel)
reltab.relocations = nrels
reltab.data = b''.join(rel.to_bin() for rel in nrels)
if not source:
# Move over relocations
for sectype in SECTIONS + ['.late_rodata']:
source = asm_objfile.find_section(sectype)
if source is None or not source.data:
continue
target_reltab = objfile.find_section('.rel' + sectype)
target_reltaba = objfile.find_section('.rela' + sectype)
target_sectype = '.rodata' if sectype == '.late_rodata' else sectype
target = objfile.find_section(target_sectype)
assert target is not None, target_sectype
target_reltab = objfile.find_section('.rel' + target_sectype)
target_reltaba = objfile.find_section('.rela' + target_sectype)
for reltab in source.relocated_by:
for rel in reltab.relocations:
rel.sym_index = asm_objfile.symtab.symbol_entries[rel.sym_index].new_index
if sectype == '.rodata' and rel.r_offset in moved_late_rodata:
if sectype == '.late_rodata':
rel.r_offset = moved_late_rodata[rel.r_offset]
new_data = b''.join(rel.to_bin() for rel in reltab.relocations)
if reltab.sh_type == SHT_REL:
if not target_reltab:
target_reltab = objfile.add_section('.rel' + sectype,
target_reltab = objfile.add_section('.rel' + target_sectype,
sh_type=SHT_REL, sh_flags=0,
sh_link=objfile.symtab.index, sh_info=target.index,
sh_addralign=4, sh_entsize=8, data=b'')
target_reltab.data += new_data
else:
if not target_reltaba:
target_reltaba = objfile.add_section('.rela' + sectype,
target_reltaba = objfile.add_section('.rela' + target_sectype,
sh_type=SHT_RELA, sh_flags=0,
sh_link=objfile.symtab.index, sh_info=target.index,
sh_addralign=4, sh_entsize=12, data=b'')
@@ -1257,9 +1418,12 @@ def run_wrapped(argv, outfile, functions):
parser.add_argument('--input-enc', default='latin1', help="input encoding (default: %(default)s)")
parser.add_argument('--output-enc', default='latin1', help="output encoding (default: %(default)s)")
parser.add_argument('--drop-mdebug-gptab', dest='drop_mdebug_gptab', action='store_true', help="drop mdebug and gptab sections")
parser.add_argument('--convert-statics', dest='convert_statics', choices=["no", "local", "global", "global-with-filename"], default="local", help="change static symbol visibility (default: %(default)s)")
parser.add_argument('--force', dest='force', action='store_true', help="force processing of files without GLOBAL_ASM blocks")
parser.add_argument('-framepointer', dest='framepointer', action='store_true')
parser.add_argument('-mips1', dest='mips1', action='store_true')
parser.add_argument('-g3', dest='g3', action='store_true')
parser.add_argument('-KPIC', dest='kpic', action='store_true')
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-O0', dest='opt', action='store_const', const='O0')
group.add_argument('-O1', dest='opt', action='store_const', const='O1')
@@ -1267,31 +1431,35 @@ def run_wrapped(argv, outfile, functions):
group.add_argument('-g', dest='opt', action='store_const', const='g')
args = parser.parse_args(argv)
opt = args.opt
pascal = any(args.filename.endswith(ext) for ext in (".p", ".pas", ".pp"))
if args.g3:
if opt != 'O2':
raise Failure("-g3 is only supported together with -O2")
opt = 'g3'
if args.mips1 and (opt != 'O2' or args.framepointer):
raise Failure("-mips1 is only supported together with -O2")
if args.mips1 and (opt not in ('O1', 'O2') or args.framepointer):
raise Failure("-mips1 is only supported together with -O1 or -O2")
if pascal and opt not in ('O1', 'O2', 'g3'):
raise Failure("Pascal is only supported together with -O1, -O2 or -O2 -g3")
opts = Opts(opt, args.framepointer, args.mips1, args.kpic, pascal, args.input_enc, args.output_enc)
if args.objfile is None:
with open(args.filename, encoding=args.input_enc) as f:
deps = []
functions = parse_source(f, opt=opt, framepointer=args.framepointer, mips1=args.mips1, input_enc=args.input_enc, output_enc=args.output_enc, out_dependencies=deps, print_source=outfile)
functions = parse_source(f, opts, out_dependencies=deps, print_source=outfile)
return functions, deps
else:
if args.assembler is None:
raise Failure("must pass assembler command")
if functions is None:
with open(args.filename, encoding=args.input_enc) as f:
functions = parse_source(f, opt=opt, framepointer=args.framepointer, mips1=args.mips1, input_enc=args.input_enc, out_dependencies=[], output_enc=args.output_enc)
if not functions:
functions = parse_source(f, opts, out_dependencies=[])
if not functions and not args.force:
return
asm_prelude = b''
if args.asm_prelude:
with open(args.asm_prelude, 'rb') as f:
asm_prelude = f.read()
fixup_objfile(args.objfile, functions, asm_prelude, args.assembler, args.output_enc, args.drop_mdebug_gptab)
fixup_objfile(args.objfile, functions, asm_prelude, args.assembler, args.output_enc, args.drop_mdebug_gptab, args.convert_statics)
def run(argv, outfile=sys.stdout.buffer, functions=None):
try: