[Audio 8/?] Check-in handwritten sequences, build sequences, automate various sfx arrays (#2137)

* [Audio 8/?] Check-in handwritten sequences, build sequences, automate various sfx arrays

* Fix whitespace in aseq.h

* Fix sequence 0 sfx id generator

* Suggested changes, adjust some MML syntax and add more instruction descriptions

* Correct some formatting in aseq.h

* Add the dir of the input .seq file to the list of includes to sequence assembling so that assembler-level includes like .include or .incbin work intuitively

* aseq.h tweaks

* MM review suggestions, aseq.h adjustments
This commit is contained in:
Tharo
2024-09-15 23:26:27 +01:00
committed by GitHub
parent 03a1502706
commit dbbeb656f8
34 changed files with 22983 additions and 1657 deletions
+315 -25
View File
@@ -8,6 +8,7 @@
*/
#include <assert.h>
#include <ctype.h>
#include <regex.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
@@ -17,24 +18,9 @@
#include "samplebank.h"
#include "soundfont.h"
#include "xml.h"
#include "elf32.h"
#include "util.h"
/* Utility */
static bool
is_xml(const char *path)
{
size_t len = strlen(path);
if (len < 4)
return false;
if (path[len - 4] == '.' && tolower(path[len - 3]) == 'x' && tolower(path[len - 2]) == 'm' &&
tolower(path[len - 1]) == 'l')
return true;
return false;
}
/* Samplebanks */
static int
@@ -46,8 +32,9 @@ tablegen_samplebanks(const char *sb_hdr_out, const char **samplebanks_paths, int
for (int i = 0; i < num_samplebank_files; i++) {
const char *path = samplebanks_paths[i];
size_t pathlen = strlen(path);
if (!is_xml(path))
if (!str_endswith(path, pathlen, ".xml"))
error("Not an xml file? (\"%s\")", path);
xmlDocPtr document = xmlReadFile(path, NULL, XML_PARSE_NONET);
@@ -189,7 +176,7 @@ validate_samplebank_index(soundfont *sf, samplebank *sb, int ptr_idx)
}
}
int
static int
tablegen_soundfonts(const char *sf_hdr_out, char **soundfonts_paths, int num_soundfont_files)
{
soundfont *soundfonts = malloc(num_soundfont_files * sizeof(soundfont));
@@ -197,8 +184,9 @@ tablegen_soundfonts(const char *sf_hdr_out, char **soundfonts_paths, int num_sou
for (int i = 0; i < num_soundfont_files; i++) {
char *path = soundfonts_paths[i];
size_t pathlen = strlen(path);
if (!is_xml(path))
if (!str_endswith(path, pathlen, ".xml"))
error("Not an xml file? (\"%s\")", path);
xmlDocPtr document = xmlReadFile(path, NULL, XML_PARSE_NONET);
@@ -213,7 +201,6 @@ tablegen_soundfonts(const char *sf_hdr_out, char **soundfonts_paths, int num_sou
// Transform the xml path into a header include path
// Assumption: replacing .xml -> .h forms a valid header include path
size_t pathlen = strlen(path);
path[pathlen - 3] = 'h';
path[pathlen - 2] = '\0';
@@ -290,6 +277,298 @@ tablegen_soundfonts(const char *sf_hdr_out, char **soundfonts_paths, int num_sou
return EXIT_SUCCESS;
}
/* Sequences */
struct seq_order_entry {
const char *name;
const char *enum_name;
bool isptr;
};
struct seq_order {
size_t num_sequences;
struct seq_order_entry *entries;
void *filedata;
};
static void
read_seq_order(struct seq_order *order, const char *path)
{
// Read from file, we assume the file has been preprocessed with cpp so that whitespace is collapsed and each line
// has the form:
// (name,enum_name) or *(name,enum_name)
UNUSED size_t data_size;
char *filedata = util_read_whole_file(path, &data_size);
// We expect one entry per line, gather the total length
size_t total_size = 0;
for (char *p = filedata; *p != '\0'; p++) {
if (*p == '\n') {
total_size++;
} else if (isspace(*p)) {
// There should be no whitespace in the input file besides newlines
goto malformed;
}
}
// Alloc entries
struct seq_order_entry *entries = malloc(total_size * sizeof(struct seq_order_entry));
enum matchno {
MATCH_ALL,
MATCH_PTR,
MATCH_NAME,
MATCH_ENUM,
MATCH_NUM
};
regmatch_t match[MATCH_NUM];
regex_t re;
// Matches either
// (<c_identifier>,<c_identifier>) for non-pointer entries
// or
// *(<c_identifier>,<c_identifier>) for pointer entries
const char *line_regexp = "^(\\*?)\\(([_a-zA-Z][_a-zA-Z0-9]*),([_a-zA-Z][_a-zA-Z0-9]*)\\)$";
int status = regcomp(&re, line_regexp, REG_EXTENDED);
assert(status == 0 && "Failed to compile sequence order regular expression?");
char *lstart = filedata;
for (size_t i = 0; i < total_size; i++) {
// find end of line
char *p = lstart;
while (*p != '\n') {
assert(*p != '\0');
p++;
}
char *lend = p;
// null-terminate the line (replaces the newline)
*lend = '\0';
// try to match the regular expression
status = regexec(&re, lstart, MATCH_NUM, match, 0);
if (status != 0) {
// failed to match, malformed input file
char ebuf[128];
regerror(status, &re, ebuf, sizeof(ebuf));
fprintf(stderr, "Failed to match line %lu: \"%s\"\nregexec error: \"%s\"\n", i, lstart, ebuf);
goto malformed;
}
// if the group is empty we're not a pointer, else we are
entries[i].isptr = match[MATCH_PTR].rm_so != match[MATCH_PTR].rm_eo;
// get the name
entries[i].name = &lstart[match[MATCH_NAME].rm_so];
lstart[match[MATCH_NAME].rm_eo] = '\0'; // replaces ,
// get the enum name
entries[i].enum_name = &lstart[match[MATCH_ENUM].rm_so];
lstart[match[MATCH_ENUM].rm_eo] = '\0'; // replaces )
// next line
lstart = lend + 1;
}
assert(*lstart == '\0');
// Write results
order->num_sequences = total_size;
order->entries = entries;
order->filedata = filedata;
return;
malformed:
error("Malformed %s?", path);
}
struct seqdata {
const char *elf_path;
const char *name;
uint32_t font_section_offset;
size_t font_section_size;
};
static int
tablegen_sequences(const char *seq_font_tbl_out, const char *seq_order_path, const char **sequences_paths,
int num_sequence_files)
{
struct seq_order order;
read_seq_order(&order, seq_order_path);
#ifdef SEQ_DEBUG
// Print the sequence order
printf("Sequence order:\n");
for (size_t i = 0; i < order.num_sequences; i++) {
printf(" name=\"%s\" enum=\"%s\" ptr=%d\n", order.entries[i].name, order.entries[i].enum_name,
order.entries[i].isptr);
}
#endif
struct seqdata *file_data = malloc(num_sequence_files * sizeof(struct seqdata));
// Read and validate the sequence object files
for (int i = 0; i < num_sequence_files; i++) {
const char *path = sequences_paths[i];
if (!str_endswith(path, strlen(path), ".o"))
error("Not a .o file? (\"%s\")", path);
// Open ELF file
size_t data_size;
void *data = elf32_read(path, &data_size);
Elf32_Shdr *symtab = elf32_get_symtab(data, data_size);
if (symtab == NULL)
error("ELF file \"%s\" has no symbol table?", path);
Elf32_Shdr *shstrtab = elf32_get_shstrtab(data, data_size);
if (shstrtab == NULL)
error("ELF file \"%s\" has no section header string table?", path);
// The .fonts and .name sections are written when assembling the sequence:
// The .fonts section contains a list of bytes for each soundfont the sequences uses
// The .name section contains the null-terminated name of the sequence as set by .startseq
Elf32_Shdr *font_section = elf32_section_forname(".fonts", shstrtab, data, data_size);
if (font_section == NULL)
error("Sequence file \"%s\" has no fonts section?", path);
uint32_t font_section_offset = elf32_read32(font_section->sh_offset);
uint32_t font_section_size = elf32_read32(font_section->sh_size);
validate_read(font_section_offset, font_section_size, data_size);
Elf32_Shdr *name_section = elf32_section_forname(".name", shstrtab, data, data_size);
if (name_section == NULL)
error("Sequence file \"%s\" has no name section?", path);
uint32_t name_section_offset = elf32_read32(name_section->sh_offset);
uint32_t name_section_size = elf32_read32(name_section->sh_size);
validate_read(name_section_offset, name_section_size, data_size);
const char *seq_name = GET_PTR(data, name_section_offset);
if (strnlen(seq_name, name_section_size + 1) >= name_section_size)
error("Sequence file \"%s\" name is not properly terminated?", path);
// Populate new data
struct seqdata *seqdata = &file_data[i];
seqdata->elf_path = strdup(path);
seqdata->name = strdup(seq_name);
seqdata->font_section_offset = font_section_offset;
seqdata->font_section_size = font_section_size;
free(data);
}
#ifdef SEQ_DEBUG
// Debugging: Print the findings for each sequence object
printf("\nNum files: %d\n\n", num_sequence_files);
for (int i = 0; i < num_sequence_files; i++) {
struct seqdata *seqdata = &file_data[i];
printf(
// clang-format off
" elf path : \"%s\"" "\n"
" name : \"%s\"" "\n"
" font offset : 0x%X" "\n"
" num fonts : %lu" "\n\n",
// clang-format on
seqdata->elf_path, seqdata->name, seqdata->font_section_offset, seqdata->font_section_size);
}
#endif
// Link against the sequence order coming from the sequence table header
struct seqdata **final_seqdata = calloc(order.num_sequences, sizeof(struct seqdata *));
for (size_t i = 0; i < order.num_sequences; i++) {
// Skip pointers for now
if (order.entries[i].isptr)
continue;
// If it's not a pointer, "name" is the name as it appears in a sequence file, find it in the list of sequences
const char *name = order.entries[i].name;
// Find the object file with this name
for (int j = 0; j < num_sequence_files; j++) {
struct seqdata *seqdata = &file_data[j];
if (strequ(name, seqdata->name)) {
// Found name, done
final_seqdata[i] = seqdata;
break;
}
}
}
for (size_t i = 0; i < order.num_sequences; i++) {
// Now we only care about pointers
if (!order.entries[i].isptr)
continue;
// If it's a pointer, "name" is the ENUM name of the sequence it points to
const char *name = order.entries[i].name;
for (size_t j = 0; j < order.num_sequences; j++) {
// Skip pointers, the system doesn't allow multiple indirection so this must point to a non-pointer entry.
if (order.entries[j].isptr)
continue;
if (strequ(name, order.entries[j].enum_name)) {
// For pointers, we just duplicate the fonts for the original into the pointer entry.
// TODO ideally we would allow fonts to be different when a sequence is accessed by pointer, but how
// to supply this info?
final_seqdata[i] = final_seqdata[j];
break;
}
}
}
// Make sure we found an object file for all declared sequences
for (size_t i = 0; i < order.num_sequences; i++) {
if (final_seqdata[i] == NULL)
error("Could not find object file for sequence %lu : %s", i, order.entries[i].name);
}
// Write the sequence font table out
FILE *out = fopen(seq_font_tbl_out, "w");
if (out == NULL)
error("Failed to open output file \"%s\" for writing", seq_font_tbl_out);
fprintf(out,
// clang-format off
"\n"
".section .rodata" "\n"
"\n"
".global gSequenceFontTable" "\n"
"gSequenceFontTable:" "\n"
// clang-format on
);
// Write the 16-bit offsets for each sequence
for (size_t i = 0; i < order.num_sequences; i++) {
fprintf(out, " .half Fonts_%lu - gSequenceFontTable\n", i);
}
fprintf(out, "\n");
// Write the fonts for each sequence: number of fonts followed by an incbin for the rest.
for (size_t i = 0; i < order.num_sequences; i++) {
fprintf(out,
// clang-format off
"Fonts_%lu:" "\n"
" .byte %ld" "\n"
" .incbin \"%s\", 0x%X, %lu" "\n"
"\n",
// clang-format on
i, final_seqdata[i]->font_section_size, final_seqdata[i]->elf_path,
final_seqdata[i]->font_section_offset, final_seqdata[i]->font_section_size);
}
fprintf(out, ".balign 16\n");
fclose(out);
return EXIT_SUCCESS;
}
/* Common */
static int
@@ -297,12 +576,13 @@ usage(const char *progname)
{
fprintf(stderr,
// clang-format off
"%s: Generate code tables for audio data" "\n"
"Usage:" "\n"
" %s --banks <samplebank_table.h> <samplebank xml files...>" "\n"
" %s --fonts <soundfont_table.h> <soundfont xml files...>" "\n",
"%s: Generate code tables for audio data" "\n"
"Usage:" "\n"
" %s --banks <samplebank_table.h> <samplebank xml files...>" "\n"
" %s --fonts <soundfont_table.h> <soundfont xml files...>" "\n"
" %s --sequences <seq_font_table.s> <sequence_order.in> <sequence object files...>" "\n",
// clang-format on
progname, progname, progname);
progname, progname, progname, progname);
return EXIT_FAILURE;
}
@@ -336,6 +616,16 @@ main(int argc, char **argv)
int num_soundfont_files = argc - 3;
ret = tablegen_soundfonts(sf_hdr_out, soundfonts_paths, num_soundfont_files);
} else if (strequ(mode, "--sequences")) {
if (argc < 5)
return usage(progname);
const char *seq_font_tbl_out = argv[2];
const char *seq_order_path = argv[3];
const char **sequences_paths = (const char **)&argv[4];
int num_sequence_files = argc - 4;
ret = tablegen_sequences(seq_font_tbl_out, seq_order_path, sequences_paths, num_sequence_files);
} else {
return usage(progname);
}
+1 -1
View File
@@ -303,7 +303,7 @@ def extract_sequences(audioseq_seg : memoryview, extracted_dir : str, version_in
disassemble_one_sequence(extracted_dir, version_info, soundfonts, seq_enum_names, *job)
dt = time.time() - t
print(f"Sequences extraction took {dt:.3f}")
print(f"Sequences extraction took {dt:.3f}s")
def extract_audio_for_version(version_info : GameVersionInfo, extracted_dir : str, read_xml : bool, write_xml : bool):
print("Setting up...")
+42 -22
View File
@@ -122,7 +122,7 @@ class MMLArg:
class MMLArgBits(MMLArg):
def read(self, disas):
return disas.read_bits(type(self).NBITS)
return disas.bits_val
class ArgU8(MMLArg):
def read(self, disas):
@@ -226,6 +226,17 @@ class ArgStereoConfig(ArgU8):
strong_rvrb_left = (self.value >> 0) & 1
return f"{type}, {strong_right}, {strong_left}, {strong_rvrb_right}, {strong_rvrb_left}"
class ArgEffectsConfig(ArgU8):
def emit(self, disas):
assert (self.value & 0b01000000) == 0
headset = str(bool((self.value >> 7) & 1)).upper()
type = (self.value >> 4) & 0b11
strong_right = (self.value >> 3) & 1
strong_left = (self.value >> 2) & 1
strong_rvrb_right = (self.value >> 1) & 1
strong_rvrb_left = (self.value >> 0) & 1
return f"{headset}, {type}, {strong_right}, {strong_left}, {strong_rvrb_right}, {strong_rvrb_left}"
class ArgPortamentoTime(ArgVar):
def read(self, disas):
if disas.portamento_is_special:
@@ -368,6 +379,20 @@ class ArgUnkPtr(ArgAddr):
def analyze(self, disas):
disas.add_ref(self.value, SqSection.UNKNOWN)
class ArgLdSampleInst(MMLArg):
def read(self, disas):
return None
def emit(self, disas):
return "LDSAMPLE_INST"
class ArgLdSampleSfx(MMLArg):
def read(self, disas):
return None
def emit(self, disas):
return "LDSAMPLE_SFX"
#
# COMMANDS
#
@@ -474,7 +499,7 @@ CMD_SPEC = (
MMLCmd(0xEC, 'vibreset', sections=(SqSection.CHAN,)),
MMLCmd(0xEB, 'fontinstr', sections=(SqSection.CHAN,), args=(ArgFontId, ArgInstr)),
MMLCmd(0xEA, 'stop', sections=(SqSection.CHAN,)),
MMLCmd(0xE9, 'notepri', sections=(SqSection.CHAN,), args=(ArgU8,)),
MMLCmd(0xE9, 'notepri', sections=(SqSection.CHAN,), args=(ArgU4x2,)),
MMLCmd(0xE8, 'params', sections=(SqSection.CHAN,), args=(ArgU8, ArgU8, ArgU8, ArgS8, ArgS8, ArgU8, ArgU8, ArgU8,)),
MMLCmd(0xE7, 'ldparams', sections=(SqSection.CHAN,), args=(ArgAddr,)),
MMLCmd(0xE6, 'samplebook', sections=(SqSection.CHAN,), args=(ArgU8,)),
@@ -497,7 +522,7 @@ CMD_SPEC = (
MMLCmd(0xD3, 'bend', sections=(SqSection.CHAN,), args=(ArgS8,)),
MMLCmd(0xD2, 'sustain', sections=(SqSection.CHAN,), args=(ArgU8,)),
MMLCmd(0xD1, 'notealloc', sections=(SqSection.CHAN,), args=(ArgU8,)),
MMLCmd(0xD0, 'effects', sections=(SqSection.CHAN,), args=(ArgU8,)),
MMLCmd(0xD0, 'effects', sections=(SqSection.CHAN,), args=(ArgEffectsConfig,)),
MMLCmd(0xCF, 'stptrtoseq', sections=(SqSection.CHAN,), args=(ArgAddr,)),
MMLCmd(0xCE, 'ldptr', sections=(SqSection.CHAN,), args=(ArgAddr,)),
MMLCmd(0xCD, 'stopchan', sections=(SqSection.CHAN,), args=(ArgU8,)),
@@ -529,20 +554,19 @@ CMD_SPEC = (
MMLCmd(0xB2, 'ldseqtoptr', sections=(SqSection.CHAN,), args=(ArgTblPtr,)),
MMLCmd(0xB1, 'freefilter', sections=(SqSection.CHAN,)),
MMLCmd(0xB0, 'ldfilter', sections=(SqSection.CHAN,), args=(ArgFilterPtr,)),
MMLCmd(0xAA, 'unk_AA', sections=(SqSection.CHAN,), args=(), version=(MMLVersion.MM,)),
MMLCmd(0xA8, 'randptr', sections=(SqSection.CHAN,), args=(ArgU16, ArgU16,), version=(MMLVersion.MM,)),
MMLCmd(0xA7, 'unk_A7', sections=(SqSection.CHAN,), args=(ArgVar,), version=(MMLVersion.MM,)),
MMLCmd(0xA6, 'unk_A6', sections=(SqSection.CHAN,), args=(ArgVar, ArgVar,), version=(MMLVersion.MM,)),
MMLCmd(0xA7, 'unk_A7', sections=(SqSection.CHAN,), args=(ArgHex8,), version=(MMLVersion.MM,)),
MMLCmd(0xA6, 'unk_A6', sections=(SqSection.CHAN,), args=(ArgU8, ArgS16,), version=(MMLVersion.MM,)),
MMLCmd(0xA5, 'unk_A5', sections=(SqSection.CHAN,), args=(), version=(MMLVersion.MM,)),
MMLCmd(0xA4, 'unk_A4', sections=(SqSection.CHAN,), args=(ArgVar,), version=(MMLVersion.MM,)),
MMLCmd(0xA4, 'unk_A4', sections=(SqSection.CHAN,), args=(ArgU8,), version=(MMLVersion.MM,)),
MMLCmd(0xA3, 'unk_A3', sections=(SqSection.CHAN,), args=(), version=(MMLVersion.MM,)),
MMLCmd(0xA2, 'unk_A2', sections=(SqSection.CHAN,), args=(ArgVar,), version=(MMLVersion.MM,)),
MMLCmd(0xA2, 'unk_A2', sections=(SqSection.CHAN,), args=(ArgS16,), version=(MMLVersion.MM,)),
MMLCmd(0xA1, 'unk_A1', sections=(SqSection.CHAN,), args=(), version=(MMLVersion.MM,)),
MMLCmd(0xA0, 'unk_A0', sections=(SqSection.CHAN,), args=(ArgVar,), version=(MMLVersion.MM,)),
MMLCmd(0xA0, 'unk_A0', sections=(SqSection.CHAN,), args=(ArgS16,), version=(MMLVersion.MM,)),
# argbits commands
MMLCmd(0x00, 'cdelay', sections=(SqSection.CHAN,), args=(ArgBits4,)),
MMLCmd(0x10, 'sample', sections=(SqSection.CHAN,), args=(ArgBits3, ArgAddr,)),
MMLCmd(0x18, 'sampleptr', sections=(SqSection.CHAN,), args=(ArgBits3, ArgAddr,)),
MMLCmd(0x10, 'ldsample', sections=(SqSection.CHAN,), args=(ArgLdSampleInst, IOPort3,)),
MMLCmd(0x18, 'ldsample', sections=(SqSection.CHAN,), args=(ArgLdSampleSfx, IOPort3,)),
MMLCmd(0x20, 'ldchan', sections=(SqSection.CHAN,), args=(ArgBits4, ArgChanPtr,)),
MMLCmd(0x30, 'stcio', sections=(SqSection.CHAN,), args=(ArgBits4, IOPort8,)),
MMLCmd(0x40, 'ldcio', sections=(SqSection.CHAN,), args=(ArgBits4, IOPort8,)),
@@ -688,10 +712,11 @@ class SequenceDisassembler:
continue
# find number of lsbits that don't contribute to the command id
if len(cmd.args) > 0 and issubclass(cmd.args[0], MMLArgBits):
nbits = cmd.args[0].NBITS
else:
nbits = 0
nbits = 0
for arg in cmd.args:
if issubclass(arg, MMLArgBits):
assert nbits == 0, f"Multiple argbits-type arguments: {cmd}"
nbits = arg.NBITS
id = cmd.cmd_id
@@ -700,6 +725,7 @@ class SequenceDisassembler:
for i in range(1 << nbits):
new = cmd
new.mask = (1 << nbits) - 1
old = cmds_s.get(id + i, None)
if old is not None:
assert old.mnemonic in ("notedvg", "notedv", "notevg"), (old.mnemonic, cmd.mnemonic)
@@ -732,9 +758,6 @@ class SequenceDisassembler:
# general helpers
def read_bits(self, nbits):
return self.bits_val
def read_u8(self):
if self.hit_eof:
raise Exception()
@@ -763,10 +786,7 @@ class SequenceDisassembler:
cmd = cmd[int(not self.large_notes)]
# part of the command byte may be an arg, save the value
mask = 0
if len(cmd.args) > 0 and issubclass(cmd.args[0], MMLArgBits):
mask = (1 << cmd.args[0].NBITS) - 1
self.bits_val = id & mask
self.bits_val = id & cmd.mask
return cmd