diff --git a/.gitignore b/.gitignore index 2d074589..2caee755 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,11 @@ dump/* *.rel *.exe *.dll +*.arc build.ninja ac-decomp.code-workspace assets/ tools/cli.ini tools/cli.py +src/data/bin1 +src/data/bin2 diff --git a/README.MD b/README.MD index 4c03267c..ae5ee4e8 100644 --- a/README.MD +++ b/README.MD @@ -20,12 +20,13 @@ Use `--recursive` when cloning to have ppcdis in the repository. 1. Dump a copy of the game and extract **main.dol** and **foresta.rel.szs**. 2. Decompress **foresta.rel.szs** with yaz0 found in *tools/* (`yaz0 -d foresta.rel.szs foresta.rel`). 3. Place **main.dol** and **foresta.rel** in *dump/*. -4. Download the [CodeWarrior 1.3.2 and 1.2.5n compilers](https://files.decomp.dev/compilers_20230715.zip) and extract them to *tools/1.3.2/* and *tools/1.2.5n/*, respectively. -5. Download the [CodeWarrior 1.3.2r compiler](https://mega.nz/file/WuBFTCLT#TmB5R4-1mEFkk4G1Vjn9_cHXRD9wOIH9CtOLaVSWEas) and extract it to *tools/1.3.2r/*. -6. Install Docker. -7. Build the Docker image (`docker build -t ac-decomp .`). -8. Run configure.py (`docker run --rm -v ${PWD}:/ac-decomp ac-decomp python3 configure.py`). -9. Run ninja (`docker run --rm -v ${PWD}:/ac-decomp ac-decomp ninja`). +4. Place **forest_1st.arc** and **forest_2nd.arc** in *dump/*. +5. Download the [CodeWarrior 1.3.2 and 1.2.5n compilers](https://files.decomp.dev/compilers_20230715.zip) and extract them to *tools/1.3.2/* and *tools/1.2.5n/*, respectively. +6. Download the [CodeWarrior 1.3.2r compiler](https://mega.nz/file/WuBFTCLT#TmB5R4-1mEFkk4G1Vjn9_cHXRD9wOIH9CtOLaVSWEas) and extract it to *tools/1.3.2r/*. +7. Install Docker. +8. Build the Docker image (`docker build -t ac-decomp .`). +9. Run configure.py (`docker run --rm -v ${PWD}:/ac-decomp ac-decomp python3 configure.py`). +10. Run build.py (`docker run --rm -v ${PWD}:/ac-decomp ac-decomp python3 build.py`). ### Build manually @@ -34,22 +35,23 @@ Use `--recursive` when cloning to have ppcdis in the repository. 1. Dump a copy of the game and extract **main.dol** and **foresta.rel.szs**. 2. Decompress **foresta.rel.szs** with yaz0 found in *tools/* (`yaz0 -d foresta.rel.szs foresta.rel`). 3. Place **main.dol** and **foresta.rel** in *dump/*. -4. Download the [CodeWarrior 1.3.2 and 1.2.5n compilers](https://files.decomp.dev/compilers_20230715.zip) and extract them to *tools/1.3.2/* and *tools/1.2.5n/*, respectively. -5. Download the [CodeWarrior 1.3.2r compiler](https://mega.nz/file/WuBFTCLT#TmB5R4-1mEFkk4G1Vjn9_cHXRD9wOIH9CtOLaVSWEas) and extract it to *tools/1.3.2r/*. -6. Install Python, pip, and [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages#package-managers) using your package manager of choice. -7. Install Python modules from requirements.txt (`pip install -r requirements.txt`). -8. Install [wibo](https://github.com/decompals/wibo) +4. Place **forest_1st.arc** and **forest_2nd.arc** in *dump/*. +5. Download the [CodeWarrior 1.3.2 and 1.2.5n compilers](https://files.decomp.dev/compilers_20230715.zip) and extract them to *tools/1.3.2/* and *tools/1.2.5n/*, respectively. +6. Download the [CodeWarrior 1.3.2r compiler](https://mega.nz/file/WuBFTCLT#TmB5R4-1mEFkk4G1Vjn9_cHXRD9wOIH9CtOLaVSWEas) and extract it to *tools/1.3.2r/*. +7. Install Python, pip, and [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages#package-managers) using your package manager of choice. +8. Install Python modules from requirements.txt (`pip install -r requirements.txt`). +9. Install [wibo](https://github.com/decompals/wibo) - Wibo is a lightweight Wine replacement that's tailor-made for use with decomp projects. Regular Wine can be used if preferred, but for the purposes of this guide, these instructions will use wibo. - Download [the latest GitHub release](https://github.com/decompals/wibo/releases/latest) and run `install ./wibo /usr/bin` to install it to your system. -9. Install devkitPPC. +10. Install devkitPPC. - To get devkitPPC, you'll need [devkitPro Pacman](https://devkitpro.org/wiki/devkitPro_pacman#Installing_devkitPro_Pacman). - Run `dkp-pacman -S devkitPPC` once dkp-pacman is installed to install devkitPPC. - Set the `DEVKITPPC` environment variable to */opt/devkitpro/devkitPPC*. -10. Set the `N64_SDK` environment variable to the path of your libultra or equivalent headers. If you need headers, you can use the ones from [ultralib](https://github.com/decompals/ultralib). +11. Set the `N64_SDK` environment variable to the path of your libultra or equivalent headers. If you need headers, you can use the ones from [ultralib](https://github.com/decompals/ultralib). - Headers should be located at `$N64_SDK/ultra/usr/include`. - You may need to modify `Gpopmtx`'s `param` member to be `unsigned int` in **gbi.h**. -11. Run `python3 configure.py`. -12. Run `ninja`. +12. Run `python3 configure.py`. +13. Run `python3 build.py`. ## Credits diff --git a/build.py b/build.py new file mode 100644 index 00000000..0381160e --- /dev/null +++ b/build.py @@ -0,0 +1,99 @@ +import subprocess +import sys +import os +import hashlib +import argparse + +# List of Ninja build targets +NINJA_BUILD_TARGETS = [ ['src/data/bin1', 'out/forest_1st.arc', 'dump/forest_1st.arc'], ['src/data/bin2', 'out/forest_2nd.arc', 'dump/forest_2nd.arc'] ] + +def calculate_directory_hash(path, hash_func): + if not os.path.isdir(path): + raise NotADirectoryError(f"{path} is not a directory") + + hasher = hash_func() + for root, dirs, files in os.walk(path): + for names in files: + filepath = os.path.join(root, names) + try: + with open(filepath, 'rb') as f: + while True: + data = f.read(65536) # Read in chunks to handle large files + if not data: + break + hasher.update(data) + except IOError: + # Handle errors as needed + pass + return hasher.hexdigest() + +def directory_changed(path, build_dir, hash_func=hashlib.md5): + hash_file = os.path.join(build_dir, f'{os.path.basename(os.path.normpath(path))}.dirhash') + current_hash = calculate_directory_hash(path, hash_func) + try: + with open(hash_file, 'r') as f: + stored_hash = f.read() + except FileNotFoundError: + stored_hash = None + + if current_hash != stored_hash: + os.makedirs(build_dir, exist_ok=True) + with open(hash_file, 'w') as f: + f.write(current_hash) + return True + return False + +def run_ninja_build(target): + try: + subprocess.run(['ninja', '-v', target], check=True) + except subprocess.CalledProcessError as e: + print(f"Error running Ninja build for target {target}: {e}") + sys.exit(1) + +def check_and_dump_arc(target, dump, verbose=False): + if not os.path.exists(target): + assert os.path.exists(dump), f"Please add missing file: {dump}" + print(f'Dumping {dump}') + + try: + if verbose: + subprocess.run(['python3', 'tools/arc_tool.py', dump, os.path.dirname(target)]) + else: + subprocess.run(['python3', 'tools/arc_tool.py', '-v', dump, os.path.dirname(target)]) + except subprocess.CalledProcessError as e: + print(f"Error running arc_tool") + sys.exit(1) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Build Animal Crossing') + parser.add_argument('--clean', help='Cleans all build artifacts', required=False, action='store_true') + parser.add_argument('-v', help='Enable verbose logging', required=False, action='store_true') + + args = parser.parse_args() + + if args.clean: + for target in NINJA_BUILD_TARGETS: + rel_path = f'{os.path.basename(os.path.normpath(target[0]))}.dirhash' + path = os.path.join('build', rel_path) + if os.path.exists(path): + os.remove(path) + try: + subprocess.run(['ninja', '-t', 'clean'], check=True) + except subprocess.CalledProcessError as e: + print(f"Error running ninja -t clean") + sys.exit(1) + else: + for target in NINJA_BUILD_TARGETS: + check_and_dump_arc(target[0], target[2], args.v) + if directory_changed(target[0], 'build'): + run_ninja_build(target[1]) + else: + print(f"No changes in {target[0]}, skipping build.") + try: + if args.v: + subprocess.run(['ninja', '-v'], check=True) + else: + subprocess.run(['ninja'], check=True) + except subprocess.CalledProcessError as e: + print(f"Error running Ninja build") + sys.exit(1) diff --git a/common.py b/common.py index 27744d08..be384a07 100644 --- a/common.py +++ b/common.py @@ -156,6 +156,12 @@ REL_SRCDIR = "src" # Include directory INCDIR = "include" +# Directory for forest_1st data +FOREST_1STDIR = "src/data/bin1" + +# Directory for forest_2nd data +FOREST_2NDDIR = "src/data/bin2" + # Build artifacts directory BUILDDIR = "build" @@ -224,6 +230,9 @@ CPP = os.path.join(DEVKITPPC, "bin", "powerpc-eabi-cpp") VTXDIS = f"{PYTHON} {TOOLS}/converters/vtxdis.py" PAL16DIS = f"{PYTHON} {TOOLS}/converters/pal16dis.py" +# JSystem JKernel archive tool +ARC_TOOL = f"{PYTHON} {TOOLS}/arc_tool.py" + ICONV = f"{PYTHON} tools/sjis.py" # TODO: get actual iconv working(?) # N64 SDK path for GBI diff --git a/configure.py b/configure.py index 74662539..c4f3f8ea 100644 --- a/configure.py +++ b/configure.py @@ -95,6 +95,7 @@ n.variable("iconv", c.ICONV) n.variable("forcefilesgen", c.FORCEFILESGEN) n.variable("vtxdis", c.VTXDIS) n.variable("pal16dis", c.PAL16DIS) +n.variable("arctool", c.ARC_TOOL) n.newline() ############## @@ -254,6 +255,12 @@ n.rule( description = "pal16dis.py $in $out" ) +n.rule( + "arctool", + command = "$arctool -v $in $out", + description = "$arctool -v $in $out" +) + ########## # Assets # ########## @@ -734,6 +741,18 @@ n.build( inputs = c.REL_YML ) +n.build( + f"{c.OUTDIR}/forest_1st.arc", + rule = "arctool", + inputs = f"{c.FOREST_1STDIR}" +) + +n.build( + f"{c.OUTDIR}/forest_2nd.arc", + rule = "arctool", + inputs = f"{c.FOREST_2NDDIR}" +) + n.build( [c.REL_LABELS, c.REL_RELOCS], rule = "analyse", diff --git a/requirements.txt b/requirements.txt index e2da4a36..b44f5418 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ ninja_syntax prettytable python-Levenshtein watchdog +pyjkernel diff --git a/tools/arc_tool.py b/tools/arc_tool.py new file mode 100644 index 00000000..fec79ab9 --- /dev/null +++ b/tools/arc_tool.py @@ -0,0 +1,71 @@ +import pyjkernel +import os +import argparse + +def unpack_dir(archive: pyjkernel.JKRArchive, dir: str, verbose=False): + if verbose: + print('Dumping dir: ' + dir) + # create all files + for file in archive.list_files(dir): + if verbose: + print('Dumping file: ' + file.name) + with open(os.path.join(dir, file.name), "wb") as f: + f.write(archive.get_file(dir + "/" + file.name).data) + + # create all subdirectories and recurse through them + for subdir in archive.list_folders(dir): + if not os.path.exists(dir + "/" + subdir): + os.mkdir(dir + "/" + subdir) + unpack_dir(archive, dir + "/" + subdir, verbose) + + +def unpack_archive(path: str, out_path: str, verbose=False): + archive = pyjkernel.from_archive_file(path, True) + orig_dir = os.path.abspath(os.curdir) + os.chdir(out_path) + if not os.path.exists(archive.root_name): + os.mkdir(archive.root_name) + unpack_dir(archive, archive.root_name, verbose) + os.chdir(orig_dir) + +def pack_dir(archive: pyjkernel.JKRArchive, path: str, verbose=False): + local_path = os.path.dirname(path) + orig_dir = os.path.abspath(os.curdir) + if local_path != "": + os.chdir(local_path) + local_root = os.path.basename(os.path.normpath(path)) + + for root, dirs, files in os.walk(local_root): + files.sort(key=lambda item: (item.lower(), item)) + for dir in dirs: + archive.create_folder(root + "/" + dir) + + for file in files: + if verbose: + print('Packing file: ' + root + '/' + file) + with open(root + "/" + file, "rb") as f: + archive.create_file(root + "/" + file, bytearray(f.read()), pyjkernel.JKRPreloadType.ARAM) + os.chdir(orig_dir) + +def pack_archive(root_path: str, out_path: str, verbose=False): + root_name = os.path.basename(os.path.normpath(root_path)) + archive = pyjkernel.create_new_archive(root_name) + pack_dir(archive, root_path, verbose) + pyjkernel.write_archive_file(archive, out_path, True, pyjkernel.jkrcomp.JKRCompression.NONE, 0) + +def main(): + parser = argparse.ArgumentParser(description='Pack or unpack JSystem JKernel archives.') + parser.add_argument('-v', help='Enable verbose logging.', required=False, action='store_true') + parser.add_argument('path', help='The path of the folder to pack or archive file to unpack.') + parser.add_argument('out', help='The path of the destination folder or file.') + + args = parser.parse_args() + if os.path.isfile(args.path): + unpack_archive(args.path, args.out, args.v) + elif os.path.isdir(args.path): + pack_archive(args.path, args.out, args.v) + else: + raise Exception('path is not a valid file or directory!') + +if __name__ == '__main__': + main()