Merge pull request #194 from Cuyler36/arc_support

Arc support
2026-05-24 23:01:25 -04:00 · 2023-12-18 22:31:05 -05:00
parent c6568474a4 4ceff2b192
commit 9dda2fdbbd
7 changed files with 219 additions and 15 deletions
@@ -10,8 +10,11 @@ dump/*
 *.rel
 *.exe
 *.dll
+*.arc
 build.ninja
 ac-decomp.code-workspace
 assets/
 tools/cli.ini
 tools/cli.py
+src/data/bin1
+src/data/bin2
@@ -20,12 +20,13 @@ Use `--recursive` when cloning to have ppcdis in the repository.
 1. Dump a copy of the game and extract **main.dol** and **foresta.rel.szs**.
 2. Decompress **foresta.rel.szs** with yaz0 found in *tools/* (`yaz0 -d foresta.rel.szs foresta.rel`).
 3. Place **main.dol** and **foresta.rel** in *dump/*.
-4. Download the [CodeWarrior 1.3.2 and 1.2.5n compilers](https://files.decomp.dev/compilers_20230715.zip) and extract them to *tools/1.3.2/* and *tools/1.2.5n/*, respectively.
-5. Download the [CodeWarrior 1.3.2r compiler](https://mega.nz/file/WuBFTCLT#TmB5R4-1mEFkk4G1Vjn9_cHXRD9wOIH9CtOLaVSWEas) and extract it to *tools/1.3.2r/*.
-6. Install Docker.
-7. Build the Docker image (`docker build -t ac-decomp .`).
-8. Run configure.py (`docker run --rm -v ${PWD}:/ac-decomp ac-decomp python3 configure.py`).
-9. Run ninja (`docker run --rm -v ${PWD}:/ac-decomp ac-decomp ninja`).
+4. Place **forest_1st.arc** and **forest_2nd.arc** in *dump/*.
+5. Download the [CodeWarrior 1.3.2 and 1.2.5n compilers](https://files.decomp.dev/compilers_20230715.zip) and extract them to *tools/1.3.2/* and *tools/1.2.5n/*, respectively.
+6. Download the [CodeWarrior 1.3.2r compiler](https://mega.nz/file/WuBFTCLT#TmB5R4-1mEFkk4G1Vjn9_cHXRD9wOIH9CtOLaVSWEas) and extract it to *tools/1.3.2r/*.
+7. Install Docker.
+8. Build the Docker image (`docker build -t ac-decomp .`).
+9. Run configure.py (`docker run --rm -v ${PWD}:/ac-decomp ac-decomp python3 configure.py`).
+10. Run build.py (`docker run --rm -v ${PWD}:/ac-decomp ac-decomp python3 build.py`).

 ### Build manually

@@ -34,22 +35,23 @@ Use `--recursive` when cloning to have ppcdis in the repository.
 1. Dump a copy of the game and extract **main.dol** and **foresta.rel.szs**.
 2. Decompress **foresta.rel.szs** with yaz0 found in *tools/* (`yaz0 -d foresta.rel.szs foresta.rel`).
 3. Place **main.dol** and **foresta.rel** in *dump/*.
-4. Download the [CodeWarrior 1.3.2 and 1.2.5n compilers](https://files.decomp.dev/compilers_20230715.zip) and extract them to *tools/1.3.2/* and *tools/1.2.5n/*, respectively.
-5. Download the [CodeWarrior 1.3.2r compiler](https://mega.nz/file/WuBFTCLT#TmB5R4-1mEFkk4G1Vjn9_cHXRD9wOIH9CtOLaVSWEas) and extract it to *tools/1.3.2r/*.
-6. Install Python, pip, and [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages#package-managers) using your package manager of choice.
-7. Install Python modules from requirements.txt (`pip install -r requirements.txt`).
-8. Install [wibo](https://github.com/decompals/wibo)
+4. Place **forest_1st.arc** and **forest_2nd.arc** in *dump/*.
+5. Download the [CodeWarrior 1.3.2 and 1.2.5n compilers](https://files.decomp.dev/compilers_20230715.zip) and extract them to *tools/1.3.2/* and *tools/1.2.5n/*, respectively.
+6. Download the [CodeWarrior 1.3.2r compiler](https://mega.nz/file/WuBFTCLT#TmB5R4-1mEFkk4G1Vjn9_cHXRD9wOIH9CtOLaVSWEas) and extract it to *tools/1.3.2r/*.
+7. Install Python, pip, and [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages#package-managers) using your package manager of choice.
+8. Install Python modules from requirements.txt (`pip install -r requirements.txt`).
+9. Install [wibo](https://github.com/decompals/wibo)
    - Wibo is a lightweight Wine replacement that's tailor-made for use with decomp projects. Regular Wine can be used if preferred, but for the purposes of this guide, these instructions will use wibo.
    - Download [the latest GitHub release](https://github.com/decompals/wibo/releases/latest) and run `install ./wibo /usr/bin` to install it to your system.
-9. Install devkitPPC.
+10. Install devkitPPC.
    - To get devkitPPC, you'll need [devkitPro Pacman](https://devkitpro.org/wiki/devkitPro_pacman#Installing_devkitPro_Pacman).
        - Run `dkp-pacman -S devkitPPC` once dkp-pacman is installed to install devkitPPC.
        - Set the `DEVKITPPC` environment variable to */opt/devkitpro/devkitPPC*.
-10. Set the `N64_SDK` environment variable to the path of your libultra or equivalent headers. If you need headers, you can use the ones from [ultralib](https://github.com/decompals/ultralib).
+11. Set the `N64_SDK` environment variable to the path of your libultra or equivalent headers. If you need headers, you can use the ones from [ultralib](https://github.com/decompals/ultralib).
    - Headers should be located at `$N64_SDK/ultra/usr/include`.
    - You may need to modify `Gpopmtx`'s `param` member to be `unsigned int` in **gbi.h**.
-11. Run `python3 configure.py`.
-12. Run `ninja`.
+12. Run `python3 configure.py`.
+13. Run `python3 build.py`.

 ## Credits

@@ -0,0 +1,99 @@
+import subprocess
+import sys
+import os
+import hashlib
+import argparse
+
+# List of Ninja build targets
+NINJA_BUILD_TARGETS = [ ['src/data/bin1', 'out/forest_1st.arc', 'dump/forest_1st.arc'], ['src/data/bin2', 'out/forest_2nd.arc', 'dump/forest_2nd.arc'] ]
+
+def calculate_directory_hash(path, hash_func):
+    if not os.path.isdir(path):
+        raise NotADirectoryError(f"{path} is not a directory")
+    
+    hasher = hash_func()
+    for root, dirs, files in os.walk(path):
+        for names in files:
+            filepath = os.path.join(root, names)
+            try:
+                with open(filepath, 'rb') as f:
+                    while True:
+                        data = f.read(65536)  # Read in chunks to handle large files
+                        if not data:
+                            break
+                        hasher.update(data)
+            except IOError:
+                # Handle errors as needed
+                pass
+    return hasher.hexdigest()
+
+def directory_changed(path, build_dir, hash_func=hashlib.md5):
+    hash_file = os.path.join(build_dir, f'{os.path.basename(os.path.normpath(path))}.dirhash')
+    current_hash = calculate_directory_hash(path, hash_func)
+    try:
+        with open(hash_file, 'r') as f:
+            stored_hash = f.read()
+    except FileNotFoundError:
+        stored_hash = None
+
+    if current_hash != stored_hash:
+        os.makedirs(build_dir, exist_ok=True)
+        with open(hash_file, 'w') as f:
+            f.write(current_hash)
+        return True
+    return False
+
+def run_ninja_build(target):
+    try:
+        subprocess.run(['ninja', '-v', target], check=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Error running Ninja build for target {target}: {e}")
+        sys.exit(1)
+
+def check_and_dump_arc(target, dump, verbose=False):
+    if not os.path.exists(target):
+        assert os.path.exists(dump), f"Please add missing file: {dump}"
+        print(f'Dumping {dump}')
+
+        try:
+            if verbose:
+                subprocess.run(['python3', 'tools/arc_tool.py', dump, os.path.dirname(target)])
+            else:
+                subprocess.run(['python3', 'tools/arc_tool.py', '-v', dump, os.path.dirname(target)])
+        except subprocess.CalledProcessError as e:
+            print(f"Error running arc_tool")
+            sys.exit(1)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Build Animal Crossing')
+    parser.add_argument('--clean', help='Cleans all build artifacts', required=False, action='store_true')
+    parser.add_argument('-v', help='Enable verbose logging', required=False, action='store_true')
+
+    args = parser.parse_args()
+
+    if args.clean:
+        for target in NINJA_BUILD_TARGETS:
+            rel_path = f'{os.path.basename(os.path.normpath(target[0]))}.dirhash'
+            path = os.path.join('build', rel_path)
+            if os.path.exists(path):
+                os.remove(path)
+        try:
+            subprocess.run(['ninja', '-t', 'clean'], check=True)
+        except subprocess.CalledProcessError as e:
+            print(f"Error running ninja -t clean")
+            sys.exit(1)
+    else:
+        for target in NINJA_BUILD_TARGETS:
+            check_and_dump_arc(target[0], target[2], args.v)
+            if directory_changed(target[0], 'build'):
+                run_ninja_build(target[1])
+            else:
+                print(f"No changes in {target[0]}, skipping build.")
+        try:
+            if args.v:
+                subprocess.run(['ninja', '-v'], check=True)
+            else:
+                subprocess.run(['ninja'], check=True)
+        except subprocess.CalledProcessError as e:
+            print(f"Error running Ninja build")
+            sys.exit(1)
@@ -156,6 +156,12 @@ REL_SRCDIR = "src"
 # Include directory
 INCDIR = "include"

+# Directory for forest_1st data
+FOREST_1STDIR = "src/data/bin1"
+
+# Directory for forest_2nd data
+FOREST_2NDDIR = "src/data/bin2"
+
 # Build artifacts directory
 BUILDDIR = "build"

@@ -224,6 +230,9 @@ CPP = os.path.join(DEVKITPPC, "bin", "powerpc-eabi-cpp")
 VTXDIS = f"{PYTHON} {TOOLS}/converters/vtxdis.py"
 PAL16DIS = f"{PYTHON} {TOOLS}/converters/pal16dis.py"

+# JSystem JKernel archive tool
+ARC_TOOL = f"{PYTHON} {TOOLS}/arc_tool.py"
+
 ICONV = f"{PYTHON} tools/sjis.py" # TODO: get actual iconv working(?)

 # N64 SDK path for GBI
@@ -95,6 +95,7 @@ n.variable("iconv", c.ICONV)
 n.variable("forcefilesgen", c.FORCEFILESGEN)
 n.variable("vtxdis", c.VTXDIS)
 n.variable("pal16dis", c.PAL16DIS)
+n.variable("arctool", c.ARC_TOOL)
 n.newline()

 ##############
@@ -254,6 +255,12 @@ n.rule(
    description = "pal16dis.py $in $out"
 )

+n.rule(
+    "arctool",
+    command = "$arctool -v $in $out",
+    description = "$arctool -v $in $out"
+)
+
 ##########
 # Assets #
 ##########
@@ -734,6 +741,18 @@ n.build(
    inputs = c.REL_YML
 )

+n.build(
+    f"{c.OUTDIR}/forest_1st.arc",
+    rule = "arctool",
+    inputs = f"{c.FOREST_1STDIR}"
+)
+
+n.build(
+    f"{c.OUTDIR}/forest_2nd.arc",
+    rule = "arctool",
+    inputs = f"{c.FOREST_2NDDIR}"
+)
+
 n.build(
    [c.REL_LABELS, c.REL_RELOCS],
    rule = "analyse",
@@ -5,3 +5,4 @@ ninja_syntax
 prettytable
 python-Levenshtein
 watchdog
+pyjkernel
@@ -0,0 +1,71 @@
+import pyjkernel
+import os
+import argparse
+
+def unpack_dir(archive: pyjkernel.JKRArchive, dir: str, verbose=False):
+  if verbose:
+    print('Dumping dir: ' + dir)
+  # create all files
+  for file in archive.list_files(dir):
+    if verbose:
+      print('Dumping file: ' + file.name)
+    with open(os.path.join(dir, file.name), "wb") as f:
+      f.write(archive.get_file(dir + "/" + file.name).data)
+
+  # create all subdirectories and recurse through them
+  for subdir in archive.list_folders(dir):
+    if not os.path.exists(dir + "/" + subdir):
+      os.mkdir(dir + "/" + subdir)
+    unpack_dir(archive, dir + "/" + subdir, verbose)
+
+
+def unpack_archive(path: str, out_path: str, verbose=False):
+  archive = pyjkernel.from_archive_file(path, True)
+  orig_dir = os.path.abspath(os.curdir)
+  os.chdir(out_path)
+  if not os.path.exists(archive.root_name):
+    os.mkdir(archive.root_name)
+  unpack_dir(archive, archive.root_name, verbose)
+  os.chdir(orig_dir)
+
+def pack_dir(archive: pyjkernel.JKRArchive, path: str, verbose=False):
+  local_path = os.path.dirname(path)
+  orig_dir = os.path.abspath(os.curdir)
+  if local_path != "":
+    os.chdir(local_path)
+  local_root = os.path.basename(os.path.normpath(path))
+
+  for root, dirs, files in os.walk(local_root):
+    files.sort(key=lambda item: (item.lower(), item))
+    for dir in dirs:
+      archive.create_folder(root + "/" + dir)
+
+    for file in files:
+      if verbose:
+        print('Packing file: ' + root + '/' + file)
+      with open(root + "/" + file, "rb") as f:
+        archive.create_file(root + "/" + file, bytearray(f.read()), pyjkernel.JKRPreloadType.ARAM)
+  os.chdir(orig_dir)
+
+def pack_archive(root_path: str, out_path: str, verbose=False):
+  root_name = os.path.basename(os.path.normpath(root_path))
+  archive = pyjkernel.create_new_archive(root_name)
+  pack_dir(archive, root_path, verbose)
+  pyjkernel.write_archive_file(archive, out_path, True, pyjkernel.jkrcomp.JKRCompression.NONE, 0)
+
+def main():
+  parser = argparse.ArgumentParser(description='Pack or unpack JSystem JKernel archives.')
+  parser.add_argument('-v', help='Enable verbose logging.', required=False, action='store_true')
+  parser.add_argument('path', help='The path of the folder to pack or archive file to unpack.')
+  parser.add_argument('out', help='The path of the destination folder or file.')
+
+  args = parser.parse_args()
+  if os.path.isfile(args.path):
+    unpack_archive(args.path, args.out, args.v)
+  elif os.path.isdir(args.path):
+    pack_archive(args.path, args.out, args.v)
+  else:
+    raise Exception('path is not a valid file or directory!')
+
+if __name__ == '__main__':
+  main()