Add the STR RPC to overlord and game code (#134)

* work in progress streaming rpc, simple test is working

* actually add the test

* debug windows failure

* windows fix maybe

* windows 2

* use str-load-status

* update types

game/common/overlord_common.h

@@ -0,0 +1,3 @@
+#pragma once
+
+constexpr int SECTOR_SIZE = 0x800;  // media sector size

game/common/str_rpc_types.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "common/common_types.h"
+#include "game/common/overlord_common.h"
+
+constexpr int STR_RPC_ID = 0xdeb5;
+constexpr int STR_RPC_CHANNEL = 4;
+
+struct RPC_Str_Cmd {
+  u16 rsvd;       // 0, seems unused
+  u16 result;     // 2, return code. see STR_RPC_RESULT_XXX
+  u32 ee_addr;    // 4, GOAL address to load to.
+  s32 chunk_id;   // 8, chunk ID for chunked file. Use -1 to load a non-chunked file, which gets the
+                  // whole file.
+  u32 length;     // 12, length that was actually loaded
+  char name[64];  // file name
+};
+
+constexpr int STR_RPC_RESULT_ERROR = 1;
+constexpr int STR_RPC_RESULT_DONE = 0;
+
+// maximum number of chunks in a chunked file.
+constexpr int SECTOR_TABLE_SIZE = 64;
+
+// the header of a chunked file
+struct StrFileHeader {
+  u32 sectors[SECTOR_TABLE_SIZE];  // start of chunk, in sectors. including this sector.
+  u32 sizes[SECTOR_TABLE_SIZE];    // size of chunk, in bytes. always an integer number of sectors
+};
+
+// the first sector of a chunked file.
+struct StrFileHeaderSector : StrFileHeader {
+  u32 pad[512 - 128];  // all zero
+};
+
+static_assert(sizeof(StrFileHeader) == 0x200, "Sector header size");
+static_assert(sizeof(StrFileHeaderSector) == SECTOR_SIZE, "Sector header size");

game/fake_iso.txt

@@ -8,3 +8,5 @@ TEST.CGO resources/TEST.CGO
 TWEAKVAL.MUS resources/TWEAKVAL.MUS
 VAGDIR.AYB resources/VAGDIR.AYB
 SCREEN1.USA resources/SCREEN1.USA
+0COMMON.TXT out/iso/0COMMON.TXT
+0TEST.TXT out/iso/0TEST.TXT

game/kernel/kdgo.cpp

@@ -16,6 +16,7 @@
 #include "game/common/ramdisk_rpc_types.h"
 #include "game/common/loader_rpc_types.h"
 #include "game/common/play_rpc_types.h"
+#include "game/common/str_rpc_types.h"
 #include "third-party/spdlog/include/spdlog/spdlog.h"
 
 using namespace ee;
@@ -49,6 +50,27 @@ s32 RpcCall(s32 rpcChannel,
                        nullptr);
 }
 
+// Terrible hack! Remove soon!
+
+namespace {
+struct RpcCallArgCache {
+  s32 rpcChannel;
+  u32 fno;
+  u32 async;
+} rpc_arg_cache;
+}  // namespace
+
+void RpcCall_wrapper_part1(s32 rpcChannel, u32 fno, u32 async) {
+  rpc_arg_cache.rpcChannel = rpcChannel;
+  rpc_arg_cache.fno = fno;
+  rpc_arg_cache.async = async;
+}
+
+u64 RpcCall_wrapper_part2(u64 send_buff, s32 send_size, u64 recv_buff, s32 recv_size) {
+  return RpcCall_wrapper(rpc_arg_cache.rpcChannel, rpc_arg_cache.fno, rpc_arg_cache.async,
+                         send_buff, send_size, recv_buff, recv_size);
+}
+
 /*!
  * GOAL Wrapper for RpcCall.
  */
@@ -59,6 +81,7 @@ u64 RpcCall_wrapper(s32 rpcChannel,
                     s32 send_size,
                     u64 recv_buff,
                     s32 recv_size) {
+  fprintf(stderr, "size in c is %d\n", recv_size);
   return sceSifCallRpc(&cd[rpcChannel], fno, async, Ptr<u8>(send_buff).c(), send_size,
                        Ptr<u8>(recv_buff).c(), recv_size, nullptr, nullptr);
 }
@@ -126,7 +149,7 @@ u32 RpcBind(s32 channel, s32 id) {
 u32 InitRPC() {
   if (!RpcBind(PLAYER_RPC_CHANNEL, PLAYER_RPC_ID) && !RpcBind(LOADER_RPC_CHANNEL, LOADER_RPC_ID) &&
       !RpcBind(RAMDISK_RPC_CHANNEL, RAMDISK_RPC_ID) && !RpcBind(DGO_RPC_CHANNEL, DGO_RPC_ID) &&
-      !RpcBind(4, 0xdeb5) && !RpcBind(PLAY_RPC_CHANNEL, PLAY_RPC_ID)) {
+      !RpcBind(STR_RPC_CHANNEL, STR_RPC_ID) && !RpcBind(PLAY_RPC_CHANNEL, PLAY_RPC_ID)) {
     return 0;
   }
   printf("Entering endless loop ... please wait\n");

game/kernel/kdgo.h

@@ -29,4 +29,6 @@ u64 RpcCall_wrapper(s32 rpcChannel,
 u32 RpcBusy(s32 channel);
 void LoadDGOTest();
 
+void RpcCall_wrapper_part1(s32 rpcChannel, u32 fno, u32 async);
+u64 RpcCall_wrapper_part2(u64 send_buff, s32 send_size, u64 recv_buff, s32 recv_size);
 #endif  // JAK_V2_KDGO_H

game/kernel/ksound.cpp

@@ -25,4 +25,8 @@ void InitSoundScheme() {
   make_function_symbol_from_c("rpc-call", (void*)RpcCall_wrapper);
   make_function_symbol_from_c("rpc-busy?", (void*)RpcBusy);
   make_function_symbol_from_c("test-load-dgo-c", (void*)LoadDGOTest);
+
+  // terrible hack!
+  make_function_symbol_from_c("rpc-call-p1", (void*)RpcCall_wrapper_part1);
+  make_function_symbol_from_c("rpc-call-p2", (void*)RpcCall_wrapper_part2);
 }

game/overlord/fake_iso.cpp

@@ -79,26 +79,10 @@ void fake_iso_init_globals() {
 int FS_Init(u8* buffer) {
   (void)buffer;
 
-  // get path to next/data/fake_iso.txt, the map file.
-  char fakeiso_path[512];
-  strcpy(fakeiso_path, file_util::get_file_path({"game", "fake_iso.txt"}).c_str());
-
-  // open the map.
-  FILE* fp = fopen(fakeiso_path, "r");
-  assert(fp);
-  fseek(fp, 0, SEEK_END);
-  size_t len = ftell(fp);
-  rewind(fp);
-  char* fakeiso = (char*)malloc(len + 1);
-  if (fread(fakeiso, len, 1, fp) != 1) {
-#ifdef __linux__
-    assert(false);
-#endif
-  }
-  fakeiso[len] = '\0';
+  auto config_str = file_util::read_text_file(file_util::get_file_path({"game", "fake_iso.txt"}));
+  const char* ptr = config_str.c_str();
 
   // loop over lines
-  char* ptr = fakeiso;
   while (*ptr) {
     // newlines
     while (*ptr && *ptr == '\n')
@@ -127,7 +111,7 @@ int FS_Init(u8* buffer) {
     }
 
     i = 0;
-    while (*ptr && (*ptr != '\n') && (*ptr != ' ') && i < 128) {
+    while (*ptr && (*ptr != '\n') && (*ptr != ' ') && (*ptr != EOF) && i < 128) {
       e->file_path[i] = *ptr;
       ptr++;
       i++;
@@ -145,8 +129,6 @@ int FS_Init(u8* buffer) {
     sFiles[i].location = i;
   }
 
-  free(fakeiso);
-
   // TODO load tweak music.
 
   return 0;
@@ -201,6 +183,7 @@ static const char* get_file_path(FileRecord* fr) {
  */
 uint32_t FS_GetLength(FileRecord* fr) {
   const char* path = get_file_path(fr);
+  file_util::assert_file_exists(path, "fake_iso FS_GetLength");
   FILE* fp = fopen(path, "rb");
   assert(fp);
   fseek(fp, 0, SEEK_END);

game/overlord/iso.cpp

@@ -43,7 +43,7 @@ s32 str_thread;
 s32 play_thread;
 u8 gVagDir[VAGDIR_SIZE];
 u32 gPlayPos;
-RPC_Dgo_Cmd sRPCBuff[1];  // todo move...
+static RPC_Dgo_Cmd sRPCBuff[1];  // todo move...
 DgoCommand scmd;
 
 void iso_init_globals() {
@@ -171,16 +171,16 @@ u32 InitISOFS(const char* fs_mode, const char* loading_screen) {
     return 1;
   }
 
-  //  thread_param.attr = TH_C;
-  //  thread_param.initPriority = 97;
-  //  thread_param.stackSize = 0x800;
-  //  thread_param.option = 0;
-  //  thread_param.entry = (void*)STRThread;
-  //  strcpy(thread_param.name, "STRThread");
-  //  str_thread = CreateThread(&thread_param);
-  //  if(str_thread <= 0) {
-  //    return 1;
-  //  }
+  thread_param.attr = TH_C;
+  thread_param.initPriority = 97;
+  thread_param.stackSize = 0x800;
+  thread_param.option = 0;
+  thread_param.entry = (void*)STRThread;
+  strcpy(thread_param.name, "STRThread");
+  str_thread = CreateThread(&thread_param);
+  if (str_thread <= 0) {
+    return 1;
+  }
   //
   //  thread_param.attr = TH_C;
   //  thread_param.initPriority = 97;
@@ -196,7 +196,7 @@ u32 InitISOFS(const char* fs_mode, const char* loading_screen) {
   // Start the threads!
   StartThread(iso_thread, 0);
   StartThread(dgo_thread, 0);
-  //  StartThread(str_thread, 0);
+  StartThread(str_thread, 0);
   //  StartThread(play_thread, 0);
 
   // wait for ISO Thread to initialize

game/overlord/iso_api.cpp

@@ -7,7 +7,7 @@ using namespace iop;
 /*!
  * Load a File to IOP memory (blocking)
  */
-void LoadISOFileToIOP(FileRecord* file, void* addr, uint32_t length) {
+s32 LoadISOFileToIOP(FileRecord* file, void* addr, uint32_t length) {
   // printf("[OVERLORD] LoadISOFileToIOP %s, %d/%d bytes\n", file->name, length, file->size);
   spdlog::debug("[OVERLORD] LoadISOFileToIOP {}, {}/{} bytes", file->name, length, file->size);
   IsoCommandLoadSingle cmd;
@@ -23,12 +23,14 @@ void LoadISOFileToIOP(FileRecord* file, void* addr, uint32_t length) {
   if (cmd.status) {
     cmd.length_to_copy = 0;
   }
+
+  return cmd.length_to_copy;
 }
 
 /*!
  * Load a File to IOP memory (blocking)
  */
-void LoadISOFileToEE(FileRecord* file, uint32_t addr, uint32_t length) {
+s32 LoadISOFileToEE(FileRecord* file, uint32_t addr, uint32_t length) {
   // printf("[OVERLORD] LoadISOFileToEE %s, %d/%d bytes\n", file->name, length, file->size);
   spdlog::debug("[OVERLORD] LoadISOFileToEE {}, {}/{} bytes", file->name, length, file->size);
   IsoCommandLoadSingle cmd;
@@ -44,4 +46,24 @@ void LoadISOFileToEE(FileRecord* file, uint32_t addr, uint32_t length) {
   if (cmd.status) {
     cmd.length_to_copy = 0;
   }
+
+  return cmd.length_to_copy;
+}
+
+s32 LoadISOFileChunkToEE(FileRecord* file, uint32_t dest_addr, uint32_t length, uint32_t offset) {
+  spdlog::debug("[OVERLORD] LoadISOFileChunkToEE {} : {} offset {}\n", file->name, length, offset);
+  IsoCommandLoadSingle cmd;
+  cmd.cmd_id = LOAD_TO_EE_OFFSET_CMD_ID;
+  cmd.messagebox_to_reply = 0;
+  cmd.thread_id = GetThreadId();
+  cmd.file_record = file;
+  cmd.dest_addr = (u8*)(u64)dest_addr;
+  cmd.length = length;
+  cmd.offset = offset;
+  SendMbx(iso_mbx, &cmd);
+  SleepThread();
+  if (cmd.status) {
+    cmd.length_to_copy = 0;
+  }
+  return cmd.length_to_copy;
 }

game/overlord/iso_api.h

@@ -1,10 +1,7 @@
 #pragma once
 
-#ifndef JAK_V2_ISO_API_H
-#define JAK_V2_ISO_API_H
 #include "isocommon.h"
 
-void LoadISOFileToIOP(FileRecord* file, void* addr, uint32_t length);
-void LoadISOFileToEE(FileRecord* file, uint32_t ee_addr, uint32_t length);
-
-#endif  // JAK_V2_ISO_API_H
+s32 LoadISOFileToIOP(FileRecord* file, void* addr, uint32_t length);
+s32 LoadISOFileToEE(FileRecord* file, uint32_t ee_addr, uint32_t length);
+s32 LoadISOFileChunkToEE(FileRecord* file, uint32_t dest_addr, uint32_t length, uint32_t offset);

game/overlord/isocommon.h

@@ -11,6 +11,7 @@
 #include <string>
 #include "common/common_types.h"
 #include "common/link_types.h"
+#include "game/common/overlord_common.h"
 
 constexpr int PRI_STACK_LENGTH = 4;  // number of queued commands per priority
 constexpr int N_PRIORITIES = 4;      // number of priorities
@@ -30,7 +31,6 @@ constexpr int LOAD_TO_IOP_CMD_ID = 0x101;        // command to load to iop
 constexpr int LOAD_TO_EE_OFFSET_CMD_ID = 0x102;  // command to load file to ee with offset.
 constexpr int LOAD_DGO_CMD_ID = 0x200;           // command to load DGO
 
-constexpr int SECTOR_SIZE = 0x800;  // media sector size
 constexpr int MAX_ISO_FILES = 350;  // maximum files on FS
 constexpr int MAX_OPEN_FILES = 16;  // maximum number of open files at a time.
 
@@ -51,7 +51,7 @@ struct FileRecord {
  */
 struct LoadStackEntry {
   FileRecord* fr;
-  uint32_t location;
+  uint32_t location;  // sectors.
 };
 
 /*!
@@ -185,5 +185,6 @@ extern IsoFs* isofs;
 extern s32 iso_mbx;
 
 void MakeISOName(char* dst, const char* src);
+void ISONameFromAnimationName(char* dst, const char* src);
 
 #endif  // JAK_V2_ISOCOMMON_H

game/overlord/stream.cpp

@@ -1,12 +1,141 @@
-#include <assert.h>
-#include "stream.h"
+/*!
+ * @file stream.cpp
+ * OVERLORD streaming driver.
+ * Supports loading a file directly to the EE, or loading chunks of a chunked file.
+ */
 
+#include <cassert>
+#include "stream.h"
+#include "game/sce/iop.h"
+#include "game/common/str_rpc_types.h"
+#include "game/overlord/isocommon.h"
+#include "game/overlord/iso_api.h"
+
+using namespace iop;
+
+static RPC_Str_Cmd sRPCBuf;
+void* RPC_STR(unsigned int fno, void* _cmd, int y);
+
+/*!
+ * We cache the chunk file headers so we can avoid seeking to the chunk header each time we
+ * need to load another chunk, even if we load chunks out of order.
+ */
+struct CacheEntry {
+  // the record for the chunk file described.
+  FileRecord* fr = nullptr;
+  // counts down from INT32_MAX - 1 each time we have a cache miss.
+  s32 countdown = 0;
+  // the actual cached data.
+  StrFileHeader header;
+};
+
+// the actual header cache.
+constexpr int STR_INDEX_CACHE_SIZE = 4;
+CacheEntry sCache[STR_INDEX_CACHE_SIZE];
+
+void stream_init_globals() {
+  memset(&sRPCBuf, 0, sizeof(RPC_Str_Cmd));
+}
+
+/*!
+ * Run the STR RPC handler.
+ */
 u32 STRThread() {
-  assert(false);
+  sceSifQueueData dq;
+  sceSifServeData serve;
+
+  CpuDisableIntr();
+  sceSifInitRpc(0);
+  sceSifSetRpcQueue(&dq, GetThreadId());
+  sceSifRegisterRpc(&serve, STR_RPC_ID, RPC_STR, &sRPCBuf, nullptr, nullptr, &dq);
+  CpuEnableIntr();
+  sceSifRpcLoop(&dq);
   return 0;
 }
 
 u32 PLAYThread() {
   assert(false);
   return 0;
+}
+
+/*!
+ * The STR RPC handler.
+ */
+void* RPC_STR(unsigned int fno, void* _cmd, int y) {
+  (void)fno;
+  (void)y;
+  auto* cmd = (RPC_Str_Cmd*)_cmd;
+  printf("RPC STR runs!\n");
+  if (cmd->chunk_id < 0) {
+    // it's _not_ a stream file. So we just treat it like a normal load.
+
+    // find the file with the given name
+    auto file_record = isofs->find(cmd->name);
+    if (file_record == nullptr) {
+      // file not found!
+      printf("[OVERLORD STR] Failed to find file %s for loading.\n", cmd->name);
+      cmd->result = STR_RPC_RESULT_ERROR;
+    } else {
+      // load directly to the EE
+      cmd->length = LoadISOFileToEE(file_record, cmd->ee_addr, cmd->length);
+      if (cmd->length) {
+        // successful load!
+        cmd->result = STR_RPC_RESULT_DONE;
+      } else {
+        // there was an error loading.
+        cmd->result = STR_RPC_RESULT_ERROR;
+      }
+    }
+  } else {
+    // it's a chunked file. These are only animations - these have a separate naming scheme.
+    char animation_iso_name[128];
+    ISONameFromAnimationName(animation_iso_name, cmd->name);
+    auto file_record = isofs->find_in(animation_iso_name);
+
+    if (!file_record) {
+      // didn't find the file
+      printf("[OVERLORD STR] Failed to find animation %s\n", cmd->name);
+      cmd->result = STR_RPC_RESULT_ERROR;
+    } else {
+      // found it! See if we've cached this animation's header.
+      int cache_entry = 0;
+      int oldest = INT32_MAX;
+      int oldest_idx = -1;
+      while (cache_entry < STR_INDEX_CACHE_SIZE && sCache[cache_entry].fr != file_record) {
+        sCache[cache_entry].countdown--;
+        if (sCache[cache_entry].countdown < oldest) {
+          oldest_idx = cache_entry;
+          oldest = sCache[cache_entry].countdown;
+        }
+        cache_entry++;
+      }
+
+      if (cache_entry == STR_INDEX_CACHE_SIZE) {
+        // cache miss, we need to load the header to the header cache on the IOP
+        cache_entry = oldest_idx;
+        sCache[oldest_idx].fr = file_record;
+        sCache[oldest_idx].countdown = INT32_MAX - 1;
+        if (!LoadISOFileToIOP(file_record, &sCache[oldest_idx].header, sizeof(StrFileHeader))) {
+          printf("[OVERLORD STR] Failed to load chunk file header for animation %s\n", cmd->name);
+          cmd->result = 1;
+          return cmd;
+        }
+      }
+
+      // load data, using the cached header to find the location of the chunk.
+      if (!LoadISOFileChunkToEE(file_record, cmd->ee_addr,
+                                sCache[cache_entry].header.sizes[cmd->chunk_id],
+                                sCache[cache_entry].header.sectors[cmd->chunk_id])) {
+        printf("[OVERLORD STR] Failed to load chunk %d for animation %s\n", cmd->chunk_id,
+               cmd->name);
+        cmd->result = 1;
+      } else {
+        // successful load!
+        cmd->length = sCache[cache_entry].header.sizes[cmd->chunk_id];
+        cmd->result = 0;
+      }
+    }
+  }
+  printf("Command result %d\n", cmd->result);
+  return cmd;
 }

game/overlord/stream.h

@@ -6,5 +6,6 @@
 #include "common/common_types.h"
 u32 STRThread();
 u32 PLAYThread();
+void stream_init_globals();
 
 #endif  // JAK_V2_STREAM_H

game/runtime.cpp

@@ -44,6 +44,7 @@
 #include "game/overlord/iso_cd.h"
 #include "game/overlord/overlord.h"
 #include "game/overlord/srpc.h"
+#include "game/overlord/stream.h"
 
 #include "common/goal_constants.h"
 
@@ -183,7 +184,7 @@ void iop_runner(SystemThreadInterface& iface) {
   // soundcommon
   srpc_init_globals();
   // ssound
-  // stream
+  stream_init_globals();
 
   iface.initialization_complete();
 

game/sce/sif_ee.cpp

@@ -85,6 +85,7 @@ s32 sceSifCallRpc(sceSifClientData* bd,
   assert(!end_para);
   assert(mode == 1);  // async
   iop->kernel.sif_rpc(bd->rpcd.id, fno, mode, send, ssize, recv, rsize);
+  iop->signal_run_iop();
   return 0;
 }