add location IDs to yaml db, get started on translating received ap data to rando data

the idea is to use the existing randomizer systems (like World, RandomizerContext, etc) in order to generate necessary data for in-game modifications such as text replacements, model swaps, etc. to do this we need to get the apworld's current settings, which atm will require loading the configured ap yaml file before connecting to archipelago.

include/d/d_item.h

@@ -250,6 +250,7 @@ void item_func_FUSED_SHADOW_1();
 void item_func_FUSED_SHADOW_2();
 void item_func_FUSED_SHADOW_3();
 void item_func_MIRROR_PIECE_1();
+void item_func_ARCHIPELAGO_ITEM();
 #endif
 void item_func_POU_SPIRIT();
 #if TARGET_PC

include/d/d_item_data.h

@@ -638,7 +638,7 @@ enum {
     /* 0xD9 */ dItemNo_Randomizer_FUSED_SHADOW_2_e,
     /* 0xDA */ dItemNo_Randomizer_FUSED_SHADOW_3_e,
     /* 0xDB */ dItemNo_Randomizer_MIRROR_PIECE_1_e,
-    /* 0xDC */ dItemNo_Randomizer_NOENTRY_220_e,
+    /* 0xDC */ dItemNo_Randomizer_ARCHIPELAGO_ITEM_e,
     /* 0xDD */ dItemNo_Randomizer_NOENTRY_221_e,
     /* 0xDE */ dItemNo_Randomizer_NOENTRY_222_e,
     /* 0xDF */ dItemNo_Randomizer_NOENTRY_223_e,

src/d/d_item.cpp

@@ -16,6 +16,8 @@
 #endif
 #include <cstring>
 
+#include "dusk/archipelago/archipelago_context.hpp"
+
 static void (*item_func_ptr[256])() = {
     item_func_HEART,
     item_func_GREEN_RUPEE,
@@ -497,7 +499,7 @@ static void (*item_func_ptr_randomizer[256])() = {
     /* 0xD9 */ item_func_FUSED_SHADOW_2,
     /* 0xDA */ item_func_FUSED_SHADOW_3,
     /* 0xDB */ item_func_MIRROR_PIECE_1,
-    /* 0xDC */ item_func_noentry,
+    /* 0xDC */ item_func_ARCHIPELAGO_ITEM,
     /* 0xDD */ item_func_noentry,
     /* 0xDE */ item_func_noentry,
     /* 0xDF */ item_func_noentry,
@@ -2180,6 +2182,13 @@ void item_func_MIRROR_PIECE_1() {
         dComIfGs_onCollectMirror(0);
     }
 }
+
+void item_func_ARCHIPELAGO_ITEM() {
+    if (randomizer_IsActive()) {
+        dusk::archi::ArchipelagoContext::UpdateCheckedLocations();
+    }
+}
+
 #endif
 
 void item_func_POU_SPIRIT() {

src/dusk/archipelago/archipelago_context.cpp

@@ -6,6 +6,7 @@
 #include "d/d_item.h"
 #include "dusk/config.hpp"
 #include "dusk/logging.h"
+#include "dusk/randomizer/game/tools.h"
 #include "dusk/ui/ui.hpp"
 
 namespace dusk::archi
@@ -13,11 +14,53 @@ namespace dusk::archi
 
 static constexpr int ARCHI_ITEM_OFFSET = 2320000;
 
+struct SettingsNameConvert {
+    std::string apName;
+    std::string dusklightName;
+    bool invert = false;
+};
+
+static auto sArchiSettingToDusklight = std::to_array<SettingsNameConvert>({
+    {"", ""},
+    {"golden_bugs_shuffled", "Golden Bugs"},
+    {"sky_characters_shuffled", "Sky Characters"},
+    {"npc_items_shuffled", "Gifts From NPCs"},
+    {"shop_items_shuffled", "Shop Items"},
+    {"hidden_skills_shuffled", "Hidden Skills"},
+    // {"poe_shuffled", ""}, // poe shuffle is Overworld, Dungeon, All, or Vanilla, so special logic is needed to convert
+    // {"heart_piece_shuffled", ""},
+    // {"overworld_shuffled", ""},
+    // {"dungeons_shuffled", ""},
+    {"dungeon_rewards_progression", "Dungeon Rewards Can Be Anywhere"},
+    {"small_keys_on_bosses", "No Small Keys on Bosses", true},
+    {"skip_prologue", "Skip Prologue"},
+    {"faron_twilight_cleared", "Faron Twilight Cleared"},
+    {"eldin_twilight_cleared", "Eldin Twilight Cleared"},
+    {"lanayru_twilight_cleared", "Lanayru Twilight Cleared"},
+    {"skip_mdh", "Skip Midna's Desparate Hour"},
+    {"open_map", "Unlock Map Regions"},
+    {"increase_wallet", "Increase Wallet Capacity"},
+    {"transform_anywhere", "Logic Transform Anywhere"},
+    {"bonks_do_damage", "Bonks Do Damage"},
+    {"skip_lakebed_entrance", "Lakebed Does Not Require Water Bombs"},
+    {"skip_arbiters_grounds_entrance", "Arbiters Does Not Require Bulblin Camp"},
+    {"skip_snowpeak_entrance", "Snowpeak Does Not Require Reekfish Scent"},
+    {"skip_city_in_the_sky_entrance", "City Does Not Require Filled Skybook"},
+});
+
 ArchipelagoContext& instance() {
     static ArchipelagoContext instance;
     return instance;
 }
 
+const SettingsNameConvert& GetAPSettingNameConvert(const std::string& apSettingName) {
+    for (const auto& entry : sArchiSettingToDusklight) {
+        if (entry.apName == apSettingName)
+            return entry;
+    }
+    return sArchiSettingToDusklight[0];
+}
+
 const char* getMessageTypeName(AP_MessageType type) {
     switch (type) {
     case AP_MessageType::Plaintext:
@@ -87,7 +130,35 @@ void ArchipelagoContext::LoadTempItemInfo() {
     }
 }
 
-void ArchipelagoContext::itemRecvImpl(int id) {
+void ArchipelagoContext::LoadTempLocationInfo() {
+    auto locDataTree = LOAD_EMBED_YAML(RANDO_DATA_PATH "locations.yaml");
+    for (const auto& locNode : locDataTree) {
+        const auto& metadata = locNode["Metadata"];
+        auto locationName =  locNode["Name"].as<std::string>();
+
+        if (!metadata.IsMap()) {
+            DuskLog.warn("Location {} missing correct Metadata field!", locationName);
+            continue;
+        }
+
+        if (!metadata["APLocationId"]) {
+            DuskLog.warn("Location {} missing APLocationId field!", locationName);
+            continue;
+        }
+
+        auto apLocationId = metadata["APLocationId"].as<int>();
+
+        if (apLocationId == -1)
+            continue;
+
+        m_apLocToGameLoc.push_back({
+            apLocationId,
+            locationName
+        });
+    }
+}
+
+void ArchipelagoContext::itemRecvImpl(int id, bool notify) {
     if (!m_apItemToGameItem.contains(id)) {
         DuskLog.warn("Got an invalid Item Id: {}", id);
         return;
@@ -104,6 +175,25 @@ void ArchipelagoContext::itemRecvImpl(int id) {
     }
 }
 
+int ArchipelagoContext::getItemIdFromApId(int apId) {
+    if (!m_apItemToGameItem.contains(apId)) {
+        DuskLog.warn("Got an invalid Item Id: {}", apId);
+        return -1;
+    }
+
+    auto& item = m_apItemToGameItem[apId];
+
+    return item.itemId;
+}
+
+std::string ArchipelagoContext::getLocationNameFromApId(int apId) const {
+    for (const auto& entry : m_apLocToGameLoc) {
+        if (entry.apId == apId)
+            return entry.locName;
+    }
+    return "";
+}
+
 ArchipelagoContext::ArchipelagoContext() {
 
 }
@@ -137,6 +227,8 @@ void ArchipelagoContext::ConnectToServer() {
 
     instance().LoadTempItemInfo();
 
+    instance().LoadTempLocationInfo();
+
     AP_Init(GetServerIp().c_str(), "Twilight Princess", GetSlotName().c_str(), GetPassword().c_str());
 
     AP_NetworkVersion ver{0, 6,7};
@@ -144,6 +236,7 @@ void ArchipelagoContext::ConnectToServer() {
 
     AP_SetItemClearCallback([]() {
         DuskLog.info("Item Clear Callback Called!");
+        // HandleResetInventory();
     });
 
     AP_SetItemRecvCallback([](int id, bool notify) {
@@ -155,6 +248,11 @@ void ArchipelagoContext::ConnectToServer() {
         DuskLog.info("Location Checked Callback Called! Location: {}", loc);
     });
 
+    AP_SetLocationInfoCallback([](std::vector<AP_NetworkItem> items) {
+        DuskLog.info("Got {} Location Scouts from Server.", items.size());
+        HandleReceiveLocationScout(items);
+    });
+
     AP_Start();
 
     if (AP_GetConnectionStatus() == AP_ConnectionStatus::ConnectionRefused) {
@@ -182,10 +280,14 @@ bool ArchipelagoContext::IsConnected() {
 
 void ArchipelagoContext::MainThreadFunc() {
     // wait a bit before checking connection state, as websocket is probably not connected yet
-    std::this_thread::sleep_for(std::chrono::seconds(1));
+    // (i really am not liking APCpp, why cant I check if the websocket is in the process of connecting???)
+    std::this_thread::sleep_for(std::chrono::seconds(2));
 
     DuskLog.info("AP Thread started.");
 
+    if (IsConnected())
+        RequestAllLocationScout();
+
     while (IsConnected()) {
         if (AP_IsMessagePending())
             ParseMessageData();
@@ -193,7 +295,7 @@ void ArchipelagoContext::MainThreadFunc() {
         instance().m_queueMutex.lock();
         if (randomizer_IsActive() && !instance().m_inactiveItemsQueue.empty()) {
             for (auto item : instance().m_inactiveItemsQueue) {
-                instance().itemRecvImpl(item);
+                instance().itemRecvImpl(item, false);
             }
 
             instance().m_inactiveItemsQueue.clear();
@@ -205,9 +307,11 @@ void ArchipelagoContext::MainThreadFunc() {
 }
 
 void ArchipelagoContext::HandleItemReceived(int id, bool notify) {
-    // TODO: instead of skipping inventory fill, we should clear the inventory when the clear item callback is called.
+    int relativeId = id - ARCHI_ITEM_OFFSET;
+
+    //  && ((relativeId >= 0 && relativeId <= 6) || relativeId == 7)
     if (!notify) {
-        DuskLog.info("Skipping Item.");
+        // skip rupee refills so players cant abuse disconnect/reconnect
         return;
     }
 
@@ -215,11 +319,187 @@ void ArchipelagoContext::HandleItemReceived(int id, bool notify) {
         DuskLog.info("Randomizer not active, adding item to queue.");
 
         instance().m_queueMutex.lock();
-        instance().m_inactiveItemsQueue.push_back(id - ARCHI_ITEM_OFFSET);
+        instance().m_inactiveItemsQueue.push_back(relativeId);
         instance().m_queueMutex.unlock();
         return;
     }
 
-    instance().itemRecvImpl(id - ARCHI_ITEM_OFFSET);
+    instance().itemRecvImpl(relativeId, notify);
+}
+
+void ArchipelagoContext::HandleResetInventory() {
+    // NOTE: this does not clear ALL things from save, so if a player managed to do something while disconnected from the archi, it might mess with things
+
+    auto& playerInfo = g_dComIfG_gameInfo.info.getPlayer();
+
+    // reset items
+    playerInfo.getItem().init();
+
+    // reset collect (poes, shards, swords)
+    playerInfo.getCollect().init();
+
+    playerInfo.getPlayerStatusA().setMaxLife(15);
+    playerInfo.getPlayerStatusA().setWalletSize(WALLET);
+    // dont reset rupees, and instead reject rupee updates while refilling inv
+
+}
+
+void ArchipelagoContext::HandleReceiveLocationScout(const std::vector<AP_NetworkItem>& items) {
+    for (const auto& item : items) {
+        int parsedItemId = dItemNo_Randomizer_ARCHIPELAGO_ITEM_e;
+        if (item.player == AP_GetPlayerID()) {
+            parsedItemId = instance().getItemIdFromApId(item.item - ARCHI_ITEM_OFFSET);
+        }
+        int locationId = item.location - ARCHI_ITEM_OFFSET;
+
+        auto locName = instance().getLocationNameFromApId(locationId);
+
+        if (locName.empty()) {
+            DuskLog.info("No location with ID {} found.", locationId);
+            continue;
+        }
+
+        instance().m_locationItemInfo[locName] = {
+            parsedItemId,
+            item.location,
+            false
+        };
+    }
+}
+
+void ArchipelagoContext::UpdateCheckedLocations() {
+    // TODO: we need a randomizer world to be able to check all valid locations and compare collection
+    // state against a previously cached value, and if collected we can send a location update packet.
+
+    return;
+
+    // non-usable example code
+
+    // replace this with however we'll be getting our world
+    std::unique_ptr<randomizer::logic::world::World> world = std::make_unique<randomizer::logic::world::World>(1, nullptr);
+
+    for (auto location : world->GetAllLocations()) {
+        auto locName = location->GetName();
+
+        if (!instance().m_locationItemInfo.contains(locName)) {
+            DuskLog.warn("No item found for ({}).", locName);
+            continue;
+        }
+
+        auto& cachedLocData = instance().m_locationItemInfo[locName];
+
+        bool isCollected = isLocationObtained(location);
+
+        if (isCollected && !cachedLocData.collected) {
+            cachedLocData.collected = true;
+            AP_SendItem(cachedLocData.apLocationId);
+        }
+    }
+}
+
+void ArchipelagoContext::RequestAllLocationScout(bool isHint) {
+    std::set<int64_t> locations;
+    // TEMP: apworld has 475 locations with ids in sequential order, so add them all individually to location set
+    // (eventually we will iterate through locations.yaml for a better data-driven solution)
+    for (int i = 0; i < 475; i++) {
+        locations.insert(ARCHI_ITEM_OFFSET + i);
+    }
+
+    AP_SendLocationScouts(locations, isHint);
+}
+
+void ArchipelagoContext::SetAPConfigYamlPath(const std::string_view& path) {
+    instance().m_apConfigPath = path;
+}
+
+bool ArchipelagoContext::GenerateConfigFromAP(randomizer::seedgen::config::Config& outConfig) {
+    if (instance().m_apConfigPath.empty()) {
+        DuskLog.warn("AP Config Path Empty!");
+        return false;
+    }
+
+    if (!std::filesystem::exists(instance().m_apConfigPath)) {
+        DuskLog.warn("AP Config Path does not exist!");
+        return false;
+    }
+
+    YAML::Node apConfigYaml;
+    try {
+        apConfigYaml = YAML::LoadFile(instance().m_apConfigPath);
+    }catch (YAML::BadFile& e) {
+        DuskLog.warn("Failed to load AP Config YAML file!");
+        return false;
+    }
+
+    outConfig.SetSeed("Archipelago");
+
+    randomizer::seedgen::settings::Settings settings;
+
+    auto defaultSettings = randomizer::seedgen::settings::GetAllSettingsInfo();
+
+    // add default settings to config first
+    for (const auto& [name, info] : *defaultSettings) {
+        if (info->GetType() == randomizer::seedgen::settings::Type::STANDARD)
+            settings.InsertSetting(name, randomizer::seedgen::settings::Setting(info.get(), info->GetDefaultOption()));
+    }
+
+    // update settings using ap config
+    for (const auto& apSettingEntry : apConfigYaml["Twilight Princess"]) {
+        auto apSettingName = apSettingEntry.first.as<std::string>();
+
+        // ignore AP-only settings
+        if (apSettingName == "progression_balancing" ||
+            apSettingName == "accessibility" ||
+            apSettingName == "local_items" ||
+            apSettingName == "non_local_items" ||
+            apSettingName == "start_inventory" ||
+            apSettingName == "start_hints" ||
+            apSettingName == "start_location_hints" ||
+            apSettingName == "exclude_locations" ||
+            apSettingName == "priority_locations" ||
+            apSettingName == "start_inventory_from_pool")
+            continue;
+
+        const auto& settingConvert = GetAPSettingNameConvert(apSettingName);
+
+        if (!settingConvert.apName.empty()) {
+            bool apSettingValue = apSettingEntry.second.as<bool>();
+
+            if (settingConvert.invert)
+                apSettingValue = !apSettingValue;
+
+            auto& setting = settings.GetMap().at(settingConvert.dusklightName);
+
+            if (apSettingValue) {
+                setting.SetCurrentOption("On");
+            }else {
+                setting.SetCurrentOption("Off");
+            }
+
+            continue;
+        }
+        // remaining settings will have string values
+
+        auto apSettingValue = apSettingEntry.second.as<std::string>();
+
+        // TODO: the rest of the translations
+
+        if (apSettingName == "castle_requirements") {
+
+        }
+
+    }
+
+    outConfig.GetSettingsList().push_back(settings);
+
+    return true;
+}
+
+int ArchipelagoContext::GetItemAtLocation(const std::string& locName) {
+    if (!instance().m_locationItemInfo.contains(locName)) {
+        DuskLog.warn("No item found for ({}).", locName);
+        return 0;
+    }
+    return instance().m_locationItemInfo[locName].itemId;
 }
 } // dusk::archi

src/dusk/archipelago/archipelago_context.hpp

@@ -3,6 +3,8 @@
 #include <mutex>
 #include <string>
 
+#include "Archipelago.h"
+
 namespace dusk::archi
 {
     class ArchipelagoContext {
@@ -13,15 +15,36 @@ namespace dusk::archi
             std::string itemName;
         };
 
+        struct TEMP_GameLocationInfo {
+            int apId;
+            std::string locName;
+        };
+
+        struct GameLocationInfo {
+            int itemId;
+            int64_t apLocationId;
+            bool collected;
+        };
+
         std::vector<int> m_inactiveItemsQueue;
         std::mutex m_queueMutex;
 
+        std::unordered_map<std::string, GameLocationInfo> m_locationItemInfo;
+
         // TEMP
         std::map<int, TEMP_GameItemInfo> m_apItemToGameItem;
+        std::vector<TEMP_GameLocationInfo> m_apLocToGameLoc;
+        std::string m_apConfigPath;
 
         void LoadTempItemInfo();
 
-        void itemRecvImpl(int id);
+        void LoadTempLocationInfo();
+
+        void itemRecvImpl(int id, bool notify);
+
+        int getItemIdFromApId(int apId);
+
+        std::string getLocationNameFromApId(int apId) const;
     public:
         ArchipelagoContext();
 
@@ -49,5 +72,23 @@ namespace dusk::archi
 
         static void HandleItemReceived(int id, bool notify);
 
+        static void HandleResetInventory();
+
+        static void HandleReceiveLocationScout(const std::vector<AP_NetworkItem>& items);
+
+        static void UpdateCheckedLocations();
+
+        // State Requesters
+
+        static void RequestAllLocationScout(bool isHint = false);
+
+        // AP -> Internal Rando Converters
+
+        static void SetAPConfigYamlPath(const std::string_view& path);
+
+        static bool GenerateConfigFromAP(randomizer::seedgen::config::Config& outConfig);
+
+        static int GetItemAtLocation(const std::string& locName);
+
     };
 } // dusk::archi

src/dusk/imgui/ImGuiArchipelagoDebug.cpp

@@ -3,9 +3,30 @@
 #include "imgui.h"
 
 #include "Archipelago.h"
+#include "aurora/lib/window.hpp"
+#include "dusk/file_select.hpp"
 #include "dusk/archipelago/archipelago_context.hpp"
 
 namespace dusk {
+
+constexpr std::array<SDL_DialogFileFilter, 2> kFileFilters{{
+    {"Archipelago Configuration File", "yaml"},
+    {"All Files", "*"},
+}};
+
+void FileDialogCallback(void*, const char* path, const char* error) {
+    if (path == nullptr || error != nullptr) {
+        return;
+    }
+
+    archi::ArchipelagoContext::SetAPConfigYamlPath(path);
+}
+
+void OpenApFilePicker() noexcept {
+    ShowFileSelect(&FileDialogCallback, nullptr, aurora::window::get_sdl_window(),
+        kFileFilters.data(), kFileFilters.size(), nullptr, false);
+}
+
 ImGuiArchipelagoDebug::ImGuiArchipelagoDebug() {
 
 }
@@ -44,6 +65,15 @@ void ImGuiArchipelagoDebug::drawWindow() {
         archi::ArchipelagoContext::SetSlotName(m_slotNameInputBuffer);
     }
 
+    if (ImGui::Button("Set Archipelago Config Path")) {
+        OpenApFilePicker();
+    }
+
+    if (ImGui::Button("Test Config Convert")) {
+        randomizer::seedgen::config::Config config;
+        archi::ArchipelagoContext::GenerateConfigFromAP(config);
+    }
+
     if (archi::ArchipelagoContext::IsConnected()) {
         if (ImGui::Button("Disconnect")) {
             archi::ArchipelagoContext::DisconnectFromServer();

src/dusk/randomizer/generator/data/items.yaml

@@ -130,6 +130,7 @@
 #- Name: Water Bombs 3
 #  Importance: Junk
 #  Id: 0x19
+#  APItemId: 16
 
 - Name: Bomblings 5
   Importance: Junk
@@ -144,6 +145,7 @@
 #- Name: Bomblings 3
 #  Importance: Junk
 #  Id: 0x1C
+#  APItemId: 20
 #
 #- Name: Bomblings 1
 #  Importance: Junk
@@ -601,7 +603,7 @@
 - Name: Horse Call
   Importance: Minor
   Id: 0x84
-  APItemId: -1
+  APItemId: 53
 
 - Name: Forest Temple Small Key
   Importance: Major
@@ -1133,7 +1135,7 @@
 - Name: North Faron Woods Gate Key
   Importance: Major
   Id: 0xEE
-  APItemId: -1
+  APItemId: 63
 
 #- Name: Blue Fire
 #  Importance: Major
@@ -1205,7 +1207,7 @@
 - Name: Coro Key
   Importance: Major
   Id: 0xFE
-  APItemId: -1
+  APItemId: 64
 
 #- Name: Invalid
 #  Importance: Major

src/dusk/settings.cpp

@@ -191,13 +191,13 @@ UserSettings g_userSettings = {
         ConfigVar<std::string>{"randomizer.file1SeedHash", ""},
          ConfigVar<std::string>{"randomizer.file2SeedHash", ""},
         ConfigVar<std::string>{"randomizer.file3SeedHash", ""},
-    }
+    },
     
     .archipelago = {
         .serverIP {"archipelago.serverIP", "archipelago.gg"},
         .serverPass {"archipelago.serverPass", ""},
         .slotName {"archipelago.slotName", ""},
-    },
+    }
 };
 
 UserSettings& getSettings() {