diff --git a/libs/blt b/libs/blt index 9ad6521..9b4d0cc 160000 --- a/libs/blt +++ b/libs/blt @@ -1 +1 @@ -Subproject commit 9ad652195b0a69f9977d313eff4dd01a7890f1df +Subproject commit 9b4d0cc9a8493c608ab0075ab2c6a2b66061f3be diff --git a/src/main.cpp b/src/main.cpp index ce422d8..e793167 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -191,7 +191,13 @@ struct db_obj private: blt::u64 guildID; std::atomic_bool loaded_channels = false; + std::atomic_bool loaded_members = false; + blt::u64 user_count = -1; + std::atomic_uint64_t loaded_users = 0; + std::queue user_load_queue; + std::mutex user_load_queue_mutex; database_type db; + std::thread* thread; void ensure_channel_exists() { @@ -202,11 +208,6 @@ struct db_obj { } - - bool loading_complete() - { - return loaded_channels.load(); - } public: explicit db_obj(blt::u64 guildID, const std::string& path): guildID(guildID), db(make_database(path + "/" + std::to_string(guildID) + "/")) @@ -216,7 +217,7 @@ struct db_obj void load(dpp::cluster& bot, const dpp::guild& guild) { - bot.channels_get(guild.id, [&bot, this, &guild](const dpp::confirmation_callback_t& event) { + bot.channels_get(guild.id, [this, guild, &bot](const dpp::confirmation_callback_t& event) { if (event.is_error()) { BLT_WARN("Failed to fetch channels for guild %ld", guildID); @@ -229,44 +230,39 @@ struct db_obj for (const auto& channel : channel_map) { - BLT_DEBUG("\tfetched channel id %ld with name '%s'", channel.first, channel.second.name.c_str()); + if (channel.second.is_category()) + continue; + BLT_DEBUG("\tFetched channel id %ld with name '%s'", channel.first, channel.second.name.c_str()); } loaded_channels = true; + BLT_INFO("Finished loading channels for guild '%s'", guild.name.c_str()); }); - while (!loaded_channels.load()) - {} - - for (const auto& member : guild.members) - { - BLT_TRACE("\tfetching user %ld -> %ld", member.first, member.second.user_id); - bot.user_get(member.first, [member, this](const dpp::confirmation_callback_t& event) { - if (event.is_error()) + bot.guild_get_members(guildID, 1000, 0, [this, &bot, guild](const dpp::confirmation_callback_t& event) { + if (event.is_error()) + { + BLT_WARN("Failed to fetch members for guild %ld", guildID); + BLT_WARN("Cause: %s", event.get_error().human_readable.c_str()); + loaded_members = true; + return; + } + auto member_map = event.get(); + BLT_INFO("Guild '%s' member count: %ld", guild.name.c_str(), member_map.size()); + { + user_count = member_map.size(); + std::scoped_lock lock(user_load_queue_mutex); + for (const auto& member : member_map) { - BLT_WARN("Failed to fetch user $ld for guild '%s'", member.first, guildID); - BLT_WARN("Cause: %s", event.get_error().human_readable.c_str()); - return; + BLT_DEBUG("\tFetched member '%s'", member.second.get_nickname().c_str()); + user_load_queue.push(member.first); } - auto user = event.get(); - - BLT_DEBUG("We got user '%s' with username '%s' and global name '%s'", user.username.c_str(), user.global_name.c_str()); - }); - bot.guild_get_member(guildID, member.first, [&guild, member](const dpp::confirmation_callback_t& event) { - if (event.is_error()) - { - BLT_WARN("Failed to fetch member %ld for guild %ld", member.first, guild.id); - BLT_WARN("Cause: %s", event.get_error().human_readable.c_str()); - return; - } - auto user = event.get(); - - BLT_DEBUG("Member of guild '%s' with nickname '%s'", guild.name.c_str(), user.get_nickname().c_str()); - }); - } + } + BLT_INFO("Finished loading members for guild '%s'", guild.name.c_str()); + loaded_members = true; + }); - while (!loading_complete()) - {} - BLT_DEBUG("Finished loading guild '%s'", guild.name.c_str()); + BLT_DEBUG("Finished requesting info for guild '%s'", guild.name.c_str()); + process_queue(bot); } void commit(const user_info_t& edited) @@ -308,6 +304,64 @@ struct db_obj { } + + void process_queue(dpp::cluster& bot) + { + thread = new std::thread([this, &bot]() { + while (user_count != loaded_users.load()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + blt::u64 member = 0; + { + std::scoped_lock lock(user_load_queue_mutex); + if (user_load_queue.empty()) + continue; + member = user_load_queue.front(); + user_load_queue.pop(); + } + bot.user_get(member, [member, this](const dpp::confirmation_callback_t& event) { + if (event.is_error()) + { + BLT_WARN("Failed to fetch user %ld for guild '%ld'", member, guildID); + BLT_WARN("Cause: %s", event.get_error().human_readable.c_str()); + BLT_INFO("Error code %d with error message '%s'", event.get_error().code, event.get_error().message.c_str()); + // requeue on rate limit + if (event.get_error().code == 0) + { + for (const auto& v : event.get_error().errors) + { + BLT_TRACE0_STREAM << "\t" << v.code << '\n'; + BLT_TRACE0_STREAM << "\t" << v.field << '\n'; + BLT_TRACE0_STREAM << "\t" << v.index << '\n'; + BLT_TRACE0_STREAM << "\t" << v.object << '\n'; + BLT_TRACE0_STREAM << "\t" << v.reason << '\n'; + } + std::scoped_lock lock(user_load_queue_mutex); + user_load_queue.push(member); + } else + loaded_users++; + BLT_INFO("%ld vs %ld", user_count, loaded_users.load()); + return; + } + auto user = event.get(); + + BLT_DEBUG("We got user '%s' with global name '%s'", user.username.c_str(), user.global_name.c_str()); + loaded_users++; + }); + } + }); + } + + bool loading_complete() + { + return loaded_channels.load() && loaded_members.load() && user_count != -1ul && user_count == loaded_users.load(); + } + + ~db_obj() + { + thread->join(); + delete thread; + } }; @@ -315,10 +369,23 @@ blt::hashmap_t> databases; std::string path; blt::u64 total_guilds = 0; std::atomic_uint64_t completed_guilds = 0; +std::atomic_bool finished_loading = false; bool loading_complete() { - return total_guilds != 0 && total_guilds == completed_guilds.load(); + bool finished = true; + if (!finished_loading.load()) + { + for (const auto& v : databases) + { + if (!v.second->loading_complete()) + finished = false; + } + finished_loading.store(finished); + if (finished) + BLT_INFO("Loading complete!"); + } + return finished && total_guilds != 0 && total_guilds == completed_guilds.load(); } db_obj& get(blt::u64 id) @@ -370,6 +437,7 @@ int main(int argc, const char** argv) completed_guilds++; }); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); } } });