updates from the last few months

- move to database for configs
- don't try to update messages in the database that are older then the max storage duration
- bunch of general bugfixes

GearBot/Bot/GearBot.py

@@ -52,6 +52,9 @@ def dispatch(self, event_name, *args, **kwargs):
 
     #### event handlers, basically bouncing everything to TheRealGearBot file so we can hotreload our listeners
 
+    async def on_connect(self):
+        await TheRealGearBot.on_connect(self)
+
     async def on_ready(self):
         await TheRealGearBot.on_ready(self)
 

GearBot/Bot/TheRealGearBot.py

@@ -22,6 +22,7 @@
 
 from Util import Configuration, GearbotLogging, Emoji, Pages, Utils, Translator, InfractionUtils, MessageUtils, \
     server_info, DashConfig
+from Util.Configuration import ConfigNotLoaded
 from Util.Permissioncheckers import NotCachedException
 from Util.Utils import to_pretty_time
 from database import DatabaseConnector, DBUtils
@@ -34,7 +35,7 @@ def prefix_callable(bot, message):
     if message.guild is None:
         prefixes.append('!') #use default ! prefix in DMs
     elif bot.STARTUP_COMPLETE:
-        prefixes.append(Configuration.get_var(message.guild.id, "GENERAL", "PREFIX"))
+        prefixes.append(Configuration.legacy_get_var(message.guild.id, "GENERAL", "PREFIX"))
     return prefixes
 
 
@@ -126,6 +127,9 @@ async def on_message(bot, message:Message):
         else:
             await bot.invoke(ctx)
 
+async def on_connect(bot):
+    await Configuration.load_bulk([guild.id for guild in bot.guilds])
+
 
 async def on_guild_join(bot, guild: Guild):
     blocked = Configuration.get_persistent_var("server_blocklist", [])
@@ -145,7 +149,7 @@ async def on_guild_join(bot, guild: Guild):
         await guild.leave()
     else:
         GearbotLogging.info(f"A new guild came up: {guild.name} ({guild.id}).")
-        Configuration.load_config(guild.id)
+        await Configuration.load_config(guild.id)
         name = await Utils.clean(guild.name)
         await guild.chunk(cache=True)
         await GearbotLogging.bot_log(f"{Emoji.get_chat_emoji('JOIN')} A new guild came up: {name} ({guild.id}).", embed=server_info.server_info_embed(guild))
@@ -178,6 +182,8 @@ def __init__(self, type, error):
 
 
 async def on_command_error(bot, ctx: commands.Context, error):
+    if isinstance(error, ConfigNotLoaded):
+        return
     if isinstance(error, NotCachedException):
         if bot.loading_task is not None:
             if bot.initial_fill_complete:
@@ -190,7 +196,7 @@ async def on_command_error(bot, ctx: commands.Context, error):
         GearbotLogging.error(f"Encountered a permission error while executing {ctx.command}: {error}")
         await send(ctx, error)
     elif isinstance(error, commands.CheckFailure):
-        if ctx.command.qualified_name != "latest" and ctx.guild is not None and Configuration.get_var(ctx.guild.id, "GENERAL", "PERM_DENIED_MESSAGE"):
+        if ctx.command.qualified_name != "latest" and ctx.guild is not None and await Configuration.get_var(ctx.guild.id, "GENERAL", "PERM_DENIED_MESSAGE"):
             await MessageUtils.send_to(ctx, 'LOCK', 'permission_denied')
     elif isinstance(error, commands.CommandOnCooldown):
         await send(ctx, error)
@@ -393,7 +399,7 @@ async def handle_exception(exception_type, bot, exception, event=None, message=N
 
 
 
-        for t in [ConnectionClosed, ClientOSError, ServerDisconnectedError]:
+        for t in [ConnectionClosed, ClientOSError, ServerDisconnectedError, ConfigNotLoaded]:
             if isinstance(exception, t):
                 return
         #nice embed for info on discord

GearBot/Cogs/Admin.py

@@ -185,7 +185,7 @@ async def reset_cache(self, ctx):
     async def thread_migration(self, ctx):
         await MessageUtils.send_to(ctx, "LOADING", "Thread migration initiated", translate=False)
         for guild in self.bot.guilds:
-            role_id = Configuration.get_var(guild.id, 'ROLES', 'MUTE_ROLE')
+            role_id = await Configuration.get_var(guild.id, 'ROLES', 'MUTE_ROLE')
             if role_id != 0:
                 role = guild.get_role(role_id)
                 if role is not None:

GearBot/Cogs/AntiRaid.py

@@ -24,7 +24,7 @@ def __init__(self, bot):
 
     @commands.Cog.listener()
     async def on_member_join(self, member: disnake.Member):
-        raid_settings = Configuration.get_var(member.guild.id, "RAID_HANDLING")
+        raid_settings = await Configuration.get_var(member.guild.id, "RAID_HANDLING")
         if not raid_settings["ENABLED"]:
             return
 
@@ -175,15 +175,15 @@ async def raid_end(self, ctx):
             await MessageUtils.send_to(ctx, 'YES', 'raid_terminated')
 
     async def terminate_raid(self, guild):
-        raid_settings = Configuration.get_var(guild, "RAID_HANDLING")
+        raid_settings = await Configuration.get_var(guild, "RAID_HANDLING")
         for shield in raid_settings["SHIELDS"]:
             if guild in self.raid_trackers and shield["id"] in self.raid_trackers[guild]["SHIELDS"]:
                 h = self.raid_trackers[guild]["SHIELDS"][shield["id"]]
                 await self.terminate_shield(guild, h, shield)
 
     @raid.command('status')
     async def raid_status(self, ctx):
-        raid_settings = Configuration.get_var(ctx.guild.id, "RAID_HANDLING")
+        raid_settings = await Configuration.get_var(ctx.guild.id, "RAID_HANDLING")
         if len(raid_settings.get('SHIELDS', [])) == 0:
             await MessageUtils.send_to(ctx, 'WRENCH', 'raid_shields_not_configured')
         elif raid_settings['ENABLED']:
@@ -193,7 +193,7 @@ async def raid_status(self, ctx):
 
     @raid.command('enable')
     async def raid_enable(self, ctx):
-        raid_settings = Configuration.get_var(ctx.guild.id, "RAID_HANDLING")
+        raid_settings = await Configuration.get_var(ctx.guild.id, "RAID_HANDLING")
         if len(raid_settings.get('SHIELDS', [])) == 0:
             await MessageUtils.send_to(ctx, 'NO', 'raid_shields_not_configured')
         else:
@@ -202,11 +202,11 @@ async def raid_enable(self, ctx):
 
     @raid.command('disable')
     async def raid_disable(self, ctx):
-        raid_settings = Configuration.get_var(ctx.guild.id, "RAID_HANDLING")
+        raid_settings = await Configuration.get_var(ctx.guild.id, "RAID_HANDLING")
         if len(raid_settings.get('SHIELDS', [])) == 0:
             await MessageUtils.send_to(ctx, 'NO', 'raid_shields_not_configured')
         else:
-            Configuration.set_var(ctx.guild.id, 'RAID_HANDLING', 'ENABLED', False)
+            await Configuration.set_var(ctx.guild.id, 'RAID_HANDLING', 'ENABLED', False)
             await MessageUtils.send_to(ctx, 'YES', 'raid_shields_disabled')
 
 

GearBot/Cogs/AntiSpam.py

@@ -100,15 +100,15 @@ def get_bucket(self, guild_id, rule_name, bucket_info):
     async def on_message(self, message: Message):
         if message.author.id == self.bot.user.id or message.guild is None:
             return  # Don't track anti-spam for ourselves or DMs
-        cfg = Configuration.get_var(message.guild.id, "ANTI_SPAM")
+        cfg = await Configuration.get_var(message.guild.id, "ANTI_SPAM")
         if not cfg.get("ENABLED", False) or message.id in self.processed:
             return
         self.processed.append(message.id)
         await self.process_message(message)
 
     async def process_message(self, message: Message):
         # print(f'{datetime.datetime.now().isoformat()} - Processing message')
-        if message.webhook_id is not None or self.is_exempt(message.guild.id, message.author):
+        if message.webhook_id is not None or await self.is_exempt(message.guild.id, message.author):
             return
 
         # Use the discord's message timestamp to hopefully not trigger false positives
@@ -131,7 +131,7 @@ async def check_bucket(check, friendly_text, amount, b):
                                            b, count)))
 
         counters = dict()
-        buckets = Configuration.get_var(message.guild.id, "ANTI_SPAM", "BUCKETS", [])
+        buckets = await Configuration.get_var(message.guild.id, "ANTI_SPAM", "BUCKETS", [])
 
         # so if someone does 20 levels of too many mentions for some stupid reason we don't end up running the same regex 20 times for nothing
         cache = dict()
@@ -235,7 +235,9 @@ async def mute_punishment(self, v: Violation, member):
         duration = v.bucket["PUNISHMENT"]["DURATION"]
         until = time.time() + duration
         reason = self.assemble_reason(v)
-        role = AntiSpam._get_mute_role(v.guild)
+        role = await AntiSpam._get_mute_role(v.guild)
+        if role is None:
+            return
         i = await Infraction.get_or_none(user_id = member.id, type = "Mute", guild_id = member.guild.id, active=True)
         if i is None:
             i = await InfractionUtils.add_infraction(v.guild.id, member.id, self.bot.user.id, 'Mute', reason,
@@ -256,7 +258,7 @@ async def mute_punishment(self, v: Violation, member):
                                        moderator_id=v.guild.me.id,
                                        duration=Utils.to_pretty_time(duration, v.guild.id),
                                        reason=reason, inf=i.id)
-                if Configuration.get_var(v.guild.id, "INFRACTIONS", "DM_ON_MUTE"):
+                if await Configuration.get_var(v.guild.id, "INFRACTIONS", "DM_ON_MUTE"):
                     await Utils.send_infraction(self.bot, member, v.guild, 'MUTE', 'mute', reason, duration=Utils.to_pretty_time(duration, v.guild.id))
         else:
             i.end += duration
@@ -282,7 +284,7 @@ async def kick_punishment(self, v: Violation, member):
                                            active=False)
         await self.bot.redis_pool.psetex(f"forced_exits:{v.guild.id}-{member.id}", 8000, "1")
         try:
-            if Configuration.get_var(v.guild.id, "INFRACTIONS", "DM_ON_KICK"):
+            if await Configuration.get_var(v.guild.id, "INFRACTIONS", "DM_ON_KICK"):
                 asyncio.create_task(Utils.send_infraction(self.bot, member, v.guild, 'BOOT', 'kick', "Spam"))
             await v.guild.kick(member, reason=reason)
         except Forbidden:
@@ -302,7 +304,7 @@ async def temp_ban_punishment(self, v: Violation, member):
         await v.guild.ban(member, reason=reason, delete_message_days=0)
         i = await InfractionUtils.add_infraction(v.guild.id, member.id, self.bot.user.id, 'Tempban', reason,
                                            end=until)
-        if Configuration.get_var(v.guild.id, "INFRACTIONS", "DM_ON_TEMPBAN"):
+        if await Configuration.get_var(v.guild.id, "INFRACTIONS", "DM_ON_TEMPBAN"):
             dur = Utils.to_pretty_time(duration, None)
             asyncio.create_task(Utils.send_infraction(self.bot, member, v.guild, 'BAN', 'tempban', "Spam", duration=dur))
         GearbotLogging.log_key(v.guild.id, 'tempban_log', user=Utils.clean_user(member),
@@ -319,7 +321,7 @@ async def ban_punishment(self, v: Violation, member):
         GearbotLogging.log_key(v.guild.id, 'ban_log', user=Utils.clean_user(member), user_id=member.id,
                                moderator=Utils.clean_user(v.guild.me), moderator_id=v.guild.me.id,
                                reason=reason, inf=i.id)
-        if Configuration.get_var(v.guild.id, "INFRACTIONS", "DM_ON_BAN"):
+        if await Configuration.get_var(v.guild.id, "INFRACTIONS", "DM_ON_BAN"):
             asyncio.create_task(Utils.send_infraction(self.bot, member, v.guild, 'BAN', 'ban', "Spam"))
 
 
@@ -335,10 +337,10 @@ async def censor_detector(self):
                     return
 
                 # make sure anti-spam is enabled
-                cfg = Configuration.get_var(message.guild.id, "ANTI_SPAM")
+                cfg = await Configuration.get_var(message.guild.id, "ANTI_SPAM")
                 if not cfg.get("ENABLED", False) or message.id in self.censor_processed:
                     continue
-                buckets = Configuration.get_var(message.guild.id, "ANTI_SPAM", "BUCKETS", [])
+                buckets = await Configuration.get_var(message.guild.id, "ANTI_SPAM", "BUCKETS", [])
                 count = 0
                 for b in buckets:
                     t = b["TYPE"]
@@ -372,10 +374,10 @@ async def voice_spam_detector(self):
                     return
 
                 # make sure anti-spam is enabled
-                cfg = Configuration.get_var(member.guild.id, "ANTI_SPAM")
-                if after.channel is None or before.channel == after.channel or member is None or not cfg.get("ENABLED", False) or self.is_exempt(member.guild.id, member):
+                cfg = await Configuration.get_var(member.guild.id, "ANTI_SPAM")
+                if after.channel is None or before.channel == after.channel or member is None or not cfg.get("ENABLED", False) or await self.is_exempt(member.guild.id, member):
                     continue
-                buckets = Configuration.get_var(member.guild.id, "ANTI_SPAM", "BUCKETS", [])
+                buckets = await Configuration.get_var(member.guild.id, "ANTI_SPAM", "BUCKETS", [])
                 count = 0
                 for b in buckets:
                     t = b["TYPE"]
@@ -405,15 +407,15 @@ async def on_raw_message_delete(self, data: RawMessageDeleteEvent):
         member = await Utils.get_member(self.bot, self.bot.get_guild(data.guild_id), message.author)
         if member is None:
             return  # user no longer present, probably already actioned
-        if self.is_exempt(data.guild_id, member):
+        if await self.is_exempt(data.guild_id, member):
             return  # don't action except users
 
-        if data.message_id in self.bot.deleted_messages and not Configuration.get_var("GENERAL", "BOT_DELETED_STILL_GHOSTS"):
+        if data.message_id in self.bot.deleted_messages and not await Configuration.get_var(data.guild_id, "GENERAL", "BOT_DELETED_STILL_GHOSTS"):
             return
 
-        ghost_message_threshold = Configuration.get_var(data.guild_id, "GENERAL", "GHOST_MESSAGE_THRESHOLD")
-        ghost_ping_threshold = Configuration.get_var(data.guild_id, "GENERAL", "GHOST_PING_THRESHOLD")
-        buckets = Configuration.get_var(data.guild_id, "ANTI_SPAM", "BUCKETS", [])
+        ghost_message_threshold = await Configuration.get_var(data.guild_id, "GENERAL", "GHOST_MESSAGE_THRESHOLD")
+        ghost_ping_threshold = await Configuration.get_var(data.guild_id, "GENERAL", "GHOST_PING_THRESHOLD")
+        buckets = await Configuration.get_var(data.guild_id, "ANTI_SPAM", "BUCKETS", [])
         mentions = len(MENTION_MATCHER.findall(message.content))
 
         msg_time = int(snowflake_time(message.messageid).timestamp())
@@ -456,9 +458,9 @@ async def on_raw_message_delete(self, data: RawMessageDeleteEvent):
 
 
     async def handle_failed_ping(self, message: disnake.Message, amount):
-        if self.is_exempt(message.guild.id, message.author) or message.author.bot or message.webhook_id is not None:
+        if await self.is_exempt(message.guild.id, message.author) or message.author.bot or message.webhook_id is not None:
             return  # don't action except users
-        buckets = Configuration.get_var(message.guild.id, "ANTI_SPAM", "BUCKETS", [])
+        buckets = await Configuration.get_var(message.guild.id, "ANTI_SPAM", "BUCKETS", [])
         msg_time = int(snowflake_time(message.id).timestamp())
         for b in buckets:
             t = b["TYPE"]
@@ -485,18 +487,18 @@ def assemble_reason(v):
                                     friendly=v.friendly)
 
     @staticmethod
-    def is_exempt(guild_id, member: Member):
+    async def is_exempt(guild_id, member: Member):
         if not hasattr(member, "roles"):
             return False
-        config = Configuration.get_var(guild_id, "ANTI_SPAM")
+        config = await Configuration.get_var(guild_id, "ANTI_SPAM")
         for role in member.roles:
             if role.id in config["EXEMPT_ROLES"]:
                 return True
         return member.id in config["EXEMPT_USERS"] or Permissioncheckers.is_mod(member)
 
     @staticmethod
-    def _get_mute_role(guild):
-        role_id = Configuration.get_var(guild.id, "ROLES", "MUTE_ROLE")
+    async def _get_mute_role(guild):
+        role_id = await Configuration.get_var(guild.id, "ROLES", "MUTE_ROLE")
         if role_id == 0:
             return None
         role = guild.get_role(role_id)

GearBot/Cogs/Basic.py

@@ -74,7 +74,7 @@ async def about(self, ctx):
 
         click_here = Translator.translate('click_here', ctx)
         embed.add_field(name=Translator.translate('support_server', ctx),
-                        value=f"[{click_here}](https://disnake.gg/vddW3D9)")
+                        value=f"[{click_here}](https://discord.gg/vddW3D9)")
         embed.add_field(name=Translator.translate('website', ctx), value=f"[{click_here}](https://gearbot.rocks)")
         embed.add_field(name=f"Github", value=f"[{click_here}](https://github.com/gearbot/GearBot)")
         embed.set_footer(text=self.bot.user.name, icon_url=self.bot.user.display_avatar.url)
@@ -115,7 +115,7 @@ async def quote(self, ctx: commands.Context, *, message: Message):
                     if message.content is None or message.content == "":
                         if attachment is not None:
                             url = Utils.assemble_attachment(message.channel.id, attachment.id, attachment.name)
-                            if attachment.isImage:
+                            if attachment.isimage:
                                 embed.set_image(url=url)
                             else:
                                 embed.add_field(name=Translator.translate("attachment_link", ctx),
@@ -128,7 +128,7 @@ async def quote(self, ctx: commands.Context, *, message: Message):
                                         value=f"[Jump to message]({message.jump_url})")
                         if attachment is not None:
                             url = Utils.assemble_attachment(message.channel.id, attachment.id, attachment.name)
-                            if attachment.isImage:
+                            if attachment.isimage:
                                 embed.set_image(url=url)
                             else:
                                 embed.add_field(name=Translator.translate("attachment_link", ctx),
@@ -175,7 +175,7 @@ async def self_role(self, ctx: commands.Context, *, role: str = None):
             except BadArgument as ex:
                 await ctx.send(Translator.translate("role_not_found", ctx))
             else:
-                roles = Configuration.get_var(ctx.guild.id, "ROLES", "SELF_ROLES")
+                roles = await Configuration.get_var(ctx.guild.id, "ROLES", "SELF_ROLES")
                 if role.id in roles:
                     try:
                         if role in ctx.author.roles:
@@ -214,7 +214,7 @@ async def uid(self, ctx, *, text: str):
 
     @commands.Cog.listener()
     async def on_guild_role_delete(self, role: disnake.Role):
-        roles = Configuration.get_var(role.guild.id, "ROLES", "SELF_ROLES")
+        roles = await Configuration.get_var(role.guild.id, "ROLES", "SELF_ROLES")
         if role.id in roles:
             roles.remove(role.id)
             Configuration.save(role.guild.id)