From 68d950a80597ed8af4e28a912b04ca39c08783cb Mon Sep 17 00:00:00 2001 From: Feyko Date: Sat, 23 Mar 2024 18:05:15 +0100 Subject: [PATCH 01/37] chore: actual README and easier hybrid setup, no more relative imports --- README.md | 59 +++++++++++++++++++++++++++- docker-compose-deps.yml | 26 ++++++++++++ fred/__init__.py | 2 +- fred/__main__.py | 4 +- fred/cogs/crashes.py | 4 +- fred/cogs/dialogflow.py | 20 +++++----- fred/cogs/levelling.py | 4 +- fred/cogs/mediaonly.py | 4 +- fred/cogs/welcome.py | 4 +- fred/fred.py | 8 ++-- fred/fred_commands/__init__.py | 4 +- fred/fred_commands/_baseclass.py | 4 +- fred/fred_commands/_command_utils.py | 2 +- fred/fred_commands/dbcommands.py | 2 +- fred/fred_commands/experience.py | 4 +- fred/libraries/__init__.py | 0 fred/libraries/common.py | 2 +- fred/libraries/createembed.py | 4 +- poetry.toml | 2 + pyproject.toml | 3 +- 20 files changed, 125 insertions(+), 37 deletions(-) create mode 100644 docker-compose-deps.yml create mode 100644 fred/libraries/__init__.py create mode 100644 poetry.toml diff --git a/README.md b/README.md index a79dbe9..3a5945d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,61 @@ # FICSIT-Fred A Discord bot for the Satisfactory Modding Discord -Thanks to Illya, creator of the bot ! +## Development +Want to contribute to Fred? Here's everything you need to know to get started + +### Prerequisites + +#### A Discord token +First of all, you'll need a Discord bot token for Fred to connect to Discord with +To get one, you can go to https://discord.com/developers/applications, register a new application, add a bot to the application and **activate the "message content intent"** +This intent is needed to receive the content of messages. Fred doesn't use slash commands so this is simply needed for him to function + +#### Docker +Docker is for making containers. If you're not sure what that means, look it up it's real cool! In our case, it helps use set up all needed dependencies for Fred without you needing to do anything other than install Docker +You can get docker [here](https://docs.docker.com/engine/install/). For Linux, make sure to **not** use Docker Desktop. For Windows, it is the easiest way +If you don't want to install Docker (especially on Windows where Docker Desktop can take up resources and requires virtualisation to be enabled), you can also manually set up a PostgreSQL DB and configure Fred to point to it. More on that later + +#### (Optional) Dialogflow auth +This is optional because this feature is currently disabled in Fred +You'll have get authentication information for dialogflow if you want to work on that + +### Setup +Two choices here: All through docker or hybrid local/docker +I recommend the hybrid way for ease of development. We don't have a proper devcontainer setup so debugging Fred when running in Docker is not great +Instead, in a hybrid setup, you'll use Docker for the postgres DB only and run Fred locally using Poetry + +#### Docker-only +Run `FRED_TOKEN= docker compose up -d` and that's it! Fred should run +(Note: This command assumes you're using bash as your shell. For Powershell or other, set the environment variable differently) + +You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show +You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose.yml` + +#### Hybrid +For this, you'll need [poetry](https://python-poetry.org/) installed, as well as `libpq` for the connection to the postgres DB +For libpq, you can install `libpq-dev` (or equivalent) on linux and [install postgres](https://www.postgresql.org/download/windows/) on Windows (test without it first actually I'm not certain this is needed) + +Once Poetry is installed, run `poetry install`. This should create a `.venv` folder that contains the virtualenv with all the necessary packages. Activate it using `poetry shell` or manually + +Now, run `docker compose up -d -f docker-compose-deps.yml`. This should spin up the postgres DB +You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show +You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose-deps.yml` + +Almost there! You'll now have to configure Fred. This just means setting the following env vars (found in `fred/__main__.py`) +``` +"FRED_IP", +"FRED_PORT", +"FRED_TOKEN", +"FRED_SQL_DB", +"FRED_SQL_USER", +"FRED_SQL_PASSWORD", +"FRED_SQL_HOST", +"FRED_SQL_PORT", +``` +For convenience, a `.env` file is included and Fred will load them automatically. It includes defaults that will work with the config of the docker-compose-deps.yml. You'll only have to set `FRED_TOKEN` and maybe change the others if the defaults don't suit you + +Finally, run `python fred/__main__.py`. Fred should run! You can now adapt this to your setup and run the script from your IDE instead. Don't forget to use the virtualenv python! + +## Thanks +Massive thanks to Borketh, Mircea and everyone else that has contributed! \ No newline at end of file diff --git a/docker-compose-deps.yml b/docker-compose-deps.yml new file mode 100644 index 0000000..405db74 --- /dev/null +++ b/docker-compose-deps.yml @@ -0,0 +1,26 @@ +version: '3' +services: + postgres: + image: postgres:14.0-alpine + environment: + POSTGRES_PASSWORD: fred + POSTGRES_USER: fred + PGDATA: /var/lib/postgresql/data/pgdata + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data/pgdata:z + - ./docker/exampledb.sql:/docker-entrypoint-initdb.d/exampledb.sql + + pgadmin: + depends_on: + - postgres + image: dpage/pgadmin4:6 + environment: + PGADMIN_DEFAULT_EMAIL: 'fred@fred.com' + PGADMIN_DEFAULT_PASSWORD: 'fred' + ports: + - "8080:80" + +volumes: + pgdata: \ No newline at end of file diff --git a/fred/__init__.py b/fred/__init__.py index 37cf71c..df474d8 100644 --- a/fred/__init__.py +++ b/fred/__init__.py @@ -1 +1 @@ -from .fred import __version__ +from fred import __version__ diff --git a/fred/__main__.py b/fred/__main__.py index f13e32d..6304683 100644 --- a/fred/__main__.py +++ b/fred/__main__.py @@ -12,14 +12,14 @@ "FRED_SQL_PASSWORD", "FRED_SQL_HOST", "FRED_SQL_PORT", - "DIALOGFLOW_AUTH", + # "DIALOGFLOW_AUTH", ] for var in ENVVARS: if getenv(var) is None: raise EnvironmentError(f"The ENV variable '{var}' isn't set") -from .fred import Bot, nextcord, __version__ +from fred import Bot, nextcord, __version__ intents = nextcord.Intents.all() diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index ea42cb4..7abf64d 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -14,8 +14,8 @@ from PIL import Image, ImageEnhance, UnidentifiedImageError from pytesseract import image_to_string, TesseractError -from .. import config -from ..libraries import createembed +import config +from libraries import createembed REGEX_LIMIT: float = 2 diff --git a/fred/cogs/dialogflow.py b/fred/cogs/dialogflow.py index 8823276..caa976a 100644 --- a/fred/cogs/dialogflow.py +++ b/fred/cogs/dialogflow.py @@ -9,15 +9,17 @@ from google.cloud import dialogflow from google.oauth2 import service_account -from .. import config -from ..libraries import common - -DIALOGFLOW_AUTH = json.loads(os.environ.get("DIALOGFLOW_AUTH")) -session_client = dialogflow.SessionsClient( - credentials=service_account.Credentials.from_service_account_info(DIALOGFLOW_AUTH) -) -DIALOGFLOW_PROJECT_ID = DIALOGFLOW_AUTH["project_id"] -SESSION_LIFETIME = 10 * 60 # 10 minutes to avoid repeated false positives +import config +from libraries import common + + +if os.environ.get('DIALOGFLOW_AUTH'): + DIALOGFLOW_AUTH = json.loads(os.environ.get("DIALOGFLOW_AUTH")) + session_client = dialogflow.SessionsClient( + credentials=service_account.Credentials.from_service_account_info(DIALOGFLOW_AUTH) + ) + DIALOGFLOW_PROJECT_ID = DIALOGFLOW_AUTH["project_id"] + SESSION_LIFETIME = 10 * 60 # 10 minutes to avoid repeated false positives class DialogFlow(commands.Cog): diff --git a/fred/cogs/levelling.py b/fred/cogs/levelling.py index 3b800b5..8d4a5a5 100644 --- a/fred/cogs/levelling.py +++ b/fred/cogs/levelling.py @@ -7,8 +7,8 @@ import nextcord.ext.commands as commands from nextcord import DMChannel -from .. import config -from ..libraries import common +import config +from libraries import common class UserProfile: diff --git a/fred/cogs/mediaonly.py b/fred/cogs/mediaonly.py index 9cf2f49..470eb60 100644 --- a/fred/cogs/mediaonly.py +++ b/fred/cogs/mediaonly.py @@ -2,8 +2,8 @@ import nextcord.ext.commands as commands -from ..libraries import common -from .. import config +from libraries import common +import config class MediaOnly(commands.Cog): diff --git a/fred/cogs/welcome.py b/fred/cogs/welcome.py index 22b2181..daae324 100644 --- a/fred/cogs/welcome.py +++ b/fred/cogs/welcome.py @@ -1,7 +1,7 @@ from nextcord.ext import commands -from .. import config -from ..libraries import common +import config +from libraries import common class Welcome(commands.Cog): diff --git a/fred/fred.py b/fred/fred.py index dc18d59..4cb2999 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -11,10 +11,10 @@ from nextcord.ext import commands import sqlobject as sql -from . import config -from .fred_commands import Commands, FredHelpEmbed -from .cogs import crashes, dialogflow, mediaonly, webhooklistener, welcome, levelling -from .libraries import createembed, common +import config +from fred_commands import Commands, FredHelpEmbed +from cogs import crashes, dialogflow, mediaonly, webhooklistener, welcome, levelling +from libraries import createembed, common __version__ = "2.20.3" diff --git a/fred/fred_commands/__init__.py b/fred/fred_commands/__init__.py index 5cee520..4a31c73 100644 --- a/fred/fred_commands/__init__.py +++ b/fred/fred_commands/__init__.py @@ -19,8 +19,8 @@ from .dialogflow import DialogflowCmds from .experience import EXPCmds from .help import HelpCmds, FredHelpEmbed -from ..libraries import createembed -from ..libraries.view.mod_picker import ModPicker +from libraries import createembed +from libraries.view.mod_picker import ModPicker class Commands(BotCmds, ChannelCmds, CommandCmds, CrashCmds, DialogflowCmds, EXPCmds, HelpCmds): diff --git a/fred/fred_commands/_baseclass.py b/fred/fred_commands/_baseclass.py index cfca9ec..f345ddc 100644 --- a/fred/fred_commands/_baseclass.py +++ b/fred/fred_commands/_baseclass.py @@ -1,8 +1,8 @@ import logging from nextcord.ext import commands -from .. import config -from ..libraries import common +import config +import libraries.common as common assert config # shut up linter, things that import this need this for convenience diff --git a/fred/fred_commands/_command_utils.py b/fred/fred_commands/_command_utils.py index cf57744..57d8e10 100644 --- a/fred/fred_commands/_command_utils.py +++ b/fred/fred_commands/_command_utils.py @@ -4,7 +4,7 @@ from regex import E, match, escape from ._baseclass import BaseCmds -from ..config import Commands, Crashes, Misc +from config import Commands, Crashes, Misc def search(table: Type[Commands | Crashes], pattern: str, column: str, force_fuzzy: bool) -> (str | list[str], bool): diff --git a/fred/fred_commands/dbcommands.py b/fred/fred_commands/dbcommands.py index 46380a1..7f53823 100644 --- a/fred/fred_commands/dbcommands.py +++ b/fred/fred_commands/dbcommands.py @@ -2,7 +2,7 @@ from ._baseclass import BaseCmds, commands, SearchFlags from ._command_utils import get_search -from .. import config +import config def _extract_prefix(string: str, prefix: str): diff --git a/fred/fred_commands/experience.py b/fred/fred_commands/experience.py index 8fa4656..907df76 100644 --- a/fred/fred_commands/experience.py +++ b/fred/fred_commands/experience.py @@ -1,6 +1,6 @@ from ._baseclass import BaseCmds, commands, common, config -from ..cogs import levelling -from ..libraries import createembed +from cogs import levelling +from libraries import createembed class EXPCmds(BaseCmds): diff --git a/fred/libraries/__init__.py b/fred/libraries/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fred/libraries/common.py b/fred/libraries/common.py index dff7050..2e06bb6 100644 --- a/fred/libraries/common.py +++ b/fred/libraries/common.py @@ -5,7 +5,7 @@ from nextcord import User, Message from nextcord.ext.commands import Context -from .. import config +import config logger = logging.Logger("PERMISSIONS") diff --git a/fred/libraries/createembed.py b/fred/libraries/createembed.py index 0d095f4..8a6f8da 100644 --- a/fred/libraries/createembed.py +++ b/fred/libraries/createembed.py @@ -7,8 +7,8 @@ from PIL import Image from nextcord.utils import format_dt -from . import common -from .. import config +import libraries.common +import config def timestamp(iso8601: str) -> str: diff --git a/poetry.toml b/poetry.toml new file mode 100644 index 0000000..efa46ec --- /dev/null +++ b/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f644f5b..b1fe26c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.poetry] -name = "FICSIT-Fred" +name = "fred" version = "2.18.10" description = "A Discord bot for the Satisfactory Modding Discord " authors = ["Feyko ", "Mircea Roata ", "Astro Floof "] @@ -17,6 +17,7 @@ algoliasearch = "^2.6.1" SQLObject = "^3.9.1" google-cloud-dialogflow = "^2.11.0" regex = "^2022.3.15" +aiohttp = "3.9.3" [tool.poetry.dev-dependencies] black = "^22.1.0" From c6cd6a632261342371373759dfd7aa4c43659e13 Mon Sep 17 00:00:00 2001 From: Feyko Date: Sat, 23 Mar 2024 18:22:03 +0100 Subject: [PATCH 02/37] Fix README.md formatting GitKraken ate my trailing spaces :( (the one bad thing in markdown) --- README.md | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 3a5945d..3ec3af9 100644 --- a/README.md +++ b/README.md @@ -7,42 +7,42 @@ Want to contribute to Fred? Here's everything you need to know to get started ### Prerequisites #### A Discord token -First of all, you'll need a Discord bot token for Fred to connect to Discord with -To get one, you can go to https://discord.com/developers/applications, register a new application, add a bot to the application and **activate the "message content intent"** -This intent is needed to receive the content of messages. Fred doesn't use slash commands so this is simply needed for him to function +First of all, you'll need a Discord bot token for Fred to connect to Discord with +To get one, you can go to https://discord.com/developers/applications, register a new application, add a bot to the application and **activate the "message content intent"** +This intent is needed to receive the content of messages. Fred doesn't use slash commands so this is simply needed for him to function #### Docker -Docker is for making containers. If you're not sure what that means, look it up it's real cool! In our case, it helps use set up all needed dependencies for Fred without you needing to do anything other than install Docker -You can get docker [here](https://docs.docker.com/engine/install/). For Linux, make sure to **not** use Docker Desktop. For Windows, it is the easiest way -If you don't want to install Docker (especially on Windows where Docker Desktop can take up resources and requires virtualisation to be enabled), you can also manually set up a PostgreSQL DB and configure Fred to point to it. More on that later +Docker is for making containers. If you're not sure what that means, look it up it's real cool! In our case, it helps use set up all needed dependencies for Fred without you needing to do anything other than install Docker +You can get docker [here](https://docs.docker.com/engine/install/). For Linux, make sure to **not** use Docker Desktop. For Windows, it is the easiest way +If you don't want to install Docker (especially on Windows where Docker Desktop can take up resources and requires virtualisation to be enabled), you can also manually set up a PostgreSQL DB and configure Fred to point to it. More on that later #### (Optional) Dialogflow auth -This is optional because this feature is currently disabled in Fred -You'll have get authentication information for dialogflow if you want to work on that +This is optional because this feature is currently disabled in Fred +You'll have get authentication information for dialogflow if you want to work on that ### Setup -Two choices here: All through docker or hybrid local/docker -I recommend the hybrid way for ease of development. We don't have a proper devcontainer setup so debugging Fred when running in Docker is not great -Instead, in a hybrid setup, you'll use Docker for the postgres DB only and run Fred locally using Poetry +Two choices here: All through docker or hybrid local/docker +I recommend the hybrid way for ease of development. We don't have a proper devcontainer setup so debugging Fred when running in Docker is not great +Instead, in a hybrid setup, you'll use Docker for the postgres DB only and run Fred locally using Poetry #### Docker-only -Run `FRED_TOKEN= docker compose up -d` and that's it! Fred should run -(Note: This command assumes you're using bash as your shell. For Powershell or other, set the environment variable differently) +Run `FRED_TOKEN= docker compose up -d` and that's it! Fred should run +(Note: This command assumes you're using bash as your shell. For Powershell or other, set the environment variable differently) -You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show -You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose.yml` +You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show +You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose.yml` #### Hybrid -For this, you'll need [poetry](https://python-poetry.org/) installed, as well as `libpq` for the connection to the postgres DB -For libpq, you can install `libpq-dev` (or equivalent) on linux and [install postgres](https://www.postgresql.org/download/windows/) on Windows (test without it first actually I'm not certain this is needed) +For this, you'll need [poetry](https://python-poetry.org/) installed, as well as `libpq` for the connection to the postgres DB +For libpq, you can install `libpq-dev` (or equivalent) on linux and [install postgres](https://www.postgresql.org/download/windows/) on Windows (test without it first actually I'm not certain this is needed) -Once Poetry is installed, run `poetry install`. This should create a `.venv` folder that contains the virtualenv with all the necessary packages. Activate it using `poetry shell` or manually +Once Poetry is installed, run `poetry install`. This should create a `.venv` folder that contains the virtualenv with all the necessary packages. Activate it using `poetry shell` or manually -Now, run `docker compose up -d -f docker-compose-deps.yml`. This should spin up the postgres DB -You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show -You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose-deps.yml` +Now, run `docker compose up -d -f docker-compose-deps.yml`. This should spin up the postgres DB +You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show +You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose-deps.yml` -Almost there! You'll now have to configure Fred. This just means setting the following env vars (found in `fred/__main__.py`) +Almost there! You'll now have to configure Fred. This just means setting the following env vars (found in `fred/__main__.py`) ``` "FRED_IP", "FRED_PORT", @@ -53,9 +53,9 @@ Almost there! You'll now have to configure Fred. This just means setting the fol "FRED_SQL_HOST", "FRED_SQL_PORT", ``` -For convenience, a `.env` file is included and Fred will load them automatically. It includes defaults that will work with the config of the docker-compose-deps.yml. You'll only have to set `FRED_TOKEN` and maybe change the others if the defaults don't suit you +For convenience, a `.env` file is included and Fred will load them automatically. It includes defaults that will work with the config of the docker-compose-deps.yml. You'll only have to set `FRED_TOKEN` and maybe change the others if the defaults don't suit you -Finally, run `python fred/__main__.py`. Fred should run! You can now adapt this to your setup and run the script from your IDE instead. Don't forget to use the virtualenv python! +Finally, run `python fred/__main__.py`. Fred should run! You can now adapt this to your setup and run the script from your IDE instead. Don't forget to use the virtualenv python! ## Thanks -Massive thanks to Borketh, Mircea and everyone else that has contributed! \ No newline at end of file +Massive thanks to Borketh, Mircea and everyone else that has contributed! From 055249d890c73caf10ce486ab3c0ed414f0405d7 Mon Sep 17 00:00:00 2001 From: borketh Date: Tue, 30 Jul 2024 18:53:27 -0400 Subject: [PATCH 03/37] fix: relative imports throughout what was there before worked before, no idea what changed but this fixes it. --- fred/__init__.py | 2 +- fred/__main__.py | 2 +- fred/cogs/crashes.py | 4 ++-- fred/cogs/dialogflow.py | 4 ++-- fred/cogs/levelling.py | 4 ++-- fred/cogs/mediaonly.py | 4 ++-- fred/cogs/welcome.py | 4 ++-- fred/fred.py | 12 +++++------- fred/fred_commands/__init__.py | 4 ++-- fred/fred_commands/_baseclass.py | 6 +++--- fred/fred_commands/_command_utils.py | 2 +- fred/fred_commands/crashes.py | 4 ++-- fred/fred_commands/dbcommands.py | 4 +--- fred/fred_commands/experience.py | 4 ++-- fred/libraries/common.py | 2 +- fred/libraries/createembed.py | 4 ++-- 16 files changed, 31 insertions(+), 35 deletions(-) diff --git a/fred/__init__.py b/fred/__init__.py index df474d8..37cf71c 100644 --- a/fred/__init__.py +++ b/fred/__init__.py @@ -1 +1 @@ -from fred import __version__ +from .fred import __version__ diff --git a/fred/__main__.py b/fred/__main__.py index 6304683..1e049a6 100644 --- a/fred/__main__.py +++ b/fred/__main__.py @@ -19,7 +19,7 @@ if getenv(var) is None: raise EnvironmentError(f"The ENV variable '{var}' isn't set") -from fred import Bot, nextcord, __version__ +from .fred import Bot, nextcord, __version__ intents = nextcord.Intents.all() diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index 7abf64d..ea42cb4 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -14,8 +14,8 @@ from PIL import Image, ImageEnhance, UnidentifiedImageError from pytesseract import image_to_string, TesseractError -import config -from libraries import createembed +from .. import config +from ..libraries import createembed REGEX_LIMIT: float = 2 diff --git a/fred/cogs/dialogflow.py b/fred/cogs/dialogflow.py index caa976a..250e2ec 100644 --- a/fred/cogs/dialogflow.py +++ b/fred/cogs/dialogflow.py @@ -9,8 +9,8 @@ from google.cloud import dialogflow from google.oauth2 import service_account -import config -from libraries import common +from .. import config +from ..libraries import common if os.environ.get('DIALOGFLOW_AUTH'): diff --git a/fred/cogs/levelling.py b/fred/cogs/levelling.py index 8d4a5a5..3b800b5 100644 --- a/fred/cogs/levelling.py +++ b/fred/cogs/levelling.py @@ -7,8 +7,8 @@ import nextcord.ext.commands as commands from nextcord import DMChannel -import config -from libraries import common +from .. import config +from ..libraries import common class UserProfile: diff --git a/fred/cogs/mediaonly.py b/fred/cogs/mediaonly.py index 470eb60..ac38250 100644 --- a/fred/cogs/mediaonly.py +++ b/fred/cogs/mediaonly.py @@ -2,8 +2,8 @@ import nextcord.ext.commands as commands -from libraries import common -import config +from .. import config +from ..libraries import common class MediaOnly(commands.Cog): diff --git a/fred/cogs/welcome.py b/fred/cogs/welcome.py index daae324..22b2181 100644 --- a/fred/cogs/welcome.py +++ b/fred/cogs/welcome.py @@ -1,7 +1,7 @@ from nextcord.ext import commands -import config -from libraries import common +from .. import config +from ..libraries import common class Welcome(commands.Cog): diff --git a/fred/fred.py b/fred/fred.py index 4a41000..463b622 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -2,20 +2,18 @@ import logging import os import sys -import textwrap import time import traceback import aiohttp import nextcord -from nextcord.ext import commands import sqlobject as sql +from nextcord.ext import commands -import config -from fred_commands import Commands, FredHelpEmbed -from cogs import crashes, dialogflow, mediaonly, webhooklistener, welcome, levelling -from libraries import createembed, common - +from . import config +from .cogs import crashes, dialogflow, mediaonly, webhooklistener, welcome, levelling +from .fred_commands import Commands, FredHelpEmbed +from .libraries import createembed, common __version__ = "2.20.4" diff --git a/fred/fred_commands/__init__.py b/fred/fred_commands/__init__.py index 73cb0f6..af58da1 100644 --- a/fred/fred_commands/__init__.py +++ b/fred/fred_commands/__init__.py @@ -19,8 +19,8 @@ from .dialogflow import DialogflowCmds from .experience import EXPCmds from .help import HelpCmds, FredHelpEmbed -from libraries import createembed -from libraries.view.mod_picker import ModPicker +from ..libraries import createembed +from ..libraries.view.mod_picker import ModPicker class Commands(BotCmds, ChannelCmds, CommandCmds, CrashCmds, DialogflowCmds, EXPCmds, HelpCmds): diff --git a/fred/fred_commands/_baseclass.py b/fred/fred_commands/_baseclass.py index f345ddc..10c2c35 100644 --- a/fred/fred_commands/_baseclass.py +++ b/fred/fred_commands/_baseclass.py @@ -1,10 +1,10 @@ import logging from nextcord.ext import commands -import config -import libraries.common as common +from .. import config +from ..libraries import common -assert config # shut up linter, things that import this need this for convenience +assert config # noqa class BaseCmds(commands.Cog): diff --git a/fred/fred_commands/_command_utils.py b/fred/fred_commands/_command_utils.py index 100afaa..17827ea 100644 --- a/fred/fred_commands/_command_utils.py +++ b/fred/fred_commands/_command_utils.py @@ -4,7 +4,7 @@ from regex import E, match, escape from ._baseclass import BaseCmds -from config import Commands, Crashes, Misc +from ..config import Commands, Crashes, Misc def search(table: Type[Commands | Crashes], pattern: str, column: str, force_fuzzy: bool) -> (str | list[str], bool): diff --git a/fred/fred_commands/crashes.py b/fred/fred_commands/crashes.py index 25aec41..9a5f438 100644 --- a/fred/fred_commands/crashes.py +++ b/fred/fred_commands/crashes.py @@ -1,6 +1,6 @@ -from ._baseclass import BaseCmds, commands, config, SearchFlags from regex import search, error as RegexError -from typing import Literal + +from ._baseclass import BaseCmds, commands, config, SearchFlags from ._command_utils import get_search diff --git a/fred/fred_commands/dbcommands.py b/fred/fred_commands/dbcommands.py index 7f53823..99c217e 100644 --- a/fred/fred_commands/dbcommands.py +++ b/fred/fred_commands/dbcommands.py @@ -1,8 +1,6 @@ -from typing import Literal - from ._baseclass import BaseCmds, commands, SearchFlags from ._command_utils import get_search -import config +from .. import config def _extract_prefix(string: str, prefix: str): diff --git a/fred/fred_commands/experience.py b/fred/fred_commands/experience.py index 907df76..8fa4656 100644 --- a/fred/fred_commands/experience.py +++ b/fred/fred_commands/experience.py @@ -1,6 +1,6 @@ from ._baseclass import BaseCmds, commands, common, config -from cogs import levelling -from libraries import createembed +from ..cogs import levelling +from ..libraries import createembed class EXPCmds(BaseCmds): diff --git a/fred/libraries/common.py b/fred/libraries/common.py index 2e06bb6..dff7050 100644 --- a/fred/libraries/common.py +++ b/fred/libraries/common.py @@ -5,7 +5,7 @@ from nextcord import User, Message from nextcord.ext.commands import Context -import config +from .. import config logger = logging.Logger("PERMISSIONS") diff --git a/fred/libraries/createembed.py b/fred/libraries/createembed.py index 21a8fd5..761e60f 100644 --- a/fred/libraries/createembed.py +++ b/fred/libraries/createembed.py @@ -7,8 +7,8 @@ from PIL import Image from nextcord.utils import format_dt -import libraries.common -import config +from .. import config +from ..libraries import common def timestamp(iso8601: str) -> str: From e6fd26ddc0f3d6d8cca1731043921f719266e90f Mon Sep 17 00:00:00 2001 From: borketh Date: Tue, 30 Jul 2024 18:54:26 -0400 Subject: [PATCH 04/37] ci: updates to tooling and versions - more to come - verified that fred works on 3.11 and can go forward using that - removed version tag from docker-compose files as it is deprecated - updated Dockerfile to use py3.11 and use new poetry profile nomenclature - updated fred version in pyproject because no one did for ages lmao --- docker-compose-deps.yml | 1 - docker-compose.yml | 1 - docker/Dockerfile | 6 +++--- pyproject.toml | 8 ++++---- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docker-compose-deps.yml b/docker-compose-deps.yml index 405db74..4c7bac2 100644 --- a/docker-compose-deps.yml +++ b/docker-compose-deps.yml @@ -1,4 +1,3 @@ -version: '3' services: postgres: image: postgres:14.0-alpine diff --git a/docker-compose.yml b/docker-compose.yml index 70920f9..90364d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: fred: depends_on: diff --git a/docker/Dockerfile b/docker/Dockerfile index 8581de4..132521d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim as runtime +FROM python:3.11-slim AS runtime ENV DEBIAN_FRONTEND=noninteractive VOLUME /config @@ -7,7 +7,7 @@ WORKDIR /app COPY docker/runtime-deps.sh . RUN bash runtime-deps.sh 1> /dev/null && rm runtime-deps.sh -FROM python:3.10-slim as build +FROM python:3.11-slim AS build WORKDIR /app @@ -16,7 +16,7 @@ RUN apt-get -qq update && apt-get -qq install curl libpq-dev gcc 1> /dev/null RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/usr/local python3 - COPY pyproject.toml . -RUN poetry install -nvvv --no-dev && mv $(poetry env info --path) /app/venv +RUN poetry install -nvvv --only main && mv $(poetry env info --path) /app/venv FROM runtime diff --git a/pyproject.toml b/pyproject.toml index b1fe26c..4009d46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] name = "fred" -version = "2.18.10" +version = "2.20.4" description = "A Discord bot for the Satisfactory Modding Discord " authors = ["Feyko ", "Mircea Roata ", "Astro Floof "] license = "MIT License" [tool.poetry.dependencies] -python = "^3.10" -nextcord = "^2.1.0" +python = "^3.11" +nextcord = "^2.6" Pillow = "^9.0.0" pytesseract = "^0.3.8" psycopg2 = "^2.9.3" @@ -24,7 +24,7 @@ black = "^22.1.0" [tool.black] line-length = 120 -target-version = ['py310'] +target-version = ['py311'] [build-system] requires = ["poetry-core>=1.0.0"] From 2825b34c1d0687916dce5ffdda81b7b06c408a0e Mon Sep 17 00:00:00 2001 From: mircearoata Date: Thu, 1 Aug 2024 23:13:38 +0300 Subject: [PATCH 05/37] ci: update action versions and variables --- .github/workflows/docker-publish.yml | 30 ++++++++++------------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 720ebce..346353d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -13,10 +13,8 @@ on: tags: [ 'v*.*.*' ] env: - # Use docker.io for Docker Hub if empty REGISTRY: ghcr.io - # github.repository as / - IMAGE_NAME: feyko/ficsit-fred + IMAGE_NAME: ${{ github.repository }} jobs: @@ -26,31 +24,25 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 - - # Login against a Docker registry except on PR + uses: actions/checkout@v4 + - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 with: registry: ${{ env.REGISTRY }} - username: feyko - password: ${{ secrets.GH_TOKEN }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - uses: docker/setup-buildx-action@v1 - id: buildx - - # Build and push Docker image with Buildx (don't push on PR) - # https://github.com/docker/build-push-action + - uses: docker/setup-buildx-action@v3 + - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: context: . file: ./docker/Dockerfile From 1e7f232462d13061134917bf8d1b1e3f64b302a3 Mon Sep 17 00:00:00 2001 From: Rob B Date: Thu, 1 Aug 2024 19:08:19 -0400 Subject: [PATCH 06/37] Fix readme typos, missing periods, and formatting --- README.md | 64 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 3ec3af9..47cf70c 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,59 @@ # FICSIT-Fred + A Discord bot for the Satisfactory Modding Discord ## Development -Want to contribute to Fred? Here's everything you need to know to get started + +Want to contribute to Fred? Here's everything you need to know to get started. ### Prerequisites #### A Discord token -First of all, you'll need a Discord bot token for Fred to connect to Discord with -To get one, you can go to https://discord.com/developers/applications, register a new application, add a bot to the application and **activate the "message content intent"** -This intent is needed to receive the content of messages. Fred doesn't use slash commands so this is simply needed for him to function + +First of all, you'll need a Discord bot token for Fred to connect to Discord with. +To get one, you can go to , register a new application, add a bot to the application and **activate the "message content intent"**. +This intent is needed to receive the content of messages. +Fred doesn't use slash commands so this is simply needed for him to function. #### Docker -Docker is for making containers. If you're not sure what that means, look it up it's real cool! In our case, it helps use set up all needed dependencies for Fred without you needing to do anything other than install Docker -You can get docker [here](https://docs.docker.com/engine/install/). For Linux, make sure to **not** use Docker Desktop. For Windows, it is the easiest way -If you don't want to install Docker (especially on Windows where Docker Desktop can take up resources and requires virtualisation to be enabled), you can also manually set up a PostgreSQL DB and configure Fred to point to it. More on that later + +Docker is for making containers. If you're not sure what that means, look it up it's real cool! In our case, it helps use set up all needed dependencies for Fred without you needing to do anything other than install Docker. +You can get docker [here](https://docs.docker.com/engine/install/). For Linux, make sure to **not** use Docker Desktop. For Windows, it is the easiest way. +If you don't want to install Docker (especially on Windows where Docker Desktop can take up resources and requires virtualisation to be enabled), you can also manually set up a PostgreSQL DB and configure Fred to point to it. More on that later. #### (Optional) Dialogflow auth -This is optional because this feature is currently disabled in Fred -You'll have get authentication information for dialogflow if you want to work on that + +This is optional because this feature is currently disabled in Fred. +You'll have get authentication information for dialogflow if you want to work on that. ### Setup -Two choices here: All through docker or hybrid local/docker -I recommend the hybrid way for ease of development. We don't have a proper devcontainer setup so debugging Fred when running in Docker is not great -Instead, in a hybrid setup, you'll use Docker for the postgres DB only and run Fred locally using Poetry + +Two choices here: All through docker or hybrid local/docker. +I recommend the hybrid way for ease of development. We don't have a proper devcontainer setup so debugging Fred when running in Docker is not great. +Instead, in a hybrid setup, you'll use Docker for the postgres DB only and run Fred locally using Poetry. #### Docker-only -Run `FRED_TOKEN= docker compose up -d` and that's it! Fred should run -(Note: This command assumes you're using bash as your shell. For Powershell or other, set the environment variable differently) -You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show -You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose.yml` +Run `FRED_TOKEN= docker compose up -d` and that's it! Fred should run. +(Note: This command assumes you're using bash as your shell. For Powershell or other, set the environment variable differently). + +You can verify that the database was properly created and manage it by going to where pgadmin should show. +You can use `fred@fred.com` for the user and `fred` for the password. All of this is customizable in `docker-compose.yml`. #### Hybrid -For this, you'll need [poetry](https://python-poetry.org/) installed, as well as `libpq` for the connection to the postgres DB -For libpq, you can install `libpq-dev` (or equivalent) on linux and [install postgres](https://www.postgresql.org/download/windows/) on Windows (test without it first actually I'm not certain this is needed) -Once Poetry is installed, run `poetry install`. This should create a `.venv` folder that contains the virtualenv with all the necessary packages. Activate it using `poetry shell` or manually +For this, you'll need [poetry](https://python-poetry.org/) installed, as well as `libpq` for the connection to the postgres DB. +For libpq, you can install `libpq-dev` (or equivalent) on linux and [install postgres](https://www.postgresql.org/download/windows/) on Windows (test without it first actually I'm not certain this is needed). -Now, run `docker compose up -d -f docker-compose-deps.yml`. This should spin up the postgres DB -You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show -You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose-deps.yml` +Once Poetry is installed, run `poetry install`. This should create a `.venv` folder that contains the virtualenv with all the necessary packages. Activate it using `poetry shell` or manually. -Almost there! You'll now have to configure Fred. This just means setting the following env vars (found in `fred/__main__.py`) -``` +Now, run `docker compose up -d -f docker-compose-deps.yml`. This should spin up the postgres DB. +You can verify that the database was properly created and manage it by going to where pgadmin should show. +You can use `fred@fred.com` for the user and `fred` for the password. All of this is customizable in `docker-compose-deps.yml`. + +Almost there! You'll now have to configure Fred. This just means setting the following env vars (found in `fred/__main__.py`). + +```sh "FRED_IP", "FRED_PORT", "FRED_TOKEN", @@ -53,9 +63,11 @@ Almost there! You'll now have to configure Fred. This just means setting the fol "FRED_SQL_HOST", "FRED_SQL_PORT", ``` -For convenience, a `.env` file is included and Fred will load them automatically. It includes defaults that will work with the config of the docker-compose-deps.yml. You'll only have to set `FRED_TOKEN` and maybe change the others if the defaults don't suit you -Finally, run `python fred/__main__.py`. Fred should run! You can now adapt this to your setup and run the script from your IDE instead. Don't forget to use the virtualenv python! +For convenience, a `.env` file is included and Fred will load them automatically. It includes defaults that will work with the config of the docker-compose-deps.yml. You'll only have to set `FRED_TOKEN` and maybe change the others if the defaults don't suit you. + +Finally, run `python fred/__main__.py`. Fred should run! You can now adapt this to your setup and run the script from your IDE instead. Don't forget to use the virtualenv python! ## Thanks + Massive thanks to Borketh, Mircea and everyone else that has contributed! From 8b4edc479256e865fd8c639c7a67c77f2d4b7cfa Mon Sep 17 00:00:00 2001 From: Rob B Date: Thu, 1 Aug 2024 19:08:53 -0400 Subject: [PATCH 07/37] Keep shell files as LF so cloning on Windows hosts doesn't break things --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitattributes b/.gitattributes index dfe0770..dada09c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,5 @@ # Auto detect text files and perform LF normalization * text=auto + +# Keep shell files as LF so Windows-host-machine users can still use Docker +*.sh text eol=lf From 81251b2ac66fce409ad4524f2cb87033f50bc694 Mon Sep 17 00:00:00 2001 From: Rob B Date: Thu, 1 Aug 2024 19:09:15 -0400 Subject: [PATCH 08/37] Spellchecker config --- cspell.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 cspell.json diff --git a/cspell.json b/cspell.json new file mode 100644 index 0000000..585b9d5 --- /dev/null +++ b/cspell.json @@ -0,0 +1,19 @@ +// https://cspell.org/configuration/ +{ + // Version of the setting file. Always 0.2 + "version": "0.2", + // language - current active spelling language + "language": "en", + // words - list of words to be always considered correct + "words": [ + "FICSIT", + "libpq", + "Mircea", + "pgadmin", + "virtualenv" + ], + // flagWords - list of words to be always considered incorrect + // This is useful for offensive words and common spelling errors. + // cSpell:disable (don't complain about the words we listed here) + "flagWords": [] +} From e995d668b16d29d1c48b59352e0b4475fe57e728 Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 4 Aug 2024 17:33:55 -0400 Subject: [PATCH 09/37] fix: extra typing so pycharm stops yelling --- fred/fred_commands/_baseclass.py | 11 +++++++++-- fred/fred_commands/experience.py | 9 +++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/fred/fred_commands/_baseclass.py b/fred/fred_commands/_baseclass.py index 10c2c35..bb72cc1 100644 --- a/fred/fred_commands/_baseclass.py +++ b/fred/fred_commands/_baseclass.py @@ -1,7 +1,14 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING from nextcord.ext import commands + from .. import config + +if TYPE_CHECKING: + from ..fred import Bot from ..libraries import common assert config # noqa @@ -11,8 +18,8 @@ class BaseCmds(commands.Cog): logger = logging.Logger("COMMANDS") - def __init__(self, bot): - self.bot = bot + def __init__(self, bot: Bot): + self.bot: Bot = bot @commands.group() @commands.check(common.l4_only) diff --git a/fred/fred_commands/experience.py b/fred/fred_commands/experience.py index 8fa4656..cc02151 100644 --- a/fred/fred_commands/experience.py +++ b/fred/fred_commands/experience.py @@ -1,3 +1,5 @@ +from nextcord import Role, User + from ._baseclass import BaseCmds, commands, common, config from ..cogs import levelling from ..libraries import createembed @@ -19,6 +21,7 @@ async def xp_give(self, ctx: commands.Context, target: commands.UserConverter, a """Usage: `xp give (user) (amount)` Purpose: gives the indicated user the specified xp Notes: don't give negative xp, use take""" + target: User profile = levelling.UserProfile(target.id, ctx.guild, self.bot) if amount < 0: await self.bot.reply_to_msg( @@ -38,6 +41,7 @@ async def xp_take(self, ctx: commands.Context, target: commands.UserConverter, a """Usage: `xp give (user) (amount)` Purpose: takes the specified xp from the indicated user Notes: don't take negative xp, use give""" + target: User profile = levelling.UserProfile(target.id, ctx.guild, self.bot) if amount < 0: await self.bot.reply_to_msg( @@ -60,6 +64,7 @@ async def xp_multiplier(self, ctx: commands.Context, target: commands.UserConver """Usage: `xp multiplier (user) (multiplier)` Purpose: sets the user's personalised xp gain multiplier from the base value Notes: a negative value will be converted to 0""" + target: User user_meta = config.Users.create_if_missing(target) amount = max(multiplier, 0) # no negative gain allowed user_meta.xp_multiplier = amount @@ -74,6 +79,7 @@ async def xp_set(self, ctx: commands.Context, target: commands.UserConverter, am """Usage: `xp set (user) (amount)` Purpose: sets the user's xp amount to the specified amount Notes: don't try negative values, it won't work""" + target: User profile = levelling.UserProfile(target.id, ctx.guild, self.bot) if amount < 0: @@ -153,6 +159,7 @@ async def level(self, ctx: commands.Context, target_user: commands.UserConverter """Usage: `level` [user] Response: Either your level or the level of the user specified Notes: the user parameter can be the user's @ mention or their UID, like 506192269557366805""" + target_user: User if target_user: user_id = target_user.id user = self.bot.get_user(user_id) @@ -169,6 +176,7 @@ async def add_level_role(self, ctx: commands.Context, role: commands.RoleConvert """Usage: `add level_role (role)` Purpose: adds a levelling role Notes: NOT IMPLEMENTED""" + role: Role role_id = role.id if config.DialogflowExceptionRoles.fetch(role_id): @@ -183,6 +191,7 @@ async def remove_level_role(self, ctx: commands.Context, role: commands.RoleConv """Usage: `remove level_role (role)` Purpose: removes a levelling role Notes: NOT IMPLEMENTED""" + role: Role role_id = role.id if config.RankRoles.fetch_by_role(role_id): From 3cdf2f461eaaacc144919c9f314875291361a1a2 Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 4 Aug 2024 17:34:17 -0400 Subject: [PATCH 10/37] fix: missing awaits caught by last commit --- fred/fred_commands/experience.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fred/fred_commands/experience.py b/fred/fred_commands/experience.py index cc02151..676e294 100644 --- a/fred/fred_commands/experience.py +++ b/fred/fred_commands/experience.py @@ -148,7 +148,7 @@ async def leaderboard(self, ctx): query = config.Users.select().orderBy("-xp_count").limit(10) results = list(query) if not results: - self.bot.reply_to_msg(ctx.message, "The database was empty. This should NEVER happen") + await self.bot.reply_to_msg(ctx.message, "The database was empty. This should NEVER happen") return data = [dict(name=user.full_name, count_and_rank=dict(count=user.xp_count, rank=user.rank)) for user in results] embed = createembed.leaderboard(data) @@ -164,7 +164,7 @@ async def level(self, ctx: commands.Context, target_user: commands.UserConverter user_id = target_user.id user = self.bot.get_user(user_id) if not user: - self.bot.reply_to_msg(ctx.message, f"Sorry, I was unable to find the user with id {user_id}") + await self.bot.reply_to_msg(ctx.message, f"Sorry, I was unable to find the user with id {user_id}") return else: user = ctx.author From 935bc2ae605071c308391bcc62dbf21e8d277103 Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 4 Aug 2024 17:35:50 -0400 Subject: [PATCH 11/37] chore: black format (why is this not automatic?) --- fred/cogs/crashes.py | 7 ++++++- fred/cogs/dialogflow.py | 2 +- fred/fred_commands/__init__.py | 3 ++- fred/fred_commands/help.py | 2 +- fred/libraries/createembed.py | 4 +--- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index ea42cb4..a1f10a5 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -307,7 +307,12 @@ async def mass_regex(self, text: str) -> AsyncIterator[dict]: command_response = command["content"] if command_response.startswith(self.bot.command_prefix): # is alias command = config.Commands.fetch(command_response.strip(self.bot.command_prefix)) - yield dict(name=command["name"], value=command["content"], attachment=command["attachment"], inline=True) + yield dict( + name=command["name"], + value=command["content"], + attachment=command["attachment"], + inline=True, + ) else: response = re.sub(r"{(\d+)}", lambda m: match.group(int(m.group(1))), str(crash["response"])) yield dict(name=crash["name"], value=response, inline=True) diff --git a/fred/cogs/dialogflow.py b/fred/cogs/dialogflow.py index 250e2ec..1fa497e 100644 --- a/fred/cogs/dialogflow.py +++ b/fred/cogs/dialogflow.py @@ -13,7 +13,7 @@ from ..libraries import common -if os.environ.get('DIALOGFLOW_AUTH'): +if os.environ.get("DIALOGFLOW_AUTH"): DIALOGFLOW_AUTH = json.loads(os.environ.get("DIALOGFLOW_AUTH")) session_client = dialogflow.SessionsClient( credentials=service_account.Credentials.from_service_account_info(DIALOGFLOW_AUTH) diff --git a/fred/fred_commands/__init__.py b/fred/fred_commands/__init__.py index af58da1..c41ca89 100644 --- a/fred/fred_commands/__init__.py +++ b/fred/fred_commands/__init__.py @@ -124,6 +124,7 @@ async def mod(self, ctx: commands.Context, *, mod_name: str) -> None: view = None msg = await self.bot.reply_to_msg(ctx.message, embed=embed, view=view, file=attachment) if view: + async def callback(interaction: nextcord.Interaction): logging.info(interaction.data.values) new_embed, new_attachment, _ = await createembed.mod_embed(interaction.data["values"][0], self.bot) @@ -140,7 +141,7 @@ async def timeout(): await view.wait() - @commands.command(aliases=['docssearch']) + @commands.command(aliases=["docssearch"]) async def docsearch(self, ctx: commands.Context, *, search: str) -> None: """Usage: `docsearch (search: str)` Response: Equivalent to using the search function on the SMR docs page; links the first search result""" diff --git a/fred/fred_commands/help.py b/fred/fred_commands/help.py index f3d390b..5b8331b 100644 --- a/fred/fred_commands/help.py +++ b/fred/fred_commands/help.py @@ -24,7 +24,7 @@ async def help(self, ctx: commands.Context) -> None: async def _send_help(self, ctx: commands.Context, **kwargs): if not ctx.author.dm_channel: await ctx.author.create_dm() - if not await self.bot.checked_DM(ctx.author, in_dm=ctx.channel == ctx.author.dm_channel,**kwargs): + if not await self.bot.checked_DM(ctx.author, in_dm=ctx.channel == ctx.author.dm_channel, **kwargs): await ctx.reply( "Help commands only work in DMs to avoid clutter. " "You have either disabled server DMs or indicated that you do not wish for Fred to DM you. " diff --git a/fred/libraries/createembed.py b/fred/libraries/createembed.py index 761e60f..e0468c2 100644 --- a/fred/libraries/createembed.py +++ b/fred/libraries/createembed.py @@ -57,9 +57,7 @@ def leaderboard(data: list[dict]) -> nextcord.Embed: embed = nextcord.Embed(title="XP Leaderboard", colour=config.ActionColours.fetch("purple"), description=desc) for user in data: - embed.add_field( - name=user["name"], value=f'XP: {user["count_and_rank"]["count"]} | Level: {user["count_and_rank"]["rank"]}' - ) + embed.add_field(name=user["name"], value=f'XP: {user["xp"]} | Level: {user["rank"]}') return embed From 7bd9d2e92fe8691cd01f253fdebbabac80f6b1a0 Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 4 Aug 2024 17:36:34 -0400 Subject: [PATCH 12/37] fix: error handling for not finding main guild --- fred/libraries/common.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fred/libraries/common.py b/fred/libraries/common.py index dff7050..611e692 100644 --- a/fred/libraries/common.py +++ b/fred/libraries/common.py @@ -28,9 +28,15 @@ async def mod_only(ctx: Context) -> bool: def permission_check(ctx: Context, level: int) -> bool: logpayload = user_info(ctx.author) logpayload["level"] = level + logger.info("Checking permissions for someone", extra=logpayload) perms = config.PermissionRoles.fetch_by_lvl(level) - main_guild = ctx.bot.get_guild(config.Misc.fetch("main_guild_id")) + + main_guild_id = int(config.Misc.fetch("main_guild_id")) + main_guild = ctx.bot.get_guild(main_guild_id) + if main_guild is None: + raise LookupError(f"Unable to retrieve the guild {main_guild_id}. Is this the guild you meant?") + if (main_guild_member := main_guild.get_member(ctx.author.id)) is None: logger.warning("Checked permissions for someone but they weren't in the main guild", extra=logpayload) return False From f25dbc47a983e3f37b813987aa335c0767a956ea Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 4 Aug 2024 17:40:45 -0400 Subject: [PATCH 13/37] fix: move error handling channel from env to db --- fred/fred.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fred/fred.py b/fred/fred.py index 463b622..f23bcdd 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -44,7 +44,7 @@ def __init__(self, *args, **kwargs): self.owo = False self.loop = asyncio.new_event_loop() - self._error_channel = int(env_val) if (env_val := os.getenv("ERROR_CHANNEL")) else 748229790825185311 + self._error_channel = int(chan) if (chan := config.Misc.fetch("error_channel")) else 748229790825185311 async def start(self, *args, **kwargs): async with aiohttp.ClientSession() as session: From 77860fe5bb6dc3f4cfbc4de3921c6ed6daaa0338 Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 4 Aug 2024 17:45:53 -0400 Subject: [PATCH 14/37] fix: prevent errors from sending too-small queries to SMR search --- fred/fred_commands/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fred/fred_commands/__init__.py b/fred/fred_commands/__init__.py index c41ca89..904d36b 100644 --- a/fred/fred_commands/__init__.py +++ b/fred/fred_commands/__init__.py @@ -114,6 +114,10 @@ async def mod(self, ctx: commands.Context, *, mod_name: str) -> None: Response: If a near-exact match is found, gives you info about that mod. If close matches are found, up to 10 of those will be listed. If nothing even comes close, I'll let you know ;)""" + if len(mod_name) < 3: + await self.bot.reply_to_msg(ctx.message, "Searching needs at least three characters!") + return + embed, attachment, multiple_mods = await createembed.mod_embed(mod_name, self.bot) if embed is None: await self.bot.reply_to_msg(ctx.message, "No mods found!") From 3d2087f3337bae4ffee906283aabac5a55aabc63 Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 4 Aug 2024 23:48:18 -0400 Subject: [PATCH 15/37] build: update dependencies nextcord's speed submodule (orjson, aiodns etc) has been added jsonpickle is only used one place in code, commented out for potential debug use, so it can be for dev. --- fred/cogs/crashes.py | 2 +- pyproject.toml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index a1f10a5..d152e42 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -280,7 +280,7 @@ async def process_file(self, file, extension) -> list[dict](): image = Image.open(file) ratio = 2160 / image.height if ratio > 1: - image = image.resize((round(image.width * ratio), round(image.height * ratio)), Image.LANCZOS) + image = image.resize((round(image.width * ratio), round(image.height * ratio)), Image.Resampling.LANCZOS) enhancer_contrast = ImageEnhance.Contrast(image) diff --git a/pyproject.toml b/pyproject.toml index 4009d46..e1a0388 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,20 +7,20 @@ license = "MIT License" [tool.poetry.dependencies] python = "^3.11" -nextcord = "^2.6" -Pillow = "^9.0.0" +nextcord = {extras = ["speed"], version = "^2.6.0"} +Pillow = "^10.4.0" pytesseract = "^0.3.8" psycopg2 = "^2.9.3" -python-dotenv = "^0.19.2" -jsonpickle = "^2.1.0" -algoliasearch = "^2.6.1" +python-dotenv = "^1.0.0" +algoliasearch = "^3.0.0" SQLObject = "^3.9.1" google-cloud-dialogflow = "^2.11.0" -regex = "^2022.3.15" -aiohttp = "3.9.3" +regex = "^2024.7.0" +aiohttp = "^3.10.0" [tool.poetry.dev-dependencies] -black = "^22.1.0" +black = "^24.8.0" +jsonpickle = "^3.0.0" [tool.black] line-length = 120 From f83b242504d88fb665992ff16006a86a02b0364e Mon Sep 17 00:00:00 2001 From: borketh Date: Mon, 5 Aug 2024 13:50:21 -0400 Subject: [PATCH 16/37] build: add black daemon and use new poetry dev profile thing --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e1a0388..7ecbb72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,8 +18,8 @@ google-cloud-dialogflow = "^2.11.0" regex = "^2024.7.0" aiohttp = "^3.10.0" -[tool.poetry.dev-dependencies] -black = "^24.8.0" +[tool.poetry.group.dev.dependencies] +black = {extras = ["d"], version = "^24.8.0"} jsonpickle = "^3.0.0" [tool.black] From 6d60067c89e743a89033c017cab7f2e94dae8ed9 Mon Sep 17 00:00:00 2001 From: Feyko Date: Mon, 5 Aug 2024 22:31:22 +0200 Subject: [PATCH 17/37] feat: basic migration setup --- fred/config.py | 32 ++++++++++++++++++++ fred/fred.py | 1 + fred/migrations/0-example_migration.down.sql | 1 + fred/migrations/0-example_migration.up.sql | 1 + pyproject.toml | 1 + 5 files changed, 36 insertions(+) create mode 100644 fred/migrations/0-example_migration.down.sql create mode 100644 fred/migrations/0-example_migration.up.sql diff --git a/fred/config.py b/fred/config.py index 08ab441..4e41df8 100644 --- a/fred/config.py +++ b/fred/config.py @@ -1,3 +1,6 @@ +import os +import pathlib + from sqlobject import * import json @@ -267,3 +270,32 @@ def change(key, value): results = list(query) if results: results[0].value = value + + @staticmethod + def create_or_change(key, value): + query = Misc.selectBy(key=key) + results = list(query) + if results: + results[0].value = value + return + + Misc(key=key, value=value) + + +def migrate(): + current_migration_rev = Misc.fetch("migration_rev") + if current_migration_rev is None: + current_migration_rev = 0 + + migrations_dir = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) + migrations_filenames = list(migrations_dir.glob('migrations/*-*.up.sql')) + valid_migrations = [migration for migration in migrations_filenames if _migration_rev(migration) > current_migration_rev] + + for migration in valid_migrations: + sqlhub.processConnection.query(migration.read_text()) + + Misc.create_or_change("migration_rev", _migration_rev(migrations_filenames[0])) + + +def _migration_rev(filepath: pathlib.Path) -> int: + return int(filepath.name.split('-')[0]) \ No newline at end of file diff --git a/fred/fred.py b/fred/fred.py index 4a41000..73dfe79 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -87,6 +87,7 @@ def setup_DB(self): time.sleep(5) else: # this happens if the loop is not broken by a successful connection raise ConnectionError("Could not connect to the DB") + config.migrate() def setup_logger(self): logging.root = logging.Logger("FRED") diff --git a/fred/migrations/0-example_migration.down.sql b/fred/migrations/0-example_migration.down.sql new file mode 100644 index 0000000..253b4ea --- /dev/null +++ b/fred/migrations/0-example_migration.down.sql @@ -0,0 +1 @@ +-- SQL code for when we want to undo a migration goes here \ No newline at end of file diff --git a/fred/migrations/0-example_migration.up.sql b/fred/migrations/0-example_migration.up.sql new file mode 100644 index 0000000..4963365 --- /dev/null +++ b/fred/migrations/0-example_migration.up.sql @@ -0,0 +1 @@ +-- SQL code for when we want to do a migration goes here \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b1fe26c..e0f7371 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ SQLObject = "^3.9.1" google-cloud-dialogflow = "^2.11.0" regex = "^2022.3.15" aiohttp = "3.9.3" +setuptools = "^72.1.0" [tool.poetry.dev-dependencies] black = "^22.1.0" From 4fa4f7781e72dbe776eee98f2b6e7ce036d8e395 Mon Sep 17 00:00:00 2001 From: borketh Date: Mon, 5 Aug 2024 18:11:52 -0400 Subject: [PATCH 18/37] fix: cleanup and fixes for minor issues and borked code --- fred/__init__.py | 22 ++++++++++++++++++++++ fred/__main__.py | 22 +--------------------- fred/cogs/crashes.py | 4 +++- fred/cogs/welcome.py | 3 ++- fred/fred.py | 2 +- fred/fred_commands/_command_utils.py | 1 - fred/fred_commands/dbcommands.py | 2 +- fred/libraries/createembed.py | 2 +- fred/libraries/view/mod_picker.py | 2 -- 9 files changed, 31 insertions(+), 29 deletions(-) diff --git a/fred/__init__.py b/fred/__init__.py index 37cf71c..4d00c2b 100644 --- a/fred/__init__.py +++ b/fred/__init__.py @@ -1 +1,23 @@ +from os import getenv + +from dotenv import load_dotenv + from .fred import __version__ + +ENVVARS = ( + "FRED_IP", + "FRED_PORT", + "FRED_TOKEN", + "FRED_SQL_DB", + "FRED_SQL_USER", + "FRED_SQL_PASSWORD", + "FRED_SQL_HOST", + "FRED_SQL_PORT", + # "DIALOGFLOW_AUTH", +) + +load_dotenv() + +for var in ENVVARS: + if getenv(var) is None: + raise EnvironmentError(f"The ENV variable '{var}' isn't set") diff --git a/fred/__main__.py b/fred/__main__.py index 1e049a6..ba81ec1 100644 --- a/fred/__main__.py +++ b/fred/__main__.py @@ -1,26 +1,6 @@ -from dotenv import load_dotenv from os import getenv -load_dotenv() - -ENVVARS = [ - "FRED_IP", - "FRED_PORT", - "FRED_TOKEN", - "FRED_SQL_DB", - "FRED_SQL_USER", - "FRED_SQL_PASSWORD", - "FRED_SQL_HOST", - "FRED_SQL_PORT", - # "DIALOGFLOW_AUTH", -] - -for var in ENVVARS: - if getenv(var) is None: - raise EnvironmentError(f"The ENV variable '{var}' isn't set") - -from .fred import Bot, nextcord, __version__ - +from .fred import Bot, nextcord intents = nextcord.Intents.all() diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index d152e42..c0730b3 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -280,7 +280,9 @@ async def process_file(self, file, extension) -> list[dict](): image = Image.open(file) ratio = 2160 / image.height if ratio > 1: - image = image.resize((round(image.width * ratio), round(image.height * ratio)), Image.Resampling.LANCZOS) + image = image.resize( + (round(image.width * ratio), round(image.height * ratio)), Image.Resampling.LANCZOS + ) enhancer_contrast = ImageEnhance.Contrast(image) diff --git a/fred/cogs/welcome.py b/fred/cogs/welcome.py index 22b2181..2d644db 100644 --- a/fred/cogs/welcome.py +++ b/fred/cogs/welcome.py @@ -1,3 +1,4 @@ +from nextcord import Member from nextcord.ext import commands from .. import config @@ -9,7 +10,7 @@ def __init__(self, bot): self.bot = bot @commands.Cog.listener() - async def on_member_join(self, member): + async def on_member_join(self, member: Member): self.bot.logger.info("Processing a member joining", extra=common.user_info(member)) if welcome := config.Misc.fetch("welcome_message"): self.bot.logger.info("Sending the welcome message to a new member", extra=common.user_info(member)) diff --git a/fred/fred.py b/fred/fred.py index f23bcdd..fd9c8cb 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -121,7 +121,7 @@ async def on_error(self, event, *args, **kwargs): channel_str = "" fred_str = f"Fred v{self.version}" - error_meta = f"{type.__name__} exception handled in {event}{channel_str}" + error_meta = f"{type.__name__} exception handled in `{event}` {channel_str}" full_error = f"\n{value}\n\n{''.join(traceback.format_tb(tb))}" logging.error(f"{fred_str}\n{error_meta}\n{full_error}") diff --git a/fred/fred_commands/_command_utils.py b/fred/fred_commands/_command_utils.py index 17827ea..2eca0a5 100644 --- a/fred/fred_commands/_command_utils.py +++ b/fred/fred_commands/_command_utils.py @@ -1,5 +1,4 @@ from typing import Type -import logging from regex import E, match, escape diff --git a/fred/fred_commands/dbcommands.py b/fred/fred_commands/dbcommands.py index c8b4843..dda1812 100644 --- a/fred/fred_commands/dbcommands.py +++ b/fred/fred_commands/dbcommands.py @@ -216,7 +216,7 @@ async def rename_command(self, ctx: commands.Context, name: str.lower, *, new_na results: list[config.Commands] if not (results := list(config.Commands.selectBy(name=name))): # this command hasn't been created yet - await self.bot.reply_to_msg("Command could not be found!") + await self.bot.reply_to_msg(ctx.message, "Command could not be found!") return if new_name is None: diff --git a/fred/libraries/createembed.py b/fred/libraries/createembed.py index e0468c2..9e7467e 100644 --- a/fred/libraries/createembed.py +++ b/fred/libraries/createembed.py @@ -80,7 +80,7 @@ def format_commit(commit: dict) -> tuple[str, str]: change_summary_icons = " ".join( [f"{em} {len(commit[k])}" for em, k in zip("✅❌📝", ["added", "removed", "modified"])] ) - return (f"{commit_message}\n", f'{change_summary_icons} - by {attribution} {ts} [{hash_id}]({commit["url"]})\n') + return f"{commit_message}\n", f'{change_summary_icons} - by {attribution} {ts} [{hash_id}]({commit["url"]})\n' def push(data: dict) -> nextcord.Embed: diff --git a/fred/libraries/view/mod_picker.py b/fred/libraries/view/mod_picker.py index fbba682..4c67128 100644 --- a/fred/libraries/view/mod_picker.py +++ b/fred/libraries/view/mod_picker.py @@ -1,5 +1,3 @@ -import logging - import nextcord.ui From 374a3ad7c151a6cc9f81f4702c0fa6a1f7ebf36f Mon Sep 17 00:00:00 2001 From: Feyko Date: Mon, 5 Aug 2024 16:31:22 -0400 Subject: [PATCH 19/37] feat: basic migration setup borketh really borked it this time, hopefully this fixes that --- fred/config.py | 33 ++++++++++++++++++++ fred/fred.py | 1 + fred/migrations/0-example_migration.down.sql | 1 + fred/migrations/0-example_migration.up.sql | 1 + 4 files changed, 36 insertions(+) create mode 100644 fred/migrations/0-example_migration.down.sql create mode 100644 fred/migrations/0-example_migration.up.sql diff --git a/fred/config.py b/fred/config.py index d3437aa..7dde6fa 100644 --- a/fred/config.py +++ b/fred/config.py @@ -2,6 +2,8 @@ from numbers import Number from typing import Optional, Any +import os +import pathlib import nextcord from sqlobject import * @@ -255,3 +257,34 @@ def change(key: str, value: JSONValue): query = Misc.selectBy(key=key).getOne(None) if query is not None: query.value = value + + @staticmethod + def create_or_change(key, value): + query = Misc.selectBy(key=key) + results = list(query) + if results: + results[0].value = value + return + + Misc(key=key, value=value) + + +def migrate(): + current_migration_rev = Misc.fetch("migration_rev") + if current_migration_rev is None: + current_migration_rev = 0 + + migrations_dir = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) + migrations_filenames = list(migrations_dir.glob("migrations/*-*.up.sql")) + valid_migrations = [ + migration for migration in migrations_filenames if _migration_rev(migration) > current_migration_rev + ] + + for migration in valid_migrations: + sqlhub.processConnection.query(migration.read_text()) + + Misc.create_or_change("migration_rev", _migration_rev(migrations_filenames[0])) + + +def _migration_rev(filepath: pathlib.Path) -> int: + return int(filepath.name.split("-")[0]) diff --git a/fred/fred.py b/fred/fred.py index fd9c8cb..9abd365 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -85,6 +85,7 @@ def setup_DB(self): time.sleep(5) else: # this happens if the loop is not broken by a successful connection raise ConnectionError("Could not connect to the DB") + config.migrate() def setup_logger(self): logging.root = logging.Logger("FRED") diff --git a/fred/migrations/0-example_migration.down.sql b/fred/migrations/0-example_migration.down.sql new file mode 100644 index 0000000..253b4ea --- /dev/null +++ b/fred/migrations/0-example_migration.down.sql @@ -0,0 +1 @@ +-- SQL code for when we want to undo a migration goes here \ No newline at end of file diff --git a/fred/migrations/0-example_migration.up.sql b/fred/migrations/0-example_migration.up.sql new file mode 100644 index 0000000..4963365 --- /dev/null +++ b/fred/migrations/0-example_migration.up.sql @@ -0,0 +1 @@ +-- SQL code for when we want to do a migration goes here \ No newline at end of file From 2ff6e7985f6afc9eac38d71b0f11141f7311922a Mon Sep 17 00:00:00 2001 From: borketh Date: Mon, 5 Aug 2024 18:52:22 -0400 Subject: [PATCH 20/37] chore: Update to 2.21 - migration for removing the users.full_name column as per 85b34cf74b37f424750da5f2dc2530160ef8e324 - update to python 3.12 - use the new type keyword in a couple places for funsies --- docker/Dockerfile | 4 ++-- fred/config.py | 18 ++++++++---------- fred/fred.py | 2 +- .../migrations/1-drop_full_name_from_users.sql | 2 ++ pyproject.toml | 9 +++++---- testing/embeds.py | 2 +- 6 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 fred/migrations/1-drop_full_name_from_users.sql diff --git a/docker/Dockerfile b/docker/Dockerfile index 132521d..afe0dac 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim AS runtime +FROM python:3.12-slim AS runtime ENV DEBIAN_FRONTEND=noninteractive VOLUME /config @@ -7,7 +7,7 @@ WORKDIR /app COPY docker/runtime-deps.sh . RUN bash runtime-deps.sh 1> /dev/null && rm runtime-deps.sh -FROM python:3.11-slim AS build +FROM python:3.12-slim AS build WORKDIR /app diff --git a/fred/config.py b/fred/config.py index 7dde6fa..23e70bb 100644 --- a/fred/config.py +++ b/fred/config.py @@ -157,7 +157,7 @@ def fetch(intent_id: str, data: dict) -> Optional[Dialogflow]: return Dialogflow.selectBy(intent_id=intent_id, data=data).getOne(None) -CommandsOrCrashesDict = dict[str, str | StringCol] +type CommandsOrCrashesDict = dict[str, str | StringCol] class Commands(SQLObject): @@ -237,7 +237,7 @@ def check(name: str) -> bool: return bool(query) -JSONValue = Number | bool | str | list | dict +type JSONValue = Number | bool | str | list | dict class Misc(SQLObject): @@ -259,14 +259,12 @@ def change(key: str, value: JSONValue): query.value = value @staticmethod - def create_or_change(key, value): - query = Misc.selectBy(key=key) - results = list(query) - if results: - results[0].value = value - return - - Misc(key=key, value=value) + def create_or_change(key: str, value: JSONValue): + query: Optional[Misc] = Misc.selectBy(key=key).getOne(None) + if query is None: + Misc(key=key, value=value) + else: + query.value = value def migrate(): diff --git a/fred/fred.py b/fred/fred.py index 9abd365..48aa76d 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -15,7 +15,7 @@ from .fred_commands import Commands, FredHelpEmbed from .libraries import createembed, common -__version__ = "2.20.4" +__version__ = "2.21.0" class Bot(commands.Bot): diff --git a/fred/migrations/1-drop_full_name_from_users.sql b/fred/migrations/1-drop_full_name_from_users.sql new file mode 100644 index 0000000..905ff6c --- /dev/null +++ b/fred/migrations/1-drop_full_name_from_users.sql @@ -0,0 +1,2 @@ +ALTER TABLE users +DROP COLUMN IF EXISTS full_name; \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 7ecbb72..224f9b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [tool.poetry] name = "fred" -version = "2.20.4" +version = "2.21.0" description = "A Discord bot for the Satisfactory Modding Discord " -authors = ["Feyko ", "Mircea Roata ", "Astro Floof "] +authors = ["Feyko ", "Mircea Roata ", "Borketh "] license = "MIT License" [tool.poetry.dependencies] -python = "^3.11" +python = "^3.12" nextcord = {extras = ["speed"], version = "^2.6.0"} Pillow = "^10.4.0" pytesseract = "^0.3.8" @@ -17,6 +17,7 @@ SQLObject = "^3.9.1" google-cloud-dialogflow = "^2.11.0" regex = "^2024.7.0" aiohttp = "^3.10.0" +setuptools = "^72.1.0" [tool.poetry.group.dev.dependencies] black = {extras = ["d"], version = "^24.8.0"} @@ -24,7 +25,7 @@ jsonpickle = "^3.0.0" [tool.black] line-length = 120 -target-version = ['py311'] +target-version = ['py312'] [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/testing/embeds.py b/testing/embeds.py index c57dd00..f0dafb5 100644 --- a/testing/embeds.py +++ b/testing/embeds.py @@ -6,7 +6,7 @@ import sys sys.path.insert(0, abspath("./bot")) -from libraries.createembed import run +from fred.libraries.createembed import run from asyncio import run as nonawait from json import load from os import getenv From f7eb7f8358d87c45315e27d10d2ba6dfb720ec69 Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 11 Aug 2024 01:13:50 -0400 Subject: [PATCH 21/37] docs: add example.env --- .gitignore | 1 + example.env | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 example.env diff --git a/.gitignore b/.gitignore index 7ec2647..b9f63b7 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,7 @@ celerybeat.pid # Environments .env +!example.env .venv .run/ env/ diff --git a/example.env b/example.env new file mode 100644 index 0000000..e271d82 --- /dev/null +++ b/example.env @@ -0,0 +1,8 @@ +FRED_IP=:: +FRED_PORT=80 +FRED_SQL_DB=fred +FRED_SQL_USER=fred +FRED_SQL_PASSWORD=fred +FRED_SQL_HOST=0.0.0.0 +FRED_SQL_PORT=5432 +FRED_LOG_LEVEL=DEBUG \ No newline at end of file From 769ecd1b51ef8f2c3e7ec5231ade3644d20b1f0b Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 11 Aug 2024 01:47:29 -0400 Subject: [PATCH 22/37] feat(logging): improve logging and make it work --- fred/__init__.py | 8 ++++- fred/cogs/crashes.py | 8 ++--- fred/cogs/dialogflow.py | 15 ++++---- fred/cogs/levelling.py | 60 ++++++++++++++++++-------------- fred/cogs/mediaonly.py | 10 ++---- fred/cogs/webhooklistener.py | 34 ++++++++++-------- fred/cogs/welcome.py | 17 +++++---- fred/fred.py | 33 +++++++----------- fred/fred_commands/_baseclass.py | 15 +------- fred/fred_commands/help.py | 11 +++--- fred/libraries/common.py | 28 ++++++++++++++- fred/libraries/createembed.py | 9 ++--- 12 files changed, 129 insertions(+), 119 deletions(-) diff --git a/fred/__init__.py b/fred/__init__.py index 4d00c2b..8278c5d 100644 --- a/fred/__init__.py +++ b/fred/__init__.py @@ -1,8 +1,14 @@ +import logging from os import getenv from dotenv import load_dotenv -from .fred import __version__ +logging.root = logging.getLogger("FRED") +logging.basicConfig(level=getenv("FRED_LOG_LEVEL", logging.DEBUG)) + +from .fred import __version__ # noqa + +logging.root.debug(f"HELLO WORLD!!! FRED version: {__version__}") ENVVARS = ( "FRED_IP", diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index c0730b3..04fb449 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -10,12 +10,12 @@ from typing import AsyncIterator import nextcord -import nextcord.ext.commands as commands from PIL import Image, ImageEnhance, UnidentifiedImageError from pytesseract import image_to_string, TesseractError from .. import config from ..libraries import createembed +from ..libraries.common import FredCog REGEX_LIMIT: float = 2 @@ -32,7 +32,7 @@ async def regex_with_timeout(*args, **kwargs): ) -class Crashes(commands.Cog): +class Crashes(FredCog): vanilla_info_patterns = [ re.compile(r"Net CL: (?P\d+)"), re.compile(r"Command Line: (?P.*)"), @@ -40,10 +40,6 @@ class Crashes(commands.Cog): re.compile(r"Launcher ID: (?P\w+)"), ] - def __init__(self, bot): - self.bot = bot - self.logger = logging.Logger("CRASH_PARSING") - @staticmethod def filter_epic_commandline(cli: str) -> str: return " ".join(filter(lambda opt: "auth" not in opt.lower(), cli.split())) diff --git a/fred/cogs/dialogflow.py b/fred/cogs/dialogflow.py index 1fa497e..37d44cc 100644 --- a/fred/cogs/dialogflow.py +++ b/fred/cogs/dialogflow.py @@ -1,11 +1,9 @@ import asyncio import json import logging -import os import uuid import nextcord -from nextcord.ext import commands from google.cloud import dialogflow from google.oauth2 import service_account @@ -22,11 +20,10 @@ SESSION_LIFETIME = 10 * 60 # 10 minutes to avoid repeated false positives -class DialogFlow(commands.Cog): - def __init__(self, bot): - self.bot = bot +class DialogFlow(common.FredCog): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.session_ids = {} - self.logger = logging.Logger("DIALOGFLOW") async def process_message(self, message): self.bot.logger.info("Processing NLP") @@ -39,8 +36,10 @@ async def process_message(self, message): # We're in a DM channel self.logger.info("Ignoring a message because it is in a DM channel", extra=common.message_info(message)) return - if not config.Misc.fetch("dialogflow_state"): self.logger.info("Ignoring a message because NLP is disabled", extra=common.message_info(message)) + self.logger.info( + "Ignoring a message because NLP is disabled or not configured", extra=common.message_info(message) + ) return if not config.Misc.fetch("dialogflow_debug_state"): self.logger.info("Checking someone's permissions", extra=common.user_info(message.author)) @@ -53,7 +52,7 @@ async def process_message(self, message): for role in roles: if role.id not in exception_roles: self.logger.info( - "Ignoring someone's message because they are authorised", extra=common.user_info(message.author) + "Ignoring someone's message because they are exempt", extra=common.user_info(message.author) ) return diff --git a/fred/cogs/levelling.py b/fred/cogs/levelling.py index d1ebde8..57d6ea6 100644 --- a/fred/cogs/levelling.py +++ b/fred/cogs/levelling.py @@ -5,11 +5,13 @@ from datetime import * import nextcord.ext.commands as commands -from nextcord import DMChannel +from nextcord.ext.commands import MemberNotFound from .. import config from ..libraries import common +logger = common.new_logger(__name__) + class UserProfile: def __init__(self, user_id, guild, bot): @@ -18,10 +20,12 @@ def __init__(self, user_id, guild, bot): self.user_id = user_id self.logger = logging.Logger("USERPROFILE") - assert bot.intents.members, "The bot has no member intents, so it cannot do levelling!!" - self.member = bot.get_user(user_id) - assert self.member is not None, "This member does not exist o.0" - self.logger.info(f"Found member id {self.member}") + self.member = guild.get_member(user_id) + if self.member is None: + logger.warning(f"Unable to retrieve information about user {user_id}") + raise MemberNotFound(f"Unable to retrieve information about user {user_id}") + + logger.info(f"Found member id {self.member}") if DB_user := config.Users.fetch(user_id): self.DB_user = DB_user @@ -33,7 +37,7 @@ def __init__(self, user_id, guild, bot): async def validate_role(self): if not self.member: - self.logger.info( + logger.info( "Could not validate someone's level role because they aren't in the main guild", extra={"user_id": self.user_id}, ) @@ -43,25 +47,26 @@ async def validate_role(self): role = self.guild.get_role(role_id) if not role: logpayload["role_id"] = role_id - self.logger.warning( + logger.warning( "Could not validate someone's level role because the role isn't in the main guild", extra=logpayload ) return self.DB_user.rank_role_id = role_id - # self.rank_role_id = role_id - if not self.member.permissions_in(self.bot.modchannel).send_messages: + + # if not self.guild.get_channel(config.Misc.fetch("mod_channel")).permissions_for(self.member).send_messages: + if not common.permission_check(self.member, level=6): for member_role in self.member.roles: if rank := config.RankRoles.fetch_by_role(member_role.id): logpayload["role_id"] = member_role.id - self.logger.info("Removing a mismatched level role from someone", extra=logpayload) + logger.info("Removing a mismatched level role from someone", extra=logpayload) await self.member.remove_roles(member_role) logpayload["role_id"] = role.id - self.logger.info("Removing a mismatched level role from someone", logpayload) + logger.info("Removing a mismatched level role from someone", logpayload) await self.member.add_roles(role) async def validate_level(self): if not self.member: - self.logger.info( + logger.info( "Could not validate someone's level because they aren't in the main guild", extra={"user_id": self.user_id}, ) @@ -78,7 +83,7 @@ async def validate_level(self): logpayload["expected_level"] = expected_level logpayload["current_level"] = self.rank if expected_level != self.rank: - self.logger.info("Correcting a mismatched level", extra=logpayload) + logger.info("Correcting a mismatched level", extra=logpayload) self.DB_user.rank = expected_level if self.DB_user.accepts_dms: if expected_level > self.rank: @@ -97,7 +102,7 @@ async def increment_xp(self): xp_gain = config.Misc.fetch("xp_gain_value") * self.DB_user.xp_multiplier * self.DB_user.role_xp_multiplier logpayload = common.user_info(self.member) logpayload["xp_increment"] = xp_gain - self.logger.info("Incrementing someone's xp", logpayload) + logger.info("Incrementing someone's xp", logpayload) await self.give_xp(xp_gain) async def give_xp(self, xp): @@ -105,7 +110,7 @@ async def give_xp(self, xp): return logpayload = common.user_info(self.member) logpayload["xp_gain"] = xp - self.logger.info("Giving someone xp", logpayload) + logger.info("Giving someone xp", logpayload) self.DB_user.xp_count += xp self.xp_count += xp await self.validate_level() @@ -117,7 +122,7 @@ async def take_xp(self, xp): logpayload = common.user_info(self.member) logpayload["xp_loss"] = xp - self.logger.info("Taking xp from someone", logpayload) + logger.info("Taking xp from someone", logpayload) self.DB_user.xp_count -= xp self.xp_count -= xp await self.validate_level() @@ -126,27 +131,28 @@ async def take_xp(self, xp): async def set_xp(self, xp): logpayload = common.user_info(self.member) logpayload["new_xp"] = xp - self.logger.info("Setting someone's xp", logpayload) + logger.info("Setting someone's xp", logpayload) self.DB_user.xp_count = xp self.xp_count = xp await self.validate_level() return True -class Levelling(commands.Cog): - def __init__(self, bot): - self.bot = bot - self.bot.xp_timers = {} - self.logger = logging.Logger("LEVELLING") +class Levelling(common.FredCog): + xp_timers = {} + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.xp_timers = {} # TODO make xp roles - # @commands.Cog.listener() + # @commonn.FredCog.listener() # async def on_member_update(self, before, after): # if before.roles != after.roles: # config.XpRoles @commands.Cog.listener() - async def on_message(self, message): + @common.FredCog.listener() self.logger.info("Levelling: Processing message", extra=common.message_info(message)) if ( message.author.bot @@ -158,12 +164,12 @@ async def on_message(self, message): profile = UserProfile(message.author.id, message.guild, self.bot) profile.DB_user.message_count += 1 - if profile.user_id in self.bot.xp_timers: - if datetime.now() >= self.bot.xp_timers[profile.user_id]: + if profile.user_id in self.xp_timers: + if datetime.now() >= self.xp_timers[profile.user_id]: await profile.increment_xp() else: self.logger.info( "Levelling: Someone sent a message too fast and will not be awarded xp", extra=common.message_info(message), ) - self.bot.xp_timers[profile.user_id] = datetime.now() + timedelta(seconds=config.Misc.fetch("xp_gain_delay")) + self.xp_timers[profile.user_id] = datetime.now() + timedelta(seconds=config.Misc.fetch("xp_gain_delay")) diff --git a/fred/cogs/mediaonly.py b/fred/cogs/mediaonly.py index d7657b9..ee603ee 100644 --- a/fred/cogs/mediaonly.py +++ b/fred/cogs/mediaonly.py @@ -1,15 +1,9 @@ -import logging - -import nextcord.ext.commands as commands - from .. import config from ..libraries import common +from ..libraries.common import FredCog -class MediaOnly(commands.Cog): - def __init__(self, bot): - self.bot = bot - self.logger = logging.Logger("MEDIA_ONLY") +class MediaOnly(FredCog): async def process_message(self, message): self.logger.info("Processing a message", extra=common.message_info(message)) diff --git a/fred/cogs/webhooklistener.py b/fred/cogs/webhooklistener.py index a380e1a..56076bd 100644 --- a/fred/cogs/webhooklistener.py +++ b/fred/cogs/webhooklistener.py @@ -1,27 +1,31 @@ import asyncio import json -import logging -import os import socket import sys import threading import traceback from http.server import BaseHTTPRequestHandler, HTTPServer -import nextcord.ext.commands as commands +from fred.libraries import common -logger = logging.Logger("GITHOOK") +logger = common.new_logger(__name__) def runServer(self, bot): logger.info("Running the webserver for the Githook") - server = HTTPServerV6((os.environ.get("FRED_IP"), int(os.environ.get("FRED_PORT"))), MakeGithookHandler(bot)) - server.serve_forever() + ip = getenv("FRED_IP") + port = int(getenv("FRED_PORT")) + try: + server = HTTPServerV6((ip, port), MakeGithookHandler(bot)) + server.serve_forever() + except PermissionError as pe: + logger.error(f"Cannot handle githooks! Permission denied to listen to {ip=} {port=}.") + logger.exception(pe) -class Githook(commands.Cog): - def __init__(self, bot): - self.bot = bot +class Githook(common.FredCog): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) # Run GitHub webhook handling server try: @@ -32,7 +36,7 @@ def __init__(self, bot): except Exception: type, value, tb = sys.exc_info() tbs = "".join(traceback.format_tb(tb)) - logger.error(f"Failed to run the webserver:\n{tbs}") + self.logger.error(f"Failed to run the webserver:\n{tbs}") # handle POST events from GitHub server @@ -55,17 +59,17 @@ def handle_check(self): case "/ready": self.respond(200 if bot.isReady else 503) case "/healthy": - logging.info("handling /healthy") + logger.info("handling /healthy") fut = asyncio.run_coroutine_threadsafe(bot.isAlive(), bot.loop) - logging.info("waiting for result from healthcheck") + logger.info("waiting for result from healthcheck") healthy = fut.result(5) - logging.info("responding") + logger.info("responding") self.respond(200 if healthy else 503) case _: self.respond(200) except Exception as e: - logging.error(f"Errored during check") - logging.exception(e) + logger.error(f"Errored during check") + logger.exception(e) def do_HEAD(self): self.handle_check() diff --git a/fred/cogs/welcome.py b/fred/cogs/welcome.py index 2d644db..365fdf0 100644 --- a/fred/cogs/welcome.py +++ b/fred/cogs/welcome.py @@ -1,21 +1,20 @@ from nextcord import Member -from nextcord.ext import commands from .. import config from ..libraries import common -class Welcome(commands.Cog): - def __init__(self, bot): - self.bot = bot - - @commands.Cog.listener() +class Welcome(common.FredCog): + @common.FredCog.listener() async def on_member_join(self, member: Member): - self.bot.logger.info("Processing a member joining", extra=common.user_info(member)) + + self.logger.info("Processing a member joining", extra=common.user_info(member)) + if welcome := config.Misc.fetch("welcome_message"): - self.bot.logger.info("Sending the welcome message to a new member", extra=common.user_info(member)) + self.logger.info("Sending the welcome message to a new member", extra=common.user_info(member)) await self.bot.send_DM(member, welcome) + if info := config.Misc.fetch("latest_info"): - self.bot.logger.info("Sending the latest information to a new member", extra=common.user_info(member)) + self.logger.info("Sending the latest information to a new member", extra=common.user_info(member)) info = f"Here's the latest information :\n\n{info}" await self.bot.send_DM(member, info) diff --git a/fred/fred.py b/fred/fred.py index 48aa76d..a43c569 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -21,21 +21,21 @@ class Bot(commands.Bot): async def isAlive(self): try: - logging.info("getting from config") + self.logger.info("Healthcheck: Attempting DB fetch") _ = config.Misc.get(1) - logging.info("creating user fetch") + self.logger.info("Healthcheck: Creating user fetch") coro = self.fetch_user(227473074616795137) - logging.info("fetching user") + self.logger.info("Healthcheck: Executing user fetch") await asyncio.wait_for(coro, timeout=5) except Exception as e: - self.logger.error(f"Healthiness check failed: {e}") + self.logger.error(f"Healthcheck failed: {e}") return False return True def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.isReady = False - self.setup_logger() + self.logger = common.new_logger(self.__class__.__name__) self.setup_DB() self.command_prefix = config.Misc.fetch("prefix") self.setup_cogs() @@ -58,7 +58,7 @@ def is_running(): async def on_ready(self): await self.change_presence(activity=nextcord.Game(f"v{self.version}")) self.isReady = True - logging.info(f"We have logged in as {self.user} with prefix {self.command_prefix}") + self.logger.info(f"We have logged in as {self.user} with prefix {self.command_prefix}") @staticmethod async def on_reaction_add(reaction: nextcord.Reaction, user: nextcord.User) -> None: @@ -85,19 +85,12 @@ def setup_DB(self): time.sleep(5) else: # this happens if the loop is not broken by a successful connection raise ConnectionError("Could not connect to the DB") + self.logger.info(f"Connected to the DB. Took {attempt} tries.") config.migrate() - - def setup_logger(self): - logging.root = logging.Logger("FRED") - logging.root.setLevel(logging.DEBUG) - - self.logger = logging.root - - self.logger.info("Successfully set up the logger") - self.logger.info(f"Prefix: {self.command_prefix}") + self.logger.debug("Applied migration.") def setup_cogs(self): - logging.info("Setting up cogs") + self.logger.info("Setting up cogs") self.add_cog(Commands(self)) self.add_cog(webhooklistener.Githook(self)) self.add_cog(mediaonly.MediaOnly(self)) @@ -105,10 +98,8 @@ def setup_cogs(self): self.add_cog(dialogflow.DialogFlow(self)) self.add_cog(welcome.Welcome(self)) self.add_cog(levelling.Levelling(self)) - self.MediaOnly = self.get_cog("MediaOnly") - self.Crashes = self.get_cog("Crashes") - self.DialogFlow = self.get_cog("DialogFlow") - logging.info("Successfully set up cogs") + + self.logger.info("Successfully set up cogs") async def on_error(self, event, *args, **kwargs): type, value, tb = sys.exc_info() @@ -124,7 +115,7 @@ async def on_error(self, event, *args, **kwargs): fred_str = f"Fred v{self.version}" error_meta = f"{type.__name__} exception handled in `{event}` {channel_str}" full_error = f"\n{value}\n\n{''.join(traceback.format_tb(tb))}" - logging.error(f"{fred_str}\n{error_meta}\n{full_error}") + self.logger.error(f"{fred_str}\n{error_meta}\n{full_error}") # error_embed = nextcord.Embed(colour=nextcord.Colour.red(), title=error_meta, description=full_error) # error_embed.set_author(name=fred_str) diff --git a/fred/fred_commands/_baseclass.py b/fred/fred_commands/_baseclass.py index bb72cc1..56e10f5 100644 --- a/fred/fred_commands/_baseclass.py +++ b/fred/fred_commands/_baseclass.py @@ -1,25 +1,12 @@ -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING - from nextcord.ext import commands from .. import config - -if TYPE_CHECKING: - from ..fred import Bot from ..libraries import common assert config # noqa -class BaseCmds(commands.Cog): - - logger = logging.Logger("COMMANDS") - - def __init__(self, bot: Bot): - self.bot: Bot = bot +class BaseCmds(common.FredCog): @commands.group() @commands.check(common.l4_only) diff --git a/fred/fred_commands/help.py b/fred/fred_commands/help.py index 5b8331b..b1d5db0 100644 --- a/fred/fred_commands/help.py +++ b/fred/fred_commands/help.py @@ -1,14 +1,16 @@ from __future__ import annotations -import logging +import re from functools import lru_cache from typing import Coroutine import attrs import nextcord -import re from ._baseclass import BaseCmds, commands, config +from ..libraries import common + +logger = common.new_logger(__name__) class HelpCmds(BaseCmds): @@ -121,7 +123,6 @@ class FredHelpEmbed(nextcord.Embed): # these are placeholder values only, they will be set up when setup() is called post-DB connection help_colour: int = 0 prefix: str = ">" - logger = logging.Logger("HELP-EMBEDS") def __init__( self: FredHelpEmbed, name: str, desc: str, /, usage: str = "", fields: list[dict] = (), **kwargs @@ -230,7 +231,7 @@ def crashes(cls, index: int = 0) -> FredHelpEmbed: "in a message, pastebin, debug file, or screenshot.*\n" ) all_crashes = list(config.Crashes.selectBy()) - cls.logger.info(f"Fetched {len(all_crashes)} crashes from database.") + logger.info(f"Fetched {len(all_crashes)} crashes from database.") global page_size, field_size # splits all crashes into groups of {page_size} @@ -278,7 +279,7 @@ def commands(cls, index: int = 0) -> FredHelpEmbed: desc = "*These are normal commands that can be called by stating their name.*\n" all_commands = list(config.Commands.selectBy()) - cls.logger.info(f"Fetched {len(all_commands)} commands from database.") + logger.info(f"Fetched {len(all_commands)} commands from database.") global page_size, field_size # splits all commands into groups of {page_size} pages = [all_commands[page : page + page_size] for page in range(0, len(all_commands), page_size)] diff --git a/fred/libraries/common.py b/fred/libraries/common.py index c6277e6..8fb7d07 100644 --- a/fred/libraries/common.py +++ b/fred/libraries/common.py @@ -1,13 +1,36 @@ +from __future__ import annotations + import logging import re from functools import lru_cache +from typing import TYPE_CHECKING, Optional from nextcord import User, Message, Member from nextcord.ext.commands import Context from .. import config -logger = logging.Logger("PERMISSIONS") +if TYPE_CHECKING: + from ..fred import Bot + + +def new_logger(name: str) -> logging.Logger: + logging.root.debug("Creating new logger for %s", name) + new_logger_ = logging.root.getChild(name) + new_logger_.setLevel(new_logger_.parent.level) + return new_logger_ + + +logger = new_logger(__name__) + + +class FredCog(commands.Cog): + bot: Bot # we can assume any cog will have a bot by the time it needs to be accessed + + def __init__(self, bot: Bot): + self.bot = bot + self.logger = new_logger(self.__class__.__name__) + self.logger.debug("Cog loaded.") def is_bot_author(user_id: int) -> bool: @@ -39,6 +62,9 @@ def permission_check(ctx: Context, level: int) -> bool: if (main_guild_member := main_guild.get_member(ctx.author.id)) is None: logger.warning("Checked permissions for someone but they weren't in the main guild", extra=logpayload) + logger.warning( + "Checked permissions for someone but they weren't in the main guild", extra=user_info(ctx.author) + ) return False user_roles = [role.id for role in main_guild_member.roles] diff --git a/fred/libraries/createembed.py b/fred/libraries/createembed.py index 9e7467e..1686a7a 100644 --- a/fred/libraries/createembed.py +++ b/fred/libraries/createembed.py @@ -1,4 +1,3 @@ -import logging from datetime import datetime from io import BytesIO from urllib.parse import quote as url_safe @@ -10,6 +9,8 @@ from .. import config from ..libraries import common +logger = common.new_logger(__name__) + def timestamp(iso8601: str) -> str: return format_dt(datetime.fromisoformat(iso8601), "R") @@ -30,7 +31,7 @@ async def run(data: dict) -> nextcord.Embed | None: repo_name, repo_full_name = "", "" if (data_type := data.get("type")) is None: - logging.error("data didn't have a type field") + logger.error("data didn't have a type field") return match data_type: @@ -47,7 +48,7 @@ async def run(data: dict) -> nextcord.Embed | None: case "issue": embed = issue(data) case _: - logging.warning("Unsupported GitHub payload", extra={"data": data}) + logger.warning("Unsupported GitHub payload", extra={"data": data}) return embed @@ -360,7 +361,7 @@ async def mod_embed(name: str, bot) -> tuple[nextcord.Embed | None, nextcord.Fil # fmt: on result = await bot.repository_query(query) mods: list[dict] = result["data"]["getMods"]["mods"] - logging.debug(mods) + logger.debug(mods) if not mods: return None, None, None From 4e123d22f3804ebab2e762d91d69c5ec3629c3fc Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 11 Aug 2024 02:06:40 -0400 Subject: [PATCH 23/37] fix: cleanup and minor fixes - fixed an issue where discord file urls weren't resolving anymore - expanded functionality of common.permission_check - handled error due to hybrid running not having permissions to open port 80 --- fred/cogs/crashes.py | 34 +++++++++++++++-------- fred/cogs/dialogflow.py | 13 +++++---- fred/cogs/levelling.py | 26 +++++++++--------- fred/cogs/webhooklistener.py | 16 +++++++---- fred/config.py | 7 +++-- fred/fred.py | 35 ++++++++++++++++-------- fred/fred_commands/__init__.py | 7 ++--- fred/libraries/common.py | 50 ++++++++++++++++++++++------------ 8 files changed, 117 insertions(+), 71 deletions(-) diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index 04fb449..d495282 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import io import json @@ -6,11 +8,14 @@ import traceback import zipfile from concurrent.futures import ThreadPoolExecutor +from os.path import split from time import strptime -from typing import AsyncIterator +from typing import AsyncIterator, IO +from urllib.parse import urlparse import nextcord from PIL import Image, ImageEnhance, UnidentifiedImageError +from nextcord import Message from pytesseract import image_to_string, TesseractError from .. import config @@ -50,7 +55,13 @@ async def parse_factory_game_log(self, text: str) -> dict[str, str | int]: vanilla_info_search_area = filter(lambda l: re.match("^LogInit", l), lines) info = {} - patterns = self.vanilla_info_patterns[:] + patterns = self.vanilla_info_patterns[:] # shallow copy + """ + This godforsaken loop sequentially finds information, + dropping patterns to look for as they are found (until it runs out of patterns). + The patterns have named regex captures which the rest of the code knows the names of. + - Borketh + """ for line in vanilla_info_search_area: if not patterns: break @@ -185,7 +196,7 @@ async def check_for_outdated_mods(self, mod_list: list) -> list[str]: if latest_compatible_loader > strptime(mod["last_version_date"], "%Y-%m-%dT%H:%M:%S.%fZ") ] - async def process_file(self, file, extension) -> list[dict](): + async def process_file(self, file: IO, extension: str) -> list[dict](): self.logger.info(f"Processing {extension} file") match extension: case "": @@ -326,7 +337,7 @@ async def complex_responses(self, log_details: dict): async def process_text(self, text: str) -> list[dict]: return [msg async for msg in self.mass_regex(text)] - async def process_message(self, message) -> bool: + async def process_message(self, message: Message) -> bool: self.bot.logger.info("Processing crashes") responses: list[dict] = [] # attachments @@ -343,9 +354,10 @@ async def process_message(self, message) -> bool: self.logger.info("Couldn't acquire file from discord API") try: self.logger.info("Attempting to acquire linked file manually") - url = cdn_link.group(1) - name = url.split("/")[-1] - async with self.bot.web_session.get(url) as response: + att_url = cdn_link.group(1) + url_path = urlparse(att_url).path + _, name = split(url_path) + async with self.bot.web_session.get(att_url) as response: assert response.status == 200, f"the web request failed with status {response.status}" file = io.BytesIO(await response.read()) except (AttributeError, AssertionError) as e: @@ -381,10 +393,10 @@ async def process_message(self, message) -> bool: await self.bot.reply_to_msg(message, embed=createembed.crashes(responses)) else: for response in responses: - attachment = response.get("attachment") - if attachment is not None: - async with self.bot.web_session.get(attachment) as resp: + att_url: str = response.get("attachment") + if att_url is not None: + async with self.bot.web_session.get(att_url) as resp: buff = io.BytesIO(await resp.read()) - attachment = nextcord.File(filename=attachment.split("/")[-1], fp=buff) + attachment = nextcord.File(filename=att_url.split("/")[-1], fp=buff) await self.bot.reply_to_msg(message, response["value"], file=attachment, propagate_reply=False) return len(responses) > 0 diff --git a/fred/cogs/dialogflow.py b/fred/cogs/dialogflow.py index 37d44cc..e0c46a4 100644 --- a/fred/cogs/dialogflow.py +++ b/fred/cogs/dialogflow.py @@ -1,7 +1,7 @@ import asyncio import json -import logging import uuid +from os import getenv import nextcord from google.cloud import dialogflow @@ -10,14 +10,17 @@ from .. import config from ..libraries import common - -if os.environ.get("DIALOGFLOW_AUTH"): - DIALOGFLOW_AUTH = json.loads(os.environ.get("DIALOGFLOW_AUTH")) +if env_auth_str := getenv("DIALOGFLOW_AUTH"): + DIALOGFLOW_AUTH = json.loads(env_auth_str) session_client = dialogflow.SessionsClient( credentials=service_account.Credentials.from_service_account_info(DIALOGFLOW_AUTH) ) DIALOGFLOW_PROJECT_ID = DIALOGFLOW_AUTH["project_id"] SESSION_LIFETIME = 10 * 60 # 10 minutes to avoid repeated false positives +else: + DIALOGFLOW_AUTH = None + session_client = None + DIALOGFLOW_PROJECT_ID = None class DialogFlow(common.FredCog): @@ -36,7 +39,7 @@ async def process_message(self, message): # We're in a DM channel self.logger.info("Ignoring a message because it is in a DM channel", extra=common.message_info(message)) return - self.logger.info("Ignoring a message because NLP is disabled", extra=common.message_info(message)) + if not config.Misc.fetch("dialogflow_state") or DIALOGFLOW_AUTH is None: self.logger.info( "Ignoring a message because NLP is disabled or not configured", extra=common.message_info(message) ) diff --git a/fred/cogs/levelling.py b/fred/cogs/levelling.py index 57d6ea6..3395b00 100644 --- a/fred/cogs/levelling.py +++ b/fred/cogs/levelling.py @@ -4,7 +4,7 @@ import math from datetime import * -import nextcord.ext.commands as commands +from nextcord import DMChannel, Message, Guild from nextcord.ext.commands import MemberNotFound from .. import config @@ -14,11 +14,9 @@ class UserProfile: - def __init__(self, user_id, guild, bot): + def __init__(self, user_id: int, guild: Guild): self.guild = guild - self.bot = bot self.user_id = user_id - self.logger = logging.Logger("USERPROFILE") self.member = guild.get_member(user_id) if self.member is None: @@ -32,8 +30,13 @@ def __init__(self, user_id, guild, bot): else: self.DB_user = config.Users(user_id=user_id) - self.rank = self.DB_user.rank - self.xp_count = self.DB_user.xp_count + @property + def rank(self): + return self.DB_user.rank + + @property + def xp_count(self): + return self.DB_user.xp_count async def validate_role(self): if not self.member: @@ -87,12 +90,12 @@ async def validate_level(self): self.DB_user.rank = expected_level if self.DB_user.accepts_dms: if expected_level > self.rank: - await self.bot.send_DM( + await Levelling.bot.send_DM( self.member, f"You went up from level {self.rank} to level {expected_level}! " f"Congratulations!", ) else: - await self.bot.send_DM( + await Levelling.bot.send_DM( self.member, f"You went down from level {self.rank} to level {expected_level}... " f"Sorry about that", ) @@ -112,7 +115,6 @@ async def give_xp(self, xp): logpayload["xp_gain"] = xp logger.info("Giving someone xp", logpayload) self.DB_user.xp_count += xp - self.xp_count += xp await self.validate_level() return True @@ -124,7 +126,6 @@ async def take_xp(self, xp): logpayload["xp_loss"] = xp logger.info("Taking xp from someone", logpayload) self.DB_user.xp_count -= xp - self.xp_count -= xp await self.validate_level() return True @@ -133,7 +134,6 @@ async def set_xp(self, xp): logpayload["new_xp"] = xp logger.info("Setting someone's xp", logpayload) self.DB_user.xp_count = xp - self.xp_count = xp await self.validate_level() return True @@ -151,8 +151,8 @@ def __init__(self, *args, **kwargs): # if before.roles != after.roles: # config.XpRoles - @commands.Cog.listener() @common.FredCog.listener() + async def on_message(self, message: Message): self.logger.info("Levelling: Processing message", extra=common.message_info(message)) if ( message.author.bot @@ -162,7 +162,7 @@ def __init__(self, *args, **kwargs): ): return - profile = UserProfile(message.author.id, message.guild, self.bot) + profile = UserProfile(message.author.id, message.guild) profile.DB_user.message_count += 1 if profile.user_id in self.xp_timers: if datetime.now() >= self.xp_timers[profile.user_id]: diff --git a/fred/cogs/webhooklistener.py b/fred/cogs/webhooklistener.py index 56076bd..7e4e913 100644 --- a/fred/cogs/webhooklistener.py +++ b/fred/cogs/webhooklistener.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import json import socket @@ -5,9 +7,14 @@ import threading import traceback from http.server import BaseHTTPRequestHandler, HTTPServer +from os import getenv +from typing import TYPE_CHECKING from fred.libraries import common +if TYPE_CHECKING: + from fred.fred import Bot + logger = common.new_logger(__name__) @@ -29,12 +36,11 @@ def __init__(self, *args, **kwargs): # Run GitHub webhook handling server try: - botargs = [bot, bot] - daemon = threading.Thread(target=runServer, args=botargs) + daemon = threading.Thread(target=runServer, args=[self.bot, self.bot]) daemon.daemon = True daemon.start() - except Exception: - type, value, tb = sys.exc_info() + except Exception: # noqa + exc_type, value, tb = sys.exc_info() tbs = "".join(traceback.format_tb(tb)) self.logger.error(f"Failed to run the webserver:\n{tbs}") @@ -47,7 +53,7 @@ def __init__(self, *args, **kwargs): EVENT_TYPE = "x-github-event" -def MakeGithookHandler(bot): +def MakeGithookHandler(bot: Bot): class MyGithookHandler(BaseHTTPRequestHandler): def respond(self, code: int, message: str | None = None): self.send_response(code, message) diff --git a/fred/config.py b/fred/config.py index 23e70bb..8272720 100644 --- a/fred/config.py +++ b/fred/config.py @@ -1,12 +1,13 @@ from __future__ import annotations -from numbers import Number -from typing import Optional, Any +import json import os import pathlib +from numbers import Number +from typing import Optional, Any import nextcord -from sqlobject import * +from sqlobject import SQLObject, IntCol, BoolCol, JSONCol, BigIntCol, StringCol, FloatCol, sqlhub class PermissionRoles(SQLObject): diff --git a/fred/fred.py b/fred/fred.py index a43c569..258bc65 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -1,9 +1,8 @@ import asyncio -import logging -import os import sys import time import traceback +from os import getenv import aiohttp import nextcord @@ -48,7 +47,7 @@ def __init__(self, *args, **kwargs): async def start(self, *args, **kwargs): async with aiohttp.ClientSession() as session: - self.web_session = session + self.web_session = session # noqa return await super().start(*args, **kwargs) @staticmethod @@ -67,11 +66,11 @@ async def on_reaction_add(reaction: nextcord.Reaction, user: nextcord.User) -> N def setup_DB(self): self.logger.info("Connecting to the database") - user = os.environ.get("FRED_SQL_USER") - password = os.environ.get("FRED_SQL_PASSWORD") - host = os.environ.get("FRED_SQL_HOST") - port = os.environ.get("FRED_SQL_PORT") - dbname = os.environ.get("FRED_SQL_DB") + user = getenv("FRED_SQL_USER") + password = getenv("FRED_SQL_PASSWORD") + host = getenv("FRED_SQL_HOST") + port = getenv("FRED_SQL_PORT") + dbname = getenv("FRED_SQL_DB") uri = f"postgresql://{user}:{password}@{host}:{port}/{dbname}" attempt = 1 while attempt < 6: @@ -101,8 +100,20 @@ def setup_cogs(self): self.logger.info("Successfully set up cogs") + @property + def MediaOnly(self) -> mediaonly.MediaOnly: + return self.get_cog("MediaOnly") # noqa + + @property + def Crashes(self) -> crashes.Crashes: + return self.get_cog("Crashes") # noqa + + @property + def DialogFlow(self) -> dialogflow.DialogFlow: + return self.get_cog("DialogFlow") # noqa + async def on_error(self, event, *args, **kwargs): - type, value, tb = sys.exc_info() + exc_type, value, tb = sys.exc_info() if event == "on_message": channel: nextcord.TextChannel | nextcord.DMChannel = args[0].channel if isinstance(channel, nextcord.DMChannel): @@ -113,7 +124,7 @@ async def on_error(self, event, *args, **kwargs): channel_str = "" fred_str = f"Fred v{self.version}" - error_meta = f"{type.__name__} exception handled in `{event}` {channel_str}" + error_meta = f"{exc_type.__name__} exception handled in `{event}` {channel_str}" full_error = f"\n{value}\n\n{''.join(traceback.format_tb(tb))}" self.logger.error(f"{fred_str}\n{error_meta}\n{full_error}") @@ -135,7 +146,7 @@ async def githook_send(self, data): async def send_DM( self, - user: nextcord.User, + user: nextcord.User | nextcord.Member, content: str = None, embed: nextcord.Embed = None, user_meta: config.Users = None, @@ -168,7 +179,7 @@ async def send_DM( content = None await user.dm_channel.send(content=content, embed=embed, **kwargs) return True - except Exception: + except Exception: # noqa self.logger.error(f"DMs: Failed to DM, reason: \n{traceback.format_exc()}") return False diff --git a/fred/fred_commands/__init__.py b/fred/fred_commands/__init__.py index d926c47..3f03392 100644 --- a/fred/fred_commands/__init__.py +++ b/fred/fred_commands/__init__.py @@ -1,21 +1,20 @@ from __future__ import annotations -import logging - import asyncio import inspect import io +import logging import re import nextcord -from nextcord.ext.commands.view import StringView from algoliasearch.search_client import SearchClient +from nextcord.ext.commands.view import StringView from ._baseclass import BaseCmds, common, config, commands from .bot_meta import BotCmds from .channels import ChannelCmds -from .dbcommands import CommandCmds from .crashes import CrashCmds +from .dbcommands import CommandCmds from .dialogflow import DialogflowCmds from .experience import EXPCmds from .help import HelpCmds, FredHelpEmbed diff --git a/fred/libraries/common.py b/fred/libraries/common.py index 8fb7d07..69bb484 100644 --- a/fred/libraries/common.py +++ b/fred/libraries/common.py @@ -2,10 +2,11 @@ import logging import re -from functools import lru_cache +from functools import lru_cache, singledispatch from typing import TYPE_CHECKING, Optional from nextcord import User, Message, Member +from nextcord.ext import commands from nextcord.ext.commands import Context from .. import config @@ -40,34 +41,45 @@ def is_bot_author(user_id: int) -> bool: async def l4_only(ctx: Context) -> bool: logger.info("Checking if someone is a T3", extra=user_info(ctx.author)) - return is_bot_author(ctx.author.id) or permission_check(ctx, 4) + return is_bot_author(ctx.author.id) or permission_check(ctx, level=4) async def mod_only(ctx: Context) -> bool: logger.info("Checking if someone is a Moderator", extra=user_info(ctx.author)) - return is_bot_author(ctx.author.id) or permission_check(ctx, 6) + return is_bot_author(ctx.author.id) or permission_check(ctx, level=6) -def permission_check(ctx: Context, level: int) -> bool: - logpayload = user_info(ctx.author) - logpayload["level"] = level - - logger.info("Checking permissions for someone", extra=logpayload) - perms = config.PermissionRoles.fetch_by_lvl(level) - - main_guild_id = int(config.Misc.fetch("main_guild_id")) +@singledispatch +def permission_check(ctx: Context, *, level: int) -> bool: + main_guild_id = config.Misc.fetch("main_guild_id") main_guild = ctx.bot.get_guild(main_guild_id) + if main_guild is None: raise LookupError(f"Unable to retrieve the guild {main_guild_id}. Is this the guild you meant?") if (main_guild_member := main_guild.get_member(ctx.author.id)) is None: - logger.warning("Checked permissions for someone but they weren't in the main guild", extra=logpayload) logger.warning( "Checked permissions for someone but they weren't in the main guild", extra=user_info(ctx.author) ) return False - user_roles = [role.id for role in main_guild_member.roles] + return permission_check(main_guild_member, level=level) + + +@permission_check.register +def permission_check(member: Member, *, level: int) -> bool: + """Checks permissions for a member assuming they are in the main guild.""" + logpayload = user_info(member) + logpayload["level"] = level + + if member.guild.id != config.Misc.fetch("main_guild_id"): + logger.warning("Checked permissions for a member of the wrong guild", extra=logpayload) + return False + + logger.info("Checking permissions for someone", extra=logpayload) + perms = config.PermissionRoles.fetch_by_lvl(level) + + user_roles = [role.id for role in member.roles] if ( # it shouldn't be possible to request a level above the defined levels but check anyway role := next( @@ -82,16 +94,18 @@ def permission_check(ctx: Context, level: int) -> bool: return False -@lru_cache(5) -def user_info(user: User | Member) -> dict: +@lru_cache(15) +def user_info(user: Optional[User | Member]) -> dict: + if user is None: + return {} return {"user_full_name": user.global_name, "user_id": user.id} -@lru_cache(5) -def message_info(message: Message) -> dict: +@lru_cache(15) +def message_info(message: Optional[Message]) -> dict: if message is None: return {} - return {"message_id": message.id, "channel_id": message.channel.id, "user_id": message.author.id} + return {"message_id": message.id, "channel_id": message.channel.id, **user_info(message.author)} def reduce_str(string: str) -> str: From 38a075673512c15266e558c6d8189f77fd7c810e Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 11 Aug 2024 02:31:03 -0400 Subject: [PATCH 24/37] fix: of course it doesn't work that way --- fred/libraries/common.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/fred/libraries/common.py b/fred/libraries/common.py index 69bb484..26d213e 100644 --- a/fred/libraries/common.py +++ b/fred/libraries/common.py @@ -50,7 +50,12 @@ async def mod_only(ctx: Context) -> bool: @singledispatch -def permission_check(ctx: Context, *, level: int) -> bool: +def permission_check(_, *, level: int) -> bool: + pass + + +@permission_check.register +def _permission_check_ctx(ctx: Context, *, level: int) -> bool: main_guild_id = config.Misc.fetch("main_guild_id") main_guild = ctx.bot.get_guild(main_guild_id) @@ -63,11 +68,11 @@ def permission_check(ctx: Context, *, level: int) -> bool: ) return False - return permission_check(main_guild_member, level=level) + return _permission_check_member(main_guild_member, level=level) @permission_check.register -def permission_check(member: Member, *, level: int) -> bool: +def _permission_check_member(member: Member, *, level: int) -> bool: """Checks permissions for a member assuming they are in the main guild.""" logpayload = user_info(member) logpayload["level"] = level From b1942dee0ee03566c48811d3b86cdf3749a0f79d Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 11 Aug 2024 03:20:33 -0400 Subject: [PATCH 25/37] perf(crashes): better debug zip responsiveness eased off regex timeout used nonblocking regex lib better async handling also fixes bug introduced in 4e123d22f3804ebab2e762d91d69c5ec3629c3fc where the attachment to a response could be undefined instead of a safe None --- fred/cogs/crashes.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index d495282..987c75d 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -4,7 +4,6 @@ import io import json import logging -import re import traceback import zipfile from concurrent.futures import ThreadPoolExecutor @@ -14,6 +13,7 @@ from urllib.parse import urlparse import nextcord +import regex from PIL import Image, ImageEnhance, UnidentifiedImageError from nextcord import Message from pytesseract import image_to_string, TesseractError @@ -22,12 +22,12 @@ from ..libraries import createembed from ..libraries.common import FredCog -REGEX_LIMIT: float = 2 +REGEX_LIMIT: float = 6.9 -async def regex_with_timeout(*args, **kwargs): +def regex_with_timeout(*args, **kwargs): try: - return await asyncio.wait_for(asyncio.to_thread(re.search, *args, **kwargs), REGEX_LIMIT) + return asyncio.wait_for(asyncio.to_thread(regex.search, *args, **kwargs), REGEX_LIMIT) except asyncio.TimeoutError: raise TimeoutError( f"A regex timed out after {REGEX_LIMIT} seconds! \n" @@ -39,10 +39,10 @@ async def regex_with_timeout(*args, **kwargs): class Crashes(FredCog): vanilla_info_patterns = [ - re.compile(r"Net CL: (?P\d+)"), - re.compile(r"Command Line: (?P.*)"), - re.compile(r"Base Directory: (?P.+)"), - re.compile(r"Launcher ID: (?P\w+)"), + regex.compile(r"Net CL: (?P\d+)"), + regex.compile(r"Command Line: (?P.*)"), + regex.compile(r"Base Directory: (?P.+)"), + regex.compile(r"Launcher ID: (?P\w+)"), ] @staticmethod @@ -52,7 +52,7 @@ def filter_epic_commandline(cli: str) -> str: async def parse_factory_game_log(self, text: str) -> dict[str, str | int]: self.logger.info("Extracting game info") lines = text.splitlines() - vanilla_info_search_area = filter(lambda l: re.match("^LogInit", l), lines) + vanilla_info_search_area = filter(lambda l: regex.match("^LogInit", l), lines) info = {} patterns = self.vanilla_info_patterns[:] # shallow copy @@ -71,7 +71,7 @@ async def parse_factory_game_log(self, text: str) -> dict[str, str | int]: else: self.logger.info("Didn't find all four pieces of information normally found in a log") - mod_loader_logs = filter(lambda l: re.match("LogSatisfactoryModLoader", l), lines) + mod_loader_logs = filter(lambda l: regex.match("LogSatisfactoryModLoader", l), lines) for line in mod_loader_logs: if match := await regex_with_timeout(r"(?<=v\.)(?P[\d.]+)", line): info |= match.groupdict() @@ -309,8 +309,15 @@ async def process_file(self, file: IO, extension: str) -> list[dict](): return [] async def mass_regex(self, text: str) -> AsyncIterator[dict]: - for crash in config.Crashes.fetch_all(): - if match := await regex_with_timeout(crash["crash"], text, flags=re.IGNORECASE): + + async def silly_async_gen(): + nonlocal text + all_crashes = config.Crashes.fetch_all() + for crash_ in all_crashes: + yield crash_, regex_with_timeout(crash_["crash"], text, flags=regex.IGNORECASE) + + async for crash, queued_regex in silly_async_gen(): + if match := await queued_regex: if str(crash["response"]).startswith(self.bot.command_prefix): if command := config.Commands.fetch(crash["response"].strip(self.bot.command_prefix)): command_response = command["content"] @@ -323,7 +330,7 @@ async def mass_regex(self, text: str) -> AsyncIterator[dict]: inline=True, ) else: - response = re.sub(r"{(\d+)}", lambda m: match.group(int(m.group(1))), str(crash["response"])) + response = regex.sub(r"{(\d+)}", lambda m: match.group(int(m.group(1))), str(crash["response"])) yield dict(name=crash["name"], value=response, inline=True) async def complex_responses(self, log_details: dict): @@ -341,7 +348,7 @@ async def process_message(self, message: Message) -> bool: self.bot.logger.info("Processing crashes") responses: list[dict] = [] # attachments - cdn_link = re.search(r"(https://cdn.discordapp.com/attachments/\S+)", message.content) + cdn_link = regex.search(r"(https://cdn.discordapp.com/attachments/\S+)", message.content) if cdn_link or message.attachments: self.logger.info("Found file in message") @@ -377,8 +384,8 @@ async def process_message(self, message: Message) -> bool: file.close() # Pastebin links - elif match := re.search(r"(https://pastebin.com/\S+)", message.content): - url = re.sub(r"(?<=bin.com)/", "/raw/", match.group(1)) + elif match := regex.search(r"(https://pastebin.com/\S+)", message.content): + url = regex.sub(r"(?<=bin.com)/", "/raw/", match.group(1)) async with self.bot.web_session.get(url) as response: text: str = await response.text() @@ -398,5 +405,7 @@ async def process_message(self, message: Message) -> bool: async with self.bot.web_session.get(att_url) as resp: buff = io.BytesIO(await resp.read()) attachment = nextcord.File(filename=att_url.split("/")[-1], fp=buff) + else: + attachment = None await self.bot.reply_to_msg(message, response["value"], file=attachment, propagate_reply=False) return len(responses) > 0 From 95db6264b25671faea689046c1b7a73c28465af3 Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 11 Aug 2024 03:33:57 -0400 Subject: [PATCH 26/37] fix(crashes): handle regex timeout correctly and remove use of timed limited regexes where unnecessary --- fred/cogs/crashes.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index 987c75d..adc760d 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -3,7 +3,6 @@ import asyncio import io import json -import logging import traceback import zipfile from concurrent.futures import ThreadPoolExecutor @@ -26,15 +25,7 @@ def regex_with_timeout(*args, **kwargs): - try: - return asyncio.wait_for(asyncio.to_thread(regex.search, *args, **kwargs), REGEX_LIMIT) - except asyncio.TimeoutError: - raise TimeoutError( - f"A regex timed out after {REGEX_LIMIT} seconds! \n" - f"pattern: ({args[0]}) \n" - f"flags: {kwargs['flags']} \n" - f"on text of length {len(args[1])}" - ) + return asyncio.wait_for(asyncio.to_thread(regex.search, *args, **kwargs), REGEX_LIMIT) class Crashes(FredCog): @@ -65,7 +56,7 @@ async def parse_factory_game_log(self, text: str) -> dict[str, str | int]: for line in vanilla_info_search_area: if not patterns: break - elif match := await regex_with_timeout(patterns[0], line): + elif match := regex.search(patterns[0], line): info |= match.groupdict() patterns.pop(0) else: @@ -73,7 +64,7 @@ async def parse_factory_game_log(self, text: str) -> dict[str, str | int]: mod_loader_logs = filter(lambda l: regex.match("LogSatisfactoryModLoader", l), lines) for line in mod_loader_logs: - if match := await regex_with_timeout(r"(?<=v\.)(?P[\d.]+)", line): + if match := regex.search(r"(?<=v\.)(?P[\d.]+)", line): info |= match.groupdict() break @@ -316,8 +307,14 @@ async def silly_async_gen(): for crash_ in all_crashes: yield crash_, regex_with_timeout(crash_["crash"], text, flags=regex.IGNORECASE) + async def patience(coro): + try: + return await coro + except asyncio.TimeoutError: + self.logger.warning("Regex timed out") + async for crash, queued_regex in silly_async_gen(): - if match := await queued_regex: + if match := await patience(queued_regex): if str(crash["response"]).startswith(self.bot.command_prefix): if command := config.Commands.fetch(crash["response"].strip(self.bot.command_prefix)): command_response = command["content"] From 8278c255e0e63ff2f30ef0720cf8f33d6e613b56 Mon Sep 17 00:00:00 2001 From: Rob B Date: Thu, 1 Aug 2024 19:08:19 -0400 Subject: [PATCH 27/37] Fix readme typos, missing periods, and formatting --- README.md | 64 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 3ec3af9..47cf70c 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,59 @@ # FICSIT-Fred + A Discord bot for the Satisfactory Modding Discord ## Development -Want to contribute to Fred? Here's everything you need to know to get started + +Want to contribute to Fred? Here's everything you need to know to get started. ### Prerequisites #### A Discord token -First of all, you'll need a Discord bot token for Fred to connect to Discord with -To get one, you can go to https://discord.com/developers/applications, register a new application, add a bot to the application and **activate the "message content intent"** -This intent is needed to receive the content of messages. Fred doesn't use slash commands so this is simply needed for him to function + +First of all, you'll need a Discord bot token for Fred to connect to Discord with. +To get one, you can go to , register a new application, add a bot to the application and **activate the "message content intent"**. +This intent is needed to receive the content of messages. +Fred doesn't use slash commands so this is simply needed for him to function. #### Docker -Docker is for making containers. If you're not sure what that means, look it up it's real cool! In our case, it helps use set up all needed dependencies for Fred without you needing to do anything other than install Docker -You can get docker [here](https://docs.docker.com/engine/install/). For Linux, make sure to **not** use Docker Desktop. For Windows, it is the easiest way -If you don't want to install Docker (especially on Windows where Docker Desktop can take up resources and requires virtualisation to be enabled), you can also manually set up a PostgreSQL DB and configure Fred to point to it. More on that later + +Docker is for making containers. If you're not sure what that means, look it up it's real cool! In our case, it helps use set up all needed dependencies for Fred without you needing to do anything other than install Docker. +You can get docker [here](https://docs.docker.com/engine/install/). For Linux, make sure to **not** use Docker Desktop. For Windows, it is the easiest way. +If you don't want to install Docker (especially on Windows where Docker Desktop can take up resources and requires virtualisation to be enabled), you can also manually set up a PostgreSQL DB and configure Fred to point to it. More on that later. #### (Optional) Dialogflow auth -This is optional because this feature is currently disabled in Fred -You'll have get authentication information for dialogflow if you want to work on that + +This is optional because this feature is currently disabled in Fred. +You'll have get authentication information for dialogflow if you want to work on that. ### Setup -Two choices here: All through docker or hybrid local/docker -I recommend the hybrid way for ease of development. We don't have a proper devcontainer setup so debugging Fred when running in Docker is not great -Instead, in a hybrid setup, you'll use Docker for the postgres DB only and run Fred locally using Poetry + +Two choices here: All through docker or hybrid local/docker. +I recommend the hybrid way for ease of development. We don't have a proper devcontainer setup so debugging Fred when running in Docker is not great. +Instead, in a hybrid setup, you'll use Docker for the postgres DB only and run Fred locally using Poetry. #### Docker-only -Run `FRED_TOKEN= docker compose up -d` and that's it! Fred should run -(Note: This command assumes you're using bash as your shell. For Powershell or other, set the environment variable differently) -You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show -You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose.yml` +Run `FRED_TOKEN= docker compose up -d` and that's it! Fred should run. +(Note: This command assumes you're using bash as your shell. For Powershell or other, set the environment variable differently). + +You can verify that the database was properly created and manage it by going to where pgadmin should show. +You can use `fred@fred.com` for the user and `fred` for the password. All of this is customizable in `docker-compose.yml`. #### Hybrid -For this, you'll need [poetry](https://python-poetry.org/) installed, as well as `libpq` for the connection to the postgres DB -For libpq, you can install `libpq-dev` (or equivalent) on linux and [install postgres](https://www.postgresql.org/download/windows/) on Windows (test without it first actually I'm not certain this is needed) -Once Poetry is installed, run `poetry install`. This should create a `.venv` folder that contains the virtualenv with all the necessary packages. Activate it using `poetry shell` or manually +For this, you'll need [poetry](https://python-poetry.org/) installed, as well as `libpq` for the connection to the postgres DB. +For libpq, you can install `libpq-dev` (or equivalent) on linux and [install postgres](https://www.postgresql.org/download/windows/) on Windows (test without it first actually I'm not certain this is needed). -Now, run `docker compose up -d -f docker-compose-deps.yml`. This should spin up the postgres DB -You can verify that the database was properly created and manage it by going to http://localhost:8080 where pgadmin should show -You can use `fred@fred.com` for the user and `fred` for the password. All of this is customisable in `docker-compose-deps.yml` +Once Poetry is installed, run `poetry install`. This should create a `.venv` folder that contains the virtualenv with all the necessary packages. Activate it using `poetry shell` or manually. -Almost there! You'll now have to configure Fred. This just means setting the following env vars (found in `fred/__main__.py`) -``` +Now, run `docker compose up -d -f docker-compose-deps.yml`. This should spin up the postgres DB. +You can verify that the database was properly created and manage it by going to where pgadmin should show. +You can use `fred@fred.com` for the user and `fred` for the password. All of this is customizable in `docker-compose-deps.yml`. + +Almost there! You'll now have to configure Fred. This just means setting the following env vars (found in `fred/__main__.py`). + +```sh "FRED_IP", "FRED_PORT", "FRED_TOKEN", @@ -53,9 +63,11 @@ Almost there! You'll now have to configure Fred. This just means setting the fol "FRED_SQL_HOST", "FRED_SQL_PORT", ``` -For convenience, a `.env` file is included and Fred will load them automatically. It includes defaults that will work with the config of the docker-compose-deps.yml. You'll only have to set `FRED_TOKEN` and maybe change the others if the defaults don't suit you -Finally, run `python fred/__main__.py`. Fred should run! You can now adapt this to your setup and run the script from your IDE instead. Don't forget to use the virtualenv python! +For convenience, a `.env` file is included and Fred will load them automatically. It includes defaults that will work with the config of the docker-compose-deps.yml. You'll only have to set `FRED_TOKEN` and maybe change the others if the defaults don't suit you. + +Finally, run `python fred/__main__.py`. Fred should run! You can now adapt this to your setup and run the script from your IDE instead. Don't forget to use the virtualenv python! ## Thanks + Massive thanks to Borketh, Mircea and everyone else that has contributed! From a3da6d9518126d190ca54b81fbea36d3adf4db8d Mon Sep 17 00:00:00 2001 From: Rob B Date: Thu, 1 Aug 2024 19:08:53 -0400 Subject: [PATCH 28/37] Keep shell files as LF so cloning on Windows hosts doesn't break things --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitattributes b/.gitattributes index dfe0770..dada09c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,5 @@ # Auto detect text files and perform LF normalization * text=auto + +# Keep shell files as LF so Windows-host-machine users can still use Docker +*.sh text eol=lf From b2514a0a1361a58b3f89d175f44904a53266b558 Mon Sep 17 00:00:00 2001 From: Rob B Date: Thu, 1 Aug 2024 19:09:15 -0400 Subject: [PATCH 29/37] Spellchecker config --- cspell.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 cspell.json diff --git a/cspell.json b/cspell.json new file mode 100644 index 0000000..585b9d5 --- /dev/null +++ b/cspell.json @@ -0,0 +1,19 @@ +// https://cspell.org/configuration/ +{ + // Version of the setting file. Always 0.2 + "version": "0.2", + // language - current active spelling language + "language": "en", + // words - list of words to be always considered correct + "words": [ + "FICSIT", + "libpq", + "Mircea", + "pgadmin", + "virtualenv" + ], + // flagWords - list of words to be always considered incorrect + // This is useful for offensive words and common spelling errors. + // cSpell:disable (don't complain about the words we listed here) + "flagWords": [] +} From 759609383f63152d7e25d5d0f98326126b8c2a16 Mon Sep 17 00:00:00 2001 From: borketh Date: Sun, 11 Aug 2024 17:32:21 -0400 Subject: [PATCH 30/37] build: simplify Dockerfile and use alpine debian version is still there and commented out this was made possible because psycopg2 started publishing arm64 and musl images (woo) on my machine this makes build time 1/3 of original and produces an image less than half the size the debian version when switched out by commenting/uncommenting takes about half the time for 3/4 of the size of previous --- docker/Dockerfile | 29 ++++++++++++++----------- docker/runtime-deps.sh | 48 ------------------------------------------ pyproject.toml | 2 +- 3 files changed, 18 insertions(+), 61 deletions(-) delete mode 100644 docker/runtime-deps.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index afe0dac..35d9c1e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,26 +1,31 @@ -FROM python:3.12-slim AS runtime +FROM python:3.12-alpine AS runtime +#FROM python:3.12-slim AS runtime -ENV DEBIAN_FRONTEND=noninteractive VOLUME /config WORKDIR /app -COPY docker/runtime-deps.sh . -RUN bash runtime-deps.sh 1> /dev/null && rm runtime-deps.sh +RUN apk update; apk add --no-cache tesseract-ocr-data-eng +#ENV DEBIAN_FRONTEND=noninteractive APT="apt-get -qq" +#RUN $APT update; \ +# $APT install tesseract-ocr; \ +# $APT clean; \ +# rm -rf /var/lib/apt/lists/* -FROM python:3.12-slim AS build +FROM python:3.12-alpine AS build +#FROM python:3.12-slim AS build -WORKDIR /app - -RUN apt-get -qq update && apt-get -qq install curl libpq-dev gcc 1> /dev/null +WORKDIR /deps -RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/usr/local python3 - +RUN pip --no-cache-dir install --progress-bar=off "poetry==1.8" COPY pyproject.toml . -RUN poetry install -nvvv --only main && mv $(poetry env info --path) /app/venv +COPY poetry.lock . +RUN poetry install -nvvv --only main --no-root; mv $(poetry env info --path) ./venv FROM runtime -COPY --from=build /app/venv /app/venv -COPY fred *.env ./fred/ +COPY --from=build /deps/venv ./venv +COPY fred ./fred/ +COPY *.env . CMD ./venv/bin/python3 -m fred \ No newline at end of file diff --git a/docker/runtime-deps.sh b/docker/runtime-deps.sh deleted file mode 100644 index c880d12..0000000 --- a/docker/runtime-deps.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# This file is meant to be run in the Dockerfile to get the runtime dependencies -# if you run it outside, good luck - -echo "Getting initial setup dependencies..." 1>&2 -apt-get update -apt-get install software-properties-common ca-certificates gnupg curl -y - -# add repo that adds libpq properly -echo "Adding repository for libpq..." 1>&2 -curl -sS https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null -add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" -apt-get update - -# install runtime dependencies -echo "Installing runtime dependencies..." 1>&2 -apt-get install -y libpq5 tesseract-ocr - - -isDocker(){ - local cgroup=/proc/1/cgroup - test -f $cgroup && [[ "$(<$cgroup)" = *:cpuset:/docker/* ]] -} - -isDockerBuildkit(){ - local cgroup=/proc/1/cgroup - test -f $cgroup && [[ "$(<$cgroup)" = *:cpuset:/docker/buildkit/* ]] -} - -isDockerContainer(){ - [ -e /.dockerenv ] -} - -# we don't need this crap anymore -echo "Cleaning up..." 1>&2 - -if isDockerBuildkit || (isDocker && ! isDockerContainer) -then - echo "Detected run within docker RUN. Removing apt cache for image size." 1>&2 - apt-get remove -y software-properties-common gnupg curl - apt-get autopurge -y - apt-get autoclean -y - rm -rf /var/lib/apt/lists/* -else - echo "Opting to not remove stuff so running on real hardware does not break things." 1>&2 -fi - -echo "All runtime dependencies handled!" 1>&2 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 224f9b7..6b0a5da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ python = "^3.12" nextcord = {extras = ["speed"], version = "^2.6.0"} Pillow = "^10.4.0" pytesseract = "^0.3.8" -psycopg2 = "^2.9.3" python-dotenv = "^1.0.0" algoliasearch = "^3.0.0" SQLObject = "^3.9.1" @@ -18,6 +17,7 @@ google-cloud-dialogflow = "^2.11.0" regex = "^2024.7.0" aiohttp = "^3.10.0" setuptools = "^72.1.0" +psycopg2-binary = "^2.9.9" [tool.poetry.group.dev.dependencies] black = {extras = ["d"], version = "^24.8.0"} From eb465a54cf0236fb3cd40e2195ba8392765eecb6 Mon Sep 17 00:00:00 2001 From: mircearoata Date: Thu, 1 Aug 2024 23:13:38 +0300 Subject: [PATCH 31/37] ci: update action versions and variables --- .github/workflows/docker-publish.yml | 30 ++++++++++------------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 720ebce..346353d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -13,10 +13,8 @@ on: tags: [ 'v*.*.*' ] env: - # Use docker.io for Docker Hub if empty REGISTRY: ghcr.io - # github.repository as / - IMAGE_NAME: feyko/ficsit-fred + IMAGE_NAME: ${{ github.repository }} jobs: @@ -26,31 +24,25 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 - - # Login against a Docker registry except on PR + uses: actions/checkout@v4 + - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 with: registry: ${{ env.REGISTRY }} - username: feyko - password: ${{ secrets.GH_TOKEN }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - uses: docker/setup-buildx-action@v1 - id: buildx - - # Build and push Docker image with Buildx (don't push on PR) - # https://github.com/docker/build-push-action + - uses: docker/setup-buildx-action@v3 + - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: context: . file: ./docker/Dockerfile From f9e47e7ab05cfbfc18a46a332fd95e738d9a006b Mon Sep 17 00:00:00 2001 From: borketh Date: Mon, 12 Aug 2024 18:16:50 -0400 Subject: [PATCH 32/37] fix: resolve minor comments from the review --- fred/cogs/levelling.py | 29 +++++++++++++++++++++-------- fred/fred_commands/help.py | 8 ++++---- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/fred/cogs/levelling.py b/fred/cogs/levelling.py index 3395b00..c12d988 100644 --- a/fred/cogs/levelling.py +++ b/fred/cogs/levelling.py @@ -30,13 +30,26 @@ def __init__(self, user_id: int, guild: Guild): else: self.DB_user = config.Users(user_id=user_id) + self._rank: int = DB_user.rank + self._xp_count: int = DB_user.xp_count + @property def rank(self): - return self.DB_user.rank + return self._rank + + @rank.setter + def rank(self, value: int): + self._rank = value + self.DB_user.rank = value @property def xp_count(self): - return self.DB_user.xp_count + return self._xp_count + + @xp_count.setter + def xp_count(self, value: int): + self._xp_count = value + self.DB_user.xp_count = value async def validate_role(self): if not self.member: @@ -59,7 +72,7 @@ async def validate_role(self): # if not self.guild.get_channel(config.Misc.fetch("mod_channel")).permissions_for(self.member).send_messages: if not common.permission_check(self.member, level=6): for member_role in self.member.roles: - if rank := config.RankRoles.fetch_by_role(member_role.id): + if config.RankRoles.fetch_by_role(member_role.id) is not None: # i.e. member_role is a rank role logpayload["role_id"] = member_role.id logger.info("Removing a mismatched level role from someone", extra=logpayload) await self.member.remove_roles(member_role) @@ -87,7 +100,6 @@ async def validate_level(self): logpayload["current_level"] = self.rank if expected_level != self.rank: logger.info("Correcting a mismatched level", extra=logpayload) - self.DB_user.rank = expected_level if self.DB_user.accepts_dms: if expected_level > self.rank: await Levelling.bot.send_DM( @@ -99,6 +111,7 @@ async def validate_level(self): self.member, f"You went down from level {self.rank} to level {expected_level}... " f"Sorry about that", ) + self.rank = expected_level await self.validate_role() async def increment_xp(self): @@ -114,18 +127,18 @@ async def give_xp(self, xp): logpayload = common.user_info(self.member) logpayload["xp_gain"] = xp logger.info("Giving someone xp", logpayload) - self.DB_user.xp_count += xp + self.xp_count += xp await self.validate_level() return True async def take_xp(self, xp): - if xp > self.DB_user.xp_count: + if xp > self.xp_count: return False # can't take more than a user has logpayload = common.user_info(self.member) logpayload["xp_loss"] = xp logger.info("Taking xp from someone", logpayload) - self.DB_user.xp_count -= xp + self.xp_count -= xp await self.validate_level() return True @@ -133,7 +146,7 @@ async def set_xp(self, xp): logpayload = common.user_info(self.member) logpayload["new_xp"] = xp logger.info("Setting someone's xp", logpayload) - self.DB_user.xp_count = xp + self.xp_count = xp await self.validate_level() return True diff --git a/fred/fred_commands/help.py b/fred/fred_commands/help.py index b1d5db0..f48db5f 100644 --- a/fred/fred_commands/help.py +++ b/fred/fred_commands/help.py @@ -1,6 +1,6 @@ from __future__ import annotations -import re +import regex from functools import lru_cache from typing import Coroutine @@ -128,9 +128,9 @@ def __init__( self: FredHelpEmbed, name: str, desc: str, /, usage: str = "", fields: list[dict] = (), **kwargs ) -> None: - desc = re.sub(r"^\s*(\S.*)$", r"\1", desc, flags=re.MULTILINE) - desc = re.sub(r"(?<=Usage: )`(.+)`", rf"`{self.prefix}\1`", desc) - desc = re.sub(r"^(\w+:) ", r"**\1** ", desc, flags=re.MULTILINE) + desc = regex.sub(r"^\s*(\S.*)$", r"\1", desc, flags=regex.MULTILINE) + desc = regex.sub(r"(?<=Usage: )`(.+)`", rf"`{self.prefix}\1`", desc) + desc = regex.sub(r"^(\w+:) ", r"**\1** ", desc, flags=regex.MULTILINE) super().__init__(title=f"**{name}**", colour=self.help_colour, description=desc, **kwargs) for f in fields: self.add_field(**f) From 4a07e91c55cc1a73d7c6f3f61bb87ef28b2a7265 Mon Sep 17 00:00:00 2001 From: borketh Date: Tue, 13 Aug 2024 21:08:26 -0400 Subject: [PATCH 33/37] fix(crashes): revert async changes --- fred/cogs/crashes.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index adc760d..fefc0ce 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -24,8 +24,16 @@ REGEX_LIMIT: float = 6.9 -def regex_with_timeout(*args, **kwargs): - return asyncio.wait_for(asyncio.to_thread(regex.search, *args, **kwargs), REGEX_LIMIT) +async def regex_with_timeout(*args, **kwargs): + try: + return await asyncio.wait_for(asyncio.to_thread(regex.search, *args, **kwargs), REGEX_LIMIT) + except asyncio.TimeoutError: + raise TimeoutError( + f"A regex timed out after {REGEX_LIMIT} seconds! \n" + f"pattern: ({args[0]}) \n" + f"flags: {kwargs['flags']} \n" + f"on text of length {len(args[1])}" + ) class Crashes(FredCog): @@ -56,7 +64,7 @@ async def parse_factory_game_log(self, text: str) -> dict[str, str | int]: for line in vanilla_info_search_area: if not patterns: break - elif match := regex.search(patterns[0], line): + elif match := await regex_with_timeout(patterns[0], line): info |= match.groupdict() patterns.pop(0) else: @@ -64,7 +72,7 @@ async def parse_factory_game_log(self, text: str) -> dict[str, str | int]: mod_loader_logs = filter(lambda l: regex.match("LogSatisfactoryModLoader", l), lines) for line in mod_loader_logs: - if match := regex.search(r"(?<=v\.)(?P[\d.]+)", line): + if match := await regex_with_timeout(r"(?<=v\.)(?P[\d.]+)", line): info |= match.groupdict() break @@ -189,6 +197,7 @@ async def check_for_outdated_mods(self, mod_list: list) -> list[str]: async def process_file(self, file: IO, extension: str) -> list[dict](): self.logger.info(f"Processing {extension} file") + match extension: case "": return [] @@ -209,7 +218,8 @@ async def process_file(self, file: IO, extension: str) -> list[dict](): return messages + [ dict( name="Bad Zip File!", - value="This zipfile is invalid! Its contents may have been changed after zipping.", + value="This zipfile is invalid! " + "Its contents may have been changed after zipping.", inline=False, ) ] @@ -300,21 +310,8 @@ async def process_file(self, file: IO, extension: str) -> list[dict](): return [] async def mass_regex(self, text: str) -> AsyncIterator[dict]: - - async def silly_async_gen(): - nonlocal text - all_crashes = config.Crashes.fetch_all() - for crash_ in all_crashes: - yield crash_, regex_with_timeout(crash_["crash"], text, flags=regex.IGNORECASE) - - async def patience(coro): - try: - return await coro - except asyncio.TimeoutError: - self.logger.warning("Regex timed out") - - async for crash, queued_regex in silly_async_gen(): - if match := await patience(queued_regex): + for crash in config.Crashes.fetch_all(): + if match := await regex_with_timeout(crash["crash"], text, flags=regex.IGNORECASE): if str(crash["response"]).startswith(self.bot.command_prefix): if command := config.Commands.fetch(crash["response"].strip(self.bot.command_prefix)): command_response = command["content"] From 3aac00457a8d5841ed73899b32c69473b016ad12 Mon Sep 17 00:00:00 2001 From: borketh Date: Tue, 13 Aug 2024 21:09:15 -0400 Subject: [PATCH 34/37] fix(crashes): better "is typing" triggers --- fred/cogs/crashes.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/fred/cogs/crashes.py b/fred/cogs/crashes.py index fefc0ce..4eb049b 100644 --- a/fred/cogs/crashes.py +++ b/fred/cogs/crashes.py @@ -347,10 +347,11 @@ async def process_message(self, message: Message) -> bool: self.logger.info("Found file in message") try: - file = await message.attachments[0].to_file() - file = file.fp - name = message.attachments[0].filename - self.logger.info("Acquired file from discord API") + with message.channel.typing(): + file = await message.attachments[0].to_file() + file = file.fp + name = message.attachments[0].filename + self.logger.info("Acquired file from discord API") except IndexError: self.logger.info("Couldn't acquire file from discord API") try: @@ -358,7 +359,7 @@ async def process_message(self, message: Message) -> bool: att_url = cdn_link.group(1) url_path = urlparse(att_url).path _, name = split(url_path) - async with self.bot.web_session.get(att_url) as response: + async with self.bot.web_session.get(att_url) as response, message.channel.typing(): assert response.status == 200, f"the web request failed with status {response.status}" file = io.BytesIO(await response.read()) except (AttributeError, AssertionError) as e: @@ -369,7 +370,7 @@ async def process_message(self, message: Message) -> bool: extension = name.rpartition(".")[-1] - if name.startswith("SMMDebug") or extension == "log": + if name.startswith("SMMDebug") or extension in ("log", "zip"): async with message.channel.typing(): responses += await self.process_file(file, extension) else: @@ -380,12 +381,12 @@ async def process_message(self, message: Message) -> bool: # Pastebin links elif match := regex.search(r"(https://pastebin.com/\S+)", message.content): url = regex.sub(r"(?<=bin.com)/", "/raw/", match.group(1)) - async with self.bot.web_session.get(url) as response: + async with self.bot.web_session.get(url) as response, message.channel.typing(): text: str = await response.text() - responses += await self.process_text(text) - maybe_log_info = await self.parse_factory_game_log(message.content) - responses += await self.complex_responses(maybe_log_info) + responses += await self.process_text(text) + maybe_log_info = await self.parse_factory_game_log(message.content) + responses += await self.complex_responses(maybe_log_info) else: responses += await self.process_text(message.content) From caaccb0cfb1726899865510b2f9445a5dac3a6f8 Mon Sep 17 00:00:00 2001 From: borketh Date: Tue, 13 Aug 2024 21:20:22 -0400 Subject: [PATCH 35/37] fix: messages that start with prefix then space are not considered commands fixes #108 --- fred/fred.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fred/fred.py b/fred/fred.py index 258bc65..d59c14d 100644 --- a/fred/fred.py +++ b/fred/fred.py @@ -228,7 +228,7 @@ async def reply_yes_or_no(self, message, question, **kwargs): await self.reply_to_msg(message, "Invalid bool string. Aborting") raise ValueError(f"Could not convert {s} to bool") - async def on_message(self, message): + async def on_message(self, message: nextcord.Message): self.logger.info("Processing a message", extra=common.message_info(message)) if (is_bot := message.author.bot) or not self.is_running(): self.logger.info( @@ -251,7 +251,9 @@ async def on_message(self, message): removed = await self.MediaOnly.process_message(message) if not removed: - if message.content.startswith(self.command_prefix): + before, space, after = message.content.partition(" ") + # if the prefix is the only thing before the space then this isn't a command + if before.startswith(self.command_prefix) and len(before) > 1: self.logger.info("Processing commands") await self.process_commands(message) else: From 7ee0fb9040761b23b7eaf8922c2dc1145b2e7da3 Mon Sep 17 00:00:00 2001 From: borketh Date: Tue, 13 Aug 2024 21:24:26 -0400 Subject: [PATCH 36/37] fix: clarity in how crash command responses work fixes #107 --- fred/fred_commands/crashes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fred/fred_commands/crashes.py b/fred/fred_commands/crashes.py index aa307e3..39cc7ee 100644 --- a/fred/fred_commands/crashes.py +++ b/fred/fred_commands/crashes.py @@ -79,8 +79,8 @@ async def modify_crash(self, ctx: commands.Context, name: str.lower, match: str if change_response := await self.bot.reply_yes_or_no(ctx.message, "Do you want to change the response?"): response, _ = await self.bot.reply_question( ctx.message, - "What response do you want it to provide? Responding with " - "a command will make the response that command", + f"What response do you want it to provide? Responding with `{self.bot.command_prefix}command_name`" + "will use the response of that command.", ) except ValueError: return From 612d30a528cff7994ba4f545bd4a9cc2f539c2d1 Mon Sep 17 00:00:00 2001 From: borketh Date: Tue, 13 Aug 2024 21:30:07 -0400 Subject: [PATCH 37/37] fix: ensure aliases are lowercased before doing anything with them fixes #105 --- fred/fred_commands/dbcommands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fred/fred_commands/dbcommands.py b/fred/fred_commands/dbcommands.py index dda1812..80872ff 100644 --- a/fred/fred_commands/dbcommands.py +++ b/fred/fred_commands/dbcommands.py @@ -156,7 +156,7 @@ async def _add_alias(self, ctx: commands.Context, target: str, aliases: list[str return user_info @BaseCmds.add.command(name="alias") - async def add_alias(self, ctx: commands.Context, target: str.lower, *aliases: str): + async def add_alias(self, ctx: commands.Context, target: str.lower, *aliases: str.lower): """Usage: `add alias (command) [aliases...]` Purpose: Adds one or more aliases to a command, checking first for overwriting stuff Notes: If an alias is not supplied you will be prompted for one with a timeout""" @@ -182,7 +182,7 @@ async def add_alias(self, ctx: commands.Context, target: str.lower, *aliases: st if not aliases: response, _ = await self.bot.reply_question(ctx.message, "Please input aliases, separated by spaces.") - aliases = response.split(" ") + aliases = response.lower().split(" ") response = await self._add_alias(ctx, target, aliases)