From 71fffae5f625722e549ff7eb29df440603d58d7b Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Thu, 29 Feb 2024 16:27:34 -0800 Subject: [PATCH 01/15] big refactor to encapsulate user functions in a user class. WIP, but tests pass. --- cli.py | 62 ++++++++++---------- cog_scn.py | 4 +- imap.py | 2 +- netbot.py | 2 +- redmine.py | 147 ++++++++++++++++++++++++++++++++--------------- test_imap.py | 2 +- test_redmine.py | 11 +++- test_synctime.py | 2 +- test_utils.py | 4 +- users.py | 65 +++++++++++++++++++++ 10 files changed, 215 insertions(+), 86 deletions(-) create mode 100644 users.py diff --git a/cli.py b/cli.py index 2a4e1e0..10f40fb 100755 --- a/cli.py +++ b/cli.py @@ -1,12 +1,9 @@ #!/usr/bin/env python3 import io -import sys import logging import datetime as dt -import math -import random -import time + import hashlib import click @@ -28,7 +25,7 @@ # redmine client # load creds into env, and init the redmine client load_dotenv() -redmine_client = redmine.Client() +redmine_client = redmine.Client.fromenv() # figure out what the term refers to # better way? @@ -94,13 +91,13 @@ def print_tickets_md(tickets, fields=["link","status","priority","age","assigned table.add_row(*row) console.print(table) - + buffer = io.StringIO(console.file.getvalue()) - + for line in buffer: print(f"{line.strip()}") - - + + def print_team(team): console = Console() table = Table(show_header=True, box=box.SIMPLE_HEAD, collapse_padding=True, header_style="bold magenta") @@ -280,8 +277,8 @@ def format_ticket(ticket, fields=["link","priority","updated","assigned", "subje def format_ticket_details(ticket): print(ticket) - - + + @click.group() def cli(): """This script showcases different terminal UI helpers in Click.""" @@ -296,38 +293,38 @@ def tickets(query): print_tickets(resolve_query_term(query)) else: print_tickets(redmine_client.my_tickets()) - - + + @cli.command() -@click.argument("id", type=int) -def resolve(id:int): - """Reslove ticket""" +@click.argument("id", type=int) +def resolve(id:int): + """Reslove ticket""" # case "resolve": redmine_client.resolve_ticket(id) print_ticket(redmine_client.get_ticket(id)) - - + + @cli.command() -@click.argument("id", type=int) -def progress(id:int): - """Mark ticket in-progress""" +@click.argument("id", type=int) +def progress(id:int): + """Mark ticket in-progress""" #case "progress": redmine_client.progress_ticket(id) print_ticket(redmine_client.get_ticket(id)) - - + + @cli.command() -@click.argument("id", type=int) -@click.argument("asignee", type=str) +@click.argument("id", type=int) +@click.argument("asignee", type=str) def assign(id:int, asignee:str): """Assign ticket to user""" # case assign redmine_client.assign_ticket(id, asignee) print_ticket(redmine_client.get_ticket(id)) - + @cli.command() -@click.argument("id", type=int) +@click.argument("id", type=int) def unassign(id:int): """Unassign ticket""" # case "unassign": @@ -342,15 +339,15 @@ def teams(): @cli.command() -@click.argument("team", type=str) +@click.argument("team", type=str) def team(team:str): """List team members""" print_team(redmine_client.get_team(team)) @cli.command() -@click.argument("user", type=str) -@click.argument("team", type=str) +@click.argument("user", type=str) +@click.argument("team", type=str) def join(user:str, team:str): """Join a team""" redmine_client.join_team(user, team) @@ -358,8 +355,8 @@ def join(user:str, team:str): @cli.command() -@click.argument("user", type=str) -@click.argument("team", type=str) +@click.argument("user", type=str) +@click.argument("team", type=str) def leave(user:str, team:str): """Leave a team""" redmine_client.leave_team(user, team) @@ -368,4 +365,3 @@ def leave(user:str, team:str): if __name__ == '__main__': cli() - diff --git a/cog_scn.py b/cog_scn.py index a2ce081..5c818f5 100644 --- a/cog_scn.py +++ b/cog_scn.py @@ -95,8 +95,8 @@ async def add(self, ctx:discord.ApplicationContext, redmine_login:str, member:di async def sync_thread(self, thread:discord.Thread): """syncronize an existing ticket thread with redmine""" # get the ticket id from the thread name - # FIXME: notice the series of calls to "self.bot": could be better encapsulated ticket_id = self.bot.parse_thread_title(thread.name) + ticket = self.redmine.get_ticket(ticket_id, include_journals=True) if ticket: completed = await self.bot.synchronize_ticket(ticket, thread) @@ -104,6 +104,8 @@ async def sync_thread(self, thread:discord.Thread): return ticket else: raise NetbotException(f"Ticket {ticket.id} is locked for syncronization.") + else: + log.debug(f"no ticket found for {thread.name}") return None diff --git a/imap.py b/imap.py index 7ec9a12..f607cee 100755 --- a/imap.py +++ b/imap.py @@ -90,7 +90,7 @@ def __init__(self): self.host = os.getenv('IMAP_HOST') self.user = os.getenv('IMAP_USER') self.passwd = os.getenv('IMAP_PASSWORD') - self.redmine = redmine.Client() + self.redmine = redmine.Client.fromenv() # note: not happy with this method of dealing with complex email address # but I don't see a better way. open to suggestions diff --git a/netbot.py b/netbot.py index 35a78d4..9d3960c 100755 --- a/netbot.py +++ b/netbot.py @@ -201,7 +201,7 @@ def main(): log.info(f"loading .env for {__name__}") load_dotenv() - client = redmine.Client() + client = redmine.Client.fromenv() bot = NetBot(client) # register cogs diff --git a/redmine.py b/redmine.py index d25435f..327aa9a 100644 --- a/redmine.py +++ b/redmine.py @@ -12,12 +12,13 @@ from dotenv import load_dotenv import synctime +from users import UserResult, User log = logging.getLogger(__name__) DEFAULT_SORT = "status:desc,priority:desc,updated_on:desc" -TIMEOUT = 2 # seconds +TIMEOUT = 10 # seconds SYNC_FIELD_NAME = "syncdata" DISCORD_ID_FIELD = "Discord ID" BLOCKED_TEAM_NAME = "blocked" @@ -32,14 +33,14 @@ def __init__(self, message: str, request_id: str) -> None: self.request_id = request_id -class Client(): ## redmine.Client() +class Client(): ## redmine.Client.fromenv() """redmine client""" - def __init__(self): - self.url = os.getenv('REDMINE_URL') + def __init__(self, url: str, token: str): + self.url = url if self.url is None: raise RedmineException("Unable to load REDMINE_URL", "[n/a]") - self.token = os.getenv('REDMINE_TOKEN') + self.token = token if self.url is None: raise RedmineException("Unable to load REDMINE_TOKEN", "__init__") @@ -51,6 +52,19 @@ def __init__(self): self.reindex() + @classmethod + def fromenv(cls): + url = os.getenv('REDMINE_URL') + if url is None: + raise RedmineException("Unable to load REDMINE_URL", "[n/a]") + + token = os.getenv('REDMINE_TOKEN') + if token is None: + raise RedmineException("Unable to load REDMINE_TOKEN", "__init__") + + return cls(url, token) + + def create_ticket(self, user, subject, body, attachments=None): """create a redmine ticket""" # https://www.redmine.org/projects/redmine/wiki/Rest_Issues#Creating-an-issue @@ -394,6 +408,7 @@ def find_ticket_from_str(self, string:str): def create_user(self, email:str, first:str, last:str): """create a new redmine user""" + # TODO: Generate JSON from User object data = { 'user': { 'login': email, @@ -411,20 +426,14 @@ def create_user(self, email:str, first:str, last:str): headers=self.get_headers()) # check status - if r.status_code == 201: - root = json.loads(r.text, object_hook= lambda x: SimpleNamespace(**x)) - user = root.user + if r.ok: + user = User(**r.json()['user']) log.info(f"created user: {user.id} {user.login} {user.mail}") - self.reindex_users() # new user! - - # add user to User group and SCN project - - #self.join_project(user.login, "scn") ### scn project key - #log.info("joined scn project") + self.reindex_users() # new user! FIXME reindix or just add? + # add user to User group self.join_team(user.login, "users") ### FIXME move default team name to defaults somewhere - log.info("joined users group") return user elif r.status_code == 403: @@ -680,14 +689,25 @@ def query(self, query_str:str, user:str=None): headers = self.get_headers(user) - r = requests.get(f"{self.url}{query_str}", headers=headers, timeout=TIMEOUT) + # TODO Detect and handle paged results + + try: + r = requests.get(f"{self.url}{query_str}", headers=headers, timeout=TIMEOUT) + + # check 200 status code + if r.ok: + # return the parsed the JSON text + return json.loads(r.text, object_hook=lambda x: SimpleNamespace(**x)) + else: + log.error(f"Status code {r.status_code} for {r.request.url}, reqid={r.headers['X-Request-Id']}: {r}") + except TimeoutError as toe: + # ticket-509: Handle timeout gracefully + log.warning(f"Timeout during {query_str}: {toe}") + except Exception as ex: + log.warning(f"Excetion during {query_str}: {ex}") + + return None - # check 200 status code - if r.status_code == 200: - return json.loads(r.text, object_hook=lambda x: SimpleNamespace(**x)) - else: - log.warning(f"{r.status_code}: {r.request.url}") - return None def assign_ticket(self, ticket_id, target, user_id=None): @@ -844,7 +864,7 @@ def get_discord_id(self, user): if user: for field in user.custom_fields: if field.name == DISCORD_ID_FIELD and field.value and len(field.value) > 0: - log.debug(f"redmine:{user.login} <==> discord:{field.value}") + #log.debug(f"redmine:{user.login} <==> discord:{field.value}") return field.value return None @@ -856,45 +876,82 @@ def is_user_or_group(self, user:str) -> bool: else: return False + + def get_all_users(self): + try: + headers = self.get_headers() + response = requests.get(f"{self.url}/users.json", headers=headers, timeout=10) ## TODO + if response.ok: + #resp_json = response.json() + user_result = UserResult(**response.json()) + + users = user_result.users + + if user_result.total_count > user_result.limit: + offset = user_result.limit + while offset < user_result.total_count: + next_req = f"{self.url}/users.json?offset={offset}&limit={user_result.limit}" + log.debug(f"next request: {next_req}") + next_resp = requests.get(next_req, headers=headers, timeout=10) + next_result = UserResult(**next_resp.json()) + users.extend(next_result.users) + offset += next_result.limit + + return users + else: + log.error(f"Status code {response.status_code} for {response.request.url}, reqid={response.headers['X-Request-Id']}: {response}") + except TimeoutError as toe: + # ticket-509: Handle timeout gracefully + log.warning(f"Timeout during get_all_users: {toe}") + except Exception as ex: + log.exception(f"Exception during get_all_users: {ex}") + + return None + # python method sync? def reindex_users(self): - # reset the indices - self.users.clear() - self.user_ids.clear() - self.user_emails.clear() - self.discord_users.clear() - # rebuild the indicies - response = self.query("/users.json?offset=0&limit=250") ## fixme max limit? paging? - if response.users: - for user in response.users: + # looking over issues in redmine and specifically https://www.redmine.org/issues/16069 + # it seems that redmine has a HARD CODED limit of 100 responses per request. + users = self.get_all_users() + if users: + # reset the indices + self.users.clear() + self.user_ids.clear() + self.user_emails.clear() + self.discord_users.clear() + + for user in users: self.users[user.login] = user.id self.user_ids[user.id] = user self.user_emails[user.mail] = user.id - discord_id = self.get_discord_id(user) - if discord_id: - self.discord_users[discord_id] = user.id - log.debug(f"### indexed {user.login} - discord={discord_id}") + #discord_id = user.get_discord_id(user) + if user.discord_id: + self.discord_users[user.discord_id] = user.id log.debug(f"indexed {len(self.users)} users") log.debug(f"discord users: {self.discord_users}") else: - log.error(f"No users: {response}") + log.error("No users to index") def get_teams(self): return self.groups.keys() + def reindex_groups(self): - # reset the indices - self.groups.clear() + # rebuild the group index + response = self.query("/groups.json") + if response and response.groups: + # reset the indices + self.groups.clear() - # rebuild the indicies - response = self.query("/groups.json?limit=1000") ## FIXME max limit? paging? - for group in response.groups: - self.groups[group.name] = group + for group in response.groups: + self.groups[group.name] = group - log.debug(f"indexed {len(self.groups)} groups") + log.debug(f"indexed {len(self.groups)} groups") + else: + log.error(f"No groups to index: {response}") def is_user_in_team(self, username:str, teamname:str) -> bool: @@ -926,6 +983,6 @@ def reindex(self): load_dotenv() # construct the client and run the email check - client = Client() + client = Client.fromenv() tickets = client.find_tickets() client.format_report(tickets) diff --git a/test_imap.py b/test_imap.py index 1574afc..c6db749 100755 --- a/test_imap.py +++ b/test_imap.py @@ -21,7 +21,7 @@ class TestMessages(unittest.TestCase): """Test suite for IMAP functions""" def setUp(self): - self.redmine = redmine.Client() + self.redmine = redmine.Client.fromenv() self.imap = imap.Client() def test_messages_stripping(self): diff --git a/test_redmine.py b/test_redmine.py index 5a1b795..1a11c8b 100755 --- a/test_redmine.py +++ b/test_redmine.py @@ -18,7 +18,7 @@ class TestRedmine(unittest.TestCase): """Test suite for Redmine client""" def setUp(self): - self.redmine = redmine.Client() + self.redmine = redmine.Client.fromenv() def test_blocked_user(self): @@ -57,6 +57,15 @@ def test_blocked_create_ticket(self): self.redmine.remove_user(user.id) + def test_client_timeout(self): + # construct an invalid client to try to get a timeout + try: + client = redmine.Client("http://192.168.1.42/", "bad-token") + log.info(client) + except TimeoutError: + self.fail("Got unexpected timeout") + + if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format="{asctime} {levelname:<8s} {name:<16} {message}", style='{') logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) diff --git a/test_synctime.py b/test_synctime.py index 6b8bc19..0990158 100755 --- a/test_synctime.py +++ b/test_synctime.py @@ -19,7 +19,7 @@ class TestTime(unittest.TestCase): """testing""" def setUp(self): - self.redmine = redmine.Client() + self.redmine = redmine.Client.fromenv() def test_redmine_times(self): #start = synctime.now() diff --git a/test_utils.py b/test_utils.py index 6c3a675..ab76bd9 100644 --- a/test_utils.py +++ b/test_utils.py @@ -72,7 +72,7 @@ class BotTestCase(unittest.IsolatedAsyncioTestCase): @classmethod def setUpClass(cls): log.info("Setting up test fixtures") - cls.redmine = Client() + cls.redmine = Client.fromenv() cls.usertag = tagstr() cls.user = create_test_user(cls.redmine, cls.usertag) log.info(f"Created test user: {cls.user}") @@ -104,7 +104,7 @@ def setUp(self): self.assertIsNotNone(self.user) self.full_name = self.user.firstname + " " + self.user.lastname - self.discord_user = self.redmine.get_discord_id(self.user) + self.discord_user = self.user.discord_id self.assertIsNotNone(self.redmine.find_user(self.user.login)) self.assertIsNotNone(self.redmine.find_user(self.discord_user)) diff --git a/users.py b/users.py new file mode 100644 index 0000000..99ec414 --- /dev/null +++ b/users.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""redmine client""" + +import datetime as dt +import logging + +from dataclasses import dataclass + + +log = logging.getLogger(__name__) + + +DISCORD_ID_FIELD = "Discord ID" + + +@dataclass +class CustomField(): + """A redmine custom field""" + id: int + name: str + value: str + +@dataclass +class User(): + """Encapsulates a redmine user""" + id: int + login: str + mail: str + custom_fields: dict + admin: bool + firstname: str + lastname: str + mail: str + created_on: dt.datetime + updated_on: dt.datetime + last_login_on: dt.datetime + passwd_changed_on: dt.datetime + twofa_scheme: str + api_key: str = "" + status: int = "" + custom_fields: list[CustomField] + + + def __post_init__(self): + self.custom_fields = [CustomField(**field) for field in self.custom_fields] + self.discord_id = self.get_custom_field(DISCORD_ID_FIELD) + + + def get_custom_field(self, name: str) -> str: + for field in self.custom_fields: + if field.name == name: + return field.value + + return None + +@dataclass +class UserResult: + """Encapsulates a set of users""" + users: list[User] + total_count: int + limit: int + offset: int + + def __post_init__(self): + self.users = [User(**user) for user in self.users] From 2ae6232f9b6b4f559c4b1eeb69016ad8ccf7d183 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Thu, 29 Feb 2024 16:44:56 -0800 Subject: [PATCH 02/15] actually using those cached teams --- redmine.py | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/redmine.py b/redmine.py index 327aa9a..673e85c 100644 --- a/redmine.py +++ b/redmine.py @@ -48,7 +48,7 @@ def __init__(self, url: str, token: str): self.user_ids = {} self.user_emails = {} self.discord_users = {} - self.groups = {} + self.teams = {} self.reindex() @@ -247,13 +247,14 @@ def upload_attachments(self, user_id, attachments): def find_team(self, name): - """find a team by name""" - response = self.query("/groups.json") - for group in response.groups: - if group.name == name: - return group - # not found - return None + return self.teams[name] + #"""find a team by name""" + #response = self.query("/groups.json") + #for group in response.groups: + # if group.name == name: + # return group + ## not found + #return None def create_team(self, teamname:str): @@ -314,8 +315,8 @@ def find_user(self, name): return self.get_user(self.users[name]) elif name in self.discord_users: return self.get_user(self.discord_users[name]) - elif name in self.groups: - return self.groups[name] #ugly. put groups in user collection? + elif name in self.teams: + return self.teams[name] #ugly. put groups in user collection? else: return None @@ -871,7 +872,7 @@ def get_discord_id(self, user): def is_user_or_group(self, user:str) -> bool: if user in self.users: return True - elif user in self.groups: + elif user in self.teams: return True else: return False @@ -880,13 +881,10 @@ def is_user_or_group(self, user:str) -> bool: def get_all_users(self): try: headers = self.get_headers() - response = requests.get(f"{self.url}/users.json", headers=headers, timeout=10) ## TODO + response = requests.get(f"{self.url}/users.json?limit=100", headers=headers, timeout=TIMEOUT) if response.ok: - #resp_json = response.json() user_result = UserResult(**response.json()) - users = user_result.users - if user_result.total_count > user_result.limit: offset = user_result.limit while offset < user_result.total_count: @@ -908,6 +906,7 @@ def get_all_users(self): return None + # python method sync? def reindex_users(self): # rebuild the indicies @@ -936,22 +935,23 @@ def reindex_users(self): def get_teams(self): - return self.groups.keys() + return self.teams.keys() - def reindex_groups(self): + # TODO: Add a dataclass for Team, and page-unrolling for "all teams" + def reindex_teams(self): # rebuild the group index - response = self.query("/groups.json") + response = self.query("/groups.json?limit=100") if response and response.groups: # reset the indices - self.groups.clear() + self.teams.clear() - for group in response.groups: - self.groups[group.name] = group + for team in response.groups: + self.teams[team.name] = team - log.debug(f"indexed {len(self.groups)} groups") + log.debug(f"indexed {len(self.teams)} team") else: - log.error(f"No groups to index: {response}") + log.error(f"No teams to index: {response}") def is_user_in_team(self, username:str, teamname:str) -> bool: @@ -961,7 +961,7 @@ def is_user_in_team(self, username:str, teamname:str) -> bool: user = self.find_user(username) if user: user_id = user.id - team = self.get_team(teamname) # requires an API call, could be cashed? only used for testing + team = self.get_team(teamname) # requires an API call if team: for team_user in team.users: @@ -974,7 +974,7 @@ def is_user_in_team(self, username:str, teamname:str) -> bool: def reindex(self): start = synctime.now() self.reindex_users() - self.reindex_groups() + self.reindex_teams() log.debug(f"reindex took {synctime.age(start)}") From 3b751755d9cd0300c6d01cfd89e1f32ef9d71ec9 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Thu, 29 Feb 2024 17:02:55 -0800 Subject: [PATCH 03/15] making sure everything uses the new User data class --- redmine.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/redmine.py b/redmine.py index 673e85c..1402207 100644 --- a/redmine.py +++ b/redmine.py @@ -248,13 +248,6 @@ def upload_attachments(self, user_id, attachments): def find_team(self, name): return self.teams[name] - #"""find a team by name""" - #response = self.query("/groups.json") - #for group in response.groups: - # if group.name == name: - # return group - ## not found - #return None def create_team(self, teamname:str): @@ -276,6 +269,7 @@ def create_team(self, teamname:str): # check status if response.ok: + self.reindex_teams() log.info(f"OK create_team {teamname}") else: raise RedmineException(f"create_team {teamname} failed", response.headers['X-Request-Id']) @@ -293,16 +287,20 @@ def lookup_user(self, username:str): log.debug("Empty user ID") return None - #response = self.query(f"/users/{user_id}.json") - response = self.query(f"/users.json?name={username}") - - log.debug(f"lookup_user: {username} -> {response.users}") - - if len(response.users) > 0: - return response.users[0] # fragile - else: - log.debug(f"Unknown user: {username}") - return None + response = requests.get(f"{self.url}/users.json?name={username}", + headers=self.get_headers(), timeout=TIMEOUT) + if response.ok: + user_result = UserResult(**response.json()) + log.debug(f"lookup_user: {username} -> {user_result.users}") + + if user_result.total_count == 1: + return user_result.users[0] + elif user_result.total_count > 1: + log.warning(f"Too many results for {username}: {user_result.users}") + return user_result.users[0] + else: + log.debug(f"Unknown user: {username}") + return None def find_user(self, name): From 7ef095cf8d3e7c69b579f46f31646ccb7f38b489 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Fri, 1 Mar 2024 13:51:11 -0800 Subject: [PATCH 04/15] test stability after major refactor. yay! --- cog_scn.py | 16 +- cog_tickets.py | 10 +- imap.py | 2 +- netbot.py | 4 +- redmine.py | 165 +++---------------- session.py | 143 +++++++++++++++++ test_cog_scn.py | 4 +- test_imap.py | 4 +- test_synctime.py | 2 +- test_utils.py | 8 +- users.py | 408 ++++++++++++++++++++++++++++++++++++++++++++++- 11 files changed, 595 insertions(+), 171 deletions(-) create mode 100644 session.py diff --git a/cog_scn.py b/cog_scn.py index 5c818f5..33cd36a 100644 --- a/cog_scn.py +++ b/cog_scn.py @@ -75,12 +75,12 @@ async def add(self, ctx:discord.ApplicationContext, redmine_login:str, member:di log.info(f"Overriding current user={ctx.user.name} with member={member.name}") discord_name = member.name - user = self.redmine.find_discord_user(discord_name) + user = self.redmine.user_cache.find_user(discord_name) if user: await ctx.respond(f"Discord user: {discord_name} is already configured as redmine user: {user.login}") else: - user = self.redmine.find_user(redmine_login) + user = self.redmine.user_cache.find_user(redmine_login) if user: self.redmine.create_discord_mapping(redmine_login, discord_name) await ctx.respond(f"Discord user: {discord_name} has been paired with redmine user: {redmine_login}") @@ -152,7 +152,7 @@ async def sync(self, ctx:discord.ApplicationContext): @scn.command() async def reindex(self, ctx:discord.ApplicationContext): """reindex the user and team information""" - self.redmine.reindex() + self.redmine.user_cache.reindex() await ctx.respond("Rebuilt redmine indices.") @@ -163,10 +163,10 @@ async def join(self, ctx:discord.ApplicationContext, teamname:str , member: disc log.info(f"Overriding current user={ctx.user.name} with member={member.name}") discord_name = member.name - user = self.redmine.find_discord_user(discord_name) + user = self.redmine.user_cache.find_user(discord_name) if user is None: await ctx.respond(f"Unknown user, no Discord mapping: {discord_name}") - elif self.redmine.find_team(teamname) is None: + elif self.redmine.user_cache.find_team(teamname) is None: await ctx.respond(f"Unknown team name: {teamname}") else: self.redmine.join_team(user.login, teamname) @@ -179,7 +179,7 @@ async def leave(self, ctx:discord.ApplicationContext, teamname:str, member: disc if member: log.info(f"Overriding current user={ctx.user.name} with member={member.name}") discord_name = member.name - user = self.redmine.find_discord_user(discord_name) + user = self.redmine.user_cache.find_user(discord_name) if user: self.redmine.leave_team(user.login, teamname) @@ -216,7 +216,7 @@ async def teams(self, ctx:discord.ApplicationContext, teamname:str=None): async def block(self, ctx:discord.ApplicationContext, username:str): log.debug(f"blocking {username}") #user = self.redmine.lookup_user(username) - user = self.redmine.find_user(username) + user = self.redmine.user_cache.find_user(username) if user: # add the user to the blocked list self.redmine.block_user(user) @@ -232,7 +232,7 @@ async def block(self, ctx:discord.ApplicationContext, username:str): @scn.command(description="unblock specific a email address") async def unblock(self, ctx:discord.ApplicationContext, username:str): log.debug(f"unblocking {username}") - user = self.redmine.find_user(username) + user = self.redmine.user_cache.find_user(username) if user: self.redmine.unblock_user(user) await ctx.respond(f"Unblocked user: {user.login}") diff --git a/cog_tickets.py b/cog_tickets.py index b738dd9..1f151ce 100644 --- a/cog_tickets.py +++ b/cog_tickets.py @@ -54,7 +54,7 @@ def resolve_query_term(self, term): return [ticket] except ValueError: # not a numeric id, check team - if self.redmine.is_user_or_group(term): + if self.redmine.user_cache.is_user_or_group(term): return self.redmine.tickets_for_team(term) else: # assume a search term @@ -70,7 +70,7 @@ async def tickets(self, ctx: discord.ApplicationContext, params: str = ""): # lookup the user log.debug(f"looking for user mapping for {ctx}") - user = self.redmine.find_discord_user(ctx.user.name) + user = self.redmine.user_cache.find_user(ctx.user.name) log.debug(f"found user mapping for {ctx.user.name}: {user}") args = params.split() @@ -88,7 +88,7 @@ async def ticket(self, ctx: discord.ApplicationContext, ticket_id:int, action:st """Update status on a ticket, using: unassign, resolve, progress""" try: # lookup the user - user = self.redmine.find_discord_user(ctx.user.name) + user = self.redmine.user_cache.find_user(ctx.user.name) log.debug(f"found user mapping for {ctx.user.name}: {user}") match action: @@ -131,7 +131,7 @@ async def ticket(self, ctx: discord.ApplicationContext, ticket_id:int, action:st @option("title", description="Title of the new SCN ticket") @option("add_thread", description="Create a Discord thread for the new ticket", default=False) async def create_new_ticket(self, ctx: discord.ApplicationContext, title:str): - user = self.redmine.find_discord_user(ctx.user.name) + user = self.redmine.user_cache.find_user(ctx.user.name) if user is None: await ctx.respond(f"Unknown user: {ctx.user.name}") return @@ -166,7 +166,7 @@ async def thread_ticket(self, ctx: discord.ApplicationContext, ticket_id:int): # update the discord flag on tickets, add a note with url of thread; thread.jump_url # TODO message templates note = f"Created Discord thread: {thread.name}: {thread.jump_url}" - user = self.redmine.find_discord_user(ctx.user.name) + user = self.redmine.user_cache.find_discord_user(ctx.user.name) self.redmine.enable_discord_sync(ticket.id, user, note) await ctx.respond(f"Created new thread for {ticket.id}: {thread}") # todo add some fancy formatting diff --git a/imap.py b/imap.py index f607cee..f042cbb 100755 --- a/imap.py +++ b/imap.py @@ -232,7 +232,7 @@ def handle_message(self, msg_id:str, message:Message): ticket = self.redmine.find_ticket_from_str(subject) # get user id from from_address - user = self.redmine.find_user(addr) + user = self.redmine.user_cache.find_user(addr) if user is None: log.debug(f"Unknown email address, no user found: {addr}, {message.from_address}") # create new user diff --git a/netbot.py b/netbot.py index 9d3960c..136938a 100755 --- a/netbot.py +++ b/netbot.py @@ -62,7 +62,7 @@ async def on_message(self, message:discord.Message): # IS a thread, check the name ticket_id = self.parse_thread_title(message.channel.name) if ticket_id > 0: - user = self.redmine.find_discord_user(message.author.name) + user = self.redmine.user_cache.find_user(message.author.name) if user: log.debug(f"known user commenting on ticket #{ticket_id}: redmine={user.login}, discord={message.author.name}") else: @@ -112,7 +112,7 @@ def append_redmine_note(self, ticket, message: discord.Message) -> None: # redmine link format: "Link Text":http://whatever # check user mapping exists - user = self.redmine.find_discord_user(message.author.name) + user = self.redmine.user_cache.find_user(message.author.name) if user: # format the note formatted = f'"Discord":{message.jump_url}: {message.content}' diff --git a/redmine.py b/redmine.py index 1402207..2e96179 100644 --- a/redmine.py +++ b/redmine.py @@ -12,7 +12,8 @@ from dotenv import load_dotenv import synctime -from users import UserResult, User +from session import RedmineSession +from users import UserResult, User, UserManager, UserCache log = logging.getLogger(__name__) @@ -44,12 +45,9 @@ def __init__(self, url: str, token: str): if self.url is None: raise RedmineException("Unable to load REDMINE_TOKEN", "__init__") - self.users = {} - self.user_ids = {} - self.user_emails = {} - self.discord_users = {} - self.teams = {} - self.reindex() + session = RedmineSession(url, token) + self.user_mgr = UserManager(session) + self.user_cache = UserCache(self.user_mgr) @classmethod @@ -246,9 +244,6 @@ def upload_attachments(self, user_id, attachments): a.set_token(token) - def find_team(self, name): - return self.teams[name] - def create_team(self, teamname:str): if teamname is None or len(teamname.strip()) == 0: @@ -269,16 +264,14 @@ def create_team(self, teamname:str): # check status if response.ok: - self.reindex_teams() + self.user_cache.reindex_teams() log.info(f"OK create_team {teamname}") else: raise RedmineException(f"create_team {teamname} failed", response.headers['X-Request-Id']) def get_user(self, user_id:int): - """get a user by ID""" - if user_id: - return self.user_ids[user_id] + return self.user_cache.get_user(user_id) def lookup_user(self, username:str): @@ -303,33 +296,6 @@ def lookup_user(self, username:str): return None - def find_user(self, name): - """find a user by name""" - # check if name is int, raw user id. then look up in userids - # check the indicies - if name in self.user_emails: - return self.get_user(self.user_emails[name]) - elif name in self.users: - return self.get_user(self.users[name]) - elif name in self.discord_users: - return self.get_user(self.discord_users[name]) - elif name in self.teams: - return self.teams[name] #ugly. put groups in user collection? - else: - return None - - def find_discord_user(self, discord_user_id:str): - """find a user by their discord ID""" - if discord_user_id is None: - return None - - if discord_user_id in self.discord_users: - user_id = self.discord_users[discord_user_id] - return self.user_ids[user_id] - else: - return None - - def is_user_blocked(self, user) -> bool: if self.is_user_in_team(user.login, BLOCKED_TEAM_NAME): return True @@ -339,7 +305,7 @@ def is_user_blocked(self, user) -> bool: def block_user(self, user) -> None: # check if the blocked team exists - blocked_team = self.find_team(BLOCKED_TEAM_NAME) + blocked_team = self.user_cache.find_team(BLOCKED_TEAM_NAME) if blocked_team is None: # create blocked team self.create_team(BLOCKED_TEAM_NAME) @@ -429,7 +395,7 @@ def create_user(self, email:str, first:str, last:str): user = User(**r.json()['user']) log.info(f"created user: {user.id} {user.login} {user.mail}") - self.reindex_users() # new user! FIXME reindix or just add? + self.user_cache.reindex_users() # new user! FIXME reindix or just add? # add user to User group self.join_team(user.login, "users") ### FIXME move default team name to defaults somewhere @@ -437,7 +403,7 @@ def create_user(self, email:str, first:str, last:str): return user elif r.status_code == 403: # can't create existing user. log err, but return from cache - user = self.find_user(email) + user = self.user_mgr.search(email) log.error(f"Trying to create existing user email: email={email}, user={user}") return user else: @@ -479,7 +445,7 @@ def remove_ticket(self, ticket_id:int): def most_recent_ticket_for(self, email): """get the most recent ticket for the user with the given email""" # get the user record for the email - user = self.find_user(email) + user = self.user_mgr.search(email) if user: # query open tickets created by user, sorted by most recently updated, limit 1 @@ -532,7 +498,7 @@ def my_tickets(self, user=None): def tickets_for_team(self, team_str:str): # validate team? - team = self.find_user(team_str) # find_user is dsigned to be broad + team = self.user_mgr.search(team_str) # find_user is dsigned to be broad query = f"/issues.json?assigned_to_id={team.id}&status_id=open&sort={DEFAULT_SORT}&limit=100" response = self.query(query) @@ -605,7 +571,7 @@ def enable_discord_sync(self, ticket_id, user, note): def create_discord_mapping(self, redmine_login:str, discord_name:str): - user = self.find_user(redmine_login) + user = self.user_mgr.search(redmine_login) field_id = 2 ## "Discord ID"search for me in cached custom fields fields = { @@ -619,12 +585,12 @@ def create_discord_mapping(self, redmine_login:str, discord_name:str): def join_team(self, username, teamname:str) -> None: # look up user ID - user = self.find_user(username) + user = self.user_mgr.search(username) if user is None: raise RedmineException(f"Unknown user name: {username}", "[n/a]") # map teamname to team - team = self.find_team(teamname) + team = self.user_cache.find_team(teamname) if team is None: raise RedmineException(f"Unknown team name: {teamname}", "[n/a]") @@ -648,13 +614,13 @@ def join_team(self, username, teamname:str) -> None: def leave_team(self, username:int, teamname:str): # look up user ID - user = self.find_user(username) + user = self.user_mgr.search(username) if user is None: log.warning(f"Unknown user name: {username}") return None # map teamname to team - team = self.find_team(teamname) + team = self.user_cache.find_team(teamname) if team is None: log.warning(f"Unknown team name: {teamname}") return None @@ -710,7 +676,7 @@ def query(self, query_str:str, user:str=None): def assign_ticket(self, ticket_id, target, user_id=None): - user = self.find_user(target) + user = self.user_mgr.search(target) if user: fields = { "assigned_to_id": user.id, @@ -753,7 +719,7 @@ def resolve_ticket(self, ticket_id, user_id=None): def get_team(self, teamname:str): - team = self.find_team(teamname) + team = self.user_cache.find_team(teamname) if team is None: log.debug(f"Unknown team name: {teamname}") return None @@ -867,96 +833,12 @@ def get_discord_id(self, user): return field.value return None - def is_user_or_group(self, user:str) -> bool: - if user in self.users: - return True - elif user in self.teams: - return True - else: - return False - - - def get_all_users(self): - try: - headers = self.get_headers() - response = requests.get(f"{self.url}/users.json?limit=100", headers=headers, timeout=TIMEOUT) - if response.ok: - user_result = UserResult(**response.json()) - users = user_result.users - if user_result.total_count > user_result.limit: - offset = user_result.limit - while offset < user_result.total_count: - next_req = f"{self.url}/users.json?offset={offset}&limit={user_result.limit}" - log.debug(f"next request: {next_req}") - next_resp = requests.get(next_req, headers=headers, timeout=10) - next_result = UserResult(**next_resp.json()) - users.extend(next_result.users) - offset += next_result.limit - - return users - else: - log.error(f"Status code {response.status_code} for {response.request.url}, reqid={response.headers['X-Request-Id']}: {response}") - except TimeoutError as toe: - # ticket-509: Handle timeout gracefully - log.warning(f"Timeout during get_all_users: {toe}") - except Exception as ex: - log.exception(f"Exception during get_all_users: {ex}") - - return None - - - # python method sync? - def reindex_users(self): - # rebuild the indicies - # looking over issues in redmine and specifically https://www.redmine.org/issues/16069 - # it seems that redmine has a HARD CODED limit of 100 responses per request. - users = self.get_all_users() - if users: - # reset the indices - self.users.clear() - self.user_ids.clear() - self.user_emails.clear() - self.discord_users.clear() - - for user in users: - self.users[user.login] = user.id - self.user_ids[user.id] = user - self.user_emails[user.mail] = user.id - - #discord_id = user.get_discord_id(user) - if user.discord_id: - self.discord_users[user.discord_id] = user.id - log.debug(f"indexed {len(self.users)} users") - log.debug(f"discord users: {self.discord_users}") - else: - log.error("No users to index") - - - def get_teams(self): - return self.teams.keys() - - - # TODO: Add a dataclass for Team, and page-unrolling for "all teams" - def reindex_teams(self): - # rebuild the group index - response = self.query("/groups.json?limit=100") - if response and response.groups: - # reset the indices - self.teams.clear() - - for team in response.groups: - self.teams[team.name] = team - - log.debug(f"indexed {len(self.teams)} team") - else: - log.error(f"No teams to index: {response}") - def is_user_in_team(self, username:str, teamname:str) -> bool: if username is None or teamname is None: return False - user = self.find_user(username) + user = self.user_mgr.search(username) if user: user_id = user.id team = self.get_team(teamname) # requires an API call @@ -969,13 +851,6 @@ def is_user_in_team(self, username:str, teamname:str) -> bool: return False - def reindex(self): - start = synctime.now() - self.reindex_users() - self.reindex_teams() - log.debug(f"reindex took {synctime.age(start)}") - - if __name__ == '__main__': # load credentials load_dotenv() diff --git a/session.py b/session.py new file mode 100644 index 0000000..a8a6603 --- /dev/null +++ b/session.py @@ -0,0 +1,143 @@ + +#!/usr/bin/env python3 +"""redmine client""" + +import os +import logging + +import requests + + +log = logging.getLogger(__name__) + + +TIMEOUT = 10 # seconds + + +class RedmineException(Exception): + """redmine exception""" + def __init__(self, message: str, request_id: str) -> None: + super().__init__(message + ", req_id=" + request_id) + self.request_id = request_id + + +class RedmineSession(): + """RedmineSession""" + url: str + token: str + session: requests.Session + + """redmine session""" + def __init__(self, url: str, token: str): + self.url = url + self.token = token + self.session = requests.Session() + + + @classmethod + def fromenv(cls): + url = os.getenv('REDMINE_URL') + if url is None: + raise RedmineException("REDMINE_URL not set in environment", __name__) + + token = os.getenv('REDMINE_TOKEN') + if token is None: + raise RedmineException("Unable to load REDMINE_TOKEN", "__init__") + + return cls(url, token) + + + def get_headers(self, impersonate_id:str=None): + headers = { + 'User-Agent': 'netbot/0.0.1', # TODO update to project version, and add version management + 'Content-Type': 'application/json', + 'X-Redmine-API-Key': self.token, + } + # insert the impersonate_id to impersonate another user + if impersonate_id: + headers['X-Redmine-Switch-User'] = impersonate_id + log.debug(f"setting redmine impersonation flag for user={impersonate_id}") + + return headers + + + def get(self, query_str:str, user:str=None): + """run a query against a redmine instance""" + + headers = self.get_headers(user) + + try: + r = self.session.get(f"{self.url}{query_str}", headers=headers, timeout=TIMEOUT) + + if r.ok: + return r.json() + else: + log.error(f"Status code {r.status_code} for {r.request.url}, reqid={r.headers['X-Request-Id']}: {r}") + except TimeoutError as toe: + # ticket-509: Handle timeout gracefully + log.warning(f"Timeout during {query_str}: {toe}") + except Exception as ex: + log.warning(f"Excetion during {query_str}: {ex}") + + return None + + + # data=json.dumps(data), + def put(self, resource: str, data, user_login: str = None): + r = self.session.put(f"{self.url}{resource}", data=data, timeout=TIMEOUT, + headers=self.get_headers(user_login)) + if r.ok: + return r.json() + else: + raise RedmineException(f"POST failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) + + + def post(self, resource: str, data, user_login: str = None): + r = self.session.post(f"{self.url}{resource}", data=data, timeout=TIMEOUT, + headers=self.get_headers(user_login)) + if r.ok: + return r.json() + else: + raise RedmineException(f"POST failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) + + + def delete(self, resource: str) -> bool: + r = self.session.delete( + url=f"{self.url}{resource}", + timeout=TIMEOUT, + headers=self.get_headers()) + + if r.ok: + return True + else: + raise RedmineException(f"DELETE failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) + + + def upload_file(self, user_id, data, filename, content_type): + """Upload a file to redmine""" + # POST /uploads.json?filename=image.png + # Content-Type: application/octet-stream + # (request body is the file content) + + headers = { + 'User-Agent': 'netbot/0.0.1', # TODO update to project version, and add version management + 'Content-Type': 'application/octet-stream', # <-- VERY IMPORTANT + 'X-Redmine-API-Key': self.token, + 'X-Redmine-Switch-User': user_id, # Make sure the comment is noted by the correct user + } + + r = self.session.post( + url=f"{self.url}/uploads.json?filename={filename}", + timeout=TIMEOUT, + files={ 'upload_file': (filename, data, content_type) }, + headers=headers) + + # 201 response: {"upload":{"token":"7167.ed1ccdb093229ca1bd0b043618d88743"}} + if r.ok: + # all good, get token + #root = json.loads(r.text, object_hook= lambda x: SimpleNamespace(**x)) + token = r.json()['upload']['token'] + log.info(f"Uploaded {filename} {content_type}, got token={token}") + return token + else: + raise RedmineException(f"UPLOAD failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) diff --git a/test_cog_scn.py b/test_cog_scn.py index f3af37b..e295696 100755 --- a/test_cog_scn.py +++ b/test_cog_scn.py @@ -49,8 +49,8 @@ async def test_team_join_leave(self): # 4.5 check reindex result, and lookup based on login and discord id ctx.respond.assert_called_with("Rebuilt redmine indices.") - self.assertIsNotNone(self.redmine.find_user(self.user.login)) - self.assertIsNotNone(self.redmine.find_user(self.discord_user)) + self.assertIsNotNone(self.redmine.user_cache.find_user(self.user.login)) + self.assertIsNotNone(self.redmine.user_cache.find_user(self.discord_user)) # join team users ctx = self.build_context() diff --git a/test_imap.py b/test_imap.py index c6db749..86891c7 100755 --- a/test_imap.py +++ b/test_imap.py @@ -130,7 +130,7 @@ def test_new_account_from_email(self): def test_subject_search(self): # create a new ticket with unique subject tag = test_utils.tagstr() - user = self.redmine.find_user("philion") # FIXME: create a relaible test_user + user = self.redmine.user_cache.find_user("philion") # FIXME: create a relaible test_user self.assertIsNotNone(user) subject = f"New ticket with unique marker {tag}" ticket = self.redmine.create_ticket(user, subject, f"This for {self.id}-{tag}") @@ -158,7 +158,7 @@ def test_ticket_query(self): # create a ticket with the tag in the body, not the subject tag = test_utils.tagstr() - user = self.redmine.find_user("admin") + user = self.redmine.user_cache.find_user("admin") self.assertIsNotNone(user) body = f"Body with {self.id} and {tag}" ticket = self.redmine.create_ticket(user, "Boring test ticket", body) diff --git a/test_synctime.py b/test_synctime.py index 0990158..3183d58 100755 --- a/test_synctime.py +++ b/test_synctime.py @@ -26,7 +26,7 @@ def test_redmine_times(self): # create a new ticket with unique subject tag = test_utils.tagstr() - user = self.redmine.find_user("philion") # FIXME: create a relaible test_user + user = self.redmine.user_cache.find_user("philion") # FIXME: create a relaible test_user self.assertIsNotNone(user) subject = f"TEST ticket {tag}" ticket = self.redmine.create_ticket(user, subject, f"This for {self.id}-{tag}") diff --git a/test_utils.py b/test_utils.py index ab76bd9..ce6653e 100644 --- a/test_utils.py +++ b/test_utils.py @@ -59,8 +59,8 @@ def create_test_user(redmine:Client, tag:str): redmine.create_discord_mapping(user.login, discord_user) # reindex users and lookup based on login - redmine.reindex_users() - return redmine.find_user(user.login) + redmine.user_cache.reindex_users() + return redmine.user_cache.find_user(user.login) class BotTestCase(unittest.IsolatedAsyncioTestCase): @@ -106,7 +106,7 @@ def setUp(self): self.full_name = self.user.firstname + " " + self.user.lastname self.discord_user = self.user.discord_id - self.assertIsNotNone(self.redmine.find_user(self.user.login)) - self.assertIsNotNone(self.redmine.find_user(self.discord_user)) + self.assertIsNotNone(self.redmine.user_cache.find_user(self.user.login)) + self.assertIsNotNone(self.redmine.user_cache.find_user(self.discord_user)) log.debug(f"setUp user {self.user.login} {self.discord_user}") diff --git a/users.py b/users.py index 99ec414..df7f809 100644 --- a/users.py +++ b/users.py @@ -3,14 +3,18 @@ import datetime as dt import logging +import json from dataclasses import dataclass +from session import RedmineSession, RedmineException log = logging.getLogger(__name__) - +USER_RESOURCE = "/users.json" +TEAM_RESOURCE = "/groups.json" DISCORD_ID_FIELD = "Discord ID" +BLOCKED_TEAM_NAME = "blocked" @dataclass @@ -53,6 +57,7 @@ def get_custom_field(self, name: str) -> str: return None + @dataclass class UserResult: """Encapsulates a set of users""" @@ -63,3 +68,404 @@ class UserResult: def __post_init__(self): self.users = [User(**user) for user in self.users] + + +@dataclass +class Team: + """Encapsulates a team""" + id: int + name: str + + +class UserManager(): + """manage redmine users""" + session: RedmineSession + + + def __init__(self, session: RedmineSession): + self.session = session + + + def get_all(self) -> list[User]: + jresp = self.session.get(f"{USER_RESOURCE}?limit=100") + if jresp: + user_result = UserResult(**jresp) + user_buffer = user_result.users + if user_result.total_count > user_result.limit: + offset = user_result.limit + while offset < user_result.total_count: + next_req = f"{USER_RESOURCE}?offset={offset}&limit={user_result.limit}" + log.debug(f"next request: {next_req}") + next_resp = self.session.get(next_req) + next_result = UserResult(**next_resp) + user_buffer.extend(next_result.users) + offset += next_result.limit + + return user_buffer + else: + log.warning("No users from get_all_users") + return [] + + + def update(self, user:User, fields:dict): + """update a user record in redmine""" + # PUT a simple JSON structure + data = {} + data['user'] = fields + + response = self.session.put(f"/users/{user.id}.json",json.dumps(data)) + + log.debug(f"update user: [{response.status_code}] {response.request.url}, fields: {fields}") + + # check status + if response.ok: + # TODO get and return the updated user + return user + else: + raise RedmineException( + f"update failed, status=[{response.status_code}] {response.reason}", + response.headers['X-Request-Id']) + + + + def search(self, username:str) -> User: + """Get a user based on ID, directly from redmine""" + if username is None or len(username) == 0: + log.debug("Empty user ID") + return None + + response = self.session.get(f"{USER_RESOURCE}?name={username}") + if response: + result = UserResult(**response) + log.debug(f"lookup_user: {username} -> {result.users}") + + if result.total_count == 1: + return result.users[0] + elif result.total_count > 1: + log.warning(f"Too many results for {username}: {result.users}") + return result.users[0] + else: + log.debug(f"Unknown user: {username}") + return None + + + def is_blocked(self, user) -> bool: + if self.is_user_in_team(user.login, BLOCKED_TEAM_NAME): + return True + else: + return False + + + def block(self, user) -> None: + # check if the blocked team exists + blocked_team = self.find_team(BLOCKED_TEAM_NAME) + if blocked_team is None: + # create blocked team + self.create_team(BLOCKED_TEAM_NAME) + + self.join_team(user.login, BLOCKED_TEAM_NAME) + + + def unblock(self, user) -> None: + self.leave_team(user.login, BLOCKED_TEAM_NAME) + + + def create(self, email:str, first:str, last:str): + """create a new redmine user""" + # TODO: Generate JSON from User object + data = { + 'user': { + 'login': email, + 'firstname': first, + 'lastname': last, + 'mail': email, + } + } + # on create, assign watcher: sender? + + r = self.session.post(USER_RESOURCE, json.dumps(data)) + + # check status + if r: + user = User(**r['user']) + + log.info(f"created user: {user.id} {user.login} {user.mail}") + + return user + else: + raise RedmineException(f"create_user {email} failed", r.headers['X-Request-Id']) + + + # used only in testing + def remove(self, user: User): + """remove user frmo redmine. used for testing""" + # DELETE to /users/{user_id}.json + r = self.session.delete(f"/users/{user.id}.json") + + # check status + if r: + log.info(f"deleted user {user.id}") + else: + log.error(f"Error removing user status={r.status_code}, url={r.request.url}, req_id={r.headers['X-Request-Id']}") + # exception? + + + def create_discord_mapping(self, redmine_login:str, discord_name:str): + user = self.search(redmine_login) + + field_id = 2 ## "Discord ID"search for me in cached custom fields + fields = { + "custom_fields": [ + { "id": field_id, "value": discord_name } # cf_4, custom field syncdata + ] + } + self.update(user, fields) + + + def get_all_teams(self) -> dict: + # this needs to be cached! + resp = self.session.get(f"{TEAM_RESOURCE}?limit=100") + # list of id, name + if resp: + teams = {} + for team in resp['groups']: + teams[team['name']] = Team(**team) + + return teams + else: + log.warning("No users from get_all_users") + return [] + + + def find_team(self, name:str) -> int: + return self.get_all_teams().get(name, None) + + + def create_team(self, teamname:str): + if teamname is None or len(teamname.strip()) == 0: + raise RedmineException(f"Invalid team name: '{teamname}'", __name__) + + # POST to /groups.json + data = { + "group": { + "name": teamname, + } + } + + response = self.session.post(TEAM_RESOURCE, json.dumps(data)) + + # check status + if response: + log.info(f"OK create_team {teamname}") + else: + raise RedmineException(f"create_team {teamname} failed", response.headers['X-Request-Id']) + + + def join_team(self, user: User, teamname:str) -> None: + # look up user ID + #user = self.find_user(username) + #if user is None: + # raise RedmineException(f"Unknown user name: {username}", "[n/a]") + + # map teamname to team + team_id = self.find_team(teamname) + if team_id is None: + raise RedmineException(f"Unknown team name: {teamname}", "[n/a]") + + # POST to /group/ID/users.json + data = { + "user_id": user.id + } + + response = self.session.post(f"/groups/{team_id}/users.json", data=json.dumps(data)) + + # check status + if response.ok: + log.info(f"OK join_team {user.login}, {teamname}") + else: + raise RedmineException(f"join_team failed, status=[{response.status_code}] {response.reason}", response.headers['X-Request-Id']) + + + def leave_team(self, user: User, teamname:str): + # look up user ID + #user = self.find_user(username) + #if user is None: + # log.warning(f"Unknown user name: {username}") + # return None + + # map teamname to team + team_id = self.find_team(teamname) + if team_id is None: + log.warning(f"Unknown team name: {teamname}") + return + + # DELETE to /groups/{team-id}/users/{user_id}.json + r = self.session.delete(f"/groups/{team_id}/users/{user.id}.json") + + # check status + if not r: + log.error(f"Error removing {user.login} from {teamname}") + + + def get_team(self, teamname:str): + team_id = self.find_team(teamname) + if team_id is None: + log.debug(f"Unknown team name: {teamname}") + return None + + # as per https://www.redmine.org/projects/redmine/wiki/Rest_Groups#GET-2 + # GET /groups/20.json?include=users + response = self.session.get(f"/groups/{team_id}.json?include=users") + if response: + return response.group + else: + #TODO exception? + return None + + + def is_user_in_team(self, user: User, teamname:str) -> bool: + if user is None or teamname is None: + return False + + team = self.get_team(teamname) + + if team: + for team_user in team.users: + if team_user.id == user.id: + return True + + return False + + +class UserCache(): + """cache of user data""" + def __init__(self, mgr:UserManager): + self.mgr = mgr + + self.users = {} + self.user_ids = {} + self.user_emails = {} + self.discord_users = {} + self.teams = {} + self.reindex() + + + def get_user(self, user_id:int): + """get a user by ID""" + if user_id: + return self.user_ids[user_id] + + + def find_user(self, name): + """find a user by name""" + # check if name is int, raw user id. then look up in userids + # check the indicies + if name in self.user_emails: + return self.get_user(self.user_emails[name]) + elif name in self.users: + return self.get_user(self.users[name]) + elif name in self.discord_users: + return self.get_user(self.discord_users[name]) + elif name in self.teams: + return self.teams[name] #ugly. put groups in user collection? + else: + return None + + + def find_team(self, name:str) -> int: + if name in self.teams: + return self.teams[name] + + + def find_discord_user(self, discord_user_id:str): + """find a user by their discord ID""" + if discord_user_id is None: + return None + + if discord_user_id in self.discord_users: + user_id = self.discord_users[discord_user_id] + return self.user_ids[user_id] + else: + return None + + + def is_user_or_group(self, name:str) -> bool: + if name in self.users: + return True + elif name in self.teams: + return True + else: + return False + + + # python method sync? + def reindex_users(self): + # rebuild the indicies + # looking over issues in redmine and specifically https://www.redmine.org/issues/16069 + # it seems that redmine has a HARD CODED limit of 100 responses per request. + all_users = self.mgr.get_all() + if all_users: + # reset the indices + self.users.clear() + self.user_ids.clear() + self.user_emails.clear() + self.discord_users.clear() + + for user in all_users: + self.users[user.login] = user.id + self.user_ids[user.id] = user + self.user_emails[user.mail] = user.id + + #discord_id = user.get_discord_id(user) + if user.discord_id: + self.discord_users[user.discord_id] = user.id + log.debug(f"indexed {len(self.users)} users") + log.debug(f"discord users: {self.discord_users}") + else: + log.error("No users to index") + + + def get_teams(self): + return self.teams.keys() + + + # TODO: Add a dataclass for Team, and page-unrolling for "all teams" + def reindex_teams(self): + # rebuild the group index + self.teams = self.mgr.get_all_teams() + + + def is_user_in_team(self, username:str, teamname:str) -> bool: + if username is None or teamname is None: + return False + + user = self.mgr.search(username) + if user: + user_id = user.id + team = self.mgr.get_team(teamname) # requires an API call + + if team: + for team_user in team.users: + if team_user.id == user_id: + return True + + return False + + + def reindex(self): + start = dt.datetime.now() + self.reindex_users() + self.reindex_teams() + log.debug(f"reindex took {dt.datetime.now() - start}") + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format="{asctime} {levelname:<8s} {name:<16} {message}", style='{') + logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) + + # load credentials + from dotenv import load_dotenv + load_dotenv() + + users = UserManager(RedmineSession.fromenv()) + print(len(users.get_all())) From 0490e2c443fddfd009b1b55a5016b74f7c6d52ca Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Sat, 2 Mar 2024 12:46:12 -0800 Subject: [PATCH 05/15] once again, test stability after users refactor. still more clean up to do. --- cli.py | 6 +- cog_scn.py | 18 +- cog_tickets.py | 10 +- imap.py | 6 +- netbot.py | 4 +- redmine.py | 254 ++------------------------- requirements.txt | 1 + session.py | 14 +- test_cog_scn.py | 13 +- test_imap.py | 24 +-- test_redmine.py | 16 +- test_synctime.py | 2 +- test_utils.py | 53 ++++-- users.py | 436 ++++++++++++++++++++++++++--------------------- 14 files changed, 359 insertions(+), 498 deletions(-) diff --git a/cli.py b/cli.py index 10f40fb..b3ab069 100755 --- a/cli.py +++ b/cli.py @@ -139,9 +139,9 @@ def hash_color(value): # consistently-hash the value into a color # hash_val = hash(value) <-- this does it inconsistantly (for security reasons) hash_val = int(hashlib.md5(value.encode('utf-8')).hexdigest(), 16) - r = (hash_val & 0xFF0000) >> 16; - g = (hash_val & 0x00FF00) >> 8; - b = hash_val & 0x0000FF; + r = (hash_val & 0xFF0000) >> 16 + g = (hash_val & 0x00FF00) >> 8 + b = hash_val & 0x0000FF return f"rgb({r},{g},{b})" def get_formatted_field(ticket, field): diff --git a/cog_scn.py b/cog_scn.py index 33cd36a..0f597c6 100644 --- a/cog_scn.py +++ b/cog_scn.py @@ -75,12 +75,12 @@ async def add(self, ctx:discord.ApplicationContext, redmine_login:str, member:di log.info(f"Overriding current user={ctx.user.name} with member={member.name}") discord_name = member.name - user = self.redmine.user_cache.find_user(discord_name) + user = self.redmine.user_mgr.find(discord_name) if user: await ctx.respond(f"Discord user: {discord_name} is already configured as redmine user: {user.login}") else: - user = self.redmine.user_cache.find_user(redmine_login) + user = self.redmine.user_mgr.find(redmine_login) if user: self.redmine.create_discord_mapping(redmine_login, discord_name) await ctx.respond(f"Discord user: {discord_name} has been paired with redmine user: {redmine_login}") @@ -152,7 +152,7 @@ async def sync(self, ctx:discord.ApplicationContext): @scn.command() async def reindex(self, ctx:discord.ApplicationContext): """reindex the user and team information""" - self.redmine.user_cache.reindex() + self.redmine.user_mgr.reindex() await ctx.respond("Rebuilt redmine indices.") @@ -163,10 +163,10 @@ async def join(self, ctx:discord.ApplicationContext, teamname:str , member: disc log.info(f"Overriding current user={ctx.user.name} with member={member.name}") discord_name = member.name - user = self.redmine.user_cache.find_user(discord_name) + user = self.redmine.user_mgr.find(discord_name) if user is None: await ctx.respond(f"Unknown user, no Discord mapping: {discord_name}") - elif self.redmine.user_cache.find_team(teamname) is None: + elif self.redmine.user_mgr.get_team_by_name(teamname) is None: await ctx.respond(f"Unknown team name: {teamname}") else: self.redmine.join_team(user.login, teamname) @@ -179,7 +179,7 @@ async def leave(self, ctx:discord.ApplicationContext, teamname:str, member: disc if member: log.info(f"Overriding current user={ctx.user.name} with member={member.name}") discord_name = member.name - user = self.redmine.user_cache.find_user(discord_name) + user = self.redmine.user_mgr.find(discord_name) if user: self.redmine.leave_team(user.login, teamname) @@ -216,10 +216,10 @@ async def teams(self, ctx:discord.ApplicationContext, teamname:str=None): async def block(self, ctx:discord.ApplicationContext, username:str): log.debug(f"blocking {username}") #user = self.redmine.lookup_user(username) - user = self.redmine.user_cache.find_user(username) + user = self.redmine.user_mgr.find(username) if user: # add the user to the blocked list - self.redmine.block_user(user) + self.redmine.user_mgr.block(user) # search and reject all tickets from that user for ticket in self.redmine.get_tickets_by(user): self.redmine.reject_ticket(ticket.id) @@ -232,7 +232,7 @@ async def block(self, ctx:discord.ApplicationContext, username:str): @scn.command(description="unblock specific a email address") async def unblock(self, ctx:discord.ApplicationContext, username:str): log.debug(f"unblocking {username}") - user = self.redmine.user_cache.find_user(username) + user = self.redmine.user_mgr.find(username) if user: self.redmine.unblock_user(user) await ctx.respond(f"Unblocked user: {user.login}") diff --git a/cog_tickets.py b/cog_tickets.py index 1f151ce..69c2879 100644 --- a/cog_tickets.py +++ b/cog_tickets.py @@ -54,7 +54,7 @@ def resolve_query_term(self, term): return [ticket] except ValueError: # not a numeric id, check team - if self.redmine.user_cache.is_user_or_group(term): + if self.redmine.user_mgr.is_user_or_group(term): return self.redmine.tickets_for_team(term) else: # assume a search term @@ -70,7 +70,7 @@ async def tickets(self, ctx: discord.ApplicationContext, params: str = ""): # lookup the user log.debug(f"looking for user mapping for {ctx}") - user = self.redmine.user_cache.find_user(ctx.user.name) + user = self.redmine.user_mgr.find(ctx.user.name) log.debug(f"found user mapping for {ctx.user.name}: {user}") args = params.split() @@ -88,7 +88,7 @@ async def ticket(self, ctx: discord.ApplicationContext, ticket_id:int, action:st """Update status on a ticket, using: unassign, resolve, progress""" try: # lookup the user - user = self.redmine.user_cache.find_user(ctx.user.name) + user = self.redmine.user_mgr.find(ctx.user.name) log.debug(f"found user mapping for {ctx.user.name}: {user}") match action: @@ -131,7 +131,7 @@ async def ticket(self, ctx: discord.ApplicationContext, ticket_id:int, action:st @option("title", description="Title of the new SCN ticket") @option("add_thread", description="Create a Discord thread for the new ticket", default=False) async def create_new_ticket(self, ctx: discord.ApplicationContext, title:str): - user = self.redmine.user_cache.find_user(ctx.user.name) + user = self.redmine.user_mgr.find(ctx.user.name) if user is None: await ctx.respond(f"Unknown user: {ctx.user.name}") return @@ -166,7 +166,7 @@ async def thread_ticket(self, ctx: discord.ApplicationContext, ticket_id:int): # update the discord flag on tickets, add a note with url of thread; thread.jump_url # TODO message templates note = f"Created Discord thread: {thread.name}: {thread.jump_url}" - user = self.redmine.user_cache.find_discord_user(ctx.user.name) + user = self.redmine.user_mgr.find_discord_user(ctx.user.name) self.redmine.enable_discord_sync(ticket.id, user, note) await ctx.respond(f"Created new thread for {ticket.id}: {thread}") # todo add some fancy formatting diff --git a/imap.py b/imap.py index f042cbb..a75823e 100755 --- a/imap.py +++ b/imap.py @@ -90,7 +90,7 @@ def __init__(self): self.host = os.getenv('IMAP_HOST') self.user = os.getenv('IMAP_USER') self.passwd = os.getenv('IMAP_PASSWORD') - self.redmine = redmine.Client.fromenv() + self.redmine:redmine.Client = redmine.Client.fromenv() # note: not happy with this method of dealing with complex email address # but I don't see a better way. open to suggestions @@ -232,11 +232,11 @@ def handle_message(self, msg_id:str, message:Message): ticket = self.redmine.find_ticket_from_str(subject) # get user id from from_address - user = self.redmine.user_cache.find_user(addr) + user = self.redmine.user_mgr.get_by_name(addr) if user is None: log.debug(f"Unknown email address, no user found: {addr}, {message.from_address}") # create new user - user = self.redmine.create_user(addr, first, last) + user = self.redmine.user_mgr.create(addr, first, last) log.info(f"Unknow user: {addr}, created new account.") # upload any attachments diff --git a/netbot.py b/netbot.py index 136938a..8253205 100755 --- a/netbot.py +++ b/netbot.py @@ -62,7 +62,7 @@ async def on_message(self, message:discord.Message): # IS a thread, check the name ticket_id = self.parse_thread_title(message.channel.name) if ticket_id > 0: - user = self.redmine.user_cache.find_user(message.author.name) + user = self.redmine.user_mgr.find(message.author.name) if user: log.debug(f"known user commenting on ticket #{ticket_id}: redmine={user.login}, discord={message.author.name}") else: @@ -112,7 +112,7 @@ def append_redmine_note(self, ticket, message: discord.Message) -> None: # redmine link format: "Link Text":http://whatever # check user mapping exists - user = self.redmine.user_cache.find_user(message.author.name) + user = self.redmine.user_mgr.find(message.author.name) if user: # format the note formatted = f'"Discord":{message.jump_url}: {message.content}' diff --git a/redmine.py b/redmine.py index 2e96179..94f49cc 100644 --- a/redmine.py +++ b/redmine.py @@ -13,7 +13,7 @@ import synctime from session import RedmineSession -from users import UserResult, User, UserManager, UserCache +from users import UserResult, UserManager log = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def __init__(self, message: str, request_id: str) -> None: self.request_id = request_id -class Client(): ## redmine.Client.fromenv() +class Client(): ## redmine.Client """redmine client""" def __init__(self, url: str, token: str): self.url = url @@ -45,21 +45,14 @@ def __init__(self, url: str, token: str): if self.url is None: raise RedmineException("Unable to load REDMINE_TOKEN", "__init__") - session = RedmineSession(url, token) - self.user_mgr = UserManager(session) - self.user_cache = UserCache(self.user_mgr) + session:RedmineSession = RedmineSession(url, token) + self.user_mgr:UserManager = UserManager(session) @classmethod def fromenv(cls): url = os.getenv('REDMINE_URL') - if url is None: - raise RedmineException("Unable to load REDMINE_URL", "[n/a]") - token = os.getenv('REDMINE_TOKEN') - if token is None: - raise RedmineException("Unable to load REDMINE_TOKEN", "__init__") - return cls(url, token) @@ -99,7 +92,7 @@ def create_ticket(self, user, subject, body, attachments=None): # ticket 484 - http://10.10.0.218/issues/484 # if the user is blocked, "reject" the new ticket - if self.is_user_blocked(user): + if self.user_mgr.is_blocked(user): log.debug(f"Rejecting ticket #{ticket.id} based on blocked user {user.login}") self.reject_ticket(ticket.id) return self.get_ticket(ticket.id) # refresh the ticket? @@ -111,30 +104,6 @@ def create_ticket(self, user, subject, body, attachments=None): response.headers['X-Request-Id']) - def update_user(self, user, fields:dict): - """update a user record in redmine""" - # PUT a simple JSON structure - data = {} - data['user'] = fields - - response = requests.put( - url=f"{self.url}/users/{user.id}.json", - timeout=TIMEOUT, - data=json.dumps(data), - headers=self.get_headers()) # removed user.login impersonation header - - log.debug(f"update user: [{response.status_code}] {response.request.url}, fields: {fields}") - - # check status - if response.ok: - # TODO get and return the updated user? - return user - else: - raise RedmineException( - f"update_user failed, status=[{response.status_code}] {response.reason}", - response.headers['X-Request-Id']) - - def update_ticket(self, ticket_id:str, fields:dict, user_login:str=None): """update a redmine ticket""" # PUT a simple JSON structure @@ -245,34 +214,6 @@ def upload_attachments(self, user_id, attachments): - def create_team(self, teamname:str): - if teamname is None or len(teamname.strip()) == 0: - raise RedmineException(f"Invalid team name: '{teamname}'", "n/a") - - # POST to /groups.json - data = { - "group": { - "name": teamname, - } - } - - response = requests.post( - url=f"{self.url}/groups.json", - data=json.dumps(data), - timeout=TIMEOUT, - headers=self.get_headers()) - - # check status - if response.ok: - self.user_cache.reindex_teams() - log.info(f"OK create_team {teamname}") - else: - raise RedmineException(f"create_team {teamname} failed", response.headers['X-Request-Id']) - - - def get_user(self, user_id:int): - return self.user_cache.get_user(user_id) - def lookup_user(self, username:str): """Get a user based on ID, directly from redmine""" @@ -296,27 +237,6 @@ def lookup_user(self, username:str): return None - def is_user_blocked(self, user) -> bool: - if self.is_user_in_team(user.login, BLOCKED_TEAM_NAME): - return True - else: - return False - - - def block_user(self, user) -> None: - # check if the blocked team exists - blocked_team = self.user_cache.find_team(BLOCKED_TEAM_NAME) - if blocked_team is None: - # create blocked team - self.create_team(BLOCKED_TEAM_NAME) - - self.join_team(user.login, BLOCKED_TEAM_NAME) - - - def unblock_user(self, user) -> None: - self.leave_team(user.login, BLOCKED_TEAM_NAME) - - def get_tickets_by(self, user): # GET /issues.json?author_id=6 response = self.query(f"/issues.json?author_id={user.id}") @@ -371,63 +291,6 @@ def find_ticket_from_str(self, string:str): return [] - def create_user(self, email:str, first:str, last:str): - """create a new redmine user""" - # TODO: Generate JSON from User object - data = { - 'user': { - 'login': email, - 'firstname': first, - 'lastname': last, - 'mail': email, - } - } - # on create, assign watcher: sender? - - r = requests.post( - url=f"{self.url}/users.json", - timeout=TIMEOUT, - data=json.dumps(data), - headers=self.get_headers()) - - # check status - if r.ok: - user = User(**r.json()['user']) - - log.info(f"created user: {user.id} {user.login} {user.mail}") - self.user_cache.reindex_users() # new user! FIXME reindix or just add? - - # add user to User group - self.join_team(user.login, "users") ### FIXME move default team name to defaults somewhere - - return user - elif r.status_code == 403: - # can't create existing user. log err, but return from cache - user = self.user_mgr.search(email) - log.error(f"Trying to create existing user email: email={email}, user={user}") - return user - else: - log.error(f"create_user, status={r.status_code}: {r.reason}, req-id={r.headers['X-Request-Id']}") - #TODO throw exception? - return None - - - # used only in testing - def remove_user(self, user_id:int): - """remove user frmo redmine. used for testing""" - # DELETE to /users/{user_id}.json - r = requests.delete( - url=f"{self.url}/users/{user_id}.json", - timeout=TIMEOUT, - headers=self.get_headers()) - - # check status - if r.ok: - log.info(f"deleted user {user_id}") - else: - log.error(f"Error removing user status={r.status_code}, url={r.request.url}, req_id={r.headers['X-Request-Id']}") - - def remove_ticket(self, ticket_id:int): """delete a ticket in redmine. used for testing""" # DELETE to /issues/{ticket_id}.json @@ -445,7 +308,7 @@ def remove_ticket(self, ticket_id:int): def most_recent_ticket_for(self, email): """get the most recent ticket for the user with the given email""" # get the user record for the email - user = self.user_mgr.search(email) + user = self.user_mgr.get_by_name(email) if user: # query open tickets created by user, sorted by most recently updated, limit 1 @@ -498,7 +361,7 @@ def my_tickets(self, user=None): def tickets_for_team(self, team_str:str): # validate team? - team = self.user_mgr.search(team_str) # find_user is dsigned to be broad + team = self.user_mgr.get_by_name(team_str) # find_user is dsigned to be broad query = f"/issues.json?assigned_to_id={team.id}&status_id=open&sort={DEFAULT_SORT}&limit=100" response = self.query(query) @@ -570,71 +433,26 @@ def enable_discord_sync(self, ticket_id, user, note): # todo: better error reporting back to discord - def create_discord_mapping(self, redmine_login:str, discord_name:str): - user = self.user_mgr.search(redmine_login) - - field_id = 2 ## "Discord ID"search for me in cached custom fields - fields = { - "custom_fields": [ - { "id": field_id, "value": discord_name } # cf_4, custom field syncdata - ] - } - self.update_user(user, fields) - # TODO rebuild user index automatically? - - def join_team(self, username, teamname:str) -> None: - # look up user ID - user = self.user_mgr.search(username) - if user is None: - raise RedmineException(f"Unknown user name: {username}", "[n/a]") - - # map teamname to team - team = self.user_cache.find_team(teamname) - if team is None: - raise RedmineException(f"Unknown team name: {teamname}", "[n/a]") - - # POST to /group/ID/users.json - data = { - "user_id": user.id - } - - response = requests.post( - url=f"{self.url}/groups/{team.id}/users.json", - data=json.dumps(data), - timeout=TIMEOUT, - headers=self.get_headers()) - - # check status - if response.ok: - log.info(f"OK join_team {username}, {teamname}") - else: - raise RedmineException(f"join_team failed, status=[{response.status_code}] {response.reason}", response.headers['X-Request-Id']) + user = self.user_mgr.get_by_name(username) + self.user_mgr.join_team(user, teamname) - def leave_team(self, username:int, teamname:str): + def leave_team(self, username:int, teamname:str) -> None: # look up user ID - user = self.user_mgr.search(username) + user = self.user_mgr.get_by_name(username) if user is None: log.warning(f"Unknown user name: {username}") return None # map teamname to team - team = self.user_cache.find_team(teamname) - if team is None: - log.warning(f"Unknown team name: {teamname}") - return None + #team = self.user_mgr.get_team_by_name(teamname) + #if team is None: + # log.warning(f"Unknown team name: {teamname}") + # return None - # DELETE to /groups/{team-id}/users/{user_id}.json - r = requests.delete( - url=f"{self.url}/groups/{team.id}/users/{user.id}.json", - timeout=TIMEOUT, - headers=self.get_headers()) + self.user_mgr.leave_team(user, teamname) - # check status - if r.status_code != 204: - log.error(f"Error removing user from group status={r.status_code}, url={r.request.url}") - return None def get_headers(self, impersonate_id:str=None): headers = { @@ -674,9 +492,8 @@ def query(self, query_str:str, user:str=None): return None - def assign_ticket(self, ticket_id, target, user_id=None): - user = self.user_mgr.search(target) + user = self.user_mgr.get_by_name(target) if user: fields = { "assigned_to_id": user.id, @@ -719,19 +536,7 @@ def resolve_ticket(self, ticket_id, user_id=None): def get_team(self, teamname:str): - team = self.user_cache.find_team(teamname) - if team is None: - log.debug(f"Unknown team name: {teamname}") - return None - - # as per https://www.redmine.org/projects/redmine/wiki/Rest_Groups#GET-2 - # GET /groups/20.json?include=users - response = self.query(f"/groups/{team.id}.json?include=users") - if response: - return response.group - else: - #TODO exception? - return None + return self.user_mgr.get_team_by_name(teamname) # FIXME consistent naming def get_sync_record(self, ticket, expected_channel: int) -> synctime.SyncRecord: @@ -825,30 +630,7 @@ def get_field(self, ticket, fieldname): except AttributeError: return "" # or None? - def get_discord_id(self, user): - if user: - for field in user.custom_fields: - if field.name == DISCORD_ID_FIELD and field.value and len(field.value) > 0: - #log.debug(f"redmine:{user.login} <==> discord:{field.value}") - return field.value - return None - - - def is_user_in_team(self, username:str, teamname:str) -> bool: - if username is None or teamname is None: - return False - - user = self.user_mgr.search(username) - if user: - user_id = user.id - team = self.get_team(teamname) # requires an API call - - if team: - for team_user in team.users: - if team_user.id == user_id: - return True - return False if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt index 1646ac1..c6765a6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ python-dotenv==1.0.0 python-gnupg==0.5.2 requests==2.31.0 rich==13.6.0 +ruff==0.3.0 tomlkit==0.12.3 urllib3==2.0.4 yarl==1.9.2 diff --git a/session.py b/session.py index a8a6603..aa3256e 100644 --- a/session.py +++ b/session.py @@ -72,22 +72,23 @@ def get(self, query_str:str, user:str=None): if r.ok: return r.json() else: - log.error(f"Status code {r.status_code} for {r.request.url}, reqid={r.headers['X-Request-Id']}: {r}") + log.error(f"GET {r.status_code} for {r.request.url}, reqid={r.headers['X-Request-Id']}: {r}") except TimeoutError as toe: # ticket-509: Handle timeout gracefully log.warning(f"Timeout during {query_str}: {toe}") except Exception as ex: - log.warning(f"Excetion during {query_str}: {ex}") + log.exception(f"Exception during {query_str}: {ex}") return None # data=json.dumps(data), - def put(self, resource: str, data, user_login: str = None): + def put(self, resource: str, data, user_login: str = None) -> bool: r = self.session.put(f"{self.url}{resource}", data=data, timeout=TIMEOUT, headers=self.get_headers(user_login)) if r.ok: - return r.json() + log.debug(f"PUT {resource}: {data} - {r}") + return True else: raise RedmineException(f"POST failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) @@ -95,8 +96,11 @@ def put(self, resource: str, data, user_login: str = None): def post(self, resource: str, data, user_login: str = None): r = self.session.post(f"{self.url}{resource}", data=data, timeout=TIMEOUT, headers=self.get_headers(user_login)) - if r.ok: + if r.status_code == 201: + #log.debug(f"POST {resource}: {data} - {vars(r)}") return r.json() + elif r.status_code == 204: + return None else: raise RedmineException(f"POST failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) diff --git a/test_cog_scn.py b/test_cog_scn.py index e295696..79a24ad 100755 --- a/test_cog_scn.py +++ b/test_cog_scn.py @@ -49,19 +49,20 @@ async def test_team_join_leave(self): # 4.5 check reindex result, and lookup based on login and discord id ctx.respond.assert_called_with("Rebuilt redmine indices.") - self.assertIsNotNone(self.redmine.user_cache.find_user(self.user.login)) - self.assertIsNotNone(self.redmine.user_cache.find_user(self.discord_user)) + self.assertIsNotNone(self.redmine.user_mgr.find(self.user.login)) + self.assertIsNotNone(self.redmine.user_mgr.find(self.discord_user)) # join team users ctx = self.build_context() #member = unittest.mock.AsyncMock(discord.Member) # for forced use case #member.name = discord_user await self.cog.join(ctx, test_team_name) + self.redmine.user_mgr.reindex_teams() # confirm via mock callback and API #ctx.respond.assert_called_with(f"Unknown team name: {test_team_name}") # unknown team response! ctx.respond.assert_called_with(f"**{self.discord_user}** has joined *{test_team_name}*") - self.assertTrue(self.redmine.is_user_in_team(self.user.login, test_team_name), f"{self.user.login} not in team {test_team_name}") + self.assertTrue(self.redmine.user_mgr.is_user_in_team(self.user, test_team_name), f"{self.user.login} not in team {test_team_name}") # confirm in team via cog teams response ctx = self.build_context() @@ -71,9 +72,10 @@ async def test_team_join_leave(self): # leave team users ctx = self.build_context() await self.cog.leave(ctx, test_team_name) + self.redmine.user_mgr.reindex_teams() # confirm via API and callback - self.assertFalse(self.redmine.is_user_in_team(self.user.login, test_team_name), f"{self.user.login} *in* team {test_team_name}") + self.assertFalse(self.redmine.user_mgr.is_user_in_team(self.user, test_team_name), f"{self.user.login} *in* team {test_team_name}") ctx.respond.assert_called_with(f"**{self.discord_user}** has left *{test_team_name}*") # confirm not in team via cog teams response @@ -107,9 +109,10 @@ async def test_block_user(self): # call block ctx = self.build_context() await self.cog.block(ctx, self.user.login) + self.redmine.user_mgr.reindex_teams() # confirmed blocked - self.assertTrue(self.redmine.is_user_blocked(self.user)) + self.assertTrue(self.redmine.user_mgr.is_blocked(self.user)) # confirm ticket rejected check_ticket = self.redmine.get_ticket(ticket.id) diff --git a/test_imap.py b/test_imap.py index 86891c7..514e266 100755 --- a/test_imap.py +++ b/test_imap.py @@ -21,8 +21,8 @@ class TestMessages(unittest.TestCase): """Test suite for IMAP functions""" def setUp(self): - self.redmine = redmine.Client.fromenv() - self.imap = imap.Client() + self.redmine: redmine.Client = redmine.Client.fromenv() + self.imap: imap.Client = imap.Client() def test_messages_stripping(self): # open @@ -124,22 +124,22 @@ def test_new_account_from_email(self): self.redmine.remove_ticket(tickets[0].id) # remove the user after the test - self.redmine.remove_user(user.id) + self.redmine.user_mgr.remove(user) def test_subject_search(self): # create a new ticket with unique subject tag = test_utils.tagstr() - user = self.redmine.user_cache.find_user("philion") # FIXME: create a relaible test_user + user = self.redmine.user_mgr.get_by_name("admin") # FIXME: create_test_user in test_utils self.assertIsNotNone(user) - subject = f"New ticket with unique marker {tag}" + subject = f"Test {tag} {tag} {tag}" ticket = self.redmine.create_ticket(user, subject, f"This for {self.id}-{tag}") self.assertIsNotNone(ticket) # search for the ticket tickets = self.redmine.match_subject(subject) - #for check in tickets: - # log.debug(f"### tickets: {check.subject}") + for check in tickets: + log.debug(f"### tickets: {check.subject}") self.assertIsNotNone(tickets) self.assertEqual(1, len(tickets)) self.assertEqual(ticket.id, tickets[0].id) @@ -158,7 +158,7 @@ def test_ticket_query(self): # create a ticket with the tag in the body, not the subject tag = test_utils.tagstr() - user = self.redmine.user_cache.find_user("admin") + user = self.redmine.user_mgr.get_by_name("admin") self.assertIsNotNone(user) body = f"Body with {self.id} and {tag}" ticket = self.redmine.create_ticket(user, "Boring test ticket", body) @@ -167,14 +167,6 @@ def test_ticket_query(self): # search for the ticket tickets = self.redmine.search_tickets(tag) - #for check in tickets: - # log.debug(f"### tickets: {check}") - - self.assertIsNotNone(tickets) - self.assertEqual(1, len(tickets)) - self.assertEqual(ticket.id, tickets[0].id) - - tickets = self.redmine.search_tickets(self.id) self.assertIsNotNone(tickets) self.assertEqual(1, len(tickets)) self.assertEqual(ticket.id, tickets[0].id) diff --git a/test_redmine.py b/test_redmine.py index 1a11c8b..d201b32 100755 --- a/test_redmine.py +++ b/test_redmine.py @@ -27,15 +27,15 @@ def test_blocked_user(self): user = test_utils.create_test_user(self.redmine, tag) # block - self.redmine.block_user(user) - self.assertTrue(self.redmine.is_user_blocked(user)) + self.redmine.user_mgr.block(user) + self.assertTrue(self.redmine.user_mgr.is_blocked(user)) # unblock - self.redmine.unblock_user(user) - self.assertFalse(self.redmine.is_user_blocked(user)) + self.redmine.user_mgr.unblock(user) + self.assertFalse(self.redmine.user_mgr.is_blocked(user)) # remove the test user - self.redmine.remove_user(user.id) + self.redmine.user_mgr.remove(user) def test_blocked_create_ticket(self): @@ -45,8 +45,8 @@ def test_blocked_create_ticket(self): try: # block - self.redmine.block_user(user) - self.assertTrue(self.redmine.is_user_blocked(user)) + self.redmine.user_mgr.block(user) + self.assertTrue(self.redmine.user_mgr.is_blocked(user)) # create ticket for blocked ticket = self.redmine.create_ticket(user, "subject", "body") @@ -54,7 +54,7 @@ def test_blocked_create_ticket(self): finally: # remove the test user - self.redmine.remove_user(user.id) + self.redmine.user_mgr.remove(user) def test_client_timeout(self): diff --git a/test_synctime.py b/test_synctime.py index 3183d58..23bd686 100755 --- a/test_synctime.py +++ b/test_synctime.py @@ -26,7 +26,7 @@ def test_redmine_times(self): # create a new ticket with unique subject tag = test_utils.tagstr() - user = self.redmine.user_cache.find_user("philion") # FIXME: create a relaible test_user + user = self.redmine.user_mgr.find("admin") # FIXME: create a relaible test_user self.assertIsNotNone(user) subject = f"TEST ticket {tag}" ticket = self.redmine.create_ticket(user, subject, f"This for {self.id}-{tag}") diff --git a/test_utils.py b/test_utils.py index ce6653e..6c94a78 100644 --- a/test_utils.py +++ b/test_utils.py @@ -8,9 +8,12 @@ import unittest from unittest import mock +from dotenv import load_dotenv + import discord from discord import ApplicationContext from redmine import Client +from users import User log = logging.getLogger(__name__) @@ -52,36 +55,45 @@ def create_test_user(redmine:Client, tag:str): email = first + "@example.com" # create new redmine user, using redmine api - user = redmine.create_user(email, first, last) + user = redmine.user_mgr.create(email, first, last) # create temp discord mapping with redmine api, assert + # create_discord_mapping will cache the new user discord_user = "discord-" + tag ### <-- - redmine.create_discord_mapping(user.login, discord_user) + redmine.user_mgr.create_discord_mapping(user, discord_user) + + # lookup based on login + return redmine.user_mgr.get_by_name(user.login) + + +def remove_test_users(redmine:Client): + for user in redmine.user_mgr.get_all(): + if user.login.startswith("test-") or user.login == "philion@acmerocket.com": + log.info(f"Removing test user: {user.login}") + redmine.user_mgr.remove(user) - # reindex users and lookup based on login - redmine.user_cache.reindex_users() - return redmine.user_cache.find_user(user.login) +# TODO delete test tickets and "Search for subject match in email threading" ticket. TAG with test too? class BotTestCase(unittest.IsolatedAsyncioTestCase): """Abstract base class for testing Bot features""" - redmine = None - usertag = None - user = None + redmine: Client = None + usertag: str = None + user: User = None @classmethod def setUpClass(cls): log.info("Setting up test fixtures") - cls.redmine = Client.fromenv() - cls.usertag = tagstr() - cls.user = create_test_user(cls.redmine, cls.usertag) + cls.redmine:Client = Client.fromenv() + cls.usertag:str = tagstr() + cls.user:User = create_test_user(cls.redmine, cls.usertag) log.info(f"Created test user: {cls.user}") @classmethod def tearDownClass(cls): log.info(f"Tearing down test fixtures: {cls.user}") - cls.redmine.remove_user(cls.user.id) + cls.redmine.user_mgr.remove(cls.user) def build_context(self) -> ApplicationContext: @@ -106,7 +118,20 @@ def setUp(self): self.full_name = self.user.firstname + " " + self.user.lastname self.discord_user = self.user.discord_id - self.assertIsNotNone(self.redmine.user_cache.find_user(self.user.login)) - self.assertIsNotNone(self.redmine.user_cache.find_user(self.discord_user)) + self.assertIsNotNone(self.redmine.user_mgr.find(self.user.login)) + self.assertIsNotNone(self.redmine.user_mgr.find(self.discord_user)) log.debug(f"setUp user {self.user.login} {self.discord_user}") + + +if __name__ == '__main__': + # when running this main, turn on DEBUG + logging.basicConfig(level=logging.DEBUG, format="{asctime} {levelname:<8s} {name:<16} {message}", style='{') + logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) + + # load credentials + load_dotenv() + + # construct the client and run the email check + client = Client.fromenv() + remove_test_users(client) diff --git a/users.py b/users.py index df7f809..eaa8586 100644 --- a/users.py +++ b/users.py @@ -24,6 +24,26 @@ class CustomField(): name: str value: str + +@dataclass +class NamedId: + '''named ID in redmine''' + id: int + name: str + + +@dataclass +class Team: + """Encapsulates a team""" + id: int + name: str + users: list[NamedId] = None + + def __post_init__(self): + if self.users: + self.users = [NamedId(**name) for name in self.users] + + @dataclass class User(): """Encapsulates a redmine user""" @@ -44,7 +64,6 @@ class User(): status: int = "" custom_fields: list[CustomField] - def __post_init__(self): self.custom_fields = [CustomField(**field) for field in self.custom_fields] self.discord_id = self.get_custom_field(DISCORD_ID_FIELD) @@ -70,21 +89,121 @@ def __post_init__(self): self.users = [User(**user) for user in self.users] -@dataclass -class Team: - """Encapsulates a team""" - id: int - name: str +class UserCache(): + """cache of user data""" + def __init__(self): + self.users: dict[str, int] = {} + self.user_ids: dict[int, User] = {} + self.user_emails: dict[str, int] = {} + self.discord_users: dict[str, int] = {} + self.teams: dict[str, Team] = {} + + + def clear(self): + # reset the indices + self.users.clear() + self.user_ids.clear() + self.user_emails.clear() + self.discord_users.clear() + + + def cache_user(self, user: User) -> None: + """add the user to the cache""" + #log.debug(f"caching: {user.id} {user.login} {user.discord_id}") + + self.user_ids[user.id] = user + self.users[user.login] = user.id + self.user_emails[user.mail] = user.id + if user.discord_id: + self.discord_users[user.discord_id] = user.id + + + def cache_team(self, team: Team) -> None: + """add the team to the cache""" + self.teams[team.name] = team + + + def get(self, user_id:int): + """get a user by ID""" + return self.user_ids.get(user_id) + + + def get_by_name(self, username:str) -> User: + return self.find(username) + + + def find(self, name): + """find a user by name""" + # check if name is int, raw user id. then look up in userids + # check the indicies + if name in self.user_emails: + return self.get(self.user_emails[name]) + elif name in self.users: + return self.get(self.users[name]) + elif name in self.discord_users: + return self.get(self.discord_users[name]) + elif name in self.teams: + return self.teams[name] #ugly. put groups in user collection? + else: + return None + + + def get_team_by_name(self, name:str) -> Team: + if name in self.teams: + return self.teams[name] + + + def find_discord_user(self, discord_user_id:str) -> User: + """find a user by their discord ID""" + if discord_user_id is None: + return None + + if discord_user_id in self.discord_users: + user_id = self.discord_users[discord_user_id] + return self.user_ids[user_id] + else: + return None + + + def is_user_or_group(self, name:str) -> bool: + if name in self.users: + return True + elif name in self.teams: + return True + else: + return False + + + def get_teams(self) -> list[Team]: + return self.teams.values() + + + def is_user_in_team(self, user:User, teamname:str) -> bool: + if user is None or teamname is None: + return False + + team = self.get_team_by_name(teamname) + + if team: + for team_user in team.users: + if team_user.id == user.id: + return True + + return False + + + class UserManager(): """manage redmine users""" session: RedmineSession + cache: UserCache def __init__(self, session: RedmineSession): self.session = session - + self.cache = UserCache() def get_all(self) -> list[User]: jresp = self.session.get(f"{USER_RESOURCE}?limit=100") @@ -107,19 +226,19 @@ def get_all(self) -> list[User]: return [] - def update(self, user:User, fields:dict): + def update(self, user:User, fields:dict) -> User: """update a user record in redmine""" # PUT a simple JSON structure data = {} data['user'] = fields - response = self.session.put(f"/users/{user.id}.json",json.dumps(data)) - - log.debug(f"update user: [{response.status_code}] {response.request.url}, fields: {fields}") + response = self.session.put(f"/users/{user.id}.json", json.dumps(data)) # check status - if response.ok: - # TODO get and return the updated user + if response: + # get and return the updated user + user = self.get(user.id) + log.debug(f"updated id={user.id}: user: {user}") return user else: raise RedmineException( @@ -127,9 +246,8 @@ def update(self, user:User, fields:dict): response.headers['X-Request-Id']) - - def search(self, username:str) -> User: - """Get a user based on ID, directly from redmine""" + def get_by_name(self, username:str) -> User: + """Get a user based on name, directly from redmine""" if username is None or len(username) == 0: log.debug("Empty user ID") return None @@ -149,27 +267,6 @@ def search(self, username:str) -> User: return None - def is_blocked(self, user) -> bool: - if self.is_user_in_team(user.login, BLOCKED_TEAM_NAME): - return True - else: - return False - - - def block(self, user) -> None: - # check if the blocked team exists - blocked_team = self.find_team(BLOCKED_TEAM_NAME) - if blocked_team is None: - # create blocked team - self.create_team(BLOCKED_TEAM_NAME) - - self.join_team(user.login, BLOCKED_TEAM_NAME) - - - def unblock(self, user) -> None: - self.leave_team(user.login, BLOCKED_TEAM_NAME) - - def create(self, email:str, first:str, last:str): """create a new redmine user""" # TODO: Generate JSON from User object @@ -196,6 +293,36 @@ def create(self, email:str, first:str, last:str): raise RedmineException(f"create_user {email} failed", r.headers['X-Request-Id']) + def find(self, name: str) -> User: + """get a user by ID""" + if not name: + return None + + # check cache first + user = self.cache.find(name) + if not user: + # not found in cache, try a name search + user = self.get_by_name(name) + if user: + log.info(f"found uncached user for {name}: {user.login}, caching") + self.cache.cache_user(user) + return user + + + def find_discord_user(self, discord_user_id:str) -> User: + # just a proxy + return self.cache.find_discord_user(discord_user_id) + + def is_user_or_group(self, term:str): + return self.cache.is_user_or_group(term) + + def get(self, user_id:int): + """get a user by ID, directly from redmine""" + jresp = self.session.get(f"/users/{user_id}.json") + if jresp: + return User(**jresp['user']) + + # used only in testing def remove(self, user: User): """remove user frmo redmine. used for testing""" @@ -210,26 +337,28 @@ def remove(self, user: User): # exception? - def create_discord_mapping(self, redmine_login:str, discord_name:str): - user = self.search(redmine_login) - + def create_discord_mapping(self, user:User, discord_name:str) -> User: field_id = 2 ## "Discord ID"search for me in cached custom fields fields = { "custom_fields": [ { "id": field_id, "value": discord_name } # cf_4, custom field syncdata ] } - self.update(user, fields) + return self.update(user, fields) - def get_all_teams(self) -> dict: - # this needs to be cached! + def get_all_teams(self, include_users: bool = True) -> dict[str, Team]: resp = self.session.get(f"{TEAM_RESOURCE}?limit=100") # list of id, name if resp: teams = {} - for team in resp['groups']: - teams[team['name']] = Team(**team) + for team_rec in resp['groups']: + # create dict mapping team name -> full team record + # calling "get_team" here, as it's the only way to get users in the team + if include_users: + teams[team_rec['name']] = self.get_team(team_rec['id']) # requires an additional API call + else: + teams[team_rec['name']] = Team(**team_rec) return teams else: @@ -237,10 +366,6 @@ def get_all_teams(self) -> dict: return [] - def find_team(self, name:str) -> int: - return self.get_all_teams().get(name, None) - - def create_team(self, teamname:str): if teamname is None or len(teamname.strip()) == 0: raise RedmineException(f"Invalid team name: '{teamname}'", __name__) @@ -261,73 +386,30 @@ def create_team(self, teamname:str): raise RedmineException(f"create_team {teamname} failed", response.headers['X-Request-Id']) - def join_team(self, user: User, teamname:str) -> None: - # look up user ID - #user = self.find_user(username) - #if user is None: - # raise RedmineException(f"Unknown user name: {username}", "[n/a]") - - # map teamname to team - team_id = self.find_team(teamname) - if team_id is None: - raise RedmineException(f"Unknown team name: {teamname}", "[n/a]") - - # POST to /group/ID/users.json - data = { - "user_id": user.id - } - - response = self.session.post(f"/groups/{team_id}/users.json", data=json.dumps(data)) - - # check status - if response.ok: - log.info(f"OK join_team {user.login}, {teamname}") - else: - raise RedmineException(f"join_team failed, status=[{response.status_code}] {response.reason}", response.headers['X-Request-Id']) - - - def leave_team(self, user: User, teamname:str): - # look up user ID - #user = self.find_user(username) - #if user is None: - # log.warning(f"Unknown user name: {username}") - # return None - - # map teamname to team - team_id = self.find_team(teamname) - if team_id is None: - log.warning(f"Unknown team name: {teamname}") - return - - # DELETE to /groups/{team-id}/users/{user_id}.json - r = self.session.delete(f"/groups/{team_id}/users/{user.id}.json") - - # check status - if not r: - log.error(f"Error removing {user.login} from {teamname}") - - - def get_team(self, teamname:str): - team_id = self.find_team(teamname) - if team_id is None: - log.debug(f"Unknown team name: {teamname}") - return None - + def get_team(self, team_id: int) -> Team: + """get a full team record from redmine. only way to get team membership""" # as per https://www.redmine.org/projects/redmine/wiki/Rest_Groups#GET-2 # GET /groups/20.json?include=users response = self.session.get(f"/groups/{team_id}.json?include=users") if response: - return response.group + return Team(**response['group']) else: #TODO exception? return None + def get_team_by_name(self, name:str) -> Team: + # need to get all team, which builds a dicts of names + teams = self.get_all_teams(include_users=False) + if name in teams: + return self.get_team(teams[name].id) + + def is_user_in_team(self, user: User, teamname:str) -> bool: if user is None or teamname is None: return False - team = self.get_team(teamname) + team = self.get_team_by_name(teamname) if team: for team_user in team.users: @@ -337,126 +419,96 @@ def is_user_in_team(self, user: User, teamname:str) -> bool: return False -class UserCache(): - """cache of user data""" - def __init__(self, mgr:UserManager): - self.mgr = mgr + def is_blocked(self, user:User) -> bool: + if self.is_user_in_team(user, BLOCKED_TEAM_NAME): + return True + else: + return False - self.users = {} - self.user_ids = {} - self.user_emails = {} - self.discord_users = {} - self.teams = {} - self.reindex() + def block(self, user) -> None: + # check if the blocked team exists + blocked_team = self.get_team_by_name(BLOCKED_TEAM_NAME) + if blocked_team is None: + # create blocked team + self.create_team(BLOCKED_TEAM_NAME) - def get_user(self, user_id:int): - """get a user by ID""" - if user_id: - return self.user_ids[user_id] + self.join_team(user, BLOCKED_TEAM_NAME) - def find_user(self, name): - """find a user by name""" - # check if name is int, raw user id. then look up in userids - # check the indicies - if name in self.user_emails: - return self.get_user(self.user_emails[name]) - elif name in self.users: - return self.get_user(self.users[name]) - elif name in self.discord_users: - return self.get_user(self.discord_users[name]) - elif name in self.teams: - return self.teams[name] #ugly. put groups in user collection? - else: - return None + def unblock(self, user) -> None: + self.leave_team(user, BLOCKED_TEAM_NAME) - def find_team(self, name:str) -> int: - if name in self.teams: - return self.teams[name] + def join_team(self, user: User, teamname:str) -> None: + # look up user ID + #user = self.find_user(username) + #if user is None: + # raise RedmineException(f"Unknown user name: {username}", "[n/a]") + # map teamname to team + team = self.get_team_by_name(teamname) + if team.id is None: + raise RedmineException(f"Unknown team name: {teamname}", "[n/a]") - def find_discord_user(self, discord_user_id:str): - """find a user by their discord ID""" - if discord_user_id is None: - return None + # POST to /group/ID/users.json + data = { + "user_id": user.id + } - if discord_user_id in self.discord_users: - user_id = self.discord_users[discord_user_id] - return self.user_ids[user_id] - else: - return None + self.session.post(f"/groups/{team.id}/users.json", data=json.dumps(data)) - def is_user_or_group(self, name:str) -> bool: - if name in self.users: - return True - elif name in self.teams: - return True - else: - return False + def leave_team(self, user: User, teamname:str): + # map teamname to team + team = self.get_team_by_name(teamname) + if team is None: + log.warning(f"Unknown team name: {teamname}") + return + + # DELETE to /groups/{team-id}/users/{user_id}.json + r = self.session.delete(f"/groups/{team.id}/users/{user.id}.json") # encapsulation + # check status + if not r: + log.error(f"Error removing {user.login} from {teamname}") + + +#### ---- indexing stuff # python method sync? def reindex_users(self): # rebuild the indicies # looking over issues in redmine and specifically https://www.redmine.org/issues/16069 # it seems that redmine has a HARD CODED limit of 100 responses per request. - all_users = self.mgr.get_all() + all_users = self.get_all() if all_users: - # reset the indices - self.users.clear() - self.user_ids.clear() - self.user_emails.clear() - self.discord_users.clear() + self.cache.clear() for user in all_users: - self.users[user.login] = user.id - self.user_ids[user.id] = user - self.user_emails[user.mail] = user.id - - #discord_id = user.get_discord_id(user) - if user.discord_id: - self.discord_users[user.discord_id] = user.id - log.debug(f"indexed {len(self.users)} users") - log.debug(f"discord users: {self.discord_users}") + self.cache.cache_user(user) # several internal indicies + + log.debug(f"indexed {len(all_users)} users") + log.debug(f"discord users: {self.cache.discord_users}") else: log.error("No users to index") - def get_teams(self): - return self.teams.keys() - - - # TODO: Add a dataclass for Team, and page-unrolling for "all teams" def reindex_teams(self): - # rebuild the group index - self.teams = self.mgr.get_all_teams() - - - def is_user_in_team(self, username:str, teamname:str) -> bool: - if username is None or teamname is None: - return False - - user = self.mgr.search(username) - if user: - user_id = user.id - team = self.mgr.get_team(teamname) # requires an API call - - if team: - for team_user in team.users: - if team_user.id == user_id: - return True - - return False + all_teams = self.get_all_teams() + if all_teams: + self.cache.teams = all_teams # replace all the cached teams + log.debug(f"indexed {len(all_teams)} teams") + else: + log.error("No teams to index") def reindex(self): start = dt.datetime.now() self.reindex_users() self.reindex_teams() - log.debug(f"reindex took {dt.datetime.now() - start}") + log.info(f"reindex took {dt.datetime.now() - start}") + if __name__ == '__main__': @@ -468,4 +520,6 @@ def reindex(self): load_dotenv() users = UserManager(RedmineSession.fromenv()) - print(len(users.get_all())) + for teamname in users.get_all_teams(): + team = users.get_team_by_name(teamname) + print(team) From 655f5cb8017da2f1b738dba73fed8b3f0a79fef1 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Sat, 2 Mar 2024 14:46:39 -0800 Subject: [PATCH 06/15] starting ticket refactor. all TCs pass so far --- imap.py | 8 + session.py | 13 +- tickets.py | 508 +++++++++++++++++++++++++++++++++++++++++++++++++++++ users.py | 10 +- 4 files changed, 522 insertions(+), 17 deletions(-) create mode 100644 tickets.py diff --git a/imap.py b/imap.py index a75823e..5c4946b 100755 --- a/imap.py +++ b/imap.py @@ -254,6 +254,14 @@ def handle_message(self, msg_id:str, message:Message): self.redmine.create_ticket(user, subject, message.note, message.attachments) log.info(f"Created new ticket for: {user.login}, {subject}, with {len(message.attachments)} attachments") + """ + # check status + if self.user_mgr.is_blocked(user): + log.debug(f"Rejecting ticket #{ticket.id} based on blocked user {user.login}") + self.reject_ticket(ticket.id) + refresh. + """ + def synchronize(self): try: diff --git a/session.py b/session.py index aa3256e..91c5ddd 100644 --- a/session.py +++ b/session.py @@ -83,17 +83,16 @@ def get(self, query_str:str, user:str=None): # data=json.dumps(data), - def put(self, resource: str, data, user_login: str = None) -> bool: + def put(self, resource: str, data:dict, user_login: str = None) -> None: r = self.session.put(f"{self.url}{resource}", data=data, timeout=TIMEOUT, headers=self.get_headers(user_login)) if r.ok: log.debug(f"PUT {resource}: {data} - {r}") - return True else: - raise RedmineException(f"POST failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) + raise RedmineException(f"POST {resource} by {user_login} failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) - def post(self, resource: str, data, user_login: str = None): + def post(self, resource: str, data:dict, user_login: str = None) -> dict: r = self.session.post(f"{self.url}{resource}", data=data, timeout=TIMEOUT, headers=self.get_headers(user_login)) if r.status_code == 201: @@ -105,15 +104,13 @@ def post(self, resource: str, data, user_login: str = None): raise RedmineException(f"POST failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) - def delete(self, resource: str) -> bool: + def delete(self, resource: str) -> None: r = self.session.delete( url=f"{self.url}{resource}", timeout=TIMEOUT, headers=self.get_headers()) - if r.ok: - return True - else: + if not r.ok: raise RedmineException(f"DELETE failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) diff --git a/tickets.py b/tickets.py new file mode 100644 index 0000000..d34e2d8 --- /dev/null +++ b/tickets.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +"""redmine ticket handling""" + +import datetime as dt +import logging +import re + +from dataclasses import dataclass + +from session import RedmineSession, RedmineException +from users import CustomField, User, Team +import synctime + + +log = logging.getLogger(__name__) + + +ISSUES_RESOURCE="/issues.json" +ISSUE_RESOURCE="/issues/" +DEFAULT_SORT = "status:desc,priority:desc,updated_on:desc" +SCN_PROJECT_ID = 1 # could lookup scn in projects +SYNC_FIELD_NAME = "syncdata" + + +@dataclass +class Ticket(): + """Encapsulates a redmine ticket""" + id: int + login: str + mail: str + custom_fields: dict + admin: bool + firstname: str + lastname: str + mail: str + created_on: dt.datetime + updated_on: dt.datetime + last_login_on: dt.datetime + passwd_changed_on: dt.datetime + twofa_scheme: str + api_key: str = "" + status: int = "" + custom_fields: list[CustomField] + + def __post_init__(self): + self.custom_fields = [CustomField(**field) for field in self.custom_fields] + + + def get_custom_field(self, name: str) -> str: + for field in self.custom_fields: + if field.name == name: + return field.value + + return None + + +@dataclass +class TicketsResult: + """Encapsulates a set of tickets""" + total_count: int + limit: int + offset: int + tickets: list[Ticket] + + def __post_init__(self): + self.tickets = [Ticket(**ticket) for ticket in self.tickets] + + +class TicketManager(): + """manage redmine tickets""" + def __init__(self, session: RedmineSession): + self.session: RedmineSession = session + + + def create(self, user:User, subject, body, attachments=None) -> Ticket: + """create a redmine ticket""" + # https://www.redmine.org/projects/redmine/wiki/Rest_Issues#Creating-an-issue + # would need full param handling to pass that thru discord to get to this invocation + # this would be resolved by a Ticket class to emcapsulate. + + data = { + 'issue': { + 'project_id': SCN_PROJECT_ID, #FIXME hard-coded project ID MOVE project ID to API + 'subject': subject, + 'description': body, + } + } + + if attachments and len(attachments) > 0: + data['issue']['uploads'] = [] + for a in attachments: + data['issue']['uploads'].append({ + "token": a.token, + "filename": a.name, + "content_type": a.content_type, + }) + + response = self.session.post(ISSUES_RESOURCE, data, user.login) + + # check status + if response: + return Ticket(**response['issue']) + else: + raise RedmineException( + f"create_ticket failed, status=[{response.status_code}] {response.reason}", + response.headers['X-Request-Id']) + + + def update(self, ticket_id:str, fields:dict, user_login:str=None) -> Ticket: + """update a redmine ticket""" + # PUT a simple JSON structure + data = { + 'issue': {} + } + + data['issue'] = fields + + response = self.session.put(f"{ISSUE_RESOURCE}{ticket_id}.json", data, user_login) + + log.debug(f"update: [{response.status_code}] {response.request.url}, fields: {fields}") + + # check status + if response.ok: + # no body, so re-get the updated tickets? + return self.get(ticket_id) + else: + raise RedmineException( + f"update_ticket failed, status=[{response.status_code}] {response.reason}", + response.headers['X-Request-Id']) + + + def append_message(self, ticket_id:int, user_login:str, note:str, attachments=None): + """append a note to a ticket""" + # PUT a simple JSON structure + data = { + 'issue': { + 'notes': note, + } + } + + # add the attachments + if attachments and len(attachments) > 0: + data['issue']['uploads'] = [] + for a in attachments: + data['issue']['uploads'].append({ + "token": a.token, + "filename": a.name, + "content_type": a.content_type, + }) + + self.session.put(f"{ISSUE_RESOURCE}{ticket_id}.json", data, user_login) + # no return, excepion thrown in case of failure + + + def upload_file(self, user:User, data:dict, filename:str, content_type) -> str: + """Upload a file to redmine""" + # POST /uploads.json?filename=image.png + # Content-Type: application/octet-stream + # (request body is the file content) + + r = self.session.post(f"/uploads.json?filename={filename}", + files={'upload_file': (filename, data, content_type)}, + user_login=user.login) + + # valid response: {"upload":{"token":"7167.ed1ccdb093229ca1bd0b043618d88743"}} + if r: + # all good, get token + token = r['upload']['token'] + log.info(f"Uploaded {filename}/{content_type}, got token={token}") + return token + else: + raise RedmineException(f"upload failed {filename}: {r.reason}/{r.status_code}", r.headers['X-Request-Id']) + + + def upload_attachments(self, user_id, attachments): + """Upload a list of attachments""" + # uploads all the attachments, + # sets the upload token for each + for a in attachments: + token = self.upload_file(user_id, a.payload, a.name, a.content_type) + a.set_token(token) + + + def get_tickets_by(self, user): + # GET /issues.json?author_id=6 + response = self.session.get(f"/issues.json?author_id={user.id}") + if response: + return response['issues'] + else: + log.debug(f"Unknown user: {user}") + return None + + + def get(self, ticket_id:int, include_journals:bool = False): + """get a ticket by ID""" + if ticket_id is None or ticket_id == 0: + #log.debug(f"Invalid ticket number: {ticket_id}") + return None + + query = f"/issues/{ticket_id}.json" + if include_journals: + query += "?include=journals" # as per https://www.redmine.org/projects/redmine/wiki/Rest_IssueJournals + + response = self.session.get(query) + if response: + return response['issue'] + else: + log.debug(f"Unknown ticket number: {ticket_id}") + return None + + + #GET /issues.json?issue_id=1,2 + def get_tickets(self, ticket_ids: list[int]): + """get several tickets based on a list of IDs""" + if ticket_ids is None or len(ticket_ids) == 0: + log.debug("No ticket numbers supplied to get_tickets.") + return [] + + response = self.session.get(f"/issues.json?issue_id={','.join(ticket_ids)}&status_id=*") # &sort={DEFAULT_SORT} needed? + log.debug(f"query response: {response}") + if response is not None and response.total_count > 0: + return response['issues'] + else: + log.info(f"Unknown ticket numbers: {ticket_ids}") + return [] + + def find_ticket_from_str(self, string:str): + """parse a ticket number from a string and get the associated ticket""" + # for now, this is a trivial REGEX to match '#nnn' in a string, and return ticket #nnn + match = re.search(r'#(\d+)', string) + if match: + ticket_num = int(match.group(1)) + return self.get(ticket_num) + else: + log.debug(f"Unable to match ticket number in: {string}") + return [] + + + def remove_ticket(self, ticket_id:int) -> None: + """delete a ticket in redmine. used for testing""" + # DELETE to /issues/{ticket_id}.json + self.session.delete(f"/issues/{ticket_id}.json") + + + def most_recent_ticket_for(self, user:User): + """get the most recent ticket for the user with the given email""" + # get the user record for the email + if not user: + return None + + # query open tickets created by user, sorted by most recently updated, limit 1 + response = self.session.get(f"/issues.json?author_id={user.id}&status_id=open&sort=updated_on:desc&limit=1") + + if response.total_count > 0: + return response.issues[0] + else: + log.info(f"No recent open ticket found for: {user}") + return None + + + def new_tickets_since(self, timestamp:dt.datetime): + """get new tickets since provided timestamp""" + # query for new tickets since date + # To fetch issues created after a certain timestamp (uncrypted filter is ">=2014-01-02T08:12:32Z") : + # GET /issues.xml?created_on=%3E%3D2014-01-02T08:12:32Z + timestr = dt.datetime.isoformat(timestamp) # time-format. + query = f"/issues.json?created_on=%3E%3D{timestr}&sort={DEFAULT_SORT}&limit=100" + response = self.session.get(query) + + if response.total_count > 0: + return response.issues + else: + log.debug(f"No tickets created since {timestamp}") + return None + + + def find_tickets(self) -> list[Ticket]: + """default ticket query""" + # "kanban" query: all ticket open or closed recently + project=1 + tracker=4 + query = f"/issues.json?project_id={project}&tracker_id={tracker}&status_id=*&sort={DEFAULT_SORT}&limit=100" + response = self.session.get(query) + return TicketsResult(**response).tickets + + + def my_tickets(self, user=None) -> list[Ticket]: + """get my tickets""" + jresp = self.session.get(f"/issues.json?assigned_to_id=me&status_id=open&sort={DEFAULT_SORT}&limit=100", user) + + if not jresp: + return None + + response = TicketsResult(**jresp) + if response.total_count > 0: + return response.tickets + else: + log.info("No open ticket for me.") + return None + + + def tickets_for_team(self, team:Team) -> list[Ticket]: + # validate team? + #team = self.user_mgr.get_by_name(team_str) # find_user is dsigned to be broad + response = self.session.get(f"/issues.json?assigned_to_id={team.id}&status_id=open&sort={DEFAULT_SORT}&limit=100") + + if not response: + return None + + result = TicketsResult(**response) + if result.total_count > 0: + return result.tickets + else: + log.info("No open ticket for me.") + return None + + + def search(self, term) -> list[Ticket]: + """search all text of all tickets (not just open) for the supplied terms""" + # todo url-encode term? + # note: sort doesn't seem to be working for search + query = f"/search.json?q={term}&issues=1&limit=100&sort={DEFAULT_SORT}" + + response = self.session.get(query) + if not response: + return None + + # the response has only IDs.... + ids = [result.id for result in response.results] + # but there's a call to get several tickets + return self.get_tickets(ids) + + + def match_subject(self, subject): + # todo url-encode term? + # note: sort doesn't seem to be working for search + query = f"/search.json?q={subject}&all_words=1&titles_only=1&open_issues=1&limit=100" + + response = self.session.get(query) + if not response: + return None + + # the response has only IDs.... + ids = [result.id for result in response.results] + # but there's a call to get several tickets + return self.get_tickets(ids) + + + def get_notes_since(self, ticket_id:int, timestamp:dt.datetime=None) -> list[str]: + notes = [] + + ticket = self.get(ticket_id, include_journals=True) + log.debug(f"got ticket {ticket_id} with {len(ticket.journals)} notes") + + for note in ticket.journals: + # note.notes is a text field with notes, or empty. if there are no notes, ignore the journal + if note.notes and timestamp: + created = synctime.parse_str(note.created_on) + if created > timestamp: + notes.append(note) + elif note.notes: + notes.append(note) # append all notes when there's no timestamp + + return notes + + + def enable_discord_sync(self, ticket_id, user, note): + fields = { + "note": note, #f"Created Discord thread: {thread.name}: {thread.jump_url}", + "cf_1": "1", + } + + self.update(ticket_id, fields, user.login) + # currently doesn't return or throw anything + # todo: better error reporting back to discord + + + def assign_ticket(self, ticket_id, user:User, user_id=None): + + fields = { + "assigned_to_id": user.id, + #"status_id": "1", # New + } + if user_id is None: + # use the user-id to self-assign + user_id = user.login + self.update(ticket_id, fields, user_id) + + + def progress_ticket(self, ticket_id, user_id=None): # TODO notes + fields = { + "assigned_to_id": "me", + "status_id": "2", # "In Progress" + } + self.update(ticket_id, fields, user_id) + + + def reject_ticket(self, ticket_id, user_id=None): # TODO notes + fields = { + "assigned_to_id": "", + "status_id": "5", # "Reject" + } + self.update(ticket_id, fields, user_id) + + + def unassign_ticket(self, ticket_id, user_id=None): + fields = { + "assigned_to_id": "", # FIXME this *should* be the team it was assigned to, but there's no way to calculate. + "status_id": "1", # New + } + self.update(ticket_id, fields, user_id) + + + def resolve_ticket(self, ticket_id, user_id=None): + self.update(ticket_id, {"status_id": "3"}, user_id) # '3' is the status_id, it doesn't accept "Resolved" + + + def get_sync_record(self, ticket, expected_channel: int) -> synctime.SyncRecord: + # Parse custom_field into datetime + # lookup field by name + token = None + try : + for field in ticket.custom_fields: + if field.name == SYNC_FIELD_NAME: + token = field.value + log.debug(f"found {field.name} => '{field.value}'") + break + except AttributeError: + # custom_fields not set, handle same as no sync field + pass + + if token: + record = synctime.SyncRecord.from_token(ticket.id, token) + log.debug(f"created sync_rec from token: {record}") + if record: + # check channel + if record.channel_id == 0: + # no valid channel set in sync data, assume lagacy + record.channel_id = expected_channel + # update the record in redmine after adding the channel info + self.update_sync_record(record) + return record + elif record.channel_id != expected_channel: + log.debug(f"channel mismatch: rec={record.channel_id} =/= {expected_channel}, token={token}") + return None + else: + return record + else: + # no token implies not-yet-initialized + record = synctime.SyncRecord(ticket.id, expected_channel, synctime.epoch_datetime()) + # apply the new sync record back to redmine + self.update_sync_record(record) + return record + + + def update_sync_record(self, record:synctime.SyncRecord): + log.debug(f"Updating sync record in redmine: {record}") + fields = { + "custom_fields": [ + { "id": 4, "value": record.token_str() } # cf_4, custom field syncdata, #TODO search for it + ] + } + self.update(record.ticket_id, fields) + + + def get_updated_field(self, ticket) -> dt.datetime: + return synctime.parse_str(ticket.updated_on) + + + # NOTE: This implies that ticket should be a full object with methods. + # Starting to move fields out to their own methods, to eventually move to + # their own Ticket class. + def get_field(self, ticket, fieldname): + try: + match fieldname: + case "id": + return f"{ticket.id}" + case "url": + return f"{self.url}/issues/{ticket.id}" + case "link": + return f"[{ticket.id}]({self.url}/issues/{ticket.id})" + case "priority": + return ticket.priority.name + case "updated": + return ticket.updated_on # string, or dt? + case "assigned": + return ticket.assigned_to.name + case "status": + return ticket.status.name + case "subject": + return ticket.subject + case "title": + return ticket.title + #case "age": + # updated = dt.datetime.fromisoformat(ticket.updated_on) ### UTC + # age = dt.datetime.now(dt.timezone.utc) - updated + # return humanize.naturaldelta(age) + #case "sync": + # try: + # # Parse custom_field into datetime + # # FIXME: this is fragile: relies on specific index of custom field, add custom field lookup by name + # timestr = ticket.custom_fields[1].value + # return dt.datetime.fromisoformat(timestr) ### UTC + # except Exception as e: + # log.debug(f"sync tag not set") + # return None + except AttributeError: + return "" # or None? diff --git a/users.py b/users.py index eaa8586..8158793 100644 --- a/users.py +++ b/users.py @@ -192,9 +192,6 @@ def is_user_in_team(self, user:User, teamname:str) -> bool: return False - - - class UserManager(): """manage redmine users""" session: RedmineSession @@ -241,9 +238,7 @@ def update(self, user:User, fields:dict) -> User: log.debug(f"updated id={user.id}: user: {user}") return user else: - raise RedmineException( - f"update failed, status=[{response.status_code}] {response.reason}", - response.headers['X-Request-Id']) + return None def get_by_name(self, username:str) -> User: @@ -332,9 +327,6 @@ def remove(self, user: User): # check status if r: log.info(f"deleted user {user.id}") - else: - log.error(f"Error removing user status={r.status_code}, url={r.request.url}, req_id={r.headers['X-Request-Id']}") - # exception? def create_discord_mapping(self, user:User, discord_name:str) -> User: From c21c69898764738154c0d554b112e09116de9c37 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Sun, 3 Mar 2024 15:50:17 -0800 Subject: [PATCH 07/15] Adding initial test case for tickets.py. Working, but more work needed for more test. This is just the first. --- test_tickets.py | 68 +++++++++++++++++++++++++++++ test_utils.py | 16 +++---- tickets.py | 112 +++++++++++++++++++++++++++++++++++------------- 3 files changed, 158 insertions(+), 38 deletions(-) create mode 100755 test_tickets.py diff --git a/test_tickets.py b/test_tickets.py new file mode 100755 index 0000000..18ea833 --- /dev/null +++ b/test_tickets.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Redmine tickets manager test cases""" + +import unittest +import logging + +from dotenv import load_dotenv + +import session +import tickets +import users +import test_utils + + +log = logging.getLogger(__name__) + + +@unittest.skipUnless(load_dotenv(), "ENV settings not available") +class TestTicketManager(unittest.TestCase): + """Test suite for Redmine ticket manager""" + + def setUp(self): + redmine_seesion = session.RedmineSession.fromenv() + self.tickets_mgr = tickets.TicketManager(redmine_seesion) + self.user_mgr = users.UserManager(redmine_seesion) + + + def test_create_ticket(self): + # create test user + tag = test_utils.tagstr() + subject = f"Test {tag} subject" + body = f"Test {tag} body" + + user = test_utils.create_test_user(self.user_mgr, tag) + + ticket = None + try: + # create ticket + ticket = self.tickets_mgr.create(user, subject, body) + self.assertIsNotNone(ticket) + self.assertEqual(subject, ticket.subject) + self.assertEqual(body, ticket.description) + + check = self.tickets_mgr.get(ticket.id) + self.assertIsNotNone(check) + self.assertEqual(subject, check.subject) + self.assertEqual(body, check.description) + + check2 = self.tickets_mgr.search(subject) # returns list + self.assertIsNotNone(check2) + self.assertEqual(1, len(check2)) + self.assertEqual(ticket.id, check2[0].id) + finally: + # delete ticket + if ticket: + self.tickets_mgr.remove(ticket.id) + check3 = self.tickets_mgr.get(ticket.id) + self.assertIsNone(check3) + + # remove the test user + self.user_mgr.remove(user) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format="{asctime} {levelname:<8s} {name:<16} {message}", style='{') + logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) + + unittest.main() diff --git a/test_utils.py b/test_utils.py index 6c94a78..1c09f93 100644 --- a/test_utils.py +++ b/test_utils.py @@ -13,7 +13,7 @@ import discord from discord import ApplicationContext from redmine import Client -from users import User +from users import User, UserManager log = logging.getLogger(__name__) @@ -47,7 +47,7 @@ def randstr(length:int=12) -> str: return ''.join(random.choices(string.ascii_uppercase + string.digits, k=length)) -def create_test_user(redmine:Client, tag:str): +def create_test_user(user_mgr:UserManager, tag:str): # create new test user name: test-12345@example.com, login test-12345 first = "test-" + tag last = "Testy" @@ -55,22 +55,22 @@ def create_test_user(redmine:Client, tag:str): email = first + "@example.com" # create new redmine user, using redmine api - user = redmine.user_mgr.create(email, first, last) + user = user_mgr.create(email, first, last) # create temp discord mapping with redmine api, assert # create_discord_mapping will cache the new user discord_user = "discord-" + tag ### <-- - redmine.user_mgr.create_discord_mapping(user, discord_user) + user_mgr.create_discord_mapping(user, discord_user) # lookup based on login - return redmine.user_mgr.get_by_name(user.login) + return user_mgr.get_by_name(user.login) -def remove_test_users(redmine:Client): - for user in redmine.user_mgr.get_all(): +def remove_test_users(user_mgr:UserManager): + for user in user_mgr.get_all(): if user.login.startswith("test-") or user.login == "philion@acmerocket.com": log.info(f"Removing test user: {user.login}") - redmine.user_mgr.remove(user) + user_mgr.remove(user) # TODO delete test tickets and "Search for subject match in email threading" ticket. TAG with test too? diff --git a/tickets.py b/tickets.py index d34e2d8..89dccfc 100644 --- a/tickets.py +++ b/tickets.py @@ -4,11 +4,12 @@ import datetime as dt import logging import re +import json from dataclasses import dataclass from session import RedmineSession, RedmineException -from users import CustomField, User, Team +from users import CustomField, User, Team, NamedId import synctime @@ -22,28 +23,66 @@ SYNC_FIELD_NAME = "syncdata" +@dataclass +class TicketStatus(): + """status of a ticket""" + id: int + name: str + id_closed: bool + + +@dataclass +class PropertyChange(): # https://www.redmine.org/projects/redmine/wiki/Rest_IssueJournals + """a documented change in a single property""" + property: str + name: str + old_value: str + new_value: str + + +@dataclass +class TicketNote(): # https://www.redmine.org/projects/redmine/wiki/Rest_IssueJournals + """a message sent to a ticket""" + user: NamedId + notes: str + created_on: dt.datetime + details: list[PropertyChange] + + def __post_init__(self): + if self.details: + self.details = [PropertyChange(**change) for change in self.details] + + @dataclass class Ticket(): """Encapsulates a redmine ticket""" id: int - login: str - mail: str - custom_fields: dict - admin: bool - firstname: str - lastname: str - mail: str + project: NamedId + tracker: NamedId + status: TicketStatus + priority: NamedId + author: NamedId + assigned_to: NamedId + subject: str + description: str + done_ratio: float + is_private: bool + estimated_hours: float + total_estimated_hours: float + start_date: dt.date + due_date: dt.date created_on: dt.datetime updated_on: dt.datetime - last_login_on: dt.datetime - passwd_changed_on: dt.datetime - twofa_scheme: str - api_key: str = "" - status: int = "" - custom_fields: list[CustomField] + closed_on: dt.datetime + spent_hours: float = 0.0 + total_spent_hours: float = 0.0 + custom_fields: list[CustomField] = None + journals: list[PropertyChange] = None + def __post_init__(self): - self.custom_fields = [CustomField(**field) for field in self.custom_fields] + if self.custom_fields: + self.custom_fields = [CustomField(**field) for field in self.custom_fields] def get_custom_field(self, name: str) -> str: @@ -60,10 +99,12 @@ class TicketsResult: total_count: int limit: int offset: int - tickets: list[Ticket] + issues: list[Ticket] + def __post_init__(self): - self.tickets = [Ticket(**ticket) for ticket in self.tickets] + if self.issues: + self.issues = [Ticket(**ticket) for ticket in self.issues] class TicketManager(): @@ -95,11 +136,16 @@ def create(self, user:User, subject, body, attachments=None) -> Ticket: "content_type": a.content_type, }) - response = self.session.post(ISSUES_RESOURCE, data, user.login) + ## NEW response = self.session.post(ISSUES_RESOURCE, data, user.login) + response = self.session.session.post( + url=f"{self.session.url}/issues.json", + data=json.dumps(data), + headers=self.session.get_headers(user.login), + timeout=5) # check status if response: - return Ticket(**response['issue']) + return Ticket(**response.json()['issue']) else: raise RedmineException( f"create_ticket failed, status=[{response.status_code}] {response.reason}", @@ -191,7 +237,7 @@ def get_tickets_by(self, user): return None - def get(self, ticket_id:int, include_journals:bool = False): + def get(self, ticket_id:int, include_journals:bool = False) -> Ticket: """get a ticket by ID""" if ticket_id is None or ticket_id == 0: #log.debug(f"Invalid ticket number: {ticket_id}") @@ -203,23 +249,28 @@ def get(self, ticket_id:int, include_journals:bool = False): response = self.session.get(query) if response: - return response['issue'] + return Ticket(**response['issue']) else: log.debug(f"Unknown ticket number: {ticket_id}") return None #GET /issues.json?issue_id=1,2 - def get_tickets(self, ticket_ids: list[int]): + def get_tickets(self, ticket_ids: list[int]) -> list[Ticket]: """get several tickets based on a list of IDs""" if ticket_ids is None or len(ticket_ids) == 0: log.debug("No ticket numbers supplied to get_tickets.") return [] - response = self.session.get(f"/issues.json?issue_id={','.join(ticket_ids)}&status_id=*") # &sort={DEFAULT_SORT} needed? + response = self.session.get(f"/issues.json?issue_id={','.join(map(str, ticket_ids))}&status_id=*&sort={DEFAULT_SORT}") log.debug(f"query response: {response}") - if response is not None and response.total_count > 0: - return response['issues'] + if response: + result = TicketsResult(**response) + if result.total_count > 0: + return result.issues + else: + return [] + else: log.info(f"Unknown ticket numbers: {ticket_ids}") return [] @@ -236,7 +287,7 @@ def find_ticket_from_str(self, string:str): return [] - def remove_ticket(self, ticket_id:int) -> None: + def remove(self, ticket_id:int) -> None: """delete a ticket in redmine. used for testing""" # DELETE to /issues/{ticket_id}.json self.session.delete(f"/issues/{ticket_id}.json") @@ -281,7 +332,7 @@ def find_tickets(self) -> list[Ticket]: tracker=4 query = f"/issues.json?project_id={project}&tracker_id={tracker}&status_id=*&sort={DEFAULT_SORT}&limit=100" response = self.session.get(query) - return TicketsResult(**response).tickets + return TicketsResult(**response).issues def my_tickets(self, user=None) -> list[Ticket]: @@ -293,7 +344,7 @@ def my_tickets(self, user=None) -> list[Ticket]: response = TicketsResult(**jresp) if response.total_count > 0: - return response.tickets + return response.issues else: log.info("No open ticket for me.") return None @@ -309,7 +360,7 @@ def tickets_for_team(self, team:Team) -> list[Ticket]: result = TicketsResult(**response) if result.total_count > 0: - return result.tickets + return result.issues else: log.info("No open ticket for me.") return None @@ -326,7 +377,8 @@ def search(self, term) -> list[Ticket]: return None # the response has only IDs.... - ids = [result.id for result in response.results] + log.debug(f"SEARCH {response}") + ids = [result['id'] for result in response['results']] # but there's a call to get several tickets return self.get_tickets(ids) From 138943fd81ca666dc67e0a41b937062f053721ae Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Sun, 3 Mar 2024 17:07:04 -0800 Subject: [PATCH 08/15] slow progress on migrating tickets --- session.py | 8 +++++--- test_redmine.py | 34 ++++++++++------------------------ test_tickets.py | 29 +++++++++-------------------- test_utils.py | 40 +++++++++++++++++++++++++++++++++------- tickets.py | 9 ++------- 5 files changed, 59 insertions(+), 61 deletions(-) diff --git a/session.py b/session.py index 91c5ddd..6765494 100644 --- a/session.py +++ b/session.py @@ -72,7 +72,7 @@ def get(self, query_str:str, user:str=None): if r.ok: return r.json() else: - log.error(f"GET {r.status_code} for {r.request.url}, reqid={r.headers['X-Request-Id']}: {r}") + log.info(f"GET {r.reason}/{r.status_code} url={r.request.url}, reqid={r.headers['X-Request-Id']}") except TimeoutError as toe: # ticket-509: Handle timeout gracefully log.warning(f"Timeout during {query_str}: {toe}") @@ -93,8 +93,10 @@ def put(self, resource: str, data:dict, user_login: str = None) -> None: def post(self, resource: str, data:dict, user_login: str = None) -> dict: - r = self.session.post(f"{self.url}{resource}", data=data, timeout=TIMEOUT, - headers=self.get_headers(user_login)) + r = self.session.post(f"{self.url}{resource}", + data=data, + timeout=TIMEOUT, + headers=self.get_headers(user_login)) if r.status_code == 201: #log.debug(f"POST {resource}: {data} - {vars(r)}") return r.json() diff --git a/test_redmine.py b/test_redmine.py index d201b32..26f569f 100755 --- a/test_redmine.py +++ b/test_redmine.py @@ -14,47 +14,33 @@ @unittest.skipUnless(load_dotenv(), "ENV settings not available") -class TestRedmine(unittest.TestCase): +class TestRedmine(test_utils.RedmineTestCase): """Test suite for Redmine client""" - def setUp(self): - self.redmine = redmine.Client.fromenv() - - def test_blocked_user(self): - # create test user - tag = test_utils.tagstr() - user = test_utils.create_test_user(self.redmine, tag) - # block - self.redmine.user_mgr.block(user) - self.assertTrue(self.redmine.user_mgr.is_blocked(user)) + self.user_mgr.block(self.user) + self.assertTrue(self.user_mgr.is_blocked(self.user)) # unblock - self.redmine.user_mgr.unblock(user) - self.assertFalse(self.redmine.user_mgr.is_blocked(user)) - - # remove the test user - self.redmine.user_mgr.remove(user) - + self.user_mgr.unblock(self.user) + self.assertFalse(self.user_mgr.is_blocked(self.user)) + """ def test_blocked_create_ticket(self): - # create test user - tag = test_utils.tagstr() - user = test_utils.create_test_user(self.redmine, tag) - try: # block - self.redmine.user_mgr.block(user) - self.assertTrue(self.redmine.user_mgr.is_blocked(user)) + self.user_mgr.block(self.user) + self.assertTrue(self.redmine.user_mgr.is_blocked(self.user)) # create ticket for blocked - ticket = self.redmine.create_ticket(user, "subject", "body") + ticket = self.create_ticket(self.user, "subject", "body") self.assertEqual("Reject", ticket.status.name) finally: # remove the test user self.redmine.user_mgr.remove(user) + """ def test_client_timeout(self): diff --git a/test_tickets.py b/test_tickets.py index 18ea833..74df5f3 100755 --- a/test_tickets.py +++ b/test_tickets.py @@ -6,9 +6,7 @@ from dotenv import load_dotenv -import session -import tickets -import users + import test_utils @@ -16,27 +14,18 @@ @unittest.skipUnless(load_dotenv(), "ENV settings not available") -class TestTicketManager(unittest.TestCase): +class TestTicketManager(test_utils.RedmineTestCase): """Test suite for Redmine ticket manager""" - def setUp(self): - redmine_seesion = session.RedmineSession.fromenv() - self.tickets_mgr = tickets.TicketManager(redmine_seesion) - self.user_mgr = users.UserManager(redmine_seesion) - - def test_create_ticket(self): # create test user - tag = test_utils.tagstr() - subject = f"Test {tag} subject" - body = f"Test {tag} body" - - user = test_utils.create_test_user(self.user_mgr, tag) + subject = f"Test {self.tag} subject" + body = f"Test {self.tag} body" ticket = None try: # create ticket - ticket = self.tickets_mgr.create(user, subject, body) + ticket = self.tickets_mgr.create(self.user, subject, body) self.assertIsNotNone(ticket) self.assertEqual(subject, ticket.subject) self.assertEqual(body, ticket.description) @@ -57,12 +46,12 @@ def test_create_ticket(self): check3 = self.tickets_mgr.get(ticket.id) self.assertIsNone(check3) - # remove the test user - self.user_mgr.remove(user) - if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG, format="{asctime} {levelname:<8s} {name:<16} {message}", style='{') + logging.basicConfig(level=logging.DEBUG, + format="{asctime} {levelname:<8s} {name:<16} {message}", + datefmt='%Y-%m-%d %H:%M:%S', + style='{') logging.getLogger("urllib3.connectionpool").setLevel(logging.INFO) unittest.main() diff --git a/test_utils.py b/test_utils.py index 1c09f93..7d02f7c 100644 --- a/test_utils.py +++ b/test_utils.py @@ -12,9 +12,10 @@ import discord from discord import ApplicationContext -from redmine import Client from users import User, UserManager - +import session +import tickets +import redmine log = logging.getLogger(__name__) @@ -75,18 +76,42 @@ def remove_test_users(user_mgr:UserManager): # TODO delete test tickets and "Search for subject match in email threading" ticket. TAG with test too? +class RedmineTestCase(unittest.TestCase): + """Abstract base class for testing redmine features""" + user_mgr: UserManager + tickets_mgr: tickets.TicketManager + tag: str + user: User + + + @classmethod + def setUpClass(cls): + sess = session.RedmineSession.fromenv() + cls.user_mgr = UserManager(sess) + cls.tickets_mgr = tickets.TicketManager(sess) + cls.tag:str = tagstr() + cls.user:User = create_test_user(cls.user_mgr, cls.tag) + log.info(f"SETUP created test user: {cls.user}") + + + @classmethod + def tearDownClass(cls): + cls.user_mgr.remove(cls.user) + log.info(f"TEARDOWN removed test user: {cls.user}") + + class BotTestCase(unittest.IsolatedAsyncioTestCase): """Abstract base class for testing Bot features""" - redmine: Client = None + redmine: session.RedmineSession = None usertag: str = None user: User = None @classmethod def setUpClass(cls): log.info("Setting up test fixtures") - cls.redmine:Client = Client.fromenv() + cls.redmine:redmine.Client = redmine.Client.fromenv() cls.usertag:str = tagstr() - cls.user:User = create_test_user(cls.redmine, cls.usertag) + cls.user:User = create_test_user(cls.redmine.user_mgr, cls.usertag) log.info(f"Created test user: {cls.user}") @@ -133,5 +158,6 @@ def setUp(self): load_dotenv() # construct the client and run the email check - client = Client.fromenv() - remove_test_users(client) + client = session.RedmineSession.fromenv() + users = UserManager(client) + remove_test_users(users) diff --git a/tickets.py b/tickets.py index 89dccfc..f010ecb 100644 --- a/tickets.py +++ b/tickets.py @@ -136,16 +136,11 @@ def create(self, user:User, subject, body, attachments=None) -> Ticket: "content_type": a.content_type, }) - ## NEW response = self.session.post(ISSUES_RESOURCE, data, user.login) - response = self.session.session.post( - url=f"{self.session.url}/issues.json", - data=json.dumps(data), - headers=self.session.get_headers(user.login), - timeout=5) + response = self.session.post(ISSUES_RESOURCE, json.dumps(data), user.login) # check status if response: - return Ticket(**response.json()['issue']) + return Ticket(**response['issue']) else: raise RedmineException( f"create_ticket failed, status=[{response.status_code}] {response.reason}", From 406b95042af76c3a21ee2e26c0f90c0571438a45 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Sun, 3 Mar 2024 17:12:28 -0800 Subject: [PATCH 09/15] slow progress on migrating tickets --- redmine.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/redmine.py b/redmine.py index 94f49cc..23cae16 100644 --- a/redmine.py +++ b/redmine.py @@ -14,6 +14,8 @@ import synctime from session import RedmineSession from users import UserResult, UserManager +from tickets import TicketManager + log = logging.getLogger(__name__) @@ -47,6 +49,7 @@ def __init__(self, url: str, token: str): session:RedmineSession = RedmineSession(url, token) self.user_mgr:UserManager = UserManager(session) + self.ticket_mgr:TicketManager = TicketManager(session) @classmethod @@ -57,6 +60,9 @@ def fromenv(cls): def create_ticket(self, user, subject, body, attachments=None): + return self.ticket_mgr.create(user, subject, body, attachments) + + def DELETE_create_ticket(self, user, subject, body, attachments=None): """create a redmine ticket""" # https://www.redmine.org/projects/redmine/wiki/Rest_Issues#Creating-an-issue # would need full param handling to pass that thru discord to get to this invocation From 6f906f38e28841ab948ae6bbc959f46ee56595ff Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Sun, 3 Mar 2024 18:33:53 -0800 Subject: [PATCH 10/15] some stability after getting create ticket. there was a lot of fall out from returning an object vs dict. --- cog_tickets.py | 2 +- netbot.py | 12 ++++-- redmine.py | 11 ++++-- test_cog_tickets.py | 6 +-- test_synctime.py | 3 +- tickets.py | 89 ++++++++++++++++++++++++++++++++++----------- users.py | 3 ++ 7 files changed, 90 insertions(+), 36 deletions(-) diff --git a/cog_tickets.py b/cog_tickets.py index 69c2879..7e724d6 100644 --- a/cog_tickets.py +++ b/cog_tickets.py @@ -219,5 +219,5 @@ def format_ticket(self, ticket, fields=None): fields = ["link","priority","updated","assigned","subject"] for field in fields: - section += self.redmine.get_field(ticket, field) + " " # spacer, one space + section += str(self.redmine.get_field(ticket, field)) + " " # spacer, one space return section.strip() # remove trailing whitespace diff --git a/netbot.py b/netbot.py index 8253205..daa177b 100755 --- a/netbot.py +++ b/netbot.py @@ -9,6 +9,7 @@ from dotenv import load_dotenv from discord.ext import commands +from tickets import TicketNote import synctime import redmine @@ -91,12 +92,13 @@ async def gather_discord_notes(self, thread: discord.Thread, sync_rec:synctime.S def format_discord_note(self, note): """Format a note for Discord""" - age = synctime.age_str(synctime.parse_str(note.created_on)) - message = f"> **{note.user.name}** *{age} ago*\n> {note.notes}"[:MAX_MESSAGE_LEN] + age = synctime.age_str(note.created_on) + log.info(f"### {note} {age} {note.user}") + message = f"> **{note.user}** *{age} ago*\n> {note.notes}"[:MAX_MESSAGE_LEN] return message - def gather_redmine_notes(self, ticket, sync_rec:synctime.SyncRecord): + def gather_redmine_notes(self, ticket, sync_rec:synctime.SyncRecord) -> list[TicketNote]: notes = [] # get the new notes from the redmine ticket redmine_notes = self.redmine.get_notes_since(ticket.id, sync_rec.last_sync) @@ -149,7 +151,9 @@ async def synchronize_ticket(self, ticket, thread:discord.Thread) -> bool: # start of the process, will become "last update" sync_start = synctime.now() - sync_rec = self.redmine.get_sync_record(ticket, expected_channel=thread.id) + #sync_rec = self.redmine.get_sync_record(ticket, expected_channel=thread.id) + sync_rec = ticket.get_sync_record(expected_channel=thread.id) + if sync_rec: log.debug(f"sync record: {sync_rec}") diff --git a/redmine.py b/redmine.py index 23cae16..03d74ad 100644 --- a/redmine.py +++ b/redmine.py @@ -254,6 +254,9 @@ def get_tickets_by(self, user): def get_ticket(self, ticket_id:int, include_journals:bool = False): + return self.ticket_mgr.get(ticket_id, include_journals) + + def DELETE_get_ticket(self, ticket_id:int, include_journals:bool = False): """get a ticket by ID""" if ticket_id is None or ticket_id == 0: #log.debug(f"Invalid ticket number: {ticket_id}") @@ -409,8 +412,11 @@ def match_subject(self, subject): log.debug(f"subject matched nothing: {subject}") return [] - # get the + def get_notes_since(self, ticket_id, timestamp=None): + return self.ticket_mgr.get_notes_since(ticket_id, timestamp) + + def DELETE_get_notes_since(self, ticket_id, timestamp=None): notes = [] ticket = self.get_ticket(ticket_id, include_journals=True) @@ -592,9 +598,6 @@ def update_sync_record(self, record:synctime.SyncRecord): } self.update_ticket(record.ticket_id, fields) - def get_updated_field(self, ticket) -> dt.datetime: - return synctime.parse_str(ticket.updated_on) - # NOTE: This implies that ticket should be a full object with methods. # Starting to move fields out to their own methods, to eventually move to diff --git a/test_cog_tickets.py b/test_cog_tickets.py index 56088ce..0106be4 100755 --- a/test_cog_tickets.py +++ b/test_cog_tickets.py @@ -68,7 +68,7 @@ async def test_new_ticket(self): response_str = ctx.respond.call_args.args[0] self.assertIn(ticket_id, response_str) self.assertIn(url, response_str) - self.assertIn(self.full_name, response_str) + self.assertIn(test_title, response_str) # "progress" the ticket, setting it in-progress and assigning it to "me" ctx = self.build_context() @@ -76,7 +76,7 @@ async def test_new_ticket(self): response_str = ctx.respond.call_args.args[0] self.assertIn(ticket_id, response_str) self.assertIn(url, response_str) - self.assertIn(self.full_name, response_str) + self.assertIn(test_title, response_str) # resolve the ticket ctx = self.build_context() @@ -84,7 +84,7 @@ async def test_new_ticket(self): response_str = ctx.respond.call_args.args[0] self.assertIn(ticket_id, response_str) self.assertIn(url, response_str) - self.assertIn(self.full_name, response_str) + self.assertIn(test_title, response_str) # delete ticket with redmine api, assert self.redmine.remove_ticket(int(ticket_id)) diff --git a/test_synctime.py b/test_synctime.py index 23bd686..69047b0 100755 --- a/test_synctime.py +++ b/test_synctime.py @@ -30,7 +30,6 @@ def test_redmine_times(self): self.assertIsNotNone(user) subject = f"TEST ticket {tag}" ticket = self.redmine.create_ticket(user, subject, f"This for {self.id}-{tag}") - updated = self.redmine.get_updated_field(ticket) test_channel = 4321 sync_rec = self.redmine.get_sync_record(ticket, expected_channel=test_channel) @@ -44,7 +43,7 @@ def test_redmine_times(self): # refetch ticket ticket2 = self.redmine.get_ticket(ticket.id) sync_rec2 = self.redmine.get_sync_record(ticket2, expected_channel=1111) # NOT the test_channel - log.info(f"ticket updated={updated}, {synctime.age(updated)} ago, sync: {sync_rec}") + log.info(f"ticket updated={ticket.updated_on}, {synctime.age(ticket.updated_on)} ago, sync: {sync_rec}") self.assertIsNone(sync_rec2) diff --git a/tickets.py b/tickets.py index f010ecb..d9a0a03 100644 --- a/tickets.py +++ b/tickets.py @@ -28,7 +28,7 @@ class TicketStatus(): """status of a ticket""" id: int name: str - id_closed: bool + is_closed: bool @dataclass @@ -43,12 +43,16 @@ class PropertyChange(): # https://www.redmine.org/projects/redmine/wiki/Rest_Iss @dataclass class TicketNote(): # https://www.redmine.org/projects/redmine/wiki/Rest_IssueJournals """a message sent to a ticket""" + id: int user: NamedId notes: str created_on: dt.datetime + private_notes: bool details: list[PropertyChange] def __post_init__(self): + self.user = NamedId(**self.user) + self.created_on = synctime.parse_str(self.created_on) if self.details: self.details = [PropertyChange(**change) for change in self.details] @@ -62,7 +66,6 @@ class Ticket(): status: TicketStatus priority: NamedId author: NamedId - assigned_to: NamedId subject: str description: str done_ratio: float @@ -76,23 +79,76 @@ class Ticket(): closed_on: dt.datetime spent_hours: float = 0.0 total_spent_hours: float = 0.0 + assigned_to: NamedId = None custom_fields: list[CustomField] = None - journals: list[PropertyChange] = None + journals: list[TicketNote] = None def __post_init__(self): + self.status = TicketStatus(**self.status) + if self.created_on: + self.created_on = synctime.parse_str(self.created_on) + if self.updated_on: + self.updated_on = synctime.parse_str(self.updated_on) + if self.closed_on: + self.closed_on = synctime.parse_str(self.closed_on) + if self.start_date: + self.start_date = synctime.parse_str(self.start_date) + if self.due_date: + self.due_date = synctime.parse_str(self.due_date) if self.custom_fields: self.custom_fields = [CustomField(**field) for field in self.custom_fields] - + if self.journals: + self.journals = [TicketNote(**note) for note in self.journals] def get_custom_field(self, name: str) -> str: - for field in self.custom_fields: - if field.name == name: - return field.value - + if self.custom_fields: + for field in self.custom_fields: + if field.name == name: + return field.value return None + def get_sync_record(self, expected_channel: int) -> synctime.SyncRecord: + # Parse custom_field into datetime + # lookup field by name + token = self.get_custom_field(SYNC_FIELD_NAME) + if token: + record = synctime.SyncRecord.from_token(self.id, token) + log.debug(f"created sync_rec from token: {record}") + if record: + # check channel + if record.channel_id == 0: + # no valid channel set in sync data, assume lagacy + record.channel_id = expected_channel + # update the record in redmine after adding the channel info + # self.update_sync_record(record) REALLY needed? should be handled when token created + return record + elif record.channel_id != expected_channel: + log.debug(f"channel mismatch: rec={record.channel_id} =/= {expected_channel}, token={token}") + return None + else: + return record + else: + # no token implies not-yet-initialized + record = synctime.SyncRecord(self.id, expected_channel, synctime.epoch_datetime()) + # apply the new sync record back to redmine + # self.update_sync_record(record) same REALLY as above ^^^^ + return record + + + def get_notes(self, since:dt.datetime=None) -> list[TicketNote]: + notes = [] + + for note in self.journals: + # note.notes is a text field with notes, or empty. if there are no notes, ignore the journal + if note.notes: + if not since or since < note.created_on: + notes.append(note) + + return notes + + @dataclass class TicketsResult: """Encapsulates a set of tickets""" @@ -393,22 +449,11 @@ def match_subject(self, subject): return self.get_tickets(ids) - def get_notes_since(self, ticket_id:int, timestamp:dt.datetime=None) -> list[str]: - notes = [] - + def get_notes_since(self, ticket_id:int, timestamp:dt.datetime=None) -> list[TicketNote]: + # get the ticket, with journals ticket = self.get(ticket_id, include_journals=True) log.debug(f"got ticket {ticket_id} with {len(ticket.journals)} notes") - - for note in ticket.journals: - # note.notes is a text field with notes, or empty. if there are no notes, ignore the journal - if note.notes and timestamp: - created = synctime.parse_str(note.created_on) - if created > timestamp: - notes.append(note) - elif note.notes: - notes.append(note) # append all notes when there's no timestamp - - return notes + return ticket.get_notes(since=timestamp) def enable_discord_sync(self, ticket_id, user, note): diff --git a/users.py b/users.py index 8158793..b95b686 100644 --- a/users.py +++ b/users.py @@ -31,6 +31,9 @@ class NamedId: id: int name: str + def __str__(self) -> str: + return self.name + @dataclass class Team: From 8c564a6bc3174d11519f70dbce487fa84f806fe3 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Sun, 3 Mar 2024 20:30:19 -0800 Subject: [PATCH 11/15] another step closer --- imap.py | 4 +- redmine.py | 143 ++++----------------------------------------------- session.py | 42 +++++++++++++-- test_imap.py | 3 +- tickets.py | 37 +++---------- 5 files changed, 59 insertions(+), 170 deletions(-) diff --git a/imap.py b/imap.py index 5c4946b..fd42d6f 100755 --- a/imap.py +++ b/imap.py @@ -36,8 +36,8 @@ def __init__(self, name:str, content_type:str, payload): self.payload = payload self.token = None - def upload(self, client, user_id): - self.token = client.upload_file(user_id, self.payload, self.name, self.content_type) + def upload(self, client, user): + self.token = client.upload_file(user, self.payload, self.name, self.content_type) def set_token(self, token): self.token = token diff --git a/redmine.py b/redmine.py index 03d74ad..12e5cf7 100644 --- a/redmine.py +++ b/redmine.py @@ -13,7 +13,7 @@ import synctime from session import RedmineSession -from users import UserResult, UserManager +from users import User, UserResult, UserManager from tickets import TicketManager @@ -62,123 +62,19 @@ def fromenv(cls): def create_ticket(self, user, subject, body, attachments=None): return self.ticket_mgr.create(user, subject, body, attachments) - def DELETE_create_ticket(self, user, subject, body, attachments=None): - """create a redmine ticket""" - # https://www.redmine.org/projects/redmine/wiki/Rest_Issues#Creating-an-issue - # would need full param handling to pass that thru discord to get to this invocation - # this would be resolved by a Ticket class to emcapsulate. - - data = { - 'issue': { - 'project_id': SCN_PROJECT_ID, #FIXME hard-coded project ID - 'subject': subject, - 'description': body, - } - } - if attachments and len(attachments) > 0: - data['issue']['uploads'] = [] - for a in attachments: - data['issue']['uploads'].append({ - "token": a.token, - "filename": a.name, - "content_type": a.content_type, - }) - - response = requests.post( - url=f"{self.url}/issues.json", - data=json.dumps(data), - headers=self.get_headers(user.login), - timeout=TIMEOUT) - - # check status - if response.ok: - root = json.loads(response.text, object_hook= lambda x: SimpleNamespace(**x)) - ticket = root.issue - - # ticket 484 - http://10.10.0.218/issues/484 - # if the user is blocked, "reject" the new ticket - if self.user_mgr.is_blocked(user): - log.debug(f"Rejecting ticket #{ticket.id} based on blocked user {user.login}") - self.reject_ticket(ticket.id) - return self.get_ticket(ticket.id) # refresh the ticket? - else: - return ticket - else: - raise RedmineException( - f"create_ticket failed, status=[{response.status_code}] {response.reason}", - response.headers['X-Request-Id']) + def update_ticket(self, ticket_id:int, fields:dict, user_login:str=None): + return self.ticket_mgr.update(ticket_id, fields, user_login) - def update_ticket(self, ticket_id:str, fields:dict, user_login:str=None): - """update a redmine ticket""" - # PUT a simple JSON structure - data = { - 'issue': {} - } - - data['issue'] = fields + def append_message(self, ticket_id:int, user_login:str, note:str, attachments=None): # Could be TicketNote + return self.ticket_mgr.append_message(ticket_id, user_login, note, attachments) - response = requests.put( - url=f"{self.url}/issues/{ticket_id}.json", - timeout=TIMEOUT, - data=json.dumps(data), - headers=self.get_headers(user_login)) - # ASIDE: this is a great example of lint standards that just make the code more difficult - # to read. There are no good answers for string-too-long. - log.debug( - f"update ticket: [{response.status_code}] {response.request.url}, fields: {fields}") + def upload_file(self, user:User, data, filename, content_type) -> str: + return self.ticket_mgr.upload_file(user, data, filename, content_type) - # check status - if response.ok: - # no body, so re-get the updated tickets? - return self.get_ticket(ticket_id) - else: - raise RedmineException( - f"update_ticket failed, status=[{response.status_code}] {response.reason}", - response.headers['X-Request-Id']) - - - def append_message(self, ticket_id:int, user_login:str, note:str, attachments=None): - """append a note to a ticket""" - # PUT a simple JSON structure - data = { - 'issue': { - 'notes': note, - } - } - - # add the attachments - if attachments and len(attachments) > 0: - data['issue']['uploads'] = [] - for a in attachments: - data['issue']['uploads'].append({ - "token": a.token, - "filename": a.name, - "content_type": a.content_type, - }) - - r = requests.put( - url=f"{self.url}/issues/{ticket_id}.json", - timeout=TIMEOUT, - data=json.dumps(data), - headers=self.get_headers(user_login)) - - # check status - if r.status_code == 204: - # all good - pass - elif r.status_code == 403: - # no access - #print(f"#### {vars(r)}") - log.error(f"{user_login} has no access to add note to ticket #{ticket_id}, req-id={r.headers['X-Request-Id']}") - else: - log.error(f"append_message, status={r.status_code}: {r.reason}, req-id={r.headers['X-Request-Id']}") - #TODO throw exception to show update failed, and why - - - def upload_file(self, user_id, data, filename, content_type): + def XXXupload_file(self, user_id, data, filename, content_type): """Upload a file to redmine""" # POST /uploads.json?filename=image.png # Content-Type: application/octet-stream @@ -210,17 +106,16 @@ def upload_file(self, user_id, data, filename, content_type): # todo throw exception #TODO throw exception to show upload failed, and why - def upload_attachments(self, user_id, attachments): + + def upload_attachments(self, user:User, attachments): """Upload a list of attachments""" # uploads all the attachments, # sets the upload token for each for a in attachments: - token = self.upload_file(user_id, a.payload, a.name, a.content_type) + token = self.upload_file(user, a.payload, a.name, a.content_type) a.set_token(token) - - def lookup_user(self, username:str): """Get a user based on ID, directly from redmine""" if username is None or len(username) == 0: @@ -256,22 +151,6 @@ def get_tickets_by(self, user): def get_ticket(self, ticket_id:int, include_journals:bool = False): return self.ticket_mgr.get(ticket_id, include_journals) - def DELETE_get_ticket(self, ticket_id:int, include_journals:bool = False): - """get a ticket by ID""" - if ticket_id is None or ticket_id == 0: - #log.debug(f"Invalid ticket number: {ticket_id}") - return None - - query = f"/issues/{ticket_id}.json" - if include_journals: - query += "?include=journals" # as per https://www.redmine.org/projects/redmine/wiki/Rest_IssueJournals - - response = self.query(query) - if response: - return response.issue - else: - log.debug(f"Unknown ticket number: {ticket_id}") - return None #GET /issues.xml?issue_id=1,2 def get_tickets(self, ticket_ids): diff --git a/session.py b/session.py index 6765494..7935189 100644 --- a/session.py +++ b/session.py @@ -84,17 +84,20 @@ def get(self, query_str:str, user:str=None): # data=json.dumps(data), def put(self, resource: str, data:dict, user_login: str = None) -> None: - r = self.session.put(f"{self.url}{resource}", data=data, timeout=TIMEOUT, - headers=self.get_headers(user_login)) + r = self.session.put(f"{self.url}{resource}", + data=data, + timeout=TIMEOUT, + headers=self.get_headers(user_login)) if r.ok: log.debug(f"PUT {resource}: {data} - {r}") else: raise RedmineException(f"POST {resource} by {user_login} failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) - def post(self, resource: str, data:dict, user_login: str = None) -> dict: + def post(self, resource: str, data:dict = None, user_login: str = None, files = None) -> dict: r = self.session.post(f"{self.url}{resource}", data=data, + files=files, timeout=TIMEOUT, headers=self.get_headers(user_login)) if r.status_code == 201: @@ -116,7 +119,7 @@ def delete(self, resource: str) -> None: raise RedmineException(f"DELETE failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) - def upload_file(self, user_id, data, filename, content_type): + def xxxxupload_file(self, user_login: str, data, filename, content_type): """Upload a file to redmine""" # POST /uploads.json?filename=image.png # Content-Type: application/octet-stream @@ -126,7 +129,7 @@ def upload_file(self, user_id, data, filename, content_type): 'User-Agent': 'netbot/0.0.1', # TODO update to project version, and add version management 'Content-Type': 'application/octet-stream', # <-- VERY IMPORTANT 'X-Redmine-API-Key': self.token, - 'X-Redmine-Switch-User': user_id, # Make sure the comment is noted by the correct user + 'X-Redmine-Switch-User': user_login, # Make sure the comment is noted by the correct user } r = self.session.post( @@ -144,3 +147,32 @@ def upload_file(self, user_id, data, filename, content_type): return token else: raise RedmineException(f"UPLOAD failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) + + def upload_file(self, user_login:str, data, filename:str, content_type:str): + """Upload a file to redmine""" + # POST /uploads.json?filename=image.png + # Content-Type: application/octet-stream + # (request body is the file content) + + headers = { + 'User-Agent': 'netbot/0.0.1', # TODO update to project version, and add version management + 'Content-Type': 'application/octet-stream', # <-- VERY IMPORTANT + 'X-Redmine-API-Key': self.token, + 'X-Redmine-Switch-User': user_login, # Make sure the comment is noted by the correct user + } + + r = self.session.post( + url=f"{self.url}/uploads.json?filename={filename}", + timeout=TIMEOUT, + files={ 'upload_file': (filename, data, content_type) }, + headers=headers) + + # 201 response: {"upload":{"token":"7167.ed1ccdb093229ca1bd0b043618d88743"}} + if r.status_code == 201: + # all good, get token + #root = json.loads(r.text, object_hook= lambda x: SimpleNamespace(**x)) + token = r.json()['upload']['token'] + log.info(f"Uploaded {filename} {content_type}, got token={token}") + return token + else: + raise RedmineException(f"UPLOAD {r.request.url} {r.reason}/{r.status_code} - {filename}/{content_type}", r.headers['X-Request-Id']) diff --git a/test_imap.py b/test_imap.py index 514e266..aa51334 100755 --- a/test_imap.py +++ b/test_imap.py @@ -60,7 +60,8 @@ def test_email_address_parsing(self): def test_upload(self): with open("test/message-161.eml", 'rb') as file: message = self.imap.parse_message(file.read()) - self.redmine.upload_attachments("philion", message.attachments) + user = self.redmine.user_mgr.get_by_name('admin') + self.redmine.upload_attachments(user, message.attachments) def test_doctype_head(self): diff --git a/tickets.py b/tickets.py index d9a0a03..633046d 100644 --- a/tickets.py +++ b/tickets.py @@ -212,18 +212,10 @@ def update(self, ticket_id:str, fields:dict, user_login:str=None) -> Ticket: data['issue'] = fields - response = self.session.put(f"{ISSUE_RESOURCE}{ticket_id}.json", data, user_login) - - log.debug(f"update: [{response.status_code}] {response.request.url}, fields: {fields}") - - # check status - if response.ok: + response = self.session.put(f"{ISSUE_RESOURCE}{ticket_id}.json", json.dumps(data), user_login) + if response: # no body, so re-get the updated tickets? return self.get(ticket_id) - else: - raise RedmineException( - f"update_ticket failed, status=[{response.status_code}] {response.reason}", - response.headers['X-Request-Id']) def append_message(self, ticket_id:int, user_login:str, note:str, attachments=None): @@ -245,36 +237,21 @@ def append_message(self, ticket_id:int, user_login:str, note:str, attachments=No "content_type": a.content_type, }) - self.session.put(f"{ISSUE_RESOURCE}{ticket_id}.json", data, user_login) + self.session.put(f"{ISSUE_RESOURCE}{ticket_id}.json", json.dumps(data), user_login) # no return, excepion thrown in case of failure - def upload_file(self, user:User, data:dict, filename:str, content_type) -> str: + def upload_file(self, user:User, data, filename:str, content_type) -> str: """Upload a file to redmine""" - # POST /uploads.json?filename=image.png - # Content-Type: application/octet-stream - # (request body is the file content) - - r = self.session.post(f"/uploads.json?filename={filename}", - files={'upload_file': (filename, data, content_type)}, - user_login=user.login) - - # valid response: {"upload":{"token":"7167.ed1ccdb093229ca1bd0b043618d88743"}} - if r: - # all good, get token - token = r['upload']['token'] - log.info(f"Uploaded {filename}/{content_type}, got token={token}") - return token - else: - raise RedmineException(f"upload failed {filename}: {r.reason}/{r.status_code}", r.headers['X-Request-Id']) + return self.session.upload_file(user.login, data, filename, content_type) - def upload_attachments(self, user_id, attachments): + def upload_attachments(self, user:User, attachments): """Upload a list of attachments""" # uploads all the attachments, # sets the upload token for each for a in attachments: - token = self.upload_file(user_id, a.payload, a.name, a.content_type) + token = self.upload_file(user.login, a.payload, a.name, a.content_type) a.set_token(token) From 14b1a50ae20fb32324eb728a3baedad5ddca5cb8 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Sun, 3 Mar 2024 21:04:16 -0800 Subject: [PATCH 12/15] checkpoint. test pass. more work in redmine, but it's late --- redmine.py | 187 ++++++--------------------------------------------- test_imap.py | 3 +- tickets.py | 15 +++-- 3 files changed, 32 insertions(+), 173 deletions(-) diff --git a/redmine.py b/redmine.py index 12e5cf7..de46c6f 100644 --- a/redmine.py +++ b/redmine.py @@ -14,7 +14,7 @@ import synctime from session import RedmineSession from users import User, UserResult, UserManager -from tickets import TicketManager +from tickets import Ticket, TicketManager log = logging.getLogger(__name__) @@ -74,48 +74,12 @@ def append_message(self, ticket_id:int, user_login:str, note:str, attachments=No def upload_file(self, user:User, data, filename, content_type) -> str: return self.ticket_mgr.upload_file(user, data, filename, content_type) - def XXXupload_file(self, user_id, data, filename, content_type): - """Upload a file to redmine""" - # POST /uploads.json?filename=image.png - # Content-Type: application/octet-stream - # (request body is the file content) - - headers = { - 'User-Agent': 'netbot/0.0.1', # TODO update to project version, and add version management - 'Content-Type': 'application/octet-stream', # <-- VERY IMPORTANT - 'X-Redmine-API-Key': self.token, - 'X-Redmine-Switch-User': user_id, # Make sure the comment is noted by the correct user - } - - r = requests.post( - url=f"{self.url}/uploads.json?filename={filename}", - timeout=TIMEOUT, - files={ 'upload_file': (filename, data, content_type) }, - headers=headers) - - # 201 response: {"upload":{"token":"7167.ed1ccdb093229ca1bd0b043618d88743"}} - if r.status_code == 201: - # all good, get token - root = json.loads(r.text, object_hook= lambda x: SimpleNamespace(**x)) - token = root.upload.token - log.info(f"Uploaded {filename} {content_type}, got token={token}") - return token - else: - #print(vars(r)) - log.error(f"upload_file, file={filename} {content_type}, status={r.status_code}: {r.reason}, req-id={r.headers['X-Request-Id']}") - # todo throw exception - #TODO throw exception to show upload failed, and why - def upload_attachments(self, user:User, attachments): - """Upload a list of attachments""" - # uploads all the attachments, - # sets the upload token for each - for a in attachments: - token = self.upload_file(user, a.payload, a.name, a.content_type) - a.set_token(token) + self.ticket_mgr.upload_attachments(user, attachments) + ## FIXME user logic def lookup_user(self, username:str): """Get a user based on ID, directly from redmine""" if username is None or len(username) == 0: @@ -138,36 +102,20 @@ def lookup_user(self, username:str): return None - def get_tickets_by(self, user): - # GET /issues.json?author_id=6 - response = self.query(f"/issues.json?author_id={user.id}") - if response: - return response.issues - else: - log.debug(f"Unknown user: {user}") - return None + def get_tickets_by(self, user) -> list[Ticket]: + return self.ticket_mgr.get_tickets_by(user) - def get_ticket(self, ticket_id:int, include_journals:bool = False): + def get_ticket(self, ticket_id:int, include_journals:bool = False) -> Ticket: return self.ticket_mgr.get(ticket_id, include_journals) #GET /issues.xml?issue_id=1,2 - def get_tickets(self, ticket_ids): - """get several tickets based on a list of IDs""" - if ticket_ids is None or len(ticket_ids) == 0: - log.debug("No ticket numbers supplied to get_tickets.") - return [] + def get_tickets(self, ticket_ids) -> list[Ticket]: + return self.ticket_mgr.get_tickets(ticket_ids) - response = self.query(f"/issues.json?issue_id={','.join(ticket_ids)}&status_id=*&sort={DEFAULT_SORT}") - log.debug(f"query response: {response}") - if response is not None and response.total_count > 0: - return response.issues - else: - log.info(f"Unknown ticket numbers: {ticket_ids}") - return [] - def find_ticket_from_str(self, string:str): + def find_ticket_from_str(self, string:str) -> Ticket: """parse a ticket number from a string and get the associated ticket""" # for now, this is a trivial REGEX to match '#nnn' in a string, and return ticket #nnn match = re.search(r'#(\d+)', string) @@ -180,99 +128,25 @@ def find_ticket_from_str(self, string:str): def remove_ticket(self, ticket_id:int): - """delete a ticket in redmine. used for testing""" - # DELETE to /issues/{ticket_id}.json - response = requests.delete( - url=f"{self.url}/issues/{ticket_id}.json", - timeout=TIMEOUT, - headers=self.get_headers()) - - if response.ok: - log.info(f"remove_ticket {ticket_id}") - else: - raise RedmineException(f"remove_ticket failed, status=[{response.status_code}] {response.reason}", response.headers['X-Request-Id']) - - - def most_recent_ticket_for(self, email): - """get the most recent ticket for the user with the given email""" - # get the user record for the email - user = self.user_mgr.get_by_name(email) - - if user: - # query open tickets created by user, sorted by most recently updated, limit 1 - response = self.query(f"/issues.json?author_id={user.id}&status_id=open&sort=updated_on:desc&limit=1") - - if response.total_count > 0: - return response.issues[0] - else: - log.info(f"No recent open ticket found for: {user}") - return None - else: - log.warning(f"Unknown email: {email}") - return None + self.ticket_mgr.remove(ticket_id) - def new_tickets_since(self, timestamp:dt.datetime): - """get new tickets since provided timestamp""" - # query for new tickets since date - # To fetch issues created after a certain timestamp (uncrypted filter is ">=2014-01-02T08:12:32Z") : - # GET /issues.xml?created_on=%3E%3D2014-01-02T08:12:32Z - timestr = dt.datetime.isoformat(timestamp) # time-format. - query = f"/issues.json?created_on=%3E%3D{timestr}&sort={DEFAULT_SORT}&limit=100" - response = self.query(query) + def most_recent_ticket_for(self, email: str) -> Ticket: + return self.ticket_mgr.most_recent_ticket_for(email) - if response.total_count > 0: - return response.issues - else: - log.debug(f"No tickets created since {timestamp}") - return None + def new_tickets_since(self, timestamp:dt.datetime) -> list[Ticket]: + return self.ticket_mgr.new_tickets_since(timestamp) + def find_tickets(self) -> list[Ticket]: + return self.ticket_mgr.find_tickets() - def find_tickets(self): - """default ticket query""" - # "kanban" query: all ticket open or closed recently - project=1 - tracker=4 - query = f"/issues.json?project_id={project}&tracker_id={tracker}&status_id=*&sort={DEFAULT_SORT}&limit=100" - response = self.query(query) + def my_tickets(self, user=None) -> list[Ticket]: + return self.ticket_mgr.my_tickets(user) - return response.issues + def tickets_for_team(self, team_str:str) -> list[Ticket]: + return self.ticket_mgr.tickets_for_team(team_str) - def my_tickets(self, user=None): - """get my tickets""" - response = self.query(f"/issues.json?assigned_to_id=me&status_id=open&sort={DEFAULT_SORT}&limit=100", user) - - if response.total_count > 0: - return response.issues - else: - log.info("No open ticket for me.") - return None - - def tickets_for_team(self, team_str:str): - # validate team? - team = self.user_mgr.get_by_name(team_str) # find_user is dsigned to be broad - - query = f"/issues.json?assigned_to_id={team.id}&status_id=open&sort={DEFAULT_SORT}&limit=100" - response = self.query(query) - - if response.total_count > 0: - return response.issues - else: - log.info(f"No open ticket found for: {team}") - return None - - def search_tickets(self, term): - """search all text of all tickets (not just open) for the supplied terms""" - # todo url-encode term? - # note: sort doesn't seem to be working for search - query = f"/search.json?q={term}&issues=1&limit=100&sort={DEFAULT_SORT}" - - response = self.query(query) - - ids = [] - for result in response.results: - ids.append(str(result.id)) - - return self.get_tickets(ids) + def search_tickets(self, term) -> list[Ticket]: + return self.ticket_mgr.search(term) def match_subject(self, subject): # todo url-encode term? @@ -295,23 +169,6 @@ def match_subject(self, subject): def get_notes_since(self, ticket_id, timestamp=None): return self.ticket_mgr.get_notes_since(ticket_id, timestamp) - def DELETE_get_notes_since(self, ticket_id, timestamp=None): - notes = [] - - ticket = self.get_ticket(ticket_id, include_journals=True) - log.debug(f"got ticket {ticket_id} with {len(ticket.journals)} notes") - - for note in ticket.journals: - # note.notes is a text field with notes, or empty. if there are no notes, ignore the journal - if note.notes and timestamp: - created = synctime.parse_str(note.created_on) - if created > timestamp: - notes.append(note) - elif note.notes: - notes.append(note) # append all notes when there's no timestamp - - return notes - def enable_discord_sync(self, ticket_id, user, note): fields = { diff --git a/test_imap.py b/test_imap.py index aa51334..da5ecc8 100755 --- a/test_imap.py +++ b/test_imap.py @@ -74,7 +74,8 @@ def test_doctype_head(self): def test_more_recent_ticket(self): - ticket = self.redmine.most_recent_ticket_for("philion") + user = self.redmine.user_mgr.get_by_name('admin') + ticket = self.redmine.most_recent_ticket_for(user) self.assertIsNotNone(ticket) diff --git a/tickets.py b/tickets.py index 633046d..0468fe9 100644 --- a/tickets.py +++ b/tickets.py @@ -86,6 +86,7 @@ class Ticket(): def __post_init__(self): self.status = TicketStatus(**self.status) + self.author = NamedId(**self.author) if self.created_on: self.created_on = synctime.parse_str(self.created_on) if self.updated_on: @@ -251,15 +252,16 @@ def upload_attachments(self, user:User, attachments): # uploads all the attachments, # sets the upload token for each for a in attachments: - token = self.upload_file(user.login, a.payload, a.name, a.content_type) + token = self.upload_file(user, a.payload, a.name, a.content_type) a.set_token(token) - def get_tickets_by(self, user): + def get_tickets_by(self, user) -> list[Ticket]: # GET /issues.json?author_id=6 response = self.session.get(f"/issues.json?author_id={user.id}") if response: - return response['issues'] + result = TicketsResult(**response) + return result.issues else: log.debug(f"Unknown user: {user}") return None @@ -321,7 +323,7 @@ def remove(self, ticket_id:int) -> None: self.session.delete(f"/issues/{ticket_id}.json") - def most_recent_ticket_for(self, user:User): + def most_recent_ticket_for(self, user:User) -> Ticket: """get the most recent ticket for the user with the given email""" # get the user record for the email if not user: @@ -329,9 +331,8 @@ def most_recent_ticket_for(self, user:User): # query open tickets created by user, sorted by most recently updated, limit 1 response = self.session.get(f"/issues.json?author_id={user.id}&status_id=open&sort=updated_on:desc&limit=1") - - if response.total_count > 0: - return response.issues[0] + if response: + return TicketsResult(**response) else: log.info(f"No recent open ticket found for: {user}") return None From 06a9ac4190cf54dcf625ce7ca7cce4b3489e7827 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Mon, 4 Mar 2024 11:23:03 -0800 Subject: [PATCH 13/15] stability after major refactor. all tests pass --- cog_scn.py | 4 +- cog_tickets.py | 4 +- netbot.py | 2 +- redmine.py | 247 ++++---------------------------------------- test_cog_scn.py | 2 + test_cog_tickets.py | 3 +- test_imap.py | 10 +- test_netbot.py | 3 + test_synctime.py | 13 +-- test_utils.py | 2 +- tickets.py | 12 ++- users.py | 2 +- 12 files changed, 54 insertions(+), 250 deletions(-) mode change 100644 => 100755 test_utils.py diff --git a/cog_scn.py b/cog_scn.py index 0f597c6..197193c 100644 --- a/cog_scn.py +++ b/cog_scn.py @@ -169,7 +169,7 @@ async def join(self, ctx:discord.ApplicationContext, teamname:str , member: disc elif self.redmine.user_mgr.get_team_by_name(teamname) is None: await ctx.respond(f"Unknown team name: {teamname}") else: - self.redmine.join_team(user.login, teamname) + self.redmine.user_mgr.join_team(user, teamname) await ctx.respond(f"**{discord_name}** has joined *{teamname}*") @@ -182,7 +182,7 @@ async def leave(self, ctx:discord.ApplicationContext, teamname:str, member: disc user = self.redmine.user_mgr.find(discord_name) if user: - self.redmine.leave_team(user.login, teamname) + self.redmine.user_mgr.leave_team(user, teamname) await ctx.respond(f"**{discord_name}** has left *{teamname}*") else: await ctx.respond(f"Unknown Discord user: {discord_name}.") diff --git a/cog_tickets.py b/cog_tickets.py index 7e724d6..17688a3 100644 --- a/cog_tickets.py +++ b/cog_tickets.py @@ -199,7 +199,7 @@ def format_tickets(self, title, tickets, fields=None, max_len=2000): return "No tickets found." if fields is None: - fields = ["link","priority","updated","assigned","subject"] + fields = ["link","priority","updated_on","assigned_to","subject"] section = "**" + title + "**\n" for ticket in tickets: @@ -216,7 +216,7 @@ def format_tickets(self, title, tickets, fields=None, max_len=2000): def format_ticket(self, ticket, fields=None): section = "" if fields is None: - fields = ["link","priority","updated","assigned","subject"] + fields = ["link","priority","updated_on","assigned_to","subject"] for field in fields: section += str(self.redmine.get_field(ticket, field)) + " " # spacer, one space diff --git a/netbot.py b/netbot.py index daa177b..1e19d0a 100755 --- a/netbot.py +++ b/netbot.py @@ -195,7 +195,7 @@ async def on_application_command_error(self, context: discord.ApplicationContext if isinstance(exception, commands.CommandOnCooldown): await context.respond("This command is currently on cooldown!") else: - log.error(f"{context} - {exception}", exc_info=True) + log.warning(f"{context} - {exception}", exc_info=True) #raise error # Here we raise other errors to ensure they aren't ignored await context.respond(f"Error processing your request: {exception}") diff --git a/redmine.py b/redmine.py index de46c6f..40aa091 100644 --- a/redmine.py +++ b/redmine.py @@ -3,17 +3,14 @@ import os import re -import json import logging import datetime as dt -from types import SimpleNamespace -import requests from dotenv import load_dotenv import synctime from session import RedmineSession -from users import User, UserResult, UserManager +from users import User, UserManager from tickets import Ticket, TicketManager @@ -78,38 +75,12 @@ def upload_file(self, user:User, data, filename, content_type) -> str: def upload_attachments(self, user:User, attachments): self.ticket_mgr.upload_attachments(user, attachments) - - ## FIXME user logic - def lookup_user(self, username:str): - """Get a user based on ID, directly from redmine""" - if username is None or len(username) == 0: - log.debug("Empty user ID") - return None - - response = requests.get(f"{self.url}/users.json?name={username}", - headers=self.get_headers(), timeout=TIMEOUT) - if response.ok: - user_result = UserResult(**response.json()) - log.debug(f"lookup_user: {username} -> {user_result.users}") - - if user_result.total_count == 1: - return user_result.users[0] - elif user_result.total_count > 1: - log.warning(f"Too many results for {username}: {user_result.users}") - return user_result.users[0] - else: - log.debug(f"Unknown user: {username}") - return None - - def get_tickets_by(self, user) -> list[Ticket]: return self.ticket_mgr.get_tickets_by(user) - def get_ticket(self, ticket_id:int, include_journals:bool = False) -> Ticket: return self.ticket_mgr.get(ticket_id, include_journals) - #GET /issues.xml?issue_id=1,2 def get_tickets(self, ticket_ids) -> list[Ticket]: return self.ticket_mgr.get_tickets(ticket_ids) @@ -149,27 +120,11 @@ def search_tickets(self, term) -> list[Ticket]: return self.ticket_mgr.search(term) def match_subject(self, subject): - # todo url-encode term? - # note: sort doesn't seem to be working for search - query = f"/search.json?q={subject}&all_words=1&titles_only=1&open_issues=1&limit=100" - - response = self.query(query) - - if response: - ids = [] - for result in response.results: - ids.append(str(result.id)) - - return self.get_tickets(ids) - else: - log.debug(f"subject matched nothing: {subject}") - return [] - + return self.ticket_mgr.match_subject(subject) def get_notes_since(self, ticket_id, timestamp=None): return self.ticket_mgr.get_notes_since(ticket_id, timestamp) - def enable_discord_sync(self, ticket_id, user, note): fields = { "note": note, #f"Created Discord thread: {thread.name}: {thread.jump_url}", @@ -181,201 +136,39 @@ def enable_discord_sync(self, ticket_id, user, note): # todo: better error reporting back to discord - def join_team(self, username, teamname:str) -> None: - user = self.user_mgr.get_by_name(username) - self.user_mgr.join_team(user, teamname) - - - def leave_team(self, username:int, teamname:str) -> None: - # look up user ID - user = self.user_mgr.get_by_name(username) - if user is None: - log.warning(f"Unknown user name: {username}") - return None - - # map teamname to team - #team = self.user_mgr.get_team_by_name(teamname) - #if team is None: - # log.warning(f"Unknown team name: {teamname}") - # return None - - self.user_mgr.leave_team(user, teamname) - - - def get_headers(self, impersonate_id:str=None): - headers = { - 'User-Agent': 'netbot/0.0.1', # TODO update to project version, and add version management - 'Content-Type': 'application/json', - 'X-Redmine-API-Key': self.token, - } - # insert the impersonate_id to impersonate another user - if impersonate_id: - headers['X-Redmine-Switch-User'] = impersonate_id # Make sure the comment is noted by the correct user - log.debug(f"setting redmine impersonation flag for user={impersonate_id}") - return headers - - - def query(self, query_str:str, user:str=None): - """run a query against a redmine instance""" - - headers = self.get_headers(user) - - # TODO Detect and handle paged results - - try: - r = requests.get(f"{self.url}{query_str}", headers=headers, timeout=TIMEOUT) - - # check 200 status code - if r.ok: - # return the parsed the JSON text - return json.loads(r.text, object_hook=lambda x: SimpleNamespace(**x)) - else: - log.error(f"Status code {r.status_code} for {r.request.url}, reqid={r.headers['X-Request-Id']}: {r}") - except TimeoutError as toe: - # ticket-509: Handle timeout gracefully - log.warning(f"Timeout during {query_str}: {toe}") - except Exception as ex: - log.warning(f"Excetion during {query_str}: {ex}") - - return None - - def assign_ticket(self, ticket_id, target, user_id=None): - user = self.user_mgr.get_by_name(target) + user = self.user_mgr.find(target) if user: - fields = { - "assigned_to_id": user.id, - #"status_id": "1", # New - } - if user_id is None: - # use the user-id to self-assign - user_id = user.login - self.update_ticket(ticket_id, fields, user_id) + self.ticket_mgr.assign_ticket(ticket_id, user, user_id) else: - log.error(f"unknow user: {target}") - + log.error(f"unknow user: {target}") # Exception? def progress_ticket(self, ticket_id, user_id=None): # TODO notes - fields = { - "assigned_to_id": "me", - "status_id": "2", # "In Progress" - } - self.update_ticket(ticket_id, fields, user_id) - - - def reject_ticket(self, ticket_id, user_id=None): # TODO notes - fields = { - "assigned_to_id": "", - "status_id": "5", # "Reject" - } - self.update_ticket(ticket_id, fields, user_id) + self.ticket_mgr.progress_ticket(ticket_id, user_id) + def reject_ticket(self, ticket_id, user_id=None): + self.ticket_mgr.reject_ticket(ticket_id, user_id) def unassign_ticket(self, ticket_id, user_id=None): - fields = { - "assigned_to_id": "", # FIXME this *should* be the team it was assigned to, but there's no way to calculate. - "status_id": "1", # New - } - self.update_ticket(ticket_id, fields, user_id) - + self.ticket_mgr.unassign_ticket(ticket_id, user_id) def resolve_ticket(self, ticket_id, user_id=None): - self.update_ticket(ticket_id, {"status_id": "3"}, user_id) # '3' is the status_id, it doesn't accept "Resolved" - + return self.ticket_mgr.resolve_ticket(ticket_id, user_id) def get_team(self, teamname:str): return self.user_mgr.get_team_by_name(teamname) # FIXME consistent naming - - def get_sync_record(self, ticket, expected_channel: int) -> synctime.SyncRecord: - # Parse custom_field into datetime - # lookup field by name - token = None - try : - for field in ticket.custom_fields: - if field.name == SYNC_FIELD_NAME: - token = field.value - log.debug(f"found {field.name} => '{field.value}'") - break - except AttributeError: - # custom_fields not set, handle same as no sync field - pass - - if token: - record = synctime.SyncRecord.from_token(ticket.id, token) - log.debug(f"created sync_rec from token: {record}") - if record: - # check channel - if record.channel_id == 0: - # no valid channel set in sync data, assume lagacy - record.channel_id = expected_channel - # update the record in redmine after adding the channel info - self.update_sync_record(record) - return record - elif record.channel_id != expected_channel: - log.debug(f"channel mismatch: rec={record.channel_id} =/= {expected_channel}, token={token}") - return None - else: - return record - else: - # no token implies not-yet-initialized - record = synctime.SyncRecord(ticket.id, expected_channel, synctime.epoch_datetime()) - # apply the new sync record back to redmine - self.update_sync_record(record) - return record - - def update_sync_record(self, record:synctime.SyncRecord): - log.debug(f"Updating sync record in redmine: {record}") - fields = { - "custom_fields": [ - { "id": 4, "value": record.token_str() } # cf_4, custom field syncdata, #TODO search for it - ] - } - self.update_ticket(record.ticket_id, fields) - - - # NOTE: This implies that ticket should be a full object with methods. - # Starting to move fields out to their own methods, to eventually move to - # their own Ticket class. - def get_field(self, ticket, fieldname): - try: - match fieldname: - case "id": - return f"{ticket.id}" - case "url": - return f"{self.url}/issues/{ticket.id}" - case "link": - return f"[{ticket.id}]({self.url}/issues/{ticket.id})" - case "priority": - return ticket.priority.name - case "updated": - return ticket.updated_on # string, or dt? - case "assigned": - return ticket.assigned_to.name - case "status": - return ticket.status.name - case "subject": - return ticket.subject - case "title": - return ticket.title - #case "age": - # updated = dt.datetime.fromisoformat(ticket.updated_on) ### UTC - # age = dt.datetime.now(dt.timezone.utc) - updated - # return humanize.naturaldelta(age) - #case "sync": - # try: - # # Parse custom_field into datetime - # # FIXME: this is fragile: relies on specific index of custom field, add custom field lookup by name - # timestr = ticket.custom_fields[1].value - # return dt.datetime.fromisoformat(timestr) ### UTC - # except Exception as e: - # log.debug(f"sync tag not set") - # return None - except AttributeError: - return "" # or None? - - + self.ticket_mgr.update_sync_record(record) + + def get_field(self, ticket:Ticket, fieldname:str): + match fieldname: + case "url": + return f"{self.url}/issues/{ticket.id}" + case "link": + return f"[{ticket.id}]({self.url}/issues/{ticket.id})" + case _: + return ticket.get_field(fieldname) if __name__ == '__main__': diff --git a/test_cog_scn.py b/test_cog_scn.py index 79a24ad..c31cc41 100755 --- a/test_cog_scn.py +++ b/test_cog_scn.py @@ -12,6 +12,8 @@ import test_utils +logging.getLogger().setLevel(logging.ERROR) + log = logging.getLogger(__name__) diff --git a/test_cog_tickets.py b/test_cog_tickets.py index 0106be4..6975125 100755 --- a/test_cog_tickets.py +++ b/test_cog_tickets.py @@ -12,7 +12,8 @@ import test_utils -logging.basicConfig(level=logging.FATAL) +logging.getLogger().setLevel(logging.ERROR) + log = logging.getLogger(__name__) diff --git a/test_imap.py b/test_imap.py index da5ecc8..4db1823 100755 --- a/test_imap.py +++ b/test_imap.py @@ -95,28 +95,28 @@ def test_email_address_parsing2(self): self.assertEqual("philion@acmerocket.com", email) + # FIXME This is very fragile def test_new_account_from_email(self): # make sure neither the email or subject exist # note: these are designed to fail-fast, because trying to manage the user and subject as part of the test failed. test_email = "philion@acmerocket.com" - user = self.redmine.lookup_user(test_email) - self.assertIsNone(user, "Found existing user: {test_email}") + user = self.redmine.user_mgr.get_by_name(test_email) + self.assertIsNone(user, f"Found existing user: {test_email}") subject = "Search for subject match in email threading" tickets = self.redmine.match_subject(subject) - self.assertEqual(0, len(tickets), "Found ticket matching: '{subject}' - {tickets[0].id}, please delete.") + self.assertEqual(0, len(tickets), f"Found ticket matching: '{subject}' - {tickets}, please delete.") with open("test/message-190.eml", 'rb') as file: message = self.imap.parse_message(file.read()) log.debug(f"loaded message: {message}") self.imap.handle_message("test", message) - user = self.redmine.lookup_user(test_email) + user = self.redmine.user_mgr.find(test_email) self.assertIsNotNone(user, f"Couldn't find user for {test_email}") self.assertEqual(test_email, user.mail) # validate the ticket created by message-190 - #subject = "Search for subject match in email threading" tickets = self.redmine.match_subject(subject) self.assertEqual(1, len(tickets)) self.assertEqual(subject, tickets[0].subject) diff --git a/test_netbot.py b/test_netbot.py index 5604b42..587b4f5 100755 --- a/test_netbot.py +++ b/test_netbot.py @@ -12,6 +12,9 @@ import test_utils +logging.getLogger().setLevel(logging.ERROR) + + log = logging.getLogger(__name__) diff --git a/test_synctime.py b/test_synctime.py index 69047b0..70f22dd 100755 --- a/test_synctime.py +++ b/test_synctime.py @@ -29,21 +29,22 @@ def test_redmine_times(self): user = self.redmine.user_mgr.find("admin") # FIXME: create a relaible test_user self.assertIsNotNone(user) subject = f"TEST ticket {tag}" - ticket = self.redmine.create_ticket(user, subject, f"This for {self.id}-{tag}") + ticket = self.redmine.create_ticket(user, subject, f"This for {self.id}-{tag}") # FIXME standard way to create test ticket! test_channel = 4321 - sync_rec = self.redmine.get_sync_record(ticket, expected_channel=test_channel) + sync_rec = ticket.get_sync_record(expected_channel=test_channel) self.assertIsNotNone(sync_rec) self.assertEqual(sync_rec.ticket_id, ticket.id) self.assertEqual(sync_rec.channel_id, test_channel) - #### NOTE to morning self: catch 42 with get_sync_record returning None or a valid new erc with the wrong channel. - #### FIX IN MORNING. + # apply the new sync back to the ticket in test context! + # happens aytomatically in sync context + self.redmine.ticket_mgr.update_sync_record(sync_rec) # refetch ticket ticket2 = self.redmine.get_ticket(ticket.id) - sync_rec2 = self.redmine.get_sync_record(ticket2, expected_channel=1111) # NOT the test_channel - log.info(f"ticket updated={ticket.updated_on}, {synctime.age(ticket.updated_on)} ago, sync: {sync_rec}") + sync_rec2 = ticket2.get_sync_record(expected_channel=1111) # NOT the test_channel + log.info(f"ticket2 updated={ticket2.updated_on}, {synctime.age_str(ticket2.updated_on)} ago, channel: {sync_rec.channel_id}") self.assertIsNone(sync_rec2) diff --git a/test_utils.py b/test_utils.py old mode 100644 new mode 100755 index 7d02f7c..4bee36a --- a/test_utils.py +++ b/test_utils.py @@ -1,4 +1,4 @@ -#test utils +#!/usr/bin/env python3 """Utilities to help testing""" import time diff --git a/tickets.py b/tickets.py index 0468fe9..1fb05f1 100644 --- a/tickets.py +++ b/tickets.py @@ -83,7 +83,6 @@ class Ticket(): custom_fields: list[CustomField] = None journals: list[TicketNote] = None - def __post_init__(self): self.status = TicketStatus(**self.status) self.author = NamedId(**self.author) @@ -109,11 +108,12 @@ def get_custom_field(self, name: str) -> str: return field.value return None - def get_sync_record(self, expected_channel: int) -> synctime.SyncRecord: # Parse custom_field into datetime # lookup field by name token = self.get_custom_field(SYNC_FIELD_NAME) + #log.info(f"### found '{token}' for #{self.id}:{SYNC_FIELD_NAME}") + #log.info(f"### custom field: {self.custom_fields}") if token: record = synctime.SyncRecord.from_token(self.id, token) log.debug(f"created sync_rec from token: {record}") @@ -135,6 +135,7 @@ def get_sync_record(self, expected_channel: int) -> synctime.SyncRecord: record = synctime.SyncRecord(self.id, expected_channel, synctime.epoch_datetime()) # apply the new sync record back to redmine # self.update_sync_record(record) same REALLY as above ^^^^ + log.debug(f"created new sync record, none found: {record}") return record @@ -149,6 +150,10 @@ def get_notes(self, since:dt.datetime=None) -> list[TicketNote]: return notes + def get_field(self, fieldname): + return getattr(self, fieldname) + + @dataclass class TicketsResult: @@ -422,7 +427,7 @@ def match_subject(self, subject): return None # the response has only IDs.... - ids = [result.id for result in response.results] + ids = [result['id'] for result in response['results']] # but there's a call to get several tickets return self.get_tickets(ids) @@ -446,7 +451,6 @@ def enable_discord_sync(self, ticket_id, user, note): def assign_ticket(self, ticket_id, user:User, user_id=None): - fields = { "assigned_to_id": user.id, #"status_id": "1", # New diff --git a/users.py b/users.py index b95b686..da4599c 100644 --- a/users.py +++ b/users.py @@ -466,7 +466,7 @@ def leave_team(self, user: User, teamname:str): # check status if not r: - log.error(f"Error removing {user.login} from {teamname}") + log.warning(f"Error removing {user.login} from {teamname}") #### ---- indexing stuff From 59caf17a278830dc62acbd46211eb684a4fe0517 Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Mon, 4 Mar 2024 15:47:32 -0800 Subject: [PATCH 14/15] more tweaking, more testing. had to add __str__ to the dataclasses --- cog_tickets.py | 2 -- netbot.py | 4 ++-- redmine.py | 5 ++++- session.py | 11 ++++++----- test_netbot.py | 8 ++++++-- test_redmine.py | 10 +++++++--- test_utils.py | 8 +++++--- tickets.py | 36 +++++++++++++++++++++++++++++------- users.py | 30 +++++++++++++++++++++++++----- 9 files changed, 84 insertions(+), 30 deletions(-) diff --git a/cog_tickets.py b/cog_tickets.py index 17688a3..f7d849b 100644 --- a/cog_tickets.py +++ b/cog_tickets.py @@ -68,8 +68,6 @@ async def tickets(self, ctx: discord.ApplicationContext, params: str = ""): # add groups to users. # lookup the user - log.debug(f"looking for user mapping for {ctx}") - user = self.redmine.user_mgr.find(ctx.user.name) log.debug(f"found user mapping for {ctx.user.name}: {user}") diff --git a/netbot.py b/netbot.py index 1e19d0a..8bb1cbb 100755 --- a/netbot.py +++ b/netbot.py @@ -195,9 +195,9 @@ async def on_application_command_error(self, context: discord.ApplicationContext if isinstance(exception, commands.CommandOnCooldown): await context.respond("This command is currently on cooldown!") else: - log.warning(f"{context} - {exception}", exc_info=True) + log.warning(f"{context.user}/{context.command} - {exception.__cause__}", exc_info=exception.__cause__) #raise error # Here we raise other errors to ensure they aren't ignored - await context.respond(f"Error processing your request: {exception}") + await context.respond(f"Error processing due to: {exception.__cause__}") def main(): diff --git a/redmine.py b/redmine.py index 40aa091..b2d7f0f 100644 --- a/redmine.py +++ b/redmine.py @@ -48,6 +48,8 @@ def __init__(self, url: str, token: str): self.user_mgr:UserManager = UserManager(session) self.ticket_mgr:TicketManager = TicketManager(session) + self.user_mgr.reindex() # build the cache when starting + @classmethod def fromenv(cls): @@ -161,7 +163,8 @@ def get_team(self, teamname:str): def update_sync_record(self, record:synctime.SyncRecord): self.ticket_mgr.update_sync_record(record) - def get_field(self, ticket:Ticket, fieldname:str): + # mostly for formatting + def get_field(self, ticket:Ticket, fieldname:str) -> str: match fieldname: case "url": return f"{self.url}/issues/{ticket.id}" diff --git a/session.py b/session.py index 7935189..c83cb69 100644 --- a/session.py +++ b/session.py @@ -4,14 +4,15 @@ import os import logging - +from urllib3.exceptions import ConnectTimeoutError import requests +from requests.exceptions import ConnectTimeout, ConnectionError log = logging.getLogger(__name__) -TIMEOUT = 10 # seconds +TIMEOUT = 5 # seconds class RedmineException(Exception): @@ -73,11 +74,11 @@ def get(self, query_str:str, user:str=None): return r.json() else: log.info(f"GET {r.reason}/{r.status_code} url={r.request.url}, reqid={r.headers['X-Request-Id']}") - except TimeoutError as toe: + except (TimeoutError, ConnectTimeoutError, ConnectTimeout, ConnectionError): # ticket-509: Handle timeout gracefully - log.warning(f"Timeout during {query_str}: {toe}") + log.warning(f"TIMEOUT ({TIMEOUT}s) during {query_str}") except Exception as ex: - log.exception(f"Exception during {query_str}: {ex}") + log.exception(f"{type(ex)} during {query_str}: {ex}") return None diff --git a/test_netbot.py b/test_netbot.py index 587b4f5..332a857 100755 --- a/test_netbot.py +++ b/test_netbot.py @@ -2,6 +2,7 @@ """NetBot Test Suite""" import unittest +from unittest import mock import logging import discord @@ -103,12 +104,15 @@ async def test_sync_ticket_long_message(self): async def test_on_application_command_error(self): ctx = self.build_context() - error = discord.DiscordException("this is exception " + self.tag) - await self.bot.on_application_command_error(ctx, error) + error = netbot.NetbotException("this is exception " + self.tag) + wrapper = discord.DiscordException("Discord Ex Wrapper") + wrapper.__cause__ = error + await self.bot.on_application_command_error(ctx, wrapper) self.assertIn(self.tag, ctx.respond.call_args.args[0]) + if __name__ == '__main__': # when running this main, turn on DEBUG logging.basicConfig(level=logging.DEBUG, format="{asctime} {levelname:<8s} {name:<16} {message}", style='{') diff --git a/test_redmine.py b/test_redmine.py index 26f569f..a5a59a1 100755 --- a/test_redmine.py +++ b/test_redmine.py @@ -10,6 +10,9 @@ import test_utils +logging.getLogger().setLevel(logging.ERROR) + + log = logging.getLogger(__name__) @@ -17,7 +20,7 @@ class TestRedmine(test_utils.RedmineTestCase): """Test suite for Redmine client""" - def test_blocked_user(self): + def test_block_user(self): # block self.user_mgr.block(self.user) self.assertTrue(self.user_mgr.is_blocked(self.user)) @@ -47,8 +50,9 @@ def test_client_timeout(self): # construct an invalid client to try to get a timeout try: client = redmine.Client("http://192.168.1.42/", "bad-token") - log.info(client) - except TimeoutError: + self.assertIsNotNone(client) + #log.info(client) + except Exception: self.fail("Got unexpected timeout") diff --git a/test_utils.py b/test_utils.py index 4bee36a..8d7a458 100755 --- a/test_utils.py +++ b/test_utils.py @@ -15,7 +15,7 @@ from users import User, UserManager import session import tickets -import redmine +from redmine import Client log = logging.getLogger(__name__) @@ -102,14 +102,14 @@ def tearDownClass(cls): class BotTestCase(unittest.IsolatedAsyncioTestCase): """Abstract base class for testing Bot features""" - redmine: session.RedmineSession = None + redmine: Client = None usertag: str = None user: User = None @classmethod def setUpClass(cls): log.info("Setting up test fixtures") - cls.redmine:redmine.Client = redmine.Client.fromenv() + cls.redmine:Client = Client.fromenv() cls.usertag:str = tagstr() cls.user:User = create_test_user(cls.redmine.user_mgr, cls.usertag) log.info(f"Created test user: {cls.user}") @@ -125,6 +125,8 @@ def build_context(self) -> ApplicationContext: ctx = mock.AsyncMock(ApplicationContext) ctx.user = mock.AsyncMock(discord.Member) ctx.user.name = self.discord_user + ctx.command = mock.AsyncMock(discord.ApplicationCommand) + ctx.command.name = unittest.TestCase.id(self) log.debug(f"created ctx with {self.discord_user}: {ctx}") return ctx diff --git a/tickets.py b/tickets.py index 1fb05f1..7a8dd39 100644 --- a/tickets.py +++ b/tickets.py @@ -30,6 +30,9 @@ class TicketStatus(): name: str is_closed: bool + def __str__(self): + return self.name + @dataclass class PropertyChange(): # https://www.redmine.org/projects/redmine/wiki/Rest_IssueJournals @@ -39,16 +42,19 @@ class PropertyChange(): # https://www.redmine.org/projects/redmine/wiki/Rest_Iss old_value: str new_value: str + def __str__(self): + return f"{self.name}/{self.property} {self.old_value} -> {self.new_value}" + @dataclass class TicketNote(): # https://www.redmine.org/projects/redmine/wiki/Rest_IssueJournals """a message sent to a ticket""" id: int - user: NamedId notes: str created_on: dt.datetime private_notes: bool details: list[PropertyChange] + user: NamedId = None def __post_init__(self): self.user = NamedId(**self.user) @@ -56,16 +62,13 @@ def __post_init__(self): if self.details: self.details = [PropertyChange(**change) for change in self.details] + def __str__(self): + return f"#{self.id} - {self.user}: {self.notes}" @dataclass class Ticket(): """Encapsulates a redmine ticket""" id: int - project: NamedId - tracker: NamedId - status: TicketStatus - priority: NamedId - author: NamedId subject: str description: str done_ratio: float @@ -77,8 +80,15 @@ class Ticket(): created_on: dt.datetime updated_on: dt.datetime closed_on: dt.datetime + project: NamedId = None + tracker: NamedId = None + priority: NamedId = None + author: NamedId = None + status: TicketStatus = None + parent: NamedId = None spent_hours: float = 0.0 total_spent_hours: float = 0.0 + category: str = None assigned_to: NamedId = None custom_fields: list[CustomField] = None journals: list[TicketNote] = None @@ -86,6 +96,12 @@ class Ticket(): def __post_init__(self): self.status = TicketStatus(**self.status) self.author = NamedId(**self.author) + self.priority = NamedId(**self.priority) + self.project = NamedId(**self.project) + self.tracker = NamedId(**self.tracker) + + if self.assigned_to: + self.assigned_to = NamedId(**self.assigned_to) if self.created_on: self.created_on = synctime.parse_str(self.created_on) if self.updated_on: @@ -108,6 +124,9 @@ def get_custom_field(self, name: str) -> str: return field.value return None + def __str__(self): + return f"#{self.id} {self.project} {self.status} {self.priority} {self.assigned_to}: {self.subject}" + def get_sync_record(self, expected_channel: int) -> synctime.SyncRecord: # Parse custom_field into datetime # lookup field by name @@ -151,7 +170,9 @@ def get_notes(self, since:dt.datetime=None) -> list[TicketNote]: return notes def get_field(self, fieldname): - return getattr(self, fieldname) + val = getattr(self, fieldname) + #log.debug(f">>> {fieldname} = {val}, type={type(val)}") + return val @@ -376,6 +397,7 @@ def my_tickets(self, user=None) -> list[Ticket]: if not jresp: return None + #log.debug(f"### json: {jresp}") response = TicketsResult(**jresp) if response.total_count > 0: return response.issues diff --git a/users.py b/users.py index da4599c..25cdaf0 100644 --- a/users.py +++ b/users.py @@ -24,15 +24,20 @@ class CustomField(): name: str value: str + def __str__(self) -> str: + return f"field-{self.id}:{self.name}={self.value}" @dataclass class NamedId: '''named ID in redmine''' id: int - name: str + name: str = None def __str__(self) -> str: - return self.name + if self.name: + return self.name + else: + return str(self.id) @dataclass @@ -46,6 +51,9 @@ def __post_init__(self): if self.users: self.users = [NamedId(**name) for name in self.users] + def __str__(self) -> str: + return self.name + @dataclass class User(): @@ -71,7 +79,6 @@ def __post_init__(self): self.custom_fields = [CustomField(**field) for field in self.custom_fields] self.discord_id = self.get_custom_field(DISCORD_ID_FIELD) - def get_custom_field(self, name: str) -> str: for field in self.custom_fields: if field.name == name: @@ -79,6 +86,16 @@ def get_custom_field(self, name: str) -> str: return None + def full_name(self) -> str: + if self.firstname is None or len(self.firstname) < 2: + return self.lastname + if self.lastname is None or len(self.lastname) < 2: + return self.firstname + return self.firstname + " " + self.lastname + + def __str__(self): + return f"#{self.id} {self.full_name()} login={self.login} discord={self.discord_id}" + @dataclass class UserResult: @@ -91,6 +108,9 @@ class UserResult: def __post_init__(self): self.users = [User(**user) for user in self.users] + def __str__(self): + return f"users:({[u.login + ',' for u in self.users]}), total={self.total_count}, {self.limit}/{self.offset}" + class UserCache(): """cache of user data""" @@ -486,7 +506,7 @@ def reindex_users(self): log.debug(f"indexed {len(all_users)} users") log.debug(f"discord users: {self.cache.discord_users}") else: - log.error("No users to index") + log.warning("No users to index") def reindex_teams(self): @@ -495,7 +515,7 @@ def reindex_teams(self): self.cache.teams = all_teams # replace all the cached teams log.debug(f"indexed {len(all_teams)} teams") else: - log.error("No teams to index") + log.warning("No teams to index") def reindex(self): From a7f5cab977a2d82a6e91c705dc6140b7849b82ed Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Tue, 5 Mar 2024 14:10:30 -0800 Subject: [PATCH 15/15] test stability once more, after completely refactoring tests base classes --- cog_tickets.py | 1 + imap.py | 17 +++++------- redmine.py | 35 ++++++++++++----------- session.py | 26 ++++++++++-------- test_cog_scn.py | 14 +++++----- test_cog_tickets.py | 2 +- test_netbot.py | 7 ++--- test_redmine.py | 28 ++++++++++--------- test_utils.py | 67 +++++++++++---------------------------------- tickets.py | 51 ++++++++++++++++------------------ users.py | 17 ++++++------ 11 files changed, 115 insertions(+), 150 deletions(-) diff --git a/cog_tickets.py b/cog_tickets.py index f7d849b..ee79d69 100644 --- a/cog_tickets.py +++ b/cog_tickets.py @@ -165,6 +165,7 @@ async def thread_ticket(self, ctx: discord.ApplicationContext, ticket_id:int): # TODO message templates note = f"Created Discord thread: {thread.name}: {thread.jump_url}" user = self.redmine.user_mgr.find_discord_user(ctx.user.name) + log.debug(f">>> found {user} for {ctx.user.name}") self.redmine.enable_discord_sync(ticket.id, user, note) await ctx.respond(f"Created new thread for {ticket.id}: {thread}") # todo add some fancy formatting diff --git a/imap.py b/imap.py index fd42d6f..33a1f90 100755 --- a/imap.py +++ b/imap.py @@ -45,6 +45,11 @@ def set_token(self, token): class Message(): """email message""" + from_address: str + subject:str + attachments: list[Attachment] + note: str + def __init__(self, from_addr:str, subject:str): self.from_address = from_addr self.subject = subject @@ -251,16 +256,8 @@ def handle_message(self, msg_id:str, message:Message): log.info(f"Updated ticket #{ticket.id} with message from {user.login} and {len(message.attachments)} attachments") else: # no open tickets, create new ticket for the email message - self.redmine.create_ticket(user, subject, message.note, message.attachments) - log.info(f"Created new ticket for: {user.login}, {subject}, with {len(message.attachments)} attachments") - - """ - # check status - if self.user_mgr.is_blocked(user): - log.debug(f"Rejecting ticket #{ticket.id} based on blocked user {user.login}") - self.reject_ticket(ticket.id) - refresh. - """ + ticket = self.redmine.create_ticket(user, subject, message.note, message.attachments) + log.info(f"Created new ticket for: {ticket}, with {len(message.attachments)} attachments") def synchronize(self): diff --git a/redmine.py b/redmine.py index b2d7f0f..ffa59b5 100644 --- a/redmine.py +++ b/redmine.py @@ -28,23 +28,15 @@ class RedmineException(Exception): """redmine exception""" - def __init__(self, message: str, request_id: str) -> None: + def __init__(self, message: str, request_id: str = "-") -> None: super().__init__(message + ", req_id=" + request_id) self.request_id = request_id -class Client(): ## redmine.Client +class Client(): """redmine client""" - def __init__(self, url: str, token: str): - self.url = url - if self.url is None: - raise RedmineException("Unable to load REDMINE_URL", "[n/a]") - - self.token = token - if self.url is None: - raise RedmineException("Unable to load REDMINE_TOKEN", "__init__") - - session:RedmineSession = RedmineSession(url, token) + def __init__(self, session:RedmineSession): + self.url = session.url self.user_mgr:UserManager = UserManager(session) self.ticket_mgr:TicketManager = TicketManager(session) @@ -54,15 +46,26 @@ def __init__(self, url: str, token: str): @classmethod def fromenv(cls): url = os.getenv('REDMINE_URL') + if url is None: + raise RedmineException("Unable to load REDMINE_URL") + token = os.getenv('REDMINE_TOKEN') - return cls(url, token) + if token is None: + raise RedmineException("Unable to load REDMINE_TOKEN") + + return cls(RedmineSession(url, token)) - def create_ticket(self, user, subject, body, attachments=None): - return self.ticket_mgr.create(user, subject, body, attachments) + def create_ticket(self, user, subject, body, attachments=None) -> Ticket: + ticket = self.ticket_mgr.create(user, subject, body, attachments) + # check user status, reject the ticket if blocked + if self.user_mgr.is_blocked(user): + log.debug(f"Rejecting ticket #{ticket.id} based on blocked user {user.login}") + ticket = self.ticket_mgr.reject_ticket(ticket.id) + return ticket - def update_ticket(self, ticket_id:int, fields:dict, user_login:str=None): + def update_ticket(self, ticket_id:int, fields:dict, user_login:str|None=None): return self.ticket_mgr.update(ticket_id, fields, user_login) diff --git a/session.py b/session.py index c83cb69..6dcdcfe 100644 --- a/session.py +++ b/session.py @@ -6,8 +6,9 @@ import logging from urllib3.exceptions import ConnectTimeoutError import requests -from requests.exceptions import ConnectTimeout, ConnectionError +from requests.exceptions import ConnectTimeout +import dotenv log = logging.getLogger(__name__) @@ -47,8 +48,12 @@ def fromenv(cls): return cls(url, token) + @classmethod + def fromenvfile(cls): + dotenv.load_dotenv() + return cls.fromenv() - def get_headers(self, impersonate_id:str=None): + def get_headers(self, impersonate_id:str|None=None): headers = { 'User-Agent': 'netbot/0.0.1', # TODO update to project version, and add version management 'Content-Type': 'application/json', @@ -62,11 +67,9 @@ def get_headers(self, impersonate_id:str=None): return headers - def get(self, query_str:str, user:str=None): + def get(self, query_str:str, impersonate_id:str|None=None): """run a query against a redmine instance""" - - headers = self.get_headers(user) - + headers = self.get_headers(impersonate_id) try: r = self.session.get(f"{self.url}{query_str}", headers=headers, timeout=TIMEOUT) @@ -83,19 +86,18 @@ def get(self, query_str:str, user:str=None): return None - # data=json.dumps(data), - def put(self, resource: str, data:dict, user_login: str = None) -> None: + def put(self, resource: str, data:str, impersonate_id:str|None=None) -> None: r = self.session.put(f"{self.url}{resource}", data=data, timeout=TIMEOUT, - headers=self.get_headers(user_login)) + headers=self.get_headers(impersonate_id)) if r.ok: - log.debug(f"PUT {resource}: {data} - {r}") + log.debug(f"PUT {resource}: {data}") else: - raise RedmineException(f"POST {resource} by {user_login} failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) + raise RedmineException(f"POST {resource} by {impersonate_id} failed, status=[{r.status_code}] {r.reason}", r.headers['X-Request-Id']) - def post(self, resource: str, data:dict = None, user_login: str = None, files = None) -> dict: + def post(self, resource: str, data:str, user_login: str|None = None, files: list|None = None) -> dict|None: r = self.session.post(f"{self.url}{resource}", data=data, files=files, diff --git a/test_cog_scn.py b/test_cog_scn.py index c31cc41..00c7bde 100755 --- a/test_cog_scn.py +++ b/test_cog_scn.py @@ -42,7 +42,7 @@ async def test_team_join_leave(self): # check add result #ctx.respond.assert_called_with( - # f"Discord user: {self.discord_user} has been paired with redmine user: {self.user.login}") + # f"Discord user: {self.user.discord_id} has been paired with redmine user: {self.user.login}") # reindex using cog ctx = self.build_context() @@ -52,7 +52,7 @@ async def test_team_join_leave(self): # 4.5 check reindex result, and lookup based on login and discord id ctx.respond.assert_called_with("Rebuilt redmine indices.") self.assertIsNotNone(self.redmine.user_mgr.find(self.user.login)) - self.assertIsNotNone(self.redmine.user_mgr.find(self.discord_user)) + self.assertIsNotNone(self.redmine.user_mgr.find(self.user.discord_id)) # join team users ctx = self.build_context() @@ -63,13 +63,13 @@ async def test_team_join_leave(self): # confirm via mock callback and API #ctx.respond.assert_called_with(f"Unknown team name: {test_team_name}") # unknown team response! - ctx.respond.assert_called_with(f"**{self.discord_user}** has joined *{test_team_name}*") + ctx.respond.assert_called_with(f"**{self.user.discord_id}** has joined *{test_team_name}*") self.assertTrue(self.redmine.user_mgr.is_user_in_team(self.user, test_team_name), f"{self.user.login} not in team {test_team_name}") # confirm in team via cog teams response ctx = self.build_context() await self.cog.teams(ctx, test_team_name) - self.assertIn(self.full_name, str(ctx.respond.call_args)) + self.assertIn(self.user.full_name(), str(ctx.respond.call_args)) # leave team users ctx = self.build_context() @@ -78,12 +78,12 @@ async def test_team_join_leave(self): # confirm via API and callback self.assertFalse(self.redmine.user_mgr.is_user_in_team(self.user, test_team_name), f"{self.user.login} *in* team {test_team_name}") - ctx.respond.assert_called_with(f"**{self.discord_user}** has left *{test_team_name}*") + ctx.respond.assert_called_with(f"**{self.user.discord_id}** has left *{test_team_name}*") # confirm not in team via cog teams response ctx = self.build_context() await self.cog.teams(ctx, test_team_name) - self.assertNotIn(self.full_name, str(ctx.respond.call_args)) + self.assertNotIn(self.user.full_name(), str(ctx.respond.call_args)) async def test_thread_sync(self): @@ -148,7 +148,7 @@ async def test_locked_during_sync_ticket(self): message = unittest.mock.AsyncMock(discord.Message) message.content = f"This is a new note about ticket #{ticket.id} for test {self.tag}" message.author = unittest.mock.AsyncMock(discord.Member) - message.author.name = self.discord_user + message.author.name = self.user.discord_id thread = unittest.mock.AsyncMock(discord.Thread) thread.name = f"Ticket #{ticket.id}: {ticket.subject}" diff --git a/test_cog_tickets.py b/test_cog_tickets.py index 6975125..1c56f50 100755 --- a/test_cog_tickets.py +++ b/test_cog_tickets.py @@ -115,7 +115,7 @@ async def test_thread_sync(self): thread.name = f"Ticket #{ticket.id}: {subject}" member = unittest.mock.AsyncMock(discord.Member) - member.name=self.discord_user + member.name=self.user.discord_id message = unittest.mock.AsyncMock(discord.Message) message.channel = ctx.channel diff --git a/test_netbot.py b/test_netbot.py index 332a857..b06643a 100755 --- a/test_netbot.py +++ b/test_netbot.py @@ -2,7 +2,6 @@ """NetBot Test Suite""" import unittest -from unittest import mock import logging import discord @@ -48,7 +47,7 @@ async def test_synchronize_ticket(self): message = unittest.mock.AsyncMock(discord.Message) message.content = f"This is a new note about ticket #{ticket.id} for test {self.tag}" message.author = unittest.mock.AsyncMock(discord.Member) - message.author.name = self.discord_user + message.author.name = self.user.discord_id thread = unittest.mock.AsyncMock(discord.Thread) thread.name = f"Ticket #{ticket.id}" @@ -62,7 +61,7 @@ async def test_synchronize_ticket(self): # assert method send called on mock thread, with the correct values self.assertIn(self.tag, thread.send.call_args.args[0]) - self.assertIn(self.full_name, thread.send.call_args.args[0]) + self.assertIn(self.user.full_name(), thread.send.call_args.args[0]) self.assertIn(body, thread.send.call_args.args[0]) # get notes from redmine, assert tags in most recent @@ -85,7 +84,7 @@ async def test_sync_ticket_long_message(self): message = unittest.mock.AsyncMock(discord.Message) message.content = f"This is a new note about ticket #{ticket.id} for test {self.tag}" message.author = unittest.mock.AsyncMock(discord.Member) - message.author.name = self.discord_user + message.author.name = self.user.discord_id thread = unittest.mock.AsyncMock(discord.Thread) thread.name = f"Ticket #{ticket.id}" diff --git a/test_redmine.py b/test_redmine.py index a5a59a1..bd01654 100755 --- a/test_redmine.py +++ b/test_redmine.py @@ -7,6 +7,7 @@ from dotenv import load_dotenv import redmine +import session import test_utils @@ -29,27 +30,28 @@ def test_block_user(self): self.user_mgr.unblock(self.user) self.assertFalse(self.user_mgr.is_blocked(self.user)) - """ + def test_blocked_create_ticket(self): - try: - # block - self.user_mgr.block(self.user) - self.assertTrue(self.redmine.user_mgr.is_blocked(self.user)) + # block + self.user_mgr.block(self.user) + self.assertTrue(self.user_mgr.is_blocked(self.user)) - # create ticket for blocked - ticket = self.create_ticket(self.user, "subject", "body") - self.assertEqual("Reject", ticket.status.name) + # create ticket for blocked + ticket = self.create_test_ticket() + self.assertIsNotNone(ticket) + self.assertEqual("Reject", ticket.status.name) - finally: - # remove the test user - self.redmine.user_mgr.remove(user) - """ + # remove the ticket and unbluck the user + self.tickets_mgr.remove(ticket.id) + self.user_mgr.unblock(self.user) + self.assertFalse(self.user_mgr.is_blocked(self.user)) def test_client_timeout(self): # construct an invalid client to try to get a timeout try: - client = redmine.Client("http://192.168.1.42/", "bad-token") + bad_session = session.RedmineSession("http://192.168.1.42/", "bad-token") + client = redmine.Client(bad_session) self.assertIsNotNone(client) #log.info(client) except Exception: diff --git a/test_utils.py b/test_utils.py index 8d7a458..cb68c5d 100755 --- a/test_utils.py +++ b/test_utils.py @@ -78,79 +78,44 @@ def remove_test_users(user_mgr:UserManager): class RedmineTestCase(unittest.TestCase): """Abstract base class for testing redmine features""" - user_mgr: UserManager - tickets_mgr: tickets.TicketManager - tag: str - user: User - @classmethod def setUpClass(cls): sess = session.RedmineSession.fromenv() - cls.user_mgr = UserManager(sess) - cls.tickets_mgr = tickets.TicketManager(sess) + cls.redmine = Client(sess) + cls.user_mgr = cls.redmine.user_mgr + cls.tickets_mgr = cls.redmine.ticket_mgr cls.tag:str = tagstr() cls.user:User = create_test_user(cls.user_mgr, cls.tag) + cls.user_mgr.cache.cache_user(cls.user) log.info(f"SETUP created test user: {cls.user}") - @classmethod def tearDownClass(cls): - cls.user_mgr.remove(cls.user) - log.info(f"TEARDOWN removed test user: {cls.user}") - - -class BotTestCase(unittest.IsolatedAsyncioTestCase): - """Abstract base class for testing Bot features""" - redmine: Client = None - usertag: str = None - user: User = None - - @classmethod - def setUpClass(cls): - log.info("Setting up test fixtures") - cls.redmine:Client = Client.fromenv() - cls.usertag:str = tagstr() - cls.user:User = create_test_user(cls.redmine.user_mgr, cls.usertag) - log.info(f"Created test user: {cls.user}") + if cls.user: + cls.user_mgr.remove(cls.user) + log.info(f"TEARDOWN removed test user: {cls.user}") + def create_test_ticket(self) -> tickets.Ticket: + subject = f"TEST {self.tag} {unittest.TestCase.id(self)}" + text = f"This is a ticket for {unittest.TestCase.id(self)} with {self.tag}." + ticket = self.redmine.create_ticket(self.user, subject, text) + return ticket - @classmethod - def tearDownClass(cls): - log.info(f"Tearing down test fixtures: {cls.user}") - cls.redmine.user_mgr.remove(cls.user) +class BotTestCase(RedmineTestCase, unittest.IsolatedAsyncioTestCase): + """Abstract base class for testing Bot features""" def build_context(self) -> ApplicationContext: ctx = mock.AsyncMock(ApplicationContext) ctx.user = mock.AsyncMock(discord.Member) - ctx.user.name = self.discord_user + ctx.user.name = self.user.discord_id ctx.command = mock.AsyncMock(discord.ApplicationCommand) ctx.command.name = unittest.TestCase.id(self) - log.debug(f"created ctx with {self.discord_user}: {ctx}") + log.debug(f"created ctx with {self.user.discord_id}: {ctx}") return ctx - def create_test_ticket(self): - subject = f"{unittest.TestCase.id(self)} {self.tag}" - text = f"This is a ticket for {unittest.TestCase.id(self)} with {self.tag}." - return self.redmine.create_ticket(self.user, subject, text) - - - def setUp(self): - self.tag = self.__class__.usertag # TODO just rename usertag to tag - represents the suite run - self.assertIsNotNone(self.tag) - self.assertIsNotNone(self.user) - - self.full_name = self.user.firstname + " " + self.user.lastname - self.discord_user = self.user.discord_id - - self.assertIsNotNone(self.redmine.user_mgr.find(self.user.login)) - self.assertIsNotNone(self.redmine.user_mgr.find(self.discord_user)) - - log.debug(f"setUp user {self.user.login} {self.discord_user}") - - if __name__ == '__main__': # when running this main, turn on DEBUG logging.basicConfig(level=logging.DEBUG, format="{asctime} {levelname:<8s} {name:<16} {message}", style='{') diff --git a/tickets.py b/tickets.py index 7a8dd39..3c56071 100644 --- a/tickets.py +++ b/tickets.py @@ -54,7 +54,7 @@ class TicketNote(): # https://www.redmine.org/projects/redmine/wiki/Rest_IssueJo created_on: dt.datetime private_notes: bool details: list[PropertyChange] - user: NamedId = None + user: NamedId | None = None def __post_init__(self): self.user = NamedId(**self.user) @@ -80,18 +80,18 @@ class Ticket(): created_on: dt.datetime updated_on: dt.datetime closed_on: dt.datetime - project: NamedId = None - tracker: NamedId = None - priority: NamedId = None - author: NamedId = None - status: TicketStatus = None - parent: NamedId = None + project: NamedId|None = None + tracker: NamedId|None = None + priority: NamedId|None = None + author: NamedId|None = None + status: TicketStatus|None = None + parent: NamedId|None = None spent_hours: float = 0.0 total_spent_hours: float = 0.0 - category: str = None - assigned_to: NamedId = None - custom_fields: list[CustomField] = None - journals: list[TicketNote] = None + category: str|None = None + assigned_to: NamedId|None = None + custom_fields: list[CustomField]|None = None + journals: list[TicketNote]|None = None def __post_init__(self): self.status = TicketStatus(**self.status) @@ -117,7 +117,7 @@ def __post_init__(self): if self.journals: self.journals = [TicketNote(**note) for note in self.journals] - def get_custom_field(self, name: str) -> str: + def get_custom_field(self, name: str) -> str | None: if self.custom_fields: for field in self.custom_fields: if field.name == name: @@ -127,7 +127,7 @@ def get_custom_field(self, name: str) -> str: def __str__(self): return f"#{self.id} {self.project} {self.status} {self.priority} {self.assigned_to}: {self.subject}" - def get_sync_record(self, expected_channel: int) -> synctime.SyncRecord: + def get_sync_record(self, expected_channel: int) -> synctime.SyncRecord | None: # Parse custom_field into datetime # lookup field by name token = self.get_custom_field(SYNC_FIELD_NAME) @@ -156,9 +156,10 @@ def get_sync_record(self, expected_channel: int) -> synctime.SyncRecord: # self.update_sync_record(record) same REALLY as above ^^^^ log.debug(f"created new sync record, none found: {record}") return record + return None - def get_notes(self, since:dt.datetime=None) -> list[TicketNote]: + def get_notes(self, since:dt.datetime|None=None) -> list[TicketNote]: notes = [] for note in self.journals: @@ -230,25 +231,21 @@ def create(self, user:User, subject, body, attachments=None) -> Ticket: response.headers['X-Request-Id']) - def update(self, ticket_id:str, fields:dict, user_login:str=None) -> Ticket: + def update(self, ticket_id:int, fields:dict[str,str], user_login:str|None=None) -> Ticket|None: """update a redmine ticket""" # PUT a simple JSON structure data = { - 'issue': {} + 'issue': fields } - data['issue'] = fields - - response = self.session.put(f"{ISSUE_RESOURCE}{ticket_id}.json", json.dumps(data), user_login) - if response: - # no body, so re-get the updated tickets? - return self.get(ticket_id) + self.session.put(f"{ISSUE_RESOURCE}{ticket_id}.json", json.dumps(data), user_login) + return self.get(ticket_id) def append_message(self, ticket_id:int, user_login:str, note:str, attachments=None): """append a note to a ticket""" # PUT a simple JSON structure - data = { + data:dict = { 'issue': { 'notes': note, } @@ -293,7 +290,7 @@ def get_tickets_by(self, user) -> list[Ticket]: return None - def get(self, ticket_id:int, include_journals:bool = False) -> Ticket: + def get(self, ticket_id:int, include_journals:bool = False) -> Ticket|None: """get a ticket by ID""" if ticket_id is None or ticket_id == 0: #log.debug(f"Invalid ticket number: {ticket_id}") @@ -483,7 +480,7 @@ def assign_ticket(self, ticket_id, user:User, user_id=None): self.update(ticket_id, fields, user_id) - def progress_ticket(self, ticket_id, user_id=None): # TODO notes + def progress_ticket(self, ticket_id, user_id=None): fields = { "assigned_to_id": "me", "status_id": "2", # "In Progress" @@ -491,12 +488,12 @@ def progress_ticket(self, ticket_id, user_id=None): # TODO notes self.update(ticket_id, fields, user_id) - def reject_ticket(self, ticket_id, user_id=None): # TODO notes + def reject_ticket(self, ticket_id, user_id=None) -> Ticket: fields = { "assigned_to_id": "", "status_id": "5", # "Reject" } - self.update(ticket_id, fields, user_id) + return self.update(ticket_id, fields, user_id) def unassign_ticket(self, ticket_id, user_id=None): diff --git a/users.py b/users.py index 25cdaf0..74f073f 100644 --- a/users.py +++ b/users.py @@ -118,7 +118,7 @@ def __init__(self): self.users: dict[str, int] = {} self.user_ids: dict[int, User] = {} self.user_emails: dict[str, int] = {} - self.discord_users: dict[str, int] = {} + self.discord_ids: dict[str, int] = {} self.teams: dict[str, Team] = {} @@ -127,7 +127,7 @@ def clear(self): self.users.clear() self.user_ids.clear() self.user_emails.clear() - self.discord_users.clear() + self.discord_ids.clear() def cache_user(self, user: User) -> None: @@ -138,7 +138,7 @@ def cache_user(self, user: User) -> None: self.users[user.login] = user.id self.user_emails[user.mail] = user.id if user.discord_id: - self.discord_users[user.discord_id] = user.id + self.discord_ids[user.discord_id] = user.id def cache_team(self, team: Team) -> None: @@ -163,8 +163,8 @@ def find(self, name): return self.get(self.user_emails[name]) elif name in self.users: return self.get(self.users[name]) - elif name in self.discord_users: - return self.get(self.discord_users[name]) + elif name in self.discord_ids: + return self.get(self.discord_ids[name]) elif name in self.teams: return self.teams[name] #ugly. put groups in user collection? else: @@ -181,8 +181,8 @@ def find_discord_user(self, discord_user_id:str) -> User: if discord_user_id is None: return None - if discord_user_id in self.discord_users: - user_id = self.discord_users[discord_user_id] + if discord_user_id in self.discord_ids: + user_id = self.discord_ids[discord_user_id] return self.user_ids[user_id] else: return None @@ -504,7 +504,7 @@ def reindex_users(self): self.cache.cache_user(user) # several internal indicies log.debug(f"indexed {len(all_users)} users") - log.debug(f"discord users: {self.cache.discord_users}") + log.debug(f"discord users: {self.cache.discord_ids}") else: log.warning("No users to index") @@ -533,7 +533,6 @@ def reindex(self): # load credentials from dotenv import load_dotenv load_dotenv() - users = UserManager(RedmineSession.fromenv()) for teamname in users.get_all_teams(): team = users.get_team_by_name(teamname)