From 012a27819ead4203f75201dd10763f2ba3ee666f Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Thu, 25 Apr 2024 18:20:16 -0700 Subject: [PATCH 1/3] refactored TicketManager to require a default_project, and added a project_id to ticket create. updated tests. all tests pass --- redmine.py | 8 ++++---- test_tickets.py | 2 +- test_utils.py | 5 +++-- tickets.py | 16 +++++++++++----- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/redmine.py b/redmine.py index f48d8a8..7f95796 100644 --- a/redmine.py +++ b/redmine.py @@ -23,7 +23,6 @@ SYNC_FIELD_NAME = "syncdata" DISCORD_ID_FIELD = "Discord ID" BLOCKED_TEAM_NAME = "blocked" -SCN_PROJECT_ID = 1 # could lookup scn in projects STATUS_REJECT = 5 # could to status lookup, based on "reject" @@ -36,10 +35,10 @@ def __init__(self, message: str, request_id: str = "-") -> None: class Client(): """redmine client""" - def __init__(self, session:RedmineSession): + def __init__(self, session:RedmineSession, default_project: int): self.url = session.url self.user_mgr:UserManager = UserManager(session) - self.ticket_mgr:TicketManager = TicketManager(session) + self.ticket_mgr:TicketManager = TicketManager(session, default_project=default_project) self.user_mgr.reindex() # build the cache when starting @@ -54,7 +53,8 @@ def fromenv(cls): if token is None: raise RedmineException("Unable to load REDMINE_TOKEN") - return cls(RedmineSession(url, token)) + default_project = os.getenv("DEFAULT_PROJECT_ID", default="1") + return cls(RedmineSession(url, token), default_project) def create_ticket(self, user:User, message:Message) -> Ticket: diff --git a/test_tickets.py b/test_tickets.py index ebd164c..090c58a 100755 --- a/test_tickets.py +++ b/test_tickets.py @@ -17,7 +17,7 @@ class TestTicketManager(unittest.TestCase): """Mocked testing of ticket manager""" def mock_mgr(self): - return TicketManager(test_utils.mock_session()) + return TicketManager(test_utils.mock_session(), "1") @patch('tickets.TicketManager.load_custom_fields') @patch('session.RedmineSession.get') diff --git a/test_utils.py b/test_utils.py index 444217d..38edd40 100755 --- a/test_utils.py +++ b/test_utils.py @@ -16,6 +16,7 @@ import model from users import UserManager from model import Message, User +from tickets import SCN_PROJECT_ID import session from redmine import Client @@ -110,7 +111,7 @@ class RedmineTestCase(unittest.TestCase): @classmethod def setUpClass(cls): sess = session.RedmineSession.fromenv() - cls.redmine = Client(sess) + cls.redmine = Client(sess, SCN_PROJECT_ID) cls.user_mgr = cls.redmine.user_mgr cls.tickets_mgr = cls.redmine.ticket_mgr cls.tag:str = tagstr() @@ -149,7 +150,7 @@ def build_context(self) -> ApplicationContext: def audit_expected_values(): - redmine = Client(session.RedmineSession.fromenv()) + redmine = Client(session.RedmineSession.fromenv(), SCN_PROJECT_ID) # audit checks... # 1. make sure admin use exists diff --git a/tickets.py b/tickets.py index e3c2140..7b8b22d 100644 --- a/tickets.py +++ b/tickets.py @@ -20,7 +20,7 @@ ISSUES_RESOURCE="/issues.json" ISSUE_RESOURCE="/issues/" DEFAULT_SORT = "status:desc,priority:desc,updated_on:desc" -SCN_PROJECT_ID = 1 # could lookup scn in projects +SCN_PROJECT_ID = "1" # could lookup scn in projects INTAKE_TEAM = "ticket-intake" INTAKE_TEAM_ID = 19 # FIXME @@ -31,9 +31,10 @@ class TicketManager(): """manage redmine tickets""" - def __init__(self, session: RedmineSession): + def __init__(self, session: RedmineSession, default_project): self.session: RedmineSession = session self.custom_fields = self.load_custom_fields() + self.default_project = default_project def load_custom_fields(self) -> dict[str,NamedId]: @@ -67,15 +68,19 @@ def get_field_id(self, name:str) -> int | None: return None - def create(self, user: User, message: Message) -> Ticket: + def create(self, user: User, message: Message, project_id: int = None) -> Ticket: """create a redmine ticket""" # https://www.redmine.org/projects/redmine/wiki/Rest_Issues#Creating-an-issue # would need full param handling to pass that thru discord to get to this invocation # this would be resolved by a Ticket class to emcapsulate. + # check default project + if not project_id: + project_id = self.default_project + data = { 'issue': { - 'project_id': SCN_PROJECT_ID, #FIXME hard-coded project ID MOVE project ID to API + 'project_id': project_id, 'subject': message.subject, 'description': message.note, # ticket-485: adding custom field for To//Cc headers. @@ -490,7 +495,7 @@ def get_field(self, ticket:Ticket, fieldname:str) -> str: def main(): - ticket_mgr = TicketManager(RedmineSession.fromenv()) + ticket_mgr = TicketManager(RedmineSession.fromenv(), SCN_PROJECT_ID) #ticket_mgr.expire_expired_tickets() #for ticket in ticket_mgr.older_than(7): #ticket_mgr.expired_tickets(): @@ -500,6 +505,7 @@ def main(): print(ticket_mgr.due()) + # for testing the redmine if __name__ == '__main__': # load credentials From a69dd743e486fdf634950aa7d762b0064cb2e65a Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Fri, 26 Apr 2024 09:19:37 -0700 Subject: [PATCH 2/3] cleanup and noting some constants --- redmine.py | 9 +++++---- test_tickets.py | 4 ++-- tickets.py | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/redmine.py b/redmine.py index 7f95796..9c3e9c0 100644 --- a/redmine.py +++ b/redmine.py @@ -12,7 +12,7 @@ from session import RedmineSession from model import Message, Ticket, User from users import UserManager -from tickets import TicketManager +from tickets import TicketManager, SCN_PROJECT_ID log = logging.getLogger(__name__) @@ -53,7 +53,7 @@ def fromenv(cls): if token is None: raise RedmineException("Unable to load REDMINE_TOKEN") - default_project = os.getenv("DEFAULT_PROJECT_ID", default="1") + default_project = os.getenv("DEFAULT_PROJECT_ID", default=SCN_PROJECT_ID) return cls(RedmineSession(url, token), default_project) @@ -134,7 +134,7 @@ def get_notes_since(self, ticket_id, timestamp=None): def enable_discord_sync(self, ticket_id, user, note): fields = { "note": note, #f"Created Discord thread: {thread.name}: {thread.jump_url}", - "cf_1": "1", + "cf_1": "1", # TODO: read from custom fields via } self.update_ticket(ticket_id, fields, user.login) @@ -168,7 +168,8 @@ def update_sync_record(self, record:synctime.SyncRecord): log.debug(f"Updating sync record in redmine: {record}") fields = { "custom_fields": [ - { "id": 4, "value": record.token_str() } # cf_4, custom field syncdata, #TODO search for it + { "id": 4, "value": record.token_str() } # cf_4, custom field syncdata, + #TODO search for custom field ID with TicketManager.get_field_id ] } self.update_ticket(record.ticket_id, fields) diff --git a/test_tickets.py b/test_tickets.py index 090c58a..71ee947 100755 --- a/test_tickets.py +++ b/test_tickets.py @@ -7,7 +7,7 @@ from dotenv import load_dotenv -from tickets import TicketManager +from tickets import TicketManager, SCN_PROJECT_ID import test_utils @@ -17,7 +17,7 @@ class TestTicketManager(unittest.TestCase): """Mocked testing of ticket manager""" def mock_mgr(self): - return TicketManager(test_utils.mock_session(), "1") + return TicketManager(test_utils.mock_session(), SCN_PROJECT_ID) @patch('tickets.TicketManager.load_custom_fields') @patch('session.RedmineSession.get') diff --git a/tickets.py b/tickets.py index 7b8b22d..d478cfe 100644 --- a/tickets.py +++ b/tickets.py @@ -232,7 +232,7 @@ def expire(self, ticket:Ticket): # Ideally, this would @ the owner and collaborators. fields = { "assigned_to_id": INTAKE_TEAM_ID, - "status_id": "1", # New + "status_id": "1", # New, TODO lookup using status lookup table. "notes": f"Ticket automatically expired after {TICKET_MAX_AGE} days due to inactivity.", } self.update(ticket.id, fields) @@ -414,7 +414,7 @@ def get_notes_since(self, ticket_id:int, timestamp:dt.datetime=None) -> list[Tic def enable_discord_sync(self, ticket_id, user, note): fields = { "note": note, #f"Created Discord thread: {thread.name}: {thread.jump_url}", - "cf_1": "1", + "cf_1": "1", # TODO: lookup in self.get_field_id } self.update(ticket_id, fields, user.login) @@ -452,7 +452,7 @@ def reject_ticket(self, ticket_id, user_id=None) -> Ticket: def unassign_ticket(self, ticket_id, user_id=None): fields = { "assigned_to_id": INTAKE_TEAM_ID, - "status_id": "1", # New + "status_id": "1", # New, TODO lookup in status table } self.update(ticket_id, fields, user_id) From 17851e8e755bd1e011fa7deb16849aada6f339cd Mon Sep 17 00:00:00 2001 From: Paul Philion Date: Sat, 27 Apr 2024 14:23:36 -0700 Subject: [PATCH 3/3] adding test cases and clean up mock behavior for redmine. must more useful redmine mock now --- model.py | 4 ++ redmine.py | 17 +++++++-- session.py | 2 +- test_tickets.py | 34 ++++++++++++----- test_utils.py | 98 ++++++++++++++++++++++++++++++++++++++++++++++--- tickets.py | 4 ++ 6 files changed, 138 insertions(+), 21 deletions(-) diff --git a/model.py b/model.py index 0bfc8ad..4d6edb6 100644 --- a/model.py +++ b/model.py @@ -200,6 +200,10 @@ def full_name(self) -> str: def __str__(self): return f"#{self.id} {self.full_name()} login={self.login} discord={self.discord_id}" + def set_field(self, fieldname:str, value): + setattr(self, fieldname, value) + log.debug(f"@{self.login}: {fieldname} <= {value}") + @dataclass class UserResult: diff --git a/redmine.py b/redmine.py index 9c3e9c0..3fa877c 100644 --- a/redmine.py +++ b/redmine.py @@ -35,14 +35,22 @@ def __init__(self, message: str, request_id: str = "-") -> None: class Client(): """redmine client""" - def __init__(self, session:RedmineSession, default_project: int): + def __init__(self, session:RedmineSession, user_mgr:UserManager, ticket_mgr:TicketManager): self.url = session.url - self.user_mgr:UserManager = UserManager(session) - self.ticket_mgr:TicketManager = TicketManager(session, default_project=default_project) + self.user_mgr = user_mgr + self.ticket_mgr = ticket_mgr self.user_mgr.reindex() # build the cache when starting + @classmethod + def from_session(cls, session:RedmineSession, default_project:int): + user_mgr = UserManager(session) + ticket_mgr = TicketManager(session, default_project=default_project) + + return cls(session, user_mgr, ticket_mgr) + + @classmethod def fromenv(cls): url = os.getenv('REDMINE_URL') @@ -54,7 +62,8 @@ def fromenv(cls): raise RedmineException("Unable to load REDMINE_TOKEN") default_project = os.getenv("DEFAULT_PROJECT_ID", default=SCN_PROJECT_ID) - return cls(RedmineSession(url, token), default_project) + + return cls.from_session(RedmineSession(url, token), default_project) def create_ticket(self, user:User, message:Message) -> Ticket: diff --git a/session.py b/session.py index 3d524d4..e82c911 100644 --- a/session.py +++ b/session.py @@ -75,7 +75,7 @@ def get(self, query_str:str, impersonate_id:str|None=None): if r.ok: return r.json() else: - log.info(f"GET {r.reason}/{r.status_code} url={r.request.url}, reqid={r.headers['X-Request-Id']}") + log.info(f"GET {r.reason}/{r.status_code} url={r.request.url}, reqid={r.headers.get('X-Request-Id','')}") except (TimeoutError, ConnectTimeoutError, ConnectTimeout, ConnectionError): # ticket-509: Handle timeout gracefully log.warning(f"TIMEOUT ({TIMEOUT}s) during {query_str}") diff --git a/test_tickets.py b/test_tickets.py index 71ee947..9de3c32 100755 --- a/test_tickets.py +++ b/test_tickets.py @@ -3,28 +3,22 @@ import unittest import logging +import json from unittest.mock import MagicMock, patch from dotenv import load_dotenv -from tickets import TicketManager, SCN_PROJECT_ID import test_utils log = logging.getLogger(__name__) -class TestTicketManager(unittest.TestCase): +class TestTicketManager(test_utils.MockRedmineTestCase): """Mocked testing of ticket manager""" - def mock_mgr(self): - return TicketManager(test_utils.mock_session(), SCN_PROJECT_ID) - @patch('tickets.TicketManager.load_custom_fields') @patch('session.RedmineSession.get') - def test_expired_tickets(self, mock_get:MagicMock, mock_cf:MagicMock): - # setup custom fields - mock_cf.return_value = test_utils.custom_fields() # TODO move to mgr setup - + def test_expired_tickets(self, mock_get:MagicMock): # setup the mock tickets ticket = test_utils.mock_ticket() result = test_utils.mock_result([ @@ -33,13 +27,33 @@ def test_expired_tickets(self, mock_get:MagicMock, mock_cf:MagicMock): ]) mock_get.return_value = result.asdict() - expired = self.mock_mgr().expired_tickets() + expired = self.tickets_mgr.expired_tickets() self.assertGreater(len(expired), 0) expired_ids = [ticket.id for ticket in expired] self.assertIn(ticket.id, expired_ids) #FIXME mock_get.assert_called_once() + @patch('session.RedmineSession.post') + def test_default_project_id(self, mock_post:MagicMock): + test_proj_id = "42" + + msg = self.create_message() + # note to future self: mock response to create should have content of crete request + + # setup the mock tickets + ticket = test_utils.mock_ticket() + mock_post.return_value = { "issue": ticket.asdict() } + + self.tickets_mgr.create(self.user, msg, test_proj_id) + + resp_ticket = json.loads(mock_post.call_args[0][1])["issue"] + #print("###", type(resp_ticket), resp_ticket) + + mock_post.assert_called_once() + self.assertEqual(test_proj_id, resp_ticket['project_id']) + + # The integration test suite is only run if the ENV settings are configured correctly @unittest.skipUnless(load_dotenv(), "ENV settings not available") class TestIntegrationTicketManager(test_utils.RedmineTestCase): diff --git a/test_utils.py b/test_utils.py index 38edd40..a62f1a4 100755 --- a/test_utils.py +++ b/test_utils.py @@ -15,8 +15,8 @@ from discord import ApplicationContext import model from users import UserManager -from model import Message, User -from tickets import SCN_PROJECT_ID +from model import Message, User, NamedId +from tickets import SCN_PROJECT_ID, TicketManager import session from redmine import Client @@ -83,12 +83,31 @@ def mock_ticket(**kwargs) -> model.Ticket: return ticket +def mock_user(tag: str) -> model.User: + return model.User( + id=5, + login=f'test-{tag}', + mail=f'{tag}@example.org', + custom_fields=[ {"id": 2, "name":'Discord ID', "value":f'discord-{tag}'} ], + admin=False, + firstname='Test', + lastname=tag, + created_on='2020-07-29T00:37:38Z', + updated_on='2020-02-21T02:14:12Z', + last_login_on='2020-03-31T01:56:05Z', + passwd_changed_on='2020-09-24T18:41:08Z', + twofa_scheme=None, + api_key='', + status='' + ) + + def mock_result(tickets: list[model.Ticket]) -> model.TicketsResult: return model.TicketsResult(len(tickets), 0, 25, tickets) def mock_session() -> session.RedmineSession: - return session.RedmineSession("","") + return session.RedmineSession("http://example.com", "TeStInG-TOK-3N") def custom_fields() -> dict: @@ -105,13 +124,74 @@ def remove_test_users(user_mgr:UserManager): # TODO delete test tickets and "Search for subject match in email threading" ticket. TAG with test too? +class MockUserManager(UserManager): + """mock""" + def get_all(self) -> list[User]: + return [] + + def get_all_teams(self, include_users: bool = True) -> dict[str, model.Team]: + return {} + + +class MockTicketManager(TicketManager): + """mock""" + def __init__(self, sess: session.RedmineSession): + super().__init__(sess, default_project=SCN_PROJECT_ID ) + + + def load_custom_fields(self) -> dict[str,NamedId]: + """ override load_custom_fields to load expected custom fields. """ + result = {} + with open('test/custom-fields.json', "r", encoding="utf-8") as fields_file: + fields = json.load(fields_file) + for name, value in fields.items(): + result[name] = NamedId(**value) + + log.debug(f"loading custom fields for test: {result}") + return result + + +def create_mock_redmine(): + sess = mock_session() + user_mgr = MockUserManager(sess) + ticket_mgr = MockTicketManager(sess) + return Client(sess, user_mgr, ticket_mgr) + + +class MockRedmineTestCase(unittest.TestCase): + """Abstract base class for mocked redmine testing""" + + @classmethod + def setUpClass(cls): + cls.redmine = create_mock_redmine() + cls.user_mgr = cls.redmine.user_mgr + cls.tickets_mgr = cls.redmine.ticket_mgr + cls.tag:str = tagstr() + + cls.user:User = mock_user(cls.tag) + cls.user_mgr.cache.cache_user(cls.user) + log.info(f"SETUP created mock user: {cls.user}") + + + def create_message(self) -> model.Message: + subject = f"TEST {self.tag} {unittest.TestCase.id(self)}" + text = f"This is a ticket for {unittest.TestCase.id(self)} with {self.tag}." + message = Message(self.user.mail, subject, f"to-{self.tag}@example.com", f"cc-{self.tag}@example.com") + message.set_note(text) + return message + + + def create_ticket(self) -> model.Ticket: + return self.redmine.create_ticket(self.user, self.create_message()) + + class RedmineTestCase(unittest.TestCase): """Abstract base class for testing redmine features""" @classmethod def setUpClass(cls): sess = session.RedmineSession.fromenv() - cls.redmine = Client(sess, SCN_PROJECT_ID) + cls.redmine = Client.from_session(sess, SCN_PROJECT_ID) cls.user_mgr = cls.redmine.user_mgr cls.tickets_mgr = cls.redmine.ticket_mgr cls.tag:str = tagstr() @@ -150,7 +230,7 @@ def build_context(self) -> ApplicationContext: def audit_expected_values(): - redmine = Client(session.RedmineSession.fromenv(), SCN_PROJECT_ID) + redmine = Client.from_session(session.RedmineSession.fromenv(), SCN_PROJECT_ID) # audit checks... # 1. make sure admin use exists @@ -177,6 +257,12 @@ def audit_expected_values(): # construct the client and run the email check #client = session.RedmineSession.fromenv() #users = UserManager(client) + + #user = users.get_by_name("philion") + + #with open('test/test-user.json', 'w') as f: + # json.dump(user, f) + #remove_test_users(users) - audit_expected_values() + #audit_expected_values() diff --git a/tickets.py b/tickets.py index d478cfe..802cb20 100644 --- a/tickets.py +++ b/tickets.py @@ -63,6 +63,10 @@ def load_trackers(self) -> dict[str,NamedId]: def get_field_id(self, name:str) -> int | None: + if not self.custom_fields: + log.warning(f"Custom field '{name}' requested, none available") + return None + if name in self.custom_fields: return self.custom_fields[name].id return None