From a67e863917c933c4c4656eb905f5ee41cc8ef2c0 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Tue, 14 Feb 2023 16:09:32 +0500 Subject: [PATCH 1/8] feat!: Update from boto to boto3 storage backend. --- cms/djangoapps/export_course_metadata/storage.py | 4 ++-- cms/djangoapps/export_course_metadata/tasks.py | 2 +- cms/djangoapps/export_course_metadata/test_signals.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cms/djangoapps/export_course_metadata/storage.py b/cms/djangoapps/export_course_metadata/storage.py index d010dd7bd2d9..02a78e4a9c08 100644 --- a/cms/djangoapps/export_course_metadata/storage.py +++ b/cms/djangoapps/export_course_metadata/storage.py @@ -5,10 +5,10 @@ from django.conf import settings from django.core.files.storage import get_storage_class -from storages.backends.s3boto import S3BotoStorage +from storages.backends.s3boto3 import S3Boto3Storage -class CourseMetadataExportS3Storage(S3BotoStorage): # pylint: disable=abstract-method +class CourseMetadataExportS3Storage(S3Boto3Storage): # pylint: disable=abstract-method """ S3 backend for course metadata export """ diff --git a/cms/djangoapps/export_course_metadata/tasks.py b/cms/djangoapps/export_course_metadata/tasks.py index 2ef37dda9e21..48ad8991ce19 100644 --- a/cms/djangoapps/export_course_metadata/tasks.py +++ b/cms/djangoapps/export_course_metadata/tasks.py @@ -27,5 +27,5 @@ def export_course_metadata_task(self, course_key_string): # pylint: disable=unu """ course_key = CourseKey.from_string(course_key_string) highlights = get_all_course_highlights(course_key) - highlights_content = ContentFile(json.dumps({'highlights': highlights})) + highlights_content = ContentFile(json.dumps({'highlights': highlights}).encode('utf-8')) course_metadata_export_storage.save(f'course_metadata_export/{course_key}.json', highlights_content) diff --git a/cms/djangoapps/export_course_metadata/test_signals.py b/cms/djangoapps/export_course_metadata/test_signals.py index 91417901443f..5e71887f2b84 100644 --- a/cms/djangoapps/export_course_metadata/test_signals.py +++ b/cms/djangoapps/export_course_metadata/test_signals.py @@ -48,7 +48,7 @@ def test_happy_path(self, patched_content, patched_storage): SignalHandler.course_published.connect(export_course_metadata) SignalHandler.course_published.send(sender=None, course_key=self.course_key) patched_content.assert_called_once_with( - '{"highlights": [["week1highlight1", "week1highlight2"], ["week1highlight1", "week1highlight2"], [], []]}' + b'{"highlights": [["week1highlight1", "week1highlight2"], ["week1highlight1", "week1highlight2"], [], []]}' ) patched_storage.save.assert_called_once_with( f'course_metadata_export/{self.course_key}.json', patched_content.return_value From a39caad04df6f00cbb1bbc4b33d052bc7c46ea80 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Tue, 14 Feb 2023 16:16:58 +0500 Subject: [PATCH 2/8] feat!: Update from boto to boto3 storage backend. --- cms/djangoapps/contentstore/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/storage.py b/cms/djangoapps/contentstore/storage.py index f3a04fbd2146..9719d052b89d 100644 --- a/cms/djangoapps/contentstore/storage.py +++ b/cms/djangoapps/contentstore/storage.py @@ -5,11 +5,11 @@ from django.conf import settings from django.core.files.storage import get_storage_class -from storages.backends.s3boto import S3BotoStorage +from storages.backends.s3boto3 import S3Boto3Storage from storages.utils import setting -class ImportExportS3Storage(S3BotoStorage): # pylint: disable=abstract-method +class ImportExportS3Storage(S3Boto3Storage): # pylint: disable=abstract-method """ S3 backend for course import and export OLX files. """ From 25471a88e17782dc1b0abaf3ba62a4d9522f4542 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Tue, 14 Feb 2023 17:48:19 +0500 Subject: [PATCH 3/8] feat!: Update from boto to boto3 storage backend. --- cms/djangoapps/cms_user_tasks/tasks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cms/djangoapps/cms_user_tasks/tasks.py b/cms/djangoapps/cms_user_tasks/tasks.py index 521d65746f6e..6f3eb3a6e3e6 100644 --- a/cms/djangoapps/cms_user_tasks/tasks.py +++ b/cms/djangoapps/cms_user_tasks/tasks.py @@ -3,7 +3,6 @@ """ import json -from boto.exception import NoAuthHandlerFound from celery import shared_task from celery.exceptions import MaxRetriesExceededError from celery.utils.log import get_task_logger @@ -52,7 +51,7 @@ def send_task_complete_email(self, task_name, task_state_text, dest_addr, detail try: mail.send_mail(subject, message, from_address, [dest_addr], fail_silently=False) LOGGER.info("Task complete email has been sent to User %s", dest_addr) - except NoAuthHandlerFound: + except exception: LOGGER.info( 'Retrying sending email to user %s, attempt # %s of %s', dest_addr, From c5a42f890fb6343d957ae141091d8a4c2ef6d2a8 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Tue, 14 Feb 2023 17:50:05 +0500 Subject: [PATCH 4/8] feat!: Update from boto to boto3 storage backend. --- cms/djangoapps/cms_user_tasks/tests.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cms/djangoapps/cms_user_tasks/tests.py b/cms/djangoapps/cms_user_tasks/tests.py index 77afca6a2c9f..d6b5b05de4b4 100644 --- a/cms/djangoapps/cms_user_tasks/tests.py +++ b/cms/djangoapps/cms_user_tasks/tests.py @@ -8,7 +8,6 @@ from uuid import uuid4 import ddt -from boto.exception import NoAuthHandlerFound from django.conf import settings from django.core import mail from django.test import override_settings @@ -303,7 +302,7 @@ def test_email_retries(self): Make sure we can succeed on retries """ with mock.patch('django.core.mail.send_mail') as mock_exception: - mock_exception.side_effect = NoAuthHandlerFound() + mock_exception.side_effect = exception() with mock.patch('cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.retry') as mock_retry: user_task_stopped.send(sender=UserTaskStatus, status=self.status) @@ -315,7 +314,7 @@ def test_queue_email_failure(self): logger.addHandler(hdlr) with mock.patch('cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.delay') as mock_delay: - mock_delay.side_effect = NoAuthHandlerFound() + mock_delay.side_effect = exception() user_task_stopped.send(sender=UserTaskStatus, status=self.status) self.assertTrue(mock_delay.called) self.assertEqual(hdlr.messages['error'][0], 'Unable to queue send_task_complete_email') From e3e9dee012558d0179381ff14ea8723ca38bd03a Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Tue, 14 Feb 2023 18:29:22 +0500 Subject: [PATCH 5/8] feat!: Update from boto to boto3 storage backend. --- cms/djangoapps/cms_user_tasks/tasks.py | 2 +- cms/djangoapps/cms_user_tasks/tests.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cms/djangoapps/cms_user_tasks/tasks.py b/cms/djangoapps/cms_user_tasks/tasks.py index 6f3eb3a6e3e6..f964df503311 100644 --- a/cms/djangoapps/cms_user_tasks/tasks.py +++ b/cms/djangoapps/cms_user_tasks/tasks.py @@ -51,7 +51,7 @@ def send_task_complete_email(self, task_name, task_state_text, dest_addr, detail try: mail.send_mail(subject, message, from_address, [dest_addr], fail_silently=False) LOGGER.info("Task complete email has been sent to User %s", dest_addr) - except exception: + except Exception: LOGGER.info( 'Retrying sending email to user %s, attempt # %s of %s', dest_addr, diff --git a/cms/djangoapps/cms_user_tasks/tests.py b/cms/djangoapps/cms_user_tasks/tests.py index d6b5b05de4b4..dbe20ad90b9e 100644 --- a/cms/djangoapps/cms_user_tasks/tests.py +++ b/cms/djangoapps/cms_user_tasks/tests.py @@ -301,8 +301,9 @@ def test_email_retries(self): """ Make sure we can succeed on retries """ + with mock.patch('django.core.mail.send_mail') as mock_exception: - mock_exception.side_effect = exception() + mock_exception.side_effect=Exception() with mock.patch('cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.retry') as mock_retry: user_task_stopped.send(sender=UserTaskStatus, status=self.status) @@ -314,7 +315,7 @@ def test_queue_email_failure(self): logger.addHandler(hdlr) with mock.patch('cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.delay') as mock_delay: - mock_delay.side_effect = exception() + mock_delay.side_effect = Exception() user_task_stopped.send(sender=UserTaskStatus, status=self.status) self.assertTrue(mock_delay.called) self.assertEqual(hdlr.messages['error'][0], 'Unable to queue send_task_complete_email') From 9d6621bcfc2947ec9c1a600599596e3fec1ea74b Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Tue, 14 Feb 2023 20:14:50 +0500 Subject: [PATCH 6/8] feat!: Update from boto to boto3 storage backend. --- cms/djangoapps/cms_user_tasks/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/djangoapps/cms_user_tasks/tests.py b/cms/djangoapps/cms_user_tasks/tests.py index dbe20ad90b9e..f11e1371bea2 100644 --- a/cms/djangoapps/cms_user_tasks/tests.py +++ b/cms/djangoapps/cms_user_tasks/tests.py @@ -303,7 +303,7 @@ def test_email_retries(self): """ with mock.patch('django.core.mail.send_mail') as mock_exception: - mock_exception.side_effect=Exception() + mock_exception.side_effect = Exception() with mock.patch('cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.retry') as mock_retry: user_task_stopped.send(sender=UserTaskStatus, status=self.status) From 86d749f68eb1850e28ef888e2bd0ac0932d7bcb5 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Wed, 15 Feb 2023 00:02:07 +0500 Subject: [PATCH 7/8] feat!: Update from boto to boto3 storage backend. --- cms/djangoapps/cms_user_tasks/tasks.py | 3 ++- cms/djangoapps/cms_user_tasks/tests.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cms/djangoapps/cms_user_tasks/tasks.py b/cms/djangoapps/cms_user_tasks/tasks.py index f964df503311..e06c58b6f081 100644 --- a/cms/djangoapps/cms_user_tasks/tasks.py +++ b/cms/djangoapps/cms_user_tasks/tasks.py @@ -3,6 +3,7 @@ """ import json +import botocore from celery import shared_task from celery.exceptions import MaxRetriesExceededError from celery.utils.log import get_task_logger @@ -51,7 +52,7 @@ def send_task_complete_email(self, task_name, task_state_text, dest_addr, detail try: mail.send_mail(subject, message, from_address, [dest_addr], fail_silently=False) LOGGER.info("Task complete email has been sent to User %s", dest_addr) - except Exception: + except botocore.exceptions.ClientError: LOGGER.info( 'Retrying sending email to user %s, attempt # %s of %s', dest_addr, diff --git a/cms/djangoapps/cms_user_tasks/tests.py b/cms/djangoapps/cms_user_tasks/tests.py index f11e1371bea2..fe1e2c296853 100644 --- a/cms/djangoapps/cms_user_tasks/tests.py +++ b/cms/djangoapps/cms_user_tasks/tests.py @@ -7,6 +7,7 @@ from unittest.mock import patch from uuid import uuid4 +import botocore import ddt from django.conf import settings from django.core import mail @@ -301,9 +302,8 @@ def test_email_retries(self): """ Make sure we can succeed on retries """ - with mock.patch('django.core.mail.send_mail') as mock_exception: - mock_exception.side_effect = Exception() + mock_exception.side_effect = botocore.exceptions.ClientError({'error_response':'a'} , {'operation_name':1} ) with mock.patch('cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.retry') as mock_retry: user_task_stopped.send(sender=UserTaskStatus, status=self.status) @@ -315,7 +315,7 @@ def test_queue_email_failure(self): logger.addHandler(hdlr) with mock.patch('cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.delay') as mock_delay: - mock_delay.side_effect = Exception() + mock_delay.side_effect = botocore.exceptions.ClientError({'error_response':'a'} , {'operation_name':1} ) user_task_stopped.send(sender=UserTaskStatus, status=self.status) self.assertTrue(mock_delay.called) self.assertEqual(hdlr.messages['error'][0], 'Unable to queue send_task_complete_email') From 8b32051f10dc31b47026109efdafa4ed69695506 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Wed, 15 Feb 2023 00:05:31 +0500 Subject: [PATCH 8/8] feat!: Update from boto to boto3 storage backend. --- cms/djangoapps/cms_user_tasks/tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/cms_user_tasks/tests.py b/cms/djangoapps/cms_user_tasks/tests.py index fe1e2c296853..f50e5002feff 100644 --- a/cms/djangoapps/cms_user_tasks/tests.py +++ b/cms/djangoapps/cms_user_tasks/tests.py @@ -303,7 +303,9 @@ def test_email_retries(self): Make sure we can succeed on retries """ with mock.patch('django.core.mail.send_mail') as mock_exception: - mock_exception.side_effect = botocore.exceptions.ClientError({'error_response':'a'} , {'operation_name':1} ) + mock_exception.side_effect = botocore.exceptions.ClientError( + {'error_response': 'error occurred'}, {'operation_name': 'test'} + ) with mock.patch('cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.retry') as mock_retry: user_task_stopped.send(sender=UserTaskStatus, status=self.status) @@ -315,7 +317,9 @@ def test_queue_email_failure(self): logger.addHandler(hdlr) with mock.patch('cms.djangoapps.cms_user_tasks.tasks.send_task_complete_email.delay') as mock_delay: - mock_delay.side_effect = botocore.exceptions.ClientError({'error_response':'a'} , {'operation_name':1} ) + mock_delay.side_effect = botocore.exceptions.ClientError( + {'error_response': 'error occurred'}, {'operation_name': 'test'} + ) user_task_stopped.send(sender=UserTaskStatus, status=self.status) self.assertTrue(mock_delay.called) self.assertEqual(hdlr.messages['error'][0], 'Unable to queue send_task_complete_email')