diff --git a/README.rst b/README.rst index e56b75f..98f2f6c 100644 --- a/README.rst +++ b/README.rst @@ -238,7 +238,7 @@ Action's uid_list arg may takes: To get uids, use the maibox methods: uids, fetch. For actions with a large number of messages imap command may be too large and will cause exception at server side, -use 'limit' argument for fetch in this case. +use ``chunks`` argument for ``copy,move,delete,flag`` OR ``limit`` argument for ``fetch`` in this case. .. code-block:: python @@ -247,8 +247,8 @@ use 'limit' argument for fetch in this case. # COPY messages with uid in 23,27 from current folder to folder1 mailbox.copy('23,27', 'folder1') - # MOVE all messages from current folder to INBOX/folder2 - mailbox.move(mailbox.uids(), 'INBOX/folder2') + # MOVE all messages from current folder to INBOX/folder2, move by 100 emails at once + mailbox.move(mailbox.uids(), 'INBOX/folder2', chunks=100) # DELETE messages with 'cat' word in its html from current folder mailbox.delete([msg.uid for msg in mailbox.fetch() if 'cat' in msg.html]) @@ -429,7 +429,8 @@ Big thanks to people who helped develop this library: `homoLudenus `_, `sphh `_, `bh `_, -`tomasmach `_ +`tomasmach `_, +`errror `_ Help the project ---------------- diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 407f38e..22617b3 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -1,3 +1,13 @@ +1.10.0 +====== +* Added: support IMAP command MOVE at BaseMailBox.move +* Added: MailboxMoveError, raises from BaseMailBox.move when MOVE command is supported +* Added: chunks argument for BaseMailBox.(copy,move,flag,delete) methods - Number of UIDs to proc at once, to avoid server errors on large set +* Changed: BaseMailBox.(copy,move,flag,delete) result types +* Changed: utils.clean_uids now returns List[str] +* Changed: utils.chunks_crop -> utils.chunked_crop, n arg renamed to chunk_size and it takes False-like vals +* Renamed: utils.chunks -> utils.chunked + 1.9.1 ===== * Replaced: functools.lru_cache to functools.cached_property diff --git a/imap_tools/__init__.py b/imap_tools/__init__.py index d9562fb..60d10cd 100644 --- a/imap_tools/__init__.py +++ b/imap_tools/__init__.py @@ -11,4 +11,4 @@ from .utils import EmailAddress from .errors import * -__version__ = '1.9.1' +__version__ = '1.10.0' diff --git a/imap_tools/consts.py b/imap_tools/consts.py index 703f409..c05742a 100644 --- a/imap_tools/consts.py +++ b/imap_tools/consts.py @@ -9,6 +9,7 @@ PYTHON_VERSION_MINOR = int(sys.version_info.minor) +MOVE_RESULT_TAG = ('_MOVE',) # const delete_result part for mailbox.move result, when server have MOVE in capabilities class MailMessageFlags: """ diff --git a/imap_tools/errors.py b/imap_tools/errors.py index ce52a73..4fe2bed 100644 --- a/imap_tools/errors.py +++ b/imap_tools/errors.py @@ -85,6 +85,10 @@ class MailboxCopyError(UnexpectedCommandStatusError): pass +class MailboxMoveError(UnexpectedCommandStatusError): + pass + + class MailboxFlagError(UnexpectedCommandStatusError): pass diff --git a/imap_tools/mailbox.py b/imap_tools/mailbox.py index 14665a0..b08c947 100644 --- a/imap_tools/mailbox.py +++ b/imap_tools/mailbox.py @@ -7,12 +7,12 @@ from .message import MailMessage from .folder import MailBoxFolderManager from .idle import IdleManager -from .consts import UID_PATTERN, PYTHON_VERSION_MINOR -from .utils import clean_uids, check_command_status, chunks, encode_folder, clean_flags, check_timeout_arg_support, \ - chunks_crop, StrOrBytes +from .consts import UID_PATTERN, PYTHON_VERSION_MINOR, MOVE_RESULT_TAG +from .utils import clean_uids, check_command_status, chunked, encode_folder, clean_flags, check_timeout_arg_support, \ + chunked_crop, StrOrBytes from .errors import MailboxStarttlsError, MailboxLoginError, MailboxLogoutError, MailboxNumbersError, \ MailboxFetchError, MailboxExpungeError, MailboxDeleteError, MailboxCopyError, MailboxFlagError, \ - MailboxAppendError, MailboxUidsError, MailboxTaggedResponseError + MailboxAppendError, MailboxUidsError, MailboxTaggedResponseError, MailboxMoveError # Maximal line length when calling readline(). This is to prevent reading arbitrary length lines. # 20Mb is enough for search response with about 2 000 000 message numbers @@ -153,7 +153,7 @@ def _fetch_in_bulk(self, uid_list: Sequence[str], message_parts: str, reverse: b return if isinstance(bulk, int) and bulk >= 2: - uid_list_seq = chunks_crop(uid_list, bulk) + uid_list_seq = chunked_crop(uid_list, bulk) elif isinstance(bulk, bool): uid_list_seq = (uid_list,) else: @@ -164,7 +164,7 @@ def _fetch_in_bulk(self, uid_list: Sequence[str], message_parts: str, reverse: b check_command_status(fetch_result, MailboxFetchError) if not fetch_result[1] or fetch_result[1][0] is None: return - for built_fetch_item in chunks((reversed if reverse else iter)(fetch_result[1]), 2): + for built_fetch_item in chunked((reversed if reverse else iter)(fetch_result[1]), 2): yield built_fetch_item def fetch(self, criteria: Criteria = 'ALL', charset: str = 'US-ASCII', limit: Optional[Union[int, slice]] = None, @@ -204,62 +204,105 @@ def expunge(self) -> tuple: check_command_status(result, MailboxExpungeError) return result - def delete(self, uid_list: Union[str, Iterable[str]]) -> Optional[Tuple[tuple, tuple]]: + def delete(self, uid_list: Union[str, Iterable[str]], chunks: Optional[int] = None) \ + -> Optional[List[Tuple[tuple, tuple]]]: """ Delete email messages Do nothing on empty uid_list + :param uid_list: UIDs for delete + :param chunks: Number of UIDs to process at once, to avoid server errors on large set. Proc all at once if None. :return: None on empty uid_list, command results otherwise """ - uid_str = clean_uids(uid_list) - if not uid_str: + cleaned_uid_list = clean_uids(uid_list) + if not cleaned_uid_list: return None - store_result = self.client.uid('STORE', uid_str, '+FLAGS', r'(\Deleted)') - check_command_status(store_result, MailboxDeleteError) - expunge_result = self.expunge() - return store_result, expunge_result - - def copy(self, uid_list: Union[str, Iterable[str]], destination_folder: StrOrBytes) -> Optional[tuple]: + results = [] + for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks): + store_result = self.client.uid('STORE', ','.join(cleaned_uid_list_i), '+FLAGS', r'(\Deleted)') + check_command_status(store_result, MailboxDeleteError) + expunge_result = self.expunge() + results.append((store_result, expunge_result)) + return results + + def copy(self, uid_list: Union[str, Iterable[str]], destination_folder: StrOrBytes, chunks: Optional[int] = None) \ + -> Optional[List[tuple]]: """ - Copy email messages into the specified folder - Do nothing on empty uid_list + Copy email messages into the specified folder. + Do nothing on empty uid_list. + :param uid_list: UIDs for copy + :param destination_folder: Folder for email copies + :param chunks: Number of UIDs to process at once, to avoid server errors on large set. Proc all at once if None. :return: None on empty uid_list, command results otherwise """ - uid_str = clean_uids(uid_list) - if not uid_str: + cleaned_uid_list = clean_uids(uid_list) + if not cleaned_uid_list: return None - copy_result = self.client.uid('COPY', uid_str, encode_folder(destination_folder)) # noqa - check_command_status(copy_result, MailboxCopyError) - return copy_result - - def move(self, uid_list: Union[str, Iterable[str]], destination_folder: StrOrBytes) -> Optional[Tuple[tuple, tuple]]: + results = [] + for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks): + copy_result = self.client.uid( + 'COPY', ','.join(cleaned_uid_list_i), encode_folder(destination_folder)) # noqa + check_command_status(copy_result, MailboxCopyError) + results.append(copy_result) + return results + + def move(self, uid_list: Union[str, Iterable[str]], destination_folder: StrOrBytes, chunks: Optional[int] = None) \ + -> Optional[List[Tuple[tuple, tuple]]]: """ - Move email messages into the specified folder - Do nothing on empty uid_list + Move email messages into the specified folder. + Do nothing on empty uid_list. + :param uid_list: UIDs for move + :param destination_folder: Folder for move to + :param chunks: Number of UIDs to process at once, to avoid server errors on large set. Proc all at once if None. :return: None on empty uid_list, command results otherwise """ - uid_str = clean_uids(uid_list) - if not uid_str: + cleaned_uid_list = clean_uids(uid_list) + if not cleaned_uid_list: return None - copy_result = self.copy(uid_str, destination_folder) - delete_result = self.delete(uid_str) - return copy_result, delete_result - - def flag(self, uid_list: Union[str, Iterable[str]], flag_set: Union[str, Iterable[str]], value: bool) \ - -> Optional[Tuple[tuple, tuple]]: + if 'MOVE' in self.client.capabilities: + # server side move + results = [] + for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks): + move_result = self.client.uid( + 'MOVE', ','.join(cleaned_uid_list_i), encode_folder(destination_folder)) # noqa + check_command_status(move_result, MailboxMoveError) + results.append((move_result, MOVE_RESULT_TAG)) + return results + else: + # client side move + results = [] + for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks): + copy_result = self.copy(cleaned_uid_list_i, destination_folder) + delete_result = self.delete(cleaned_uid_list_i) + results.append((copy_result, delete_result)) + return results + + def flag(self, uid_list: Union[str, Iterable[str]], flag_set: Union[str, Iterable[str]], value: bool, + chunks: Optional[int] = None) -> Optional[List[Tuple[tuple, tuple]]]: """ - Set/unset email flags - Do nothing on empty uid_list + Set/unset email flags. + Do nothing on empty uid_list. System flags contains in consts.MailMessageFlags.all + :param uid_list: UIDs for set flag + :param flag_set: Flags for operate + :param value: Should the flags be set: True - yes, False - no + :param chunks: Number of UIDs to process at once, to avoid server errors on large set. Proc all at once if None. :return: None on empty uid_list, command results otherwise """ - uid_str = clean_uids(uid_list) - if not uid_str: + cleaned_uid_list = clean_uids(uid_list) + if not cleaned_uid_list: return None - store_result = self.client.uid( - 'STORE', uid_str, ('+' if value else '-') + 'FLAGS', f'({" ".join(clean_flags(flag_set))})') - check_command_status(store_result, MailboxFlagError) - expunge_result = self.expunge() - return store_result, expunge_result + results = [] + for cleaned_uid_list_i in chunked_crop(cleaned_uid_list, chunks): + store_result = self.client.uid( + 'STORE', + ','.join(cleaned_uid_list_i), + ('+' if value else '-') + 'FLAGS', + f'({" ".join(clean_flags(flag_set))})' + ) + check_command_status(store_result, MailboxFlagError) + expunge_result = self.expunge() + results.append((store_result, expunge_result)) + return results def append(self, message: Union[MailMessage, bytes], folder: StrOrBytes = 'INBOX', diff --git a/imap_tools/query.py b/imap_tools/query.py index f8b38c0..88bc741 100644 --- a/imap_tools/query.py +++ b/imap_tools/query.py @@ -209,7 +209,7 @@ def cleaned_uid(key: str, value: Union[str, Iterable[str], UidRange]) -> str: return str(value) # set try: - return clean_uids(value) + return ','.join(clean_uids(value)) except TypeError as e: raise TypeError(f'{key} parse error: {str(e)}') diff --git a/imap_tools/utils.py b/imap_tools/utils.py index d215e23..e1713e3 100644 --- a/imap_tools/utils.py +++ b/imap_tools/utils.py @@ -4,7 +4,7 @@ from itertools import zip_longest from email.utils import getaddresses, parsedate_to_datetime from email.header import decode_header, Header -from typing import Union, Optional, Tuple, Iterable, Any, List, Dict, Iterator +from typing import Union, Optional, Tuple, Iterable, Any, List, Dict, Iterator, Sequence from .consts import SHORT_MONTH_NAMES, MailMessageFlags from .imap_utf7 import utf7_encode @@ -12,19 +12,19 @@ StrOrBytes = Union[str, bytes] -def clean_uids(uid_set: Union[str, Iterable[str]]) -> str: +def clean_uids(uid_set: Union[str, Iterable[str]]) -> List[str]: """ Prepare set of uid for use in IMAP commands uid RE patterns are not strict and allow invalid combinations, but simple. Example: 2,4:7,9,12:* :param uid_set: str, that is comma separated uids Iterable, that contains str uids - :return: str - uids, concatenated by a comma + :return: list of str - cleaned uids """ # str if type(uid_set) is str: if re.search(r'^([\d*:]+,)*[\d*:]+$', uid_set): # *optimization for already good str - return uid_set + return uid_set.split(',') uid_set = uid_set.split(',') # check uid types for uid in uid_set: @@ -32,7 +32,7 @@ def clean_uids(uid_set: Union[str, Iterable[str]]) -> str: raise TypeError(f'uid "{str(uid)}" is not string') if not re.match(r'^[\d*:]+$', uid.strip()): raise TypeError(f'Wrong uid: "{uid}"') - return ','.join(i.strip() for i in uid_set) + return [i.strip() for i in uid_set] def check_command_status(command_result: tuple, exception: type, expected='OK'): @@ -205,23 +205,27 @@ def replace_html_ct_charset(html: str, new_charset: str) -> str: return html -def chunks(iterable: Iterable[Any], n: int, fill_value: Optional[Any] = None) -> Iterator[Tuple[Any, ...]]: +def chunked(iterable: Iterable[Any], n: int, fill_value: Optional[Any] = None) -> Iterator[Tuple[Any, ...]]: """ Group data into fixed-length chunks or blocks [iter(iterable)]*n creates one iterator, repeated n times in the list izip_longest then effectively performs a round-robin of "each" (same) iterator Examples: - chunks('ABCDEFGH', 3, '?') --> [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'H', '?')] - chunks([1, 2, 3, 4, 5], 2) --> [(1, 2), (3, 4), (5, None)] + chunked('ABCDEFGH', 3, '?') --> [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'H', '?')] + chunked([1, 2, 3, 4, 5], 2) --> [(1, 2), (3, 4), (5, None)] """ return zip_longest(*[iter(iterable)] * n, fillvalue=fill_value) -def chunks_crop(lst: iter, n: int) -> iter: +def chunked_crop(seq: Sequence, chunk_size: Optional[int]) -> Iterator[list]: """ - Yield successive n-sized chunks from lst. + Yield successive n-sized chunks from seq. + Yield seq if chunk_size is False-like + :param seq: Sequence to chunks + :param chunk_size: chunk size + :return: Iterator import pprint - pprint.pprint(list(chunks(range(10, 75), 10))) + pprint.pprint(list(chunked_crop(range(10, 75), 10))) [[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], [30, 31, 32, 33, 34, 35, 36, 37, 38, 39], @@ -230,5 +234,10 @@ def chunks_crop(lst: iter, n: int) -> iter: [60, 61, 62, 63, 64, 65, 66, 67, 68, 69], [70, 71, 72, 73, 74]] """ - for i in range(0, len(lst), n): - yield lst[i:i + n] + if not chunk_size: + yield seq + return + if chunk_size < 0: + raise ValueError('False-like or int>=0 expected') + for i in range(0, len(seq), chunk_size): + yield seq[i:i + chunk_size] diff --git a/tests/test_mailbox.py b/tests/test_mailbox.py index f1cac5f..fd73a0f 100644 --- a/tests/test_mailbox.py +++ b/tests/test_mailbox.py @@ -10,6 +10,7 @@ class MailboxTest(MailboxTestCase): base_test_msg_cnt = 6 + test_chunks_size = 4 @classmethod def setUpClass(cls): @@ -68,6 +69,16 @@ def test_action(self): self.base_test_msg_cnt ) + # APPEND + if mailbox.mailbox_name not in ('MAIL_RU', 'YANDEX'): + mailbox.folder.set('INBOX') + q = A(subject='_append_') + mailbox.delete(mailbox.uids(q)) + self.assertEqual(len(list(mailbox.numbers(q))), 0) + mailbox.append(TEST_MESSAGE_DATA) + self.assertEqual(len(list(mailbox.numbers(q))), 1) # YANDEX 0!=1 in test only, strange + mailbox.delete(mailbox.uids(q)) + # COPY mailbox.folder.set(mailbox.folder_test_base) mailbox.copy(mailbox.uids(), mailbox.folder_test_temp1) @@ -99,15 +110,27 @@ def test_action(self): mailbox.delete(mailbox.uids()) self.assertEqual(len(list(mailbox.numbers())), 0) - # APPEND - if mailbox.mailbox_name not in ('MAIL_RU', 'YANDEX'): - mailbox.folder.set('INBOX') - q = A(subject='_append_') - mailbox.delete(mailbox.uids(q)) - self.assertEqual(len(list(mailbox.numbers(q))), 0) - mailbox.append(TEST_MESSAGE_DATA) - self.assertEqual(len(list(mailbox.numbers(q))), 1) # YANDEX 0!=1 in test only, strange - mailbox.delete(mailbox.uids(q)) + # COPY chunks + mailbox.folder.set(mailbox.folder_test_base) + mailbox.copy(mailbox.uids(), mailbox.folder_test_temp1, self.test_chunks_size) + self.assertEqual(len(list(mailbox.numbers())), self.base_test_msg_cnt) + mailbox.folder.set(mailbox.folder_test_temp1) + self.assertEqual(len(list(mailbox.numbers())), self.base_test_msg_cnt) + # MOVE chunks + mailbox.folder.set(mailbox.folder_test_temp1) + mailbox.move(mailbox.uids(), mailbox.folder_test_temp2, self.test_chunks_size) + self.assertEqual(len(list(mailbox.numbers())), 0) + mailbox.folder.set(mailbox.folder_test_temp2) + self.assertEqual(len(list(mailbox.numbers())), self.base_test_msg_cnt) + # FLAG chunks + mailbox.folder.set(mailbox.folder_test_temp2) + mailbox.flag(mailbox.uids(), MailMessageFlags.FLAGGED, True, self.test_chunks_size) + self.assertTrue( + all([MailMessageFlags.FLAGGED in msg.flags for msg in mailbox.fetch(bulk=True, headers_only=1)])) + # DELETE chunks + mailbox.folder.set(mailbox.folder_test_temp2) + mailbox.delete(mailbox.uids(), self.test_chunks_size) + self.assertEqual(len(list(mailbox.numbers())), 0) if __name__ == "__main__": diff --git a/tests/test_utils.py b/tests/test_utils.py index 870fea6..ba30ec6 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,8 +4,8 @@ from imap_tools.errors import ImapToolsError, UnexpectedCommandStatusError, MailboxCopyError from imap_tools.consts import MailMessageFlags -from imap_tools.utils import clean_flags, chunks, quote, pairs_to_dict, decode_value, check_command_status, \ - parse_email_date, parse_email_addresses, EmailAddress, clean_uids, replace_html_ct_charset, chunks_crop, \ +from imap_tools.utils import clean_flags, chunked, quote, pairs_to_dict, decode_value, check_command_status, \ + parse_email_date, parse_email_addresses, EmailAddress, clean_uids, replace_html_ct_charset, chunked_crop, \ remove_non_printable @@ -21,13 +21,14 @@ def test_remove_non_printable(self): def test_clean_uids(self): # *clean_uids also implicitly tested in test_query.py - self.assertEqual(clean_uids('11'), '11') - self.assertEqual(clean_uids('1,2'), '1,2') - self.assertEqual(clean_uids('1,2:*'), '1,2:*') - with self.assertRaises(TypeError): - clean_uids(1) # noqa - with self.assertRaises(TypeError): - clean_uids(dict(a=1)) # noqa + self.assertEqual(clean_uids('11'), ['11']) + self.assertEqual(clean_uids('1,2'), ['1', '2']) + self.assertEqual(clean_uids(' 1,2, 4 '), ['1', '2', '4']) + self.assertEqual(clean_uids('1,222'), ['1', '222']) + self.assertEqual(clean_uids('1,2:*'), ['1', '2:*']) + for i in ['', 1, 1.0, dict(a=1)]: + with self.assertRaises(TypeError): + clean_uids(i) # noqa def test_clean_flags(self): self.assertEqual(clean_flags([MailMessageFlags.FLAGGED, MailMessageFlags.SEEN]), ['\\Flagged', '\\Seen']) @@ -39,17 +40,22 @@ def test_clean_flags(self): with self.assertRaises(ValueError): clean_flags([MailMessageFlags.FLAGGED, '\\CUSTOM_TAG_WITH_SLASH']) - def test_chunks(self): - self.assertEqual(list(chunks('ABCDE', 2, '=')), [('A', 'B'), ('C', 'D'), ('E', '=')]) - self.assertEqual(list(chunks([1, 2, 3, 4, 5, 6], 3)), [(1, 2, 3), (4, 5, 6)]) - self.assertEqual(list(chunks([], 4)), []) - self.assertEqual(list(chunks([1, 2], 0)), []) - self.assertEqual(list(chunks(['0', '0'], 1)), [('0',), ('0',)]) - - def test_chunks_crop(self): - self.assertEqual(list(chunks_crop([1, 2, 3, 4, 5, 6, 7], 3)), [[1, 2, 3], [4, 5, 6], [7]]) - self.assertEqual(list(chunks_crop([1, 2, 3, 4, 5, 6], 3)), [[1, 2, 3], [4, 5, 6]]) - self.assertEqual(list(chunks_crop([1, ], 3)), [[1]]) + def test_chunked(self): + self.assertEqual(list(chunked('ABCDE', 2, '=')), [('A', 'B'), ('C', 'D'), ('E', '=')]) + self.assertEqual(list(chunked([1, 2, 3, 4, 5, 6], 3)), [(1, 2, 3), (4, 5, 6)]) + self.assertEqual(list(chunked([], 4)), []) + self.assertEqual(list(chunked([1, 2], 0)), []) + self.assertEqual(list(chunked(['0', '0'], 1)), [('0',), ('0',)]) + + def test_chunked_crop(self): + self.assertEqual(list(chunked_crop([1, 2, 3, 4, 5, 6, 7], 3)), [[1, 2, 3], [4, 5, 6], [7]]) + self.assertEqual(list(chunked_crop([1, 2, 3, 4, 5, 6], 3)), [[1, 2, 3], [4, 5, 6]]) + self.assertEqual(list(chunked_crop([1, ], 3)), [[1]]) + self.assertEqual(list(chunked_crop([1], 0)), [[1]]) + self.assertEqual(list(chunked_crop([1, 2], False)), [[1, 2]]) + self.assertEqual(list(chunked_crop([1, 2, 3], None)), [[1, 2, 3]]) + with self.assertRaises(ValueError): + list(chunked_crop([1], -1)) def test_quote(self): self.assertEqual(quote('str привет'), '"str привет"')