From 52c07629572d537d077d68e295b3d47343f9bb41 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Wed, 5 Apr 2023 17:56:13 +0200 Subject: [PATCH 01/47] start modernization of massmail make massmail installable using more modern techniques: - pyproject.toml (supports editable installation since at least setuptools 66.1.1, but maybe even earlier) - automatically generate massmail script on installation --- .gitignore | 1 + LICENSE | 24 +++++++++++++++++++ massmail.py | 1 - massmail/__init__.py | 0 massmail => massmail/massmail.py | 4 +++- test_massmail.py => massmail/test_massmail.py | 0 test_sending.py => massmail/test_sending.py | 0 pyproject.toml | 18 ++++++++++++++ 8 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 LICENSE delete mode 120000 massmail.py create mode 100644 massmail/__init__.py rename massmail => massmail/massmail.py (99%) rename test_massmail.py => massmail/test_massmail.py (100%) rename test_sending.py => massmail/test_sending.py (100%) create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..11041c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.egg-info diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7230ab9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +Copyright (C) 2003-2017 Tiziano Zito , Jakob Jordan +Copyright (C) 2017-2023 ASPP + +# This is free software and comes without any warranty, to +# the extent permitted by applicable law. You can redistribute it +# and/or modify it under the terms of the Do What The Fuck You Want +# To Public License, Version 2, as published by Sam Hocevar. +# http://www.wtfpl.net + +Full license text: + +DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. diff --git a/massmail.py b/massmail.py deleted file mode 120000 index d662c24..0000000 --- a/massmail.py +++ /dev/null @@ -1 +0,0 @@ -massmail \ No newline at end of file diff --git a/massmail/__init__.py b/massmail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/massmail b/massmail/massmail.py similarity index 99% rename from massmail rename to massmail/massmail.py index 662997a..f2d5f57 100755 --- a/massmail +++ b/massmail/massmail.py @@ -316,7 +316,9 @@ def send_messages(options, msgs): server.close() -def main(arguments): +def main(arguments=None): + if arguments is None: + arguments = sys.argv[1:] options = parse_command_line_options(arguments) keywords, email_count = parse_parameter_file(options) msgs = create_email_bodies(options, keywords, email_count, sys.stdin.read()) diff --git a/test_massmail.py b/massmail/test_massmail.py similarity index 100% rename from test_massmail.py rename to massmail/test_massmail.py diff --git a/test_sending.py b/massmail/test_sending.py similarity index 100% rename from test_sending.py rename to massmail/test_sending.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..29d5f79 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "massmail" +version = "0.2" +description = "Send mass mail" +authors = [ { name="ASPP", email ="info@aspp.school" } ] +license = { file = "LICENSE" } +requires-python = ">=3.6" +dependencies = [ "aiosmtpd" ] + +[project.scripts] +massmail = "massmail.massmail:main" + +[tool.setuptools] +packages = ["massmail"] + +[build-system] +requires = ["setuptools>=42"] +build-backend = "setuptools.build_meta" From d927cac33b9270fb5c3681436a7f3bce876d56e8 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Wed, 5 Apr 2023 18:20:42 +0200 Subject: [PATCH 02/47] fix test_massmail --- massmail/test_massmail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 5ba1203..f845d69 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -1,6 +1,6 @@ import tempfile -import massmail +import massmail.massmail as massmail def test_dummy(): pass From 492aebce7c5b3457e62750714a5162292e7cddc3 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Wed, 5 Apr 2023 18:21:08 +0200 Subject: [PATCH 03/47] fix test_local_sending --- massmail/test_sending.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/massmail/test_sending.py b/massmail/test_sending.py index 5f9a859..97ad30b 100644 --- a/massmail/test_sending.py +++ b/massmail/test_sending.py @@ -8,7 +8,7 @@ import base64 import tempfile -import massmail +import massmail.massmail as massmail @contextlib.contextmanager def replace_stdin(text): @@ -23,8 +23,8 @@ def replace_stdin(text): @contextlib.contextmanager def fake_smtp_server(address): devnull = open(os.devnull, 'w') - server = subprocess.Popen(['python2', - '-m', 'smtpd', + server = subprocess.Popen(['python', + '-m', 'aiosmtpd', '-n', '-d', '-c', 'DebuggingServer', @@ -65,7 +65,22 @@ def test_local_sending(): keywords, email_count = massmail.parse_parameter_file(options) msgs = massmail.create_email_bodies(options, keywords, email_count, email_body) massmail.add_email_headers(options, msgs) - assert msgs['testrecv@test.org'].as_string() == expected_email.as_string() + # we should find a Date header and a message-id header in the email + # we can't replicate them here, so get rid of them before comparing + generated_lines = msgs['testrecv@test.org'].as_string().splitlines() + generated = [] + found_date = False + found_messageid = False + for line in generated_lines: + if line.startswith('Date:'): + found_date = True + elif line.startswith('Message-ID:'): + found_messageid = True + else: + generated.append(line) + assert found_date + assert found_messageid + assert generated == expected_email.as_string().splitlines() def test_fake_sending(): address = 'localhost:8025' From ea0e6fb8b73d57608decefaa2aa79c2b207e1380 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Wed, 5 Apr 2023 21:52:18 +0200 Subject: [PATCH 04/47] implement a hidden option '-f' to allow for testing --- massmail/massmail.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index f2d5f57..cdcae65 100755 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -122,7 +122,7 @@ def parse_command_line_options(arguments): Arguments are checked for validity. """ try: - opts, args = getopt.getopt(arguments, "hs:F:S:B:C:R:e:u:p:P:z:") + opts, args = getopt.getopt(arguments, "hs:F:S:B:C:R:e:u:p:P:z:f") except getopt.GetoptError as err: error(str(err)+USAGE) @@ -139,6 +139,7 @@ def parse_command_line_options(arguments): 'server': None, 'port': 0, 'in_reply_to': '', + 'force': False, } for option, value in opts: @@ -167,6 +168,8 @@ def parse_command_line_options(arguments): options['port'] = int(value) elif option == "-z": options['server'] = value + elif option == '-f': + options['force'] = True if len(args) == 0: error('You must specify a parameter file') @@ -278,14 +281,15 @@ def send_messages(options, msgs): print() print(msgs[emailaddr].get_payload(decode=True).decode(options['encoding'])) - # ask for confirmation before really sending stuff - # we need to read input from the terminal, because stdin is taken already - sys.stdin = open('/dev/tty') - ans = input('Send the emails above? Type "Y" to confirm! ') - if ans != 'Y': - error('OK, exiting without sending anything!') + if not options['force']: + # ask for confirmation before really sending stuff + # we need to read input from the terminal, because stdin is taken already + sys.stdin = open('/dev/tty') + ans = input('Send the emails above? Type "Y" to confirm! ') + if ans != 'Y': + error('OK, exiting without sending anything!') - print() + print() server = smtplib.SMTP(options['server'], port=options['port']) From f9360808760a83a1211effd1b457c54390756860 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Wed, 5 Apr 2023 21:59:51 +0200 Subject: [PATCH 05/47] define additional hidden option '-t' to disable TLS (useful for testing) --- massmail/massmail.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index cdcae65..c08792a 100755 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -122,7 +122,7 @@ def parse_command_line_options(arguments): Arguments are checked for validity. """ try: - opts, args = getopt.getopt(arguments, "hs:F:S:B:C:R:e:u:p:P:z:f") + opts, args = getopt.getopt(arguments, "hs:F:S:B:C:R:e:u:p:P:z:ft") except getopt.GetoptError as err: error(str(err)+USAGE) @@ -140,6 +140,7 @@ def parse_command_line_options(arguments): 'port': 0, 'in_reply_to': '', 'force': False, + 'tls': True, } for option, value in opts: @@ -170,6 +171,8 @@ def parse_command_line_options(arguments): options['server'] = value elif option == '-f': options['force'] = True + elif option == '-t': + options['tls'] = False if len(args) == 0: error('You must specify a parameter file') @@ -292,10 +295,11 @@ def send_messages(options, msgs): print() server = smtplib.SMTP(options['server'], port=options['port']) + if options['tls']: + server.starttls() if options['smtpuser'] is not None: try: - server.starttls() # get password if needed if options['smtppassword'] is None: options['smtppassword'] = getpass.getpass('Enter password for %s: '%options['smtpuser']) From cfbe4808137fdff8750747656371db43ff957298 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Wed, 5 Apr 2023 22:00:27 +0200 Subject: [PATCH 06/47] fix test with fake smtp server --- massmail/test_sending.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/massmail/test_sending.py b/massmail/test_sending.py index 97ad30b..92d1068 100644 --- a/massmail/test_sending.py +++ b/massmail/test_sending.py @@ -22,16 +22,17 @@ def replace_stdin(text): @contextlib.contextmanager def fake_smtp_server(address): - devnull = open(os.devnull, 'w') server = subprocess.Popen(['python', '-m', 'aiosmtpd', '-n', '-d', - '-c', 'DebuggingServer', - address], - stdin=devnull, - stdout=devnull, - stderr=subprocess.PIPE) + '-l', address, + '-c', 'aiosmtpd.handlers.Debugging', 'stderr'], + stdin=None, + text=False, + stderr=subprocess.PIPE, + stdout=None, + bufsize=0) try: time.sleep(1) yield server @@ -85,18 +86,17 @@ def test_local_sending(): def test_fake_sending(): address = 'localhost:8025' with tempfile.NamedTemporaryFile('wt') as f: - f.write('$EMAIL$;$VALUE$\ntestrecv@test;this is a test') + f.write('$EMAIL$;$VALUE$\ntestrecv@test;this is a test\n') f.flush() - with fake_smtp_server(address) as server: - with replace_stdin('EMAIL=$EMAIL$\nVALUE=$VALUE$'): + with replace_stdin('EMAIL=$EMAIL$\nVALUE=$VALUE$\n'): massmail.main(['-F', 'fake@foobar.com', - '-z', address, - f.name]) + '-z', address, '-f', '-t', + f.name]) - output = server.stderr.read() - assert b'MAIL FROM:' in output - assert b'RCPT TO:' in output + stderr = server.stderr.read() + assert b'sender: fake@foobar.com' in stderr + assert b'recip: testrecv@test' in stderr - encoded = base64.b64encode(b'EMAIL=testrecv@test\nVALUE=this is a test') - assert encoded in output + encoded = base64.b64encode(b'EMAIL=testrecv@test\nVALUE=this is a test\n') + assert encoded in stderr From dd5344b3b5fa88c70cc0c4857c8db113cdc731dd Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Thu, 6 Apr 2023 14:48:10 +0200 Subject: [PATCH 07/47] use the python local to the environment to run the smtp server --- massmail/test_sending.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massmail/test_sending.py b/massmail/test_sending.py index 92d1068..6cef599 100644 --- a/massmail/test_sending.py +++ b/massmail/test_sending.py @@ -22,7 +22,7 @@ def replace_stdin(text): @contextlib.contextmanager def fake_smtp_server(address): - server = subprocess.Popen(['python', + server = subprocess.Popen([sys.executable, '-m', 'aiosmtpd', '-n', '-d', From 1a1681c45c571fd50afe0c4898fd5c93709b9f05 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Thu, 6 Apr 2023 16:30:40 +0200 Subject: [PATCH 08/47] use pytest for testing and click for the CLI interface --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 29d5f79..bea8a16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Send mass mail" authors = [ { name="ASPP", email ="info@aspp.school" } ] license = { file = "LICENSE" } requires-python = ">=3.6" -dependencies = [ "aiosmtpd" ] +dependencies = [ "aiosmtpd", "click", "pytest" ] [project.scripts] massmail = "massmail.massmail:main" From 2493c850bdda65884f5a1c48bda0d20ee1863a5f Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Thu, 6 Apr 2023 19:52:52 +0200 Subject: [PATCH 09/47] biiiig rewrite all in one commit because the rewrite was not functional in the development stages: - use click for the command line interface [this makes for a much more consistent and clear CLI] - port to the modern interface of the email stdlib module [takes care automatically of encoding issues and it is way more pythonic] - do not use stdin anymore for the body, we use required options for both the parameter file and the body text: no more confusion about which is which like in the old interface! - line count cut by almost half :-))) TODO: - write a callback for checking that CLI arguments that should be email address are indeed email addresses! - rewrite the tests so that they work and add more tests - detect use of stdin -> this means we are called using the old interace: we can generate a good error message! --- massmail/massmail.py | 417 ++++++++++++++----------------------------- 1 file changed, 132 insertions(+), 285 deletions(-) mode change 100755 => 100644 massmail/massmail.py diff --git a/massmail/massmail.py b/massmail/massmail.py old mode 100755 new mode 100644 index c08792a..18b4c72 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -1,337 +1,184 @@ -#!/usr/bin/env python -# Script to send mass email -# -# Copyright (C) 2003-2017 Tiziano Zito , Jakob Jordan -# -# This script is free software and comes without any warranty, to -# the extent permitted by applicable law. You can redistribute it -# and/or modify it under the terms of the Do What The Fuck You Want -# To Public License, Version 2, as published by Sam Hocevar. -# http://www.wtfpl.net -# -# Full license text: -# -# DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE -# Version 2, December 2004. -# -# Everyone is permitted to copy and distribute verbatim or modified -# copies of this license document, and changing it is allowed as long -# as the name is changed. -# -# DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE -# TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION -# -# 0. You just DO WHAT THE FUCK YOU WANT TO. - -import smtplib, getopt, sys, os, email, getpass -import email.header -import email.mime.text -import email.utils -import io -import hashlib -import contextlib -import tempfile -import time -import subprocess -import base64 +import email import re +import smtplib -PROGNAME = os.path.basename(sys.argv[0]) -USAGE = """Send mass mail -Usage: - %s [...] PARAMETER_FILE < BODY +import click -Options: - -F FROM set the From: header for all messages. - Must be ASCII. This argument is required - -S SUBJECT set the Subject: header for all messages - - -B BCC set the Bcc: header for all messages. - Must be ASCII - - -C CC set the Cc: header for all messages. - Must be ASCII - - -R In-Reply-To set the In-Reply-To: takes a Message-ID as input - - -s SEPARATOR set field separator in parameter file, - default: ";" - - -e ENCODING set PARAMETER_FILE *and* BODY character set - encoding, default: "UTF-8". Note that if you fuck - up this one, your email will be full of rubbish: - You have been warned! - - -z SERVER the SMTP server to use. This argument is required - - -P PORT the SMTP port to use. Must be between 1 and 65535. - - -u SMTP user name. If not set, use anonymous SMTP - connection - - -p SMTP password. If not set you will be prompted for one - - -h show this usage message - -Notes: - The message body is read from standard input or - typed in interactively (exit with ^D) and keywords are subsituted with values - read from a parameter file. The first line of the parameter file defines - the keywords. The keyword $EMAIL$ must always be present and contains a comma - separated list of email addresses. - Keep in mind shell escaping when setting headers with white spaces or special - characters. - Character set encodings are those supported by python. - -Examples: - -* Example of a parameter file: - -$NAME$; $SURNAME$; $EMAIL$ -John; Smith; j@guys.com -Anne; Joyce; a@girls.com - -* Example of body: - -Dear $NAME$ $SURNAME$, - -I think you are a great guy/girl! - -Cheers, - -My self. - -* Example usage: - -%s -F "Great Guy " -S "You are a great guy" -B "Not so great Guy " parameter-file < body - -"""%(PROGNAME, PROGNAME) - -def error(s): - sys.stderr.write(PROGNAME+': ') - sys.stderr.write(s+'\n') - sys.stderr.flush() - sys.exit(-1) - -def parse_command_line_options(arguments): - """Parse options. - - Returns a dictionary of options. - - Arguments are checked for validity. - """ - try: - opts, args = getopt.getopt(arguments, "hs:F:S:B:C:R:e:u:p:P:z:ft") - except getopt.GetoptError as err: - error(str(err)+USAGE) - - # set default options - options = { - 'sep': u';', - 'from': '', - 'subject': '', - 'bcc': '', - 'cc': '', - 'encoding': 'utf-8', - 'smtpuser': None, - 'smtppassword': None, - 'server': None, - 'port': 0, - 'in_reply_to': '', - 'force': False, - 'tls': True, - } - - for option, value in opts: - if option == "-e": - options['encoding'] = value - if option == "-s": - options['sep'] = value - elif option == "-F": - options['from'] = value - elif option == "-S": - options['subject'] = value - elif option == "-B": - options['bcc'] = value - elif option == "-C": - options['cc'] = value - elif option == "-R": - options['in_reply_to'] = value - elif option == "-h": - print(USAGE) - exit(0) - elif option == "-u": - options['smtpuser'] = value - elif option == "-p": - options['smtppassword'] = value - elif option == "-P": - options['port'] = int(value) - elif option == "-z": - options['server'] = value - elif option == '-f': - options['force'] = True - elif option == '-t': - options['tls'] = False - - if len(args) == 0: - error('You must specify a parameter file') - - if len(options['from']) == 0: - error('You must set a from address with option -F') - - if options['server'] is None: - error('You must set a SMTP server with option -z') - - if options['sep'] == ",": - error('Separator can not be a comma') - - # set filenames of parameter and mail body - options['fn_parameters'] = args[0] - - return options - -def parse_parameter_file(options): - pars_fh = open(options['fn_parameters'],'rt') - pars = pars_fh.read() - pars_fh.close() - - if options['in_reply_to'] and not options['in_reply_to'].startswith('<'): - options['in_reply_to'] = '<{}>'.format(options['in_reply_to']) - - # split lines +def parse_parameter_file(parameter_file, separator): + name = parameter_file.name + pars = parameter_file.read() pars = pars.splitlines() # get keywords from first line - keywords_list = [key.strip() for key in pars[0].split(options['sep'])] + key_list = [key.strip() for key in pars[0].split(separator)] # fail immediately if no EMAIL keyword is found - if '$EMAIL$' not in keywords_list: - error('No $EMAIL$ keyword found in %s'%options['fn_parameters']) + if '$EMAIL$' not in key_list: + raise click.ClickException(f'No $EMAIL$ keyword found in {name}') # check that all keywords start and finish with a '$' character - for key in keywords_list: + for key in key_list: if not key.startswith('$') or not key.endswith('$'): - error('Keyword "%s" malformed: should be $KEYWORD$'%key) + raise click.ClickException(f'Keyword {key=} malformed in {name}: should be $KEY$') # gather all values - email_count = 0 - keywords = dict([(key, []) for key in keywords_list]) + keys = dict([(key, []) for key in key_list]) for count, line in enumerate(pars[1:]): # ignore empty lines if len(line) == 0: continue - values = [key.strip() for key in line.split(options['sep'])] - if len(values) != len(keywords_list): - error(('Line %d in "%s" malformed: %d values found instead of' - ' %d: %s'%(count+1,options['fn_parameters'],len(values),len(keywords_list),line)).encode(options['encoding'])) - for i, key in enumerate(keywords_list): - keywords[key].append(values[i]) - email_count += 1 - - return keywords, email_count - -def create_email_bodies(options, keywords, email_count, body): - # find keywords and substitute with values - # we need to create email_count bodies - msgs = {} - - for i in range(email_count): - lbody = re.sub(r'\$\w+\$', lambda m: keywords[m.group(0)][i], body) - - # encode body - lbody = email.mime.text.MIMEText(lbody.encode(options['encoding']), 'plain', options['encoding']) - msgs[keywords['$EMAIL$'][i]] = lbody - - return msgs + values = [key.strip() for key in line.split(separator)] + if len(values) != len(key_list): + raise click.ClickException(f'Line {count+1} in {name} malformed: ' + f'{len(values)} found instead of {len(key_list)}') + for i, key in enumerate(key_list): + keys[key].append(values[i]) -def add_email_headers(options, msgs): - # msgs is now a dictionary with {emailaddr:body} - # we need to add the headers + return keys - for emailaddr in msgs: - msg = msgs[emailaddr] - msg['To'] = str(emailaddr) - msg['From'] = email.header.Header(options['from']) - if options['subject']: - msg['Subject'] = email.header.Header(options['subject'].encode(options['encoding']), options['encoding']) - if options['in_reply_to']: - msg['In-Reply-To'] = email.header.Header(options['in_reply_to']) - if options['cc']: - msg['Cc'] = email.header.Header(options['cc']) +def create_email_bodies(body_file, keys, fromh, subject, cc, reply_to): + msgs = {} + body_text = body_file.read() + for i, emails in enumerate(keys['$EMAIL$']): + # find keywords and substitute with values + body = re.sub(r'\$\w+\$', lambda m: keys[m.group(0)][i], body_text) + msg = email.message.EmailMessage() + msg.set_content(body) + msg['To'] = emails + msg['From'] = fromh + msg['Subject'] = subject + if reply_to: + msg['In-Reply-To'] = reply_to + if cc: + msg['Cc'] = cc # add the required date header - msg['Date'] = email.utils.formatdate(localtime=True) + msg['Date'] = email.utils.localtime() # add a unique message-id - msg['Message-ID'] = email.header.Header(generate_unique_id(msg)) - msgs[emailaddr] = msg - - return None + msg['Message-ID'] = email.utils.make_msgid() + msgs[emails] = msg -def generate_unique_id(msg): - # Get headers and body in the message, hash it and convert to base64 - prefix = hashlib.sha256(msg.get_payload(decode=True)).hexdigest()[:16] - message_id = email.utils.make_msgid(idstring=prefix) - return message_id - -def send_messages(options, msgs): + return msgs +def send_messages(msgs, fromh, bcc, force, server, tls, smtpuser, smtppassword): for emailaddr in msgs: emails = [e.strip() for e in emailaddr.split(',')] - if len(options['bcc']) > 0: - emails.append(options['bcc']) + if bcc: + emails.append(bcc) print('This email will be sent to:', ', '.join(emails)) [print(hdr+':', value) for hdr, value in msgs[emailaddr].items()] print() - print(msgs[emailaddr].get_payload(decode=True).decode(options['encoding'])) + print(msgs[emailaddr].get_content()) - if not options['force']: + if not force: # ask for confirmation before really sending stuff - # we need to read input from the terminal, because stdin is taken already - sys.stdin = open('/dev/tty') - ans = input('Send the emails above? Type "Y" to confirm! ') - if ans != 'Y': - error('OK, exiting without sending anything!') + if not click.confirm('Send the emails above?', default=None): + raise click.ClickException('Aborted! We did not send anything!') print() - server = smtplib.SMTP(options['server'], port=options['port']) - if options['tls']: + servername = server.split(':')[0] + try: + server = smtplib.SMTP(server) + except Exception as err: + raise click.ClickException(f'Can not connect to "{servername}": {err}') + + if tls: server.starttls() - if options['smtpuser'] is not None: + if smtpuser is not None: try: # get password if needed - if options['smtppassword'] is None: - options['smtppassword'] = getpass.getpass('Enter password for %s: '%options['smtpuser']) - server.login(options['smtpuser'], options['smtppassword']) + if smtppassword is None: + smtppassword = click.prompt(f'Enter password for {smtpuser} on {servername}', + hide_input=True) + server.login(smtpuser, smtppassword) except Exception as err: - error(str(err)) + raise click.ClickException(f'Can not login to {servername}: {err}') print() for emailaddr in msgs: print('Sending email to:', emailaddr) emails = [e.strip() for e in emailaddr.split(',')] - if len(options['bcc']) > 0: - emails.append(options['bcc']) + if bcc: + emails.append(bcc) try: - out = server.sendmail(options['from'], emails, msgs[emailaddr].as_string()) + out = server.sendmail(fromh, emails, msgs[emailaddr].as_string()) except Exception as err: - error(str(err)) + raise click.ClickException(f'Can not send email: {err}') if len(out) != 0: - error(str(out)) + raise click.ClickException(f'Can not send email: {err}') server.close() -def main(arguments=None): - if arguments is None: - arguments = sys.argv[1:] - options = parse_command_line_options(arguments) - keywords, email_count = parse_parameter_file(options) - msgs = create_email_bodies(options, keywords, email_count, sys.stdin.read()) - add_email_headers(options, msgs) - send_messages(options, msgs) - -if __name__ == '__main__': - main(sys.argv[1:]) + +def validate_separator(context, param, value): + # only one single character is allowedm and it can not be a comma [because we + # want to use a comma separated list in the $EMAIL$ parameter] + if len(value) > 1 or value == ',': + raise click.BadParameter(f"only once char different from comma is permitted: '{value}'!") + return value + +def validate_reply_to(context, param, value): + if value is None: + return None + if not (value.startswith('<') and value.endswith('>')): + raise click.BadParameter(f"must be enclosed in brackets (): {value}!") + return value + +@click.command(context_settings={'help_option_names': ['-h', '--help'], + 'max_content_width': 120}) +@click.option('-F', '--from', 'fromh', required=True, help='set the From: header') +@click.option('-S', '--subject', required=True, help='set the Subject: header') +@click.option('-Z', '--server', required=True, help='the SMTP server to use') +@click.option('-P', '--parameter', 'parameter_file', required=True, + type=click.File(mode='rt', encoding='utf8', errors='strict'), + help='set the parameter file (see above for an example)') +@click.option('-B', '--body', 'body_file', required=True, + type=click.File(mode='rt', encoding='utf8', errors='strict'), + help='set the email body file (see above for an example)') +@click.option('-b', '--bcc', help='set the Bcc: header') +@click.option('-c', '--cc', help='set the Cc: header') +@click.option('-r', '--reply-to', callback=validate_reply_to, metavar="", + help='set the In-Reply-to: header. Set it to a Message-ID.') +@click.option('-u', '--smtpuser', help='SMTP user name. If not set, use anonymous SMTP connection') +@click.option('-p', '--smtppassword', help='SMTP password. If not set you will be prompted for one') +@click.option('-f', '--force', is_flag=True, default=False, help='do not ask for confirmation before sending messages (use with care!)') +@click.option('--tls/--no-tls', default=True, show_default=True, + help='encrypt SMTP connection with TLS (disable only if you know what you are doing!)') +@click.option('--separator', help='set field separator in parameter file [comma "," is not permitted]', + default=';', show_default=True, callback=validate_separator, metavar="CHAR") +def main(fromh, subject, server, parameter_file, body_file, bcc, cc, reply_to, + smtpuser, smtppassword, force, tls, separator): + """Send mass mail + + Example: + + \b + massmail --from "Blushing Gorilla " --subject "Invitation to the jungle" --server smtp.gmail.com:587 -P parm.csv -B body.txt + + parm.csv: + + \b + $NAME$; $SURNAME$; $EMAIL$ + John; Smith; j@monkeys.com + Anne; Joyce; a@donkeys.com + + body.txt: + + \b + Dear $NAME$ $SURNAME$, + we kindly invite you to join us in the jungle + Cheers, + Gorilla + + Notes: + + Keywords from the parameter file (parm.csv) are subsituted in the body text. The keyword $EMAIL$ must always be present in the parameter files and contains a comma separated list of email addresses. Keep in mind shell escaping when setting headers with white spaces or special characters. Both files must be UTF8 encoded! + """ + keys = parse_parameter_file(parameter_file, separator) + msgs = create_email_bodies(body_file, keys, fromh, subject, cc, reply_to) + send_messages(msgs, fromh, bcc, force, server, tls, smtpuser, smtppassword) + From ffaeab92a8764f3e407173c32e9ac9c9b8b87694 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Fri, 14 Apr 2023 13:33:36 +0200 Subject: [PATCH 10/47] first version of the test for the modernized API... ...we make a heavy use of the pytest fixture system, which makes for much clearer and compact test code --- massmail/test_massmail.py | 117 ++++++++++++++++++++++++++++++-------- massmail/test_sending.py | 102 --------------------------------- 2 files changed, 92 insertions(+), 127 deletions(-) delete mode 100644 massmail/test_sending.py diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index f845d69..ce16f8b 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -1,25 +1,92 @@ -import tempfile - -import massmail.massmail as massmail - -def test_dummy(): - pass - -def test_command_help(): - import pytest - with pytest.raises(SystemExit): - massmail.main(['-h']) - -def test_parse_parameter_file(): - expected_keywords = {u'$VALUE$': [u'this is a test'], u'$EMAIL$': [u'testrecv@test']} - with tempfile.NamedTemporaryFile('wt') as f: - f.write('$EMAIL$;$VALUE$\ntestrecv@test;this is a test') - f.flush() - cmd_options = [ - '-F', 'testfrom@test', - '-z', 'localhost', - f.name, - ] - options = massmail.parse_command_line_options(cmd_options) - keywords, email_count = massmail.parse_parameter_file(options) - assert keywords == expected_keywords +import contextlib +import email +import io +import sys +import os +import time +import subprocess +import base64 + +from massmail.massmail import main as massmail +import click.testing +import pytest + +# run the script from the command line through the click-internal testing interface +@pytest.fixture +def cli(): + return click.testing.CliRunner() + +# return the list of monimal required options to run massmail without errors +@pytest.fixture +def opts(): + return ['--from', 'Blushing Gorilla ', + '--subject', 'Invitation to the jungle', + '--server', 'localhost:8025', + '--force', + '--no-tls'] + + +# return a locally running SMTP server. This fixture kills the server after the +# test run is finished, i.e. not after every test! +# The resulting server runs at localhost:8025 +@pytest.fixture(scope="module") +def server(): + server = subprocess.Popen([sys.executable, + '-m', 'aiosmtpd', + '-n', + '-d', + '-l', 'localhost:8025', + '-c', 'aiosmtpd.handlers.Debugging', 'stderr'], + stdin=None, + text=False, + stderr=subprocess.PIPE, + stdout=None, + bufsize=0, + env={'AIOSMTPD_CONTROLLER_TIMEOUT':'0'}) + # give the smtp server 100 milliseconds to startup + time.sleep(0.1) + yield server + server.terminate() + +# return a "good" parameter file +@pytest.fixture +def good_parm(tmp_path): + header = '$NAME$;$SURNAME$;$EMAIL$' + row1 = 'Alice;Joyce;donkeys@jungle.com' + f = tmp_path / 'parms.csv' + f.write_text(header+'\n'+row1) + yield f + f.unlink() + +# return a "good" body file +@pytest.fixture +def good_body(tmp_path): + body = """Dear $NAME$ $SURNAME$, + + we kindly invite you to join us in the jungle + + Cheers, + Gorilla + """ + f = tmp_path / 'body.txt' + f.write_text(body) + yield f + f.unlink() + +# just test that the cli is working and we get the right help text +def test_help(cli): + result = cli.invoke(massmail, ['-h']) + assert result.exit_code == 0 + assert 'Usage:' in result.output + assert 'Example:' in result.output + + +def test_regular_sending(cli, opts, server, good_parm, good_body): + inp = ['--parameter', str(good_parm), '--body', str(good_body)] + result = cli.invoke(massmail, opts + inp) + # we can not just issue a blank .read() because that would would block until + # server.stderr is closed, which only happens after the server has exited + smtp = server.stderr.read(100000).decode('ascii') + assert result.exit_code == 0 + assert 'Dear Alice Joyce' in smtp + diff --git a/massmail/test_sending.py b/massmail/test_sending.py deleted file mode 100644 index 6cef599..0000000 --- a/massmail/test_sending.py +++ /dev/null @@ -1,102 +0,0 @@ -import contextlib -import email -import io -import sys -import os -import time -import subprocess -import base64 -import tempfile - -import massmail.massmail as massmail - -@contextlib.contextmanager -def replace_stdin(text): - input = io.StringIO(text) - old = sys.stdin - sys.stdin, old = input, sys.stdin - try: - yield - finally: - sys.stdin = old - -@contextlib.contextmanager -def fake_smtp_server(address): - server = subprocess.Popen([sys.executable, - '-m', 'aiosmtpd', - '-n', - '-d', - '-l', address, - '-c', 'aiosmtpd.handlers.Debugging', 'stderr'], - stdin=None, - text=False, - stderr=subprocess.PIPE, - stdout=None, - bufsize=0) - try: - time.sleep(1) - yield server - finally: - server.terminate() - -def test_local_sending(): - parameter_string = '$EMAIL$;$NAME$;$VALUE$\ntestrecv@test.org;TestName;531' - email_body = 'Dear $NAME$,\nthis is a test: $VALUE$\nBest regards' - email_to = 'testrecv@test.org' - email_from = 'testfrom@test.org' - email_subject = 'Test Subject' - email_encoding = 'utf-8' - - expected_email = email.mime.text.MIMEText('Dear TestName,\nthis is a test: 531\nBest regards'.encode(email_encoding), 'plain', email_encoding) - expected_email['To'] = email_to - expected_email['From'] = email_from - expected_email['Subject'] = email.header.Header(email_subject.encode(email_encoding), email_encoding) - - with tempfile.NamedTemporaryFile('wt') as f: - f.write(parameter_string) - f.flush() - cmd_options = [ - '-F', email_from, - '-S', email_subject, - '-z', 'localhost', - '-e', email_encoding, - f.name - ] - options = massmail.parse_command_line_options(cmd_options) - keywords, email_count = massmail.parse_parameter_file(options) - msgs = massmail.create_email_bodies(options, keywords, email_count, email_body) - massmail.add_email_headers(options, msgs) - # we should find a Date header and a message-id header in the email - # we can't replicate them here, so get rid of them before comparing - generated_lines = msgs['testrecv@test.org'].as_string().splitlines() - generated = [] - found_date = False - found_messageid = False - for line in generated_lines: - if line.startswith('Date:'): - found_date = True - elif line.startswith('Message-ID:'): - found_messageid = True - else: - generated.append(line) - assert found_date - assert found_messageid - assert generated == expected_email.as_string().splitlines() - -def test_fake_sending(): - address = 'localhost:8025' - with tempfile.NamedTemporaryFile('wt') as f: - f.write('$EMAIL$;$VALUE$\ntestrecv@test;this is a test\n') - f.flush() - with fake_smtp_server(address) as server: - with replace_stdin('EMAIL=$EMAIL$\nVALUE=$VALUE$\n'): - massmail.main(['-F', 'fake@foobar.com', - '-z', address, '-f', '-t', - f.name]) - - stderr = server.stderr.read() - assert b'sender: fake@foobar.com' in stderr - assert b'recip: testrecv@test' in stderr - - encoded = base64.b64encode(b'EMAIL=testrecv@test\nVALUE=this is a test\n') - assert encoded in stderr From 02bc042b767c2f43bd41a57795878733badc597a Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 12:50:32 +0200 Subject: [PATCH 11/47] use new SMTP.send_message method, which allows us to simplify the send-mail logic --- massmail/massmail.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 18b4c72..29ebefb 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -37,7 +37,7 @@ def parse_parameter_file(parameter_file, separator): return keys -def create_email_bodies(body_file, keys, fromh, subject, cc, reply_to): +def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, reply_to): msgs = {} body_text = body_file.read() for i, emails in enumerate(keys['$EMAIL$']): @@ -52,6 +52,8 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, reply_to): msg['In-Reply-To'] = reply_to if cc: msg['Cc'] = cc + if bcc: + msg['Bcc'] = bcc # add the required date header msg['Date'] = email.utils.localtime() # add a unique message-id @@ -60,11 +62,9 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, reply_to): return msgs -def send_messages(msgs, fromh, bcc, force, server, tls, smtpuser, smtppassword): +def send_messages(msgs, force, server, tls, smtpuser, smtppassword): for emailaddr in msgs: emails = [e.strip() for e in emailaddr.split(',')] - if bcc: - emails.append(bcc) print('This email will be sent to:', ', '.join(emails)) [print(hdr+':', value) for hdr, value in msgs[emailaddr].items()] print() @@ -100,18 +100,15 @@ def send_messages(msgs, fromh, bcc, force, server, tls, smtpuser, smtppassword): for emailaddr in msgs: print('Sending email to:', emailaddr) - emails = [e.strip() for e in emailaddr.split(',')] - if bcc: - emails.append(bcc) try: - out = server.sendmail(fromh, emails, msgs[emailaddr].as_string()) + out = server.send_message(msgs[emailaddr]) except Exception as err: raise click.ClickException(f'Can not send email: {err}') if len(out) != 0: raise click.ClickException(f'Can not send email: {err}') - server.close() + server.quit() def validate_separator(context, param, value): @@ -179,6 +176,6 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, reply_to, Keywords from the parameter file (parm.csv) are subsituted in the body text. The keyword $EMAIL$ must always be present in the parameter files and contains a comma separated list of email addresses. Keep in mind shell escaping when setting headers with white spaces or special characters. Both files must be UTF8 encoded! """ keys = parse_parameter_file(parameter_file, separator) - msgs = create_email_bodies(body_file, keys, fromh, subject, cc, reply_to) - send_messages(msgs, fromh, bcc, force, server, tls, smtpuser, smtppassword) + msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, reply_to) + send_messages(msgs, force, server, tls, smtpuser, smtppassword) From 31559748f42514495f391c4713f9f12b8e9e70e0 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 14:13:24 +0200 Subject: [PATCH 12/47] use get_payload instead of get_content to get a decode string of the email body ... get_content is documented, but does not automatically handle decoding of 8bit transferred emails. get_payload is not documented, but is used internally by the email module and it handles correctly the decoding... --- massmail/massmail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 29ebefb..806a6ed 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -68,7 +68,7 @@ def send_messages(msgs, force, server, tls, smtpuser, smtppassword): print('This email will be sent to:', ', '.join(emails)) [print(hdr+':', value) for hdr, value in msgs[emailaddr].items()] print() - print(msgs[emailaddr].get_content()) + print(msgs[emailaddr].get_payload()) if not force: # ask for confirmation before really sending stuff From 42afd21277a0a48f551200b732a78146db1d57f9 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 14:13:39 +0200 Subject: [PATCH 13/47] added a bunch of tests --- massmail/test_massmail.py | 114 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 5 deletions(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index ce16f8b..1a78a59 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -1,5 +1,5 @@ import contextlib -import email +import email as email_module import io import sys import os @@ -73,6 +73,34 @@ def good_body(tmp_path): yield f f.unlink() +def parse_smtp(server): + # we can not just issue a blank .read() because that would would block until + # server.stderr is closed, which only happens after the server has exited + # so we request 1MB (1024*1024 = 2^20 bytes = 1048576) to be sure + smtp = server.stderr.read(1048576).decode() + protocol = [] + emails = [] + for line in smtp.splitlines(): + if line.startswith('INFO:mail.log:'): + # this is a line of protocol: SMTP server <-> SMTP client + protocol.append(line) + elif line.startswith('---------- MESSAGE FOLLOWS ----------'): + # this lines starts a new email message + email = [] + elif line.startswith('------------ END MESSAGE ------------'): + # this line closes an email message + # join all collected lines + email = '\n'.join(email) + # parse the whole thing as a string to get a Message object from + # the email module + emails.append(email_module.message_from_string(email, + policy=email_module.policy.default)) + else: + # collect this line, it belongs to the current email message + email.append(line) + protocol = '\n'.join(protocol) + return protocol, emails + # just test that the cli is working and we get the right help text def test_help(cli): result = cli.invoke(massmail, ['-h']) @@ -84,9 +112,85 @@ def test_help(cli): def test_regular_sending(cli, opts, server, good_parm, good_body): inp = ['--parameter', str(good_parm), '--body', str(good_body)] result = cli.invoke(massmail, opts + inp) - # we can not just issue a blank .read() because that would would block until - # server.stderr is closed, which only happens after the server has exited - smtp = server.stderr.read(100000).decode('ascii') assert result.exit_code == 0 - assert 'Dear Alice Joyce' in smtp + protocol, emails = parse_smtp(server) + email = emails[0] + + # check that the envelope is correct + assert 'sender: gorilla@jungle.com' in protocol + assert 'recip: donkeys@jungle.com' in protocol + + # check that the headers are correct + assert email['From'] == 'Blushing Gorilla ' + assert email['To'] == 'donkeys@jungle.com' + assert email['Subject'] == 'Invitation to the jungle' + assert 'Date' in email + assert 'Message-ID' in email + assert email.get_charsets() == ['utf-8'] + assert email.get_content_type() == 'text/plain' + + # check that we have insert the right values for our keys + body = email.get_payload() + assert 'Dear Alice Joyce' in body + assert 'we kindly invite you to join us in the jungle' in body + + +def test_unicode_body_sending(cli, opts, server, good_parm, good_body): + # add some unicode text to the body + with good_body.open('at') as bodyf: + bodyf.write('\nÜni©ödę¿\n') + inp = ['--parameter', str(good_parm), '--body', str(good_body)] + result = cli.invoke(massmail, opts + inp) + assert result.exit_code == 0 + protocol, emails = parse_smtp(server) + email = emails[0] + body = email.get_payload() + assert 'Üni©ödę¿' in body + +def test_unicode_subject(cli, opts, server, good_parm, good_body): + opts.extend(('--subject', 'Üni©ödę¿')) + inp = ['--parameter', str(good_parm), '--body', str(good_body)] + result = cli.invoke(massmail, opts + inp) + assert result.exit_code == 0 + protocol, emails = parse_smtp(server) + email = emails[0] + assert email['Subject'] == 'Üni©ödę¿' + +def test_unicode_from(cli, opts, server, good_parm, good_body): + opts.extend(('--from', 'Üni©ödę¿ ')) + inp = ['--parameter', str(good_parm), '--body', str(good_body)] + result = cli.invoke(massmail, opts + inp) + assert result.exit_code == 0 + protocol, emails = parse_smtp(server) + email = emails[0] + assert email['From'] == 'Üni©ödę¿ ' + +def test_unicode_several_reciepients(cli, opts, server, good_parm, good_body): + # add some unicode text to the body + with good_parm.open('at') as parmf: + parmf.write('\nJohn; Smith; j@monkeys.com\n') + inp = ['--parameter', str(good_parm), '--body', str(good_body)] + result = cli.invoke(massmail, opts + inp) + assert result.exit_code == 0 + protocol, emails = parse_smtp(server) + + assert len(emails) == 2 + assert 'sender: gorilla@jungle.com' in protocol + assert 'recip: donkeys@jungle.com' in protocol + assert 'recip: j@monkeys.com' in protocol + + assert emails[0]['To'] == 'donkeys@jungle.com' + assert emails[1]['To'] == 'j@monkeys.com' + assert 'Dear Alice Joyce' in emails[0].get_payload() + assert 'Dear John Smith' in emails[1].get_payload() + +def test_unicode_parm(cli, opts, server, good_parm, good_body): + # add some unicode text to the body + with good_parm.open('at') as parmf: + parmf.write('\nÜni©ödę¿; Smith; j@monkeys.com\n') + inp = ['--parameter', str(good_parm), '--body', str(good_body)] + result = cli.invoke(massmail, opts + inp) + assert result.exit_code == 0 + protocol, emails = parse_smtp(server) + assert 'Dear Üni©ödę¿ Smith' in emails[1].get_payload() From 9fc15a0cb66576140996c24f22cedd6baf84ab58 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 14:15:53 +0200 Subject: [PATCH 14/47] remove redundant imports --- massmail/test_massmail.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 1a78a59..527bed3 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -1,11 +1,7 @@ -import contextlib import email as email_module -import io +import subprocess import sys -import os import time -import subprocess -import base64 from massmail.massmail import main as massmail import click.testing From b28d479891a93ffff58508566d379b1d951abef2 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 18:43:43 +0200 Subject: [PATCH 15/47] refactor tests and add test with wild unicode characters --- massmail/test_massmail.py | 119 +++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 60 deletions(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 527bed3..3da0ecb 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -7,19 +7,7 @@ import click.testing import pytest -# run the script from the command line through the click-internal testing interface -@pytest.fixture -def cli(): - return click.testing.CliRunner() -# return the list of monimal required options to run massmail without errors -@pytest.fixture -def opts(): - return ['--from', 'Blushing Gorilla ', - '--subject', 'Invitation to the jungle', - '--server', 'localhost:8025', - '--force', - '--no-tls'] # return a locally running SMTP server. This fixture kills the server after the @@ -46,7 +34,7 @@ def server(): # return a "good" parameter file @pytest.fixture -def good_parm(tmp_path): +def parm(tmp_path): header = '$NAME$;$SURNAME$;$EMAIL$' row1 = 'Alice;Joyce;donkeys@jungle.com' f = tmp_path / 'parms.csv' @@ -56,8 +44,8 @@ def good_parm(tmp_path): # return a "good" body file @pytest.fixture -def good_body(tmp_path): - body = """Dear $NAME$ $SURNAME$, +def body(tmp_path): + text = """Dear $NAME$ $SURNAME$, we kindly invite you to join us in the jungle @@ -65,7 +53,7 @@ def good_body(tmp_path): Gorilla """ f = tmp_path / 'body.txt' - f.write_text(body) + f.write_text(text) yield f f.unlink() @@ -97,25 +85,36 @@ def parse_smtp(server): protocol = '\n'.join(protocol) return protocol, emails +def cli(server, parm, body, opts=[]): + options = ['--from', 'Blushing Gorilla ', + '--subject', 'Invitation to the jungle', + '--server', 'localhost:8025', + '--force', + '--no-tls'] + opts + ['--parameter', str(parm), '--body', str(body)] + script = click.testing.CliRunner() + result = script.invoke(massmail, options) + assert result.exit_code == 0 + protocol, emails = parse_smtp(server) + return protocol, emails + # just test that the cli is working and we get the right help text -def test_help(cli): - result = cli.invoke(massmail, ['-h']) +def test_help(): + result = click.testing.CliRunner().invoke(massmail, ['-h']) assert result.exit_code == 0 assert 'Usage:' in result.output assert 'Example:' in result.output - -def test_regular_sending(cli, opts, server, good_parm, good_body): - inp = ['--parameter', str(good_parm), '--body', str(good_body)] - result = cli.invoke(massmail, opts + inp) - assert result.exit_code == 0 - protocol, emails = parse_smtp(server) +def test_regular_sending(server, parm, body): + protocol, emails = cli(server, parm, body) email = emails[0] # check that the envelope is correct assert 'sender: gorilla@jungle.com' in protocol assert 'recip: donkeys@jungle.com' in protocol + # email is ASCII, so the transfer encoding should be 7bit + assert email['Content-Transfer-Encoding'] == '7bit' + # check that the headers are correct assert email['From'] == 'Blushing Gorilla ' assert email['To'] == 'donkeys@jungle.com' @@ -126,50 +125,53 @@ def test_regular_sending(cli, opts, server, good_parm, good_body): assert email.get_content_type() == 'text/plain' # check that we have insert the right values for our keys - body = email.get_payload() - assert 'Dear Alice Joyce' in body - assert 'we kindly invite you to join us in the jungle' in body + text = email.get_payload() + assert 'Dear Alice Joyce' in text + assert 'we kindly invite you to join us in the jungle' in text -def test_unicode_body_sending(cli, opts, server, good_parm, good_body): +def test_unicode_body_sending(server, parm, body): # add some unicode text to the body - with good_body.open('at') as bodyf: + with body.open('at') as bodyf: bodyf.write('\nÜni©ödę¿\n') - inp = ['--parameter', str(good_parm), '--body', str(good_body)] - result = cli.invoke(massmail, opts + inp) - assert result.exit_code == 0 - protocol, emails = parse_smtp(server) + protocol, emails = cli(server, parm, body) email = emails[0] - body = email.get_payload() - assert 'Üni©ödę¿' in body - + # unicode characters force the transfer encoding to 8bit + assert email['Content-Transfer-Encoding'] == '8bit' + text = email.get_payload() + assert 'Üni©ödę¿' in text -def test_unicode_subject(cli, opts, server, good_parm, good_body): - opts.extend(('--subject', 'Üni©ödę¿')) - inp = ['--parameter', str(good_parm), '--body', str(good_body)] - result = cli.invoke(massmail, opts + inp) - assert result.exit_code == 0 - protocol, emails = parse_smtp(server) +def test_wild_unicode_body_sending(server, parm, body): + # add some unicode text to the body + with body.open('at') as bodyf: + bodyf.write('\nœ´®†¥¨ˆøπ¬˚∆˙©ƒ∂ßåΩ≈ç√∫˜µ≤ユーザーコードa😀\n') + protocol, emails = cli(server, parm, body) + email = emails[0] + # because we use unicode characters that don't fit in one byte, + # the email will be encoded in base64 for tranfer + assert email['Content-Transfer-Encoding'] == 'base64' + # because the text contains, we trust the internal email_module machinery + # to perform the proper decoding + text = email.get_content() + assert 'œ´®†¥¨ˆøπ¬˚∆˙©ƒ∂ßåΩ≈ç√∫˜µ≤ユーザーコードa😀' in text + +def test_unicode_subject(server, parm, body): + opts = ['--subject', 'Üni©ödę¿'] + protocol, emails = cli(server, parm, body, opts=opts) email = emails[0] assert email['Subject'] == 'Üni©ödę¿' -def test_unicode_from(cli, opts, server, good_parm, good_body): - opts.extend(('--from', 'Üni©ödę¿ ')) - inp = ['--parameter', str(good_parm), '--body', str(good_body)] - result = cli.invoke(massmail, opts + inp) - assert result.exit_code == 0 - protocol, emails = parse_smtp(server) +def test_unicode_from(server, parm, body): + opts = ['--from', 'Üni©ödę¿ '] + protocol, emails = cli(server, parm, body, opts=opts) email = emails[0] assert email['From'] == 'Üni©ödę¿ ' -def test_unicode_several_reciepients(cli, opts, server, good_parm, good_body): +def test_unicode_several_reciepients(server, parm, body): # add some unicode text to the body - with good_parm.open('at') as parmf: + with parm.open('at') as parmf: parmf.write('\nJohn; Smith; j@monkeys.com\n') - inp = ['--parameter', str(good_parm), '--body', str(good_body)] - result = cli.invoke(massmail, opts + inp) - assert result.exit_code == 0 - protocol, emails = parse_smtp(server) + protocol, emails = cli(server, parm, body) assert len(emails) == 2 assert 'sender: gorilla@jungle.com' in protocol @@ -181,12 +183,9 @@ def test_unicode_several_reciepients(cli, opts, server, good_parm, good_body): assert 'Dear Alice Joyce' in emails[0].get_payload() assert 'Dear John Smith' in emails[1].get_payload() -def test_unicode_parm(cli, opts, server, good_parm, good_body): +def test_unicode_parm(server, parm, body): # add some unicode text to the body - with good_parm.open('at') as parmf: + with parm.open('at') as parmf: parmf.write('\nÜni©ödę¿; Smith; j@monkeys.com\n') - inp = ['--parameter', str(good_parm), '--body', str(good_body)] - result = cli.invoke(massmail, opts + inp) - assert result.exit_code == 0 - protocol, emails = parse_smtp(server) + protocol, emails = cli(server, parm, body) assert 'Dear Üni©ödę¿ Smith' in emails[1].get_payload() From 614b990ad97dfc24803244416e678328e785de69 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:05:31 +0200 Subject: [PATCH 16/47] Revert "use get_payload instead of get_content to get a decode string of the email body" This commit was based on some wrong assumptions. Actually get_content should always take care of the proper transfer encoding. If it doesn't there is a bug in the email module and we shouldn't try to work around it here. This reverts commit 31559748f42514495f391c4713f9f12b8e9e70e0. --- massmail/massmail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 806a6ed..29ebefb 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -68,7 +68,7 @@ def send_messages(msgs, force, server, tls, smtpuser, smtppassword): print('This email will be sent to:', ', '.join(emails)) [print(hdr+':', value) for hdr, value in msgs[emailaddr].items()] print() - print(msgs[emailaddr].get_payload()) + print(msgs[emailaddr].get_content()) if not force: # ask for confirmation before really sending stuff From 1bd0b839ee8a41a59d83f16f2a5360f688b5b6f0 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:08:05 +0200 Subject: [PATCH 17/47] use a better way of representing options while testing --- massmail/test_massmail.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 3da0ecb..dd0b3c7 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -85,16 +85,29 @@ def parse_smtp(server): protocol = '\n'.join(protocol) return protocol, emails -def cli(server, parm, body, opts=[]): - options = ['--from', 'Blushing Gorilla ', - '--subject', 'Invitation to the jungle', - '--server', 'localhost:8025', - '--force', - '--no-tls'] + opts + ['--parameter', str(parm), '--body', str(body)] +# wrapper for running the massmail script and parse the SMTP server output +def cli(server, parm, body, opts={}): + options = { + '--from' : 'Blushing Gorilla ', + '--subject' : 'Invitation to the jungle', + '--server' : 'localhost:8025', + '--parameter' : str(parm), + '--body' : str(body), + } + options.update(opts) + opts = [] + for option, value in options.items(): + opts.extend((option, value)) + # now we have all default options + options passed by the test + # instantiate a click Runner script = click.testing.CliRunner() - result = script.invoke(massmail, options) + # invoke the script, add the no-tls options (our SMTP does not support TLS) + # and do not ask for confirmation to send emails + result = script.invoke(massmail, opts + ['--force', '--no-tls']) assert result.exit_code == 0 + # parse the output of the SMTP server which is running in the background protocol, emails = parse_smtp(server) + # return the protocol text and a list of emails return protocol, emails # just test that the cli is working and we get the right help text From 99ec1f0a8160012c972534bd5356e261194cf366 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:08:31 +0200 Subject: [PATCH 18/47] properly handle SMTP server binary output in tests --- massmail/test_massmail.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index dd0b3c7..5f82dcc 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -61,28 +61,30 @@ def parse_smtp(server): # we can not just issue a blank .read() because that would would block until # server.stderr is closed, which only happens after the server has exited # so we request 1MB (1024*1024 = 2^20 bytes = 1048576) to be sure - smtp = server.stderr.read(1048576).decode() + smtp = server.stderr.read(1048576) protocol = [] emails = [] for line in smtp.splitlines(): - if line.startswith('INFO:mail.log:'): + if line.startswith(b'INFO:mail.log:'): # this is a line of protocol: SMTP server <-> SMTP client protocol.append(line) - elif line.startswith('---------- MESSAGE FOLLOWS ----------'): + elif line.startswith(b'---------- MESSAGE FOLLOWS ----------'): # this lines starts a new email message email = [] - elif line.startswith('------------ END MESSAGE ------------'): + elif line.startswith(b'------------ END MESSAGE ------------'): # this line closes an email message # join all collected lines - email = '\n'.join(email) - # parse the whole thing as a string to get a Message object from - # the email module - emails.append(email_module.message_from_string(email, + email = b'\n'.join(email) # we are dealing with bytes here + # parse the whole thing to get a EmailMessage object from the email module + emails.append(email_module.message_from_bytes(email, policy=email_module.policy.default)) else: # collect this line, it belongs to the current email message email.append(line) - protocol = '\n'.join(protocol) + protocol = b'\n'.join(protocol) + # protocol chat should always be ascii, if we get a UnicodeDecodeError here + # we'll have to examine the server output to understand what's up + protocol = protocol.decode('ascii') return protocol, emails # wrapper for running the massmail script and parse the SMTP server output @@ -137,8 +139,8 @@ def test_regular_sending(server, parm, body): assert email.get_charsets() == ['utf-8'] assert email.get_content_type() == 'text/plain' - # check that we have insert the right values for our keys - text = email.get_payload() + # check that we have inserted the right values for our keys + text = email.get_content() assert 'Dear Alice Joyce' in text assert 'we kindly invite you to join us in the jungle' in text @@ -151,7 +153,7 @@ def test_unicode_body_sending(server, parm, body): email = emails[0] # unicode characters force the transfer encoding to 8bit assert email['Content-Transfer-Encoding'] == '8bit' - text = email.get_payload() + text = email.get_content() assert 'Üni©ödę¿' in text def test_wild_unicode_body_sending(server, parm, body): @@ -163,19 +165,19 @@ def test_wild_unicode_body_sending(server, parm, body): # because we use unicode characters that don't fit in one byte, # the email will be encoded in base64 for tranfer assert email['Content-Transfer-Encoding'] == 'base64' - # because the text contains, we trust the internal email_module machinery + # we have to trust the internal email_module machinery # to perform the proper decoding text = email.get_content() assert 'œ´®†¥¨ˆøπ¬˚∆˙©ƒ∂ßåΩ≈ç√∫˜µ≤ユーザーコードa😀' in text def test_unicode_subject(server, parm, body): - opts = ['--subject', 'Üni©ödę¿'] + opts = {'--subject' : 'Üni©ödę¿' } protocol, emails = cli(server, parm, body, opts=opts) email = emails[0] assert email['Subject'] == 'Üni©ödę¿' def test_unicode_from(server, parm, body): - opts = ['--from', 'Üni©ödę¿ '] + opts = { '--from' : 'Üni©ödę¿ ' } protocol, emails = cli(server, parm, body, opts=opts) email = emails[0] assert email['From'] == 'Üni©ödę¿ ' @@ -193,12 +195,12 @@ def test_unicode_several_reciepients(server, parm, body): assert emails[0]['To'] == 'donkeys@jungle.com' assert emails[1]['To'] == 'j@monkeys.com' - assert 'Dear Alice Joyce' in emails[0].get_payload() - assert 'Dear John Smith' in emails[1].get_payload() + assert 'Dear Alice Joyce' in emails[0].get_content() + assert 'Dear John Smith' in emails[1].get_content() def test_unicode_parm(server, parm, body): # add some unicode text to the body with parm.open('at') as parmf: parmf.write('\nÜni©ödę¿; Smith; j@monkeys.com\n') protocol, emails = cli(server, parm, body) - assert 'Dear Üni©ödę¿ Smith' in emails[1].get_payload() + assert 'Dear Üni©ödę¿ Smith' in emails[1].get_content() From 13ac9f814e20f8b127e998c1f43dfe787f7a96e7 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:21:39 +0200 Subject: [PATCH 19/47] allow for testing errors in script --- massmail/test_massmail.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 5f82dcc..fb80a69 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -88,7 +88,7 @@ def parse_smtp(server): return protocol, emails # wrapper for running the massmail script and parse the SMTP server output -def cli(server, parm, body, opts={}): +def cli(server, parm, body, opts={}, errs=False): options = { '--from' : 'Blushing Gorilla ', '--subject' : 'Invitation to the jungle', @@ -106,6 +106,10 @@ def cli(server, parm, body, opts={}): # invoke the script, add the no-tls options (our SMTP does not support TLS) # and do not ask for confirmation to send emails result = script.invoke(massmail, opts + ['--force', '--no-tls']) + if errs: + # we expect errors, so do not interact with the SMTP server at all + # and read the errors from the script instead + return result.exit_code, result.output assert result.exit_code == 0 # parse the output of the SMTP server which is running in the background protocol, emails = parse_smtp(server) From dd985a520333b3a48227c25207db355008aac47f Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:31:11 +0200 Subject: [PATCH 20/47] rename smtpuser and smtppassword options --- massmail/massmail.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 29ebefb..c680e3f 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -62,7 +62,7 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, reply_to): return msgs -def send_messages(msgs, force, server, tls, smtpuser, smtppassword): +def send_messages(msgs, force, server, tls, user, password): for emailaddr in msgs: emails = [e.strip() for e in emailaddr.split(',')] print('This email will be sent to:', ', '.join(emails)) @@ -86,13 +86,13 @@ def send_messages(msgs, force, server, tls, smtpuser, smtppassword): if tls: server.starttls() - if smtpuser is not None: + if user is not None: try: # get password if needed - if smtppassword is None: - smtppassword = click.prompt(f'Enter password for {smtpuser} on {servername}', + if password is None: + password = click.prompt(f'Enter password for {user} on {servername}', hide_input=True) - server.login(smtpuser, smtppassword) + server.login(user, password) except Exception as err: raise click.ClickException(f'Can not login to {servername}: {err}') @@ -140,15 +140,15 @@ def validate_reply_to(context, param, value): @click.option('-c', '--cc', help='set the Cc: header') @click.option('-r', '--reply-to', callback=validate_reply_to, metavar="", help='set the In-Reply-to: header. Set it to a Message-ID.') -@click.option('-u', '--smtpuser', help='SMTP user name. If not set, use anonymous SMTP connection') -@click.option('-p', '--smtppassword', help='SMTP password. If not set you will be prompted for one') +@click.option('-u', '--user', help='SMTP user name. If not set, use anonymous SMTP connection') +@click.option('-p', '--password', help='SMTP password. If not set you will be prompted for one') @click.option('-f', '--force', is_flag=True, default=False, help='do not ask for confirmation before sending messages (use with care!)') @click.option('--tls/--no-tls', default=True, show_default=True, help='encrypt SMTP connection with TLS (disable only if you know what you are doing!)') @click.option('--separator', help='set field separator in parameter file [comma "," is not permitted]', default=';', show_default=True, callback=validate_separator, metavar="CHAR") def main(fromh, subject, server, parameter_file, body_file, bcc, cc, reply_to, - smtpuser, smtppassword, force, tls, separator): + user, password, force, tls, separator): """Send mass mail Example: @@ -177,5 +177,5 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, reply_to, """ keys = parse_parameter_file(parameter_file, separator) msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, reply_to) - send_messages(msgs, force, server, tls, smtpuser, smtppassword) + send_messages(msgs, force, server, tls, user, password) From 9c0af7443ca7a2d2bccffc0cc19525981ffa5da6 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:32:55 +0200 Subject: [PATCH 21/47] rename smtpuser and smtppassword options --- massmail/massmail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index c680e3f..52a3f35 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -156,12 +156,12 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, reply_to, \b massmail --from "Blushing Gorilla " --subject "Invitation to the jungle" --server smtp.gmail.com:587 -P parm.csv -B body.txt - parm.csv: + parm.csv (semi-colon separated): \b $NAME$; $SURNAME$; $EMAIL$ John; Smith; j@monkeys.com - Anne; Joyce; a@donkeys.com + Anne and Mary; Joyce; a@donkeys.com, m@donkeys.com body.txt: From 3f0945c6f7edc87f0f2b4213e152a03c917e6ae4 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:40:26 +0200 Subject: [PATCH 22/47] remove --separator option. We just force the use of ';' --- massmail/massmail.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 52a3f35..3132b42 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -5,13 +5,13 @@ import click -def parse_parameter_file(parameter_file, separator): +def parse_parameter_file(parameter_file): name = parameter_file.name pars = parameter_file.read() pars = pars.splitlines() # get keywords from first line - key_list = [key.strip() for key in pars[0].split(separator)] + key_list = [key.strip() for key in pars[0].split(';')] # fail immediately if no EMAIL keyword is found if '$EMAIL$' not in key_list: @@ -28,7 +28,7 @@ def parse_parameter_file(parameter_file, separator): # ignore empty lines if len(line) == 0: continue - values = [key.strip() for key in line.split(separator)] + values = [key.strip() for key in line.split(';')] if len(values) != len(key_list): raise click.ClickException(f'Line {count+1} in {name} malformed: ' f'{len(values)} found instead of {len(key_list)}') @@ -110,14 +110,6 @@ def send_messages(msgs, force, server, tls, user, password): server.quit() - -def validate_separator(context, param, value): - # only one single character is allowedm and it can not be a comma [because we - # want to use a comma separated list in the $EMAIL$ parameter] - if len(value) > 1 or value == ',': - raise click.BadParameter(f"only once char different from comma is permitted: '{value}'!") - return value - def validate_reply_to(context, param, value): if value is None: return None @@ -145,10 +137,8 @@ def validate_reply_to(context, param, value): @click.option('-f', '--force', is_flag=True, default=False, help='do not ask for confirmation before sending messages (use with care!)') @click.option('--tls/--no-tls', default=True, show_default=True, help='encrypt SMTP connection with TLS (disable only if you know what you are doing!)') -@click.option('--separator', help='set field separator in parameter file [comma "," is not permitted]', - default=';', show_default=True, callback=validate_separator, metavar="CHAR") def main(fromh, subject, server, parameter_file, body_file, bcc, cc, reply_to, - user, password, force, tls, separator): + user, password, force, tls): """Send mass mail Example: @@ -175,7 +165,7 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, reply_to, Keywords from the parameter file (parm.csv) are subsituted in the body text. The keyword $EMAIL$ must always be present in the parameter files and contains a comma separated list of email addresses. Keep in mind shell escaping when setting headers with white spaces or special characters. Both files must be UTF8 encoded! """ - keys = parse_parameter_file(parameter_file, separator) + keys = parse_parameter_file(parameter_file) msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, reply_to) send_messages(msgs, force, server, tls, user, password) From 20e62e87f10272aa1ca1401e01037a9214ddfc8c Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:47:36 +0200 Subject: [PATCH 23/47] renamed in-reply-to option --- massmail/massmail.py | 14 ++++----- massmail/test_massmail.py | 63 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 3132b42..d551d13 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -37,7 +37,7 @@ def parse_parameter_file(parameter_file): return keys -def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, reply_to): +def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to): msgs = {} body_text = body_file.read() for i, emails in enumerate(keys['$EMAIL$']): @@ -48,8 +48,8 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, reply_to): msg['To'] = emails msg['From'] = fromh msg['Subject'] = subject - if reply_to: - msg['In-Reply-To'] = reply_to + if inreply_to: + msg['In-Reply-To'] = inreply_to if cc: msg['Cc'] = cc if bcc: @@ -110,7 +110,7 @@ def send_messages(msgs, force, server, tls, user, password): server.quit() -def validate_reply_to(context, param, value): +def validate_inreply_to(context, param, value): if value is None: return None if not (value.startswith('<') and value.endswith('>')): @@ -130,14 +130,14 @@ def validate_reply_to(context, param, value): help='set the email body file (see above for an example)') @click.option('-b', '--bcc', help='set the Bcc: header') @click.option('-c', '--cc', help='set the Cc: header') -@click.option('-r', '--reply-to', callback=validate_reply_to, metavar="", +@click.option('-r', '--inreply-to', callback=validate_inreply_to, metavar="", help='set the In-Reply-to: header. Set it to a Message-ID.') @click.option('-u', '--user', help='SMTP user name. If not set, use anonymous SMTP connection') @click.option('-p', '--password', help='SMTP password. If not set you will be prompted for one') @click.option('-f', '--force', is_flag=True, default=False, help='do not ask for confirmation before sending messages (use with care!)') @click.option('--tls/--no-tls', default=True, show_default=True, help='encrypt SMTP connection with TLS (disable only if you know what you are doing!)') -def main(fromh, subject, server, parameter_file, body_file, bcc, cc, reply_to, +def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, user, password, force, tls): """Send mass mail @@ -166,6 +166,6 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, reply_to, Keywords from the parameter file (parm.csv) are subsituted in the body text. The keyword $EMAIL$ must always be present in the parameter files and contains a comma separated list of email addresses. Keep in mind shell escaping when setting headers with white spaces or special characters. Both files must be UTF8 encoded! """ keys = parse_parameter_file(parameter_file) - msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, reply_to) + msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to) send_messages(msgs, force, server, tls, user, password) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index fb80a69..3fb076f 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -7,9 +7,6 @@ import click.testing import pytest - - - # return a locally running SMTP server. This fixture kills the server after the # test run is finished, i.e. not after every test! # The resulting server runs at localhost:8025 @@ -148,7 +145,6 @@ def test_regular_sending(server, parm, body): assert 'Dear Alice Joyce' in text assert 'we kindly invite you to join us in the jungle' in text - def test_unicode_body_sending(server, parm, body): # add some unicode text to the body with body.open('at') as bodyf: @@ -208,3 +204,62 @@ def test_unicode_parm(server, parm, body): parmf.write('\nÜni©ödę¿; Smith; j@monkeys.com\n') protocol, emails = cli(server, parm, body) assert 'Dear Üni©ödę¿ Smith' in emails[1].get_content() + +def test_multiple_recipients_in_one_row(server, parm, body): + # add some unicode text to the body + with parm.open('at') as parmf: + parmf.write('\nAnne and Mary; Joyce; a@donkeys.com, m@donkeys.com\n') + protocol, emails = cli(server, parm, body) + assert len(emails) == 2 + assert 'sender: gorilla@jungle.com' in protocol + assert 'recip: donkeys@jungle.com' in protocol + assert 'recip: a@donkeys.com' in protocol + assert 'recip: m@donkeys.com' in protocol + + assert emails[0]['To'] == 'donkeys@jungle.com' + assert emails[1]['To'] == 'a@donkeys.com, m@donkeys.com' + assert 'Dear Alice Joyce' in emails[0].get_content() + assert 'Dear Anne and Mary Joyce' in emails[1].get_content() + + +def test_parm_malformed_keys(server, parm, body): + parm.write_text("""$NAME;$SURNAME$;$EMAIL$ + test;test;test@test.com""") + code, output = cli(server, parm, body, errs=True) + assert code != 0 + assert '$NAME' in output + assert 'malformed' in output + parm.write_text("""$NAME$;SURNAME$;$EMAIL$ + test;test;test@test.com""") + code, output = cli(server, parm, body, errs=True) + assert code != 0 + assert 'SURNAME$' in output + assert 'malformed' in output + +def test_missing_email_in_parm(server, parm, body): + parm.write_text("""$NAME$;$SURNAME$ + test;test""") + code, output = cli(server, parm, body, errs=True) + assert code != 0 + assert 'No $EMAIL$' in output + +def test_missing_value_in_parm(server, parm, body): + with parm.open('at') as parmf: + parmf.write('\nMario;Rossi;j@monkeys.com;too much\n') + code, output = cli(server, parm, body, errs=True) + assert code != 0 + assert 'Line 2' in output + assert '4 found instead of 3' in output + +def test_server_offline(server, parm, body): + opts = {'--server' : 'noserver:25' } + code, output = cli(server, parm, body, opts=opts, errs=True) + assert code != 0 + assert 'Can not connect to' in output + +def test_server_wrong_authentication(server, parm, body): + opts = {'--user' : 'noone', '--password' : 'nopass' } + code, output = cli(server, parm, body, opts=opts, errs=True) + assert code != 0 + assert 'Can not login' in output + From 63efa253cc1f534a002ca6af478a6c7179d9c8e7 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:47:56 +0200 Subject: [PATCH 24/47] added tests for all options --- massmail/test_massmail.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 3fb076f..021fe11 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -263,3 +263,34 @@ def test_server_wrong_authentication(server, parm, body): assert code != 0 assert 'Can not login' in output +def test_bcc(server, parm, body): + opts = {'--bcc' : 'x@monkeys.com'} + protocol, emails = cli(server, parm, body, opts=opts) + + assert len(emails) == 1 + assert 'sender: gorilla@jungle.com' in protocol + assert 'recip: donkeys@jungle.com' in protocol + assert 'recip: x@monkeys.com' in protocol + + assert emails[0]['To'] == 'donkeys@jungle.com' + assert 'Bcc' not in emails[0] + +def test_cc(server, parm, body): + opts = {'--cc' : 'Markus Murkis '} + protocol, emails = cli(server, parm, body, opts=opts) + + assert len(emails) == 1 + assert 'sender: gorilla@jungle.com' in protocol + assert 'recip: donkeys@jungle.com' in protocol + assert 'recip: x@monkeys.com' in protocol + + assert emails[0]['To'] == 'donkeys@jungle.com' + assert emails[0]['Cc'] == 'Markus Murkis ' + +def test_in_reply_to(server, parm, body): + opts = {'--inreply-to' : ''} + protocol, emails = cli(server, parm, body, opts=opts) + + assert len(emails) == 1 + assert emails[0]['In-Reply-To'] == '' + From c7568a41774ad673f4993b0e994d78d0d0eac3c0 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sat, 15 Apr 2023 19:50:42 +0200 Subject: [PATCH 25/47] added test for invalid inreply-to --- massmail/test_massmail.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 021fe11..7ac61c4 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -294,3 +294,9 @@ def test_in_reply_to(server, parm, body): assert len(emails) == 1 assert emails[0]['In-Reply-To'] == '' +def test_invalid_in_reply_to(server, parm, body): + opts = {'--inreply-to' : 'abc'} + code, output = cli(server, parm, body, opts=opts, errs=True) + assert code != 0 + assert 'Invalid value' in output + assert 'brackets' in output From c9ec4e2ee4c0acde05fd4db9997c62531381aa6f Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 10:37:54 +0200 Subject: [PATCH 26/47] added an email address validator --- massmail/massmail.py | 40 +++++++++++++++++++++++++++++++++++++--- pyproject.toml | 2 +- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index d551d13..698c010 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -3,6 +3,7 @@ import smtplib import click +import email_validator def parse_parameter_file(parameter_file): @@ -117,9 +118,42 @@ def validate_inreply_to(context, param, value): raise click.BadParameter(f"must be enclosed in brackets (): {value}!") return value +# a custom click parameter type to represent email addresses +class Email(click.ParamType): + name = "Email" + + def convert(self, value, param, ctx): + # we support two kind of email address: + # 1. x@y.org + # 2. Blushing Gorilla + if address := re.match(r'(.+)<(\S+)>', value): + # we are dealing with form 2 + # extract the email address + prefix = address.group(1) # here we get "Blushing Gorilla " + address = address.group(2) # here we get x@y.org + else: + # we are dealing with form 1 + prefix = None + address = value + # validate the email address + try: + emailinfo = email_validator.validate_email(address, check_deliverability=False) + except email_validator.EmailNotValidError as e: + self.fail(f"{value!r} is not a valid email address:\n{str(e)}", param, ctx) + # support different versions of email-validator + try: + email = emailinfo.normalized # version >= 2.0 + except AttributeError: + email = emailinfo.email # version <= 1.3 + if prefix: + return f'{prefix}<{email}>' + else: + return email + + @click.command(context_settings={'help_option_names': ['-h', '--help'], 'max_content_width': 120}) -@click.option('-F', '--from', 'fromh', required=True, help='set the From: header') +@click.option('-F', '--from', 'fromh', required=True, type=Email(), help='set the From: header') @click.option('-S', '--subject', required=True, help='set the Subject: header') @click.option('-Z', '--server', required=True, help='the SMTP server to use') @click.option('-P', '--parameter', 'parameter_file', required=True, @@ -128,8 +162,8 @@ def validate_inreply_to(context, param, value): @click.option('-B', '--body', 'body_file', required=True, type=click.File(mode='rt', encoding='utf8', errors='strict'), help='set the email body file (see above for an example)') -@click.option('-b', '--bcc', help='set the Bcc: header') -@click.option('-c', '--cc', help='set the Cc: header') +@click.option('-b', '--bcc', type=Email(), help='set the Bcc: header') +@click.option('-c', '--cc', type=Email(), help='set the Cc: header') @click.option('-r', '--inreply-to', callback=validate_inreply_to, metavar="", help='set the In-Reply-to: header. Set it to a Message-ID.') @click.option('-u', '--user', help='SMTP user name. If not set, use anonymous SMTP connection') diff --git a/pyproject.toml b/pyproject.toml index bea8a16..45e82dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Send mass mail" authors = [ { name="ASPP", email ="info@aspp.school" } ] license = { file = "LICENSE" } requires-python = ">=3.6" -dependencies = [ "aiosmtpd", "click", "pytest" ] +dependencies = [ "aiosmtpd", "click", "email-validator", "pytest" ] [project.scripts] massmail = "massmail.massmail:main" From 170aeaff965feefcf29f9a40a26a1f7fbb36f7ff Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 10:38:42 +0200 Subject: [PATCH 27/47] refactor error management in the tests --- massmail/test_massmail.py | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 7ac61c4..549cd44 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -106,12 +106,14 @@ def cli(server, parm, body, opts={}, errs=False): if errs: # we expect errors, so do not interact with the SMTP server at all # and read the errors from the script instead - return result.exit_code, result.output - assert result.exit_code == 0 - # parse the output of the SMTP server which is running in the background - protocol, emails = parse_smtp(server) - # return the protocol text and a list of emails - return protocol, emails + assert result.exit_code != 0 + return result.output + else: + assert result.exit_code == 0 + # parse the output of the SMTP server which is running in the background + protocol, emails = parse_smtp(server) + # return the protocol text and a list of emails + return protocol, emails # just test that the cli is working and we get the right help text def test_help(): @@ -225,43 +227,34 @@ def test_multiple_recipients_in_one_row(server, parm, body): def test_parm_malformed_keys(server, parm, body): parm.write_text("""$NAME;$SURNAME$;$EMAIL$ test;test;test@test.com""") - code, output = cli(server, parm, body, errs=True) - assert code != 0 + output = cli(server, parm, body, errs=True) assert '$NAME' in output assert 'malformed' in output parm.write_text("""$NAME$;SURNAME$;$EMAIL$ test;test;test@test.com""") - code, output = cli(server, parm, body, errs=True) - assert code != 0 + output = cli(server, parm, body, errs=True) assert 'SURNAME$' in output assert 'malformed' in output def test_missing_email_in_parm(server, parm, body): parm.write_text("""$NAME$;$SURNAME$ test;test""") - code, output = cli(server, parm, body, errs=True) - assert code != 0 - assert 'No $EMAIL$' in output + assert 'No $EMAIL$' in cli(server, parm, body, errs=True) def test_missing_value_in_parm(server, parm, body): with parm.open('at') as parmf: parmf.write('\nMario;Rossi;j@monkeys.com;too much\n') - code, output = cli(server, parm, body, errs=True) - assert code != 0 + output = cli(server, parm, body, errs=True) assert 'Line 2' in output assert '4 found instead of 3' in output def test_server_offline(server, parm, body): opts = {'--server' : 'noserver:25' } - code, output = cli(server, parm, body, opts=opts, errs=True) - assert code != 0 - assert 'Can not connect to' in output + assert 'Can not connect to' in cli(server, parm, body, opts=opts, errs=True) def test_server_wrong_authentication(server, parm, body): opts = {'--user' : 'noone', '--password' : 'nopass' } - code, output = cli(server, parm, body, opts=opts, errs=True) - assert code != 0 - assert 'Can not login' in output + assert 'Can not login' in cli(server, parm, body, opts=opts, errs=True) def test_bcc(server, parm, body): opts = {'--bcc' : 'x@monkeys.com'} @@ -296,7 +289,6 @@ def test_in_reply_to(server, parm, body): def test_invalid_in_reply_to(server, parm, body): opts = {'--inreply-to' : 'abc'} - code, output = cli(server, parm, body, opts=opts, errs=True) - assert code != 0 + output = cli(server, parm, body, opts=opts, errs=True) assert 'Invalid value' in output assert 'brackets' in output From 3c1b88cd9fa4d272a06844b2a1c07785c060e46b Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 10:38:51 +0200 Subject: [PATCH 28/47] add tests for email validator --- massmail/test_massmail.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 549cd44..24cbc45 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -292,3 +292,11 @@ def test_invalid_in_reply_to(server, parm, body): output = cli(server, parm, body, opts=opts, errs=True) assert 'Invalid value' in output assert 'brackets' in output + +def test_validate_from(server, parm, body): + opts = {'--from' : 'invalid@email'} + assert 'is not a valid email' in cli(server, parm, body, opts=opts, errs=True) + opts = {'--from' : 'Blushing Gorilla'} + assert 'is not a valid email' in cli(server, parm, body, opts=opts, errs=True) + opts = {'--from' : 'Blushing Gorilla '} + assert 'is not a valid email' in cli(server, parm, body, opts=opts, errs=True) From ac1726ff8835d774a9f93fc572786841dd8bad4e Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 10:44:25 +0200 Subject: [PATCH 29/47] factored out the email validator so that it can be used everywhere in the code --- massmail/massmail.py | 57 +++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 698c010..0991fa2 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -118,38 +118,45 @@ def validate_inreply_to(context, param, value): raise click.BadParameter(f"must be enclosed in brackets (): {value}!") return value +def validate_email_address(value): + # we support two kind of email address: + # 1. x@y.org + # 2. Blushing Gorilla + if address := re.match(r'(.+)<(\S+)>', value): + # we are dealing with form 2 + # extract the email address + prefix = address.group(1) # here we get "Blushing Gorilla " + address = address.group(2) # here we get x@y.org + else: + # we are dealing with form 1 + prefix = None + address = value + # validate the email address + try: + emailinfo = email_validator.validate_email(address, check_deliverability=False) + except email_validator.EmailNotValidError as e: + raise click.BadParameter(f"{value!r} is not a valid email address:\n{str(e)}") + # support different versions of email-validator + try: + email = emailinfo.normalized # version >= 2.0 + except AttributeError: + email = emailinfo.email # version <= 1.3 + if prefix: + return f'{prefix}<{email}>' + else: + return email + + # a custom click parameter type to represent email addresses class Email(click.ParamType): name = "Email" def convert(self, value, param, ctx): - # we support two kind of email address: - # 1. x@y.org - # 2. Blushing Gorilla - if address := re.match(r'(.+)<(\S+)>', value): - # we are dealing with form 2 - # extract the email address - prefix = address.group(1) # here we get "Blushing Gorilla " - address = address.group(2) # here we get x@y.org - else: - # we are dealing with form 1 - prefix = None - address = value # validate the email address try: - emailinfo = email_validator.validate_email(address, check_deliverability=False) - except email_validator.EmailNotValidError as e: - self.fail(f"{value!r} is not a valid email address:\n{str(e)}", param, ctx) - # support different versions of email-validator - try: - email = emailinfo.normalized # version >= 2.0 - except AttributeError: - email = emailinfo.email # version <= 1.3 - if prefix: - return f'{prefix}<{email}>' - else: - return email - + return validate_email_address(value) + except click.BadParameter as e: + self.fail(str(e), param, ctx) @click.command(context_settings={'help_option_names': ['-h', '--help'], 'max_content_width': 120}) From ad37ad6141806d51872b692b6c18aab6ef465c1d Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 11:02:02 +0200 Subject: [PATCH 30/47] help debugging failing test by printing the script output on unexpected error --- massmail/test_massmail.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 24cbc45..d0ba0d0 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -109,6 +109,10 @@ def cli(server, parm, body, opts={}, errs=False): assert result.exit_code != 0 return result.output else: + # in case of unexpected error, let's print the output so we have a chance + # to debug without touching the code here + if result.exit_code != 0: + print(result.output) assert result.exit_code == 0 # parse the output of the SMTP server which is running in the background protocol, emails = parse_smtp(server) From 0a8ed2a454e69ff8d0c1828524d266981df1ba42 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 11:02:46 +0200 Subject: [PATCH 31/47] validate email addresses from parameter file --- massmail/massmail.py | 15 +++++++++++++-- massmail/test_massmail.py | 7 +++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 0991fa2..32abef4 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -36,6 +36,17 @@ def parse_parameter_file(parameter_file): for i, key in enumerate(key_list): keys[key].append(values[i]) + # validate email addresses + for idx, emails in enumerate(keys['$EMAIL$']): + # split into individual email addresses + validated = [] + for email in emails.split(','): + # remove spaces + email = email.strip() + validated.append(validate_email_address(email, + errstr=f'Line {idx+1} in {name}:\n')) + # store the validated and normalized email addresses back into the dict + keys['$EMAIL$'][idx] = ','.join(validated) return keys def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to): @@ -118,7 +129,7 @@ def validate_inreply_to(context, param, value): raise click.BadParameter(f"must be enclosed in brackets (): {value}!") return value -def validate_email_address(value): +def validate_email_address(value, errstr=''): # we support two kind of email address: # 1. x@y.org # 2. Blushing Gorilla @@ -135,7 +146,7 @@ def validate_email_address(value): try: emailinfo = email_validator.validate_email(address, check_deliverability=False) except email_validator.EmailNotValidError as e: - raise click.BadParameter(f"{value!r} is not a valid email address:\n{str(e)}") + raise click.BadParameter(errstr+f"{value!r} is not a valid email address:\n{str(e)}") # support different versions of email-validator try: email = emailinfo.normalized # version >= 2.0 diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index d0ba0d0..fd03fd8 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -304,3 +304,10 @@ def test_validate_from(server, parm, body): assert 'is not a valid email' in cli(server, parm, body, opts=opts, errs=True) opts = {'--from' : 'Blushing Gorilla '} assert 'is not a valid email' in cli(server, parm, body, opts=opts, errs=True) + +def test_invalid_email_in_parm(server, parm, body): + with parm.open('at') as parmf: + parmf.write('\nMario;Rossi;j@monkeys\n') + output = cli(server, parm, body, errs=True) + assert 'is not a valid email' in output + assert 'Line 2' in output From 2c3cdb801f0caca31cc474d7d57c3ab49096b64f Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 11:08:36 +0200 Subject: [PATCH 32/47] added more email address validation tests --- massmail/test_massmail.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index fd03fd8..98f6019 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -311,3 +311,11 @@ def test_invalid_email_in_parm(server, parm, body): output = cli(server, parm, body, errs=True) assert 'is not a valid email' in output assert 'Line 2' in output + +def test_rich_email_address_in_parm(server, parm, body): + with parm.open('at') as parmf: + parmf.write('\nMario;Rossi;Mario Rossi \n') + protocol, emails = cli(server, parm, body) + assert 'recip: j@monkeys.org' in protocol + assert 'Mario Rossi' in emails[1]['To'] + assert 'j@monkeys.org' in emails[1]['To'] From 5432140e4dee968e2587894a925c1995cb628c7d Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 11:21:34 +0200 Subject: [PATCH 33/47] hide options from --help output that we only should use in tests --- massmail/massmail.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 32abef4..76db78b 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -171,6 +171,8 @@ def convert(self, value, param, ctx): @click.command(context_settings={'help_option_names': ['-h', '--help'], 'max_content_width': 120}) + +### REQUIRED OPTIONS ### @click.option('-F', '--from', 'fromh', required=True, type=Email(), help='set the From: header') @click.option('-S', '--subject', required=True, help='set the Subject: header') @click.option('-Z', '--server', required=True, help='the SMTP server to use') @@ -180,15 +182,22 @@ def convert(self, value, param, ctx): @click.option('-B', '--body', 'body_file', required=True, type=click.File(mode='rt', encoding='utf8', errors='strict'), help='set the email body file (see above for an example)') + +### OPTIONALS ### @click.option('-b', '--bcc', type=Email(), help='set the Bcc: header') @click.option('-c', '--cc', type=Email(), help='set the Cc: header') @click.option('-r', '--inreply-to', callback=validate_inreply_to, metavar="", help='set the In-Reply-to: header. Set it to a Message-ID.') @click.option('-u', '--user', help='SMTP user name. If not set, use anonymous SMTP connection') @click.option('-p', '--password', help='SMTP password. If not set you will be prompted for one') -@click.option('-f', '--force', is_flag=True, default=False, help='do not ask for confirmation before sending messages (use with care!)') -@click.option('--tls/--no-tls', default=True, show_default=True, - help='encrypt SMTP connection with TLS (disable only if you know what you are doing!)') + +### INTERNAL OPTIONS ### +# do not ask for confirmation before sending messages (to be used in tests) +@click.option('-f', '--force', is_flag=True, default=False, hidden=True) +# do not TLS encrypt connection to SMTP server (to be used in tests) +@click.option('--tls/--no-tls', default=True, hidden=True) + +### MAIN SCRIPT ### def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, user, password, force, tls): """Send mass mail @@ -196,7 +205,7 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, Example: \b - massmail --from "Blushing Gorilla " --subject "Invitation to the jungle" --server smtp.gmail.com:587 -P parm.csv -B body.txt + massmail --from "Blushing Gorilla " --subject "Invitation to the jungle" --server mail.example.com:587 --user user@example.com -P parm.csv -B body.txt parm.csv (semi-colon separated): @@ -215,7 +224,7 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, Notes: - Keywords from the parameter file (parm.csv) are subsituted in the body text. The keyword $EMAIL$ must always be present in the parameter files and contains a comma separated list of email addresses. Keep in mind shell escaping when setting headers with white spaces or special characters. Both files must be UTF8 encoded! + Values from the parameter file (parm.csv) are inserted in the body text (body.txt). The keyword $EMAIL$ must always be present in the parameter files and contains a comma separated list of email addresses. Keep in mind shell escaping when setting headers with white spaces or special characters. Both files must be UTF8 encoded! """ keys = parse_parameter_file(parameter_file) msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to) From a03ff54f603eaa520006d80b36d25a30389ea17f Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 12:43:48 +0200 Subject: [PATCH 34/47] new feature: it is now possible to add mutiple attachments --- massmail/massmail.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 76db78b..eb44d77 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -1,4 +1,6 @@ import email +import mimetypes +import pathlib import re import smtplib @@ -49,9 +51,22 @@ def parse_parameter_file(parameter_file): keys['$EMAIL$'][idx] = ','.join(validated) return keys -def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to): +def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, attachment): msgs = {} body_text = body_file.read() + + # collect attachments once and then attach them to every single message + attachments = {} + for path in attachment: + # guess the MIME type based on file extension only... + mime, encoding = mimetypes.guess_type(path, strict=False) + # if no guess or if the type is already encoded, the MIME type is octet-stream + if mime is None or encoding is not None: + mime = 'application/octet-stream' + maintype, subtype = mime.split('/', 1) + data = path.read_bytes() + attachments[path.name] = (data, maintype, subtype) + for i, emails in enumerate(keys['$EMAIL$']): # find keywords and substitute with values body = re.sub(r'\$\w+\$', lambda m: keys[m.group(0)][i], body_text) @@ -70,6 +85,9 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to): msg['Date'] = email.utils.localtime() # add a unique message-id msg['Message-ID'] = email.utils.make_msgid() + # add attachments + for name, (data, mtyp, styp) in attachments.items(): + msg.add_attachment(data, filename=name, maintype=mtyp, subtype=styp) msgs[emails] = msg return msgs @@ -78,9 +96,14 @@ def send_messages(msgs, force, server, tls, user, password): for emailaddr in msgs: emails = [e.strip() for e in emailaddr.split(',')] print('This email will be sent to:', ', '.join(emails)) - [print(hdr+':', value) for hdr, value in msgs[emailaddr].items()] - print() - print(msgs[emailaddr].get_content()) + for hdr, value in msgs[emailaddr].items(): + print(f'{hdr}: {value}') + for attachment in msgs[emailaddr].iter_attachments(): + name = attachment.get_filename() + content_type = attachment.get_content_type() + print(f'Attachment ({content_type}): {name}') + body = msgs[emailaddr].get_body().get_content() + print(f'\n{body}') if not force: # ask for confirmation before really sending stuff @@ -190,6 +213,9 @@ def convert(self, value, param, ctx): help='set the In-Reply-to: header. Set it to a Message-ID.') @click.option('-u', '--user', help='SMTP user name. If not set, use anonymous SMTP connection') @click.option('-p', '--password', help='SMTP password. If not set you will be prompted for one') +@click.option('-a', '--attachment', help='add attachment [repeat for multiple attachments]', + multiple=True, type=click.Path(exists=True, dir_okay=False, + readable=True, path_type=pathlib.Path)) ### INTERNAL OPTIONS ### # do not ask for confirmation before sending messages (to be used in tests) @@ -199,7 +225,7 @@ def convert(self, value, param, ctx): ### MAIN SCRIPT ### def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, - user, password, force, tls): + user, password, attachment, force, tls): """Send mass mail Example: @@ -227,6 +253,6 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, Values from the parameter file (parm.csv) are inserted in the body text (body.txt). The keyword $EMAIL$ must always be present in the parameter files and contains a comma separated list of email addresses. Keep in mind shell escaping when setting headers with white spaces or special characters. Both files must be UTF8 encoded! """ keys = parse_parameter_file(parameter_file) - msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to) + msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, attachment) send_messages(msgs, force, server, tls, user, password) From f79831b299cdd8e8358aba2678ae4846a5ec337d Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 13:25:18 +0200 Subject: [PATCH 35/47] add tests for the attachment feature --- massmail/test_massmail.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 98f6019..db202dd 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -85,7 +85,7 @@ def parse_smtp(server): return protocol, emails # wrapper for running the massmail script and parse the SMTP server output -def cli(server, parm, body, opts={}, errs=False): +def cli(server, parm, body, opts={}, opts_list=[], errs=False): options = { '--from' : 'Blushing Gorilla ', '--subject' : 'Invitation to the jungle', @@ -93,10 +93,15 @@ def cli(server, parm, body, opts={}, errs=False): '--parameter' : str(parm), '--body' : str(body), } + # options can be passed by a test as a dictionary or as a list + # as a dictionary options.update(opts) opts = [] for option, value in options.items(): opts.extend((option, value)) + # we need to allow passing options as a list for those options that can be + # passed multiple times, e.g. --attachment, where a dictionary wouldn't work + opts.extend(opts_list) # now we have all default options + options passed by the test # instantiate a click Runner script = click.testing.CliRunner() @@ -319,3 +324,29 @@ def test_rich_email_address_in_parm(server, parm, body): assert 'recip: j@monkeys.org' in protocol assert 'Mario Rossi' in emails[1]['To'] assert 'j@monkeys.org' in emails[1]['To'] + +def test_attach_png_and_pdf(server, parm, body, tmp_path): + # create a little PNG (greyscale, 1x1 pixel) from: + # https://garethrees.org/2007/11/14/pngcrush/ + png_bytes = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x01\x00\x00\x00\x007n\xf9$\x00\x00\x00\nIDATx\x9cc`\x00\x00\x00\x02\x00\x01H\xaf\xa4q\x00\x00\x00\x00IEND\xaeB`\x82' + png_file = tmp_path / 'test.png' + png_file.write_bytes(png_bytes) + + # create a little PDF (stackoverflow + adaptations) + pdf_bytes = b"%PDF-1.2 \n9 0 obj\n<<\n>>\nstream\nBT/ 32 Tf(text)' ET\nendstream\nendobj\n4 0 obj\n<<\n/Type /Page\n/Parent 5 0 R\n/Contents 9 0 R\n>>\nendobj\n5 0 obj\n<<\n/Kids [4 0 R ]\n/Count 1\n/Type /Pages\n/MediaBox [ 0 0 250 50 ]\n>>\nendobj\n3 0 obj\n<<\n/Pages 5 0 R\n/Type /Catalog\n>>\nendobj\ntrailer\n<<\n/Root 3 0 R\n>>\n%%EOF" + pdf_file = tmp_path / 'test.pdf' + pdf_file.write_bytes(pdf_bytes) + + opts = ('--attachment', str(png_file), '--attachment', str(pdf_file)) + protocol, emails = cli(server, parm, body, opts_list=opts) + email = emails[0] + assert 'Dear Alice Joyce' in email.get_body().get_content() + attachments = list(email.iter_attachments()) + assert attachments[0].get_content_type() == 'image/png' + assert attachments[0].get_filename() == 'test.png' + assert attachments[0].get_content() == png_bytes + assert attachments[1].get_content_type() == 'application/pdf' + assert attachments[1].get_filename() == 'test.pdf' + assert attachments[1].get_content() == pdf_bytes + + From 415eabb4675bbf37ebd29a13b811b6a504b36f69 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 13:31:49 +0200 Subject: [PATCH 36/47] added a test for an unkown attachment type --- massmail/test_massmail.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index db202dd..73f5cde 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -349,4 +349,13 @@ def test_attach_png_and_pdf(server, parm, body, tmp_path): assert attachments[1].get_filename() == 'test.pdf' assert attachments[1].get_content() == pdf_bytes +def test_unknown_attachment_type(server, parm, body, tmp_path): + random_bytes = b'\x9diou\xd5\x12\xdf/\x03\xf8' + fl = tmp_path / 'dummy' + fl.write_bytes(random_bytes) + opts = {'--attachment' : str(fl)} + protocol, emails = cli(server, parm, body, opts=opts) + attachments = list(emails[0].iter_attachments()) + assert attachments[0].get_content_type() == 'application/octet-stream' + assert attachments[0].get_content() == random_bytes From 30a05ca1fb9bf225e787b3d0eb5b1bdcebc06f97 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 14:04:32 +0200 Subject: [PATCH 37/47] removed unused --force option --- massmail/massmail.py | 17 +++++++---------- massmail/test_massmail.py | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index eb44d77..92ea215 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -92,7 +92,7 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, at return msgs -def send_messages(msgs, force, server, tls, user, password): +def send_messages(msgs, server, tls, user, password): for emailaddr in msgs: emails = [e.strip() for e in emailaddr.split(',')] print('This email will be sent to:', ', '.join(emails)) @@ -105,12 +105,11 @@ def send_messages(msgs, force, server, tls, user, password): body = msgs[emailaddr].get_body().get_content() print(f'\n{body}') - if not force: - # ask for confirmation before really sending stuff - if not click.confirm('Send the emails above?', default=None): - raise click.ClickException('Aborted! We did not send anything!') + # ask for confirmation before really sending stuff + if not click.confirm('Send the emails above?', default=None): + raise click.ClickException('Aborted! We did not send anything!') - print() + print() servername = server.split(':')[0] try: @@ -218,14 +217,12 @@ def convert(self, value, param, ctx): readable=True, path_type=pathlib.Path)) ### INTERNAL OPTIONS ### -# do not ask for confirmation before sending messages (to be used in tests) -@click.option('-f', '--force', is_flag=True, default=False, hidden=True) # do not TLS encrypt connection to SMTP server (to be used in tests) @click.option('--tls/--no-tls', default=True, hidden=True) ### MAIN SCRIPT ### def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, - user, password, attachment, force, tls): + user, password, attachment, tls): """Send mass mail Example: @@ -254,5 +251,5 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, """ keys = parse_parameter_file(parameter_file) msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, attachment) - send_messages(msgs, force, server, tls, user, password) + send_messages(msgs, server, tls, user, password) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 73f5cde..e851ac9 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -107,7 +107,7 @@ def cli(server, parm, body, opts={}, opts_list=[], errs=False): script = click.testing.CliRunner() # invoke the script, add the no-tls options (our SMTP does not support TLS) # and do not ask for confirmation to send emails - result = script.invoke(massmail, opts + ['--force', '--no-tls']) + result = script.invoke(massmail, opts + ['--no-tls'], input='y\n') if errs: # we expect errors, so do not interact with the SMTP server at all # and read the errors from the script instead From d2aea3ac5495f9123703b96fda5b747c0a83a0cc Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 14:28:52 +0200 Subject: [PATCH 38/47] remove --no-tls option. We can now test using TLS --- massmail/massmail.py | 14 +++-- massmail/test_massmail.py | 109 +++++++++++++++++++++++++++++++++++--- 2 files changed, 107 insertions(+), 16 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 92ea215..a2e9e85 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -92,7 +92,7 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, at return msgs -def send_messages(msgs, server, tls, user, password): +def send_messages(msgs, server, user, password): for emailaddr in msgs: emails = [e.strip() for e in emailaddr.split(',')] print('This email will be sent to:', ', '.join(emails)) @@ -117,8 +117,10 @@ def send_messages(msgs, server, tls, user, password): except Exception as err: raise click.ClickException(f'Can not connect to "{servername}": {err}') - if tls: + try: server.starttls() + except Exception as err: + raise click.ClickException(f'Could not STARTTLS with "{servername}": {err}') if user is not None: try: @@ -216,13 +218,9 @@ def convert(self, value, param, ctx): multiple=True, type=click.Path(exists=True, dir_okay=False, readable=True, path_type=pathlib.Path)) -### INTERNAL OPTIONS ### -# do not TLS encrypt connection to SMTP server (to be used in tests) -@click.option('--tls/--no-tls', default=True, hidden=True) - ### MAIN SCRIPT ### def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, - user, password, attachment, tls): + user, password, attachment): """Send mass mail Example: @@ -251,5 +249,5 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, """ keys = parse_parameter_file(parameter_file) msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, attachment) - send_messages(msgs, server, tls, user, password) + send_messages(msgs, server, user, password) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index e851ac9..9e2b11f 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -7,17 +7,112 @@ import click.testing import pytest +# 4096 bit, valid until year 2123 +# generated with: +# openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 36500 -nodes -subj '/CN=localhost' +TLS_KEY="""-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDL+HYMRZ21L8Eh +aR0wBgiK9ww/51iNYSKVffXphnt77B1o5A80m98B3v9epY48jE+Cbytl9PxlWHcn +jlGWqO/xSFiETmJokRcBr1s8Ij0tZYc2PuPwpLm+wN3rn1JX4wuskSXaxnDtUgZg +t7GYIsVxpg5qIdvMiV/68sho0pG9Y4LSw+mwy2rIXOwpCWuV0PKOq64++oK2P1N7 +x+B0iyWHvQd+SLKK7yLQr2Sh5iPq14KJkBe2mWjoujtGxo8Cp4aYE1nVAF1VV0Bx +Psp8xsKo6DEcljIw3MfI4gdYUYaxQPWmWb9141iMCoeCkAqrnSSUFdRf6QUhfesu +mNGyIx/d7iE5LsTT9i0kMdemM5R3b9IEwD6EXlJDOVKTZnw2tKqbBnzeIpeRF/qZ +HevOnhLc6AewJacEURFG2TNgkxCJ3SwFdzgIWbYz64J85Q8Bu3e0s4WUjl2XRgsa +IDAYn6tYWndMbGiHfjx60R153lIxHWAu+bAsV15WDI+s1wJT7RjG0Xs37WEcGhK4 +4eFdMtb9cKmWGFIFEhWx/6iFwRk/uPptE+QgMiuEkydUidw/6hQ0vJKnwFRCOcFa +ri4SrUgpgMjKsxNyAosGvi+cE7U7XmtbOOpKGeL1y05JFnUucaQfuCLzaiiI1V/R +HlaeU0ZNk3EQl09coNCzJtjxhHDvcwIDAQABAoICAFoP48Poa0npA5xmhuJBD72Y +dvqygnmupbAfdZk+7cBakePSK1qd5pqzZcvbSxI2HBdqUd3LjjSLmtVG9ISTJJtD +x/3nhHFKez+dt6m2LpAgb9MGcuw7N97f1z1mVFwFHw776hyPGabYXIORKKQV2luj +qGK5f41xLQWn9NDABWT8DvRUWBfdwdEloos+Ixh8MdXIPYCGaXfiP1D9AQFEvXYR +g8EBhYBuNc+yWjtYXIyhyvxFuQrB8z4rmOfX3aac5QO6K6Su7Ac2Jvi58nuk4afm +GmVWdmP34Gk1UGvxV4lltvHUWANMNrljHtGKG4QKN3ABsYwF3mOa72DcTl8bPkKc +peQxa3ZD0BqGeZhD7vUBsLKi1lzk9vxPoAWHcQGFztUSkPu2Nzmi1SJmhu3rPNa/ +eCJD8TzSotVR7sph2ERb5qtTe3kv1vM3pRCGKr8/yx5BoYokzsT1vqDOmUQLN+YF +zopLAlmC2iyPwhYVL1UsfH8L1+acffmlXZ7G2+RXIRzyq3IOHVi499UvLnnhP2E3 +QQmPziKNr1zMuabg0kr9vDlToUY9r6VRlx/ioOs6QlfUBJc7B6/2ElEFupFR2TlF +Lo+JZPgmybE9kAg2dYtQMLoVn9LEkDH1XzXM+cZvIgox7hBCVUvoKO6woA93fLGd +YB71OFyw15FVzsQI+cyNAoIBAQDcFY3a9qPSs3vQahQ2PtYOEv3FcWSJsDvYX8Dh +EeSNu+qCJhzHTXCoFwI91L0IOiOWI2xxBor/5ma2pH5toRfx6Vw9ygskKGK3gPlO +OXkDAypVWky9is09yp80cyx+1M5f/USZfhphN3rQhqWRyvV5M75Ibe3Sis60llS1 +aIV4hmTNZ9sQ0jAGnbL8Ym5Yq55eNX/whEEi1xZSfAuho+ChEBkh8/syRKdTlERc +8EGUip7PIr1KqAxXembcYVP+MbmIPlypzL5Th7vsIj5eBbcLdrhdDY/sBq5730p3 +19pMye49fdkx33eNSkBe6eanUdy4a77VnL9yuItwRDJtdGRPAoIBAQDtQbpmOQgn +WMfUaUuTDIEm9DcCAuRp4IPEHdZ8GpmAgYpjhrKAVeF72OjNglyCtYBHQ3GLOFZt +TqtibPHg+WS6JeBeXJze7STP4IO3JgaspEbO5Rj8wJEwHTZn7BZIX85RJrUduMUQ +oxKgBtMsSrN1IUjHGgOz7NMQRP/j+Beiej1beP/8BJIhGkONlvN9hUM1V6jSM6qy +Vyzm/vJjkT11THe7J36oB23Bncf3hC8Ii14TTBEohHDAbEjSeu0ZQGMc81zKQuyI +zRatFGNKwh52LHyw9zKI0C9bfXUVXe8zNe4e9dpdmsKBx3NoRirYiT/d2YDcrnzh +DjAULX+hcCWdAoIBADaM0SBYybpL6oB6CpB3eq76XhQ2Sukl2W+ELFadDL1kuneP +4sozk5zWNyQEOuZzIqbwGMzbBlDvVr4mf3/E0h6P7OET1zcbG3zIZwLQlAH/ItsN +CsBgSwbp1hQ2B+1X6d8482voKbm2+qX8+cTtPXLRNHTXan8pEJsKN+zO/2YkSY/w +EghVULoTFG4iJ5+qyhInyJJg9ZQhI9NGE8v4xpClYNVdmAGZqq+4rEks89RRl5NX +1PtQM97q49vz89GpmYb/jhA4Q2SI3DdnNXYwjHI29vN5jRa/gTgK3HZf9ifaVUbA +jrkh3owSv2nHJ/iI/eBoNGDV/U3+F/G3tZgTpVkCggEBAIZjjrPMZkPzU+2LXxWC +Jb3s4yOug7c9RyXVSOKvJnfV6I+LgpyTCM/gA640wzX+nRTArRYQ6VOtFgMAdtna +KiYOwlJw3yKe7RUatUEOtwUfYERdHJQ+d37rbR/caJrCOdlZtYmKWYWc+TXP59nU +zmXwXor4v1QxNzSmANQeeTS9TPf9R/J2nFdHyy/uaymUTIdwid3XCj9Ohc6qZp3j +bQ5+K+vE6UdAPflH6DbZltKeLsF7etSagEteirk+jAKbqAiECPFAiz7J/Kg5Pizg +W+TQOij7PJKmaczG+YUK2i0FxUWgOPqAaOCeG07bP/W7eIOvagCWjYHlSXKEeyD0 +pzkCggEBAKHAvy4SKqDZdEZcOZPMEmRIPX+BO0Ld58RE5Guc/X4zAA+CjWmUT6Cn +aHT3YQ/6bKaLKBLmsthRfKEyOA6tPlYqsRWGLv62AK0ZaYqzTypd/NGNO8jCtP07 +IkNlwVYq+Bx2mwqnCZev7R2QeJSzUpAAIRlNt0oCP3yAaoJXY3ulluBZSgjwrds7 +w+ZLd6cs7RoOwMYxfHmKAy+rlcpkds+IDEj0Gxtzy+S9GjN6Yxve9eumN6cepmCn +/qQRqjk4wJOo0oH9JS62L+0b75NRoVa7iciW8Ao22+aZYpRFY+27Nh3pR7PnJsHM +sBuZkVRS+PJe57phZv06aFFrv2VcXXg= +-----END PRIVATE KEY----- +""" + +TLS_CERT="""-----BEGIN CERTIFICATE----- +MIIFCzCCAvOgAwIBAgIULLLAwLBYIJVgJ/39zhjZq+YnzkcwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTIzMDQxNjEyMjIyMFoYDzIxMjMw +MzIzMTIyMjIwWjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDL+HYMRZ21L8EhaR0wBgiK9ww/51iNYSKVffXphnt7 +7B1o5A80m98B3v9epY48jE+Cbytl9PxlWHcnjlGWqO/xSFiETmJokRcBr1s8Ij0t +ZYc2PuPwpLm+wN3rn1JX4wuskSXaxnDtUgZgt7GYIsVxpg5qIdvMiV/68sho0pG9 +Y4LSw+mwy2rIXOwpCWuV0PKOq64++oK2P1N7x+B0iyWHvQd+SLKK7yLQr2Sh5iPq +14KJkBe2mWjoujtGxo8Cp4aYE1nVAF1VV0BxPsp8xsKo6DEcljIw3MfI4gdYUYax +QPWmWb9141iMCoeCkAqrnSSUFdRf6QUhfesumNGyIx/d7iE5LsTT9i0kMdemM5R3 +b9IEwD6EXlJDOVKTZnw2tKqbBnzeIpeRF/qZHevOnhLc6AewJacEURFG2TNgkxCJ +3SwFdzgIWbYz64J85Q8Bu3e0s4WUjl2XRgsaIDAYn6tYWndMbGiHfjx60R153lIx +HWAu+bAsV15WDI+s1wJT7RjG0Xs37WEcGhK44eFdMtb9cKmWGFIFEhWx/6iFwRk/ +uPptE+QgMiuEkydUidw/6hQ0vJKnwFRCOcFari4SrUgpgMjKsxNyAosGvi+cE7U7 +XmtbOOpKGeL1y05JFnUucaQfuCLzaiiI1V/RHlaeU0ZNk3EQl09coNCzJtjxhHDv +cwIDAQABo1MwUTAdBgNVHQ4EFgQU3PuRab4JVPITf+ICU5HVOMpW0gIwHwYDVR0j +BBgwFoAU3PuRab4JVPITf+ICU5HVOMpW0gIwDwYDVR0TAQH/BAUwAwEB/zANBgkq +hkiG9w0BAQsFAAOCAgEAOkEwnaMyxc7NdJedwk5sl6yoP/yY/rKDKA1yCEwgolkF +k73F5VcaOcVhwoA/a2h8AwYOQTGv2YNZnRxLskooyKkLrbHoZn34/YO1kgaXAm/p +mtiVsdaoFxUrcE7GXkJDTz4cC8wHc8BFrK6xESk8FYl07DCjIK9kFBzTGLV234kR +OLpvFAoFbvVWlWDT3VLIGjP6EIp7JFE/neTpB8PguZ7dcNdgtoiWz19Pv3JYoWWj +DTSZXJgu6OUjVxWC0T1VBz/uv5n9GNEjLa23hdzhztigmcrOKgrX43cTzNhnu62R +6Tx1HUtkswvxNzhaEKt3TP6Jde7mP5J6DKi1HpfKuGDizFJzoEkmWQNfeCzWniw8 +W01qwU2xZ9AETQB2uAWoHF0vKRX8ZuzOBhK1vW9HVtSZFHJZ3OaS5/5TNvts8fvs +eK5DIjBOr2RQqw+V9JYoBPr3t8EUE1cBSI6THlkWMDvXm5wal0tZOF7HGjCxXnHY +Z7tS2M2dLHwsnOultNrtZn+zjSmmce2/YQzNqZcax5G9kTuU88OguqcDdB3pcWzY +U1iY1r9bTWpfIASnDpYz3RbE9jp5+ZgNjhA6l5lXk9/CHBQWyxL9d2Khp+3IV0t9 +TfvhHNbB/2KiW9Si3G4bfcyPrH60O9yCfmbLDUjRgH2UjbOIefWKcKyAz/ZA5Y0= +-----END CERTIFICATE----- +""" + # return a locally running SMTP server. This fixture kills the server after the # test run is finished, i.e. not after every test! # The resulting server runs at localhost:8025 @pytest.fixture(scope="module") -def server(): +def server(tmp_path_factory): + tlsdir = tmp_path_factory.mktemp('tlsdir') + key = tlsdir / 'key' + key.write_text(TLS_KEY) + cert = tlsdir / 'cert' + cert.write_text(TLS_CERT) server = subprocess.Popen([sys.executable, '-m', 'aiosmtpd', - '-n', - '-d', - '-l', 'localhost:8025', - '-c', 'aiosmtpd.handlers.Debugging', 'stderr'], + '--nosetuid', + '--debug', + '--tlscert', str(cert), + '--tlskey', str(key), + '--listen', 'localhost:8025', + '--class', 'aiosmtpd.handlers.Debugging', 'stderr'], stdin=None, text=False, stderr=subprocess.PIPE, @@ -105,9 +200,7 @@ def cli(server, parm, body, opts={}, opts_list=[], errs=False): # now we have all default options + options passed by the test # instantiate a click Runner script = click.testing.CliRunner() - # invoke the script, add the no-tls options (our SMTP does not support TLS) - # and do not ask for confirmation to send emails - result = script.invoke(massmail, opts + ['--no-tls'], input='y\n') + result = script.invoke(massmail, opts, input='y\n') if errs: # we expect errors, so do not interact with the SMTP server at all # and read the errors from the script instead From 340d532236692260eb8e841a653f1171beb19fac Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 15:04:00 +0200 Subject: [PATCH 39/47] do not allow for empty values in parameter file --- massmail/massmail.py | 8 +++++++- massmail/test_massmail.py | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index a2e9e85..84bc790 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -31,10 +31,16 @@ def parse_parameter_file(parameter_file): # ignore empty lines if len(line) == 0: continue - values = [key.strip() for key in line.split(';')] + values = [value.strip() for value in line.split(';')] if len(values) != len(key_list): raise click.ClickException(f'Line {count+1} in {name} malformed: ' f'{len(values)} found instead of {len(key_list)}') + for idx, value in enumerate(values): + # do not allow for empty values + if value == '': + raise click.ClickException(f'Line {count+1} in {name} malformed: ' + f'empty value for key {key_list[idx]}') + for i, key in enumerate(key_list): keys[key].append(values[i]) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 9e2b11f..3f49e01 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -343,13 +343,27 @@ def test_missing_email_in_parm(server, parm, body): test;test""") assert 'No $EMAIL$' in cli(server, parm, body, errs=True) -def test_missing_value_in_parm(server, parm, body): +def test_too_many_values_in_parm(server, parm, body): with parm.open('at') as parmf: parmf.write('\nMario;Rossi;j@monkeys.com;too much\n') output = cli(server, parm, body, errs=True) assert 'Line 2' in output assert '4 found instead of 3' in output +def test_missing_values_in_parm(server, parm, body): + with parm.open('at') as parmf: + parmf.write('\nMario;j@monkeys.com\n') + output = cli(server, parm, body, errs=True) + assert 'Line 2' in output + assert '2 found instead of 3' in output + +def test_empty_values_in_parm(server, parm, body): + with parm.open('at') as parmf: + parmf.write('\nMario;;j@monkeys.com\n') + output = cli(server, parm, body, errs=True) + assert 'Line 2' in output + assert 'empty value for key $SURNAME$' in output + def test_server_offline(server, parm, body): opts = {'--server' : 'noserver:25' } assert 'Can not connect to' in cli(server, parm, body, opts=opts, errs=True) From 117333bf03e665c30fd9da31b6a747f7e87de789 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 15:04:25 +0200 Subject: [PATCH 40/47] also print SMTP protocol log when unexpectedly failing a test --- massmail/test_massmail.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 3f49e01..622ec90 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -181,6 +181,7 @@ def parse_smtp(server): # wrapper for running the massmail script and parse the SMTP server output def cli(server, parm, body, opts={}, opts_list=[], errs=False): + # set default options options = { '--from' : 'Blushing Gorilla ', '--subject' : 'Invitation to the jungle', @@ -207,13 +208,14 @@ def cli(server, parm, body, opts={}, opts_list=[], errs=False): assert result.exit_code != 0 return result.output else: + # parse the output of the SMTP server which is running in the background + protocol, emails = parse_smtp(server) # in case of unexpected error, let's print the output so we have a chance # to debug without touching the code here if result.exit_code != 0: print(result.output) + print(protocol) assert result.exit_code == 0 - # parse the output of the SMTP server which is running in the background - protocol, emails = parse_smtp(server) # return the protocol text and a list of emails return protocol, emails From b066e337bba00893c5bb153e3b824b236070639e Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 15:48:50 +0200 Subject: [PATCH 41/47] add detailed explanation about the magic happening in re.sub --- massmail/massmail.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 84bc790..4a1b055 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -40,7 +40,6 @@ def parse_parameter_file(parameter_file): if value == '': raise click.ClickException(f'Line {count+1} in {name} malformed: ' f'empty value for key {key_list[idx]}') - for i, key in enumerate(key_list): keys[key].append(values[i]) @@ -75,6 +74,32 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, at for i, emails in enumerate(keys['$EMAIL$']): # find keywords and substitute with values + + # The following line does this: + # - Assume that the parameter file has an header like + # $NAME$;$SURNAME$;$EMAIL$ + # - Assume that line "i" from the parameter file look like this + # Gorilla; Blushing; email@gorillas.org + # - Then the following like is equivalent to having a loop over the keys + # from the parameter file: ($NAME$, $SURNAME$) + # - The m in the lambda is going to be the re.match object for $NAME$ at the + # first iteration, and the re.match object for $SURNAME$ at the second one + # - at every iteration, m.group(0) is the string of the matching key, i.e. + # it is "$NAME$" at the first iteration and "$SURNAME$" at the second one + # - keys[m.group(0)] is then equivalent, at the first iteration, to + # keys["$NAME$"] == ["Gorilla", "Othername1", "Othername2"] + # - keys[.group(0)][i] is then equivalent, at the first iteration, to + # keys["$NAME$"][i] == "Gorilla" + # if we assume that "Gorilla" is on line i + # - at the second iteration, we will have: + # keys[m.group(0)] == keys["$SURNAME$"] == ["Blushing", "Othersurname1",....] + # keys[m.group(0)][i] = keys["$SURNAME$"][i] == "Blushing" + # - we only have two iterations for re.sub, because we only have two keys + # in the body matching the regexp r'\$\w+\$' + # - so at the end all the values corresponding to the keys will be inserted + # into the body_text in place of the key + # - at the next iteration of i, we are going to select a different line from + # the parameter file, and generate a new email with different substitutions body = re.sub(r'\$\w+\$', lambda m: keys[m.group(0)][i], body_text) msg = email.message.EmailMessage() msg.set_content(body) From 628a92c37283920a2a5057cfd580150396e9ecb7 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 15:57:33 +0200 Subject: [PATCH 42/47] complain if an unknown key is found in the body text --- massmail/massmail.py | 5 ++++- massmail/test_massmail.py | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 4a1b055..51dc0cb 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -100,7 +100,10 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, at # into the body_text in place of the key # - at the next iteration of i, we are going to select a different line from # the parameter file, and generate a new email with different substitutions - body = re.sub(r'\$\w+\$', lambda m: keys[m.group(0)][i], body_text) + try: + body = re.sub(r'\$\w+\$', lambda m: keys[m.group(0)][i], body_text) + except KeyError as err: + raise click.ClickException(f'Unknown key in body file {body_file.name}: {err}') msg = email.message.EmailMessage() msg.set_content(body) msg['To'] = emails diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 622ec90..2eccf45 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -366,6 +366,14 @@ def test_empty_values_in_parm(server, parm, body): assert 'Line 2' in output assert 'empty value for key $SURNAME$' in output +def test_unknown_key_in_body(server, parm, body): + # add some unknown key to the body + with body.open('at') as bodyf: + bodyf.write('\n$UNKNOWN$\n') + output = cli(server, parm, body, errs=True) + assert 'Unknown key in body' in output + assert '$UNKNOWN$' in output + def test_server_offline(server, parm, body): opts = {'--server' : 'noserver:25' } assert 'Can not connect to' in cli(server, parm, body, opts=opts, errs=True) From 4b7e354244718170b93e537a1c7b679fda656f33 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 16:05:14 +0200 Subject: [PATCH 43/47] warn if no keys are found in the body text --- massmail/massmail.py | 5 +++++ massmail/test_massmail.py | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 51dc0cb..1b59c34 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -103,7 +103,12 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, at try: body = re.sub(r'\$\w+\$', lambda m: keys[m.group(0)][i], body_text) except KeyError as err: + # an unknown key was detected raise click.ClickException(f'Unknown key in body file {body_file.name}: {err}') + # warn if no keys were found + if body == body_text: + print('WARNING: no keys found in body file {body_file.name}') + msg = email.message.EmailMessage() msg.set_content(body) msg['To'] = emails diff --git a/massmail/test_massmail.py b/massmail/test_massmail.py index 2eccf45..bb7e160 100644 --- a/massmail/test_massmail.py +++ b/massmail/test_massmail.py @@ -180,7 +180,7 @@ def parse_smtp(server): return protocol, emails # wrapper for running the massmail script and parse the SMTP server output -def cli(server, parm, body, opts={}, opts_list=[], errs=False): +def cli(server, parm, body, opts={}, opts_list=[], errs=False, output=False): # set default options options = { '--from' : 'Blushing Gorilla ', @@ -216,8 +216,12 @@ def cli(server, parm, body, opts={}, opts_list=[], errs=False): print(result.output) print(protocol) assert result.exit_code == 0 - # return the protocol text and a list of emails - return protocol, emails + if output: + # return the protocol text and a list of emails and the output of the script + return protocol, emails, result.output + else: + # return the protocol text and a list of emails + return protocol, emails # just test that the cli is working and we get the right help text def test_help(): @@ -374,6 +378,15 @@ def test_unknown_key_in_body(server, parm, body): assert 'Unknown key in body' in output assert '$UNKNOWN$' in output +def test_no_key_in_body(server, parm, body): + # add some unknown key to the body + with body.open('wt') as bodyf: + bodyf.write('No keys here!\n') + protocol, emails, output = cli(server, parm, body, output=True) + assert 'WARNING: no keys found in body file' in output + assert 'No keys here!' in emails[0].get_content() + + def test_server_offline(server, parm, body): opts = {'--server' : 'noserver:25' } assert 'Can not connect to' in cli(server, parm, body, opts=opts, errs=True) From baba19fa0352742e5bd7d982c627f55cb6364c94 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 16:58:56 +0200 Subject: [PATCH 44/47] use rich for color and progress report --- massmail/massmail.py | 52 ++++++++++++++++++++++++++++---------------- pyproject.toml | 2 +- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 1b59c34..4f1151d 100644 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -4,6 +4,10 @@ import re import smtplib +import rich.prompt +import rich.panel +import rich.progress +from rich import print as rprint import click import email_validator @@ -59,7 +63,7 @@ def parse_parameter_file(parameter_file): def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, attachment): msgs = {} body_text = body_file.read() - + warned_once = False # collect attachments once and then attach them to every single message attachments = {} for path in attachment: @@ -106,8 +110,9 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, at # an unknown key was detected raise click.ClickException(f'Unknown key in body file {body_file.name}: {err}') # warn if no keys were found - if body == body_text: - print('WARNING: no keys found in body file {body_file.name}') + if body == body_text and not warned_once: + rprint(f'[bold][red]WARNING:[/red] no keys found in body file {body_file.name}[/bold]') + warned_once = True msg = email.message.EmailMessage() msg.set_content(body) @@ -132,20 +137,27 @@ def create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, at return msgs def send_messages(msgs, server, user, password): - for emailaddr in msgs: - emails = [e.strip() for e in emailaddr.split(',')] - print('This email will be sent to:', ', '.join(emails)) - for hdr, value in msgs[emailaddr].items(): - print(f'{hdr}: {value}') - for attachment in msgs[emailaddr].iter_attachments(): - name = attachment.get_filename() - content_type = attachment.get_content_type() - print(f'Attachment ({content_type}): {name}') - body = msgs[emailaddr].get_body().get_content() - print(f'\n{body}') + # print one example email for confirmation + ex_addr = list(msgs.keys())[0] + msg = msgs[ex_addr] + panel = [] + for hdr, value in msg.items(): + if hdr in ('From', 'Subject', 'Cc', 'Bcc', 'In-Reply-To'): + panel.append(f'[yellow]{hdr}[/yellow]: [red bold]{value}[/red bold]') + else: + panel.append(f'[yellow]{hdr}[/yellow]: {value}') + for attachment in msg.iter_attachments(): + name = attachment.get_filename() + content_type = attachment.get_content_type() + panel.append(f'[magenta]Attachment[/magenta] ([cyan]{content_type}[/cyan]): {name}') + body = msg.get_body().get_content() + panel.append(f'\n{body}') + rprint(rich.panel.Panel.fit('\n'.join(panel))) # ask for confirmation before really sending stuff - if not click.confirm('Send the emails above?', default=None): + rprint(f'[bold]About to send {len(msgs)} email messages like the one above…[/bold]') + if not rich.prompt.Confirm.ask(f'[bold]Send?[/bold]'): + #if not click.confirm('Send the emails above?', default=None): raise click.ClickException('Aborted! We did not send anything!') print() @@ -165,16 +177,18 @@ def send_messages(msgs, server, user, password): try: # get password if needed if password is None: - password = click.prompt(f'Enter password for {user} on {servername}', - hide_input=True) + password = rich.prompt.Prompt.ask(f'Enter password for ' + f'[bold]{user}[/bold] ' + f'on [bold]{servername}[/bold]', + password=True) server.login(user, password) except Exception as err: raise click.ClickException(f'Can not login to {servername}: {err}') print() - for emailaddr in msgs: - print('Sending email to:', emailaddr) + for emailaddr in rich.progress.track(msgs, description="[green]Sending:[/green]"): + rprint(f'Sending to: [bold]{emailaddr}[/bold]') try: out = server.send_message(msgs[emailaddr]) except Exception as err: diff --git a/pyproject.toml b/pyproject.toml index 45e82dd..544660e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Send mass mail" authors = [ { name="ASPP", email ="info@aspp.school" } ] license = { file = "LICENSE" } requires-python = ">=3.6" -dependencies = [ "aiosmtpd", "click", "email-validator", "pytest" ] +dependencies = [ "aiosmtpd", "click", "email-validator", "rich", "pytest" ] [project.scripts] massmail = "massmail.massmail:main" From 3830a9497c7d3b75f9beead275847118821b3b15 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Sun, 16 Apr 2023 17:36:59 +0200 Subject: [PATCH 45/47] allow for standalone running withoout installation --- massmail/massmail.py | 4 ++++ 1 file changed, 4 insertions(+) mode change 100644 => 100755 massmail/massmail.py diff --git a/massmail/massmail.py b/massmail/massmail.py old mode 100644 new mode 100755 index 4f1151d..03ba2bc --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import email import mimetypes import pathlib @@ -304,3 +305,6 @@ def main(fromh, subject, server, parameter_file, body_file, bcc, cc, inreply_to, msgs = create_email_bodies(body_file, keys, fromh, subject, cc, bcc, inreply_to, attachment) send_messages(msgs, server, user, password) +if __name__ == '__main__': + import sys + main(sys.argv[1:]) From 5df4a5ead414811d94263bd937be86cf02187f6f Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Thu, 22 Feb 2024 17:31:14 +0100 Subject: [PATCH 46/47] the malformed line count error did not take into account the header of the csv file --- massmail/massmail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index 03ba2bc..c0a6748 100755 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -38,7 +38,7 @@ def parse_parameter_file(parameter_file): continue values = [value.strip() for value in line.split(';')] if len(values) != len(key_list): - raise click.ClickException(f'Line {count+1} in {name} malformed: ' + raise click.ClickException(f'Line {count+2} in {name} malformed: ' f'{len(values)} found instead of {len(key_list)}') for idx, value in enumerate(values): # do not allow for empty values From 4239241d366a005fea9e7dd7ae61d093260272e9 Mon Sep 17 00:00:00 2001 From: Tiziano Zito Date: Tue, 20 Aug 2024 17:50:05 +0200 Subject: [PATCH 47/47] allow for empty values --- massmail/massmail.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/massmail/massmail.py b/massmail/massmail.py index c0a6748..3e7936e 100755 --- a/massmail/massmail.py +++ b/massmail/massmail.py @@ -40,11 +40,11 @@ def parse_parameter_file(parameter_file): if len(values) != len(key_list): raise click.ClickException(f'Line {count+2} in {name} malformed: ' f'{len(values)} found instead of {len(key_list)}') - for idx, value in enumerate(values): - # do not allow for empty values - if value == '': - raise click.ClickException(f'Line {count+1} in {name} malformed: ' - f'empty value for key {key_list[idx]}') + #for idx, value in enumerate(values): + # # do not allow for empty values + # if value == '': + # raise click.ClickException(f'Line {count+1} in {name} malformed: ' + # f'empty value for key {key_list[idx]}') for i, key in enumerate(key_list): keys[key].append(values[i])