From df85d4432b0c2544ae069de94a021414975a1dc7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Fri, 2 Jun 2023 12:57:55 +0200 Subject: [PATCH] Add support for AWS SES as mail sending provider This commits adds the support for AWS Simple Email Service as an alternative transport for sending notification emails from nextflow. The email is sent via AWS SES when using the nf-amazon plugin, and: - the NXF_ENABLE_AWS_SES=true environment variable is set - or, not `mail.smtp` setting is provided and the AWS_REGION or AWS_DEFAULT_REGION is set in the launching environment Signed-off-by: Paolo Di Tommaso --- .../nextflow/mail/BaseMailProvider.groovy | 69 ++++++++++ .../nextflow/mail/JavaMailProvider.groovy | 58 ++++++++ .../groovy/nextflow/mail/MailProvider.groovy | 37 +++++ .../main/groovy/nextflow/mail/Mailer.groovy | 127 +++++++----------- .../nextflow/mail/SendMailProvider.groovy | 36 +++++ .../nextflow/mail/SimpleMailProvider.groovy | 36 +++++ .../nextflow/script/WorkflowNotifier.groovy | 2 +- .../main/resources/META-INF/extensions.idx | 3 + .../groovy/nextflow/mail/MailerTest.groovy | 16 ++- plugins/nf-amazon/build.gradle | 1 + .../cloud/aws/mail/AwsMailProvider.groovy | 71 ++++++++++ .../src/resources/META-INF/extensions.idx | 1 + 12 files changed, 369 insertions(+), 88 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/mail/BaseMailProvider.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/mail/JavaMailProvider.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/mail/MailProvider.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/mail/SendMailProvider.groovy create mode 100644 modules/nextflow/src/main/groovy/nextflow/mail/SimpleMailProvider.groovy create mode 100644 plugins/nf-amazon/src/main/nextflow/cloud/aws/mail/AwsMailProvider.groovy diff --git a/modules/nextflow/src/main/groovy/nextflow/mail/BaseMailProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/mail/BaseMailProvider.groovy new file mode 100644 index 0000000000..ee72e12d0c --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/mail/BaseMailProvider.groovy @@ -0,0 +1,69 @@ +/* + * Copyright 2020-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.mail + +import javax.mail.MessagingException +import javax.mail.internet.MimeMessage + +import groovy.transform.CompileStatic +import nextflow.util.Duration + +/** + * Base class or sending an email via sys command `mail` or `sendmail` + * + * @author Paolo Di Tommaso + */ +@CompileStatic +abstract class BaseMailProvider implements MailProvider { + + private long SEND_MAIL_TIMEOUT = 15_000 + + /** + * Send a email message by using system tool such as `sendmail` or `mail` + * + * @param message A {@link MimeMessage} object representing the email to send + */ + void send(MimeMessage message, Mailer mailer) { + final cmd = [name(), '-t'] + final proc = new ProcessBuilder() + .command(cmd) + .redirectErrorStream(true) + .start() + // pipe the message to the sendmail stdin + final stdout = new StringBuilder() + final stdin = proc.getOutputStream() + message.writeTo(stdin); + stdin.close() // <-- don't forget otherwise it hangs + // wait for the sending to complete + final consumer = proc.consumeProcessOutputStream(stdout) + proc.waitForOrKill(sendTimeout(mailer)) + def status = proc.exitValue() + if( status != 0 ) { + consumer.join() + throw new MessagingException("Unable to send mail message\n $mailer exit status: $status\n reported error: $stdout") + } + } + + private long sendTimeout(Mailer mailer) { + final timeout = mailer.config.sendMailTimeout as Duration + return timeout ? timeout.toMillis() : SEND_MAIL_TIMEOUT + } + + + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/mail/JavaMailProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/mail/JavaMailProvider.groovy new file mode 100644 index 0000000000..cc2c0c4e32 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/mail/JavaMailProvider.groovy @@ -0,0 +1,58 @@ +/* + * Copyright 2020-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.mail + +import javax.mail.internet.MimeMessage + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j + +/** + * Implements a send mail provider based on Java Mail API + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class JavaMailProvider implements MailProvider { + @Override + void send(MimeMessage message, Mailer mailer) { + if( !message.getAllRecipients() ) + throw new IllegalArgumentException("Missing mail message recipient") + + final transport = mailer.getSession().getTransport() + transport.connect(mailer.host, mailer.port as int, mailer.user, mailer.password) + log.trace("Connected to host=$mailer.host port=$mailer.port") + try { + transport.sendMessage(message, message.getAllRecipients()) + } + finally { + transport.close() + } + } + + @Override + String name() { + return 'javamail' + } + + @Override + boolean textOnly() { + return false + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/mail/MailProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/mail/MailProvider.groovy new file mode 100644 index 0000000000..a4316225d3 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/mail/MailProvider.groovy @@ -0,0 +1,37 @@ +/* + * Copyright 2020-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.mail + +import javax.mail.internet.MimeMessage + +import org.pf4j.ExtensionPoint + +/** + * Define a generic interface to send an email modelled as a Mime message object + * + * @author Paolo Di Tommaso + */ +interface MailProvider extends ExtensionPoint { + + void send(MimeMessage message, Mailer mailer) + + String name() + + boolean textOnly() + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/mail/Mailer.groovy b/modules/nextflow/src/main/groovy/nextflow/mail/Mailer.groovy index 7114098081..d7c09b13de 100644 --- a/modules/nextflow/src/main/groovy/nextflow/mail/Mailer.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/mail/Mailer.groovy @@ -15,6 +15,7 @@ */ package nextflow.mail + import javax.activation.DataHandler import javax.activation.URLDataSource import javax.mail.Message @@ -34,7 +35,7 @@ import groovy.transform.CompileStatic import groovy.transform.Memoized import groovy.util.logging.Slf4j import nextflow.io.LogOutputStream -import nextflow.util.Duration +import nextflow.plugin.Plugins import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.parser.Parser @@ -61,8 +62,6 @@ class Mailer { private final static Pattern HTML_PATTERN = Pattern.compile("("+TAG_START+".*"+TAG_END+")|("+TAG_SELF_CLOSING+")|("+HTML_ENTITY+")", Pattern.DOTALL ) - private long SEND_MAIL_TIMEOUT = 15_000 - private static String DEF_CHARSET = Charset.defaultCharset().toString() /** @@ -87,6 +86,8 @@ class Mailer { return this } + Map getConfig() { config } + protected String getSysMailer() { if( !fMailer ) fMailer = findSysMailer() @@ -200,7 +201,7 @@ class Mailer { getConfig('password') } - protected getConfig(String name ) { + protected getConfig(String name) { def key = "smtp.${name}" def value = config.navigate(key) if( !value ) { @@ -210,58 +211,6 @@ class Mailer { return value } - /** - * Send a email message by using the Java API - * - * @param message A {@link MimeMessage} object representing the email to send - */ - protected void sendViaJavaMail(MimeMessage message) { - if( !message.getAllRecipients() ) - throw new IllegalArgumentException("Missing mail message recipient") - - final transport = getSession().getTransport() - transport.connect(host, port as int, user, password) - log.trace("Connected to host=$host port=$port") - try { - transport.sendMessage(message, message.getAllRecipients()) - } - finally { - transport.close() - } - } - - protected long getSendTimeout() { - def timeout = config.sendMailTimeout as Duration - return timeout ? timeout.toMillis() : SEND_MAIL_TIMEOUT - } - - /** - * Send a email message by using system tool such as `sendmail` or `mail` - * - * @param message A {@link MimeMessage} object representing the email to send - */ - protected void sendViaSysMail(MimeMessage message) { - final mailer = getSysMailer() - final cmd = [mailer, '-t'] - final proc = new ProcessBuilder() - .command(cmd) - .redirectErrorStream(true) - .start() - // pipe the message to the sendmail stdin - final stdout = new StringBuilder() - final stdin = proc.getOutputStream() - message.writeTo(stdin); - stdin.close() // <-- don't forget otherwise it hangs - // wait for the sending to complete - final consumer = proc.consumeProcessOutputStream(stdout) - proc.waitForOrKill(sendTimeout) - def status = proc.exitValue() - if( status != 0 ) { - consumer.join() - throw new MessagingException("Unable to send mail message\n $mailer exit status: $status\n reported error: $stdout") - } - } - /** * @return A multipart mime message representing the mail message to send */ @@ -407,41 +356,55 @@ class Mailer { guessHtml(str) ? 'text/html' : 'text/plain' } - /** - * Send the mail given the provided config setting - */ - void send(Mail mail) { - log.trace "Mailer config: $config -- mail: $mail" + protected boolean detectAwsEnv() { + if( env.get('AWS_REGION') ) return true + if( env.get('AWS_DEFAULT_REGION') ) return true + return false + } + + protected MailProvider provider() { + // load all providers + final providers = Plugins.getExtensions(MailProvider) + // find the AWS provider + final awsProvider = providers.find(it -> it.name()=='aws-ses') + // check if it can use the aws provider + if( env.get('NXF_ENABLE_AWS_SES')=='true' ) { + if( awsProvider ) + return awsProvider + else + log.warn "Unable to load AWS Simple Email Service (SES) client" + } - // if the user provided required configuration - // send via Java Mail API if( config.containsKey('smtp') ) { - log.trace "Mailer send via `javamail`" - def msg = createMimeMessage(mail) - sendViaJavaMail(msg) - return + return providers.find(it -> it.name()=='javamail') } - final mailer = getSysMailer() - // otherwise fallback on system sendmail - if( mailer == 'sendmail' ) { - log.trace "Mailer send via `sendmail`" - def msg = createMimeMessage(mail) - sendViaSysMail(msg) - return + if( awsProvider && detectAwsEnv() ) { + return awsProvider } - if( mailer == 'mail' ) { - log.trace "Mailer send via `mail`" - def msg = createTextMessage(mail) - sendViaSysMail(msg) + // detect the mailer type + final type = getSysMailer() + return providers.find(it -> it.name()==type) + } + + /** + * Send the mail given the provided config setting + */ + void send(Mail mail) { + log.trace "Mailer config: $config -- mail: $mail" + + final p = provider() + if( p != null ) { + log.debug "Sending mail via `${p.name()}`" + final msg = p.textOnly() + ? createTextMessage(mail) + : createMimeMessage(mail) + p.send(msg, this) return } - String msg = (mailer - ? "Unknown system mail tool: $mailer" - : "Cannot send email message -- Make sure you have installed `sendmail` or `mail` program or configure a mail SMTP server in the nextflow config file" - ) + final msg = "Cannot send email message -- Make sure you have installed `sendmail` or `mail` program or configure a mail SMTP server in the nextflow config file" throw new IllegalArgumentException(msg) } diff --git a/modules/nextflow/src/main/groovy/nextflow/mail/SendMailProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/mail/SendMailProvider.groovy new file mode 100644 index 0000000000..7fc367dd37 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/mail/SendMailProvider.groovy @@ -0,0 +1,36 @@ +/* + * Copyright 2020-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.mail + +/** + * Send a mail using the `sendmail` sys tool + * + * @author Paolo Di Tommaso + */ +class SendMailProvider extends BaseMailProvider { + + @Override + String name() { + return 'sendmail' + } + + @Override + boolean textOnly() { + return false + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/mail/SimpleMailProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/mail/SimpleMailProvider.groovy new file mode 100644 index 0000000000..1ffd476b3b --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/mail/SimpleMailProvider.groovy @@ -0,0 +1,36 @@ +/* + * Copyright 2020-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.mail + +/** + * Send a mail using the `mail` sys tool + * + * @author Paolo Di Tommaso + */ +class SimpleMailProvider extends BaseMailProvider { + + @Override + String name() { + return 'mail' + } + + @Override + boolean textOnly() { + return true + } +} diff --git a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy index bf8a0b57ef..028e224dec 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/WorkflowNotifier.groovy @@ -74,7 +74,7 @@ class WorkflowNotifier { /** * Creates {@link Mailer} object that sends the actual email message * - * @param config The {@link Mailer} settings correspoding to the content of the {@code mail} configuration file scope + * @param config The {@link Mailer} settings corresponding to the content of the {@code mail} configuration file scope * @return A {@link Mailer} object */ protected Mailer createMailer(Map config) { diff --git a/modules/nextflow/src/main/resources/META-INF/extensions.idx b/modules/nextflow/src/main/resources/META-INF/extensions.idx index 860b8d7c60..73ade8f597 100644 --- a/modules/nextflow/src/main/resources/META-INF/extensions.idx +++ b/modules/nextflow/src/main/resources/META-INF/extensions.idx @@ -20,3 +20,6 @@ nextflow.secret.LocalSecretsProvider nextflow.cache.DefaultCacheFactory nextflow.scm.RepositoryFactory nextflow.container.resolver.DefaultContainerResolver +nextflow.mail.SendMailProvider +nextflow.mail.SimpleMailProvider +nextflow.mail.JavaMailProvider diff --git a/modules/nextflow/src/test/groovy/nextflow/mail/MailerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/mail/MailerTest.groovy index bfefcf8b6b..a5bb2276df 100644 --- a/modules/nextflow/src/test/groovy/nextflow/mail/MailerTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/mail/MailerTest.groovy @@ -160,14 +160,16 @@ class MailerTest extends Specification { def mailer = Spy(Mailer) def MSG = Mock(MimeMessage) def mail = new Mail() + def provider = Spy(new JavaMailProvider()) when: mailer.config = [smtp: [host:'foo.com'] ] mailer.send(mail) then: 0 * mailer.getSysMailer() >> null + 1 * mailer.provider() >> provider 1 * mailer.createMimeMessage(mail) >> MSG - 1 * mailer.sendViaJavaMail(MSG) >> null + 1 * provider.send(MSG, mailer) >> null } @@ -176,13 +178,15 @@ class MailerTest extends Specification { def mailer = Spy(Mailer) def MSG = Mock(MimeMessage) def mail = new Mail() + and: + def provider = Spy(new SendMailProvider()) when: mailer.send(mail) then: - 1 * mailer.getSysMailer() >> 'sendmail' + 1 * mailer.provider() >> provider 1 * mailer.createMimeMessage(mail) >> MSG - 1 * mailer.sendViaSysMail(MSG) >> null + 1 * provider.send(MSG, mailer) >> null } def 'should throw an exception' () { @@ -201,12 +205,14 @@ class MailerTest extends Specification { def mailer = Spy(Mailer) def MSG = Mock(MimeMessage) def mail = new Mail() + and: + def provider = Spy(new SimpleMailProvider()) when: mailer.send(mail) then: - 1 * mailer.getSysMailer() >> 'mail' + 1 * mailer.provider() >> provider 1 * mailer.createTextMessage(mail) >> MSG - 1 * mailer.sendViaSysMail(MSG) >> null + 1 * provider.send(MSG, mailer) >> null } diff --git a/plugins/nf-amazon/build.gradle b/plugins/nf-amazon/build.gradle index 9884a66e3e..8abde02593 100644 --- a/plugins/nf-amazon/build.gradle +++ b/plugins/nf-amazon/build.gradle @@ -46,6 +46,7 @@ dependencies { api ('com.amazonaws:aws-java-sdk-logs:1.12.429') api ('com.amazonaws:aws-java-sdk-codecommit:1.12.429') api ('com.amazonaws:aws-java-sdk-sts:1.12.429') + api ('com.amazonaws:aws-java-sdk-ses:1.12.429') constraints { api 'com.fasterxml.jackson.core:jackson-databind:2.12.7.1' diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/mail/AwsMailProvider.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/mail/AwsMailProvider.groovy new file mode 100644 index 0000000000..cc18cd7ba9 --- /dev/null +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/mail/AwsMailProvider.groovy @@ -0,0 +1,71 @@ +/* + * Copyright 2020-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.cloud.aws.mail + +import javax.mail.internet.MimeMessage +import java.nio.ByteBuffer + +import com.amazonaws.services.simpleemail.AmazonSimpleEmailService +import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClientBuilder +import com.amazonaws.services.simpleemail.model.RawMessage +import com.amazonaws.services.simpleemail.model.SendRawEmailRequest +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.mail.MailProvider +import nextflow.mail.Mailer + +/** + * Send a mime message via AWS SES raw API + * + * https://docs.aws.amazon.com/ses/latest/dg/send-email-raw.html + * + * @author Paolo Di Tommaso + */ +@CompileStatic +@Slf4j +class AwsMailProvider implements MailProvider { + + @Override + String name() { + return 'aws-ses' + } + + @Override + boolean textOnly() { + return false + } + + @Override + void send(MimeMessage message, Mailer mailer) { + final client = getEmailClient() + // dump the message to a buffer + final outputStream = new ByteArrayOutputStream() + message.writeTo(outputStream) + // send the email + final rawMessage = new RawMessage(ByteBuffer.wrap(outputStream.toByteArray())) + final result = client.sendRawEmail(new SendRawEmailRequest(rawMessage)); + log.debug "Mail message sent: ${result}" + } + + AmazonSimpleEmailService getEmailClient() { + return AmazonSimpleEmailServiceClientBuilder + .standard() + .build() + } + +} diff --git a/plugins/nf-amazon/src/resources/META-INF/extensions.idx b/plugins/nf-amazon/src/resources/META-INF/extensions.idx index 369aaedcf4..8f769f6f23 100644 --- a/plugins/nf-amazon/src/resources/META-INF/extensions.idx +++ b/plugins/nf-amazon/src/resources/META-INF/extensions.idx @@ -18,3 +18,4 @@ nextflow.cloud.aws.batch.AwsBatchExecutor nextflow.cloud.aws.util.S3PathSerializer nextflow.cloud.aws.util.S3PathFactory nextflow.cloud.aws.fusion.AwsFusionEnv +nextflow.cloud.aws.mail.AwsMailProvider