fix: email helper now uses Emails.queue instead of SMTP transport (wi…

…th smart fallback), added bounce webhook test, updated FAQ, fixed content/styling for bots

app/controllers/web/admin/inquiries.js

@@ -15,7 +15,7 @@ const previewEmail = require('preview-email');
 const nodemailer = require('nodemailer');
 const Axe = require('axe');
 
-const { Inquiries, Users } = require('#models');
+const { Emails, Inquiries, Users } = require('#models');
 const config = require('#config');
 const emailHelper = require('#helpers/email');
 
@@ -222,7 +222,7 @@ async function reply(ctx) {
 
   // rely on historical user.email or fall back to newer sender_email
   // for those sending direct emails instead of creating an inquiry
-  const email = user?.email ?? inquiry.sender_email;
+  const address = user?.email ?? inquiry.sender_email;
 
   const { body, files } = ctx.request;
 
@@ -242,25 +242,32 @@ async function reply(ctx) {
   const lastMessage = inquiry.messages[lastMessageIndex];
 
   // https://github.com/nodemailer/nodemailer/issues/1312#issuecomment-891237590
-  const info = await emailHelper({
+  const { email, info } = await emailHelper({
     template: 'inquiry-response',
     message: {
-      to: email,
+      to: address,
       cc: config.email.message.from,
       inReplyTo: lastMessage.reference,
       references: lastMessage.reference,
       subject: inquiry.subject,
       attachments: resolvedAttachments
     },
     locals: {
-      user: { email },
+      user: { email: address },
       inquiry,
       response: { message: body?.message }
     }
   });
 
-  const raw = await transporter.sendMail(info.originalMessage);
-  inquiry.messages.push({ raw: raw.message });
+  let raw;
+  if (email) {
+    raw = await Emails.getMessage(email.message);
+  } else {
+    const obj = await transporter.sendMail(info.originalMessage);
+    raw = obj.message;
+  }
+
+  inquiry.messages.push({ raw });
 
   await inquiry.save();
 
@@ -299,36 +306,44 @@ async function bulkReply(ctx) {
 
       // rely on historical user.email or fall back to newer sender_email
       // for those sending direct emails instead of creating an inquiry
-      const email = user?.email ?? inquiry.sender_email;
+      const address = user?.email ?? inquiry.sender_email;
 
       // if the user has multiple inquiries and we've just responded
       // in bulk to a previous message then let's skip the email
-      if (!repliedTo.has(email)) {
+      if (!repliedTo.has(address)) {
         // eslint-disable-next-line no-await-in-loop
-        const info = await emailHelper({
+        const { email, info } = await emailHelper({
           template: 'inquiry-response',
           message: {
-            to: email,
+            to: address,
             cc: config.email.message.from,
             inReplyTo: inquiry?.messages[inquiry.messages.length - 1] || '',
             references: inquiry.references,
             subject: inquiry.subject
           },
           locals: {
-            user: { email },
+            user: { email: address },
             inquiry,
             response: { message }
           }
         });
 
-        // eslint-disable-next-line no-await-in-loop
-        const raw = await transporter.sendMail(info?.originalMessage);
+        let raw;
+        if (email) {
+          // eslint-disable-next-line no-await-in-loop
+          raw = await Emails.getMessage(email.message);
+        } else {
+          // eslint-disable-next-line no-await-in-loop
+          const obj = await transporter.sendMail(info.originalMessage);
+          raw = obj.message;
+        }
+
+        inquiry.messages.push({ raw });
 
-        inquiry.messages.push({ raw: raw.message });
         // eslint-disable-next-line no-await-in-loop
         await inquiry.save();
 
-        repliedTo.add(email);
+        repliedTo.add(address);
       }
     }
   } catch (err) {

app/controllers/web/help.js

@@ -11,7 +11,7 @@ const nodemailer = require('nodemailer');
 const Axe = require('axe');
 
 const emailHelper = require('#helpers/email');
-const { Domains, Inquiries } = require('#models');
+const { Domains, Emails, Inquiries } = require('#models');
 const config = require('#config');
 
 const transporter = nodemailer.createTransport({
@@ -61,7 +61,7 @@ async function help(ctx) {
       user.plan === 'free' ? '' : 'Premium Support: '
     }${ctx.translate('YOUR_HELP_REQUEST')} #${inquiry.reference}`;
 
-    const email = await emailHelper({
+    const { email, info } = await emailHelper({
       template: 'inquiry',
       message: {
         to: ctx.state.user[config.userFields.fullEmail],
@@ -75,12 +75,18 @@ async function help(ctx) {
       }
     });
 
-    const info = await transporter.sendMail(email.originalMessage);
+    let raw;
+    if (email) {
+      raw = await Emails.getMessage(email.message);
+    } else {
+      const obj = await transporter.sendMail(info.originalMessage);
+      raw = obj.message;
+    }
 
     inquiry.original_message = body.message;
     inquiry.subject = subject;
     inquiry.messages.push({
-      raw: info.message,
+      raw,
       text: body.message
     });
     await inquiry.save();

app/models/domains.js

@@ -951,6 +951,8 @@ Domains.pre('validate', function (next) {
 //
 Domains.pre('save', async function (next) {
   if (!isSANB(this.bounce_webhook)) return next();
+  if (config.env === 'test' && this.bounce_webhook.endsWith('?test=true'))
+    return next();
   try {
     if (!isPrivateIP) await pWaitFor(() => Boolean(isPrivateIP));
     const value = this.bounce_webhook

app/models/emails.js

@@ -841,7 +841,8 @@ Emails.statics.getMessage = async function (obj, returnString = false) {
   return raw;
 };
 
-// options.message (Buffer or nodemailer Object)
+// options.info (Nodemailer transport response *optional*)
+// options.message (Buffer or nodemailer Object) (required if `options.info` not provided)
 // options.alias
 // options.domain
 // options.user (from `ctx.state.user` or `alias.user`)
@@ -858,7 +859,8 @@ Emails.statics.queue = async function (
   //       nodemailer does not abort opened and not finished stream
   //
 
-  const info = await transporter.sendMail(options.message);
+  // this allow us to pass an already created nodemailer transport stream for parsing
+  const info = options?.info || (await transporter.sendMail(options.message));
 
   const messageSplitter = new MessageSplitter({
     maxBytes: MAX_BYTES

app/views/admin/domains/_table.pug

@@ -30,7 +30,7 @@ include ../../_pagination
       tbody
         if domains.length === 0
           tr
-            td.alert.alert-info(colspan="11")
+            td.alert.alert-info(colspan="12")
               = t("No domains exist for that search.")
         else
           each domain in domains

app/views/docs/index.pug

@@ -7,7 +7,7 @@ block body
     .container
       .row
         .col-12
-          .text-center.text-white
+          .text-center(class=isBot(ctx.get("User-Agent")) ? "" : "text-white")
             h1= t("Free Email Developer Tools and Resources")
             p!= t("Developer-focused email API, tools, and resources to send email, trigger webhooks, forward messages, and more.")
             include ../_author

app/views/faq/index.md

@@ -2763,7 +2763,7 @@ Note that if you upgrade or downgrade between paid plans within a 30-day window
   <strong class="font-weight-bold">
     Tip:
   </strong>
-    Looking for documentation on email webhooks?  See <a href="/faq#do-you-support-webhooks">Do you support webhooks?</a> for more insight.
+    Looking for documentation on email webhooks?  See <a href="/faq#do-you-support-webhooks" class="alert-link">Do you support webhooks?</a> for more insight.
   <span>
   </span>
 </div>
@@ -2837,7 +2837,7 @@ Here are a few additional notes regarding bounce webhooks:
   <strong class="font-weight-bold">
     Tip:
   </strong>
-    Looking for documentation on bounce webhooks?  See <a href="/faq#do-you-support-bounce-webhooks">Do you support bounce webhooks?</a> for more insight.
+    Looking for documentation on bounce webhooks?  See <a href="/faq#do-you-support-bounce-webhooks" class="alert-link">Do you support bounce webhooks?</a> for more insight.
   <span>
   </span>
 </div>

app/views/home.pug

@@ -208,14 +208,15 @@ block body
                     = " "
                     = t("Developers")
 
-    a.text-decoration-none.pt-1.pb-4.pt-lg-4.text-uppercase.text-white.mx-auto.bounce-animation(
-      href="#learn-more"
-    )
-      i.fa.fa-angle-double-down
-      = " "
-      = t("Learn more")
-      = " "
-      i.fa.fa-angle-double-down
+    if !isBot(ctx.get('User-Agent'))
+      a.text-decoration-none.pt-1.pb-4.pt-lg-4.text-uppercase.text-white.mx-auto.bounce-animation(
+        href="#learn-more"
+      )
+        i.fa.fa-angle-double-down
+        = " "
+        = t("Learn more")
+        = " "
+        i.fa.fa-angle-double-down
   #learn-more.bg-dark.text-white.py-3.d-block.overflow-hidden.no-search(
     data-ignore-hash-change
   )

app/views/pricing.pug

@@ -132,15 +132,15 @@ block body
                       i.fa.fa-code
                       = " "
                       = t("Developers")
-      a.text-decoration-none.pt-1.pb-4.pt-lg-4.text-uppercase.text-white.mx-auto.bounce-animation(
-        href="#learn-more"
-      )
-        i.fa.fa-angle-double-down
-        = " "
-        = t("Learn more")
-        = " "
-        i.fa.fa-angle-double-down
-
+      if !isBot(ctx.get('User-Agent'))
+        a.text-decoration-none.pt-1.pb-4.pt-lg-4.text-uppercase.text-white.mx-auto.bounce-animation(
+          href="#learn-more"
+        )
+          i.fa.fa-angle-double-down
+          = " "
+          = t("Learn more")
+          = " "
+          i.fa.fa-angle-double-down
     #learn-more.bg-dark.text-white.py-3.d-block.overflow-hidden.min-vh-100.no-search(
       data-ignore-hash-change
     )

config/index.js

@@ -1546,13 +1546,8 @@ config.views.locals.config = _.pick(config, [
 // <https://nodemailer.com/transports/>
 // <https://github.com/nodemailer/nodemailer/pull/1539>
 config.email.transport = nodemailer.createTransport({
-  host: env.SMTP_TRANSPORT_HOST,
-  port: env.SMTP_TRANSPORT_PORT,
-  secure: env.SMTP_TRANSPORT_SECURE,
-  auth: {
-    user: env.SMTP_TRANSPORT_USER,
-    pass: env.SMTP_TRANSPORT_PASS
-  },
+  streamTransport: true,
+  buffer: false,
   logger,
   debug: boolean(env.TRANSPORT_DEBUG)
 });

helpers/email.js

@@ -3,28 +3,100 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const Email = require('email-templates');
+const EmailTemplates = require('email-templates');
 const _ = require('lodash');
+const mongoose = require('mongoose');
+const nodemailer = require('nodemailer');
 const striptags = require('striptags');
+const { boolean } = require('boolean');
 const { decode } = require('html-entities');
 
 const getEmailLocals = require('./get-email-locals');
 const logger = require('./logger');
+
 const config = require('#config');
+const env = require('#config/env');
+
+const conn = mongoose.connections.find(
+  (conn) => conn[Symbol.for('connection.name')] === 'MONGO_URI'
+);
+if (!conn) {
+  throw new Error('Mongoose connection does not exist');
+}
+
+const emailConn = mongoose.connections.find(
+  (conn) => conn[Symbol.for('connection.name')] === 'EMAILS_MONGO_URI'
+);
+if (!emailConn) throw new Error('Mongoose connection does not exist');
 
-const email = new Email(config.email);
+const emailTemplates = new EmailTemplates(config.email);
+const emailTemplatesFallback = new EmailTemplates({
+  ...config.email,
+  transport: nodemailer.createTransport({
+    host: env.SMTP_TRANSPORT_HOST,
+    port: env.SMTP_TRANSPORT_PORT,
+    secure: env.SMTP_TRANSPORT_SECURE,
+    auth: {
+      user: env.SMTP_TRANSPORT_USER,
+      pass: env.SMTP_TRANSPORT_PASS
+    },
+    logger,
+    debug: boolean(env.TRANSPORT_DEBUG)
+  })
+});
 
 // TODO: email-templates should strip tags from HTML in subject
 module.exports = async (data) => {
   try {
+    if (
+      !conn?.models?.Users ||
+      !conn?.models?.Domains ||
+      !emailConn?.models?.Emails
+    )
+      throw new TypeError('Models were not available');
+
     logger.info('sending email', { data });
     if (!_.isObject(data.locals)) data.locals = {};
     const emailLocals = await getEmailLocals();
     Object.assign(data.locals, emailLocals);
     if (data?.message?.subject)
       data.message.subject = decode(striptags(data.message.subject));
-    const info = await email.send(data);
-    return info;
+
+    const adminIds = await conn.models.Users.distinct('_id', {
+      group: 'admin'
+    });
+
+    const domain = await conn.models.Domains.findOne({
+      name: env.WEB_HOST,
+      'members.user': { $in: adminIds },
+      has_txt_record: true
+    }).populate(
+      'members.user',
+      `id plan ${config.userFields.isBanned} ${config.userFields.hasVerifiedEmail} ${config.userFields.planExpiresAt} ${config.userFields.stripeSubscriptionID} ${config.userFields.paypalSubscriptionID}`
+    );
+
+    // alert admins they can configure and verify domain for faster email queueing
+    if (!domain) {
+      const err = new TypeError(
+        `Configure and verify a new admin-owned domain for ${env.WEB_HOST} for faster email queuing`
+      );
+      err.isCodeBug = true;
+      logger.fatal(err);
+    }
+
+    // info.message is a stream
+    const info = domain
+      ? emailTemplates.send(data)
+      : emailTemplatesFallback.send(data);
+    let email = null;
+
+    if (domain)
+      email = await emailConn.models.Emails.queue({
+        info,
+        domain,
+        catchall: true
+      });
+    return { info, email };
   } catch (err) {
     logger.error(err, { data });
     throw err;