fix: MX server WIP monorepo merge

.env.defaults

@@ -11,6 +11,14 @@ APPLE_ID_HASHED_PASSWORD=
 DKIM_DOMAIN_NAME={{WEB_HOST}}
 DKIM_KEY_SELECTOR=default
 DKIM_PRIVATE_KEY_PATH=
+#
+# instead of using DKIM_PRIVATE_KEY_PATH
+# you can use DKIM_PRIVATE_KEY_VALUE which
+# is the DKIM private key as a string with line breaks
+# (this is useful for tests and GitHub CI)
+# openssl genrsa -f4 -out private.key 2048
+#
+DKIM_PRIVATE_KEY_VALUE=
 
 ###############
 ## Microsoft ##
@@ -124,6 +132,7 @@ PREVIEW_EMAIL=true
 # HELPER_ENCRYPTION_KEY=xxxxx
 # SRS_SECRET=xxxxx
 EMAIL_ABUSE="abuse@{{WEB_HOST}}"
+EMAIL_FRIENDLY_FROM="no-reply@{{WEB_HOST}}"
 EMAIL_DEFAULT_FROM_EMAIL="support@{{WEB_HOST}}"
 EMAIL_DEFAULT_FROM="{{APP_NAME}} <{{EMAIL_DEFAULT_FROM_EMAIL}}>"
 REMOVED_EMAIL_DOMAIN="removed.{{WEB_HOST}}"
@@ -319,6 +328,11 @@ TXT_ENCRYPTION_KEY=
 ###########################
 API_RESTRICTED_SYMBOL=API_RESTRICTED_SYMBOL
 
+########
+## mx ##
+########
+MX_PORT=2525
+
 ##########################
 ## smtp mirrored config ##
 ##########################
@@ -331,6 +345,8 @@ SMTP_TRANSPORT_SECURE=true
 SMTP_HOST=localhost
 SMTP_PORT=2587
 SMTP_MESSAGE_MAX_SIZE=50mb
+ALLOWLIST=
+DENYLIST=
 TRUTH_SOURCES=
 MAX_RECIPIENTS=50
 

.env.schema

@@ -11,6 +11,14 @@ APPLE_ID_HASHED_PASSWORD=
 DKIM_DOMAIN_NAME={{env.WEB_HOST}}
 DKIM_KEY_SELECTOR=default
 DKIM_PRIVATE_KEY_PATH=
+#
+# instead of using DKIM_PRIVATE_KEY_PATH
+# you can use DKIM_PRIVATE_KEY_VALUE which
+# is the DKIM private key as a string with line breaks
+# (this is useful for tests and GitHub CI)
+# openssl genrsa -f4 -out private.key 2048
+#
+DKIM_PRIVATE_KEY_VALUE=
 
 ###############
 ## Microsoft ##
@@ -126,6 +134,7 @@ SEND_EMAIL=
 PREVIEW_EMAIL=
 TRANSPORT_DEBUG=
 EMAIL_ABUSE=
+EMAIL_FRIENDLY_FROM=
 EMAIL_DEFAULT_FROM_EMAIL=
 EMAIL_DEFAULT_FROM=
 REMOVED_EMAIL_DOMAIN=
@@ -309,6 +318,11 @@ TXT_ENCRYPTION_KEY=
 ###########################
 API_RESTRICTED_SYMBOL=
 
+########
+## mx ##
+########
+MX_PORT=
+
 ##########################
 ## smtp mirrored config ##
 ##########################
@@ -322,6 +336,8 @@ SMTP_HOST=
 SMTP_PORT=
 SMTP_MESSAGE_MAX_SIZE=
 SMTP_EXCHANGE_DOMAINS=
+ALLOWLIST=
+DENYLIST=
 TRUTH_SOURCES=
 MAX_RECIPIENTS=
 

README.md

@@ -164,6 +164,7 @@ You can start any of the services using our pre-built commands to make it easy.
 | API          | `npm start api`    |          `4000`          | <http://localhost:4000> |
 | Bree         | `npm start bree`   |           None           | None                    |
 | SMTP         | `npm start smtp`   |          `2432`          | `telnet localhost 2432` |
+| MX           | `npm start mx`     |          `2525`          | `telnet localhost 2525` |
 | IMAP         | `npm start imap`   |          `2113`          | `telnet localhost 2113` |
 | POP3         | `npm start pop3`   |          `2115`          | `telnet localhost 2115` |
 | SQLite       | `npm start sqlite` |          `3456`          | `telnet localhost 3456` |

app/controllers/web/denylist.js

@@ -202,6 +202,14 @@ async function remove(ctx) {
     }
   }
 
+  // TODO: redo this to check permanent hard-coded denylist and email admins of this
+  // TODO: redo this to check permanent hard-coded denylist and email admins of this
+  // TODO: redo this to check permanent hard-coded denylist and email admins of this
+  // TODO: redo this to check permanent hard-coded denylist and email admins of this
+  // TODO: redo this to check permanent hard-coded denylist and email admins of this
+  // TODO: redo this to check permanent hard-coded denylist and email admins of this
+  // TODO: redo this to check permanent hard-coded denylist and email admins of this
+
   // if user is on free plan then send an email
   // with link for admins to /denylist?q=ctx.state.q
   if (ctx.state.user.group !== 'admin') {

app/models/emails.js

@@ -42,6 +42,7 @@ const emailHelper = require('#helpers/email');
 const env = require('#config/env');
 const getBlockedHashes = require('#helpers/get-blocked-hashes');
 const getErrorCode = require('#helpers/get-error-code');
+const getHeaders = require('#helpers/get-headers');
 const i18n = require('#helpers/i18n');
 const isCodeBug = require('#helpers/is-code-bug');
 const logger = require('#helpers/logger');
@@ -266,7 +267,10 @@ Emails.pre('validate', function (next) {
   try {
     if (!_.isObject(this.envelope)) throw Boom.badRequest('Envelope missing');
 
-    if (!isSANB(this.envelope.from) || !isEmail(this.envelope.from))
+    if (
+      !isSANB(this.envelope.from) ||
+      !isEmail(this.envelope.from, { ignore_max_length: true })
+    )
       throw Boom.badRequest('Envelope from missing');
 
     if (
@@ -927,7 +931,7 @@ Emails.statics.queue = async function (
   if (
     info.envelope.from === false ||
     typeof info.envelope.from !== 'string' ||
-    !isEmail(info.envelope.from)
+    !isEmail(info.envelope.from, { ignore_max_length: true })
   )
     throw Boom.forbidden(i18n.translateError('ENVELOPE_FROM_MISSING', locale));
 
@@ -1075,7 +1079,9 @@ Emails.statics.queue = async function (
   if (_.isObject(fromHeader) && Array.isArray(fromHeader.value)) {
     fromHeader.value = fromHeader.value.filter(
       (addr) =>
-        _.isObject(addr) && isSANB(addr.address) && isEmail(addr.address)
+        _.isObject(addr) &&
+        isSANB(addr.address) &&
+        isEmail(addr.address, { ignore_max_length: true })
     );
     if (fromHeader.value.length === 1)
       from = fromHeader.value[0].address.toLowerCase();
@@ -1133,7 +1139,7 @@ Emails.statics.queue = async function (
   //
   if (
     isSANB(info?.envelope?.from) &&
-    isEmail(info.envelope.from) &&
+    isEmail(info.envelope.from, { ignore_max_length: true }) &&
     config.supportEmail !== info.envelope.from.toLowerCase() &&
     //
     // we don't want to scan messages sent with our own SMTP service
@@ -1230,42 +1236,7 @@ Emails.statics.queue = async function (
     }
   }
 
-  const headers = {};
-  for (const headerLine of parsed.headerLines) {
-    const index = headerLine.line.indexOf(': ');
-    const header = parsed.headers.get(headerLine.key);
-    const key = headerLine.line.slice(0, index);
-    let value = headerLine.line.slice(index + 2);
-    if (header) {
-      switch (headerLine.key) {
-        case 'content-type':
-        case 'content-disposition':
-        case 'dkim-signature':
-        case 'subject':
-        case 'references':
-        case 'message-id':
-        case 'in-reply-to':
-        case 'priority':
-        case 'x-priority':
-        case 'x-msmail-priority':
-        case 'importance': {
-          if (isSANB(header?.value)) {
-            value = header.value;
-          } else if (isSANB(header)) {
-            value = header;
-          }
-
-          break;
-        }
-
-        default: {
-          break;
-        }
-      }
-    }
-
-    headers[key] = value;
-  }
+  const headers = await getHeaders(parsed);
 
   const status =
     _.isDate(domain.smtp_suspended_sent_at) || options?.isPending === true

app/views/faq/index.md

@@ -1588,7 +1588,7 @@ Email relies on the [SMTP protocol](https://en.wikipedia.org/wiki/Simple_Mail_Tr
 
 * `MAIL FROM` - This indicates the envelope mail from address of the email.  If a value is entered, it must be a valid RFC 5322 email address.  Empty values are permitted.  We [check for backscatter](#how-do-you-protect-against-backscatter) here, and we also check the MAIL FROM against our [denylist](#do-you-have-a-denylist).  We finally check senders that are not on the allowlist for rate limiting (see the section on [Rate Limiting](#do-you-have-rate-limiting) and [allowlist](#do-you-have-an-allowlist) for more information).
 
-* `RCPT TO` - This indicates the recipient(s) of the email.  These must be valid RFC 5322 email addresses.  We only permit up to 50 envelope recipients per message (this is different than the "To" header from an email).  We also check for a valid [Sender Rewriting Scheme](https://en.wikipedia.org/wiki/Sender_Rewriting_Scheme) ("SRS") address here to protect against spoofing with our SRS domain name.  Recipients provided that contain a "no-reply" address will receive a 553 error.  See the [complete list of "no-reply" addresses below](#what-are-no-reply-addresses).  We also check the recipient against our [denylist](#do-you-have-a-denylist).
+* `RCPT TO` - This indicates the recipient(s) of the email.  These must be valid RFC 5322 email addresses.  We only permit up to 50 envelope recipients per message (this is different than the "To" header from an email).  We also check for a valid [Sender Rewriting Scheme](https://en.wikipedia.org/wiki/Sender_Rewriting_Scheme) ("SRS") address here to protect against spoofing with our SRS domain name.  We also check the recipient against our [denylist](#do-you-have-a-denylist).
 
 * `DATA` - This is the core part of our service which processes an email.  See the section [How do you process an email for forwarding](#how-do-you-process-an-email-for-forwarding) below for more insight.
 
@@ -1618,10 +1618,10 @@ This section describes our process related to the SMTP protocol command `DATA` i
    * `X-Original-To` - the original `RCPT TO` email address for the message.
      * This header's value has `Bcc` header parsed addresses removed from it.
      * This is useful for determining where an email was originally delivered to.
-     * Existing value if any is preserved as `X-Original-Preserved-To`.
    * `X-ForwardEmail-Version` - the current [SemVer](https://semver.org/) version from `package.json` of our codebase.
    * `X-ForwardEmail-Session-ID` - a session ID value used for debug purposes (only applies in non-production environments).
    * `X-ForwardEmail-Sender` - a comma separated list containing the original envelope MAIL FROM address (if it was not blank), the reverse PTR client FQDN (if it exists), and the sender's IP address.
+   * `X-ForwardEmail-ID` - this is only applicable for outbound SMTP and correlates to the email ID stored in My Account → Emails
    * `X-Report-Abuse` - with a value of `[email protected]`.
    * `X-Report-Abuse-To` - with a value of `[email protected]`.
    * `X-Complaints-To` - with a value of `[email protected]`.
@@ -1691,8 +1691,6 @@ Our IP addresses are publicly available, [see this section below for more insigh
 
 ## What are no-reply addresses
 
-We do not forward emails to "no-reply" addresses, and any sender attempting to will receive a 553 error.
-
 Email usernames equal to any of the following (case-insensitive) are considered to be no-reply addresses:
 
 * `do-not-reply`

config/index.js

@@ -3,8 +3,9 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 
-const path = require('node:path');
+const fs = require('node:fs');
 const os = require('node:os');
+const path = require('node:path');
 
 const Axe = require('axe');
 const Boom = require('@hapi/boom');
@@ -16,6 +17,7 @@ const isSANB = require('is-string-and-not-blank');
 const manifestRev = require('manifest-rev');
 const ms = require('ms');
 const nodemailer = require('nodemailer');
+const tlds = require('tlds');
 const { Iconv } = require('iconv');
 const { boolean } = require('boolean');
 
@@ -181,9 +183,57 @@ const STRIPE_LOCALES = new Set([
   'zh-TW'
 ]);
 
+const POSTMASTER_USERNAMES = new Set([
+  'automailer',
+  'autoresponder',
+  'bounce',
+  'bounce-notification',
+  'bounce-notifications',
+  'bounces',
+  'e-bounce',
+  'ebounce',
+  'host-master',
+  'host.master',
+  'hostmaster',
+  'localhost',
+  'mail-daemon',
+  'mail.daemon',
+  'maildaemon',
+  'mailer',
+  'mailer-daemon',
+  'mailer.daemon',
+  'mailerdaemon',
+  'post-master',
+  'post.master',
+  'postmaster'
+  // NOTE: excluding these because we try to go by SAFE MODE
+  // <https://www.backscatterer.org/?target=usage>
+  // 'www',
+  // 'www-data'
+  // 'root',
+  // 'abuse',
+  // 'admin',
+  // 'admini',
+  // ...noReplyList
+]);
+
 const config = {
   ...metaConfig,
+
+  signatureData: {
+    signingDomain: env.DKIM_DOMAIN_NAME,
+    selector: env.DKIM_KEY_SELECTOR,
+    privateKey: isSANB(env.DKIM_PRIVATE_KEY_PATH)
+      ? fs.readFileSync(env.DKIM_PRIVATE_KEY_PATH, 'utf8')
+      : isSANB(env.DKIM_PRIVATE_KEY_VALUE)
+      ? env.DKIM_PRIVATE_KEY_VALUE
+      : undefined,
+    algorithm: 'rsa-sha256',
+    canonicalization: 'relaxed/relaxed'
+  },
+
   socketTimeout: ms('3m'),
+  POSTMASTER_USERNAMES,
   ubuntuTeamMapping: {
     'ubuntu.com': '~ubuntumembers',
     'kubuntu.org': '~kubuntu-members',
@@ -252,7 +302,24 @@ const config = {
       ? 1
       : os.cpus().length,
 
-  // truth sources (mirrors smtp)
+  allowlist: new Set(
+    _.isArray(env.ALLOWLIST)
+      ? env.ALLOWLIST.map((key) => key.toLowerCase().trim())
+      : isSANB(env.ALLOWLIST)
+      ? env.ALLOWLIST.split(',').map((key) => key.toLowerCase().trim())
+      : []
+  ),
+
+  fingerprintPrefix: 'f',
+
+  denylist: new Set(
+    _.isArray(env.DENYLIST)
+      ? env.DENYLIST.map((key) => key.toLowerCase().trim())
+      : isSANB(env.DENYLIST)
+      ? env.DENYLIST.split(',').map((key) => key.toLowerCase().trim())
+      : []
+  ),
+
   truthSources: new Set(
     _.isArray(env.TRUTH_SOURCES)
       ? env.TRUTH_SOURCES.map((key) => key.toLowerCase().trim())
@@ -261,6 +328,9 @@ const config = {
       : []
   ),
 
+  greylistTimeout: ms('5m'),
+  greylistTtlMs: ms('5d'),
+
   emailRetention: env.EMAIL_RETENTION,
   logRetention: env.LOG_RETENTION,
 
@@ -347,6 +417,7 @@ const config = {
   dkimKeySelector: 'forwardemail', // forwardemail._domainkey.example.com
   supportRequestMaxLength: env.SUPPORT_REQUEST_MAX_LENGTH,
   abuseEmail: env.EMAIL_ABUSE,
+  friendlyFromEmail: env.EMAIL_FRIENDLY_FROM,
   email: {
     preview: {
       open: env.PREVIEW_EMAIL,
@@ -1401,6 +1472,13 @@ const config = {
   }
 };
 
+// arbitrarily add domains to the denylist
+for (const tld of tlds) {
+  if (config.restrictedDomains.includes(tld)) continue;
+  // postline.* (e.g. "postline.ml")
+  config.denylist.add(`postline.${tld}`);
+}
+
 // sanity test against validDurations and durationMapping length
 if (config.validDurations.length !== Object.keys(config.durationMapping).length)
   throw new Error('validDurations and durationMapping must be aligned');

ecosystem-mx.json

@@ -0,0 +1,27 @@
+{
+  "apps": [
+    {
+      "name": "mx",
+      "script": "mx.js",
+      "max_restarts": 999,
+      "exec_mode": "cluster",
+      "wait_ready": true,
+      "instances": "max",
+      "pmx": false,
+      "env_production": {
+        "NODE_ENV": "production"
+      }
+    }
+  ],
+  "deploy": {
+    "production": {
+      "user": "deploy",
+      "host": ["138.197.213.185","104.248.224.170"],
+      "ref": "origin/master",
+      "repo": "[email protected]:forwardemail/forwardemail.net.git",
+      "path": "/var/www/production",
+      "pre-deploy": "git reset --hard",
+      "post-deploy": "pnpm install && NODE_ENV=production npm start build && pm2 startOrGracefulReload ecosystem-mx.json --env production --update-env"
+    }
+  }
+}