diff --git a/.env.defaults b/.env.defaults index 890c4bb795..d12f5b377b 100644 --- a/.env.defaults +++ b/.env.defaults @@ -11,6 +11,14 @@ APPLE_ID_HASHED_PASSWORD= DKIM_DOMAIN_NAME={{WEB_HOST}} DKIM_KEY_SELECTOR=default DKIM_PRIVATE_KEY_PATH= +# +# instead of using DKIM_PRIVATE_KEY_PATH +# you can use DKIM_PRIVATE_KEY_VALUE which +# is the DKIM private key as a string with line breaks +# (this is useful for tests and GitHub CI) +# openssl genrsa -f4 -out private.key 2048 +# +DKIM_PRIVATE_KEY_VALUE= ############### ## Microsoft ## @@ -124,6 +132,7 @@ PREVIEW_EMAIL=true # HELPER_ENCRYPTION_KEY=xxxxx # SRS_SECRET=xxxxx EMAIL_ABUSE="abuse@{{WEB_HOST}}" +EMAIL_FRIENDLY_FROM="no-reply@{{WEB_HOST}}" EMAIL_DEFAULT_FROM_EMAIL="support@{{WEB_HOST}}" EMAIL_DEFAULT_FROM="{{APP_NAME}} <{{EMAIL_DEFAULT_FROM_EMAIL}}>" REMOVED_EMAIL_DOMAIN="removed.{{WEB_HOST}}" @@ -319,6 +328,11 @@ TXT_ENCRYPTION_KEY= ########################### API_RESTRICTED_SYMBOL=API_RESTRICTED_SYMBOL +######## +## mx ## +######## +MX_PORT=2525 + ########################## ## smtp mirrored config ## ########################## @@ -331,6 +345,8 @@ SMTP_TRANSPORT_SECURE=true SMTP_HOST=localhost SMTP_PORT=2587 SMTP_MESSAGE_MAX_SIZE=50mb +ALLOWLIST= +DENYLIST= TRUTH_SOURCES= MAX_RECIPIENTS=50 diff --git a/.env.schema b/.env.schema index 1ce1515221..defa6a5007 100644 --- a/.env.schema +++ b/.env.schema @@ -11,6 +11,14 @@ APPLE_ID_HASHED_PASSWORD= DKIM_DOMAIN_NAME={{env.WEB_HOST}} DKIM_KEY_SELECTOR=default DKIM_PRIVATE_KEY_PATH= +# +# instead of using DKIM_PRIVATE_KEY_PATH +# you can use DKIM_PRIVATE_KEY_VALUE which +# is the DKIM private key as a string with line breaks +# (this is useful for tests and GitHub CI) +# openssl genrsa -f4 -out private.key 2048 +# +DKIM_PRIVATE_KEY_VALUE= ############### ## Microsoft ## @@ -126,6 +134,7 @@ SEND_EMAIL= PREVIEW_EMAIL= TRANSPORT_DEBUG= EMAIL_ABUSE= +EMAIL_FRIENDLY_FROM= EMAIL_DEFAULT_FROM_EMAIL= EMAIL_DEFAULT_FROM= REMOVED_EMAIL_DOMAIN= @@ -309,6 +318,11 @@ TXT_ENCRYPTION_KEY= ########################### API_RESTRICTED_SYMBOL= +######## +## mx ## +######## +MX_PORT= + ########################## ## smtp mirrored config ## ########################## @@ -322,6 +336,8 @@ SMTP_HOST= SMTP_PORT= SMTP_MESSAGE_MAX_SIZE= SMTP_EXCHANGE_DOMAINS= +ALLOWLIST= +DENYLIST= TRUTH_SOURCES= MAX_RECIPIENTS= diff --git a/README.md b/README.md index ab636c88de..c4998c415d 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ You can start any of the services using our pre-built commands to make it easy. | API | `npm start api` | `4000` | | | Bree | `npm start bree` | None | None | | SMTP | `npm start smtp` | `2432` | `telnet localhost 2432` | +| MX | `npm start mx` | `2525` | `telnet localhost 2525` | | IMAP | `npm start imap` | `2113` | `telnet localhost 2113` | | POP3 | `npm start pop3` | `2115` | `telnet localhost 2115` | | SQLite | `npm start sqlite` | `3456` | `telnet localhost 3456` | diff --git a/app/controllers/web/denylist.js b/app/controllers/web/denylist.js index d51e7d13e8..1c90b1917c 100644 --- a/app/controllers/web/denylist.js +++ b/app/controllers/web/denylist.js @@ -202,6 +202,14 @@ async function remove(ctx) { } } + // TODO: redo this to check permanent hard-coded denylist and email admins of this + // TODO: redo this to check permanent hard-coded denylist and email admins of this + // TODO: redo this to check permanent hard-coded denylist and email admins of this + // TODO: redo this to check permanent hard-coded denylist and email admins of this + // TODO: redo this to check permanent hard-coded denylist and email admins of this + // TODO: redo this to check permanent hard-coded denylist and email admins of this + // TODO: redo this to check permanent hard-coded denylist and email admins of this + // if user is on free plan then send an email // with link for admins to /denylist?q=ctx.state.q if (ctx.state.user.group !== 'admin') { diff --git a/app/models/emails.js b/app/models/emails.js index 05ac241c0b..9fadc534f6 100644 --- a/app/models/emails.js +++ b/app/models/emails.js @@ -42,6 +42,7 @@ const emailHelper = require('#helpers/email'); const env = require('#config/env'); const getBlockedHashes = require('#helpers/get-blocked-hashes'); const getErrorCode = require('#helpers/get-error-code'); +const getHeaders = require('#helpers/get-headers'); const i18n = require('#helpers/i18n'); const isCodeBug = require('#helpers/is-code-bug'); const logger = require('#helpers/logger'); @@ -266,7 +267,10 @@ Emails.pre('validate', function (next) { try { if (!_.isObject(this.envelope)) throw Boom.badRequest('Envelope missing'); - if (!isSANB(this.envelope.from) || !isEmail(this.envelope.from)) + if ( + !isSANB(this.envelope.from) || + !isEmail(this.envelope.from, { ignore_max_length: true }) + ) throw Boom.badRequest('Envelope from missing'); if ( @@ -927,7 +931,7 @@ Emails.statics.queue = async function ( if ( info.envelope.from === false || typeof info.envelope.from !== 'string' || - !isEmail(info.envelope.from) + !isEmail(info.envelope.from, { ignore_max_length: true }) ) throw Boom.forbidden(i18n.translateError('ENVELOPE_FROM_MISSING', locale)); @@ -1075,7 +1079,9 @@ Emails.statics.queue = async function ( if (_.isObject(fromHeader) && Array.isArray(fromHeader.value)) { fromHeader.value = fromHeader.value.filter( (addr) => - _.isObject(addr) && isSANB(addr.address) && isEmail(addr.address) + _.isObject(addr) && + isSANB(addr.address) && + isEmail(addr.address, { ignore_max_length: true }) ); if (fromHeader.value.length === 1) from = fromHeader.value[0].address.toLowerCase(); @@ -1133,7 +1139,7 @@ Emails.statics.queue = async function ( // if ( isSANB(info?.envelope?.from) && - isEmail(info.envelope.from) && + isEmail(info.envelope.from, { ignore_max_length: true }) && config.supportEmail !== info.envelope.from.toLowerCase() && // // we don't want to scan messages sent with our own SMTP service @@ -1230,42 +1236,7 @@ Emails.statics.queue = async function ( } } - const headers = {}; - for (const headerLine of parsed.headerLines) { - const index = headerLine.line.indexOf(': '); - const header = parsed.headers.get(headerLine.key); - const key = headerLine.line.slice(0, index); - let value = headerLine.line.slice(index + 2); - if (header) { - switch (headerLine.key) { - case 'content-type': - case 'content-disposition': - case 'dkim-signature': - case 'subject': - case 'references': - case 'message-id': - case 'in-reply-to': - case 'priority': - case 'x-priority': - case 'x-msmail-priority': - case 'importance': { - if (isSANB(header?.value)) { - value = header.value; - } else if (isSANB(header)) { - value = header; - } - - break; - } - - default: { - break; - } - } - } - - headers[key] = value; - } + const headers = await getHeaders(parsed); const status = _.isDate(domain.smtp_suspended_sent_at) || options?.isPending === true diff --git a/app/views/faq/index.md b/app/views/faq/index.md index 58551e6b6c..1c551319a4 100644 --- a/app/views/faq/index.md +++ b/app/views/faq/index.md @@ -1588,7 +1588,7 @@ Email relies on the [SMTP protocol](https://en.wikipedia.org/wiki/Simple_Mail_Tr * `MAIL FROM` - This indicates the envelope mail from address of the email. If a value is entered, it must be a valid RFC 5322 email address. Empty values are permitted. We [check for backscatter](#how-do-you-protect-against-backscatter) here, and we also check the MAIL FROM against our [denylist](#do-you-have-a-denylist). We finally check senders that are not on the allowlist for rate limiting (see the section on [Rate Limiting](#do-you-have-rate-limiting) and [allowlist](#do-you-have-an-allowlist) for more information). -* `RCPT TO` - This indicates the recipient(s) of the email. These must be valid RFC 5322 email addresses. We only permit up to 50 envelope recipients per message (this is different than the "To" header from an email). We also check for a valid [Sender Rewriting Scheme](https://en.wikipedia.org/wiki/Sender_Rewriting_Scheme) ("SRS") address here to protect against spoofing with our SRS domain name. Recipients provided that contain a "no-reply" address will receive a 553 error. See the [complete list of "no-reply" addresses below](#what-are-no-reply-addresses). We also check the recipient against our [denylist](#do-you-have-a-denylist). +* `RCPT TO` - This indicates the recipient(s) of the email. These must be valid RFC 5322 email addresses. We only permit up to 50 envelope recipients per message (this is different than the "To" header from an email). We also check for a valid [Sender Rewriting Scheme](https://en.wikipedia.org/wiki/Sender_Rewriting_Scheme) ("SRS") address here to protect against spoofing with our SRS domain name. We also check the recipient against our [denylist](#do-you-have-a-denylist). * `DATA` - This is the core part of our service which processes an email. See the section [How do you process an email for forwarding](#how-do-you-process-an-email-for-forwarding) below for more insight. @@ -1618,10 +1618,10 @@ This section describes our process related to the SMTP protocol command `DATA` i * `X-Original-To` - the original `RCPT TO` email address for the message. * This header's value has `Bcc` header parsed addresses removed from it. * This is useful for determining where an email was originally delivered to. - * Existing value if any is preserved as `X-Original-Preserved-To`. * `X-ForwardEmail-Version` - the current [SemVer](https://semver.org/) version from `package.json` of our codebase. * `X-ForwardEmail-Session-ID` - a session ID value used for debug purposes (only applies in non-production environments). * `X-ForwardEmail-Sender` - a comma separated list containing the original envelope MAIL FROM address (if it was not blank), the reverse PTR client FQDN (if it exists), and the sender's IP address. + * `X-ForwardEmail-ID` - this is only applicable for outbound SMTP and correlates to the email ID stored in My Account → Emails * `X-Report-Abuse` - with a value of `abuse@forwardemail.net`. * `X-Report-Abuse-To` - with a value of `abuse@forwardemail.net`. * `X-Complaints-To` - with a value of `abuse@forwardemail.net`. @@ -1691,8 +1691,6 @@ Our IP addresses are publicly available, [see this section below for more insigh ## What are no-reply addresses -We do not forward emails to "no-reply" addresses, and any sender attempting to will receive a 553 error. - Email usernames equal to any of the following (case-insensitive) are considered to be no-reply addresses: * `do-not-reply` diff --git a/config/index.js b/config/index.js index 0d9d02d92e..65092bebcc 100644 --- a/config/index.js +++ b/config/index.js @@ -3,8 +3,9 @@ * SPDX-License-Identifier: BUSL-1.1 */ -const path = require('node:path'); +const fs = require('node:fs'); const os = require('node:os'); +const path = require('node:path'); const Axe = require('axe'); const Boom = require('@hapi/boom'); @@ -16,6 +17,7 @@ const isSANB = require('is-string-and-not-blank'); const manifestRev = require('manifest-rev'); const ms = require('ms'); const nodemailer = require('nodemailer'); +const tlds = require('tlds'); const { Iconv } = require('iconv'); const { boolean } = require('boolean'); @@ -181,9 +183,57 @@ const STRIPE_LOCALES = new Set([ 'zh-TW' ]); +const POSTMASTER_USERNAMES = new Set([ + 'automailer', + 'autoresponder', + 'bounce', + 'bounce-notification', + 'bounce-notifications', + 'bounces', + 'e-bounce', + 'ebounce', + 'host-master', + 'host.master', + 'hostmaster', + 'localhost', + 'mail-daemon', + 'mail.daemon', + 'maildaemon', + 'mailer', + 'mailer-daemon', + 'mailer.daemon', + 'mailerdaemon', + 'post-master', + 'post.master', + 'postmaster' + // NOTE: excluding these because we try to go by SAFE MODE + // + // 'www', + // 'www-data' + // 'root', + // 'abuse', + // 'admin', + // 'admini', + // ...noReplyList +]); + const config = { ...metaConfig, + + signatureData: { + signingDomain: env.DKIM_DOMAIN_NAME, + selector: env.DKIM_KEY_SELECTOR, + privateKey: isSANB(env.DKIM_PRIVATE_KEY_PATH) + ? fs.readFileSync(env.DKIM_PRIVATE_KEY_PATH, 'utf8') + : isSANB(env.DKIM_PRIVATE_KEY_VALUE) + ? env.DKIM_PRIVATE_KEY_VALUE + : undefined, + algorithm: 'rsa-sha256', + canonicalization: 'relaxed/relaxed' + }, + socketTimeout: ms('3m'), + POSTMASTER_USERNAMES, ubuntuTeamMapping: { 'ubuntu.com': '~ubuntumembers', 'kubuntu.org': '~kubuntu-members', @@ -252,7 +302,24 @@ const config = { ? 1 : os.cpus().length, - // truth sources (mirrors smtp) + allowlist: new Set( + _.isArray(env.ALLOWLIST) + ? env.ALLOWLIST.map((key) => key.toLowerCase().trim()) + : isSANB(env.ALLOWLIST) + ? env.ALLOWLIST.split(',').map((key) => key.toLowerCase().trim()) + : [] + ), + + fingerprintPrefix: 'f', + + denylist: new Set( + _.isArray(env.DENYLIST) + ? env.DENYLIST.map((key) => key.toLowerCase().trim()) + : isSANB(env.DENYLIST) + ? env.DENYLIST.split(',').map((key) => key.toLowerCase().trim()) + : [] + ), + truthSources: new Set( _.isArray(env.TRUTH_SOURCES) ? env.TRUTH_SOURCES.map((key) => key.toLowerCase().trim()) @@ -261,6 +328,9 @@ const config = { : [] ), + greylistTimeout: ms('5m'), + greylistTtlMs: ms('5d'), + emailRetention: env.EMAIL_RETENTION, logRetention: env.LOG_RETENTION, @@ -347,6 +417,7 @@ const config = { dkimKeySelector: 'forwardemail', // forwardemail._domainkey.example.com supportRequestMaxLength: env.SUPPORT_REQUEST_MAX_LENGTH, abuseEmail: env.EMAIL_ABUSE, + friendlyFromEmail: env.EMAIL_FRIENDLY_FROM, email: { preview: { open: env.PREVIEW_EMAIL, @@ -1401,6 +1472,13 @@ const config = { } }; +// arbitrarily add domains to the denylist +for (const tld of tlds) { + if (config.restrictedDomains.includes(tld)) continue; + // postline.* (e.g. "postline.ml") + config.denylist.add(`postline.${tld}`); +} + // sanity test against validDurations and durationMapping length if (config.validDurations.length !== Object.keys(config.durationMapping).length) throw new Error('validDurations and durationMapping must be aligned'); diff --git a/ecosystem-mx.json b/ecosystem-mx.json new file mode 100644 index 0000000000..36f8896996 --- /dev/null +++ b/ecosystem-mx.json @@ -0,0 +1,27 @@ +{ + "apps": [ + { + "name": "mx", + "script": "mx.js", + "max_restarts": 999, + "exec_mode": "cluster", + "wait_ready": true, + "instances": "max", + "pmx": false, + "env_production": { + "NODE_ENV": "production" + } + } + ], + "deploy": { + "production": { + "user": "deploy", + "host": ["138.197.213.185","104.248.224.170"], + "ref": "origin/master", + "repo": "git@github.com:forwardemail/forwardemail.net.git", + "path": "/var/www/production", + "pre-deploy": "git reset --hard", + "post-deploy": "pnpm install && NODE_ENV=production npm start build && pm2 startOrGracefulReload ecosystem-mx.json --env production --update-env" + } + } +} diff --git a/helpers/combine-errors.js b/helpers/combine-errors.js index 05cc159aeb..7bb385a9d3 100644 --- a/helpers/combine-errors.js +++ b/helpers/combine-errors.js @@ -36,6 +36,13 @@ function combineErrors(errors) { '\n\n' ); + // if all errors had `name` and they were all the same then preserve it + if ( + typeof errors[0].name !== 'undefined' && + errors.every((e) => e.name === errors[0].name) + ) + err.name = errors[0].name; + // if all errors had `code` and they were all the same then preserve it if ( typeof errors[0].code !== 'undefined' && diff --git a/helpers/denylist-error.js b/helpers/denylist-error.js new file mode 100644 index 0000000000..2d680e1b1d --- /dev/null +++ b/helpers/denylist-error.js @@ -0,0 +1,22 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +class DenylistError extends Error { + constructor( + message = 'Denylisted', + responseCode = 554, + denylistValue = '', + ...parameters + ) { + super(...parameters); + Error.captureStackTrace(this, DenylistError); + this.name = 'DenylistError'; + this.message = message; + this.responseCode = responseCode; + this.denylistValue = denylistValue.toLowerCase(); + } +} + +module.exports = DenylistError; diff --git a/helpers/get-attributes.js b/helpers/get-attributes.js new file mode 100644 index 0000000000..d53d43a145 --- /dev/null +++ b/helpers/get-attributes.js @@ -0,0 +1,112 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const _ = require('lodash'); +const addrs = require('email-addresses'); +const addressParser = require('nodemailer/lib/addressparser'); +const isSANB = require('is-string-and-not-blank'); +const { isEmail } = require('validator'); + +const checkSRS = require('#helpers/check-srs'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); +const parseRootDomain = require('#helpers/parse-root-domain'); + +function getAttributes(headers, session) { + const replyTo = headers.getFirst('reply-to'); + + let replyToAddresses = + addrs.parseAddressList({ input: replyTo, partial: true }) || []; + + if (replyToAddresses.length === 0) + replyToAddresses = addrs.parseAddressList({ input: replyTo }) || []; + + // safeguard + if (replyToAddresses.length === 0) replyToAddresses = addressParser(replyTo); + + replyToAddresses = replyToAddresses.filter( + (addr) => + _.isObject(addr) && + isSANB(addr.address) && + isEmail(addr.address, { ignore_max_length: true }) + ); + + // + // check if the From, Reply-To, MAIL FROM, sender IP/host, or RCPT TO were silent banned + // (and filter out RCPT TO while parsing so we don't send twice) + // (and also check root domains as well for each of these) + // + return _.uniq( + _.compact( + [ + // NOTE: we don't check HELO because it's arbitrary + // check the From header + session.originalFromAddress, + // check the From header domains + parseHostFromDomainOrAddress(session.originalFromAddress), + // check the From header root domains + parseRootDomain( + parseHostFromDomainOrAddress(session.originalFromAddress) + ), + // check the Reply-To header + ...replyToAddresses.map((addr) => checkSRS(addr.address).toLowerCase()), + // check the Reply-To header domains + ...replyToAddresses.map((addr) => + parseHostFromDomainOrAddress(checkSRS(addr.address)) + ), + // check the Reply-To header root domains + ...replyToAddresses.map((addr) => + parseRootDomain(parseHostFromDomainOrAddress(checkSRS(addr.address))) + ), + // check the sender client host (if provided) + // (only applicable if not allowlisted) + !session.isAllowlisted && session.resolvedClientHostname + ? session.resolvedClientHostname + : null, + // check the sender client host root (if provided) + // (only applicable if not allowlisted) + // (but only if the root domain was not equal to the parsed host) + !session.isAllowlisted && + session.resolvedClientHostname && + session.resolvedClientHostname !== session.resolvedRootClientHostname + ? session.resolvedRootClientHostname + : null, + // check the sender client IP address + // (only applicable if not allowlisted) + session.isAllowlisted ? null : session.remoteAddress, + // check the MAIL FROM (if provided; lowercased) + isSANB(session.envelope.mailFrom.address) + ? checkSRS(session.envelope.mailFrom.address).toLowerCase() + : null, + // check the MAIL FROM host (if provided; lowercased) + isSANB(session.envelope.mailFrom.address) + ? parseHostFromDomainOrAddress( + checkSRS(session.envelope.mailFrom.address).toLowerCase() + ) + : null, + // check the MAIL FROM host root (if provided; lowercased) + // (but only if the root domain was not equal to the parsed host) + isSANB(session.envelope.mailFrom.address) && + parseRootDomain( + parseHostFromDomainOrAddress( + checkSRS(session.envelope.mailFrom.address).toLowerCase() + ) + ) !== + parseHostFromDomainOrAddress( + checkSRS(session.envelope.mailFrom.address).toLowerCase() + ) + ? parseRootDomain( + parseHostFromDomainOrAddress( + checkSRS(session.envelope.mailFrom.address).toLowerCase() + ) + ) + : null + ].map((str) => + typeof str === 'string' ? str.toLowerCase().trim() : null + ) + ) + ); +} + +module.exports = getAttributes; diff --git a/helpers/get-forwarding-addresses.js b/helpers/get-forwarding-addresses.js new file mode 100644 index 0000000000..dfa8ed7c65 --- /dev/null +++ b/helpers/get-forwarding-addresses.js @@ -0,0 +1,711 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const { isIP } = require('node:net'); +const { Buffer } = require('node:buffer'); + +const RE2 = require('re2'); +const _ = require('lodash'); +const addressParser = require('nodemailer/lib/addressparser'); +const isBase64 = require('is-base64'); +const isFQDN = require('is-fqdn'); +const isSANB = require('is-string-and-not-blank'); +const ms = require('ms'); +const regexParser = require('regex-parser'); +const { boolean } = require('boolean'); +const { isURL, isEmail } = require('validator'); + +const SMTPError = require('#helpers/smtp-error'); +const config = require('#config'); +const env = require('#config/env'); +const getErrorCode = require('#helpers/get-error-code'); +const logger = require('#helpers/logger'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); +const parseRootDomain = require('#helpers/parse-root-domain'); +const parseUsername = require('#helpers/parse-username'); +const { decrypt } = require('#helpers/encrypt-decrypt'); + +const USER_AGENT = `${config.pkg.name}/${config.pkg.version}`; + +function parseFilter(address) { + ({ address } = addressParser(address)[0]); + return address.includes('+') ? address.split('+')[1].split('@')[0] : ''; +} + +// +const REGEX_INTERPOLATED_DOLLAR = new RE2(/(\$)([1-9]\d*|[$&`'])/); + +// eslint-disable-next-line complexity +async function getForwardingAddresses( + address, + recursive = [], + ignoreBilling = false, + session +) { + let hasIMAP = false; + let aliasIds; + + const domain = parseHostFromDomainOrAddress(address); + const rootDomain = parseRootDomain(domain); + + // if it is a truth source then don't bother fetching + let records = []; + if (domain !== rootDomain || !config.truthSources.has(rootDomain)) { + try { + records = await this.resolver.resolveTxt(domain); + } catch (err) { + logger.warn(err, { address }); + // support retries + // TODO: rewrite `err.response` and `err.message` if either/both start with diagnostic code + err.responseCode = getErrorCode(err); + + throw err; + } + + try { + const mxRecords = await this.resolver.resolveMx(domain); + + if (!mxRecords || mxRecords.length === 0) { + records = []; + } else { + // let hasExchanges = false; + // for (const record of mxRecords) { + // hasExchanges = config.exchanges.some( + // (exchange) => exchange === record.exchange.toLowerCase() + // ); + // if (hasExchanges) break; + // } + // if (!hasExchanges) records = []; + } + } catch (err) { + // support retries + // TODO: rewrite `err.response` and `err.message` if either/both start with diagnostic code + logger.warn(err, { address }); + + if (err.code === 'ENOTFOUND' || err.code === 'ENODATA') { + records = []; + } else { + // support retries + // TODO: rewrite `err.response` and `err.message` if either/both start with diagnostic code + err.responseCode = getErrorCode(err); + throw err; + } + } + } + + // dns TXT record must contain `forward-email=` prefix + const validRecords = []; + + // verifications must start with `forward-email-site-verification=` prefix + const verifications = []; + + // add support for multi-line TXT records + for (let i = 0; i < records.length; i++) { + records[i] = records[i].join('').trim(); // join and trim chunks together + if (records[i].indexOf(`${config.recordPrefix}=`) === 0) { + let value = records[i].replace(`${config.recordPrefix}=`, ''); + if (isBase64(value)) { + try { + value = decrypt( + Buffer.from(value, 'base64').toString('utf8'), + env.TXT_ENCRYPTION_KEY + ); + } catch (err) { + logger.debug(err); + try { + value = decrypt( + Buffer.from(value, 'base64').toString('hex'), + env.TXT_ENCRYPTION_KEY + ); + } catch (err) { + logger.debug(err); + } + } + } + + validRecords.push(value); + } + + if (records[i].indexOf(`${config.recordPrefix}-site-verification=`) === 0) + verifications.push( + records[i].replace(`${config.recordPrefix}-site-verification=`, '') + ); + } + + // check if we have a specific redirect and store global redirects (if any) + // get username from recipient email address + // (e.g. user@example.com => hello) + const username = parseUsername(address); + + // + // store if the domain was bad and not on paid plan (required for bad domains) + // + let badDomainExtension = true; + for (const tld of config.goodDomains) { + if (rootDomain.endsWith(`.${tld}`)) { + badDomainExtension = false; + break; + } + } + + if (!badDomainExtension) { + for (const tld of config.restrictedDomains) { + if (rootDomain === tld || rootDomain.endsWith(`.${tld}`)) { + badDomainExtension = false; + break; + } + } + } + + if (verifications.length > 0) { + if (verifications.length > 1) + throw new SMTPError( + // TODO: we may want to replace with "Invalid Recipients" + `Domain ${domain} has multiple verification TXT records of "${config.recordPrefix}-site-verification" and should only have one`, + { responseCode: 421 } + ); + + // TODO: cache responses in redis and purge on web-side changes + + // if there was a verification record then perform lookup + const response = await this.apiClient.request({ + path: '/v1/lookup', + method: 'GET', + headers: { + 'User-Agent': USER_AGENT, + Accept: 'application/json', + Authorization: + 'Basic ' + Buffer.from(env.API_SECRETS[0] + ':').toString('base64') + }, + query: { + username, + ignore_billing: ignoreBilling, + verification_record: verifications[0] + } + }); + + const body = await response.body.json(); + // body = { + // has_imap: boolean, + // mapping: array + // } + hasIMAP = boolean(body.has_imap); + if (hasIMAP) badDomainExtension = false; + if ( + body.alias_ids && + Array.isArray(body.alias_ids) && + body.alias_ids.length > 0 + ) + aliasIds = body.alias_ids; + // body is an Array of records that are formatted like TXT records + if (_.isArray(body.mapping) && body.mapping.length > 0) { + // update that domain was bad but on paid plan so OK + badDomainExtension = false; + + // combine with any existing TXT records (ensures graceful DNS propagation) + for (const element of body.mapping) { + validRecords.push(element); + } + } + } + + // join multi-line TXT records together and replace double w/single commas + const record = validRecords.join(',').replace(/,+/g, ',').trim(); + + // if the record was blank then throw an error + if (!isSANB(record) && !hasIMAP) + throw new SMTPError( + `${address} is not yet configured with its email service provider ${config.urls.web} ;`, + { responseCode: 421 } + ); + + // e.g. user@example.com => user@gmail.com + // record = "forward-email=hello:user@gmail.com" + // e.g. hello+test@example.com => user+test@gmail.com + // record = "forward-email=hello:user@gmail.com" + // e.g. *@example.com => user@gmail.com + // record = "forward-email=user@gmail.com" + // e.g. *+test@example.com => user@gmail.com + // record = "forward-email=user@gmail.com" + + function splitString(str) { + if (str.indexOf('/') === 0) { + // it can either be split by ",/" or "," + const index = str.includes(',/') + ? str.lastIndexOf('/:', str.indexOf(',/')) + : str.indexOf('/:'); + const lastComma = str.indexOf(',', index); + if (lastComma === -1) return [str]; + if (lastComma === str.length - 1) return [str.slice(0, lastComma)]; + return [ + str.slice(0, lastComma), + ...splitString(str.slice(lastComma + 1)) + ]; + } + + return str.includes(',') + ? [ + str.slice(0, str.indexOf(',')), + ...splitString(str.slice(str.indexOf(',') + 1)) + ] + : [str]; + } + + // remove trailing whitespaces from each address listed + const addresses = _.uniq( + _.compact( + record + .split({ + [Symbol.split](str) { + return splitString(str); + } + }) + .map((str) => str.trim()) + ) + ); + + if (addresses.length === 0 && !hasIMAP) + throw new SMTPError( + // TODO: we may want to replace with "Invalid Recipients" + `${address} domain of ${domain} has zero forwarded addresses configured in the TXT record with "${config.recordPrefix}"`, + { responseCode: 421 } + ); + + // store if address is ignored or not + let ignored = false; // 250 + let softRejected = false; // 421 + let hardRejected = false; // 550 + + // store if we have a forwarding address or not + let forwardingAddresses = []; + + // store if we have a global redirect or not + const globalForwardingAddresses = []; + + for (let element of addresses) { + // convert addresses to lowercase + const lowerCaseAddress = element.toLowerCase(); + + // must start with / and end with /: and not have the same index for the last index + // forward-email=/^(support|info)$/:user+$1@gmail.com + // -> would forward to user+support@gmail.com if email sent to support@ + + // it either ends with: + // "/gi:" + // "/ig:" + // "/g:" + // "/i:" + // "/:" + // + let lastIndex; + const REGEX_FLAG_ENDINGS = ['/gi:', '/ig:', '/g:', '/i:', '/:']; + + // regex ignore support + let wasIgnoredRegex = false; + // ! -> 250 + if ( + element.startsWith('!') && + element.indexOf('/') === 1 && + element.endsWith('/') + ) { + element = element.slice(1) + ':'; + wasIgnoredRegex = true; + } + + // !! -> 421 + if ( + element.startsWith('!!') && + element.indexOf('/') === 2 && + element.endsWith('/') + ) { + element = element.slice(2) + ':'; + wasIgnoredRegex = true; + } + + // !!! -> 550 + if ( + element.startsWith('!!!') && + element.indexOf('/') === 3 && + element.endsWith('/') + ) { + element = element.slice(3) + ':'; + wasIgnoredRegex = true; + } + + const hasTwoSlashes = element.lastIndexOf('/') !== 0; + const startsWithSlash = element.indexOf('/') === 0; + + if (startsWithSlash && hasTwoSlashes) { + for (const ending of REGEX_FLAG_ENDINGS) { + if ( + element.lastIndexOf(ending) !== -1 && + element.lastIndexOf(ending) !== 0 + ) { + lastIndex = ending; + break; + } + } + } + + // + // regular expression support + // + // (with added support for regex gi flags) + // + if (startsWithSlash && hasTwoSlashes && lastIndex) { + const elementWithoutRegex = element.slice( + Math.max(0, element.lastIndexOf(lastIndex) + lastIndex.length) + ); + + let parsedRegex = element.slice( + 0, + Math.max(0, element.lastIndexOf(lastIndex) + 1) + ); + + // add case insensitive flag since email addresses are case insensitive + if (lastIndex === '/g:' || lastIndex === '/:') parsedRegex += 'i'; + // + // `forward-email=/^(support|info)$/:user+$1@gmil.coail.com` + // support@mydomain.com -> user+support@gmail.com + // + // `forward-email=/^(support|info)$/:example.com/$1` + // info@mydomain.com -> POST to example.com/info + // + // `forward-email=/Support/g:example.com` + // + // `forward-email=/SUPPORT/gi:example.com` + // + let regex; + try { + // NOTE: catches errors like "Invalid regular expression": + regex = new RE2(regexParser(parsedRegex)); + } catch (err) { + logger.fatal(err, { address }); + } + + if (regex && regex.test(username.toLowerCase())) { + const hasDollarInterpolation = + REGEX_INTERPOLATED_DOLLAR.test(elementWithoutRegex); + + const substitutedAlias = hasDollarInterpolation + ? username.toLowerCase().replace(regex, elementWithoutRegex) + : elementWithoutRegex; + + if ( + (wasIgnoredRegex && !substitutedAlias) || + substitutedAlias.indexOf('!!!') === 0 + ) { + hardRejected = true; + break; + } + + if ( + (wasIgnoredRegex && !substitutedAlias) || + substitutedAlias.indexOf('!!') === 0 + ) { + softRejected = true; + break; + } + + if ( + (wasIgnoredRegex && !substitutedAlias) || + substitutedAlias.indexOf('!') === 0 + ) { + ignored = true; + break; + } + + if ( + !isFQDN(substitutedAlias) && + !isIP(substitutedAlias) && + !isEmail(substitutedAlias, { ignore_max_length: true }) && + !isURL(substitutedAlias, config.isURLOptions) + ) + throw new SMTPError( + // TODO: we may want to replace with "Invalid Recipients" + `Domain of ${domain} has an invalid "${config.recordPrefix}" TXT record due to an invalid regular expression email address match` + ); + + if (isURL(substitutedAlias, config.isURLOptions)) + forwardingAddresses.push(substitutedAlias); + else forwardingAddresses.push(substitutedAlias.toLowerCase()); + } + } else if ( + (element.includes(':') || element.indexOf('!') === 0) && + !isURL(element, config.isURLOptions) + ) { + // > const str = 'foo:https://foo.com' + // > str.slice(0, str.indexOf(':')) + // 'foo' + // > str.slice(str.indexOf(':') + 1) + // 'https://foo.com' + const index = element.indexOf(':'); + const addr = + index === -1 + ? [element] + : [element.slice(0, index), element.slice(index + 1)]; + + // addr[0] = hello (username) + // addr[1] = user@gmail.com (forwarding email) + // check if we have a match (and if it is ignored) + if (_.isString(addr[0]) && addr[0].indexOf('!') === 0) { + // !!! -> 550 + if ( + addr[0].indexOf('!!!') === 0 && + username === addr[0].toLowerCase().slice(3) + ) { + hardRejected = true; + break; + } + + // !! -> 421 + if ( + addr[0].indexOf('!!') === 0 && + username === addr[0].toLowerCase().slice(2) + ) { + softRejected = true; + break; + } + + // ! -> 250 + if (username === addr[0].toLowerCase().slice(1)) { + ignored = true; + break; + } + + continue; + } + + if ( + addr.length !== 2 || + !_.isString(addr[1]) || + (!isFQDN(addr[1]) && + !isIP(addr[1]) && + !isEmail(addr[1], { ignore_max_length: true }) && + !isURL(addr[1], config.isURLOptions)) + ) + throw new SMTPError( + // TODO: we may want to replace with "Invalid Recipients" + `${lowerCaseAddress} domain of ${domain} has an invalid "${config.recordPrefix}" TXT record due to an invalid email address of "${element}"` + ); + + if (_.isString(addr[0]) && username === addr[0].toLowerCase()) { + if (isURL(addr[1], config.isURLOptions)) + forwardingAddresses.push(addr[1]); + else forwardingAddresses.push(addr[1].toLowerCase()); + } + } else if (isFQDN(lowerCaseAddress) || isIP(lowerCaseAddress)) { + // allow domain alias forwarding + // (e.. the record is just "b.com" if it's not a valid email) + globalForwardingAddresses.push(`${username}@${lowerCaseAddress}`); + } else if (isEmail(lowerCaseAddress, { ignore_max_length: true })) { + globalForwardingAddresses.push(lowerCaseAddress); + } else if (isURL(element, config.isURLOptions)) { + globalForwardingAddresses.push(element); + } + } + + // if it was ignored then return early with false indicating it's disabled + if (ignored) return { ignored }; + if (softRejected) return { softRejected }; + if (hardRejected) return { hardRejected }; + + // if we don't have a specific forwarding address try the global redirect + if ( + forwardingAddresses.length === 0 && + globalForwardingAddresses.length > 0 && + !hasIMAP + ) { + for (const address of globalForwardingAddresses) { + forwardingAddresses.push(address); + } + } + + // + // if the domain does not have any verifications + // and if the domain ended with a bad domain + // then we can reject the message and inform + // the recipient with a one-time courtesy email + // + if (badDomainExtension && forwardingAddresses.length > 0) + throw new SMTPError( + `${address} requires an upgrade to Enhanced Protection at ${config.urls.web} ; Please read ${config.urls.web}/faq#what-domain-name-extensions-can-be-used-for-free for more information` + ); + + // if we don't have a forwarding address then throw an error + if (forwardingAddresses.length === 0 && !hasIMAP) { + throw new SMTPError( + `${address} is not yet configured with its email service provider ${config.urls.web} ;`, + { responseCode: 421 } + ); + } + + // + // ensure forwarding addresses are unique to prevent additional hops + // TODO: we could also do indexOf or includes check above before pushing + // + forwardingAddresses = _.uniq(forwardingAddresses); + + // TODO: isn't actually utilized since `recursive` arg not used + // (e.g. we don't do MX lookup on the TXT we're forwarding to) + // allow one recursive lookup on forwarding addresses + const recursivelyForwardedAddresses = []; + + const { length } = forwardingAddresses; + for (let x = 0; x < length; x++) { + const forwardingAddress = forwardingAddresses[x]; + try { + // TODO: is the culprit + if (recursive.includes(forwardingAddress)) continue; + if (isURL(forwardingAddress, config.isURLOptions)) continue; + + const newRecursive = [...forwardingAddresses, ...recursive]; + + // prevent a double-lookup if user is using + symbols + if (forwardingAddress.includes('+')) + newRecursive.push( + `${parseUsername(address)}@${parseHostFromDomainOrAddress(address)}` + ); + + // support recursive IMAP lookup + // eslint-disable-next-line no-await-in-loop + const data = await getForwardingAddresses.call( + this, + forwardingAddress, + newRecursive, + ignoreBilling, + session + ); + + if (data.hasIMAP) hasIMAP = true; + if ( + data.aliasIds && + Array.isArray(data.aliasIds) && + data.aliasIds.length > 0 + ) { + if (!aliasIds) aliasIds = []; + for (const id of data.aliasIds) { + if (!aliasIds.includes(id)) aliasIds.push(id); + } + } + + // if address was ignored then skip adding it + if (data.ignored) continue; + if (data.softRejected) continue; + if (data.hardRejected) continue; + + // if it was recursive then remove the original + if (data.addresses.length > 0 || data.hasIMAP) + recursivelyForwardedAddresses.push(forwardingAddress); + // add the recursively forwarded addresses + for (const element of data.addresses) { + forwardingAddresses.push(element); + } + } catch (err) { + logger.error(err); + } + } + + // make the forwarding addresses unique + // (and omit the recursively forwarded addresses) + forwardingAddresses = _.uniq( + _.compact( + forwardingAddresses.map((addr) => { + if (!recursivelyForwardedAddresses.includes(addr)) return addr; + return null; + }) + ) + ); + + // lookup here to determine max forwarded addresses on the domain + // if max number of forwarding addresses exceeded + let { maxForwardedAddresses } = config; + + // attempt to get cached value for domain + let value = false; + try { + value = await this.client.get(`v1_max_forwarded:${domain}`); + if (value) { + value = Number.parseInt(value, 10); + if (Number.isNaN(value) || !Number.isFinite(value)) { + value = false; + await this.client.del(`v1_max_forwarded:${domain}`); + } + } + } catch (err) { + value = false; + logger.fatal(err); + } + + if (value) { + maxForwardedAddresses = value; + } else { + try { + const response = await this.apiClient.request({ + path: '/v1/max-forwarded-addresses', + method: 'GET', + headers: { + 'User-Agent': USER_AGENT, + Accept: 'application/json', + Authorization: + 'Basic ' + Buffer.from(env.API_SECRETS[0] + ':').toString('base64') + }, + query: { + domain + } + }); + + const body = await response.body.json(); + // body is an Object with `max_forwarded_addresses` Number + if ( + _.isObject(body) && + _.isNumber(body.max_forwarded_addresses) && + body.max_forwarded_addresses > 0 + ) + maxForwardedAddresses = body.max_forwarded_addresses; + await this.client.set( + `v1_max_forwarded:${domain}`, + maxForwardedAddresses, + 'PX', + ms('1h') + ); + } catch (err) { + err.isCodeBug = true; + logger.error(err); + } + } + + if (forwardingAddresses.length > maxForwardedAddresses) { + throw new SMTPError( + `The address ${address} is attempted to be forwarded to (${forwardingAddresses.length}) addresses which exceeds the maximum of (${maxForwardedAddresses})`, + { responseCode: 421 } + ); + } + + // otherwise transform the + symbol filter if we had it + // and then resolve with the newly formatted forwarding address + // (we can return early here if there was no + symbol) + if (!address.includes('+')) + return { aliasIds, hasIMAP, addresses: forwardingAddresses }; + + return { + aliasIds, + hasIMAP, + addresses: forwardingAddresses.map((forwardingAddress) => { + if ( + isFQDN(forwardingAddress) || + isIP(forwardingAddress) || + isURL(forwardingAddress, config.isURLOptions) + ) + return forwardingAddress; + + return `${parseUsername(forwardingAddress)}+${parseFilter( + address + )}@${parseHostFromDomainOrAddress(forwardingAddress)}`; + }) + }; +} + +module.exports = getForwardingAddresses; diff --git a/helpers/get-from-address.js b/helpers/get-from-address.js new file mode 100644 index 0000000000..7f30f88b86 --- /dev/null +++ b/helpers/get-from-address.js @@ -0,0 +1,61 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const _ = require('lodash'); +const addressParser = require('nodemailer/lib/addressparser'); +const addrs = require('email-addresses'); +const isSANB = require('is-string-and-not-blank'); +const { isEmail } = require('validator'); + +const SMTPError = require('#helpers/smtp-error'); +const checkSRS = require('#helpers/check-srs'); + +function getFromAddress(originalFrom) { + if (!originalFrom) + throw new SMTPError( + 'Your message is not RFC 5322 compliant, please include a valid "From" header' + ); + + // + // parse the original from and ensure that there is one valid email address + // + // + // + // + // TODO: we probably should rewrite this with something else (!!!!) + // + let originalFromAddresses = + addrs.parseAddressList({ input: originalFrom, partial: true }) || []; + + if (originalFromAddresses.length === 0) + originalFromAddresses = + addrs.parseAddressList({ input: originalFrom }) || []; + + // safeguard + if (originalFromAddresses.length === 0) + originalFromAddresses = addressParser(originalFrom); + + const originalLength = originalFromAddresses.length; + + originalFromAddresses = originalFromAddresses.filter( + (addr) => + _.isObject(addr) && + isSANB(addr.address) && + isEmail(addr.address, { ignore_max_length: true }) + ); + + if ( + originalFromAddresses.length !== 1 || + originalLength !== originalFromAddresses.length + ) + throw new SMTPError( + 'Your message must contain one valid email address in the "From" header' + ); + + // set original from address that was parsed + return checkSRS(originalFromAddresses[0].address).toLowerCase(); +} + +module.exports = getFromAddress; diff --git a/helpers/get-greylist-key.js b/helpers/get-greylist-key.js new file mode 100644 index 0000000000..ee655bc184 --- /dev/null +++ b/helpers/get-greylist-key.js @@ -0,0 +1,12 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const revHash = require('rev-hash'); + +function getGreylistKey(clientRootDomainOrIP) { + return `greylist:${revHash(clientRootDomainOrIP)}`; +} + +module.exports = getGreylistKey; diff --git a/helpers/get-headers.js b/helpers/get-headers.js new file mode 100644 index 0000000000..eeb4c38a55 --- /dev/null +++ b/helpers/get-headers.js @@ -0,0 +1,20 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +function getHeaders(headers) { + const _headers = {}; + const lines = headers.headers + .toString('binary') + .replace(/[\r\n]+$/, '') + .split(/\r?\n/); + for (const line of lines) { + const index = line.indexOf(': '); + _headers[line.slice(0, index)] = line.slice(index + 2); + } + + return _headers; +} + +module.exports = getHeaders; diff --git a/helpers/get-recipients.js b/helpers/get-recipients.js new file mode 100644 index 0000000000..94d1f82a40 --- /dev/null +++ b/helpers/get-recipients.js @@ -0,0 +1,596 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const { Buffer } = require('node:buffer'); + +const pMap = require('p-map'); +const _ = require('lodash'); +const safeStringify = require('fast-safe-stringify'); +const ms = require('ms'); +const isSANB = require('is-string-and-not-blank'); +const { isPort, isURL, isIP, isFQDN } = require('validator'); + +const DenylistError = require('#helpers/denylist-error'); +const SMTPError = require('#helpers/smtp-error'); +const combineErrors = require('#helpers/combine-errors'); +const config = require('#config'); +const env = require('#config/env'); +const getErrorCode = require('#helpers/get-error-code'); +const getForwardingAddresses = require('#helpers/get-forwarding-addresses'); +const isDenylisted = require('#helpers/is-denylisted'); +const isSilentBanned = require('#helpers/is-silent-banned'); +const logger = require('#helpers/logger'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); +const parseRootDomain = require('#helpers/parse-root-domain'); +const parseUsername = require('#helpers/parse-username'); +const { encrypt } = require('#helpers/encrypt-decrypt'); + +const USER_AGENT = `${config.pkg.name}/${config.pkg.version}`; + +// eslint-disable-next-line complexity +async function getRecipients(session, scan) { + const bounces = []; + const normalized = []; + const imap = []; + + // lookup forwarding recipients recursively + let recipients = await pMap( + session.envelope.rcptTo, + // eslint-disable-next-line complexity + async (to) => { + let port = '25'; + try { + let hasAdultContentProtection = true; + let hasPhishingProtection = true; + let hasExecutableProtection = true; + let hasVirusProtection = true; + let customAllowlist = []; + let customDenylist = []; + let webhookKey; + + // get all forwarding addresses for individual address + const { + aliasIds, + addresses, + hasIMAP, + ignored, + softRejected, + hardRejected + } = await getForwardingAddresses.call( + this, + to.address, + [], + session.originalFromAddressRootDomain === env.WEB_HOST, + session + ); + + if (ignored) + return { + address: to.address, + addresses: [], + ignored: true, + hasIMAP: false + }; + + if (softRejected) + return { + address: to.address, + addresses: [], + ignored: false, + hasIMAP: false, + softRejected: true + }; + + if (hardRejected) + return { + address: to.address, + addresses: [], + ignored: false, + hasIMAP: false, + hardRejected: true + }; + + // lookup the port (e.g. if `forward-email-port=` or custom set on the domain) + const domain = parseHostFromDomainOrAddress(to.address); + + // attempt to get cached value for domain + let value = false; + try { + value = await this.client.get(`v1_settings:${domain}`); + if (value) value = JSON.parse(value); + } catch (err) { + value = false; + logger.fatal(err); + } + + let body; + if (value) { + body = value; + } else { + try { + const response = await this.apiClient.request({ + path: '/v1/settings', + method: 'GET', + headers: { + 'User-Agent': USER_AGENT, + Accept: 'application/json', + Authorization: + 'Basic ' + + Buffer.from(env.API_SECRETS[0] + ':').toString('base64') + }, + query: { domain } + }); + + body = await response.body.json(); + + await this.client.set( + `v1_settings:${domain}`, + safeStringify(body), + 'PX', + ms('1h') + ); + } catch (err) { + err.isCodeBug = true; + logger.err(err); + } + } + + // body is an Object + if (_.isObject(body)) { + // `port` (String) - a valid port number, defaults to 25 + if (isSANB(body.port) && isPort(body.port) && body.port !== '25') { + port = body.port; + logger.debug(`Custom port for ${to.address} detected`, { + port, + session + }); + } + + // Spam Scanner boolean values adjusted by user in Advanced Settings page + if (_.isBoolean(body.has_adult_content_protection)) + hasAdultContentProtection = body.has_adult_content_protection; + + if (_.isBoolean(body.has_phishing_protection)) + hasPhishingProtection = body.has_phishing_protection; + + if (_.isBoolean(body.has_executable_protection)) + hasExecutableProtection = body.has_executable_protection; + + if (_.isBoolean(body.has_virus_protection)) + hasVirusProtection = body.has_virus_protection; + + if (Array.isArray(body.allowlist)) customAllowlist = body.allowlist; + + if (Array.isArray(body.denylist)) customDenylist = body.denylist; + + if (isSANB(body.webhook_key)) webhookKey = body.webhook_key; + } + + // + // NOTE: here is where we check if Spam Scanner settings + // were either enabled or disabled, and if they were enabled + // and the respective policy did not pass, then throw that error as a bounce + // + if (_.isObject(scan)) { + // + // NOTE: until we are confident with the accuracy + // we are not utilizing classification right now + // however we still want to use other detections + // + const messages = []; + + if ( + hasPhishingProtection && + _.isArray(scan?.results?.phishing) && + !_.isEmpty(scan.results.phishing) + ) { + for (const message of scan.results.phishing) { + // if we're not filtering for adult-related content then continue early + + if ( + !hasAdultContentProtection && + message.includes('adult-related content') + ) + continue; + + if (message.includes('adult-related content')) + messages.push( + 'Links were detected that may contain adult-related content' + ); + else + messages.push( + 'Links were detected that may contain phishing and/or malware' + ); + // + // NOTE: we do not want to push the link in the response + // (otherwise bounce emails may never arrive to sender) + // + // messages.push(message); + } + } + + if ( + hasExecutableProtection && + _.isArray(scan?.results?.executables) && + !_.isEmpty(scan.results.executables) + ) { + for (const message of scan.results.executables.slice(0, 2)) { + messages.push(message); + } + + messages.push( + `You may want to re-send your attachment in a compressed archive format (e.g. a ZIP file)` + ); + } + + if ( + hasVirusProtection && + _.isArray(scan?.results?.viruses) && + !_.isEmpty(scan.results.viruses) + ) { + for (const message of scan.results.viruses.slice(0, 2)) { + messages.push(message); + } + } + + if (messages.length > 0) { + messages.push( + 'For more information on Spam Scanner visit https://spamscanner.net' + ); + throw new SMTPError(_.uniq(messages).join(' '), { + responseCode: 554 + }); + } + } + + // + // NOTE: section is for a domain's specific allowlist and denylist + // + // has to match at least one of the following for allowlist to pass: + // - session.remoteAddress + // - session.resolvedClientHostname + // - session.resolvedRootClientHostname + // - session.originalFromAddress + // + if (customAllowlist.length > 0) { + let pass = false; + if (customAllowlist.includes(session.remoteAddress)) pass = true; + else if ( + session.resolvedClientHostname && + customAllowlist.includes(session.resolvedClientHostname) + ) + pass = true; + else if ( + session.resolvedRootClientHostname && + customAllowlist.includes(session.resolvedRootClientHostname) + ) + pass = true; + else if ( + session.originalFromAddress && + customAllowlist.includes(session.originalFromAddress) + ) + pass = true; + if (!pass && session.originalFromAddress) { + // check if the domain or root portion of the `session.originalFromAddress` matches + const domain = session.originalFromAddress.split('@')[1]; + const root = parseRootDomain(domain); + if (customAllowlist.includes(domain)) pass = true; + else if (customAllowlist.includes(root)) pass = true; + } + + if (!pass) + throw new SMTPError( + `Your IP address, client hostname, or From header is not yet allowlisted by admins of ${domain}` + ); + } + + if (customDenylist.length > 0) { + let pass = true; + if (customDenylist.includes(session.remoteAddress)) pass = false; + else if ( + session.resolvedClientHostname && + customDenylist.includes(session.resolvedClientHostname) + ) + pass = false; + else if ( + session.resolvedRootClientHostname && + customDenylist.includes(session.resolvedRootClientHostname) + ) + pass = false; + else if ( + session.originalFromAddress && + customDenylist.includes(session.originalFromAddress) + ) + pass = false; + if (pass && session.originalFromAddress) { + // check if the domain or root portion of the `session.originalFromAddress` matches + const domain = session.originalFromAddress.split('@')[1]; + const root = parseRootDomain(domain); + if (customDenylist.includes(domain)) pass = false; + else if (customDenylist.includes(root)) pass = false; + } + + if (!pass) + throw new SMTPError( + `Your IP address, client hostname, or From header was denylisted by admins of ${domain}` + ); + } + + return { + address: to.address, + addresses, + port, + hasIMAP, + aliasIds, + webhookKey + }; + } catch (err) { + logger.warn(err, { session }); + err.responseCode = getErrorCode(err); + bounces.push({ + address: to.address, + err + }); + } + }, + { concurrency: config.concurrency } + ); + + // + // TODO: look into this note below (?) + // + // NOTE: if user has both plain TXT and encrypted + // then only the first match will be used + // (probably unwanted, we should just merge) + // + // flatten the recipients and make them unique + recipients = _.uniqBy(_.compact(recipients.flat()), 'address'); + + // TODO: we can probably remove now + // go through recipients and if we have a user+xyz@domain + // AND we also have user@domain then honor the user@domain only + // (helps to alleviate bulk spam with services like Gmail) + for (const recipient of recipients) { + const filtered = []; + for (const address of recipient.addresses) { + if (!address.includes('+')) { + filtered.push(address); + continue; + } + + if ( + !recipient.addresses.includes( + `${parseUsername(address)}@${parseHostFromDomainOrAddress(address)}` + ) + ) + filtered.push(address); + } + + recipient.addresses = filtered; + } + + let hasSilentBannedRecipients = false; + + recipients = await pMap( + recipients, + async (recipient) => { + const errors = []; + const addresses = []; + await pMap( + recipient.addresses, + async (address) => { + try { + // check if the recipient was silent banned + const silentBanned = await isSilentBanned( + address, + this.client, + this.resolver + ); + if (silentBanned) { + hasSilentBannedRecipients = true; + // logger.debug('silent banned', { + // session, + // value: address.toLowerCase() + // }); + return; + } + + // check if the address was denylisted + try { + await isDenylisted(address, this.client, this.resolver); + } catch (err) { + err.message = `The address ${ + recipient.address + } is denylisted by ${ + config.website + } ; To request removal, you must visit ${ + config.website + }/denylist?q=${encrypt(address.toLowerCase())} ;`; + err.address = address; + throw err; + } + + // if it was a URL webhook then return early + if (isURL(address, config.isURLOptions)) { + addresses.push({ to: address, is_webhook: true }); + } else if (isIP(address) || isFQDN(address)) { + // if it was an IP or FQDN then rewrite it (since it's a catch-all) + addresses.push({ + to: `${parseUsername(recipient.address)}@${address}`, + host: address + }); + } else { + addresses.push({ + to: address, + host: parseHostFromDomainOrAddress(address) + }); + } + } catch (err) { + // TODO: e.g. if the MX servers don't exist for recipient + // then obviously there should be an error + logger.warn(err, { session }); + errors.push(err); + } + }, + { concurrency: config.concurrency } + ); + + // map it back + recipient.addresses = addresses; + + // TODO: how does work with IMAP (?) + // custom port support + if (recipient.addresses.length === 0 && recipient.port !== '25') { + recipient.addresses.push({ + to: recipient.address, + host: parseHostFromDomainOrAddress(recipient.address) + }); + } + + if (recipient.addresses.length > 0 || recipient.hasIMAP) return recipient; + if (errors.length === 0) return; + for (const err of errors) { + logger.warn(err, { session }); + } + + const err = combineErrors(errors); + if (errors.some((err) => err instanceof DenylistError)) + err.name = 'DenylistError'; + // TODO: rewrite `err.response` and `err.message` if either/both start with diagnostic code + err.responseCode = _.sortBy(errors.map((err) => getErrorCode(err)))[0]; + bounces.push({ + address: recipient.address, + err, + recipient + }); + }, + { concurrency: config.concurrency } + ); + + recipients = _.compact(recipients); + + if (_.isEmpty(recipients)) { + if (_.isEmpty(bounces)) { + // return early if silent banned recipients + if (hasSilentBannedRecipients) return; + throw new SMTPError('Invalid recipients'); + } + + // if there was only one bounce then throw it by itself + if (bounces.length === 1) throw bounces[0].err; + + // + // otherwise combine the bounce errors into one error + // + + // go by lowest code (e.g. 421 retry instead of 5xx if one still hasn't sent yet) + const errors = []; + const codes = []; + for (const bounce of bounces) { + // NOTE: we also have `bounce.host` and `bounce.address` to use if needed for more verbosity + errors.push(bounce.err); + codes.push(getErrorCode(bounce.err)); + } + + // join the messages together + const err = combineErrors(errors); + + // + // NOTE: we only do this because in the web UI we render a removal button + // and there could be multiple RCPT TO errors and not all could be denylist + // (e.g. some could be Redis/Mongo, but we want to make it easy for the user) + // + // NOTE: `combineErrors` will additionally set `err` properties such as `err.name` + // on the resulting combined error object returned `err` + // if all of the errors being combined have the same value for `err.name` + // + if (errors.some((err) => err instanceof DenylistError)) + err.name = 'DenylistError'; + + err.responseCode = _.sortBy(codes)[0]; + err.bounces = bounces; + throw err; + } + + for (const recipient of recipients) { + // if it's ignored then don't bother + if (recipient.ignored) continue; + + if (recipient.softRejected) { + bounces.push({ + address: recipient.address, + err: new SMTPError('Mailbox is disabled, try again later', { + responseCode: 421 + }) + }); + continue; + } + + if (recipient.hardRejected) { + bounces.push({ + address: recipient.address, + err: new SMTPError('Mailbox is disabled') + }); + continue; + } + + // if it has imap then push it + if (recipient.hasIMAP && recipient.aliasIds) { + for (const aliasId of recipient.aliasIds) { + if ( + imap.some( + (obj) => obj.address === recipient.address && obj.id === aliasId + ) + ) + continue; + imap.push({ address: recipient.address, id: aliasId }); + } + } + + for (const address of recipient.addresses) { + // if it's a webhook then return early + if (address.is_webhook) { + // + // NOTE: we group webhooks based off their endpoint + // to reduce the number of requests sent across + // + const match = normalized.find((r) => r.webhook === address.to); + + if (match) { + if (!match.to.includes(address.to)) match.to.push(address.to); + + if (!match.replacements[recipient.address]) + match.replacements[recipient.address] = address.to; // normal; + } else { + const replacements = {}; + replacements[recipient.address] = address.to; // normal; + normalized.push({ + webhookKey: recipient.webhookKey, + webhook: address.to, + to: [address.to], + recipient: recipient.address, + replacements + }); + } + + continue; + } + + const replacements = {}; + replacements[recipient.address] = address.to; + normalized.push({ + host: address.host, + port: recipient.port, + recipient: recipient.address, + to: [address.to], + replacements + }); + } + } + + return { bounces, normalized, imap }; +} + +module.exports = getRecipients; diff --git a/helpers/has-fingerprint-expired.js b/helpers/has-fingerprint-expired.js new file mode 100644 index 0000000000..d4b4fb90d2 --- /dev/null +++ b/helpers/has-fingerprint-expired.js @@ -0,0 +1,43 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const prettyMilliseconds = require('pretty-ms'); +const _ = require('lodash'); + +const SMTPError = require('#helpers/smtp-error'); +const config = require('#config'); + +async function hasFingerprintExpired(session, client) { + if (!session.fingerprint) throw new TypeError('Fingerprint missing'); + const key = `${config.fingerprintPrefix}:${session.fingerprint}`; + let value = await client.get(key); + if (value) { + value = new Date(value); + + // reset if cache invalid + if (!_.isDate(value)) value = null; + } + + if (value) { + if (value.getTime() + config.maxRetryDuration <= session.arrivalTime) { + throw new SMTPError( + `This message has been retried for the maximum period of ${prettyMilliseconds( + config.maxRetryDuration, + { verbose: true, secondsDecimalDigits: 0 } + )} and has permanently failed` + ); + } + } else { + // after 5 days the message fingerprint is deleted (to save on storage) + await client.set( + key, + new Date(session.arrivalTime).toISOString(), + 'PX', + config.greylistTtlMs + ); + } +} + +module.exports = hasFingerprintExpired; diff --git a/helpers/index.js b/helpers/index.js index 7a7781f74a..4cc29ee18f 100644 --- a/helpers/index.js +++ b/helpers/index.js @@ -51,7 +51,6 @@ const RetryClient = require('./retry-client'); const retryRequest = require('./retry-request'); const getBounceInfo = require('./get-bounce-info'); const getLogsCsv = require('./get-logs-csv'); -const smtp = require('./smtp'); const imap = require('./imap'); const pop3 = require('./pop3'); const ServerShutdownError = require('./server-shutdown-error'); @@ -99,6 +98,32 @@ const sendApn = require('./send-apn'); const getApnCerts = require('./get-apn-certs'); const WKD = require('./wkd'); const asctime = require('./asctime'); +const onConnect = require('./on-connect'); +const onClose = require('./on-close'); +const onMailFrom = require('./on-mail-from'); +const onData = require('./on-data'); +const onRcptTo = require('./on-rcpt-to'); +const onDataSMTP = require('./on-data-smtp'); +const onDataMX = require('./on-data-mx'); +const parseHostFromDomainOrAddress = require('./parse-host-from-domain-or-address'); +const isAllowlisted = require('./is-allowlisted'); +const isGreylisted = require('./is-greylisted'); +const getGreylistKey = require('./get-greylist-key'); +const getHeaders = require('./get-headers'); +const getFromAddress = require('./get-from-address'); +const updateSession = require('./update-session'); +const getAttributes = require('./get-attributes'); +const isSilentBanned = require('./is-silent-banned'); +const isBackscatterer = require('./is-backscatterer'); +const DenylistError = require('./denylist-error'); +const isDenylisted = require('./is-denylisted'); +const hasFingerprintExpired = require('./has-fingerprint-expired'); +const isArbitrary = require('./is-arbitrary'); +const updateHeaders = require('./update-headers'); +const getRecipients = require('./get-recipients'); +const isAuthenticatedMessage = require('./is-authenticated-message'); +const getForwardingAddresses = require('./get-forwarding-addresses'); +const MessageSplitter = require('./message-splitter'); module.exports = { decrypt, @@ -151,7 +176,6 @@ module.exports = { retryRequest, getBounceInfo, getLogsCsv, - smtp, imap, pop3, ServerShutdownError, @@ -198,5 +222,31 @@ module.exports = { sendApn, getApnCerts, WKD, - asctime + asctime, + onConnect, + onClose, + onMailFrom, + onData, + onRcptTo, + parseHostFromDomainOrAddress, + isAllowlisted, + isGreylisted, + getGreylistKey, + onDataSMTP, + onDataMX, + getHeaders, + getFromAddress, + updateSession, + getAttributes, + isSilentBanned, + isBackscatterer, + DenylistError, + isDenylisted, + hasFingerprintExpired, + isArbitrary, + updateHeaders, + getRecipients, + isAuthenticatedMessage, + getForwardingAddresses, + MessageSplitter }; diff --git a/helpers/is-allowlisted.js b/helpers/is-allowlisted.js new file mode 100644 index 0000000000..da682cca76 --- /dev/null +++ b/helpers/is-allowlisted.js @@ -0,0 +1,98 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const punycode = require('node:punycode'); +const { isIP } = require('node:net'); + +const isLocalhost = require('is-localhost-ip'); +const localhostUrl = require('localhost-url-regex'); +const pWaitFor = require('p-wait-for'); +const { boolean } = require('boolean'); +const isFQDN = require('is-fqdn'); +const { isEmail } = require('validator'); + +const config = require('#config'); +const env = require('#config/env'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); +const parseRootDomain = require('#helpers/parse-root-domain'); + +// dynamically import private-ip +let isPrivateIP; +import('private-ip').then((obj) => { + isPrivateIP = obj.default; +}); + +// eslint-disable-next-line complexity +async function isAllowlisted(val, client, resolver) { + const lowerCased = punycode.toASCII(val).toLowerCase().trim(); + + // check hard-coded allowlist + if (config.allowlist.has(lowerCased)) return true; + + // check hard-coded truth source list + if (config.truthSources.has(lowerCased)) return true; + + if (!isPrivateIP) await pWaitFor(() => Boolean(isPrivateIP)); + + // if it's localhost or local IP address then return early + if (localhostUrl().test(val) || isPrivateIP(val) || (await isLocalhost(val))) + return true; + + // if it is a FQDN and ends with restricted domain + if ( + isFQDN(val) && + !val.endsWith('.edu.cn') && + !val.endsWith('.edu.eg') && + !val.endsWith('.edu.ge') && + !val.endsWith('.edu.gr') && + !val.endsWith('.edu.gt') && + !val.endsWith('.edu.hk') && + !val.endsWith('.edu.kg') && + !val.endsWith('.edu.lk') && + !val.endsWith('.edu.my') && + !val.endsWith('.edu.om') && + !val.endsWith('.edu.pe') && + !val.endsWith('.edu.pk') && + !val.endsWith('.edu.pl') && + !val.endsWith('.edu.sg') && + !val.endsWith('.edu.vn') && + !val.endsWith('.edu.za') && + !val.endsWith('.edu.eu.org') && + config.restrictedDomains.some( + (ext) => lowerCased === ext || lowerCased.endsWith(`.${ext}`) + ) + ) + return true; + + // if it was an email address or domain and was our domain then whitelist + if (isEmail(val, { ignore_max_length: true })) { + const domain = parseHostFromDomainOrAddress(val); + const root = parseRootDomain(domain); + if (root === env.WEB_HOST) return true; + } else if (isFQDN(val)) { + const root = parseRootDomain(val); + if (root === env.WEB_HOST) return true; + } else if (isIP(val)) { + // reverse lookup IP and if it was allowlisted then return early + const [clientHostname] = await resolver.reverse(val); + if (isFQDN(clientHostname)) { + // check domain + if (await isAllowlisted(clientHostname, client, resolver)) return true; + // check root domain (if differed) + const root = parseRootDomain(clientHostname); + if ( + clientHostname !== root && + (await isAllowlisted(root, client, resolver)) + ) + return true; + } + } + + const result = await client.get(`allowlist:${lowerCased}`); + + return boolean(result); +} + +module.exports = isAllowlisted; diff --git a/helpers/is-arbitrary.js b/helpers/is-arbitrary.js new file mode 100644 index 0000000000..6770d66026 --- /dev/null +++ b/helpers/is-arbitrary.js @@ -0,0 +1,210 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const RE2 = require('re2'); +const isSANB = require('is-string-and-not-blank'); + +const config = require('#config'); +const env = require('#config/env'); +const SMTPError = require('#helpers/smtp-error'); +const checkSRS = require('#helpers/check-srs'); +const logger = require('#helpers/logger'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); +const parseRootDomain = require('#helpers/parse-root-domain'); + +const BLOCKED_PHRASES = new RE2( + /recorded you|account is hacked|personal data has leaked/im +); + +const REGEX_BITCOIN = new RE2(/bitcoin|btc/im); +const REGEX_PASSWORD_MALWARE_INFECTED_VIDEO = new RE2( + /hacked|malware|infected|trojan|recorded you/im +); + +// TODO: remove yum here and wrap these with spaces or something +const REGEX_SYSADMIN_SUBJECT = new RE2( + /docker|system events|monit alert|cron|yum|exim|backup|logwatch|unattended-upgrades/im +); + +/* +// NOTE: not being used but keeping it here in case we need it for future +const YAHOO_DOMAINS = new Set([ + 'yahoo.com', + 'aol.com', + 'verizon.net', + 'yahoo.de', + 'yahoo.ca', + 'rocketmail.com', + 'yahoo.com.mx', + 'yahoo.co.in', + 'yahoo.co.uk', + 'yahoo.com.au', + 'yahoo.com.br', + 'sky.com' +]); +*/ + +const REGEX_DOMAIN = new RE2(new RegExp(env.WEB_HOST, 'im')); +const REGEX_APP_NAME = new RE2(new RegExp(env.APP_NAME, 'im')); + +// eslint-disable-next-line complexity +function isArbitrary(session, headers, bodyStr) { + let subject = headers.getFirst('subject'); + if (!isSANB(subject)) subject = null; + + const from = headers.getFirst('from'); + + // rudimentary blocking + if (subject && BLOCKED_PHRASES.test(subject)) + throw new SMTPError( + `Blocked phrase, please forward this to ${config.abuseEmail}` + ); + + // check for btc crypto scam + if ( + isSANB(bodyStr) && + REGEX_BITCOIN.test(bodyStr) && + REGEX_PASSWORD_MALWARE_INFECTED_VIDEO.test(bodyStr) + ) + throw new SMTPError( + `Blocked crypto scam, please forward this to ${config.abuseEmail}` + ); + + // + // NOTE: due to unprecendented spam from Microsoft's "onmicrosoft.com" domain + // we had to implement arbitrary rule to block spam from them + // + // + // + // + // + if (session.originalFromAddressRootDomain === 'onmicrosoft.com') { + const msHeader = headers.getFirst('X-MS-Exchange-Authentication-Results'); + if ( + msHeader && + (msHeader.includes('spf=fail') || + msHeader.includes('spf=none') || + msHeader.includes('dmarc=fail') || + msHeader.includes('spf=softfail')) + ) + throw new SMTPError( + 'Due to spam from onmicrosoft.com we have implemented restrictions; see https://old.reddit.com/r/msp/comments/16n8p0j/spam_increase_from_onmicrosoftcom_addresses/ ;' + ); + } + + // + // due to high amount of Microsoft spam we are blocking their bounces + // if from postmaster@outlook.com and message is "Undeliverable: " + // + if ( + session.originalFromAddress === 'postmaster@outlook.com' && + subject && + subject.startsWith('Undeliverable: ') + ) + throw new SMTPError( + 'Due to spam from onmicrosoft.com we have implemented restrictions; see https://old.reddit.com/r/msp/comments/16n8p0j/spam_increase_from_onmicrosoftcom_addresses/ ;' + ); + + // + // due to high spam from 163.com we are blocking their bounces + // + if ( + session.originalFromAddress === 'postmaster@163.com' && + subject && + subject.includes('系统退信') + ) + throw new SMTPError( + 'Due to spam from postmaster@163.com we have implemented bounce block restrictions' + ); + + // + // due to microsoft and docusign scam + // + if ( + session.originalFromAddress === 'dse_na4@docusign.net' && + (session?.spf?.domain.endsWith('.onmicrosoft.com') || + session?.spf?.domain === 'onmicrosoft.com') + ) + throw new SMTPError( + 'Due to spam from onmicrosoft.com and docusign.net SPF we have implemented restrictions; see https://old.sp/comments/16n8p0j/spam_increase_from_onmicrosoftcom_addresses/ ;' + ); + + // + // this checks for messages that aren't coming from us + // and contain a spoofed "From" address that looks like it's from us + // + if ( + (!session.hadAlignedAndPassingDKIM || + (session.hadAlignedAndPassingDKIM && + session.originalFromAddressRootDomain !== env.WEB_HOST)) && + (session.spfFromHeader.status.result !== 'pass' || + session.originalFromAddressRootDomain !== env.WEB_HOST) && + (REGEX_DOMAIN.test(from) || REGEX_APP_NAME.test(from)) + ) + throw new SMTPError( + `Blocked spoofing, please forward this to ${config.abuseEmail}` + ); + + // + // here is where we attempt to protect users from spammers + // that impersonate spoofing the "From" address in an email + // as if it's from their domain name, which is a common attack + // + // note that we only check this if DKIM wasn't aligned and passing + // and if the sender's hostname is not same as From header's hostname + // so we use `session.hasSameHostnameAsFrom` for this (which is set in `helpers/update-session.js`) + // because that's an obvious signal that it's coming from the same address + // due to the resolved client hostname of the reverse lookup on the `session.remoteAddress` + // + // the way that we check this is quite simple: + // all we need to do is check if any of the RCPT TO values have a matching root domain as From header + // AND if the From header was not SPF aligned, then throw the error + // + // NOTE: we do have one exception to this, and it is that often + // system administrators will set up cron jobs to send them alerts + // (which often are sent to the same domain name) and are lacking passing SPF + // (or the administrator simply never configured an SPF policy) + // therefore we check for those cases with a simple regular expression against the Subject line + // and if the SPF policy was not strictly failing, then it's probably a legitimate message + // + if (!session.hasSameHostnameAsFrom && !session.hadAlignedAndPassingDKIM) { + const hasSameRcptToAsFrom = session.envelope.rcptTo.some( + (to) => + parseRootDomain(parseHostFromDomainOrAddress(checkSRS(to.address))) === + session.originalFromAddressRootDomain + ); + if ( + hasSameRcptToAsFrom && + session.spfFromHeader.status.result !== 'pass' && + !( + session.spfFromHeader.status.result !== 'fail' && + subject && + REGEX_SYSADMIN_SUBJECT.test(subject) + ) + ) { + // TODO: until we're certain this is properly working we're going to monitor it with code bug to admins + const err = new TypeError( + `Spoofing detected: ${session.originalFromAddressRootDomain}` + ); + err.session = session; + logger.fatal(err); + + throw new SMTPError( + 'Message likely to be spoofing attack and was rejected due to lack of SPF alignment with From header', + { responseCode: 421 } + ); + } + } + + // + // NOTE: we may want to handle Reply-To attack where the reply address is different than the From address + // BUT... this is typically handled with the logic above + // (for example, someone sends an email "We have your password, send us BTC") + // (and the From is you@yourdomain.com and the To is you@yourdomain.com, but the Reply-To needs to be different) + // (otherwise the spammer/attacker would never get the response to the email) + // +} + +module.exports = isArbitrary; diff --git a/helpers/is-authenticated-message.js b/helpers/is-authenticated-message.js new file mode 100644 index 0000000000..f91dd77ffb --- /dev/null +++ b/helpers/is-authenticated-message.js @@ -0,0 +1,163 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const os = require('node:os'); + +const _ = require('lodash'); +const isSANB = require('is-string-and-not-blank'); +const { spf } = require('mailauth/lib/spf'); +const { authenticate } = require('mailauth'); + +const SMTPError = require('#helpers/smtp-error'); +const config = require('#config'); +const parseRootDomain = require('#helpers/parse-root-domain'); + +const HOSTNAME = os.hostname(); + +const UBUNTU_DOMAINS = Object.keys(config.ubuntuTeamMapping); + +// eslint-disable-next-line complexity +async function isAuthenticatedMessage(raw, session, resolver) { + const options = { + ip: session.remoteAddress, + helo: session.hostNameAppearsAs, + mta: HOSTNAME, + resolver: resolver.resolve + }; + + const [results, spfFromHeader] = await Promise.all([ + authenticate(raw, { + ...options, + sender: session.envelope.mailFrom.address, + seal: config.signatureData + }), + spf({ + ...options, + sender: session.originalFromAddress + }) + ]); + + session.dkim = results.dkim; + session.spf = results.spf; + + // + // safeguard in case mailauth is not accurate or breaks + // + if ( + !_.isObject(session.spf) || + !_.isObject(session.spf.status) || + !isSANB(session.spf.status.result) + ) { + session.spf = { + status: { + // + result: 'temperror' + } + }; + } + + session.arc = results.arc; + session.dmarc = results.dmarc; + session.arcSealedHeaders = results.headers; + session.bimi = results.bimi; + session.receivedChain = results.receivedChain; + + // we check that if SPF was aligned with "From" header + // (this is similar to DKIM alignment, but an extra step we take to prevent spam and spoofing) + // + session.spfFromHeader = spfFromHeader; + + // + // safeguard in case mailauth is not accurate or breaks + // + if ( + !_.isObject(session.spfFromHeader) || + !_.isObject(session.spfFromHeader.status) || + !isSANB(session.spfFromHeader.status.result) + ) { + session.spfFromHeader = { + status: { + // + result: 'temperror' + } + }; + } + + session.signingDomains = new Set(); + session.alignedDKIMResults = []; + + if ( + _.isObject(session.dkim) && + _.isArray(session.dkim.results) && + !_.isEmpty(session.dkim.results) + ) { + for (const result of session.dkim.results) { + if ( + _.isObject(result) && + _.isObject(result.status) && + result.status.result === 'pass' + ) { + if (isSANB(result.status.aligned)) + session.alignedDKIMResults.push(result); + + // + // check DKIM signature domain against denylist and silent ban + // + + if (isSANB(result.signingDomain)) { + const rootSigningDomain = parseRootDomain(result.signingDomain); + session.signingDomains.add(result.signingDomain); + session.signingDomains.add(rootSigningDomain); + } + } + } + } + + session.hadAlignedAndPassingDKIM = session.alignedDKIMResults.length > 0; + + // + // NOTE: since our SPF check does not have PTR support + // we need to conditionally set SPF passing to true + // in order to override DMARC and strict SPF checks + // + + // + // only reject if ARC was not passing + // and DMARC fail with p=reject policy + // + if ( + // session.spf.status.result !== 'pass' && + session.dmarc && + session.dmarc.policy === 'reject' && + session.dmarc.status && + session.dmarc.status.result === 'fail' + ) { + throw new SMTPError( + "The email sent has failed DMARC validation and is rejected due to the domain's DMARC policy", + { + // if spf status was temperror then retry + responseCode: session.spf.status.result === 'temperror' ? 421 : 550 + } + ); + } + + // if no DMARC and SPF had hardfail and no aligned DKIM then reject + // NOTE: it'd be nice if we alerted admins of SPF permerror due to SPF misconfiguration + if ( + session.spf.status.result === 'fail' && + session.dmarc && + session.dmarc.status && + session.dmarc.status.result === 'none' && + !session.hadAlignedAndPassingDKIM && + // NOTE: this is an exception for Ubuntu since they have custom postfix setup + (!session.resolvedRootClientHostname || + !UBUNTU_DOMAINS.includes(session.resolvedRootClientHostname)) + ) + throw new SMTPError( + "The email sent has failed SPF validation and is rejected due to the domain's SPF hard fail policy" + ); +} + +module.exports = isAuthenticatedMessage; diff --git a/helpers/is-backscatterer.js b/helpers/is-backscatterer.js new file mode 100644 index 0000000000..848e6d926c --- /dev/null +++ b/helpers/is-backscatterer.js @@ -0,0 +1,50 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const { isIP } = require('node:net'); + +const pFilter = require('p-filter'); +const { boolean } = require('boolean'); + +const DenylistError = require('#helpers/denylist-error'); +const config = require('#config'); +const isAllowlisted = require('#helpers/is-allowlisted'); + +async function isBackscatterer(value, client, resolver) { + if (!Array.isArray(value)) value = [value]; + + const filtered = await pFilter( + value, + async (v) => { + // filter out IP only values + if (!isIP(v)) return false; + + // filter out IP's that were allowlisted + // (this includes local IP's and our own servers) + // (isAllowlisted function will perform reverse lookup on IP address) + if (await isAllowlisted(v, client, resolver)) return false; + + return true; + }, + { concurrency: config.concurrency } + ); + + const results = + filtered.length > 0 + ? await client.mget(filtered.map((v) => `backscatter:${v}`)) + : []; + + for (const [i, result] of results.entries()) { + if (!boolean(result)) continue; + const v = filtered[i]; + throw new DenylistError( + `The IP ${v} is listed on https://www.backscatterer.org. To request removal, you must visit https://www.backscatterer.org/index.php?target=test&ip=${v} ;`, + 421, + v + ); + } +} + +module.exports = isBackscatterer; diff --git a/helpers/is-denylisted.js b/helpers/is-denylisted.js new file mode 100644 index 0000000000..56e88b9f95 --- /dev/null +++ b/helpers/is-denylisted.js @@ -0,0 +1,113 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const punycode = require('node:punycode'); +const { isIP } = require('node:net'); + +const isFQDN = require('is-fqdn'); +const { boolean } = require('boolean'); +const { isEmail } = require('validator'); + +const DenylistError = require('#helpers/denylist-error'); +const config = require('#config'); +const isAllowlisted = require('#helpers/is-allowlisted'); +const parseRootDomain = require('#helpers/parse-root-domain'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); + +function createDenylistError(val) { + let str = 'value'; + if (isEmail(val, { ignore_max_length: true })) str = 'address'; + else if (isIP(val)) str = 'IP'; + else if (isFQDN(val)) str = 'domain'; + return new DenylistError( + `The ${str} ${val} is denylisted by ${config.urls.web} ; To request removal, you must visit ${config.urls.web}/denylist?q=${val} ;`, + 421, + val + ); +} + +// eslint-disable-next-line complexity +async function isDenylisted(value, client, resolver) { + if (!Array.isArray(value)) value = [value]; + + // lowercase and trim all the values + value = value.map((v) => punycode.toASCII(v).toLowerCase().trim()); + + for (const v of value) { + // if the value was in hard-coded denylist then exit early + if (config.denylist.has(v)) throw createDenylistError(v); + + // if it was an email address then check domain and root domain (if differs) against hard-coded denylist + if (isEmail(v, { ignore_max_length: true })) { + const domain = parseHostFromDomainOrAddress(v); + if (config.denylist.has(domain)) throw createDenylistError(domain); + const root = parseRootDomain(domain); + if (domain !== root && config.denylist.has(root)) + throw createDenylistError(root); + } + + // if it was a domain name then check root domain against hard-coded denylist if differs + if (isFQDN(v)) { + if (config.denylist.has(v)) throw createDenylistError(v); + const root = parseRootDomain(v); + if (v !== root && config.denylist.has(root)) + throw createDenylistError(root); + } + + // if allowlisted return early + // (note this does a reverse lookup on IP address to check hostname of IP against allowlist too) + // eslint-disable-next-line no-await-in-loop + if (await isAllowlisted(v, client, resolver)) return false; + + // TODO: if it was a FQDN then lookup A records for domain and root domain (?) + + // + // check if the root domain is allowlisted but IFF the value was different + // (only applies to email and FQDN values) + // + if (isEmail(v, { ignore_max_length: true }) || isFQDN(v)) { + const root = parseRootDomain( + isFQDN(v) ? v : parseHostFromDomainOrAddress(v) + ); + + if (root !== v) { + if (config.denylist.has(root)) throw createDenylistError(root); + + const isRootDomainAllowlisted = client + ? // eslint-disable-next-line no-await-in-loop + await isAllowlisted(root, client, resolver) + : false; + // + // if the root domain was allowlisted and it was an email + // then we need to check the combination of: + // `denylist:domain:email` against the denylist + // + // (this is a safeguard in case the email is not denylisted but domain:email is) + // + if (isRootDomainAllowlisted) { + if (isEmail(v, { ignore_max_length: true })) { + // eslint-disable-next-line no-await-in-loop + const result = await client.get(`denylist:${root}:${v}`); + + if (boolean(result)) throw createDenylistError(v); + } + } else { + // check redis denylist on root domain + // eslint-disable-next-line no-await-in-loop + const isRootDomainDenylisted = await client.get(`denylist:${root}`); + + if (boolean(isRootDomainDenylisted)) throw createDenylistError(root); + } + } + } + + // eslint-disable-next-line no-await-in-loop + const denylisted = await client.get(`denylist:${v}`); + + if (boolean(denylisted)) throw createDenylistError(v); + } +} + +module.exports = isDenylisted; diff --git a/helpers/is-greylisted.js b/helpers/is-greylisted.js new file mode 100644 index 0000000000..cd2fd0277d --- /dev/null +++ b/helpers/is-greylisted.js @@ -0,0 +1,63 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const prettyMilliseconds = require('pretty-ms'); + +const getGreylistKey = require('#helpers/get-greylist-key'); +const SMTPError = require('#helpers/smtp-error'); +const config = require('#config'); + +async function isGreylisted(session, client) { + // safeguard to return early if the sender was allowlisted + if (session.isAllowlisted) return; + + const key = getGreylistKey( + session.resolvedRootClientHostname || session.remoteAddress + ); + let value = await client.get(key); + if (value) { + // parse the value from a string to an integer (date) + const time = new Date(Number.parseInt(value, 10)).getTime(); + // validate date stored is not NaN and is numeric positive time + if (Number.isFinite(time) && time > 0) { + // example: + // - value stored in redis is 4:05pm (original arrival time + 5m) + // - currently it's 4:08pm + // - msToGo = 4:05pm + 5m - 4:08pm = 4:10pm - 4:08pm = 2m + // + const msToGo = time + config.greylistTimeout - session.arrivalTime; + + if (msToGo > 0 && msToGo <= config.greylistTimeout) { + throw new SMTPError( + `Message was greylisted, try again in ${prettyMilliseconds(msToGo, { + verbose: true, + secondsDecimalDigits: 0 + })}; see https://forwardemail.net/faq#do-you-have-a-greylist for more information`, + { responseCode: 450, ignoreHook: true } + ); + } + } else { + // value stored was invalid so we need to reset it + value = null; + } + } + + // if there was no value stored then set one and throw an error + if (!value) { + await client.set(key, session.arrivalTime, 'PX', config.greylistTtlMs); + throw new SMTPError( + `Message was greylisted, try again in ${prettyMilliseconds( + config.greylistTimeout, + { + verbose: true, + secondsDecimalDigits: 0 + } + )}; see https://forwardemail.net/faq#do-you-have-a-greylist for more information`, + { responseCode: 450, ignoreHook: true } + ); + } +} + +module.exports = isGreylisted; diff --git a/helpers/is-silent-banned.js b/helpers/is-silent-banned.js new file mode 100644 index 0000000000..3165041418 --- /dev/null +++ b/helpers/is-silent-banned.js @@ -0,0 +1,41 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const punycode = require('node:punycode'); + +const pFilter = require('p-filter'); +const { boolean } = require('boolean'); + +const config = require('#config'); +const isAllowlisted = require('#helpers/is-allowlisted'); + +async function isSilentBanned(value, client, resolver) { + if (!Array.isArray(value)) value = [value]; + + // lowercase and trim all the values + value = value.map((v) => punycode.toASCII(v).toLowerCase().trim()); + + // `value` can be anything arbitrary (IP, hostname, email address) + // and it can be an Array of Strings or a String + + // filter out values from array if they were allowlisted + const filtered = await pFilter( + value, + async (v) => { + const allowlisted = await isAllowlisted(v, client, resolver); + return !allowlisted; + }, + { concurrency: config.concurrency } + ); + + const results = + filtered.length > 0 + ? await client.mget(filtered.map((v) => `silent:${v}`)) + : []; + + return results.some((result) => boolean(result)); +} + +module.exports = isSilentBanned; diff --git a/helpers/message-splitter.js b/helpers/message-splitter.js new file mode 100644 index 0000000000..68503dde84 --- /dev/null +++ b/helpers/message-splitter.js @@ -0,0 +1,185 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const { Buffer } = require('node:buffer'); +const { Transform } = require('node:stream'); + +// +// Many thanks to Andris Reissman +// +// +const bytes = require('bytes'); +const { Headers } = require('mailsplit'); +const { Iconv } = require('iconv'); + +/** + * MessageSplitter instance is a transform stream that separates message headers + * from the rest of the body. Headers are emitted with the 'headers' event. Message + * body is passed on as the resulting stream. + */ +class MessageSplitter extends Transform { + constructor(options) { + super(options); + this.lastBytes = Buffer.alloc(4); + this.headersParsed = false; + this.headerBytes = 0; + this.headerChunks = []; + this.rawHeaders = false; + // this.bodySize = 0; + this.dataBytes = 0; + this._maxBytes = + (options.maxBytes && Number(options.maxBytes)) || + Number.POSITIVE_INFINITY; + this.sizeExceeded = false; + } + + /** + * Keeps count of the last 4 bytes in order to detect line breaks on chunk boundaries + * + * @param {Buffer} data Next data chunk from the stream + */ + _updateLastBytes(data) { + const lblen = this.lastBytes.length; + const nblen = Math.min(data.length, lblen); + + // shift existing bytes + for (let i = 0, length = lblen - nblen; i < length; i++) { + this.lastBytes[i] = this.lastBytes[i + nblen]; + } + + // add new bytes + for (let i = 1; i <= nblen; i++) { + this.lastBytes[lblen - i] = data[data.length - i]; + } + } + + /** + * Finds and removes message headers from the remaining body. We want to keep + * headers separated until final delivery to be able to modify these + * + * @param {Buffer} data Next chunk of data + * @return {Boolean} Returns true if headers are already found or false otherwise + */ + _checkHeaders(data) { + if (this.headersParsed) { + return true; + } + + const lblen = this.lastBytes.length; + let headerPos = 0; + this.curLinePos = 0; + for ( + let i = 0, length = this.lastBytes.length + data.length; + i < length; + i++ + ) { + const chr = i < lblen ? this.lastBytes[i] : data[i - lblen]; + + if (chr === 0x0a && i) { + const pr1 = i - 1 < lblen ? this.lastBytes[i - 1] : data[i - 1 - lblen]; + const pr2 = + i > 1 + ? i - 2 < lblen + ? this.lastBytes[i - 2] + : data[i - 2 - lblen] + : false; + if (pr1 === 0x0a) { + this.headersParsed = true; + headerPos = i - lblen + 1; + this.headerBytes += headerPos; + break; + } else if (pr1 === 0x0d && pr2 === 0x0a) { + this.headersParsed = true; + headerPos = i - lblen + 1; + this.headerBytes += headerPos; + break; + } + } + } + + if (this.headersParsed) { + this.headerChunks.push(data.slice(0, headerPos)); + this.rawHeaders = Buffer.concat(this.headerChunks, this.headerBytes); + this.headerChunks = null; + this.headers = new Headers(this.rawHeaders, { Iconv }); + // this.emit('headers', this.headers); + if (data.length - 1 > headerPos) { + const chunk = data.slice(headerPos); + // this.bodySize += chunk.length; + + // this would be the first chunk of data sent downstream + // from now on we keep header and body separated until final delivery + setImmediate(() => this.push(chunk)); + } + + return false; + } + + this.headerBytes += data.length; + this.headerChunks.push(data); + + // store last 4 bytes to catch header break + this._updateLastBytes(data); + + return false; + } + + _transform(chunk, encoding, callback) { + if (!chunk || chunk.length === 0) return callback(); + + // stop reading if max size reached + this.dataBytes += chunk.length; + this.sizeExceeded = this.dataBytes > this._maxBytes; + if (this.sizeExceeded) { + const err = new Error( + `Maximum allowed message size ${bytes(this._maxBytes)} exceeded` + ); + err.responseCode = 552; + return callback(err); + } + + if (typeof chunk === 'string') chunk = Buffer.from(chunk, encoding); + + let headersFound; + + try { + headersFound = this._checkHeaders(chunk); + } catch (err) { + return callback(err); + } + + if (headersFound) { + // this.bodySize += chunk.length; + this.push(chunk); + } + + setImmediate(callback); + } + + _flush(callback) { + if (this.headerChunks) { + // all chunks are checked but we did not find where the body starts + // so emit all we got as headers and push empty line as body + this.headersParsed = true; + // add header terminator + this.headerChunks.push(Buffer.from('\r\n\r\n')); + this.headerBytes += 4; + // join all chunks into a header block + this.rawHeaders = Buffer.concat(this.headerChunks, this.headerBytes); + + this.headers = new Headers(this.rawHeaders); + + // this.emit('headers', this.headers); + this.headerChunks = null; + + // this is our body + this.push(Buffer.from('\r\n')); + } + + callback(); + } +} + +module.exports = MessageSplitter; diff --git a/helpers/on-auth.js b/helpers/on-auth.js index 2485749fe3..ed15e76391 100644 --- a/helpers/on-auth.js +++ b/helpers/on-auth.js @@ -31,7 +31,7 @@ const env = require('#config/env'); const getQueryResponse = require('#helpers/get-query-response'); const i18n = require('#helpers/i18n'); const isValidPassword = require('#helpers/is-valid-password'); -const onConnect = require('#helpers/smtp/on-connect'); +const onConnect = require('#helpers/on-connect'); const { encrypt } = require('#helpers/encrypt-decrypt'); const onConnectPromise = pify(onConnect); diff --git a/helpers/on-close.js b/helpers/on-close.js new file mode 100644 index 0000000000..c1825ee81a --- /dev/null +++ b/helpers/on-close.js @@ -0,0 +1,52 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const config = require('#config'); +const logger = require('#helpers/logger'); + +async function onClose(session) { + // NOTE: do not change this prefix unless you also change it in `helpers/on-connect.js` + const prefix = `concurrent_${this.constructor.name.toLowerCase()}_${ + config.env + }`; + await Promise.all([ + // + // decrease # concurrent connections for + // client hostname or remote address + // + (async () => { + if (!session?.resolvedRootClientHostname && !session?.remoteAddress) + return; + try { + const key = `${prefix}:${ + session.resolvedRootClientHostname || session.remoteAddress + }`; + const count = await this.client.incrby(key, 0); + if (count > 0) await this.client.decr(key); + } catch (err) { + logger.fatal(err); + } + })(), + // + // decrease # concurrent connections for + // the logged in alias or domain (if using catch-all password) + // + (async () => { + // ignore unauthenticated sessions + if (!session?.user?.alias_id && !session?.user?.domain_id) return; + try { + const key = `${prefix}:${ + session.user.alias_id || session.user.domain_id + }`; + const count = await this.client.incrby(key, 0); + if (count > 0) await this.client.decr(key); + } catch (err) { + logger.fatal(err); + } + })() + ]); +} + +module.exports = onClose; diff --git a/helpers/on-connect.js b/helpers/on-connect.js new file mode 100644 index 0000000000..ad7589a1d9 --- /dev/null +++ b/helpers/on-connect.js @@ -0,0 +1,140 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const punycode = require('node:punycode'); + +const isFQDN = require('is-fqdn'); + +const SMTPError = require('#helpers/smtp-error'); +const ServerShutdownError = require('#helpers/server-shutdown-error'); +const config = require('#config'); +const env = require('#config/env'); +const isAllowlisted = require('#helpers/is-allowlisted'); +const parseRootDomain = require('#helpers/parse-root-domain'); +const refineAndLogError = require('#helpers/refine-and-log-error'); + +async function onConnect(session, fn) { + this.logger.debug('CONNECT', { session }); + + if (this.isClosing) return fn(new ServerShutdownError()); + + // this is used for setting Date header if missing on SMTP submission + session.arrivalDate = new Date(); + + // this is used for sending bounces for MX server + session.arrivalDateFormatted = session.arrivalDate + .toISOString() + .split('T')[0]; + + // this is used for greylisting and in other places + session.arrivalTime = session.arrivalDate.getTime(); + + // lookup the client hostname + try { + const [clientHostname] = await this.resolver.reverse(session.remoteAddress); + if (isFQDN(clientHostname)) { + // do we need this still (?) + let domain = clientHostname.toLowerCase().trim(); + try { + domain = punycode.toASCII(domain); + } catch { + // ignore punycode conversion errors + } + + session.resolvedClientHostname = domain; + session.resolvedRootClientHostname = parseRootDomain( + session.resolvedClientHostname + ); + } + } catch (err) { + // + // NOTE: the native Node.js DNS module would throw an error previously + // + // + if (env.NODE_ENV !== 'production') this.logger.debug(err, { session }); + } + + // + // check if the connecting remote IP address is allowlisted + // + session.isAllowlisted = false; + if (session.resolvedClientHostname && session.resolvedRootClientHostname) { + // check the root domain + session.isAllowlisted = await isAllowlisted( + session.resolvedRootClientHostname, + this.client, + this.resolver + ); + if (session.isAllowlisted) { + session.allowlistValue = session.resolvedRootClientHostname; + } else if ( + session.resolvedRootClientHostname !== session.resolvedClientHostname + ) { + // if differed, check the sub-domain + session.isAllowlisted = await isAllowlisted( + session.resolvedClientHostname, + this.client, + this.resolver + ); + + if (session.isAllowlisted) + session.allowlistValue = session.resolvedClientHostname; + } + } + + if (!session.isAllowlisted) { + session.isAllowlisted = await isAllowlisted( + session.remoteAddress, + this.client, + this.resolver + ); + if (session.isAllowlisted) session.allowlistValue = session.remoteAddress; + } + + // + // NOTE: we do not check in onConnect for denylist/silent/backscatter in MX server + // because we need to let users know of a given email/sender that was rejected + // so we need to store a log for it with subject, RCPT TO, MAIL FROM, etc + // and if we were to throw an error here if someone was denylisted for instance, + // then the connection would not proceed to onRcptTo, onMailFrom, etc + // and therefore it would not be possible to store an error log for this case + // + // additionally, we do not check against denylist/silent/backscatter for SMTP server + // because AUTH is required for a user to access the SMTP server anyways + // + + // + // NOTE: we return early here because we do not want to limit concurrent connections from allowlisted values + // + // + if (session.isAllowlisted) return fn(); + + // + // do not allow more than 10 concurrent connections using constructor + // + try { + // NOTE: do not change this prefix unless you also change it in `helpers/on-close.js` + const prefix = `concurrent_${this.constructor.name.toLowerCase()}_${ + config.env + }`; + const key = `${prefix}:${ + session.resolvedRootClientHostname || session.remoteAddress + }`; + const count = await this.client.incr(key); + await this.client.pexpire(key, config.socketTimeout); + if (count >= 10) + throw new SMTPError( + `Too many concurrent connections from ${ + session.resolvedRootClientHostname || session.remoteAddress + }`, + { responseCode: 421, ignoreHook: true } + ); + fn(); + } catch (err) { + fn(refineAndLogError(err, session, false, this)); + } +} + +module.exports = onConnect; diff --git a/helpers/on-data-mx.js b/helpers/on-data-mx.js new file mode 100644 index 0000000000..a922f492a0 --- /dev/null +++ b/helpers/on-data-mx.js @@ -0,0 +1,296 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const { Buffer } = require('node:buffer'); + +const SpamScanner = require('spamscanner'); +const _ = require('lodash'); +const bytes = require('bytes'); +const isSANB = require('is-string-and-not-blank'); +const { SRS } = require('sender-rewriting-scheme'); +const { isEmail } = require('validator'); +const { sealMessage } = require('mailauth'); + +const DenylistError = require('#helpers/denylist-error'); +const SMTPError = require('#helpers/smtp-error'); +const checkSRS = require('#helpers/check-srs'); +const config = require('#config'); +const env = require('#config/env'); +const getRecipients = require('#helpers/get-recipients'); +const hasFingerprintExpired = require('#helpers/has-fingerprint-expired'); +const isArbitrary = require('#helpers/is-arbitrary'); +const isAuthenticatedMessage = require('#helpers/is-authenticated-message'); +const isBackscatterer = require('#helpers/is-backscatterer'); +const isDenylisted = require('#helpers/is-denylisted'); +const isGreylisted = require('#helpers/is-greylisted'); +const isSilentBanned = require('#helpers/is-silent-banned'); +const logger = require('#helpers/logger'); +const parseUsername = require('#helpers/parse-username'); +const updateHeaders = require('#helpers/update-headers'); + +const srs = new SRS(config.srs); + +const scanner = new SpamScanner({ + logger, + clamscan: config.env === 'test', + memoize: { + // since memoizee doesn't support supplying mb or gb of cache size + // we can calculate how much the maximum could potentially be + // the max length of a domain name is 253 characters (bytes) + // and if we want to store up to 1 GB in memory, that's + // `Math.floor(bytes('1GB') / 253)` = 4244038 (domains) + // note that this is per thread, so if you have 4 core server + // you will have 4 threads, and therefore need 4 GB of free memory + size: Math.floor(bytes('0.5GB') / 253) + } +}); + +// +// TODO: all counters should be reflected in new deliverability dashboard for users +// + +async function updateMXHeaders(session, headers, body) { + headers.remove('x-forwardemail-sender'); + const senderHeader = []; + if ( + isSANB(session.envelope.mailFrom.address) && + isEmail(session.envelope.mailFrom.address, { ignore_max_length: true }) + ) + senderHeader.push(checkSRS(session.envelope.mailFrom.address)); + if (session.resolvedClientHostname) + senderHeader.push(session.resolvedClientHostname); + senderHeader.push(session.remoteAddress); + headers.add( + 'X-ForwardEmail-Sender', + `rfc822; ${senderHeader.join(', ')}`, + headers.lines.length + ); + if (config.env !== 'production') { + headers.remove('x-forwardemail-session-id'); + headers.add('X-ForwardEmail-Session-ID', session.id, headers.lines.length); + } + + // + // perform a friendly-from rewrite if necessary using mailauth data + // (basically if no aligned DKIM and if strict DMARC we can assume it's relying on SPF) + // + // TODO: remove other instances of session.dmarc.policy and rely on status.result + if ( + session.dmarc?.status?.result === 'pass' && + !session.hadAlignedAndPassingDKIM + ) { + session.rewriteFriendlyFrom = true; + + if (isSANB(session.envelope.mailFrom.address)) + session.envelope.mailFrom.address = srs.forward( + checkSRS(session.envelope.mailFrom.address), + env.WEB_HOST + ); + + // TODO: if sender was allowlisted then we should notify them of their issue (?) + + headers.update( + 'From', + `"${session.originalFromAddress}" <${config.friendlyFromEmail}>` + ); + headers.add('X-Original-From', session.originalFromAddress); + // + // if there was an original reply-to on the email + // then we don't want to modify it of course + // + if (!headers.getFirst('reply-to')) + headers.update('Reply-To', session.originalFromAddress); + + // rewrite ARC sealed headers with updated headers object value + session.arcSealedHeaders = await sealMessage( + Buffer.concat([headers.build(), body]), + { + ...config.signatureData, + // values from the authentication step + authResults: session.arc.authResults, + cv: session.arc.status.result + } + ); + } +} + +// TODO: add X-Original-To and Received headers to outbound SMTP (to top of message) +// and to MX below on a per-message basis for accuracy + +// eslint-disable-next-line complexity +async function onDataMX(raw, session, headers, body) { + // + // determine if we should check against backscatterer list + // (only if blank, mailer-daemon@, postmaster@, or another standard) + // (and if not allowlisted) + // + // + // + if (!session.isAllowlisted) { + let checkBackscatterer = false; + // check against MAIL FROM + if ( + isSANB(session.envelope.mailFrom.address) && + isEmail(session.envelope.mailFrom.address, { ignore_max_length: true }) + ) { + const username = parseUsername( + checkSRS(session.envelope.mailFrom.address) + ); + if (config.POSTMASTER_USERNAMES.has(username)) checkBackscatterer = true; + } else { + // MAIL FROM was <> (empty) + checkBackscatterer = true; + } + + // check against From header + if (!checkBackscatterer) { + const username = parseUsername(checkSRS(session.originalFromAddress)); + if (config.POSTMASTER_USERNAMES.has(username)) checkBackscatterer = true; + } + + // check against backscatterer list + // (it will throw a DenylistError if so) + if (checkBackscatterer) { + try { + await isBackscatterer( + session.remoteAddress, + this.client, + this.resolver + ); + } catch (err) { + // store a counter + if (err instanceof DenylistError) + await this.client.incr( + `backscatter_prevented:${session.arrivalDateFormatted}` + ); + throw err; + } + } + } + + // + // NOTE: here is where we check against denylist + // (we simply check if any of the `session.attributes` were denylisted) + // (this includes added RCPT TO values as parsed in `helpers/on-data.js`) + // (it will throw a DenylistError if so) + // + try { + await isDenylisted(session.attributes, this.client, this.resolver); + } catch (err) { + // store a counter + if (err instanceof DenylistError) + await this.client.incr( + `denylist_prevented:${session.arrivalDateFormatted}` + ); + throw err; + } + + // only let this message retry for up to 5 days + // (this throws an error if it exceeds duration) + await hasFingerprintExpired(session, this.client); + + // TODO: possibly store a counter here too + // check if the message needs to be greylisted + // (this throws an error if so) + await isGreylisted(session, this.client); + + // + // check message against DKIM, SPF, DMARC + // (this populates `session.spf`, `session.dmarc`, etc) + // (it also throws an error if it was found to be unauthenticated) + // + await isAuthenticatedMessage(raw, session, this.resolver); + + // arbitrary spam checks + // (this throws an error if any arbitrary checks were detected) + // (this relies on `isAuthenticatedMessage` to populate `session.spf` etc) + await isArbitrary(session, headers, body.toString()); + + // if there were DKIM signing domains then check them + // against the silent ban and denylists + if (session.signingDomains.size > 0) { + let silentBanned = false; + for (const signingDomain of session.signingDomains) { + // eslint-disable-next-line no-await-in-loop + silentBanned = await isSilentBanned( + signingDomain, + this.client, + this.resolver + ); + if (silentBanned) break; // break early + try { + // eslint-disable-next-line no-await-in-loop + await isDenylisted(signingDomain, this.client, this.resolver); + } catch (err) { + // store a counter + if (err instanceof DenylistError) + // eslint-disable-next-line no-await-in-loop + await this.client.incr( + `denylist_prevented:${session.arrivalDateFormatted}` + ); + throw err; + } + } + + // return early if it was silent banned + if (silentBanned) return; + } + + // TODO: session.arcSealedHeaders + + const scan = await scanner.scan(raw); + + // arbitrary tests (e.g. EICAR) always should throw + if (_.isArray(scan?.results?.arbitrary) && !_.isEmpty(scan.results.arbitrary)) + throw new SMTPError(scan.results.arbitrary.join(' '), { + responseCode: 554 + }); + + // + // NOTE: however the other spamscanner tests including these should be on a per-domain basis + // - phishing + // - executables + // - viruses + // + // (see `helpers/get-recipients.js` as these args are passed) + // + + // add X-* headers (e.g. version + report-to) + await updateHeaders(headers); + + // additional headers to add specifically for MX + // (this also does a friendly-from rewrite if necessary) + await updateMXHeaders(session, headers, body); + + // this is the core logic that determines where to forward and deliver emails to + const data = await getRecipients.call(this, session, scan); + + // return early if necessary (e.g. all recipients were silent banned) + if ( + !data || + (data.bounces.length === 0 && + data.normalized.length === 0 && + data.imap.length === 0) + ) + return; + + console.log('WIP'); + + // TODO: add opt-in logging in Domain > Settings and log for each if enabled + // logger.info('email processed', { + // // TODO: session.headers etc need set (see createSession) + // session, + // // TODO: figure this out + // // user: email.user, + // // email: email._id, + // // domains: [email.domain], + // ignore_hook: false + // }); + // TODO: send email here and log "email delivered" (or) "email forwarded" + + // TODO: fill this in +} + +module.exports = onDataMX; diff --git a/helpers/on-data-smtp.js b/helpers/on-data-smtp.js new file mode 100644 index 0000000000..866e3750d5 --- /dev/null +++ b/helpers/on-data-smtp.js @@ -0,0 +1,454 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const _ = require('lodash'); +const dayjs = require('dayjs-with-plugins'); +const mongoose = require('mongoose'); +const { isEmail } = require('validator'); + +const Aliases = require('#models/aliases'); +const Domains = require('#models/domains'); +const Emails = require('#models/emails'); +const SMTPError = require('#helpers/smtp-error'); +const Users = require('#models/users'); +const config = require('#config'); +const createSession = require('#helpers/create-session'); +const emailHelper = require('#helpers/email'); +const i18n = require('#helpers/i18n'); +const isValidPassword = require('#helpers/is-valid-password'); +const logger = require('#helpers/logger'); +const validateAlias = require('#helpers/validate-alias'); +const validateDomain = require('#helpers/validate-domain'); +const { decrypt } = require('#helpers/encrypt-decrypt'); + +async function sendRateLimitEmail(user) { + // if the user received rate limit email in past 30d + if ( + _.isDate(user.smtp_rate_limit_sent_at) && + dayjs().isBefore(dayjs(user.smtp_rate_limit_sent_at).add(30, 'days')) + ) { + logger.info('user was already rate limited'); + return; + } + + await emailHelper({ + template: 'alert', + message: { + to: user[config.userFields.fullEmail], + bcc: config.email.message.from, + locale: user[config.lastLocaleField], + subject: i18n.translate( + 'SMTP_RATE_LIMIT_EXCEEDED', + user[config.lastLocaleField] + ) + }, + locals: { + locale: user[config.lastLocaleField], + message: i18n.translate( + 'SMTP_RATE_LIMIT_EXCEEDED', + user[config.lastLocaleField] + ) + } + }); + + // otherwise send the user an email and update the user record + await Users.findByIdAndUpdate(user._id, { + $set: { + smtp_rate_limit_sent_at: new Date() + } + }); +} + +// eslint-disable-next-line complexity +async function onDataSMTP(raw, session) { + // + // NOTE: we don't share the full alias and domain object + // in between onAuth and onData because there could + // be a time gap between the SMTP commands are sent + // (we want the most real-time information) + // + // ensure that user is authenticated + if ( + !isEmail(session?.user?.username) || + typeof session?.user?.password !== 'string' || + typeof session?.user?.domain_id !== 'string' || + typeof session?.user?.domain_name !== 'string' + ) + throw new SMTPError(config.authRequiredMessage, { + responseCode: 530 + }); + + // + // NOTE: we validate that the in-memory password is still active for + // the given user or the domain-wide catch-all generated password + // (e.g. edge case where AUTH done, a few seconds go by, then pass removed by user, and email would've gone through) + // + + let alias; + let isValid = false; + if (session.user.alias_id) { + alias = await Aliases.findOne({ + _id: new mongoose.Types.ObjectId(session.user.alias_id), + domain: new mongoose.Types.ObjectId(session.user.domain_id) + }) + .populate( + 'user', + `id ${config.userFields.isBanned} ${config.userFields.smtpLimit} smtp_rate_limit_sent_at ${config.userFields.fullEmail} ${config.lastLocaleField}` + ) + .select('+tokens.hash +tokens.salt') + .lean() + .exec(); + + // alias must exist + if (!alias) throw new Error('Alias does not exist'); + + // validate alias + validateAlias(alias, session.user.domain_name, session.user.alias_name); + + // ensure the token is still valid + if (Array.isArray(alias.tokens) && alias.tokens.length > 0) + isValid = await isValidPassword( + alias.tokens, + decrypt(session.user.password) + ); + } + + const domain = await Domains.findOne({ + id: session.user.domain_id, + plan: { $in: ['enhanced_protection', 'team'] } + }) + .populate( + 'members.user', + `id plan email ${config.userFields.isBanned} ${config.userFields.hasVerifiedEmail} ${config.userFields.planExpiresAt} ${config.userFields.smtpLimit} ${config.userFields.stripeSubscriptionID} ${config.userFields.paypalSubscriptionID} smtp_rate_limit_sent_at ${config.userFields.fullEmail} ${config.lastLocaleField}` + ) + .select('+tokens +tokens.hash +tokens.salt') + .exec(); + + if (!domain) + throw new Error( + 'Domain does not exist with current TXT verification record' + ); + + // validate domain + validateDomain(domain, session.user.domain_name); + + // + // NOTE: this is only applicable to SMTP servers (outbound mail) + // we allow users to use a generated token for the domain + // + if (!isValid && Array.isArray(domain.tokens) && domain.tokens.length > 0) + isValid = await isValidPassword( + domain.tokens, + decrypt(session.user.password) + ); + + if (!isValid) + throw new SMTPError( + `Invalid password, please try again or go to ${config.urls.web}/my-account/domains/${session.user.domain_name}/aliases and click "Generate Password"`, + { + responseCode: 535 + // ignoreHook: true + } + ); + + // + // NOTE: if the domain is suspended then the state is "pending" not queued + // + if (_.isDate(domain.smtp_suspended_sent_at)) + throw new SMTPError( + `Domain is suspended from outbound SMTP access, contact us at ${config.supportEmail}` + ); + + if (!domain.has_smtp) { + if (!_.isDate(domain.smtp_verified_at)) + throw new SMTPError( + `Domain is not configured for outbound SMTP, go to ${config.urls.web}/my-account/domains/${domain.name}/verify-smtp and click "Verify"`, + { + responseCode: 535, + ignoreHook: true + } + ); + + throw new SMTPError( + `Domain is pending admin approval for outbound SMTP access. Approval typically takes less than 24 hours; please check your inbox soon as we may be requesting additional information`, + { + responseCode: 535, + ignoreHook: true + } + ); + } + + // TODO: document storage of outbound SMTP email in FAQ/Privacy + // (it will be retained for 30d after + enable 30d expiry) + // TODO: document suspension process in Terms of Use + // (e.g. after 30d unpaid access, API access restrictions, etc) + // TODO: suspend domains with has_smtp that have past due balance + + // prepare envelope + const envelope = {}; + + if ( + isEmail(session?.envelope?.mailFrom?.address, { + ignore_max_length: true + }) + ) + envelope.from = session.envelope.mailFrom.address; + + if ( + Array.isArray(session?.envelope?.rcptTo) && + session.envelope.rcptTo.length > 0 + ) { + const to = []; + for (const rcpt of session.envelope.rcptTo) { + if (isEmail(rcpt.address, { ignore_max_length: true })) to.push(rcpt); + } + + if (to.length > 0) envelope.to = to; + } + + // if any of the domain admins are admins then don't rate limit + const adminExists = await Users.exists({ + _id: { + $in: domain.members + .filter((m) => m.group === 'admin' && typeof m.user === 'object') + .map((m) => + typeof m.user === 'object' && typeof m?.user?._id === 'object' + ? m.user._id + : m.user + ) + }, + group: 'admin' + }); + + let user; + if (alias) { + if (!alias.user) throw new TypeError('Alias user does not exist'); + user = alias.user; + } else { + // + // NOTE: if no alias was found then we can assume it's a domain catch-all password + // so we will assign the user to be either the user that generated the token + // or if that user no longer an admin of the domain then we'll email admins + // (and leave it up to them if they want to purge it, but at least we will re-assign) + // + // this also yields us the opportunity to validate that the in-memory password is still valid + // + let isValid = false; + let tokenUsed; + if (Array.isArray(domain.tokens) && domain.tokens.length > 0) { + for (const token of domain.tokens) { + // eslint-disable-next-line no-await-in-loop + isValid = await isValidPassword( + [token], + decrypt(session.user.password) + ); + if (isValid) { + tokenUsed = token; + break; // break out early if we found one that is valid + } + } + } + + if (!isValid) + throw new SMTPError( + `Invalid password, please try again or go to ${config.urls.web}/my-account/domains/${domain.name}/aliases and click "Generate Password"`, + { + responseCode: 535 + // ignoreHook: true + } + ); + + // now that we have `tokenUsed` we can perform a lookup on `tokenUsed.user` + user = await Users.findById(tokenUsed.user) + .select( + `id email ${config.userFields.isBanned} ${config.userFields.smtpLimit} smtp_rate_limit_sent_at ${config.userFields.fullEmail} ${config.lastLocaleField}` + ) + .lean() + .exec(); + + let reassign = false; // should we re-assign token and alert admins (?) + + // if user exists then ensure they are not banned and still an admin of domain + if (user) { + // user must not be banned + if (user[config.userFields.isBanned]) reassign = true; + // alias must still be an admin + else if ( + !domain.members.some( + (m) => m.user && m.user.id === user.id && m.group === 'admin' + ) + ) + reassign = true; + } else { + reassign = true; + } + + // + // if we need to reassign then find first admin that is not banned + // reassign the token to the first admin found + // alert admins of the token reassignment (if user then share email otherwise don't) + // + if (reassign) { + const admin = domain.members.find( + (m) => + m.group === 'admin' && m.user && !m.user[config.userFields.isBanned] + ); + // if no admin exists then purge token as a safeguard and alert system admins + if (admin) { + const token = domain.tokens.id(tokenUsed._id); + token.user = admin.user._id; + domain.skip_verification = true; + await domain.save(); + const { to, locale } = await Domains.getToAndMajorityLocaleByDomain( + domain + ); + // email the admins of the domain + email({ + template: 'alert', + message: { + to, + locale, + subject: 'Domain catch-all generated password re-assigned' + }, + locals: { + locale, + message: `Domain catch-all generated password has been re-assigned from ${ + user ? user.email : '' + } to ${ + admin.email + } since the user no longer existed or is no longer an admin of the domain.` + } + }) + .then() + .catch((err) => logger.fatal(err, { session })); + + // + // reassign user to the admin + // (after we send email to keep preservation of variables for message/subject) + // + user = admin; + } else { + domain.tokens.id(tokenUsed._id).remove(); + domain.skip_verification = true; + await domain.save(); + // alert admins of the edge case + const err = new TypeError( + `Domain name ${domain.name} (ID ${domain.id}) was using a catch-all alias that no longer has a valid admin/user assigned and SMTP onData attempted` + ); + logger.error(err, { session }); + // throw an error that password is not valid + throw new SMTPError('Catch-all password no longer exists', { + responseCode: 535 + }); + } + } + } + + // if for any reason there isn't a user then throw an error + if (!user) throw new TypeError('User does not exist'); + + // + // TODO: this should probably be moved to after `queue()` is invoked + // (we could use `zcard(key)` like we do in list emails controller) + // + const max = user[config.userFields.smtpLimit] || config.smtpLimitMessages; + if (!adminExists) { + // rate limit to X emails per day by domain id then denylist + { + const count = await this.client.zcard( + `${config.smtpLimitNamespace}:${domain.id}` + ); + // return 550 error code + if (count >= max) { + // send one-time email alert to admin + user + sendRateLimitEmail(user) + .then() + .catch((err) => logger.fatal(err, { session })); + throw new SMTPError('Rate limit exceeded', { ignoreHook: true }); + } + } + + // rate limit to X emails per day by alias user id then denylist + { + const count = await this.client.zcard( + `${config.smtpLimitNamespace}:${user.id}` + ); + // return 550 error code + if (count >= max) { + // send one-time email alert to admin + user + sendRateLimitEmail(user) + .then() + .catch((err) => logger.fatal(err, { session })); + throw new SMTPError('Rate limit exceeded', { ignoreHook: true }); + } + } + } + + // queue the email + const email = await Emails.queue({ + message: { + envelope, + raw + }, + alias, + domain, + user, + date: new Date(session.arrivalDate), + catchall: typeof session?.user?.alias_id !== 'string', + isPending: true + }); + + if (!adminExists) { + try { + // rate limit to X emails per day by domain id then denylist + { + const limit = await this.rateLimiter.get({ + id: domain.id, + max + }); + + // return 550 error code + if (!limit.remaining) + throw new SMTPError('Rate limit exceeded', { ignoreHook: true }); + } + + // rate limit to X emails per day by alias user id then denylist + const limit = await this.rateLimiter.get({ + id: user.id, + max + }); + + // return 550 error code + if (!limit.remaining) + throw new SMTPError('Rate limit exceeded', { ignoreHook: true }); + } catch (err) { + // remove the job from the queue + Emails.findByIdAndRemove(email._id) + .then() + .catch((err) => logger.fatal(err)); + throw err; + } + } + + if (!_.isDate(domain.smtp_suspended_sent_at)) { + email.status = 'queued'; + await email.save(); + } + + // TODO: implement credit system + + logger.info('email created', { + session: { + ...session, + ...createSession(email) + }, + user: email.user, + email: email._id, + domains: [email.domain], + ignore_hook: false + }); +} + +module.exports = onDataSMTP; diff --git a/helpers/on-data.js b/helpers/on-data.js new file mode 100644 index 0000000000..b5f8bd7bc2 --- /dev/null +++ b/helpers/on-data.js @@ -0,0 +1,202 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const { Buffer } = require('node:buffer'); + +const _ = require('lodash'); +const addressParser = require('nodemailer/lib/addressparser'); +const addrs = require('email-addresses'); +const bytes = require('bytes'); +const getStream = require('get-stream'); +const isSANB = require('is-string-and-not-blank'); +const pFilter = require('p-filter'); +const safeStringify = require('fast-safe-stringify'); +const { isEmail } = require('validator'); + +const MessageSplitter = require('#helpers/message-splitter'); +const SMTPError = require('#helpers/smtp-error'); +const ServerShutdownError = require('#helpers/server-shutdown-error'); +const checkSRS = require('#helpers/check-srs'); +const config = require('#config'); +const env = require('#config/env'); +const isSilentBanned = require('#helpers/is-silent-banned'); +const onDataMX = require('#helpers/on-data-mx'); +const onDataSMTP = require('#helpers/on-data-smtp'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); +const parseRootDomain = require('#helpers/parse-root-domain'); +const refineAndLogError = require('#helpers/refine-and-log-error'); +const updateSession = require('#helpers/update-session'); + +const MAX_BYTES = bytes(env.SMTP_MESSAGE_MAX_SIZE); + +// TODO: check for `this.isClosing` before heavy/slow operations in onDataMX + +// eslint-disable-next-line complexity +async function onData(stream, _session, fn) { + if (this.isClosing) return setImmediate(() => fn(new ServerShutdownError())); + + // store clone of session since it gets modified/destroyed + const session = JSON.parse(safeStringify(_session)); + + this.logger.debug('DATA', { session }); + + try { + const messageSplitter = new MessageSplitter({ + maxBytes: MAX_BYTES + }); + + const body = await getStream.buffer(stream.pipe(messageSplitter)); + + if (messageSplitter.sizeExceeded) + throw new SMTPError('Size exceeded', { responseCode: 552 }); + + if (!messageSplitter.headersParsed) + throw new SMTPError('Headers unable to be parsed'); + + const { headers } = messageSplitter; + const raw = Buffer.concat([headers.build(), body]); + + // update session object with useful debug info for error logs + // (also subsequently throws an error if "From" header was not valid per RFC 5322) + await updateSession.call(this, raw, headers, session); + + // prevent messages from being stuck in a redirect loop + // + if (headers.get('received').length > 25) + throw new SMTPError('Message was stuck in a redirect loop'); + + // check against silent ban list + const silentBanned = await isSilentBanned( + session.attributes, + this.client, + this.resolver + ); + if (silentBanned) return setImmediate(fn); + + // + // filter out RCPT TO for silent banned users + // + const rcptTo = + Array.isArray(session.envelope.rcptTo) && + session.envelope.rcptTo.length > 0 + ? await pFilter( + // + // NOTE: if an invalid SRS signature was specified in RCPT TO + // then the RCPT TO command will already throw an error (see `helpers/on-rcpt-to.js`) + // however there could be a delay long enough to cause invalidation of SRS + // in between when the RCPT TO command and the DATA command is received + // so we similarly thrown an error here + // (and also rewrite the RCPT TO as needed; since this is not possible in RCPT TO command handler in `smtp-server`) + // + session.envelope.rcptTo.map((to) => { + const shouldThrow = + parseRootDomain(parseHostFromDomainOrAddress(to.address)) === + env.WEB_HOST; + to.address = checkSRS(to.address, shouldThrow); + return to; + }), + async (to) => { + const arr = _.uniq( + _.compact([ + // check the TO + to.address, + // check the TO hostname + parseHostFromDomainOrAddress(to.address), + // check the TO hostname root + // (but only if it was not equal to the hostname) + parseRootDomain(parseHostFromDomainOrAddress(to.address)) === + parseHostFromDomainOrAddress(to.address) + ? null + : parseRootDomain(parseHostFromDomainOrAddress(to.address)) + ]).map((str) => str.toLowerCase().trim()) + ); + if (arr.length === 0) return false; + const silentBanned = await isSilentBanned( + arr, + this.client, + this.resolver + ); + if (!silentBanned) { + for (const v of arr) { + if (!session.attributes.includes(v)) + session.attributes.push(v); + } + } + + return !silentBanned; + }, + { concurrency: config.concurrency } + ) + : []; + + // if no RCPT TO remaining after filtering then return early + if (rcptTo.length === 0) return setImmediate(fn); + + // rudimentary debugging for silent banned users + // (in case someone ever reaches out that a message not delivered) + if (rcptTo.length !== session.envelope.rcptTo.length) { + session.hasSilentBanned = true; + session.originalRcptTo = [...session.envelope.rcptTo]; + } + + // + // re-assign RCPT TO with values that were not silent banned and also with SRS rewritten addresses + // (because sometimes improperly configured servers will send a response to the MAIL FROM) + // (which could be an SRS forwarded address, which we need to rewrite so it goes to its actual destination) + // + session.envelope.rcptTo = rcptTo; + + // + // in addition to RCPT TO being incorrect due to improperly configured server sending to SRS forwarded address + // we also need to rewrite the "To" header an rewrite any SRS forwarded addresses with their actual ones + // + let to = headers.getFirst('to'); + if (isSANB(to)) { + let originalToAddresses = + addrs.parseAddressList({ input: to, partial: true }) || []; + if (originalToAddresses.length === 0) + originalToAddresses = addrs.parseAddressList({ input: to }) || []; + // safeguard + if (originalToAddresses.length === 0) + originalToAddresses = addressParser(to); + originalToAddresses = originalToAddresses.filter( + (addr) => + _.isObject(addr) && + isSANB(addr.address) && + isEmail(addr.address, { ignore_max_length: true }) + ); + for (const obj of originalToAddresses) { + const shouldThrow = + parseRootDomain(parseHostFromDomainOrAddress(obj.address)) === + env.WEB_HOST; + // rewrite the to line + let isModified = false; + if (checkSRS(obj.address, shouldThrow) !== obj.address) { + isModified = true; + to = to.replaceAll(obj.address, checkSRS(obj.address, shouldThrow)); + } + + if (isModified) headers.update('to', to); + } + } + + if (this.constructor.name === 'SMTP') { + await onDataSMTP.call(this, raw, session); + return setImmediate(fn); + } + + if (this.constructor.name === 'MX') { + await onDataMX.call(this, raw, session, headers, body); + return setImmediate(fn); + } + + // safeguard in case unknown constructor + throw new TypeError('Unknown constructor'); + } catch (err) { + setImmediate(() => fn(refineAndLogError(err, session, false, this))); + } +} + +module.exports = onData; diff --git a/helpers/smtp/on-mail-from.js b/helpers/on-mail-from.js similarity index 68% rename from helpers/smtp/on-mail-from.js rename to helpers/on-mail-from.js index edf53fe4df..7c517a2830 100644 --- a/helpers/smtp/on-mail-from.js +++ b/helpers/on-mail-from.js @@ -8,6 +8,9 @@ const { isEmail } = require('validator'); const SMTPError = require('#helpers/smtp-error'); const ServerShutdownError = require('#helpers/server-shutdown-error'); +const checkSRS = require('#helpers/check-srs'); +const env = require('#config/env'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); const refineAndLogError = require('#helpers/refine-and-log-error'); function onMailFrom(address, session, fn) { @@ -35,6 +38,16 @@ function onMailFrom(address, session, fn) { ) ); + // + // check if it was invalid SRS (pass `shouldThrow` as `true`) + // + try { + if (parseHostFromDomainOrAddress(address.address) === env.WEB_HOST) + checkSRS(address.address, true, true); + } catch (err) { + return setImmediate(() => fn(refineAndLogError(err, session, false, this))); + } + setImmediate(fn); } diff --git a/helpers/smtp/on-rcpt-to.js b/helpers/on-rcpt-to.js similarity index 52% rename from helpers/smtp/on-rcpt-to.js rename to helpers/on-rcpt-to.js index aed4f32658..5f5bf3867c 100644 --- a/helpers/smtp/on-rcpt-to.js +++ b/helpers/on-rcpt-to.js @@ -8,10 +8,14 @@ const { isEmail } = require('validator'); const SMTPError = require('#helpers/smtp-error'); const ServerShutdownError = require('#helpers/server-shutdown-error'); +const checkSRS = require('#helpers/check-srs'); const config = require('#config'); +const env = require('#config/env'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); +const parseRootDomain = require('#helpers/parse-root-domain'); const refineAndLogError = require('#helpers/refine-and-log-error'); -function onRcptTo(address, session, fn) { +async function onRcptTo(address, session, fn) { this.logger.debug('RCPT TO', { address, session }); if (this.isClosing) return setImmediate(() => fn(new ServerShutdownError())); @@ -32,7 +36,6 @@ function onRcptTo(address, session, fn) { false, this ) - // session ) ); @@ -53,10 +56,39 @@ function onRcptTo(address, session, fn) { false, this ) - // session ) ); + try { + // + // check if attempted spoofed or invalid SRS (e.g. fake bounces) + // + if ( + parseRootDomain(parseHostFromDomainOrAddress(address.address)) === + env.WEB_HOST + ) + checkSRS(address.address, true, true); + + // + // if we're on the MX server then we perform a very rudimentary check + // on the RCPT domain name to see that it actually is set up to receive mail + // (they do not necessarily need to be ours, but this helps thwart spammers) + // + if (this?.constructor?.name === 'MX') { + const domain = parseHostFromDomainOrAddress(checkSRS(address.address)); + const records = await this.resolver.resolveMx(domain); + if (!records || records.length === 0) + throw new SMTPError( + `${checkSRS( + address.address + )} does not have any MX records configured on its domain ${domain}`, + { ignoreHook: true } + ); + } + } catch (err) { + return setImmediate(() => fn(refineAndLogError(err, session, false, this))); + } + setImmediate(fn); } diff --git a/helpers/parse-host-from-domain-or-address.js b/helpers/parse-host-from-domain-or-address.js new file mode 100644 index 0000000000..bf14016149 --- /dev/null +++ b/helpers/parse-host-from-domain-or-address.js @@ -0,0 +1,51 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const punycode = require('node:punycode'); +const { isIP } = require('node:net'); + +const _ = require('lodash'); +const addressParser = require('nodemailer/lib/addressparser'); +const isFQDN = require('is-fqdn'); +const isSANB = require('is-string-and-not-blank'); + +const SMTPError = require('#helpers/smtp-error'); + +function parseHostFromDomainOrAddress(address) { + const parsedAddresses = addressParser(address); + let domain = address; + + if ( + _.isArray(parsedAddresses) && + _.isObject(parsedAddresses[0]) && + isSANB(parsedAddresses[0].address) + ) { + domain = parsedAddresses[0].address; + } + + const atPos = domain.indexOf('@'); + if (atPos !== -1) domain = domain.slice(atPos + 1); + + domain = domain.toLowerCase().trim(); + + try { + domain = punycode.toASCII(domain); + } catch { + // ignore punycode conversion errors + } + + // ensure fully qualified domain name or IP address + if (!domain || (!isFQDN(domain) && !isIP(domain))) + throw new SMTPError( + `${ + domain || address + } does not contain a fully qualified domain name ("FQDN") nor IP address`, + { responseCode: 550 } + ); + + return domain; +} + +module.exports = parseHostFromDomainOrAddress; diff --git a/helpers/parse-payload.js b/helpers/parse-payload.js index 6bc3bd5454..d5a12e985c 100644 --- a/helpers/parse-payload.js +++ b/helpers/parse-payload.js @@ -29,7 +29,6 @@ const pify = require('pify'); const prettyBytes = require('pretty-bytes'); const safeStringify = require('fast-safe-stringify'); const { Iconv } = require('iconv'); -const { boolean } = require('boolean'); const { isEmail } = require('validator'); const Boom = require('@hapi/boom'); @@ -47,6 +46,7 @@ const getFingerprint = require('#helpers/get-fingerprint'); const getPathToDatabase = require('#helpers/get-path-to-database'); const getTemporaryDatabase = require('#helpers/get-temporary-database'); const i18n = require('#helpers/i18n'); +const isAllowlisted = require('#helpers/is-allowlisted'); const isCodeBug = require('#helpers/is-code-bug'); const isRetryableError = require('#helpers/is-retryable-error'); const logger = require('#helpers/logger'); @@ -808,10 +808,12 @@ async function parsePayload(data, ws) { throw err; } } else { - const isAllowlisted = await this.client.get( - `allowlist:${sender}` + const allowlisted = await isAllowlisted( + sender, + this.client, + this.resolver ); - if (boolean(isAllowlisted)) { + if (allowlisted) { // 2) Senders that are allowlisted are limited to sending 10 GB per day. if (size >= bytes('10GB')) { const err = new SMTPError( diff --git a/helpers/parse-username.js b/helpers/parse-username.js new file mode 100644 index 0000000000..3e4c296498 --- /dev/null +++ b/helpers/parse-username.js @@ -0,0 +1,20 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const punycode = require('node:punycode'); + +const addressParser = require('nodemailer/lib/addressparser'); + +function parseUsername(address) { + ({ address } = addressParser(address)[0]); + let username = address.includes('+') + ? address.split('+')[0] + : address.split('@')[0]; + + username = punycode.toASCII(username).toLowerCase(); + return username; +} + +module.exports = parseUsername; diff --git a/helpers/process-email.js b/helpers/process-email.js index 7a46b3eb19..bf6f2c1ba9 100644 --- a/helpers/process-email.js +++ b/helpers/process-email.js @@ -48,6 +48,7 @@ const sendEmail = require('./send-email'); const { encrypt, decrypt } = require('./encrypt-decrypt'); const isMessageEncrypted = require('#helpers/is-message-encrypted'); const encryptMessage = require('#helpers/encrypt-message'); +const updateHeaders = require('#helpers/update-headers'); const config = require('#config'); const env = require('#config/env'); @@ -449,42 +450,16 @@ async function processEmail({ email, port = 25, resolver, client }) { } // add X-* headers (e.g. version + report-to) - for (const key of [ - 'x-report-abuse-to', - 'x-report-abuse', - 'x-complaints-to', - 'x-forwardemail-version', - 'x-forwardemail-sender', - 'x-forwardemail-id' - ]) { - data.headers.remove(key); - } + updateHeaders(data.headers); - data.headers.add( - 'X-Report-Abuse-To', - config.abuseEmail, - data.headers.lines.length - ); - data.headers.add( - 'X-Report-Abuse', - config.abuseEmail, - data.headers.lines.length - ); - data.headers.add( - 'X-Complaints-To', - config.abuseEmail, - data.headers.lines.length - ); - data.headers.add( - 'X-ForwardEmail-Version', - config.pkg.version, - data.headers.lines.length - ); + // additional headers to add specifically for outbound smtp + data.headers.remove('x-forwardemail-sender'); data.headers.add( 'X-ForwardEmail-Sender', `rfc822; ${[email.envelope.from, HOSTNAME, IP_ADDRESS].join(', ')}`, data.headers.lines.length ); + data.headers.remove('x-forwardemail-id'); data.headers.add( 'X-ForwardEmail-ID', email.id, diff --git a/helpers/smtp/index.js b/helpers/smtp/index.js deleted file mode 100644 index 30e795ab0c..0000000000 --- a/helpers/smtp/index.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Copyright (c) Forward Email LLC - * SPDX-License-Identifier: BUSL-1.1 - */ - -const onData = require('./on-data'); -const onConnect = require('./on-connect'); -const onMailFrom = require('./on-mail-from'); -const onRcptTo = require('./on-rcpt-to'); - -module.exports = { - onData, - onConnect, - onMailFrom, - onRcptTo -}; diff --git a/helpers/smtp/on-connect.js b/helpers/smtp/on-connect.js deleted file mode 100644 index 888d9dd38b..0000000000 --- a/helpers/smtp/on-connect.js +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Copyright (c) Forward Email LLC - * SPDX-License-Identifier: BUSL-1.1 - */ - -const punycode = require('node:punycode'); - -const isFQDN = require('is-fqdn'); -const { boolean } = require('boolean'); - -// const SMTPError = require('#helpers/smtp-error'); -const ServerShutdownError = require('#helpers/server-shutdown-error'); -// const config = require('#config'); -const env = require('#config/env'); -const parseRootDomain = require('#helpers/parse-root-domain'); -const refineAndLogError = require('#helpers/refine-and-log-error'); - -async function onConnect(session, fn) { - this.logger.debug('CONNECT', { session }); - - if (this.isClosing) return fn(new ServerShutdownError()); - - // this is used for setting Date header if missing on SMTP submission - session.arrivalDate = new Date(); - - // lookup the client hostname - try { - const [clientHostname] = await this.resolver.reverse(session.remoteAddress); - if (isFQDN(clientHostname)) { - // do we need this still (?) - let domain = clientHostname.toLowerCase().trim(); - try { - domain = punycode.toASCII(domain); - } catch { - // ignore punycode conversion errors - } - - session.resolvedClientHostname = domain; - } - } catch (err) { - // - // NOTE: the native Node.js DNS module would throw an error previously - // - // - if (env.NODE_ENV !== 'production') this.logger.debug(err, { session }); - } - - try { - // get root domain if available - let rootDomain; - if (session.resolvedClientHostname) - rootDomain = parseRootDomain(session.resolvedClientHostname); - - // check if allowlisted - const result = await this.client.get( - `allowlist:${rootDomain || session.remoteAddress}` - ); - - if (boolean(result)) { - session.allowlistValue = rootDomain || session.remoteAddress; - } else { - // - // NOTE: because there are too many false positives with actual users - // we're not going to do denylist/silent/backscatter lookup anymore - /* - // - // prevent connections from backscatter, silent ban, and denylist - // - const arr = [ - `backscatter:${session.remoteAddress}`, - `denylist:${session.remoteAddress}`, - `silent:${session.remoteAddress}` - ]; - - if (rootDomain) - arr.push( - `backscatter:${rootDomain}`, - `denylist:${rootDomain}`, - `silent:${rootDomain}` - ); - - const results = await this.client.mget(arr); - - if (results.some((result) => boolean(result))) { - throw new SMTPError( - `The ${rootDomain ? 'domain' : 'IP'} ${ - rootDomain || session.remoteAddress - } is denylisted by ${ - config.urls.web - }. To request removal, you must visit ${config.urls.web}/denylist?q=${ - rootDomain || session.remoteAddress - }.`, - { ignoreHook: true } - ); - } - */ - } - - fn(); - } catch (err) { - fn(refineAndLogError(err, session, false, this)); - } -} - -module.exports = onConnect; diff --git a/helpers/smtp/on-data.js b/helpers/smtp/on-data.js deleted file mode 100644 index 41c6b970b8..0000000000 --- a/helpers/smtp/on-data.js +++ /dev/null @@ -1,482 +0,0 @@ -/** - * Copyright (c) Forward Email LLC - * SPDX-License-Identifier: BUSL-1.1 - */ - -const _ = require('lodash'); -const bytes = require('bytes'); -const dayjs = require('dayjs-with-plugins'); -const getStream = require('get-stream'); -const mongoose = require('mongoose'); -const safeStringify = require('fast-safe-stringify'); -const { isEmail } = require('validator'); - -const Aliases = require('#models/aliases'); -const Domains = require('#models/domains'); -const Emails = require('#models/emails'); -const SMTPError = require('#helpers/smtp-error'); -const ServerShutdownError = require('#helpers/server-shutdown-error'); -const Users = require('#models/users'); -const config = require('#config'); -const createSession = require('#helpers/create-session'); -const emailHelper = require('#helpers/email'); -const env = require('#config/env'); -const i18n = require('#helpers/i18n'); -const isValidPassword = require('#helpers/is-valid-password'); -const logger = require('#helpers/logger'); -const refineAndLogError = require('#helpers/refine-and-log-error'); -const validateAlias = require('#helpers/validate-alias'); -const validateDomain = require('#helpers/validate-domain'); -const { decrypt } = require('#helpers/encrypt-decrypt'); - -const MAX_BYTES = bytes(env.SMTP_MESSAGE_MAX_SIZE); - -async function sendRateLimitEmail(user) { - // if the user received rate limit email in past 30d - if ( - _.isDate(user.smtp_rate_limit_sent_at) && - dayjs().isBefore(dayjs(user.smtp_rate_limit_sent_at).add(30, 'days')) - ) { - logger.info('user was already rate limited'); - return; - } - - await emailHelper({ - template: 'alert', - message: { - to: user[config.userFields.fullEmail], - bcc: config.email.message.from, - locale: user[config.lastLocaleField], - subject: i18n.translate( - 'SMTP_RATE_LIMIT_EXCEEDED', - user[config.lastLocaleField] - ) - }, - locals: { - locale: user[config.lastLocaleField], - message: i18n.translate( - 'SMTP_RATE_LIMIT_EXCEEDED', - user[config.lastLocaleField] - ) - } - }); - - // otherwise send the user an email and update the user record - await Users.findByIdAndUpdate(user._id, { - $set: { - smtp_rate_limit_sent_at: new Date() - } - }); -} - -// -// NOTE: we can merge SMTP/FE codebase in future and simply check if auth disabled -// then this will act as a forwarding server only (MTA) -// -// eslint-disable-next-line complexity -async function onData(stream, _session, fn) { - if (this.isClosing) return setImmediate(() => fn(new ServerShutdownError())); - - // store clone of session since it gets modified/destroyed - const session = JSON.parse(safeStringify(_session)); - - this.logger.debug('DATA', { session }); - - try { - // we have to consume the stream - const raw = await getStream.buffer(stream, { - maxBuffer: MAX_BYTES - }); - - // - // NOTE: we don't share the full alias and domain object - // in between onAuth and onData because there could - // be a time gap between the SMTP commands are sent - // (we want the most real-time information) - // - // ensure that user is authenticated - if ( - !isEmail(session?.user?.username) || - typeof session?.user?.password !== 'string' || - typeof session?.user?.domain_id !== 'string' || - typeof session?.user?.domain_name !== 'string' - ) - throw new SMTPError(config.authRequiredMessage, { - responseCode: 530 - }); - - // - // NOTE: we validate that the in-memory password is still active for - // the given user or the domain-wide catch-all generated password - // (e.g. edge case where AUTH done, a few seconds go by, then pass removed by user, and email would've gone through) - // - - let alias; - let isValid = false; - if (session.user.alias_id) { - alias = await Aliases.findOne({ - _id: new mongoose.Types.ObjectId(session.user.alias_id), - domain: new mongoose.Types.ObjectId(session.user.domain_id) - }) - .populate( - 'user', - `id ${config.userFields.isBanned} ${config.userFields.smtpLimit} smtp_rate_limit_sent_at ${config.userFields.fullEmail} ${config.lastLocaleField}` - ) - .select('+tokens.hash +tokens.salt') - .lean() - .exec(); - - // alias must exist - if (!alias) throw new Error('Alias does not exist'); - - // validate alias - validateAlias(alias, session.user.domain_name, session.user.alias_name); - - // ensure the token is still valid - if (Array.isArray(alias.tokens) && alias.tokens.length > 0) - isValid = await isValidPassword( - alias.tokens, - decrypt(session.user.password) - ); - } - - const domain = await Domains.findOne({ - id: session.user.domain_id, - plan: { $in: ['enhanced_protection', 'team'] } - }) - .populate( - 'members.user', - `id plan email ${config.userFields.isBanned} ${config.userFields.hasVerifiedEmail} ${config.userFields.planExpiresAt} ${config.userFields.smtpLimit} ${config.userFields.stripeSubscriptionID} ${config.userFields.paypalSubscriptionID} smtp_rate_limit_sent_at ${config.userFields.fullEmail} ${config.lastLocaleField}` - ) - .select('+tokens +tokens.hash +tokens.salt') - .exec(); - - if (!domain) - throw new Error( - 'Domain does not exist with current TXT verification record' - ); - - // validate domain - validateDomain(domain, session.user.domain_name); - - // - // NOTE: this is only applicable to SMTP servers (outbound mail) - // we allow users to use a generated token for the domain - // - if (!isValid && Array.isArray(domain.tokens) && domain.tokens.length > 0) - isValid = await isValidPassword( - domain.tokens, - decrypt(session.user.password) - ); - - if (!isValid) - throw new SMTPError( - `Invalid password, please try again or go to ${config.urls.web}/my-account/domains/${session.user.domain_name}/aliases and click "Generate Password"`, - { - responseCode: 535 - // ignoreHook: true - } - ); - - // - // NOTE: if the domain is suspended then the state is "pending" not queued - // - if (_.isDate(domain.smtp_suspended_sent_at)) - throw new SMTPError( - `Domain is suspended from outbound SMTP access, contact us at ${config.supportEmail}` - ); - - if (!domain.has_smtp) { - if (!_.isDate(domain.smtp_verified_at)) - throw new SMTPError( - `Domain is not configured for outbound SMTP, go to ${config.urls.web}/my-account/domains/${domain.name}/verify-smtp and click "Verify"`, - { - responseCode: 535, - ignoreHook: true - } - ); - - throw new SMTPError( - `Domain is pending admin approval for outbound SMTP access. Approval typically takes less than 24 hours; please check your inbox soon as we may be requesting additional information`, - { - responseCode: 535, - ignoreHook: true - } - ); - } - - // TODO: document storage of outbound SMTP email in FAQ/Privacy - // (it will be retained for 30d after + enable 30d expiry) - // TODO: document suspension process in Terms of Use - // (e.g. after 30d unpaid access, API access restrictions, etc) - // TODO: suspend domains with has_smtp that have past due balance - - // prepare envelope - const envelope = {}; - - if ( - isEmail(session?.envelope?.mailFrom?.address, { ignore_max_length: true }) - ) - envelope.from = session.envelope.mailFrom.address; - - if ( - Array.isArray(session?.envelope?.rcptTo) && - session.envelope.rcptTo.length > 0 - ) { - const to = []; - for (const rcpt of session.envelope.rcptTo) { - if (isEmail(rcpt.address, { ignore_max_length: true })) to.push(rcpt); - } - - if (to.length > 0) envelope.to = to; - } - - // if any of the domain admins are admins then don't rate limit - const adminExists = await Users.exists({ - _id: { - $in: domain.members - .filter((m) => m.group === 'admin' && typeof m.user === 'object') - .map((m) => - typeof m.user === 'object' && typeof m?.user?._id === 'object' - ? m.user._id - : m.user - ) - }, - group: 'admin' - }); - - let user; - if (alias) { - if (!alias.user) throw new TypeError('Alias user does not exist'); - user = alias.user; - } else { - // - // NOTE: if no alias was found then we can assume it's a domain catch-all password - // so we will assign the user to be either the user that generated the token - // or if that user no longer an admin of the domain then we'll email admins - // (and leave it up to them if they want to purge it, but at least we will re-assign) - // - // this also yields us the opportunity to validate that the in-memory password is still valid - // - let isValid = false; - let tokenUsed; - if (Array.isArray(domain.tokens) && domain.tokens.length > 0) { - for (const token of domain.tokens) { - // eslint-disable-next-line no-await-in-loop - isValid = await isValidPassword( - [token], - decrypt(session.user.password) - ); - if (isValid) { - tokenUsed = token; - break; // break out early if we found one that is valid - } - } - } - - if (!isValid) - throw new SMTPError( - `Invalid password, please try again or go to ${config.urls.web}/my-account/domains/${domain.name}/aliases and click "Generate Password"`, - { - responseCode: 535 - // ignoreHook: true - } - ); - - // now that we have `tokenUsed` we can perform a lookup on `tokenUsed.user` - user = await Users.findById(tokenUsed.user) - .select( - `id email ${config.userFields.isBanned} ${config.userFields.smtpLimit} smtp_rate_limit_sent_at ${config.userFields.fullEmail} ${config.lastLocaleField}` - ) - .lean() - .exec(); - - let reassign = false; // should we re-assign token and alert admins (?) - - // if user exists then ensure they are not banned and still an admin of domain - if (user) { - // user must not be banned - if (user[config.userFields.isBanned]) reassign = true; - // alias must still be an admin - else if ( - !domain.members.some( - (m) => m.user && m.user.id === user.id && m.group === 'admin' - ) - ) - reassign = true; - } else { - reassign = true; - } - - // - // if we need to reassign then find first admin that is not banned - // reassign the token to the first admin found - // alert admins of the token reassignment (if user then share email otherwise don't) - // - if (reassign) { - const admin = domain.members.find( - (m) => - m.group === 'admin' && m.user && !m.user[config.userFields.isBanned] - ); - // if no admin exists then purge token as a safeguard and alert system admins - if (admin) { - const token = domain.tokens.id(tokenUsed._id); - token.user = admin.user._id; - domain.skip_verification = true; - await domain.save(); - const { to, locale } = await Domains.getToAndMajorityLocaleByDomain( - domain - ); - // email the admins of the domain - email({ - template: 'alert', - message: { - to, - locale, - subject: 'Domain catch-all generated password re-assigned' - }, - locals: { - locale, - message: `Domain catch-all generated password has been re-assigned from ${ - user ? user.email : '' - } to ${ - admin.email - } since the user no longer existed or is no longer an admin of the domain.` - } - }) - .then() - .catch((err) => logger.fatal(err, { session })); - - // - // reassign user to the admin - // (after we send email to keep preservation of variables for message/subject) - // - user = admin; - } else { - domain.tokens.id(tokenUsed._id).remove(); - domain.skip_verification = true; - await domain.save(); - // alert admins of the edge case - const err = new TypeError( - `Domain name ${domain.name} (ID ${domain.id}) was using a catch-all alias that no longer has a valid admin/user assigned and SMTP onData attempted` - ); - logger.error(err, { session }); - // throw an error that password is not valid - throw new SMTPError('Catch-all password no longer exists', { - responseCode: 535 - }); - } - } - } - - // if for any reason there isn't a user then throw an error - if (!user) throw new TypeError('User does not exist'); - - // - // TODO: this should probably be moved to after `queue()` is invoked - // (we could use `zcard(key)` like we do in list emails controller) - // - const max = user[config.userFields.smtpLimit] || config.smtpLimitMessages; - if (!adminExists) { - // rate limit to X emails per day by domain id then denylist - { - const count = await this.client.zcard( - `${config.smtpLimitNamespace}:${domain.id}` - ); - // return 550 error code - if (count >= max) { - // send one-time email alert to admin + user - sendRateLimitEmail(user) - .then() - .catch((err) => logger.fatal(err, { session })); - throw new SMTPError('Rate limit exceeded', { ignoreHook: true }); - } - } - - // rate limit to X emails per day by alias user id then denylist - { - const count = await this.client.zcard( - `${config.smtpLimitNamespace}:${user.id}` - ); - // return 550 error code - if (count >= max) { - // send one-time email alert to admin + user - sendRateLimitEmail(user) - .then() - .catch((err) => logger.fatal(err, { session })); - throw new SMTPError('Rate limit exceeded', { ignoreHook: true }); - } - } - } - - // queue the email - const email = await Emails.queue({ - message: { - envelope, - raw - }, - alias, - domain, - user, - date: new Date(session.arrivalDate), - catchall: typeof session?.user?.alias_id !== 'string', - isPending: true - }); - - if (!adminExists) { - try { - // rate limit to X emails per day by domain id then denylist - { - const limit = await this.rateLimiter.get({ - id: domain.id, - max - }); - - // return 550 error code - if (!limit.remaining) - throw new SMTPError('Rate limit exceeded', { ignoreHook: true }); - } - - // rate limit to X emails per day by alias user id then denylist - const limit = await this.rateLimiter.get({ - id: user.id, - max - }); - - // return 550 error code - if (!limit.remaining) - throw new SMTPError('Rate limit exceeded', { ignoreHook: true }); - } catch (err) { - // remove the job from the queue - Emails.findByIdAndRemove(email._id) - .then() - .catch((err) => logger.fatal(err)); - throw err; - } - } - - if (!_.isDate(domain.smtp_suspended_sent_at)) { - email.status = 'queued'; - await email.save(); - } - - // TODO: implement credit system - - logger.info('email created', { - session: { - ...session, - ...createSession(email) - }, - user: email.user, - email: email._id, - domains: [email.domain], - ignore_hook: false - }); - - setImmediate(fn); - } catch (err) { - setImmediate(() => fn(refineAndLogError(err, session, false, this))); - } -} - -module.exports = onData; diff --git a/helpers/update-headers.js b/helpers/update-headers.js new file mode 100644 index 0000000000..0d6c79c780 --- /dev/null +++ b/helpers/update-headers.js @@ -0,0 +1,28 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const config = require('#config'); + +function updateHeaders(headers) { + for (const key of [ + 'x-report-abuse-to', + 'x-report-abuse', + 'x-complaints-to', + 'x-forwardemail-version' + ]) { + headers.remove(key); + } + + headers.add('X-Report-Abuse-To', config.abuseEmail, headers.lines.length); + headers.add('X-Report-Abuse', config.abuseEmail, headers.lines.length); + headers.add('X-Complaints-To', config.abuseEmail, headers.lines.length); + headers.add( + 'X-ForwardEmail-Version', + config.pkg.version, + headers.lines.length + ); +} + +module.exports = updateHeaders; diff --git a/helpers/update-session.js b/helpers/update-session.js new file mode 100644 index 0000000000..6c3744b48b --- /dev/null +++ b/helpers/update-session.js @@ -0,0 +1,72 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const getAttributes = require('#helpers/get-attributes'); +const getFromAddress = require('#helpers/get-from-address'); +const getFingerprint = require('#helpers/get-fingerprint'); +const getHeaders = require('#helpers/get-headers'); +const isAllowlisted = require('#helpers/is-allowlisted'); +const parseHostFromDomainOrAddress = require('#helpers/parse-host-from-domain-or-address'); +const parseRootDomain = require('#helpers/parse-root-domain'); + +async function updateSession(raw, headers, session) { + // + // NOTE: we set `session.headers` and `session.originalFromAddress` + // so that if an error is thrown, then the error log + // will have this information in the `session` object + // and therefore it will be useful debugging information for the end user + // (since our error log UI renders these values in My Account > Logs) + // + session.headers = getHeaders(headers); + + // getFromAddress will thrown an error if it's not RFC 5322 compliant + session.originalFromAddress = getFromAddress(headers.getFirst('from')); + session.originalFromAddressDomain = parseHostFromDomainOrAddress( + session.originalFromAddress + ); + session.originalFromAddressRootDomain = parseRootDomain( + session.originalFromAddressDomain + ); + + session.isOriginalFromAddressAllowlisted = await isAllowlisted( + session.originalFromAddress, + this.client, + this.resolver + ); + + if (!session.isOriginalFromAddressAllowlisted) + session.isOriginalFromAddressAllowlisted = await isAllowlisted( + session.originalFromAddressDomain, + this.client, + this.resolver + ); + + if ( + !session.isOriginalFromAddressAllowlisted && + session.originalFromAddressDomain !== session.originalFromAddressRootDomain + ) + session.isOriginalFromAddressAllowlisted = await isAllowlisted( + session.originalFromAddressRootDomain, + this.client, + this.resolver + ); + + // store if the From had same sender hostname (used for spam prevention) + session.hasSameHostnameAsFrom = Boolean( + session.resolvedClientHostname === session.originalFromAddressDomain || + session.resolvedRootClientHostname === + session.originalFromAddressRootDomain + ); + + // get all sender attributes (e.g. email, domain, root domain) + session.attributes = getAttributes(headers, session); + + // get message fingerprint + session.fingerprint = getFingerprint(session, headers, raw); + + return session; +} + +module.exports = updateSession; diff --git a/jobs/check-pm2.js b/jobs/check-pm2.js index f919f41acd..5bf178e59e 100644 --- a/jobs/check-pm2.js +++ b/jobs/check-pm2.js @@ -18,7 +18,7 @@ const ip = require('ip'); const mongoose = require('mongoose'); const parseErr = require('parse-err'); const pm2 = require('pm2'); -const prettyMs = require('pretty-ms'); +const prettyMilliseconds = require('pretty-ms'); const ms = require('ms'); const config = require('#config'); @@ -116,7 +116,7 @@ graceful.listen(); (p) => `${p.name} with status "${ p.pm2_env.status - }" and uptime of ${prettyMs( + }" and uptime of ${prettyMilliseconds( Date.now() - p.pm2_env.pm_uptime )}` ) diff --git a/jobs/tti.js b/jobs/tti.js index 9f31aba90c..5c8af5ced6 100644 --- a/jobs/tti.js +++ b/jobs/tti.js @@ -7,7 +7,6 @@ require('#config/env'); const os = require('node:os'); -const fs = require('node:fs'); const { Buffer } = require('node:buffer'); // eslint-disable-next-line import/no-unassigned-import @@ -18,7 +17,6 @@ const Redis = require('@ladjs/redis'); const _ = require('lodash'); const falso = require('@ngneat/falso'); const ip = require('ip'); -const isSANB = require('is-string-and-not-blank'); const mongoose = require('mongoose'); const ms = require('ms'); const prettyMilliseconds = require('pretty-ms'); @@ -34,25 +32,12 @@ const createMtaStsCache = require('#helpers/create-mta-sts-cache'); const createSession = require('#helpers/create-session'); const createTangerine = require('#helpers/create-tangerine'); const emailHelper = require('#helpers/email'); -const env = require('#config/env'); const getMessage = require('#helpers/get-message'); const logger = require('#helpers/logger'); const sendEmail = require('#helpers/send-email'); const setupMongoose = require('#helpers/setup-mongoose'); const monitorServer = require('#helpers/monitor-server'); -const signatureData = [ - { - signingDomain: env.DKIM_DOMAIN_NAME, - selector: env.DKIM_KEY_SELECTOR, - privateKey: isSANB(env.DKIM_PRIVATE_KEY_PATH) - ? fs.readFileSync(env.DKIM_PRIVATE_KEY_PATH, 'utf8') - : undefined, - algorithm: 'rsa-sha256', - canonicalization: 'relaxed/relaxed' - } -]; - monitorServer(); // @@ -242,7 +227,7 @@ Forward Email canonicalization: 'relaxed/relaxed', algorithm: 'rsa-sha256', signTime: new Date(), - signatureData + signatureData: [config.signatureData] }); if (signResult.errors.length > 0) { @@ -298,7 +283,7 @@ Forward Email canonicalization: 'relaxed/relaxed', algorithm: 'rsa-sha256', signTime: new Date(), - signatureData + signatureData: [config.signatureData] }); if (signResult.errors.length > 0) { diff --git a/mx-server.js b/mx-server.js new file mode 100644 index 0000000000..0e41cefdfa --- /dev/null +++ b/mx-server.js @@ -0,0 +1,116 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const fs = require('node:fs'); + +const bytes = require('bytes'); +const ms = require('ms'); +const pify = require('pify'); +const { SMTPServer } = require('smtp-server'); + +const RetryClient = require('#helpers/retry-client'); +const config = require('#config'); +const createTangerine = require('#helpers/create-tangerine'); +const env = require('#config/env'); +const logger = require('#helpers/logger'); +const onClose = require('#helpers/on-close'); +const onConnect = require('#helpers/on-connect'); +const onData = require('#helpers/on-data'); +const onMailFrom = require('#helpers/on-mail-from'); +const onRcptTo = require('#helpers/on-rcpt-to'); + +const MAX_BYTES = bytes(env.SMTP_MESSAGE_MAX_SIZE); + +// TODO: remove try/catch for isDenylisted/isSilent/isBackscatter +// and replace with catch (err) for onData to detect and store counter +// based off err.name detected or if it was combined then err.errors + +// TODO: we probably should disable spam scanner +// TODO: we probably should disable spam scanner +// TODO: we probably should disable spam scanner +// TODO: we probably should disable spam scanner +// TODO: we probably should disable spam scanner +// TODO: we probably should disable spam scanner +// TODO: we probably should disable spam scanner +// TODO: we probably should disable spam scanner +// TODO: we probably should disable spam scanner +class MX { + constructor(options = {}) { + this.client = options.client; + this.resolver = createTangerine(this.client, logger); + + // NOTE: this is useful for tests since we can pass `apiEndpoint` in test options + // TODO: remove API and replace with MongoDB calls (and then we can remove API from MX tests) + this.apiClient = new RetryClient(options.apiEndpoint || config.urls.api); + + // TODO: rate limiting (?) + + this.logger = logger; + + // setup our smtp server which listens for incoming email + // TODO: + this.server = new SMTPServer({ + // + // most of these options mirror the FE forwarding server options + // + size: MAX_BYTES, + onData: onData.bind(this), + onConnect: onConnect.bind(this), + onClose: onClose.bind(this), + onMailFrom: onMailFrom.bind(this), + onRcptTo: onRcptTo.bind(this), + // NOTE: we don't need to set a value for maxClients + // since we have rate limiting enabled by IP + // maxClients: Infinity, // default is Infinity + // allow 3m to process bulk RCPT TO + socketTimeout: config.socketTimeout, + // default closeTimeout is 30s + closeTimeout: ms('30s'), + // + disableReverseLookup: true, + logger: this.logger, + + disabledCommands: ['AUTH'], + secure: false, + needsUpgrade: false, + + // + // hide8BITMIME: true, + + // keys + ...(config.env === 'production' + ? { + key: fs.readFileSync(env.WEB_SSL_KEY_PATH), + cert: fs.readFileSync(env.WEB_SSL_CERT_PATH), + ca: fs.readFileSync(env.WEB_SSL_CA_PATH) + } + : {}) + }); + + // override logger + this.server.logger = this.logger; + + // kind of hacky but I filed a GH issue + // + this.server.address = this.server.server.address.bind(this.server.server); + + this.server.on('error', (err) => { + logger.error(err); + }); + + this.listen = this.listen.bind(this); + this.close = this.close.bind(this); + } + + async listen(port = env.MX_PORT, host = '::', ...args) { + await pify(this.server.listen).bind(this.server)(port, host, ...args); + } + + async close() { + await pify(this.server.close).bind(this.server); + } +} + +module.exports = MX; diff --git a/mx.js b/mx.js new file mode 100644 index 0000000000..6454e83893 --- /dev/null +++ b/mx.js @@ -0,0 +1,67 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const process = require('node:process'); + +// eslint-disable-next-line import/no-unassigned-import +require('#config/env'); +// eslint-disable-next-line import/no-unassigned-import +require('#config/mongoose'); + +const Graceful = require('@ladjs/graceful'); +const Redis = require('@ladjs/redis'); +const delay = require('delay'); +const ip = require('ip'); +const mongoose = require('mongoose'); +const ms = require('ms'); +const sharedConfig = require('@ladjs/shared-config'); + +const MX = require('./mx-server'); +const logger = require('#helpers/logger'); +const monitorServer = require('#helpers/monitor-server'); +const setupMongoose = require('#helpers/setup-mongoose'); + +const breeSharedConfig = sharedConfig('BREE'); +const client = new Redis(breeSharedConfig.redis, logger); + +const mx = new MX({ client }); + +const graceful = new Graceful({ + mongooses: [mongoose], + servers: [mx.server], + redisClients: [client], + customHandlers: [ + () => { + mx.isClosing = true; + }, + async () => { + // wait for connection rate limiter to finish + // (since `onClose` is run in the background) + await delay(ms('3s')); + } + ], + logger +}); +graceful.listen(); +monitorServer(); + +(async () => { + try { + await mx.listen(); + if (process.send) process.send('ready'); + logger.info( + `MX server listening on ${ + mx.server.address().port + } (LAN: ${ip.address()}:${mx.server.address().port})`, + { hide_meta: true } + ); + await setupMongoose(logger); + } catch (err) { + await logger.error(err); + process.exit(1); + } +})(); + +logger.info('MX server started', { hide_meta: true }); diff --git a/package.json b/package.json index ddd9109f0f..6345a21219 100644 --- a/package.json +++ b/package.json @@ -271,6 +271,7 @@ "sweetalert2": "8.19.1", "tangerine": "1.5.9", "titleize": "2", + "tlds": "1.254.0", "tsdav": "2.1.1", "twilio": "4.23.0", "typed.js": "2.1.0", diff --git a/smtp-server.js b/smtp-server.js index f839eb5fba..a59d802e13 100644 --- a/smtp-server.js +++ b/smtp-server.js @@ -16,25 +16,14 @@ const createTangerine = require('#helpers/create-tangerine'); const env = require('#config/env'); const logger = require('#helpers/logger'); const onAuth = require('#helpers/on-auth'); -const smtp = require('#helpers/smtp'); +const onClose = require('#helpers/on-close'); +const onConnect = require('#helpers/on-connect'); +const onData = require('#helpers/on-data'); +const onMailFrom = require('#helpers/on-mail-from'); +const onRcptTo = require('#helpers/on-rcpt-to'); const MAX_BYTES = bytes(env.SMTP_MESSAGE_MAX_SIZE); -async function onClose(session) { - // ignore unauthenticated sessions - if (!session?.user?.alias_id && !session?.user?.domain_id) return; - // decrease # connections for this alias (or domain if using catch-all) - try { - const key = `connections_${config.env}:${ - session.user.alias_id || session.user.domain_id - }`; - const count = await this.client.incrby(key, 0); - if (count > 0) await this.client.decr(key); - } catch (err) { - logger.fatal(err); - } -} - class SMTP { constructor( options = {}, @@ -66,12 +55,12 @@ class SMTP { // most of these options mirror the FE forwarding server options // size: MAX_BYTES, - onData: smtp.onData.bind(this), - onConnect: smtp.onConnect.bind(this), + onData: onData.bind(this), + onConnect: onConnect.bind(this), onClose: onClose.bind(this), onAuth: onAuth.bind(this), - onMailFrom: smtp.onMailFrom.bind(this), - onRcptTo: smtp.onRcptTo.bind(this), + onMailFrom: onMailFrom.bind(this), + onRcptTo: onRcptTo.bind(this), // NOTE: we don't need to set a value for maxClients // since we have rate limiting enabled by IP // maxClients: Infinity, // default is Infinity diff --git a/smtp.js b/smtp.js index eddfd065f8..b19129ad3d 100644 --- a/smtp.js +++ b/smtp.js @@ -33,6 +33,9 @@ const graceful = new Graceful({ servers: [smtp.server], redisClients: [client], customHandlers: [ + () => { + smtp.isClosing = true; + }, async () => { // wait for connection rate limiter to finish // (since `onClose` is run in the background) diff --git a/test/mx/index.js b/test/mx/index.js new file mode 100644 index 0000000000..2bd3cb7a4c --- /dev/null +++ b/test/mx/index.js @@ -0,0 +1,239 @@ +/** + * Copyright (c) Forward Email LLC + * SPDX-License-Identifier: BUSL-1.1 + */ + +const util = require('node:util'); + +const API = require('@ladjs/api'); +const Redis = require('ioredis-mock'); +const dayjs = require('dayjs-with-plugins'); +const getPort = require('get-port'); +const ip = require('ip'); +const ms = require('ms'); +const mxConnect = require('mx-connect'); +const nodemailer = require('nodemailer'); +const pify = require('pify'); +const test = require('ava'); + +const utils = require('../utils'); +const MX = require('../../mx-server'); + +const { Users } = require('#models'); +const apiConfig = require('#config/api'); +const env = require('#config/env'); +const config = require('#config'); +const logger = require('#helpers/logger'); + +const asyncMxConnect = pify(mxConnect); +const IP_ADDRESS = ip.address(); +const client = new Redis(); +client.setMaxListeners(0); +const tls = { rejectUnauthorized: false }; + +test.before(utils.setupMongoose); +test.after.always(utils.teardownMongoose); +test.beforeEach(utils.setupSMTPServer); + +// setup API server so we can configure MX with it +// (similar to `utils.setupApiServer`) +test.beforeEach(async (t) => { + const api = new API( + { + ...apiConfig, + redis: t.context.client + }, + Users + ); + const port = await getPort(); + t.context.apiPort = port; + await api.listen(port); +}); + +test('connects', async (t) => { + const smtp = new MX({ + client: t.context.client, + apiEndpoint: `http://${IP_ADDRESS}:${t.context.apiPort}` + }); + const { resolver } = smtp; + const port = await getPort(); + await smtp.listen(port); + + const user = await utils.userFactory + .withState({ + plan: 'enhanced_protection', + [config.userFields.planSetAt]: dayjs().startOf('day').toDate() + }) + .create(); + + await utils.paymentFactory + .withState({ + user: user._id, + amount: 300, + invoice_at: dayjs().startOf('day').toDate(), + method: 'free_beta_program', + duration: ms('30d'), + plan: user.plan, + kind: 'one-time' + }) + .create(); + + await user.save(); + + const domain = await utils.domainFactory + .withState({ + members: [{ user: user._id, group: 'admin' }], + plan: user.plan, + has_smtp: true, + resolver + }) + .create(); + + await utils.aliasFactory + .withState({ + name: '*', // catch-all + user: user._id, + domain: domain._id, + recipients: [user.email] + }) + .create(); + + // spoof dns records + const map = new Map(); + + // spoof test@test.com mx records + map.set( + 'mx:test.com', + resolver.spoofPacket( + 'test.com', + 'MX', + [{ exchange: IP_ADDRESS, priority: 0 }], + true + ) + ); + + map.set( + `mx:${domain.name}`, + resolver.spoofPacket( + domain.name, + 'MX', + [{ exchange: IP_ADDRESS, priority: 0 }], + true + ) + ); + + map.set( + `txt:${domain.name}`, + resolver.spoofPacket( + domain.name, + 'TXT', + [`${config.paidPrefix}${domain.verification_record}`], + true + ) + ); + + // dkim + map.set( + `txt:${domain.dkim_key_selector}._domainkey.${domain.name}`, + resolver.spoofPacket( + `${domain.dkim_key_selector}._domainkey.${domain.name}`, + 'TXT', + [`v=DKIM1; k=rsa; p=${domain.dkim_public_key.toString('base64')};`], + true + ) + ); + + // spf + map.set( + `txt:${env.WEB_HOST}`, + resolver.spoofPacket( + `${env.WEB_HOST}`, + 'TXT', + [`v=spf1 ip4:${IP_ADDRESS} -all`], + true + ) + ); + + // cname + map.set( + `cname:${domain.return_path}.${domain.name}`, + resolver.spoofPacket( + `${domain.return_path}.${domain.name}`, + 'CNAME', + [env.WEB_HOST], + true + ) + ); + + // cname -> txt + map.set( + `txt:${domain.return_path}.${domain.name}`, + resolver.spoofPacket( + `${domain.return_path}.${domain.name}`, + 'TXT', + [`v=spf1 ip4:${IP_ADDRESS} -all`], + true + ) + ); + + // dmarc + map.set( + `txt:_dmarc.${domain.name}`, + resolver.spoofPacket( + `_dmarc.${domain.name}`, + 'TXT', + [ + // TODO: consume dmarc reports and parse dmarc-$domain + `v=DMARC1; p=reject; pct=100; rua=mailto:dmarc-${domain.id}@forwardemail.net;` + ], + true + ) + ); + + // store spoofed dns cache + await resolver.options.cache.mset(map); + + const mx = await asyncMxConnect({ + target: IP_ADDRESS, + port: smtp.server.address().port, + dnsOptions: { + // + resolve: util.callbackify(resolver.resolve.bind(resolver)) + } + }); + + const transporter = nodemailer.createTransport({ + logger, + debug: true, + host: mx.host, + port: mx.port, + connection: mx.socket, + // TODO: ignoreTLS: true, + secure: false, + tls + }); + + await t.notThrowsAsync( + transporter.sendMail({ + envelope: { + from: 'test@test.com', + to: `test@${domain.name}` + }, + raw: ` +To: test@${domain.name} +From: test@test.com +Subject: test +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +Test`.trim() + }) + ); + + await smtp.close(); +}); + +// TODO: test forwarding +// TODO: test IMAP +// TODO: test EICAR +// TODO: other tests to copy over (e.g. webhooks)