feat: added API endpoint to download logs, updated email verbiage, sy…

…nc locales

app/controllers/web/my-account/list-logs.js

@@ -322,34 +322,36 @@ async function listLogs(ctx) {
   }
 
   // in the future we can move this to a background job
-  if (ctx.pathWithoutLocale === '/my-account/logs/download') {
+  if (
+    ctx.pathWithoutLocale === '/my-account/logs/download' ||
+    (ctx.api &&
+      ctx.pathWithoutLocale === '/v1/logs/download' &&
+      ctx.method === 'POST')
+  ) {
     // download in background and email to users
     const now = new Date();
     getLogsCsv(now, query)
       .then((results) => {
         // if no results return early
-        if (results.count === 0) return;
+        if (!ctx.api && results.count === 0) return;
         // email the spreadsheet to admins
         emailHelper({
           template: 'alert',
           message: {
             to: ctx.state.user[config.userFields.fullEmail],
             bcc: config.email.message.from,
-            subject: `(${results.count}) Email Deliverability Logs for ${dayjs(
-              now
-            ).format('M/D/YY h:mm A z')} (${
-              results.set.size
-            } trusted hosts blocked)`,
-            attachments: [
-              {
-                filename: `email-deliverability-logs-${dayjs(now).format(
-                  'YYYY-MM-DD-h-mm-A-z'
-                )}.csv.gz`.toLowerCase(),
-                content: zlib.gzipSync(Buffer.from(results.csv, 'utf8'), {
-                  level: 9
-                })
-              }
-            ]
+            subject: results.subject,
+            attachments:
+              results.count > 0
+                ? [
+                    {
+                      filename: results.filename + '.gz',
+                      content: zlib.gzipSync(Buffer.from(results.csv, 'utf8'), {
+                        level: 9
+                      })
+                    }
+                  ]
+                : []
           },
           locals: {
             message: results.message
@@ -369,7 +371,9 @@ async function listLogs(ctx) {
     const message = ctx.translate('LOG_DOWNLOAD_IN_PROGRESS');
     const redirectTo = ctx.state.l('/my-account/logs');
 
-    if (ctx.accepts('html')) {
+    if (ctx.api) {
+      ctx.body = message;
+    } else if (ctx.accepts('html')) {
       ctx.flash('success', message);
       ctx.redirect(redirectTo);
     } else {

app/views/api/index.md

@@ -9,6 +9,8 @@
 * [Errors](#errors)
 * [Localization](#localization)
 * [Pagination](#pagination)
+* [Logs](#logs)
+  * [Retrieve logs](#retrieve-logs)
 * [Account](#account)
   * [Create account](#create-account)
   * [Retrieve account](#retrieve-account)
@@ -105,6 +107,45 @@ Our service is translated to over 25 different languages. All API response messa
 If you would like to be notified when pagination is available, then please email <[email protected]>.
 
 
+## Logs
+
+### Retrieve logs
+
+Our API programmatically allows you to download logs for your account.  Submitting a request to this endpoint will process all logs for your account and email them to you as an attachment ([Gzip](https://en.wikipedia.org/wiki/Gzip) compressed [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) spreadsheet file) once complete.
+
+This allows you to create background jobs with a [Cron job](https://en.wikipedia.org/wiki/Cron) or using our [Node.js job scheduling software Bree](https://github.com/breejs/bree) to receive logs whenever you desire.  Note that this endpoint is limited to `10` requests per day.
+
+The attachment is the lowercase form of `email-deliverability-logs-YYYY-MM-DD-h-mm-A-z.csv.gz` and the email itself contains a brief summary of the logs retrieved.  You can also download logs at any time from [My Account → Logs](/my-account/logs)
+
+> `GET /v1/logs/download`
+
+| Querystring Parameter | Required | Type          | Description                                                                                                                     |
+| --------------------- | -------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------- |
+| `domain`              | No       | String (FQDN) | Filter logs by fully qualified domain ("FQDN").  If you do not provide this then all logs across all domains will be retrieved. |
+| `q`                   | No       | String        | Search for logs by email, domain, alias name, IP address, or date (`M/Y`, `M/D/YY`, `M-D`, `M-D-YY`, or `M.D.YY` format).       |
+
+> Example Request:
+
+```sh
+curl -X POST BASE_URI/v1/logs/download \
+  -u API_TOKEN:
+```
+
+> Example Cron job (at midnight every day):
+
+```sh
+0 0 * * * /usr/bin/curl BASE_URI/v1/logs/download -u API_TOKEN: &>/dev/null
+```
+
+Note that you can use services such as [Crontab.guru](https://crontab.guru/) to validate your cron job expression syntax.
+
+> Example Cron job (at midnight every day **and with logs for previous day**):
+
+```sh
+0 0 * * * /usr/bin/curl BASE_URI/v1/logs/download?q=`date -v-1d -u "+%-m/%-d/%y"` -u API_TOKEN: &>/dev/null
+```
+
+
 ## Account
 
 ### Create account

app/views/my-account/logs/_table.pug

@@ -89,7 +89,11 @@ table.table.table-hover.table-bordered.table-sm
                         li.list-inline-item.d-inline
                           span.badge.badge-pill(class=badgeClass)= statusCode
                     li.list-inline-item.small.d-inline
-                      small.text-monospace!= log.err && log.err.message ? ansiHTML(log.err.message) : ansiHTML(log.message)
+                      small.text-monospace
+                        if log.err && log.err.isCodeBug === true
+                          = 'An unexpected internal server error has occurred'
+                        else
+                          != log.err && log.err.message ? ansiHTML(log.err.message) : ansiHTML(log.message)
             td.align-middle.text-center
               ul.list-inline.mb-0
                 li.list-inline-item

app/views/my-account/logs/retrieve.pug

@@ -69,7 +69,11 @@ block body
             li.list-inline-item.d-inline
               span.badge.badge-pill(class=badgeClass)= statusCode
           li.list-inline-item.small.d-inline
-            small.text-monospace!= log.err && log.err.message ? ansiHTML(log.err.message) : ansiHTML(log.message)
+            small.text-monospace
+              if log.err && log.err.isCodeBug === true
+                = 'An unexpected internal server error has occurred'
+              else
+                != log.err && log.err.message ? ansiHTML(log.err.message) : ansiHTML(log.message)
         if _.isObject(log.meta) && _.isObject(log.meta.session) && _.isObject(log.meta.session.headers) && !_.isEmpty(log.meta.session.headers)
           h3.h5= t("Message Headers")
           table.table.table-hover.table-bordered.table-striped

emails/self-test/html.pug

@@ -21,32 +21,110 @@ block content
                 = t("No, we only send it the first time as a courtesy.")
             .p-3
               h2.h5= t("Why are you sending this email?")
-              p.card-text
+              p.card-text.small
                 != t("It is a widely known issue that <u>only</u> happens when you send an email to yourself <i>as</i> yourself.")
                 = " "
                 = t('The emails do not show up twice in your inbox because they have the same "Message-ID" value in the email headers.')
                 = " "
                 = t("If you're using a service such as Gmail, then the email will only be shown in your Sent folder.")
                 = " "
                 = t("We are simply letting you know in advance of this issue!")
-              a.btn.btn-lg.btn-danger(
+              a.btn.btn-sm.btn-danger(
                 href="https://support.google.com/a/answer/1703601",
                 target="_blank",
                 rel="noopener noreferrer"
               )
                 = t("Read the official Gmail answer")
-            .p-3
-              h2.h5= t("Is there a workaround?")
-              p.card-text
-                = t("We spent a lot of time researching, testing, and implementing alternatives since 2017.")
                 = " "
-                = t("Unfortunately we did not discover any reasonable alternatives that respect privacy and security.")
+                != "&rarr;"
             .p-3
-              h2.h5= t("What is the technical reason?")
+              h2= t("What is Forward Email?")
               p.card-text
-                = t('The core reason why this happens is because emails with duplicate "Message-ID" headers only show up once in the inbox.')
-              p.card-text
-                != t('In the past we had a workaround where we rewrote the "Message-ID" and removed the "DKIM-Signature" header.')
+                != t('For <span class="notranslate">%d</span> years and counting, we are the go-to email service for hundreds of thousands of creators, developers, and businesses.', dayjs().endOf("year").diff(dayjs("1/1/17", "M-D/YY"), "year"))
                 = " "
-                = t('This workaround led to a confusing "Be careful with this message" alert and sometimes marked it as spam in Gmail.')
+                != t('Send and receive email as <span class="notranslate font-weight-bold text-nowrap">[email protected]</span>.')
+              ul.list-unstyled.text-left.mb-3.d-inline-block.mx-auto
+                li
+                  = emoji("white_check_mark")
+                  = " "
+                  = t("Unlimited domains and aliases")
+                  = " "
+                  span.badge.badge-success
+                    = t("100% Free")
+                li
+                  = emoji("white_check_mark")
+                  = " "
+                  = t("Privacy-focused email")
+                  = " "
+                  a.badge.badge-dark.align-middle(
+                    href=`${config.urls.web}/privacy`
+                  )
+                    = t("Privacy Policy")
+                    = " "
+                    != "&rarr;"
+                li
+                  = emoji("white_check_mark")
+                  = " "
+                  = t("Send outbound SMTP email")
+                  = " "
+                  a.badge.badge-dark(
+                    href=`${config.urls.web}/guides/send-email-with-custom-domain-smtp`
+                  )
+                    = t("SMTP Guide")
+                    = " "
+                    != "&rarr;"
+                li
+                  = emoji("white_check_mark")
+                  = " "
+                  = t("Error logs and real-time alerts")
+                  = " "
+                  a.badge.badge-dark(
+                    href=`${config.urls.web}/faq#do-you-store-error-logs`
+                  )
+                    = t("View FAQ")
+                    = " "
+                    != "&rarr;"
+                li
+                  = emoji("white_check_mark")
+                  = " "
+                  = t("Powered by bare metal servers")
+                  = " "
+                  a.badge.badge-dark(
+                    href="https://status.forwardemail.net",
+                    target="_blank",
+                    rel="noopener noreferrer"
+                  )
+                    = t("Status Page")
+                    = " "
+                    != "&rarr;"
+                li
+                  = emoji("white_check_mark")
+                  = " "
+                  = t("100% open-source software")
+                  = " "
+                  a.badge.badge-dark(
+                    href="https://github.com/forwardemail",
+                    target="_blank",
+                    rel="noopener noreferrer"
+                  )
+                    = t("View")
+                    = " "
+                    = t("GitHub")
+                    = " "
+                    != "&rarr;"
+                li
+                  = emoji("white_check_mark")
+                  = " "
+                  = t("Email API designed for developers")
+                  = " "
+                  a.badge.badge-dark(
+                    href=`${config.urls.web}/email-api`
+                  )
+                    = t("Email API")
+                    = " "
+                    != "&rarr;"
+              a.btn.btn-lg.btn-success.text-uppercase.font-weight-bold(
+                href=config.urls.web
+              )
+                = t("Try for free")
           .card-footer.small.text-muted= t("If you have any questions or comments, then please let us know.")

helpers/get-logs-csv.js

@@ -13,7 +13,7 @@ function makeDelimitedString(arr) {
 }
 
 // eslint-disable-next-line complexity
-async function getLogsCsv(now = new Date(), query = {}) {
+async function getLogsCsv(now = new Date(), query = {}, isAdmin = false) {
   if (!_.isObject(query) || _.isEmpty(query)) throw new Error('Invalid query');
 
   //
@@ -69,6 +69,9 @@ async function getLogsCsv(now = new Date(), query = {}) {
     .cursor()
     .addCursorFlag('noCursorTimeout', true)) {
     if (!log?.meta?.session?.id) continue;
+    let response = log?.err?.response || log?.err?.message || log.message;
+    if (!isAdmin && log?.err?.isCodeBug === true)
+      response = 'An unexpected internal server error has occurred';
     // add new row to spreadsheet
     csv.push(
       makeDelimitedString([
@@ -89,7 +92,7 @@ async function getLogsCsv(now = new Date(), query = {}) {
           ? log.err.truthSource
           : '',
         // SMTP Response
-        log?.err?.response || log?.err?.message || log.message,
+        response,
         // SMTP Code
         log?.err?.responseCode,
         // From
@@ -172,9 +175,19 @@ async function getLogsCsv(now = new Date(), query = {}) {
     );
   }
 
-  const message = [
-    `<p>Log download from ${dayjs(now).format('M/D/YY h:mm A z')}:</p>`
-  ];
+  const message = [];
+  const count = csv.length - 1;
+
+  if (count === 0)
+    message.push(
+      `<p>No logs were available to download from ${dayjs(now).format(
+        'M/D/YY h:mm A z'
+      )}.</p>`
+    );
+  else
+    message.push(
+      `<p>Log download from ${dayjs(now).format('M/D/YY h:mm A z')}:</p>`
+    );
 
   if (list.length > 0) {
     message.push(`<ul>`, ...list, `</ul>`);
@@ -190,8 +203,18 @@ async function getLogsCsv(now = new Date(), query = {}) {
     );
   }
 
+  const subject = `(${count}) Email Deliverability Logs for ${dayjs(now).format(
+    'M/D/YY h:mm A z'
+  )} (${set.size} trusted hosts blocked)`;
+
+  const filename = `email-deliverability-logs-${dayjs(now).format(
+    'YYYY-MM-DD-h-mm-A-z'
+  )}.csv`.toLowerCase();
+
   return {
-    count: csv.length - 1,
+    subject,
+    filename,
+    count,
     csv: csv.join('\n'),
     set,
     message: message.join('\n')