fix: fixed API responses, fixed tests, added pagination support to AP…

…I, updated API docs for existing and new WIP endpoints, added HTTP pagination headers and documentation, fixed punycode domain name support, fixed header issues, added bank transfer/wire transfer payment methods, fixed issue with Date header being incorrect with SMTP/Emails.queue, ensured punycode in allowlist, sync locales and update translations

app/controllers/api/v1/domains.js

@@ -7,9 +7,16 @@ const _ = require('lodash');
 const pickOriginal = require('@ladjs/pick-original');
 
 const config = require('#config');
+const populateDomainStorage = require('#helpers/populate-domain-storage');
 const toObject = require('#helpers/to-object');
 const { Users, Aliases, Domains } = require('#models');
 
+const VIRTUAL_KEYS = [
+  'storage_used',
+  'storage_used_by_aliases',
+  'storage_quota'
+];
+
 function json(domain, isList = false) {
   const object = toObject(Domains, domain);
   // map max recipients per alias
@@ -47,37 +54,33 @@ function json(domain, isList = false) {
       });
   }
 
+  // preserve virtuals added in `app/controllers/web/my-account/list-domains.js`
+  // - storage_used
+  // - storage_used_by_aliases
+  // - storage_quota
+  const virtuals = {};
+  for (const key of VIRTUAL_KEYS) {
+    if (typeof domain[key] !== 'undefined') virtuals[key] = domain[key];
+  }
+
   return {
     ...pickOriginal(
       object,
       _.isFunction(domain.toObject) ? domain.toObject() : domain
     ),
+    ...virtuals,
     // add a helper url
     link: `${config.urls.web}/my-account/domains/${domain.name}`
   };
 }
 
 async function list(ctx) {
-  // hide global domain names if not admin of
-  // the global domain and had zero aliases
-  const data = ctx.state.domains
-    .filter((domain) => {
-      if (!domain.is_global) return true;
-      const member = domain.members.find(
-        (m) => m.user.id === ctx.state.user.id
-      );
-      if (!member) return false;
-      if (member.group === 'admin') return true;
-      if (domain.has_global_aliases) return true;
-      return false;
-    })
-    .map((d) => json(d, true));
-
-  ctx.body = data;
+  ctx.body = ctx.state.domains.map((d) => json(d, true));
 }
 
 async function retrieve(ctx) {
-  const data = json(ctx.state.domain);
+  // populate storage quota stuff
+  const data = json(await populateDomainStorage(ctx.state.domain, ctx.locale));
   ctx.body = data;
 }
 

app/controllers/api/v1/emails.js

@@ -12,19 +12,58 @@ const config = require('#config');
 const createSession = require('#helpers/create-session');
 const toObject = require('#helpers/to-object');
 
+const REJECTED_ERROR_KEYS = [
+  'recipient',
+  'responseCode',
+  'response',
+  'message'
+];
+
 function json(email, isList = false) {
   const object = toObject(Emails, email);
+
+  //
+  // NOTE: we always rewrite rejectedErrors
+  //       since we don't want to show code bugs
+  //       to user via API response
+  //
+  delete object.rejectedErrors;
+
   if (isList) {
+    delete object.message;
     delete object.headers;
-    delete object.accepted;
-    delete object.rejectedErrors;
+  } else {
+    //
+    // instead we render it similarly as we do in My Account > Emails
+    // (and we only render these fields to the user)
+    //
+    // - recipient
+    // - responseCode
+    // - response
+    // - message
+    //
+    // (not the full error object which contains stack trace etc.)
+    //
+    object.rejectedErrors = email.rejectedErrors.map((err) => {
+      const error = {};
+      for (const key of REJECTED_ERROR_KEYS) {
+        if (typeof err[key] !== 'undefined') error[key] = err[key];
+      }
+
+      return error;
+    });
   }
 
+  //
+  // safeguard to always add `rejectedErrors` since
+  // we have it listed in omitExtraFields in emails model
+  // (we never want to accidentally render it to a user)
+  //
+  const keys = _.isFunction(email.toObject) ? email.toObject() : email;
+  if (!isList) keys.rejectedErrors = object.rejectedErrors;
+
   return {
-    ...pickOriginal(
-      object,
-      _.isFunction(email.toObject) ? email.toObject() : email
-    ),
+    ...pickOriginal(object, keys),
     // add a helper url
     link: `${config.urls.web}/my-account/emails/${email.id}`
   };
@@ -37,7 +76,7 @@ async function list(ctx) {
 async function retrieve(ctx) {
   const body = json(ctx.state.email);
   // we want to return the `message` property
-  body.message = await Emails.getMessage(ctx.state.email.message);
+  body.message = await Emails.getMessage(ctx.state.email.message, true);
   ctx.body = body;
 }
 
@@ -156,7 +195,10 @@ async function create(ctx) {
       ignore_hook: false
     });
 
-    ctx.body = email;
+    // we want to return the `message` property
+    const body = json(email);
+    body.message = await Emails.getMessage(email.message, true);
+    ctx.body = body;
   } catch (err) {
     ctx.logger.error(err);
     ctx.throw(err);

app/controllers/web/my-account/create-domain.js

@@ -6,8 +6,7 @@
 const Boom = require('@hapi/boom');
 const { boolean } = require('boolean');
 
-const toObject = require('#helpers/to-object');
-const { Users, Domains, Aliases } = require('#models');
+const { Domains, Aliases } = require('#models');
 
 const config = require('#config');
 const logger = require('#helpers/logger');
@@ -94,11 +93,7 @@ async function createDomain(ctx, next) {
       });
     }
 
-    if (ctx.api) {
-      ctx.state.domain = toObject(Domains, ctx.state.domain);
-      ctx.state.domain.members[0].user = toObject(Users, ctx.state.user);
-      return next();
-    }
+    if (ctx.api) return next();
 
     // TODO: flash messages logic in @ladjs/assets doesn't support both
     // custom and regular flash message yet

app/controllers/web/my-account/list-billing.js

@@ -10,6 +10,7 @@ const isSANB = require('is-string-and-not-blank');
 const paginate = require('koa-ctx-paginate');
 
 const config = require('#config');
+const setPaginationHeaders = require('#helpers/set-pagination-headers');
 
 const REGEX_AMOUNT_FORMATTED = new RE2('amount_formatted', 'i');
 
@@ -21,10 +22,11 @@ async function listBilling(ctx) {
 
   // sort payments
   let sortFn;
-  if (REGEX_AMOUNT_FORMATTED.test(ctx.query.sort))
-    sortFn = (p) => p.amount_formatted.replace(/[^\d.]/, '');
-  else if (isSANB(ctx.query.sort))
-    sortFn = (p) => p[ctx.query.sort.replace(/^-/, '')];
+  if (isSANB(ctx.query.sort)) {
+    sortFn = REGEX_AMOUNT_FORMATTED.test(ctx.query.sort)
+      ? (p) => p.amount_formatted.replace(/[^\d.]/, '')
+      : (p) => p[ctx.query.sort.replace(/^-/, '')];
+  }
 
   payments = _.sortBy(payments, sortFn ? [sortFn] : ['invoice_at']);
 
@@ -48,6 +50,18 @@ async function listBilling(ctx) {
     )
   );
 
+  //
+  // set HTTP headers for pagination
+  // <https://forwardemail.net/api#pagination>
+  //
+  setPaginationHeaders(
+    ctx,
+    pageCount,
+    ctx.query.page,
+    payments.length,
+    itemCount
+  );
+
   if (ctx.accepts('html'))
     return ctx.render('my-account/billing', {
       payments,