diff --git a/ghost/core/core/server/services/members/exporter/query.js b/ghost/core/core/server/services/members/exporter/query.js index 81594279acf0..7d7fbdfaf41d 100644 --- a/ghost/core/core/server/services/members/exporter/query.js +++ b/ghost/core/core/server/services/members/exporter/query.js @@ -1,9 +1,11 @@ const models = require('../../../models'); const {knex} = require('../../../data/db'); const moment = require('moment'); +const logging = require('@tryghost/logging'); module.exports = async function (options) { const hasFilter = options.limit !== 'all' || options.filter || options.search; + const start = Date.now(); let ids = null; if (hasFilter) { @@ -31,58 +33,120 @@ module.exports = async function (options) { */ } - const allProducts = await models.Product.fetchAll(); - const allLabels = await models.Label.fetchAll(); + logging.info('[MembersExporter] Fetching products and labels'); - let query = knex('members') + const startFetchingProducts = Date.now(); + + const allProducts = await knex('products').select('id', 'name').then(rows => rows.reduce((acc, product) => { + acc[product.id] = product.name; + return acc; + }, {}) + ); + + const allLabels = await knex('labels').select('id', 'name').then(rows => rows.reduce((acc, label) => { + acc[label.id] = label.name; + return acc; + }, {}) + ); + + logging.info('[MembersExporter] Fetched products and labels in ' + (Date.now() - startFetchingProducts) + 'ms'); + + const startFetchingMembers = Date.now(); + + const members = await knex('members') .select('id', 'email', 'name', 'note', 'status', 'created_at') - .select(knex.raw(` - (CASE WHEN EXISTS (SELECT 1 FROM members_newsletters n WHERE n.member_id = members.id) - THEN TRUE ELSE FALSE - END) as subscribed - `)) - .select(knex.raw(` - (SELECT GROUP_CONCAT(product_id) FROM members_products f WHERE f.member_id = members.id) as tiers - `)) - .select(knex.raw(` - (SELECT GROUP_CONCAT(label_id) FROM members_labels f WHERE f.member_id = members.id) as labels - `)) - .select(knex.raw(` - (SELECT customer_id FROM members_stripe_customers f WHERE f.member_id = members.id limit 1) as stripe_customer_id - `)); + .modify((query) => { + if (hasFilter) { + query.whereIn('id', ids); + } + }); - if (hasFilter) { - query = query.whereIn('id', ids); - } + logging.info('[MembersExporter] Fetched members in ' + (Date.now() - startFetchingMembers) + 'ms'); + + const startFetchingTiers = Date.now(); + + const tiers = await knex('members_products') + .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers')) + .groupBy('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }); + + logging.info('[MembersExporter] Fetched tiers in ' + (Date.now() - startFetchingTiers) + 'ms'); + + const startFetchingLabels = Date.now(); + + const labels = await knex('members_labels') + .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels')) + .groupBy('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }); - const rows = await query; - for (const row of rows) { - const tierIds = row.tiers ? row.tiers.split(',') : []; - const tiers = tierIds.map((id) => { - const tier = allProducts.find(p => p.id === id); + logging.info('[MembersExporter] Fetched labels in ' + (Date.now() - startFetchingLabels) + 'ms'); + + const startFetchingStripeCustomers = Date.now(); + + const stripeCustomers = await knex('members_stripe_customers') + .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id')) + .groupBy('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }); + + logging.info('[MembersExporter] Fetched stripe customers in ' + (Date.now() - startFetchingStripeCustomers) + 'ms'); + + const startFetchingSubscriptions = Date.now(); + + const subscriptions = await knex('members_newsletters') + .distinct('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }); + + logging.info('[MembersExporter] Fetched subscriptions in ' + (Date.now() - startFetchingSubscriptions) + 'ms'); + + const startInMemoryProcessing = Date.now(); + + const tiersMap = new Map(tiers.map(row => [row.member_id, row.tiers])); + const labelsMap = new Map(labels.map(row => [row.member_id, row.labels])); + const stripeCustomerMap = new Map(stripeCustomers.map(row => [row.member_id, row.stripe_customer_id])); + const subscribedSet = new Set(subscriptions.map(row => row.member_id)); + + for (const row of members) { + const tierIds = tiersMap.get(row.id) ? tiersMap.get(row.id).split(',') : []; + const tierDetails = tierIds.map((id) => { return { - name: tier.get('name') + name: allProducts[id] }; }); - row.tiers = tiers; + row.tiers = tierDetails; - const labelIds = row.labels ? row.labels.split(',') : []; - const labels = labelIds.map((id) => { - const label = allLabels.find(l => l.id === id); + const labelIds = labelsMap.get(row.id) ? labelsMap.get(row.id).split(',') : []; + const labelDetails = labelIds.map((id) => { return { - name: label.get('name') + name: allLabels[id] }; }); - row.labels = labels; - } + row.labels = labelDetails; - for (const member of rows) { - // Note: we don't modify the array or change/duplicate objects - // to increase performance - member.subscribed = !!member.subscribed; - member.comped = member.status === 'comped'; - member.created_at = moment(member.created_at).toISOString(); + row.subscribed = subscribedSet.has(row.id); + row.comped = row.status === 'comped'; + row.stripe_customer_id = stripeCustomerMap.get(row.id) || null; + row.created_at = moment(row.created_at).toISOString(); } - return rows; + logging.info('[MembersExporter] In memory processing finished in ' + (Date.now() - startInMemoryProcessing) + 'ms'); + + logging.info('[MembersExporter] Total time taken: ' + (Date.now() - start) / 1000 + 's'); + + return members; };