From 374ae21f4c2c43014ae962becbfd5b92b38203a1 Mon Sep 17 00:00:00 2001 From: Princi Vershwal Date: Tue, 14 Jan 2025 22:54:54 +0530 Subject: [PATCH 1/6] Initial commit --- ghost/core/core/server/services/members/exporter/query.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ghost/core/core/server/services/members/exporter/query.js b/ghost/core/core/server/services/members/exporter/query.js index 81594279acf..9859c18b314 100644 --- a/ghost/core/core/server/services/members/exporter/query.js +++ b/ghost/core/core/server/services/members/exporter/query.js @@ -2,6 +2,7 @@ const models = require('../../../models'); const {knex} = require('../../../data/db'); const moment = require('moment'); + module.exports = async function (options) { const hasFilter = options.limit !== 'all' || options.filter || options.search; From 6d0005fe4dcc001b4c8e01e8f577d1a29955b273 Mon Sep 17 00:00:00 2001 From: Princi Vershwal Date: Tue, 14 Jan 2025 23:01:47 +0530 Subject: [PATCH 2/6] Initial Commit --- ghost/core/core/server/services/members/exporter/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghost/core/core/server/services/members/exporter/query.js b/ghost/core/core/server/services/members/exporter/query.js index 9859c18b314..9783fc85c28 100644 --- a/ghost/core/core/server/services/members/exporter/query.js +++ b/ghost/core/core/server/services/members/exporter/query.js @@ -2,7 +2,6 @@ const models = require('../../../models'); const {knex} = require('../../../data/db'); const moment = require('moment'); - module.exports = async function (options) { const hasFilter = options.limit !== 'all' || options.filter || options.search; @@ -10,6 +9,7 @@ module.exports = async function (options) { if (hasFilter) { // do a very minimal query, only to fetch the ids of the filtered values // should be quite fast + // hello options.withRelated = []; options.columns = ['id']; From f0ae3d68e3357ac7fac2da6621f29b693d683486 Mon Sep 17 00:00:00 2001 From: Princi Vershwal Date: Wed, 15 Jan 2025 17:37:43 +0530 Subject: [PATCH 3/6] Optimised sql query --- .../server/services/members/exporter/query.js | 94 +++++++++++-------- 1 file changed, 57 insertions(+), 37 deletions(-) diff --git a/ghost/core/core/server/services/members/exporter/query.js b/ghost/core/core/server/services/members/exporter/query.js index 9783fc85c28..5989fca49d0 100644 --- a/ghost/core/core/server/services/members/exporter/query.js +++ b/ghost/core/core/server/services/members/exporter/query.js @@ -9,7 +9,6 @@ module.exports = async function (options) { if (hasFilter) { // do a very minimal query, only to fetch the ids of the filtered values // should be quite fast - // hello options.withRelated = []; options.columns = ['id']; @@ -35,55 +34,76 @@ module.exports = async function (options) { const allProducts = await models.Product.fetchAll(); const allLabels = await models.Label.fetchAll(); - let query = knex('members') - .select('id', 'email', 'name', 'note', 'status', 'created_at') - .select(knex.raw(` - (CASE WHEN EXISTS (SELECT 1 FROM members_newsletters n WHERE n.member_id = members.id) - THEN TRUE ELSE FALSE - END) as subscribed - `)) - .select(knex.raw(` - (SELECT GROUP_CONCAT(product_id) FROM members_products f WHERE f.member_id = members.id) as tiers - `)) - .select(knex.raw(` - (SELECT GROUP_CONCAT(label_id) FROM members_labels f WHERE f.member_id = members.id) as labels - `)) - .select(knex.raw(` - (SELECT customer_id FROM members_stripe_customers f WHERE f.member_id = members.id limit 1) as stripe_customer_id - `)); + const [members, tiers, labels, stripeCustomers, subscriptions] = await Promise.all([ + knex('members') + .select('id', 'email', 'name', 'note', 'status', 'created_at') + .modify((query) => { + if (hasFilter) { + query.whereIn('id', ids); + } + }), + knex('members_products') + .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers')) + .groupBy('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }), + knex('members_labels') + .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels')) + .groupBy('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }), + knex('members_stripe_customers') + .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id')) + .groupBy('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }), + knex('members_newsletters') + .distinct('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }) + ]); - if (hasFilter) { - query = query.whereIn('id', ids); - } + const tiersMap = new Map(tiers.map(row => [row.member_id, row.tiers])); + const labelsMap = new Map(labels.map(row => [row.member_id, row.labels])); + const stripeCustomerMap = new Map(stripeCustomers.map(row => [row.member_id, row.stripe_customer_id])); + const subscribedSet = new Set(subscriptions.map(row => row.member_id)); - const rows = await query; - for (const row of rows) { - const tierIds = row.tiers ? row.tiers.split(',') : []; - const tiers = tierIds.map((id) => { + for (const row of members) { + const tierIds = tiersMap.get(row.id) ? tiersMap.get(row.id).split(',') : []; + const tierDetails = tierIds.map((id) => { const tier = allProducts.find(p => p.id === id); return { name: tier.get('name') }; }); - row.tiers = tiers; + row.tiers = tierDetails; - const labelIds = row.labels ? row.labels.split(',') : []; - const labels = labelIds.map((id) => { + const labelIds = labelsMap.get(row.id) ? labelsMap.get(row.id).split(',') : []; + const labelDetails = labelIds.map((id) => { const label = allLabels.find(l => l.id === id); return { name: label.get('name') }; }); - row.labels = labels; - } + row.labels = labelDetails; - for (const member of rows) { - // Note: we don't modify the array or change/duplicate objects - // to increase performance - member.subscribed = !!member.subscribed; - member.comped = member.status === 'comped'; - member.created_at = moment(member.created_at).toISOString(); + row.subscribed = subscribedSet.has(row.id); + row.comped = row.status === 'comped'; + row.stripe_customer_id = stripeCustomerMap.get(row.id) || null; + row.created_at = moment(row.created_at).toISOString(); } - return rows; -}; + return members; +}; \ No newline at end of file From ad16d4ef9e6dfbe81094970a5917a0683a7ac3e5 Mon Sep 17 00:00:00 2001 From: Princi Vershwal Date: Thu, 16 Jan 2025 11:37:49 +0530 Subject: [PATCH 4/6] Testing with optimised query --- .../server/services/members/exporter/query.js | 105 ++++++++++-------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/ghost/core/core/server/services/members/exporter/query.js b/ghost/core/core/server/services/members/exporter/query.js index 5989fca49d0..607697697fe 100644 --- a/ghost/core/core/server/services/members/exporter/query.js +++ b/ghost/core/core/server/services/members/exporter/query.js @@ -31,49 +31,60 @@ module.exports = async function (options) { */ } - const allProducts = await models.Product.fetchAll(); - const allLabels = await models.Label.fetchAll(); - - const [members, tiers, labels, stripeCustomers, subscriptions] = await Promise.all([ - knex('members') - .select('id', 'email', 'name', 'note', 'status', 'created_at') - .modify((query) => { - if (hasFilter) { - query.whereIn('id', ids); - } - }), - knex('members_products') - .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers')) - .groupBy('member_id') - .modify((query) => { - if (hasFilter) { - query.whereIn('member_id', ids); - } - }), - knex('members_labels') - .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels')) - .groupBy('member_id') - .modify((query) => { - if (hasFilter) { - query.whereIn('member_id', ids); - } - }), - knex('members_stripe_customers') - .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id')) - .groupBy('member_id') - .modify((query) => { - if (hasFilter) { - query.whereIn('member_id', ids); - } - }), - knex('members_newsletters') - .distinct('member_id') - .modify((query) => { - if (hasFilter) { - query.whereIn('member_id', ids); - } - }) - ]); + const allProducts = await knex('products').select('id', 'name').then(rows => rows.reduce((acc, product) => { + acc[product.id] = product.name; + return acc; + }, {}) + ); + + const allLabels = await knex('labels').select('id', 'name').then(rows => rows.reduce((acc, label) => { + acc[label.id] = label.name; + return acc; + }, {}) + ); + + const members = await knex('members') + .select('id', 'email', 'name', 'note', 'status', 'created_at') + .modify((query) => { + if (hasFilter) { + query.whereIn('id', ids); + } + }); + + const tiers = await knex('members_products') + .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers')) + .groupBy('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }); + + const labels = await knex('members_labels') + .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels')) + .groupBy('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }); + + const stripeCustomers = await knex('members_stripe_customers') + .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id')) + .groupBy('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }); + + const subscriptions = await knex('members_newsletters') + .distinct('member_id') + .modify((query) => { + if (hasFilter) { + query.whereIn('member_id', ids); + } + }); const tiersMap = new Map(tiers.map(row => [row.member_id, row.tiers])); const labelsMap = new Map(labels.map(row => [row.member_id, row.labels])); @@ -83,18 +94,16 @@ module.exports = async function (options) { for (const row of members) { const tierIds = tiersMap.get(row.id) ? tiersMap.get(row.id).split(',') : []; const tierDetails = tierIds.map((id) => { - const tier = allProducts.find(p => p.id === id); return { - name: tier.get('name') + name: allProducts[id] }; }); row.tiers = tierDetails; const labelIds = labelsMap.get(row.id) ? labelsMap.get(row.id).split(',') : []; const labelDetails = labelIds.map((id) => { - const label = allLabels.find(l => l.id === id); return { - name: label.get('name') + name: allLabels[id] }; }); row.labels = labelDetails; @@ -106,4 +115,4 @@ module.exports = async function (options) { } return members; -}; \ No newline at end of file +}; From 344236af0d99e990f48e675e842bef79c68bdff4 Mon Sep 17 00:00:00 2001 From: Princi Vershwal Date: Thu, 16 Jan 2025 13:51:49 +0530 Subject: [PATCH 5/6] Added time logs --- .../server/services/members/exporter/query.js | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ghost/core/core/server/services/members/exporter/query.js b/ghost/core/core/server/services/members/exporter/query.js index 607697697fe..53be7c0eef2 100644 --- a/ghost/core/core/server/services/members/exporter/query.js +++ b/ghost/core/core/server/services/members/exporter/query.js @@ -1,9 +1,11 @@ const models = require('../../../models'); const {knex} = require('../../../data/db'); const moment = require('moment'); +const logging = require('@tryghost/logging'); module.exports = async function (options) { const hasFilter = options.limit !== 'all' || options.filter || options.search; + const start = Date.now(); let ids = null; if (hasFilter) { @@ -31,6 +33,10 @@ module.exports = async function (options) { */ } + logging.info('[MembersExporter] Fetching products and labels'); + + const startFetchingProducts = Date.now(); + const allProducts = await knex('products').select('id', 'name').then(rows => rows.reduce((acc, product) => { acc[product.id] = product.name; return acc; @@ -43,6 +49,10 @@ module.exports = async function (options) { }, {}) ); + logging.info('[MembersExporter] Fetched products and labels in ' + (Date.now() - startFetchingProducts) + 'ms'); + + const startFetchingMembers = Date.now(); + const members = await knex('members') .select('id', 'email', 'name', 'note', 'status', 'created_at') .modify((query) => { @@ -51,6 +61,10 @@ module.exports = async function (options) { } }); + logging.info('[MembersExporter] Fetched members in ' + (Date.now() - startFetchingMembers) + 'ms'); + + const startFetchingTiers = Date.now(); + const tiers = await knex('members_products') .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers')) .groupBy('member_id') @@ -60,6 +74,10 @@ module.exports = async function (options) { } }); + logging.info('[MembersExporter] Fetched tiers in ' + (Date.now() - startFetchingTiers) + 'ms'); + + const startFetchingLabels = Date.now(); + const labels = await knex('members_labels') .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels')) .groupBy('member_id') @@ -69,6 +87,10 @@ module.exports = async function (options) { } }); + logging.info('[MembersExporter] Fetched labels in ' + (Date.now() - startFetchingLabels) + 'ms'); + + const startFetchingStripeCustomers = Date.now(); + const stripeCustomers = await knex('members_stripe_customers') .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id')) .groupBy('member_id') @@ -78,6 +100,10 @@ module.exports = async function (options) { } }); + logging.info('[MembersExporter] Fetched stripe customers in ' + (Date.now() - startFetchingStripeCustomers) + 'ms'); + + const startFetchingSubscriptions = Date.now(); + const subscriptions = await knex('members_newsletters') .distinct('member_id') .modify((query) => { @@ -86,6 +112,10 @@ module.exports = async function (options) { } }); + logging.info('[MembersExporter] Fetched subscriptions in ' + (Date.now() - startFetchingSubscriptions) + 'ms'); + + const startInMemoryProcessing = Date.now(); + const tiersMap = new Map(tiers.map(row => [row.member_id, row.tiers])); const labelsMap = new Map(labels.map(row => [row.member_id, row.labels])); const stripeCustomerMap = new Map(stripeCustomers.map(row => [row.member_id, row.stripe_customer_id])); @@ -114,5 +144,9 @@ module.exports = async function (options) { row.created_at = moment(row.created_at).toISOString(); } + logging.info('[MembersExporter] In memory processing finished in ' + (Date.now() - startInMemoryProcessing) + 'ms'); + + logging.info('[MembersExporter] Total time taken: ' + (Date.now() - start)/1000 + 's'); + return members; }; From adbf5ce199cb530f9a777539537a9ab3f3e0fe00 Mon Sep 17 00:00:00 2001 From: Princi Vershwal Date: Thu, 16 Jan 2025 13:55:48 +0530 Subject: [PATCH 6/6] lint fix --- ghost/core/core/server/services/members/exporter/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ghost/core/core/server/services/members/exporter/query.js b/ghost/core/core/server/services/members/exporter/query.js index 53be7c0eef2..7d7fbdfaf41 100644 --- a/ghost/core/core/server/services/members/exporter/query.js +++ b/ghost/core/core/server/services/members/exporter/query.js @@ -146,7 +146,7 @@ module.exports = async function (options) { logging.info('[MembersExporter] In memory processing finished in ' + (Date.now() - startInMemoryProcessing) + 'ms'); - logging.info('[MembersExporter] Total time taken: ' + (Date.now() - start)/1000 + 's'); + logging.info('[MembersExporter] Total time taken: ' + (Date.now() - start) / 1000 + 's'); return members; };