From cd9719637c88cdd544fc643ab2bcdf4b1e864cb2 Mon Sep 17 00:00:00 2001 From: diosmosis Date: Wed, 8 May 2024 21:21:54 -0700 Subject: [PATCH] add new dimensions for week of year, month of year and year --- package-lock.json | 73 ++++++++++++++++++++++++- package.json | 4 +- rollup.config.js | 2 + src/data.ts | 100 ++++++++++++++++++++++++++++------- tests/appscript/data.spec.ts | 40 +++++++------- 5 files changed, 178 insertions(+), 41 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4782b3a..7567ba2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,11 +10,13 @@ "license": "GPL-3.0-or-later", "dependencies": { "@types/axios": "^0.14.0", - "@types/google-apps-script": "^1.0.58" + "@types/google-apps-script": "^1.0.58", + "dayjs": "^1.11.11" }, "devDependencies": { "@google/clasp": "^2.4.2", "@jest/globals": "^29.5.0", + "@rollup/plugin-node-resolve": "^15.2.3", "@rollup/plugin-typescript": "^11.0.0", "@types/inquirer": "^8.2.10", "@types/jest": "^29.5.11", @@ -1932,6 +1934,31 @@ "integrity": "sha512-AanzbulOHljrku1NGfafxdpTCfw2ENaWzH01N2vqQM+cUFbk868Cgh0xylz0JIM9BoKbfI++bdD6EYX0Q/UTEw==", "dev": true }, + "node_modules/@rollup/plugin-node-resolve": { + "version": "15.2.3", + "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-15.2.3.tgz", + "integrity": "sha512-j/lym8nf5E21LwBT4Df1VD6hRO2L2iwUeUmP7litikRsVp1H6NWx20NEp0Y7su+7XGc476GnXXc4kFeZNGmaSQ==", + "dev": true, + "dependencies": { + "@rollup/pluginutils": "^5.0.1", + "@types/resolve": "1.20.2", + "deepmerge": "^4.2.2", + "is-builtin-module": "^3.2.1", + "is-module": "^1.0.0", + "resolve": "^1.22.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^2.78.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, "node_modules/@rollup/plugin-replace": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/@rollup/plugin-replace/-/plugin-replace-5.0.5.tgz", @@ -2427,6 +2454,12 @@ "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", "dev": true }, + "node_modules/@types/resolve": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", + "integrity": "sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==", + "dev": true + }, "node_modules/@types/responselike": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.3.tgz", @@ -2967,6 +3000,18 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/builtin-modules": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", + "integrity": "sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==", + "dev": true, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/cache-content-type": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz", @@ -3436,6 +3481,11 @@ "node": ">=0.10.0" } }, + "node_modules/dayjs": { + "version": "1.11.11", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.11.tgz", + "integrity": "sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg==" + }, "node_modules/debounce": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/debounce/-/debounce-1.2.1.tgz", @@ -4938,6 +4988,21 @@ "node": ">=8" } }, + "node_modules/is-builtin-module": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", + "integrity": "sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A==", + "dev": true, + "dependencies": { + "builtin-modules": "^3.3.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-core-module": { "version": "2.13.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", @@ -5034,6 +5099,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz", + "integrity": "sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==", + "dev": true + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", diff --git a/package.json b/package.json index 738521a..515c4cb 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "devDependencies": { "@google/clasp": "^2.4.2", "@jest/globals": "^29.5.0", + "@rollup/plugin-node-resolve": "^15.2.3", "@rollup/plugin-typescript": "^11.0.0", "@types/inquirer": "^8.2.10", "@types/jest": "^29.5.11", @@ -55,6 +56,7 @@ }, "dependencies": { "@types/axios": "^0.14.0", - "@types/google-apps-script": "^1.0.58" + "@types/google-apps-script": "^1.0.58", + "dayjs": "^1.11.11" } } diff --git a/rollup.config.js b/rollup.config.js index f5edf60..c48fb63 100644 --- a/rollup.config.js +++ b/rollup.config.js @@ -8,6 +8,7 @@ const typescript = require('@rollup/plugin-typescript'); const copy = require('rollup-plugin-copy'); const dotenv = require('rollup-plugin-dotenv').default; +const { nodeResolve } = require('@rollup/plugin-node-resolve'); // removes export statements since they are not recognized by apps script, but rollup always puts them in for esm output const removeExports = () => { @@ -27,6 +28,7 @@ module.exports = { name: 'MatomoLookerStudio', }, plugins: [ + nodeResolve(), typescript(), dotenv(), copy({ diff --git a/src/data.ts b/src/data.ts index 476b9db..210ec5b 100644 --- a/src/data.ts +++ b/src/data.ts @@ -5,6 +5,8 @@ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later */ +import dayjs from 'dayjs/esm'; +import weekOfYear from 'dayjs/esm/plugin/weekOfYear'; import cc, { ConnectorParams } from './connector'; import * as Api from './api'; import env from './env'; @@ -16,6 +18,8 @@ import { import { DataTableRow } from './api'; import { debugLog } from './log'; +dayjs.extend(weekOfYear); + const pastScriptRuntimeLimitErrorMessage = 'It\'s taking too long to get the requested data. This may be a momentary issue with ' + 'your Matomo, but if it continues to occur for this report, then you may be requesting too much data. In this ' + 'case, limit the data you are requesting to see it in Looker Studio.'; @@ -212,13 +216,29 @@ function getReportData(request: GoogleAppsScript.Data_Studio.Request f.name === 'date')); + const dateMetricIfPresent = request.fields && request.fields + .filter((f) => DATE_METRIC_IDS.includes(f.name)) + .pop(); // always use the last occurrence, since 'date' will be requested along with the other dimension let period = 'range'; let date = `${request.dateRange.startDate},${request.dateRange.endDate}`; - if (hasDate) { - period = 'day'; + if (dateMetricIfPresent) { + switch (dateMetricIfPresent.name) { + case 'date_year': + period = 'year'; + break; + case 'date_month': + period = 'month'; + break; + case 'date_week': + period = 'week'; + break; + case 'date': + default: + period = 'day'; + break; + } // note: this calculation doesn't work every time, but it's good enough for determining row counts const MS_IN_DAY = 1000 * 60 * 60 * 24; @@ -275,7 +295,7 @@ function getReportData(request: GoogleAppsScript.Data_Studio.Request : { [date]: partialResponseRaw as DataTableRow[] }; + const partialResponse = dateMetricIfPresent ? partialResponseRaw as Record : { [date]: partialResponseRaw as DataTableRow[] }; Object.entries(partialResponse).forEach(([date, rows]) => { if (!rows) { rows = []; @@ -285,7 +305,7 @@ function getReportData(request: GoogleAppsScript.Data_Studio.Request ({ ...r, date }))); } else { response[date].push(...rows); @@ -323,12 +343,43 @@ function addDimension(fields: GoogleAppsScript.Data_Studio.Fields, id: string, d .setType(cc.FieldType.TEXT); } -function addDateDimension(fields: GoogleAppsScript.Data_Studio.Fields) { - fields - .newDimension() - .setId('date') - .setName('Date') - .setType(cc.FieldType.YEAR_MONTH_DAY); +const DATE_METRIC_IDS = ['date', 'date_month', 'date_week', 'date_year']; + +function addDateDimensions( + fields: GoogleAppsScript.Data_Studio.Fields, + includeOnly: string[] = DATE_METRIC_IDS, +) { + if (includeOnly.includes('date')) { + fields + .newDimension() + .setId('date') + .setName('Date') + .setType(cc.FieldType.YEAR_MONTH_DAY); + } + + if (includeOnly.includes('date_month')) { + fields + .newDimension() + .setId('date_month') + .setName('Month') + .setType(cc.FieldType.YEAR_MONTH); + } + + if (includeOnly.includes('date_week')) { + fields + .newDimension() + .setId('date_week') + .setName('Week (Mon - Sun)') + .setType(cc.FieldType.YEAR_WEEK); + } + + if (includeOnly.includes('date_year')) { + fields + .newDimension() + .setId('date_year') + .setName('Year') + .setType(cc.FieldType.YEAR); + } } function metricsForEachGoal(metrics: Record, goals: Record) { @@ -401,8 +452,8 @@ function getFieldsFromReportMetadata(reportMetadata: Api.ReportMetadata, goals: return; } - if (metricId === 'date') { - addDateDimension(fields); + if (DATE_METRIC_IDS.includes(metricId)) { + addDateDimensions(fields, [metricId]); } if (reportMetadata.dimensions?.[metricId]) { @@ -456,7 +507,7 @@ export function getSchema(request: GoogleAppsScript.Data_Studio.Request { const fieldValues = requestedFields .map(({ name }, index) => { + // perform any transformations on the value required by the Matomo type + let matomoType = reportMetadata?.metricTypes?.[name]; + if (DATE_METRIC_IDS.includes(name)) { + matomoType = name; + name = 'date'; + } + if (typeof row[name] !== 'undefined' && row[name] !== false // edge case that can happen in some report output ) { let value = row[name]; - // perform any transformations on the value required by the Matomo type - let matomoType = reportMetadata?.metricTypes?.[name]; - if (name === 'date') { - matomoType = 'date'; - } - if (matomoType === 'duration_ms') { value = parseInt(value as string, 10) / 1000; } else if (matomoType === 'date') { // value is in YYYY-MM-DD format, but must be converted to YYYYMMDD value = value.toString().replace(/-/g, ''); + } else if (matomoType === 'date_month') { + // value is in YYYY-MM-DD format, but must be converted to YYYYMM + value = value.toString().split('-').slice(0, 2).join(''); + } else if (matomoType === 'date_week') { + // value is in YYYY-MM-DD, but must be converted to YYYYww + value = value.toString().split('-').shift() + dayjs(value).week(); + } else if (matomoType === 'date_year') { + value = value.toString().split('-').shift(); } else if (matomoType === 'datetime') { // value is in YYYY-MM-DD HH:MM:SS format, but must be converted to YYYYMMDDHHMMSS value = value.toString().replace(/[-:\s]/g, ''); diff --git a/tests/appscript/data.spec.ts b/tests/appscript/data.spec.ts index 2ab3b71..13288a0 100644 --- a/tests/appscript/data.spec.ts +++ b/tests/appscript/data.spec.ts @@ -388,26 +388,28 @@ describe('data', () => { }); } - it('should fetch data by day when the date dimension is requested', async () => { - let result = await Clasp.run('getData', { - configParams: { - idsite: env.APPSCRIPT_TEST_IDSITE, - report: JSON.stringify({ apiModule: 'Events', apiAction: 'getName' }), - filter_limit: 5, - }, - dateRange: { - startDate: RANGE_START_DATE_TO_TEST, - endDate: RANGE_END_DATE_TO_TEST, - }, - fields: [ - { name: 'date' }, - { name: 'nb_events' }, - { name: 'Events_EventAction' }, - { name: 'Events_EventName' }, - { name: 'max_event_value' }, - ], + ['date', 'date_week', 'date_month', 'date_year'].forEach((dimensionName) => { + it(`should fetch data by day when the ${dimensionName} dimension is requested`, async () => { + let result = await Clasp.run('getData', { + configParams: { + idsite: env.APPSCRIPT_TEST_IDSITE, + report: JSON.stringify({ apiModule: 'Events', apiAction: 'getName' }), + filter_limit: 5, + }, + dateRange: { + startDate: RANGE_START_DATE_TO_TEST, + endDate: RANGE_END_DATE_TO_TEST, + }, + fields: [ + { name: dimensionName }, + { name: 'nb_events' }, + { name: 'Events_EventAction' }, + { name: 'Events_EventName' }, + { name: 'max_event_value' }, + ], + }); + expect(result).toEqual(getExpectedResponse(result, 'data', `Events.getName_withDateDimension_${dimensionName}`)); }); - expect(result).toEqual(getExpectedResponse(result, 'data', 'Events.getName_withDateDimension')); }); it('should paginate correctly when fetch data by day when the date dimension is requested', async () => {