From d64a3659bff6ee2bda8c806544563f5c93bb0156 Mon Sep 17 00:00:00 2001 From: Pierre-Gilles Leymarie Date: Thu, 8 Aug 2024 19:20:44 +0200 Subject: [PATCH] Purge states should purge duckDB states + remove purge state agregates + improve shutdown DB closing --- .../actions/signup/signupSetPreferences.js | 6 -- .../SettingsSystemKeepAggregatedStates.jsx | 98 ------------------- .../settings-system/SettingsSystemPage.jsx | 2 - server/index.js | 25 +++-- .../lib/device/device.onPurgeStatesEvent.js | 1 - .../lib/device/device.purgeAggregateStates.js | 44 --------- server/lib/device/device.purgeStates.js | 13 ++- server/lib/device/index.js | 2 - .../device.purgeAggregateStates.test.js | 54 ---------- .../lib/device/device.purgeStates.test.js | 19 ++++ 10 files changed, 42 insertions(+), 222 deletions(-) delete mode 100644 front/src/routes/settings/settings-system/SettingsSystemKeepAggregatedStates.jsx delete mode 100644 server/lib/device/device.purgeAggregateStates.js delete mode 100644 server/test/lib/device/device.purgeAggregateStates.test.js diff --git a/front/src/actions/signup/signupSetPreferences.js b/front/src/actions/signup/signupSetPreferences.js index 02397986d4..08a4e0de56 100644 --- a/front/src/actions/signup/signupSetPreferences.js +++ b/front/src/actions/signup/signupSetPreferences.js @@ -59,12 +59,6 @@ function createActions(store) { await state.httpClient.post(`/api/v1/variable/${SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HISTORY_IN_DAYS}`, { value: state.signupSystemPreferences[SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HISTORY_IN_DAYS] }); - await state.httpClient.post( - `/api/v1/variable/${SYSTEM_VARIABLE_NAMES.DEVICE_AGGREGATE_STATE_HISTORY_IN_DAYS}`, - { - value: state.signupSystemPreferences[SYSTEM_VARIABLE_NAMES.DEVICE_STATE_HISTORY_IN_DAYS] - } - ); store.setState({ signupSaveSystemPreferences: RequestStatus.Success }); diff --git a/front/src/routes/settings/settings-system/SettingsSystemKeepAggregatedStates.jsx b/front/src/routes/settings/settings-system/SettingsSystemKeepAggregatedStates.jsx deleted file mode 100644 index ded8ce8a76..0000000000 --- a/front/src/routes/settings/settings-system/SettingsSystemKeepAggregatedStates.jsx +++ /dev/null @@ -1,98 +0,0 @@ -import { connect } from 'unistore/preact'; -import { Component } from 'preact'; -import { Text } from 'preact-i18n'; -import { SYSTEM_VARIABLE_NAMES } from '../../../../../server/utils/constants'; -import get from 'get-value'; - -class SettingsSystemKeepAggregatedStates extends Component { - getDeviceAggregateStateHistoryPreference = async () => { - try { - const { value } = await this.props.httpClient.get( - `/api/v1/variable/${SYSTEM_VARIABLE_NAMES.DEVICE_AGGREGATE_STATE_HISTORY_IN_DAYS}` - ); - this.setState({ - deviceAggregateStateHistoryInDays: value - }); - } catch (e) { - console.error(e); - const status = get(e, 'response.status'); - if (status === 404) { - // Default value is -1 - this.setState({ - deviceAggregateStateHistoryInDays: '-1' - }); - } - } - }; - - updateDeviceAggregateStateHistory = async e => { - await this.setState({ - deviceAggregateStateHistoryInDays: e.target.value, - savingDeviceStateHistory: true - }); - try { - await this.props.httpClient.post( - `/api/v1/variable/${SYSTEM_VARIABLE_NAMES.DEVICE_AGGREGATE_STATE_HISTORY_IN_DAYS}`, - { - value: e.target.value - } - ); - } catch (e) { - console.error(e); - } - await this.setState({ - savingDeviceStateHistory: false - }); - }; - - componentDidMount() { - this.getDeviceAggregateStateHistoryPreference(); - } - - render({}, { deviceAggregateStateHistoryInDays }) { - return ( -
-

- -

- -
-
-

- -

- -
-
-
- ); - } -} - -export default connect('httpClient', null)(SettingsSystemKeepAggregatedStates); diff --git a/front/src/routes/settings/settings-system/SettingsSystemPage.jsx b/front/src/routes/settings/settings-system/SettingsSystemPage.jsx index e42578e5c1..a9cfa2db58 100644 --- a/front/src/routes/settings/settings-system/SettingsSystemPage.jsx +++ b/front/src/routes/settings/settings-system/SettingsSystemPage.jsx @@ -5,7 +5,6 @@ import SettingsSystemContainers from './SettingsSystemContainers'; import SettingsSystemOperations from './SettingsSystemOperations'; import SettingsSystemTimezone from './SettingsSystemTimezone'; import SettingsSystemKeepDeviceHistory from './SettingsSystemKeepDeviceHistory'; -import SettingsSystemKeepAggregatedStates from './SettingsSystemKeepAggregatedStates'; import SettingsSystemTimeExpiryState from './SettingsSystemTimeExpiryState'; import SettingsSystemDatabaseCleaning from './SettingsSystemDatabaseCleaning'; import SettingsSystemDuckDbMigration from './SettingsSystemDuckDbMigration'; @@ -89,7 +88,6 @@ const SystemPage = ({ children, ...props }) => ( -
diff --git a/server/index.js b/server/index.js index 7040bff181..89dc38ed23 100644 --- a/server/index.js +++ b/server/index.js @@ -21,26 +21,35 @@ process.on('uncaughtException', (error, promise) => { logger.error(error); }); -const shutdown = async (signal) => { - logger.info(`${signal} received.`); - // We give Gladys 10 seconds to properly shutdown, otherwise we do it - setTimeout(() => { - logger.info('Timeout to shutdown expired, forcing shut down.'); - process.exit(); - }, 10 * 1000); - logger.info('Closing database connections.'); +const closeSQLite = async () => { try { await db.sequelize.close(); + logger.info('SQLite closed.'); } catch (e) { logger.info('SQLite database is probably already closed'); logger.warn(e); } +}; + +const closeDuckDB = async () => { try { await db.duckDb.close(); + logger.info('DuckDB closed.'); } catch (e) { logger.info('DuckDB database is probably already closed'); logger.warn(e); } +}; + +const shutdown = async (signal) => { + logger.info(`${signal} received.`); + // We give Gladys 10 seconds to properly shutdown, otherwise we do it + setTimeout(() => { + logger.info('Timeout to shutdown expired, forcing shut down.'); + process.exit(); + }, 10 * 1000); + logger.info('Closing database connections.'); + await Promise.all([closeSQLite(), closeDuckDB()]); process.exit(); }; diff --git a/server/lib/device/device.onPurgeStatesEvent.js b/server/lib/device/device.onPurgeStatesEvent.js index 17cb296b67..87ed7bb0af 100644 --- a/server/lib/device/device.onPurgeStatesEvent.js +++ b/server/lib/device/device.onPurgeStatesEvent.js @@ -8,7 +8,6 @@ const { JOB_TYPES } = require('../../utils/constants'); async function onPurgeStatesEvent() { const purgeAllStates = this.job.wrapper(JOB_TYPES.DEVICE_STATES_PURGE, async () => { await this.purgeStates(); - await this.purgeAggregateStates(); }); await purgeAllStates(); } diff --git a/server/lib/device/device.purgeAggregateStates.js b/server/lib/device/device.purgeAggregateStates.js deleted file mode 100644 index e3313db178..0000000000 --- a/server/lib/device/device.purgeAggregateStates.js +++ /dev/null @@ -1,44 +0,0 @@ -const { Op } = require('sequelize'); -const db = require('../../models'); -const logger = require('../../utils/logger'); -const { SYSTEM_VARIABLE_NAMES } = require('../../utils/constants'); - -/** - * @description Purge device aggregate states. - * @returns {Promise} Resolve when finished. - * @example - * device.purgeAggregateStates(); - */ -async function purgeAggregateStates() { - logger.debug('Purging device feature aggregate states...'); - const deviceAggregateStateHistoryInDays = await this.variable.getValue( - SYSTEM_VARIABLE_NAMES.DEVICE_AGGREGATE_STATE_HISTORY_IN_DAYS, - ); - const deviceAggregateStateHistoryInDaysInt = parseInt(deviceAggregateStateHistoryInDays, 10); - if (Number.isNaN(deviceAggregateStateHistoryInDaysInt)) { - logger.debug('Not purging device feature aggregate states, deviceAggregateStateHistoryInDays is not an integer.'); - return Promise.resolve(false); - } - if (deviceAggregateStateHistoryInDaysInt === -1) { - logger.debug('Not purging device feature aggregate states, deviceAggregateStateHistoryInDays = -1'); - return Promise.resolve(false); - } - const queryInterface = db.sequelize.getQueryInterface(); - const now = new Date().getTime(); - // all date before this timestamp will be removed - const timstampLimit = now - deviceAggregateStateHistoryInDaysInt * 24 * 60 * 60 * 1000; - const dateLimit = new Date(timstampLimit); - logger.info( - `Purging device feature states of the last ${deviceAggregateStateHistoryInDaysInt} days. States older than ${dateLimit} will be purged.`, - ); - await queryInterface.bulkDelete('t_device_feature_state_aggregate', { - created_at: { - [Op.lte]: dateLimit, - }, - }); - return true; -} - -module.exports = { - purgeAggregateStates, -}; diff --git a/server/lib/device/device.purgeStates.js b/server/lib/device/device.purgeStates.js index cd95f8e9e5..a0e8985d6f 100644 --- a/server/lib/device/device.purgeStates.js +++ b/server/lib/device/device.purgeStates.js @@ -1,4 +1,3 @@ -const { Op } = require('sequelize'); const db = require('../../models'); const logger = require('../../utils/logger'); const { SYSTEM_VARIABLE_NAMES } = require('../../utils/constants'); @@ -21,7 +20,6 @@ async function purgeStates() { logger.debug('Not purging device feature states, deviceStateHistoryInDays = -1'); return Promise.resolve(false); } - const queryInterface = db.sequelize.getQueryInterface(); const now = new Date().getTime(); // all date before this timestamp will be removed const timstampLimit = now - deviceStateHistoryInDaysInt * 24 * 60 * 60 * 1000; @@ -29,11 +27,12 @@ async function purgeStates() { logger.info( `Purging device feature states of the last ${deviceStateHistoryInDaysInt} days. States older than ${dateLimit} will be purged.`, ); - await queryInterface.bulkDelete('t_device_feature_state', { - created_at: { - [Op.lte]: dateLimit, - }, - }); + await db.duckDbWriteConnectionAllAsync( + ` + DELETE FROM t_device_feature_state WHERE created_at < ? + `, + dateLimit, + ); return true; } diff --git a/server/lib/device/index.js b/server/lib/device/index.js index b559946650..e77ec4db51 100644 --- a/server/lib/device/index.js +++ b/server/lib/device/index.js @@ -20,7 +20,6 @@ const { getDeviceFeaturesAggregates } = require('./device.getDeviceFeaturesAggre const { getDeviceFeaturesAggregatesMulti } = require('./device.getDeviceFeaturesAggregatesMulti'); const { onPurgeStatesEvent } = require('./device.onPurgeStatesEvent'); const { purgeStates } = require('./device.purgeStates'); -const { purgeAggregateStates } = require('./device.purgeAggregateStates'); const { purgeStatesByFeatureId } = require('./device.purgeStatesByFeatureId'); const { poll } = require('./device.poll'); const { pollAll } = require('./device.pollAll'); @@ -114,7 +113,6 @@ DeviceManager.prototype.getDeviceFeaturesAggregates = getDeviceFeaturesAggregate DeviceManager.prototype.getDeviceFeaturesAggregatesMulti = getDeviceFeaturesAggregatesMulti; DeviceManager.prototype.onPurgeStatesEvent = onPurgeStatesEvent; DeviceManager.prototype.purgeStates = purgeStates; -DeviceManager.prototype.purgeAggregateStates = purgeAggregateStates; DeviceManager.prototype.purgeStatesByFeatureId = purgeStatesByFeatureId; DeviceManager.prototype.poll = poll; DeviceManager.prototype.pollAll = pollAll; diff --git a/server/test/lib/device/device.purgeAggregateStates.test.js b/server/test/lib/device/device.purgeAggregateStates.test.js deleted file mode 100644 index 4fcb2a4d06..0000000000 --- a/server/test/lib/device/device.purgeAggregateStates.test.js +++ /dev/null @@ -1,54 +0,0 @@ -const { expect } = require('chai'); -const EventEmitter = require('events'); -const { fake } = require('sinon'); - -const Device = require('../../../lib/device'); - -const StateManager = require('../../../lib/state'); -const Job = require('../../../lib/job'); - -const event = new EventEmitter(); -const job = new Job(event); - -describe('Device.purgeAggregateStates', () => { - it('should purgeAggregateStates', async () => { - const variable = { - getValue: fake.resolves(30), - }; - const stateManager = new StateManager(event); - const service = {}; - const device = new Device(event, {}, stateManager, service, {}, variable, job); - const devicePurged = await device.purgeAggregateStates(); - expect(devicePurged).to.equal(true); - }); - it('should not purgeAggregateStates, invalid date', async () => { - const variable = { - getValue: fake.resolves('NOT A DATE'), - }; - const stateManager = new StateManager(event); - const service = {}; - const device = new Device(event, {}, stateManager, service, {}, variable, job); - const devicePurged = await device.purgeAggregateStates(); - expect(devicePurged).to.equal(false); - }); - it('should not purgeAggregateStates, null', async () => { - const variable = { - getValue: fake.resolves(null), - }; - const stateManager = new StateManager(event); - const service = {}; - const device = new Device(event, {}, stateManager, service, {}, variable, job); - const devicePurged = await device.purgeAggregateStates(); - expect(devicePurged).to.equal(false); - }); - it('should not purgeAggregateStates, date = -1', async () => { - const variable = { - getValue: fake.resolves('-1'), - }; - const stateManager = new StateManager(event); - const service = {}; - const device = new Device(event, {}, stateManager, service, {}, variable, job); - const devicePurged = await device.purgeAggregateStates(); - expect(devicePurged).to.equal(false); - }); -}); diff --git a/server/test/lib/device/device.purgeStates.test.js b/server/test/lib/device/device.purgeStates.test.js index 3f300c7f92..e97c512d83 100644 --- a/server/test/lib/device/device.purgeStates.test.js +++ b/server/test/lib/device/device.purgeStates.test.js @@ -1,6 +1,7 @@ const { expect } = require('chai'); const EventEmitter = require('events'); const { fake } = require('sinon'); +const db = require('../../../models'); const Device = require('../../../lib/device'); @@ -15,11 +16,29 @@ describe('Device', () => { const variable = { getValue: fake.resolves(30), }; + const currentDate = new Date(); + const fortyDaysAgo = new Date(currentDate); + fortyDaysAgo.setDate(currentDate.getDate() - 40); + await db.duckDbBatchInsertState('ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', [ + { + value: 1, + created_at: fortyDaysAgo, + }, + { + value: 10, + created_at: currentDate, + }, + ]); const stateManager = new StateManager(event); const service = {}; const device = new Device(event, {}, stateManager, service, {}, variable, job); const devicePurged = await device.purgeStates(); expect(devicePurged).to.equal(true); + const res = await db.duckDbReadConnectionAllAsync( + 'SELECT value FROM t_device_feature_state WHERE device_feature_id = ?', + 'ca91dfdf-55b2-4cf8-a58b-99c0fbf6f5e4', + ); + expect(res).to.deep.equal([{ value: 10 }]); }); it('should not purgeStates, invalid date', async () => { const variable = {