6915 - Telemetry daily frequency (#7095)

* Telemetry frequency changed from monthly to daily.
* The id of a telemetry record now looks like `telemetry-<year>-<month>-<day>-<username>-<uuid>`. Also in the metadata section a new field `day` is added with the day of the month the data belongs to.
* Bug fix: when `record()` was called for the first time in the period (a period was a month, now is a day), the record was added to the telemetry DB and then the aggregation executed with the data that was stored previously, so the aggregation process use to aggregate all the records from the previous period + a record that was actually from the current period. Now the last record is not taken into account in the aggregation submitted, instead is stored to be used in the following aggregation.
* Add more telemetry tests. Refactor.
* Improvements in the script to collect meta data from a terminal:
  - Output errors in the standard error stream to see the errors in the console while piping right results to a JSON file.
  - Add Unix exec permission and header to execute the script without the `node ` prefix.

scripts/get_users_meta_docs.js

@@ -1,3 +1,5 @@
+#!/usr/bin/env node
+
 const inquirer = require('inquirer');
 const PouchDB = require('pouchdb-core');
 const fs = require('fs');
@@ -86,7 +88,7 @@ const actionQuestions = [{
           }
           docs.forEach(doc => console.log(JSON.stringify(doc, null, 2) + ','));
         } else if (i === 0) {
-          console.log('\x1b[31m%s\x1b[0m', `There are no documents of type ${type}`);
+          console.error('\x1b[31m%s\x1b[0m', `There are no documents of type ${type}`);
           break;
         } else {
           console.log('{}]');
@@ -100,7 +102,7 @@ const actionQuestions = [{
       let docIndex = 0;
 
       if (docs.length === 0) {
-        console.log('\x1b[31m%s\x1b[0m', `There are no documents of type ${type}`);
+        console.error('\x1b[31m%s\x1b[0m', `There are no documents of type ${type}`);
       } else {
         console.log(JSON.stringify(docs[docIndex], null, 2));
 
@@ -125,7 +127,7 @@ const actionQuestions = [{
 
             console.log(JSON.stringify(docs[docIndex], null, 2));
             if (printMessage) {
-              console.log('\x1b[31m%s\x1b[0m', `No next document. This is the last one.`);
+              console.error('\x1b[31m%s\x1b[0m', `No next document. This is the last one.`);
             }
           } else if (response.action === 'save_current') {
             const filePath = path.join(path.resolve(__dirname), docs[docIndex]._id + '.json');
@@ -154,6 +156,6 @@ const actionQuestions = [{
       }
     }
   } catch(err) {
-    console.log(err);
+    console.error(err);
   }
 })();

webapp/src/ts/services/telemetry.service.ts

@@ -9,13 +9,13 @@ import { SessionService } from '@mm-services/session.service';
   providedIn: 'root'
 })
 /**
- * TelemetryService: Records, aggregates, and submits telemetry data
+ * TelemetryService: Records, aggregates, and submits telemetry data.
  */
 export class TelemetryService {
   // Intentionally scoped to the whole browser (for this domain). We can then tell if multiple users use the same device
   private readonly DEVICE_ID_KEY = 'medic-telemetry-device-id';
   private DB_ID_KEY;
-  private LAST_AGGREGATED_DATE_KEY;
+  private FIRST_AGGREGATED_DATE_KEY;
 
   private queue = Promise.resolve();
 
@@ -27,7 +27,7 @@ export class TelemetryService {
     // Intentionally scoped to the specific user, as they may perform a
     // different role (online vs. offline being being the most obvious) with different performance implications
     this.DB_ID_KEY = ['medic', this.sessionService.userCtx().name, 'telemetry-db'].join('-');
-    this.LAST_AGGREGATED_DATE_KEY = ['medic', this.sessionService.userCtx().name, 'telemetry-date'].join('-');
+    this.FIRST_AGGREGATED_DATE_KEY = ['medic', this.sessionService.userCtx().name, 'telemetry-date'].join('-');
   }
 
   private getDb() {
@@ -54,15 +54,21 @@ export class TelemetryService {
     return uniqueDeviceId;
   }
 
-  private getLastAggregatedDate() {
-    let date = parseInt(window.localStorage.getItem(this.LAST_AGGREGATED_DATE_KEY));
+  /**
+   * Returns a Moment object when the first telemetry record was created.
+   *
+   * This date is computed and stored in milliseconds (since Unix epoch)
+   * when we call this method for the first time and after every aggregation.
+   */
+  private getFirstAggregatedDate() {
+    let date = parseInt(window.localStorage.getItem(this.FIRST_AGGREGATED_DATE_KEY));
 
     if (!date) {
       date = Date.now();
-      window.localStorage.setItem(this.LAST_AGGREGATED_DATE_KEY, date.toString());
+      window.localStorage.setItem(this.FIRST_AGGREGATED_DATE_KEY, date.toString());
     }
 
-    return date;
+    return moment(date);
   }
 
   private storeIt(db, key, value) {
@@ -73,11 +79,17 @@ export class TelemetryService {
     });
   }
 
+  // moment when the aggregation starts (the beginning of the current day)
+  private aggregateStartsAt() {
+    return moment().startOf('day');
+  }
+
+  // if there is telemetry data from previous days, aggregation is performed and the data destroyed
   private submitIfNeeded(db) {
-    const monthStart = moment().startOf('month');
-    const dbDate = moment(this.getLastAggregatedDate());
+    const startOf = this.aggregateStartsAt();
+    const dbDate = this.getFirstAggregatedDate();
 
-    if (dbDate.isBefore(monthStart)) {
+    if (dbDate.isBefore(startOf)) {
       return this
         .aggregate(db)
         .then(() => this.reset(db));
@@ -97,6 +109,7 @@ export class TelemetryService {
       'telemetry',
       metadata.year,
       metadata.month,
+      metadata.day,
       metadata.user,
       metadata.deviceId,
     ].join('-');
@@ -109,7 +122,7 @@ export class TelemetryService {
         this.dbService.get().query('medic-client/doc_by_type', { key: ['form'], include_docs: true })
       ])
       .then(([ddoc, formResults]) => {
-        const date = moment(this.getLastAggregatedDate());
+        const date = this.getFirstAggregatedDate();
         const version = (ddoc.deploy_info && ddoc.deploy_info.version) || 'unknown';
         const forms = formResults.rows.reduce((keyToVersion, row) => {
           keyToVersion[row.doc.internalId] = row.doc._rev;
@@ -120,6 +133,7 @@ export class TelemetryService {
         return {
           year: date.year(),
           month: date.month() + 1,
+          day: date.date(),
           user: this.sessionService.userCtx().name,
           deviceId: this.getUniqueDeviceId(),
           versions: {
@@ -204,7 +218,7 @@ export class TelemetryService {
 
   private reset(db) {
     window.localStorage.removeItem(this.DB_ID_KEY);
-    window.localStorage.removeItem(this.LAST_AGGREGATED_DATE_KEY);
+    window.localStorage.removeItem(this.FIRST_AGGREGATED_DATE_KEY);
 
     return db.destroy();
   }
@@ -225,7 +239,7 @@ export class TelemetryService {
    *   metric_b:  { sum: -16, min: -4, max: -4, count: 4, sumsqr: 64 }
    * }
    *
-   * See: https://wiki.apache.org/couchdb/Built-In_Reduce_Functions#A_stats
+   * See: https://docs.couchdb.org/en/stable/ddocs/ddocs.html#_stats
    *
    * This single month aggregate document is of type 'telemetry', and is
    * stored in the user's meta DB (which replicates up to the main server)
@@ -255,14 +269,14 @@ export class TelemetryService {
     let db;
     this.queue = this.queue
       .then(() => db = this.getDb())
-      .then(() => this.storeIt(db, key, value))
       .then(() => this.submitIfNeeded(db))
+      .then(() => db = this.getDb())  // db is fetched again in case submitIfNeeded dropped the old reference
+      .then(() => this.storeIt(db, key, value))
       .catch(err => console.error('Error in telemetry service', err))
       .finally(() => {
         if (!db || db._destroyed || db._closed) {
           return;
         }
-
         try {
           db.close();
         } catch (err) {