Completing reports migration

definitions/output/reports/reports_dynamic.js

@@ -1,47 +1,74 @@
 const configs = new reports.HTTPArchiveReports()
 const metrics = configs.listMetrics()
 
+// Adjust start and end dates to update reports retrospectively
+const startDate = '2024-12-01' // constants.currentMonth;
+const endDate = '2024-12-01' // constants.currentMonth;
+
+const lenses = {
+  all: '',
+  top1k: 'AND rank <= 1000',
+  top10k: 'AND rank <= 10000',
+  top100k: 'AND rank <= 100000',
+  top1m: 'AND rank <= 1000000',
+  drupal: 'AND \'Drupal\' IN UNNEST(technologies.technology)',
+  magento: 'AND \'Magento\' IN UNNEST(technologies.technology)',
+  wordpress: 'AND \'WordPress\' IN UNNEST(technologies.technology)'
+}
+
 const iterations = []
+// dates
 for (
-  let month = constants.currentMonth; month >= constants.currentMonth; month = constants.fnPastMonth(month)) {
-  iterations.push({
-    date: month,
-    devRankFilter: constants.devRankFilter
+  let date = endDate;
+  date >= startDate;
+  date = constants.fnPastMonth(date)
+) {
+  // metrics
+  metrics.forEach(metric => {
+    // timeseries and histograms
+    metric.SQL.forEach(sql => {
+      // lenses
+      for (const [key, value] of Object.entries(lenses)) {
+        iterations.push({
+          date,
+          metric,
+          sql,
+          lense: { name: key, sql: value },
+          devRankFilter: constants.devRankFilter
+        })
+      }
+    })
   })
 }
 
-if (iterations.length === 1) {
-  const params = iterations[0]
-  metrics.forEach(metric => {
-    metric.SQL.forEach(sql => {
-      publish(metric.id + '_' + sql.type, {
-        type: 'incremental',
-        protected: true,
-        bigquery: sql.type === 'histogram' ? { partitionBy: 'date', clusterBy: ['client'] } : {},
-        schema: 'reports',
-        tags: ['crawl_complete']
-      }).preOps(ctx => `
+if (startDate === endDate) {
+  iterations.forEach((params, i) => {
+    publish(params.metric.id + '_' + params.sql.type + '_' + params.lense.name, {
+      type: 'incremental',
+      protected: true,
+      bigquery: params.sql.type === 'histogram' ? { partitionBy: 'date', clusterBy: ['client'] } : {},
+      schema: 'reports',
+      tags: ['crawl_complete', 'reports']
+    }).preOps(ctx => `
 --DELETE FROM ${ctx.self()}
 --WHERE date = '${params.date}';
-  `).query(ctx => `
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */` +
-sql.query(ctx, params))
-    })
+    `).query(ctx => `
+/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${params.metric.id}", "type": "${params.sql.type}", "lense": "${params.lense.name}"} */` +
+params.sql.query(ctx, params)
+    )
   })
 } else {
   iterations.forEach((params, i) => {
-    metrics.forEach(metric => {
-      metric.SQL.forEach(sql => {
-        operate(metric.id + '_' + sql.type + '_' + params.date, {
-          tags: ['crawl_complete']
-        }).queries(ctx => `
-DELETE FROM reports.${metric.id}_${sql.type}
+    operate(
+      params.metric.id + '_' + params.sql.type + '_' + params.lense.name + '_' + params.date)
+      .tags(['crawl_complete', 'reports'])
+      .queries(ctx => `
+DELETE FROM reports.${params.metric.id}_${params.sql.type}
 WHERE date = '${params.date}';
 
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */
-INSERT INTO reports.${metric.id}_${sql.type}` +
-        sql.query(ctx, params))
-      })
-    })
+/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${params.metric.id}", "type": "${params.sql.type}", "lense": "${params.lense.name}"} */
+INSERT INTO reports.${params.metric.id}_${params.sql.type}` +
+params.sql.query(ctx, params)
+      )
   })
 }

includes/reports.js

@@ -22,19 +22,37 @@ FROM (
       COUNT(0) AS volume
     FROM ${ctx.ref('crawl', 'pages')}
     WHERE
-      date = '${params.date}' ${params.devRankFilter}
+      date = '${params.date}' ${params.devRankFilter} ${params.lense.sql} AND
+      is_root_page AND
+      INT64(summary.bytesTotal) > 0
     GROUP BY
       date,
       client,
       bin
     HAVING bin IS NOT NULL
   )
 )
+ORDER BY
+  date,
+  bin,
+  client
 `)
         },
         {
           type: 'timeseries',
           query: DataformTemplateBuilder.create((ctx, params) => `
+WITH pages AS (
+  SELECT
+    date,
+    client,
+    INT64(summary.bytesTotal) AS bytesTotal
+  FROM ${ctx.ref('crawl', 'pages')}
+  WHERE
+    date = '${params.date}' ${params.devRankFilter} ${params.lense.sql} AND
+    is_root_page AND
+    INT64(summary.bytesTotal) > 0
+)
+
 SELECT
   date,
   client,
@@ -44,16 +62,7 @@ SELECT
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(501)] / 1024, 2) AS p50,
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(751)] / 1024, 2) AS p75,
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(901)] / 1024, 2) AS p90
-FROM (
-  SELECT
-    date,
-    client,
-    INT64(summary.bytesTotal) AS bytesTotal
-  FROM ${ctx.ref('crawl', 'pages')}
-  WHERE
-    date = '${params.date}' ${params.devRankFilter} AND
-    INT64(summary.bytesTotal) > 0
-)
+FROM pages
 GROUP BY
   date,
   client,

infra/bigquery-export/reports.js

@@ -6,6 +6,7 @@ export class ReportsExporter {
   constructor () {
     this.bigquery = new BigQueryExport()
     this.storage = new StorageExport()
+    this.storagePath = 'reports/dev/' // TODO change to prod
   }
 
   // Export timeseries reports
@@ -18,7 +19,7 @@ SELECT
 FROM reports.${metric}_timeseries
 `
     const rows = await this.bigquery.queryResults(query)
-    await this.storage.exportToJson(rows, metric)
+    await this.storage.exportToJson(rows, `${this.storagePath}${metric}.json`)
   }
 
   // Export monthly histogram report
@@ -41,6 +42,10 @@ WHERE date = '${date}'
       return
     }
 
+    if (exportData.lense && exportData.lense !== 'all') {
+      this.storagePath = this.storagePath + `${exportData.lense}/`
+    }
+
     if (exportData.type === 'histogram') {
       await this.exportHistogram(exportData)
     } else if (exportData.type === 'timeseries') {

infra/bigquery-export/storage.js

@@ -7,16 +7,13 @@ const storage = new Storage()
 export class StorageExport {
   constructor (bucket = 'httparchive') {
     this.bucket = bucket
-    this.storagePath = 'reports/dev/' // TODO change to prod
     this.stream = new Readable({
       objectMode: true,
       read () {}
     })
   }
 
   async exportToJson (data, fileName) {
-    fileName = this.storagePath + fileName + '.json'
-
     const bucket = storage.bucket(this.bucket)
     const file = bucket.file(fileName)