Completing reports migration

definitions/output/reports/reports_dynamic.js

@@ -1,47 +1,66 @@
 const configs = new reports.HTTPArchiveReports()
 const metrics = configs.listMetrics()
 
-const iterations = []
-for (
-  let month = constants.currentMonth; month >= constants.currentMonth; month = constants.fnPastMonth(month)) {
-  iterations.push({
-    date: month,
-    devRankFilter: constants.devRankFilter
-  })
+// Adjust start and end dates to update reports retrospectively
+const startDate = '2024-12-01' // constants.currentMonth;
+const endDate = '2024-12-01' // constants.currentMonth;
+
+const lenses = {
+  all: '',
+  top1k: 'AND rank <= 1000',
+  top10k: 'AND rank <= 10000',
+  top100k: 'AND rank <= 100000',
+  top1m: 'AND rank <= 1000000',
+  drupal: 'AND \'Drupal\' IN UNNEST(technologies.technology)',
+  magento: 'AND \'Magento\' IN UNNEST(technologies.technology)',
+  wordpress: 'AND \'WordPress\' IN UNNEST(technologies.technology)'
 }
 
-if (iterations.length === 1) {
-  const params = iterations[0]
+const iterations = []
+// dates
+for (
+  let date = endDate;
+  date >= startDate;
+  date = constants.fnPastMonth(date)
+) {
+  // metrics
   metrics.forEach(metric => {
+    // timeseries and histograms
     metric.SQL.forEach(sql => {
-      publish(metric.id + '_' + sql.type, {
-        type: 'incremental',
-        protected: true,
-        bigquery: sql.type === 'histogram' ? { partitionBy: 'date', clusterBy: ['client'] } : {},
-        schema: 'reports',
-        tags: ['crawl_complete', 'http_reports']
-      }).preOps(ctx => `
---DELETE FROM ${ctx.self()}
---WHERE date = '${params.date}';
-  `).query(ctx => `
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */` +
-sql.query(ctx, params))
+      // lenses
+      for (const [key, value] of Object.entries(lenses)) {
+        iterations.push({
+          date,
+          metric,
+          sql,
+          lens: { name: key, sql: value },
+          devRankFilter: constants.devRankFilter
+        })
+      }
     })
   })
-} else {
-  iterations.forEach((params, i) => {
-    metrics.forEach(metric => {
-      metric.SQL.forEach(sql => {
-        operate(metric.id + '_' + sql.type + '_' + params.date, {
-          tags: ['crawl_complete']
-        }).queries(ctx => `
-DELETE FROM reports.${metric.id}_${sql.type}
+}
+
+iterations.forEach((params, i) => {
+  operate(
+    params.metric.id + '_' + params.sql.type + '_' + params.lens.name + '_' + params.date)
+    .tags(['crawl_complete', 'reports'])
+    .queries(ctx => `
+CREATE TABLE IF NOT EXISTS reports.${params.sql.type} (
+  date DATE,
+  lens STRING,
+  metric STRING,
+  client STRING,
+  data JSON
+)
+PARTITION BY date
+CLUSTER BY client, lens;
+
+DELETE FROM reports.${params.sql.type}
 WHERE date = '${params.date}';
 
-/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${metric.id}", "type": "${sql.type}"} */
-INSERT INTO reports.${metric.id}_${sql.type}` +
-        sql.query(ctx, params))
-      })
-    })
-  })
-}
+/* {"dataform_trigger": "report_complete", "date": "${params.date}", "name": "${params.metric.id}", "type": "${params.sql.type}", "lense": "${params.lens.name}"} */
+INSERT INTO reports.${params.sql.type}` +
+params.sql.query(ctx, params)
+    )
+})

includes/reports.js

@@ -7,6 +7,18 @@ const config = {
         {
           type: 'histogram',
           query: DataformTemplateBuilder.create((ctx, params) => `
+WITH pages AS (
+  SELECT
+    date,
+    client,
+    CAST(FLOOR(INT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin
+  FROM crawl.pages
+  WHERE
+    date = '${params.date}' ${params.lens.sql} AND
+    is_root_page AND
+    INT64(summary.bytesTotal) > 0
+)
+
 SELECT
   *,
   SUM(pdf) OVER (PARTITION BY client ORDER BY bin) AS cdf
@@ -16,25 +28,37 @@ FROM (
     volume / SUM(volume) OVER (PARTITION BY client) AS pdf
   FROM (
     SELECT
-      date,
-      client,
-      CAST(FLOOR(INT64(summary.bytesTotal) / 1024 / 100) * 100 AS INT64) AS bin,
+      *,
       COUNT(0) AS volume
-    FROM ${ctx.ref('crawl', 'pages')}
-    WHERE
-      date = '${params.date}' ${params.devRankFilter}
+    FROM pages
     GROUP BY
       date,
       client,
       bin
     HAVING bin IS NOT NULL
   )
 )
+ORDER BY
+  date,
+  bin,
+  client
 `)
         },
         {
           type: 'timeseries',
           query: DataformTemplateBuilder.create((ctx, params) => `
+WITH pages AS (
+  SELECT
+    date,
+    client,
+    INT64(summary.bytesTotal) AS bytesTotal
+  FROM crawl.pages
+  WHERE
+    date = '${params.date}' $ ${params.lens.sql} AND
+    is_root_page AND
+    INT64(summary.bytesTotal) > 0
+)
+
 SELECT
   date,
   client,
@@ -44,16 +68,7 @@ SELECT
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(501)] / 1024, 2) AS p50,
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(751)] / 1024, 2) AS p75,
   ROUND(APPROX_QUANTILES(bytesTotal, 1001)[OFFSET(901)] / 1024, 2) AS p90
-FROM (
-  SELECT
-    date,
-    client,
-    INT64(summary.bytesTotal) AS bytesTotal
-  FROM ${ctx.ref('crawl', 'pages')}
-  WHERE
-    date = '${params.date}' ${params.devRankFilter} AND
-    INT64(summary.bytesTotal) > 0
-)
+FROM pages
 GROUP BY
   date,
   client,

infra/bigquery-export/reports.js

@@ -18,7 +18,7 @@ SELECT
 FROM reports.${metric}_timeseries
 `
     const rows = await this.bigquery.queryResults(query)
-    await this.storage.exportToJson(rows, metric)
+    await this.storage.exportToJson(rows, `${this.storagePath}${metric}.json`)
   }
 
   // Export monthly histogram report
@@ -41,6 +41,12 @@ WHERE date = '${date}'
       return
     }
 
+    this.storagePath = 'reports/' + exportConfig.environment !== 'prod' ? 'dev/' : ''
+
+    if (exportConfig.lense && exportConfig.lense !== 'all') {
+      this.storagePath = this.storagePath + `${exportConfig.lense}/`
+    }
+
     if (exportConfig.type === 'histogram') {
       await this.exportHistogram(exportConfig)
     } else if (exportConfig.type === 'timeseries') {

infra/bigquery-export/storage.js

@@ -7,16 +7,13 @@ const storage = new Storage()
 export class StorageExport {
   constructor (bucket = 'httparchive') {
     this.bucket = bucket
-    this.storagePath = 'reports/dev/' // TODO change to prod
     this.stream = new Readable({
       objectMode: true,
       read () {}
     })
   }
 
   async exportToJson (data, fileName) {
-    fileName = this.storagePath + fileName + '.json'
-
     const bucket = storage.bucket(this.bucket)
     const file = bucket.file(fileName)