feat: cwv audits

package.json

@@ -57,7 +57,8 @@
     "@adobe/helix-shared-secrets": "2.1.2",
     "@adobe/helix-shared-wrap": "2.0.0",
     "@adobe/helix-status": "10.0.10",
-    "@adobe/helix-universal-logger": "3.0.11"
+    "@adobe/helix-universal-logger": "3.0.11",
+    "@aws-sdk/client-sqs": "3.450.0"
   },
   "devDependencies": {
     "@adobe/eslint-config-helix": "2.0.3",
@@ -70,6 +71,8 @@
     "@semantic-release/exec": "6.0.3",
     "@semantic-release/git": "10.0.1",
     "c8": "8.0.1",
+    "chai": "4.3.10",
+    "chai-as-promised": "7.1.1",
     "dotenv": "16.3.1",
     "eslint": "8.50.0",
     "esmock": "2.5.1",
@@ -82,6 +85,8 @@
     "nock": "13.3.3",
     "nodemon": "3.0.1",
     "semantic-release": "22.0.5",
+    "sinon": "17.0.1",
+    "sinon-chai": "3.7.0",
     "yaml": "2.3.2"
   },
   "lint-staged": {

src/cwv/handler.js

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { createUrl, Response } from '@adobe/fetch';
+import { fetch } from '../support/utils.js';
+
+export const DEFAULT_PARAMS = {
+  interval: 7,
+  offset: 0,
+  limit: 100,
+};
+
+// weekly pageview threshold to eliminate urls with lack of samples
+const PAGEVIEW_THRESHOLD = 7000;
+
+export async function getRUMUrl(url) {
+  const urlWithScheme = url.startsWith('http') ? url : `https://${url}`;
+  const resp = await fetch(urlWithScheme);
+  return resp.url.split('://')[1];
+}
+
+const DOMAIN_LIST_URL = 'https://helix-pages.anywhere.run/helix-services/run-query@v3/rum-dashboard';
+
+export default async function auditCWV(message, context) {
+  const { type, url, auditContext } = message;
+  const { log, sqs } = context;
+  const { AUDIT_JOBS_QUEUE_URL: queueUrl } = context.env;
+
+  log.info(`Received audit req for domain: ${url}`);
+
+  const finalUrl = await getRUMUrl(url);
+
+  const params = {
+    ...DEFAULT_PARAMS,
+    domainkey: context.env.RUM_DOMAIN_KEY,
+    url: finalUrl,
+  };
+
+  const resp = await fetch(createUrl(DOMAIN_LIST_URL, params));
+  const respJson = await resp.json();
+
+  const auditResult = respJson?.results?.data
+    .filter((row) => row.pageviews > PAGEVIEW_THRESHOLD)
+    .filter((row) => row.url.toLowerCase() !== 'other')
+    .map((row) => ({
+      url: row.url,
+      pageviews: row.pageviews,
+      avglcp: row.avglcp,
+    }));
+
+  await sqs.sendMessage(queueUrl, {
+    type,
+    url,
+    auditContext,
+    auditResult,
+  });
+
+  log.info(`Successfully audited ${url} for ${type} type audit`);
+
+  return new Response('');
+}

src/index.js

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019 Adobe. All rights reserved.
+ * Copyright 2023 Adobe. All rights reserved.
  * This file is licensed to you under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License. You may obtain a copy
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
@@ -13,19 +13,79 @@ import wrap from '@adobe/helix-shared-wrap';
 import { helixStatus } from '@adobe/helix-status';
 import { Response } from '@adobe/fetch';
 import secrets from '@adobe/helix-shared-secrets';
+import sqs from './support/sqs.js';
+import cwv from './cwv/handler.js';
+
+const HANDLERS = {
+  cwv,
+};
+
+/**
+ * Wrapper to turn an SQS record into a function param
+ * Inspired by https://github.com/adobe/helix-admin/blob/main/src/index.js#L104C1-L128C5
+ *
+ * @param {UniversalAction} fn
+ * @returns {function(object, UniversalContext): Promise<Response>}
+ */
+function sqsEventAdapter(fn) {
+  return async (req, context) => {
+    const { log } = context;
+    let message;
+
+    try {
+      log.info(`number of records in message: ${context.invocation?.event?.Records.length}`);
+      // currently not publishing batch messages
+      message = JSON.parse(context.invocation?.event?.Records[0]?.body);
+    } catch (e) {
+      log.error('Function was not invoked properly, message body is not a valid JSON');
+      return new Response('', {
+        status: 400,
+        headers: {
+          'x-error': 'Event does not contain a valid message body',
+        },
+      });
+    }
+    return fn(message, context);
+  };
+}
 
 /**
  * This is the main function
- * @param {Request} request the request object (see fetch api)
+ * @param {object} message the message object received from SQS
  * @param {UniversalContext} context the context of the universal serverless function
  * @returns {Response} a response
  */
-function run(request, context) {
-  const name = new URL(request.url).searchParams.get('name') || process.env.SECRET;
-  context.log.info(`Saying hello to: ${name}.`);
-  return new Response(`Hello, ${name}.`);
+async function run(message, context) {
+  const { log } = context;
+  const { type, url } = message;
+
+  log.info(`Audit req received for url: ${url}`);
+
+  const handler = HANDLERS[type];
+  if (!handler) {
+    const msg = `no such audit type: ${type}`;
+    log.error(msg);
+    return new Response('', { status: 404 });
+  }
+
+  const t0 = Date.now();
+
+  try {
+    return await handler(message, context);
+  } catch (e) {
+    const t1 = Date.now();
+    log.error(`handler exception after ${t1 - t0}ms`, e);
+    return new Response('', {
+      status: e.statusCode || 500,
+      headers: {
+        'x-error': 'internal server error',
+      },
+    });
+  }
 }
 
 export const main = wrap(run)
-  .with(helixStatus)
-  .with(secrets);
+  .with(sqsEventAdapter)
+  .with(sqs)
+  .with(secrets)
+  .with(helixStatus);

src/support/sqs.js

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { SendMessageCommand, SQSClient } from '@aws-sdk/client-sqs';
+
+/**
+ * @class SQS utility to send messages to SQS
+ * @param {string} region - AWS region
+ * @param {object} log - log object
+ */
+class SQS {
+  constructor(region, log) {
+    this.sqsClient = new SQSClient({ region });
+    this.log = log;
+  }
+
+  async sendMessage(queueUrl, message) {
+    const body = {
+      ...message,
+      timestamp: new Date().toISOString(),
+    };
+
+    const msgCommand = new SendMessageCommand({
+      MessageBody: JSON.stringify(body),
+      QueueUrl: queueUrl,
+    });
+
+    try {
+      const data = await this.sqsClient.send(msgCommand);
+      this.log.info(`Success, message sent. MessageID:  ${data.MessageId}`);
+    } catch (e) {
+      const { type, code, message: msg } = e;
+      this.log.error(`Message sent failed. Type: ${type}, Code: ${code}, Message: ${msg}`);
+      throw e;
+    }
+  }
+}
+
+export default function sqsWrapper(fn) {
+  return async (request, context) => {
+    if (!context.sqs) {
+      const { log } = context;
+      const { region } = context.runtime;
+      context.sqs = new SQS(region, log);
+    }
+
+    return fn(request, context);
+  };
+}

src/support/utils.js

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+import { context as h2, h1 } from '@adobe/fetch';
+
+/* c8 ignore next 3 */
+export const { fetch } = process.env.HELIX_FETCH_FORCE_HTTP1
+  ? h1()
+  : h2();

test/audits/cwv.test.js

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2023 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/* eslint-env mocha */
+/* eslint-disable no-unused-expressions */ // expect statements
+
+import chai from 'chai';
+import sinon from 'sinon';
+import sinonChai from 'sinon-chai';
+import { Request } from '@adobe/fetch';
+import nock from 'nock';
+import { main } from '../../src/index.js';
+import { DEFAULT_PARAMS, getRUMUrl } from '../../src/cwv/handler.js';
+import { expectedAuditResult, rumData } from '../rum-data.js';
+
+chai.use(sinonChai);
+const { expect } = chai;
+
+const sandbox = sinon.createSandbox();
+describe('Index Tests', () => {
+  const request = new Request('https://space.cat');
+  let context;
+  let messageBodyJson;
+
+  beforeEach('setup', () => {
+    messageBodyJson = {
+      type: 'cwv',
+      url: 'adobe.com',
+      auditContext: {
+        key: 'value',
+      },
+    };
+    context = {
+      log: console,
+      runtime: {
+        region: 'us-east-1',
+      },
+      env: {
+        AUDIT_JOBS_QUEUE_URL: 'queueUrl',
+        RUM_DOMAIN_KEY: 'domainkey',
+      },
+      invocation: {
+        event: {
+          Records: [{
+            body: JSON.stringify(messageBodyJson),
+          }],
+        },
+      },
+      sqs: {
+        sendMessage: sandbox.stub().resolves(),
+      },
+    };
+  });
+
+  it('fetch cwv for base url > process > send results', async () => {
+    nock('https://adobe.com')
+      .get('/')
+      .reply(200);
+    nock('https://helix-pages.anywhere.run')
+      .get('/helix-services/run-query@v3/rum-dashboard')
+      .query({
+        ...DEFAULT_PARAMS,
+        domainkey: context.env.RUM_DOMAIN_KEY,
+        url: 'adobe.com/',
+      })
+      .reply(200, rumData);
+
+    const resp = await main(request, context);
+
+    const expectedMessage = {
+      ...messageBodyJson,
+      auditResult: expectedAuditResult,
+    };
+
+    expect(resp.status).to.equal(200);
+    expect(context.sqs.sendMessage).to.have.been.calledOnce;
+    expect(context.sqs.sendMessage).to.have.been
+      .calledWith(context.env.AUDIT_JOBS_QUEUE_URL, expectedMessage);
+  });
+
+  it('getRUMUrl do not add scheme to urls with a scheme already', async () => {
+    nock('http://space.cat')
+      .get('/')
+      .reply(200);
+
+    const finalUrl = await getRUMUrl('http://space.cat');
+    expect(finalUrl).to.eql('space.cat/');
+  });
+});