feat: cwv audit revival

src/cwv/handler.js

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Adobe. All rights reserved.
+ * Copyright 2024 Adobe. All rights reserved.
  * This file is licensed to you under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License. You may obtain a copy
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
@@ -10,83 +10,33 @@
  * governing permissions and limitations under the License.
  */
 
-import RUMAPIClient, { createRUMURL } from '@adobe/spacecat-shared-rum-api-client-v1';
-import { internalServerError, noContent } from '@adobe/spacecat-shared-http-utils';
-import { composeAuditURL } from '@adobe/spacecat-shared-utils';
-import { retrieveSiteBySiteId } from '../utils/data-access.js';
-
-const PAGEVIEW_THRESHOLD = 35000;
-
-export function filterRUMData(data) {
-  return data.pageviews > PAGEVIEW_THRESHOLD // ignore the pages with low pageviews
-      && data.url.toLowerCase() !== 'other'; // ignore the combined result
+import RUMAPIClient from '@adobe/spacecat-shared-rum-api-client';
+import { getRUMDomainkey } from '../support/utils.js';
+import { AuditBuilder } from '../common/audit-builder.js';
+
+const DAILY_THRESHOLD = 1000;
+const INTERVAL = 7; // days
+
+export async function CWVRunner(auditUrl, context, site) {
+  const rumAPIClient = RUMAPIClient.createFrom(context);
+  const domainkey = await getRUMDomainkey(site.getBaseURL(), context);
+  const options = {
+    domain: auditUrl,
+    domainkey,
+    interval: INTERVAL,
+    granularity: 'hourly',
+  };
+  const cwvData = await rumAPIClient.query('cwv', options);
+  const auditResult = {
+    cwv: cwvData.filter((data) => data.pageviews >= DAILY_THRESHOLD * INTERVAL),
+  };
+
+  return {
+    auditResult,
+    fullAuditRef: auditUrl,
+  };
 }
 
-/**
- * url param in run-query@v3/rum-dashboard works in a 'startsWith' fashion. url=domain.com returns
- * an empty result whereas url=www.domain.com/ returns the desired result. To catch the redirects
- * to subdomains we issue a GET call to the domain, then use the final url after redirects
- * @param url
- * @returns finalUrl {Promise<string>}
- */
-
-function processRUMResponse(data) {
-  return data
-    .filter(filterRUMData)
-    .map((row) => ({
-      url: row.url,
-      pageviews: row.pageviews,
-      CLS: row.avgcls,
-      INP: row.avginp,
-      LCP: row.avglcp,
-    }));
-}
-export default async function auditCWV(message, context) {
-  const { type, url: siteId, auditContext = {} } = message;
-  const { dataAccess, log, sqs } = context;
-  const {
-    AUDIT_RESULTS_QUEUE_URL: queueUrl,
-  } = context.env;
-  try {
-    const site = await retrieveSiteBySiteId(dataAccess, siteId, log);
-    const url = site.getBaseURL();
-
-    log.info(`Received audit req for domain: ${url}`);
-
-    const rumAPIClient = RUMAPIClient.createFrom(context);
-    const finalUrl = await composeAuditURL(url);
-    auditContext.finalUrl = finalUrl;
-
-    const params = {
-      url: finalUrl,
-    };
-
-    const data = await rumAPIClient.getRUMDashboard(params);
-    const auditResult = processRUMResponse(data);
-    const fullAuditRef = createRUMURL({ ...params, domainkey: '' });
-
-    const auditData = {
-      siteId: site.getId(),
-      isLive: site.isLive(),
-      auditedAt: new Date().toISOString(),
-      auditType: type,
-      fullAuditRef,
-      auditResult,
-    };
-
-    await dataAccess.addAudit(auditData);
-
-    await sqs.sendMessage(queueUrl, {
-      type,
-      url,
-      auditContext,
-      auditResult,
-    });
-
-    log.info(`Successfully audited ${url} for ${type} type audit`);
-    return noContent();
-  } catch (e) {
-    log.info(`CWV audit failed for ${siteId} failed due to ${e.message}`);
-    return internalServerError(`Internal server error: ${e.message}`);
-  }
-}
+export default new AuditBuilder()
+  .withRunner(CWVRunner)
+  .build();

src/index.js

@@ -11,10 +11,10 @@
  */
 import wrap from '@adobe/helix-shared-wrap';
 import { helixStatus } from '@adobe/helix-status';
-import { Response } from '@adobe/fetch';
 import secrets from '@adobe/helix-shared-secrets';
 import dataAccess from '@adobe/spacecat-shared-data-access';
 import { resolveSecretsName, sqsEventAdapter } from '@adobe/spacecat-shared-utils';
+import { internalServerError, notFound, ok } from '@adobe/spacecat-shared-http-utils';
 
 import sqs from './support/sqs.js';
 import apex from './apex/handler.js';
@@ -45,6 +45,7 @@ const HANDLERS = {
   'experimentation-ess-daily': essExperimentationDaily,
   'experimentation-ess-all': essExperimentationAll,
   costs,
+  dummy: (message) => ok(message),
 };
 
 function getElapsedSeconds(startTime) {
@@ -69,7 +70,7 @@ async function run(message, context) {
   if (!handler) {
     const msg = `no such audit type: ${type}`;
     log.error(msg);
-    return new Response('', { status: 404 });
+    return notFound();
   }
 
   const startTime = process.hrtime();
@@ -82,12 +83,7 @@ async function run(message, context) {
     return result;
   } catch (e) {
     log.error(`Audit failed after ${getElapsedSeconds(startTime)} seconds`, e);
-    return new Response('', {
-      status: e.statusCode || 500,
-      headers: {
-        'x-error': 'internal server error',
-      },
-    });
+    return internalServerError();
   }
 }
 

src/support/utils.js

@@ -11,7 +11,7 @@
  */
 
 import { context as h2, h1 } from '@adobe/fetch';
-import { hasText, resolveCustomerSecretsName } from '@adobe/spacecat-shared-utils';
+import { hasText, prependSchema, resolveCustomerSecretsName } from '@adobe/spacecat-shared-utils';
 import URI from 'urijs';
 import { JSDOM } from 'jsdom';
 import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager';
@@ -26,7 +26,7 @@ export const { fetch } = process.env.HELIX_FETCH_FORCE_HTTP1
 // weekly pageview threshold to eliminate urls with lack of samples
 
 export async function getRUMUrl(url) {
-  const urlWithScheme = url.startsWith('http') ? url : `https://${url}`;
+  const urlWithScheme = prependSchema(url);
   const resp = await fetch(urlWithScheme);
   const finalUrl = resp.url.split('://')[1];
   return finalUrl.endsWith('/') ? finalUrl.slice(0, -1) : /* c8 ignore next */ finalUrl;

test/audits/cwv.test.js

@@ -15,156 +15,56 @@
 import { expect, use } from 'chai';
 import sinon from 'sinon';
 import sinonChai from 'sinon-chai';
-import { Request } from '@adobe/fetch';
 import nock from 'nock';
 import { createSite } from '@adobe/spacecat-shared-data-access/src/models/site.js';
-import { main } from '../../src/index.js';
-import { getRUMUrl } from '../../src/support/utils.js';
-import { expectedAuditResult, rumData } from '../fixtures/rum-data.js';
+import { CWVRunner } from '../../src/cwv/handler.js';
+import { rumData } from '../fixtures/rum-data.js';
 
 use(sinonChai);
 
 const sandbox = sinon.createSandbox();
+
+const baseURL = 'https://spacecat.com';
+const auditUrl = 'www.spacecat.com';
 const DOMAIN_REQUEST_DEFAULT_PARAMS = {
+  domain: auditUrl,
+  domainkey: 42,
   interval: 7,
-  offset: 0,
-  limit: 101,
+  granularity: 'hourly',
 };
 
-const mockDate = '2023-11-27T12:30:01.124Z';
 describe('Index Tests', () => {
-  const request = new Request('https://space.cat');
-  let mockDataAccess;
-  let context;
-  let messageBodyJson;
-  let site;
-
-  before('init', function () {
-    this.clock = sandbox.useFakeTimers({
-      now: new Date(mockDate).getTime(),
-    });
-  });
+  const site = createSite({ baseURL });
+  const context = {
+    runtime: { name: 'aws-lambda', region: 'us-east-1' },
+    func: { package: 'spacecat-services', version: 'ci', name: 'test' },
+    rumApiClient: {
+      query: sandbox.stub().withArgs('variable-1', sinon.match(DOMAIN_REQUEST_DEFAULT_PARAMS)).resolves(rumData),
+    },
+  };
 
   beforeEach('setup', () => {
-    site = createSite({
-      baseURL: 'https://adobe.com',
-    });
-
-    mockDataAccess = {
-      getSiteByID: sinon.stub(),
-      addAudit: sinon.stub(),
-    };
-    mockDataAccess.getSiteByID = sinon.stub().withArgs('site-id').resolves(site);
-
-    messageBodyJson = {
-      type: 'cwv',
-      url: 'site-id',
-      auditContext: {
-        finalUrl: 'adobe.com',
-      },
-    };
-    context = {
-      log: console,
-      runtime: {
-        region: 'us-east-1',
-      },
-      dataAccess: mockDataAccess,
-      env: {
-        AUDIT_RESULTS_QUEUE_URL: 'queueUrl',
-        RUM_DOMAIN_KEY: 'domainkey',
-      },
-      invocation: {
-        event: {
-          Records: [{
-            body: JSON.stringify(messageBodyJson),
-          }],
-        },
-      },
-      sqs: {
-        sendMessage: sandbox.stub().resolves(),
-      },
-    };
-  });
-
-  after(function () {
-    this.clock.uninstall();
+    nock('https://secretsmanager.us-east-1.amazonaws.com/')
+      .post('/', (body) => body.SecretId === '/helix-deploy/spacecat-services/customer-secrets/spacecat_com/ci')
+      .reply(200, {
+        SecretString: JSON.stringify({
+          RUM_DOMAIN_KEY: '42',
+        }),
+      });
   });
 
   afterEach(() => {
     nock.cleanAll();
     sinon.restore();
   });
 
-  it('fetch cwv for base url > process > send results', async () => {
-    nock('https://adobe.com')
-      .get('/')
-      .reply(200);
-    nock('https://helix-pages.anywhere.run')
-      .get('/helix-services/run-query@v3/rum-dashboard')
-      .query({
-        ...DOMAIN_REQUEST_DEFAULT_PARAMS,
-        domainkey: context.env.RUM_DOMAIN_KEY,
-        url: 'adobe.com',
-      })
-      .reply(200, rumData);
-
-    const resp = await main(request, context);
-
-    const expectedMessage = {
-      ...messageBodyJson,
-      url: site.getBaseURL(),
-      auditResult: expectedAuditResult,
-    };
-
-    expect(resp.status).to.equal(204);
-    expect(mockDataAccess.addAudit).to.have.been.calledOnce;
-    expect(mockDataAccess.addAudit).to.have.been.calledWith({
-      siteId: site.getId(),
-      isLive: false,
-      auditedAt: mockDate,
-      auditType: 'cwv',
-      fullAuditRef: 'https://helix-pages.anywhere.run/helix-services/run-query@v3/rum-dashboard?interval=7&offset=0&limit=101&url=adobe.com&domainkey=',
-      auditResult: expectedAuditResult,
+  it('cwv audit runs rum api client cwv query', async () => {
+    const result = await CWVRunner('www.spacecat.com', context, site);
+    expect(result).to.deep.equal({
+      auditResult: {
+        cwv: rumData.filter((data) => data.pageviews >= 7000),
+      },
+      fullAuditRef: auditUrl,
     });
-    expect(context.sqs.sendMessage).to.have.been.calledOnce;
-    expect(context.sqs.sendMessage).to.have.been
-      .calledWith(context.env.AUDIT_RESULTS_QUEUE_URL, expectedMessage);
-  });
-
-  it('fetch cwv for base url for base url > process > reject', async () => {
-    nock('https://adobe.com')
-      .get('/')
-      .reply(200);
-    nock('https://helix-pages.anywhere.run')
-      .get('/helix-services/run-query@v3/rum-dashboard')
-      .query({
-        ...DOMAIN_REQUEST_DEFAULT_PARAMS,
-        domainkey: context.env.RUM_DOMAIN_KEY,
-        checkpoint: 404,
-        url: 'adobe.com',
-      })
-      .replyWithError('Bad request');
-
-    const resp = await main(request, context);
-
-    expect(resp.status).to.equal(500);
-  });
-
-  it('getRUMUrl do not add scheme to urls with a scheme already', async () => {
-    nock('http://space.cat')
-      .get('/')
-      .reply(200);
-
-    const finalUrl = await getRUMUrl('http://space.cat');
-    expect(finalUrl).to.eql('space.cat');
-  });
-
-  it('getRUMUrl adds scheme to urls without a scheme', async () => {
-    nock('https://space.cat')
-      .get('/')
-      .reply(200);
-
-    const finalUrl = await getRUMUrl('space.cat');
-    expect(finalUrl).to.eql('space.cat');
   });
 });