Merge branch 'main' into messagesize

Author: AndreiAlexandruParaschiv

CHANGELOG.md

@@ -1,3 +1,17 @@
+# [1.27.0](https://github.com/adobe/spacecat-audit-worker/compare/v1.26.5...v1.27.0) (2024-08-26)
+
+
+### Features
+
+* cwv audit revival ([#378](https://github.com/adobe/spacecat-audit-worker/issues/378)) ([3a27bde](https://github.com/adobe/spacecat-audit-worker/commit/3a27bded17a6efe7354b5533a217e47fc14c2285))
+
+## [1.26.5](https://github.com/adobe/spacecat-audit-worker/compare/v1.26.4...v1.26.5) (2024-08-24)
+
+
+### Bug Fixes
+
+* **deps:** update external fixes ([#376](https://github.com/adobe/spacecat-audit-worker/issues/376)) ([923ad80](https://github.com/adobe/spacecat-audit-worker/commit/923ad80f45b2ea2462857c997c377260349629f7))
+
 ## [1.26.4](https://github.com/adobe/spacecat-audit-worker/compare/v1.26.3...v1.26.4) (2024-08-23)
 
 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-audit-worker",
-  "version": "1.26.4",
+  "version": "1.27.0",
   "description": "SpaceCat Audit Worker",
   "main": "src/index.js",
   "type": "module",
@@ -51,32 +51,32 @@
   },
   "dependencies": {
     "@adobe/fetch": "4.1.8",
-    "@adobe/helix-shared-secrets": "2.2.8",
+    "@adobe/helix-shared-secrets": "2.2.9",
     "@adobe/helix-shared-wrap": "2.0.2",
     "@adobe/helix-status": "10.1.2",
     "@adobe/helix-universal": "5.0.5",
     "@adobe/helix-universal-logger": "3.0.18",
-    "@adobe/spacecat-shared-ahrefs-client": "1.5.0",
-    "@adobe/spacecat-shared-data-access": "1.43.2",
+    "@adobe/spacecat-shared-ahrefs-client": "1.5.1",
+    "@adobe/spacecat-shared-data-access": "1.43.3",
     "@adobe/spacecat-shared-http-utils": "1.6.8",
-    "@adobe/spacecat-shared-rum-api-client": "2.7.3",
+    "@adobe/spacecat-shared-rum-api-client": "2.7.4",
     "@adobe/spacecat-shared-rum-api-client-v1": "npm:@adobe/[email protected]",
-    "@aws-sdk/client-lambda": "3.632.0",
-    "@aws-sdk/credential-provider-node": "3.632.0",
-    "@adobe/spacecat-shared-utils": "1.19.5",
-    "@aws-sdk/client-secrets-manager": "3.632.0",
-    "@aws-sdk/client-sqs": "3.632.0",
+    "@aws-sdk/client-lambda": "3.637.0",
+    "@aws-sdk/credential-provider-node": "3.637.0",
+    "@adobe/spacecat-shared-utils": "1.19.6",
+    "@aws-sdk/client-secrets-manager": "3.637.0",
+    "@aws-sdk/client-sqs": "3.637.0",
     "diff": "5.2.0",
     "jsdom": "24.1.1",
     "urijs": "1.19.11"
   },
   "devDependencies": {
     "@adobe/eslint-config-helix": "2.0.6",
-    "@adobe/helix-deploy": "12.0.9",
+    "@adobe/helix-deploy": "12.0.10",
     "@adobe/helix-universal": "5.0.5",
     "@adobe/semantic-release-coralogix": "1.1.31",
     "@adobe/semantic-release-skms-cmr": "1.1.2",
-    "@redocly/cli": "1.19.0",
+    "@redocly/cli": "1.21.0",
     "@semantic-release/changelog": "6.0.3",
     "@semantic-release/exec": "6.0.3",
     "@semantic-release/git": "10.0.1",
@@ -88,13 +88,13 @@
     "chai-as-promised": "8.0.0",
     "dotenv": "16.4.5",
     "eslint": "8.57.0",
-    "husky": "9.1.4",
+    "husky": "9.1.5",
     "junit-report-builder": "5.0.0",
     "lint-staged": "15.2.9",
     "mocha": "10.7.3",
     "mocha-multi-reporters": "1.5.1",
     "mocha-suppress-logs": "0.5.1",
-    "nock": "13.5.4",
+    "nock": "13.5.5",
     "nodemon": "3.1.4",
     "semantic-release": "24.1.0",
     "sinon": "18.0.0",

package.json

@@ -1,5 +1,5 @@
 /*
- * Copyright 2023 Adobe. All rights reserved.
+ * Copyright 2024 Adobe. All rights reserved.
  * This file is licensed to you under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License. You may obtain a copy
  * of the License at http://www.apache.org/licenses/LICENSE-2.0
@@ -10,83 +10,33 @@
  * governing permissions and limitations under the License.
  */
 
-import RUMAPIClient, { createRUMURL } from '@adobe/spacecat-shared-rum-api-client-v1';
-import { internalServerError, noContent } from '@adobe/spacecat-shared-http-utils';
-import { composeAuditURL } from '@adobe/spacecat-shared-utils';
-import { retrieveSiteBySiteId } from '../utils/data-access.js';
-
-const PAGEVIEW_THRESHOLD = 35000;
-
-export function filterRUMData(data) {
-  return data.pageviews > PAGEVIEW_THRESHOLD // ignore the pages with low pageviews
-      && data.url.toLowerCase() !== 'other'; // ignore the combined result
+import RUMAPIClient from '@adobe/spacecat-shared-rum-api-client';
+import { getRUMDomainkey } from '../support/utils.js';
+import { AuditBuilder } from '../common/audit-builder.js';
+
+const DAILY_THRESHOLD = 1000;
+const INTERVAL = 7; // days
+
+export async function CWVRunner(auditUrl, context, site) {
+  const rumAPIClient = RUMAPIClient.createFrom(context);
+  const domainkey = await getRUMDomainkey(site.getBaseURL(), context);
+  const options = {
+    domain: auditUrl,
+    domainkey,
+    interval: INTERVAL,
+    granularity: 'hourly',
+  };
+  const cwvData = await rumAPIClient.query('cwv', options);
+  const auditResult = {
+    cwv: cwvData.filter((data) => data.pageviews >= DAILY_THRESHOLD * INTERVAL),
+  };
+
+  return {
+    auditResult,
+    fullAuditRef: auditUrl,
+  };
 }
 
-/**
- * url param in run-query@v3/rum-dashboard works in a 'startsWith' fashion. url=domain.com returns
- * an empty result whereas url=www.domain.com/ returns the desired result. To catch the redirects
- * to subdomains we issue a GET call to the domain, then use the final url after redirects
- * @param url
- * @returns finalUrl {Promise<string>}
- */
-
-function processRUMResponse(data) {
-  return data
-    .filter(filterRUMData)
-    .map((row) => ({
-      url: row.url,
-      pageviews: row.pageviews,
-      CLS: row.avgcls,
-      INP: row.avginp,
-      LCP: row.avglcp,
-    }));
-}
-export default async function auditCWV(message, context) {
-  const { type, url: siteId, auditContext = {} } = message;
-  const { dataAccess, log, sqs } = context;
-  const {
-    AUDIT_RESULTS_QUEUE_URL: queueUrl,
-  } = context.env;
-  try {
-    const site = await retrieveSiteBySiteId(dataAccess, siteId, log);
-    const url = site.getBaseURL();
-
-    log.info(`Received audit req for domain: ${url}`);
-
-    const rumAPIClient = RUMAPIClient.createFrom(context);
-    const finalUrl = await composeAuditURL(url);
-    auditContext.finalUrl = finalUrl;
-
-    const params = {
-      url: finalUrl,
-    };
-
-    const data = await rumAPIClient.getRUMDashboard(params);
-    const auditResult = processRUMResponse(data);
-    const fullAuditRef = createRUMURL({ ...params, domainkey: '' });
-
-    const auditData = {
-      siteId: site.getId(),
-      isLive: site.isLive(),
-      auditedAt: new Date().toISOString(),
-      auditType: type,
-      fullAuditRef,
-      auditResult,
-    };
-
-    await dataAccess.addAudit(auditData);
-
-    await sqs.sendMessage(queueUrl, {
-      type,
-      url,
-      auditContext,
-      auditResult,
-    });
-
-    log.info(`Successfully audited ${url} for ${type} type audit`);
-    return noContent();
-  } catch (e) {
-    log.info(`CWV audit failed for ${siteId} failed due to ${e.message}`);
-    return internalServerError(`Internal server error: ${e.message}`);
-  }
-}
+export default new AuditBuilder()
+  .withRunner(CWVRunner)
+  .build();

src/cwv/handler.js

@@ -11,10 +11,10 @@
  */
 import wrap from '@adobe/helix-shared-wrap';
 import { helixStatus } from '@adobe/helix-status';
-import { Response } from '@adobe/fetch';
 import secrets from '@adobe/helix-shared-secrets';
 import dataAccess from '@adobe/spacecat-shared-data-access';
 import { resolveSecretsName, sqsEventAdapter } from '@adobe/spacecat-shared-utils';
+import { internalServerError, notFound, ok } from '@adobe/spacecat-shared-http-utils';
 
 import sqs from './support/sqs.js';
 import apex from './apex/handler.js';
@@ -45,6 +45,7 @@ const HANDLERS = {
   'experimentation-ess-daily': essExperimentationDaily,
   'experimentation-ess-all': essExperimentationAll,
   costs,
+  dummy: (message) => ok(message),
 };
 
 function getElapsedSeconds(startTime) {
@@ -69,7 +70,7 @@ async function run(message, context) {
   if (!handler) {
     const msg = `no such audit type: ${type}`;
     log.error(msg);
-    return new Response('', { status: 404 });
+    return notFound();
   }
 
   const startTime = process.hrtime();
@@ -82,12 +83,7 @@ async function run(message, context) {
     return result;
   } catch (e) {
     log.error(`Audit failed after ${getElapsedSeconds(startTime)} seconds`, e);
-    return new Response('', {
-      status: e.statusCode || 500,
-      headers: {
-        'x-error': 'internal server error',
-      },
-    });
+    return internalServerError();
   }
 }
 

src/index.js

@@ -11,7 +11,7 @@
  */
 
 import { context as h2, h1 } from '@adobe/fetch';
-import { hasText, resolveCustomerSecretsName } from '@adobe/spacecat-shared-utils';
+import { hasText, prependSchema, resolveCustomerSecretsName } from '@adobe/spacecat-shared-utils';
 import URI from 'urijs';
 import { JSDOM } from 'jsdom';
 import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager';
@@ -26,7 +26,7 @@ export const { fetch } = process.env.HELIX_FETCH_FORCE_HTTP1
 // weekly pageview threshold to eliminate urls with lack of samples
 
 export async function getRUMUrl(url) {
-  const urlWithScheme = url.startsWith('http') ? url : `https://${url}`;
+  const urlWithScheme = prependSchema(url);
   const resp = await fetch(urlWithScheme);
   const finalUrl = resp.url.split('://')[1];
   return finalUrl.endsWith('/') ? finalUrl.slice(0, -1) : /* c8 ignore next */ finalUrl;