Feature: Enhanced handling of GDPR banner acceptance in CLI

packages/analysis-utils/src/browserManagement/index.ts

@@ -28,6 +28,7 @@
   delay,
   RESPONSE_EVENT,
   REQUEST_EVENT,
+  type Selectors,
 } from '@google-psat/common';
 
 /**
@@ -42,6 +43,7 @@
 } from './types';
 import { parseNetworkDataToCookieData } from './parseNetworkDataToCookieData';
 import collateCookieData from './collateCookieData';
+import { CMP_SELECTORS, CMP_TEXT_SELECTORS } from '../constants';
 
 export class BrowserManagement {
   viewportConfig: ViewportConfig;
@@ -56,13 +58,15 @@
   shouldLogDebug: boolean;
   spinnies: Spinnies | undefined;
   indent = 0;
+  selectors: Selectors | undefined;
   constructor(
     viewportConfig: ViewportConfig,
     isHeadless: boolean,
     pageWaitTime: number,
     shouldLogDebug: boolean,
     indent: number,
-    spinnies?: Spinnies
+    spinnies?: Spinnies,
+    selectors?: Selectors
   ) {
     this.viewportConfig = viewportConfig;
     this.browser = null;
@@ -76,13 +80,13 @@
     this.pageResourcesMaps = {};
     this.spinnies = spinnies;
     this.indent = indent;
+    this.selectors = selectors;
   }
 
   debugLog(msg: any) {
     if (this.shouldLogDebug && this.spinnies) {
       this.spinnies.add(msg, {
         text: msg,
-        //@ts-ignore
         succeedColor: 'white',
         status: 'non-spinnable',
         indent: this.indent,
@@ -105,50 +109,186 @@
       headless: this.isHeadless,
       args,
     });
+
     this.debugLog('Browser initialized');
   }
 
+  async clickOnButtonUsingCMPSelectors(page: Page): Promise<boolean> {
+    let clickedOnButton = false;
+
+    try {
+      await Promise.all(
+        CMP_SELECTORS.map(async (selector) => {
+          const buttonToClick = await page.$(selector);
+          if (buttonToClick) {
+            await buttonToClick.click();
+            clickedOnButton = true;
+          }
+        })
+      );
+      return clickedOnButton;
+    } catch (error) {
+      return clickedOnButton;
+    }
+  }
+
+  async clickOnGDPRUsingTextSelectors(
+    page: Page,
+    textSelectors: string[]
+  ): Promise<boolean> {
+    if (textSelectors.length === 0) {
+      return false;
+    }
+
+    try {
+      const result = await page.evaluate((args: string[]) => {
+        let clickedOnButton = false;
+
+        const bannerNodes: Element[] = Array.from(
+          (document.querySelector('body')?.childNodes || []) as Element[]
+        )
+          ?.filter((node: Element) => node && node?.tagName === 'DIV')
+          ?.filter((node) => {
+            if (!node || !node?.textContent) {
+              return false;
+            }
+            const regex =
+              /\b(consent|policy|cookie policy|privacy policy|personalize|preferences|cookies)\b/;
+
+            return regex.test(node.textContent.toLowerCase());
+          });
+
+        bannerNodes?.forEach((node: Element) => {
+          const buttonNodes = Array.from(node?.getElementsByTagName('button'));
+
+          buttonNodes?.forEach((cnode) => {
+            if (!cnode?.textContent) {
+              return;
+            }
+
+            args.forEach((text) => {
+              if (cnode?.textContent?.toLowerCase().includes(text)) {
+                clickedOnButton = true;
+                cnode?.click();
+              }
+            });
+          });
+        });
+
+        return clickedOnButton;
+      }, textSelectors);
+
+      return result;
+    } catch (error) {
+      return false;
+    }
+  }
+
   async clickOnAcceptBanner(url: string) {
     const page = this.pages[url];
 
     if (!page) {
       throw new Error('No page with the provided id was found');
     }
 
-    await page.evaluate(() => {
-      const bannerNodes: Element[] = Array.from(
-        (document.querySelector('body')?.childNodes || []) as Element[]
-      )
-        .filter((node: Element) => node && node?.tagName === 'DIV')
-        .filter((node) => {
-          if (!node || !node?.textContent) {
-            return false;
+    const didSelectorsFromUserWork = await this.useSelectorsToSelectGDPRBanner(
+      page
+    );
+
+    if (didSelectorsFromUserWork) {
+      this.debugLog('GDPR banner found and accepted');
+      await delay(this.pageWaitTime / 2);
+      return;
+    }
+
+    // Click using CSS selectors.
+    const clickedUsingCMPCSSSelectors =
+      await this.clickOnButtonUsingCMPSelectors(page);
+
+    if (clickedUsingCMPCSSSelectors) {
+      this.debugLog('GDPR banner found and accepted');
+      await delay(this.pageWaitTime / 2);
+      return;
+    }
+
+    const buttonClicked = await this.clickOnGDPRUsingTextSelectors(
+      page,
+      CMP_TEXT_SELECTORS
+    );
+
+    if (buttonClicked) {
+      this.debugLog('GDPR banner found and accepted');
+      await delay(this.pageWaitTime / 2);
+      return;
+    }
+
+    this.debugLog('GDPR banner could not be found');
+    await delay(this.pageWaitTime / 2);
+    return;
+  }
+
+  async useSelectorsToSelectGDPRBanner(page: Page): Promise<boolean> {
+    let clickedOnButton = false;
+
+    if (!this.selectors) {
+      return false;
+    }
+
+    try {
+      await Promise.all(
+        this.selectors?.cssSelectors.map(async (selector) => {
+          const buttonToClick = await page.$(selector);
+          if (buttonToClick) {
+            clickedOnButton = true;
+            this.debugLog('GDPR banner found and accepted');
+            await buttonToClick.click();
+          }
+        })
+      );
+
+      if (clickedOnButton) {
+        return clickedOnButton;
+      }
+
+      clickedOnButton = await page.evaluate((xPaths: string[]) => {
+        const rootElement = document.querySelector('html');
+        let bannerAccepted = false;
+
+        if (!rootElement) {
+          return false;
+        }
+
+        xPaths.forEach((xPath) => {
+          const _acceptButton = document
+            .evaluate(xPath, rootElement)
+            .iterateNext();
+
+          if (!_acceptButton) {
+            return;
           }
-          const regex =
-            /\b(consent|policy|cookie policy|privacy policy|personalize|preferences)\b/;
 
-          return regex.test(node.textContent.toLowerCase());
+          if (_acceptButton instanceof HTMLElement) {
+            _acceptButton?.click();
+            bannerAccepted = true;
+          }
         });
 
-      const buttonToClick: HTMLButtonElement[] = bannerNodes
-        .map((node: Element) => {
-          const buttonNodes = Array.from(node.getElementsByTagName('button'));
-          const isButtonForAccept = buttonNodes.filter(
-            (cnode) =>
-              cnode.textContent &&
-              (cnode.textContent.toLowerCase().includes('accept') ||
-                cnode.textContent.toLowerCase().includes('allow') ||
-                cnode.textContent.toLowerCase().includes('ok') ||
-                cnode.textContent.toLowerCase().includes('agree'))
-          );
+        return bannerAccepted;
+      }, this.selectors?.xPath);
 
-          return isButtonForAccept[0];
-        })
-        .filter((button) => button);
-      buttonToClick[0]?.click();
-    });
+      if (clickedOnButton) {
+        return clickedOnButton;
+      }
 
-    await delay(this.pageWaitTime / 2);
+      clickedOnButton = await this.clickOnGDPRUsingTextSelectors(
+        page,
+        this.selectors?.textSelectors
+      );
+
+      return clickedOnButton;
+    } catch (error) {
+      return clickedOnButton;
+    }
   }
 
   async openPage(): Promise<Page> {
@@ -584,7 +724,7 @@
   }
 
   getResources(urls: string[]) {
     const allFetchedResources: { [key: string]: any } = {};
 
     urls.forEach((url) => {
       const page = this.pages[url];

packages/analysis-utils/src/constants.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export const CMP_SELECTORS = [
+  '.fc-cta-consent',
+  '.cky-btn-accept',
+  '.cc-accept-all',
+  '.cmplz-accept',
+  '.cc-allow',
+  '#CybotCookiebotDialogBodyLevelButtonLevelOptinAllowAll',
+  '#CybotCookiebotDialog button[value=”accept”]',
+  '#onetrust-accept-btn-handler',
+  '.iubenda-cs-accept-btn',
+];
+
+export const CMP_TEXT_SELECTORS = ['accept', 'allow', 'agree', 'ok'];

packages/analysis-utils/src/procedures/analyzeCookiesUrlsAndFetchResources.ts

@@ -23,6 +23,7 @@ import {
   type CookieDatabase,
   type LibraryMatchers,
   deriveBlockingStatus,
+  type Selectors,
 } from '@google-psat/common';
 
 /**
@@ -38,6 +39,7 @@ export const analyzeCookiesUrlsAndFetchResources = async (
   cookieDictionary: CookieDatabase,
   shouldSkipAcceptBanner: boolean,
   verbose: boolean,
+  selectors?: Selectors,
   spinnies?: Spinnies,
   indent = 4
 ) => {
@@ -51,7 +53,8 @@ export const analyzeCookiesUrlsAndFetchResources = async (
     delayTime,
     verbose,
     indent,
-    spinnies
+    spinnies,
+    selectors
   );
 
   await browser.initializeBrowser(true);

packages/analysis-utils/src/procedures/analyzeCookiesUrlsInBatchesAndFetchResources.ts

@@ -23,6 +23,7 @@ import {
   LibraryData,
   type LibraryMatchers,
   removeAndAddNewSpinnerText,
+  type Selectors,
 } from '@google-psat/common';
 
 /**
@@ -40,7 +41,8 @@ export const analyzeCookiesUrlsInBatchesAndFetchResources = async (
   spinnies?: Spinnies,
   shouldSkipAcceptBanner = false,
   verbose = false,
-  indent = 4
+  indent = 4,
+  selectors?: Selectors
 ) => {
   let report: {
     url: string;
@@ -81,6 +83,7 @@ export const analyzeCookiesUrlsInBatchesAndFetchResources = async (
         cookieDictionary,
         shouldSkipAcceptBanner,
         verbose,
+        selectors,
         spinnies
       );
 

packages/cli/src/e2e-tests/index.ts

@@ -26,7 +26,7 @@ describe('CLI E2E Test', () => {
 
   it('Should run site analysis', () => {
     return coffee
-      .fork(cli, ['-u https://bbc.com', '-w 1'])
+      .fork(cli, ['-u https://bbc.com', '-w 100'])
       .includes('stdout', '/out/bbc-com/report_')
       .end();
   }, 60000);