Resolve Primary Scan crashes

apps/cli/src/queue.controller.ts

@@ -47,7 +47,7 @@ export class QueueController {
     this.logger.log(`queueing site: ${url}`);
 
     try {
-      let website = await this.websiteService.findByUrl(url);
+      const website = await this.websiteService.findByUrl(url);
 
       const coreInput: CoreInputDto = {
         websiteId: website.id,

libs/core-scanner/src/scans/cms.ts

@@ -6,8 +6,9 @@ import { CmsScan } from 'entities/scan-data.entity';
 import { Logger } from 'pino';
 
 export async function buildCmsResult( parentLogger: Logger, mainResponse: HTTPResponse ): Promise<CmsScan> {
-  const htmlMatches = await getHtmlMatches(mainResponse);
-  const headerMatches = await getHeaderMatches(mainResponse);
+  parentLogger.info(`Building CMS scan result for: ${mainResponse.url()}`);
+  const htmlMatches = await getHtmlMatches(parentLogger, mainResponse);
+  const headerMatches = await getHeaderMatches(parentLogger, mainResponse);
 
   let cms = null;
 
@@ -20,8 +21,14 @@ export async function buildCmsResult( parentLogger: Logger, mainResponse: HTTPRe
   return { cms };
 };
 
-const getHtmlMatches = async (response: HTTPResponse) => {
-  const actualHtml = await response.text();
+const getHtmlMatches = async (logger: Logger, response: HTTPResponse) => {
+  logger.info(`Getting HTML matches for: ${response.url()}`);
+  let actualHtml = '';
+  try {
+    actualHtml = await response.text();
+  } catch (error) {
+    logger.error({ error }, `Error getting HTML from response: ${error.message}`);
+  }
 
   return cmsData.filter((obj) => {
     if (obj.html) {
@@ -42,7 +49,7 @@ const getHtmlMatches = async (response: HTTPResponse) => {
   });
 };
 
-const getHeaderMatches = async (response: HTTPResponse) => {
+const getHeaderMatches = async (logger: Logger, response: HTTPResponse) => {
   const actualHeaders = await response.headers();
   const formattedActualHeaders = _.transform(
     actualHeaders,

libs/core-scanner/src/scans/login.ts

@@ -36,16 +36,25 @@ const loginProviderStrings = [
 ];
 
 export async function buildLoginResult ( parentLogger: Logger, mainResponse: HTTPResponse ): Promise<LoginScan> {
-  const html = await mainResponse.text();
-  const htmlLower = html.toLowerCase();
+  parentLogger.info(`Building login scan result for: ${mainResponse.url()}`);
+  try {
+    const html = await mainResponse.text();
+    const htmlLower = html.toLowerCase();
 
-  const loginDetected = getLoginDetectedResults(htmlLower);
-  const loginProvider = getLoginProviderResults(htmlLower);
+    const loginDetected = getLoginDetectedResults(htmlLower);
+    const loginProvider = getLoginProviderResults(htmlLower);
 
-  return {
-    loginDetected,
-    loginProvider,
-  };
+    return {
+      loginDetected,
+      loginProvider,
+    };
+  } catch (error) {
+    parentLogger.error({ error }, `Error scanning for login: ${error.message}`);
+    return {
+      loginDetected: null,
+      loginProvider: null,
+    };
+  }
 };
 
 const getLoginDetectedResults = (html: string): string | null => {

libs/core-scanner/src/scans/mobile.ts

@@ -7,21 +7,26 @@ export const buildMobileResult = async (
   logger: Logger,
   page: Page,
 ): Promise<MobileScan> => {
-  const viewportMetaTag = await getHasViewportMetaTag(page);
+  const viewportMetaTag = await getHasViewportMetaTag(logger, page);
 
   return {
     viewportMetaTag,
   };
 };
 
-async function getHasViewportMetaTag(page: Page): Promise<boolean> {
-  const result = await page.evaluate(() => {
-    const el = document.querySelector(
-      "head > meta[name='viewport'][content*='width=']",
-    );
-
-    return el === null ? false : true;
-  });
+async function getHasViewportMetaTag(logger: Logger, page: Page): Promise<boolean> {
+  let result = false;
+  try {
+    result = await page.evaluate(() => {
+      const el = document.querySelector(
+        "head > meta[name='viewport'][content*='width=']",
+      );
+
+      return el === null ? false : true;
+    });
+  } catch(error) {
+    logger.error({error}, `Error getting viewport meta tag: ${error}`);
+  };
 
   return result;
 }

libs/core-scanner/src/scans/required-links.ts

@@ -5,71 +5,79 @@ import { RequiredLinksScan } from 'entities/scan-data.entity';
 import { Logger } from 'pino';
 
 export async function buildRequiredLinksResult( parentLogger: Logger, page: Page ): Promise<RequiredLinksScan> {
-  const requiredLinksResults = await page.evaluate(() => {
-    const requiredLinksUrlContents = [
-      'about',
-      'fear',
-      'foia',
-      'inspector',
-      'privacy',
-      'usa.gov',
-      'spanish',
-      'espanol',
-      'español',
-      '/es',
-    ];
-
-    const requiredLinksUrl = requiredLinksUrlContents
-      .filter((string) => {
-        let stringDetected = false;
-
-        document.querySelectorAll('a').forEach((el) => {
-          const href = el.getAttribute('href');
-          if (href && href.toLowerCase().includes(string)) {
-            stringDetected = true;
-          }
-        });
-
-        return stringDetected;
-      })
-      .join(',');
-
-    const requiredLinksTextContents = [
-      'about us',
-      'accessibility',
-      'budget and performance',
-      'no fear act',
-      'foia',
-      'freedom of information act',
-      'inspector general',
-      'privacy policy',
-      'vulnerability disclosure',
-      'usa.gov',
-      'espanol',
-      'español',
-      'espa&ntilde;ol',
-      'spanish',
-    ];
-
-    const requiredLinksText = requiredLinksTextContents
-      .filter((string) => {
-        let stringDetected = false;
-
-        document.querySelectorAll('a').forEach((el) => {
-          if (el.textContent.toLowerCase().includes(string)) {
-            stringDetected = true;
-          }
-        });
-
-        return stringDetected;
-      })
-      .join(',');
-
-    return {
-      requiredLinksUrl,
-      requiredLinksText,
-    };
-  });
+  let requiredLinksResults = {
+    requiredLinksUrl: '',
+    requiredLinksText: '',
+  };
+  try {
+    requiredLinksResults = await page.evaluate(() => {
+      const requiredLinksUrlContents = [
+        'about',
+        'fear',
+        'foia',
+        'inspector',
+        'privacy',
+        'usa.gov',
+        'spanish',
+        'espanol',
+        'español',
+        '/es',
+      ];
+
+      const requiredLinksUrl = requiredLinksUrlContents
+        .filter((string) => {
+          let stringDetected = false;
+
+          document.querySelectorAll('a').forEach((el) => {
+            const href = el.getAttribute('href');
+            if (href && href.toLowerCase().includes(string)) {
+              stringDetected = true;
+            }
+          });
+
+          return stringDetected;
+        })
+        .join(',');
+
+      const requiredLinksTextContents = [
+        'about us',
+        'accessibility',
+        'budget and performance',
+        'no fear act',
+        'foia',
+        'freedom of information act',
+        'inspector general',
+        'privacy policy',
+        'vulnerability disclosure',
+        'usa.gov',
+        'espanol',
+        'español',
+        'espa&ntilde;ol',
+        'spanish',
+      ];
+
+      const requiredLinksText = requiredLinksTextContents
+        .filter((string) => {
+          let stringDetected = false;
+
+          document.querySelectorAll('a').forEach((el) => {
+            if (el.textContent.toLowerCase().includes(string)) {
+              stringDetected = true;
+            }
+          });
+
+          return stringDetected;
+        })
+        .join(',');
+
+      return {
+        requiredLinksUrl,
+        requiredLinksText,
+      };
+    });
+  } catch (error) {
+    parentLogger.error({ error }, 'Error building required links scan');
+  };
 
   return requiredLinksResults;
 };