From 86dbee668aa3b69f78cee9b17fb3b67cd98308d1 Mon Sep 17 00:00:00 2001
From: Andy Kuny <andrew.kuny@gsa.gov>
Date: Fri, 15 Dec 2023 08:43:40 -0500
Subject: [PATCH] Move isLive to util and apply to sitemap and robots scans

---
 libs/core-scanner/src/pages/primary.spec.ts | 2 +-
 libs/core-scanner/src/pages/robots-txt.ts   | 6 +++---
 libs/core-scanner/src/pages/sitemap-xml.ts  | 7 +++----
 libs/core-scanner/src/scans/url-scan.ts     | 6 +-----
 libs/core-scanner/src/util.ts               | 5 +++++
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/libs/core-scanner/src/pages/primary.spec.ts b/libs/core-scanner/src/pages/primary.spec.ts
index 071d08f5..1c608c7e 100644
--- a/libs/core-scanner/src/pages/primary.spec.ts
+++ b/libs/core-scanner/src/pages/primary.spec.ts
@@ -61,7 +61,7 @@ describe('primary scanner', () => {
           uswdsUsFlagInCss: 0,
           uswdsStringInCss: 20,
           uswdsPublicSansFont: 0,
-          uswdsSemanticVersion: 'v3.7.0',
+          uswdsSemanticVersion: 'v3.7.1',
           uswdsVersion: 100,
           uswdsCount: 197,
         },
diff --git a/libs/core-scanner/src/pages/robots-txt.ts b/libs/core-scanner/src/pages/robots-txt.ts
index e62a4aac..b6a92d33 100644
--- a/libs/core-scanner/src/pages/robots-txt.ts
+++ b/libs/core-scanner/src/pages/robots-txt.ts
@@ -6,6 +6,7 @@ import { RobotsTxtScan } from 'entities/scan-data.entity';
 import { RobotsTxtPageScans } from 'entities/scan-page.entity';
 
 import { getHttpsUrl, getMIMEType } from '../util';
+import { isLive } from '../util';
 
 export const createRobotsTxtScanner = (logger: Logger, input: CoreInputDto) => {
   const url = getHttpsUrl(input.url);
@@ -35,8 +36,7 @@ const buildRobotTxtResult = (
   robotsText: string,
 ): RobotsTxtScan => {
   const robotsUrl = new URL(robotsResponse.url());
-  const robotsStatus = robotsResponse.status();
-  const robotsLive = robotsStatus / 100 === 2;
+  const robotsLive = isLive(robotsResponse);
   const robotsTxtDetected = robotsUrl.pathname === '/robots.txt' && robotsLive;
 
   return {
@@ -45,7 +45,7 @@ const buildRobotTxtResult = (
     robotsTxtTargetUrlRedirects:
       robotsResponse.request().redirectChain().length > 0,
     robotsTxtFinalUrlMimeType: getMIMEType(robotsResponse),
-    robotsTxtStatusCode: robotsStatus,
+    robotsTxtStatusCode: robotsResponse.status(),
 
     robotsTxtDetected,
     ...(robotsTxtDetected
diff --git a/libs/core-scanner/src/pages/sitemap-xml.ts b/libs/core-scanner/src/pages/sitemap-xml.ts
index 30944d29..e4583d49 100644
--- a/libs/core-scanner/src/pages/sitemap-xml.ts
+++ b/libs/core-scanner/src/pages/sitemap-xml.ts
@@ -5,7 +5,7 @@ import { CoreInputDto } from '@app/core-scanner/core.input.dto';
 import { SitemapXmlScan } from 'entities/scan-data.entity';
 import { SitemapXmlPageScans } from 'entities/scan-page.entity';
 
-import { getHttpsUrl, getMIMEType } from '../util';
+import { getHttpsUrl, getMIMEType, isLive } from '../util';
 
 export const createSitemapXmlScanner = (
   logger: Logger,
@@ -39,8 +39,7 @@ const buildSitemapResult = async (
   sitemapPage: Page,
 ): Promise<SitemapXmlScan> => {
   const sitemapUrl = new URL(sitemapResponse.url());
-  const sitemapStatus = sitemapResponse.status();
-  const sitemapLive = sitemapStatus / 100 === 2;
+  const sitemapLive = isLive(sitemapResponse);
 
   const sitemapXmlDetected =
     sitemapUrl.pathname === '/sitemap.xml' && sitemapLive;
@@ -51,7 +50,7 @@ const buildSitemapResult = async (
     sitemapTargetUrlRedirects:
       sitemapResponse.request().redirectChain().length > 0,
     sitemapXmlFinalUrlMimeType: getMIMEType(sitemapResponse),
-    sitemapXmlStatusCode: sitemapStatus,
+    sitemapXmlStatusCode: sitemapResponse.status(),
 
     sitemapXmlDetected,
     ...(sitemapXmlDetected
diff --git a/libs/core-scanner/src/scans/url-scan.ts b/libs/core-scanner/src/scans/url-scan.ts
index 61d8c373..ec12a982 100644
--- a/libs/core-scanner/src/scans/url-scan.ts
+++ b/libs/core-scanner/src/scans/url-scan.ts
@@ -3,6 +3,7 @@ import { Page, HTTPRequest, HTTPResponse } from 'puppeteer';
 
 import { CoreInputDto } from '@app/core-scanner/core.input.dto';
 import { UrlScan } from 'entities/scan-data.entity';
+import { isLive } from '../util';
 
 import {
   getBaseDomain,
@@ -43,8 +44,3 @@ const getFinalUrl = (page: Page) => {
   const finalUrl = page.url();
   return finalUrl;
 };
-
-const isLive = (res: HTTPResponse) => {
-  const http200FamilyCodes = [200, 201, 202, 203, 204, 205, 206];
-  return _.includes(http200FamilyCodes, res.status());
-};
diff --git a/libs/core-scanner/src/util.ts b/libs/core-scanner/src/util.ts
index 20801ba1..421a754e 100644
--- a/libs/core-scanner/src/util.ts
+++ b/libs/core-scanner/src/util.ts
@@ -62,3 +62,8 @@ export const getTopLevelDomain = (url: string): string | null => {
 
   return tldMatch ? tldMatch[1] : null;
 };
+
+export const isLive = (res: HTTPResponse): boolean => {
+  const http200FamilyCodes = [200, 201, 202, 203, 204, 205, 206];
+  return _.includes(http200FamilyCodes, res.status());
+};