Merge pull request #292 from GSA/issue-702

Move isLive to util and apply to sitemap and robots scans
GSA · Dec 15, 2023 · 5d114dd · 5d114dd
2 parents 9a8eba8 + 86dbee6
commit 5d114dd
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 13 deletions.
diff --git a/libs/core-scanner/src/pages/primary.spec.ts b/libs/core-scanner/src/pages/primary.spec.ts
@@ -61,7 +61,7 @@ describe('primary scanner', () => {
           uswdsUsFlagInCss: 0,
           uswdsStringInCss: 20,
           uswdsPublicSansFont: 0,
-          uswdsSemanticVersion: 'v3.7.0',
+          uswdsSemanticVersion: 'v3.7.1',
           uswdsVersion: 100,
           uswdsCount: 197,
         },

diff --git a/libs/core-scanner/src/pages/robots-txt.ts b/libs/core-scanner/src/pages/robots-txt.ts
@@ -6,6 +6,7 @@ import { RobotsTxtScan } from 'entities/scan-data.entity';
 import { RobotsTxtPageScans } from 'entities/scan-page.entity';
 
 import { getHttpsUrl, getMIMEType } from '../util';
+import { isLive } from '../util';
 
 export const createRobotsTxtScanner = (logger: Logger, input: CoreInputDto) => {
   const url = getHttpsUrl(input.url);
@@ -35,8 +36,7 @@ const buildRobotTxtResult = (
   robotsText: string,
 ): RobotsTxtScan => {
   const robotsUrl = new URL(robotsResponse.url());
-  const robotsStatus = robotsResponse.status();
-  const robotsLive = robotsStatus / 100 === 2;
+  const robotsLive = isLive(robotsResponse);
   const robotsTxtDetected = robotsUrl.pathname === '/robots.txt' && robotsLive;
 
   return {
@@ -45,7 +45,7 @@ const buildRobotTxtResult = (
     robotsTxtTargetUrlRedirects:
       robotsResponse.request().redirectChain().length > 0,
     robotsTxtFinalUrlMimeType: getMIMEType(robotsResponse),
-    robotsTxtStatusCode: robotsStatus,
+    robotsTxtStatusCode: robotsResponse.status(),
 
     robotsTxtDetected,
     ...(robotsTxtDetected

diff --git a/libs/core-scanner/src/pages/sitemap-xml.ts b/libs/core-scanner/src/pages/sitemap-xml.ts
@@ -5,7 +5,7 @@ import { CoreInputDto } from '@app/core-scanner/core.input.dto';
 import { SitemapXmlScan } from 'entities/scan-data.entity';
 import { SitemapXmlPageScans } from 'entities/scan-page.entity';
 
-import { getHttpsUrl, getMIMEType } from '../util';
+import { getHttpsUrl, getMIMEType, isLive } from '../util';
 
 export const createSitemapXmlScanner = (
   logger: Logger,
@@ -39,8 +39,7 @@ const buildSitemapResult = async (
   sitemapPage: Page,
 ): Promise<SitemapXmlScan> => {
   const sitemapUrl = new URL(sitemapResponse.url());
-  const sitemapStatus = sitemapResponse.status();
-  const sitemapLive = sitemapStatus / 100 === 2;
+  const sitemapLive = isLive(sitemapResponse);
 
   const sitemapXmlDetected =
     sitemapUrl.pathname === '/sitemap.xml' && sitemapLive;
@@ -51,7 +50,7 @@ const buildSitemapResult = async (
     sitemapTargetUrlRedirects:
       sitemapResponse.request().redirectChain().length > 0,
     sitemapXmlFinalUrlMimeType: getMIMEType(sitemapResponse),
-    sitemapXmlStatusCode: sitemapStatus,
+    sitemapXmlStatusCode: sitemapResponse.status(),
 
     sitemapXmlDetected,
     ...(sitemapXmlDetected

diff --git a/libs/core-scanner/src/scans/url-scan.ts b/libs/core-scanner/src/scans/url-scan.ts
@@ -3,6 +3,7 @@ import { Page, HTTPRequest, HTTPResponse } from 'puppeteer';
 
 import { CoreInputDto } from '@app/core-scanner/core.input.dto';
 import { UrlScan } from 'entities/scan-data.entity';
+import { isLive } from '../util';
 
 import {
   getBaseDomain,
@@ -43,8 +44,3 @@ const getFinalUrl = (page: Page) => {
   const finalUrl = page.url();
   return finalUrl;
 };
-
-const isLive = (res: HTTPResponse) => {
-  const http200FamilyCodes = [200, 201, 202, 203, 204, 205, 206];
-  return _.includes(http200FamilyCodes, res.status());
-};
diff --git a/libs/core-scanner/src/util.ts b/libs/core-scanner/src/util.ts
@@ -62,3 +62,8 @@ export const getTopLevelDomain = (url: string): string | null => {
 
   return tldMatch ? tldMatch[1] : null;
 };
+
+export const isLive = (res: HTTPResponse): boolean => {
+  const http200FamilyCodes = [200, 201, 202, 203, 204, 205, 206];
+  return _.includes(http200FamilyCodes, res.status());
+};