Feature: High level aggregated CSV download for sitemaps. (#814)

* Add report.CSV generator to the cli out directory. * Add report.csv to report downloader. * Move extract cookies to common. * Move reshape cookies to common. * Fix csv generator. * Refactor code move data mock to test utils. * Fix type errors. * Fix analytics cookies with issues count. * Add retry times for flaky tests. * Remove console.log. Add ts ignore reasons. Use type keyword. * Make return more readable. * Fix flaky tests.
GoogleChromeLabs · Aug 28, 2024 · d42100e · d42100e
1 parent c4cab25
commit d42100e
Show file tree

Hide file tree

Showing 29 changed files with 1,193 additions and 1,400 deletions.
diff --git a/packages/cli-dashboard/src/app.tsx b/packages/cli-dashboard/src/app.tsx
@@ -17,19 +17,16 @@
  * External dependencies
  */
 import React, { useEffect, useState } from 'react';
-import type {
-  CompleteJson,
-  CookieFrameStorageType,
-  LibraryData,
-  TechnologyData,
-} from '@google-psat/common';
-import { I18n } from '@google-psat/i18n';
 import {
+  type CompleteJson,
+  type CookieFrameStorageType,
+  type LibraryData,
+  type TechnologyData,
   extractReportData,
   extractCookies,
-  SiteMapReport,
-  SiteReport,
-} from '@google-psat/report';
+} from '@google-psat/common';
+import { I18n } from '@google-psat/i18n';
+import { SiteMapReport, SiteReport } from '@google-psat/report';
 
 /**
  * Internal dependencies

diff --git a/packages/cli/src/utils/saveReports.ts b/packages/cli/src/utils/saveReports.ts
@@ -17,7 +17,10 @@
 /**
  * External dependencies.
  */
-import { type CompleteJson } from '@google-psat/common';
+import {
+  generateRootSummaryDataCSV,
+  type CompleteJson,
+} from '@google-psat/common';
 import { ensureFile, writeFile } from 'fs-extra';
 /**
  * Internal dependencies.
@@ -54,6 +57,10 @@ const saveReports = async (
       'report.html',
       sitemapUrl
     );
+
+    const rootSummaryData = generateRootSummaryDataCSV(result);
+    await ensureFile(path.join(outDir, 'report.csv'));
+    await writeFile(path.join(outDir, 'report.csv'), rootSummaryData);
     // Sitemap report
     await Promise.all(
       result.map(async (siteReport) => {

diff --git a/packages/common/src/cookies.types.ts b/packages/common/src/cookies.types.ts
@@ -153,10 +153,11 @@ export interface TabCookies {
   [key: string]: CookieTableData;
 }
 
+export type FrameType = 'outermost_frame' | 'fenced_frame' | 'sub_frame';
 export interface TabFrames {
   [key: string]: {
     frameIds: string[];
-    frameType?: 'outermost_frame' | 'fenced_frame' | 'sub_frame';
+    frameType?: FrameType;
   };
 }
 

diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts
@@ -44,6 +44,9 @@ export { default as getCurrentDateAndTime } from './utils/getCurrentDateAndTime'
 export { default as resolveWithTimeout } from './utils/resolveWithTimeout';
 export { default as deriveBlockingStatus } from './utils/deriveBlockingStatus';
 export { default as getLegendDescription } from './utils/getLegendDescription';
+export { default as extractCookies } from './utils/extractCookies';
+export { default as extractReportData } from './utils/extractReportData';
+export { default as reshapeCookies } from './utils/reshapeCookies';
 export * from './worker/enums';
 export * from './utils/generateReports';
 export * from './cookies.types';

diff --git a/.../utils/generateReports/tests/data.mock.ts → packages/common/src/test-utils/data.mock.ts b/.../utils/generateReports/tests/data.mock.ts → packages/common/src/test-utils/data.mock.ts
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { CompleteJson } from '../../../cookies.types';
+import { CompleteJson } from '../cookies.types';
 
 export const mockData1: CompleteJson = {
   pageUrl: 'https://edition.cnn.com/',
@@ -63,7 +63,7 @@ export const mockData1: CompleteJson = {
       version: '4.3.2',
       icon: 'styled-components.svg',
       website: 'https://styled-components.com',
-      cpe: null,
+      cpe: '',
       categories: [
         {
           id: 12,
@@ -80,7 +80,6 @@ export const mockData1: CompleteJson = {
   ],
   cookieData: {
     'https://edition.cnn.com': {
-      cookiesCount: 2,
       frameCookies: {
         'aniC:.aniview.com:/': {
           parsedCookie: {
@@ -89,9 +88,10 @@ export const mockData1: CompleteJson = {
             domain: '.aniview.com',
             path: '/',
             expires: '2023-12-18T04:40:15.773Z',
-            httpOnly: true,
+            httponly: true,
             secure: true,
-            sameSite: 'None',
+            samesite: 'None',
+            partitionKey: '',
           },
           analytics: {
             platform: 'Unknown',
@@ -110,9 +110,10 @@ export const mockData1: CompleteJson = {
             domain: '.aniview.com',
             path: '/',
             expires: '2023-12-18T04:25:51.777Z',
-            httpOnly: false,
+            httponly: false,
             secure: true,
-            sameSite: 'None',
+            samesite: 'None',
+            partitionKey: '',
           },
           analytics: {
             platform: 'Unknown',
@@ -127,7 +128,6 @@ export const mockData1: CompleteJson = {
       },
     },
     'https://s.amazon-adsystem.com': {
-      cookiesCount: 1,
       frameCookies: {
         'ad-id:.amazon-adsystem.com:/': {
           parsedCookie: {
@@ -136,9 +136,10 @@ export const mockData1: CompleteJson = {
             domain: '.amazon-adsystem.com',
             path: '/',
             expires: '2023-12-18T09:22:30.184Z',
-            httpOnly: true,
+            httponly: true,
+            partitionKey: '',
             secure: true,
-            sameSite: 'None',
+            samesite: 'None',
           },
           analytics: {
             platform: 'Amazon',
@@ -154,14 +155,14 @@ export const mockData1: CompleteJson = {
       },
     },
   },
+  libraryMatches: {},
 };
 
 export const mockData2: CompleteJson = {
   pageUrl: 'https://edition.cnn.com/',
   technologyData: [],
   cookieData: {
     'https://edition.cnn.com': {
-      cookiesCount: 2,
       frameCookies: {
         'aniC:.aniview.com:/': {
           parsedCookie: {
@@ -170,9 +171,10 @@ export const mockData2: CompleteJson = {
             domain: '.aniview.com',
             path: '/',
             expires: '2023-12-18T04:40:15.773Z',
-            httpOnly: true,
+            httponly: true,
+            partitionKey: '',
             secure: true,
-            sameSite: 'None',
+            samesite: 'None',
           },
           analytics: {
             platform: 'Unknown',
@@ -191,9 +193,10 @@ export const mockData2: CompleteJson = {
             domain: '.aniview.com',
             path: '/',
             expires: '2023-12-18T04:25:51.777Z',
-            httpOnly: false,
+            httponly: false,
+            partitionKey: '',
             secure: true,
-            sameSite: 'None',
+            samesite: 'None',
           },
           analytics: {
             platform: 'Unknown',
@@ -212,9 +215,10 @@ export const mockData2: CompleteJson = {
             domain: '.cnn.com',
             path: '/',
             expires: '2023-11-28T11:38:36.290Z',
-            httpOnly: false,
+            httponly: false,
+            partitionKey: '',
             secure: true,
-            sameSite: 'None',
+            samesite: 'None',
           },
           analytics: {
             platform: 'ShareThis',
@@ -230,7 +234,6 @@ export const mockData2: CompleteJson = {
       },
     },
     'https://s.amazon-adsystem.com': {
-      cookiesCount: 2,
       frameCookies: {
         'ad-id:.amazon-adsystem.com:/': {
           parsedCookie: {
@@ -239,9 +242,10 @@ export const mockData2: CompleteJson = {
             domain: '.amazon-adsystem.com',
             path: '/',
             expires: '2023-12-18T09:22:30.184Z',
-            httpOnly: true,
+            httponly: true,
+            partitionKey: '',
             secure: true,
-            sameSite: 'None',
+            samesite: 'None',
           },
           analytics: {
             platform: 'Amazon',
@@ -261,9 +265,10 @@ export const mockData2: CompleteJson = {
             domain: '.aniview.com',
             path: '/',
             expires: '2023-12-18T04:25:51.777Z',
-            httpOnly: false,
+            httponly: false,
+            partitionKey: '',
             secure: true,
-            sameSite: 'None',
+            samesite: 'None',
           },
           analytics: {
             platform: 'Unknown',
@@ -278,4 +283,163 @@ export const mockData2: CompleteJson = {
       },
     },
   },
+  libraryMatches: {},
 };
+export const tempSinglePageData: CompleteJson = {
+  pageUrl: 'https://edition.cnn.com/sitemaps/sitemap-section.xml',
+  libraryMatches: {},
+  cookieData: {
+    'https://edition.cnn.com': {
+      frameCookies: {
+        'countryCode:.cnn.com:/': {
+          parsedCookie: {
+            name: 'countryCode',
+            domain: '.cnn.com',
+            path: '/',
+            value: 'IN',
+            samesite: 'None',
+            expires: 'Session',
+            httponly: false,
+            secure: true,
+            partitionKey: '',
+          },
+          analytics: {
+            platform: 'Unknown',
+            category: 'Uncategorized',
+            GDPR: '',
+            description: '',
+          },
+          url: '',
+          isBlocked: false,
+          isFirstParty: true,
+        },
+      },
+    },
+  },
+  technologyData: [
+    {
+      slug: 'varnish',
+      name: 'Varnish',
+      description: 'Varnish is a reverse caching proxy.',
+      confidence: 100,
+      version: null,
+      icon: 'Varnish.svg',
+      website: 'https://www.varnish-cache.org',
+      cpe: 'cpe:2.3:a:varnish-software:varnish_cache:*:*:*:*:*:*:*:*',
+      categories: [
+        {
+          id: 23,
+          slug: 'caching',
+          name: 'Caching',
+        },
+      ],
+    },
+  ],
+};
+
+export const tempMultiPageData: CompleteJson[] = [
+  {
+    libraryMatches: {},
+    pageUrl: 'https://www.cnn.com/index.html',
+    technologyData: [
+      {
+        slug: 'varnish',
+        name: 'Varnish',
+        description: 'Varnish is a reverse caching proxy.',
+        confidence: 100,
+        version: null,
+        icon: 'Varnish.svg',
+        website: 'https://www.varnish-cache.org',
+        cpe: 'cpe:2.3:a:varnish-software:varnish_cache:*:*:*:*:*:*:*:*',
+        categories: [
+          {
+            id: 23,
+            slug: 'caching',
+            name: 'Caching',
+          },
+        ],
+      },
+    ],
+    cookieData: {
+      'https://edition.cnn.com': {
+        frameCookies: {
+          'countryCode:.cnn.com:/': {
+            parsedCookie: {
+              name: 'countryCode',
+              domain: '.cnn.com',
+              path: '/',
+              value: 'IN',
+              samesite: 'None',
+              expires: 'Session',
+              httponly: false,
+              secure: true,
+              partitionKey: '',
+            },
+            analytics: {
+              platform: 'Unknown',
+              category: 'Uncategorized',
+              GDPR: '',
+              description: '',
+            },
+            url: '',
+            isBlocked: false,
+            blockedReasons: ['ThirdPartyPhaseout'],
+            isFirstParty: true,
+          },
+        },
+      },
+    },
+  },
+  {
+    libraryMatches: {},
+    pageUrl: 'https://edition.cnn.com/index.html',
+    technologyData: [
+      {
+        slug: 'varnish',
+        name: 'Varnish',
+        description: 'Varnish is a reverse caching proxy.',
+        confidence: 100,
+        version: null,
+        icon: 'Varnish.svg',
+        website: 'https://www.varnish-cache.org',
+        cpe: 'cpe:2.3:a:varnish-software:varnish_cache:*:*:*:*:*:*:*:*',
+        categories: [
+          {
+            id: 23,
+            slug: 'caching',
+            name: 'Caching',
+          },
+        ],
+      },
+    ],
+    cookieData: {
+      'https://edition.cnn.com': {
+        frameCookies: {
+          'countryCode:.cnn.com:/': {
+            parsedCookie: {
+              name: 'countryCode',
+              domain: '.cnn.com',
+              path: '/',
+              value: 'IN',
+              samesite: 'None',
+              expires: 'Session',
+              httponly: false,
+              secure: true,
+              partitionKey: '',
+            },
+            analytics: {
+              platform: 'Unknown',
+              category: 'Uncategorized',
+              GDPR: '',
+              description: '',
+            },
+            url: '',
+            isBlocked: false,
+            blockedReasons: ['ThirdPartyPhaseout'],
+            isFirstParty: true,
+          },
+        },
+      },
+    },
+  },
+];