Prerelease linting

Author: eaton

src/cli/commands/crawl.ts

@@ -1,11 +1,6 @@
 import { Flags, Args } from '@oclif/core';
 import { LogLevel } from 'crawlee';
-import {
-  Spidergram,
-  Spider,
-  EntityQuery,
-  UniqueUrl,
-} from '../../index.js';
+import { Spidergram, Spider, EntityQuery, UniqueUrl } from '../../index.js';
 import { QueryFragments } from '../../model/queries/query-fragments.js';
 import { CLI, OutputLevel, SgCommand } from '../index.js';
 import { filterUrl } from '../../tools/urls/filter-url.js';
@@ -44,10 +39,12 @@ export default class Crawl extends SgCommand {
     const sg = await Spidergram.load();
     const { argv: urls, flags } = await this.parse(Crawl);
 
-    const crawlTargets = [...sg.config.spider?.seed ?? [], ...urls ?? []];
+    const crawlTargets = [...(sg.config.spider?.seed ?? []), ...(urls ?? [])];
 
     if (crawlTargets.length == 0) {
-      this.error('Crawl URLs must be provided via the command line, or via the configuration file.');
+      this.error(
+        'Crawl URLs must be provided via the command line, or via the configuration file.',
+      );
     }
 
     if (flags.verbose) {

src/cli/commands/go.ts

@@ -33,10 +33,15 @@ export default class Go extends SgCommand {
     const sg = await Spidergram.load();
     const { argv: urls, flags } = await this.parse(Go);
 
-    const crawlTargets: string[] = [...sg.config.spider?.seed ?? [], ...urls ?? []];
+    const crawlTargets: string[] = [
+      ...(sg.config.spider?.seed ?? []),
+      ...(urls ?? []),
+    ];
 
     if (crawlTargets.length == 0) {
-      this.error('Crawl URLs must be provided via the command line, or via the configuration file.');
+      this.error(
+        'Crawl URLs must be provided via the command line, or via the configuration file.',
+      );
     }
 
     if (flags.erase) {
@@ -84,8 +89,14 @@ export default class Go extends SgCommand {
 
       this.ux.action.start('Crawl reports');
       await r.run();
-      this.log(`Saved ${(r.status.files.length === 1) ? '1 report' : r.status.files.length + ' reports'}`);
-      r.status.files.map(file => this.log(`  ${file}`))
+      this.log(
+        `Saved ${
+          r.status.files.length === 1
+            ? '1 report'
+            : r.status.files.length + ' reports'
+        }`,
+      );
+      r.status.files.map(file => this.log(`  ${file}`));
     }
 
     // We should perform some kin of wrapup step here.

src/cli/commands/init.ts

@@ -20,7 +20,7 @@ export default class Initialize extends SgCommand {
       char: 'f',
       summary: 'Configuration file format',
       options: ['json', 'yaml', 'json5'],
-      default: 'json5'
+      default: 'json5',
     }),
     dbaddress: Flags.string({
       char: 'a',
@@ -60,8 +60,8 @@ export default class Initialize extends SgCommand {
     populate: Flags.boolean({
       char: 'p',
       summary: 'Populate the config file with common defaults',
-      default: true
-    })
+      default: true,
+    }),
   };
 
   async run() {
@@ -71,22 +71,25 @@ export default class Initialize extends SgCommand {
 
     if (sg.configFile) {
       // Is there already a config file? Prompt to ensure the user really wants to generate this.
-      await CLI.confirm(`A configuration file already exists. Create one anyways?`)
-        .then(confirmed => { if (!confirmed) this.exit(0); })
+      await CLI.confirm(
+        `A configuration file already exists. Create one anyways?`,
+      ).then(confirmed => {
+        if (!confirmed) this.exit(0);
+      });
     }
 
     const settings: SpidergramConfig = {
       configVersion: this.config.version,
       storageDirectory: flags.storage,
-      outputDirectory: flags.output,      
+      outputDirectory: flags.output,
       arango: {
         databaseName: flags.dbname,
         url: flags.dbaddress,
         auth: {
           username: flags.dbuser,
-          password: flags.dbpass
-        }
-      }
+          password: flags.dbpass,
+        },
+      },
     };
 
     if (flags.populate) {
@@ -116,11 +119,11 @@ export default class Initialize extends SgCommand {
         configData = JSON.stringify(settings, undefined, 4);
         break;
 
-      case 'json5': 
+      case 'json5':
         configData = stringify(settings, undefined, 4);
         break;
-        
-      case 'yaml': 
+
+      case 'yaml':
         configData = dump(settings);
         break;
     }
@@ -129,42 +132,47 @@ export default class Initialize extends SgCommand {
       this.error("The configuration file couln't be generated.");
     }
 
-    await writeFile(filePath, configData)
-      .then(() => this.log('Config file generated!'));
+    await writeFile(filePath, configData).then(() =>
+      this.log('Config file generated!'),
+    );
   }
 }
 
 type ArangoStatus = {
-  error?: boolean,
-  server: boolean,
-  auth: boolean,
-  db: boolean
-}
-
-export async function testConnection(name: string, host: string, user: string, pass: string): Promise<ArangoStatus> {
+  error?: boolean;
+  server: boolean;
+  auth: boolean;
+  db: boolean;
+};
+
+export async function testConnection(
+  name: string,
+  host: string,
+  user: string,
+  pass: string,
+): Promise<ArangoStatus> {
   const results = {
     error: true,
     server: true,
     auth: true,
-    db: true
+    db: true,
   };
 
   const connection = await ArangoStore.open(undefined, {
     url: host,
     auth: {
       username: user,
-      password: pass
-    }
-  })
-  .catch(() => results);
+      password: pass,
+    },
+  }).catch(() => results);
 
   if ('error' in connection) {
     return Promise.resolve(connection);
   } else {
     return Promise.resolve({
       server: true,
       auth: true,
-      db: true
-    })
+      db: true,
+    });
   }
 }

src/cli/commands/report.ts

@@ -5,7 +5,6 @@ import _ from 'lodash';
 import { queryFilterFlag } from '../shared/index.js';
 import { buildFilter } from '../shared/flag-query-tools.js';
 
-
 export default class DoReport extends SgCommand {
   static summary = 'Build and save a crawl report';
 
@@ -34,12 +33,12 @@ export default class DoReport extends SgCommand {
     output: Flags.string({
       char: 'o',
       summary: 'Output file type',
-      options: ['csv', 'tsv', 'json', 'json5', 'xlsx', 'debug']
+      options: ['csv', 'tsv', 'json', 'json5', 'xlsx', 'debug'],
     }),
     setting: Flags.string({
       char: 's',
       summary: 'Add custom report setting',
-      multiple: true
+      multiple: true,
     }),
   };
 
@@ -69,7 +68,10 @@ export default class DoReport extends SgCommand {
       } else {
         const data: Record<string, unknown>[] = Object.entries(reports).map(
           ([name, report]) => {
-            const r = report instanceof ReportRunner ? report : new ReportRunner(report);
+            const r =
+              report instanceof ReportRunner
+                ? report
+                : new ReportRunner(report);
             return {
               report: name,
               category: r.config.group,
@@ -93,7 +95,9 @@ export default class DoReport extends SgCommand {
 
     const definition = sg.config.reports?.[args.report ?? ''];
     const report =
-      definition instanceof ReportRunner ? definition : new ReportRunner(definition);
+      definition instanceof ReportRunner
+        ? definition
+        : new ReportRunner(definition);
 
     if (flags.filter) {
       const filters: AqFilter[] = [];
@@ -110,11 +114,15 @@ export default class DoReport extends SgCommand {
     if (flags.name) report.config.name = flags.name;
     if (flags.path) report.config.settings.path = flags.path;
     for (const s of flags.setting ?? []) {
-      _.set(report.config.settings, s.split('=').shift() ?? '', s.split('=').pop() ?? true);
+      _.set(
+        report.config.settings,
+        s.split('=').shift() ?? '',
+        s.split('=').pop() ?? true,
+      );
     }
 
     if (flags.output === 'debug') {
-      this.ux.styledHeader('Report structure')
+      this.ux.styledHeader('Report structure');
       this.ux.styledJSON(report.config);
     } else {
       if (flags.output) report.config.settings.type = flags.output;
@@ -126,11 +134,17 @@ export default class DoReport extends SgCommand {
 
       this.ux.action.start('Running report');
       await report.run();
-      this.log(`Saved ${(report.status.files.length === 1) ? '1 report' : report.status.files.length + ' reports'}`);
-      report.status.files.map(file => this.log(`  ${file}`))
+      this.log(
+        `Saved ${
+          report.status.files.length === 1
+            ? '1 report'
+            : report.status.files.length + ' reports'
+        }`,
+      );
+      report.status.files.map(file => this.log(`  ${file}`));
 
       if (report.status.lastError) {
-        this.log('At least one error was encountered during processing:')
+        this.log('At least one error was encountered during processing:');
         this.log(report.status.lastError);
       }
     }

src/cli/shared/flags.ts

@@ -94,7 +94,7 @@ export const analysisFlags = {
     char: 'r',
     summary: 'Reprocess already-analyzed pages',
     allowNo: true,
-    default: false
+    default: false,
   }),
   site: Flags.boolean({
     char: 's',

src/config/defaults.ts

@@ -114,7 +114,7 @@ export const analyzePageDefaults: PageAnalysisOptions = {
   links: false,
   site: 'parsed.hostname',
   properties: {},
-  patterns: []
+  patterns: [],
 };
 
 export const spidergramDefaults: SpidergramConfig = {

src/config/global-normalizer.ts

@@ -1,7 +1,15 @@
 import { UrlMutators, ParsedUrl } from '@autogram/url-tools';
 import minimatch from 'minimatch';
 
-type urlStringProps = 'protocol' | 'subdomain' | 'domain' | 'host' | 'hostname' | 'pathname' | 'search' | 'hash';
+type urlStringProps =
+  | 'protocol'
+  | 'subdomain'
+  | 'domain'
+  | 'host'
+  | 'hostname'
+  | 'pathname'
+  | 'search'
+  | 'hash';
 
 export interface NormalizerOptions {
   /**
@@ -79,11 +87,13 @@ export function globalNormalizer(
   if (opts.forceLowercase) {
     if (opts.forceLowercase === true) {
       url.href = url.href.toLocaleLowerCase();
-    }
-    else {
-      const props = Array.isArray(opts.forceLowercase) ? opts.forceLowercase : [opts.forceLowercase];
+    } else {
+      const props = Array.isArray(opts.forceLowercase)
+        ? opts.forceLowercase
+        : [opts.forceLowercase];
       for (const prop of props) {
-        url[prop as urlStringProps] = url[prop as urlStringProps].toLocaleLowerCase();
+        url[prop as urlStringProps] =
+          url[prop as urlStringProps].toLocaleLowerCase();
       }
     }
   }

src/config/spidergram-config.ts

@@ -7,9 +7,7 @@ import {
   PageContentOptions,
   PageDataOptions,
 } from '../tools/html/index.js';
-import {
-  PageAnalysisOptions,
-} from '../tools/graph/analyze-page.js';
+import { PageAnalysisOptions } from '../tools/graph/analyze-page.js';
 import { TechAuditOptions } from '../tools/browser/index.js';
 import { Configuration as FileConfiguration } from 'typefs';
 import { Config as ArangoConfig } from 'arangojs/connection';
@@ -30,7 +28,6 @@ import { SpiderCli } from '../cli/shared/index.js';
  * get more precise contextual control.
  */
 export interface SpidergramConfig extends Record<string, unknown> {
-  
   /**
    * The version of Spidergram the configuration data was originally created for.
    * This allows Spidergram to warn you if your config is out of date.