From b56af8ed252cda82da11eaafd65e96895160f84d Mon Sep 17 00:00:00 2001 From: Jeff Eaton Date: Fri, 28 Apr 2023 23:03:06 -0500 Subject: [PATCH] Prerelease linting --- src/cli/commands/crawl.ts | 13 +- src/cli/commands/go.ts | 19 ++- src/cli/commands/init.ts | 64 +++---- src/cli/commands/report.ts | 34 ++-- src/cli/shared/flags.ts | 2 +- src/config/defaults.ts | 2 +- src/config/global-normalizer.ts | 20 ++- src/config/spidergram-config.ts | 5 +- src/config/spidergram.ts | 94 ++++++++--- src/model/arango-store.ts | 15 +- src/model/entities/named-entity.ts | 10 +- src/model/entities/resource.ts | 2 +- src/model/entities/site.ts | 12 +- src/model/queries/query-fragments.ts | 33 ++-- src/model/queries/query-inheritance.ts | 44 +++-- src/model/queries/query.ts | 18 +- src/model/relationships/appears-on.ts | 5 +- src/reports/output-csv.ts | 51 +++--- src/reports/output-json.ts | 56 +++---- src/reports/output-xlsx.ts | 211 +++++++++++++++--------- src/reports/report-types.ts | 45 +++-- src/reports/report.ts | 39 +++-- src/spider/handlers/download-handler.ts | 3 +- src/spider/handlers/page-handler.ts | 6 +- src/tools/browser/axe-auditor.ts | 8 +- src/tools/file/spreadsheet.ts | 39 ++--- src/tools/graph/analyze-page.ts | 24 ++- src/tools/graph/filter-by-property.ts | 26 ++- src/tools/graph/get-resource-site.ts | 6 +- src/tools/graph/index.ts | 2 +- src/tools/graph/merge-site.ts | 19 ++- src/tools/html/find-patterns.ts | 64 ++++--- src/tools/html/get-page-content.ts | 10 +- src/tools/map-properties.ts | 45 ++--- src/tools/screenshot.ts | 24 +-- 35 files changed, 654 insertions(+), 416 deletions(-) diff --git a/src/cli/commands/crawl.ts b/src/cli/commands/crawl.ts index cfa5567..2e8804b 100644 --- a/src/cli/commands/crawl.ts +++ b/src/cli/commands/crawl.ts @@ -1,11 +1,6 @@ import { Flags, Args } from '@oclif/core'; import { LogLevel } from 'crawlee'; -import { - Spidergram, - Spider, - EntityQuery, - UniqueUrl, -} from '../../index.js'; +import { Spidergram, Spider, EntityQuery, UniqueUrl } from '../../index.js'; import { QueryFragments } from '../../model/queries/query-fragments.js'; import { CLI, OutputLevel, SgCommand } from '../index.js'; import { filterUrl } from '../../tools/urls/filter-url.js'; @@ -44,10 +39,12 @@ export default class Crawl extends SgCommand { const sg = await Spidergram.load(); const { argv: urls, flags } = await this.parse(Crawl); - const crawlTargets = [...sg.config.spider?.seed ?? [], ...urls ?? []]; + const crawlTargets = [...(sg.config.spider?.seed ?? []), ...(urls ?? [])]; if (crawlTargets.length == 0) { - this.error('Crawl URLs must be provided via the command line, or via the configuration file.'); + this.error( + 'Crawl URLs must be provided via the command line, or via the configuration file.', + ); } if (flags.verbose) { diff --git a/src/cli/commands/go.ts b/src/cli/commands/go.ts index c7c43c9..a21576e 100644 --- a/src/cli/commands/go.ts +++ b/src/cli/commands/go.ts @@ -33,10 +33,15 @@ export default class Go extends SgCommand { const sg = await Spidergram.load(); const { argv: urls, flags } = await this.parse(Go); - const crawlTargets: string[] = [...sg.config.spider?.seed ?? [], ...urls ?? []]; + const crawlTargets: string[] = [ + ...(sg.config.spider?.seed ?? []), + ...(urls ?? []), + ]; if (crawlTargets.length == 0) { - this.error('Crawl URLs must be provided via the command line, or via the configuration file.'); + this.error( + 'Crawl URLs must be provided via the command line, or via the configuration file.', + ); } if (flags.erase) { @@ -84,8 +89,14 @@ export default class Go extends SgCommand { this.ux.action.start('Crawl reports'); await r.run(); - this.log(`Saved ${(r.status.files.length === 1) ? '1 report' : r.status.files.length + ' reports'}`); - r.status.files.map(file => this.log(` ${file}`)) + this.log( + `Saved ${ + r.status.files.length === 1 + ? '1 report' + : r.status.files.length + ' reports' + }`, + ); + r.status.files.map(file => this.log(` ${file}`)); } // We should perform some kin of wrapup step here. diff --git a/src/cli/commands/init.ts b/src/cli/commands/init.ts index d80c980..271f3b8 100644 --- a/src/cli/commands/init.ts +++ b/src/cli/commands/init.ts @@ -20,7 +20,7 @@ export default class Initialize extends SgCommand { char: 'f', summary: 'Configuration file format', options: ['json', 'yaml', 'json5'], - default: 'json5' + default: 'json5', }), dbaddress: Flags.string({ char: 'a', @@ -60,8 +60,8 @@ export default class Initialize extends SgCommand { populate: Flags.boolean({ char: 'p', summary: 'Populate the config file with common defaults', - default: true - }) + default: true, + }), }; async run() { @@ -71,22 +71,25 @@ export default class Initialize extends SgCommand { if (sg.configFile) { // Is there already a config file? Prompt to ensure the user really wants to generate this. - await CLI.confirm(`A configuration file already exists. Create one anyways?`) - .then(confirmed => { if (!confirmed) this.exit(0); }) + await CLI.confirm( + `A configuration file already exists. Create one anyways?`, + ).then(confirmed => { + if (!confirmed) this.exit(0); + }); } const settings: SpidergramConfig = { configVersion: this.config.version, storageDirectory: flags.storage, - outputDirectory: flags.output, + outputDirectory: flags.output, arango: { databaseName: flags.dbname, url: flags.dbaddress, auth: { username: flags.dbuser, - password: flags.dbpass - } - } + password: flags.dbpass, + }, + }, }; if (flags.populate) { @@ -116,11 +119,11 @@ export default class Initialize extends SgCommand { configData = JSON.stringify(settings, undefined, 4); break; - case 'json5': + case 'json5': configData = stringify(settings, undefined, 4); break; - - case 'yaml': + + case 'yaml': configData = dump(settings); break; } @@ -129,34 +132,39 @@ export default class Initialize extends SgCommand { this.error("The configuration file couln't be generated."); } - await writeFile(filePath, configData) - .then(() => this.log('Config file generated!')); + await writeFile(filePath, configData).then(() => + this.log('Config file generated!'), + ); } } type ArangoStatus = { - error?: boolean, - server: boolean, - auth: boolean, - db: boolean -} - -export async function testConnection(name: string, host: string, user: string, pass: string): Promise { + error?: boolean; + server: boolean; + auth: boolean; + db: boolean; +}; + +export async function testConnection( + name: string, + host: string, + user: string, + pass: string, +): Promise { const results = { error: true, server: true, auth: true, - db: true + db: true, }; const connection = await ArangoStore.open(undefined, { url: host, auth: { username: user, - password: pass - } - }) - .catch(() => results); + password: pass, + }, + }).catch(() => results); if ('error' in connection) { return Promise.resolve(connection); @@ -164,7 +172,7 @@ export async function testConnection(name: string, host: string, user: string, p return Promise.resolve({ server: true, auth: true, - db: true - }) + db: true, + }); } } diff --git a/src/cli/commands/report.ts b/src/cli/commands/report.ts index b4f5c65..b2acbb8 100644 --- a/src/cli/commands/report.ts +++ b/src/cli/commands/report.ts @@ -5,7 +5,6 @@ import _ from 'lodash'; import { queryFilterFlag } from '../shared/index.js'; import { buildFilter } from '../shared/flag-query-tools.js'; - export default class DoReport extends SgCommand { static summary = 'Build and save a crawl report'; @@ -34,12 +33,12 @@ export default class DoReport extends SgCommand { output: Flags.string({ char: 'o', summary: 'Output file type', - options: ['csv', 'tsv', 'json', 'json5', 'xlsx', 'debug'] + options: ['csv', 'tsv', 'json', 'json5', 'xlsx', 'debug'], }), setting: Flags.string({ char: 's', summary: 'Add custom report setting', - multiple: true + multiple: true, }), }; @@ -69,7 +68,10 @@ export default class DoReport extends SgCommand { } else { const data: Record[] = Object.entries(reports).map( ([name, report]) => { - const r = report instanceof ReportRunner ? report : new ReportRunner(report); + const r = + report instanceof ReportRunner + ? report + : new ReportRunner(report); return { report: name, category: r.config.group, @@ -93,7 +95,9 @@ export default class DoReport extends SgCommand { const definition = sg.config.reports?.[args.report ?? '']; const report = - definition instanceof ReportRunner ? definition : new ReportRunner(definition); + definition instanceof ReportRunner + ? definition + : new ReportRunner(definition); if (flags.filter) { const filters: AqFilter[] = []; @@ -110,11 +114,15 @@ export default class DoReport extends SgCommand { if (flags.name) report.config.name = flags.name; if (flags.path) report.config.settings.path = flags.path; for (const s of flags.setting ?? []) { - _.set(report.config.settings, s.split('=').shift() ?? '', s.split('=').pop() ?? true); + _.set( + report.config.settings, + s.split('=').shift() ?? '', + s.split('=').pop() ?? true, + ); } if (flags.output === 'debug') { - this.ux.styledHeader('Report structure') + this.ux.styledHeader('Report structure'); this.ux.styledJSON(report.config); } else { if (flags.output) report.config.settings.type = flags.output; @@ -126,11 +134,17 @@ export default class DoReport extends SgCommand { this.ux.action.start('Running report'); await report.run(); - this.log(`Saved ${(report.status.files.length === 1) ? '1 report' : report.status.files.length + ' reports'}`); - report.status.files.map(file => this.log(` ${file}`)) + this.log( + `Saved ${ + report.status.files.length === 1 + ? '1 report' + : report.status.files.length + ' reports' + }`, + ); + report.status.files.map(file => this.log(` ${file}`)); if (report.status.lastError) { - this.log('At least one error was encountered during processing:') + this.log('At least one error was encountered during processing:'); this.log(report.status.lastError); } } diff --git a/src/cli/shared/flags.ts b/src/cli/shared/flags.ts index ac9cd7d..3e3f43b 100644 --- a/src/cli/shared/flags.ts +++ b/src/cli/shared/flags.ts @@ -94,7 +94,7 @@ export const analysisFlags = { char: 'r', summary: 'Reprocess already-analyzed pages', allowNo: true, - default: false + default: false, }), site: Flags.boolean({ char: 's', diff --git a/src/config/defaults.ts b/src/config/defaults.ts index 72e9615..a9adcf6 100644 --- a/src/config/defaults.ts +++ b/src/config/defaults.ts @@ -114,7 +114,7 @@ export const analyzePageDefaults: PageAnalysisOptions = { links: false, site: 'parsed.hostname', properties: {}, - patterns: [] + patterns: [], }; export const spidergramDefaults: SpidergramConfig = { diff --git a/src/config/global-normalizer.ts b/src/config/global-normalizer.ts index 9ffce92..ab7ae4a 100644 --- a/src/config/global-normalizer.ts +++ b/src/config/global-normalizer.ts @@ -1,7 +1,15 @@ import { UrlMutators, ParsedUrl } from '@autogram/url-tools'; import minimatch from 'minimatch'; -type urlStringProps = 'protocol' | 'subdomain' | 'domain' | 'host' | 'hostname' | 'pathname' | 'search' | 'hash'; +type urlStringProps = + | 'protocol' + | 'subdomain' + | 'domain' + | 'host' + | 'hostname' + | 'pathname' + | 'search' + | 'hash'; export interface NormalizerOptions { /** @@ -79,11 +87,13 @@ export function globalNormalizer( if (opts.forceLowercase) { if (opts.forceLowercase === true) { url.href = url.href.toLocaleLowerCase(); - } - else { - const props = Array.isArray(opts.forceLowercase) ? opts.forceLowercase : [opts.forceLowercase]; + } else { + const props = Array.isArray(opts.forceLowercase) + ? opts.forceLowercase + : [opts.forceLowercase]; for (const prop of props) { - url[prop as urlStringProps] = url[prop as urlStringProps].toLocaleLowerCase(); + url[prop as urlStringProps] = + url[prop as urlStringProps].toLocaleLowerCase(); } } } diff --git a/src/config/spidergram-config.ts b/src/config/spidergram-config.ts index d03e735..855ab5b 100644 --- a/src/config/spidergram-config.ts +++ b/src/config/spidergram-config.ts @@ -7,9 +7,7 @@ import { PageContentOptions, PageDataOptions, } from '../tools/html/index.js'; -import { - PageAnalysisOptions, -} from '../tools/graph/analyze-page.js'; +import { PageAnalysisOptions } from '../tools/graph/analyze-page.js'; import { TechAuditOptions } from '../tools/browser/index.js'; import { Configuration as FileConfiguration } from 'typefs'; import { Config as ArangoConfig } from 'arangojs/connection'; @@ -30,7 +28,6 @@ import { SpiderCli } from '../cli/shared/index.js'; * get more precise contextual control. */ export interface SpidergramConfig extends Record { - /** * The version of Spidergram the configuration data was originally created for. * This allows Spidergram to warn you if your config is out of date. diff --git a/src/config/spidergram.ts b/src/config/spidergram.ts index f4f87bf..79ade52 100644 --- a/src/config/spidergram.ts +++ b/src/config/spidergram.ts @@ -16,7 +16,14 @@ import is from '@sindresorhus/is'; import _ from 'lodash'; import * as defaults from './defaults.js'; -import { ArangoStore, Query, QueryFragments, QueryInput, ReportConfig, Resource } from '../index.js'; +import { + ArangoStore, + Query, + QueryFragments, + QueryInput, + ReportConfig, + Resource, +} from '../index.js'; import { globalNormalizer } from './global-normalizer.js'; import { SpidergramConfig } from './spidergram-config.js'; import { setTimeout } from 'timers/promises'; @@ -154,7 +161,7 @@ export class Spidergram { if (this._loadedConfig?.value.finalizer) { await this._loadedConfig?.value.finalizer(this); } - + this._initializing = false; this._needsInit = false; return Promise.resolve(this); @@ -363,22 +370,22 @@ export class Spidergram { group: 'builtin', description: 'Summary of pages, downloadable media, and errors', settings: { - type: 'xlsx' + type: 'xlsx', }, queries: { - 'Overview': 'summary', + Overview: 'summary', 'Crawled Pages': 'pages', - 'Downloads': 'media', - 'Errors': 'errors' - } - } - } + Downloads: 'media', + Errors: 'errors', + }, + }, + }; this._activeConfig.reports ??= {}; this._activeConfig.reports = { ...reports, - ...this._activeConfig.reports - } + ...this._activeConfig.reports, + }; } buildDefaultQueries() { @@ -390,7 +397,7 @@ export class Spidergram { .collect('content', 'mime') .collect('status', 'code') .sortBy('site', 'asc'), - + pages: new Query(QueryFragments.pages_linked) .category('builtin') .description('Successfully crawled HTML pages') @@ -401,32 +408,57 @@ export class Spidergram { .return('path', 'parsed.pathname') .return('title', 'data.title') .return('words', 'content.readability.words') - .return({ document: false, name: 'Inlinks', path: 'inlinks', function: 'length' }) - .return({ document: false, name: 'Outlinks', path: 'outlinks', function: 'length' }), - + .return({ + document: false, + name: 'Inlinks', + path: 'inlinks', + function: 'length', + }) + .return({ + document: false, + name: 'Outlinks', + path: 'outlinks', + function: 'length', + }), + media: new Query(QueryFragments.pages_linked) .category('builtin') .description('Successfully crawled non-HTML content') .filterBy('code', 200) - .filterBy({ path: 'mime', eq: 'text/html', negate: true}) + .filterBy({ path: 'mime', eq: 'text/html', negate: true }) .sortBy('url', 'asc') .return('site', 'parsed.hostname') .return('path', 'parsed.pathname') .return('type', 'mime') .return('size', 'size') - .return({ document: false, name: 'Inlinks', path: 'inlinks', function: 'length' }), + .return({ + document: false, + name: 'Inlinks', + path: 'inlinks', + function: 'length', + }), errors: new Query(QueryFragments.pages_linked) .category('builtin') .description('Errors encountered while crawling') .filterBy({ path: 'code', eq: 200, negate: true }) - .sortBy({ document: false, path: 'inlinks', function: 'length', direction: 'desc' }) + .sortBy({ + document: false, + path: 'inlinks', + function: 'length', + direction: 'desc', + }) .return('site', 'parsed.hostname') .return('path', 'parsed.pathname') .return('status', 'code') .return('message', 'message') - .return({ document: false, name: 'Inlinks', path: 'inlinks', function: 'length' }), - + .return({ + document: false, + name: 'Inlinks', + path: 'inlinks', + function: 'length', + }), + network: new Query({ collection: 'resources', subqueries: [ @@ -438,20 +470,30 @@ export class Spidergram { { path: '_id', join: 'lt._from' }, { document: 'lt', path: '_to', join: 'rw._from' }, { document: 'rw', path: '_to', join: 'target._id' }, - { path: 'parsed.hostname', eq: 'target.parsed.hostname', negate: true, value: 'dynamic' } + { + path: 'parsed.hostname', + eq: 'target.parsed.hostname', + negate: true, + value: 'dynamic', + }, ], aggregates: [ { name: 'source', path: 'parsed.hostname', function: 'collect' }, - { name: 'destination', document: 'target', path: 'parsed.hostname', function: 'collect' }, + { + name: 'destination', + document: 'target', + path: 'parsed.hostname', + function: 'collect', + }, ], - count: 'strength' - }) + count: 'strength', + }), }; this._activeConfig.queries ??= {}; this._activeConfig.queries = { ...queries, - ...this._activeConfig.queries - } + ...this._activeConfig.queries, + }; } } diff --git a/src/model/arango-store.ts b/src/model/arango-store.ts index 1df6740..24a66cf 100644 --- a/src/model/arango-store.ts +++ b/src/model/arango-store.ts @@ -310,19 +310,28 @@ export class ArangoStore { } type validDelimiter = '-' | '_' | '.'; -export function sanitizeDbName(input: string, delimiter: validDelimiter = NAME_SEPARATOR): string { +export function sanitizeDbName( + input: string, + delimiter: validDelimiter = NAME_SEPARATOR, +): string { return input .replaceAll(INVALID_COLLECTION_CHARS_REGEX, delimiter) .replaceAll(/-+/g, delimiter); } -export function sanitizeCollectionName(input: string, delimiter: validDelimiter = NAME_SEPARATOR): string { +export function sanitizeCollectionName( + input: string, + delimiter: validDelimiter = NAME_SEPARATOR, +): string { return input .replaceAll(INVALID_COLLECTION_CHARS_REGEX, delimiter) .replaceAll(/-+/g, delimiter); } -export function sanitizeKey(input: string, delimiter: validDelimiter = NAME_SEPARATOR): string { +export function sanitizeKey( + input: string, + delimiter: validDelimiter = NAME_SEPARATOR, +): string { return input .replaceAll(INVALID_KEY_CHARS_REGEX, delimiter) .replaceAll(/-+/g, delimiter); diff --git a/src/model/entities/named-entity.ts b/src/model/entities/named-entity.ts index 036482e..523e82a 100644 --- a/src/model/entities/named-entity.ts +++ b/src/model/entities/named-entity.ts @@ -1,10 +1,8 @@ -import { - Entity, - EntityConstructorOptions, -} from './entity.js'; +import { Entity, EntityConstructorOptions } from './entity.js'; import { sanitizeKey } from '../index.js'; -export interface NamedEntityConstructorOptions extends EntityConstructorOptions { +export interface NamedEntityConstructorOptions + extends EntityConstructorOptions { key?: string; name?: string; description?: string; @@ -21,7 +19,7 @@ export abstract class NamedEntity extends Entity { constructor(data: NamedEntityConstructorOptions = {}) { const { key, name, description, ...dataForSuper } = data; super(dataForSuper); - + // We should test for missing key/name, but things get squirrelly during // serialization (IE, the incoming value is actaully '_key') diff --git a/src/model/entities/resource.ts b/src/model/entities/resource.ts index e5ce9ca..5294012 100644 --- a/src/model/entities/resource.ts +++ b/src/model/entities/resource.ts @@ -99,7 +99,7 @@ export class Resource extends Entity { if (site) { this.site = Entity.idFromReference(site); - } + } this.message = message ?? ''; this.headers = headers ?? {}; diff --git a/src/model/entities/site.ts b/src/model/entities/site.ts index 492bcb5..52765ff 100644 --- a/src/model/entities/site.ts +++ b/src/model/entities/site.ts @@ -1,8 +1,4 @@ -import { - Entity, - Expose, - Transform -} from './entity.js'; +import { Entity, Expose, Transform } from './entity.js'; import { NamedEntity, NamedEntityConstructorOptions } from './named-entity.js'; export interface SiteConstructorOptions extends NamedEntityConstructorOptions { @@ -12,7 +8,7 @@ export interface SiteConstructorOptions extends NamedEntityConstructorOptions { export class Site extends NamedEntity { readonly _collection = 'sites'; - + /** * A list of URLs that can be used to access this site */ @@ -37,9 +33,7 @@ export class Site extends NamedEntity { const { urls, ...dataForSuper } = data; super(dataForSuper); - this.urls = new Set( - [...(urls ?? []).map(u => u.toString())] - ); + this.urls = new Set([...(urls ?? []).map(u => u.toString())]); } } diff --git a/src/model/queries/query-fragments.ts b/src/model/queries/query-fragments.ts index f913345..f61c233 100644 --- a/src/model/queries/query-fragments.ts +++ b/src/model/queries/query-fragments.ts @@ -11,14 +11,14 @@ export class QueryFragments { pages_crawled: this.pages_crawled, pages_linked: this.pages_linked, urls_uncrawled: this.urls_uncrawled, - urls_redirected: this.urls_redirected - } + urls_redirected: this.urls_redirected, + }; } /** * Returns a collection of Resources, each with the first URL request record that - * led to the resource. - * + * led to the resource. + * * @example Unfiltered return structure: * ``` * [{ @@ -39,7 +39,7 @@ export class QueryFragments { static pages_crawled: AqQuery = { metadata: { category: 'partial', - description: "Crawled resources with URL and request data", + description: 'Crawled resources with URL and request data', }, collection: 'resources', document: 'resource', @@ -55,13 +55,13 @@ export class QueryFragments { ], filters: [ { document: 'request', path: '_to', join: 'resource._id' }, - { document: 'request', path: '_from', join: 'url._id' } + { document: 'request', path: '_from', join: 'url._id' }, ], }; /** * Returns a collection of Resources with URLs for all inbound and outbound links. - * + * * @example Unfiltered return structure: * ``` * [{ @@ -81,7 +81,7 @@ export class QueryFragments { static pages_linked: AqQuery = { metadata: { category: 'partial', - description: "Crawled resources with inlinks and outlinks", + description: 'Crawled resources with inlinks and outlinks', }, collection: 'resources', document: 'resource', @@ -105,7 +105,7 @@ export class QueryFragments { filters: [{ path: '_id', join: 'lt._from' }], }, ], - } + }, ], return: [{ path: 'url', document: 'source' }], }, @@ -122,7 +122,7 @@ export class QueryFragments { collection: 'unique_urls', document: 'target', filters: [{ path: '_id', join: 'lt._to' }], - } + }, ], return: [{ path: 'url', document: 'target' }], }, @@ -140,7 +140,7 @@ export class QueryFragments { static urls_uncrawled: AqQuery = { metadata: { category: 'partial', - description: "URLs found but not yet visited", + description: 'URLs found but not yet visited', }, collection: 'unique_urls', document: 'url', @@ -194,7 +194,7 @@ export class QueryFragments { static urls_redirected: AqQuery = { metadata: { category: 'partial', - description: "Redirected URLs and received page data", + description: 'Redirected URLs and received page data', }, collection: 'unique_urls', document: 'requested', @@ -211,9 +211,14 @@ export class QueryFragments { filters: [ { document: 'request', path: '_from', join: 'requested._id' }, { document: 'request', path: '_to', join: 'received._id' }, - { document: 'requested', path: 'parsed.href', eq: 'received.parsed.href', value: 'dynamic', negate: true }, + { + document: 'requested', + path: 'parsed.href', + eq: 'received.parsed.href', + value: 'dynamic', + negate: true, + }, { document: 'request', path: 'redirects', function: 'count', gt: 1 }, ], }; } - diff --git a/src/model/queries/query-inheritance.ts b/src/model/queries/query-inheritance.ts index 8802420..3ba6796 100644 --- a/src/model/queries/query-inheritance.ts +++ b/src/model/queries/query-inheritance.ts @@ -1,20 +1,32 @@ -import { AqQuery, isAqAggregate, isAqFilter, isAqProperty, isAqQuery, isAqSort } from "aql-builder"; -import { GeneratedAqlQuery, aql, isGeneratedAqlQuery, literal } from "arangojs/aql.js"; -import { Query } from "../index.js"; -import { Spidergram } from "../../config/index.js"; -import { JsonPrimitive } from "@salesforce/ts-types"; +import { + AqQuery, + isAqAggregate, + isAqFilter, + isAqProperty, + isAqQuery, + isAqSort, +} from 'aql-builder'; +import { + GeneratedAqlQuery, + aql, + isGeneratedAqlQuery, + literal, +} from 'arangojs/aql.js'; +import { Query } from '../index.js'; +import { Spidergram } from '../../config/index.js'; +import { JsonPrimitive } from '@salesforce/ts-types'; export type AqQueryFragment = Partial; -export type AqBindVars = Record +export type AqBindVars = Record; export type QueryInput = string | AqQuery | Query | GeneratedAqlQuery; /** * A query definition that can (optionally) include a foundational base query. - * + * * If the base query is a {@link Query} or an {@link AqQuery}, the properties of * the {@link ChildQuery} will be combined with those of the base query to * generate a new derivitive query. - * + * * If the base query is a {@link GeneratedAqlQuery}, the list of named bind variables * will be injected into the base query before execution. */ @@ -34,15 +46,14 @@ export type ChildQuery = AqQueryFragment & { bind?: AqBindVars; }; - export function isChildQuery(input: unknown): input is ChildQuery { - return (isAqQueryFragment(input) || isAqQuery(input)); + return isAqQueryFragment(input) || isAqQuery(input); } export function isAqQueryFragment(input: unknown): input is AqQueryFragment { if (input) { if (typeof input !== 'object') return false; - return ('parent' in input); + return 'parent' in input; } return false; } @@ -69,8 +80,9 @@ export async function buildQueryWithParents( } else if (input instanceof Query) { return Promise.resolve(new Query(input.spec)); } else if (isChildQuery(input)) { - return buildQueryWithParents(input.parent) - .then(q => q ? addToParentQuery(q, input) : undefined) + return buildQueryWithParents(input.parent).then(q => + q ? addToParentQuery(q, input) : undefined, + ); } else { return Promise.resolve(undefined); } @@ -78,10 +90,10 @@ export async function buildQueryWithParents( /** * Apply child-query modifications to a parent query. -*/ + */ export function addToParentQuery( base: Query | GeneratedAqlQuery, - mods?: ChildQuery + mods?: ChildQuery, ) { let aq: GeneratedAqlQuery | undefined; @@ -112,7 +124,7 @@ export function addToParentQuery( } else { aq = base; } - + if (aq && mods?.bind) { for (const [key, val] of Object.entries(mods.bind)) { aq.bindVars[key] = val; diff --git a/src/model/queries/query.ts b/src/model/queries/query.ts index 56b725f..ba11cff 100644 --- a/src/model/queries/query.ts +++ b/src/model/queries/query.ts @@ -1,4 +1,10 @@ -import { ChildQuery, Spidergram, addToParentQuery, buildQueryWithParents, isChildQuery } from '../../index.js'; +import { + ChildQuery, + Spidergram, + addToParentQuery, + buildQueryWithParents, + isChildQuery, +} from '../../index.js'; import { QueryOptions } from 'arangojs/database.js'; import { GeneratedAqlQuery, @@ -25,8 +31,9 @@ export class Query extends AqBuilder { } if (isChildQuery(input)) { - const modified = await(buildQueryWithParents(input)) - .then(q => q ? addToParentQuery(q, input as ChildQuery) : undefined); + const modified = await buildQueryWithParents(input).then(q => + q ? addToParentQuery(q, input as ChildQuery) : undefined, + ); if (modified) aq = modified; } else if (typeof input === 'string') { aq = aql`${literal(input)}`; @@ -43,7 +50,10 @@ export class Query extends AqBuilder { .then(db => db.query(aq, options).then(cursor => cursor.all())); } - constructor(input: string | ArangoCollection | AqStrict | AqQuery, document?: string) { + constructor( + input: string | ArangoCollection | AqStrict | AqQuery, + document?: string, + ) { // This avoids unpleasant situations where a base query spec is modified, // each time it's used, affecting all of the other queries based on it. if (isAqQuery(input)) { diff --git a/src/model/relationships/appears-on.ts b/src/model/relationships/appears-on.ts index 412d10e..1c31c65 100644 --- a/src/model/relationships/appears-on.ts +++ b/src/model/relationships/appears-on.ts @@ -4,10 +4,11 @@ import { Entity, Reference, Resource, - Pattern + Pattern, } from '../index.js'; -export interface AppearsOnConstructorOptions extends RelationshipConstructorOptions { +export interface AppearsOnConstructorOptions + extends RelationshipConstructorOptions { page?: Reference; pattern?: Reference; } diff --git a/src/reports/output-csv.ts b/src/reports/output-csv.ts index 592dee6..c03cae2 100644 --- a/src/reports/output-csv.ts +++ b/src/reports/output-csv.ts @@ -1,9 +1,8 @@ -import { BaseReportSettings, ReportConfig } from "./report-types.js"; +import { BaseReportSettings, ReportConfig } from './report-types.js'; import { write as writeCsv } from '@fast-csv/format'; import { ReportRunner, Spidergram } from '../index.js'; -import { JsonCollection, isJsonArray, isJsonMap } from "@salesforce/ts-types"; -import path from "path"; - +import { JsonCollection, isJsonArray, isJsonMap } from '@salesforce/ts-types'; +import path from 'path'; /** * Output options specific to Comma and Tab delimited files @@ -12,42 +11,45 @@ export type CsvReportSettings = BaseReportSettings & { /** * File format to generate. CSV and TSV are almost identical, differing only in * filename and the type of delimiter used to separate columns. - * + * * @defaultValue 'csv' */ - type: 'csv' | 'tsv', + type: 'csv' | 'tsv'; /** - * The delmiter used to separate individual columns. + * The delmiter used to separate individual columns. * * @defaultValue `,` when `csv` is selected, `\t` when `tsv` is selected */ - delimiter?: string, + delimiter?: string; /** - * The delmiter used to separate each record. + * The delmiter used to separate each record. * * @defaultValue `\n` */ - rowDelimiter?: string, - quote?: string | boolean, - escape?: string, - quoteColumns?: boolean, - quoteHeaders?: boolean, - headers?: boolean, - writeHeaders?: boolean, - includeEndRowDelimiter?: boolean, - writeBOM?: boolean, - alwaysWriteHeaders?: boolean + rowDelimiter?: string; + quote?: string | boolean; + escape?: string; + quoteColumns?: boolean; + quoteHeaders?: boolean; + headers?: boolean; + writeHeaders?: boolean; + includeEndRowDelimiter?: boolean; + writeBOM?: boolean; + alwaysWriteHeaders?: boolean; }; -export async function outputCsvReport(config: ReportConfig, runner: ReportRunner): Promise { +export async function outputCsvReport( + config: ReportConfig, + runner: ReportRunner, +): Promise { const sg = await Spidergram.load(); const datasets = config.data ?? {}; const settings = (config.settings ?? {}) as CsvReportSettings; - settings.delimiter ??= (settings.type === 'tsv' ? '\t' : ','); + settings.delimiter ??= settings.type === 'tsv' ? '\t' : ','; const outputPath = settings.path ?? ''; if (!(await sg.files('output').exists(outputPath))) { @@ -59,7 +61,10 @@ export async function outputCsvReport(config: ReportConfig, runner: ReportRunner if (isJsonArray(data) || isJsonMap(data)) { const rows = data as JsonCollection[]; - const curFilePath = path.join(outputPath ?? '', `${name}.${settings.type}`); + const curFilePath = path.join( + outputPath ?? '', + `${name}.${settings.type}`, + ); const stream = writeCsv(rows, settings); await sg.files('output').writeStream(curFilePath, stream); @@ -69,4 +74,4 @@ export async function outputCsvReport(config: ReportConfig, runner: ReportRunner } return Promise.resolve(); -} \ No newline at end of file +} diff --git a/src/reports/output-json.ts b/src/reports/output-json.ts index 0fba0f7..49c8fe2 100644 --- a/src/reports/output-json.ts +++ b/src/reports/output-json.ts @@ -1,7 +1,7 @@ -import { BaseReportSettings, ReportConfig } from "./report-types.js"; +import { BaseReportSettings, ReportConfig } from './report-types.js'; import { ReportRunner, Spidergram } from '../index.js'; -import JSON5 from 'json5' -import path from "path"; +import JSON5 from 'json5'; +import path from 'path'; /** * Report output settings for JSON and JSON5 file formats @@ -10,29 +10,32 @@ export type JsonReportSettings = BaseReportSettings & { /** * File format to generate. JSON5 files are structurally similar to JSON, but allow * unquoted string keys, trailing commas, and inline comments like Javascript code. - * + * * @defaultValue 'json' */ - type: 'json' | 'json5', + type: 'json' | 'json5'; /** * Format the JSON output file with linebreaks and indentation. Output files are * larger, but easier to read. - * + * * @defaultValue false */ - readable?: boolean, + readable?: boolean; /** * If the report contains multiple queries, combine all results into a single * JSON output file. - * + * * @defaultValue false */ - combine?: boolean, + combine?: boolean; }; -export async function outputJsonReport(config: ReportConfig, runner: ReportRunner): Promise { +export async function outputJsonReport( + config: ReportConfig, + runner: ReportRunner, +): Promise { const sg = await Spidergram.load(); const datasets = config.data ?? {}; @@ -43,41 +46,36 @@ export async function outputJsonReport(config: ReportConfig, runner: ReportRunne if (settings.combine) { const curFilePath = `${outputPath}.${settings.type}`; const b = Buffer.from( - settings.type === 'json' ? - JSON.stringify(datasets, undefined, settings.readable ? 2 : 0) : - JSON5.stringify(datasets, undefined, settings.readable ? 2 : 0) + settings.type === 'json' + ? JSON.stringify(datasets, undefined, settings.readable ? 2 : 0) + : JSON5.stringify(datasets, undefined, settings.readable ? 2 : 0), ); - await sg - .files('output') - .write(curFilePath, b); + await sg.files('output').write(curFilePath, b); runner.status.finished++; runner.status.files.push(curFilePath); - } else { if (!(await sg.files('output').exists(outputPath))) { await sg.files('output').createDirectory(outputPath); } - + for (const [name, data] of Object.entries(datasets)) { if (data.length === 0 && settings.includeEmptyResults === false) continue; - + const curFilePath = path.join(outputPath, `${name}.${settings.type}`); - + const b = Buffer.from( - settings.type === 'json' ? - JSON.stringify(data, undefined, settings.readable ? 2 : 0) : - JSON5.stringify(data, undefined, settings.readable ? 2 : 0) + settings.type === 'json' + ? JSON.stringify(data, undefined, settings.readable ? 2 : 0) + : JSON5.stringify(data, undefined, settings.readable ? 2 : 0), ); - - await sg - .files('output') - .write(curFilePath, b); - + + await sg.files('output').write(curFilePath, b); + runner.status.finished++; runner.status.files.push(curFilePath); } } return Promise.resolve(); -} \ No newline at end of file +} diff --git a/src/reports/output-xlsx.ts b/src/reports/output-xlsx.ts index d832300..1f8f767 100644 --- a/src/reports/output-xlsx.ts +++ b/src/reports/output-xlsx.ts @@ -1,61 +1,75 @@ -import { Spidergram } from "../config/index.js"; -import { FileTools } from "../tools/index.js"; -import { BaseReportSettings, ReportConfig } from "./report-types.js"; -import { Properties, ColInfo, CellObject, CellStyle, ExcelDataType, NumberFormat, RowInfo } from "xlsx-js-style"; +import { Spidergram } from '../config/index.js'; +import { FileTools } from '../tools/index.js'; +import { BaseReportSettings, ReportConfig } from './report-types.js'; +import { + Properties, + ColInfo, + CellObject, + CellStyle, + ExcelDataType, + NumberFormat, + RowInfo, +} from 'xlsx-js-style'; import xlspkg from 'xlsx-js-style'; const { utils } = xlspkg; -import { ReportRunner } from "./report.js"; -import { JsonCollection, JsonMap, JsonPrimitive, isJsonArray, isJsonMap } from "@salesforce/ts-types"; -import { DateTime } from "luxon"; -import is from "@sindresorhus/is"; -import _ from "lodash"; +import { ReportRunner } from './report.js'; +import { + JsonCollection, + JsonMap, + JsonPrimitive, + isJsonArray, + isJsonMap, +} from '@salesforce/ts-types'; +import { DateTime } from 'luxon'; +import is from '@sindresorhus/is'; +import _ from 'lodash'; /** * Output options specific to workbook-style spreadsheet files */ export type XlsReportSettings = BaseReportSettings & { - type: 'xlsx', + type: 'xlsx'; /** * Internal metadata about the spreadsheet, usually displayed by a program's * "document information" command. */ - metadata?: Properties + metadata?: Properties; /** * A dictionary of per-sheet configuration options. Each key corresponds to a key * in the report's `data` property; if it exists, the `default` entry will be used * as a fallback for sheets with no specific settings. */ - sheets?: Record + sheets?: Record; }; type SheetSettings = { /** * A human-friendly name for the sheet. */ - name?: string, + name?: string; /** * A human-friendly description of the data on the sheet. */ - description?: string, + description?: string; /** * An output template to use when generating the sheet. - * + * * - table: A traditional columns/rows data sheet, with styled header - * - cover: An array of strings to turned into + * - cover: An array of strings to turned into * - inspector: Key/Value pairs, optionally grouped under subheadings. */ - template?: 'table' | 'cover' | 'inspector' + template?: 'table' | 'cover' | 'inspector'; /** * Settings for individual columns. The settings in the 'default' entry will be * applied to all columns. */ - columns?: Record, + columns?: Record; /** * Default styling information for all cells in the sheet @@ -65,55 +79,55 @@ type SheetSettings = { /** * Additional style information for heading columns and rows. */ - headingStyle?: CellStyle, -} + headingStyle?: CellStyle; +}; type ColumnSettings = { /** * Override the text of the column's first row. */ - title?: string, + title?: string; /** * Add a hover/tooltip comment to the column's first row. */ - comment?: string, + comment?: string; /** * Hide the column from view; its data will still be present and usable in formulas. */ - hidden?: boolean, + hidden?: boolean; /** * Adjust the column to the width of its widest row. - * + * * @defaultValue true */ - autoFit?: boolean, + autoFit?: boolean; /** * Hard-code the width of the column. This measurement is in 'approxomate characters,' not pixels. */ - width?: number, + width?: number; /** * The max width for the column. This measurement is in 'approxomate characters,' not pixels. - * + * * @defaultValue 80 */ - maxWidth?: number, + maxWidth?: number; /** * The minimum width for the column. This measurement is in 'approxomate characters,' not pixels. - * + * * @defaultValue 5 */ - minWidth?: number, + minWidth?: number; /** * Wrap the contents of this column rather than truncating or overflowing. */ - wrap?: boolean + wrap?: boolean; /** * If the column is populated and wrapping is turned on, its Row's height should @@ -123,19 +137,19 @@ type ColumnSettings = { /** * Override data type auto-detection. Possible values: - * + * * - "b": boolean * - "n": number * - "s": string * - "d": date */ - type?: ExcelDataType, + type?: ExcelDataType; /** * If the data type is set to 'n' or 'd', this format will be used used by Excel * when displaying the data. it does not change the underlying cell value. */ - format?: NumberFormat, + format?: NumberFormat; /** * Attempt to parse and/or coerce the column's data before creating the sheet. @@ -143,9 +157,8 @@ type ColumnSettings = { * and 'parse' to 'true' means Spidergram will ATTEMPT to turn timestamps, ISO dates, * and more into "clean" dates. */ - parse?: boolean, -} - + parse?: boolean; +}; /** * Default fallback settings for a sheet. Incoming values are merged with these settings. @@ -157,17 +170,20 @@ const sheetDefaults: SheetSettings = { alignment: { vertical: 'top', horizontal: 'left', - } + }, }, columns: { default: { autoFit: true, - maxWidth: 80 - } - } -} + maxWidth: 80, + }, + }, +}; -export async function outputXlsxReport(config: ReportConfig, runner: ReportRunner): Promise { +export async function outputXlsxReport( + config: ReportConfig, + runner: ReportRunner, +): Promise { const sg = await Spidergram.load(); const datasets = config.data ?? {}; @@ -181,9 +197,12 @@ export async function outputXlsxReport(config: ReportConfig, runner: ReportRunne for (const [name, data] of Object.entries(datasets)) { if (data.length === 0 && !settings.includeEmptyResults) continue; - const sheetSettings = _.defaultsDeep(settings.sheets[name] || settings.sheets.default, sheetDefaults); + const sheetSettings = _.defaultsDeep( + settings.sheets[name] || settings.sheets.default, + sheetDefaults, + ); - switch (sheetSettings.template) { + switch (sheetSettings.template) { case 'table': buildTabularSheet(name, data, sheetSettings, rpt); break; @@ -199,7 +218,14 @@ export async function outputXlsxReport(config: ReportConfig, runner: ReportRunne await sg .files('output') - .write(curFilePath, rpt.toBuffer({ cellStyles: true, Props: settings.metadata, compression: true })) + .write( + curFilePath, + rpt.toBuffer({ + cellStyles: true, + Props: settings.metadata, + compression: true, + }), + ) .then(() => { runner.status.finished++; runner.status.files.push(curFilePath); @@ -208,7 +234,12 @@ export async function outputXlsxReport(config: ReportConfig, runner: ReportRunne return Promise.resolve(); } -function buildTabularSheet(name: string, data: JsonCollection, settings: SheetSettings, rpt: FileTools.Spreadsheet) { +function buildTabularSheet( + name: string, + data: JsonCollection, + settings: SheetSettings, + rpt: FileTools.Spreadsheet, +) { if (!isJsonArray(data) || !isJsonMap(data[0])) { // We can only work with arrays of maps here; in the future we might be able to // expand it to deal with other stuff. @@ -226,28 +257,32 @@ function buildTabularSheet(name: string, data: JsonCollection, settings: SheetSe settings.columns ??= {}; - if (sheet["!ref"]) { - const range = utils.decode_range(sheet["!ref"]); - const dense = sheet["!data"]; + if (sheet['!ref']) { + const range = utils.decode_range(sheet['!ref']); + const dense = sheet['!data']; const colSettings: ColumnSettings[] = []; for (const c of Object.keys(firstRow)) { colSettings.push({ ...settings.columns[c], - ...settings.columns['default'] + ...settings.columns['default'], }); } - for(let R = 0; R <= range.e.r; ++R) { + for (let R = 0; R <= range.e.r; ++R) { const ri: RowInfo = {}; - for(let C = 0; C <= range.e.c; ++C) { - const cell: CellObject = dense ? sheet["!data"]?.[R]?.[C] : sheet[utils.encode_cell({r:R, c:C})]; + for (let C = 0; C <= range.e.c; ++C) { + const cell: CellObject = dense + ? sheet['!data']?.[R]?.[C] + : sheet[utils.encode_cell({ r: R, c: C })]; const cs = colSettings[C]; - + cell.s = settings.style; if (R === 0) { - if (cs.comment) { cell.c = [{ t: cs.comment }] } + if (cs.comment) { + cell.c = [{ t: cs.comment }]; + } if (cs.title) cell.v = cs.title; if (settings.headingStyle) { cell.s = { ...cell.s, ...settings.headingStyle }; @@ -270,22 +305,27 @@ function buildTabularSheet(name: string, data: JsonCollection, settings: SheetSe } for (const cs of colSettings) { - cs.width = Math.min(cs.maxWidth ?? 120, cs.width ?? 5) - cs.width = Math.max(cs.minWidth ?? 5, cs.width) + cs.width = Math.min(cs.maxWidth ?? 120, cs.width ?? 5); + cs.width = Math.max(cs.minWidth ?? 5, cs.width); colInfo.push({ wch: cs.width, hidden: cs.hidden, - }) + }); } - sheet["!cols"] = colInfo; - sheet["!rows"] = rowInfo; + sheet['!cols'] = colInfo; + sheet['!rows'] = rowInfo; } } -function buildInspectorSheet(name: string, input: JsonCollection, settings: SheetSettings, rpt: FileTools.Spreadsheet) { +function buildInspectorSheet( + name: string, + input: JsonCollection, + settings: SheetSettings, + rpt: FileTools.Spreadsheet, +) { const cells: JsonPrimitive[][] = []; - const data = (isJsonArray(input)) ? input[0] : input; + const data = isJsonArray(input) ? input[0] : input; if (!isJsonMap(data)) { return; } @@ -297,18 +337,20 @@ function buildInspectorSheet(name: string, input: JsonCollection, settings: Shee // Now go through and align everything const sheet = rpt.workbook.Sheets[displayName]; - if (sheet["!ref"]) { - const range = utils.decode_range(sheet["!ref"]); - const dense = sheet["!data"]; + if (sheet['!ref']) { + const range = utils.decode_range(sheet['!ref']); + const dense = sheet['!data']; const rowInfo: RowInfo[] = []; - for(let R = 0; R <= range.e.r; ++R) { + for (let R = 0; R <= range.e.r; ++R) { const ri: RowInfo = {}; - for(let C = 0; C <= range.e.c; ++C) { - const cell: CellObject = dense ? sheet["!data"]?.[R]?.[C] : sheet[utils.encode_cell({r:R, c:C})]; + for (let C = 0; C <= range.e.c; ++C) { + const cell: CellObject = dense + ? sheet['!data']?.[R]?.[C] + : sheet[utils.encode_cell({ r: R, c: C })]; if (!cell) continue; const cs = settings.columns?.[cell.v?.toString() ?? 'default'] ?? {}; - + cell.s = settings.style; if (C === 0) { @@ -327,7 +369,9 @@ function buildInspectorSheet(name: string, input: JsonCollection, settings: Shee const maxHeight = 10; const ptSize = 12; cell.s = { alignment: { wrapText: true } }; - const lines = Math.floor((cell.v?.toLocaleString().length ?? 0) / (cs.width ?? 80)); + const lines = Math.floor( + (cell.v?.toLocaleString().length ?? 0) / (cs.width ?? 80), + ); ri.hpt = ptSize * Math.min(maxHeight, lines); } } @@ -335,41 +379,44 @@ function buildInspectorSheet(name: string, input: JsonCollection, settings: Shee rowInfo.push(ri); } - sheet["!cols"] = [{ wch: 20 }, { wch: 80 }]; - sheet["!rows"] = rowInfo; + sheet['!cols'] = [{ wch: 20 }, { wch: 80 }]; + sheet['!rows'] = rowInfo; } } function objectToArray(data: JsonMap, cells: JsonPrimitive[][]) { for (const [key, value] of Object.entries(data)) { - if (isJsonArray(value)) { const rs = [...value]; cells.push([key, rs.shift()?.toString() ?? null]); while (rs.length) { cells.push(['', rs.shift()?.toString() ?? null]); } - } else if (isJsonMap(value)) { cells.push(['', '']); cells.push([key, '']); objectToArray(value, cells); - } - else if (is.string(value)) { + } else if (is.string(value)) { cells.push([key, value.slice(0, 30_000)]); - } - else { - cells.push([key, value ?? null]) + } else { + cells.push([key, value ?? null]); } } } -function buildCoverSheet(name: string, data: JsonCollection, settings: SheetSettings, rpt: FileTools.Spreadsheet) { +function buildCoverSheet( + name: string, + data: JsonCollection, + settings: SheetSettings, + rpt: FileTools.Spreadsheet, +) { // Not yet implemented console.log(name, data, settings, rpt); } -function desperatelyAttemptToParseDate(input: string | number | boolean | Date | undefined) { +function desperatelyAttemptToParseDate( + input: string | number | boolean | Date | undefined, +) { let value = input; if (is.numericString(value)) { value = Number.parseInt(value); diff --git a/src/reports/report-types.ts b/src/reports/report-types.ts index 40c4852..4565982 100644 --- a/src/reports/report-types.ts +++ b/src/reports/report-types.ts @@ -1,13 +1,16 @@ -import { JsonCollection, JsonPrimitive } from "@salesforce/ts-types"; -import { XlsReportSettings } from "./output-xlsx.js"; -import { CsvReportSettings } from "./output-csv.js"; -import { JsonReportSettings } from "./output-json.js"; -import { ReportRunner } from "./report.js"; -import { ChildQuery, QueryInput } from "../model/queries/query-inheritance.js"; -import { AqFilter } from "aql-builder"; +import { JsonCollection, JsonPrimitive } from '@salesforce/ts-types'; +import { XlsReportSettings } from './output-xlsx.js'; +import { CsvReportSettings } from './output-csv.js'; +import { JsonReportSettings } from './output-json.js'; +import { ReportRunner } from './report.js'; +import { ChildQuery, QueryInput } from '../model/queries/query-inheritance.js'; +import { AqFilter } from 'aql-builder'; -export type ReportResult = { messages?: string[], errors?: Error[] }; -export type ReportWorker = (report: ReportConfig, runner: ReportRunner) => Promise +export type ReportResult = { messages?: string[]; errors?: Error[] }; +export type ReportWorker = ( + report: ReportConfig, + runner: ReportRunner, +) => Promise; export type TransformOptions = Record; export type BaseReportSettings = Record & { @@ -29,10 +32,14 @@ export type BaseReportSettings = Record & { /** * Include datasets in final output even when they're empty. */ - includeEmptyResults?: boolean, + includeEmptyResults?: boolean; }; -export type ReportSettings = BaseReportSettings | CsvReportSettings | JsonReportSettings | XlsReportSettings; +export type ReportSettings = + | BaseReportSettings + | CsvReportSettings + | JsonReportSettings + | XlsReportSettings; /** * Configuration for a specific Spidergram report @@ -65,11 +72,11 @@ export interface ReportConfig extends Record { /** * Run the report multiple times, once for each entry in the 'repeat' property. - * + * * The key of each repeat entry will be used as the new report name, and the value * (one or more AqFilters) will be applied to every alterable query in the report. */ - repeat?: Record + repeat?: Record; /** * A keyed list of queries to be run when building this report's data. Query data can @@ -90,7 +97,7 @@ export interface ReportConfig extends Record { * Keyed JSON data containing results from the reports queries. Additional datasets * can also be included manually; this can be useful for explanitory/overview sheets, * or pre-built supplementary data used during the output process. - * + * * Generally this collection's records correspond directly to the queries; transform * callbacks or query alteration flags may result in multiple datasets from a single * query, or multiple queries combined into a single dataset. @@ -102,7 +109,7 @@ export interface ReportConfig extends Record { * data after all queries have been run, but before output is generated. This can * be useful for date and number formatting and other cleanup. */ - alterData?: ReportWorker | Record + alterData?: ReportWorker | Record; /** * A custom report generation function that assumes responsibility for processing @@ -112,5 +119,9 @@ export interface ReportConfig extends Record { } export type ReportDataTransform = ReportDataSplit | ReportDataPivot; -export type ReportDataSplit = { action: 'split', property: string, mustMatch?: JsonPrimitive[] }; -export type ReportDataPivot = { action: 'pivot', property: string }; \ No newline at end of file +export type ReportDataSplit = { + action: 'split'; + property: string; + mustMatch?: JsonPrimitive[]; +}; +export type ReportDataPivot = { action: 'pivot'; property: string }; diff --git a/src/reports/report.ts b/src/reports/report.ts index 5ac5168..75fadf8 100644 --- a/src/reports/report.ts +++ b/src/reports/report.ts @@ -1,6 +1,11 @@ import { Query, JobStatus, Spidergram, AqFilter } from '../index.js'; import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter'; -import { AnyJson, isAnyJson, isJsonArray, isJsonMap } from '@salesforce/ts-types'; +import { + AnyJson, + isAnyJson, + isJsonArray, + isJsonMap, +} from '@salesforce/ts-types'; import { DateTime } from 'luxon'; import { ReportConfig } from './report-types.js'; import { buildQueryWithParents } from '../model/queries/query-inheritance.js'; @@ -72,7 +77,10 @@ export class ReportRunner { } } - async build(customConfig?: ReportConfig, filters: AqFilter[] = []): Promise { + async build( + customConfig?: ReportConfig, + filters: AqFilter[] = [], + ): Promise { const config = customConfig ?? this.config; // Give custom builder functions a chance to alter the queries, // populate custom data, or, you know, whatevs. @@ -80,7 +88,7 @@ export class ReportRunner { await config.alterQueries(config, this); } - this.status.total += Object.keys(config.queries ?? {}).length + this.status.total += Object.keys(config.queries ?? {}).length; // Iterate over every query, modify it if necessary, and run it. for (const [name, query] of Object.entries(config.queries ?? {})) { @@ -107,12 +115,12 @@ export class ReportRunner { const config = customConfig ?? this.config; config.data ??= {}; config.settings ??= {}; - config.settings.sheets ??= {} + config.settings.sheets ??= {}; if (is.function_(config.alterData)) { await config.alterData(config, this); } else if (config.alterData) { - for (const [key, data] of Object.entries(config.data)) { + for (const [key, data] of Object.entries(config.data)) { const op = config.alterData[key] ?? config.alterData['default']; if (op) { if (op.action === 'split' && isJsonArray(data)) { @@ -146,19 +154,25 @@ export class ReportRunner { const config = customConfig ?? this.config; this.events.emit('progress', this.status, 'Generating files'); - config.settings ??= {} + config.settings ??= {}; // Supply defaults and do token-replacement on the output path - let loc = config?.settings?.path as string ?? '{{date}} {{name}}'; + let loc = (config?.settings?.path as string) ?? '{{date}} {{name}}'; loc = loc.replace('{{name}}', config.name ?? 'report'); loc = loc.replace('{{date}}', DateTime.now().toISODate() ?? ''); config.settings.path = loc; if (is.function_(config.output)) { await config.output(config, this); - } else if (config.settings?.type === 'csv' || config.settings?.type === 'tsv') { + } else if ( + config.settings?.type === 'csv' || + config.settings?.type === 'tsv' + ) { await outputCsvReport(config, this); - } else if (this.config.settings?.type === 'json' || config.settings?.type === 'json5') { + } else if ( + this.config.settings?.type === 'json' || + config.settings?.type === 'json5' + ) { await outputJsonReport(config, this); } else { // Default to XLSX? @@ -191,10 +205,13 @@ export class ReportRunner { return Promise.resolve(this.status); } - async _runReport(config: ReportConfig, filters: AqFilter[] = []): Promise { + async _runReport( + config: ReportConfig, + filters: AqFilter[] = [], + ): Promise { await Spidergram.load(); return this.build(config, filters) .then(() => this.transform(config)) - .then(() => this.output(config)) + .then(() => this.output(config)); } } diff --git a/src/spider/handlers/download-handler.ts b/src/spider/handlers/download-handler.ts index da4da8d..e79acf7 100644 --- a/src/spider/handlers/download-handler.ts +++ b/src/spider/handlers/download-handler.ts @@ -30,7 +30,8 @@ export async function downloadHandler(context: SpiderContext): Promise { path.join(proj.config.storageDirectory ?? './storage', directory), ); const fullPath = path.join(directory, fileName); - const results = await files().writeStream(fullPath, Duplex.from(response.body)) + const results = await files() + .writeStream(fullPath, Duplex.from(response.body)) .then(() => ({ bucket: 'downloads', path: fullPath })) .catch((error: unknown) => { if (error instanceof Error) { diff --git a/src/spider/handlers/page-handler.ts b/src/spider/handlers/page-handler.ts index 083814a..6337e19 100644 --- a/src/spider/handlers/page-handler.ts +++ b/src/spider/handlers/page-handler.ts @@ -9,7 +9,6 @@ export async function pageHandler(context: SpiderContext) { ? await page.context().cookies() : undefined; - const timing = context.savePerformance ? await BrowserTools.getPageTiming(page) : undefined; @@ -19,7 +18,10 @@ export async function pageHandler(context: SpiderContext) { : undefined; const accessibility = context?.auditAccessibility - ? await BrowserTools.AxeAuditor.getAuditResults(page, context?.auditAccessibility) + ? await BrowserTools.AxeAuditor.getAuditResults( + page, + context?.auditAccessibility, + ) : undefined; await saveResource({ body, cookies, xhr, accessibility, timing }); diff --git a/src/tools/browser/axe-auditor.ts b/src/tools/browser/axe-auditor.ts index d337747..9ebe96c 100644 --- a/src/tools/browser/axe-auditor.ts +++ b/src/tools/browser/axe-auditor.ts @@ -123,10 +123,14 @@ export class AxeAuditor { }; if (page.isClosed()) { - return Promise.resolve({ error: new SpidergramError('Page was closed before action') }); + return Promise.resolve({ + error: new SpidergramError('Page was closed before action'), + }); } - const kv = await KeyValueStore.open(typeof opt.save === 'string' ? opt.save : 'axe_audits'); + const kv = await KeyValueStore.open( + typeof opt.save === 'string' ? opt.save : 'axe_audits', + ); return AxeAuditor.run(page) .then(results => { diff --git a/src/tools/file/spreadsheet.ts b/src/tools/file/spreadsheet.ts index 2aa2580..13f47e9 100644 --- a/src/tools/file/spreadsheet.ts +++ b/src/tools/file/spreadsheet.ts @@ -51,24 +51,19 @@ export class Spreadsheet { addSheet(input: unknown, name?: string) { if (isSimpleSheet(input)) { - utils.book_append_sheet( - this.workbook, - utils.json_to_sheet(input), - name, - ); + utils.book_append_sheet(this.workbook, utils.json_to_sheet(input), name); } else if (isStructuredSheet(input)) { - const data = (isJsonArray(input.data) && isJsonArray(input.data[0])) ? - utils.aoa_to_sheet(input.data as (JsonPrimitive | Date | undefined)[][]) : - utils.json_to_sheet(input.data, { - header: input.header, - skipHeader: input.skipHeader, - }); - - utils.book_append_sheet( - this.workbook, - data, - input.name ?? name, - ); + const data = + isJsonArray(input.data) && isJsonArray(input.data[0]) + ? utils.aoa_to_sheet( + input.data as (JsonPrimitive | Date | undefined)[][], + ) + : utils.json_to_sheet(input.data, { + header: input.header, + skipHeader: input.skipHeader, + }); + + utils.book_append_sheet(this.workbook, data, input.name ?? name); } else { throw new TypeError( 'Input must be a SimpleSheet array or StructuredSheet object', @@ -108,10 +103,16 @@ export class Spreadsheet { ...customOptions, }; - return write(this.workbook, { Props: this.workbook.Props, ...options, type: 'buffer' }) as Buffer; + return write(this.workbook, { + Props: this.workbook.Props, + ...options, + type: 'buffer', + }) as Buffer; } toStream(customOptions: Partial = {}): Readable { - return Readable.from(this.toBuffer({ Props: this.workbook.Props, ...customOptions })); + return Readable.from( + this.toBuffer({ Props: this.workbook.Props, ...customOptions }), + ); } } diff --git a/src/tools/graph/analyze-page.ts b/src/tools/graph/analyze-page.ts index 5d50f3c..32425dd 100644 --- a/src/tools/graph/analyze-page.ts +++ b/src/tools/graph/analyze-page.ts @@ -1,6 +1,18 @@ -import { Spidergram, Resource, HtmlTools, BrowserTools, EnqueueUrlOptions } from '../../index.js'; +import { + Spidergram, + Resource, + HtmlTools, + BrowserTools, + EnqueueUrlOptions, +} from '../../index.js'; import { rebuildResourceLinks } from './rebuild-resource-links.js'; -import { PageDataOptions, PageContentOptions, PatternDefinition, findAndSavePagePatterns, ConditionalPatternGroup } from '../html/index.js'; +import { + PageDataOptions, + PageContentOptions, + PatternDefinition, + findAndSavePagePatterns, + ConditionalPatternGroup, +} from '../html/index.js'; import { TechAuditOptions } from '../browser/index.js'; import { PropertyMap, mapProperties } from '../map-properties.js'; import _ from 'lodash'; @@ -37,8 +49,8 @@ export interface PageAnalysisOptions extends Record { /** * One or more {@link PropertyMap} rules that determine what {@link Site} * the {@link Resource} belongs to. - * - * The value here corresponds to the unique key of a {@link Site}; + * + * The value here corresponds to the unique key of a {@link Site}; */ site?: PropertyMap | PropertyMap[] | false; @@ -67,11 +79,11 @@ export interface PageAnalysisOptions extends Record { /** * A dictionary used to map existing data on a {@link Resource} to new properties. * If this property is set to `false`, property mapping is skipped entirely. - * + * * The key of each entry is the destination name or dot-notation path of a property * Resource, and the value of each entry is one or more {@link PropertyMap} * rules describing where the new property's value should be found. - * + * * If an array is given, the individual {@link PropertyMap} records will be * checked in order; the first one to produce a value will be used. If no value is * produced, the destination property will remain undefined. diff --git a/src/tools/graph/filter-by-property.ts b/src/tools/graph/filter-by-property.ts index 0ca1312..6b992be 100644 --- a/src/tools/graph/filter-by-property.ts +++ b/src/tools/graph/filter-by-property.ts @@ -4,19 +4,19 @@ import minimatch from 'minimatch'; export type ExactFilter = { property?: string; eq: string; - reject?: true + reject?: true; }; export type InFilter = { property?: string; in: string[]; - reject?: true + reject?: true; }; export type GlobFilter = { property?: string; glob: string; - reject?: true + reject?: true; }; export type RegexFilter = { @@ -30,21 +30,29 @@ export type PropertyFilter = ExactFilter | InFilter | GlobFilter | RegexFilter; /** * A too-clever for its own good filtering function. */ -export function filterByProperty(input: object, filter: PropertyFilter): boolean { +export function filterByProperty( + input: object, + filter: PropertyFilter, +): boolean { let result: boolean | undefined = undefined; // Populate 'value' with the property value. If there's no property pointer, // toString() the input and use it for comparison. - const value = (filter.property ? _.get(input, filter.property, false) : input).toString(); + const value = ( + filter.property ? _.get(input, filter.property, false) : input + ).toString(); if ('eq' in filter && filter.eq) { - result = value === filter.eq + result = value === filter.eq; } else if ('in' in filter && filter.in) { - result = filter.in?.includes(value) + result = filter.in?.includes(value); } else if ('glob' in filter && filter.glob) { result = minimatch(value, filter.glob); } else if ('regex' in filter && filter.regex) { - const regex = typeof filter.regex === 'string' ? new RegExp(filter.regex) : filter.regex; + const regex = + typeof filter.regex === 'string' + ? new RegExp(filter.regex) + : filter.regex; result = regex.test(value); } @@ -61,4 +69,4 @@ export function filterByProperty(input: object, filter: PropertyFilter): boolean */ export function byProperty(filter: PropertyFilter) { return (input: object) => filterByProperty(input, filter); -} \ No newline at end of file +} diff --git a/src/tools/graph/get-resource-site.ts b/src/tools/graph/get-resource-site.ts index cb1d800..98d2f00 100644 --- a/src/tools/graph/get-resource-site.ts +++ b/src/tools/graph/get-resource-site.ts @@ -4,12 +4,14 @@ import { findPropertyValue } from '../../index.js'; export async function getResourceSite( input: Resource, source: PropertyMap | PropertyMap[], - saveSite = true + saveSite = true, ) { const siteKey = findPropertyValue(input, source); if (typeof siteKey === 'string' && saveSite) { const sg = await Spidergram.load(); sg.arango.push(new Site({ key: siteKey }), false); } - return Promise.resolve(typeof siteKey === 'string' ? 'sites/' + siteKey : undefined); + return Promise.resolve( + typeof siteKey === 'string' ? 'sites/' + siteKey : undefined, + ); } diff --git a/src/tools/graph/index.ts b/src/tools/graph/index.ts index 81caad6..7bd9396 100644 --- a/src/tools/graph/index.ts +++ b/src/tools/graph/index.ts @@ -6,4 +6,4 @@ export * from './analyze-page.js'; export * from './rebuild-resource-links.js'; -export * from './get-resource-site.js' \ No newline at end of file +export * from './get-resource-site.js'; diff --git a/src/tools/graph/merge-site.ts b/src/tools/graph/merge-site.ts index eee3e05..e70940f 100644 --- a/src/tools/graph/merge-site.ts +++ b/src/tools/graph/merge-site.ts @@ -1,21 +1,24 @@ -import { Entity, Resource, Query, Reference, Site, aql } from '../../index.js'; +import { Entity, Query, Reference, Site, aql } from '../../index.js'; /** * Delete a {@link Site} and remove references to it from {@link Resource.site}. * If a replacement site is offered, update the affected resources to point to * the new site. */ -export async function deleteSite(site: Reference, replacement?: Reference) { +export async function deleteSite( + site: Reference, + replacement?: Reference, +) { const sid = Entity.idFromReference(site); - + if (replacement) { const rid = Entity.idFromReference(replacement); return Query.run( - aql`FOR r IN resources FILTER r.site == ${sid} UPDATE r WITH { site: ${rid} } IN resources` - ).then(() => Query.run(aql`REMOVE ${sid}`)) + aql`FOR r IN resources FILTER r.site == ${sid} UPDATE r WITH { site: ${rid} } IN resources`, + ).then(() => Query.run(aql`REMOVE ${sid}`)); } else { return Query.run( - aql`FOR r IN resources FILTER r.site == ${sid} UPDATE r WITH { site: null } IN resources` - ).then(() => Query.run(aql`REMOVE ${sid}`)) + aql`FOR r IN resources FILTER r.site == ${sid} UPDATE r WITH { site: null } IN resources`, + ).then(() => Query.run(aql`REMOVE ${sid}`)); } -} \ No newline at end of file +} diff --git a/src/tools/html/find-patterns.ts b/src/tools/html/find-patterns.ts index 4331d84..1dceec7 100644 --- a/src/tools/html/find-patterns.ts +++ b/src/tools/html/find-patterns.ts @@ -3,7 +3,10 @@ import _ from 'lodash'; import { Pattern, AppearsOn, Query, Resource, aql } from '../../model/index.js'; import { getCheerio } from './get-cheerio.js'; import { Spidergram } from '../../config/index.js'; -import { PropertyFilter, filterByProperty } from '../graph/filter-by-property.js'; +import { + PropertyFilter, + filterByProperty, +} from '../graph/filter-by-property.js'; import is from '@sindresorhus/is'; export interface FoundPattern extends HtmlTools.ElementData { @@ -12,16 +15,20 @@ export interface FoundPattern extends HtmlTools.ElementData { } export type ConditionalPatternGroup = PropertyFilter & { - patterns?: PatternDefinition[], - pattern?: PatternDefinition, + patterns?: PatternDefinition[]; + pattern?: PatternDefinition; }; -export function isConditionalPatternGroup(input: unknown): input is ConditionalPatternGroup { - return (is.plainObject(input) && ('patterns' in input || 'pattern' in input)); +export function isConditionalPatternGroup( + input: unknown, +): input is ConditionalPatternGroup { + return is.plainObject(input) && ('patterns' in input || 'pattern' in input); } -export function isPatternDefinition(input: unknown): input is PatternDefinition { - return (is.plainObject(input) && 'name' in input && 'selector' in input); +export function isPatternDefinition( + input: unknown, +): input is PatternDefinition { + return is.plainObject(input) && 'name' in input && 'selector' in input; } /** @@ -38,7 +45,7 @@ export interface PatternDefinition extends HtmlTools.ElementDataOptions { patternKey?: string; /** - * Indicates that the definition is a named variation of a more generic pattern. + * Indicates that the definition is a named variation of a more generic pattern. */ variant?: string; @@ -48,7 +55,7 @@ export interface PatternDefinition extends HtmlTools.ElementDataOptions { selector: string; /** - * When an instance of this pattern is found, remove its markup from the DOM + * When an instance of this pattern is found, remove its markup from the DOM * so it won't be matched multiple times. */ exclusive?: boolean; @@ -70,22 +77,31 @@ const defaults: HtmlTools.ElementDataOptions = { export async function findAndSavePagePatterns( input: Resource, patterns: PatternDefinition | (PatternDefinition | ConditionalPatternGroup)[], -) { +) { const defs: PatternDefinition[] = []; if (Array.isArray(patterns)) { for (const pattern of patterns) { if (isPatternDefinition(pattern)) { defs.push(pattern); - } - else if (isConditionalPatternGroup(pattern) && filterByProperty(input, pattern)) { + } else if ( + isConditionalPatternGroup(pattern) && + filterByProperty(input, pattern) + ) { if (pattern.pattern) defs.push(pattern.pattern); if (pattern.patterns) defs.push(...pattern.patterns); } } } - const pts = defs.map(p => new Pattern({ key: p.patternKey, name: p.name, description: p.description })); - + const pts = defs.map( + p => + new Pattern({ + key: p.patternKey, + name: p.name, + description: p.description, + }), + ); + const instances = findPagePatterns(input, defs); const sg = await Spidergram.load(); @@ -110,16 +126,14 @@ export function findPagePatterns( const resource = input instanceof Resource ? input : undefined; for (const pattern of list) { results.push( - ...findPatternInstances(input, pattern).map( - fp => { - const pi = new AppearsOn({ - ...fp, - pattern: `patterns/${ fp.patternKey ?? fp.pattern ?? 'null'}`, - page: resource ?? 'resources/null', - }); - return pi; - } - ), + ...findPatternInstances(input, pattern).map(fp => { + const pi = new AppearsOn({ + ...fp, + pattern: `patterns/${fp.patternKey ?? fp.pattern ?? 'null'}`, + page: resource ?? 'resources/null', + }); + return pi; + }), ); } @@ -151,7 +165,7 @@ export function findPatternInstances( ), }; if (pattern.properties) { - mapProperties(found, pattern.properties) + mapProperties(found, pattern.properties); } if (pattern.exclusive) { $(element).remove(); diff --git a/src/tools/html/get-page-content.ts b/src/tools/html/get-page-content.ts index 27c2011..d186821 100644 --- a/src/tools/html/get-page-content.ts +++ b/src/tools/html/get-page-content.ts @@ -96,12 +96,10 @@ export async function getPageContent( ? options.selector : [options.selector]; const $ = getCheerio(input); - markup = selectors.reduce( - (html, selector) => { - if (html.length === 0) html = $(selector).html() ?? ''; - return html; - }, '' - ); + markup = selectors.reduce((html, selector) => { + if (html.length === 0) html = $(selector).html() ?? ''; + return html; + }, ''); } else { markup = getMarkup(input); } diff --git a/src/tools/map-properties.ts b/src/tools/map-properties.ts index a63f873..a9c5e58 100644 --- a/src/tools/map-properties.ts +++ b/src/tools/map-properties.ts @@ -12,7 +12,10 @@ import is from '@sindresorhus/is'; * transformation rules * - a function that accepts the object and returns a property value or `undefined` */ -export type PropertyMap = string | PropertyMapRule | ((input: T) => unknown); +export type PropertyMap = + | string + | PropertyMapRule + | ((input: T) => unknown); /** * Describes the location of a piece of data on a source object. @@ -33,12 +36,12 @@ export interface PropertyMapRule extends Record { /** * If a selector is used, return the number of matches rather than the content. */ - count?: boolean - + count?: boolean; + /** - * If a selector is used, return the value of an attribute rather than the element text. + * If a selector is used, return the value of an attribute rather than the element text. */ - attribute?: string + attribute?: string; /** * If the property value is found and is an array, limit the number of results @@ -133,9 +136,9 @@ export interface PropertyMapRule extends Record { export function mapProperties( obj: object, - map: Record + map: Record, ) { - const domDictionary: Record = {} + const domDictionary: Record = {}; for (const [prop, rule] of Object.entries(map)) { _.set(obj, prop, findPropertyValue(obj, rule, domDictionary)); @@ -153,7 +156,7 @@ export function mapProperties( export function findPropertyValue( object: T, locations: PropertyMap | PropertyMap[], - domDictionary: Record = {} + domDictionary: Record = {}, ): unknown | undefined { const sources = Array.isArray(locations) ? locations : [locations]; for (const source of sources) { @@ -175,13 +178,12 @@ export function findPropertyValue( v = matches.length; } else { if (matches.length > 0) { - v = matches - .slice(0, source.limit) - .map(e => { - if (source.attribute) return $(e).attr(source.attribute)?.trim(); - else return $(e).text().trim(); - }); - v = (source.join || v.length === 1) ? v.join(source.join) : v; + v = matches.slice(0, source.limit).map(e => { + if (source.attribute) + return $(e).attr(source.attribute)?.trim(); + else return $(e).text().trim(); + }); + v = source.join || v.length === 1 ? v.join(source.join) : v; if (v?.length === 0) v = undefined; } else { v = undefined; @@ -194,8 +196,7 @@ export function findPropertyValue( if (undef(v, source)) { if (source.fallback) return source.fallback; return undefined; - } - else return v; + } else return v; } } } @@ -278,13 +279,15 @@ function checkPropertyValue( function undef(value: unknown, rules?: PropertyMapRule): value is undefined { const nok = rules?.acceptNull ?? false; const eok = rules?.acceptEmpty ?? false; - - if (value === undefined || + + if ( + value === undefined || (value === null && !nok) || (is.emptyArray(value) && !eok) || - (is.emptyString(value) && !eok) || + (is.emptyString(value) && !eok) || (is.emptyObject(value) && !eok) - ) return true; + ) + return true; return false; } diff --git a/src/tools/screenshot.ts b/src/tools/screenshot.ts index f5dd223..e99fb9f 100644 --- a/src/tools/screenshot.ts +++ b/src/tools/screenshot.ts @@ -183,14 +183,18 @@ export class ScreenshotTool { const filename = `${directory}/${this.getFilename( page.url(), v, - undefined + undefined, )}.${type}`; if (fullPage === false) { pwOptions.clip = { x: 0, y: 0, ...materializedViewports[v] }; } const buffer = await page.screenshot(pwOptions); - const bin = path.join(sg.config.outputDirectory ?? sg.config.storageDirectory ?? './storage'); + const bin = path.join( + sg.config.outputDirectory ?? + sg.config.storageDirectory ?? + './storage', + ); await ensureDir(path.join(bin, path.dirname(filename))); await storage.writeStream(filename, Readable.from(buffer)); @@ -201,7 +205,7 @@ export class ScreenshotTool { let filename = `${directory}/${this.getFilename( page.url(), v, - selector + selector, )}.${type}`; const max = Math.min(limit, await page.locator(selector).count()); @@ -221,11 +225,15 @@ export class ScreenshotTool { filename = `${directory}/${this.getFilename( page.url(), v, - selector + selector, )}-${l}.${type}`; } const buffer = await locator.screenshot(pwOptions); - const bin = path.join(sg.config.outputDirectory ?? sg.config.storageDirectory ?? './storage'); + const bin = path.join( + sg.config.outputDirectory ?? + sg.config.storageDirectory ?? + './storage', + ); await ensureDir(path.join(bin, path.dirname(filename))); await storage.writeStream(filename, Readable.from(buffer)); @@ -285,11 +293,7 @@ export class ScreenshotTool { } // In theory, we could use this for subdirectories in addition to long filenames. - protected getFilename( - url: string, - viewport: string, - selector?: string, - ) { + protected getFilename(url: string, viewport: string, selector?: string) { let path = new URL(url).pathname.replaceAll('/', '-').slice(1); if (path.length === 0) path = 'index'; const components = [new URL(url).hostname, path, viewport];