From 76cdeac10bf6bea3b23b864b4f9059bcfe0dfda5 Mon Sep 17 00:00:00 2001 From: Vjacheslav Trushkin Date: Sun, 26 Nov 2023 23:43:30 +0200 Subject: [PATCH] chore: concurrent downloading from Figma API --- @iconify/tools/package.json | 5 + @iconify/tools/src/download/api/queue.ts | 113 +++++++++++ @iconify/tools/src/import/figma/query.ts | 184 +++++++++++------- @iconify/tools/tests/misc/concurrency-test.ts | 45 +++++ 4 files changed, 274 insertions(+), 73 deletions(-) create mode 100644 @iconify/tools/src/download/api/queue.ts create mode 100644 @iconify/tools/tests/misc/concurrency-test.ts diff --git a/@iconify/tools/package.json b/@iconify/tools/package.json index 9e916f7..bfadd70 100644 --- a/@iconify/tools/package.json +++ b/@iconify/tools/package.json @@ -147,6 +147,11 @@ "require": "./lib/download/api/index.cjs", "import": "./lib/download/api/index.mjs" }, + "./lib/download/api/queue": { + "types": "./lib/download/api/queue.d.ts", + "require": "./lib/download/api/queue.cjs", + "import": "./lib/download/api/queue.mjs" + }, "./lib/download/api/types": { "types": "./lib/download/api/types.d.ts", "require": "./lib/download/api/types.cjs", diff --git a/@iconify/tools/src/download/api/queue.ts b/@iconify/tools/src/download/api/queue.ts new file mode 100644 index 0000000..a92b551 --- /dev/null +++ b/@iconify/tools/src/download/api/queue.ts @@ -0,0 +1,113 @@ +/** + * Concurrent queries limit + */ +let queriesLimit = 5; + +/** + * Concurrent queries retries count + */ +let maxRetries = 3; + +/** + * Set concurrent queries default limit + */ +export function setConcurrentQueriesDefaultLimit(value: number) { + queriesLimit = value; +} + +/** + * Set concurrent queries default retries count + */ +export function setConcurrentQueriesDefaultRetries(value: number) { + maxRetries = value; +} + +/** + * Callback to get query + */ +export type GetConcurrentQueryCallback = (index: number) => Promise; + +/** + * Runs concurrent async operations + */ +export function runConcurrentQueries( + count: number, + callback: GetConcurrentQueryCallback, + limit = 0, + retries = 0 +): Promise { + // Set limit and retries count + limit = Math.max(1, Math.min(limit || queriesLimit, count)); + retries = Math.max(1, retries || maxRetries); + + // Results + const results: T[] = Array(count).fill(null as unknown as T); + + // Queue + let nextIndex = 0; + const resolving = new Set(); + let rejected = false; + let resolved = false; + + return new Promise((resolve, reject) => { + // Function to call after item is resolved + function resolvedItem() { + if (rejected || resolved) { + return; + } + + if (!resolving.size && nextIndex > count) { + resolved = true; + resolve(results); + return; + } + + if (resolving.size < limit && nextIndex <= count) { + startNext(); + } + } + + // Run item + function run(index: number, retry: number) { + // Mark as resolving + resolving.add(index); + + // Get promise and run it + const p = callback(index); + p.then((value) => { + resolving.delete(index); + results[index] = value; + resolvedItem(); + }).catch((err) => { + if (retry < retries) { + // try again on next tick + setTimeout(() => { + run(index, retry + 1); + }); + } else if (!rejected) { + rejected = true; + reject(err); + } + }); + } + + // Start next item + function startNext() { + // Get next item + const index = nextIndex++; + if (index >= count) { + // Out of queue items + resolvedItem(); + return; + } + + // Run query + run(index, 0); + } + + // Queue items up to a limit + for (let i = 0; i < limit; i++) { + startNext(); + } + }); +} diff --git a/@iconify/tools/src/import/figma/query.ts b/@iconify/tools/src/import/figma/query.ts index 9fb7b66..75d9060 100644 --- a/@iconify/tools/src/import/figma/query.ts +++ b/@iconify/tools/src/import/figma/query.ts @@ -4,6 +4,7 @@ import { clearAPICache, getAPICache, } from '../../download/api/cache'; +import { runConcurrentQueries } from '../../download/api/queue'; import type { APICacheOptions, APIQueryParams } from '../../download/api/types'; import type { DocumentNotModified } from '../../download/types/modified'; import type { @@ -16,7 +17,7 @@ import type { FigmaFilesQueryOptions, FigmaImagesQueryOptions, } from './types/options'; -import type { FigmaNodesImportResult } from './types/result'; +import type { FigmaIconNode, FigmaNodesImportResult } from './types/result'; /** * Compare last modified dates @@ -180,77 +181,97 @@ export async function figmaImagesQuery( const maxLength = 2048 - uri.length; const svgOptions = options.svgOptions || {}; - let ids: string[] = []; - let idsLength = 0; let lastError: number | undefined; let found = 0; // Send query - const query = async () => { - const params = new URLSearchParams({ - ids: ids.join(','), - format: 'svg', - }); - if (options.version) { - params.set('version', options.version); - } - if (svgOptions.includeID) { - params.set('svg_include_id', 'true'); - } - if (svgOptions.simplifyStroke) { - params.set('svg_simplify_stroke', 'true'); - } - if (svgOptions.useAbsoluteBounds) { - params.set('use_absolute_bounds', 'true'); - } + const query = (ids: string[]): Promise => { + return new Promise((resolve, reject) => { + const params = new URLSearchParams({ + ids: ids.join(','), + format: 'svg', + }); + if (options.version) { + params.set('version', options.version); + } + if (svgOptions.includeID) { + params.set('svg_include_id', 'true'); + } + if (svgOptions.simplifyStroke) { + params.set('svg_simplify_stroke', 'true'); + } + if (svgOptions.useAbsoluteBounds) { + params.set('use_absolute_bounds', 'true'); + } - const data = await sendAPIQuery( - { - uri, - params, - headers: { - 'X-FIGMA-TOKEN': options.token, + sendAPIQuery( + { + uri, + params, + headers: { + 'X-FIGMA-TOKEN': options.token, + }, }, - }, - cache - ); - if (typeof data === 'number') { - lastError = data; - return; - } - try { - const parsedData = JSON.parse(data) as FigmaAPIImagesResponse; - const images = parsedData.images; - for (const id in images) { - const node = nodes.icons[id]; - const target = images[id]; - if (node && target) { - node.url = target; - found++; - } - } - } catch (err) { - return; - } + cache + ) + .then((data) => { + if (typeof data === 'number') { + reject(data); + return; + } + + let parsedData: FigmaAPIImagesResponse; + try { + parsedData = JSON.parse(data) as FigmaAPIImagesResponse; + } catch { + reject('Bad API response'); + return; + } + + resolve(parsedData); + }) + .catch(reject); + }); }; - // Loop all ids + // Generate queue + let ids: string[] = []; + let idsLength = 0; const allKeys = Object.keys(nodes.icons); + const queue: string[][] = []; for (let i = 0; i < allKeys.length; i++) { const id = allKeys[i]; ids.push(id); idsLength += id.length + 1; if (idsLength >= maxLength) { - await query(); + queue.push(ids.slice(0)); ids = []; idsLength = 0; } } if (idsLength) { - await query(); + queue.push(ids.slice(0)); } - // Check data + // Get data + const results = await runConcurrentQueries(queue.length, (index) => + query(queue[index]) + ); + + // Parse data + results.forEach((data) => { + const images = data.images; + for (const id in images) { + const node = nodes.icons[id]; + const target = images[id]; + if (node && target) { + node.url = target; + found++; + } + } + }); + + // Validate results if (!found) { if (lastError) { throw new Error( @@ -276,37 +297,54 @@ export async function figmaDownloadImages( const icons = nodes.icons; const ids = Object.keys(icons); let count = 0; - let lastError: number | undefined; + // Filter data + interface FigmaIconNodeWithURL extends FigmaIconNode { + url: string; + } + const filtered = Object.create(null) as Record< + string, + FigmaIconNodeWithURL + >; for (let i = 0; i < ids.length; i++) { const id = ids[i]; const item = icons[id]; - if (!item.url) { - continue; - } - const result = await sendAPIQuery( - { - uri: item.url, - }, - cache - ); - if (typeof result === 'number') { - lastError = result; - continue; - } - if (typeof result === 'string') { - count++; - item.content = result; + if (item.url) { + filtered[id] = item as FigmaIconNodeWithURL; } } + const keys = Object.keys(filtered); + // Download everything + await runConcurrentQueries(keys.length, (index) => { + return new Promise((resolve, reject) => { + const id = keys[index]; + const item = filtered[id]; + sendAPIQuery( + { + uri: item.url, + }, + cache + ) + .then((data) => { + if (typeof data === 'string') { + count++; + item.content = data; + resolve(true); + } else { + reject(data); + } + }) + .catch(reject); + }); + }); + + // Make sure something was downloaded if (!count) { - throw new Error( - `Error retrieving images${ - lastError ? ': ' + lastError.toString() : '' - }` - ); + throw new Error('Error retrieving images'); } + + // Update counter and return node nodes.downloadedIconsCount = count; return nodes; } diff --git a/@iconify/tools/tests/misc/concurrency-test.ts b/@iconify/tools/tests/misc/concurrency-test.ts new file mode 100644 index 0000000..71a9038 --- /dev/null +++ b/@iconify/tools/tests/misc/concurrency-test.ts @@ -0,0 +1,45 @@ +import { runConcurrentQueries } from '../../lib/download/api/queue'; + +describe('Testing concurrency', () => { + test('Simple queue', async () => { + const tests: Record = { + test1: 2, + test2: 100, + test3: 50, + test4: 1, + }; + + const keys = Object.keys(tests); + const callbacks: Set = new Set(); + const resolved: Set = new Set(); + + const result = await runConcurrentQueries( + keys.length, + (index) => { + expect(callbacks.has(index)).toBeFalsy(); + expect(resolved.has(index)).toBeFalsy(); + + if (index < 3) { + // When first 3 items are called, nothing should be resolved + expect(Array.from(resolved)).toEqual([]); + } + if (index === 3) { + // When 4th item is called, first one should be resolved + expect(Array.from(resolved)).toEqual([0]); + } + + const key = keys[index] as keyof typeof tests; + const delay = tests[key]; + return new Promise((resolve) => { + setTimeout(() => { + resolved.add(index); + resolve(delay); + }, delay); + }); + }, + 3 + ); + + expect(result).toEqual(Object.values(tests)); + }); +});