From 7d4f5132bf22806bdee80f225f86dfad35b3b7bb Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Wed, 13 Dec 2023 21:21:58 -0500 Subject: [PATCH 01/21] Change requestGraph cache key --- packages/core/core/src/RequestTracker.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js index 79efd368e93..c46c7e99a60 100644 --- a/packages/core/core/src/RequestTracker.js +++ b/packages/core/core/src/RequestTracker.js @@ -1110,7 +1110,8 @@ export default class RequestTracker { async writeToCache() { let cacheKey = getCacheKey(this.options); - let requestGraphKey = hashString(`${cacheKey}:requestGraph`); + let requestGraphKey = + hashString(`${cacheKey}:requestGraph`) + '-RequestGraph'; let snapshotKey = hashString(`${cacheKey}:snapshot`); if (this.options.shouldDisableCache) { @@ -1206,7 +1207,8 @@ async function loadRequestGraph(options): Async { } let cacheKey = getCacheKey(options); - let requestGraphKey = hashString(`${cacheKey}:requestGraph`); + let requestGraphKey = + hashString(`${cacheKey}:requestGraph`) + '-RequestGraph'; if (await options.cache.hasLargeBlob(requestGraphKey)) { let requestGraph: RequestGraph = deserialize( await options.cache.getLargeBlob(requestGraphKey), From 6e85be331bc4e9567341f0a01a31519eee507eee Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Wed, 13 Dec 2023 21:22:34 -0500 Subject: [PATCH 02/21] Change bundleGraph cache key --- packages/core/core/src/requests/BundleGraphRequest.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/core/core/src/requests/BundleGraphRequest.js b/packages/core/core/src/requests/BundleGraphRequest.js index 2696835dba3..8a420adb1ec 100644 --- a/packages/core/core/src/requests/BundleGraphRequest.js +++ b/packages/core/core/src/requests/BundleGraphRequest.js @@ -189,11 +189,12 @@ class BundlerRunner { this.pluginOptions = new PluginOptions( optionsProxy(this.options, api.invalidateOnOptionChange), ); - this.cacheKey = hashString( - `${PARCEL_VERSION}:BundleGraph:${JSON.stringify(options.entries) ?? ''}${ - options.mode - }`, - ); + this.cacheKey = + hashString( + `${PARCEL_VERSION}:BundleGraph:${ + JSON.stringify(options.entries) ?? '' + }${options.mode}`, + ) + '-BundleGraph'; } async loadConfigs() { From 4465ae24c9088e970e20f37be1ddada8e2ad0622 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Wed, 13 Dec 2023 21:22:51 -0500 Subject: [PATCH 03/21] Change assetGraph cache key --- packages/core/core/src/requests/AssetGraphRequest.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/core/core/src/requests/AssetGraphRequest.js b/packages/core/core/src/requests/AssetGraphRequest.js index 067d1912fb1..67a117a951e 100644 --- a/packages/core/core/src/requests/AssetGraphRequest.js +++ b/packages/core/core/src/requests/AssetGraphRequest.js @@ -157,9 +157,12 @@ export class AssetGraphBuilder { this.shouldBuildLazily = shouldBuildLazily ?? false; this.lazyIncludes = lazyIncludes ?? []; this.lazyExcludes = lazyExcludes ?? []; - this.cacheKey = hashString( - `${PARCEL_VERSION}${name}${JSON.stringify(entries) ?? ''}${options.mode}`, - ); + this.cacheKey = + hashString( + `${PARCEL_VERSION}${name}${JSON.stringify(entries) ?? ''}${ + options.mode + }`, + ) + '-AssetGraph'; this.isSingleChangeRebuild = api From 32beac29835614176ff51992079013d812369031 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Wed, 13 Dec 2023 22:32:52 -0500 Subject: [PATCH 04/21] Change manner in which requestGraph is found --- packages/dev/query/src/index.js | 85 +++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index ee33d54f7c9..6bc80e6f2f4 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -27,56 +27,71 @@ export async function loadGraphs(cacheDir: string): Promise<{| bundleInfo: ?Map, cacheInfo: ?Map>, |}> { - function filesBySizeAndModifiedTime() { - let files = fs.readdirSync(cacheDir).map(f => { - let stat = fs.statSync(path.join(cacheDir, f)); - return [path.join(cacheDir, f), stat.size, stat.mtime]; + function filesByTypeAndModifiedTime() { + let files = fs.readdirSync(cacheDir); + + let requestGraphFiles = []; + let bundleGraphFiles = []; + let assetGraphFiles = []; + + files.forEach(f => { + if (f.endsWith('-0')) { + let stat = fs.statSync(path.join(cacheDir, f)); + let info = [path.join(cacheDir, f), stat.size, stat.mtime]; + + if (f.endsWith('RequestGraph-0')) { + requestGraphFiles.push(info); + } else if (f.endsWith('BundleGraph-0')) { + bundleGraphFiles.push(info); + } else if (f.endsWith('AssetGraph-0')) { + assetGraphFiles.push(info); + } + } }); - files.sort(([, a], [, b]) => b - a); - files.sort(([, , a], [, , b]) => b - a); + requestGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime); + bundleGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime); + assetGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime); - return files.map(([f]) => f); + return { + requestGraphFiles: requestGraphFiles.map(([f]) => f), + bundleGraphFiles: bundleGraphFiles.map(([f]) => f), + assetGraphFiles: assetGraphFiles.map(([f]) => f), + }; } let cacheInfo: Map> = new Map(); - let timeToDeserialize = 0; - let requestTracker; + let {requestGraphFiles, bundleGraphFiles, assetGraphFiles} = + filesByTypeAndModifiedTime(); const cache = new LMDBCache(cacheDir); - for (let f of filesBySizeAndModifiedTime()) { - // Empty filename or not the first chunk - if (path.extname(f) !== '' || !f.endsWith('-0')) continue; + + // Get requestTracker + let requestTracker; + if (requestGraphFiles.length > 0) { try { let file = await cache.getLargeBlob( - path.basename(f).slice(0, -'-0'.length), + path.basename(requestGraphFiles[0]).slice(0, -'-0'.length), ); - cacheInfo.set('RequestGraph', [Buffer.byteLength(file)]); - - timeToDeserialize = Date.now(); + let timeToDeserialize = Date.now(); let obj = v8.deserialize(file); timeToDeserialize = Date.now() - timeToDeserialize; - /* if (obj.assetGraph != null && obj.assetGraph.value.hash != null) { - assetGraph = AssetGraph.deserialize(obj.assetGraph.value); - } else if (obj.bundleGraph != null) { - bundleGraph = BundleGraph.deserialize(obj.bundleGraph.value); - } else */ - if (obj['$$type']?.endsWith('RequestGraph')) { - let date = Date.now(); - requestTracker = new RequestTracker({ - graph: RequestGraph.deserialize(obj.value), - // $FlowFixMe - farm: null, - // $FlowFixMe - options: null, - }); - timeToDeserialize += Date.now() - date; - break; - } + invariant(obj['$$type']?.endsWith('RequestGraph')); + let date = Date.now(); + requestTracker = new RequestTracker({ + graph: RequestGraph.deserialize(obj.value), + // $FlowFixMe + farm: null, + // $FlowFixMe + options: null, + }); + timeToDeserialize += Date.now() - date; + cacheInfo.set('RequestGraph', [Buffer.byteLength(file)]); + cacheInfo.get('RequestGraph')?.push(timeToDeserialize); } catch (e) { - // noop + throw new Error('Issue with identifying Request Graph'); } } @@ -124,7 +139,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| ); } } - cacheInfo.get('RequestGraph')?.push(timeToDeserialize); + let writeBundlesRequest = buildRequestSubRequests.find( n => n.type === 1 && n.requestType === 11, ); From 331a50ca725778221c8e3441a752cfa035a64387 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Wed, 13 Dec 2023 22:35:53 -0500 Subject: [PATCH 05/21] Remove assetGraph reliance on requestTracker --- packages/dev/query/src/index.js | 34 +++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index 6bc80e6f2f4..4428d5b4008 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -95,6 +95,27 @@ export async function loadGraphs(cacheDir: string): Promise<{| } } + // Get assetGraph + let assetGraph; + if (assetGraphFiles.length > 0) { + try { + let file = await cache.getLargeBlob( + path.basename(assetGraphFiles[0]).slice(0, -'-0'.length), + ); + + let timeToDeserialize = Date.now(); + let obj = v8.deserialize(file); + invariant(obj.assetGraph != null); + assetGraph = AssetGraph.deserialize(obj.assetGraph.value); + timeToDeserialize = Date.now() - timeToDeserialize; + + cacheInfo.set('AssetGraph', [Buffer.byteLength(file)]); + cacheInfo.get('AssetGraph')?.push(timeToDeserialize); + } catch (e) { + throw new Error('Issue with identifying Asset Graph'); + } + } + function getSubRequests(id: NodeId) { return requestTracker.graph .getNodeIdsConnectedFrom(id, requestGraphEdgeTypes.subrequest) @@ -102,9 +123,8 @@ export async function loadGraphs(cacheDir: string): Promise<{| } // Load graphs by finding the main subrequests and loading their results - let assetGraph, bundleGraph, bundleInfo; + let bundleGraph, bundleInfo; cacheInfo.set('BundleGraph', []); - cacheInfo.set('AssetGraph', []); invariant(requestTracker); let buildRequestId = requestTracker.graph.getNodeIdByContentKey( 'parcel_build_request', @@ -128,16 +148,6 @@ export async function loadGraphs(cacheDir: string): Promise<{| ) ).bundleGraph.value, ); - - let assetGraphRequest = getSubRequests( - requestTracker.graph.getNodeIdByContentKey(bundleGraphRequestNode.id), - ).find(n => n.type === 1 && n.requestType === 3); - if (assetGraphRequest != null) { - assetGraph = AssetGraph.deserialize( - (await loadLargeBlobRequestRequest(cache, assetGraphRequest, cacheInfo)) - .assetGraph.value, - ); - } } let writeBundlesRequest = buildRequestSubRequests.find( From ec8b2c12bf92896231552ee00845e684fc8c2ff7 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Wed, 13 Dec 2023 22:40:14 -0500 Subject: [PATCH 06/21] Remove bundleGraph reliance requestTracker --- packages/dev/query/src/index.js | 39 +++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index 4428d5b4008..a8973e2b6ba 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -95,6 +95,27 @@ export async function loadGraphs(cacheDir: string): Promise<{| } } + // Get bundleGraph + let bundleGraph; + if (bundleGraphFiles.length > 0) { + try { + let file = await cache.getLargeBlob( + path.basename(bundleGraphFiles[0]).slice(0, -'-0'.length), + ); + + let timeToDeserialize = Date.now(); + let obj = v8.deserialize(file); + invariant(obj.bundleGraph != null); + bundleGraph = BundleGraph.deserialize(obj.bundleGraph.value); + timeToDeserialize = Date.now() - timeToDeserialize; + + cacheInfo.set('BundleGraph', [Buffer.byteLength(file)]); + cacheInfo.get('BundleGraph')?.push(timeToDeserialize); + } catch (e) { + throw new Error('Issue with identifying Bundle Graph'); + } + } + // Get assetGraph let assetGraph; if (assetGraphFiles.length > 0) { @@ -123,8 +144,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| } // Load graphs by finding the main subrequests and loading their results - let bundleGraph, bundleInfo; - cacheInfo.set('BundleGraph', []); + let bundleInfo; invariant(requestTracker); let buildRequestId = requestTracker.graph.getNodeIdByContentKey( 'parcel_build_request', @@ -135,21 +155,6 @@ export async function loadGraphs(cacheDir: string): Promise<{| invariant(buildRequestNode.type === 1 && buildRequestNode.requestType === 1); let buildRequestSubRequests = getSubRequests(buildRequestId); - let bundleGraphRequestNode = buildRequestSubRequests.find( - n => n.type === 1 && n.requestType === 2, - ); - if (bundleGraphRequestNode != null) { - bundleGraph = BundleGraph.deserialize( - ( - await loadLargeBlobRequestRequest( - cache, - bundleGraphRequestNode, - cacheInfo, - ) - ).bundleGraph.value, - ); - } - let writeBundlesRequest = buildRequestSubRequests.find( n => n.type === 1 && n.requestType === 11, ); From 48384759c98169c5ca66a78f41885111a74f7c94 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Wed, 13 Dec 2023 22:48:23 -0500 Subject: [PATCH 07/21] Clean up --- packages/dev/query/src/index.js | 65 ++++++++++++--------------------- 1 file changed, 23 insertions(+), 42 deletions(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index a8973e2b6ba..9014586e136 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -145,49 +145,30 @@ export async function loadGraphs(cacheDir: string): Promise<{| // Load graphs by finding the main subrequests and loading their results let bundleInfo; - invariant(requestTracker); - let buildRequestId = requestTracker.graph.getNodeIdByContentKey( - 'parcel_build_request', - ); - let buildRequestNode = nullthrows( - requestTracker.graph.getNode(buildRequestId), - ); - invariant(buildRequestNode.type === 1 && buildRequestNode.requestType === 1); - let buildRequestSubRequests = getSubRequests(buildRequestId); - - let writeBundlesRequest = buildRequestSubRequests.find( - n => n.type === 1 && n.requestType === 11, - ); - if (writeBundlesRequest != null) { - invariant(writeBundlesRequest.type === 1); - // $FlowFixMe[incompatible-cast] - bundleInfo = (nullthrows(writeBundlesRequest.result): Map< - ContentKey, - PackagedBundleInfo, - >); + if (requestTracker) { + let buildRequestId = requestTracker.graph.getNodeIdByContentKey( + 'parcel_build_request', + ); + let buildRequestNode = nullthrows( + requestTracker.graph.getNode(buildRequestId), + ); + invariant( + buildRequestNode.type === 1 && buildRequestNode.requestType === 1, + ); + let buildRequestSubRequests = getSubRequests(buildRequestId); + + let writeBundlesRequest = buildRequestSubRequests.find( + n => n.type === 1 && n.requestType === 11, + ); + if (writeBundlesRequest != null) { + invariant(writeBundlesRequest.type === 1); + // $FlowFixMe[incompatible-cast] + bundleInfo = (nullthrows(writeBundlesRequest.result): Map< + ContentKey, + PackagedBundleInfo, + >); + } } return {assetGraph, bundleGraph, requestTracker, bundleInfo, cacheInfo}; } - -async function loadLargeBlobRequestRequest(cache, node, cacheInfo) { - invariant(node.type === 1); - - let cachedFile = await cache.getLargeBlob(nullthrows(node.resultCacheKey)); - - let TTD = Date.now(); - let result = v8.deserialize(cachedFile); - TTD = Date.now() - TTD; - - if (node.requestType === 2) { - cacheInfo.get('BundleGraph')?.push(cachedFile.byteLength); //Add size - cacheInfo.get('BundleGraph')?.push(TTD); - } - - if (node.requestType === 3) { - cacheInfo.get('AssetGraph')?.push(cachedFile.byteLength); - cacheInfo.get('AssetGraph')?.push(TTD); - } - - return result; -} From e1562613387fdf7e958dfe4677f7db8f5de23dc5 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Wed, 13 Dec 2023 23:24:57 -0500 Subject: [PATCH 08/21] Change throws to logs --- packages/dev/query/src/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index 9014586e136..005b2678d08 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -91,7 +91,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| cacheInfo.set('RequestGraph', [Buffer.byteLength(file)]); cacheInfo.get('RequestGraph')?.push(timeToDeserialize); } catch (e) { - throw new Error('Issue with identifying Request Graph'); + console.log('Issue with identifying Request Graph'); } } @@ -112,7 +112,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| cacheInfo.set('BundleGraph', [Buffer.byteLength(file)]); cacheInfo.get('BundleGraph')?.push(timeToDeserialize); } catch (e) { - throw new Error('Issue with identifying Bundle Graph'); + console.log('Issue with identifying Bundle Graph'); } } @@ -133,7 +133,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| cacheInfo.set('AssetGraph', [Buffer.byteLength(file)]); cacheInfo.get('AssetGraph')?.push(timeToDeserialize); } catch (e) { - throw new Error('Issue with identifying Asset Graph'); + console.log('Issue with identifying Asset Graph'); } } From 8eb9e38da31f8391491bdf80c793a4990cf7492e Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 00:14:06 -0500 Subject: [PATCH 09/21] Fix unit test --- packages/core/integration-tests/test/cache.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/core/integration-tests/test/cache.js b/packages/core/integration-tests/test/cache.js index 3bf1a51457b..25fb5098358 100644 --- a/packages/core/integration-tests/test/cache.js +++ b/packages/core/integration-tests/test/cache.js @@ -6214,11 +6214,12 @@ describe('cache', function () { getParcelOptions(entries, options), ); - let bundleGraphCacheKey = hashString( - `${version}:BundleGraph:${ - JSON.stringify(resolvedOptions.entries) ?? '' - }${resolvedOptions.mode}`, - ); + let bundleGraphCacheKey = + hashString( + `${version}:BundleGraph:${ + JSON.stringify(resolvedOptions.entries) ?? '' + }${resolvedOptions.mode}`, + ) + '-BundleGraph'; assert( deserialize( From 648cb54d2a0c1e459f0ae57f6dc3734ad2828445 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 00:14:19 -0500 Subject: [PATCH 10/21] Ignore log statement --- packages/dev/query/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index 005b2678d08..f3d94c0f884 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -1,5 +1,5 @@ // @flow strict-local -/* eslint-disable monorepo/no-internal-import */ +/* eslint-disable no-console monorepo/no-internal-import */ import type {ContentKey, NodeId} from '@parcel/graph'; import type {PackagedBundleInfo} from '@parcel/core/src/types'; From 8cd73c098abf31bb38ce5dbf1c55d964cc24b256 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 00:24:00 -0500 Subject: [PATCH 11/21] Fix lint comment --- packages/dev/query/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index f3d94c0f884..e01e6669c73 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -1,5 +1,5 @@ // @flow strict-local -/* eslint-disable no-console monorepo/no-internal-import */ +/* eslint-disable no-console, monorepo/no-internal-import */ import type {ContentKey, NodeId} from '@parcel/graph'; import type {PackagedBundleInfo} from '@parcel/core/src/types'; From 9176ddebc6d1eedd6d681b0f9017784e0134db2c Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 08:37:35 -0500 Subject: [PATCH 12/21] Better error logging --- packages/dev/query/src/index.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index e01e6669c73..d83a5415584 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -91,7 +91,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| cacheInfo.set('RequestGraph', [Buffer.byteLength(file)]); cacheInfo.get('RequestGraph')?.push(timeToDeserialize); } catch (e) { - console.log('Issue with identifying Request Graph'); + console.log('Error loading Request Graph\n', e); } } @@ -112,7 +112,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| cacheInfo.set('BundleGraph', [Buffer.byteLength(file)]); cacheInfo.get('BundleGraph')?.push(timeToDeserialize); } catch (e) { - console.log('Issue with identifying Bundle Graph'); + console.log('Error loading Bundle Graph\n', e); } } @@ -133,7 +133,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| cacheInfo.set('AssetGraph', [Buffer.byteLength(file)]); cacheInfo.get('AssetGraph')?.push(timeToDeserialize); } catch (e) { - console.log('Issue with identifying Asset Graph'); + console.log('Error loading Asset Graph\n', e); } } @@ -145,7 +145,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| // Load graphs by finding the main subrequests and loading their results let bundleInfo; - if (requestTracker) { + try { let buildRequestId = requestTracker.graph.getNodeIdByContentKey( 'parcel_build_request', ); @@ -168,6 +168,8 @@ export async function loadGraphs(cacheDir: string): Promise<{| PackagedBundleInfo, >); } + } catch (e) { + console.log('Error loading bundleInfo\n', e); } return {assetGraph, bundleGraph, requestTracker, bundleInfo, cacheInfo}; From 928762f3a3f60793e4332cd20cad8492612c0570 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 08:45:49 -0500 Subject: [PATCH 13/21] Make flow happy --- packages/dev/query/src/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index d83a5415584..a68702bb9fd 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -146,6 +146,7 @@ export async function loadGraphs(cacheDir: string): Promise<{| // Load graphs by finding the main subrequests and loading their results let bundleInfo; try { + invariant(requestTracker); let buildRequestId = requestTracker.graph.getNodeIdByContentKey( 'parcel_build_request', ); From 5a9b01a7122ac2f60697af7645a44fc2fa53379e Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 09:40:16 -0500 Subject: [PATCH 14/21] Initial splitting of functions --- packages/dev/query/src/index.js | 127 +++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 45 deletions(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index a68702bb9fd..39ca5a45f06 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -20,53 +20,47 @@ const { LMDBCache, } = require('./deep-imports.js'); -export async function loadGraphs(cacheDir: string): Promise<{| - assetGraph: ?AssetGraph, - bundleGraph: ?BundleGraph, - requestTracker: ?RequestTracker, - bundleInfo: ?Map, - cacheInfo: ?Map>, -|}> { - function filesByTypeAndModifiedTime() { - let files = fs.readdirSync(cacheDir); - - let requestGraphFiles = []; - let bundleGraphFiles = []; - let assetGraphFiles = []; - - files.forEach(f => { - if (f.endsWith('-0')) { - let stat = fs.statSync(path.join(cacheDir, f)); - let info = [path.join(cacheDir, f), stat.size, stat.mtime]; - - if (f.endsWith('RequestGraph-0')) { - requestGraphFiles.push(info); - } else if (f.endsWith('BundleGraph-0')) { - bundleGraphFiles.push(info); - } else if (f.endsWith('AssetGraph-0')) { - assetGraphFiles.push(info); - } +let cacheInfo: Map> = new Map(); + +function filesByTypeAndModifiedTime(cacheDir: string) { + let files = fs.readdirSync(cacheDir); + + let requestGraphFiles = []; + let bundleGraphFiles = []; + let assetGraphFiles = []; + + files.forEach(f => { + if (f.endsWith('-0')) { + let stat = fs.statSync(path.join(cacheDir, f)); + let info = [path.join(cacheDir, f), stat.size, stat.mtime]; + + if (f.endsWith('RequestGraph-0')) { + requestGraphFiles.push(info); + } else if (f.endsWith('BundleGraph-0')) { + bundleGraphFiles.push(info); + } else if (f.endsWith('AssetGraph-0')) { + assetGraphFiles.push(info); } - }); - - requestGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime); - bundleGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime); - assetGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime); - - return { - requestGraphFiles: requestGraphFiles.map(([f]) => f), - bundleGraphFiles: bundleGraphFiles.map(([f]) => f), - assetGraphFiles: assetGraphFiles.map(([f]) => f), - }; - } + } + }); - let cacheInfo: Map> = new Map(); + requestGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime); + bundleGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime); + assetGraphFiles.sort(([, , aTime], [, , bTime]) => bTime - aTime); - let {requestGraphFiles, bundleGraphFiles, assetGraphFiles} = - filesByTypeAndModifiedTime(); - const cache = new LMDBCache(cacheDir); + return { + requestGraphFiles: requestGraphFiles.map(([f]) => f), + bundleGraphFiles: bundleGraphFiles.map(([f]) => f), + assetGraphFiles: assetGraphFiles.map(([f]) => f), + }; +} - // Get requestTracker +export async function loadRequestTracker( + requestGraphFiles: string[], + cache: LMDBCache, +): Promise<{| + requestTracker: ?RequestTracker, +|}> { let requestTracker; if (requestGraphFiles.length > 0) { try { @@ -90,12 +84,19 @@ export async function loadGraphs(cacheDir: string): Promise<{| timeToDeserialize += Date.now() - date; cacheInfo.set('RequestGraph', [Buffer.byteLength(file)]); cacheInfo.get('RequestGraph')?.push(timeToDeserialize); + return requestTracker; } catch (e) { console.log('Error loading Request Graph\n', e); } } +} - // Get bundleGraph +export async function loadBundleGraph( + bundleGraphFiles: string[], + cache: LMDBCache, +): Promise<{| + bundleGraph: ?BundleGraph, +|}> { let bundleGraph; if (bundleGraphFiles.length > 0) { try { @@ -111,12 +112,19 @@ export async function loadGraphs(cacheDir: string): Promise<{| cacheInfo.set('BundleGraph', [Buffer.byteLength(file)]); cacheInfo.get('BundleGraph')?.push(timeToDeserialize); + return bundleGraph; } catch (e) { console.log('Error loading Bundle Graph\n', e); } } +} - // Get assetGraph +export async function loadAssetGraph( + assetGraphFiles: string[], + cache: LMDBCache, +): Promise<{| + assetGraph: ?AssetGraph, +|}> { let assetGraph; if (assetGraphFiles.length > 0) { try { @@ -132,11 +140,16 @@ export async function loadGraphs(cacheDir: string): Promise<{| cacheInfo.set('AssetGraph', [Buffer.byteLength(file)]); cacheInfo.get('AssetGraph')?.push(timeToDeserialize); + return assetGraph; } catch (e) { console.log('Error loading Asset Graph\n', e); } } +} +export async function loadBundleInfo(requestTracker: RequestTracker): Promise<{| + bundleInfo: ?Map, +|}> { function getSubRequests(id: NodeId) { return requestTracker.graph .getNodeIdsConnectedFrom(id, requestGraphEdgeTypes.subrequest) @@ -168,10 +181,34 @@ export async function loadGraphs(cacheDir: string): Promise<{| ContentKey, PackagedBundleInfo, >); + return bundleInfo; } } catch (e) { console.log('Error loading bundleInfo\n', e); } +} + +export async function loadGraphs(cacheDir: string): Promise<{| + assetGraph: ?AssetGraph, + bundleGraph: ?BundleGraph, + requestTracker: ?RequestTracker, + bundleInfo: ?Map, + cacheInfo: ?Map>, +|}> { + let {requestGraphFiles, bundleGraphFiles, assetGraphFiles} = + filesByTypeAndModifiedTime(cacheDir); + const cache = new LMDBCache(cacheDir); + + let requestTracker = loadRequestTracker(requestGraphFiles, cache); + let bundleGraph = loadBundleGraph(bundleGraphFiles, cache); + let assetGraph = loadAssetGraph(assetGraphFiles, cache); + + requestTracker = await requestTracker; + let bundleInfo = loadBundleInfo(requestTracker); + bundleGraph = await bundleGraph; + assetGraph = await assetGraph; + + bundleInfo = await bundleInfo; return {assetGraph, bundleGraph, requestTracker, bundleInfo, cacheInfo}; } From fdafab5bf41018979d2844e550412f655fe2458b Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 11:58:48 -0500 Subject: [PATCH 15/21] Check individual function calls --- packages/dev/query/src/cli.js | 23 +++++++++++++++++++---- packages/dev/query/src/index.js | 15 ++++++++------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/dev/query/src/cli.js b/packages/dev/query/src/cli.js index a653791b303..f2dfcd96658 100644 --- a/packages/dev/query/src/cli.js +++ b/packages/dev/query/src/cli.js @@ -14,12 +14,20 @@ import {serialize} from 'v8'; // $FlowFixMe import {table} from 'table'; -import {loadGraphs} from './index.js'; +import { + filesByTypeAndModifiedTime, + loadAssetGraph, + loadBundleGraph, + loadBundleInfo, + loadGraphs, + loadRequestTracker, +} from './index.js'; const { BundleGraph: {bundleGraphEdgeTypes: bundleGraphEdgeTypes}, Priority, fromProjectPathRelative, + LMDBCache, } = require('./deep-imports.js'); export async function run(input: string[]) { @@ -38,9 +46,16 @@ export async function run(input: string[]) { process.exit(1); } - console.log('Loading graphs...'); - let {assetGraph, bundleGraph, bundleInfo, requestTracker, cacheInfo} = - await loadGraphs(cacheDir); + const cache = new LMDBCache(cacheDir); + let {requestGraphFiles, bundleGraphFiles, assetGraphFiles} = + filesByTypeAndModifiedTime(cacheDir); + let cacheInfo: Map> = new Map(); + + //let {assetGraph, bundleGraph, requestTracker, bundleInfo} = await loadGraphs(cache, requestGraphFiles, bundleGraphFiles, assetGraphFiles); + let bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); + let assetGraph = await loadAssetGraph(assetGraphFiles, cache); + let requestTracker = await loadRequestTracker(requestGraphFiles, cache); + let bundleInfo = await loadBundleInfo(requestTracker); function hasRequestTracker() { if (requestTracker == null) { diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index 39ca5a45f06..d5b10cbb77c 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -5,7 +5,7 @@ import type {PackagedBundleInfo} from '@parcel/core/src/types'; import fs from 'fs'; import path from 'path'; -import v8 from 'v8'; +import v8, {deserialize} from 'v8'; import nullthrows from 'nullthrows'; import invariant from 'assert'; @@ -22,7 +22,7 @@ const { let cacheInfo: Map> = new Map(); -function filesByTypeAndModifiedTime(cacheDir: string) { +export function filesByTypeAndModifiedTime(cacheDir: string) { let files = fs.readdirSync(cacheDir); let requestGraphFiles = []; @@ -188,17 +188,18 @@ export async function loadBundleInfo(requestTracker: RequestTracker): Promise<{| } } -export async function loadGraphs(cacheDir: string): Promise<{| +export async function loadGraphs( + cache: LMDBCache, + requestGraphFiles: string[], + bundleGraphFiles: string[], + assetGraphFiles: string[], +): Promise<{| assetGraph: ?AssetGraph, bundleGraph: ?BundleGraph, requestTracker: ?RequestTracker, bundleInfo: ?Map, cacheInfo: ?Map>, |}> { - let {requestGraphFiles, bundleGraphFiles, assetGraphFiles} = - filesByTypeAndModifiedTime(cacheDir); - const cache = new LMDBCache(cacheDir); - let requestTracker = loadRequestTracker(requestGraphFiles, cache); let bundleGraph = loadBundleGraph(bundleGraphFiles, cache); let assetGraph = loadAssetGraph(assetGraphFiles, cache); From 3437e7f3566c668f5baeaf9a0a3a0c7f905312af Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 13:15:03 -0500 Subject: [PATCH 16/21] [WIP] load required graph on request --- packages/dev/query/src/cli.js | 120 ++++++++++++++++++++++++++++------ 1 file changed, 101 insertions(+), 19 deletions(-) diff --git a/packages/dev/query/src/cli.js b/packages/dev/query/src/cli.js index f2dfcd96658..53e5e965f8b 100644 --- a/packages/dev/query/src/cli.js +++ b/packages/dev/query/src/cli.js @@ -52,43 +52,125 @@ export async function run(input: string[]) { let cacheInfo: Map> = new Map(); //let {assetGraph, bundleGraph, requestTracker, bundleInfo} = await loadGraphs(cache, requestGraphFiles, bundleGraphFiles, assetGraphFiles); - let bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); - let assetGraph = await loadAssetGraph(assetGraphFiles, cache); - let requestTracker = await loadRequestTracker(requestGraphFiles, cache); - let bundleInfo = await loadBundleInfo(requestTracker); - - function hasRequestTracker() { - if (requestTracker == null) { - console.error('Request Graph could not be found'); + // let bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); + // let assetGraph = await loadAssetGraph(assetGraphFiles, cache); + // let requestTracker = await loadRequestTracker(requestGraphFiles, cache); + // let bundleInfo = await loadBundleInfo(requestTracker); + + let assetGraph, bundleGraph, requestTracker, bundleInfo; + let assetGraphLoaded = false, + bundleGraphLoaded = false, + requestTrackerLoaded = false, + bundleInfoLoaded = false; + + async function hasRequestTracker() { + if (requestTracker) return true; + + if (!requestTrackerLoaded) { + console.log('Loading request tracker'); + requestTracker = await loadRequestTracker(requestGraphFiles, cache); + requestTrackerLoaded = true; + } + + if (!requestTracker) { + console.error('Request tracker could not be found'); return false; } + return true; } - function hasBundleGraph() { - if (bundleGraph == null) { - console.error('Bundle Graph could not be found'); + async function hasBundleGraph() { + if (bundleGraph) return true; + + if (!bundleGraphLoaded) { + console.log('Loading bundle graph'); + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); + // let timeToDeserialize, fileSize; + // ({bundleGraph, timeToDeserialize, fileSize}= await loadBundleGraph(bundleGraphFiles, cache)); + // cacheInfo.get('BundleGraph')?.push(fileSize); + // cacheInfo.get('BundleGraph')?.push(timeToDeserialize); + bundleGraphLoaded = true; + } + + if (!bundleGraph) { + console.error('Bundle graph could not be found'); return false; } + return true; } - function hasAssetGraph() { - if (assetGraph == null) { - console.error('Asset Graph could not be found'); + async function hasAssetGraph() { + if (assetGraph) return true; + + if (!assetGraphLoaded) { + console.log('Loading asset graph'); + assetGraph = await loadAssetGraph(assetGraphFiles, cache); + assetGraphLoaded = true; + } + + if (!assetGraph) { + console.error('Asset graph could not be found'); return false; } + return true; } - function hasBundleInfo() { - if (bundleInfo == null) { - console.error('Bundle Info could not be found'); + async function hasBundleInfo() { + if (bundleInfo) return true; + + if (!bundleInfo) { + console.log('Loading bundle info'); + + if (hasRequestTracker()) { + bundleInfo = await loadBundleInfo(requestTracker); + } + + bundleInfoLoaded = true; + } + + if (!bundleInfo) { + console.error('Bundle info could not be found'); return false; } + return true; } + // function hasRequestTracker() { + // if (requestTracker == null) { + // console.error('Request Graph could not be found'); + // return false; + // } + // return true; + // } + + // function hasBundleGraph() { + // if (bundleGraph == null) { + // console.error('Bundle Graph could not be found'); + // return false; + // } + // return true; + // } + + // function hasAssetGraph() { + // if (assetGraph == null) { + // console.error('Asset Graph could not be found'); + // return false; + // } + // return true; + // } + + // function hasBundleInfo() { + // if (bundleInfo == null) { + // console.error('Bundle Info could not be found'); + // return false; + // } + // return true; + // } + // ------------------------------------------------------- function getBundleFilePath(id: ContentKey) { @@ -449,8 +531,8 @@ export async function run(input: string[]) { } // eslint-disable-next-line no-unused-vars - function getBundles(_) { - if (!hasBundleGraph()) { + async function getBundles(_) { + if (!(await hasBundleGraph())) { return; } invariant(bundleGraph != null); From 850e07351842e330de80174509957a439e31468c Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 17:19:49 -0500 Subject: [PATCH 17/21] Edit load graph functions --- packages/dev/query/src/index.js | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index d5b10cbb77c..891b01b46fd 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -20,6 +20,7 @@ const { LMDBCache, } = require('./deep-imports.js'); +let assetGraph, bundleGraph, requestTracker, bundleInfo; let cacheInfo: Map> = new Map(); export function filesByTypeAndModifiedTime(cacheDir: string) { @@ -61,7 +62,10 @@ export async function loadRequestTracker( ): Promise<{| requestTracker: ?RequestTracker, |}> { - let requestTracker; + if (requestTracker) { + return requestTracker; + } + if (requestGraphFiles.length > 0) { try { let file = await cache.getLargeBlob( @@ -97,7 +101,10 @@ export async function loadBundleGraph( ): Promise<{| bundleGraph: ?BundleGraph, |}> { - let bundleGraph; + if (bundleGraph) { + return bundleGraph; + } + if (bundleGraphFiles.length > 0) { try { let file = await cache.getLargeBlob( @@ -125,7 +132,10 @@ export async function loadAssetGraph( ): Promise<{| assetGraph: ?AssetGraph, |}> { - let assetGraph; + if (assetGraph) { + return assetGraph; + } + if (assetGraphFiles.length > 0) { try { let file = await cache.getLargeBlob( @@ -156,8 +166,11 @@ export async function loadBundleInfo(requestTracker: RequestTracker): Promise<{| .map(n => nullthrows(requestTracker.graph.getNode(n))); } + if (bundleInfo) { + return bundleInfo; + } + // Load graphs by finding the main subrequests and loading their results - let bundleInfo; try { invariant(requestTracker); let buildRequestId = requestTracker.graph.getNodeIdByContentKey( @@ -200,12 +213,12 @@ export async function loadGraphs( bundleInfo: ?Map, cacheInfo: ?Map>, |}> { - let requestTracker = loadRequestTracker(requestGraphFiles, cache); - let bundleGraph = loadBundleGraph(bundleGraphFiles, cache); - let assetGraph = loadAssetGraph(assetGraphFiles, cache); + requestTracker = loadRequestTracker(requestGraphFiles, cache); + bundleGraph = loadBundleGraph(bundleGraphFiles, cache); + assetGraph = loadAssetGraph(assetGraphFiles, cache); requestTracker = await requestTracker; - let bundleInfo = loadBundleInfo(requestTracker); + bundleInfo = loadBundleInfo(requestTracker); bundleGraph = await bundleGraph; assetGraph = await assetGraph; From 28b9ad7b0342686ca6a93a4ebe2e8a48ee62a012 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 17:24:14 -0500 Subject: [PATCH 18/21] Load graphs on request --- packages/dev/query/src/cli.js | 348 +++++++++------------------------- 1 file changed, 88 insertions(+), 260 deletions(-) diff --git a/packages/dev/query/src/cli.js b/packages/dev/query/src/cli.js index 53e5e965f8b..77c5d14a798 100644 --- a/packages/dev/query/src/cli.js +++ b/packages/dev/query/src/cli.js @@ -50,145 +50,23 @@ export async function run(input: string[]) { let {requestGraphFiles, bundleGraphFiles, assetGraphFiles} = filesByTypeAndModifiedTime(cacheDir); let cacheInfo: Map> = new Map(); - - //let {assetGraph, bundleGraph, requestTracker, bundleInfo} = await loadGraphs(cache, requestGraphFiles, bundleGraphFiles, assetGraphFiles); - // let bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); - // let assetGraph = await loadAssetGraph(assetGraphFiles, cache); - // let requestTracker = await loadRequestTracker(requestGraphFiles, cache); - // let bundleInfo = await loadBundleInfo(requestTracker); - let assetGraph, bundleGraph, requestTracker, bundleInfo; - let assetGraphLoaded = false, - bundleGraphLoaded = false, - requestTrackerLoaded = false, - bundleInfoLoaded = false; - - async function hasRequestTracker() { - if (requestTracker) return true; - - if (!requestTrackerLoaded) { - console.log('Loading request tracker'); - requestTracker = await loadRequestTracker(requestGraphFiles, cache); - requestTrackerLoaded = true; - } - - if (!requestTracker) { - console.error('Request tracker could not be found'); - return false; - } - - return true; - } - - async function hasBundleGraph() { - if (bundleGraph) return true; - - if (!bundleGraphLoaded) { - console.log('Loading bundle graph'); - bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); - // let timeToDeserialize, fileSize; - // ({bundleGraph, timeToDeserialize, fileSize}= await loadBundleGraph(bundleGraphFiles, cache)); - // cacheInfo.get('BundleGraph')?.push(fileSize); - // cacheInfo.get('BundleGraph')?.push(timeToDeserialize); - bundleGraphLoaded = true; - } - - if (!bundleGraph) { - console.error('Bundle graph could not be found'); - return false; - } - - return true; - } - - async function hasAssetGraph() { - if (assetGraph) return true; - - if (!assetGraphLoaded) { - console.log('Loading asset graph'); - assetGraph = await loadAssetGraph(assetGraphFiles, cache); - assetGraphLoaded = true; - } - - if (!assetGraph) { - console.error('Asset graph could not be found'); - return false; - } - - return true; - } - - async function hasBundleInfo() { - if (bundleInfo) return true; - - if (!bundleInfo) { - console.log('Loading bundle info'); - - if (hasRequestTracker()) { - bundleInfo = await loadBundleInfo(requestTracker); - } - - bundleInfoLoaded = true; - } - - if (!bundleInfo) { - console.error('Bundle info could not be found'); - return false; - } - - return true; - } - - // function hasRequestTracker() { - // if (requestTracker == null) { - // console.error('Request Graph could not be found'); - // return false; - // } - // return true; - // } - - // function hasBundleGraph() { - // if (bundleGraph == null) { - // console.error('Bundle Graph could not be found'); - // return false; - // } - // return true; - // } - - // function hasAssetGraph() { - // if (assetGraph == null) { - // console.error('Asset Graph could not be found'); - // return false; - // } - // return true; - // } - - // function hasBundleInfo() { - // if (bundleInfo == null) { - // console.error('Bundle Info could not be found'); - // return false; - // } - // return true; - // } // ------------------------------------------------------- - function getBundleFilePath(id: ContentKey) { - if (!hasBundleInfo()) { - return; - } + async function getBundleFilePath(id: ContentKey) { + requestTracker = await loadRequestTracker(requestGraphFiles, cache); + bundleInfo = await loadBundleInfo(requestTracker); invariant(bundleInfo != null); return fromProjectPathRelative(nullthrows(bundleInfo.get(id)?.filePath)); } - function parseAssetLocator(v: string) { + async function parseAssetLocator(v: string) { let id: ?string = null; if (v.length === 16) { id = v; } else { - if (!hasBundleGraph()) { - return; - } + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); for (let [assetId, publicId] of bundleGraph._publicIdByAssetId) { if (publicId === v) { @@ -199,9 +77,7 @@ export async function run(input: string[]) { } if (id == null && v.length > 0) { - if (!hasAssetGraph()) { - return; - } + assetGraph = await loadAssetGraph(assetGraphFiles, cache); invariant(assetGraph != null); let assetRegex = new RegExp(v); for (let node of assetGraph.nodes.values()) { @@ -217,14 +93,12 @@ export async function run(input: string[]) { return id; } - function parseBundleLocator(v: string) { - if (!hasBundleGraph()) { - return; - } + async function parseBundleLocator(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let bundleRegex = new RegExp(v); for (let b of bundleGraph.getBundles()) { - let bundleFilePath = getBundleFilePath(b.id); + let bundleFilePath = await getBundleFilePath(b.id); if ( (bundleFilePath !== undefined && bundleRegex.test(bundleFilePath)) || b.id === v @@ -234,24 +108,20 @@ export async function run(input: string[]) { } } - function getAsset(v: string) { - let id: ?string = parseAssetLocator(v); + async function getAsset(v: string) { + let id: ?string = await parseAssetLocator(v); if (id == null) { console.log(null); } else { try { - if (!hasBundleGraph()) { - return; - } + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let asset = bundleGraph.getAssetById(id); console.log('Public id', bundleGraph.getAssetPublicId(asset)); console.log(asset); } catch (e) { - if (!hasAssetGraph()) { - return; - } + assetGraph = await loadAssetGraph(assetGraphFiles, cache); invariant(assetGraph != null); let node = nullthrows(assetGraph.getNodeByContentKey(id)); invariant(node.type === 'asset'); @@ -260,10 +130,8 @@ export async function run(input: string[]) { } } - function _findAssetNode(v: string) { - if (!hasAssetGraph()) { - return; - } + async function _findAssetNode(v: string) { + assetGraph = await loadAssetGraph(assetGraphFiles, cache); invariant(assetGraph != null); let assetRegex = new RegExp(v); for (let node of assetGraph.nodes.values()) { @@ -276,13 +144,11 @@ export async function run(input: string[]) { } } - function findAsset(v: string) { - let node = _findAssetNode(v); + async function findAsset(v: string) { + let node = await _findAssetNode(v); if (node) { try { - if (!hasBundleGraph()) { - return; - } + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); console.log( `${bundleGraph.getAssetPublicId( @@ -295,10 +161,9 @@ export async function run(input: string[]) { } } - function findAssetWithSymbol(local: string) { - if (!hasBundleGraph() || !hasAssetGraph()) { - return; - } + async function findAssetWithSymbol(local: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); + assetGraph = await loadAssetGraph(assetGraphFiles, cache); invariant(bundleGraph != null); invariant(assetGraph != null); let [, assetId, binding, ref] = nullthrows( @@ -351,7 +216,7 @@ export async function run(input: string[]) { if (symbol.local === local) { if (symbol.loc) { let locPath = symbol.loc.filePath; - let locAsset = _findAssetNode(String(locPath)); + let locAsset = await _findAssetNode(String(locPath)); if (locAsset != null) { try { console.log( @@ -383,17 +248,13 @@ export async function run(input: string[]) { } } - function getNodeAssetGraph(v: string) { - if (!hasAssetGraph()) { - return; - } + async function getNodeAssetGraph(v: string) { + assetGraph = await loadAssetGraph(assetGraphFiles, cache); invariant(assetGraph != null); console.log(assetGraph.getNodeByContentKey(v)); } - function getNodeBundleGraph(v: string) { - if (!hasBundleGraph()) { - return; - } + async function getNodeBundleGraph(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); console.log(bundleGraph._graph.getNodeByContentKey(v)); } @@ -423,13 +284,13 @@ export async function run(input: string[]) { } } - function _findEntries( + async function _findEntries( graph: | ContentGraph | ContentGraph, v: string, ) { - let asset = nullthrows(parseAssetLocator(v), 'Asset not found'); + let asset = nullthrows(await parseAssetLocator(v), 'Asset not found'); let paths = new Paths(graph.getNodeIdByContentKey(asset), ' '); let cb = (id, ctx, revisiting) => { @@ -472,34 +333,28 @@ export async function run(input: string[]) { }); } - function findEntriesAssetGraph(v: string) { - if (!hasAssetGraph()) { - return; - } + async function findEntriesAssetGraph(v: string) { + assetGraph = await loadAssetGraph(assetGraphFiles, cache); invariant(assetGraph != null); - _findEntries(assetGraph, v); + await _findEntries(assetGraph, v); } - function findEntriesBundleGraph(v: string) { - if (!hasBundleGraph()) { - return; - } + async function findEntriesBundleGraph(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); - _findEntries(bundleGraph._graph, v); + await _findEntries(bundleGraph._graph, v); } - function findEntries(v: string) { - findEntriesBundleGraph(v); + async function findEntries(v: string) { + await findEntriesBundleGraph(v); } - function getBundlesWithAsset(v: string) { - if (!hasBundleGraph()) { - return; - } + async function getBundlesWithAsset(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); - let asset = nullthrows(parseAssetLocator(v), 'Asset not found'); + let asset = nullthrows(await parseAssetLocator(v), 'Asset not found'); for (let b of bundleGraph.getBundlesWithAsset( bundleGraph.getAssetById(asset), )) { - let bundleFilePath = getBundleFilePath(b.id); + let bundleFilePath = await getBundleFilePath(b.id); if (bundleFilePath !== undefined) { console.log( `${b.id} ${bundleFilePath} ${ @@ -510,16 +365,14 @@ export async function run(input: string[]) { } } - function getBundlesWithDependency(v: string) { - if (!hasBundleGraph()) { - return; - } + async function getBundlesWithDependency(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let node = nullthrows(bundleGraph._graph.getNodeByContentKey(v)); invariant(node.type === 'dependency'); for (let b of bundleGraph.getBundlesWithDependency(node.value)) { - let bundleFilePath = getBundleFilePath(b.id); + let bundleFilePath = await getBundleFilePath(b.id); if (bundleFilePath !== undefined) { console.log( `${b.id} ${bundleFilePath} ${ @@ -532,12 +385,10 @@ export async function run(input: string[]) { // eslint-disable-next-line no-unused-vars async function getBundles(_) { - if (!(await hasBundleGraph())) { - return; - } + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); for (let b of bundleGraph.getBundles()) { - let bundleFilePath = getBundleFilePath(b.id); + let bundleFilePath = await getBundleFilePath(b.id); if (bundleFilePath !== undefined) { console.log( `${b.id} ${bundleFilePath} ${ @@ -548,10 +399,8 @@ export async function run(input: string[]) { } } - function getReferencingBundles(v: string) { - if (!hasBundleGraph()) { - return; - } + async function getReferencingBundles(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let bundleId = nullthrows(parseBundleLocator(v), 'Bundle not found'); let bundleNodeId = bundleGraph._graph.getNodeIdByContentKey(bundleId); @@ -562,7 +411,7 @@ export async function run(input: string[]) { invariant(bundleNode.type === 'bundle', 'Not a bundle'); for (let b of bundleGraph.getReferencingBundles(bundleNode.value)) { - let bundleFilePath = getBundleFilePath(b.id); + let bundleFilePath = await getBundleFilePath(b.id); if (bundleFilePath !== undefined) { console.log( `${b.id} ${bundleFilePath} ${ @@ -573,23 +422,19 @@ export async function run(input: string[]) { } } - function getIncomingDependenciesAssetGraph(v: string) { - if (!hasAssetGraph()) { - return; - } + async function getIncomingDependenciesAssetGraph(v: string) { + assetGraph = await loadAssetGraph(assetGraphFiles, cache); invariant(assetGraph != null); - let asset = nullthrows(parseAssetLocator(v), 'Asset not found'); + let asset = nullthrows(await parseAssetLocator(v), 'Asset not found'); let node = nullthrows(assetGraph.getNodeByContentKey(asset)); invariant(node.type === 'asset'); console.log(assetGraph.getIncomingDependencies(node.value)); } - function getIncomingDependenciesBundleGraph(v: string) { - if (!hasBundleGraph()) { - return; - } + async function getIncomingDependenciesBundleGraph(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); - let asset = nullthrows(parseAssetLocator(v), 'Asset not found'); + let asset = nullthrows(await parseAssetLocator(v), 'Asset not found'); let value = nullthrows(bundleGraph.getAssetById(asset)); console.log(bundleGraph.getIncomingDependencies(value)); @@ -599,10 +444,8 @@ export async function run(input: string[]) { getIncomingDependenciesBundleGraph(v); } - function getResolvedAsset(v: string) { - if (!hasBundleGraph()) { - return; - } + async function getResolvedAsset(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let node = nullthrows( bundleGraph._graph.getNodeByContentKey(v), @@ -615,10 +458,8 @@ export async function run(input: string[]) { console.log(bundleGraph.getResolvedAsset(node.value)); } - function getAssetWithDependency(v: string) { - if (!hasBundleGraph()) { - return; - } + async function getAssetWithDependency(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let node = nullthrows( bundleGraph._graph.getNodeByContentKey(v), @@ -631,10 +472,8 @@ export async function run(input: string[]) { console.log(bundleGraph.getAssetWithDependency(node.value)); } - function traverseAssets(v: string) { - if (!hasBundleGraph()) { - return; - } + async function traverseAssets(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let bundleId = nullthrows(parseBundleLocator(v), 'Bundle not found'); let node = nullthrows( @@ -650,10 +489,8 @@ export async function run(input: string[]) { console.log(asset.id, asset.filePath); }); } - function traverseBundle(v: string) { - if (!hasBundleGraph()) { - return; - } + async function traverseBundle(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let bundleId = nullthrows(parseBundleLocator(v), 'Bundle not found'); let node = nullthrows( @@ -683,27 +520,23 @@ export async function run(input: string[]) { }); } - function getBundle(v: string) { - if (!hasBundleGraph()) { - return; - } + async function getBundle(v: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let bundleRegex = new RegExp(v); for (let b of bundleGraph.getBundles()) { - let bundleFilePath = getBundleFilePath(b.id); + let bundleFilePath = await getBundleFilePath(b.id); if ( (bundleFilePath !== undefined && bundleRegex.test(bundleFilePath)) || b.id === v ) { - console.log(getBundleFilePath(b.id), b); + console.log(await getBundleFilePath(b.id), b); } } } - function findBundleReason(bundle: string, asset: string) { - if (!hasBundleGraph()) { - return; - } + async function findBundleReason(bundle: string, asset: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let bundleId = nullthrows(parseBundleLocator(bundle), 'Bundle not found'); let bundleNodeId = bundleGraph._graph.getNodeIdByContentKey(bundleId); @@ -712,7 +545,7 @@ export async function run(input: string[]) { 'Bundle not found', ); invariant(bundleNode.type === 'bundle', 'Not a bundle'); - let assetId = nullthrows(parseAssetLocator(asset), 'Asset not found'); + let assetId = nullthrows(await parseAssetLocator(asset), 'Asset not found'); let assetNodeId = bundleGraph._graph.getNodeIdByContentKey(assetId); let assetNode = nullthrows( bundleGraph._graph.getNode(assetNodeId), @@ -777,10 +610,8 @@ export async function run(input: string[]) { } } - function _getIncomingNodeOfType(bundleGraph, node, type: string) { - if (!hasBundleGraph()) { - return; - } + async function _getIncomingNodeOfType(bundleGraph, node, type: string) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); const bundleGraphNodeId = bundleGraph._graph.getNodeIdByContentKey(node.id); return bundleGraph._graph @@ -792,17 +623,15 @@ export async function run(input: string[]) { // We find the priority of a Bundle or BundleGroup by looking at its incoming dependencies. // If a Bundle does not have an incoming dependency, we look for an incoming BundleGroup and its dependency // e.g. Dep(priority = 1) -> BundleGroup -> Bundle means that the Bundle has priority 1. - function _getBundlePriority(bundleGraph, bundle) { - if (!hasBundleGraph()) { - return; - } + async function _getBundlePriority(bundleGraph, bundle) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); - let node = _getIncomingNodeOfType(bundleGraph, bundle, 'dependency'); + let node = await _getIncomingNodeOfType(bundleGraph, bundle, 'dependency'); if (node == null) { - node = _getIncomingNodeOfType(bundleGraph, bundle, 'bundle_group'); + node = await _getIncomingNodeOfType(bundleGraph, bundle, 'bundle_group'); if (node == null) return null; - node = _getIncomingNodeOfType(bundleGraph, node, 'dependency'); + node = await _getIncomingNodeOfType(bundleGraph, node, 'dependency'); } if (node == null) return null; @@ -812,10 +641,8 @@ export async function run(input: string[]) { return node.value.priority; } - function _findEntryBundle(bundleGraph, node) { - if (!hasBundleGraph()) { - return; - } + async function _findEntryBundle(bundleGraph, node) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); const bundleGraphNodeId = bundleGraph._graph.getNodeIdByContentKey(node.id); const entryBundleGroup = bundleGraph._graph @@ -898,14 +725,15 @@ export async function run(input: string[]) { } // eslint-disable-next-line no-unused-vars - function stats(_) { + async function stats(_) { let ag = { asset: 0, dependency: 0, asset_group: 0, }; - if (hasAssetGraph()) { + assetGraph = await loadAssetGraph(assetGraphFiles, cache); + if (assetGraph) { invariant(assetGraph != null); for (let n of assetGraph.nodes) { if (n && n.type in ag) { @@ -916,9 +744,7 @@ export async function run(input: string[]) { _printStatsTable('# Asset Graph Node Counts', Object.entries(ag)); } - if (!hasBundleGraph()) { - return; - } + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); invariant(bundleGraph != null); let bg = { dependency: 0, @@ -950,7 +776,7 @@ export async function run(input: string[]) { b_ext[n.value.type] = (b_ext[n.value.type] || 0) + 1; // $FlowFixMe - const entry_group = _findEntryBundle(bundleGraph, n); + const entry_group = await _findEntryBundle(bundleGraph, n); if (entry_group != null && !entries.has(entry_group.id)) { b_type.entry++; @@ -959,7 +785,7 @@ export async function run(input: string[]) { // In general, !bundle.mainEntryId means that it is shared. In the case of an async and shared bundle, only count it as shared. b_type.shared++; } else { - const priority = _getBundlePriority(bundleGraph, n); + const priority = await _getBundlePriority(bundleGraph, n); if (priority == Priority.lazy) { b_type.async++; @@ -1012,8 +838,10 @@ export async function run(input: string[]) { // ------------------------------------------------------- if (initialCmd != null) { - eval(initialCmd); - process.exit(0); + (async () => { + await eval(initialCmd); + process.exit(0); + })(); } else { console.log( 'See .help. The graphs can be accessed via `assetGraph`, `bundleGraph` and `requestTracker`.', From 2b2d457f5e6f00acaa81e11e50d90defca234cdf Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Thu, 14 Dec 2023 18:31:33 -0500 Subject: [PATCH 19/21] Handle cacheInfo --- packages/dev/query/src/cli.js | 31 ++++++++++------------------- packages/dev/query/src/index.js | 35 ++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/packages/dev/query/src/cli.js b/packages/dev/query/src/cli.js index 77c5d14a798..f9bc5d82b0c 100644 --- a/packages/dev/query/src/cli.js +++ b/packages/dev/query/src/cli.js @@ -19,6 +19,7 @@ import { loadAssetGraph, loadBundleGraph, loadBundleInfo, + loadCacheInfo, loadGraphs, loadRequestTracker, } from './index.js'; @@ -657,8 +658,14 @@ export async function run(input: string[]) { return entryBundleGroup; } // eslint-disable-next-line no-unused-vars - function inspectCache(_) { + async function inspectCache(_) { // displays sizing of various entries of the cache + cacheInfo = await loadCacheInfo( + requestGraphFiles, + bundleGraphFiles, + assetGraphFiles, + cache, + ); let table: Array> = []; table.push([ 'Graphs', @@ -666,21 +673,9 @@ export async function run(input: string[]) { 'Deserialize (ms)', 'Serialize (ms)', ]); - let serialized: Map = new Map(); - serialized.set('RequestGraph', timeSerialize(requestTracker)); - serialized.set('BundleGraph', timeSerialize(bundleGraph)); - serialized.set('AssetGraph', timeSerialize(assetGraph)); + for (let [name, info] of nullthrows(cacheInfo).entries()) { - if ( - (name === 'RequestGraph' && !hasRequestTracker()) || - (name === 'BundleGraph' && !hasBundleGraph()) || - (name === 'AssetGraph' && !hasAssetGraph()) - ) { - continue; - } - let s = serialized.get(name); - invariant(s != null); - table.push([name, ...info, s]); + table.push([name, ...info]); } function getColumnSum(t: Array>, col: number) { if (t == null) { @@ -704,12 +699,6 @@ export async function run(input: string[]) { _printStatsTable('Cache Info', table); } - function timeSerialize(graph) { - let date = Date.now(); - serialize(graph); - date = Date.now() - date; - return date; - } function _printStatsTable(header, data) { const config = { columnDefault: { diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index 891b01b46fd..608fb83267a 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -5,7 +5,7 @@ import type {PackagedBundleInfo} from '@parcel/core/src/types'; import fs from 'fs'; import path from 'path'; -import v8, {deserialize} from 'v8'; +import v8, {deserializ, serialize} from 'v8'; import nullthrows from 'nullthrows'; import invariant from 'assert'; @@ -201,6 +201,39 @@ export async function loadBundleInfo(requestTracker: RequestTracker): Promise<{| } } +function timeSerialize(graph) { + let date = Date.now(); + serialize(graph); + date = Date.now() - date; + return date; +} + +export async function loadCacheInfo( + requestGraphFiles: string[], + bundleGraphFiles: string[], + assetGraphFiles: string[], + cache: LMDBCache, +): Promise<{| + cacheInfo: ?Map>, +|}> { + if (!requestTracker) { + await loadRequestTracker(requestGraphFiles, cache); + cacheInfo.get('RequestGraph')?.push(timeSerialize(requestTracker)); + } + + if (!bundleGraph) { + bundleGraph = await loadBundleGraph(bundleGraphFiles, cache); + cacheInfo.get('BundleGraph')?.push(timeSerialize(bundleGraph)); + } + + if (!assetGraph) { + assetGraph = await loadAssetGraph(assetGraphFiles, cache); + cacheInfo.get('AssetGraph')?.push(timeSerialize(assetGraph)); + } + + return cacheInfo; +} + export async function loadGraphs( cache: LMDBCache, requestGraphFiles: string[], From e780f709ce7ba9dc6e026b5402cff812290df661 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Fri, 15 Dec 2023 12:26:30 -0500 Subject: [PATCH 20/21] Fix linting errors --- packages/dev/query/src/cli.js | 6 ++---- packages/dev/query/src/index.js | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/dev/query/src/cli.js b/packages/dev/query/src/cli.js index f9bc5d82b0c..5c45611f430 100644 --- a/packages/dev/query/src/cli.js +++ b/packages/dev/query/src/cli.js @@ -10,7 +10,6 @@ import repl from 'repl'; import os from 'os'; import nullthrows from 'nullthrows'; import invariant from 'assert'; -import {serialize} from 'v8'; // $FlowFixMe import {table} from 'table'; @@ -20,7 +19,6 @@ import { loadBundleGraph, loadBundleInfo, loadCacheInfo, - loadGraphs, loadRequestTracker, } from './index.js'; @@ -31,7 +29,7 @@ const { LMDBCache, } = require('./deep-imports.js'); -export async function run(input: string[]) { +export function run(input: string[]) { let args = input; let cacheDir = path.join(process.cwd(), '.parcel-cache'); if (args[0] === '--cache') { @@ -57,7 +55,7 @@ export async function run(input: string[]) { async function getBundleFilePath(id: ContentKey) { requestTracker = await loadRequestTracker(requestGraphFiles, cache); - bundleInfo = await loadBundleInfo(requestTracker); + bundleInfo = loadBundleInfo(requestTracker); invariant(bundleInfo != null); return fromProjectPathRelative(nullthrows(bundleInfo.get(id)?.filePath)); } diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index 608fb83267a..f1319a8eb0c 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -5,7 +5,7 @@ import type {PackagedBundleInfo} from '@parcel/core/src/types'; import fs from 'fs'; import path from 'path'; -import v8, {deserializ, serialize} from 'v8'; +import v8, {serialize} from 'v8'; import nullthrows from 'nullthrows'; import invariant from 'assert'; @@ -157,7 +157,7 @@ export async function loadAssetGraph( } } -export async function loadBundleInfo(requestTracker: RequestTracker): Promise<{| +export function loadBundleInfo(requestTracker: RequestTracker): Promise<{| bundleInfo: ?Map, |}> { function getSubRequests(id: NodeId) { From c8c86f2c571134889a0018290ae5470966373af2 Mon Sep 17 00:00:00 2001 From: Iris Moini Date: Fri, 15 Dec 2023 12:49:00 -0500 Subject: [PATCH 21/21] Fix flow error --- packages/dev/query/src/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/dev/query/src/index.js b/packages/dev/query/src/index.js index f1319a8eb0c..89f273901b8 100644 --- a/packages/dev/query/src/index.js +++ b/packages/dev/query/src/index.js @@ -157,9 +157,9 @@ export async function loadAssetGraph( } } -export function loadBundleInfo(requestTracker: RequestTracker): Promise<{| - bundleInfo: ?Map, -|}> { +export function loadBundleInfo( + requestTracker: RequestTracker, +): ?Map { function getSubRequests(id: NodeId) { return requestTracker.graph .getNodeIdsConnectedFrom(id, requestGraphEdgeTypes.subrequest)