diff --git a/e2e-tests/adapters/package.json b/e2e-tests/adapters/package.json index 7ddcfda4f06f3..ec2af81f43678 100644 --- a/e2e-tests/adapters/package.json +++ b/e2e-tests/adapters/package.json @@ -14,14 +14,14 @@ "develop:debug": "start-server-and-test develop http://localhost:8000 'npm run cy:open -- --config baseUrl=http://localhost:8000'", "ssat:debug": "start-server-and-test serve http://localhost:9000 cy:open", "test:template": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX node ../../scripts/cypress-run-with-conditional-record-flag.js --browser chrome --e2e --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX", - "test:template:debug": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX npm run cy:open -- --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX", + "test:template:debug": "cross-env-shell CYPRESS_GROUP_NAME=\"adapter:$ADAPTER / trailingSlash:${TRAILING_SLASH:-always} / pathPrefix:${PATH_PREFIX:--} / excludeDatastoreFromBundle:${GATSBY_EXCLUDE_DATASTORE_FROM_BUNDLE:-false}\" TRAILING_SLASH=$TRAILING_SLASH PATH_PREFIX=$PATH_PREFIX npm run cy:open -- --config-file \"cypress/configs/$ADAPTER.ts\" --env TRAILING_SLASH=$TRAILING_SLASH,PATH_PREFIX=$PATH_PREFIX", "test:debug": "npm-run-all -s build:debug ssat:debug", "test:netlify": "cross-env TRAILING_SLASH=always node scripts/deploy-and-run/netlify.mjs test:template", "test:smoke": "node smoke-test.mjs", "test:netlify:debug": "cross-env TRAILING_SLASH=always node scripts/deploy-and-run/netlify.mjs test:template:debug", - "test:netlify:prefix-never": "cross-env TRAILING_SLASH=never PATH_PREFIX=/prefix node scripts/deploy-and-run/netlify.mjs test:template", - "test:netlify:prefix-never:debug": "cross-env TRAILING_SLASH=never PATH_PREFIX=/prefix node scripts/deploy-and-run/netlify.mjs test:template:debug", - "test": "npm-run-all -c -s test:netlify test:netlify:prefix-never" + "test:netlify:non-defaults-variant": "cross-env TRAILING_SLASH=never PATH_PREFIX=/prefix GATSBY_EXCLUDE_DATASTORE_FROM_BUNDLE=true node scripts/deploy-and-run/netlify.mjs test:template", + "test:netlify:non-defaults-variant:debug": "cross-env TRAILING_SLASH=never PATH_PREFIX=/prefix GATSBY_EXCLUDE_DATASTORE_FROM_BUNDLE=true node scripts/deploy-and-run/netlify.mjs test:template:debug", + "test": "npm-run-all -c -s test:netlify test:netlify:non-defaults-variant" }, "dependencies": { "gatsby": "next", diff --git a/packages/gatsby-adapter-netlify/src/lambda-handler.ts b/packages/gatsby-adapter-netlify/src/lambda-handler.ts index ef70257ba2c2e..f19cd5de358d1 100644 --- a/packages/gatsby-adapter-netlify/src/lambda-handler.ts +++ b/packages/gatsby-adapter-netlify/src/lambda-handler.ts @@ -176,11 +176,13 @@ const createRequestObject = ({ event, context }) => { multiValueHeaders = {}, body, isBase64Encoded, + rawUrl } = event const newStream = new Stream.Readable() const req = Object.assign(newStream, http.IncomingMessage.prototype) req.url = path req.originalUrl = req.url + req.rawUrl = rawUrl req.query = queryStringParameters req.multiValueQuery = multiValueQueryStringParameters req.method = httpMethod diff --git a/packages/gatsby/index.d.ts b/packages/gatsby/index.d.ts index 6ef0ef1a31c02..6b7af9f0a86b0 100644 --- a/packages/gatsby/index.d.ts +++ b/packages/gatsby/index.d.ts @@ -1869,6 +1869,10 @@ export interface GatsbyFunctionRequest extends IncomingMessage { * Object of `cookies` from header */ cookies: Record + /** + * Optional field to store the full raw URL by adapters + */ + rawUrl?: string } export interface GatsbyFunctionBodyParserCommonMiddlewareConfig { diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts index 489d99ada5c6f..a4b5ca390a9a3 100644 --- a/packages/gatsby/src/commands/build.ts +++ b/packages/gatsby/src/commands/build.ts @@ -191,37 +191,6 @@ module.exports = async function build( buildActivityTimer.end() } - if (shouldGenerateEngines()) { - const state = store.getState() - const buildActivityTimer = report.activityTimer( - `Building Rendering Engines`, - { parentSpan: buildSpan } - ) - try { - buildActivityTimer.start() - // bundle graphql-engine - engineBundlingPromises.push( - createGraphqlEngineBundle(program.directory, report, program.verbose) - ) - - engineBundlingPromises.push( - createPageSSRBundle({ - rootDir: program.directory, - components: state.components, - staticQueriesByTemplate: state.staticQueriesByTemplate, - webpackCompilationHash: webpackCompilationHash as string, // we set webpackCompilationHash above - reporter: report, - isVerbose: program.verbose, - }) - ) - await Promise.all(engineBundlingPromises) - } catch (err) { - reporter.panic(err) - } finally { - buildActivityTimer.end() - } - } - const buildSSRBundleActivityProgress = report.activityTimer( `Building HTML renderer`, { parentSpan: buildSpan } @@ -295,6 +264,35 @@ module.exports = async function build( } if (shouldGenerateEngines()) { + const state = store.getState() + const buildActivityTimer = report.activityTimer( + `Building Rendering Engines`, + { parentSpan: buildSpan } + ) + try { + buildActivityTimer.start() + // bundle graphql-engine + engineBundlingPromises.push( + createGraphqlEngineBundle(program.directory, report, program.verbose) + ) + + engineBundlingPromises.push( + createPageSSRBundle({ + rootDir: program.directory, + components: state.components, + staticQueriesByTemplate: state.staticQueriesByTemplate, + webpackCompilationHash: webpackCompilationHash as string, // we set webpackCompilationHash above + reporter: report, + isVerbose: program.verbose, + }) + ) + await Promise.all(engineBundlingPromises) + } catch (err) { + reporter.panic(err) + } finally { + buildActivityTimer.end() + } + await validateEnginesWithActivity(program.directory, buildSpan) } diff --git a/packages/gatsby/src/commands/serve.ts b/packages/gatsby/src/commands/serve.ts index 1727e2c0c420e..415b689806d6b 100644 --- a/packages/gatsby/src/commands/serve.ts +++ b/packages/gatsby/src/commands/serve.ts @@ -202,7 +202,7 @@ module.exports = async (program: IServeProgram): Promise => { try { const { GraphQLEngine } = require(graphqlEnginePath) as typeof import("../schema/graphql-engine/entry") - const { getData, renderPageData, renderHTML } = + const { getData, renderPageData, renderHTML, findEnginePageByPath } = require(pageSSRModule) as typeof import("../utils/page-ssr-module/entry") const graphqlEngine = new GraphQLEngine({ dbPath: path.posix.join( @@ -222,7 +222,7 @@ module.exports = async (program: IServeProgram): Promise => { } const potentialPagePath = reverseFixedPagePath(requestedPagePath) - const page = graphqlEngine.findPageByPath(potentialPagePath) + const page = findEnginePageByPath(potentialPagePath) if (page && (page.mode === `DSG` || page.mode === `SSR`)) { const requestActivity = report.phantomActivity( @@ -272,7 +272,7 @@ module.exports = async (program: IServeProgram): Promise => { router.use(async (req, res, next) => { if (req.accepts(`html`)) { const potentialPagePath = req.path - const page = graphqlEngine.findPageByPath(potentialPagePath) + const page = findEnginePageByPath(potentialPagePath) if (page && (page.mode === `DSG` || page.mode === `SSR`)) { const requestActivity = report.phantomActivity( `request for "${req.path}"` diff --git a/packages/gatsby/src/schema/graphql-engine/entry.ts b/packages/gatsby/src/schema/graphql-engine/entry.ts index c44ce3dcaded6..039d67bf16cf9 100644 --- a/packages/gatsby/src/schema/graphql-engine/entry.ts +++ b/packages/gatsby/src/schema/graphql-engine/entry.ts @@ -195,6 +195,9 @@ export class GraphQLEngine { } } + /** + * @deprecated use findEnginePageByPath exported from page-ssr module instead + */ public findPageByPath(pathName: string): IGatsbyPage | undefined { // adapter so `findPageByPath` use SitePage nodes in datastore // instead of `pages` redux slice diff --git a/packages/gatsby/src/utils/adapter/get-route-path.ts b/packages/gatsby/src/utils/adapter/get-route-path.ts index a4018bc19684b..95a9ae60fc6cc 100644 --- a/packages/gatsby/src/utils/adapter/get-route-path.ts +++ b/packages/gatsby/src/utils/adapter/get-route-path.ts @@ -11,7 +11,9 @@ function maybeDropNamedPartOfWildcard( return path.replace(/\*.+$/, `*`) } -export function getRoutePathFromPage(page: IGatsbyPage): string { +export function getRoutePathFromPage( + page: Pick +): string { return maybeDropNamedPartOfWildcard(page.matchPath) ?? page.path } diff --git a/packages/gatsby/src/utils/adapter/manager.ts b/packages/gatsby/src/utils/adapter/manager.ts index 7879565715030..a159fa1d4b5db 100644 --- a/packages/gatsby/src/utils/adapter/manager.ts +++ b/packages/gatsby/src/utils/adapter/manager.ts @@ -26,7 +26,7 @@ import { getPageMode } from "../page-mode" import { getStaticQueryPath } from "../static-query-utils" import { getAdapterInit } from "./init" import { - LmdbOnCdnPath, + getLmdbOnCdnPath, shouldBundleDatastore, shouldGenerateEngines, } from "../engines-helpers" @@ -192,7 +192,7 @@ export async function initAdapterManager(): Promise { } // handle lmdb file - const mdbInPublicPath = `public/${LmdbOnCdnPath}` + const mdbInPublicPath = `public/${getLmdbOnCdnPath()}` if (!shouldBundleDatastore()) { const mdbPath = getDefaultDbPath() + `/data.mdb` copy(mdbPath, mdbInPublicPath) diff --git a/packages/gatsby/src/utils/engines-helpers.ts b/packages/gatsby/src/utils/engines-helpers.ts index 0e01e5e6eb7de..ab7c254d18916 100644 --- a/packages/gatsby/src/utils/engines-helpers.ts +++ b/packages/gatsby/src/utils/engines-helpers.ts @@ -32,7 +32,7 @@ function getCDNObfuscatedPath(path: string): string { return `${store.getState().status.cdnObfuscatedPrefix}-${path}` } -export const LmdbOnCdnPath = getCDNObfuscatedPath(`data.mdb`) +export const getLmdbOnCdnPath = (): string => getCDNObfuscatedPath(`data.mdb`) export interface IPlatformAndArch { platform: string diff --git a/packages/gatsby/src/utils/get-server-data.ts b/packages/gatsby/src/utils/get-server-data.ts index 730549be4c1b8..7fecfbca269e4 100644 --- a/packages/gatsby/src/utils/get-server-data.ts +++ b/packages/gatsby/src/utils/get-server-data.ts @@ -23,7 +23,7 @@ export async function getServerData( req: | Partial> | undefined, - page: IGatsbyPage, + page: Pick, pagePath: string, mod: IModuleWithServerData | undefined ): Promise { diff --git a/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts b/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts index c0641a5336d84..ba42ba25e0961 100644 --- a/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts +++ b/packages/gatsby/src/utils/page-ssr-module/bundle-webpack.ts @@ -4,7 +4,7 @@ import webpack from "webpack" import mod from "module" import { WebpackLoggingPlugin } from "../../utils/webpack/plugins/webpack-logging" import reporter from "gatsby-cli/lib/reporter" -import type { ITemplateDetails } from "./entry" +import type { EnginePage, ITemplateDetails } from "./entry" import { getScriptsAndStylesForTemplate, @@ -12,7 +12,8 @@ import { } from "../client-assets-for-template" import { IGatsbyState } from "../../redux/types" import { store } from "../../redux" -import { LmdbOnCdnPath, shouldBundleDatastore } from "../engines-helpers" +import { getLmdbOnCdnPath, shouldBundleDatastore } from "../engines-helpers" +import { getPageMode } from "../page-mode" type Reporter = typeof reporter @@ -112,6 +113,25 @@ export async function createPageSSRBundle({ } } + const pagesIterable: Array<[string, EnginePage]> = [] + for (const [pagePath, page] of state.pages) { + const mode = getPageMode(page, state) + if (mode !== `SSG`) { + pagesIterable.push([ + pagePath, + { + componentChunkName: page.componentChunkName, + componentPath: page.componentPath, + context: page.context, + matchPath: page.matchPath, + mode, + path: page.path, + slices: page.slices, + }, + ]) + } + } + const compiler = webpack({ name: `Page Engine`, mode: `none`, @@ -193,6 +213,7 @@ export async function createPageSSRBundle({ INLINED_TEMPLATE_TO_DETAILS: JSON.stringify(toInline), INLINED_HEADERS_CONFIG: JSON.stringify(state.config.headers), WEBPACK_COMPILATION_HASH: JSON.stringify(webpackCompilationHash), + GATSBY_PAGES: JSON.stringify(pagesIterable), GATSBY_SLICES: JSON.stringify(slicesStateObject), GATSBY_SLICES_BY_TEMPLATE: JSON.stringify(slicesByTemplateStateObject), GATSBY_SLICES_SCRIPT: JSON.stringify( @@ -248,9 +269,11 @@ export async function createPageSSRBundle({ functionCode = functionCode .replaceAll( `%CDN_DATASTORE_PATH%`, - shouldBundleDatastore() - ? `` - : `${state.adapter.config.deployURL ?? ``}/${LmdbOnCdnPath}` + shouldBundleDatastore() ? `` : getLmdbOnCdnPath() + ) + .replaceAll( + `%CDN_DATASTORE_ORIGIN%`, + shouldBundleDatastore() ? `` : state.adapter.config.deployURL ?? `` ) .replaceAll(`%PATH_PREFIX%`, pathPrefix) .replaceAll( diff --git a/packages/gatsby/src/utils/page-ssr-module/entry.ts b/packages/gatsby/src/utils/page-ssr-module/entry.ts index de6c333f53d13..0d74eb0914f66 100644 --- a/packages/gatsby/src/utils/page-ssr-module/entry.ts +++ b/packages/gatsby/src/utils/page-ssr-module/entry.ts @@ -34,15 +34,28 @@ import { ICollectedSlice } from "../babel/find-slices" import { createHeadersMatcher } from "../adapter/create-headers" import { MUST_REVALIDATE_HEADERS } from "../adapter/constants" import { getRoutePathFromPage } from "../adapter/get-route-path" +import { findPageByPath } from "../find-page-by-path" export interface ITemplateDetails { query: string staticQueryHashes: Array assets: IScriptsAndStyles } + +export type EnginePage = Pick< + IGatsbyPage, + | "componentChunkName" + | "componentPath" + | "context" + | "matchPath" + | "mode" + | "path" + | "slices" +> + export interface ISSRData { results: IExecutionResult - page: IGatsbyPage + page: EnginePage templateDetails: ITemplateDetails potentialPagePath: string /** @@ -61,6 +74,7 @@ declare global { const INLINED_HEADERS_CONFIG: Array | undefined const WEBPACK_COMPILATION_HASH: string const GATSBY_SLICES_SCRIPT: string + const GATSBY_PAGES: Array<[string, EnginePage]> } const tracerReadyPromise = initTracer( @@ -73,19 +87,35 @@ type MaybePhantomActivity = const createHeaders = createHeadersMatcher(INLINED_HEADERS_CONFIG) -export async function getData({ - pathName, - graphqlEngine, - req, - spanContext, - telemetryResolverTimings, -}: { - graphqlEngine: GraphQLEngine +interface IGetDataBaseArgs { pathName: string req?: Partial> spanContext?: Span | SpanContext telemetryResolverTimings?: Array -}): Promise { +} + +interface IGetDataEagerEngineArgs extends IGetDataBaseArgs { + graphqlEngine: GraphQLEngine +} + +interface IGetDataLazyEngineArgs extends IGetDataBaseArgs { + getGraphqlEngine: () => Promise +} + +type IGetDataArgs = IGetDataEagerEngineArgs | IGetDataLazyEngineArgs + +function isEagerGraphqlEngine( + arg: IGetDataArgs +): arg is IGetDataEagerEngineArgs { + return typeof (arg as IGetDataEagerEngineArgs).graphqlEngine !== `undefined` +} + +export async function getData(arg: IGetDataArgs): Promise { + const getGraphqlEngine = isEagerGraphqlEngine(arg) + ? (): Promise => Promise.resolve(arg.graphqlEngine) + : arg.getGraphqlEngine + + const { pathName, req, spanContext, telemetryResolverTimings } = arg await tracerReadyPromise let getDataWrapperActivity: MaybePhantomActivity @@ -97,7 +127,7 @@ export async function getData({ getDataWrapperActivity.start() } - let page: IGatsbyPage + let page: EnginePage let templateDetails: ITemplateDetails let potentialPagePath: string let findMetaActivity: MaybePhantomActivity @@ -114,7 +144,7 @@ export async function getData({ potentialPagePath = getPagePathFromPageDataPath(pathName) || pathName // 1. Find a page for pathname - const maybePage = graphqlEngine.findPageByPath(potentialPagePath) + const maybePage = findEnginePageByPath(potentialPagePath) if (!maybePage) { // page not found, nothing to run query for @@ -151,44 +181,46 @@ export async function getData({ runningQueryActivity.start() } executionPromises.push( - graphqlEngine - .runQuery( - templateDetails.query, - { - ...page, - ...page.context, - }, - { - queryName: page.path, - componentPath: page.componentPath, - parentSpan: runningQueryActivity?.span, - forceGraphqlTracing: !!runningQueryActivity, - telemetryResolverTimings, - } - ) - .then(queryResults => { - if (queryResults.errors && queryResults.errors.length > 0) { - const e = queryResults.errors[0] - const codeFrame = getCodeFrame( - templateDetails.query, - e.locations && e.locations[0].line, - e.locations && e.locations[0].column - ) - - const queryRunningError = new Error( - e.message + `\n\n` + codeFrame - ) - queryRunningError.stack = e.stack - throw queryRunningError - } else { - results = queryResults - } - }) - .finally(() => { - if (runningQueryActivity) { - runningQueryActivity.end() - } - }) + getGraphqlEngine().then(graphqlEngine => + graphqlEngine + .runQuery( + templateDetails.query, + { + ...page, + ...page.context, + }, + { + queryName: page.path, + componentPath: page.componentPath, + parentSpan: runningQueryActivity?.span, + forceGraphqlTracing: !!runningQueryActivity, + telemetryResolverTimings, + } + ) + .then(queryResults => { + if (queryResults.errors && queryResults.errors.length > 0) { + const e = queryResults.errors[0] + const codeFrame = getCodeFrame( + templateDetails.query, + e.locations && e.locations[0].line, + e.locations && e.locations[0].column + ) + + const queryRunningError = new Error( + e.message + `\n\n` + codeFrame + ) + queryRunningError.stack = e.stack + throw queryRunningError + } else { + results = queryResults + } + }) + .finally(() => { + if (runningQueryActivity) { + runningQueryActivity.end() + } + }) + ) ) } @@ -489,3 +521,11 @@ export async function renderHTML({ } } } + +const stateWithPages = { + pages: new Map(GATSBY_PAGES), +} as unknown as IGatsbyState + +export function findEnginePageByPath(pathName: string): EnginePage | undefined { + return findPageByPath(stateWithPages, pathName, false) +} diff --git a/packages/gatsby/src/utils/page-ssr-module/lambda.ts b/packages/gatsby/src/utils/page-ssr-module/lambda.ts index 81ae8f6938eff..0aab7f4a69675 100644 --- a/packages/gatsby/src/utils/page-ssr-module/lambda.ts +++ b/packages/gatsby/src/utils/page-ssr-module/lambda.ts @@ -8,11 +8,13 @@ import { pipeline } from "stream" import { URL } from "url" import { promisify } from "util" -import type { IGatsbyPage } from "../../internal" -import type { ISSRData } from "./entry" +import type { ISSRData, EnginePage } from "./entry" import { link, rewritableMethods as linkRewritableMethods } from "linkfs" -const cdnDatastore = `%CDN_DATASTORE_PATH%` +const cdnDatastorePath = `%CDN_DATASTORE_PATH%` +// this is fallback origin, we will prefer to extract it from first request instead +// as in some cases one reported by adapter might not be correct +const cdnDatastoreOrigin = `%CDN_DATASTORE_ORIGIN%` const PATH_PREFIX = `%PATH_PREFIX%` function setupFsWrapper(): string { @@ -175,7 +177,7 @@ function setupFsWrapper(): string { // @ts-ignore __promisify__ stuff global._fsWrapper = lfs - if (!cdnDatastore) { + if (!cdnDatastorePath) { const dir = `data` if ( !process.env.NETLIFY_LOCAL && @@ -223,7 +225,7 @@ type GraphQLEngineType = const { GraphQLEngine } = require(`../query-engine`) as typeof import("../../schema/graphql-engine/entry") -const { getData, renderPageData, renderHTML } = +const { getData, renderPageData, renderHTML, findEnginePageByPath } = require(`./index`) as typeof import("./entry") const streamPipeline = promisify(pipeline) @@ -237,47 +239,66 @@ function get( : httpGet(url, callback) } -async function getEngine(): Promise { - if (cdnDatastore) { - // if this variable is set we need to download the datastore from the CDN - const downloadPath = dbPath + `/data.mdb` - console.log( - `Downloading datastore from CDN (${cdnDatastore} -> ${downloadPath})` - ) - - await fs.ensureDir(dbPath) - await new Promise((resolve, reject) => { - const req = get(cdnDatastore, response => { - if ( - !response.statusCode || - response.statusCode < 200 || - response.statusCode > 299 - ) { - reject( - new Error( - `Failed to download ${cdnDatastore}: ${response.statusCode} ${ - response.statusMessage || `` - }` - ) - ) - return - } +interface IEngineError extends Error { + downloadError?: boolean +} - const fileStream = fs.createWriteStream(downloadPath) - streamPipeline(response, fileStream) - .then(resolve) - .catch(error => { - console.log(`Error downloading ${cdnDatastore}`, error) - reject(error) - }) - }) +function shouldDownloadDatastoreFromCDN(): boolean { + return !!cdnDatastorePath +} - req.on(`error`, error => { - console.log(`Error downloading ${cdnDatastore}`, error) - reject(error) - }) +async function downloadDatastoreFromCDN(origin: string): Promise { + const cdnDatastore = `${origin}/${cdnDatastorePath}` + // if this variable is set we need to download the datastore from the CDN + const downloadPath = dbPath + `/data.mdb` + console.log( + `Downloading datastore from CDN (${cdnDatastore} -> ${downloadPath})` + ) + + await fs.ensureDir(dbPath) + await new Promise((resolve, reject) => { + const req = get(cdnDatastore, response => { + if ( + !response.statusCode || + response.statusCode < 200 || + response.statusCode > 299 + ) { + const engineError = new Error( + `Failed to download ${cdnDatastore}: ${response.statusCode} ${ + response.statusMessage || `` + }` + ) as IEngineError + engineError.downloadError = true + reject(engineError) + return + } + + const fileStream = fs.createWriteStream(downloadPath) + streamPipeline(response, fileStream) + .then(resolve) + .catch(error => { + console.log(`Error downloading ${cdnDatastore}`, error) + const engineError = error as IEngineError + engineError.downloadError = true + reject(engineError) + }) + }) + + req.on(`error`, error => { + console.log(`Error downloading ${cdnDatastore}`, error) + const engineError = error as IEngineError + engineError.downloadError = true + reject(engineError) }) - console.log(`Downloaded datastore from CDN`) + }) + console.log(`Downloaded datastore from CDN`) +} + +async function initializeGraphqlEngine( + origin: string +): Promise { + if (shouldDownloadDatastoreFromCDN()) { + await downloadDatastoreFromCDN(origin) } const graphqlEngine = new GraphQLEngine({ @@ -289,7 +310,67 @@ async function getEngine(): Promise { return graphqlEngine } -const engineReadyPromise = getEngine() +let memoizedGraphqlEnginePromise: Promise | null = null +const originToGraphqlEnginePromise = new Map< + string, + Promise | null | Error +>() + +function tryToInitializeGraphqlEngineFromCollectedOrigins(): Promise { + for (const [origin, originEngineState] of originToGraphqlEnginePromise) { + if (!(originEngineState instanceof Error)) { + if (originEngineState === null) { + const engineForOriginPromise = initializeGraphqlEngine(origin).catch( + e => { + originToGraphqlEnginePromise.set( + origin, + e instanceof Error ? e : new Error(e) + ) + + if (e.downloadError) { + return tryToInitializeGraphqlEngineFromCollectedOrigins() + } + + throw e + } + ) + originToGraphqlEnginePromise.set(origin, engineForOriginPromise) + return engineForOriginPromise + } else { + return originEngineState + } + } + } + + return Promise.reject(new Error(`No engine available`)) +} + +function memoizedInitializeGraphqlEngine( + origin: string +): Promise { + if (!originToGraphqlEnginePromise.has(origin)) { + // register origin, but for now don't init anything + originToGraphqlEnginePromise.set(origin, null) + } + + if (!memoizedGraphqlEnginePromise) { + // pick first non-errored entry + memoizedGraphqlEnginePromise = + tryToInitializeGraphqlEngineFromCollectedOrigins().catch(e => { + // at this point we don't have any origin that work, but maybe we will get one in future + // so unset memoizedGraphqlEnginePromise as it would be not allowing any more attempts once it settled + memoizedGraphqlEnginePromise = null + throw e + }) + } + return memoizedGraphqlEnginePromise +} + +memoizedInitializeGraphqlEngine(cdnDatastoreOrigin).catch( + () => + // we don't want to crash the process if we can't get the engine without a request + null +) function reverseFixedPagePath(pageDataRequestPath: string): string { return pageDataRequestPath === `index` ? `/` : pageDataRequestPath @@ -321,7 +402,7 @@ function setStatusAndHeaders({ data, res, }: { - page: IGatsbyPage + page: EnginePage data: ISSRData res: GatsbyFunctionResponse }): void { @@ -354,15 +435,12 @@ function getErrorBody(statusCode: number): string { } interface IPageInfo { - page: IGatsbyPage + page: EnginePage isPageData: boolean pagePath: string } -function getPage( - pathname: string, - graphqlEngine: GraphQLEngineType -): IPageInfo | undefined { +function getPage(pathname: string): IPageInfo | undefined { const pathInfo = getPathInfo(pathname) if (!pathInfo) { return undefined @@ -370,7 +448,7 @@ function getPage( const { isPageData, pagePath } = pathInfo - const page = graphqlEngine.findPageByPath(pagePath) + const page = findEnginePageByPath(pagePath) if (!page) { return undefined } @@ -387,18 +465,17 @@ async function engineHandler( res: GatsbyFunctionResponse ): Promise { try { - const graphqlEngine = await engineReadyPromise let pageInfo: IPageInfo | undefined const originalPathName = req.url ?? `` if (PATH_PREFIX && originalPathName.startsWith(PATH_PREFIX)) { const maybePath = originalPathName.slice(PATH_PREFIX.length) - pageInfo = getPage(maybePath, graphqlEngine) + pageInfo = getPage(maybePath) } if (!pageInfo) { - pageInfo = getPage(originalPathName, graphqlEngine) + pageInfo = getPage(originalPathName) } if (!pageInfo) { @@ -410,7 +487,10 @@ async function engineHandler( const data = await getData({ pathName: pagePath, - graphqlEngine, + getGraphqlEngine: () => + memoizedInitializeGraphqlEngine( + req?.rawUrl ? new URL(req.rawUrl).origin : cdnDatastoreOrigin + ), req, })