Concurrently generate pages

[chaegumi]

Summary

Can I generate multiple pages at the same time when astro build?

Background & Motivation

I have a website with 9352 pages. All the page URLs are known. It takes a long time to generate them one by one. If 10 or more pages can be generated in parallel, it will save a lot of time.

Goals

Concurrent generation

Example

18:46:48 [build] 9343 page(s) built in 11355.91s

[ematipico]

Please restore the template, it's really important to us. For example, at the moment, I can't understand what you're proposing

[chaegumi]

This can meet my needs. Can anyone integrate it into the system?
change file https://github.com/withastro/astro/blob/main/packages/astro/src/core/build/generate.ts#L212

// lodash slice function
function slice(array, start, end) {
  let length = array == null ? 0 : array.length;
  if (!length) {
    return [];
  }
  start = start == null ? 0 : start;
  end = end === undefined ? length : end;

  if (start < 0) {
    start = -start > length ? 0 : length + start;
  }
  end = end > length ? length : end;
  if (end < 0) {
    end += length;
  }
  length = start > end ? 0 : (end - start) >>> 0;
  start >>>= 0;

  let index = -1;
  const result = new Array(length);
  while (++index < length) {
    result[index] = array[index + start];
  }
  return result;
}
// lodash chunk function
function chunk(array, size = 1) {
  size = Math.max(size, 0);
  const length = array == null ? 0 : array.length;
  if (!length || size < 1) {
    return [];
  }
  let index = 0;
  let resIndex = 0;
  const result = new Array(Math.ceil(length / size));

  while (index < length) {
    result[resIndex++] = slice(array, index, (index += size));
  }
  return result;
}
// https://github.com/withastro/astro/blob/main/packages/astro/src/core/build/generate.ts#L212
async function generatePage(pageData, ssrEntry, builtPaths, pipeline) {
  const { config, logger } = pipeline;
  const pageModulePromise = ssrEntry.page;
  const styles = pageData.styles.sort(cssOrder).map(({ sheet }) => sheet).reduce(mergeInlineCss, []);
  const linkIds = [];
  const scripts = pageData.hoistedScript ?? null;
  if (!pageModulePromise) {
    throw new Error(
      `Unable to find the module for ${pageData.component}. This is unexpected and likely a bug in Astro, please report.`
    );
  }
  const pageModule = await pageModulePromise();
  const generationOptions = {
    pageData,
    linkIds,
    scripts,
    styles,
    mod: pageModule
  };
  for (const route of eachRouteInRouteData(pageData)) {
    const icon = route.type === "page" || route.type === "redirect" || route.type === "fallback" ? green("\u25B6") : magenta("\u03BB");
    logger.info(null, `${icon} ${getPrettyRouteName(route)}`);
    const paths = await getPathsForRoute(route, pageModule, pipeline, builtPaths);
    let timeStart = performance.now();
    let prevTimeEnd = timeStart;
    const arr1 = chunk(paths, 10);
    for (const arr of arr1) {
      let promises = [];
      for (let i in arr) {
        // console.log(arr[i]);

        const path = arr[i];
        // console.log(path);
        pipeline.logger.debug("build", `Generating: ${path}`);
        const filePath = getOutputFilename(config, path, pageData.route.type);
        const lineIcon = i === arr.length - 1 ? "\u2514\u2500" : "\u251C\u2500";
        logger.info(null, `  ${blue(lineIcon)} ${dim(filePath)}`, false);
        promises.push(generatePath(path, pipeline, generationOptions, route));
        const timeEnd = performance.now();
        const timeChange = getTimeStat(prevTimeEnd, timeEnd);
        const timeIncrease = `(+${timeChange})`;
        let timeIncreaseLabel;
        if (timeEnd - prevTimeEnd > THRESHOLD_SLOW_RENDER_TIME_MS) {
          timeIncreaseLabel = red(timeIncrease);
        } else {
          timeIncreaseLabel = dim(timeIncrease);
        }
        logger.info("SKIP_FORMAT", ` ${timeIncreaseLabel}`);
        prevTimeEnd = timeEnd;


      }
      await Promise.all(promises);
    }

  }
}

[chaegumi]

It has been supported since 4.16.0