feat: add Frameworks API (#5668)

* feat: add new config properties to Frameworks API

* feat: add Blobs directory

* feat: add functions endpoint

* feat: add edge functions endpoint

* refactor: clean up feature flag

* chore: fix tests

* feat: add new Blobs structure

* chore: add comments

* chore: update snapshots

* fix: pass `featureFlags` around

* chore: update snapshots

* chore: update snapshots

* refactor: define precedence of functions directories

packages/build/src/core/build.ts

@@ -636,7 +636,7 @@ const runBuild = async function ({
     timeline === 'dev' ? getDevSteps(devCommand, pluginsSteps, eventHandlers) : getSteps(pluginsSteps, eventHandlers)
 
   if (dry) {
-    await doDryRun({ buildDir, steps, netlifyConfig, constants, buildbotServerSocket, logs })
+    await doDryRun({ buildDir, steps, netlifyConfig, constants, buildbotServerSocket, logs, featureFlags })
     return { netlifyConfig }
   }
 

packages/build/src/core/dry.js

@@ -4,22 +4,28 @@ import { logDryRunStart, logDryRunStep, logDryRunEnd } from '../log/messages/dry
 import { runsOnlyOnBuildFailure } from '../plugins/events.js'
 
 // If the `dry` flag is specified, do a dry run
-export const doDryRun = async function ({ buildDir, steps, netlifyConfig, constants, buildbotServerSocket, logs }) {
+export const doDryRun = async function ({
+  buildDir,
+  steps,
+  netlifyConfig,
+  constants,
+  buildbotServerSocket,
+  logs,
+  featureFlags,
+}) {
   const successSteps = await pFilter(steps, ({ event, condition }) =>
-    shouldIncludeStep({ buildDir, event, condition, netlifyConfig, constants, buildbotServerSocket }),
+    shouldIncludeStep({ buildDir, event, condition, netlifyConfig, constants, buildbotServerSocket, featureFlags }),
   )
   const eventWidth = Math.max(...successSteps.map(getEventLength))
   const stepsCount = successSteps.length
 
   logDryRunStart({ logs, eventWidth, stepsCount })
 
-  successSteps.forEach((step, index) => {
-    if (step.quiet) {
-      return
-    }
-
-    logDryRunStep({ logs, step, index, netlifyConfig, eventWidth, stepsCount })
-  })
+  successSteps
+    .filter((step) => !step.quiet)
+    .forEach((step, index) => {
+      logDryRunStep({ logs, step, index, netlifyConfig, eventWidth, stepsCount })
+    })
 
   logDryRunEnd(logs)
 }
@@ -31,10 +37,12 @@ const shouldIncludeStep = async function ({
   netlifyConfig,
   constants,
   buildbotServerSocket,
+  featureFlags,
 }) {
   return (
     !runsOnlyOnBuildFailure(event) &&
-    (condition === undefined || (await condition({ buildDir, constants, netlifyConfig, buildbotServerSocket })))
+    (condition === undefined ||
+      (await condition({ buildDir, constants, netlifyConfig, buildbotServerSocket, featureFlags })))
   )
 }
 

packages/build/src/core/feature_flags.ts

@@ -23,5 +23,6 @@ export const DEFAULT_FEATURE_FLAGS: FeatureFlags = {
   edge_functions_system_logger: false,
   netlify_build_reduced_output: false,
   netlify_build_updated_plugin_compatibility: false,
+  netlify_build_frameworks_api: false,
   netlify_build_plugin_system_log: false,
 }

packages/build/src/log/messages/core_steps.js

@@ -59,13 +59,23 @@ export const logFunctionsToBundle = function ({
   userFunctionsSrcExists,
   internalFunctions,
   internalFunctionsSrc,
+  frameworkFunctions,
   type = 'Functions',
 }) {
   if (internalFunctions.length !== 0) {
     log(logs, `Packaging ${type} from ${THEME.highlightWords(internalFunctionsSrc)} directory:`)
     logArray(logs, internalFunctions, { indent: false })
   }
 
+  if (frameworkFunctions.length !== 0) {
+    if (internalFunctions.length !== 0) {
+      log(logs, '')
+    }
+
+    log(logs, `Packaging ${type} generated by your framework:`)
+    logArray(logs, frameworkFunctions, { indent: false })
+  }
+
   if (!userFunctionsSrcExists) {
     return
   }
@@ -76,7 +86,7 @@ export const logFunctionsToBundle = function ({
     return
   }
 
-  if (internalFunctions.length !== 0) {
+  if (internalFunctions.length !== 0 || frameworkFunctions.length !== 0) {
     log(logs, '')
   }
 

packages/build/src/plugins_core/blobs_upload/index.ts

@@ -7,6 +7,7 @@ import semver from 'semver'
 import { DEFAULT_API_HOST } from '../../core/normalize_flags.js'
 import { logError } from '../../log/logger.js'
 import { getFileWithMetadata, getKeysToUpload, scanForBlobs } from '../../utils/blobs.js'
+import { getBlobs } from '../../utils/frameworks_api.js'
 import { type CoreStep, type CoreStepCondition, type CoreStepFunction } from '../types.js'
 
 const coreStep: CoreStepFunction = async function ({
@@ -46,30 +47,31 @@ const coreStep: CoreStepFunction = async function ({
     return {}
   }
 
-  // If using the deploy config API, configure the store to use the region that
-  // was configured for the deploy.
-  if (!blobs.isLegacyDirectory) {
+  // If using the deploy config API or the Frameworks API, configure the store
+  // to use the region that was configured for the deploy. We don't do it for
+  // the legacy file-based upload API since that would be a breaking change.
+  if (blobs.apiVersion > 1) {
     storeOpts.experimentalRegion = 'auto'
   }
 
   const blobStore = getDeployStore(storeOpts)
-  const keys = await getKeysToUpload(blobs.directory)
+  const blobsToUpload = blobs.apiVersion >= 3 ? await getBlobs(blobs.directory) : await getKeysToUpload(blobs.directory)
 
-  if (keys.length === 0) {
+  if (blobsToUpload.length === 0) {
     systemLog('No blobs to upload to deploy store.')
 
     return {}
   }
 
-  systemLog(`Uploading ${keys.length} blobs to deploy store`)
+  systemLog(`Uploading ${blobsToUpload.length} blobs to deploy store...`)
 
   try {
     await pMap(
-      keys,
-      async (key: string) => {
+      blobsToUpload,
+      async ({ key, contentPath, metadataPath }) => {
         systemLog(`Uploading blob ${key}`)
 
-        const { data, metadata } = await getFileWithMetadata(blobs.directory, key)
+        const { data, metadata } = await getFileWithMetadata(key, contentPath, metadataPath)
         await blobStore.set(key, data, { metadata })
       },
       { concurrency: 10 },

packages/build/src/plugins_core/deploy_config/index.ts

@@ -1,18 +1,30 @@
-import { promises as fs } from 'fs'
-import { resolve } from 'path'
-
 import { mergeConfigs } from '@netlify/config'
 
 import type { NetlifyConfig } from '../../index.js'
 import { getConfigMutations } from '../../plugins/child/diff.js'
 import { CoreStep, CoreStepFunction } from '../types.js'
 
-import { filterConfig } from './util.js'
+import { filterConfig, loadConfigFile } from './util.js'
 
 // The properties that can be set using this API. Each element represents a
 // path using dot-notation — e.g. `["build", "functions"]` represents the
 // `build.functions` property.
-const ALLOWED_PROPERTIES = [['images', 'remote_images']]
+const ALLOWED_PROPERTIES = [
+  ['build', 'functions'],
+  ['build', 'publish'],
+  ['functions', '*'],
+  ['functions', '*', '*'],
+  ['headers'],
+  ['images', 'remote_images'],
+  ['redirects'],
+]
+
+// For array properties, any values set in this API will be merged with the
+// main configuration file in such a way that user-defined values always take
+// precedence. The exception are these properties that let frameworks set
+// values that should be evaluated before any user-defined values. They use
+// a special notation where `redirects!` represents "forced redirects", etc.
+const OVERRIDE_PROPERTIES = new Set(['redirects!'])
 
 const coreStep: CoreStepFunction = async function ({
   buildDir,
@@ -22,30 +34,42 @@ const coreStep: CoreStepFunction = async function ({
     // no-op
   },
 }) {
-  const configPath = resolve(buildDir, packagePath ?? '', '.netlify/deploy/v1/config.json')
-
-  let config: Partial<NetlifyConfig> = {}
+  let config: Partial<NetlifyConfig> | undefined
 
   try {
-    const data = await fs.readFile(configPath, 'utf8')
-
-    config = JSON.parse(data) as Partial<NetlifyConfig>
+    config = await loadConfigFile(buildDir, packagePath)
   } catch (err) {
-    // If the file doesn't exist, this is a non-error.
-    if (err.code === 'ENOENT') {
-      return {}
-    }
-
-    systemLog(`Failed to read Deploy Configuration API: ${err.message}`)
+    systemLog(`Failed to read Frameworks API: ${err.message}`)
 
     throw new Error('An error occured while processing the platform configurarion defined by your framework')
   }
 
+  if (!config) {
+    return {}
+  }
+
+  const configOverrides: Partial<NetlifyConfig> = {}
+
+  for (const key in config) {
+    // If the key uses the special notation for defining mutations that should
+    // take precedence over user-defined properties, extract the canonical
+    // property, set it on a different object, and delete it from the main one.
+    if (OVERRIDE_PROPERTIES.has(key)) {
+      const canonicalKey = key.slice(0, -1)
+
+      configOverrides[canonicalKey] = config[key]
+
+      delete config[key]
+    }
+  }
+
   // Filtering out any properties that can't be mutated using this API.
   const filteredConfig = filterConfig(config, [], ALLOWED_PROPERTIES, systemLog)
 
   // Merging the config extracted from the API with the initial config.
-  const newConfig = mergeConfigs([filteredConfig, netlifyConfig], { concatenateArrays: true }) as Partial<NetlifyConfig>
+  const newConfig = mergeConfigs([filteredConfig, netlifyConfig, configOverrides], {
+    concatenateArrays: true,
+  }) as Partial<NetlifyConfig>
 
   // Diffing the initial and the new configs to compute the mutations (what
   // changed between them).
@@ -59,9 +83,8 @@ const coreStep: CoreStepFunction = async function ({
 export const applyDeployConfig: CoreStep = {
   event: 'onBuild',
   coreStep,
-  coreStepId: 'deploy_config',
-  coreStepName: 'Applying Deploy Configuration',
+  coreStepId: 'frameworks_api_config',
+  coreStepName: 'Applying configuration from Frameworks API',
   coreStepDescription: () => '',
-  condition: ({ featureFlags }) => featureFlags?.netlify_build_deploy_configuration_api,
   quiet: true,
 }

packages/build/src/plugins_core/deploy_config/util.ts

@@ -1,8 +1,44 @@
+import { promises as fs } from 'fs'
+import { resolve } from 'path'
+
 import isPlainObject from 'is-plain-obj'
 import mapObject, { mapObjectSkip } from 'map-obj'
 
+import type { NetlifyConfig } from '../../index.js'
+import { FRAMEWORKS_API_CONFIG_ENDPOINT } from '../../utils/frameworks_api.js'
 import { SystemLogger } from '../types.js'
 
+export const loadConfigFile = async (buildDir: string, packagePath?: string) => {
+  const configPath = resolve(buildDir, packagePath ?? '', FRAMEWORKS_API_CONFIG_ENDPOINT)
+
+  try {
+    const data = await fs.readFile(configPath, 'utf8')
+
+    return JSON.parse(data) as Partial<NetlifyConfig>
+  } catch (err) {
+    // If the file doesn't exist, this is a non-error.
+    if (err.code !== 'ENOENT') {
+      throw err
+    }
+  }
+
+  // The first version of this API was called "Deploy Configuration API" and it
+  // had `.netlify/deploy` as its base directory. For backwards-compatibility,
+  // we need to support that path for the config file.
+  const legacyConfigPath = resolve(buildDir, packagePath ?? '', '.netlify/deploy/v1/config.json')
+
+  try {
+    const data = await fs.readFile(legacyConfigPath, 'utf8')
+
+    return JSON.parse(data) as Partial<NetlifyConfig>
+  } catch (err) {
+    // If the file doesn't exist, this is a non-error.
+    if (err.code !== 'ENOENT') {
+      throw err
+    }
+  }
+}
+
 /**
  * Checks whether a property matches a template that may contain wildcards.
  * Both the property and the template use a dot-notation represented as an

packages/build/src/plugins_core/dev_blobs_upload/index.ts

@@ -6,6 +6,7 @@ import semver from 'semver'
 
 import { log, logError } from '../../log/logger.js'
 import { getFileWithMetadata, getKeysToUpload, scanForBlobs } from '../../utils/blobs.js'
+import { getBlobs } from '../../utils/frameworks_api.js'
 import { type CoreStep, type CoreStepCondition, type CoreStepFunction } from '../types.js'
 
 const coreStep: CoreStepFunction = async function ({
@@ -47,34 +48,35 @@ const coreStep: CoreStepFunction = async function ({
     return {}
   }
 
-  // If using the deploy config API, configure the store to use the region that
-  // was configured for the deploy.
-  if (!blobs.isLegacyDirectory) {
+  // If using the deploy config API or the Frameworks API, configure the store
+  // to use the region that was configured for the deploy. We don't do it for
+  // the legacy file-based upload API since that would be a breaking change.
+  if (blobs.apiVersion > 1) {
     storeOpts.experimentalRegion = 'auto'
   }
 
   const blobStore = getDeployStore(storeOpts)
-  const keys = await getKeysToUpload(blobs.directory)
+  const blobsToUpload = blobs.apiVersion >= 3 ? await getBlobs(blobs.directory) : await getKeysToUpload(blobs.directory)
 
-  if (keys.length === 0) {
+  if (blobsToUpload.length === 0) {
     if (!quiet) {
       log(logs, 'No blobs to upload to deploy store.')
     }
     return {}
   }
 
   if (!quiet) {
-    log(logs, `Uploading ${keys.length} blobs to deploy store...`)
+    log(logs, `Uploading ${blobsToUpload.length} blobs to deploy store...`)
   }
 
   try {
     await pMap(
-      keys,
-      async (key: string) => {
+      blobsToUpload,
+      async ({ key, contentPath, metadataPath }) => {
         if (debug && !quiet) {
           log(logs, `- Uploading blob ${key}`, { indent: true })
         }
-        const { data, metadata } = await getFileWithMetadata(blobs.directory, key)
+        const { data, metadata } = await getFileWithMetadata(key, contentPath, metadataPath)
         await blobStore.set(key, data, { metadata })
       },
       { concurrency: 10 },