feat(integration): inject Google Drive OAuth details to pipeline and …

…console (#1143)

Because

- Google Drive component will need the OAuth client secret and ID
injected
as an environment variable.
- These values are required in console, too. We need to define them in a
  single place.
- It can be interesting to define other component values such as the AI
  component API keys.

This commit

- Defines the global component secrets as environment values and injects
  them into the required services.

.env

@@ -192,12 +192,3 @@ MINIO_EXTERNAL_PORT=19000
 MINIO_CONSOLE_PORT=9001
 MINIO_CONSOLE_EXTERNAL_PORT=19001
 MINIO_DATA_DIR=~/.local/instill/minio/data
-
-# Component global secrets
-# Provide your API key for the AI vendors so that you can use
-# ${secret.INSTILL_SECRET} in the pipeline to access these shared keys.
-OPENAI_SECRET_KEY=
-STABILITYAI_SECRET_KEY=
-ANTHROPIC_SECRET_KEY=
-COHERE_SECRET_KEY=
-MISTRALAI_SECRET_KEY=

.env.component

@@ -0,0 +1,21 @@
+# Provide your API key for the AI vendors so that you can set the components up
+# with default credentials.
+CFG_COMPONENT_SECRETS_OPENAI_APIKEY=
+CFG_COMPONENT_SECRETS_STABILITYAI_APIKEY=
+CFG_COMPONENT_SECRETS_ANTHROPIC_APIKEY=
+CFG_COMPONENT_SECRETS_COHERE_APIKEY=
+CFG_COMPONENT_SECRETS_MISTRALAI_APIKEY=
+CFG_COMPONENT_SECRETS_GROQ_APIKEY=
+CFG_COMPONENT_SECRETS_FIREWORKSAI_APIKEY=
+
+# Numbers Protocol API key.
+CFG_COMPONENT_SECRETS_NUMBERS_XAPIKEY=
+
+# OAuth secrets. When these are filled, the specified component will support
+# OAuth integrations.
+CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTID=
+CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTSECRET=
+CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTID=
+CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTSECRET=
+CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTID=
+CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTSECRET=

.env.console

@@ -0,0 +1,8 @@
+# Client variables for OAuth integrations.
+# These values depend on .env.component.
+INTEGRATION_GOOGLE_DRIVE_CLIENT_ID=${CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTID}
+INTEGRATION_GOOGLE_DRIVE_CLIENT_SECRET=${CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTSECRET}
+INTEGRATION_SLACK_CLIENT_ID=${CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTID}
+INTEGRATION_SLACK_CLIENT_SECRET=${CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTSECRET}
+INTEGRATION_GITHUB_CLIENT_ID=${CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTID}
+INTEGRATION_GITHUB_CLIENT_SECRET=${CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTSECRET}

.github/CONTRIBUTING.md

@@ -35,6 +35,22 @@ The env variable `PROFILE` is intended to specify which service component you wa
 
   When you set `PROFILE=exclude-{service}`, in which `{service}` can be  `pipeline`, it means you want to develop on that particular service. The `make` command will launch the corresponding stack **WITHOUT** that service component and **WITH** all other services. Given that, you can later on spin up and down the `{service}` in your dev container. Please take the [pipeline-backend](https://github.com/instill-ai/pipeline-backend#local-dev) as an example.
 
+#### Component environment variables
+
+Some components can be configured with global secrets. This has several
+applications:
+
+- By accepting a global API key, some components have a default setup. When
+  the `setup` block is omitted in the recipe, this API key will be used.
+- In order to connect to 3rd party vendors via OAuth, the application
+  client ID and secret must be injected.
+
+You can set the values of these global secrets in
+[`.env.component`](./.env.component) before running the Docker container in
+order to add a global configuration to your components. These values will
+be injected into `pipeline-backend`. Additionally, `console` will also
+receive the OAuth configuration values.
+
 ### Tear down the local dev system
 
 Simply run:

docker-compose.yml

@@ -137,6 +137,7 @@ services:
     container_name: ${PIPELINE_BACKEND_HOST}
     image: ${PIPELINE_BACKEND_IMAGE}:${PIPELINE_BACKEND_VERSION}
     restart: unless-stopped
+    env_file: .env.component
     environment:
       CFG_SERVER_PRIVATEPORT: ${PIPELINE_BACKEND_PRIVATEPORT}
       CFG_SERVER_PUBLICPORT: ${PIPELINE_BACKEND_PUBLICPORT}
@@ -157,11 +158,6 @@ services:
       CFG_CACHE_REDIS_REDISOPTIONS_ADDR: ${REDIS_HOST}:${REDIS_PORT}
       CFG_LOG_EXTERNAL: ${OBSERVE_ENABLED}
       CFG_LOG_OTELCOLLECTOR_PORT: ${OTEL_COLLECTOR_PORT}
-      CFG_CONNECTOR_SECRETS_OPENAI_APIKEY: ${OPENAI_SECRET_KEY}
-      CFG_CONNECTOR_SECRETS_STABILITYAI_APIKEY: ${STABILITYAI_SECRET_KEY}
-      CFG_CONNECTOR_SECRETS_ANTHROPIC_APIKEY: ${ANTHROPIC_SECRET_KEY}
-      CFG_CONNECTOR_SECRETS_COHERE_APIKEY: ${COHERE_SECRET_KEY}
-      CFG_CONNECTOR_SECRETS_MISTRALAI_APIKEY: ${MISTRALAI_SECRET_KEY}
     command:
       - /bin/sh
       - -c
@@ -193,6 +189,7 @@ services:
     container_name: ${PIPELINE_BACKEND_HOST}-worker
     image: ${PIPELINE_BACKEND_IMAGE}:${PIPELINE_BACKEND_VERSION}
     restart: unless-stopped
+    env_file: .env.component
     environment:
       CFG_SERVER_PRIVATEPORT: ${PIPELINE_BACKEND_PRIVATEPORT}
       CFG_SERVER_PUBLICPORT: ${PIPELINE_BACKEND_PUBLICPORT}
@@ -212,11 +209,6 @@ services:
       CFG_CACHE_REDIS_REDISOPTIONS_ADDR: ${REDIS_HOST}:${REDIS_PORT}
       CFG_LOG_EXTERNAL: ${OBSERVE_ENABLED}
       CFG_LOG_OTELCOLLECTOR_PORT: ${OTEL_COLLECTOR_PORT}
-      CFG_CONNECTOR_SECRETS_OPENAI_APIKEY: ${OPENAI_SECRET_KEY}
-      CFG_CONNECTOR_SECRETS_STABILITYAI_APIKEY: ${STABILITYAI_SECRET_KEY}
-      CFG_CONNECTOR_SECRETS_ANTHROPIC_APIKEY: ${ANTHROPIC_SECRET_KEY}
-      CFG_CONNECTOR_SECRETS_COHERE_APIKEY: ${COHERE_SECRET_KEY}
-      CFG_CONNECTOR_SECRETS_MISTRALAI_APIKEY: ${MISTRALAI_SECRET_KEY}
     entrypoint: ./pipeline-backend-worker
     depends_on:
       pipeline_backend:
@@ -364,6 +356,9 @@ services:
     container_name: ${CONSOLE_HOST}
     image: ${CONSOLE_IMAGE}:${CONSOLE_VERSION}
     restart: unless-stopped
+    env_file:
+      - path: .env.component
+      - path: .env.console
     environment:
       NEXT_PUBLIC_GENERAL_API_VERSION: v1beta
       NEXT_PUBLIC_MODEL_API_VERSION: v1alpha