From 7ccf5ae40a5a080b955de23a95559df0ae4469ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Vall=C3=A9s?= <3977183+jvallesm@users.noreply.github.com> Date: Thu, 31 Oct 2024 08:00:58 +0100 Subject: [PATCH] feat(integration): inject Google Drive OAuth details to pipeline and console (#1143) Because - Google Drive component will need the OAuth client secret and ID injected as an environment variable. - These values are required in console, too. We need to define them in a single place. - It can be interesting to define other component values such as the AI component API keys. This commit - Defines the global component secrets as environment values and injects them into the required services. --- .env | 9 --------- .env.component | 21 +++++++++++++++++++++ .env.console | 8 ++++++++ .github/CONTRIBUTING.md | 16 ++++++++++++++++ docker-compose.yml | 15 +++++---------- 5 files changed, 50 insertions(+), 19 deletions(-) create mode 100644 .env.component create mode 100644 .env.console diff --git a/.env b/.env index ba540bac..95e9739b 100644 --- a/.env +++ b/.env @@ -192,12 +192,3 @@ MINIO_EXTERNAL_PORT=19000 MINIO_CONSOLE_PORT=9001 MINIO_CONSOLE_EXTERNAL_PORT=19001 MINIO_DATA_DIR=~/.local/instill/minio/data - -# Component global secrets -# Provide your API key for the AI vendors so that you can use -# ${secret.INSTILL_SECRET} in the pipeline to access these shared keys. -OPENAI_SECRET_KEY= -STABILITYAI_SECRET_KEY= -ANTHROPIC_SECRET_KEY= -COHERE_SECRET_KEY= -MISTRALAI_SECRET_KEY= diff --git a/.env.component b/.env.component new file mode 100644 index 00000000..b0f788f5 --- /dev/null +++ b/.env.component @@ -0,0 +1,21 @@ +# Provide your API key for the AI vendors so that you can set the components up +# with default credentials. +CFG_COMPONENT_SECRETS_OPENAI_APIKEY= +CFG_COMPONENT_SECRETS_STABILITYAI_APIKEY= +CFG_COMPONENT_SECRETS_ANTHROPIC_APIKEY= +CFG_COMPONENT_SECRETS_COHERE_APIKEY= +CFG_COMPONENT_SECRETS_MISTRALAI_APIKEY= +CFG_COMPONENT_SECRETS_GROQ_APIKEY= +CFG_COMPONENT_SECRETS_FIREWORKSAI_APIKEY= + +# Numbers Protocol API key. +CFG_COMPONENT_SECRETS_NUMBERS_XAPIKEY= + +# OAuth secrets. When these are filled, the specified component will support +# OAuth integrations. +CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTID= +CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTSECRET= +CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTID= +CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTSECRET= +CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTID= +CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTSECRET= diff --git a/.env.console b/.env.console new file mode 100644 index 00000000..c287c1e4 --- /dev/null +++ b/.env.console @@ -0,0 +1,8 @@ +# Client variables for OAuth integrations. +# These values depend on .env.component. +INTEGRATION_GOOGLE_DRIVE_CLIENT_ID=${CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTID} +INTEGRATION_GOOGLE_DRIVE_CLIENT_SECRET=${CFG_COMPONENT_SECRETS_GOOGLEDRIVE_OAUTHCLIENTSECRET} +INTEGRATION_SLACK_CLIENT_ID=${CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTID} +INTEGRATION_SLACK_CLIENT_SECRET=${CFG_COMPONENT_SECRETS_SLACK_OAUTHCLIENTSECRET} +INTEGRATION_GITHUB_CLIENT_ID=${CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTID} +INTEGRATION_GITHUB_CLIENT_SECRET=${CFG_COMPONENT_SECRETS_GITHUB_OAUTHCLIENTSECRET} diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 1c64d8ef..24ed22bb 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -35,6 +35,22 @@ The env variable `PROFILE` is intended to specify which service component you wa When you set `PROFILE=exclude-{service}`, in which `{service}` can be `pipeline`, it means you want to develop on that particular service. The `make` command will launch the corresponding stack **WITHOUT** that service component and **WITH** all other services. Given that, you can later on spin up and down the `{service}` in your dev container. Please take the [pipeline-backend](https://github.com/instill-ai/pipeline-backend#local-dev) as an example. +#### Component environment variables + +Some components can be configured with global secrets. This has several +applications: + +- By accepting a global API key, some components have a default setup. When + the `setup` block is omitted in the recipe, this API key will be used. +- In order to connect to 3rd party vendors via OAuth, the application + client ID and secret must be injected. + +You can set the values of these global secrets in +[`.env.component`](./.env.component) before running the Docker container in +order to add a global configuration to your components. These values will +be injected into `pipeline-backend`. Additionally, `console` will also +receive the OAuth configuration values. + ### Tear down the local dev system Simply run: diff --git a/docker-compose.yml b/docker-compose.yml index 855b4a34..1aab5843 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -137,6 +137,7 @@ services: container_name: ${PIPELINE_BACKEND_HOST} image: ${PIPELINE_BACKEND_IMAGE}:${PIPELINE_BACKEND_VERSION} restart: unless-stopped + env_file: .env.component environment: CFG_SERVER_PRIVATEPORT: ${PIPELINE_BACKEND_PRIVATEPORT} CFG_SERVER_PUBLICPORT: ${PIPELINE_BACKEND_PUBLICPORT} @@ -157,11 +158,6 @@ services: CFG_CACHE_REDIS_REDISOPTIONS_ADDR: ${REDIS_HOST}:${REDIS_PORT} CFG_LOG_EXTERNAL: ${OBSERVE_ENABLED} CFG_LOG_OTELCOLLECTOR_PORT: ${OTEL_COLLECTOR_PORT} - CFG_CONNECTOR_SECRETS_OPENAI_APIKEY: ${OPENAI_SECRET_KEY} - CFG_CONNECTOR_SECRETS_STABILITYAI_APIKEY: ${STABILITYAI_SECRET_KEY} - CFG_CONNECTOR_SECRETS_ANTHROPIC_APIKEY: ${ANTHROPIC_SECRET_KEY} - CFG_CONNECTOR_SECRETS_COHERE_APIKEY: ${COHERE_SECRET_KEY} - CFG_CONNECTOR_SECRETS_MISTRALAI_APIKEY: ${MISTRALAI_SECRET_KEY} command: - /bin/sh - -c @@ -193,6 +189,7 @@ services: container_name: ${PIPELINE_BACKEND_HOST}-worker image: ${PIPELINE_BACKEND_IMAGE}:${PIPELINE_BACKEND_VERSION} restart: unless-stopped + env_file: .env.component environment: CFG_SERVER_PRIVATEPORT: ${PIPELINE_BACKEND_PRIVATEPORT} CFG_SERVER_PUBLICPORT: ${PIPELINE_BACKEND_PUBLICPORT} @@ -212,11 +209,6 @@ services: CFG_CACHE_REDIS_REDISOPTIONS_ADDR: ${REDIS_HOST}:${REDIS_PORT} CFG_LOG_EXTERNAL: ${OBSERVE_ENABLED} CFG_LOG_OTELCOLLECTOR_PORT: ${OTEL_COLLECTOR_PORT} - CFG_CONNECTOR_SECRETS_OPENAI_APIKEY: ${OPENAI_SECRET_KEY} - CFG_CONNECTOR_SECRETS_STABILITYAI_APIKEY: ${STABILITYAI_SECRET_KEY} - CFG_CONNECTOR_SECRETS_ANTHROPIC_APIKEY: ${ANTHROPIC_SECRET_KEY} - CFG_CONNECTOR_SECRETS_COHERE_APIKEY: ${COHERE_SECRET_KEY} - CFG_CONNECTOR_SECRETS_MISTRALAI_APIKEY: ${MISTRALAI_SECRET_KEY} entrypoint: ./pipeline-backend-worker depends_on: pipeline_backend: @@ -364,6 +356,9 @@ services: container_name: ${CONSOLE_HOST} image: ${CONSOLE_IMAGE}:${CONSOLE_VERSION} restart: unless-stopped + env_file: + - path: .env.component + - path: .env.console environment: NEXT_PUBLIC_GENERAL_API_VERSION: v1beta NEXT_PUBLIC_MODEL_API_VERSION: v1alpha