diff --git a/.gitignore b/.gitignore index d9c13c35..4ff8aba5 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,7 @@ yarn.lock dist/ # Ignore the Azure credentials file -azure_creds.json +*_creds.json # Ignore the .env file .env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f7b7b77c..88c24b57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update && \ apt-get install -y --no-install-recommends \ tzdata=2024a-3ubuntu1.1 \ - curl=8.5.0-2ubuntu10.3 \ + curl=8.5.0-2ubuntu10.4 \ bash=5.2.21-2ubuntu4 \ gettext=0.21-14ubuntu2 \ gnupg=2.4.4-2ubuntu17 \ diff --git a/Makefile b/Makefile index 4f5d543e..1b5bb435 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ log: # Delete the running container clean: - @echo "Deleting Docker container..." + @echo "Deleting Docker container if exists..." @docker rm -f $(CONTAINER_NAME) || true # Run the validation tests diff --git a/lib/auth/aws.sh b/lib/auth/aws.sh new file mode 100644 index 00000000..04d210cd --- /dev/null +++ b/lib/auth/aws.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Example usage of the function +# aws_authenticate "/path/to/your/aws_creds.json" +# +# Example AWS credentials JSON file: +# +# { +# "AccessKeyId": "your-access-key-id", +# "SecretAccessKey": "your-secret-access-key", +# "Region": "your-aws-region" +# } + +# Function to authenticate AWS using IAM user credentials +aws_authenticate() { + echo "Not supported yet. Is in progress" +} + +# Example usage of the function +# aws_authenticate "/path/to/your/aws_creds.json" diff --git a/lib/auth/azure.sh b/lib/auth/azure.sh index d265b753..467b0082 100644 --- a/lib/auth/azure.sh +++ b/lib/auth/azure.sh @@ -1,5 +1,20 @@ #!/bin/bash +# Function to authenticate Azure accounts +# +# Example usage of the function +# azure_authenticate "/path/to/your/azure_creds.json" +# +# Example Azure credentials JSON file: +# +# { +# "clientId": "your-client-id", +# "clientSecret": "your-client-secret", +# "subscriptionId": "your-subscription-id", +# "tenantId": "your-tenant-id" +# } +# + # Function to authenticate Azure accounts azure_authenticate() { local creds_json="$1" diff --git a/lib/auth/bitwarden.sh b/lib/auth/bitwarden.sh new file mode 100644 index 00000000..d083cf48 --- /dev/null +++ b/lib/auth/bitwarden.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Function to authenticate Bitwarden using API key or master password +# +# Example usage of the function +# bitwarden_authenticate "/path/to/your/bitwarden_creds.json" +# +# Example Bitwarden credentials JSON file: +# +# { +# "clientId": "your-client-id", +# "clientSecret": "your-client-secret", +# "masterPassword": "your-master-password" +# } +# + +# Function to authenticate Bitwarden using API key or master password +bitwarden_authenticate() { + echo "Not supported yet. Is in progress" +} + +# Example usage of the function +# bitwarden_authenticate "/path/to/your/bitwarden_creds.json" diff --git a/lib/auth/gcp.sh b/lib/auth/gcp.sh new file mode 100644 index 00000000..74dbb88a --- /dev/null +++ b/lib/auth/gcp.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Function to authenticate GCP service accounts +# +# Example usage of the function +# gcp_authenticate "/path/to/your/gcp_creds.json" +# +# Example GCP credentials JSON file: +# +# { +# "type": "service_account", +# "project_id": "your-project-id", +# "private_key_id": "your-private-key-id", +# "private_key": "your-private-key", +# "client_email": "your-client-email", +# "client_id": "your-client-id", +# "auth_uri": "https://accounts.google.com/o/oauth2/auth", +# "token_uri": "https://oauth2.googleapis.com/token", +# "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", +# "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/your-client-email" +# } +# + +# Function to authenticate GCP service accounts +gcp_authenticate() { + local creds_json="$1" + + # Read the contents of the file + local creds_content + creds_content=$(cat "$creds_json") + + if [[ -z "$creds_content" ]]; then + echo "[ERROR] No GCP credentials provided." >&2 + return 1 + fi + + # Extract necessary fields from the JSON credentials + local clientEmail privateKey projectId + + clientEmail=$(echo "$creds_content" | jq -r '.client_email') + privateKey=$(echo "$creds_content" | jq -r '.private_key') + projectId=$(echo "$creds_content" | jq -r '.project_id') + + if [[ -z "$clientEmail" || -z "$privateKey" || -z "$projectId" ]]; then + echo "[ERROR] Missing required GCP credentials." >&2 + return 1 + fi + + # Create a temporary credentials file for gcloud authentication + local temp_creds_file="/tmp/gcp_creds.json" + echo "$creds_content" > "$temp_creds_file" + + echo "[INFO] Authenticating GCP service account..." + if ! gcloud auth activate-service-account "$clientEmail" --key-file="$temp_creds_file" >/dev/null 2>&1; then + echo "[ERROR] GCP service account authentication failed." >&2 + rm -f "$temp_creds_file" + return 1 + fi + + if ! gcloud config set project "$projectId" >/dev/null 2>&1; then + echo "[ERROR] Failed to set GCP project." >&2 + rm -f "$temp_creds_file" + return 1 + fi + + echo "[INFO] GCP service account authenticated and project set." + + # Clean up temporary credentials file + rm -f "$temp_creds_file" +} diff --git a/lib/auth/readme.md b/lib/auth/readme.md index 8b580c0c..f3e90e2b 100644 --- a/lib/auth/readme.md +++ b/lib/auth/readme.md @@ -27,12 +27,9 @@ Note the appId, password, and tenant. Update your `worker.yml` configuration file to include the Azure Service Principal credentials: ```yaml -workerActors: - - type: azure-service-principal - subscription: "YOUR_SUBSCRIPTION_ID" - tenant: "YOUR_TENANT_ID" - application: "YOUR_APP_ID" - password: "YOUR_CLIENT_SECRET" +actors: + - type: azure + creds: "${AZURE_CREDS}" ``` ### AWS IAM Role (TBD) @@ -43,6 +40,10 @@ Instructions for setting up AWS IAM Role will be provided here. Instructions for setting up GCP Service Account will be provided here. +### Bitwarden Service Account (TBD) + +Instructions for setting up GCP Service Account will be provided here. + ## Best Practices - **Use least privilege**: Assign the minimum required permissions to service accounts and roles. diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 0493622e..4af48ec4 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -6,71 +6,34 @@ source /usr/local/lib/utils.sh # shellcheck source=/dev/null source /usr/local/lib/worker_config.sh -# Function to clean up Azure authentication -cleanup_azure() { - log_info "Cleaning up Azure authentication" - if command -v az > /dev/null; then - if az account show > /dev/null 2>&1; then - if ! az logout; then - log_error "Failed to log out of Azure" - fi - else - log_info "No active Azure accounts found." - fi - else - log_warn "Azure CLI not found. Skipping Azure cleanup." +# Generic function to clean up authentication for any provider +cleanup_provider() { + local provider=$1 + local logout_cmd=$2 + local list_cmd=$3 + local name=$4 + + log_info "Cleaning up $name authentication" + + if ! command -v "$provider" > /dev/null; then + log_warn "$name CLI not found. Skipping $name cleanup." + return 0 fi -} -# Function to clean up GCP authentication -cleanup_gcp() { - log_info "Cleaning up GCP authentication" - if command -v gcloud > /dev/null; then - if gcloud auth list --format="value(account)" > /dev/null 2>&1; then - if ! gcloud auth revoke --all; then - log_error "Failed to revoke GCP authentication" - fi - else - log_info "No active GCP accounts found." - fi - else - log_warn "GCP CLI not found. Skipping GCP cleanup." + if ! eval "$list_cmd" > /dev/null 2>&1; then + log_info "No active $name accounts or sessions found." + return 0 fi -} -# Function to clean up AWS authentication -cleanup_aws() { - log_info "Cleaning up AWS authentication" - if command -v aws > /dev/null; then - if aws sso list-accounts > /dev/null 2>&1; then - if ! aws sso logout; then - log_warn "AWS SSO logout not configured or failed" - fi - else - log_info "No active AWS SSO sessions found." - fi - else - log_warn "AWS CLI not found. Skipping AWS cleanup." + if ! eval "$logout_cmd"; then + log_error "Failed to log out of $name" + return 1 fi -} -# Function to clean up Bitwarden authentication -cleanup_bitwarden() { - log_info "Cleaning up Bitwarden authentication" - if command -v bw > /dev/null; then - if bw status > /dev/null 2>&1; then - if ! bw logout --force; then - log_error "Failed to log out of Bitwarden" - fi - else - log_info "No active Bitwarden sessions found." - fi - else - log_warn "Bitwarden CLI not found. Skipping Bitwarden cleanup." - fi + log_info "$name authentication cleaned up successfully." } -# Function to clean up actors +# Function to clean up actors based on the worker configuration cleanup_actors() { log_info "Starting cleanup of actors" @@ -88,51 +51,47 @@ cleanup_actors() { return 0 fi - # Process each actor in the configuration + # Process each actor type echo "$actors_json" | jq -c '.[]' | while IFS= read -r actor; do local type type=$(echo "$actor" | jq -r '.type') - - local cleanup_function="cleanup_${type//[-]/_}" - if declare -F "$cleanup_function" > /dev/null; then - $cleanup_function - else - log_warn "Unsupported or unavailable actor type for cleanup: $type" - fi + + case "$type" in + azure) + cleanup_provider "az" "az logout" "az account show" "Azure" + ;; + gcp) + cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" + ;; + aws) + cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" + ;; + bitwarden) + cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" + ;; + *) + log_warn "Unsupported or unavailable actor type for cleanup: $type" + ;; + esac done } -# Function to clean up sensitive environment variables +# Function to clean up sensitive environment variables based on a pattern cleanup_sensitive_env_vars() { log_info "Cleaning up sensitive environment variables" - local env_config - if ! env_config=$(get_worker_config_path); then - log_error "Failed to retrieve configuration path." - return 1 - fi - - # Extract environment variable names defined in worker.yml (both variables and secrets) - local defined_vars - defined_vars=$(yq e -o=json '.config.variables, .config.secrets' "$env_config" 2>/dev/null | jq -r 'to_entries[].key') - - if [[ -z "$defined_vars" ]]; then - log_info "No sensitive environment variables found." - return 0 - fi + # Define a pattern for sensitive environment variables (e.g., AZURE_CREDS, GCP_CREDS, etc.) + local pattern="_CREDS" - # Unset the defined environment variables - for var in $defined_vars; do - unset "$var" || log_warn "Failed to unset environment variable: $var" + # Loop through environment variables that match the pattern + for var in $(env | grep "${pattern}" | cut -d'=' -f1); do + unset "$var" + log_info "Unset sensitive environment variable: $var" done log_info "Sensitive environment variables cleaned up successfully." } -# Example usage: -# cleanup_azure -# cleanup_gcp -# cleanup_aws -# cleanup_bitwarden +# Example usage # cleanup_actors # cleanup_sensitive_env_vars diff --git a/lib/environment.sh b/lib/environment.sh index 438e7541..8b59f359 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -8,21 +8,8 @@ source /usr/local/lib/secrets.sh source /usr/local/lib/cleanup.sh source /usr/local/lib/worker_config.sh -# Load environment variables from .env file if it exists -load_env_file() { - if [ -f .env ]; then - log_info "Loading environment variables from .env file." - # Quote the command substitution to prevent word splitting - export "$(grep -v '^#' .env | xargs -r)" - else - log_info "No .env file found. Proceeding with environment variables from the host." - fi -} - # Main function to coordinate environment setup configure_environment() { - # Load environment variables from .env file if it exists - load_env_file # Load and resolve the worker configuration local resolved_config diff --git a/lib/secrets.sh b/lib/secrets.sh index b86e9e99..aae52f4a 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -20,38 +20,59 @@ source_provider_module() { # Fetch secrets and set them as environment variables fetch_secrets() { - local secrets_json="$1" # The .config.secrets section is passed as an argument - + local secrets_json="$1" + log_info "Fetching secrets and setting them as environment variables." - local secrets_env_file - secrets_env_file=$(mktemp /tmp/secret_vars.XXXXXX) - echo "# Secrets environment variables" > "$secrets_env_file" - if [[ -z "$secrets_json" || "$secrets_json" == "null" ]]; then log_info "No worker secrets found in the configuration." - clean_up_files "$secrets_env_file" return 0 fi + # Create a temporary file to store environment variables + local secrets_env_file + secrets_env_file=$(mktemp /tmp/secret_vars.XXXXXX) + echo "# Secrets environment variables" > "$secrets_env_file" + + # Process each secret in the JSON object echo "$secrets_json" | jq -c 'to_entries[]' | while IFS= read -r secret; do - local name url value provider key_vault_name secret_name + local name url value provider secret_name key_vault_name name=$(echo "$secret" | jq -r '.key') url=$(resolve_env_vars "$(echo "$secret" | jq -r '.value')") - # Extract provider, key_vault_name, and secret_name from the URL + # Extract provider from the URL (first part before '/') provider=$(echo "$url" | cut -d '/' -f 1) - key_vault_name=$(echo "$url" | cut -d '/' -f 2) - secret_name=$(echo "$url" | cut -d '/' -f 3) - - # Ensure provider, key_vault_name, and secret_name are all set - if [[ -z "$provider" || -z "$key_vault_name" || -z "$secret_name" ]]; then - log_error "Invalid secret format: $url" - continue - fi + # Handle secrets based on the provider + case "$provider" in + gcp) + # Extract secret name and pass to GCP resolver + key_vault_name=$(echo "$url" | cut -d '/' -f 2) + secret_name=$(echo "$url" | cut -d '/' -f 3) + if [[ -z "$secret_name" ]]; then + log_error "Invalid GCP secret name: $url" + continue + fi + ;; + azure|bitwarden) + # Extract key vault and secret name + key_vault_name=$(echo "$url" | cut -d '/' -f 2) + secret_name=$(echo "$url" | cut -d '/' -f 3) + if [[ -z "$key_vault_name" || -z "$secret_name" ]]; then + log_error "Invalid secret format for $provider: $url" + continue + fi + ;; + *) + log_warn "Unsupported provider: $provider" + continue + ;; + esac + + # Source the provider module dynamically source_provider_module "$provider" + # Determine the resolve function for the provider local resolve_function="resolve_${provider}_secret" if command -v "$resolve_function" > /dev/null; then value=$("$resolve_function" "$key_vault_name" "$secret_name") @@ -60,6 +81,7 @@ fetch_secrets() { continue fi + # Export the secret as an environment variable if [[ -n "$value" ]]; then echo "export $name=\"$value\"" >> "$secrets_env_file" log_info "Resolved secret for $name from $provider." @@ -68,14 +90,15 @@ fetch_secrets() { fi done - if [[ -f "$secrets_env_file" ]]; then + # Source the environment file if it exists + if [[ -s "$secrets_env_file" ]]; then set -a - # shellcheck source=/tmp/secret_vars.XXXXXX disable=SC1091 + # shellcheck disable=SC1090 source "$secrets_env_file" set +a log_info "Secrets environment variables sourced successfully." else - log_error "Secrets environment file not found: $secrets_env_file" + log_error "No secrets were written to the environment file." return 1 fi @@ -93,3 +116,6 @@ clean_up_files() { fi done } + +# Example usage: +# fetch_secrets '{"TEST": "gcp/new_relic_api_key"}' diff --git a/lib/secrets/aws.sh b/lib/secrets/aws.sh new file mode 100644 index 00000000..ed381229 --- /dev/null +++ b/lib/secrets/aws.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Function to resolve AWS secret +resolve_aws_secret(){ + echo "Not supported yet. Is in progress" +} \ No newline at end of file diff --git a/lib/secrets/bitwarden.sh b/lib/secrets/bitwarden.sh new file mode 100644 index 00000000..6c3f1921 --- /dev/null +++ b/lib/secrets/bitwarden.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# Function to resolve Bitwarden secret +resolve_bitwarden_secret(){ + echo "Not supported yet. Is in progress" +} \ No newline at end of file diff --git a/lib/secrets/gcp.sh b/lib/secrets/gcp.sh new file mode 100644 index 00000000..4dd1d2dd --- /dev/null +++ b/lib/secrets/gcp.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Function to resolve GCP secret +resolve_gcp_secret() { + local project_id="$1" + local secret_name="$2" + local secret_value + + # Validate input arguments + if [[ -z "$project_id" || -z "$secret_name" ]]; then + log_error "Invalid GCP project ID or secret name. project_id: $project_id, secret_name: $secret_name" + return 1 + fi + + # log_info "Retrieving secret from GCP Secret Manager: project_id=$project_id, secret_name=$secret_name" + + # Retrieve the latest version of the secret + if ! secret_value=$(gcloud secrets versions access latest --secret="$secret_name" --project="$project_id" 2>/tmp/gcp_secret_error.log); then + log_error "Failed to retrieve secret from GCP Secret Manager for secret: $secret_name" + log_error "GCP CLI output: $(cat /tmp/gcp_secret_error.log)" + return 1 + fi + + # Check if the secret value is empty + if [[ -z "$secret_value" ]]; then + log_error "Secret value is empty for secret: $secret_name in project: $project_id" + return 1 + fi + + # Output only the secret value + printf "%s" "$secret_value" + return 0 +} diff --git a/lib/secrets/readme.md b/lib/secrets/readme.md index 02519052..12ef5768 100644 --- a/lib/secrets/readme.md +++ b/lib/secrets/readme.md @@ -25,8 +25,9 @@ az keyvault secret set --vault-name "your-vault-name" --name "your-secret-name" Update your worker.yml configuration file to include the Azure Key Vault secrets: ```yaml -workerSecrets: - MY_SECRET: "https://your-vault-name.vault.azure.net/secrets/your-secret-name" +secrets: + NEW_RELIC_API_KEY: "azure/kv-udx-worker/new-relic-api-key" + HEALTHCHECK_IO_API_KEY: "azure/kv-udx-worker/healthcheck-io-api-key" ``` ### AWS IAM Role (TBD) @@ -37,6 +38,10 @@ Instructions for setting up AWS IAM Role will be provided here. Instructions for setting up GCP Service Account will be provided here. +### Bitwarden Service Account (TBD) + +Instructions for setting up GCP Service Account will be provided here. + ## Best Practices - **Encrypt secrets**: Ensure secrets are encrypted at rest and in transit. diff --git a/lib/utils.sh b/lib/utils.sh index c555f8b4..cf034c12 100644 --- a/lib/utils.sh +++ b/lib/utils.sh @@ -11,19 +11,19 @@ if [ -z "${UTILS_SH_INCLUDED+x}" ]; then # Simple logging functions log_info() { - echo "[INFO] $1" + echo "[INFO] $1" >&2 # Send to stderr } log_error() { - echo "[ERROR] $1" + echo "[ERROR] $1" >&2 # Send to stderr } log_warn() { - echo "[WARN] $1" + echo "[WARN] $1" >&2 # Send to stderr } - + log_debug() { - echo "[DEBUG] $1" + echo "[DEBUG] $1" >&2 # Send to stderr } # Function to resolve placeholders with environment variables diff --git a/src/configs/readme.md b/src/configs/readme.md index 913a794e..e0e5dd55 100644 --- a/src/configs/readme.md +++ b/src/configs/readme.md @@ -21,7 +21,8 @@ config: variables: DOCKER_IMAGE_NAME: "udx-worker" secrets: - NEW_RELIC_API_KEY: "azure/kv-udx-worker/udx-worker-secret-one" + NEW_RELIC_API_KEY: "azure/kv-udx-worker/new-relic-api-key" + HEALTHCHECK_IO_API_KEY: "azure/kv-udx-worker/healthcheck-io-api-key" actors: - type: azure creds: "${AZURE_CREDS}" diff --git a/src/configs/worker.yml b/src/configs/worker.yml index 8d5d0e5d..3a0dd41b 100644 --- a/src/configs/worker.yml +++ b/src/configs/worker.yml @@ -5,14 +5,34 @@ config: variables: DOCKER_IMAGE_NAME: "udx-worker" secrets: - # supported - NEW_RELIC_API_KEY: "azure/kv-udx-worker/udx-worker-secret-one" - # for testing - OCTOPUS_API_KEY: "bitwarden/octetopus_api_key" + NEW_RELIC_API_KEY: "gcp/udx-worker-project/new_relic_api_key" + + # Supported: + # + # NEW_RELIC_API_KEY: "azure/kv-udx-worker/new_relic_api_key" + # + + # To be supported: + # + # NEW_RELIC_API_KEY: "aws/secrets-manager/new_relic_api_key" + # NEW_RELIC_API_KEY: "bitwarden/collection/new_relic_api_key" + # + actors: - # supported - - type: azure - creds: "${AZURE_CREDS}" - # for testing - type: gcp creds: "${GCP_CREDS}" + + # Supported: + # + # - type: azure + # creds: "${AZURE_CREDS}" + # + + # To be supported: + # + # - type: aws + # creds: "${AWS_CREDS}" + # + # - type: bitwarden + # creds: "${BITWARDEN_CREDS}" + #