diff --git a/.dockerignore b/.dockerignore index a0c0a5b1..1c9975e5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -32,9 +32,6 @@ Makefile.variables *.bak *.old -# Unnecessary configuration or text files -etc/home/logo.txt - # Readme files in the secrets and auth directories lib/auth/readme.md lib/secrets/readme.md diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 0bd62120..3929ddd0 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,9 +1,17 @@ ---- name: Build and Test Docker Image on: push: branches-ignore: latest + paths: + - '.github/workflows/build-and-test.yml' + - 'Dockerfile' + - 'bin/**' + - 'lib/**' + - 'src/**' + - 'etc/**' + - 'Makefile' + - 'Makefile.variables' jobs: build: @@ -24,14 +32,6 @@ jobs: run: make dev-pipeline working-directory: . - - name: Cache Trivy DB - uses: actions/cache@v4 - with: - path: ~/.cache/trivy - key: ${{ runner.os }}-trivy-db-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-trivy-db- - - name: Install Trivy run: | curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ @@ -54,28 +54,28 @@ jobs: # Run the Trivy scan and capture the exit status trivy image --severity CRITICAL --exit-code 1 --quiet \ - udx-worker/udx-worker:latest | tee trivy.log | grep -v 'INFO' + usabilitydynamics/udx-worker:latest | tee trivy.log | grep -v 'INFO' scan_exit_code=$? - # Check for CRITICAL vulnerabilities + # Check if CRITICAL vulnerabilities were detected if grep -E "Total: [1-9]" trivy.log; then - echo "CRITICAL vulnerabilities detected!" + echo "CRITICAL vulnerabilities detected! Exiting." exit 1 fi - # Check if Trivy exited with an error + # Handle a successful scan (no critical vulnerabilities found) if [ $scan_exit_code -eq 0 ]; then echo "No CRITICAL vulnerabilities found." success=true break else - echo "Trivy scan failed, retrying in 2 minutes..." + echo "Trivy scan encountered an error, retrying in 2 minutes..." sleep 120 attempt=$((attempt+1)) fi done - # If all retries fail, exit with an error + # Exit if all retries fail without a successful scan if [ "$success" = false ]; then echo "Failed to complete Trivy scan after $max_retries attempts." exit 1 @@ -83,15 +83,13 @@ jobs: - name: Trivy SBOM Generation run: | - # Suppress verbose notices and informational messages export TRIVY_DISABLE_VEX_NOTICE=true - trivy image --format spdx-json --output sbom.json udx-worker/udx-worker:latest 2>/dev/null + trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:latest 2>/dev/null echo "SBOM Top Packages Summary:" echo "| Package Name | Version |" echo "|-------------------|-----------|" - # Use jq to extract name and versionInfo, excluding packages with null versions, and pipe to column for formatting jq -r '.packages[] | select(.versionInfo != null) | "\(.name) | \(.versionInfo)"' sbom.json | sort | uniq | head -n 20 | column -t -s '|' - name: Upload SBOM Artifact diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d35f87bd..35bcaa8d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,4 @@ ---- -name: Release UDX Worker +name: Release on: push: @@ -7,14 +6,16 @@ on: - "latest" jobs: - test-pipeline: + docker-release: runs-on: ubuntu-latest permissions: id-token: write - contents: read + contents: write + outputs: semVer: ${{ steps.gitversion.outputs.semVer }} changelog: ${{ steps.changelog.outputs.changelog }} + steps: - name: Checkout code uses: actions/checkout@v4 @@ -26,25 +27,14 @@ jobs: with: driver: docker-container - - name: Prepare Docker cache directory - run: mkdir -p /tmp/.buildx-cache - - - name: Cache Docker layers - uses: actions/cache@v4 + - name: Install GitVersion + uses: gittools/actions/gitversion/setup@v3.0.0 with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} - restore-keys: | - ${{ runner.os }}-buildx- + versionSpec: "6.0.0" - name: Clear GitVersion Cache run: rm -rf .git/gitversion_cache - - name: Install GitVersion - uses: gittools/actions/gitversion/setup@v3.0.0 - with: - versionSpec: "5.12.0" - - name: Determine Version id: gitversion uses: gittools/actions/gitversion/execute@v3.0.0 @@ -52,25 +42,66 @@ jobs: useConfigFile: true configFilePath: ci/git-version.yml - - name: Multi-arch build - id: build + - name: Generate Changelog + id: changelog + run: | + git log $(git describe --tags --abbrev=0)..HEAD -- . \ + --pretty=format:"- %s" > changelog.txt + CHANGELOG=$(cat changelog.txt | jq -sRr @uri) + echo "changelog<> $GITHUB_ENV + echo "$CHANGELOG" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: "usabilitydynamics" + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and Push Docker Image + id: docker_push uses: docker/build-push-action@v6 with: context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: false - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache,mode=max + platforms: linux/amd64, linux/arm64 + push: true + sbom: true + provenance: true tags: | usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} + usabilitydynamics/udx-worker:latest - - name: Generate SBOM + - name: Install Trivy run: | - export TRIVY_DISABLE_VEX_NOTICE=true - trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} 2>/dev/null - # Save SBOM for later upload + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | \ + sudo sh -s -- -b /usr/local/bin + + - name: Generate SBOM with Retry Logic id: generate-sbom + run: | + export TRIVY_DISABLE_VEX_NOTICE=true + max_retries=10 + attempt=1 + success=false + while [ $attempt -le $max_retries ]; do + echo "Generating SBOM, attempt $attempt..." + output=$(trivy image --format spdx-json --output sbom.json usabilitydynamics/udx-worker:${{ steps.gitversion.outputs.semVer }} 2>&1) + sbom_exit_code=$? + if [ $sbom_exit_code -eq 0 ]; then + echo "SBOM generation successful." + success=true + break + else + echo "Retrying in 120 seconds..." + sleep 120 + attempt=$((attempt+1)) + fi + done + if [ "$success" = false ]; then + exit 1 + fi - name: Upload SBOM Artifact uses: actions/upload-artifact@v4 @@ -78,112 +109,41 @@ jobs: name: sbom path: sbom.json - - name: Generate changelog - id: changelog - run: | - git log $(git describe --tags --abbrev=0)..HEAD -- . \ - --pretty=format:"- %s" > changelog.txt - CHANGELOG=$(cat changelog.txt | jq -sRr @uri) - echo "changelog<> $GITHUB_ENV - echo "$CHANGELOG" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV + - name: Log out from Docker Hub + run: docker logout github-release: runs-on: ubuntu-latest - needs: [test-pipeline] + needs: docker-release permissions: contents: write + steps: - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Configure git for pushing + - name: Configure Git for Pushing run: | git config --global user.email "worker@udx.io" git config --global user.name "UDX Worker" - - name: Create GitHub Tag - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - run: | - git tag ${{ needs.test-pipeline.outputs.semVer }} - git push origin ${{ needs.test-pipeline.outputs.semVer }} - - name: Download SBOM Artifact uses: actions/download-artifact@v4 with: name: sbom - - name: Create GitHub release + - name: Create GitHub Release uses: softprops/action-gh-release@v2 with: - tag_name: ${{ needs.test-pipeline.outputs.semVer }} - body: ${{ needs.test-pipeline.outputs.changelog }} + tag_name: ${{ needs.docker-release.outputs.semVer }} + body: | + Release version ${{ needs.docker-release.outputs.semVer }}. + [View on Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker/tags?page=1&ordering=last_updated). + ${{ needs.docker-release.outputs.changelog }} draft: false prerelease: false files: sbom.json env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - docker-release: - runs-on: ubuntu-latest - needs: [test-pipeline] - permissions: - contents: write - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker-container - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Load Docker cache - uses: actions/cache@v4 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }} - - - name: Push Docker image - uses: docker/build-push-action@v6 - with: - context: . - file: ./Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - cache-from: type=local,src=/tmp/.buildx-cache - tags: | - usabilitydynamics/udx-worker:${{ needs.test-pipeline.outputs.semVer }} - usabilitydynamics/udx-worker:latest - - - name: Download SBOM Artifact - uses: actions/download-artifact@v4 - with: - name: sbom - - - name: Install Cosign - uses: sigstore/cosign-installer@v3.7.0 - - - name: Sign SBOM with Cosign - env: - COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} - run: | - cosign attest \ - --key env://COSIGN_PRIVATE_KEY \ - --type sbom \ - --predicate sbom.json \ - usabilitydynamics/udx-worker:${{ needs.test-pipeline.outputs.semVer }} - - - name: Log out from Docker Hub - run: docker logout diff --git a/.yamllint b/.yamllint index 315d44fc..b9759552 100644 --- a/.yamllint +++ b/.yamllint @@ -2,11 +2,11 @@ extends: default rules: line-length: - max: 105 # Set a longer line length if 80 is too restrictive + max: 150 # Set a longer line length if 80 is too restrictive level: warning # Level should be either "error" or "warning" truthy: level: warning # Correct the level to "error" or "warning" instead of any invalid value comments-indentation: - level: warning # Level should be either "error" or "warning" + level: warning # Level should be either "error" or "warning" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 5fc7db0d..350c3073 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update && \ apt-get install -y --no-install-recommends \ tzdata=2024a-3ubuntu1.1 \ - curl=8.5.0-2ubuntu10.4 \ + curl=8.5.0-2ubuntu10.5 \ bash=5.2.21-2ubuntu4 \ apt-utils=2.7.14build2 \ gettext=0.21-14ubuntu2 \ @@ -33,7 +33,7 @@ RUN apt-get update && \ zip=3.0-13build1 \ unzip=6.0-28ubuntu4 \ nano=7.2-2build1 \ - vim=2:9.1.0016-1ubuntu7.3 && \ + vim=2:9.1.0016-1ubuntu7.5 && \ ln -fs /usr/share/zoneinfo/$TZ /etc/localtime && \ dpkg-reconfigure --frontend noninteractive tzdata && \ apt-get clean && \ @@ -85,25 +85,27 @@ RUN curl -Lso /usr/local/bin/bw "https://vault.bitwarden.com/download/?app=cli&p RUN groupadd -g ${GID} ${USER} && \ useradd -l -m -u ${UID} -g ${GID} -s /bin/bash ${USER} +# Prepare directories for the user and worker configuration +RUN mkdir -p /etc/worker /home/${USER}/.cd/bin /home/${USER}/.cd/configs && \ + touch /home/${USER}/.cd/configs/merged_worker.yml && \ + chown -R ${UID}:${GID} /etc/worker /home/${USER}/.cd && \ + chmod 600 /home/${USER}/.cd/configs/merged_worker.yml + # Switch to the user directory WORKDIR /home/${USER} -# Create necessary directories and set permissions for GPG and other files -RUN mkdir -p /home/${USER}/.gnupg && \ - chmod 700 /home/${USER}/.gnupg && \ - mkdir -p /home/${USER}/etc /home/${USER}/.cd/configs && \ - chown -R ${USER}:${USER} /home/${USER} +# Copy built-in worker.yml to the container +COPY ./src/configs/worker.yml /etc/worker/worker.yml # Copy the bin, etc, and lib directories COPY ./etc/home /home/${USER}/etc -COPY ./src/configs /home/${USER}/.cd/configs COPY ./lib /usr/local/lib COPY ./bin/entrypoint.sh /usr/local/bin/entrypoint.sh COPY ./bin/test.sh /usr/local/bin/test.sh -# Set executable permissions and ownership for scripts -RUN chmod +x /usr/local/lib/* /usr/local/bin/entrypoint.sh /usr/local/bin/test.sh && \ - chown -R ${USER}:${USER} /usr/local/lib /home/${USER}/etc /home/${USER}/.cd/configs +# Set permissions during build +RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/test.sh && \ + chown -R ${UID}:${GID} /usr/local/lib /etc/worker /home/${USER}/etc /home/${USER}/.cd # Switch to non-root user USER ${USER} diff --git a/Makefile.variables b/Makefile.variables index 56494988..0e558242 100644 --- a/Makefile.variables +++ b/Makefile.variables @@ -1,5 +1,5 @@ # Variables -IMAGE_NAME ?= udx-worker/udx-worker +IMAGE_NAME ?= usabilitydynamics/udx-worker TAG ?= latest DOCKER_IMAGE := $(IMAGE_NAME):$(TAG) CONTAINER_NAME ?= udx-worker-container diff --git a/bin/test.sh b/bin/test.sh index 6b2500da..19c63b46 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -1,58 +1,8 @@ #!/bin/bash -echo "Starting validation tests..." - -# Dynamically redact sensitive environment variables for output -redact_sensitive_vars() { - local var_name="$1" - local value="$2" - local SENSITIVE_PATTERN="PASSWORD|SECRET|KEY|TOKEN" - if echo "$var_name" | grep -Eq "$SENSITIVE_PATTERN"; then - echo "$var_name=********" - else - echo "$var_name=$value" - fi -} - -# Extract required environment variables from the configuration -WORKER_CONFIG="/home/$USER/.cd/configs/worker.yml" -REQUIRED_VARS=$(yq e '.config.env | to_entries | .[].key' "$WORKER_CONFIG") -echo "Required environment variables: $REQUIRED_VARS" -# Ensure all required environment variables are set -for var in $REQUIRED_VARS; do - value=$(printenv "$var") - if [ -z "$value" ]; then - echo "Error: Environment variable $var is not set." - exit 1 - else - echo "Environment variable $var is set." - fi -done - -# Verify secrets are fetched (assuming secrets are set as environment variables) -SECRETS=$(yq e '.config.workerSecrets | to_entries | .[].key' "$WORKER_CONFIG") -for secret in $SECRETS; do - value=$(printenv "$secret") - if [ -z "$value" ]; then - echo "Error: Secret $secret is not resolved correctly." - exit 1 - else - echo "Secret $secret is resolved correctly." - fi -done +echo "Starting validation tests..." -# Confirm that sensitive actor variables are not set -ACTOR_VARS=$(yq e '.config.workerActors[] | to_entries | .[].value' "$WORKER_CONFIG" | grep -E "PASSWORD|SECRET|KEY|TOKEN") -for var in $ACTOR_VARS; do - var_name="${var//*\{/\{}" - var_name="${var_name//\}*/\}}" - if [ -n "$(printenv "$var_name")" ]; then - echo "Error: Sensitive variable $var_name is still set after cleanup." - exit 1 - else - echo "Sensitive variable $var_name is not set as expected." - fi -done +##@TODO: Add validation tests echo "All validation tests passed successfully." diff --git a/ci/git-version.yml b/ci/git-version.yml index 5bed307f..f0c01219 100644 --- a/ci/git-version.yml +++ b/ci/git-version.yml @@ -1,11 +1,17 @@ --- -mode: Mainline +mode: ContinuousDeployment branches: latest: regex: ^latest$ - tag: '' increment: Minor source-branches: [] + other: + regex: ^(?!latest).*$ + increment: Patch + source-branches: [] + tracks-release-branches: false + is-release-branch: false + assembly-versioning-scheme: MajorMinorPatch diff --git a/lib/auth.sh b/lib/auth.sh index 9cf834d1..38d517f2 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -3,6 +3,9 @@ # shellcheck source=/usr/local/lib/utils.sh disable=SC1091 source /usr/local/lib/utils.sh +# Array to track configured providers +declare -a configured_providers=() + # Function to authenticate actors authenticate_actors() { local actors_json="$1" # Expect the extracted actors JSON as a parameter @@ -12,8 +15,17 @@ authenticate_actors() { return 0 fi - # Process each actor in the configuration - echo "$actors_json" | jq -c '.[]' | while IFS= read -r actor; do + # Use a temporary file to avoid a subshell + local actors_file + actors_file=$(mktemp) + echo "$actors_json" | jq -c '.[]' > "$actors_file" + + # Read all lines into an array, then delete the file + mapfile -t actors_array < "$actors_file" + rm -f "$actors_file" + + # Process each actor in the array + for actor in "${actors_array[@]}"; do local type provider creds auth_script auth_function # Extract the type and provider from the actor data @@ -48,6 +60,8 @@ authenticate_actors() { log_error "Authentication failed for provider $provider" return 1 fi + # Add provider to configured list if authentication succeeds + configured_providers+=("$provider") else log_error "Authentication function $auth_function not found for provider $provider" return 1 @@ -88,5 +102,69 @@ authenticate_provider() { return 0 } -# Example usage (expects actors JSON to be passed): +# Generic function to clean up authentication for any provider +cleanup_provider() { + local provider=$1 + local logout_cmd=$2 + local list_cmd=$3 + local name=$4 + local cleaned_up=false + + # Check if the provider's CLI is available + if ! command -v "$provider" > /dev/null; then + return 0 # Skip silently if CLI is not available + fi + + # Check if there are active sessions/accounts to clean up + if ! eval "$list_cmd" > /dev/null 2>&1; then + return 0 # Skip silently if no active sessions are found + fi + + log_info "Cleaning up $name authentication" + + # Run the logout command and capture any output or errors + local logout_output + logout_output=$(eval "$logout_cmd" 2>&1) + local logout_status=$? + + # Check if the logout was successful or if an expected error message was returned + if [[ $logout_status -ne 0 ]]; then + if echo "$logout_output" | grep -q -E "No credentials available to revoke|No active sessions|No active accounts"; then + log_info "No active $name credentials to revoke." + else + log_error "Failed to log out of $name: $logout_output" + return 1 + fi + else + log_info "$name authentication cleaned up successfully." + cleaned_up=true + fi + + # Return the cleanup status for summary reporting + if [[ "$cleaned_up" == true ]]; then + return 0 + else + return 1 + fi +} + +# Function to clean up sensitive environment variables based on a pattern +cleanup_sensitive_env_vars() { + log_info "Cleaning up sensitive environment variables" + + # Define a pattern for sensitive environment variables (e.g., AZURE_CREDS, GCP_CREDS, etc.) + local pattern="_CREDS" + + # Loop through environment variables that match the pattern + for var in $(env | grep "${pattern}" | cut -d'=' -f1); do + unset "$var" + log_info "Unset sensitive environment variable: $var" + done + + log_info "Sensitive environment variables cleaned up successfully." +} + +# Example usage: # authenticate_actors "$actors_json" +# cleanup_actors +# cleanup_sensitive_env_vars diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 4af48ec4..d6c26624 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -12,68 +12,81 @@ cleanup_provider() { local logout_cmd=$2 local list_cmd=$3 local name=$4 - - log_info "Cleaning up $name authentication" - + local cleaned_up=false + + # Check if the provider's CLI is available if ! command -v "$provider" > /dev/null; then - log_warn "$name CLI not found. Skipping $name cleanup." - return 0 + return 0 # Skip silently if CLI is not available fi + # Check if there are active sessions/accounts to clean up if ! eval "$list_cmd" > /dev/null 2>&1; then - log_info "No active $name accounts or sessions found." - return 0 + return 0 # Skip silently if no active sessions are found fi - if ! eval "$logout_cmd"; then - log_error "Failed to log out of $name" - return 1 + log_info "Cleaning up $name authentication" + + # Run the logout command and capture any output or errors + local logout_output + logout_output=$(eval "$logout_cmd" 2>&1) + local logout_status=$? + + # Check if the logout was successful or if an expected error message was returned + if [[ $logout_status -ne 0 ]]; then + if echo "$logout_output" | grep -q -E "No credentials available to revoke|No active sessions|No active accounts"; then + log_info "No active $name credentials to revoke." + else + log_error "Failed to log out of $name: $logout_output" + return 1 + fi + else + log_info "$name authentication cleaned up successfully." + cleaned_up=true fi - log_info "$name authentication cleaned up successfully." + # Return the cleanup status for summary reporting + if [[ "$cleaned_up" == true ]]; then + return 0 + else + return 1 + fi } -# Function to clean up actors based on the worker configuration +# Function to clean up actors based on the providers configured during authentication cleanup_actors() { log_info "Starting cleanup of actors" - local worker_config - if ! worker_config=$(get_worker_config_path); then - log_error "Failed to retrieve worker configuration path." - return 1 - fi - - local actors_json - actors_json=$(yq e -o=json '.config.actors' "$worker_config" 2>/dev/null) - - if [[ -z "$actors_json" || "$actors_json" == "null" ]]; then - log_info "No actors found for cleanup." - return 0 - fi - - # Process each actor type - echo "$actors_json" | jq -c '.[]' | while IFS= read -r actor; do - local type - type=$(echo "$actor" | jq -r '.type') + # Accept configured providers as arguments + local configured_providers=("$@") + + # Track if any actual cleanup was performed + local any_cleanup=false - case "$type" in + # Loop through each configured provider only + for provider in "${configured_providers[@]}"; do + case "$provider" in azure) - cleanup_provider "az" "az logout" "az account show" "Azure" + cleanup_provider "az" "az logout" "az account show" "Azure" && any_cleanup=true ;; gcp) - cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" + cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" && any_cleanup=true ;; aws) - cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" + cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" && any_cleanup=true ;; bitwarden) - cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" + cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" && any_cleanup=true ;; *) - log_warn "Unsupported or unavailable actor type for cleanup: $type" + log_warn "Unsupported or unavailable actor type for cleanup: $provider" ;; esac done + + # Log a summary if no cleanup actions were needed + if [[ "$any_cleanup" == false ]]; then + log_info "No active sessions found for any configured providers." + fi } # Function to clean up sensitive environment variables based on a pattern diff --git a/lib/environment.sh b/lib/environment.sh index 8b59f359..c1ca2aa2 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -1,65 +1,81 @@ #!/bin/bash -# Include necessary modules -# shellcheck source=/dev/null -source /usr/local/lib/utils.sh -source /usr/local/lib/auth.sh -source /usr/local/lib/secrets.sh -source /usr/local/lib/cleanup.sh -source /usr/local/lib/worker_config.sh +# Get the directory of this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Source a file if it exists +source_if_exists() { + local file_path="$1" + if [[ -f "$file_path" ]]; then + # shellcheck disable=SC1090 + source "$file_path" + else + echo "[ERROR] Missing file: $file_path" >&2 + exit 1 + fi +} + +# Include necessary modules from the same directory +# shellcheck source=./utils.sh +source_if_exists "$SCRIPT_DIR/utils.sh" +# shellcheck source=./auth.sh +source_if_exists "$SCRIPT_DIR/auth.sh" +# shellcheck source=./secrets.sh +source_if_exists "$SCRIPT_DIR/secrets.sh" +# shellcheck source=./cleanup.sh +source_if_exists "$SCRIPT_DIR/cleanup.sh" +# shellcheck source=./worker_config.sh +source_if_exists "$SCRIPT_DIR/worker_config.sh" # Main function to coordinate environment setup configure_environment() { + log_info "Starting environment configuration..." # Load and resolve the worker configuration local resolved_config - resolved_config=$(load_and_resolve_worker_config) - # Directly check the command success, not using $? - if [ -z "$resolved_config" ]; then - log_error "Failed to resolve worker configuration." + resolved_config=$(load_and_parse_config) + if [[ -z "$resolved_config" ]]; then + log_error "Configuration loading failed. Exiting..." return 1 fi - # Verify the config file exists at the expected path - local config_path - config_path=$(get_worker_config_path) - if [[ ! -f "$config_path" ]]; then - log_error "Configuration file not found at: $config_path" - return 1 - fi - log_info "Config file found: $config_path" + log_info "Worker configuration loaded successfully." - # Extract actors section from the resolved configuration - local actors - actors=$(get_worker_section "$resolved_config" "config.actors") - # log_debug "Extracted actors: $actors" - if [ -z "$actors" ]; then - log_error "No actors found in the configuration." + # Export variables from the configuration + log_info "Exporting variables from configuration to environment..." + if ! export_variables_from_config "$resolved_config"; then + log_error "Failed to export variables." return 1 fi - # Authenticate actors using the extracted actors section - if ! authenticate_actors "$actors"; then - log_error "Failed to authenticate actors." - return 1 + # Extract and authenticate actors + local actors + actors=$(get_config_section "$resolved_config" "actors") + if [[ $? -eq 0 && -n "$actors" ]]; then + log_info "Authenticating actors from configuration..." + if ! authenticate_actors "$actors"; then + log_error "Failed to authenticate actors." + return 1 + fi + else + log_info "No actors defined in the configuration." fi - # Extract secrets section from the resolved configuration + # Extract and fetch secrets local secrets - secrets=$(get_worker_section "$resolved_config" "config.secrets") - # log_debug "Extracted secrets: $secrets" - if [ -z "$secrets" ]; then - log_error "No secrets found in the configuration." - return 1 - fi - - # Fetch secrets using the resolved configuration - if ! fetch_secrets "$secrets"; then - log_error "Failed to fetch secrets." - return 1 + secrets=$(get_config_section "$resolved_config" "secrets") + if [[ $? -eq 0 && -n "$secrets" ]]; then + log_info "Fetching secrets from configuration..." + if ! fetch_secrets "$secrets"; then + log_error "Failed to fetch secrets." + return 1 + fi + else + log_info "No secrets defined in the configuration." fi - # Clean up actors and sensitive environment variables + # Perform cleanup + log_info "Cleaning up sensitive data..." if ! cleanup_actors; then log_error "Failed to clean up actors." return 1 diff --git a/lib/secrets.sh b/lib/secrets.sh index aae52f4a..fdd860ad 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -21,14 +21,21 @@ source_provider_module() { # Fetch secrets and set them as environment variables fetch_secrets() { local secrets_json="$1" - + log_info "Fetching secrets and setting them as environment variables." + # Exit early if secrets_json is empty, null, or invalid JSON if [[ -z "$secrets_json" || "$secrets_json" == "null" ]]; then log_info "No worker secrets found in the configuration." return 0 fi + # Confirm secrets_json is valid JSON before processing + if ! echo "$secrets_json" | jq empty > /dev/null 2>&1; then + log_error "Invalid JSON format for secrets configuration." + return 1 + fi + # Create a temporary file to store environment variables local secrets_env_file secrets_env_file=$(mktemp /tmp/secret_vars.XXXXXX) @@ -40,22 +47,26 @@ fetch_secrets() { name=$(echo "$secret" | jq -r '.key') url=$(resolve_env_vars "$(echo "$secret" | jq -r '.value')") + # Check if the secret has a valid name and URL + if [[ -z "$name" || -z "$url" ]]; then + log_error "Secret name or URL is missing or empty." + continue + fi + # Extract provider from the URL (first part before '/') provider=$(echo "$url" | cut -d '/' -f 1) # Handle secrets based on the provider case "$provider" in gcp) - # Extract secret name and pass to GCP resolver key_vault_name=$(echo "$url" | cut -d '/' -f 2) secret_name=$(echo "$url" | cut -d '/' -f 3) if [[ -z "$secret_name" ]]; then - log_error "Invalid GCP secret name: $url" + log_error "Invalid GCP secret name format: $url" continue fi ;; azure|bitwarden) - # Extract key vault and secret name key_vault_name=$(echo "$url" | cut -d '/' -f 2) secret_name=$(echo "$url" | cut -d '/' -f 3) if [[ -z "$key_vault_name" || -z "$secret_name" ]]; then diff --git a/lib/secrets/aws.sh b/lib/secrets/aws.sh index ed381229..b89fc012 100644 --- a/lib/secrets/aws.sh +++ b/lib/secrets/aws.sh @@ -1,6 +1,29 @@ #!/bin/bash # Function to resolve AWS secret -resolve_aws_secret(){ - echo "Not supported yet. Is in progress" +resolve_aws_secret() { + local secret_name="$1" + local region="$2" + local secret_value + + if [[ -z "$secret_name" || -z "$region" ]]; then + echo "[ERROR] Invalid AWS secret name or region" >&2 + return 1 + fi + + # Retrieve the secret value using AWS CLI with detailed logging + echo "[INFO] Retrieving secret from AWS Secrets Manager: secret_name=$secret_name, region=$region" >&2 + if ! secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query 'SecretString' --output text --region "$region" 2>&1); then + echo "[ERROR] Failed to retrieve secret from AWS Secrets Manager: secret_name=$secret_name, region=$region" >&2 + echo "[DEBUG] AWS CLI output: $secret_value" >&2 + return 1 + fi + + if [ -z "$secret_value" ]; then + echo "[ERROR] Secret value is empty for $secret_name in region $region" >&2 + return 1 + fi + + printf "%s" "$secret_value" + return 0 } \ No newline at end of file diff --git a/lib/secrets/bitwarden.sh b/lib/secrets/bitwarden.sh index 6c3f1921..f6665568 100644 --- a/lib/secrets/bitwarden.sh +++ b/lib/secrets/bitwarden.sh @@ -1,6 +1,29 @@ #!/bin/bash # Function to resolve Bitwarden secret -resolve_bitwarden_secret(){ - echo "Not supported yet. Is in progress" +resolve_bitwarden_secret() { + local collection_name="$1" + local secret_name="$2" + local secret_value + + if [[ -z "$collection_name" || -z "$secret_name" ]]; then + echo "[ERROR] Invalid Bitwarden collection name or secret name" >&2 + return 1 + fi + + # Retrieve the secret using Bitwarden CLI + echo "[INFO] Retrieving secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" >&2 + if ! secret_value=$(bw get item "$secret_name" --organizationid "$collection_name" --output text 2>&1); then + echo "[ERROR] Failed to retrieve secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" >&2 + echo "[DEBUG] Bitwarden CLI output: $secret_value" >&2 + return 1 + fi + + if [ -z "$secret_value" ]; then + echo "[ERROR] Secret value is empty for $secret_name in collection $collection_name" >&2 + return 1 + fi + + printf "%s" "$secret_value" + return 0 } \ No newline at end of file diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 038e5c07..1c4dc08f 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -1,53 +1,96 @@ #!/bin/bash +# Paths for configurations +BUILT_IN_CONFIG="/etc/worker/worker.yml" +USER_CONFIG="/home/udx/.cd/configs/worker.yml" +MERGED_CONFIG="/home/udx/.cd/configs/merged_worker.yml" + # Utility functions for logging log_info() { - echo "[INFO] $1" + echo "[INFO] $1" >&2 } log_error() { - echo "[ERROR] $1" + echo "[ERROR] $1" >&2 } -# Function to get the path to the worker.yml configuration file -get_worker_config_path() { - local config_path="/home/${USER}/.cd/configs/worker.yml" - - if [[ ! -f "$config_path" ]]; then - log_error "Configuration file not found: $config_path" +# Ensure `yq` is available +if ! command -v yq >/dev/null 2>&1; then + log_error "yq is not installed. Please ensure it is available in the PATH." + exit 1 +fi + +# Ensure configuration file exists +ensure_config_exists() { + local config_path="$1" + if [[ ! -s "$config_path" ]]; then + log_error "Configuration file not found or empty: $config_path" return 1 fi - - echo "$config_path" } -# Function to load the worker configuration from YAML and convert it to JSON -load_and_resolve_worker_config() { - local config_path - config_path=$(get_worker_config_path) +# Merge built-in and user-provided configurations +merge_worker_configs() { + log_info "Merging worker configurations..." - # Check if the config_path retrieval was successful - if [[ -z "$config_path" ]]; then - return 1 + # Ensure built-in config exists + ensure_config_exists "$BUILT_IN_CONFIG" || return 1 + + # If a user-provided configuration exists, merge it + if [[ -f "$USER_CONFIG" ]]; then + log_info "User configuration detected. Merging with the built-in configuration." + + if ! yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' "$BUILT_IN_CONFIG" "$USER_CONFIG" > "$MERGED_CONFIG"; then + log_error "Failed to merge configurations. yq returned an error." + return 1 + fi + else + log_info "No user configuration provided. Using built-in configuration only." + + # Copy the built-in configuration to the merged configuration + if ! cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG"; then + log_error "Failed to copy built-in configuration to merged configuration." + return 1 + fi fi +} + +# Load and parse the merged configuration +load_and_parse_config() { + merge_worker_configs || return 1 - # Convert the YAML configuration to JSON using yq + # Parse the merged configuration into JSON local json_output - if ! json_output=$(yq eval -o=json "$config_path" 2>/dev/null); then - log_error "Failed to parse YAML from $config_path. yq returned an error." + if ! json_output=$(yq eval -o=json "$MERGED_CONFIG" 2>/dev/null); then + log_error "Failed to parse merged YAML from $MERGED_CONFIG. yq returned an error." return 1 fi - if [[ -z "$json_output" ]]; then - log_error "YAML parsed to an empty JSON output." - return 1 + echo "$json_output" +} + +# Export variables from the configuration +export_variables_from_config() { + local config_json="$1" + + log_info "Exporting variables from configuration..." + + # Extract the `variables` section + local variables + variables=$(echo "$config_json" | jq -r '.config.env // empty') + if [[ -z "$variables" || "$variables" == "null" ]]; then + log_info "No variables found in the configuration." + return 0 fi - echo "$json_output" + # Iterate over variables and export them into the main shell + while IFS="=" read -r key value; do + eval "export $key=\"$value\"" + done < <(echo "$variables" | jq -r 'to_entries[] | "\(.key)=\(.value)"') } # Function to extract a specific section from the JSON configuration -get_worker_section() { +get_config_section() { local config_json="$1" local section="$2" @@ -56,22 +99,12 @@ get_worker_section() { return 1 fi + # Attempt to extract the section and handle missing/null cases local extracted_section - if ! extracted_section=$(echo "$config_json" | jq -r ".${section}"); then - log_error "Failed to extract section '$section' from JSON." - return 1 - fi - - if [[ -z "$extracted_section" || "$extracted_section" == "null" ]]; then - log_error "Section '$section' is empty or null." + if ! extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null); then + log_error "Failed to parse section '${section}' from configuration." return 1 fi echo "$extracted_section" -} - -# Example usage of the above functions -# You can comment this out if it’s just a library -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - configure_environment -fi +} \ No newline at end of file diff --git a/src/configs/worker.yml b/src/configs/worker.yml index 1ff0e0f6..f2700e65 100644 --- a/src/configs/worker.yml +++ b/src/configs/worker.yml @@ -2,29 +2,15 @@ kind: workerConfig version: udx.io/worker-v1/config config: - variables: - DOCKER_IMAGE_NAME: 'udx-worker' - - secrets: - NEW_RELIC_API_KEY: 'gcp/udx-worker-project/new_relic_api_key' - - # Supported: - # NEW_RELIC_API_KEY: "azure/kv-udx-worker/new_relic_api_key" - - # To be supported: - # NEW_RELIC_API_KEY: "aws/secrets-manager/new_relic_api_key" - # NEW_RELIC_API_KEY: "bitwarden/collection/new_relic_api_key" + env: + DOCKER_IMAGE_NAME: "udx-worker" actors: - type: gcp - creds: '${GCP_CREDS}' - - # Supported: - # - type: azure - # creds: "${AZURE_CREDS}" - - # To be supported: - # - type: aws - # creds: "${AWS_CREDS}" - # - type: bitwarden - # creds: "${BITWARDEN_CREDS}" + creds: "${GCP_CREDS}" + - type: azure + creds: "${AZURE_CREDS}" + - type: aws + creds: "${AWS_CREDS}" + - type: bitwarden + creds: "${BITWARDEN_CREDS}"