diff --git a/.yamllint b/.yamllint index b9759552..3457a8a3 100644 --- a/.yamllint +++ b/.yamllint @@ -2,7 +2,7 @@ extends: default rules: line-length: - max: 150 # Set a longer line length if 80 is too restrictive + max: 170 # Set a longer line length if 80 is too restrictive level: warning # Level should be either "error" or "warning" truthy: diff --git a/Dockerfile b/Dockerfile index 1ed22e61..dd21a372 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ USER root RUN apt-get update && \ apt-get install -y \ tzdata=2024b-6ubuntu1 \ - curl=8.11.1-1ubuntu1 \ + curl=8.12.0+git20250209.89ed161+ds-1ubuntu1 \ bash=5.2.37-1ubuntu1 \ apt-utils=2.9.28 \ gettext=0.23.1-1 \ @@ -58,9 +58,9 @@ RUN ARCH=$(uname -m) && \ ENV CLOUDSDK_CONFIG=/usr/local/configs/gcloud RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "x86_64" ]; then \ - curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-508.0.0-linux-x86_64.tar.gz" -o google-cloud-sdk.tar.gz; \ + curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-510.0.0-linux-x86_64.tar.gz" -o google-cloud-sdk.tar.gz; \ elif [ "$ARCH" = "aarch64" ]; then \ - curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-508.0.0-linux-arm.tar.gz" -o google-cloud-sdk.tar.gz; \ + curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-510.0.0-linux-arm.tar.gz" -o google-cloud-sdk.tar.gz; \ fi && \ tar -xzf google-cloud-sdk.tar.gz && \ ./google-cloud-sdk/install.sh -q && \ @@ -86,7 +86,7 @@ RUN mkdir -p $GNUPGHOME && \ gpg --export EB3E94ADBE1229CF | tee /usr/share/keyrings/microsoft-archive-keyring.gpg && \ echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/azure-cli/ jammy main" | tee /etc/apt/sources.list.d/azure-cli.list && \ apt-get update && \ - apt-get install -y --no-install-recommends azure-cli=2.68.0-1~jammy && \ + apt-get install -y --no-install-recommends azure-cli=2.69.0-1~jammy && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -116,17 +116,21 @@ RUN chmod +x /usr/local/bin/worker_mgmt && \ ln -s /usr/local/bin/worker_mgmt /usr/local/bin/worker # Copy the bin, etc, and lib directories -COPY etc/home /etc COPY etc/configs /usr/local/configs COPY lib /usr/local/lib COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh # Set permissions during build -RUN chmod +x /usr/local/bin/entrypoint.sh && \ - chown -R ${UID}:${GID} /usr/local/configs && \ - chown -R ${UID}:${GID} /usr/local/bin && \ - chown -R ${UID}:${GID} /usr/local/lib && \ - chmod -R g-w,o-w /usr/local/configs /usr/local/bin /usr/local/lib +# Set ownership +RUN chown -R ${UID}:${GID} /usr/local/configs /usr/local/bin /usr/local/lib && \ + # Make specific scripts executable + chmod 755 /usr/local/bin/entrypoint.sh /usr/local/lib/process_manager.sh && \ + # Set read-only permissions for config files + find /usr/local/configs -type f -exec chmod 644 {} + && \ + # Set read-only permissions for library files + find /usr/local/lib -type f ! -name process_manager.sh -exec chmod 644 {} + && \ + # Ensure directories are accessible + find /usr/local/configs /usr/local/bin /usr/local/lib -type d -exec chmod 755 {} + # Create a symbolic link for the supervisord configuration file RUN ln -sf /usr/local/configs/supervisor/supervisord.conf /etc/supervisord.conf diff --git a/Makefile b/Makefile index 44843247..ddfe6d1c 100644 --- a/Makefile +++ b/Makefile @@ -7,68 +7,93 @@ include Makefile.help .PHONY: run run-it clean build exec log test dev-pipeline -# Build the Docker image -MULTIPLATFORM ?= false +# Use BuildKit for better output +export DOCKER_BUILDKIT=1 + + +.SILENT: build run run-it clean exec log test dev-pipeline + +# Common shell function for error handling and output +define exec_with_status +@bash -c 'set -eo pipefail; \ + printf "$(COLOR_BLUE)$(SYM_ARROW) %s$(COLOR_RESET)\n" "$(1)"; \ + { $(2); } && \ + printf "$(COLOR_GREEN)$(SYM_SUCCESS) %s$(COLOR_RESET)\n" "$(3)" || \ + { printf "$(COLOR_RED)$(SYM_ERROR) %s$(COLOR_RESET)\n" "$(4)"; exit 1; }' +endef build: - @echo "Building Docker image..." - @if [ "$(MULTIPLATFORM)" = "true" ]; then \ - echo "Multiple platforms: [linux/amd64, linux/arm64]..."; \ - docker buildx build --platform linux/amd64,linux/arm64 -t $(DOCKER_IMAGE) --load .; \ - else \ - echo "Only local platform..."; \ - docker build -t $(DOCKER_IMAGE) .; \ - fi - @echo "Docker image build completed." + @bash -c 'set -eo pipefail; \ + filter="(error|Error|ERROR|failed|Failed|FAILED|\\[.*[0-9]+/[0-9]+\\]|^#[0-9]+ DONE|sha256|CACHED)"; \ + printf "$(COLOR_BLUE)$(SYM_ARROW) Starting Docker build...$(COLOR_RESET)\n"; \ + if [ "$(MULTIPLATFORM)" = "true" ]; then \ + printf "$(COLOR_BLUE)$(SYM_ARROW) Building for multiple platforms: [linux/amd64, linux/arm64]$(COLOR_RESET)\n"; \ + docker buildx build --progress=plain \ + --platform linux/amd64,linux/arm64 \ + -t $(DOCKER_IMAGE) \ + --load . 2>&1 | grep -E "$$filter" || exit 1; \ + else \ + printf "$(COLOR_BLUE)$(SYM_ARROW) Building for local platform$(COLOR_RESET)\n"; \ + DOCKER_BUILDKIT=1 docker build \ + --progress=plain \ + -t $(DOCKER_IMAGE) . 2>&1 | grep -E "$$filter" || exit 1; \ + fi && \ + printf "$(COLOR_GREEN)$(SYM_SUCCESS) Docker image build completed$(COLOR_RESET)\n" || \ + { printf "$(COLOR_RED)$(SYM_ERROR) Docker build failed$(COLOR_RESET)\n"; exit 1; }' -# Run Docker container (supports interactive mode) run: clean - @echo "Running Docker container..." - + @printf "$(COLOR_BLUE)$(SYM_ARROW) Starting container...$(COLOR_RESET)\n" @if [ ! -f $(ENV_FILE) ]; then \ - echo "Creating environment file..."; \ + printf "$(COLOR_YELLOW)$(SYM_WARNING) Creating environment file...$(COLOR_RESET)\n" && \ touch $(ENV_FILE); \ - else \ - echo "Environment file exists..."; \ fi - @docker run $(if $(INTERACTIVE),-it,-d) --rm --name $(CONTAINER_NAME) \ --env-file $(ENV_FILE) \ $(foreach vol,$(VOLUMES),-v $(vol)) \ $(DOCKER_IMAGE) $(COMMAND) - $(if $(filter false,$(INTERACTIVE)),docker logs -f $(CONTAINER_NAME);) + @if [ "$(INTERACTIVE)" = "true" ]; then \ + :; \ + else \ + printf "$(COLOR_BLUE)$(SYM_ARROW) Container started in detached mode. Use 'make log' to view logs$(COLOR_RESET)\n"; \ + fi + @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Container started successfully$(COLOR_RESET)\n" -# Run Docker container in interactive mode run-it: + @printf "$(COLOR_BLUE)$(SYM_ARROW) Starting interactive container...$(COLOR_RESET)\n" @$(MAKE) run INTERACTIVE=true COMMAND=/bin/bash + @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Interactive container started$(COLOR_RESET)\n" -# Exec into the running container exec: - @echo "Executing into Docker container..." - @docker exec -it $(CONTAINER_NAME) /bin/bash + @printf "$(COLOR_BLUE)$(SYM_ARROW) Executing into container...$(COLOR_RESET)\n" + @docker exec -it $(CONTAINER_NAME) /bin/bash || \ + { printf "$(COLOR_RED)$(SYM_ERROR) Failed to execute into container$(COLOR_RESET)\n"; exit 1; } + @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Successfully executed into container$(COLOR_RESET)\n" -# View the container logs log: - @echo "Viewing Docker container logs..." + @printf "$(COLOR_BLUE)$(SYM_ARROW) Fetching container logs...$(COLOR_RESET)\n" @if [ "$(FOLLOW_LOGS)" = "true" ]; then \ - docker logs -f $(CONTAINER_NAME); \ + docker logs -f $(CONTAINER_NAME) || \ + { printf "$(COLOR_RED)$(SYM_ERROR) Failed to retrieve logs$(COLOR_RESET)\n"; exit 1; }; \ else \ - docker logs $(CONTAINER_NAME); \ + docker logs $(CONTAINER_NAME) || \ + { printf "$(COLOR_RED)$(SYM_ERROR) Failed to retrieve logs$(COLOR_RESET)\n"; exit 1; }; \ fi + @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Logs retrieved successfully$(COLOR_RESET)\n" -# Delete the running container clean: - @echo "Deleting Docker container if exists..." + @printf "$(COLOR_BLUE)$(SYM_ARROW) Cleaning up containers...$(COLOR_RESET)\n" @docker stop $(CONTAINER_NAME) 2>/dev/null || true @docker rm -f $(CONTAINER_NAME) 2>/dev/null || true + @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Cleanup completed$(COLOR_RESET)\n" -# Test Docker container test: clean - @echo "Setting up test environment..." - @$(MAKE) run VOLUMES="$(TEST_WORKER_CONFIG):/home/$(USER)/worker.yaml:ro $(TESTS_TASKS_DIR):/home/$(USER)/tasks:ro $(TESTS_MAIN_SCRIPT):/home/$(USER)/main.sh:ro" COMMAND="/home/$(USER)/main.sh" - @$(MAKE) log FOLLOW_LOGS=true - @$(MAKE) clean + @printf "$(COLOR_BLUE)$(SYM_ARROW) Running tests...$(COLOR_RESET)\n" + @$(MAKE) run \ + VOLUMES="$(TEST_WORKER_CONFIG):/home/$(USER)/worker.yaml:ro $(TEST_SERVICES_CONFIG):/home/$(USER)/services.yaml:ro $(TESTS_TASKS_DIR):/home/$(USER)/tasks:ro $(TESTS_MAIN_SCRIPT):/home/$(USER)/main.sh:ro" \ + COMMAND="/home/$(USER)/main.sh" || exit 1 + @$(MAKE) log FOLLOW_LOGS=true || exit 1 + @$(MAKE) clean || exit 1 + @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Tests completed successfully$(COLOR_RESET)\n" -# Development pipeline dev-pipeline: build test - @echo "Development pipeline completed successfully." \ No newline at end of file + @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Development pipeline completed successfully$(COLOR_RESET)\n" \ No newline at end of file diff --git a/Makefile.variables b/Makefile.variables index 39ec891c..ca79ceea 100644 --- a/Makefile.variables +++ b/Makefile.variables @@ -7,9 +7,23 @@ ENV_FILE ?= .env TESTS_MAIN_SCRIPT ?= ./src/tests/main.sh TESTS_TASKS_DIR ?= ./src/tests/tasks TEST_WORKER_CONFIG ?= ./src/tests/configs/worker.yaml +TEST_SERVICES_CONFIG ?= ./src/tests/configs/services.yaml USER = udx VOLUMES ?= ./src/scripts:/home/$(USER) DEBUG ?= false COMMAND ?= MULTIPLATFORM ?= false -FOLLOW_LOGS ?= false \ No newline at end of file +FOLLOW_LOGS ?= false + +# Colors for better visibility +COLOR_RESET=\033[0m +COLOR_BLUE=\033[34m +COLOR_GREEN=\033[32m +COLOR_RED=\033[31m +COLOR_YELLOW=\033[33m + +# Symbols +SYM_ARROW=➜ +SYM_SUCCESS=✔ +SYM_ERROR=✖ +SYM_WARNING=⚠ \ No newline at end of file diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 71aac06d..16611f82 100644 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -3,62 +3,22 @@ # shellcheck disable=SC1091 source /usr/local/lib/utils.sh -udx_logo - log_info "Welcome to UDX Worker Container. Initializing environment..." # shellcheck disable=SC1091 source /usr/local/lib/environment.sh -handle_services() { - - if check_active_services; then - wait_for_services - log_info "Services are fully running." - else - log_warn "No services are active." - fi -} - -check_active_services() { - log_info "Checking for active or starting services..." - if supervisorctl status | grep -Eq 'RUNNING|STARTING'; then - log_info "Active or starting services found." - return 0 - else - log_warn "No active or starting services detected." - return 1 - fi -} +# Start the process manager in the background +log_info "Starting process manager..." +/usr/local/lib/process_manager.sh & -wait_for_services() { - local attempts=0 max_attempts=10 - log_info "Waiting for services to be fully running..." - while [ $attempts -lt $max_attempts ]; do - if supervisorctl status | grep -q "RUNNING"; then - log_info "All services are now running." - return 0 - fi - log_info "Waiting for services to be fully running... (Attempt: $attempts)" - attempts=$((attempts + 1)) - sleep 5 - done - log_warn "Services are not fully running after $max_attempts attempts." - return 1 -} - -# Main execution path +# Main execution logic if [ "$#" -gt 0 ]; then + # Log the command being executed log_info "Executing command: $*" - - if [[ "$1" =~ \.sh$ ]]; then - "$@" # Execute the provided command - log_info "Shell script execution completed. Exiting." - exit 0 - else - handle_services - "$@" # Execute the provided command - fi -else - handle_services -fi \ No newline at end of file + # Execute the command passed to the container + exec "$@" +fi + +# Keep the container running +wait \ No newline at end of file diff --git a/docs/CLI.md b/docs/CLI.md index c00b2377..10f07d0b 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -1,40 +1,91 @@ -### Commands +# UDX Worker CLI Documentation -#### Service +The UDX Worker provides a command-line interface for managing services, environment variables, and generating Software Bill of Materials (SBOM). -* `worker service list` -Lists all available services. +## Commands + +### Service Management + +* `worker service list` + - Lists of configured services. + - Shows service name, status, PID, and uptime * `worker service status ` -Displays the status of a specified service. + - Displays detailed status of a service + - Shows service state, uptime, and process information -* `worker service logs ` -Tails the output logs for a specific service. +* `worker service logs [options]` + - Views service output logs + - Options: + - `--lines N`: Show last N lines (default: 20) + - `--nostream`: Show logs without following -* `worker service errors ` -Tails the error logs for a specific service. +* `worker service errors [options]` + - Views service error logs + - Options: + - `--lines N`: Show last N lines (default: 20) + - `--nostream`: Show logs without following * `worker service config` -Shows the services configuration settings. + - Shows the services configuration settings + - Displays supervisor configuration for all services * `worker service start ` -Starts a specified service. + - Starts a specified service + - Creates necessary log files and directories * `worker service stop ` -Stops a specified service. + - Stops a specified service + - Service can be restarted later * `worker service restart ` -Restarts a specified service. + - Restarts a specified service + - Equivalent to stop followed by start -#### ENV +### Environment Variables * `worker env set ` -Sets an environment variable. + - Sets an environment variable + - Variable will be available to all services * `worker env get [key]` -Retrieves an environment variable. Show all environment variables if no key is provided. + - Retrieves environment variable(s) + - If key is provided, shows specific variable + - If no key, shows all environment variables -#### SBOM +### Software Bill of Materials * `worker sbom generate` -Generates container SBOM. \ No newline at end of file + - Generates container Software Bill of Materials + - Lists all installed packages with: + - Package name + - Version + - Architecture + +## Examples + +```bash +# View service status +worker service status my_service + +# Get last 100 lines of logs without following +worker service logs my_service --lines=100 --nostream + +# Set and verify environment variable +worker env set MY_VAR "my value" +worker env get MY_VAR + +# Generate SBOM +worker sbom generate +``` + +## Exit Codes + +- 0: Command completed successfully +- 1: Command failed or invalid usage + +## Notes + +- All commands use logging levels: INFO, DEBUG, WARN, ERROR +- Service logs are stored in `/var/log/supervisor/` +- Configuration files are in `/etc/supervisord.conf` \ No newline at end of file diff --git a/etc/configs/supervisor/common.conf b/etc/configs/supervisor/common.conf index e212090d..f2916455 100644 --- a/etc/configs/supervisor/common.conf +++ b/etc/configs/supervisor/common.conf @@ -1,18 +1,17 @@ [supervisord] -logfile=/var/log/supervisor/supervisord.log -logfile_maxbytes=50MB -logfile_backups=10 -loglevel=info -pidfile=/var/run/supervisor/supervisord.pid -nodaemon=false +logfile=/dev/null +loglevel=critical +silent=true +nodaemon=true minfds=1024 minprocs=200 +pidfile=/var/run/supervisor/supervisord.pid + +[unix_http_server] +file=/var/run/supervisor/supervisord.sock [supervisorctl] serverurl=unix:///var/run/supervisor/supervisord.sock -[unix_http_server] -file=/var/run/supervisor/supervisord.sock ; path to your socket file - [rpcinterface:supervisor] supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface \ No newline at end of file diff --git a/etc/configs/supervisor/program.conf b/etc/configs/supervisor/program.conf index 7b6f5373..33586c30 100644 --- a/etc/configs/supervisor/program.conf +++ b/etc/configs/supervisor/program.conf @@ -1,7 +1,17 @@ [program:${process_name}] +# Process command and restart settings command=${command} autostart=${autostart} autorestart=${autorestart} -stderr_logfile=/var/log/supervisor/${process_name}.err.log +startretries=3 + +# Logging settings stdout_logfile=/var/log/supervisor/${process_name}.out.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=10 +stderr_logfile=/var/log/supervisor/${process_name}.err.log +stderr_logfile_maxbytes=50MB +stderr_logfile_backups=10 + +# Environment environment=${envs} \ No newline at end of file diff --git a/etc/home/logo.txt b/etc/home/logo.txt deleted file mode 100644 index 92184d28..00000000 --- a/etc/home/logo.txt +++ /dev/null @@ -1,6 +0,0 @@ - _ _ - | | | | - _ _ __| |_ ____ _____ _ __| | _____ _ __ - | | | |/ _` \ \/ /\ \ /\ / / _ \| '__| |/ / _ \ '__| - | |_| | (_| |> < _\ V V / (_) | | | < __/ | - \__,_|\__,_/_/\_(_)\_/\_/ \___/|_| |_|\_\___|_| diff --git a/lib/auth.sh b/lib/auth.sh index f22f6839..1ebc851c 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -39,7 +39,7 @@ authenticate_actors() { if [[ -z "$creds" ]]; then continue else - log_info "Detected credentials for $type" + log_success "Authentication" "Detected credentials for $type" fi # Explicitly check for JSON format @@ -54,7 +54,7 @@ authenticate_actors() { log_info "Reading base64 encoded JSON credentials" creds=$(echo "$creds" | base64 --decode) else - log_error "Credentials format not recognized for $provider. Skipping..." + log_error "Authentication" "Credentials format not recognized for $provider. Skipping..." continue fi @@ -70,20 +70,20 @@ authenticate_actors() { if command -v "$auth_function" > /dev/null; then if ! authenticate_provider "$provider" "$auth_function" "$creds"; then - log_error "Authentication failed for provider $provider." + log_error "Authentication" "Authentication failed for provider $provider." return 1 fi configured_providers+=("$provider") else - log_error "Authentication function $auth_function not found for $provider. Skipping..." + log_error "Authentication" "Authentication function $auth_function not found for $provider. Skipping..." continue fi else - log_error "Authentication script $auth_script not found for $provider. Skipping..." + log_error "Authentication" "Authentication script $auth_script not found for $provider. Skipping..." continue fi else - log_error "Invalid JSON credentials for $provider. Skipping..." + log_error "Authentication" "Invalid JSON credentials for $provider. Skipping..." continue fi done @@ -121,7 +121,6 @@ authenticate_provider() { # Set an environment variable to mark successful authorization export "${provider^^}_AUTHORIZED=true" - log_info "Authorization successful for provider $provider." return 0 } diff --git a/lib/auth/aws.sh b/lib/auth/aws.sh index 46c5b29d..05b41592 100644 --- a/lib/auth/aws.sh +++ b/lib/auth/aws.sh @@ -1,5 +1,8 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Example usage of the function # aws_authenticate "/path/to/your/aws_creds.json" @@ -12,7 +15,7 @@ aws_authenticate() { creds_content=$(cat "$creds_json") if [[ -z "$creds_content" ]]; then - echo "[ERROR] No AWS credentials provided." >&2 + log_error "AWS Authentication" "No AWS credentials provided." return 1 fi @@ -24,7 +27,7 @@ aws_authenticate() { sessionToken=$(echo "$creds_content" | jq -r '.SessionToken') if [[ -z "$accessKeyId" || -z "$secretAccessKey" ]]; then - echo "[ERROR] Missing required AWS credentials." >&2 + log_error "AWS Authentication" "Missing required AWS credentials." return 1 fi @@ -35,5 +38,5 @@ aws_authenticate() { export AWS_SESSION_TOKEN="$sessionToken" fi - echo "[INFO] AWS credentials set successfully." + log_success "AWS Authentication" "AWS credentials set successfully." } \ No newline at end of file diff --git a/lib/auth/azure.sh b/lib/auth/azure.sh index 9fe3ae71..7286fd7b 100644 --- a/lib/auth/azure.sh +++ b/lib/auth/azure.sh @@ -17,7 +17,7 @@ azure_authenticate() { creds_content=$(cat "$creds_json") if [[ -z "$creds_content" ]]; then - echo "[ERROR] No Azure credentials provided." >&2 + log_error "Azure Authentication" "No Azure credentials provided." return 1 fi @@ -30,20 +30,20 @@ azure_authenticate() { tenantId=$(echo "$creds_content" | jq -r '.tenantId') if [[ -z "$clientId" || -z "$clientSecret" || -z "$subscriptionId" || -z "$tenantId" ]]; then - echo "[ERROR] Missing required Azure credentials." >&2 + log_error "Azure Authentication" "Missing required Azure credentials." return 1 fi log_info "Authenticating Azure service principal..." if ! az login --service-principal -u "$clientId" -p "$clientSecret" --tenant "$tenantId" >/dev/null 2>&1; then - echo "[ERROR] Azure service principal authentication failed." >&2 + log_error "Azure Authentication" "Azure service principal authentication failed." return 1 fi if ! az account set --subscription "$subscriptionId" >/dev/null 2>&1; then - echo "[ERROR] Failed to set Azure subscription." >&2 + log_error "Azure Authentication" "Failed to set Azure subscription." return 1 fi - log_info "Azure service principal authenticated and subscription set." + log_success "Azure Authentication" "Azure service principal authenticated and subscription set." } diff --git a/lib/auth/bitwarden.sh b/lib/auth/bitwarden.sh index b9f7b838..57dbd7d6 100644 --- a/lib/auth/bitwarden.sh +++ b/lib/auth/bitwarden.sh @@ -1,5 +1,8 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Function to authenticate Bitwarden using API key or master password # # Example usage of the function @@ -9,7 +12,7 @@ bitwarden_authenticate() { local creds_file="$1" if [[ ! -f "$creds_file" ]]; then - echo "[ERROR] Credentials file not found: $creds_file" >&2 + log_error "Bitwarden Authentication" "Credentials file not found: $creds_file" return 1 fi @@ -18,7 +21,7 @@ bitwarden_authenticate() { creds_content=$(cat "$creds_file") if [[ -z "$creds_content" ]]; then - echo "[ERROR] Credentials file is empty: $creds_file" >&2 + log_error "Bitwarden Authentication" "Credentials file is empty: $creds_file" return 1 fi @@ -28,7 +31,7 @@ bitwarden_authenticate() { master_password=$(echo "$creds_content" | jq -r '.masterPassword // empty') if [[ -z "$api_key" && -z "$master_password" ]]; then - echo "[ERROR] Either API key or master password must be provided in the credentials file." >&2 + log_error "Bitwarden Authentication" "Either API key or master password must be provided in the credentials file." return 1 fi @@ -40,18 +43,18 @@ bitwarden_authenticate() { local email email=$(echo "$creds_content" | jq -r '.email // empty') if [[ -z "$email" ]]; then - echo "[ERROR] Email must be provided with the master password." >&2 + log_error "Bitwarden Authentication" "Email must be provided with the master password." return 1 fi session_key=$(bw login "$email" "$master_password" --raw 2>/dev/null) fi if [[ -z "$session_key" ]]; then - echo "[ERROR] Failed to authenticate with Bitwarden." >&2 + log_error "Bitwarden Authentication" "Failed to authenticate with Bitwarden." return 1 fi - echo "[INFO] Bitwarden authentication successful. Session key obtained." + log_success "Bitwarden Authentication" "Authenticated with Bitwarden." export BW_SESSION="$session_key" return 0 } \ No newline at end of file diff --git a/lib/auth/gcp.sh b/lib/auth/gcp.sh index fbe6d9da..0299d3df 100644 --- a/lib/auth/gcp.sh +++ b/lib/auth/gcp.sh @@ -1,5 +1,8 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Function to authenticate GCP service accounts # # Example usage of the function @@ -15,7 +18,7 @@ gcp_authenticate() { creds_content=$(cat "$creds_json") if [[ -z "$creds_content" ]]; then - echo "[ERROR] No GCP credentials provided." >&2 + log_error "GCP Authentication" "No GCP credentials provided." return 1 fi @@ -27,7 +30,7 @@ gcp_authenticate() { projectId=$(echo "$creds_content" | jq -r '.project_id') if [[ -z "$clientEmail" || -z "$privateKey" || -z "$projectId" ]]; then - echo "[ERROR] Missing required GCP credentials." >&2 + log_error "GCP Authentication" "Missing required GCP credentials." return 1 fi @@ -41,20 +44,20 @@ gcp_authenticate() { jq -n --arg clientEmail "$clientEmail" --arg privateKey "$privateKey" --arg projectId "$projectId" \ '{client_email: $clientEmail, private_key: $privateKey, project_id: $projectId}' > "$temp_creds_file" - echo "[INFO] Authenticating GCP service account..." + log_info "GCP Authentication" "Authenticating GCP service account..." if ! gcloud auth activate-service-account "$clientEmail" --key-file="$temp_creds_file" >/dev/null 2>&1; then - echo "[ERROR] GCP service account authentication failed." >&2 + log_error "GCP Authentication" "GCP service account authentication failed." rm -f "$temp_creds_file" return 1 fi if ! gcloud config set project "$projectId" >/dev/null 2>&1; then - echo "[ERROR] Failed to set GCP project." >&2 + log_error "GCP Authentication" "Failed to set GCP project." rm -f "$temp_creds_file" return 1 fi - echo "[INFO] GCP service account authenticated and project set." + log_success "GCP Authentication" "GCP service account authenticated and project set." # Clean up temporary credentials file rm -f "$temp_creds_file" diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 2f0bab74..9a716936 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -1,11 +1,12 @@ #!/bin/bash -# Include utility functions and worker config utilities -# shellcheck source=/dev/null -source /usr/local/lib/utils.sh +# Include worker config utilities first # shellcheck source=/dev/null source /usr/local/lib/worker_config.sh +# shellcheck source=/dev/null +source /usr/local/lib/utils.sh + # Generic function to clean up authentication for any provider cleanup_provider() { local provider=$1 @@ -46,11 +47,11 @@ cleanup_provider() { if echo "$logout_output" | grep -q -E "No credentials available to revoke|No active sessions|No active accounts"; then log_info "No active $name credentials to revoke." else - log_error "Failed to log out of $name: $logout_output" + log_error "Cleanup" "Failed to log out of $name: $logout_output" return 1 fi else - log_info "$name authentication cleaned up successfully." + log_success "Cleanup" "$name authentication cleaned up successfully." cleaned_up=true fi diff --git a/lib/cli.sh b/lib/cli.sh index 5839bcea..c9152153 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -6,6 +6,9 @@ for module in /usr/local/lib/cli/*.sh; do source "$module" done +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # CLI Interface case $1 in env) @@ -21,7 +24,7 @@ case $1 in service_handler "$@" ;; *) - echo "Usage: $0 {service|env|...}" + log_error "CLI" "Unknown command: $1" exit 1 ;; esac \ No newline at end of file diff --git a/lib/cli/env.sh b/lib/cli/env.sh index c3241495..b3b68bc5 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -1,23 +1,25 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Function to display current environment settings or a specific variable get_environment() { if [ $# -eq 0 ]; then - echo "Current Environment Settings:" - env + log_debug "Current Environment Settings:" "$(env)" else - echo "$1=${!1}" + log_debug "$1" "${!1}" fi } # Function to set a new environment variable set_environment() { if [ $# -ne 2 ]; then - echo "Usage: $0 env set " + log_warn "CLI" "Usage: $0 env set " return 1 fi export "$1=$2" - echo "Set $1 to '$2'." + log_debug "CLI" "Set $1 to '$2'." } # Handle environment commands @@ -32,12 +34,12 @@ env_handler() { if [ $# -eq 2 ]; then set_environment "$@" else - echo "Usage: $0 env set " + log_warn "CLI" "Usage: $0 env set " exit 1 fi ;; *) - echo "Usage: $0 env {show [VARIABLE_NAME]|set }" + log_warn "CLI" "Usage: $0 env {show [VARIABLE_NAME]|set }" exit 1 ;; esac diff --git a/lib/cli/sbom.sh b/lib/cli/sbom.sh index 566b752c..3ba15d70 100644 --- a/lib/cli/sbom.sh +++ b/lib/cli/sbom.sh @@ -1,12 +1,16 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Function to display dpkg packages in a table format using awk generate_sbom() { - echo "Package Name | Version | Architecture" - echo "----------------------------------------------------------" + log_debug "Package Name | Version | Architecture" + log_debug "----------------------------------------------------------" dpkg-query -W -f='${binary:Package} | ${Version} | ${Architecture}\n' | awk -F'|' '{ printf("%-30s | %-20s | %-10s\n", $1, $2, $3) }' + log_debug "----------------------------------------------------------" } # Handler for the sbom command @@ -17,7 +21,7 @@ sbom_handler() { generate_sbom "$@" ;; *) - echo "Usage: $0 sbom show" + log_warn "CLI" "Usage: $0 sbom {generate}" exit 1 ;; esac diff --git a/lib/cli/service.sh b/lib/cli/service.sh index 3f565ac8..ed764f1e 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -1,27 +1,33 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + service_handler() { - case $1 in + local cmd=$1 + shift # Remove the command from args + + case $cmd in list) list_services ;; status) - check_status "$2" + check_status "$1" ;; logs) - follow_logs "$2" + follow_logs "$@" ;; errors) - follow_logs "$2" "err" + follow_logs "$1" "err" ;; config) show_config ;; start|stop|restart) - manage_service "$1" "$2" + manage_service "$cmd" "$1" ;; *) - echo "Usage: $0 {list|status|logs|config|start|stop|restart}" + log_warn "CLI" "Usage: $0 {list|status|logs|config|start|stop|restart}" exit 1 ;; esac @@ -35,14 +41,14 @@ list_services() { # Check if Supervisor is not running, not accessible, or if there are no managed services if [[ -z "$services_status" ]] || echo "$services_status" | grep -Eq 'no such|ERROR'; then - echo "No services are currently managed." + log_warn "Service" "No services are currently managed." exit 1 fi - echo "Listing all managed services:" + log_debug "Service" "Listing all managed services:" local i=1 echo "$services_status" | while read -r line; do - echo "$i. $line" + log_debug "Service" "$i. $line" ((i++)) done } @@ -51,8 +57,8 @@ list_services() { check_status() { # Require a service name for this function if [ -z "$1" ]; then - echo "Error: No service name provided." - echo "Usage: $0 status " + log_warn "Service" "Error: No service name provided." + log_warn "Service" "Usage: $0 status " exit 1 fi @@ -62,7 +68,7 @@ check_status() { # Check if Supervisor is not running, not accessible, or if the service does not exist if [[ -z "$service_status" ]] || echo "$service_status" | grep -Eq 'no such|ERROR'; then - echo "The service '$1' does not exist." + log_warn "Service" "The service '$1' does not exist." exit 1 fi @@ -72,28 +78,71 @@ check_status() { # Function to follow logs for a specific service follow_logs() { - if [ -z "$1" ]; then - echo "Error: No service name provided." - echo "Usage: $0 logs " + local service_name="" + local type="out" + local lines=20 + local nostream=false + + # Parse arguments + while [[ $# -gt 0 ]]; do + case "$1" in + --lines=*) + lines="${1#*=}" + ;; + --lines) + shift + if [[ -n "$1" && "$1" =~ ^[0-9]+$ ]]; then + lines="$1" + fi + ;; + --nostream) + nostream=true + ;; + err) + type="err" + ;; + *) + if [[ -z "$service_name" ]]; then + service_name="$1" + fi + ;; + esac + shift + done + + if [[ -z "$service_name" ]]; then + log_error "Service" "Error: No service name provided." + log_error "Service" "Usage: $0 logs [--lines N] [--nostream]" exit 1 fi - local logfile="/var/log/supervisor/$1" - local type=${2:-out} # Default to 'out' if not specified + local logfile="/var/log/supervisor/$service_name" logfile="$logfile.$type.log" - if [ ! -f "$logfile" ]; then - echo "Log file does not exist: $logfile" + if [[ ! -f "$logfile" ]]; then + log_error "Service" "Log file does not exist: $logfile" + exit 1 + fi + + # Ensure lines is a valid number + if ! [[ "$lines" =~ ^[0-9]+$ ]]; then + log_error "Service" "Invalid line count: $lines" exit 1 fi - tail -f "$logfile" + if [ "$nostream" = true ]; then + # Just show the last N lines without following + tail -n "$lines" "$logfile" + else + # Show the last N lines and follow + exec tail -n "$lines" -f "$logfile" + fi } # Function to show supervisor configuration show_config() { if [ ! -f "/etc/supervisord.conf" ]; then - echo "Configuration file is not generated since no services are managed." + log_error "Service" "Configuration file is not generated since no services are managed." exit 1 fi cat /etc/supervisord.conf @@ -102,13 +151,13 @@ show_config() { # Function to start, stop, or restart a service manage_service() { if [ -z "$2" ]; then - echo "Error: No service name provided." - echo "Usage: $0 $1 " + log_error "Service" "Error: No service name provided." + log_warn "Service" "Usage: $0 $1 " exit 1 fi if [ ! -e "/var/run/supervisor/supervisord.sock" ]; then - echo "Error: Service doesn't exist." + log_error "Service" "Error: Service doesn't exist." exit 1 fi diff --git a/lib/environment.sh b/lib/environment.sh index bd2b8e6c..2c436af3 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -1,33 +1,17 @@ #!/bin/bash -# Get the directory of this script -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Include necessary modules +# shellcheck disable=SC1091 +source /usr/local/lib/auth.sh +# shellcheck disable=SC1091 +source /usr/local/lib/secrets.sh +# shellcheck disable=SC1091 +source /usr/local/lib/cleanup.sh +# shellcheck disable=SC1091 +source /usr/local/lib/worker_config.sh -# Source a file if it exists -source_if_exists() { - local file_path="$1" - if [[ -f "$file_path" ]]; then - # shellcheck disable=SC1090 - source "$file_path" - else - echo "[ERROR] Missing file: $file_path" >&2 - exit 1 - fi -} - -# Include necessary modules from the same directory -# shellcheck source=./utils.sh -source_if_exists "$SCRIPT_DIR/utils.sh" -# shellcheck source=./auth.sh -source_if_exists "$SCRIPT_DIR/auth.sh" -# shellcheck source=./secrets.sh -source_if_exists "$SCRIPT_DIR/secrets.sh" -# shellcheck source=./cleanup.sh -source_if_exists "$SCRIPT_DIR/cleanup.sh" -# shellcheck source=./process_manager.sh -source_if_exists "$SCRIPT_DIR/process_manager.sh" -# shellcheck source=./worker_config.sh -source_if_exists "$SCRIPT_DIR/worker_config.sh" +# shellcheck disable=SC1091 +source /usr/local/lib/utils.sh # Main function to coordinate environment setup configure_environment() { @@ -37,13 +21,13 @@ configure_environment() { local resolved_config resolved_config=$(load_and_parse_config) if [[ -z "$resolved_config" ]]; then - log_error "Configuration loading failed. Exiting..." + log_error "Environment" "Configuration loading failed. Exiting..." return 1 fi # Export variables from the configuration if ! export_variables_from_config "$resolved_config"; then - log_error "Failed to export variables." + log_error "Environment" "Failed to export variables." return 1 fi @@ -53,7 +37,7 @@ configure_environment() { if [[ $? -eq 0 && -n "$actors" ]]; then log_info "Authenticating actors from configuration..." if ! authenticate_actors "$actors"; then - log_error "Failed to authenticate actors." + log_error "Environment" "Failed to authenticate actors." return 1 fi else @@ -66,7 +50,7 @@ configure_environment() { if [[ $? -eq 0 && -n "$secrets" ]]; then log_info "Fetching secrets from configuration..." if ! fetch_secrets "$secrets"; then - log_error "Failed to fetch secrets." + log_error "Environment" "Failed to fetch secrets." return 1 fi else @@ -76,19 +60,11 @@ configure_environment() { # Perform cleanup log_info "Cleaning up sensitive data..." if ! cleanup_actors; then - log_error "Failed to clean up actors." + log_error "Environment" "Failed to clean up actors." return 1 fi - # Perform process manager setup - log_info "Setting up process manager..." - if should_generate_config; then - log_info "Generating Supervisor configuration..." - configure_and_execute_services - else - log_info "No services found in the configuration. Skipping process manager setup." - fi - + # Environment setup complete log_info "Secure environment setup completed successfully." } diff --git a/lib/process_manager.sh b/lib/process_manager.sh index a56c2a9d..4c13fa28 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -1,5 +1,8 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Define paths DEFAULT_CONFIG_FILE="/usr/local/configs/worker/services.yaml" # Define the user-specific configuration path search @@ -12,20 +15,30 @@ COMMON_TEMPLATE_FILE="/usr/local/configs/supervisor/common.conf" PROGRAM_TEMPLATE_FILE="/usr/local/configs/supervisor/program.conf" FINAL_CONFIG="/usr/local/configs/supervisor/supervisord.conf" -# Function to check for service configurations -should_generate_config() { - local enabled_services_count - # Extract services into JSON format - services_yaml=$(yq e -o=json '.services[] | select(.ignore != true)' "$CONFIG_FILE") - # Count the number of items in the JSON array, trimming any newlines or spaces - enabled_services_count=$(echo "$services_yaml" | jq -c '. | length' | tr -d '\n') +# Set up signal handling +trap 'handle_supervisor_signals SIGTERM' SIGTERM +trap 'handle_supervisor_signals SIGINT' SIGINT - # Check if the configuration file exists and there is at least one enabled service - if [ -f "$CONFIG_FILE" ] && [ "${enabled_services_count:-0}" -gt 0 ]; then - return 0 - else - return 1 +# Main execution +main() { + log_info "Process Manager" "Starting process manager..." + + if ! configure_and_execute_services; then + log_error "Process Manager" "Failed to configure and start services" + exit 1 + fi + + # Wait for services to be ready + if ! wait_for_services_ready; then + log_error "Process Manager" "Services failed to start properly" + exit 1 fi + + # Monitor services in the background + monitor_services & + + # Wait for signals + wait } # Helper function to parse and process each service configuration @@ -51,13 +64,13 @@ parse_service_info() { # Check if 'name' is set, log error and return if not if [ -z "$name" ]; then - echo "Error: 'name' not set for a service. Skipping..." + log_error "Process Manager" "Error: 'name' not set for service $name. Skipping..." return fi # Check if 'command' is set, log error and return if not if [ -z "$command" ]; then - echo "Error: 'command' not set for service $name. Skipping..." + log_error "Process Manager" "Error: 'command' not set for service $name. Skipping..." return fi @@ -71,15 +84,103 @@ parse_service_info() { s|\${envs}|$envs|g" "$PROGRAM_TEMPLATE_FILE" >> "$FINAL_CONFIG" } +# Function to check if services are active +check_services_status() { + local service_count + service_count=$(supervisorctl status | grep -c "RUNNING") + if [ "$service_count" -gt 0 ]; then + return 0 + fi + return 1 +} + +# Function to wait for services to be ready +wait_for_services_ready() { + local max_attempts=30 + local attempt=1 + + while [ $attempt -le $max_attempts ]; do + if check_services_status; then + log_info "All services are running" + return 0 + fi + log_info "Waiting for services to start (attempt $attempt/$max_attempts)..." + sleep 1 + attempt=$((attempt + 1)) + done + + log_error "ProcessManager" "Services failed to start within timeout" + return 1 +} + +# Function to monitor services and keep container running +monitor_services() { + while true; do + if ! check_services_status; then + log_error "ProcessManager" "No active services found, exiting..." + exit 1 + fi + sleep 5 + done +} + +# Function to handle graceful shutdown +handle_supervisor_signals() { + local signal=$1 + log_info "$signal received, stopping all services..." + + # First stop each service gracefully + for program in $(supervisorctl status | awk '{print $1}'); do + log_info "Gracefully stopping $program..." + supervisorctl stop "$program" + + # Give the service time to handle its shutdown + local timeout=10 + local elapsed=0 + while [ $elapsed -lt $timeout ]; do + if ! supervisorctl status "$program" | grep -q "RUNNING\|STOPPING"; then + log_info "$program stopped successfully" + break + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + done + + # Finally stop supervisor itself + log_info "All services stopped, shutting down supervisor..." + supervisorctl shutdown + exit 0 +} + # Function to start Supervisor with the generated configuration start_supervisor() { - supervisord + log_info "Starting supervisord..." + + # Start supervisord in non-daemon mode + exec supervisord -n } -# Function to configure and start the Supervisor -configure_and_execute_services() { +# Function to check for service configurations +should_generate_config() { + local enabled_services_count + # Extract services into JSON format + services_yaml=$(yq e -o=json '.services[] | select(.ignore != true)' "$CONFIG_FILE") + # Count the number of items in the JSON array, trimming any newlines or spaces + enabled_services_count=$(echo "$services_yaml" | jq -c '. | length' | tr -d '\n') + + # Check if the configuration file exists and there is at least one enabled service + if [ -f "$CONFIG_FILE" ] && [ "${enabled_services_count:-0}" -gt 0 ]; then + return 0 + else + return 1 + fi +} + +# Function to configure services +configure_services() { if ! should_generate_config; then - echo "No services found in $CONFIG_FILE. No Supervisor configuration generated." + log_warn "Process Manager" "No services found in $CONFIG_FILE. No Supervisor configuration generated." return 1 fi @@ -91,7 +192,7 @@ configure_and_execute_services() { services_yaml=$(yq e -o=json '.services[] | select(.ignore != true)' "$CONFIG_FILE" | jq -c .) if [ -z "$services_yaml" ]; then - echo "Failed to parse services from $CONFIG_FILE or no services defined." + log_error "Process Manager" "Failed to parse services from $CONFIG_FILE or no services defined." return 1 fi @@ -99,6 +200,20 @@ configure_and_execute_services() { parse_service_info "$service_json" done - # After generating the config and processing services, start Supervisor + log_info "Service configuration generated successfully." + return 0 +} + +# Function to configure and start services +configure_and_execute_services() { + # First configure the services + if ! configure_services; then + return 1 + fi + + # Then start supervisor start_supervisor -} \ No newline at end of file +} + +# Execute main function +main \ No newline at end of file diff --git a/lib/secrets.sh b/lib/secrets.sh index fdd860ad..04153cb5 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -1,6 +1,5 @@ #!/bin/bash -# Include utility functions and worker config utilities # shellcheck source=/usr/local/lib/utils.sh disable=SC1091 source /usr/local/lib/utils.sh @@ -32,7 +31,7 @@ fetch_secrets() { # Confirm secrets_json is valid JSON before processing if ! echo "$secrets_json" | jq empty > /dev/null 2>&1; then - log_error "Invalid JSON format for secrets configuration." + log_error "Secrets" "Invalid JSON format for secrets configuration." return 1 fi @@ -49,7 +48,7 @@ fetch_secrets() { # Check if the secret has a valid name and URL if [[ -z "$name" || -z "$url" ]]; then - log_error "Secret name or URL is missing or empty." + log_error "Secrets" "Secret name or URL is missing or empty." continue fi @@ -62,7 +61,7 @@ fetch_secrets() { key_vault_name=$(echo "$url" | cut -d '/' -f 2) secret_name=$(echo "$url" | cut -d '/' -f 3) if [[ -z "$secret_name" ]]; then - log_error "Invalid GCP secret name format: $url" + log_error "Secrets" "Invalid GCP secret name format: $url" continue fi ;; @@ -70,7 +69,7 @@ fetch_secrets() { key_vault_name=$(echo "$url" | cut -d '/' -f 2) secret_name=$(echo "$url" | cut -d '/' -f 3) if [[ -z "$key_vault_name" || -z "$secret_name" ]]; then - log_error "Invalid secret format for $provider: $url" + log_error "Secrets" "Invalid secret format for $provider: $url" continue fi ;; @@ -95,9 +94,9 @@ fetch_secrets() { # Export the secret as an environment variable if [[ -n "$value" ]]; then echo "export $name=\"$value\"" >> "$secrets_env_file" - log_info "Resolved secret for $name from $provider." + log_success "Secrets" "Resolved secret for $name from $provider." else - log_error "Failed to resolve secret for $name from $provider." + log_error "Secrets" "Failed to resolve secret for $name from $provider." fi done @@ -109,7 +108,7 @@ fetch_secrets() { set +a log_info "Secrets environment variables sourced successfully." else - log_error "No secrets were written to the environment file." + log_error "Secrets" "No secrets were written to the environment file." return 1 fi diff --git a/lib/secrets/aws.sh b/lib/secrets/aws.sh index b89fc012..3a34d1d7 100644 --- a/lib/secrets/aws.sh +++ b/lib/secrets/aws.sh @@ -1,5 +1,8 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Function to resolve AWS secret resolve_aws_secret() { local secret_name="$1" @@ -7,20 +10,20 @@ resolve_aws_secret() { local secret_value if [[ -z "$secret_name" || -z "$region" ]]; then - echo "[ERROR] Invalid AWS secret name or region" >&2 + log_error "AWS" "Invalid AWS secret name or region" return 1 fi # Retrieve the secret value using AWS CLI with detailed logging - echo "[INFO] Retrieving secret from AWS Secrets Manager: secret_name=$secret_name, region=$region" >&2 + log_info "AWS" "Retrieving secret from AWS Secrets Manager: secret_name=$secret_name, region=$region" if ! secret_value=$(aws secretsmanager get-secret-value --secret-id "$secret_name" --query 'SecretString' --output text --region "$region" 2>&1); then - echo "[ERROR] Failed to retrieve secret from AWS Secrets Manager: secret_name=$secret_name, region=$region" >&2 - echo "[DEBUG] AWS CLI output: $secret_value" >&2 + log_error "AWS" "Failed to retrieve secret from AWS Secrets Manager: secret_name=$secret_name, region=$region" + log_error "AWS" "AWS CLI output: $secret_value" return 1 fi if [ -z "$secret_value" ]; then - echo "[ERROR] Secret value is empty for $secret_name in region $region" >&2 + log_error "AWS" "Secret value is empty for $secret_name in region $region" return 1 fi diff --git a/lib/secrets/azure.sh b/lib/secrets/azure.sh index d58bee41..4289d357 100644 --- a/lib/secrets/azure.sh +++ b/lib/secrets/azure.sh @@ -1,5 +1,8 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Function to resolve Azure secret resolve_azure_secret() { local key_vault_name="$1" @@ -7,20 +10,20 @@ resolve_azure_secret() { local secret_value if [ -z "$key_vault_name" ] || [ -z "$secret_name" ]; then - echo "[ERROR] Invalid Azure Key Vault name or secret name" >&2 + log_error "Azure" "Invalid Azure Key Vault name or secret name" return 1 fi # Retrieve the secret value using Azure CLI with detailed logging - echo "[INFO] Retrieving secret from Azure Key Vault: vault_name=$key_vault_name, secret_name=$secret_name" >&2 + log_info "Azure" "Retrieving secret from Azure Key Vault: vault_name=$key_vault_name, secret_name=$secret_name" if ! secret_value=$(az keyvault secret show --vault-name "$key_vault_name" --name "$secret_name" --query value -o tsv 2>&1); then - echo "[ERROR] Failed to retrieve secret from Azure Key Vault: vault_name=$key_vault_name, secret_name=$secret_name" >&2 - echo "[DEBUG] Azure CLI output: $secret_value" >&2 + log_error "Azure" "Failed to retrieve secret from Azure Key Vault: vault_name=$key_vault_name, secret_name=$secret_name" + log_error "Azure" "Azure CLI output: $secret_value" return 1 fi if [ -z "$secret_value" ]; then - echo "[ERROR] Secret value is empty for $key_vault_name/$secret_name" >&2 + log_error "Azure" "Secret value is empty for $key_vault_name/$secret_name" return 1 fi diff --git a/lib/secrets/bitwarden.sh b/lib/secrets/bitwarden.sh index f6665568..6075aaf0 100644 --- a/lib/secrets/bitwarden.sh +++ b/lib/secrets/bitwarden.sh @@ -1,5 +1,8 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Function to resolve Bitwarden secret resolve_bitwarden_secret() { local collection_name="$1" @@ -7,20 +10,20 @@ resolve_bitwarden_secret() { local secret_value if [[ -z "$collection_name" || -z "$secret_name" ]]; then - echo "[ERROR] Invalid Bitwarden collection name or secret name" >&2 + log_error "Bitwarden" "Invalid Bitwarden collection name or secret name" return 1 fi # Retrieve the secret using Bitwarden CLI - echo "[INFO] Retrieving secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" >&2 + log_info "Bitwarden" "Retrieving secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" if ! secret_value=$(bw get item "$secret_name" --organizationid "$collection_name" --output text 2>&1); then - echo "[ERROR] Failed to retrieve secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" >&2 - echo "[DEBUG] Bitwarden CLI output: $secret_value" >&2 + log_error "Bitwarden" "Failed to retrieve secret from Bitwarden: collection_name=$collection_name, secret_name=$secret_name" + log_error "Bitwarden" "Bitwarden CLI output: $secret_value" return 1 fi if [ -z "$secret_value" ]; then - echo "[ERROR] Secret value is empty for $secret_name in collection $collection_name" >&2 + log_error "Bitwarden" "Secret value is empty for $secret_name in collection $collection_name" return 1 fi diff --git a/lib/secrets/gcp.sh b/lib/secrets/gcp.sh index fdc9dae1..2edff3d2 100644 --- a/lib/secrets/gcp.sh +++ b/lib/secrets/gcp.sh @@ -1,5 +1,8 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Function to resolve GCP secret resolve_gcp_secret() { local project_id="$1" @@ -8,7 +11,7 @@ resolve_gcp_secret() { # Validate input arguments if [[ -z "$project_id" || -z "$secret_name" ]]; then - log_error "Invalid GCP project ID or secret name. project_id: $project_id, secret_name: $secret_name" + log_error "GCP Secret Manager" "Invalid GCP project ID or secret name. project_id: $project_id, secret_name: $secret_name" return 1 fi @@ -16,14 +19,14 @@ resolve_gcp_secret() { # Retrieve the latest version of the secret if ! secret_value=$(gcloud secrets versions access latest --secret="$secret_name" --project="$project_id" 2>/tmp/gcp_secret_error.log); then - log_error "Failed to retrieve secret from GCP Secret Manager for secret: $secret_name" + log_error "GCP Secret Manager" "Failed to retrieve secret from GCP Secret Manager for secret: $secret_name" log_error "GCP CLI output: $(cat /tmp/gcp_secret_error.log)" return 1 fi # Check if the secret value is empty if [[ -z "$secret_value" ]]; then - log_error "Secret value is empty for secret: $secret_name in project: $project_id" + log_error "GCP Secret Manager" "Secret value is empty for secret: $secret_name in project: $project_id" return 1 fi diff --git a/lib/utils.sh b/lib/utils.sh index 37cd943f..b1a543f1 100644 --- a/lib/utils.sh +++ b/lib/utils.sh @@ -1,34 +1,31 @@ #!/bin/bash -# Prevent the script from being sourced multiple times -if [ -z "${UTILS_SH_INCLUDED+x}" ]; then - UTILS_SH_INCLUDED=true - - # Function to print the UDX logo - udx_logo() { - cat /etc/logo.txt - } - - # Simple logging functions - log_info() { - echo "[INFO] $1" >&2 # Send to stderr - } - - log_error() { - echo "[ERROR] $1" >&2 # Send to stderr - } - - log_warn() { - echo "[WARN] $1" >&2 # Send to stderr - } - - log_debug() { - echo "[DEBUG] $1" >&2 # Send to stderr - } - - # Function to resolve placeholders with environment variables - resolve_env_vars() { - local value="$1" - eval echo "$value" - } -fi +# Function to resolve placeholders with environment variables +resolve_env_vars() { + local value="$1" + eval echo "$value" +} + +log_info() { + if [ $# -eq 1 ]; then + printf "ℹ️ %s\n" "$1" >&2 + else + printf "ℹ️ %s: %s\n" "$1" "$2" >&2 + fi +} + +log_warn() { + printf "⚠️ %s: %s\n" "$1" "$2" >&2 +} + +log_error() { + printf "❌ %s: %s\n" "$1" "$2" >&2 +} + +log_success() { + printf "✅ %s: %s\n" "$1" "$2" >&2 +} + +log_debug() { + printf "%s: %s\n" "$1" "$2" >&2 +} \ No newline at end of file diff --git a/lib/worker_config.sh b/lib/worker_config.sh index ea24cfd3..fdfed67c 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -1,5 +1,8 @@ #!/bin/bash +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + # Paths for configurations BUILT_IN_CONFIG="/usr/local/configs/worker/default.yaml" # Dynamically find user configuration in any subfolder of $HOME @@ -7,18 +10,9 @@ BUILT_IN_CONFIG="/usr/local/configs/worker/default.yaml" USER_CONFIG=$(find "$HOME" -name 'worker.yaml' 2>/dev/null -print | head -n 1) MERGED_CONFIG="/usr/local/configs/worker/merged_worker.yaml" -# Utility functions for logging -log_info() { - echo "[INFO] $1" >&2 -} - -log_error() { - echo "[ERROR] $1" >&2 -} - # Ensure `yq` is available if ! command -v yq >/dev/null 2>&1; then - log_error "yq is not installed. Please ensure it is available in the PATH." + log_error "Worker configuration" "yq is not installed. Please ensure it is available in the PATH." exit 1 fi @@ -26,7 +20,7 @@ fi ensure_config_exists() { local config_path="$1" if [[ ! -s "$config_path" ]]; then - log_error "Configuration file not found or empty: $config_path" + log_error "Worker configuration" "Configuration file not found or empty: $config_path" return 1 fi } @@ -35,7 +29,7 @@ ensure_config_exists() { merge_worker_configs() { # Ensure the merged configuration file exists if [ ! -f "$MERGED_CONFIG" ]; then - touch "$MERGED_CONFIG" || { log_error "Failed to create merged configuration file at $MERGED_CONFIG"; return 1; } + touch "$MERGED_CONFIG" || { log_error "Worker configuration" "Failed to create merged configuration file at $MERGED_CONFIG"; return 1; } fi # Ensure built-in config exists @@ -43,10 +37,10 @@ merge_worker_configs() { # If a user-provided configuration exists (and path is not empty), merge it if [[ -f "$USER_CONFIG" && -n "$USER_CONFIG" ]]; then - log_info "User configuration detected at $USER_CONFIG" + log_success "Worker configuration" "User configuration detected at $USER_CONFIG" if ! yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' "$BUILT_IN_CONFIG" "$USER_CONFIG" > "$MERGED_CONFIG"; then - log_error "Failed to merge configurations. yq returned an error." + log_error "Worker configuration" "Failed to merge configurations. yq returned an error." return 1 fi else @@ -54,7 +48,7 @@ merge_worker_configs() { # Copy the built-in configuration to the merged configuration if ! cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG"; then - log_error "Failed to copy built-in configuration to merged configuration." + log_error "Worker configuration" "Failed to copy built-in configuration to merged configuration." return 1 fi fi @@ -67,7 +61,7 @@ load_and_parse_config() { # Parse the merged configuration into JSON local json_output if ! json_output=$(yq eval -o=json "$MERGED_CONFIG" 2>/dev/null); then - log_error "Failed to parse merged YAML from $MERGED_CONFIG. yq returned an error." + log_error "Worker configuration" "Failed to parse merged YAML from $MERGED_CONFIG. yq returned an error." return 1 fi @@ -85,7 +79,7 @@ export_variables_from_config() { log_info "No variables found in the configuration." return 0 else - log_info "Found variables in the configuration. Exporting..." + log_success "Worker configuration" "Found variables in the configuration. Exporting..." fi # Iterate over variables and export them into the main shell @@ -100,14 +94,14 @@ get_config_section() { local section="$2" if [[ -z "$config_json" ]]; then - log_error "Empty configuration JSON provided." + log_error "Worker configuration" "Empty configuration JSON provided." return 1 fi # Attempt to extract the section and handle missing/null cases local extracted_section if ! extracted_section=$(echo "$config_json" | jq -r ".config.${section} // empty" 2>/dev/null); then - log_error "Failed to parse section '${section}' from configuration." + log_error "Worker configuration" "Failed to parse section '${section}' from configuration." return 1 fi diff --git a/src/tests/configs/services.yaml b/src/tests/configs/services.yaml new file mode 100644 index 00000000..c197b511 --- /dev/null +++ b/src/tests/configs/services.yaml @@ -0,0 +1,7 @@ +--- +kind: workerService +version: udx.io/worker-v1/service +services: + - name: "test_service" + command: "tail -f /dev/null" + autostart: true diff --git a/src/tests/main.sh b/src/tests/main.sh index 74ef5af9..c207b879 100755 --- a/src/tests/main.sh +++ b/src/tests/main.sh @@ -1,11 +1,20 @@ #!/bin/bash -echo "Running all tests..." +# Source utils.sh for logging functions +# shellcheck disable=SC1091 +source /usr/local/lib/utils.sh -# Find and execute all test scripts in the /usr/local/tests directory +log_info "Main" "Running all test suites" + +# Find and execute all test scripts in the tasks directory for test_script in ./tasks/*.sh; do - echo "Running $(basename "$test_script")..." - bash "$test_script" + log_info "Running $(basename "$test_script")..." + if bash "$test_script"; then + log_success "$(basename "$test_script")" "Test completed successfully" + else + log_error "$(basename "$test_script")" "Test failed" + exit 1 + fi done -echo "All tests completed." \ No newline at end of file +log_success "Main" "All test suites completed successfully" \ No newline at end of file diff --git a/src/tests/tasks/10_dependencies.sh b/src/tests/tasks/10_dependencies.sh index 5cd03221..3bcc99dc 100755 --- a/src/tests/tasks/10_dependencies.sh +++ b/src/tests/tasks/10_dependencies.sh @@ -1,28 +1,34 @@ #!/bin/bash -echo "Starting validation of dependencies..." +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + +log_info "Dependencies: Validating required dependencies and their versions" # Function to test if a command is available and show its version check_command() { local cmd_path cmd_path=$(command -v "$1") if [ -x "$cmd_path" ]; then - echo "[INFO] $1 is installed at $cmd_path." + log_success "$1" "$1 is installed at $cmd_path" local version version=$($1 --version 2>&1 | head -n 1) - echo "[INFO] $1 version: $version" + log_success "$1" "Version: $version" else - echo "[ERROR] $1 is not installed or not in PATH." + log_error "$1" "Not installed or not in PATH" exit 1 fi } # Verify gcloud, aws, az, bw, yq, and jq commands are available +log_info "Checking cloud provider CLIs..." check_command gcloud check_command aws check_command az + +log_info "Checking utility tools..." check_command bw check_command yq check_command jq -echo "Dependencies tests passed successfully." \ No newline at end of file +log_success "Dependencies" "All dependencies validated successfully" \ No newline at end of file diff --git a/src/tests/tasks/20_config.sh b/src/tests/tasks/20_config.sh index 4a39eabf..8b80d78d 100644 --- a/src/tests/tasks/20_config.sh +++ b/src/tests/tasks/20_config.sh @@ -1,13 +1,16 @@ #!/bin/bash -echo "Starting validation of config..." +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + +log_info "Starting validation of config..." # Path to the merged configuration file MERGED_CONFIG="/usr/local/configs/worker/merged_worker.yaml" # Test configure_environment function test_configure_environment() { - echo "Running test: configure_environment" + log_info "Running test: configure_environment" # Load the merged configuration merged_config=$(cat "$MERGED_CONFIG") @@ -26,20 +29,21 @@ test_configure_environment() { actual_value="${!key}" if [[ "$actual_value" != "$value" ]]; then - echo "Test failed: $key is not set correctly. Expected: $value, Got: $actual_value" + log_error "Config" "$key is not set correctly. Expected: $value, Got: $actual_value" return 1 else - echo "Test passed: $key is set correctly." + log_success "Config" "$key is set correctly" fi done - echo "Test passed: configure_environment" + log_success "Config" "configure_environment test passed" + return 0 } # Run the test if test_configure_environment; then - echo "Config tests passed successfully." + log_success "Config" "All config tests passed successfully" else - echo "Config tests failed." + log_error "Config" "Config tests failed" exit 1 fi \ No newline at end of file diff --git a/src/tests/tasks/30_secrets.sh b/src/tests/tasks/30_secrets.sh index a94a088e..e0deae41 100644 --- a/src/tests/tasks/30_secrets.sh +++ b/src/tests/tasks/30_secrets.sh @@ -1,13 +1,16 @@ #!/bin/bash -echo "Starting validation of secrets fetching..." +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + +log_info "Starting validation of secrets fetching..." # Path to the merged configuration file MERGED_CONFIG="/usr/local/configs/worker/merged_worker.yaml" # Test verify_secrets function test_verify_secrets() { - echo "Running test: verify_secrets" + log_info "Running test: verify_secrets" # Load the merged configuration merged_config=$(cat "$MERGED_CONFIG") @@ -17,7 +20,7 @@ test_verify_secrets() { # Check if secrets is empty or null if [[ -z "$secrets" || "$secrets" == "null" ]]; then - echo "Info: No secrets found in the configuration." + log_info "No secrets found in the configuration." return 0 fi @@ -34,28 +37,21 @@ test_verify_secrets() { # Verify that the environment variable is set and different from the reference if [[ -z "$actual_value" || "$actual_value" == "$expected_reference" ]]; then - echo "Test failed: $secret_key is not replaced correctly. Got: $actual_value" + log_error "Secrets" "$secret_key is not replaced correctly. Got: $actual_value" return 1 else - echo "Test passed: $secret_key is resolved correctly." + log_success "Secrets" "$secret_key is resolved correctly" fi done - echo "Test passed: verify_secrets" + log_success "Secrets" "verify_secrets test passed" + return 0 } # Run the test if test_verify_secrets; then - echo "Secrets fetching tests passed successfully." -else - echo "Secrets fetching tests failed." - exit 1 -fi - -# Run the test -if test_verify_secrets; then - echo "Secrets fetching tests passed successfully." + log_success "Secrets" "All secrets fetching tests passed successfully" else - echo "Secrets fetching tests failed." + log_error "Secrets" "Secrets fetching tests failed" exit 1 fi \ No newline at end of file diff --git a/src/tests/tasks/40_test_service.sh b/src/tests/tasks/40_test_service.sh new file mode 100644 index 00000000..e06b638b --- /dev/null +++ b/src/tests/tasks/40_test_service.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 +source /usr/local/lib/utils.sh + +log_info "Test" "Starting service tests..." + +# Test 1: Check service configuration +log_info "Test" "Testing service configuration..." +if ! worker service config | grep -q "test_service"; then + log_error "Test" "Service configuration test failed: test_service not found in config" + exit 1 +fi +log_success "Test" "Service configuration test passed" + +# Test 2: Check service list and wait for it to be running +log_info "Test" "Testing service list and waiting for service to be ready..." + +# Wait for up to 30 seconds for the service to be fully running +for i in {1..30}; do + status_output=$(worker service list 2>&1) + if echo "$status_output" | grep "test_service" | grep -q "RUNNING"; then + log_success "Test" "Service is now running" + break + fi + if [ "$i" -eq 30 ]; then + log_error "Test" "Timeout waiting for service to start" + log_error "Test" "Current status: $status_output" + exit 1 + fi + sleep 1 +done +log_success "Test" "Service list test passed" + +# Test 3: Check service status +log_info "Test" "Testing service status..." +status_output=$(worker service status test_service) +if ! echo "$status_output" | grep -q "RUNNING"; then + log_error "Test" "Service status test failed: test_service not running" + log_error "Test" "Current status: $status_output" + exit 1 +fi +log_success "Test" "Service status test passed" + +# Test 4: Test service stop/start +log_info "Test" "Testing service stop..." +worker service stop test_service +sleep 2 +if worker service status test_service | grep -q "RUNNING"; then + log_error "Test" "Service stop test failed: test_service still running" + exit 1 +fi +log_success "Test" "Service stop test passed" + +log_info "Test" "Testing service start..." +worker service start test_service +sleep 2 +if ! worker service status test_service | grep -q "RUNNING"; then + log_error "Test" "Service start test failed: test_service not running" + exit 1 +fi +log_success "Test" "Service start test passed" + +# Test 5: Check service logs +log_info "Test" "Testing service logs..." + +# Wait a bit for the service to initialize after restart +sleep 2 + +# First check if log file exists +log_file="/var/log/supervisor/test_service.out.log" +if [ ! -f "$log_file" ]; then + log_error "Test" "Service logs test failed: log file not found at $log_file" + exit 1 +fi + +# Try a simple tail command first +if ! tail -n 1 "$log_file" > /dev/null 2>&1; then + log_error "Test" "Service logs test failed: cannot read log file" + exit 1 +fi + +# Now test the worker service logs command with --nostream +if ! worker service logs test_service --lines=1 --nostream > /dev/null 2>&1; then + log_error "Test" "Service logs test failed: worker service logs command failed" + exit 1 +fi + +log_success "Test" "Service logs test passed" + +log_success "Test" "All service tests completed successfully"