From 389f2c0785bbb478175856d7dd740db9163f4063 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Tue, 18 Feb 2025 19:00:19 +0300 Subject: [PATCH] update image --- Dockerfile | 119 +++++--- Makefile | 34 ++- bin/entrypoint.sh | 6 +- etc/configs/worker/services.yaml | 18 -- lib/auth.sh | 6 +- lib/auth/aws.sh | 4 +- lib/auth/azure.sh | 4 +- lib/auth/bitwarden.sh | 4 +- lib/auth/gcp.sh | 4 +- lib/cleanup.sh | 4 +- lib/cli.sh | 192 +++++++++++-- lib/cli/auth.sh | 135 +++++++++ lib/cli/config.sh | 203 ++++++++++++++ lib/cli/env.sh | 371 ++++++++++++++++++++++-- lib/cli/health.sh | 242 ++++++++++++++++ lib/cli/info.sh | 437 +++++++++++++++++++++++++++++ lib/cli/logs.sh | 271 ++++++++++++++++++ lib/cli/sbom.sh | 279 +++++++++++++++++- lib/cli/service.sh | 53 +++- lib/env_handler.sh | 150 ++++++++++ lib/environment.sh | 10 +- lib/process_manager.sh | 27 +- lib/secrets.sh | 8 +- lib/secrets/aws.sh | 4 +- lib/secrets/azure.sh | 4 +- lib/secrets/bitwarden.sh | 4 +- lib/secrets/gcp.sh | 4 +- lib/worker_config.sh | 47 ++-- src/tests/configs/services.yaml | 7 - src/tests/configs/worker.yaml | 17 -- src/tests/env.sh | 29 ++ src/tests/main.sh | 20 -- src/tests/service.sh | 27 ++ src/tests/tasks/10_dependencies.sh | 34 --- src/tests/tasks/20_config.sh | 49 ---- src/tests/tasks/30_secrets.sh | 57 ---- src/tests/tasks/40_test_service.sh | 91 ------ 37 files changed, 2508 insertions(+), 467 deletions(-) delete mode 100644 etc/configs/worker/services.yaml create mode 100644 lib/cli/auth.sh create mode 100644 lib/cli/config.sh create mode 100644 lib/cli/health.sh create mode 100644 lib/cli/info.sh create mode 100644 lib/cli/logs.sh create mode 100644 lib/env_handler.sh delete mode 100644 src/tests/configs/services.yaml delete mode 100644 src/tests/configs/worker.yaml create mode 100755 src/tests/env.sh delete mode 100755 src/tests/main.sh create mode 100755 src/tests/service.sh delete mode 100755 src/tests/tasks/10_dependencies.sh delete mode 100644 src/tests/tasks/20_config.sh delete mode 100644 src/tests/tasks/30_secrets.sh delete mode 100644 src/tests/tasks/40_test_service.sh diff --git a/Dockerfile b/Dockerfile index dd21a372..5061bc63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,27 @@ FROM ubuntu:25.04 # Set the maintainer of the image LABEL maintainer="UDX CAG Team" -# Set environment variables to avoid interactive prompts and set a fixed timezone +# Set base environment variables ENV DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC \ USER=udx \ UID=500 \ GID=500 \ - HOME=/home/udx + HOME=/home/udx \ + # Worker specific paths + WORKER_BASE_DIR=/opt/worker \ + WORKER_CONFIG_DIR=/etc/worker \ + WORKER_APP_DIR=/opt/worker/apps \ + WORKER_DATA_DIR=/opt/worker/data \ + WORKER_LIB_DIR=/usr/local/worker/lib \ + WORKER_BIN_DIR=/usr/local/worker/bin \ + WORKER_ETC_DIR=/usr/local/worker/etc \ + # Add worker bin to PATH + PATH=/usr/local/worker/bin:${PATH} \ + # Cloud SDK configurations + CLOUDSDK_CONFIG=/usr/local/configs/gcloud \ + AWS_CONFIG_FILE=/usr/local/configs/aws \ + AZURE_CONFIG_DIR=/usr/local/configs/azure # Set the shell with pipefail option SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -22,10 +36,10 @@ USER root # hadolint ignore=DL3015 RUN apt-get update && \ apt-get install -y \ - tzdata=2024b-6ubuntu1 \ + tzdata=2025a-2ubuntu1 \ curl=8.12.0+git20250209.89ed161+ds-1ubuntu1 \ bash=5.2.37-1ubuntu1 \ - apt-utils=2.9.28 \ + apt-utils=2.9.29 \ gettext=0.23.1-1 \ gnupg=2.4.4-2ubuntu22 \ ca-certificates=20241223 \ @@ -110,34 +124,73 @@ RUN groupadd -g ${GID} ${USER} && \ RUN mkdir -p /var/log/supervisor /var/run/supervisor && \ chown -R ${USER}:${USER} /var/log/supervisor /var/run/supervisor -# Copy the CLI tool into the image -COPY lib/cli.sh /usr/local/bin/worker_mgmt -RUN chmod +x /usr/local/bin/worker_mgmt && \ - ln -s /usr/local/bin/worker_mgmt /usr/local/bin/worker - -# Copy the bin, etc, and lib directories -COPY etc/configs /usr/local/configs -COPY lib /usr/local/lib -COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh - -# Set permissions during build -# Set ownership -RUN chown -R ${UID}:${GID} /usr/local/configs /usr/local/bin /usr/local/lib && \ - # Make specific scripts executable - chmod 755 /usr/local/bin/entrypoint.sh /usr/local/lib/process_manager.sh && \ - # Set read-only permissions for config files - find /usr/local/configs -type f -exec chmod 644 {} + && \ - # Set read-only permissions for library files - find /usr/local/lib -type f ! -name process_manager.sh -exec chmod 644 {} + && \ - # Ensure directories are accessible - find /usr/local/configs /usr/local/bin /usr/local/lib -type d -exec chmod 755 {} + - -# Create a symbolic link for the supervisord configuration file -RUN ln -sf /usr/local/configs/supervisor/supervisord.conf /etc/supervisord.conf - -# Prepare directories for the user and worker configuration -RUN mkdir -p ${HOME} && \ - chown -R ${USER}:${USER} ${HOME} +# Create directory structure +RUN mkdir -p \ + # Worker directories + ${WORKER_CONFIG_DIR} \ + ${WORKER_APP_DIR} \ + ${WORKER_DATA_DIR} \ + ${WORKER_LIB_DIR} \ + ${WORKER_BIN_DIR} \ + ${WORKER_ETC_DIR} \ + # Environment and secrets files directory + ${WORKER_CONFIG_DIR}/environment.d \ + # User and config directories + ${HOME}/.config/worker \ + # Cloud SDK config directories + ${CLOUDSDK_CONFIG} \ + ${AWS_CONFIG_FILE%/*} \ + ${AZURE_CONFIG_DIR} && \ + # Create and set permissions for environment files + touch ${WORKER_CONFIG_DIR}/environment ${WORKER_CONFIG_DIR}/secrets && \ + chown ${USER}:${USER} \ + ${WORKER_CONFIG_DIR}/environment \ + ${WORKER_CONFIG_DIR}/secrets && \ + chmod 644 ${WORKER_CONFIG_DIR}/environment && \ + chmod 600 ${WORKER_CONFIG_DIR}/secrets + +# Copy worker files +COPY bin/entrypoint.sh ${WORKER_BIN_DIR}/ +COPY lib ${WORKER_LIB_DIR}/ +COPY etc/configs/worker/default.yaml ${WORKER_CONFIG_DIR}/worker.yaml +COPY etc/configs/supervisor ${WORKER_CONFIG_DIR}/supervisor/ + +# Make scripts executable and initialize environment +RUN chmod +x ${WORKER_LIB_DIR}/*.sh && \ + ${WORKER_LIB_DIR}/env_handler.sh init_environment + +# Set up CLI tool +COPY lib/cli.sh ${WORKER_BIN_DIR}/worker_mgmt +RUN chmod +x ${WORKER_BIN_DIR}/worker_mgmt && \ + ln -s ${WORKER_BIN_DIR}/worker_mgmt ${WORKER_BIN_DIR}/worker + +# Set permissions +RUN \ + # Set base ownership + chown -R ${UID}:${GID} \ + ${WORKER_BASE_DIR} \ + ${WORKER_CONFIG_DIR} \ + ${WORKER_LIB_DIR} \ + ${WORKER_BIN_DIR} \ + ${HOME} \ + ${CLOUDSDK_CONFIG} \ + ${AWS_CONFIG_FILE%/*} \ + ${AZURE_CONFIG_DIR} && \ + # Set directory permissions + find ${WORKER_BASE_DIR} ${WORKER_CONFIG_DIR} ${WORKER_LIB_DIR} ${WORKER_BIN_DIR} -type d -exec chmod 755 {} + && \ + # Set file permissions + find ${WORKER_CONFIG_DIR} -type f -exec chmod 644 {} + && \ + find ${WORKER_LIB_DIR} -type f ! -name process_manager.sh -exec chmod 644 {} + && \ + # Make specific files executable + chmod 755 \ + ${WORKER_BIN_DIR}/entrypoint.sh \ + ${WORKER_BIN_DIR}/worker_mgmt \ + ${WORKER_LIB_DIR}/process_manager.sh && \ + # Set runtime directories permissions + chmod 775 ${WORKER_APP_DIR} ${WORKER_DATA_DIR} + +# Set up supervisor configuration +RUN ln -sf ${WORKER_CONFIG_DIR}/supervisor/supervisord.conf /etc/supervisord.conf # Switch to the user directory WORKDIR ${HOME} @@ -146,7 +199,7 @@ WORKDIR ${HOME} USER ${USER} # Set the entrypoint to run the entrypoint script using shell form -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +ENTRYPOINT ["/usr/local/worker/bin/entrypoint.sh"] # Set the default command CMD ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/Makefile b/Makefile index ddfe6d1c..1cc33b1f 100644 --- a/Makefile +++ b/Makefile @@ -28,15 +28,28 @@ build: printf "$(COLOR_BLUE)$(SYM_ARROW) Starting Docker build...$(COLOR_RESET)\n"; \ if [ "$(MULTIPLATFORM)" = "true" ]; then \ printf "$(COLOR_BLUE)$(SYM_ARROW) Building for multiple platforms: [linux/amd64, linux/arm64]$(COLOR_RESET)\n"; \ - docker buildx build --progress=plain \ - --platform linux/amd64,linux/arm64 \ - -t $(DOCKER_IMAGE) \ - --load . 2>&1 | grep -E "$$filter" || exit 1; \ + if [ "$(DEBUG)" = "true" ]; then \ + docker buildx build --progress=plain \ + --platform linux/amd64,linux/arm64 \ + -t $(DOCKER_IMAGE) \ + --load .; \ + else \ + docker buildx build --progress=plain \ + --platform linux/amd64,linux/arm64 \ + -t $(DOCKER_IMAGE) \ + --load . 2>&1 | grep -E "$$filter" || exit 1; \ + fi; \ else \ printf "$(COLOR_BLUE)$(SYM_ARROW) Building for local platform$(COLOR_RESET)\n"; \ - DOCKER_BUILDKIT=1 docker build \ - --progress=plain \ - -t $(DOCKER_IMAGE) . 2>&1 | grep -E "$$filter" || exit 1; \ + if [ "$(DEBUG)" = "true" ]; then \ + DOCKER_BUILDKIT=1 docker build \ + --progress=plain \ + -t $(DOCKER_IMAGE) .; \ + else \ + DOCKER_BUILDKIT=1 docker build \ + --progress=plain \ + -t $(DOCKER_IMAGE) . 2>&1 | grep -E "$$filter" || exit 1; \ + fi; \ fi && \ printf "$(COLOR_GREEN)$(SYM_SUCCESS) Docker image build completed$(COLOR_RESET)\n" || \ { printf "$(COLOR_RED)$(SYM_ERROR) Docker build failed$(COLOR_RESET)\n"; exit 1; }' @@ -88,10 +101,11 @@ clean: test: clean @printf "$(COLOR_BLUE)$(SYM_ARROW) Running tests...$(COLOR_RESET)\n" + @chmod +x src/tests/*.sh @$(MAKE) run \ - VOLUMES="$(TEST_WORKER_CONFIG):/home/$(USER)/worker.yaml:ro $(TEST_SERVICES_CONFIG):/home/$(USER)/services.yaml:ro $(TESTS_TASKS_DIR):/home/$(USER)/tasks:ro $(TESTS_MAIN_SCRIPT):/home/$(USER)/main.sh:ro" \ - COMMAND="/home/$(USER)/main.sh" || exit 1 - @$(MAKE) log FOLLOW_LOGS=true || exit 1 + INTERACTIVE=true \ + VOLUMES="$(PWD)/src/tests:/tests" \ + COMMAND="/bin/bash -c '/tests/env.sh && /tests/service.sh'" || exit 1 @$(MAKE) clean || exit 1 @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Tests completed successfully$(COLOR_RESET)\n" diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 16611f82..8e28b787 100644 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -1,16 +1,16 @@ #!/bin/bash # shellcheck disable=SC1091 -source /usr/local/lib/utils.sh +source ${WORKER_LIB_DIR}/utils.sh log_info "Welcome to UDX Worker Container. Initializing environment..." # shellcheck disable=SC1091 -source /usr/local/lib/environment.sh +source ${WORKER_LIB_DIR}/environment.sh # Start the process manager in the background log_info "Starting process manager..." -/usr/local/lib/process_manager.sh & +${WORKER_LIB_DIR}/process_manager.sh & # Main execution logic if [ "$#" -gt 0 ]; then diff --git a/etc/configs/worker/services.yaml b/etc/configs/worker/services.yaml deleted file mode 100644 index 82f0fa44..00000000 --- a/etc/configs/worker/services.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -kind: workerService -version: udx.io/worker-v1/service -services: - - name: "app_service" - ignore: "true" - command: "sh ./main.sh" - autostart: "true" - autorestart: "false" - envs: - - "KEY1=value1" - - "KEY2=value2" - - name: "another_service" - ignore: "true" - command: "sh ./main.sh" - autostart: "true" - autorestart: "true" - envs: [] diff --git a/lib/auth.sh b/lib/auth.sh index 1ebc851c..fdb4fb59 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Array to track configured providers declare -a configured_providers=() @@ -61,7 +61,7 @@ authenticate_actors() { # Proceed only if creds are valid JSON if echo "$creds" | jq empty &>/dev/null; then log_info "Processing credentials for $provider" - auth_script="/usr/local/lib/auth/${provider}.sh" + auth_script="${WORKER_LIB_DIR}/auth/${provider}.sh" auth_function="${provider}_authenticate" if [[ -f "$auth_script" ]]; then diff --git a/lib/auth/aws.sh b/lib/auth/aws.sh index 05b41592..a3234d57 100644 --- a/lib/auth/aws.sh +++ b/lib/auth/aws.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Example usage of the function # aws_authenticate "/path/to/your/aws_creds.json" diff --git a/lib/auth/azure.sh b/lib/auth/azure.sh index 7286fd7b..180b2382 100644 --- a/lib/auth/azure.sh +++ b/lib/auth/azure.sh @@ -5,8 +5,8 @@ # Example usage of the function # azure_authenticate "/path/to/your/azure_creds.json" -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to authenticate Azure accounts azure_authenticate() { diff --git a/lib/auth/bitwarden.sh b/lib/auth/bitwarden.sh index 57dbd7d6..55867cda 100644 --- a/lib/auth/bitwarden.sh +++ b/lib/auth/bitwarden.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to authenticate Bitwarden using API key or master password # diff --git a/lib/auth/gcp.sh b/lib/auth/gcp.sh index 0299d3df..1d8e3b88 100644 --- a/lib/auth/gcp.sh +++ b/lib/auth/gcp.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to authenticate GCP service accounts # diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 9a716936..505a3928 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -2,10 +2,10 @@ # Include worker config utilities first # shellcheck source=/dev/null -source /usr/local/lib/worker_config.sh +source ${WORKER_LIB_DIR}/worker_config.sh # shellcheck source=/dev/null -source /usr/local/lib/utils.sh +source ${WORKER_LIB_DIR}/utils.sh # Generic function to clean up authentication for any provider cleanup_provider() { diff --git a/lib/cli.sh b/lib/cli.sh index c9152153..aeafd12b 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -1,30 +1,172 @@ #!/bin/bash +# Version information +VERSION="1.0.0" + +# Source worker configuration +# shellcheck source=${WORKER_LIB_DIR}/worker_config.sh disable=SC1091 +source "${WORKER_LIB_DIR}/worker_config.sh" + +# Load and export configuration +config=$(load_and_parse_config) +export_variables_from_config "$config" + # Dynamically source all command modules -for module in /usr/local/lib/cli/*.sh; do - # shellcheck disable=SC1090 - source "$module" +for module in "${WORKER_LIB_DIR}/cli/"*.sh; do + # shellcheck disable=SC1090 + source "$module" done -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -# CLI Interface -case $1 in - env) - shift - env_handler "$@" - ;; - sbom) - shift - sbom_handler "$@" - ;; - service) - shift - service_handler "$@" - ;; - *) - log_error "CLI" "Unknown command: $1" - exit 1 - ;; -esac \ No newline at end of file +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" + +# Print version information +show_version() { + log_info "Worker CLI version $VERSION" +} + +# Get available commands and their descriptions +get_available_commands() { + local commands={} + declare -A commands + + # Add built-in commands + commands["help"]="Show help for any command" + commands["version"]="Show version information" + + # Scan through CLI modules to find commands and their descriptions + for module in "${WORKER_LIB_DIR}/cli/"*.sh; do + if [ -f "$module" ]; then + local name=$(basename "$module" .sh) + local description="" + + # Extract description from help function + if grep -q "${name}_help()" "$module"; then + description=$(grep -A 5 "${name}_help()" "$module" | + grep -v "${name}_help()" | + grep -v "^{" | + grep -v "cat << EOF" | + head -n 1 | + sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + commands[$name]="$description" + fi + fi + done + + echo "$(declare -p commands)" +} + +# Print help information +show_help() { + local -A commands + eval "$(get_available_commands)" + + # Find the longest command name for proper padding + local max_length=0 + for cmd in "${!commands[@]}"; do + local len=${#cmd} + if ((len > max_length)); then + max_length=$len + fi + done + + # Add padding for alignment + max_length=$((max_length + 2)) + + cat << EOF +šŸš€ Welcome to UDX Worker Container! + +This container helps you run and manage cloud services and applications. +Here's how to get started: + +1. Configure Services: + - Create service config: ${HOME}/.config/worker/services.yaml + - View service commands: worker help service + - Check service status: worker service list + +2. Configure Environment: + - Set environment vars: /etc/worker/environment + - View current settings: worker env show + - Configure cloud auth: worker auth setup + +3. Monitor & Manage: + - View container status: worker info overview + - Check service logs: worker service logs + - View system health: worker health check + +Available Commands: +EOF + + # Sort commands alphabetically and display + local sorted_commands=($(echo "${!commands[@]}" | tr ' ' '\n' | sort)) + for cmd in "${sorted_commands[@]}"; do + printf " %-${max_length}s %s\n" "$cmd" "${commands[$cmd]}" + done + + cat << EOF + +Examples: + worker help service Learn about service configuration + worker env show View current environment + worker info system Check system health + worker auth setup Configure cloud provider auth + +Tip: Run 'worker help [command]' for detailed information about any command +EOF +} + +# Show command-specific help +show_command_help() { + local cmd=$1 + local help_function="${cmd}_help" + + # Check if the help function exists + if [[ $(type -t "$help_function") == function ]]; then + "$help_function" + else + log_error "CLI" "No help available for command: $cmd" + show_help + return 1 + fi +} + +# Main CLI interface +if [ -z "$1" ] || [ "$1" = "help" ]; then + if [ -z "$2" ]; then + show_help + log_info "Container is ready. Run with a command to start services." + else + show_command_help "$2" + fi + exit 0 +fi + +if [ "$1" = "version" ]; then + show_version + exit 0 +fi + +# Handle app and service commands +if [ "$1" = "app" ] || [ "$1" = "service" ]; then + log_info "Starting process manager..." + "${WORKER_LIB_DIR}/process_manager.sh" + pm_status=$? + if [ $pm_status -ne 0 ]; then + exit $pm_status + fi + shift + exec "$@" +fi + +# Check if the command exists by looking for its handler +command=$1 +handler_function="${command}_handler" + +if [[ $(type -t "$handler_function") == function ]]; then + shift + "$handler_function" "$@" +else + log_error "CLI" "Unknown command: $command" + echo "Run 'worker help' to see available commands." + exit 1 +fi \ No newline at end of file diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh new file mode 100644 index 00000000..303259c1 --- /dev/null +++ b/lib/cli/auth.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh +source ${WORKER_LIB_DIR}/auth.sh + +# Show help for auth command +auth_help() { + cat << EOF +Manage authentication and credentials + +Usage: worker auth [command] + +Available Commands: + status Show authentication status for all providers + test Test authentication for all or specific provider + refresh Refresh credentials + rotate Rotate credentials (if supported by provider) + +Examples: + worker auth status + worker auth test aws + worker auth refresh + worker auth rotate --provider aws +EOF +} + +# Show authentication status +show_auth_status() { + log_info "Auth" "Checking authentication status..." + + # Check each provider's status + for provider in aws gcp azure bitwarden; do + if is_provider_configured "$provider"; then + log_success "Auth" "$provider: Configured and authenticated" + else + log_warn "Auth" "$provider: Not configured" + fi + done +} + +# Test authentication +test_auth() { + local provider=$1 + + if [ -z "$provider" ]; then + # Test all configured providers + for p in aws gcp azure bitwarden; do + if is_provider_configured "$p"; then + test_provider_auth "$p" + fi + done + else + # Test specific provider + if is_provider_configured "$provider"; then + test_provider_auth "$provider" + else + log_error "Auth" "Provider $provider is not configured" + return 1 + fi + fi +} + +# Refresh credentials +refresh_auth() { + log_info "Auth" "Refreshing credentials..." + + # Re-run authentication for all configured providers + local actors_json + actors_json=$(get_config_section "$(load_and_parse_config)" "actors") + + if [ -n "$actors_json" ]; then + if authenticate_actors "$actors_json"; then + log_success "Auth" "Successfully refreshed credentials" + else + log_error "Auth" "Failed to refresh credentials" + return 1 + fi + else + log_warn "Auth" "No actors configured" + return 1 + fi +} + +# Rotate credentials +rotate_auth() { + local provider=$1 + + if [ -z "$provider" ]; then + log_error "Auth" "Provider is required for credential rotation" + return 1 + fi + + if ! is_provider_configured "$provider"; then + log_error "Auth" "Provider $provider is not configured" + return 1 + fi + + # Call provider-specific rotation function + if rotate_provider_credentials "$provider"; then + log_success "Auth" "Successfully rotated credentials for $provider" + else + log_error "Auth" "Failed to rotate credentials for $provider" + return 1 + fi +} + +# Handle auth commands +auth_handler() { + local cmd=$1 + shift + + case $cmd in + status) + show_auth_status + ;; + test) + test_auth "$1" + ;; + refresh) + refresh_auth + ;; + rotate) + rotate_auth "$1" + ;; + help) + auth_help + ;; + *) + log_error "Auth" "Unknown command: $cmd" + auth_help + exit 1 + ;; + esac +} diff --git a/lib/cli/config.sh b/lib/cli/config.sh new file mode 100644 index 00000000..f5e9cea4 --- /dev/null +++ b/lib/cli/config.sh @@ -0,0 +1,203 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh +source ${WORKER_LIB_DIR}/worker_config.sh + +# Show help for config command +config_help() { + cat << EOF +Manage worker configuration + +Usage: worker config [command] + +Available Commands: + show Show current configuration + edit Edit configuration in default editor + validate Validate configuration files + locations Show configuration file locations + init Initialize a new configuration file + diff Show differences between default and current config + +Examples: + worker config show + worker config show --format json + worker config validate + worker config edit + worker config locations + worker config init +EOF +} + +# Show current configuration +show_config() { + local format=${1:-yaml} + log_info "Config" "Current configuration:" + + local config + config=$(load_and_parse_config) + + case $format in + json) + echo "$config" | yq eval -o=json '.' + ;; + yaml) + echo "$config" + ;; + *) + log_error "Config" "Unknown format: $format" + return 1 + ;; + esac +} + +# Edit configuration +edit_config() { + local config_file="${HOME}/.config/worker/worker.yaml" + + # Create directory if it doesn't exist + mkdir -p "$(dirname "$config_file")" + + # Create file if it doesn't exist + if [ ! -f "$config_file" ]; then + cp "/etc/worker/worker.yaml" "$config_file" + fi + + # Use default editor or fallback to nano + ${EDITOR:-nano} "$config_file" + + # Validate after editing + if validate_config "$config_file"; then + log_success "Config" "Configuration updated successfully" + else + log_error "Config" "Configuration validation failed after editing" + return 1 + fi +} + +# Validate configuration +validate_config() { + local config_file=${1:-} + log_info "Config" "Validating configuration..." + + # If no file specified, validate all config files + if [ -z "$config_file" ]; then + local files=( + "/etc/worker/worker.yaml" + "${HOME}/.config/worker/worker.yaml" + "/etc/worker/supervisor/supervisord.conf" + ) + + local failed=0 + for file in "${files[@]}"; do + if [ -f "$file" ]; then + if ! validate_yaml "$file"; then + log_error "Config" "Validation failed for $file" + failed=1 + else + log_success "Config" "Validation passed for $file" + fi + fi + done + + return $failed + else + # Validate specific file + if validate_yaml "$config_file"; then + log_success "Config" "Configuration is valid" + return 0 + else + log_error "Config" "Configuration is invalid" + return 1 + fi + fi +} + +# Show configuration locations +show_locations() { + cat << EOF +Configuration Locations: + System config: /etc/worker/worker.yaml + User config: ${HOME}/.config/worker/worker.yaml + Supervisor config: /etc/worker/supervisor/supervisord.conf + Services config: ${HOME}/.config/worker/services.yaml +EOF +} + +# Initialize new configuration +init_config() { + local config_dir="${HOME}/.config/worker" + local config_file="$config_dir/worker.yaml" + + # Create directory if it doesn't exist + mkdir -p "$config_dir" + + # Don't overwrite existing config without confirmation + if [ -f "$config_file" ]; then + log_warn "Config" "Configuration file already exists at $config_file" + read -r -p "Do you want to overwrite it? [y/N] " response + if [[ ! "$response" =~ ^[Yy]$ ]]; then + log_info "Config" "Initialization cancelled" + return 0 + fi + fi + + # Copy default config + cp "/etc/worker/worker.yaml" "$config_file" + + if [ -f "$config_file" ]; then + log_success "Config" "Configuration initialized at $config_file" + else + log_error "Config" "Failed to initialize configuration" + return 1 + fi +} + +# Show differences between default and current config +show_diff() { + local default_config="/etc/worker/worker.yaml" + local user_config="${HOME}/.config/worker/worker.yaml" + + if [ ! -f "$user_config" ]; then + log_error "Config" "User configuration does not exist at $user_config" + return 1 + fi + + log_info "Config" "Differences between default and current configuration:" + diff -u "$default_config" "$user_config" || true +} + +# Handle config commands +config_handler() { + local cmd=$1 + shift + + case $cmd in + show) + show_config "$@" + ;; + edit) + edit_config + ;; + validate) + validate_config "$@" + ;; + locations) + show_locations + ;; + init) + init_config + ;; + diff) + show_diff + ;; + help) + config_help + ;; + *) + log_error "Config" "Unknown command: $cmd" + config_help + exit 1 + ;; + esac +} diff --git a/lib/cli/env.sh b/lib/cli/env.sh index b3b68bc5..84a689d3 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -1,45 +1,368 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh +source ${WORKER_LIB_DIR}/worker_config.sh -# Function to display current environment settings or a specific variable -get_environment() { - if [ $# -eq 0 ]; then - log_debug "Current Environment Settings:" "$(env)" +# Show help for env command +env_help() { + cat << EOF +Manage environment variables and secrets + +Usage: worker env [command] + +Available Commands: + show Show environment variables (excludes secrets) + set Set an environment variable + unset Unset an environment variable + reload Reload environment from config + status Show environment status + validate Validate environment variables + export Export environment to file + import Import environment from file + +Options: + --format Output format (text/json) + --filter Filter variables by prefix + --file File to export to or import from + --include-secrets Include secrets in output (masked) + +Examples: + worker env show + worker env show --format json + worker env show --filter AWS_* + worker env show --include-secrets + worker env set MY_VAR "my value" + worker env reload +EOF +} + +# Show environment variables +show_environment() { + local format=${1:-text} + local filter=$2 + local include_secrets=${3:-false} + + if [ "$include_secrets" == "true" ]; then + list_env_vars "$format" else - log_debug "$1" "${!1}" + # Only show non-secret environment variables + if [ -f "$WORKER_ENV_FILE" ]; then + case $format in + json) + { + echo "{" + if [ -n "$filter" ]; then + grep "^export $filter" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/ "\1": "\2",/' + else + grep "^export" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/ "\1": "\2",/' + fi + echo "}" + } | sed 's/,}/}/' | jq '.' + ;; + text) + if [ -n "$filter" ]; then + grep "^export $filter" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' + else + grep "^export" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' + fi + ;; + *) + log_error "Env" "Unknown format: $format" + return 1 + ;; + esac + else + log_error "Env" "Environment file not found" + return 1 + fi fi } -# Function to set a new environment variable +# Set environment variable set_environment() { - if [ $# -ne 2 ]; then - log_warn "CLI" "Usage: $0 env set " + local name=$1 + local value=$2 + + if [ -z "$name" ] || [ -z "$value" ]; then + log_error "Env" "Both variable name and value are required" + return 1 + fi + + # Validate variable name + if ! [[ $name =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then + log_error "Env" "Invalid variable name: $name" + return 1 + fi + + # Add to environment file + if [ -f "$WORKER_ENV_FILE" ]; then + # Remove existing declaration if any + sed -i "/^export $name=/d" "$WORKER_ENV_FILE" + # Add new declaration + echo "export $name=\"$value\"" >> "$WORKER_ENV_FILE" + # Export in current session + export "$name=$value" + log_success "Env" "Set $name to '$value'" + else + log_error "Env" "Environment file not found" + return 1 + fi +} + +# Unset environment variable +unset_environment() { + local name=$1 + + if [ -z "$name" ]; then + log_error "Env" "Variable name is required" + return 1 + fi + + if [ -n "${!name}" ]; then + unset "$name" + log_success "Env" "Unset $name" + else + log_warn "Env" "Variable $name is not set" + fi +} + +# Export environment variables +export_environment() { + local file=$1 + local filter=$2 + + if [ -z "$file" ]; then + log_error "Env" "Output file is required" + return 1 + fi + + # Export variables + if [ -n "$filter" ]; then + env | grep "^$filter" > "$file" + else + env > "$file" + fi + + log_success "Env" "Environment variables exported to $file" +} + +# Import environment variables +import_environment() { + local file=$1 + + if [ -z "$file" ]; then + log_error "Env" "Input file is required" + return 1 + fi + + if [ ! -f "$file" ]; then + log_error "Env" "File not found: $file" return 1 fi - export "$1=$2" - log_debug "CLI" "Set $1 to '$2'." + + # Import variables + while IFS='=' read -r key value; do + if [ -n "$key" ]; then + export "$key=$value" + fi + done < "$file" + + log_success "Env" "Environment variables imported from $file" +} + +# Validate environment variables +validate_environment() { + local config + config=$(load_and_parse_config) + + if [ -z "$config" ]; then + log_error "Env" "Failed to load configuration" + return 1 + fi + + # Extract required variables from config + local required_vars + required_vars=$(echo "$config" | yq eval '.config.env | keys' -) + + local failed=0 + while IFS= read -r var; do + if [ -z "${!var}" ]; then + log_error "Env" "Required variable $var is not set" + failed=1 + fi + done <<< "$required_vars" + + if [ $failed -eq 0 ]; then + log_success "Env" "All required environment variables are set" + else + return 1 + fi +} + +# Generate environment template +generate_template() { + local config + config=$(load_and_parse_config) + + if [ -z "$config" ]; then + log_error "Env" "Failed to load configuration" + return 1 + fi + + echo "# Worker Environment Variables" + echo "# Generated on $(date)" + echo + + # Extract variables from config + echo "$config" | yq eval '.config.env | to_entries | .[] | "# " + .key + "\n" + .key + "=\"" + .value + "\""' - +} + +# Reset environment +reset_environment() { + # Clear all non-system variables + local system_vars="HOME|USER|PATH|SHELL|TERM|LANG|PWD" + + # Get all variables except system ones + local vars_to_unset + vars_to_unset=$(env | grep -vE "^($system_vars)=") + + while IFS='=' read -r key _; do + if [ -n "$key" ]; then + unset "$key" + fi + done <<< "$vars_to_unset" + + # Reconfigure environment + if configure_environment; then + log_success "Env" "Environment reset to default state" + else + log_error "Env" "Failed to reset environment" + return 1 + fi +} + +# Parse command line arguments +parse_args() { + local args=() + while [[ $# -gt 0 ]]; do + case $1 in + --format) + format=$2 + shift 2 + ;; + --filter) + filter=$2 + shift 2 + ;; + --file) + file=$2 + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac + done + set -- "${args[@]}" +} + +# Show environment status +show_status() { + local format=${1:-text} + + local env_file_exists=false + local secrets_file_exists=false + local env_count=0 + local secrets_count=0 + + [ -f "$WORKER_ENV_FILE" ] && env_file_exists=true + [ -f "$WORKER_SECRETS_FILE" ] && secrets_file_exists=true + + if [ "$env_file_exists" = true ]; then + env_count=$(grep -c "^export" "$WORKER_ENV_FILE" || echo 0) + fi + + if [ "$secrets_file_exists" = true ]; then + secrets_count=$(grep -c "^export" "$WORKER_SECRETS_FILE" || echo 0) + fi + + case $format in + json) + { + echo "{" + echo " \"environment\": {" + echo " \"file\": \"$WORKER_ENV_FILE\"," + echo " \"exists\": $env_file_exists," + echo " \"variables\": $env_count" + echo " }," + echo " \"secrets\": {" + echo " \"file\": \"$WORKER_SECRETS_FILE\"," + echo " \"exists\": $secrets_file_exists," + echo " \"variables\": $secrets_count" + echo " }" + echo "}" + } | jq '.' + ;; + text) + echo "Environment Status:" + echo "------------------" + echo "Environment File: $WORKER_ENV_FILE" + echo " - Exists: $env_file_exists" + echo " - Variables: $env_count" + echo + echo "Secrets File: $WORKER_SECRETS_FILE" + echo " - Exists: $secrets_file_exists" + echo " - Variables: $secrets_count" + ;; + *) + log_error "Env" "Unknown format: $format" + return 1 + ;; + esac } # Handle environment commands env_handler() { - case $1 in - get) - shift - get_environment "$@" + local cmd=$1 + shift + + # Parse command line arguments + parse_args "$@" + + case $cmd in + show) + show_environment "$format" "$filter" "$include_secrets" ;; set) - shift - if [ $# -eq 2 ]; then - set_environment "$@" - else - log_warn "CLI" "Usage: $0 env set " - exit 1 - fi + set_environment "$1" "$2" + ;; + unset) + unset_environment "$1" + ;; + reload) + config=$(load_and_parse_config) + export_variables_from_config "$config" + ;; + status) + show_status "$format" + ;; + validate) + validate_environment + ;; + export) + export_environment "$file" "$filter" + ;; + import) + import_environment "$file" + ;; + help) + env_help ;; *) - log_warn "CLI" "Usage: $0 env {show [VARIABLE_NAME]|set }" + log_error "Env" "Unknown command: $cmd" + env_help exit 1 ;; esac diff --git a/lib/cli/health.sh b/lib/cli/health.sh new file mode 100644 index 00000000..0ecd4fb6 --- /dev/null +++ b/lib/cli/health.sh @@ -0,0 +1,242 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh + +# Show help for health command +health_help() { + cat << EOF +Check system health and run diagnostics + +Usage: worker health [command] + +Available Commands: + check Run health check + status Show current health status + diag Run diagnostics + report Generate health report + +Examples: + worker health check + worker health status + worker health diag + worker health report --format json +EOF +} + +# Run health check +check_health() { + log_info "Health" "Running health check..." + local failed=0 + + # Check system resources + check_system_resources || failed=1 + + # Check supervisor status + check_supervisor_status || failed=1 + + # Check service health + check_services_health || failed=1 + + # Check authentication status + check_auth_status || failed=1 + + if [ $failed -eq 0 ]; then + log_success "Health" "All health checks passed" + else + log_error "Health" "Some health checks failed" + return 1 + fi +} + +# Check system resources +check_system_resources() { + local failed=0 + + # Check disk space + local disk_usage + disk_usage=$(df -h / | awk 'NR==2 {print $5}' | tr -d '%') + if [ "$disk_usage" -gt 90 ]; then + log_error "Health" "Disk usage is critical: ${disk_usage}%" + failed=1 + else + log_success "Health" "Disk usage is normal: ${disk_usage}%" + fi + + # Check memory + local mem_usage + mem_usage=$(free | awk '/Mem:/ {printf("%.0f", $3/$2 * 100)}') + if [ "$mem_usage" -gt 90 ]; then + log_error "Health" "Memory usage is critical: ${mem_usage}%" + failed=1 + else + log_success "Health" "Memory usage is normal: ${mem_usage}%" + fi + + # Check load average + local load_avg + load_avg=$(uptime | awk -F'load average:' '{print $2}' | cut -d, -f1) + if [ "${load_avg%.*}" -gt 4 ]; then + log_error "Health" "Load average is high: $load_avg" + failed=1 + else + log_success "Health" "Load average is normal: $load_avg" + fi + + return $failed +} + +# Check supervisor status +check_supervisor_status() { + if ! pgrep -f supervisord > /dev/null; then + log_error "Health" "Supervisor is not running" + return 1 + fi + + if ! supervisorctl status > /dev/null; then + log_error "Health" "Supervisor is not responding" + return 1 + fi + + log_success "Health" "Supervisor is running and responsive" + return 0 +} + +# Check services health +check_services_health() { + local failed=0 + local services_status + + services_status=$(supervisorctl status) + if [ $? -ne 0 ]; then + log_error "Health" "Failed to get services status" + return 1 + fi + + echo "$services_status" | while read -r line; do + local service_name status + service_name=$(echo "$line" | awk '{print $1}') + status=$(echo "$line" | awk '{print $2}') + + if [ "$status" != "RUNNING" ]; then + log_error "Health" "Service $service_name is not running (status: $status)" + failed=1 + else + log_success "Health" "Service $service_name is running" + fi + done + + return $failed +} + +# Check authentication status +check_auth_status() { + local failed=0 + + # Check each provider + for provider in aws gcp azure bitwarden; do + if is_provider_configured "$provider"; then + if ! test_provider_auth "$provider"; then + log_error "Health" "Authentication failed for $provider" + failed=1 + else + log_success "Health" "Authentication successful for $provider" + fi + fi + done + + return $failed +} + +# Generate health report +generate_report() { + local format=${1:-text} + local report_file + report_file=$(mktemp) + + { + echo "Worker Health Report" + echo "===================" + echo "Timestamp: $(date)" + echo + + echo "System Resources:" + echo "----------------" + df -h / + echo + free -h + echo + uptime + echo + + echo "Services Status:" + echo "---------------" + supervisorctl status + echo + + echo "Authentication Status:" + echo "--------------------" + for provider in aws gcp azure bitwarden; do + if is_provider_configured "$provider"; then + echo "$provider: Configured" + else + echo "$provider: Not configured" + fi + done + } > "$report_file" + + case $format in + json) + # Convert report to JSON format + jq -R -s '{ + timestamp: now, + system_resources: { + disk: (input | match("^/dev.*$")), + memory: (input | match("^Mem:.*$")), + load: (input | match("^load average:.*$")) + }, + services: (input | match("^RUNNING.*$")), + auth: (input | match("^.*: Configured$")) + }' "$report_file" + ;; + text) + cat "$report_file" + ;; + *) + log_error "Health" "Unknown format: $format" + rm -f "$report_file" + return 1 + ;; + esac + + rm -f "$report_file" +} + +# Handle health commands +health_handler() { + local cmd=$1 + shift + + case $cmd in + check) + check_health + ;; + status) + check_health --quiet + ;; + diag) + check_health --verbose + ;; + report) + generate_report "$@" + ;; + help) + health_help + ;; + *) + log_error "Health" "Unknown command: $cmd" + health_help + exit 1 + ;; + esac +} diff --git a/lib/cli/info.sh b/lib/cli/info.sh new file mode 100644 index 00000000..e2836d41 --- /dev/null +++ b/lib/cli/info.sh @@ -0,0 +1,437 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" + +# Show help for info command +info_help() { + cat << EOF +Display information about the UDX Worker image and runtime + +Usage: worker info [command] + +Available Commands: + overview Show a high-level overview of the worker image + system Show system information and resource usage + config Show configuration and settings + services Show available services and their status + env Show environment variables and settings + auth Show authentication and credentials status + deps Show installed dependencies and versions + features Show available features and capabilities + paths Show important filesystem paths + logs Show logging configuration and locations + security Show security settings and policies + network Show network configuration and ports + version Show detailed version information + +Options: + --format Output format (text/json) + --verbose Show detailed information + +Examples: + worker info overview # Get a quick overview of the worker + worker info system --verbose # Show detailed system information + worker info deps --format json # List dependencies in JSON format + worker info features # Show available features +EOF +} + +# Show general information +show_info() { + local format=${1:-text} + + case $format in + json) + { + echo "{" + echo " \"version\": \"$VERSION\"," + echo " \"os\": \"$(uname -s)\"," + echo " \"architecture\": \"$(uname -m)\"," + echo " \"hostname\": \"$(hostname)\"," + echo " \"user\": \"$USER\"," + echo " \"paths\": {" + echo " \"base\": \"$WORKER_BASE_DIR\"," + echo " \"config\": \"$WORKER_CONFIG_DIR\"," + echo " \"apps\": \"$WORKER_APP_DIR\"," + echo " \"data\": \"$WORKER_DATA_DIR\"," + echo " \"lib\": \"$WORKER_LIB_DIR\"," + echo " \"bin\": \"$WORKER_BIN_DIR\"" + echo " }," + echo " \"cloud\": {" + echo " \"gcp_config\": \"$CLOUDSDK_CONFIG\"," + echo " \"aws_config\": \"$AWS_CONFIG_FILE\"," + echo " \"azure_config\": \"$AZURE_CONFIG_DIR\"" + echo " }" + echo "}" + } | jq '.' + ;; + text) + echo "UDX Worker Information" + echo "=====================" + echo + echo "Version: $VERSION" + echo + echo "System" + echo "------" + echo "OS: $(uname -s)" + echo "Architecture: $(uname -m)" + echo "Hostname: $(hostname)" + echo "User: $USER" + echo + echo "Paths" + echo "-----" + echo "Base Directory: $WORKER_BASE_DIR" + echo "Config Directory: $WORKER_CONFIG_DIR" + echo "Apps Directory: $WORKER_APP_DIR" + echo "Data Directory: $WORKER_DATA_DIR" + echo "Library Directory: $WORKER_LIB_DIR" + echo "Binary Directory: $WORKER_BIN_DIR" + echo + echo "Cloud Configuration" + echo "------------------" + echo "GCP Config: $CLOUDSDK_CONFIG" + echo "AWS Config: $AWS_CONFIG_FILE" + echo "Azure Config: $AZURE_CONFIG_DIR" + ;; + *) + log_error "Info" "Unknown format: $format" + return 1 + ;; + esac +} + +# Show system information +show_system_info() { + local format=${1:-text} + + case $format in + json) + { + echo "{" + echo " \"kernel\": \"$(uname -r)\"," + echo " \"os\": {" + echo " \"name\": \"$(uname -s)\"," + echo " \"version\": \"$(cat /etc/os-release | grep VERSION= | cut -d'\"' -f2)\"" + echo " }," + echo " \"cpu\": {" + echo " \"architecture\": \"$(uname -m)\"," + echo " \"cores\": $(nproc)," + echo " \"model\": \"$(grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2 | xargs)\"" + echo " }," + echo " \"memory\": {" + echo " \"total\": $(free -b | grep Mem | awk '{print $2}')," + echo " \"free\": $(free -b | grep Mem | awk '{print $4}')" + echo " }," + echo " \"disk\": {" + echo " \"total\": $(df -B1 / | tail -1 | awk '{print $2}')," + echo " \"free\": $(df -B1 / | tail -1 | awk '{print $4}')" + echo " }" + echo "}" + } | jq '.' + ;; + text) + echo "System Information" + echo "==================" + echo + echo "Kernel: $(uname -r)" + echo + echo "Operating System" + echo "----------------" + echo "Name: $(uname -s)" + echo "Version: $(cat /etc/os-release | grep VERSION= | cut -d'\"' -f2)" + echo + echo "CPU" + echo "---" + echo "Architecture: $(uname -m)" + echo "Cores: $(nproc)" + echo "Model: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2 | xargs)" + echo + echo "Memory" + echo "------" + echo "Total: $(free -h | grep Mem | awk '{print $2}')" + echo "Free: $(free -h | grep Mem | awk '{print $4}')" + echo + echo "Disk" + echo "----" + echo "Total: $(df -h / | tail -1 | awk '{print $2}')" + echo "Free: $(df -h / | tail -1 | awk '{print $4}')" + ;; + *) + log_error "Info" "Unknown format: $format" + return 1 + ;; + esac +} + +# Get available CLI modules and their help info +get_cli_modules() { + local modules_dir="${WORKER_LIB_DIR}/cli" + local modules={} + + # Initialize modules array + declare -A modules + + # Scan through CLI modules + for module in "$modules_dir"/*.sh; do + local name=$(basename "$module" .sh) + local description="" + local commands=() + + # Extract help function content + if grep -q "${name}_help()" "$module"; then + # Get description from help function + description=$(grep -A 1 "${name}_help()" "$module" | grep -v "${name}_help()" | grep -v "^{" | head -n 1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + # Get available commands + while IFS= read -r line; do + if [[ $line =~ ^[[:space:]]+([a-zA-Z0-9_-]+)[[:space:]]+(.+)$ ]]; then + commands+=("${BASH_REMATCH[1]}:${BASH_REMATCH[2]}") + fi + done < <(grep -A 20 "Available Commands:" "$module" | grep -B 20 "Examples:" | grep "^[[:space:]]*[a-zA-Z]") + fi + + modules[$name]="$description|${commands[*]}" + done + + echo "$(declare -p modules)" +} + +# Show overview of the worker +show_overview() { + local format=${1:-text} + local -A modules + eval "$(get_cli_modules)" + + case $format in + json) + { + echo "{" + echo " \"name\": \"UDX Worker\"," + echo " \"description\": \"Universal Data Exchange Worker for managing cloud services and data pipelines\"," + echo " \"version\": \"$VERSION\"," + echo " \"modules\": {" + + local first=true + for module in "${!modules[@]}"; do + IFS='|' read -r description commands <<< "${modules[$module]}" + + if [ "$first" = true ]; then + first=false + else + echo "," + fi + + echo " \"$module\": {" + echo " \"description\": \"$description\"," + echo " \"commands\": [" + + IFS=' ' read -ra cmd_array <<< "$commands" + local first_cmd=true + for cmd in "${cmd_array[@]}"; do + IFS=':' read -r cmd_name cmd_desc <<< "$cmd" + if [ "$first_cmd" = true ]; then + first_cmd=false + else + echo "," + fi + echo " {" + echo " \"name\": \"$cmd_name\"," + echo " \"description\": \"$cmd_desc\"" + echo -n " }" + done + echo "" + echo " ]" + echo -n " }" + done + echo "" + echo " }," + echo " \"dependencies\": {" + echo " \"runtime\": \"$(cat /etc/os-release | grep PRETTY_NAME | cut -d '"' -f 2)\"," + echo " \"python\": \"$(python3 --version | cut -d ' ' -f 2)\"," + echo " \"supervisor\": \"$(supervisord -v)\"" + echo " }" + echo "}" + } | jq '.' + ;; + text) + cat << EOF +UDX Worker Overview +================== + +Description: Universal Data Exchange Worker for managing cloud services and data pipelines +Version: ${VERSION} + +Available Modules +---------------- +EOF + for module in "${!modules[@]}"; do + IFS='|' read -r description commands <<< "${modules[$module]}" + echo -e "\nā€¢ $module - $description" + + if [ -n "$commands" ]; then + echo " Commands:" + IFS=' ' read -ra cmd_array <<< "$commands" + for cmd in "${cmd_array[@]}"; do + IFS=':' read -r cmd_name cmd_desc <<< "$cmd" + echo " āœ“ $cmd_name - $cmd_desc" + done + fi + done + + echo -e "\nSystem Information +------------------" + echo "ā€¢ Runtime: $(cat /etc/os-release | grep PRETTY_NAME | cut -d '"' -f 2)" + echo "ā€¢ Python: $(python3 --version | cut -d ' ' -f 2)" + echo "ā€¢ Supervisor: $(supervisord -v)" + + echo -e "\nFor more information:" + echo "ā€¢ Run 'worker help' for usage instructions" + echo "ā€¢ Run 'worker info ' for detailed module information" + echo "ā€¢ Run 'worker help' for module-specific help" + ;; + *) + log_error "Info" "Unknown format: $format" + return 1 + ;; + esac +} + +# Show available features +show_features() { + local format=${1:-text} + local -A modules + eval "$(get_cli_modules)" + + case $format in + json) + { + echo "{" + echo " \"features\": {" + + local first=true + for module in "${!modules[@]}"; do + IFS='|' read -r description commands <<< "${modules[$module]}" + + if [ "$first" = true ]; then + first=false + else + echo "," + fi + + # Convert module name to feature key + local feature_key=$(echo "$module" | tr '-' '_') + + echo " \"$feature_key\": {" + echo " \"description\": \"$description\"," + echo " \"commands\": [" + + IFS=' ' read -ra cmd_array <<< "$commands" + local first_cmd=true + for cmd in "${cmd_array[@]}"; do + IFS=':' read -r cmd_name cmd_desc <<< "$cmd" + if [ "$first_cmd" = true ]; then + first_cmd=false + else + echo "," + fi + echo " {" + echo " \"name\": \"$cmd_name\"," + echo " \"description\": \"$cmd_desc\"" + echo -n " }" + done + echo "" + echo " ]" + echo -n " }" + done + echo "" + echo " }" + echo "}" + } | jq '.' + ;; + text) + cat << EOF +UDX Worker Features +================== + +EOF + local count=1 + for module in "${!modules[@]}"; do + IFS='|' read -r description commands <<< "${modules[$module]}" + echo -e "\n$count. ${module^}" + echo " Description: $description" + + if [ -n "$commands" ]; then + echo " Commands:" + IFS=' ' read -ra cmd_array <<< "$commands" + for cmd in "${cmd_array[@]}"; do + IFS=':' read -r cmd_name cmd_desc <<< "$cmd" + echo " ā€¢ $cmd_name - $cmd_desc" + done + fi + ((count++)) + done + echo -e "\nUse 'worker help' for detailed feature documentation" + ;; + *) + log_error "Info" "Unknown format: $format" + return 1 + ;; + esac +} + +# Handle info commands +info_handler() { + local command=$1 + shift + + case $command in + overview) + show_overview "$@" + ;; + system) + show_system_info "$@" + ;; + features) + show_features "$@" + ;; + deps) + show_deps "$@" + ;; + config) + show_config "$@" + ;; + services) + show_services "$@" + ;; + env) + show_env "$@" + ;; + auth) + show_auth "$@" + ;; + paths) + show_paths "$@" + ;; + logs) + show_logs "$@" + ;; + security) + show_security "$@" + ;; + network) + show_network "$@" + ;; + version) + show_version "$@" + ;; + help|"") + info_help + ;; + *) + log_error "Info" "Unknown command: $command" + info_help + return 1 + ;; + esac +} diff --git a/lib/cli/logs.sh b/lib/cli/logs.sh new file mode 100644 index 00000000..03a92dbf --- /dev/null +++ b/lib/cli/logs.sh @@ -0,0 +1,271 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh + +# Show help for logs command +logs_help() { + cat << EOF +View and manage logs + +Usage: worker logs [command] + +Available Commands: + show Show logs for a service + follow Follow logs in real-time + search Search logs for pattern + export Export logs to file + clean Clean old logs + +Options: + --service Service name (required for show/follow) + --type Log type (out/err/all, default: all) + --lines Number of lines (default: 100) + --since Show logs since timestamp + --until Show logs until timestamp + --pattern Search pattern + --format Output format (text/json) + +Examples: + worker logs show --service myapp + worker logs follow --service myapp --type err + worker logs search --pattern "error" + worker logs export --service myapp --since "2024-01-01" + worker logs clean --older-than 7d +EOF +} + +# Get log file path +get_log_file() { + local service=$1 + local type=${2:-all} + + case $type in + out) + echo "/var/log/supervisor/${service}-stdout.log" + ;; + err) + echo "/var/log/supervisor/${service}-stderr.log" + ;; + all) + echo "/var/log/supervisor/${service}-*.log" + ;; + *) + log_error "Logs" "Invalid log type: $type" + return 1 + ;; + esac +} + +# Show logs +show_logs() { + local service=$1 + local type=${2:-all} + local lines=${3:-100} + local since=$4 + local until=$5 + + if [ -z "$service" ]; then + log_error "Logs" "Service name is required" + return 1 + fi + + local log_file + log_file=$(get_log_file "$service" "$type") + + if [ ! -f "$log_file" ]; then + log_error "Logs" "Log file not found: $log_file" + return 1 + fi + + local cmd="tail -n $lines" + + if [ -n "$since" ]; then + cmd="$cmd | awk -v since=\"\$since\" '\$0 >= since'" + fi + + if [ -n "$until" ]; then + cmd="$cmd | awk -v until=\"\$until\" '\$0 <= until'" + fi + + eval "$cmd $log_file" +} + +# Follow logs +follow_logs() { + local service=$1 + local type=${2:-all} + + if [ -z "$service" ]; then + log_error "Logs" "Service name is required" + return 1 + fi + + local log_file + log_file=$(get_log_file "$service" "$type") + + if [ ! -f "$log_file" ]; then + log_error "Logs" "Log file not found: $log_file" + return 1 + fi + + tail -f "$log_file" +} + +# Search logs +search_logs() { + local pattern=$1 + local service=$2 + local type=${3:-all} + + if [ -z "$pattern" ]; then + log_error "Logs" "Search pattern is required" + return 1 + fi + + local log_file + if [ -n "$service" ]; then + log_file=$(get_log_file "$service" "$type") + else + log_file="/var/log/supervisor/*.log" + fi + + grep -n "$pattern" $log_file +} + +# Export logs +export_logs() { + local service=$1 + local type=${2:-all} + local since=$3 + local until=$4 + local format=${5:-text} + + if [ -z "$service" ]; then + log_error "Logs" "Service name is required" + return 1 + fi + + local log_file + log_file=$(get_log_file "$service" "$type") + + if [ ! -f "$log_file" ]; then + log_error "Logs" "Log file not found: $log_file" + return 1 + fi + + local output_file="${service}_logs_$(date +%Y%m%d_%H%M%S)" + + case $format in + json) + output_file="$output_file.json" + awk '{printf "{\\"timestamp\\":\\"%s\\",\\"message\\":\\"%s\\"}\\n", $1, substr($0,index($0,$2))}' "$log_file" > "$output_file" + ;; + text) + output_file="$output_file.log" + if [ -n "$since" ] || [ -n "$until" ]; then + awk -v since="$since" -v until="$until" ' + ($0 >= since) && ($0 <= until || until=="") + ' "$log_file" > "$output_file" + else + cp "$log_file" "$output_file" + fi + ;; + *) + log_error "Logs" "Unknown format: $format" + return 1 + ;; + esac + + log_success "Logs" "Logs exported to $output_file" +} + +# Clean old logs +clean_logs() { + local older_than=${1:-7d} # Default: 7 days + + find /var/log/supervisor -name "*.log" -type f -mtime +"${older_than%d}" -delete + + log_success "Logs" "Cleaned logs older than $older_than" +} + +# Parse command line arguments +parse_args() { + local args=() + while [[ $# -gt 0 ]]; do + case $1 in + --service) + service=$2 + shift 2 + ;; + --type) + type=$2 + shift 2 + ;; + --lines) + lines=$2 + shift 2 + ;; + --since) + since=$2 + shift 2 + ;; + --until) + until=$2 + shift 2 + ;; + --pattern) + pattern=$2 + shift 2 + ;; + --format) + format=$2 + shift 2 + ;; + --older-than) + older_than=$2 + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac + done + set -- "${args[@]}" +} + +# Handle logs commands +logs_handler() { + local cmd=$1 + shift + + # Parse command line arguments + parse_args "$@" + + case $cmd in + show) + show_logs "$service" "$type" "$lines" "$since" "$until" + ;; + follow) + follow_logs "$service" "$type" + ;; + search) + search_logs "$pattern" "$service" "$type" + ;; + export) + export_logs "$service" "$type" "$since" "$until" "$format" + ;; + clean) + clean_logs "$older_than" + ;; + help) + logs_help + ;; + *) + log_error "Logs" "Unknown command: $cmd" + logs_help + exit 1 + ;; + esac +} diff --git a/lib/cli/sbom.sh b/lib/cli/sbom.sh index 3ba15d70..0d3eb57b 100644 --- a/lib/cli/sbom.sh +++ b/lib/cli/sbom.sh @@ -1,27 +1,278 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh -# Function to display dpkg packages in a table format using awk +# Show help for sbom command +sbom_help() { + cat << EOF +Manage Software Bill of Materials (SBOM) + +Usage: worker sbom [command] + +Available Commands: + generate Generate SBOM in various formats + analyze Analyze dependencies for security issues + export Export SBOM to file + deps Show dependency tree + verify Verify package integrity + updates Check for available updates + +Options: + --format Output format (text/json/cyclonedx/spdx) + --file Output file for export + --type Package type (system/python/all) + --filter Filter packages by name pattern + +Examples: + worker sbom generate + worker sbom generate --format json + worker sbom export --format cyclonedx --file sbom.xml + worker sbom deps --type python + worker sbom verify +EOF +} + +# Generate SBOM generate_sbom() { - log_debug "Package Name | Version | Architecture" - log_debug "----------------------------------------------------------" - dpkg-query -W -f='${binary:Package} | ${Version} | ${Architecture}\n' | awk -F'|' '{ - printf("%-30s | %-20s | %-10s\n", $1, $2, $3) - }' - log_debug "----------------------------------------------------------" + local format=${1:-text} + local type=${2:-all} + local filter=$3 + + log_info "SBOM" "Generating software bill of materials..." + + # Get system packages + local system_packages + if [[ $type == "all" || $type == "system" ]]; then + if [ -n "$filter" ]; then + system_packages=$(dpkg-query -W -f='${binary:Package}\t${Version}\t${Architecture}\t${Status}\n' | grep "$filter") + else + system_packages=$(dpkg-query -W -f='${binary:Package}\t${Version}\t${Architecture}\t${Status}\n') + fi + fi + + # Get Python packages + local python_packages + if [[ $type == "all" || $type == "python" ]]; then + if [ -n "$filter" ]; then + python_packages=$(pip list --format=json | jq -r '.[] | select(.name | contains("'"$filter"'")) | [.name, .version] | @tsv') + else + python_packages=$(pip list --format=json | jq -r '.[] | [.name, .version] | @tsv') + fi + fi + + case $format in + json) + { + echo "{" + if [ -n "$system_packages" ]; then + echo ' "system_packages": {' + echo "$system_packages" | awk -F'\t' '{ + printf(" \"%s\": {\n \"version\": \"%s\",\n \"architecture\": \"%s\"\n }%s\n", $1, $2, $3, (NR==1?",":"")) + }' + echo " }," + fi + if [ -n "$python_packages" ]; then + echo ' "python_packages": {' + echo "$python_packages" | awk -F'\t' '{ + printf(" \"%s\": {\n \"version\": \"%s\"\n }%s\n", $1, $2, (NR==1?",":"")) + }' + echo " }" + fi + echo "}" + } | jq '.' + ;; + cyclonedx) + # Generate CycloneDX XML format + { + echo '' + echo '' + echo ' ' + if [ -n "$system_packages" ]; then + echo "$system_packages" | awk -F'\t' '{ + printf(" \n %s\n %s\n pkg:deb/%s@%s?arch=%s\n \n", $1, $2, $1, $2, $3) + }' + fi + if [ -n "$python_packages" ]; then + echo "$python_packages" | awk -F'\t' '{ + printf(" \n %s\n %s\n pkg:pypi/%s@%s\n \n", $1, $2, $1, $2) + }' + fi + echo ' ' + echo '' + } + ;; + text) + { + echo "Software Bill of Materials" + echo "Generated on $(date)" + echo + if [ -n "$system_packages" ]; then + echo "System Packages:" + echo "---------------" + echo "Package Name | Version | Architecture" + echo "----------------------------------------------------------" + echo "$system_packages" | awk -F'\t' '{ + printf("%-20s | %-16s | %-12s\n", $1, $2, $3) + }' + echo + fi + if [ -n "$python_packages" ]; then + echo "Python Packages:" + echo "---------------" + echo "Package Name | Version" + echo "----------------------------------" + echo "$python_packages" | awk -F'\t' '{ + printf("%-20s | %-16s\n", $1, $2) + }' + fi + } + ;; + *) + log_error "SBOM" "Unknown format: $format" + return 1 + ;; + esac +} + +# Analyze dependencies +analyze_deps() { + log_info "SBOM" "Analyzing dependencies for security issues..." + + # Check system packages + if command -v apt-get >/dev/null; then + apt-get update -qq + apt-get --simulate upgrade | grep -i security + fi + + # Check Python packages + if command -v pip-audit >/dev/null; then + pip-audit + else + log_warn "SBOM" "pip-audit not installed. Install with: pip install pip-audit" + fi +} + +# Export SBOM +export_sbom() { + local format=$1 + local file=$2 + + if [ -z "$file" ]; then + log_error "SBOM" "Output file is required" + return 1 + fi + + generate_sbom "$format" > "$file" + log_success "SBOM" "SBOM exported to $file" +} + +# Show dependency tree +show_deps() { + local type=${1:-all} + + if [[ $type == "all" || $type == "python" ]]; then + log_info "SBOM" "Python dependencies:" + pip install pipdeptree >/dev/null 2>&1 + pipdeptree + fi +} + +# Verify package integrity +verify_packages() { + log_info "SBOM" "Verifying package integrity..." + + # Verify system packages + local failed=0 + while IFS= read -r pkg; do + if ! dpkg -V "$pkg" >/dev/null 2>&1; then + log_error "SBOM" "Package integrity check failed: $pkg" + failed=1 + fi + done < <(dpkg-query -W -f='${binary:Package}\n') + + if [ $failed -eq 0 ]; then + log_success "SBOM" "All packages verified successfully" + else + return 1 + fi +} + +# Check for updates +check_updates() { + log_info "SBOM" "Checking for available updates..." + + # Check system packages + apt-get update -qq + apt-get --just-print upgrade 2>&1 | awk '/^Inst/ { print $2 " (" $3 " => " $4 ")" }' + + # Check Python packages + pip list --outdated --format=json | jq -r '.[] | "\(.name) (\(.version) => \(.latest_version))"' } -# Handler for the sbom command +# Parse command line arguments +parse_args() { + local args=() + while [[ $# -gt 0 ]]; do + case $1 in + --format) + format=$2 + shift 2 + ;; + --file) + file=$2 + shift 2 + ;; + --type) + type=$2 + shift 2 + ;; + --filter) + filter=$2 + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac + done + set -- "${args[@]}" +} + +# Handle sbom commands sbom_handler() { - case $1 in + local cmd=$1 + shift + + # Parse command line arguments + parse_args "$@" + + case $cmd in generate) - shift - generate_sbom "$@" + generate_sbom "$format" "$type" "$filter" + ;; + analyze) + analyze_deps + ;; + export) + export_sbom "$format" "$file" + ;; + deps) + show_deps "$type" + ;; + verify) + verify_packages + ;; + updates) + check_updates + ;; + help) + sbom_help ;; *) - log_warn "CLI" "Usage: $0 sbom {generate}" + log_error "SBOM" "Unknown command: $cmd" + sbom_help exit 1 ;; esac diff --git a/lib/cli/service.sh b/lib/cli/service.sh index ed764f1e..4fb32932 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh service_handler() { local cmd=$1 @@ -33,6 +33,45 @@ service_handler() { esac } +# Show help for service command +service_help() { + cat << EOF +Manage UDX Worker services and applications + +Usage: worker service [command] [options] + +Commands: + list List all configured services and their status + status [name] Show status of all services or a specific service + logs [name] View logs for all services or a specific service + errors [name] View error logs for all services or a specific service + config Show current service configuration + start [name] Start a service + stop [name] Stop a service + restart [name] Restart a service + +Configuration: + Services are configured in: ${HOME}/.config/worker/services.yaml + +Example service configuration: + version: "1.0" + services: + my-app: + name: "my-app" + command: "python app.py" + working_dir: "/opt/worker/apps" + autostart: true + autorestart: true + environment: + APP_PORT: "8080" + +Examples: + worker service list # List all services + worker service logs my-app # View logs for my-app + worker service start my-app # Start my-app service +EOF +} + # Function to list all services list_services() { # Capture the output of supervisorctl status @@ -41,14 +80,16 @@ list_services() { # Check if Supervisor is not running, not accessible, or if there are no managed services if [[ -z "$services_status" ]] || echo "$services_status" | grep -Eq 'no such|ERROR'; then - log_warn "Service" "No services are currently managed." - exit 1 + log_info "No services are currently managed." + log_info "To configure services, create ${HOME}/.config/worker/services.yaml" + log_info "Run 'worker help service' for configuration examples." + return 0 fi - log_debug "Service" "Listing all managed services:" + log_info "Managed services:" local i=1 echo "$services_status" | while read -r line; do - log_debug "Service" "$i. $line" + log_info "$i. $line" ((i++)) done } diff --git a/lib/env_handler.sh b/lib/env_handler.sh new file mode 100644 index 00000000..f253cc99 --- /dev/null +++ b/lib/env_handler.sh @@ -0,0 +1,150 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" + +# Environment file location +WORKER_ENV_FILE="/etc/worker/environment" +WORKER_SECRETS_FILE="/etc/worker/secrets" + +# Generate environment file from worker.yaml +generate_env_file() { + local config + config=$(load_and_parse_config) + + if [ -z "$config" ]; then + log_error "Environment" "Failed to load configuration" + return 1 + fi + + # Extract and evaluate environment variables + { + echo "# Worker Environment Variables" + echo "# Generated on $(date)" + echo "# DO NOT EDIT THIS FILE DIRECTLY" + echo + + # Extract environment variables + echo "$config" | yq eval '.config.env | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' - + + # Add computed variables + echo + echo "# Computed variables" + echo "export WORKER_ENV_FILE=\"$WORKER_ENV_FILE\"" + echo "export WORKER_SECRETS_FILE=\"$WORKER_SECRETS_FILE\"" + } > "$WORKER_ENV_FILE" + + # File permissions are set during build +} + +# Generate secrets file from worker.yaml +generate_secrets_file() { + local config + config=$(load_and_parse_config) + + if [ -z "$config" ]; then + log_error "Environment" "Failed to load configuration" + return 1 + fi + + # Extract and evaluate secrets + { + echo "# Worker Secrets" + echo "# Generated on $(date)" + echo "# DO NOT EDIT THIS FILE DIRECTLY" + echo "# DO NOT COMMIT THIS FILE" + echo + + # Extract secrets + echo "$config" | yq eval '.config.secrets | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' - + } > "$WORKER_SECRETS_FILE" + + # File permissions are set during build +} + +# Load environment variables +load_environment() { + if [ -f "$WORKER_ENV_FILE" ]; then + # shellcheck source=/dev/null + source "$WORKER_ENV_FILE" + else + log_warn "Environment" "Environment file not found, generating..." + generate_env_file + # shellcheck source=/dev/null + source "$WORKER_ENV_FILE" + fi +} + +# Load secrets +load_secrets() { + if [ -f "$WORKER_SECRETS_FILE" ]; then + # shellcheck source=/dev/null + source "$WORKER_SECRETS_FILE" + else + log_warn "Environment" "Secrets file not found, generating..." + generate_secrets_file + # shellcheck source=/dev/null + source "$WORKER_SECRETS_FILE" + fi +} + +# Initialize environment +init_environment() { + generate_env_file + generate_secrets_file + load_environment + load_secrets +} + +# Update environment when config changes +update_environment() { + generate_env_file + generate_secrets_file + load_environment + load_secrets +} + +# Get environment variable value +get_env_value() { + local var_name="$1" + + if [ -z "$var_name" ]; then + log_error "Environment" "Variable name not provided" + return 1 + fi + + if [ -f "$WORKER_ENV_FILE" ]; then + grep "^export $var_name=" "$WORKER_ENV_FILE" | cut -d'=' -f2- | tr -d '"' + elif [ -f "$WORKER_SECRETS_FILE" ]; then + grep "^export $var_name=" "$WORKER_SECRETS_FILE" | cut -d'=' -f2- | tr -d '"' + else + log_error "Environment" "Neither environment nor secrets file exists" + return 1 + fi +} + +# List all environment variables +list_env_vars() { + local show_secrets="$1" + local env_vars="" + local secret_vars="" + + if [ -f "$WORKER_ENV_FILE" ]; then + env_vars=$(grep "^export" "$WORKER_ENV_FILE" | cut -d'=' -f1 | cut -d' ' -f2) + fi + + if [ "$show_secrets" = "true" ] && [ -f "$WORKER_SECRETS_FILE" ]; then + secret_vars=$(grep "^export" "$WORKER_SECRETS_FILE" | cut -d'=' -f1 | cut -d' ' -f2) + fi + + if [ -n "$env_vars" ]; then + echo "Environment Variables:" + echo "$env_vars" + fi + + if [ -n "$secret_vars" ]; then + echo + echo "Secret Variables:" + echo "$secret_vars" + fi +} \ No newline at end of file diff --git a/lib/environment.sh b/lib/environment.sh index 2c436af3..02595dac 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -2,16 +2,16 @@ # Include necessary modules # shellcheck disable=SC1091 -source /usr/local/lib/auth.sh +source "${WORKER_LIB_DIR}/auth.sh" # shellcheck disable=SC1091 -source /usr/local/lib/secrets.sh +source "${WORKER_LIB_DIR}/secrets.sh" # shellcheck disable=SC1091 -source /usr/local/lib/cleanup.sh +source "${WORKER_LIB_DIR}/cleanup.sh" # shellcheck disable=SC1091 -source /usr/local/lib/worker_config.sh +source "${WORKER_LIB_DIR}/worker_config.sh" # shellcheck disable=SC1091 -source /usr/local/lib/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Main function to coordinate environment setup configure_environment() { diff --git a/lib/process_manager.sh b/lib/process_manager.sh index 4c13fa28..4a1b19ba 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -1,19 +1,23 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" # Define paths -DEFAULT_CONFIG_FILE="/usr/local/configs/worker/services.yaml" -# Define the user-specific configuration path search -# shellcheck disable=SC2227 -USER_CONFIG_PATH=$(find "$HOME" -name 'services.yaml' 2>/dev/null -print | head -n 1) +USER_CONFIG_PATH="${HOME}/.config/worker/services.yaml" +CONFIG_FILE="${USER_CONFIG_PATH}" -# Use the first user-specific config found; if none, use the default -CONFIG_FILE="${USER_CONFIG_PATH:-$DEFAULT_CONFIG_FILE}" -COMMON_TEMPLATE_FILE="/usr/local/configs/supervisor/common.conf" -PROGRAM_TEMPLATE_FILE="/usr/local/configs/supervisor/program.conf" -FINAL_CONFIG="/usr/local/configs/supervisor/supervisord.conf" +# Check if user config exists +if [[ ! -f "${USER_CONFIG_PATH}" ]]; then + log_info "No services configuration found at ${USER_CONFIG_PATH}. Services will not be started." + log_info "Run 'worker help service' for information about service configuration" + exit 0 +fi + +# Supervisor configuration paths +COMMON_TEMPLATE_FILE="${WORKER_CONFIG_DIR}/supervisor/common.conf" +PROGRAM_TEMPLATE_FILE="${WORKER_CONFIG_DIR}/supervisor/program.conf" +FINAL_CONFIG="${WORKER_CONFIG_DIR}/supervisor/supervisord.conf" # Set up signal handling trap 'handle_supervisor_signals SIGTERM' SIGTERM @@ -25,6 +29,7 @@ main() { if ! configure_and_execute_services; then log_error "Process Manager" "Failed to configure and start services" + log_info "Run 'worker help service' for information about service configuration" exit 1 fi diff --git a/lib/secrets.sh b/lib/secrets.sh index 04153cb5..4a40d428 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -1,15 +1,15 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Dynamically source the required provider-specific modules source_provider_module() { local provider="$1" - local module_path="/usr/local/lib/secrets/${provider}.sh" + local module_path="${WORKER_LIB_DIR}/secrets/${provider}.sh" if [[ -f "$module_path" ]]; then - # shellcheck source=/usr/local/lib/secrets/${provider}.sh disable=SC1091 + # shellcheck source=${WORKER_LIB_DIR}/secrets/${provider}.sh disable=SC1091 source "$module_path" log_info "Loaded module for provider: $provider" else diff --git a/lib/secrets/aws.sh b/lib/secrets/aws.sh index 3a34d1d7..67ee643e 100644 --- a/lib/secrets/aws.sh +++ b/lib/secrets/aws.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to resolve AWS secret resolve_aws_secret() { diff --git a/lib/secrets/azure.sh b/lib/secrets/azure.sh index 4289d357..80c515bd 100644 --- a/lib/secrets/azure.sh +++ b/lib/secrets/azure.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to resolve Azure secret resolve_azure_secret() { diff --git a/lib/secrets/bitwarden.sh b/lib/secrets/bitwarden.sh index 6075aaf0..962d0689 100644 --- a/lib/secrets/bitwarden.sh +++ b/lib/secrets/bitwarden.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to resolve Bitwarden secret resolve_bitwarden_secret() { diff --git a/lib/secrets/gcp.sh b/lib/secrets/gcp.sh index 2edff3d2..09bd7c67 100644 --- a/lib/secrets/gcp.sh +++ b/lib/secrets/gcp.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to resolve GCP secret resolve_gcp_secret() { diff --git a/lib/worker_config.sh b/lib/worker_config.sh index fdfed67c..9d65ca69 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -1,14 +1,14 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" +# shellcheck source=${WORKER_LIB_DIR}/env_handler.sh disable=SC1091 +source "${WORKER_LIB_DIR}/env_handler.sh" # Paths for configurations -BUILT_IN_CONFIG="/usr/local/configs/worker/default.yaml" -# Dynamically find user configuration in any subfolder of $HOME -# shellcheck disable=SC2227 -USER_CONFIG=$(find "$HOME" -name 'worker.yaml' 2>/dev/null -print | head -n 1) -MERGED_CONFIG="/usr/local/configs/worker/merged_worker.yaml" +BUILT_IN_CONFIG="${WORKER_CONFIG_DIR}/worker.yaml" # Built-in default config +USER_CONFIG="${HOME}/.config/worker/worker.yaml" # Optional user config +MERGED_CONFIG="${WORKER_CONFIG_DIR}/worker.merged.yaml" # Result of merging both configs # Ensure `yq` is available if ! command -v yq >/dev/null 2>&1; then @@ -72,20 +72,31 @@ load_and_parse_config() { export_variables_from_config() { local config_json="$1" - # Extract the `variables` section - local variables - variables=$(echo "$config_json" | jq -r '.config.env // empty') - if [[ -z "$variables" || "$variables" == "null" ]]; then - log_info "No variables found in the configuration." + # Extract environment variables and secrets + local env_vars secrets + env_vars=$(echo "$config_json" | jq -r '.config.env // empty') + secrets=$(echo "$config_json" | jq -r '.config.secrets // empty') + + if [[ -z "$env_vars" && -z "$secrets" ]]; then + log_info "No variables or secrets found in the configuration." return 0 - else - log_success "Worker configuration" "Found variables in the configuration. Exporting..." fi - # Iterate over variables and export them into the main shell - while IFS="=" read -r key value; do - eval "export $key=\"$value\"" - done < <(echo "$variables" | jq -r 'to_entries[] | "\(.key)=\(.value)"') + # Generate environment file + if [[ -n "$env_vars" && "$env_vars" != "null" ]]; then + log_success "Worker configuration" "Found environment variables in the configuration." + generate_env_file + fi + + # Generate secrets file + if [[ -n "$secrets" && "$secrets" != "null" ]]; then + log_success "Worker configuration" "Found secrets in the configuration." + generate_secrets_file + fi + + # Load both environment and secrets + load_environment + load_secrets } # Function to extract a specific section from the JSON configuration diff --git a/src/tests/configs/services.yaml b/src/tests/configs/services.yaml deleted file mode 100644 index c197b511..00000000 --- a/src/tests/configs/services.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -kind: workerService -version: udx.io/worker-v1/service -services: - - name: "test_service" - command: "tail -f /dev/null" - autostart: true diff --git a/src/tests/configs/worker.yaml b/src/tests/configs/worker.yaml deleted file mode 100644 index 1f93e288..00000000 --- a/src/tests/configs/worker.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -kind: workerConfig -version: udx.io/worker-v1/config -config: - env: - AZURE_CLIENT_ID: "12345678-1234-1234-1234-1234567890ab" - AZURE_TENANT_ID: "abcdef12-3456-7890-abcd-ef1234567890" - AZURE_SUBSCRIPTION_ID: "1234abcd-5678-90ef-abcd-12345678abcd" - AZURE_RESOURCE_GROUP: "rg-example" - APIM_SERVICE_NAME: "example-apim" - ACR_REPO_NAME: "exampleacr" - STORAGE_ACCOUNT_NAME: "examplestorage" - KEY_VAULT_NAME: "examplekv" - MANAGED_IDENTITY_NAME: "exampleidentity" - - secrets: - TEST_SECRET: "azure/kv-udx-worker-tooling/test-secret" diff --git a/src/tests/env.sh b/src/tests/env.sh new file mode 100755 index 00000000..3d0a1dbd --- /dev/null +++ b/src/tests/env.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Test environment commands + +# Test environment show +echo "Testing: env show" +worker env show +if ! worker env show | grep "Environment variables"; then + echo "FAIL: env show should display environment variables" + exit 1 +fi + +# Test environment set/get +echo "Testing: env set/get" +worker env set TEST_VAR "test value" +if ! worker env show | grep "TEST_VAR=test value"; then + echo "FAIL: env set/get not working" + exit 1 +fi + +# Test environment JSON output +echo "Testing: env show --format json" +if ! worker env show --format json | jq -e '.variables'; then + echo "FAIL: JSON output not working" + exit 1 +fi + +# All tests passed +echo "Environment tests passed" diff --git a/src/tests/main.sh b/src/tests/main.sh deleted file mode 100755 index c207b879..00000000 --- a/src/tests/main.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Source utils.sh for logging functions -# shellcheck disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Main" "Running all test suites" - -# Find and execute all test scripts in the tasks directory -for test_script in ./tasks/*.sh; do - log_info "Running $(basename "$test_script")..." - if bash "$test_script"; then - log_success "$(basename "$test_script")" "Test completed successfully" - else - log_error "$(basename "$test_script")" "Test failed" - exit 1 - fi -done - -log_success "Main" "All test suites completed successfully" \ No newline at end of file diff --git a/src/tests/service.sh b/src/tests/service.sh new file mode 100755 index 00000000..dd063901 --- /dev/null +++ b/src/tests/service.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Test service commands + +# Test service list +echo "Testing: service list" +if ! worker service list | grep "Available services"; then + echo "FAIL: service list should show available services" + exit 1 +fi + +# Test service status +echo "Testing: service status" +if ! worker service status | grep "Service status"; then + echo "FAIL: service status not working" + exit 1 +fi + +# Test service JSON output +echo "Testing: service list --format json" +if ! worker service list --format json | jq -e '.services'; then + echo "FAIL: JSON output not working" + exit 1 +fi + +# All tests passed +echo "Service tests passed" diff --git a/src/tests/tasks/10_dependencies.sh b/src/tests/tasks/10_dependencies.sh deleted file mode 100755 index 3bcc99dc..00000000 --- a/src/tests/tasks/10_dependencies.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Dependencies: Validating required dependencies and their versions" - -# Function to test if a command is available and show its version -check_command() { - local cmd_path - cmd_path=$(command -v "$1") - if [ -x "$cmd_path" ]; then - log_success "$1" "$1 is installed at $cmd_path" - local version - version=$($1 --version 2>&1 | head -n 1) - log_success "$1" "Version: $version" - else - log_error "$1" "Not installed or not in PATH" - exit 1 - fi -} - -# Verify gcloud, aws, az, bw, yq, and jq commands are available -log_info "Checking cloud provider CLIs..." -check_command gcloud -check_command aws -check_command az - -log_info "Checking utility tools..." -check_command bw -check_command yq -check_command jq - -log_success "Dependencies" "All dependencies validated successfully" \ No newline at end of file diff --git a/src/tests/tasks/20_config.sh b/src/tests/tasks/20_config.sh deleted file mode 100644 index 8b80d78d..00000000 --- a/src/tests/tasks/20_config.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Starting validation of config..." - -# Path to the merged configuration file -MERGED_CONFIG="/usr/local/configs/worker/merged_worker.yaml" - -# Test configure_environment function -test_configure_environment() { - log_info "Running test: configure_environment" - - # Load the merged configuration - merged_config=$(cat "$MERGED_CONFIG") - - # Extract environment variables from the merged configuration - env_vars=$(echo "$merged_config" | yq eval '.config.env' -) - - # Verify environment variables - for key in $(echo "$env_vars" | yq eval 'keys' -); do - # Check if key is empty or not set - if [[ -z "$key" || "$key" == "null" || "$key" == "-" ]]; then - continue - fi - - value=$(echo "$env_vars" | yq eval ".${key}" -) - actual_value="${!key}" - - if [[ "$actual_value" != "$value" ]]; then - log_error "Config" "$key is not set correctly. Expected: $value, Got: $actual_value" - return 1 - else - log_success "Config" "$key is set correctly" - fi - done - - log_success "Config" "configure_environment test passed" - return 0 -} - -# Run the test -if test_configure_environment; then - log_success "Config" "All config tests passed successfully" -else - log_error "Config" "Config tests failed" - exit 1 -fi \ No newline at end of file diff --git a/src/tests/tasks/30_secrets.sh b/src/tests/tasks/30_secrets.sh deleted file mode 100644 index e0deae41..00000000 --- a/src/tests/tasks/30_secrets.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Starting validation of secrets fetching..." - -# Path to the merged configuration file -MERGED_CONFIG="/usr/local/configs/worker/merged_worker.yaml" - -# Test verify_secrets function -test_verify_secrets() { - log_info "Running test: verify_secrets" - - # Load the merged configuration - merged_config=$(cat "$MERGED_CONFIG") - - # Extract secrets from the merged configuration - secrets=$(echo "$merged_config" | yq eval '.config.secrets' -) - - # Check if secrets is empty or null - if [[ -z "$secrets" || "$secrets" == "null" ]]; then - log_info "No secrets found in the configuration." - return 0 - fi - - # Verify secrets as environment variables - for secret_key in $(echo "$secrets" | yq eval 'keys' -); do - - # Check if the key is valid - if [[ -z "$secret_key" || "$secret_key" == "-" ]]; then - continue - fi - - expected_reference=$(echo "$secrets" | yq eval ".${secret_key}" -) - actual_value="${!secret_key}" - - # Verify that the environment variable is set and different from the reference - if [[ -z "$actual_value" || "$actual_value" == "$expected_reference" ]]; then - log_error "Secrets" "$secret_key is not replaced correctly. Got: $actual_value" - return 1 - else - log_success "Secrets" "$secret_key is resolved correctly" - fi - done - - log_success "Secrets" "verify_secrets test passed" - return 0 -} - -# Run the test -if test_verify_secrets; then - log_success "Secrets" "All secrets fetching tests passed successfully" -else - log_error "Secrets" "Secrets fetching tests failed" - exit 1 -fi \ No newline at end of file diff --git a/src/tests/tasks/40_test_service.sh b/src/tests/tasks/40_test_service.sh deleted file mode 100644 index e06b638b..00000000 --- a/src/tests/tasks/40_test_service.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Test" "Starting service tests..." - -# Test 1: Check service configuration -log_info "Test" "Testing service configuration..." -if ! worker service config | grep -q "test_service"; then - log_error "Test" "Service configuration test failed: test_service not found in config" - exit 1 -fi -log_success "Test" "Service configuration test passed" - -# Test 2: Check service list and wait for it to be running -log_info "Test" "Testing service list and waiting for service to be ready..." - -# Wait for up to 30 seconds for the service to be fully running -for i in {1..30}; do - status_output=$(worker service list 2>&1) - if echo "$status_output" | grep "test_service" | grep -q "RUNNING"; then - log_success "Test" "Service is now running" - break - fi - if [ "$i" -eq 30 ]; then - log_error "Test" "Timeout waiting for service to start" - log_error "Test" "Current status: $status_output" - exit 1 - fi - sleep 1 -done -log_success "Test" "Service list test passed" - -# Test 3: Check service status -log_info "Test" "Testing service status..." -status_output=$(worker service status test_service) -if ! echo "$status_output" | grep -q "RUNNING"; then - log_error "Test" "Service status test failed: test_service not running" - log_error "Test" "Current status: $status_output" - exit 1 -fi -log_success "Test" "Service status test passed" - -# Test 4: Test service stop/start -log_info "Test" "Testing service stop..." -worker service stop test_service -sleep 2 -if worker service status test_service | grep -q "RUNNING"; then - log_error "Test" "Service stop test failed: test_service still running" - exit 1 -fi -log_success "Test" "Service stop test passed" - -log_info "Test" "Testing service start..." -worker service start test_service -sleep 2 -if ! worker service status test_service | grep -q "RUNNING"; then - log_error "Test" "Service start test failed: test_service not running" - exit 1 -fi -log_success "Test" "Service start test passed" - -# Test 5: Check service logs -log_info "Test" "Testing service logs..." - -# Wait a bit for the service to initialize after restart -sleep 2 - -# First check if log file exists -log_file="/var/log/supervisor/test_service.out.log" -if [ ! -f "$log_file" ]; then - log_error "Test" "Service logs test failed: log file not found at $log_file" - exit 1 -fi - -# Try a simple tail command first -if ! tail -n 1 "$log_file" > /dev/null 2>&1; then - log_error "Test" "Service logs test failed: cannot read log file" - exit 1 -fi - -# Now test the worker service logs command with --nostream -if ! worker service logs test_service --lines=1 --nostream > /dev/null 2>&1; then - log_error "Test" "Service logs test failed: worker service logs command failed" - exit 1 -fi - -log_success "Test" "Service logs test passed" - -log_success "Test" "All service tests completed successfully"