From 389f2c0785bbb478175856d7dd740db9163f4063 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Tue, 18 Feb 2025 19:00:19 +0300 Subject: [PATCH 01/25] update image --- Dockerfile | 119 +++++--- Makefile | 34 ++- bin/entrypoint.sh | 6 +- etc/configs/worker/services.yaml | 18 -- lib/auth.sh | 6 +- lib/auth/aws.sh | 4 +- lib/auth/azure.sh | 4 +- lib/auth/bitwarden.sh | 4 +- lib/auth/gcp.sh | 4 +- lib/cleanup.sh | 4 +- lib/cli.sh | 192 +++++++++++-- lib/cli/auth.sh | 135 +++++++++ lib/cli/config.sh | 203 ++++++++++++++ lib/cli/env.sh | 371 ++++++++++++++++++++++-- lib/cli/health.sh | 242 ++++++++++++++++ lib/cli/info.sh | 437 +++++++++++++++++++++++++++++ lib/cli/logs.sh | 271 ++++++++++++++++++ lib/cli/sbom.sh | 279 +++++++++++++++++- lib/cli/service.sh | 53 +++- lib/env_handler.sh | 150 ++++++++++ lib/environment.sh | 10 +- lib/process_manager.sh | 27 +- lib/secrets.sh | 8 +- lib/secrets/aws.sh | 4 +- lib/secrets/azure.sh | 4 +- lib/secrets/bitwarden.sh | 4 +- lib/secrets/gcp.sh | 4 +- lib/worker_config.sh | 47 ++-- src/tests/configs/services.yaml | 7 - src/tests/configs/worker.yaml | 17 -- src/tests/env.sh | 29 ++ src/tests/main.sh | 20 -- src/tests/service.sh | 27 ++ src/tests/tasks/10_dependencies.sh | 34 --- src/tests/tasks/20_config.sh | 49 ---- src/tests/tasks/30_secrets.sh | 57 ---- src/tests/tasks/40_test_service.sh | 91 ------ 37 files changed, 2508 insertions(+), 467 deletions(-) delete mode 100644 etc/configs/worker/services.yaml create mode 100644 lib/cli/auth.sh create mode 100644 lib/cli/config.sh create mode 100644 lib/cli/health.sh create mode 100644 lib/cli/info.sh create mode 100644 lib/cli/logs.sh create mode 100644 lib/env_handler.sh delete mode 100644 src/tests/configs/services.yaml delete mode 100644 src/tests/configs/worker.yaml create mode 100755 src/tests/env.sh delete mode 100755 src/tests/main.sh create mode 100755 src/tests/service.sh delete mode 100755 src/tests/tasks/10_dependencies.sh delete mode 100644 src/tests/tasks/20_config.sh delete mode 100644 src/tests/tasks/30_secrets.sh delete mode 100644 src/tests/tasks/40_test_service.sh diff --git a/Dockerfile b/Dockerfile index dd21a372..5061bc63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,27 @@ FROM ubuntu:25.04 # Set the maintainer of the image LABEL maintainer="UDX CAG Team" -# Set environment variables to avoid interactive prompts and set a fixed timezone +# Set base environment variables ENV DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC \ USER=udx \ UID=500 \ GID=500 \ - HOME=/home/udx + HOME=/home/udx \ + # Worker specific paths + WORKER_BASE_DIR=/opt/worker \ + WORKER_CONFIG_DIR=/etc/worker \ + WORKER_APP_DIR=/opt/worker/apps \ + WORKER_DATA_DIR=/opt/worker/data \ + WORKER_LIB_DIR=/usr/local/worker/lib \ + WORKER_BIN_DIR=/usr/local/worker/bin \ + WORKER_ETC_DIR=/usr/local/worker/etc \ + # Add worker bin to PATH + PATH=/usr/local/worker/bin:${PATH} \ + # Cloud SDK configurations + CLOUDSDK_CONFIG=/usr/local/configs/gcloud \ + AWS_CONFIG_FILE=/usr/local/configs/aws \ + AZURE_CONFIG_DIR=/usr/local/configs/azure # Set the shell with pipefail option SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -22,10 +36,10 @@ USER root # hadolint ignore=DL3015 RUN apt-get update && \ apt-get install -y \ - tzdata=2024b-6ubuntu1 \ + tzdata=2025a-2ubuntu1 \ curl=8.12.0+git20250209.89ed161+ds-1ubuntu1 \ bash=5.2.37-1ubuntu1 \ - apt-utils=2.9.28 \ + apt-utils=2.9.29 \ gettext=0.23.1-1 \ gnupg=2.4.4-2ubuntu22 \ ca-certificates=20241223 \ @@ -110,34 +124,73 @@ RUN groupadd -g ${GID} ${USER} && \ RUN mkdir -p /var/log/supervisor /var/run/supervisor && \ chown -R ${USER}:${USER} /var/log/supervisor /var/run/supervisor -# Copy the CLI tool into the image -COPY lib/cli.sh /usr/local/bin/worker_mgmt -RUN chmod +x /usr/local/bin/worker_mgmt && \ - ln -s /usr/local/bin/worker_mgmt /usr/local/bin/worker - -# Copy the bin, etc, and lib directories -COPY etc/configs /usr/local/configs -COPY lib /usr/local/lib -COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh - -# Set permissions during build -# Set ownership -RUN chown -R ${UID}:${GID} /usr/local/configs /usr/local/bin /usr/local/lib && \ - # Make specific scripts executable - chmod 755 /usr/local/bin/entrypoint.sh /usr/local/lib/process_manager.sh && \ - # Set read-only permissions for config files - find /usr/local/configs -type f -exec chmod 644 {} + && \ - # Set read-only permissions for library files - find /usr/local/lib -type f ! -name process_manager.sh -exec chmod 644 {} + && \ - # Ensure directories are accessible - find /usr/local/configs /usr/local/bin /usr/local/lib -type d -exec chmod 755 {} + - -# Create a symbolic link for the supervisord configuration file -RUN ln -sf /usr/local/configs/supervisor/supervisord.conf /etc/supervisord.conf - -# Prepare directories for the user and worker configuration -RUN mkdir -p ${HOME} && \ - chown -R ${USER}:${USER} ${HOME} +# Create directory structure +RUN mkdir -p \ + # Worker directories + ${WORKER_CONFIG_DIR} \ + ${WORKER_APP_DIR} \ + ${WORKER_DATA_DIR} \ + ${WORKER_LIB_DIR} \ + ${WORKER_BIN_DIR} \ + ${WORKER_ETC_DIR} \ + # Environment and secrets files directory + ${WORKER_CONFIG_DIR}/environment.d \ + # User and config directories + ${HOME}/.config/worker \ + # Cloud SDK config directories + ${CLOUDSDK_CONFIG} \ + ${AWS_CONFIG_FILE%/*} \ + ${AZURE_CONFIG_DIR} && \ + # Create and set permissions for environment files + touch ${WORKER_CONFIG_DIR}/environment ${WORKER_CONFIG_DIR}/secrets && \ + chown ${USER}:${USER} \ + ${WORKER_CONFIG_DIR}/environment \ + ${WORKER_CONFIG_DIR}/secrets && \ + chmod 644 ${WORKER_CONFIG_DIR}/environment && \ + chmod 600 ${WORKER_CONFIG_DIR}/secrets + +# Copy worker files +COPY bin/entrypoint.sh ${WORKER_BIN_DIR}/ +COPY lib ${WORKER_LIB_DIR}/ +COPY etc/configs/worker/default.yaml ${WORKER_CONFIG_DIR}/worker.yaml +COPY etc/configs/supervisor ${WORKER_CONFIG_DIR}/supervisor/ + +# Make scripts executable and initialize environment +RUN chmod +x ${WORKER_LIB_DIR}/*.sh && \ + ${WORKER_LIB_DIR}/env_handler.sh init_environment + +# Set up CLI tool +COPY lib/cli.sh ${WORKER_BIN_DIR}/worker_mgmt +RUN chmod +x ${WORKER_BIN_DIR}/worker_mgmt && \ + ln -s ${WORKER_BIN_DIR}/worker_mgmt ${WORKER_BIN_DIR}/worker + +# Set permissions +RUN \ + # Set base ownership + chown -R ${UID}:${GID} \ + ${WORKER_BASE_DIR} \ + ${WORKER_CONFIG_DIR} \ + ${WORKER_LIB_DIR} \ + ${WORKER_BIN_DIR} \ + ${HOME} \ + ${CLOUDSDK_CONFIG} \ + ${AWS_CONFIG_FILE%/*} \ + ${AZURE_CONFIG_DIR} && \ + # Set directory permissions + find ${WORKER_BASE_DIR} ${WORKER_CONFIG_DIR} ${WORKER_LIB_DIR} ${WORKER_BIN_DIR} -type d -exec chmod 755 {} + && \ + # Set file permissions + find ${WORKER_CONFIG_DIR} -type f -exec chmod 644 {} + && \ + find ${WORKER_LIB_DIR} -type f ! -name process_manager.sh -exec chmod 644 {} + && \ + # Make specific files executable + chmod 755 \ + ${WORKER_BIN_DIR}/entrypoint.sh \ + ${WORKER_BIN_DIR}/worker_mgmt \ + ${WORKER_LIB_DIR}/process_manager.sh && \ + # Set runtime directories permissions + chmod 775 ${WORKER_APP_DIR} ${WORKER_DATA_DIR} + +# Set up supervisor configuration +RUN ln -sf ${WORKER_CONFIG_DIR}/supervisor/supervisord.conf /etc/supervisord.conf # Switch to the user directory WORKDIR ${HOME} @@ -146,7 +199,7 @@ WORKDIR ${HOME} USER ${USER} # Set the entrypoint to run the entrypoint script using shell form -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +ENTRYPOINT ["/usr/local/worker/bin/entrypoint.sh"] # Set the default command CMD ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/Makefile b/Makefile index ddfe6d1c..1cc33b1f 100644 --- a/Makefile +++ b/Makefile @@ -28,15 +28,28 @@ build: printf "$(COLOR_BLUE)$(SYM_ARROW) Starting Docker build...$(COLOR_RESET)\n"; \ if [ "$(MULTIPLATFORM)" = "true" ]; then \ printf "$(COLOR_BLUE)$(SYM_ARROW) Building for multiple platforms: [linux/amd64, linux/arm64]$(COLOR_RESET)\n"; \ - docker buildx build --progress=plain \ - --platform linux/amd64,linux/arm64 \ - -t $(DOCKER_IMAGE) \ - --load . 2>&1 | grep -E "$$filter" || exit 1; \ + if [ "$(DEBUG)" = "true" ]; then \ + docker buildx build --progress=plain \ + --platform linux/amd64,linux/arm64 \ + -t $(DOCKER_IMAGE) \ + --load .; \ + else \ + docker buildx build --progress=plain \ + --platform linux/amd64,linux/arm64 \ + -t $(DOCKER_IMAGE) \ + --load . 2>&1 | grep -E "$$filter" || exit 1; \ + fi; \ else \ printf "$(COLOR_BLUE)$(SYM_ARROW) Building for local platform$(COLOR_RESET)\n"; \ - DOCKER_BUILDKIT=1 docker build \ - --progress=plain \ - -t $(DOCKER_IMAGE) . 2>&1 | grep -E "$$filter" || exit 1; \ + if [ "$(DEBUG)" = "true" ]; then \ + DOCKER_BUILDKIT=1 docker build \ + --progress=plain \ + -t $(DOCKER_IMAGE) .; \ + else \ + DOCKER_BUILDKIT=1 docker build \ + --progress=plain \ + -t $(DOCKER_IMAGE) . 2>&1 | grep -E "$$filter" || exit 1; \ + fi; \ fi && \ printf "$(COLOR_GREEN)$(SYM_SUCCESS) Docker image build completed$(COLOR_RESET)\n" || \ { printf "$(COLOR_RED)$(SYM_ERROR) Docker build failed$(COLOR_RESET)\n"; exit 1; }' @@ -88,10 +101,11 @@ clean: test: clean @printf "$(COLOR_BLUE)$(SYM_ARROW) Running tests...$(COLOR_RESET)\n" + @chmod +x src/tests/*.sh @$(MAKE) run \ - VOLUMES="$(TEST_WORKER_CONFIG):/home/$(USER)/worker.yaml:ro $(TEST_SERVICES_CONFIG):/home/$(USER)/services.yaml:ro $(TESTS_TASKS_DIR):/home/$(USER)/tasks:ro $(TESTS_MAIN_SCRIPT):/home/$(USER)/main.sh:ro" \ - COMMAND="/home/$(USER)/main.sh" || exit 1 - @$(MAKE) log FOLLOW_LOGS=true || exit 1 + INTERACTIVE=true \ + VOLUMES="$(PWD)/src/tests:/tests" \ + COMMAND="/bin/bash -c '/tests/env.sh && /tests/service.sh'" || exit 1 @$(MAKE) clean || exit 1 @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Tests completed successfully$(COLOR_RESET)\n" diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 16611f82..8e28b787 100644 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -1,16 +1,16 @@ #!/bin/bash # shellcheck disable=SC1091 -source /usr/local/lib/utils.sh +source ${WORKER_LIB_DIR}/utils.sh log_info "Welcome to UDX Worker Container. Initializing environment..." # shellcheck disable=SC1091 -source /usr/local/lib/environment.sh +source ${WORKER_LIB_DIR}/environment.sh # Start the process manager in the background log_info "Starting process manager..." -/usr/local/lib/process_manager.sh & +${WORKER_LIB_DIR}/process_manager.sh & # Main execution logic if [ "$#" -gt 0 ]; then diff --git a/etc/configs/worker/services.yaml b/etc/configs/worker/services.yaml deleted file mode 100644 index 82f0fa44..00000000 --- a/etc/configs/worker/services.yaml +++ /dev/null @@ -1,18 +0,0 @@ ---- -kind: workerService -version: udx.io/worker-v1/service -services: - - name: "app_service" - ignore: "true" - command: "sh ./main.sh" - autostart: "true" - autorestart: "false" - envs: - - "KEY1=value1" - - "KEY2=value2" - - name: "another_service" - ignore: "true" - command: "sh ./main.sh" - autostart: "true" - autorestart: "true" - envs: [] diff --git a/lib/auth.sh b/lib/auth.sh index 1ebc851c..fdb4fb59 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Array to track configured providers declare -a configured_providers=() @@ -61,7 +61,7 @@ authenticate_actors() { # Proceed only if creds are valid JSON if echo "$creds" | jq empty &>/dev/null; then log_info "Processing credentials for $provider" - auth_script="/usr/local/lib/auth/${provider}.sh" + auth_script="${WORKER_LIB_DIR}/auth/${provider}.sh" auth_function="${provider}_authenticate" if [[ -f "$auth_script" ]]; then diff --git a/lib/auth/aws.sh b/lib/auth/aws.sh index 05b41592..a3234d57 100644 --- a/lib/auth/aws.sh +++ b/lib/auth/aws.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Example usage of the function # aws_authenticate "/path/to/your/aws_creds.json" diff --git a/lib/auth/azure.sh b/lib/auth/azure.sh index 7286fd7b..180b2382 100644 --- a/lib/auth/azure.sh +++ b/lib/auth/azure.sh @@ -5,8 +5,8 @@ # Example usage of the function # azure_authenticate "/path/to/your/azure_creds.json" -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to authenticate Azure accounts azure_authenticate() { diff --git a/lib/auth/bitwarden.sh b/lib/auth/bitwarden.sh index 57dbd7d6..55867cda 100644 --- a/lib/auth/bitwarden.sh +++ b/lib/auth/bitwarden.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to authenticate Bitwarden using API key or master password # diff --git a/lib/auth/gcp.sh b/lib/auth/gcp.sh index 0299d3df..1d8e3b88 100644 --- a/lib/auth/gcp.sh +++ b/lib/auth/gcp.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to authenticate GCP service accounts # diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 9a716936..505a3928 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -2,10 +2,10 @@ # Include worker config utilities first # shellcheck source=/dev/null -source /usr/local/lib/worker_config.sh +source ${WORKER_LIB_DIR}/worker_config.sh # shellcheck source=/dev/null -source /usr/local/lib/utils.sh +source ${WORKER_LIB_DIR}/utils.sh # Generic function to clean up authentication for any provider cleanup_provider() { diff --git a/lib/cli.sh b/lib/cli.sh index c9152153..aeafd12b 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -1,30 +1,172 @@ #!/bin/bash +# Version information +VERSION="1.0.0" + +# Source worker configuration +# shellcheck source=${WORKER_LIB_DIR}/worker_config.sh disable=SC1091 +source "${WORKER_LIB_DIR}/worker_config.sh" + +# Load and export configuration +config=$(load_and_parse_config) +export_variables_from_config "$config" + # Dynamically source all command modules -for module in /usr/local/lib/cli/*.sh; do - # shellcheck disable=SC1090 - source "$module" +for module in "${WORKER_LIB_DIR}/cli/"*.sh; do + # shellcheck disable=SC1090 + source "$module" done -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -# CLI Interface -case $1 in - env) - shift - env_handler "$@" - ;; - sbom) - shift - sbom_handler "$@" - ;; - service) - shift - service_handler "$@" - ;; - *) - log_error "CLI" "Unknown command: $1" - exit 1 - ;; -esac \ No newline at end of file +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" + +# Print version information +show_version() { + log_info "Worker CLI version $VERSION" +} + +# Get available commands and their descriptions +get_available_commands() { + local commands={} + declare -A commands + + # Add built-in commands + commands["help"]="Show help for any command" + commands["version"]="Show version information" + + # Scan through CLI modules to find commands and their descriptions + for module in "${WORKER_LIB_DIR}/cli/"*.sh; do + if [ -f "$module" ]; then + local name=$(basename "$module" .sh) + local description="" + + # Extract description from help function + if grep -q "${name}_help()" "$module"; then + description=$(grep -A 5 "${name}_help()" "$module" | + grep -v "${name}_help()" | + grep -v "^{" | + grep -v "cat << EOF" | + head -n 1 | + sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + commands[$name]="$description" + fi + fi + done + + echo "$(declare -p commands)" +} + +# Print help information +show_help() { + local -A commands + eval "$(get_available_commands)" + + # Find the longest command name for proper padding + local max_length=0 + for cmd in "${!commands[@]}"; do + local len=${#cmd} + if ((len > max_length)); then + max_length=$len + fi + done + + # Add padding for alignment + max_length=$((max_length + 2)) + + cat << EOF +πŸš€ Welcome to UDX Worker Container! + +This container helps you run and manage cloud services and applications. +Here's how to get started: + +1. Configure Services: + - Create service config: ${HOME}/.config/worker/services.yaml + - View service commands: worker help service + - Check service status: worker service list + +2. Configure Environment: + - Set environment vars: /etc/worker/environment + - View current settings: worker env show + - Configure cloud auth: worker auth setup + +3. Monitor & Manage: + - View container status: worker info overview + - Check service logs: worker service logs + - View system health: worker health check + +Available Commands: +EOF + + # Sort commands alphabetically and display + local sorted_commands=($(echo "${!commands[@]}" | tr ' ' '\n' | sort)) + for cmd in "${sorted_commands[@]}"; do + printf " %-${max_length}s %s\n" "$cmd" "${commands[$cmd]}" + done + + cat << EOF + +Examples: + worker help service Learn about service configuration + worker env show View current environment + worker info system Check system health + worker auth setup Configure cloud provider auth + +Tip: Run 'worker help [command]' for detailed information about any command +EOF +} + +# Show command-specific help +show_command_help() { + local cmd=$1 + local help_function="${cmd}_help" + + # Check if the help function exists + if [[ $(type -t "$help_function") == function ]]; then + "$help_function" + else + log_error "CLI" "No help available for command: $cmd" + show_help + return 1 + fi +} + +# Main CLI interface +if [ -z "$1" ] || [ "$1" = "help" ]; then + if [ -z "$2" ]; then + show_help + log_info "Container is ready. Run with a command to start services." + else + show_command_help "$2" + fi + exit 0 +fi + +if [ "$1" = "version" ]; then + show_version + exit 0 +fi + +# Handle app and service commands +if [ "$1" = "app" ] || [ "$1" = "service" ]; then + log_info "Starting process manager..." + "${WORKER_LIB_DIR}/process_manager.sh" + pm_status=$? + if [ $pm_status -ne 0 ]; then + exit $pm_status + fi + shift + exec "$@" +fi + +# Check if the command exists by looking for its handler +command=$1 +handler_function="${command}_handler" + +if [[ $(type -t "$handler_function") == function ]]; then + shift + "$handler_function" "$@" +else + log_error "CLI" "Unknown command: $command" + echo "Run 'worker help' to see available commands." + exit 1 +fi \ No newline at end of file diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh new file mode 100644 index 00000000..303259c1 --- /dev/null +++ b/lib/cli/auth.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh +source ${WORKER_LIB_DIR}/auth.sh + +# Show help for auth command +auth_help() { + cat << EOF +Manage authentication and credentials + +Usage: worker auth [command] + +Available Commands: + status Show authentication status for all providers + test Test authentication for all or specific provider + refresh Refresh credentials + rotate Rotate credentials (if supported by provider) + +Examples: + worker auth status + worker auth test aws + worker auth refresh + worker auth rotate --provider aws +EOF +} + +# Show authentication status +show_auth_status() { + log_info "Auth" "Checking authentication status..." + + # Check each provider's status + for provider in aws gcp azure bitwarden; do + if is_provider_configured "$provider"; then + log_success "Auth" "$provider: Configured and authenticated" + else + log_warn "Auth" "$provider: Not configured" + fi + done +} + +# Test authentication +test_auth() { + local provider=$1 + + if [ -z "$provider" ]; then + # Test all configured providers + for p in aws gcp azure bitwarden; do + if is_provider_configured "$p"; then + test_provider_auth "$p" + fi + done + else + # Test specific provider + if is_provider_configured "$provider"; then + test_provider_auth "$provider" + else + log_error "Auth" "Provider $provider is not configured" + return 1 + fi + fi +} + +# Refresh credentials +refresh_auth() { + log_info "Auth" "Refreshing credentials..." + + # Re-run authentication for all configured providers + local actors_json + actors_json=$(get_config_section "$(load_and_parse_config)" "actors") + + if [ -n "$actors_json" ]; then + if authenticate_actors "$actors_json"; then + log_success "Auth" "Successfully refreshed credentials" + else + log_error "Auth" "Failed to refresh credentials" + return 1 + fi + else + log_warn "Auth" "No actors configured" + return 1 + fi +} + +# Rotate credentials +rotate_auth() { + local provider=$1 + + if [ -z "$provider" ]; then + log_error "Auth" "Provider is required for credential rotation" + return 1 + fi + + if ! is_provider_configured "$provider"; then + log_error "Auth" "Provider $provider is not configured" + return 1 + fi + + # Call provider-specific rotation function + if rotate_provider_credentials "$provider"; then + log_success "Auth" "Successfully rotated credentials for $provider" + else + log_error "Auth" "Failed to rotate credentials for $provider" + return 1 + fi +} + +# Handle auth commands +auth_handler() { + local cmd=$1 + shift + + case $cmd in + status) + show_auth_status + ;; + test) + test_auth "$1" + ;; + refresh) + refresh_auth + ;; + rotate) + rotate_auth "$1" + ;; + help) + auth_help + ;; + *) + log_error "Auth" "Unknown command: $cmd" + auth_help + exit 1 + ;; + esac +} diff --git a/lib/cli/config.sh b/lib/cli/config.sh new file mode 100644 index 00000000..f5e9cea4 --- /dev/null +++ b/lib/cli/config.sh @@ -0,0 +1,203 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh +source ${WORKER_LIB_DIR}/worker_config.sh + +# Show help for config command +config_help() { + cat << EOF +Manage worker configuration + +Usage: worker config [command] + +Available Commands: + show Show current configuration + edit Edit configuration in default editor + validate Validate configuration files + locations Show configuration file locations + init Initialize a new configuration file + diff Show differences between default and current config + +Examples: + worker config show + worker config show --format json + worker config validate + worker config edit + worker config locations + worker config init +EOF +} + +# Show current configuration +show_config() { + local format=${1:-yaml} + log_info "Config" "Current configuration:" + + local config + config=$(load_and_parse_config) + + case $format in + json) + echo "$config" | yq eval -o=json '.' + ;; + yaml) + echo "$config" + ;; + *) + log_error "Config" "Unknown format: $format" + return 1 + ;; + esac +} + +# Edit configuration +edit_config() { + local config_file="${HOME}/.config/worker/worker.yaml" + + # Create directory if it doesn't exist + mkdir -p "$(dirname "$config_file")" + + # Create file if it doesn't exist + if [ ! -f "$config_file" ]; then + cp "/etc/worker/worker.yaml" "$config_file" + fi + + # Use default editor or fallback to nano + ${EDITOR:-nano} "$config_file" + + # Validate after editing + if validate_config "$config_file"; then + log_success "Config" "Configuration updated successfully" + else + log_error "Config" "Configuration validation failed after editing" + return 1 + fi +} + +# Validate configuration +validate_config() { + local config_file=${1:-} + log_info "Config" "Validating configuration..." + + # If no file specified, validate all config files + if [ -z "$config_file" ]; then + local files=( + "/etc/worker/worker.yaml" + "${HOME}/.config/worker/worker.yaml" + "/etc/worker/supervisor/supervisord.conf" + ) + + local failed=0 + for file in "${files[@]}"; do + if [ -f "$file" ]; then + if ! validate_yaml "$file"; then + log_error "Config" "Validation failed for $file" + failed=1 + else + log_success "Config" "Validation passed for $file" + fi + fi + done + + return $failed + else + # Validate specific file + if validate_yaml "$config_file"; then + log_success "Config" "Configuration is valid" + return 0 + else + log_error "Config" "Configuration is invalid" + return 1 + fi + fi +} + +# Show configuration locations +show_locations() { + cat << EOF +Configuration Locations: + System config: /etc/worker/worker.yaml + User config: ${HOME}/.config/worker/worker.yaml + Supervisor config: /etc/worker/supervisor/supervisord.conf + Services config: ${HOME}/.config/worker/services.yaml +EOF +} + +# Initialize new configuration +init_config() { + local config_dir="${HOME}/.config/worker" + local config_file="$config_dir/worker.yaml" + + # Create directory if it doesn't exist + mkdir -p "$config_dir" + + # Don't overwrite existing config without confirmation + if [ -f "$config_file" ]; then + log_warn "Config" "Configuration file already exists at $config_file" + read -r -p "Do you want to overwrite it? [y/N] " response + if [[ ! "$response" =~ ^[Yy]$ ]]; then + log_info "Config" "Initialization cancelled" + return 0 + fi + fi + + # Copy default config + cp "/etc/worker/worker.yaml" "$config_file" + + if [ -f "$config_file" ]; then + log_success "Config" "Configuration initialized at $config_file" + else + log_error "Config" "Failed to initialize configuration" + return 1 + fi +} + +# Show differences between default and current config +show_diff() { + local default_config="/etc/worker/worker.yaml" + local user_config="${HOME}/.config/worker/worker.yaml" + + if [ ! -f "$user_config" ]; then + log_error "Config" "User configuration does not exist at $user_config" + return 1 + fi + + log_info "Config" "Differences between default and current configuration:" + diff -u "$default_config" "$user_config" || true +} + +# Handle config commands +config_handler() { + local cmd=$1 + shift + + case $cmd in + show) + show_config "$@" + ;; + edit) + edit_config + ;; + validate) + validate_config "$@" + ;; + locations) + show_locations + ;; + init) + init_config + ;; + diff) + show_diff + ;; + help) + config_help + ;; + *) + log_error "Config" "Unknown command: $cmd" + config_help + exit 1 + ;; + esac +} diff --git a/lib/cli/env.sh b/lib/cli/env.sh index b3b68bc5..84a689d3 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -1,45 +1,368 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh +source ${WORKER_LIB_DIR}/worker_config.sh -# Function to display current environment settings or a specific variable -get_environment() { - if [ $# -eq 0 ]; then - log_debug "Current Environment Settings:" "$(env)" +# Show help for env command +env_help() { + cat << EOF +Manage environment variables and secrets + +Usage: worker env [command] + +Available Commands: + show Show environment variables (excludes secrets) + set Set an environment variable + unset Unset an environment variable + reload Reload environment from config + status Show environment status + validate Validate environment variables + export Export environment to file + import Import environment from file + +Options: + --format Output format (text/json) + --filter Filter variables by prefix + --file File to export to or import from + --include-secrets Include secrets in output (masked) + +Examples: + worker env show + worker env show --format json + worker env show --filter AWS_* + worker env show --include-secrets + worker env set MY_VAR "my value" + worker env reload +EOF +} + +# Show environment variables +show_environment() { + local format=${1:-text} + local filter=$2 + local include_secrets=${3:-false} + + if [ "$include_secrets" == "true" ]; then + list_env_vars "$format" else - log_debug "$1" "${!1}" + # Only show non-secret environment variables + if [ -f "$WORKER_ENV_FILE" ]; then + case $format in + json) + { + echo "{" + if [ -n "$filter" ]; then + grep "^export $filter" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/ "\1": "\2",/' + else + grep "^export" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/ "\1": "\2",/' + fi + echo "}" + } | sed 's/,}/}/' | jq '.' + ;; + text) + if [ -n "$filter" ]; then + grep "^export $filter" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' + else + grep "^export" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' + fi + ;; + *) + log_error "Env" "Unknown format: $format" + return 1 + ;; + esac + else + log_error "Env" "Environment file not found" + return 1 + fi fi } -# Function to set a new environment variable +# Set environment variable set_environment() { - if [ $# -ne 2 ]; then - log_warn "CLI" "Usage: $0 env set " + local name=$1 + local value=$2 + + if [ -z "$name" ] || [ -z "$value" ]; then + log_error "Env" "Both variable name and value are required" + return 1 + fi + + # Validate variable name + if ! [[ $name =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then + log_error "Env" "Invalid variable name: $name" + return 1 + fi + + # Add to environment file + if [ -f "$WORKER_ENV_FILE" ]; then + # Remove existing declaration if any + sed -i "/^export $name=/d" "$WORKER_ENV_FILE" + # Add new declaration + echo "export $name=\"$value\"" >> "$WORKER_ENV_FILE" + # Export in current session + export "$name=$value" + log_success "Env" "Set $name to '$value'" + else + log_error "Env" "Environment file not found" + return 1 + fi +} + +# Unset environment variable +unset_environment() { + local name=$1 + + if [ -z "$name" ]; then + log_error "Env" "Variable name is required" + return 1 + fi + + if [ -n "${!name}" ]; then + unset "$name" + log_success "Env" "Unset $name" + else + log_warn "Env" "Variable $name is not set" + fi +} + +# Export environment variables +export_environment() { + local file=$1 + local filter=$2 + + if [ -z "$file" ]; then + log_error "Env" "Output file is required" + return 1 + fi + + # Export variables + if [ -n "$filter" ]; then + env | grep "^$filter" > "$file" + else + env > "$file" + fi + + log_success "Env" "Environment variables exported to $file" +} + +# Import environment variables +import_environment() { + local file=$1 + + if [ -z "$file" ]; then + log_error "Env" "Input file is required" + return 1 + fi + + if [ ! -f "$file" ]; then + log_error "Env" "File not found: $file" return 1 fi - export "$1=$2" - log_debug "CLI" "Set $1 to '$2'." + + # Import variables + while IFS='=' read -r key value; do + if [ -n "$key" ]; then + export "$key=$value" + fi + done < "$file" + + log_success "Env" "Environment variables imported from $file" +} + +# Validate environment variables +validate_environment() { + local config + config=$(load_and_parse_config) + + if [ -z "$config" ]; then + log_error "Env" "Failed to load configuration" + return 1 + fi + + # Extract required variables from config + local required_vars + required_vars=$(echo "$config" | yq eval '.config.env | keys' -) + + local failed=0 + while IFS= read -r var; do + if [ -z "${!var}" ]; then + log_error "Env" "Required variable $var is not set" + failed=1 + fi + done <<< "$required_vars" + + if [ $failed -eq 0 ]; then + log_success "Env" "All required environment variables are set" + else + return 1 + fi +} + +# Generate environment template +generate_template() { + local config + config=$(load_and_parse_config) + + if [ -z "$config" ]; then + log_error "Env" "Failed to load configuration" + return 1 + fi + + echo "# Worker Environment Variables" + echo "# Generated on $(date)" + echo + + # Extract variables from config + echo "$config" | yq eval '.config.env | to_entries | .[] | "# " + .key + "\n" + .key + "=\"" + .value + "\""' - +} + +# Reset environment +reset_environment() { + # Clear all non-system variables + local system_vars="HOME|USER|PATH|SHELL|TERM|LANG|PWD" + + # Get all variables except system ones + local vars_to_unset + vars_to_unset=$(env | grep -vE "^($system_vars)=") + + while IFS='=' read -r key _; do + if [ -n "$key" ]; then + unset "$key" + fi + done <<< "$vars_to_unset" + + # Reconfigure environment + if configure_environment; then + log_success "Env" "Environment reset to default state" + else + log_error "Env" "Failed to reset environment" + return 1 + fi +} + +# Parse command line arguments +parse_args() { + local args=() + while [[ $# -gt 0 ]]; do + case $1 in + --format) + format=$2 + shift 2 + ;; + --filter) + filter=$2 + shift 2 + ;; + --file) + file=$2 + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac + done + set -- "${args[@]}" +} + +# Show environment status +show_status() { + local format=${1:-text} + + local env_file_exists=false + local secrets_file_exists=false + local env_count=0 + local secrets_count=0 + + [ -f "$WORKER_ENV_FILE" ] && env_file_exists=true + [ -f "$WORKER_SECRETS_FILE" ] && secrets_file_exists=true + + if [ "$env_file_exists" = true ]; then + env_count=$(grep -c "^export" "$WORKER_ENV_FILE" || echo 0) + fi + + if [ "$secrets_file_exists" = true ]; then + secrets_count=$(grep -c "^export" "$WORKER_SECRETS_FILE" || echo 0) + fi + + case $format in + json) + { + echo "{" + echo " \"environment\": {" + echo " \"file\": \"$WORKER_ENV_FILE\"," + echo " \"exists\": $env_file_exists," + echo " \"variables\": $env_count" + echo " }," + echo " \"secrets\": {" + echo " \"file\": \"$WORKER_SECRETS_FILE\"," + echo " \"exists\": $secrets_file_exists," + echo " \"variables\": $secrets_count" + echo " }" + echo "}" + } | jq '.' + ;; + text) + echo "Environment Status:" + echo "------------------" + echo "Environment File: $WORKER_ENV_FILE" + echo " - Exists: $env_file_exists" + echo " - Variables: $env_count" + echo + echo "Secrets File: $WORKER_SECRETS_FILE" + echo " - Exists: $secrets_file_exists" + echo " - Variables: $secrets_count" + ;; + *) + log_error "Env" "Unknown format: $format" + return 1 + ;; + esac } # Handle environment commands env_handler() { - case $1 in - get) - shift - get_environment "$@" + local cmd=$1 + shift + + # Parse command line arguments + parse_args "$@" + + case $cmd in + show) + show_environment "$format" "$filter" "$include_secrets" ;; set) - shift - if [ $# -eq 2 ]; then - set_environment "$@" - else - log_warn "CLI" "Usage: $0 env set " - exit 1 - fi + set_environment "$1" "$2" + ;; + unset) + unset_environment "$1" + ;; + reload) + config=$(load_and_parse_config) + export_variables_from_config "$config" + ;; + status) + show_status "$format" + ;; + validate) + validate_environment + ;; + export) + export_environment "$file" "$filter" + ;; + import) + import_environment "$file" + ;; + help) + env_help ;; *) - log_warn "CLI" "Usage: $0 env {show [VARIABLE_NAME]|set }" + log_error "Env" "Unknown command: $cmd" + env_help exit 1 ;; esac diff --git a/lib/cli/health.sh b/lib/cli/health.sh new file mode 100644 index 00000000..0ecd4fb6 --- /dev/null +++ b/lib/cli/health.sh @@ -0,0 +1,242 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh + +# Show help for health command +health_help() { + cat << EOF +Check system health and run diagnostics + +Usage: worker health [command] + +Available Commands: + check Run health check + status Show current health status + diag Run diagnostics + report Generate health report + +Examples: + worker health check + worker health status + worker health diag + worker health report --format json +EOF +} + +# Run health check +check_health() { + log_info "Health" "Running health check..." + local failed=0 + + # Check system resources + check_system_resources || failed=1 + + # Check supervisor status + check_supervisor_status || failed=1 + + # Check service health + check_services_health || failed=1 + + # Check authentication status + check_auth_status || failed=1 + + if [ $failed -eq 0 ]; then + log_success "Health" "All health checks passed" + else + log_error "Health" "Some health checks failed" + return 1 + fi +} + +# Check system resources +check_system_resources() { + local failed=0 + + # Check disk space + local disk_usage + disk_usage=$(df -h / | awk 'NR==2 {print $5}' | tr -d '%') + if [ "$disk_usage" -gt 90 ]; then + log_error "Health" "Disk usage is critical: ${disk_usage}%" + failed=1 + else + log_success "Health" "Disk usage is normal: ${disk_usage}%" + fi + + # Check memory + local mem_usage + mem_usage=$(free | awk '/Mem:/ {printf("%.0f", $3/$2 * 100)}') + if [ "$mem_usage" -gt 90 ]; then + log_error "Health" "Memory usage is critical: ${mem_usage}%" + failed=1 + else + log_success "Health" "Memory usage is normal: ${mem_usage}%" + fi + + # Check load average + local load_avg + load_avg=$(uptime | awk -F'load average:' '{print $2}' | cut -d, -f1) + if [ "${load_avg%.*}" -gt 4 ]; then + log_error "Health" "Load average is high: $load_avg" + failed=1 + else + log_success "Health" "Load average is normal: $load_avg" + fi + + return $failed +} + +# Check supervisor status +check_supervisor_status() { + if ! pgrep -f supervisord > /dev/null; then + log_error "Health" "Supervisor is not running" + return 1 + fi + + if ! supervisorctl status > /dev/null; then + log_error "Health" "Supervisor is not responding" + return 1 + fi + + log_success "Health" "Supervisor is running and responsive" + return 0 +} + +# Check services health +check_services_health() { + local failed=0 + local services_status + + services_status=$(supervisorctl status) + if [ $? -ne 0 ]; then + log_error "Health" "Failed to get services status" + return 1 + fi + + echo "$services_status" | while read -r line; do + local service_name status + service_name=$(echo "$line" | awk '{print $1}') + status=$(echo "$line" | awk '{print $2}') + + if [ "$status" != "RUNNING" ]; then + log_error "Health" "Service $service_name is not running (status: $status)" + failed=1 + else + log_success "Health" "Service $service_name is running" + fi + done + + return $failed +} + +# Check authentication status +check_auth_status() { + local failed=0 + + # Check each provider + for provider in aws gcp azure bitwarden; do + if is_provider_configured "$provider"; then + if ! test_provider_auth "$provider"; then + log_error "Health" "Authentication failed for $provider" + failed=1 + else + log_success "Health" "Authentication successful for $provider" + fi + fi + done + + return $failed +} + +# Generate health report +generate_report() { + local format=${1:-text} + local report_file + report_file=$(mktemp) + + { + echo "Worker Health Report" + echo "===================" + echo "Timestamp: $(date)" + echo + + echo "System Resources:" + echo "----------------" + df -h / + echo + free -h + echo + uptime + echo + + echo "Services Status:" + echo "---------------" + supervisorctl status + echo + + echo "Authentication Status:" + echo "--------------------" + for provider in aws gcp azure bitwarden; do + if is_provider_configured "$provider"; then + echo "$provider: Configured" + else + echo "$provider: Not configured" + fi + done + } > "$report_file" + + case $format in + json) + # Convert report to JSON format + jq -R -s '{ + timestamp: now, + system_resources: { + disk: (input | match("^/dev.*$")), + memory: (input | match("^Mem:.*$")), + load: (input | match("^load average:.*$")) + }, + services: (input | match("^RUNNING.*$")), + auth: (input | match("^.*: Configured$")) + }' "$report_file" + ;; + text) + cat "$report_file" + ;; + *) + log_error "Health" "Unknown format: $format" + rm -f "$report_file" + return 1 + ;; + esac + + rm -f "$report_file" +} + +# Handle health commands +health_handler() { + local cmd=$1 + shift + + case $cmd in + check) + check_health + ;; + status) + check_health --quiet + ;; + diag) + check_health --verbose + ;; + report) + generate_report "$@" + ;; + help) + health_help + ;; + *) + log_error "Health" "Unknown command: $cmd" + health_help + exit 1 + ;; + esac +} diff --git a/lib/cli/info.sh b/lib/cli/info.sh new file mode 100644 index 00000000..e2836d41 --- /dev/null +++ b/lib/cli/info.sh @@ -0,0 +1,437 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" + +# Show help for info command +info_help() { + cat << EOF +Display information about the UDX Worker image and runtime + +Usage: worker info [command] + +Available Commands: + overview Show a high-level overview of the worker image + system Show system information and resource usage + config Show configuration and settings + services Show available services and their status + env Show environment variables and settings + auth Show authentication and credentials status + deps Show installed dependencies and versions + features Show available features and capabilities + paths Show important filesystem paths + logs Show logging configuration and locations + security Show security settings and policies + network Show network configuration and ports + version Show detailed version information + +Options: + --format Output format (text/json) + --verbose Show detailed information + +Examples: + worker info overview # Get a quick overview of the worker + worker info system --verbose # Show detailed system information + worker info deps --format json # List dependencies in JSON format + worker info features # Show available features +EOF +} + +# Show general information +show_info() { + local format=${1:-text} + + case $format in + json) + { + echo "{" + echo " \"version\": \"$VERSION\"," + echo " \"os\": \"$(uname -s)\"," + echo " \"architecture\": \"$(uname -m)\"," + echo " \"hostname\": \"$(hostname)\"," + echo " \"user\": \"$USER\"," + echo " \"paths\": {" + echo " \"base\": \"$WORKER_BASE_DIR\"," + echo " \"config\": \"$WORKER_CONFIG_DIR\"," + echo " \"apps\": \"$WORKER_APP_DIR\"," + echo " \"data\": \"$WORKER_DATA_DIR\"," + echo " \"lib\": \"$WORKER_LIB_DIR\"," + echo " \"bin\": \"$WORKER_BIN_DIR\"" + echo " }," + echo " \"cloud\": {" + echo " \"gcp_config\": \"$CLOUDSDK_CONFIG\"," + echo " \"aws_config\": \"$AWS_CONFIG_FILE\"," + echo " \"azure_config\": \"$AZURE_CONFIG_DIR\"" + echo " }" + echo "}" + } | jq '.' + ;; + text) + echo "UDX Worker Information" + echo "=====================" + echo + echo "Version: $VERSION" + echo + echo "System" + echo "------" + echo "OS: $(uname -s)" + echo "Architecture: $(uname -m)" + echo "Hostname: $(hostname)" + echo "User: $USER" + echo + echo "Paths" + echo "-----" + echo "Base Directory: $WORKER_BASE_DIR" + echo "Config Directory: $WORKER_CONFIG_DIR" + echo "Apps Directory: $WORKER_APP_DIR" + echo "Data Directory: $WORKER_DATA_DIR" + echo "Library Directory: $WORKER_LIB_DIR" + echo "Binary Directory: $WORKER_BIN_DIR" + echo + echo "Cloud Configuration" + echo "------------------" + echo "GCP Config: $CLOUDSDK_CONFIG" + echo "AWS Config: $AWS_CONFIG_FILE" + echo "Azure Config: $AZURE_CONFIG_DIR" + ;; + *) + log_error "Info" "Unknown format: $format" + return 1 + ;; + esac +} + +# Show system information +show_system_info() { + local format=${1:-text} + + case $format in + json) + { + echo "{" + echo " \"kernel\": \"$(uname -r)\"," + echo " \"os\": {" + echo " \"name\": \"$(uname -s)\"," + echo " \"version\": \"$(cat /etc/os-release | grep VERSION= | cut -d'\"' -f2)\"" + echo " }," + echo " \"cpu\": {" + echo " \"architecture\": \"$(uname -m)\"," + echo " \"cores\": $(nproc)," + echo " \"model\": \"$(grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2 | xargs)\"" + echo " }," + echo " \"memory\": {" + echo " \"total\": $(free -b | grep Mem | awk '{print $2}')," + echo " \"free\": $(free -b | grep Mem | awk '{print $4}')" + echo " }," + echo " \"disk\": {" + echo " \"total\": $(df -B1 / | tail -1 | awk '{print $2}')," + echo " \"free\": $(df -B1 / | tail -1 | awk '{print $4}')" + echo " }" + echo "}" + } | jq '.' + ;; + text) + echo "System Information" + echo "==================" + echo + echo "Kernel: $(uname -r)" + echo + echo "Operating System" + echo "----------------" + echo "Name: $(uname -s)" + echo "Version: $(cat /etc/os-release | grep VERSION= | cut -d'\"' -f2)" + echo + echo "CPU" + echo "---" + echo "Architecture: $(uname -m)" + echo "Cores: $(nproc)" + echo "Model: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2 | xargs)" + echo + echo "Memory" + echo "------" + echo "Total: $(free -h | grep Mem | awk '{print $2}')" + echo "Free: $(free -h | grep Mem | awk '{print $4}')" + echo + echo "Disk" + echo "----" + echo "Total: $(df -h / | tail -1 | awk '{print $2}')" + echo "Free: $(df -h / | tail -1 | awk '{print $4}')" + ;; + *) + log_error "Info" "Unknown format: $format" + return 1 + ;; + esac +} + +# Get available CLI modules and their help info +get_cli_modules() { + local modules_dir="${WORKER_LIB_DIR}/cli" + local modules={} + + # Initialize modules array + declare -A modules + + # Scan through CLI modules + for module in "$modules_dir"/*.sh; do + local name=$(basename "$module" .sh) + local description="" + local commands=() + + # Extract help function content + if grep -q "${name}_help()" "$module"; then + # Get description from help function + description=$(grep -A 1 "${name}_help()" "$module" | grep -v "${name}_help()" | grep -v "^{" | head -n 1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + # Get available commands + while IFS= read -r line; do + if [[ $line =~ ^[[:space:]]+([a-zA-Z0-9_-]+)[[:space:]]+(.+)$ ]]; then + commands+=("${BASH_REMATCH[1]}:${BASH_REMATCH[2]}") + fi + done < <(grep -A 20 "Available Commands:" "$module" | grep -B 20 "Examples:" | grep "^[[:space:]]*[a-zA-Z]") + fi + + modules[$name]="$description|${commands[*]}" + done + + echo "$(declare -p modules)" +} + +# Show overview of the worker +show_overview() { + local format=${1:-text} + local -A modules + eval "$(get_cli_modules)" + + case $format in + json) + { + echo "{" + echo " \"name\": \"UDX Worker\"," + echo " \"description\": \"Universal Data Exchange Worker for managing cloud services and data pipelines\"," + echo " \"version\": \"$VERSION\"," + echo " \"modules\": {" + + local first=true + for module in "${!modules[@]}"; do + IFS='|' read -r description commands <<< "${modules[$module]}" + + if [ "$first" = true ]; then + first=false + else + echo "," + fi + + echo " \"$module\": {" + echo " \"description\": \"$description\"," + echo " \"commands\": [" + + IFS=' ' read -ra cmd_array <<< "$commands" + local first_cmd=true + for cmd in "${cmd_array[@]}"; do + IFS=':' read -r cmd_name cmd_desc <<< "$cmd" + if [ "$first_cmd" = true ]; then + first_cmd=false + else + echo "," + fi + echo " {" + echo " \"name\": \"$cmd_name\"," + echo " \"description\": \"$cmd_desc\"" + echo -n " }" + done + echo "" + echo " ]" + echo -n " }" + done + echo "" + echo " }," + echo " \"dependencies\": {" + echo " \"runtime\": \"$(cat /etc/os-release | grep PRETTY_NAME | cut -d '"' -f 2)\"," + echo " \"python\": \"$(python3 --version | cut -d ' ' -f 2)\"," + echo " \"supervisor\": \"$(supervisord -v)\"" + echo " }" + echo "}" + } | jq '.' + ;; + text) + cat << EOF +UDX Worker Overview +================== + +Description: Universal Data Exchange Worker for managing cloud services and data pipelines +Version: ${VERSION} + +Available Modules +---------------- +EOF + for module in "${!modules[@]}"; do + IFS='|' read -r description commands <<< "${modules[$module]}" + echo -e "\nβ€’ $module - $description" + + if [ -n "$commands" ]; then + echo " Commands:" + IFS=' ' read -ra cmd_array <<< "$commands" + for cmd in "${cmd_array[@]}"; do + IFS=':' read -r cmd_name cmd_desc <<< "$cmd" + echo " βœ“ $cmd_name - $cmd_desc" + done + fi + done + + echo -e "\nSystem Information +------------------" + echo "β€’ Runtime: $(cat /etc/os-release | grep PRETTY_NAME | cut -d '"' -f 2)" + echo "β€’ Python: $(python3 --version | cut -d ' ' -f 2)" + echo "β€’ Supervisor: $(supervisord -v)" + + echo -e "\nFor more information:" + echo "β€’ Run 'worker help' for usage instructions" + echo "β€’ Run 'worker info ' for detailed module information" + echo "β€’ Run 'worker help' for module-specific help" + ;; + *) + log_error "Info" "Unknown format: $format" + return 1 + ;; + esac +} + +# Show available features +show_features() { + local format=${1:-text} + local -A modules + eval "$(get_cli_modules)" + + case $format in + json) + { + echo "{" + echo " \"features\": {" + + local first=true + for module in "${!modules[@]}"; do + IFS='|' read -r description commands <<< "${modules[$module]}" + + if [ "$first" = true ]; then + first=false + else + echo "," + fi + + # Convert module name to feature key + local feature_key=$(echo "$module" | tr '-' '_') + + echo " \"$feature_key\": {" + echo " \"description\": \"$description\"," + echo " \"commands\": [" + + IFS=' ' read -ra cmd_array <<< "$commands" + local first_cmd=true + for cmd in "${cmd_array[@]}"; do + IFS=':' read -r cmd_name cmd_desc <<< "$cmd" + if [ "$first_cmd" = true ]; then + first_cmd=false + else + echo "," + fi + echo " {" + echo " \"name\": \"$cmd_name\"," + echo " \"description\": \"$cmd_desc\"" + echo -n " }" + done + echo "" + echo " ]" + echo -n " }" + done + echo "" + echo " }" + echo "}" + } | jq '.' + ;; + text) + cat << EOF +UDX Worker Features +================== + +EOF + local count=1 + for module in "${!modules[@]}"; do + IFS='|' read -r description commands <<< "${modules[$module]}" + echo -e "\n$count. ${module^}" + echo " Description: $description" + + if [ -n "$commands" ]; then + echo " Commands:" + IFS=' ' read -ra cmd_array <<< "$commands" + for cmd in "${cmd_array[@]}"; do + IFS=':' read -r cmd_name cmd_desc <<< "$cmd" + echo " β€’ $cmd_name - $cmd_desc" + done + fi + ((count++)) + done + echo -e "\nUse 'worker help' for detailed feature documentation" + ;; + *) + log_error "Info" "Unknown format: $format" + return 1 + ;; + esac +} + +# Handle info commands +info_handler() { + local command=$1 + shift + + case $command in + overview) + show_overview "$@" + ;; + system) + show_system_info "$@" + ;; + features) + show_features "$@" + ;; + deps) + show_deps "$@" + ;; + config) + show_config "$@" + ;; + services) + show_services "$@" + ;; + env) + show_env "$@" + ;; + auth) + show_auth "$@" + ;; + paths) + show_paths "$@" + ;; + logs) + show_logs "$@" + ;; + security) + show_security "$@" + ;; + network) + show_network "$@" + ;; + version) + show_version "$@" + ;; + help|"") + info_help + ;; + *) + log_error "Info" "Unknown command: $command" + info_help + return 1 + ;; + esac +} diff --git a/lib/cli/logs.sh b/lib/cli/logs.sh new file mode 100644 index 00000000..03a92dbf --- /dev/null +++ b/lib/cli/logs.sh @@ -0,0 +1,271 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh + +# Show help for logs command +logs_help() { + cat << EOF +View and manage logs + +Usage: worker logs [command] + +Available Commands: + show Show logs for a service + follow Follow logs in real-time + search Search logs for pattern + export Export logs to file + clean Clean old logs + +Options: + --service Service name (required for show/follow) + --type Log type (out/err/all, default: all) + --lines Number of lines (default: 100) + --since Show logs since timestamp + --until Show logs until timestamp + --pattern Search pattern + --format Output format (text/json) + +Examples: + worker logs show --service myapp + worker logs follow --service myapp --type err + worker logs search --pattern "error" + worker logs export --service myapp --since "2024-01-01" + worker logs clean --older-than 7d +EOF +} + +# Get log file path +get_log_file() { + local service=$1 + local type=${2:-all} + + case $type in + out) + echo "/var/log/supervisor/${service}-stdout.log" + ;; + err) + echo "/var/log/supervisor/${service}-stderr.log" + ;; + all) + echo "/var/log/supervisor/${service}-*.log" + ;; + *) + log_error "Logs" "Invalid log type: $type" + return 1 + ;; + esac +} + +# Show logs +show_logs() { + local service=$1 + local type=${2:-all} + local lines=${3:-100} + local since=$4 + local until=$5 + + if [ -z "$service" ]; then + log_error "Logs" "Service name is required" + return 1 + fi + + local log_file + log_file=$(get_log_file "$service" "$type") + + if [ ! -f "$log_file" ]; then + log_error "Logs" "Log file not found: $log_file" + return 1 + fi + + local cmd="tail -n $lines" + + if [ -n "$since" ]; then + cmd="$cmd | awk -v since=\"\$since\" '\$0 >= since'" + fi + + if [ -n "$until" ]; then + cmd="$cmd | awk -v until=\"\$until\" '\$0 <= until'" + fi + + eval "$cmd $log_file" +} + +# Follow logs +follow_logs() { + local service=$1 + local type=${2:-all} + + if [ -z "$service" ]; then + log_error "Logs" "Service name is required" + return 1 + fi + + local log_file + log_file=$(get_log_file "$service" "$type") + + if [ ! -f "$log_file" ]; then + log_error "Logs" "Log file not found: $log_file" + return 1 + fi + + tail -f "$log_file" +} + +# Search logs +search_logs() { + local pattern=$1 + local service=$2 + local type=${3:-all} + + if [ -z "$pattern" ]; then + log_error "Logs" "Search pattern is required" + return 1 + fi + + local log_file + if [ -n "$service" ]; then + log_file=$(get_log_file "$service" "$type") + else + log_file="/var/log/supervisor/*.log" + fi + + grep -n "$pattern" $log_file +} + +# Export logs +export_logs() { + local service=$1 + local type=${2:-all} + local since=$3 + local until=$4 + local format=${5:-text} + + if [ -z "$service" ]; then + log_error "Logs" "Service name is required" + return 1 + fi + + local log_file + log_file=$(get_log_file "$service" "$type") + + if [ ! -f "$log_file" ]; then + log_error "Logs" "Log file not found: $log_file" + return 1 + fi + + local output_file="${service}_logs_$(date +%Y%m%d_%H%M%S)" + + case $format in + json) + output_file="$output_file.json" + awk '{printf "{\\"timestamp\\":\\"%s\\",\\"message\\":\\"%s\\"}\\n", $1, substr($0,index($0,$2))}' "$log_file" > "$output_file" + ;; + text) + output_file="$output_file.log" + if [ -n "$since" ] || [ -n "$until" ]; then + awk -v since="$since" -v until="$until" ' + ($0 >= since) && ($0 <= until || until=="") + ' "$log_file" > "$output_file" + else + cp "$log_file" "$output_file" + fi + ;; + *) + log_error "Logs" "Unknown format: $format" + return 1 + ;; + esac + + log_success "Logs" "Logs exported to $output_file" +} + +# Clean old logs +clean_logs() { + local older_than=${1:-7d} # Default: 7 days + + find /var/log/supervisor -name "*.log" -type f -mtime +"${older_than%d}" -delete + + log_success "Logs" "Cleaned logs older than $older_than" +} + +# Parse command line arguments +parse_args() { + local args=() + while [[ $# -gt 0 ]]; do + case $1 in + --service) + service=$2 + shift 2 + ;; + --type) + type=$2 + shift 2 + ;; + --lines) + lines=$2 + shift 2 + ;; + --since) + since=$2 + shift 2 + ;; + --until) + until=$2 + shift 2 + ;; + --pattern) + pattern=$2 + shift 2 + ;; + --format) + format=$2 + shift 2 + ;; + --older-than) + older_than=$2 + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac + done + set -- "${args[@]}" +} + +# Handle logs commands +logs_handler() { + local cmd=$1 + shift + + # Parse command line arguments + parse_args "$@" + + case $cmd in + show) + show_logs "$service" "$type" "$lines" "$since" "$until" + ;; + follow) + follow_logs "$service" "$type" + ;; + search) + search_logs "$pattern" "$service" "$type" + ;; + export) + export_logs "$service" "$type" "$since" "$until" "$format" + ;; + clean) + clean_logs "$older_than" + ;; + help) + logs_help + ;; + *) + log_error "Logs" "Unknown command: $cmd" + logs_help + exit 1 + ;; + esac +} diff --git a/lib/cli/sbom.sh b/lib/cli/sbom.sh index 3ba15d70..0d3eb57b 100644 --- a/lib/cli/sbom.sh +++ b/lib/cli/sbom.sh @@ -1,27 +1,278 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh -# Function to display dpkg packages in a table format using awk +# Show help for sbom command +sbom_help() { + cat << EOF +Manage Software Bill of Materials (SBOM) + +Usage: worker sbom [command] + +Available Commands: + generate Generate SBOM in various formats + analyze Analyze dependencies for security issues + export Export SBOM to file + deps Show dependency tree + verify Verify package integrity + updates Check for available updates + +Options: + --format Output format (text/json/cyclonedx/spdx) + --file Output file for export + --type Package type (system/python/all) + --filter Filter packages by name pattern + +Examples: + worker sbom generate + worker sbom generate --format json + worker sbom export --format cyclonedx --file sbom.xml + worker sbom deps --type python + worker sbom verify +EOF +} + +# Generate SBOM generate_sbom() { - log_debug "Package Name | Version | Architecture" - log_debug "----------------------------------------------------------" - dpkg-query -W -f='${binary:Package} | ${Version} | ${Architecture}\n' | awk -F'|' '{ - printf("%-30s | %-20s | %-10s\n", $1, $2, $3) - }' - log_debug "----------------------------------------------------------" + local format=${1:-text} + local type=${2:-all} + local filter=$3 + + log_info "SBOM" "Generating software bill of materials..." + + # Get system packages + local system_packages + if [[ $type == "all" || $type == "system" ]]; then + if [ -n "$filter" ]; then + system_packages=$(dpkg-query -W -f='${binary:Package}\t${Version}\t${Architecture}\t${Status}\n' | grep "$filter") + else + system_packages=$(dpkg-query -W -f='${binary:Package}\t${Version}\t${Architecture}\t${Status}\n') + fi + fi + + # Get Python packages + local python_packages + if [[ $type == "all" || $type == "python" ]]; then + if [ -n "$filter" ]; then + python_packages=$(pip list --format=json | jq -r '.[] | select(.name | contains("'"$filter"'")) | [.name, .version] | @tsv') + else + python_packages=$(pip list --format=json | jq -r '.[] | [.name, .version] | @tsv') + fi + fi + + case $format in + json) + { + echo "{" + if [ -n "$system_packages" ]; then + echo ' "system_packages": {' + echo "$system_packages" | awk -F'\t' '{ + printf(" \"%s\": {\n \"version\": \"%s\",\n \"architecture\": \"%s\"\n }%s\n", $1, $2, $3, (NR==1?",":"")) + }' + echo " }," + fi + if [ -n "$python_packages" ]; then + echo ' "python_packages": {' + echo "$python_packages" | awk -F'\t' '{ + printf(" \"%s\": {\n \"version\": \"%s\"\n }%s\n", $1, $2, (NR==1?",":"")) + }' + echo " }" + fi + echo "}" + } | jq '.' + ;; + cyclonedx) + # Generate CycloneDX XML format + { + echo '' + echo '' + echo ' ' + if [ -n "$system_packages" ]; then + echo "$system_packages" | awk -F'\t' '{ + printf(" \n %s\n %s\n pkg:deb/%s@%s?arch=%s\n \n", $1, $2, $1, $2, $3) + }' + fi + if [ -n "$python_packages" ]; then + echo "$python_packages" | awk -F'\t' '{ + printf(" \n %s\n %s\n pkg:pypi/%s@%s\n \n", $1, $2, $1, $2) + }' + fi + echo ' ' + echo '' + } + ;; + text) + { + echo "Software Bill of Materials" + echo "Generated on $(date)" + echo + if [ -n "$system_packages" ]; then + echo "System Packages:" + echo "---------------" + echo "Package Name | Version | Architecture" + echo "----------------------------------------------------------" + echo "$system_packages" | awk -F'\t' '{ + printf("%-20s | %-16s | %-12s\n", $1, $2, $3) + }' + echo + fi + if [ -n "$python_packages" ]; then + echo "Python Packages:" + echo "---------------" + echo "Package Name | Version" + echo "----------------------------------" + echo "$python_packages" | awk -F'\t' '{ + printf("%-20s | %-16s\n", $1, $2) + }' + fi + } + ;; + *) + log_error "SBOM" "Unknown format: $format" + return 1 + ;; + esac +} + +# Analyze dependencies +analyze_deps() { + log_info "SBOM" "Analyzing dependencies for security issues..." + + # Check system packages + if command -v apt-get >/dev/null; then + apt-get update -qq + apt-get --simulate upgrade | grep -i security + fi + + # Check Python packages + if command -v pip-audit >/dev/null; then + pip-audit + else + log_warn "SBOM" "pip-audit not installed. Install with: pip install pip-audit" + fi +} + +# Export SBOM +export_sbom() { + local format=$1 + local file=$2 + + if [ -z "$file" ]; then + log_error "SBOM" "Output file is required" + return 1 + fi + + generate_sbom "$format" > "$file" + log_success "SBOM" "SBOM exported to $file" +} + +# Show dependency tree +show_deps() { + local type=${1:-all} + + if [[ $type == "all" || $type == "python" ]]; then + log_info "SBOM" "Python dependencies:" + pip install pipdeptree >/dev/null 2>&1 + pipdeptree + fi +} + +# Verify package integrity +verify_packages() { + log_info "SBOM" "Verifying package integrity..." + + # Verify system packages + local failed=0 + while IFS= read -r pkg; do + if ! dpkg -V "$pkg" >/dev/null 2>&1; then + log_error "SBOM" "Package integrity check failed: $pkg" + failed=1 + fi + done < <(dpkg-query -W -f='${binary:Package}\n') + + if [ $failed -eq 0 ]; then + log_success "SBOM" "All packages verified successfully" + else + return 1 + fi +} + +# Check for updates +check_updates() { + log_info "SBOM" "Checking for available updates..." + + # Check system packages + apt-get update -qq + apt-get --just-print upgrade 2>&1 | awk '/^Inst/ { print $2 " (" $3 " => " $4 ")" }' + + # Check Python packages + pip list --outdated --format=json | jq -r '.[] | "\(.name) (\(.version) => \(.latest_version))"' } -# Handler for the sbom command +# Parse command line arguments +parse_args() { + local args=() + while [[ $# -gt 0 ]]; do + case $1 in + --format) + format=$2 + shift 2 + ;; + --file) + file=$2 + shift 2 + ;; + --type) + type=$2 + shift 2 + ;; + --filter) + filter=$2 + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac + done + set -- "${args[@]}" +} + +# Handle sbom commands sbom_handler() { - case $1 in + local cmd=$1 + shift + + # Parse command line arguments + parse_args "$@" + + case $cmd in generate) - shift - generate_sbom "$@" + generate_sbom "$format" "$type" "$filter" + ;; + analyze) + analyze_deps + ;; + export) + export_sbom "$format" "$file" + ;; + deps) + show_deps "$type" + ;; + verify) + verify_packages + ;; + updates) + check_updates + ;; + help) + sbom_help ;; *) - log_warn "CLI" "Usage: $0 sbom {generate}" + log_error "SBOM" "Unknown command: $cmd" + sbom_help exit 1 ;; esac diff --git a/lib/cli/service.sh b/lib/cli/service.sh index ed764f1e..4fb32932 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh service_handler() { local cmd=$1 @@ -33,6 +33,45 @@ service_handler() { esac } +# Show help for service command +service_help() { + cat << EOF +Manage UDX Worker services and applications + +Usage: worker service [command] [options] + +Commands: + list List all configured services and their status + status [name] Show status of all services or a specific service + logs [name] View logs for all services or a specific service + errors [name] View error logs for all services or a specific service + config Show current service configuration + start [name] Start a service + stop [name] Stop a service + restart [name] Restart a service + +Configuration: + Services are configured in: ${HOME}/.config/worker/services.yaml + +Example service configuration: + version: "1.0" + services: + my-app: + name: "my-app" + command: "python app.py" + working_dir: "/opt/worker/apps" + autostart: true + autorestart: true + environment: + APP_PORT: "8080" + +Examples: + worker service list # List all services + worker service logs my-app # View logs for my-app + worker service start my-app # Start my-app service +EOF +} + # Function to list all services list_services() { # Capture the output of supervisorctl status @@ -41,14 +80,16 @@ list_services() { # Check if Supervisor is not running, not accessible, or if there are no managed services if [[ -z "$services_status" ]] || echo "$services_status" | grep -Eq 'no such|ERROR'; then - log_warn "Service" "No services are currently managed." - exit 1 + log_info "No services are currently managed." + log_info "To configure services, create ${HOME}/.config/worker/services.yaml" + log_info "Run 'worker help service' for configuration examples." + return 0 fi - log_debug "Service" "Listing all managed services:" + log_info "Managed services:" local i=1 echo "$services_status" | while read -r line; do - log_debug "Service" "$i. $line" + log_info "$i. $line" ((i++)) done } diff --git a/lib/env_handler.sh b/lib/env_handler.sh new file mode 100644 index 00000000..f253cc99 --- /dev/null +++ b/lib/env_handler.sh @@ -0,0 +1,150 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" + +# Environment file location +WORKER_ENV_FILE="/etc/worker/environment" +WORKER_SECRETS_FILE="/etc/worker/secrets" + +# Generate environment file from worker.yaml +generate_env_file() { + local config + config=$(load_and_parse_config) + + if [ -z "$config" ]; then + log_error "Environment" "Failed to load configuration" + return 1 + fi + + # Extract and evaluate environment variables + { + echo "# Worker Environment Variables" + echo "# Generated on $(date)" + echo "# DO NOT EDIT THIS FILE DIRECTLY" + echo + + # Extract environment variables + echo "$config" | yq eval '.config.env | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' - + + # Add computed variables + echo + echo "# Computed variables" + echo "export WORKER_ENV_FILE=\"$WORKER_ENV_FILE\"" + echo "export WORKER_SECRETS_FILE=\"$WORKER_SECRETS_FILE\"" + } > "$WORKER_ENV_FILE" + + # File permissions are set during build +} + +# Generate secrets file from worker.yaml +generate_secrets_file() { + local config + config=$(load_and_parse_config) + + if [ -z "$config" ]; then + log_error "Environment" "Failed to load configuration" + return 1 + fi + + # Extract and evaluate secrets + { + echo "# Worker Secrets" + echo "# Generated on $(date)" + echo "# DO NOT EDIT THIS FILE DIRECTLY" + echo "# DO NOT COMMIT THIS FILE" + echo + + # Extract secrets + echo "$config" | yq eval '.config.secrets | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' - + } > "$WORKER_SECRETS_FILE" + + # File permissions are set during build +} + +# Load environment variables +load_environment() { + if [ -f "$WORKER_ENV_FILE" ]; then + # shellcheck source=/dev/null + source "$WORKER_ENV_FILE" + else + log_warn "Environment" "Environment file not found, generating..." + generate_env_file + # shellcheck source=/dev/null + source "$WORKER_ENV_FILE" + fi +} + +# Load secrets +load_secrets() { + if [ -f "$WORKER_SECRETS_FILE" ]; then + # shellcheck source=/dev/null + source "$WORKER_SECRETS_FILE" + else + log_warn "Environment" "Secrets file not found, generating..." + generate_secrets_file + # shellcheck source=/dev/null + source "$WORKER_SECRETS_FILE" + fi +} + +# Initialize environment +init_environment() { + generate_env_file + generate_secrets_file + load_environment + load_secrets +} + +# Update environment when config changes +update_environment() { + generate_env_file + generate_secrets_file + load_environment + load_secrets +} + +# Get environment variable value +get_env_value() { + local var_name="$1" + + if [ -z "$var_name" ]; then + log_error "Environment" "Variable name not provided" + return 1 + fi + + if [ -f "$WORKER_ENV_FILE" ]; then + grep "^export $var_name=" "$WORKER_ENV_FILE" | cut -d'=' -f2- | tr -d '"' + elif [ -f "$WORKER_SECRETS_FILE" ]; then + grep "^export $var_name=" "$WORKER_SECRETS_FILE" | cut -d'=' -f2- | tr -d '"' + else + log_error "Environment" "Neither environment nor secrets file exists" + return 1 + fi +} + +# List all environment variables +list_env_vars() { + local show_secrets="$1" + local env_vars="" + local secret_vars="" + + if [ -f "$WORKER_ENV_FILE" ]; then + env_vars=$(grep "^export" "$WORKER_ENV_FILE" | cut -d'=' -f1 | cut -d' ' -f2) + fi + + if [ "$show_secrets" = "true" ] && [ -f "$WORKER_SECRETS_FILE" ]; then + secret_vars=$(grep "^export" "$WORKER_SECRETS_FILE" | cut -d'=' -f1 | cut -d' ' -f2) + fi + + if [ -n "$env_vars" ]; then + echo "Environment Variables:" + echo "$env_vars" + fi + + if [ -n "$secret_vars" ]; then + echo + echo "Secret Variables:" + echo "$secret_vars" + fi +} \ No newline at end of file diff --git a/lib/environment.sh b/lib/environment.sh index 2c436af3..02595dac 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -2,16 +2,16 @@ # Include necessary modules # shellcheck disable=SC1091 -source /usr/local/lib/auth.sh +source "${WORKER_LIB_DIR}/auth.sh" # shellcheck disable=SC1091 -source /usr/local/lib/secrets.sh +source "${WORKER_LIB_DIR}/secrets.sh" # shellcheck disable=SC1091 -source /usr/local/lib/cleanup.sh +source "${WORKER_LIB_DIR}/cleanup.sh" # shellcheck disable=SC1091 -source /usr/local/lib/worker_config.sh +source "${WORKER_LIB_DIR}/worker_config.sh" # shellcheck disable=SC1091 -source /usr/local/lib/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Main function to coordinate environment setup configure_environment() { diff --git a/lib/process_manager.sh b/lib/process_manager.sh index 4c13fa28..4a1b19ba 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -1,19 +1,23 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" # Define paths -DEFAULT_CONFIG_FILE="/usr/local/configs/worker/services.yaml" -# Define the user-specific configuration path search -# shellcheck disable=SC2227 -USER_CONFIG_PATH=$(find "$HOME" -name 'services.yaml' 2>/dev/null -print | head -n 1) +USER_CONFIG_PATH="${HOME}/.config/worker/services.yaml" +CONFIG_FILE="${USER_CONFIG_PATH}" -# Use the first user-specific config found; if none, use the default -CONFIG_FILE="${USER_CONFIG_PATH:-$DEFAULT_CONFIG_FILE}" -COMMON_TEMPLATE_FILE="/usr/local/configs/supervisor/common.conf" -PROGRAM_TEMPLATE_FILE="/usr/local/configs/supervisor/program.conf" -FINAL_CONFIG="/usr/local/configs/supervisor/supervisord.conf" +# Check if user config exists +if [[ ! -f "${USER_CONFIG_PATH}" ]]; then + log_info "No services configuration found at ${USER_CONFIG_PATH}. Services will not be started." + log_info "Run 'worker help service' for information about service configuration" + exit 0 +fi + +# Supervisor configuration paths +COMMON_TEMPLATE_FILE="${WORKER_CONFIG_DIR}/supervisor/common.conf" +PROGRAM_TEMPLATE_FILE="${WORKER_CONFIG_DIR}/supervisor/program.conf" +FINAL_CONFIG="${WORKER_CONFIG_DIR}/supervisor/supervisord.conf" # Set up signal handling trap 'handle_supervisor_signals SIGTERM' SIGTERM @@ -25,6 +29,7 @@ main() { if ! configure_and_execute_services; then log_error "Process Manager" "Failed to configure and start services" + log_info "Run 'worker help service' for information about service configuration" exit 1 fi diff --git a/lib/secrets.sh b/lib/secrets.sh index 04153cb5..4a40d428 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -1,15 +1,15 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Dynamically source the required provider-specific modules source_provider_module() { local provider="$1" - local module_path="/usr/local/lib/secrets/${provider}.sh" + local module_path="${WORKER_LIB_DIR}/secrets/${provider}.sh" if [[ -f "$module_path" ]]; then - # shellcheck source=/usr/local/lib/secrets/${provider}.sh disable=SC1091 + # shellcheck source=${WORKER_LIB_DIR}/secrets/${provider}.sh disable=SC1091 source "$module_path" log_info "Loaded module for provider: $provider" else diff --git a/lib/secrets/aws.sh b/lib/secrets/aws.sh index 3a34d1d7..67ee643e 100644 --- a/lib/secrets/aws.sh +++ b/lib/secrets/aws.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to resolve AWS secret resolve_aws_secret() { diff --git a/lib/secrets/azure.sh b/lib/secrets/azure.sh index 4289d357..80c515bd 100644 --- a/lib/secrets/azure.sh +++ b/lib/secrets/azure.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to resolve Azure secret resolve_azure_secret() { diff --git a/lib/secrets/bitwarden.sh b/lib/secrets/bitwarden.sh index 6075aaf0..962d0689 100644 --- a/lib/secrets/bitwarden.sh +++ b/lib/secrets/bitwarden.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to resolve Bitwarden secret resolve_bitwarden_secret() { diff --git a/lib/secrets/gcp.sh b/lib/secrets/gcp.sh index 2edff3d2..09bd7c67 100644 --- a/lib/secrets/gcp.sh +++ b/lib/secrets/gcp.sh @@ -1,7 +1,7 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source ${WORKER_LIB_DIR}/utils.sh # Function to resolve GCP secret resolve_gcp_secret() { diff --git a/lib/worker_config.sh b/lib/worker_config.sh index fdfed67c..9d65ca69 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -1,14 +1,14 @@ #!/bin/bash -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" +# shellcheck source=${WORKER_LIB_DIR}/env_handler.sh disable=SC1091 +source "${WORKER_LIB_DIR}/env_handler.sh" # Paths for configurations -BUILT_IN_CONFIG="/usr/local/configs/worker/default.yaml" -# Dynamically find user configuration in any subfolder of $HOME -# shellcheck disable=SC2227 -USER_CONFIG=$(find "$HOME" -name 'worker.yaml' 2>/dev/null -print | head -n 1) -MERGED_CONFIG="/usr/local/configs/worker/merged_worker.yaml" +BUILT_IN_CONFIG="${WORKER_CONFIG_DIR}/worker.yaml" # Built-in default config +USER_CONFIG="${HOME}/.config/worker/worker.yaml" # Optional user config +MERGED_CONFIG="${WORKER_CONFIG_DIR}/worker.merged.yaml" # Result of merging both configs # Ensure `yq` is available if ! command -v yq >/dev/null 2>&1; then @@ -72,20 +72,31 @@ load_and_parse_config() { export_variables_from_config() { local config_json="$1" - # Extract the `variables` section - local variables - variables=$(echo "$config_json" | jq -r '.config.env // empty') - if [[ -z "$variables" || "$variables" == "null" ]]; then - log_info "No variables found in the configuration." + # Extract environment variables and secrets + local env_vars secrets + env_vars=$(echo "$config_json" | jq -r '.config.env // empty') + secrets=$(echo "$config_json" | jq -r '.config.secrets // empty') + + if [[ -z "$env_vars" && -z "$secrets" ]]; then + log_info "No variables or secrets found in the configuration." return 0 - else - log_success "Worker configuration" "Found variables in the configuration. Exporting..." fi - # Iterate over variables and export them into the main shell - while IFS="=" read -r key value; do - eval "export $key=\"$value\"" - done < <(echo "$variables" | jq -r 'to_entries[] | "\(.key)=\(.value)"') + # Generate environment file + if [[ -n "$env_vars" && "$env_vars" != "null" ]]; then + log_success "Worker configuration" "Found environment variables in the configuration." + generate_env_file + fi + + # Generate secrets file + if [[ -n "$secrets" && "$secrets" != "null" ]]; then + log_success "Worker configuration" "Found secrets in the configuration." + generate_secrets_file + fi + + # Load both environment and secrets + load_environment + load_secrets } # Function to extract a specific section from the JSON configuration diff --git a/src/tests/configs/services.yaml b/src/tests/configs/services.yaml deleted file mode 100644 index c197b511..00000000 --- a/src/tests/configs/services.yaml +++ /dev/null @@ -1,7 +0,0 @@ ---- -kind: workerService -version: udx.io/worker-v1/service -services: - - name: "test_service" - command: "tail -f /dev/null" - autostart: true diff --git a/src/tests/configs/worker.yaml b/src/tests/configs/worker.yaml deleted file mode 100644 index 1f93e288..00000000 --- a/src/tests/configs/worker.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -kind: workerConfig -version: udx.io/worker-v1/config -config: - env: - AZURE_CLIENT_ID: "12345678-1234-1234-1234-1234567890ab" - AZURE_TENANT_ID: "abcdef12-3456-7890-abcd-ef1234567890" - AZURE_SUBSCRIPTION_ID: "1234abcd-5678-90ef-abcd-12345678abcd" - AZURE_RESOURCE_GROUP: "rg-example" - APIM_SERVICE_NAME: "example-apim" - ACR_REPO_NAME: "exampleacr" - STORAGE_ACCOUNT_NAME: "examplestorage" - KEY_VAULT_NAME: "examplekv" - MANAGED_IDENTITY_NAME: "exampleidentity" - - secrets: - TEST_SECRET: "azure/kv-udx-worker-tooling/test-secret" diff --git a/src/tests/env.sh b/src/tests/env.sh new file mode 100755 index 00000000..3d0a1dbd --- /dev/null +++ b/src/tests/env.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Test environment commands + +# Test environment show +echo "Testing: env show" +worker env show +if ! worker env show | grep "Environment variables"; then + echo "FAIL: env show should display environment variables" + exit 1 +fi + +# Test environment set/get +echo "Testing: env set/get" +worker env set TEST_VAR "test value" +if ! worker env show | grep "TEST_VAR=test value"; then + echo "FAIL: env set/get not working" + exit 1 +fi + +# Test environment JSON output +echo "Testing: env show --format json" +if ! worker env show --format json | jq -e '.variables'; then + echo "FAIL: JSON output not working" + exit 1 +fi + +# All tests passed +echo "Environment tests passed" diff --git a/src/tests/main.sh b/src/tests/main.sh deleted file mode 100755 index c207b879..00000000 --- a/src/tests/main.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Source utils.sh for logging functions -# shellcheck disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Main" "Running all test suites" - -# Find and execute all test scripts in the tasks directory -for test_script in ./tasks/*.sh; do - log_info "Running $(basename "$test_script")..." - if bash "$test_script"; then - log_success "$(basename "$test_script")" "Test completed successfully" - else - log_error "$(basename "$test_script")" "Test failed" - exit 1 - fi -done - -log_success "Main" "All test suites completed successfully" \ No newline at end of file diff --git a/src/tests/service.sh b/src/tests/service.sh new file mode 100755 index 00000000..dd063901 --- /dev/null +++ b/src/tests/service.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Test service commands + +# Test service list +echo "Testing: service list" +if ! worker service list | grep "Available services"; then + echo "FAIL: service list should show available services" + exit 1 +fi + +# Test service status +echo "Testing: service status" +if ! worker service status | grep "Service status"; then + echo "FAIL: service status not working" + exit 1 +fi + +# Test service JSON output +echo "Testing: service list --format json" +if ! worker service list --format json | jq -e '.services'; then + echo "FAIL: JSON output not working" + exit 1 +fi + +# All tests passed +echo "Service tests passed" diff --git a/src/tests/tasks/10_dependencies.sh b/src/tests/tasks/10_dependencies.sh deleted file mode 100755 index 3bcc99dc..00000000 --- a/src/tests/tasks/10_dependencies.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Dependencies: Validating required dependencies and their versions" - -# Function to test if a command is available and show its version -check_command() { - local cmd_path - cmd_path=$(command -v "$1") - if [ -x "$cmd_path" ]; then - log_success "$1" "$1 is installed at $cmd_path" - local version - version=$($1 --version 2>&1 | head -n 1) - log_success "$1" "Version: $version" - else - log_error "$1" "Not installed or not in PATH" - exit 1 - fi -} - -# Verify gcloud, aws, az, bw, yq, and jq commands are available -log_info "Checking cloud provider CLIs..." -check_command gcloud -check_command aws -check_command az - -log_info "Checking utility tools..." -check_command bw -check_command yq -check_command jq - -log_success "Dependencies" "All dependencies validated successfully" \ No newline at end of file diff --git a/src/tests/tasks/20_config.sh b/src/tests/tasks/20_config.sh deleted file mode 100644 index 8b80d78d..00000000 --- a/src/tests/tasks/20_config.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Starting validation of config..." - -# Path to the merged configuration file -MERGED_CONFIG="/usr/local/configs/worker/merged_worker.yaml" - -# Test configure_environment function -test_configure_environment() { - log_info "Running test: configure_environment" - - # Load the merged configuration - merged_config=$(cat "$MERGED_CONFIG") - - # Extract environment variables from the merged configuration - env_vars=$(echo "$merged_config" | yq eval '.config.env' -) - - # Verify environment variables - for key in $(echo "$env_vars" | yq eval 'keys' -); do - # Check if key is empty or not set - if [[ -z "$key" || "$key" == "null" || "$key" == "-" ]]; then - continue - fi - - value=$(echo "$env_vars" | yq eval ".${key}" -) - actual_value="${!key}" - - if [[ "$actual_value" != "$value" ]]; then - log_error "Config" "$key is not set correctly. Expected: $value, Got: $actual_value" - return 1 - else - log_success "Config" "$key is set correctly" - fi - done - - log_success "Config" "configure_environment test passed" - return 0 -} - -# Run the test -if test_configure_environment; then - log_success "Config" "All config tests passed successfully" -else - log_error "Config" "Config tests failed" - exit 1 -fi \ No newline at end of file diff --git a/src/tests/tasks/30_secrets.sh b/src/tests/tasks/30_secrets.sh deleted file mode 100644 index e0deae41..00000000 --- a/src/tests/tasks/30_secrets.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Starting validation of secrets fetching..." - -# Path to the merged configuration file -MERGED_CONFIG="/usr/local/configs/worker/merged_worker.yaml" - -# Test verify_secrets function -test_verify_secrets() { - log_info "Running test: verify_secrets" - - # Load the merged configuration - merged_config=$(cat "$MERGED_CONFIG") - - # Extract secrets from the merged configuration - secrets=$(echo "$merged_config" | yq eval '.config.secrets' -) - - # Check if secrets is empty or null - if [[ -z "$secrets" || "$secrets" == "null" ]]; then - log_info "No secrets found in the configuration." - return 0 - fi - - # Verify secrets as environment variables - for secret_key in $(echo "$secrets" | yq eval 'keys' -); do - - # Check if the key is valid - if [[ -z "$secret_key" || "$secret_key" == "-" ]]; then - continue - fi - - expected_reference=$(echo "$secrets" | yq eval ".${secret_key}" -) - actual_value="${!secret_key}" - - # Verify that the environment variable is set and different from the reference - if [[ -z "$actual_value" || "$actual_value" == "$expected_reference" ]]; then - log_error "Secrets" "$secret_key is not replaced correctly. Got: $actual_value" - return 1 - else - log_success "Secrets" "$secret_key is resolved correctly" - fi - done - - log_success "Secrets" "verify_secrets test passed" - return 0 -} - -# Run the test -if test_verify_secrets; then - log_success "Secrets" "All secrets fetching tests passed successfully" -else - log_error "Secrets" "Secrets fetching tests failed" - exit 1 -fi \ No newline at end of file diff --git a/src/tests/tasks/40_test_service.sh b/src/tests/tasks/40_test_service.sh deleted file mode 100644 index e06b638b..00000000 --- a/src/tests/tasks/40_test_service.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -# shellcheck source=/usr/local/lib/utils.sh disable=SC1091 -source /usr/local/lib/utils.sh - -log_info "Test" "Starting service tests..." - -# Test 1: Check service configuration -log_info "Test" "Testing service configuration..." -if ! worker service config | grep -q "test_service"; then - log_error "Test" "Service configuration test failed: test_service not found in config" - exit 1 -fi -log_success "Test" "Service configuration test passed" - -# Test 2: Check service list and wait for it to be running -log_info "Test" "Testing service list and waiting for service to be ready..." - -# Wait for up to 30 seconds for the service to be fully running -for i in {1..30}; do - status_output=$(worker service list 2>&1) - if echo "$status_output" | grep "test_service" | grep -q "RUNNING"; then - log_success "Test" "Service is now running" - break - fi - if [ "$i" -eq 30 ]; then - log_error "Test" "Timeout waiting for service to start" - log_error "Test" "Current status: $status_output" - exit 1 - fi - sleep 1 -done -log_success "Test" "Service list test passed" - -# Test 3: Check service status -log_info "Test" "Testing service status..." -status_output=$(worker service status test_service) -if ! echo "$status_output" | grep -q "RUNNING"; then - log_error "Test" "Service status test failed: test_service not running" - log_error "Test" "Current status: $status_output" - exit 1 -fi -log_success "Test" "Service status test passed" - -# Test 4: Test service stop/start -log_info "Test" "Testing service stop..." -worker service stop test_service -sleep 2 -if worker service status test_service | grep -q "RUNNING"; then - log_error "Test" "Service stop test failed: test_service still running" - exit 1 -fi -log_success "Test" "Service stop test passed" - -log_info "Test" "Testing service start..." -worker service start test_service -sleep 2 -if ! worker service status test_service | grep -q "RUNNING"; then - log_error "Test" "Service start test failed: test_service not running" - exit 1 -fi -log_success "Test" "Service start test passed" - -# Test 5: Check service logs -log_info "Test" "Testing service logs..." - -# Wait a bit for the service to initialize after restart -sleep 2 - -# First check if log file exists -log_file="/var/log/supervisor/test_service.out.log" -if [ ! -f "$log_file" ]; then - log_error "Test" "Service logs test failed: log file not found at $log_file" - exit 1 -fi - -# Try a simple tail command first -if ! tail -n 1 "$log_file" > /dev/null 2>&1; then - log_error "Test" "Service logs test failed: cannot read log file" - exit 1 -fi - -# Now test the worker service logs command with --nostream -if ! worker service logs test_service --lines=1 --nostream > /dev/null 2>&1; then - log_error "Test" "Service logs test failed: worker service logs command failed" - exit 1 -fi - -log_success "Test" "Service logs test passed" - -log_success "Test" "All service tests completed successfully" From c40a45fdb4c7891039364bb9cf5626984872321f Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Tue, 18 Feb 2025 19:09:13 +0300 Subject: [PATCH 02/25] security improvements --- .dockerignore | 13 +++++++++++++ Dockerfile | 12 +++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index bd055df2..c3127dc1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,18 @@ # CI/CD and GitHub-related files ci/* + +# macOS system files +.DS_Store +**/.DS_Store +.AppleDouble +.LSOverride + +# Editor and IDE files +.idea/ +.vscode/ +*.swp +*.swo .github/* # Documentation files @@ -35,3 +47,4 @@ src/* .vscode/ *.iml .prettierignore +.DS_Store diff --git a/Dockerfile b/Dockerfile index 5061bc63..279cc33f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -159,10 +159,10 @@ COPY etc/configs/supervisor ${WORKER_CONFIG_DIR}/supervisor/ RUN chmod +x ${WORKER_LIB_DIR}/*.sh && \ ${WORKER_LIB_DIR}/env_handler.sh init_environment -# Set up CLI tool +# Set up CLI tool and create symlink COPY lib/cli.sh ${WORKER_BIN_DIR}/worker_mgmt -RUN chmod +x ${WORKER_BIN_DIR}/worker_mgmt && \ - ln -s ${WORKER_BIN_DIR}/worker_mgmt ${WORKER_BIN_DIR}/worker +RUN chmod 755 ${WORKER_BIN_DIR}/worker_mgmt && \ + ln -sf ${WORKER_BIN_DIR}/worker_mgmt ${WORKER_BIN_DIR}/worker # Set permissions RUN \ @@ -178,7 +178,7 @@ RUN \ ${AZURE_CONFIG_DIR} && \ # Set directory permissions find ${WORKER_BASE_DIR} ${WORKER_CONFIG_DIR} ${WORKER_LIB_DIR} ${WORKER_BIN_DIR} -type d -exec chmod 755 {} + && \ - # Set file permissions + # Set base file permissions find ${WORKER_CONFIG_DIR} -type f -exec chmod 644 {} + && \ find ${WORKER_LIB_DIR} -type f ! -name process_manager.sh -exec chmod 644 {} + && \ # Make specific files executable @@ -187,7 +187,9 @@ RUN \ ${WORKER_BIN_DIR}/worker_mgmt \ ${WORKER_LIB_DIR}/process_manager.sh && \ # Set runtime directories permissions - chmod 775 ${WORKER_APP_DIR} ${WORKER_DATA_DIR} + chmod 775 ${WORKER_APP_DIR} ${WORKER_DATA_DIR} && \ + # Set sensitive file permissions + chmod 600 ${WORKER_CONFIG_DIR}/secrets # Set up supervisor configuration RUN ln -sf ${WORKER_CONFIG_DIR}/supervisor/supervisord.conf /etc/supervisord.conf From e18dd5a2030ba1a3b5f5bb6a8d5a1fc3c2f58b85 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Wed, 19 Feb 2025 12:58:11 +0300 Subject: [PATCH 03/25] improve cli --- lib/cli/auth.sh | 14 ++- lib/cli/config.sh | 21 +++-- lib/cli/env.sh | 153 +++++++++++++++++++++--------- lib/cli/health.sh | 19 ++-- lib/cli/info.sh | 230 ++++++++++++++++++++++++++++++--------------- lib/cli/logs.sh | 20 +++- lib/cli/sbom.sh | 24 +++-- lib/cli/service.sh | 16 +++- lib/env_handler.sh | 32 +++++++ 9 files changed, 374 insertions(+), 155 deletions(-) diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh index 303259c1..4dc412c3 100644 --- a/lib/cli/auth.sh +++ b/lib/cli/auth.sh @@ -25,7 +25,8 @@ Examples: EOF } -# Show authentication status +# Description: Display authentication status for all cloud providers +# Example: worker auth status show_auth_status() { log_info "Auth" "Checking authentication status..." @@ -39,7 +40,9 @@ show_auth_status() { done } -# Test authentication +# Description: Test authentication for cloud providers +# Options: --provider aws|gcp|azure|bitwarden +# Example: worker auth test --provider aws test_auth() { local provider=$1 @@ -61,7 +64,8 @@ test_auth() { fi } -# Refresh credentials +# Description: Refresh credentials for all configured providers +# Example: worker auth refresh refresh_auth() { log_info "Auth" "Refreshing credentials..." @@ -82,7 +86,9 @@ refresh_auth() { fi } -# Rotate credentials +# Description: Rotate credentials for a specific provider +# Options: --provider aws|gcp|azure|bitwarden +# Example: worker auth rotate --provider aws rotate_auth() { local provider=$1 diff --git a/lib/cli/config.sh b/lib/cli/config.sh index f5e9cea4..55574068 100644 --- a/lib/cli/config.sh +++ b/lib/cli/config.sh @@ -29,7 +29,9 @@ Examples: EOF } -# Show current configuration +# Description: Display current worker configuration +# Options: --format yaml|json +# Example: worker config show --format json show_config() { local format=${1:-yaml} log_info "Config" "Current configuration:" @@ -51,7 +53,8 @@ show_config() { esac } -# Edit configuration +# Description: Edit configuration in default editor +# Example: worker config edit edit_config() { local config_file="${HOME}/.config/worker/worker.yaml" @@ -75,7 +78,9 @@ edit_config() { fi } -# Validate configuration +# Description: Validate all configuration files or a specific one +# Options: --file PATH +# Example: worker config validate --file ~/.config/worker/worker.yaml validate_config() { local config_file=${1:-} log_info "Config" "Validating configuration..." @@ -113,7 +118,8 @@ validate_config() { fi } -# Show configuration locations +# Description: Display paths of all configuration files +# Example: worker config locations show_locations() { cat << EOF Configuration Locations: @@ -124,7 +130,8 @@ Configuration Locations: EOF } -# Initialize new configuration +# Description: Initialize a new configuration file with defaults +# Example: worker config init init_config() { local config_dir="${HOME}/.config/worker" local config_file="$config_dir/worker.yaml" @@ -153,7 +160,9 @@ init_config() { fi } -# Show differences between default and current config +# Description: Show differences between default and current configuration +# Options: --format unified|context|git +# Example: worker config diff --format git show_diff() { local default_config="/etc/worker/worker.yaml" local user_config="${HOME}/.config/worker/worker.yaml" diff --git a/lib/cli/env.sh b/lib/cli/env.sh index 84a689d3..34032f78 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -37,49 +37,61 @@ Examples: EOF } -# Show environment variables +# Description: Display environment variables with optional filtering +# Options: --format text|json, --filter PATTERN, --include-secrets +# Example: worker env show --format json --filter AWS_* --include-secrets show_environment() { local format=${1:-text} local filter=$2 local include_secrets=${3:-false} - if [ "$include_secrets" == "true" ]; then - list_env_vars "$format" - else - # Only show non-secret environment variables - if [ -f "$WORKER_ENV_FILE" ]; then - case $format in - json) - { - echo "{" - if [ -n "$filter" ]; then - grep "^export $filter" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/ "\1": "\2",/' - else - grep "^export" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/ "\1": "\2",/' - fi - echo "}" - } | sed 's/,}/}/' | jq '.' - ;; - text) - if [ -n "$filter" ]; then - grep "^export $filter" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' - else - grep "^export" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' + # Check if environment file exists + if [ ! -f "$WORKER_ENV_FILE" ]; then + log_error "Env" "Environment file not found" + return 1 + fi + + case $format in + json) + # Get variables and convert to JSON + local vars + if [ -n "$filter" ]; then + vars=$(grep "^export $filter" "$WORKER_ENV_FILE") + else + vars=$(grep "^export" "$WORKER_ENV_FILE") + fi + + # Convert to JSON + local json="{" + while IFS= read -r line; do + if [[ $line =~ ^export[[:space:]]+([^=]+)=\"([^\"]*)\" ]]; then + if [ -n "$json" ] && [ "$json" != "{" ]; then + json="$json," fi - ;; - *) - log_error "Env" "Unknown format: $format" - return 1 - ;; - esac - else - log_error "Env" "Environment file not found" + key=${BASH_REMATCH[1]} + value=${BASH_REMATCH[2]} + json="$json\"$key\":\"$value\"" + fi + done <<< "$vars" + json="$json}" + echo "$json" | jq . + ;; + text) + if [ -n "$filter" ]; then + grep "^export $filter" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' + else + grep "^export" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' + fi + ;; + *) + log_error "Env" "Unknown format: $format" return 1 - fi - fi + ;; + esac } -# Set environment variable +# Description: Set a new environment variable or update existing one +# Example: worker env set MY_VAR "my value" set_environment() { local name=$1 local value=$2 @@ -110,7 +122,8 @@ set_environment() { fi } -# Unset environment variable +# Description: Remove an environment variable +# Example: worker env unset MY_VAR unset_environment() { local name=$1 @@ -119,15 +132,32 @@ unset_environment() { return 1 fi - if [ -n "${!name}" ]; then + # Check if variable exists in environment file + if grep -q "^export $name=" "$WORKER_ENV_FILE"; then + # Create a temporary file + local tmpfile + tmpfile=$(mktemp) + + # Remove the variable from environment file + grep -v "^export $name=" "$WORKER_ENV_FILE" > "$tmpfile" + cat "$tmpfile" > "$WORKER_ENV_FILE" + rm -f "$tmpfile" + + # Also remove from current environment unset "$name" + log_success "Env" "Unset $name" + + # Reload environment to ensure consistency + source "$WORKER_ENV_FILE" else log_warn "Env" "Variable $name is not set" fi } -# Export environment variables +# Description: Export environment variables to a file +# Options: --file PATH, --include-secrets +# Example: worker env export --file env.backup --include-secrets export_environment() { local file=$1 local filter=$2 @@ -147,7 +177,9 @@ export_environment() { log_success "Env" "Environment variables exported to $file" } -# Import environment variables +# Description: Import environment variables from a file +# Options: --file PATH +# Example: worker env import --file env.backup import_environment() { local file=$1 @@ -171,7 +203,9 @@ import_environment() { log_success "Env" "Environment variables imported from $file" } -# Validate environment variables +# Description: Validate environment variables against schema +# Options: --format text|json +# Example: worker env validate --format json validate_environment() { local config config=$(load_and_parse_config) @@ -200,7 +234,9 @@ validate_environment() { fi } -# Generate environment template +# Description: Generate a template environment file +# Options: --file PATH +# Example: worker env template --file .env.template generate_template() { local config config=$(load_and_parse_config) @@ -218,7 +254,8 @@ generate_template() { echo "$config" | yq eval '.config.env | to_entries | .[] | "# " + .key + "\n" + .key + "=\"" + .value + "\""' - } -# Reset environment +# Description: Reset environment to default values +# Example: worker env reset reset_environment() { # Clear all non-system variables local system_vars="HOME|USER|PATH|SHELL|TERM|LANG|PWD" @@ -244,21 +281,43 @@ reset_environment() { # Parse command line arguments parse_args() { + # Initialize variables with defaults + format="text" + filter="" + file="" + include_secrets="false" + local args=() while [[ $# -gt 0 ]]; do case $1 in - --format) - format=$2 + --format=*|--type=*) + format="${1#*=}" + shift + ;; + --format|--type) + format="$2" shift 2 ;; + --filter=*) + filter="${1#*=}" + shift + ;; --filter) - filter=$2 + filter="$2" shift 2 ;; + --file=*) + file="${1#*=}" + shift + ;; --file) - file=$2 + file="$2" shift 2 ;; + --include-secrets) + include_secrets="true" + shift + ;; *) args+=("$1") shift @@ -268,7 +327,9 @@ parse_args() { set -- "${args[@]}" } -# Show environment status +# Description: Show environment status and validation results +# Options: --format text|json +# Example: worker env status --format json show_status() { local format=${1:-text} diff --git a/lib/cli/health.sh b/lib/cli/health.sh index 0ecd4fb6..39d97be1 100644 --- a/lib/cli/health.sh +++ b/lib/cli/health.sh @@ -24,7 +24,8 @@ Examples: EOF } -# Run health check +# Description: Run comprehensive health check of the worker +# Example: worker health check check_health() { log_info "Health" "Running health check..." local failed=0 @@ -49,7 +50,8 @@ check_health() { fi } -# Check system resources +# Description: Check system resource usage (disk, memory, CPU) +# Example: worker health check system check_system_resources() { local failed=0 @@ -86,7 +88,8 @@ check_system_resources() { return $failed } -# Check supervisor status +# Description: Check if supervisor is running and responsive +# Example: worker health check supervisor check_supervisor_status() { if ! pgrep -f supervisord > /dev/null; then log_error "Health" "Supervisor is not running" @@ -102,7 +105,8 @@ check_supervisor_status() { return 0 } -# Check services health +# Description: Check health status of all managed services +# Example: worker health check services check_services_health() { local failed=0 local services_status @@ -129,7 +133,8 @@ check_services_health() { return $failed } -# Check authentication status +# Description: Check authentication status for all providers +# Example: worker health check auth check_auth_status() { local failed=0 @@ -148,7 +153,9 @@ check_auth_status() { return $failed } -# Generate health report +# Description: Generate detailed health report +# Options: --format text|json +# Example: worker health report --format json generate_report() { local format=${1:-text} local report_file diff --git a/lib/cli/info.sh b/lib/cli/info.sh index e2836d41..b5c384c9 100644 --- a/lib/cli/info.sh +++ b/lib/cli/info.sh @@ -3,41 +3,135 @@ # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 source "${WORKER_LIB_DIR}/utils.sh" +# Get command metadata by scanning function contents +get_command_metadata() { + local commands={} + declare -A commands + local current_file="${BASH_SOURCE[0]}" + + # Scan the current file for show_* functions and their metadata + while IFS= read -r line; do + # Match show_* function definitions + if [[ $line =~ ^show_([a-z_]+)[[:space:]]*\(\)[[:space:]]*\{[[:space:]]*$ ]]; then + local cmd=${BASH_REMATCH[1]} + local desc="" + local options="" + local example="" + + # Skip internal functions like show_help + [[ $cmd == "help" ]] && continue + + # Look for metadata in comments above function + local temp_file=$(mktemp) + grep -B 10 "^show_${cmd}()" "$current_file" > "$temp_file" + + # Get description + desc=$(grep -B 10 "^show_${cmd}()" "$temp_file" | grep '^# Description:' | tail -n 1 | sed 's/^# Description:[[:space:]]*//') + if [ -z "$desc" ]; then + desc=$(grep -B 10 "^show_${cmd}()" "$temp_file" | grep '^#[[:space:]]' | tail -n 1 | sed 's/^#[[:space:]]*//') + fi + if [ -z "$desc" ]; then + desc="Show ${cmd//_/ } information" + fi + + # Get options + options=$(grep -B 10 "^show_${cmd}()" "$temp_file" | grep '^# Options:' | tail -n 1 | sed 's/^# Options:[[:space:]]*//') + if [ -z "$options" ] && grep -q 'format=' "$temp_file"; then + options="--format text|json" + fi + if grep -q 'verbose=' "$temp_file"; then + [[ -n "$options" ]] && options="$options, " + options+="--verbose" + fi + + # Get example + example=$(grep -B 10 "^show_${cmd}()" "$temp_file" | grep '^# Example:' | tail -n 1 | sed 's/^# Example:[[:space:]]*//') + if [ -z "$example" ] && [ -n "$options" ]; then + if [[ $options == *"format"* ]]; then + example="worker info $cmd --format json" + elif [[ $options == *"verbose"* ]]; then + example="worker info $cmd --verbose" + fi + fi + + rm "$temp_file" + + # Store metadata + commands[$cmd]="$desc|$options|$example" + fi + done < "$current_file" + + echo "$(declare -p commands)" +} + # Show help for info command info_help() { + local -A commands + eval "$(get_command_metadata)" + + # Find the longest command name for proper padding + local max_length=0 + for cmd in "${!commands[@]}"; do + local len=${#cmd} + if ((len > max_length)); then + max_length=$len + fi + done + + # Add padding for alignment + max_length=$((max_length + 2)) + cat << EOF Display information about the UDX Worker image and runtime Usage: worker info [command] Available Commands: - overview Show a high-level overview of the worker image - system Show system information and resource usage - config Show configuration and settings - services Show available services and their status - env Show environment variables and settings - auth Show authentication and credentials status - deps Show installed dependencies and versions - features Show available features and capabilities - paths Show important filesystem paths - logs Show logging configuration and locations - security Show security settings and policies - network Show network configuration and ports - version Show detailed version information - -Options: - --format Output format (text/json) - --verbose Show detailed information - -Examples: - worker info overview # Get a quick overview of the worker - worker info system --verbose # Show detailed system information - worker info deps --format json # List dependencies in JSON format - worker info features # Show available features EOF + + # Sort commands alphabetically and display + local sorted_commands=($(echo "${!commands[@]}" | tr ' ' '\n' | sort)) + for cmd in "${sorted_commands[@]}"; do + IFS='|' read -r desc options example <<< "${commands[$cmd]}" + printf " %-${max_length}s %s\n" "$cmd" "$desc" + done + + # Collect unique options from all commands + local all_options="" + for cmd in "${!commands[@]}"; do + IFS='|' read -r _ options _ <<< "${commands[$cmd]}" + if [ -n "$options" ]; then + all_options="$all_options $options" + fi + done + + # Display unique options + if [ -n "$all_options" ]; then + echo -e "\nOptions:" + echo "$all_options" | tr ',' '\n' | tr ' ' '\n' | sort -u | grep -v '^$' | while read -r opt; do + printf " %s\n" "$opt" + done + fi + + # Display examples for commands that have them + local has_examples=false + for cmd in "${sorted_commands[@]}"; do + IFS='|' read -r desc options example <<< "${commands[$cmd]}" + if [ -n "$example" ]; then + if ! $has_examples; then + echo -e "\nExamples:" + has_examples=true + fi + printf " %-45s # %s\n" "$example" "$desc" + fi + done + + echo } -# Show general information +# Description: Display general information about the worker +# Options: --format text|json +# Example: worker info --format json show_info() { local format=${1:-text} @@ -101,7 +195,9 @@ show_info() { esac } -# Show system information +# Description: Display detailed system information and resource usage +# Options: --format text|json, --verbose +# Example: worker info system --verbose --format json show_system_info() { local format=${1:-text} @@ -197,7 +293,9 @@ get_cli_modules() { echo "$(declare -p modules)" } -# Show overview of the worker +# Description: Display a high-level overview of the worker image and its capabilities +# Options: --format text|json +# Example: worker info overview --format text show_overview() { local format=${1:-text} local -A modules @@ -297,7 +395,9 @@ EOF esac } -# Show available features +# Description: Display available features and their capabilities +# Options: --format text|json +# Example: worker info features --format json show_features() { local format=${1:-text} local -A modules @@ -384,54 +484,30 @@ EOF info_handler() { local command=$1 shift - - case $command in - overview) - show_overview "$@" - ;; - system) - show_system_info "$@" - ;; - features) - show_features "$@" - ;; - deps) - show_deps "$@" - ;; - config) - show_config "$@" - ;; - services) - show_services "$@" - ;; - env) - show_env "$@" - ;; - auth) - show_auth "$@" - ;; - paths) - show_paths "$@" - ;; - logs) - show_logs "$@" - ;; - security) - show_security "$@" - ;; - network) - show_network "$@" - ;; - version) - show_version "$@" - ;; - help|"") - info_help - ;; - *) - log_error "Info" "Unknown command: $command" - info_help - return 1 - ;; - esac + + # If no command provided or help requested, show help + if [ -z "$command" ] || [ "$command" = "help" ]; then + info_help + return + fi + + # Get available commands + local -A commands + eval "$(get_command_metadata)" + + # Check if command exists + if [ -z "${commands[$command]+x}" ]; then + log_error "Info" "Unknown command: $command" + info_help + return 1 + fi + + # Try to execute the command function + local func_name="show_${command}" + if declare -F "$func_name" > /dev/null; then + "$func_name" "$@" + else + log_error "Info" "Command handler not implemented: $command" + return 1 + fi } diff --git a/lib/cli/logs.sh b/lib/cli/logs.sh index 03a92dbf..7d8d28f2 100644 --- a/lib/cli/logs.sh +++ b/lib/cli/logs.sh @@ -57,7 +57,9 @@ get_log_file() { esac } -# Show logs +# Description: Display logs for a specific service +# Options: --service NAME, --type out|err|all, --lines N, --since TIME, --until TIME +# Example: worker logs show --service myapp --type err --lines 100 --since "2024-01-01" show_logs() { local service=$1 local type=${2:-all} @@ -91,7 +93,9 @@ show_logs() { eval "$cmd $log_file" } -# Follow logs +# Description: Follow logs in real-time for a service +# Options: --service NAME, --type out|err|all +# Example: worker logs follow --service myapp --type err follow_logs() { local service=$1 local type=${2:-all} @@ -112,7 +116,9 @@ follow_logs() { tail -f "$log_file" } -# Search logs +# Description: Search logs for a specific pattern +# Options: --pattern PATTERN, --service NAME, --type out|err|all +# Example: worker logs search --pattern "error" --service myapp search_logs() { local pattern=$1 local service=$2 @@ -133,7 +139,9 @@ search_logs() { grep -n "$pattern" $log_file } -# Export logs +# Description: Export logs to a file +# Options: --service NAME, --type out|err|all, --since TIME, --until TIME, --format text|json +# Example: worker logs export --service myapp --since "2024-01-01" --format json export_logs() { local service=$1 local type=${2:-all} @@ -180,7 +188,9 @@ export_logs() { log_success "Logs" "Logs exported to $output_file" } -# Clean old logs +# Description: Clean old log files +# Options: --older-than DURATION +# Example: worker logs clean --older-than 7d clean_logs() { local older_than=${1:-7d} # Default: 7 days diff --git a/lib/cli/sbom.sh b/lib/cli/sbom.sh index 0d3eb57b..29c55616 100644 --- a/lib/cli/sbom.sh +++ b/lib/cli/sbom.sh @@ -33,7 +33,9 @@ Examples: EOF } -# Generate SBOM +# Description: Generate Software Bill of Materials in various formats +# Options: --format text|json|cyclonedx|spdx, --type system|python|all, --filter PATTERN +# Example: worker sbom generate --format json --type python --filter requests generate_sbom() { local format=${1:-text} local type=${2:-all} @@ -135,7 +137,9 @@ generate_sbom() { esac } -# Analyze dependencies +# Description: Analyze dependencies for security vulnerabilities +# Options: --type system|python|all, --severity low|medium|high|critical +# Example: worker sbom analyze --type python --severity high analyze_deps() { log_info "SBOM" "Analyzing dependencies for security issues..." @@ -153,7 +157,9 @@ analyze_deps() { fi } -# Export SBOM +# Description: Export SBOM to a file in specified format +# Options: --format text|json|cyclonedx|spdx, --file PATH +# Example: worker sbom export --format cyclonedx --file sbom.xml export_sbom() { local format=$1 local file=$2 @@ -167,7 +173,9 @@ export_sbom() { log_success "SBOM" "SBOM exported to $file" } -# Show dependency tree +# Description: Display dependency tree for installed packages +# Options: --type system|python|all, --format text|json +# Example: worker sbom deps --type python --format json show_deps() { local type=${1:-all} @@ -178,7 +186,9 @@ show_deps() { fi } -# Verify package integrity +# Description: Verify integrity of installed packages +# Options: --type system|python|all +# Example: worker sbom verify --type all verify_packages() { log_info "SBOM" "Verifying package integrity..." @@ -198,7 +208,9 @@ verify_packages() { fi } -# Check for updates +# Description: Check for available package updates +# Options: --type system|python|all, --format text|json +# Example: worker sbom updates --type all --format json check_updates() { log_info "SBOM" "Checking for available updates..." diff --git a/lib/cli/service.sh b/lib/cli/service.sh index 4fb32932..b2baadc9 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -72,7 +72,8 @@ Examples: EOF } -# Function to list all services +# Description: List all configured services and their status +# Example: worker service list list_services() { # Capture the output of supervisorctl status local services_status @@ -94,7 +95,8 @@ list_services() { done } -# Function to check the status of one or all services +# Description: Show detailed status of a specific service +# Example: worker service status my-app check_status() { # Require a service name for this function if [ -z "$1" ]; then @@ -117,7 +119,9 @@ check_status() { echo "$service_status" } -# Function to follow logs for a specific service +# Description: View and follow logs for a service +# Options: --tail N, --follow, --error-only +# Example: worker service logs my-app --tail 100 --follow follow_logs() { local service_name="" local type="out" @@ -180,7 +184,8 @@ follow_logs() { fi } -# Function to show supervisor configuration +# Description: Display current service configuration +# Example: worker service config show_config() { if [ ! -f "/etc/supervisord.conf" ]; then log_error "Service" "Configuration file is not generated since no services are managed." @@ -189,7 +194,8 @@ show_config() { cat /etc/supervisord.conf } -# Function to start, stop, or restart a service +# Description: Start, stop, or restart a service +# Example: worker service restart my-app manage_service() { if [ -z "$2" ]; then log_error "Service" "Error: No service name provided." diff --git a/lib/env_handler.sh b/lib/env_handler.sh index f253cc99..54a4bbc6 100644 --- a/lib/env_handler.sh +++ b/lib/env_handler.sh @@ -88,6 +88,38 @@ load_secrets() { fi } +# Format environment variables as JSON or text +format_env_vars() { + local vars=$1 + local format=${2:-text} + + case $format in + json) + # Convert to JSON + local json="{" + while IFS= read -r line; do + if [[ $line =~ ^export[[:space:]]+([^=]+)=\"([^\"]*)\" ]]; then + if [ -n "$json" ] && [ "$json" != "{" ]; then + json="$json," + fi + key=${BASH_REMATCH[1]} + value=${BASH_REMATCH[2]} + json="$json\"$key\":\"$value\"" + fi + done <<< "$vars" + json="$json}" + echo "$json" | jq . + ;; + text) + echo "$vars" | sed 's/export \([^=]*\)=\"\([^\"]*\)"/\1=\2/' + ;; + *) + log_error "Environment" "Unknown format: $format" + return 1 + ;; + esac +} + # Initialize environment init_environment() { generate_env_file From 74c5f9044a98eb09aa0ab551ca55deeff33ab175 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Wed, 19 Feb 2025 19:09:32 +0300 Subject: [PATCH 04/25] auth cli improvements --- lib/auth.sh | 64 +++++++++++ lib/cleanup.sh | 41 ++++--- lib/cli/auth.sh | 263 ++++++++++++++++++++++++++++++------------- lib/worker_config.sh | 2 +- 4 files changed, 277 insertions(+), 93 deletions(-) diff --git a/lib/auth.sh b/lib/auth.sh index fdb4fb59..4b15b186 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -6,6 +6,70 @@ source ${WORKER_LIB_DIR}/utils.sh # Array to track configured providers declare -a configured_providers=() +# Function to get env var names for a provider from config +get_provider_env_vars() { + local provider=$1 + local config + config=$(load_and_parse_config) + + # Get all env var names from actor creds that match ${VAR} pattern + if echo "$config" | jq -e '.config.actors' >/dev/null 2>&1; then + echo "$config" | jq -r ".config.actors[] | select(.type | startswith(\"$provider\")) | .creds" 2>/dev/null | \ + grep -o '\${[^}]*}' | sed 's/[\${}]//g' || true + fi +} + +# Function to check if a provider is configured +is_provider_configured() { + local provider=$1 + + # Get all possible env var names from config + local env_vars + mapfile -t env_vars < <(get_provider_env_vars "$provider") + + # First check all possible env vars + for env_var in "${env_vars[@]}"; do + if [ -n "${!env_var}" ]; then + return 0 + fi + done + + # Fallback to config if no env vars are set + local config + config=$(load_and_parse_config) + + # Check if we have a valid config with actors + if echo "$config" | jq -e '.config.actors' >/dev/null 2>&1; then + # Get all actors for this provider + local actors + actors=$(echo "$config" | jq -r ".config.actors[] | select(.type | startswith(\"$provider-\"))" 2>/dev/null) + + if [ -n "$actors" ] && [ "$actors" != "null" ]; then + # Check each actor's credentials + while IFS= read -r actor; do + [ -z "$actor" ] && continue + + local creds + creds=$(echo "$actor" | jq -r '.creds' 2>/dev/null) + [ "$creds" = "null" ] && continue + + # Evaluate creds as a reference to an environment variable + if [[ "$creds" =~ ^\$\{(.+)\}$ ]]; then + local env_var_name="${BASH_REMATCH[1]}" + creds="${!env_var_name}" + fi + + # If we find any valid credentials, return success + if [ -n "$creds" ]; then + return 0 + fi + done <<< "$actors" + fi + fi + + return 1 +} + # Function to authenticate actors authenticate_actors() { local actors_json="$1" diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 505a3928..9082ed73 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -65,32 +65,42 @@ cleanup_provider() { # Function to clean up actors based on the providers configured during authentication cleanup_actors() { - log_info "Starting cleanup of actors" + # Skip cleanup if no providers were configured during authentication + if [[ ${#configured_providers[@]} -eq 0 ]]; then + return 0 + fi - # Accept configured providers as arguments - local configured_providers=('azure' 'gcp' 'aws' 'bitwarden') + log_info "Starting cleanup of actors" # Track if any actual cleanup was performed local any_cleanup=false - # Loop through each configured provider only + # Only clean up providers that were actually configured for provider in "${configured_providers[@]}"; do case "$provider" in azure) - cleanup_provider "az" "az logout" "az account show" "Azure" && any_cleanup=true - ;; + if cleanup_provider "az" "az logout" "az account show" "Azure"; then + any_cleanup=true + fi + ;; gcp) - cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" && any_cleanup=true - ;; + if cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP"; then + any_cleanup=true + fi + ;; aws) - cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" && any_cleanup=true - ;; + if cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS"; then + any_cleanup=true + fi + ;; bitwarden) - cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" && any_cleanup=true - ;; + if cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden"; then + any_cleanup=true + fi + ;; *) log_warn "Unsupported or unavailable actor type for cleanup: $provider" - ;; + ;; esac done @@ -98,6 +108,11 @@ cleanup_actors() { if [[ "$any_cleanup" == false ]]; then log_info "No active sessions found for any configured providers." fi + + # Clear the configured providers array + configured_providers=() + + return 0 } # Example usage diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh index 4dc412c3..da42c49c 100644 --- a/lib/cli/auth.sh +++ b/lib/cli/auth.sh @@ -9,133 +9,238 @@ auth_help() { cat << EOF Manage authentication and credentials -Usage: worker auth [command] +Usage: worker auth [command] [provider] Available Commands: - status Show authentication status for all providers - test Test authentication for all or specific provider - refresh Refresh credentials - rotate Rotate credentials (if supported by provider) + status Show authentication status for all or specific provider + login Re-authenticate with provider(s) using available credentials + logout Log out from provider(s) Examples: - worker auth status - worker auth test aws - worker auth refresh - worker auth rotate --provider aws + worker auth status # Show status of all providers + worker auth status azure # Show status of Azure only + worker auth login # Re-auth all providers with available creds + worker auth login azure # Re-auth Azure only + worker auth logout # Log out from all providers EOF } # Description: Display authentication status for all cloud providers # Example: worker auth status show_auth_status() { + local target_provider=$1 log_info "Auth" "Checking authentication status..." - # Check each provider's status - for provider in aws gcp azure bitwarden; do - if is_provider_configured "$provider"; then - log_success "Auth" "$provider: Configured and authenticated" - else - log_warn "Auth" "$provider: Not configured" - fi - done -} - -# Description: Test authentication for cloud providers -# Options: --provider aws|gcp|azure|bitwarden -# Example: worker auth test --provider aws -test_auth() { - local provider=$1 - - if [ -z "$provider" ]; then - # Test all configured providers - for p in aws gcp azure bitwarden; do - if is_provider_configured "$p"; then - test_provider_auth "$p" + # Function to check a specific provider + check_provider_status() { + local provider=$1 + local has_env_creds=false + local has_config_creds=false + local types="" + + # First check env vars from config + local env_vars + mapfile -t env_vars < <(get_provider_env_vars "$provider") + + for env_var in "${env_vars[@]}"; do + if [ -n "${!env_var}" ]; then + has_env_creds=true + break fi done - else - # Test specific provider - if is_provider_configured "$provider"; then - test_provider_auth "$provider" + + # Then check config + local config + config=$(load_and_parse_config) + + if echo "$config" | jq -e '.config.actors' >/dev/null 2>&1; then + local actors + actors=$(echo "$config" | jq -r ".config.actors[] | select(.type | startswith(\"$provider-\"))" 2>/dev/null) + + if [ -n "$actors" ] && [ "$actors" != "null" ]; then + types=$(echo "$actors" | jq -r '.type' 2>/dev/null | grep . | tr '\n' ' ') + + while IFS= read -r actor; do + [ -z "$actor" ] && continue + + local creds + creds=$(echo "$actor" | jq -r '.creds' 2>/dev/null) + [ "$creds" = "null" ] && continue + + # Evaluate creds as a reference to an environment variable + if [[ "$creds" =~ ^\$\{(.+)\}$ ]]; then + local env_var_name="${BASH_REMATCH[1]}" + creds="${!env_var_name}" + fi + + if [ -n "$creds" ]; then + has_config_creds=true + break + fi + done <<< "$actors" + fi + fi + + # Check if currently authenticated + local is_authenticated=false + if check_provider_auth "$provider"; then + is_authenticated=true + fi + + # Show status based on env vars and config + if [ "$has_env_creds" = true ] || [ "$has_config_creds" = true ]; then + if [ "$is_authenticated" = true ]; then + log_success "Auth" "$provider: Active session${types:+ (actors: $types)}" + else + log_info "Auth" "$provider: Has credentials${types:+ (actors: $types)}, needs re-auth" + fi else - log_error "Auth" "Provider $provider is not configured" - return 1 + if [ -n "$types" ]; then + log_warn "Auth" "$provider: Missing credentials (actors: $types)" + else + log_info "Auth" "$provider: Not configured" + fi fi + } + + if [ -n "$target_provider" ]; then + check_provider_status "$target_provider" + else + for provider in aws gcp azure bitwarden; do + check_provider_status "$provider" + done fi } -# Description: Refresh credentials for all configured providers -# Example: worker auth refresh -refresh_auth() { - log_info "Auth" "Refreshing credentials..." +# Login to provider(s) +login_provider() { + local target_provider=$1 + log_info "Auth" "Authenticating providers..." - # Re-run authentication for all configured providers - local actors_json - actors_json=$(get_config_section "$(load_and_parse_config)" "actors") + # Load config and get actors + local config actors_json + config=$(load_and_parse_config) + actors_json=$(echo "$config" | jq -r '.config.actors') - if [ -n "$actors_json" ]; then - if authenticate_actors "$actors_json"; then - log_success "Auth" "Successfully refreshed credentials" - else - log_error "Auth" "Failed to refresh credentials" + if [[ -z "$actors_json" || "$actors_json" == "null" ]]; then + log_warn "Auth" "No providers found in configuration" + return 1 + fi + + # Filter actors by provider if specified + if [[ -n "$target_provider" ]]; then + actors_json=$(echo "$config" | jq -r ".config.actors | map(select(.type | startswith(\"$target_provider-\")))") + if [[ "$actors_json" == "[]" ]]; then + log_warn "Auth" "$target_provider: Not configured" return 1 fi + fi + + # Use the same authentication flow as entrypoint + if authenticate_actors "$actors_json"; then + log_success "Auth" "Authentication complete" + return 0 else - log_warn "Auth" "No actors configured" + log_warn "Auth" "No providers were authenticated" return 1 fi } -# Description: Rotate credentials for a specific provider -# Options: --provider aws|gcp|azure|bitwarden -# Example: worker auth rotate --provider aws -rotate_auth() { - local provider=$1 +# Logout from provider(s) +logout_provider() { + local target_provider=$1 + log_info "Auth" "Logging out providers..." - if [ -z "$provider" ]; then - log_error "Auth" "Provider is required for credential rotation" - return 1 - fi + # Source cleanup utilities + source ${WORKER_LIB_DIR}/cleanup.sh - if ! is_provider_configured "$provider"; then - log_error "Auth" "Provider $provider is not configured" - return 1 - fi + # Function to logout from a specific provider + do_provider_logout() { + local provider=$1 + + case "$provider" in + aws) + cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" + ;; + azure) + cleanup_provider "az" "az logout" "az account show" "Azure" + ;; + gcp) + cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" + ;; + bitwarden) + cleanup_provider "bw" "bw logout --force" "bw status" "Bitwarden" + ;; + esac + } - # Call provider-specific rotation function - if rotate_provider_credentials "$provider"; then - log_success "Auth" "Successfully rotated credentials for $provider" + if [ -n "$target_provider" ]; then + do_provider_logout "$target_provider" else - log_error "Auth" "Failed to rotate credentials for $provider" - return 1 + for provider in aws gcp azure bitwarden; do + do_provider_logout "$provider" + done fi } + + +# Check if a provider is currently authenticated +check_provider_auth() { + local provider=$1 + + case "$provider" in + aws) + if aws sts get-caller-identity &>/dev/null; then + return 0 + fi + ;; + azure) + if az account show &>/dev/null; then + return 0 + fi + ;; + gcp) + if gcloud auth list --format="value(account)" 2>/dev/null | grep -q .; then + return 0 + fi + ;; + bitwarden) + if bw status | grep -q "unlocked"; then + return 0 + fi + ;; + esac + + return 1 +} + # Handle auth commands auth_handler() { local cmd=$1 - shift + local provider=$2 case $cmd in status) - show_auth_status + show_auth_status "$provider" + return $? ;; - test) - test_auth "$1" + login) + login_provider "$provider" + return $? ;; - refresh) - refresh_auth + logout) + logout_provider "$provider" + return $? ;; - rotate) - rotate_auth "$1" - ;; - help) + "" | help) auth_help + return 0 ;; *) - log_error "Auth" "Unknown command: $cmd" + log_error "CLI" "Unknown command: auth" auth_help - exit 1 + return 1 ;; esac } diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 9d65ca69..a65826ff 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -44,7 +44,7 @@ merge_worker_configs() { return 1 fi else - log_info "No worker configuration provided." + # Using default config # Copy the built-in configuration to the merged configuration if ! cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG"; then From a31abce2fcdba8ed59b12cf8f16c886acdc04fdc Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 20 Feb 2025 00:20:08 +0300 Subject: [PATCH 05/25] simplify health cli --- lib/cli/health.sh | 231 +++++++++++++--------------------------------- 1 file changed, 63 insertions(+), 168 deletions(-) diff --git a/lib/cli/health.sh b/lib/cli/health.sh index 39d97be1..d52cc5e9 100644 --- a/lib/cli/health.sh +++ b/lib/cli/health.sh @@ -11,47 +11,80 @@ Check system health and run diagnostics Usage: worker health [command] Available Commands: - check Run health check status Show current health status - diag Run diagnostics - report Generate health report + +Options: + --format Output format (text|json) Examples: - worker health check worker health status - worker health diag - worker health report --format json + worker health status --format json EOF } -# Description: Run comprehensive health check of the worker -# Example: worker health check +# Description: Check system health metrics +# Example: worker health status [--format json] check_health() { - log_info "Health" "Running health check..." + local format="text" local failed=0 - # Check system resources - check_system_resources || failed=1 - - # Check supervisor status - check_supervisor_status || failed=1 + while [ $# -gt 0 ]; do + case $1 in + --format) + format=$2 + shift 2 + ;; + *) + log_error "Health" "Unknown option: $1" + return 1 + ;; + esac + done - # Check service health - check_services_health || failed=1 + if [ "$format" != "json" ]; then + log_info "Health" "Running health check..." + fi - # Check authentication status - check_auth_status || failed=1 + # Check system resources + check_system_resources || failed=1 - if [ $failed -eq 0 ]; then - log_success "Health" "All health checks passed" + # Gather all data + local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + local disk_usage=$(df -h / | awk 'NR==2 {print $5}' | tr -d '%') + local mem_total=$(free -b | awk '/Mem:/ {printf "%.2f", $2/1024/1024/1024}') + local mem_used=$(free -b | awk '/Mem:/ {printf "%.2f", $3/1024/1024/1024}') + local mem_usage=$(free | awk '/Mem:/ {printf("%.0f", $3/$2 * 100)}') + local load_avg=$(uptime | awk -F'load average:' '{print $2}' | cut -d, -f1 | tr -d ' ') + + if [ "$format" = "json" ]; then + { + echo "{" + echo " \"timestamp\": \"$timestamp\"," + echo " \"status\": \"$([ $failed -eq 0 ] && echo "healthy" || echo "unhealthy")\"," + echo " \"disk\": {" + echo " \"usage_percent\": $disk_usage" + echo " }," + echo " \"memory\": {" + echo " \"total_gb\": $mem_total," + echo " \"used_gb\": $mem_used," + echo " \"usage_percent\": $mem_usage" + echo " }," + echo " \"load_average\": $load_avg" + echo "}" + } else - log_error "Health" "Some health checks failed" - return 1 + if [ $failed -eq 0 ]; then + log_success "Health" "All health checks passed" + else + log_error "Health" "Some health checks failed" + fi fi + + return $failed } # Description: Check system resource usage (disk, memory, CPU) -# Example: worker health check system +# Example: worker health status check_system_resources() { local failed=0 @@ -88,162 +121,24 @@ check_system_resources() { return $failed } -# Description: Check if supervisor is running and responsive -# Example: worker health check supervisor -check_supervisor_status() { - if ! pgrep -f supervisord > /dev/null; then - log_error "Health" "Supervisor is not running" - return 1 - fi - - if ! supervisorctl status > /dev/null; then - log_error "Health" "Supervisor is not responding" - return 1 - fi - - log_success "Health" "Supervisor is running and responsive" - return 0 -} - -# Description: Check health status of all managed services -# Example: worker health check services -check_services_health() { - local failed=0 - local services_status - - services_status=$(supervisorctl status) - if [ $? -ne 0 ]; then - log_error "Health" "Failed to get services status" - return 1 - fi - - echo "$services_status" | while read -r line; do - local service_name status - service_name=$(echo "$line" | awk '{print $1}') - status=$(echo "$line" | awk '{print $2}') - - if [ "$status" != "RUNNING" ]; then - log_error "Health" "Service $service_name is not running (status: $status)" - failed=1 - else - log_success "Health" "Service $service_name is running" - fi - done - - return $failed -} - -# Description: Check authentication status for all providers -# Example: worker health check auth -check_auth_status() { - local failed=0 - - # Check each provider - for provider in aws gcp azure bitwarden; do - if is_provider_configured "$provider"; then - if ! test_provider_auth "$provider"; then - log_error "Health" "Authentication failed for $provider" - failed=1 - else - log_success "Health" "Authentication successful for $provider" - fi - fi - done - - return $failed -} -# Description: Generate detailed health report -# Options: --format text|json -# Example: worker health report --format json -generate_report() { - local format=${1:-text} - local report_file - report_file=$(mktemp) - - { - echo "Worker Health Report" - echo "===================" - echo "Timestamp: $(date)" - echo - - echo "System Resources:" - echo "----------------" - df -h / - echo - free -h - echo - uptime - echo - - echo "Services Status:" - echo "---------------" - supervisorctl status - echo - - echo "Authentication Status:" - echo "--------------------" - for provider in aws gcp azure bitwarden; do - if is_provider_configured "$provider"; then - echo "$provider: Configured" - else - echo "$provider: Not configured" - fi - done - } > "$report_file" - - case $format in - json) - # Convert report to JSON format - jq -R -s '{ - timestamp: now, - system_resources: { - disk: (input | match("^/dev.*$")), - memory: (input | match("^Mem:.*$")), - load: (input | match("^load average:.*$")) - }, - services: (input | match("^RUNNING.*$")), - auth: (input | match("^.*: Configured$")) - }' "$report_file" - ;; - text) - cat "$report_file" - ;; - *) - log_error "Health" "Unknown format: $format" - rm -f "$report_file" - return 1 - ;; - esac - - rm -f "$report_file" -} # Handle health commands health_handler() { - local cmd=$1 + local command=$1 shift - - case $cmd in - check) - check_health - ;; + + case $command in status) - check_health --quiet + check_health "$@" ;; - diag) - check_health --verbose - ;; - report) - generate_report "$@" - ;; - help) + ""|-h|--help) health_help ;; *) - log_error "Health" "Unknown command: $cmd" + log_error "Health" "Unknown command: $command" health_help - exit 1 + return 1 ;; esac } From 50749a0133b2b91da75b9a9c7f3fb7b1f38c5f2a Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 20 Feb 2025 13:31:41 +0300 Subject: [PATCH 06/25] simplified sbom cli --- lib/cli.sh | 17 ++-- lib/cli/sbom.sh | 243 ++++++++++++++++++++---------------------------- 2 files changed, 107 insertions(+), 153 deletions(-) diff --git a/lib/cli.sh b/lib/cli.sh index aeafd12b..d34786bb 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -3,13 +3,8 @@ # Version information VERSION="1.0.0" -# Source worker configuration -# shellcheck source=${WORKER_LIB_DIR}/worker_config.sh disable=SC1091 -source "${WORKER_LIB_DIR}/worker_config.sh" - -# Load and export configuration -config=$(load_and_parse_config) -export_variables_from_config "$config" +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" # Dynamically source all command modules for module in "${WORKER_LIB_DIR}/cli/"*.sh; do @@ -17,9 +12,6 @@ for module in "${WORKER_LIB_DIR}/cli/"*.sh; do source "$module" done -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - # Print version information show_version() { log_info "Worker CLI version $VERSION" @@ -148,6 +140,11 @@ fi # Handle app and service commands if [ "$1" = "app" ] || [ "$1" = "service" ]; then + # Source and load configuration for app/service commands + source "${WORKER_LIB_DIR}/worker_config.sh" + config=$(load_and_parse_config) + export_variables_from_config "$config" + log_info "Starting process manager..." "${WORKER_LIB_DIR}/process_manager.sh" pm_status=$? diff --git a/lib/cli/sbom.sh b/lib/cli/sbom.sh index 29c55616..67323f35 100644 --- a/lib/cli/sbom.sh +++ b/lib/cli/sbom.sh @@ -12,44 +12,37 @@ Usage: worker sbom [command] Available Commands: generate Generate SBOM in various formats - analyze Analyze dependencies for security issues - export Export SBOM to file - deps Show dependency tree verify Verify package integrity - updates Check for available updates Options: - --format Output format (text/json/cyclonedx/spdx) - --file Output file for export + --format Output format (text/json) --type Package type (system/python/all) --filter Filter packages by name pattern Examples: worker sbom generate worker sbom generate --format json - worker sbom export --format cyclonedx --file sbom.xml - worker sbom deps --type python worker sbom verify EOF } # Description: Generate Software Bill of Materials in various formats -# Options: --format text|json|cyclonedx|spdx, --type system|python|all, --filter PATTERN +# Options: --format text|json, --type system|python|all, --filter PATTERN # Example: worker sbom generate --format json --type python --filter requests generate_sbom() { local format=${1:-text} local type=${2:-all} local filter=$3 - log_info "SBOM" "Generating software bill of materials..." + log_info "SBOM" "Generating software bill of materials..." >&2 # Get system packages local system_packages if [[ $type == "all" || $type == "system" ]]; then if [ -n "$filter" ]; then - system_packages=$(dpkg-query -W -f='${binary:Package}\t${Version}\t${Architecture}\t${Status}\n' | grep "$filter") + system_packages=$(dpkg-query -W -f='${binary:Package}\t${Version}\t${Architecture}\t${Status}\n' | grep "$filter" 2>/dev/null) else - system_packages=$(dpkg-query -W -f='${binary:Package}\t${Version}\t${Architecture}\t${Status}\n') + system_packages=$(dpkg-query -W -f='${binary:Package}\t${Version}\t${Architecture}\t${Status}\n' 2>/dev/null) fi fi @@ -57,53 +50,76 @@ generate_sbom() { local python_packages if [[ $type == "all" || $type == "python" ]]; then if [ -n "$filter" ]; then - python_packages=$(pip list --format=json | jq -r '.[] | select(.name | contains("'"$filter"'")) | [.name, .version] | @tsv') + python_packages=$(pip list --format=json 2>/dev/null | jq -r '.[] | select(.name | contains("'"$filter"'")) | [.name, .version] | @tsv') else - python_packages=$(pip list --format=json | jq -r '.[] | [.name, .version] | @tsv') + python_packages=$(pip list --format=json 2>/dev/null | jq -r '.[] | [.name, .version] | @tsv') fi fi case $format in json) { + # Start JSON object echo "{" + + # Track if we need a comma between sections + local need_comma=false + + # System packages section if [ -n "$system_packages" ]; then echo ' "system_packages": {' - echo "$system_packages" | awk -F'\t' '{ - printf(" \"%s\": {\n \"version\": \"%s\",\n \"architecture\": \"%s\"\n }%s\n", $1, $2, $3, (NR==1?",":"")) - }' - echo " }," + # Convert to array for processing + mapfile -t packages <<< "$system_packages" + local total=${#packages[@]} + local count=0 + + for pkg in "${packages[@]}"; do + count=$((count + 1)) + IFS=$'\t' read -r name version arch _ <<< "$pkg" + printf ' "%s": { + "version": "%s", + "architecture": "%s" + }' "$name" "$version" "$arch" + if [ $count -lt $total ]; then + echo "," + else + echo "" + fi + done + echo " }" + need_comma=true fi + + # Python packages section if [ -n "$python_packages" ]; then + if $need_comma; then + echo "," + fi echo ' "python_packages": {' - echo "$python_packages" | awk -F'\t' '{ - printf(" \"%s\": {\n \"version\": \"%s\"\n }%s\n", $1, $2, (NR==1?",":"")) - }' + # Convert to array for processing + mapfile -t packages <<< "$python_packages" + local total=${#packages[@]} + local count=0 + + for pkg in "${packages[@]}"; do + count=$((count + 1)) + IFS=$'\t' read -r name version <<< "$pkg" + printf ' "%s": { + "version": "%s" + }' "$name" "$version" + if [ $count -lt $total ]; then + echo "," + else + echo "" + fi + done echo " }" fi + + # Close JSON object echo "}" } | jq '.' ;; - cyclonedx) - # Generate CycloneDX XML format - { - echo '' - echo '' - echo ' ' - if [ -n "$system_packages" ]; then - echo "$system_packages" | awk -F'\t' '{ - printf(" \n %s\n %s\n pkg:deb/%s@%s?arch=%s\n \n", $1, $2, $1, $2, $3) - }' - fi - if [ -n "$python_packages" ]; then - echo "$python_packages" | awk -F'\t' '{ - printf(" \n %s\n %s\n pkg:pypi/%s@%s\n \n", $1, $2, $1, $2) - }' - fi - echo ' ' - echo '' - } - ;; text) { echo "Software Bill of Materials" @@ -131,60 +147,12 @@ generate_sbom() { } ;; *) - log_error "SBOM" "Unknown format: $format" + log_error "SBOM" "Unknown format: $format" >&2 return 1 ;; esac } -# Description: Analyze dependencies for security vulnerabilities -# Options: --type system|python|all, --severity low|medium|high|critical -# Example: worker sbom analyze --type python --severity high -analyze_deps() { - log_info "SBOM" "Analyzing dependencies for security issues..." - - # Check system packages - if command -v apt-get >/dev/null; then - apt-get update -qq - apt-get --simulate upgrade | grep -i security - fi - - # Check Python packages - if command -v pip-audit >/dev/null; then - pip-audit - else - log_warn "SBOM" "pip-audit not installed. Install with: pip install pip-audit" - fi -} - -# Description: Export SBOM to a file in specified format -# Options: --format text|json|cyclonedx|spdx, --file PATH -# Example: worker sbom export --format cyclonedx --file sbom.xml -export_sbom() { - local format=$1 - local file=$2 - - if [ -z "$file" ]; then - log_error "SBOM" "Output file is required" - return 1 - fi - - generate_sbom "$format" > "$file" - log_success "SBOM" "SBOM exported to $file" -} - -# Description: Display dependency tree for installed packages -# Options: --type system|python|all, --format text|json -# Example: worker sbom deps --type python --format json -show_deps() { - local type=${1:-all} - - if [[ $type == "all" || $type == "python" ]]; then - log_info "SBOM" "Python dependencies:" - pip install pipdeptree >/dev/null 2>&1 - pipdeptree - fi -} # Description: Verify integrity of installed packages # Options: --type system|python|all @@ -196,96 +164,85 @@ verify_packages() { local failed=0 while IFS= read -r pkg; do if ! dpkg -V "$pkg" >/dev/null 2>&1; then - log_error "SBOM" "Package integrity check failed: $pkg" + log_error "SBOM" "Package integrity check failed: $pkg" >&2 failed=1 fi - done < <(dpkg-query -W -f='${binary:Package}\n') + done < <(dpkg-query -f '${Package}\n' -W) if [ $failed -eq 0 ]; then - log_success "SBOM" "All packages verified successfully" - else - return 1 + log_success "SBOM" "All packages verified successfully" >&2 fi } -# Description: Check for available package updates -# Options: --type system|python|all, --format text|json -# Example: worker sbom updates --type all --format json -check_updates() { - log_info "SBOM" "Checking for available updates..." - - # Check system packages - apt-get update -qq - apt-get --just-print upgrade 2>&1 | awk '/^Inst/ { print $2 " (" $3 " => " $4 ")" }' - - # Check Python packages - pip list --outdated --format=json | jq -r '.[] | "\(.name) (\(.version) => \(.latest_version))"' -} - # Parse command line arguments parse_args() { - local args=() + local command=$1 + shift + + local format="text" + local type="all" + local filter="" + while [[ $# -gt 0 ]]; do case $1 in --format) - format=$2 - shift 2 - ;; - --file) - file=$2 + format="$2" shift 2 ;; --type) - type=$2 + type="$2" shift 2 ;; --filter) - filter=$2 + filter="$2" shift 2 ;; *) - args+=("$1") - shift + log_error "SBOM" "Unknown option: $1" >&2 + return 1 ;; esac done - set -- "${args[@]}" + + echo "$command" "$format" "$type" "$filter" } # Handle sbom commands sbom_handler() { - local cmd=$1 - shift - - # Parse command line arguments - parse_args "$@" + if [ $# -eq 0 ]; then + sbom_help + return 0 + fi + + local args + read -r command format type filter <<< "$(parse_args "$@")" - case $cmd in + case $command in generate) - generate_sbom "$format" "$type" "$filter" - ;; - analyze) - analyze_deps - ;; - export) - export_sbom "$format" "$file" - ;; - deps) - show_deps "$type" + if [ "$format" = "json" ]; then + # Capture all output + output=$(generate_sbom "$format" "$type" "$filter" 2>&1) + # Extract only the JSON part (everything between first { and last }) + echo "$output" | awk '/^{/,/^}$/ {print}' + else + generate_sbom "$format" "$type" "$filter" + fi ;; verify) verify_packages ;; - updates) - check_updates - ;; - help) + help|"") sbom_help ;; *) - log_error "SBOM" "Unknown command: $cmd" + log_error "SBOM" "Unknown command: $command" >&2 sbom_help - exit 1 + return 1 ;; esac -} \ No newline at end of file +} + +# If script is being executed directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + sbom_handler "$@" +fi From 083028663260d3fddbc677eeb278009c3c09a3e3 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 20 Feb 2025 15:23:30 +0300 Subject: [PATCH 07/25] fixed env cli --- lib/cli/auth.sh | 87 ++++++++++++++++++---- lib/cli/env.sh | 189 ++++++++++++++++++------------------------------ 2 files changed, 145 insertions(+), 131 deletions(-) diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh index da42c49c..ae6db8b7 100644 --- a/lib/cli/auth.sh +++ b/lib/cli/auth.sh @@ -9,27 +9,38 @@ auth_help() { cat << EOF Manage authentication and credentials -Usage: worker auth [command] [provider] +Usage: worker auth [command] [provider] [options] Available Commands: status Show authentication status for all or specific provider login Re-authenticate with provider(s) using available credentials logout Log out from provider(s) +Options: + --format Output format for status command (e.g. json) + Examples: - worker auth status # Show status of all providers - worker auth status azure # Show status of Azure only - worker auth login # Re-auth all providers with available creds + worker auth status # Show status of all providers + worker auth status azure # Show status of Azure only + worker auth status --format json # Show status in JSON format + worker auth login # Re-auth all providers with available creds worker auth login azure # Re-auth Azure only worker auth logout # Log out from all providers EOF } # Description: Display authentication status for all cloud providers -# Example: worker auth status +# Example: worker auth status [--format json] show_auth_status() { local target_provider=$1 - log_info "Auth" "Checking authentication status..." + local format=$2 + + if [ "$format" != "json" ]; then + log_info "Auth" "Checking authentication status..." + fi + + # Initialize JSON array if json format + local json_output="[" # Function to check a specific provider check_provider_status() { @@ -87,20 +98,38 @@ show_auth_status() { is_authenticated=true fi - # Show status based on env vars and config + # Prepare status message and state + local status_msg state if [ "$has_env_creds" = true ] || [ "$has_config_creds" = true ]; then if [ "$is_authenticated" = true ]; then - log_success "Auth" "$provider: Active session${types:+ (actors: $types)}" + status_msg="Active session${types:+ (actors: $types)}" + state="active" else - log_info "Auth" "$provider: Has credentials${types:+ (actors: $types)}, needs re-auth" + status_msg="Has credentials${types:+ (actors: $types)}, needs re-auth" + state="needs_reauth" fi else if [ -n "$types" ]; then - log_warn "Auth" "$provider: Missing credentials (actors: $types)" + status_msg="Missing credentials (actors: $types)" + state="missing_creds" else - log_info "Auth" "$provider: Not configured" + status_msg="Not configured" + state="not_configured" fi fi + + # Output based on format + if [ "$format" = "json" ]; then + [ "$json_output" != "[" ] && json_output+="," + json_output+="{\"provider\":\"$provider\",\"state\":\"$state\",\"message\":\"$status_msg\"}" + else + case "$state" in + "active") log_success "Auth" "$provider: $status_msg" ;; + "needs_reauth") log_info "Auth" "$provider: $status_msg" ;; + "missing_creds") log_warn "Auth" "$provider: $status_msg" ;; + "not_configured") log_info "Auth" "$provider: $status_msg" ;; + esac + fi } if [ -n "$target_provider" ]; then @@ -110,6 +139,12 @@ show_auth_status() { check_provider_status "$provider" done fi + + # Close and output JSON if json format + if [ "$format" = "json" ]; then + json_output+="]" + echo "$json_output" + fi } # Login to provider(s) @@ -218,11 +253,37 @@ check_provider_auth() { # Handle auth commands auth_handler() { local cmd=$1 - local provider=$2 + shift case $cmd in status) - show_auth_status "$provider" + local provider="" + local format="" + + # Parse arguments + while [ $# -gt 0 ]; do + case "$1" in + --format) + format="$2" + shift 2 + ;; + --*) + log_error "CLI" "Unknown option: $1" + return 1 + ;; + *) + if [ -z "$provider" ]; then + provider="$1" + else + log_error "CLI" "Unexpected argument: $1" + return 1 + fi + shift + ;; + esac + done + + show_auth_status "$provider" "$format" return $? ;; login) diff --git a/lib/cli/env.sh b/lib/cli/env.sh index 34032f78..0518ad75 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -17,23 +17,19 @@ Available Commands: unset Unset an environment variable reload Reload environment from config status Show environment status - validate Validate environment variables - export Export environment to file - import Import environment from file Options: --format Output format (text/json) --filter Filter variables by prefix - --file File to export to or import from --include-secrets Include secrets in output (masked) Examples: - worker env show - worker env show --format json - worker env show --filter AWS_* - worker env show --include-secrets - worker env set MY_VAR "my value" - worker env reload + worker env show # Show all environment variables + worker env show --format json # Show variables in JSON format + worker env show --filter AWS_* # Show only AWS variables + worker env set MY_VAR "my value" # Set a new variable + worker env unset MY_VAR # Remove a variable + worker env reload # Reload from config EOF } @@ -45,6 +41,7 @@ show_environment() { local filter=$2 local include_secrets=${3:-false} + # Check if environment file exists if [ ! -f "$WORKER_ENV_FILE" ]; then log_error "Env" "Environment file not found" @@ -63,9 +60,12 @@ show_environment() { # Convert to JSON local json="{" + local first=true while IFS= read -r line; do if [[ $line =~ ^export[[:space:]]+([^=]+)=\"([^\"]*)\" ]]; then - if [ -n "$json" ] && [ "$json" != "{" ]; then + if [ "$first" = true ]; then + first=false + else json="$json," fi key=${BASH_REMATCH[1]} @@ -74,7 +74,11 @@ show_environment() { fi done <<< "$vars" json="$json}" - echo "$json" | jq . + if command -v jq >/dev/null 2>&1; then + echo "$json" | jq . + else + echo "$json" + fi ;; text) if [ -n "$filter" ]; then @@ -96,8 +100,13 @@ set_environment() { local name=$1 local value=$2 - if [ -z "$name" ] || [ -z "$value" ]; then - log_error "Env" "Both variable name and value are required" + if [ -z "$name" ]; then + log_error "Env" "Variable name is required" + return 1 + fi + + if [ -z "$value" ] && [ "$#" -lt 2 ]; then + log_error "Env" "Variable value is required" return 1 fi @@ -155,53 +164,7 @@ unset_environment() { fi } -# Description: Export environment variables to a file -# Options: --file PATH, --include-secrets -# Example: worker env export --file env.backup --include-secrets -export_environment() { - local file=$1 - local filter=$2 - - if [ -z "$file" ]; then - log_error "Env" "Output file is required" - return 1 - fi - - # Export variables - if [ -n "$filter" ]; then - env | grep "^$filter" > "$file" - else - env > "$file" - fi - - log_success "Env" "Environment variables exported to $file" -} -# Description: Import environment variables from a file -# Options: --file PATH -# Example: worker env import --file env.backup -import_environment() { - local file=$1 - - if [ -z "$file" ]; then - log_error "Env" "Input file is required" - return 1 - fi - - if [ ! -f "$file" ]; then - log_error "Env" "File not found: $file" - return 1 - fi - - # Import variables - while IFS='=' read -r key value; do - if [ -n "$key" ]; then - export "$key=$value" - fi - done < "$file" - - log_success "Env" "Environment variables imported from $file" -} # Description: Validate environment variables against schema # Options: --format text|json @@ -280,52 +243,7 @@ reset_environment() { } # Parse command line arguments -parse_args() { - # Initialize variables with defaults - format="text" - filter="" - file="" - include_secrets="false" - - local args=() - while [[ $# -gt 0 ]]; do - case $1 in - --format=*|--type=*) - format="${1#*=}" - shift - ;; - --format|--type) - format="$2" - shift 2 - ;; - --filter=*) - filter="${1#*=}" - shift - ;; - --filter) - filter="$2" - shift 2 - ;; - --file=*) - file="${1#*=}" - shift - ;; - --file) - file="$2" - shift 2 - ;; - --include-secrets) - include_secrets="true" - shift - ;; - *) - args+=("$1") - shift - ;; - esac - done - set -- "${args[@]}" -} + # Description: Show environment status and validation results # Options: --format text|json @@ -389,15 +307,59 @@ env_handler() { local cmd=$1 shift - # Parse command line arguments - parse_args "$@" + # Store original arguments for set/unset commands + local orig_args=("$@") + + # Default values + local format="text" + local filter="" + local include_secrets="false" + + # Parse arguments for show/status commands + if [[ "$cmd" == "show" || "$cmd" == "status" ]]; then + while [[ $# -gt 0 ]]; do + case $1 in + --format=*|--type=*) + format="${1#*=}" + shift + ;; + --format|--type) + if [[ -n "$2" && ! "$2" =~ ^-- ]]; then + format="$2" + shift 2 + else + shift + fi + ;; + --filter=*) + filter="${1#*=}" + shift + ;; + --filter) + if [[ -n "$2" && ! "$2" =~ ^-- ]]; then + filter="$2" + shift 2 + else + shift + fi + ;; + --include-secrets) + include_secrets="true" + shift + ;; + *) + shift + ;; + esac + done + fi case $cmd in show) show_environment "$format" "$filter" "$include_secrets" ;; set) - set_environment "$1" "$2" + set_environment "${orig_args[0]}" "${orig_args[1]}" ;; unset) unset_environment "$1" @@ -409,15 +371,6 @@ env_handler() { status) show_status "$format" ;; - validate) - validate_environment - ;; - export) - export_environment "$file" "$filter" - ;; - import) - import_environment "$file" - ;; help) env_help ;; From 6660a17033b72cff225edc445795f7cf297b878a Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 20 Feb 2025 16:10:03 +0300 Subject: [PATCH 08/25] config cli --- Makefile.variables | 2 +- lib/cli/config.sh | 149 ++++++++++++++++++++++--------------------- lib/cli/service.sh | 4 +- lib/worker_config.sh | 38 +++++++---- 4 files changed, 106 insertions(+), 87 deletions(-) diff --git a/Makefile.variables b/Makefile.variables index ca79ceea..c4853a5f 100644 --- a/Makefile.variables +++ b/Makefile.variables @@ -9,7 +9,7 @@ TESTS_TASKS_DIR ?= ./src/tests/tasks TEST_WORKER_CONFIG ?= ./src/tests/configs/worker.yaml TEST_SERVICES_CONFIG ?= ./src/tests/configs/services.yaml USER = udx -VOLUMES ?= ./src/scripts:/home/$(USER) +VOLUMES ?= DEBUG ?= false COMMAND ?= MULTIPLATFORM ?= false diff --git a/lib/cli/config.sh b/lib/cli/config.sh index 55574068..8abd2c90 100644 --- a/lib/cli/config.sh +++ b/lib/cli/config.sh @@ -14,7 +14,6 @@ Usage: worker config [command] Available Commands: show Show current configuration edit Edit configuration in default editor - validate Validate configuration files locations Show configuration file locations init Initialize a new configuration file diff Show differences between default and current config @@ -22,7 +21,6 @@ Available Commands: Examples: worker config show worker config show --format json - worker config validate worker config edit worker config locations worker config init @@ -32,19 +30,49 @@ EOF # Description: Display current worker configuration # Options: --format yaml|json # Example: worker config show --format json -show_config() { - local format=${1:-yaml} +config_show() { + local format="yaml" + local args=("$@") + local i=0 + + # Parse arguments + while [ $i -lt ${#args[@]} ]; do + case "${args[$i]}" in + --format) + i=$((i + 1)) + format="${args[$i]}" + ;; + *) + log_error "Config" "Unknown argument: ${args[$i]}" + return 1 + ;; + esac + i=$((i + 1)) + done + log_info "Config" "Current configuration:" + # Check if user config exists + if [ ! -f "$USER_CONFIG" ] || [ ! -s "$USER_CONFIG" ]; then + log_info "Config" "No user configuration found at $USER_CONFIG" + log_info "Config" "Use 'worker config init' to create one" + return 0 + fi + + # Parse user config local config - config=$(load_and_parse_config) - + if ! config=$(yq eval -o=json "$USER_CONFIG" 2>/dev/null); then + log_error "Config" "Failed to parse user configuration" + return 1 + fi + + # Show config based on format case $format in json) - echo "$config" | yq eval -o=json '.' + echo "$config" ;; yaml) - echo "$config" + yq eval "$USER_CONFIG" ;; *) log_error "Config" "Unknown format: $format" @@ -63,61 +91,37 @@ edit_config() { # Create file if it doesn't exist if [ ! -f "$config_file" ]; then - cp "/etc/worker/worker.yaml" "$config_file" + # Copy built-in config first to ensure actors section is preserved + if [ -f "$BUILT_IN_CONFIG" ]; then + cp "$BUILT_IN_CONFIG" "$config_file" || { + log_error "Config" "Failed to copy built-in configuration" + return 1 + } + else + # If built-in config doesn't exist, create minimal structure + cat > "$config_file" << EOF +kind: workerConfig +version: udx.io/worker-v1/config +config: {} +EOF + fi fi # Use default editor or fallback to nano ${EDITOR:-nano} "$config_file" - # Validate after editing - if validate_config "$config_file"; then - log_success "Config" "Configuration updated successfully" - else - log_error "Config" "Configuration validation failed after editing" + # Basic YAML validation + if ! yq eval '.' "$config_file" >/dev/null 2>&1; then + log_error "Config" "Configuration is invalid YAML" return 1 fi -} - -# Description: Validate all configuration files or a specific one -# Options: --file PATH -# Example: worker config validate --file ~/.config/worker/worker.yaml -validate_config() { - local config_file=${1:-} - log_info "Config" "Validating configuration..." - # If no file specified, validate all config files - if [ -z "$config_file" ]; then - local files=( - "/etc/worker/worker.yaml" - "${HOME}/.config/worker/worker.yaml" - "/etc/worker/supervisor/supervisord.conf" - ) - - local failed=0 - for file in "${files[@]}"; do - if [ -f "$file" ]; then - if ! validate_yaml "$file"; then - log_error "Config" "Validation failed for $file" - failed=1 - else - log_success "Config" "Validation passed for $file" - fi - fi - done - - return $failed - else - # Validate specific file - if validate_yaml "$config_file"; then - log_success "Config" "Configuration is valid" - return 0 - else - log_error "Config" "Configuration is invalid" - return 1 - fi - fi + log_success "Config" "Configuration saved successfully" + return 0 } + + # Description: Display paths of all configuration files # Example: worker config locations show_locations() { @@ -125,8 +129,6 @@ show_locations() { Configuration Locations: System config: /etc/worker/worker.yaml User config: ${HOME}/.config/worker/worker.yaml - Supervisor config: /etc/worker/supervisor/supervisord.conf - Services config: ${HOME}/.config/worker/services.yaml EOF } @@ -136,26 +138,29 @@ init_config() { local config_dir="${HOME}/.config/worker" local config_file="$config_dir/worker.yaml" - # Create directory if it doesn't exist - mkdir -p "$config_dir" - - # Don't overwrite existing config without confirmation + # Skip if config already exists if [ -f "$config_file" ]; then - log_warn "Config" "Configuration file already exists at $config_file" - read -r -p "Do you want to overwrite it? [y/N] " response - if [[ ! "$response" =~ ^[Yy]$ ]]; then - log_info "Config" "Initialization cancelled" - return 0 - fi + log_info "Config" "Configuration already exists at $config_file" + return 0 fi - # Copy default config - cp "/etc/worker/worker.yaml" "$config_file" + # Create directory if it doesn't exist + mkdir -p "$config_dir" + # Create new config with timestamp + cat > "$config_file" << EOF +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + CREATED: "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +EOF + if [ -f "$config_file" ]; then log_success "Config" "Configuration initialized at $config_file" + return 0 else - log_error "Config" "Failed to initialize configuration" + log_error "Config" "Failed to create configuration file" return 1 fi } @@ -183,14 +188,11 @@ config_handler() { case $cmd in show) - show_config "$@" + config_show "$@" ;; edit) edit_config ;; - validate) - validate_config "$@" - ;; locations) show_locations ;; @@ -203,6 +205,9 @@ config_handler() { help) config_help ;; + "") + config_help + ;; *) log_error "Config" "Unknown command: $cmd" config_help diff --git a/lib/cli/service.sh b/lib/cli/service.sh index b2baadc9..419499f5 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -21,7 +21,7 @@ service_handler() { follow_logs "$1" "err" ;; config) - show_config + service_show_config ;; start|stop|restart) manage_service "$cmd" "$1" @@ -186,7 +186,7 @@ follow_logs() { # Description: Display current service configuration # Example: worker service config -show_config() { +service_show_config() { if [ ! -f "/etc/supervisord.conf" ]; then log_error "Service" "Configuration file is not generated since no services are managed." exit 1 diff --git a/lib/worker_config.sh b/lib/worker_config.sh index a65826ff..c4e67ff3 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -32,26 +32,40 @@ merge_worker_configs() { touch "$MERGED_CONFIG" || { log_error "Worker configuration" "Failed to create merged configuration file at $MERGED_CONFIG"; return 1; } fi - # Ensure built-in config exists - ensure_config_exists "$BUILT_IN_CONFIG" || return 1 + # Ensure built-in config exists and has actors section + if ! ensure_config_exists "$BUILT_IN_CONFIG"; then + log_error "Worker configuration" "Built-in configuration not found" + return 1 + fi - # If a user-provided configuration exists (and path is not empty), merge it - if [[ -f "$USER_CONFIG" && -n "$USER_CONFIG" ]]; then - log_success "Worker configuration" "User configuration detected at $USER_CONFIG" + # First copy built-in config (with actors) to merged config + if ! cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG"; then + log_error "Worker configuration" "Failed to copy built-in configuration" + return 1 + fi - if ! yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' "$BUILT_IN_CONFIG" "$USER_CONFIG" > "$MERGED_CONFIG"; then - log_error "Worker configuration" "Failed to merge configurations. yq returned an error." + # If user config exists, merge env and secrets sections + if [[ -f "$USER_CONFIG" && -s "$USER_CONFIG" ]]; then + # Use yq to merge configs, preserving actors from built-in + if ! yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' "$MERGED_CONFIG" "$USER_CONFIG" > "${MERGED_CONFIG}.tmp"; then + log_error "Worker configuration" "Failed to merge configurations" return 1 fi - else - # Using default config - # Copy the built-in configuration to the merged configuration - if ! cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG"; then - log_error "Worker configuration" "Failed to copy built-in configuration to merged configuration." + # Check if merge was successful + if [ -s "${MERGED_CONFIG}.tmp" ]; then + mv "${MERGED_CONFIG}.tmp" "$MERGED_CONFIG" + log_success "Worker configuration" "Using user configuration at $USER_CONFIG" + return 0 + else + rm -f "${MERGED_CONFIG}.tmp" + log_error "Worker configuration" "Failed to merge configurations - empty result" return 1 fi fi + + # No user config + log_info "Worker configuration" "No user configuration found at $USER_CONFIG, using built-in defaults" } # Load and parse the merged configuration From dc88df0bd6040be9d6fcac8ae48c74b3b80361e4 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 20 Feb 2025 17:37:49 +0300 Subject: [PATCH 09/25] improvements --- lib/auth.sh | 71 +++--- lib/cli.sh | 6 - lib/cli/auth.sh | 140 +++++------ lib/cli/config.sh | 65 ++++-- lib/cli/env.sh | 17 +- lib/cli/info.sh | 513 ----------------------------------------- lib/cli/logs.sh | 281 ---------------------- lib/env_handler.sh | 6 - lib/process_manager.sh | 2 +- lib/worker_config.sh | 4 - 10 files changed, 148 insertions(+), 957 deletions(-) delete mode 100644 lib/cli/info.sh delete mode 100644 lib/cli/logs.sh diff --git a/lib/auth.sh b/lib/auth.sh index 4b15b186..e3bd4878 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -6,66 +6,59 @@ source ${WORKER_LIB_DIR}/utils.sh # Array to track configured providers declare -a configured_providers=() -# Function to get env var names for a provider from config +# Function to get env var names for a provider from actors JSON get_provider_env_vars() { local provider=$1 - local config - config=$(load_and_parse_config) + local actors_json=$2 # Get all env var names from actor creds that match ${VAR} pattern - if echo "$config" | jq -e '.config.actors' >/dev/null 2>&1; then - echo "$config" | jq -r ".config.actors[] | select(.type | startswith(\"$provider\")) | .creds" 2>/dev/null | \ + echo "$actors_json" | jq -r ".[].creds" 2>/dev/null | \ grep -o '\${[^}]*}' | sed 's/[\${}]//g' || true - fi } # Function to check if a provider is configured is_provider_configured() { local provider=$1 + local actors_json=$2 + + # Get provider's actors + local provider_actors + provider_actors=$(echo "$actors_json" | jq -r "[.[] | select(.type | startswith(\"$provider\"))]" 2>/dev/null) + + if [ -z "$provider_actors" ] || [ "$provider_actors" = "[]" ]; then + return 1 + fi - # Get all possible env var names from config + # Get all possible env var names from actors local env_vars - mapfile -t env_vars < <(get_provider_env_vars "$provider") + mapfile -t env_vars < <(get_provider_env_vars "$provider" "$provider_actors") - # First check all possible env vars + # Check all possible env vars for env_var in "${env_vars[@]}"; do if [ -n "${!env_var}" ]; then return 0 fi done - # Fallback to config if no env vars are set - local config - config=$(load_and_parse_config) - - # Check if we have a valid config with actors - if echo "$config" | jq -e '.config.actors' >/dev/null 2>&1; then - # Get all actors for this provider - local actors - actors=$(echo "$config" | jq -r ".config.actors[] | select(.type | startswith(\"$provider-\"))" 2>/dev/null) + # Check each actor's credentials + while IFS= read -r actor; do + [ -z "$actor" ] && continue - if [ -n "$actors" ] && [ "$actors" != "null" ]; then - # Check each actor's credentials - while IFS= read -r actor; do - [ -z "$actor" ] && continue - - local creds - creds=$(echo "$actor" | jq -r '.creds' 2>/dev/null) - [ "$creds" = "null" ] && continue - - # Evaluate creds as a reference to an environment variable - if [[ "$creds" =~ ^\$\{(.+)\}$ ]]; then - local env_var_name="${BASH_REMATCH[1]}" - creds="${!env_var_name}" - fi - - # If we find any valid credentials, return success - if [ -n "$creds" ]; then - return 0 - fi - done <<< "$actors" + local creds + creds=$(echo "$actor" | jq -r '.creds' 2>/dev/null) + [ "$creds" = "null" ] && continue + + # Evaluate creds as a reference to an environment variable + if [[ "$creds" =~ ^\$\{(.+)\}$ ]]; then + local env_var_name="${BASH_REMATCH[1]}" + creds="${!env_var_name}" fi - fi + + # If we find any valid credentials, return success + if [ -n "$creds" ]; then + return 0 + fi + done <<< "$(echo "$provider_actors" | jq -r '.[]')" return 1 } diff --git a/lib/cli.sh b/lib/cli.sh index d34786bb..d93f370f 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -97,12 +97,6 @@ EOF cat << EOF -Examples: - worker help service Learn about service configuration - worker env show View current environment - worker info system Check system health - worker auth setup Configure cloud provider auth - Tip: Run 'worker help [command]' for detailed information about any command EOF } diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh index ae6db8b7..82b089df 100644 --- a/lib/cli/auth.sh +++ b/lib/cli/auth.sh @@ -42,105 +42,73 @@ show_auth_status() { # Initialize JSON array if json format local json_output="[" + # Load config once and extract actors section + local config actors + config=$(load_and_parse_config) + actors=$(get_config_section "$config" "actors") + # Function to check a specific provider check_provider_status() { local provider=$1 - local has_env_creds=false - local has_config_creds=false + local status="Not configured" local types="" - # First check env vars from config - local env_vars - mapfile -t env_vars < <(get_provider_env_vars "$provider") - - for env_var in "${env_vars[@]}"; do - if [ -n "${!env_var}" ]; then - has_env_creds=true - break - fi - done - - # Then check config - local config - config=$(load_and_parse_config) + # Get provider's actors + local provider_actors + provider_actors=$(echo "$actors" | jq -r "[.[] | select(.type | startswith(\"$provider\"))]" 2>/dev/null) - if echo "$config" | jq -e '.config.actors' >/dev/null 2>&1; then - local actors - actors=$(echo "$config" | jq -r ".config.actors[] | select(.type | startswith(\"$provider-\"))" 2>/dev/null) + if [ -n "$provider_actors" ] && [ "$provider_actors" != "[]" ]; then + types=$(echo "$provider_actors" | jq -r '.[].type' 2>/dev/null | tr '\n' ' ') - if [ -n "$actors" ] && [ "$actors" != "null" ]; then - types=$(echo "$actors" | jq -r '.type' 2>/dev/null | grep . | tr '\n' ' ') - - while IFS= read -r actor; do - [ -z "$actor" ] && continue - - local creds - creds=$(echo "$actor" | jq -r '.creds' 2>/dev/null) - [ "$creds" = "null" ] && continue - - # Evaluate creds as a reference to an environment variable - if [[ "$creds" =~ ^\$\{(.+)\}$ ]]; then - local env_var_name="${BASH_REMATCH[1]}" - creds="${!env_var_name}" - fi - - if [ -n "$creds" ]; then - has_config_creds=true - break - fi - done <<< "$actors" - fi - fi - - # Check if currently authenticated - local is_authenticated=false - if check_provider_auth "$provider"; then - is_authenticated=true - fi - - # Prepare status message and state - local status_msg state - if [ "$has_env_creds" = true ] || [ "$has_config_creds" = true ]; then - if [ "$is_authenticated" = true ]; then - status_msg="Active session${types:+ (actors: $types)}" - state="active" + # First check if provider has credentials + if is_provider_configured "$provider" "$provider_actors"; then + # Then check if it's authenticated + if check_provider_auth "$provider"; then + status="Authenticated" + state="active" + else + status="Needs re-auth" + state="needs_reauth" + fi else - status_msg="Has credentials${types:+ (actors: $types)}, needs re-auth" - state="needs_reauth" - fi - else - if [ -n "$types" ]; then - status_msg="Missing credentials (actors: $types)" + status="Not configured" state="missing_creds" - else - status_msg="Not configured" - state="not_configured" fi + else + status="Not configured" + state="not_configured" fi - - # Output based on format + + # Output status based on format if [ "$format" = "json" ]; then - [ "$json_output" != "[" ] && json_output+="," - json_output+="{\"provider\":\"$provider\",\"state\":\"$state\",\"message\":\"$status_msg\"}" + [ -n "$json_output" ] && [ "$json_output" != "[" ] && json_output+="," + json_output+=$(jq -n \ + --arg provider "$provider" \ + --arg state "$state" \ + --arg status "$status" \ + --arg types "$types" \ + '{provider: $provider, state: $state, status: $status, types: $types}') else case "$state" in - "active") log_success "Auth" "$provider: $status_msg" ;; - "needs_reauth") log_info "Auth" "$provider: $status_msg" ;; - "missing_creds") log_warn "Auth" "$provider: $status_msg" ;; - "not_configured") log_info "Auth" "$provider: $status_msg" ;; + "active") log_success "Auth" "$provider: $status" ;; + "needs_reauth") log_info "Auth" "$provider: $status" ;; + "missing_creds") log_warn "Auth" "$provider: $status" ;; + "not_configured") log_info "Auth" "$provider: $status" ;; esac fi } + # Check status for specific provider or all providers if [ -n "$target_provider" ]; then check_provider_status "$target_provider" else - for provider in aws gcp azure bitwarden; do - check_provider_status "$provider" - done + check_provider_status "aws" + check_provider_status "gcp" + check_provider_status "azure" + check_provider_status "bitwarden" fi - - # Close and output JSON if json format + + # Close JSON array if json format if [ "$format" = "json" ]; then json_output+="]" echo "$json_output" @@ -152,27 +120,27 @@ login_provider() { local target_provider=$1 log_info "Auth" "Authenticating providers..." - # Load config and get actors - local config actors_json + # Load config once and extract actors section + local config actors config=$(load_and_parse_config) - actors_json=$(echo "$config" | jq -r '.config.actors') + actors=$(get_config_section "$config" "actors") - if [[ -z "$actors_json" || "$actors_json" == "null" ]]; then + if [[ -z "$actors" || "$actors" == "null" ]]; then log_warn "Auth" "No providers found in configuration" return 1 fi # Filter actors by provider if specified if [[ -n "$target_provider" ]]; then - actors_json=$(echo "$config" | jq -r ".config.actors | map(select(.type | startswith(\"$target_provider-\")))") - if [[ "$actors_json" == "[]" ]]; then + actors=$(echo "$actors" | jq -r "[.[] | select(.type | startswith(\"$target_provider\"))]") + if [[ "$actors" == "[]" ]]; then log_warn "Auth" "$target_provider: Not configured" return 1 fi fi # Use the same authentication flow as entrypoint - if authenticate_actors "$actors_json"; then + if authenticate_actors "$actors"; then log_success "Auth" "Authentication complete" return 0 else @@ -231,7 +199,9 @@ check_provider_auth() { fi ;; azure) - if az account show &>/dev/null; then + # Azure CLI can return non-zero exit code even when it succeeds + # so we check if the output contains valid JSON + if output=$(az account show 2>/dev/null) && echo "$output" | jq empty &>/dev/null; then return 0 fi ;; diff --git a/lib/cli/config.sh b/lib/cli/config.sh index 8abd2c90..2cc313ea 100644 --- a/lib/cli/config.sh +++ b/lib/cli/config.sh @@ -17,6 +17,7 @@ Available Commands: locations Show configuration file locations init Initialize a new configuration file diff Show differences between default and current config + apply Parse and apply the configuration Examples: worker config show @@ -84,7 +85,7 @@ config_show() { # Description: Edit configuration in default editor # Example: worker config edit edit_config() { - local config_file="${HOME}/.config/worker/worker.yaml" + local config_file="$USER_CONFIG" # Create directory if it doesn't exist mkdir -p "$(dirname "$config_file")" @@ -127,28 +128,26 @@ EOF show_locations() { cat << EOF Configuration Locations: - System config: /etc/worker/worker.yaml - User config: ${HOME}/.config/worker/worker.yaml + Built-in config: $BUILT_IN_CONFIG + User config: $USER_CONFIG + Merged config: $MERGED_CONFIG EOF } # Description: Initialize a new configuration file with defaults # Example: worker config init init_config() { - local config_dir="${HOME}/.config/worker" - local config_file="$config_dir/worker.yaml" - # Skip if config already exists - if [ -f "$config_file" ]; then - log_info "Config" "Configuration already exists at $config_file" + if [ -f "$USER_CONFIG" ]; then + log_info "Config" "Configuration already exists at $USER_CONFIG" return 0 fi # Create directory if it doesn't exist - mkdir -p "$config_dir" + mkdir -p "$(dirname "$USER_CONFIG")" - # Create new config with timestamp - cat > "$config_file" << EOF + # Create minimal config with timestamp + cat > "$USER_CONFIG" << EOF kind: workerConfig version: udx.io/worker-v1/config config: @@ -156,8 +155,9 @@ config: CREATED: "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" EOF - if [ -f "$config_file" ]; then - log_success "Config" "Configuration initialized at $config_file" + if [ -f "$USER_CONFIG" ]; then + log_success "Config" "Configuration initialized at $USER_CONFIG" + merge_worker_configs # Merge with built-in config return 0 else log_error "Config" "Failed to create configuration file" @@ -169,16 +169,40 @@ EOF # Options: --format unified|context|git # Example: worker config diff --format git show_diff() { - local default_config="/etc/worker/worker.yaml" - local user_config="${HOME}/.config/worker/worker.yaml" + # Check if configs exist + if ! ensure_config_exists "$BUILT_IN_CONFIG"; then + return 1 + fi - if [ ! -f "$user_config" ]; then - log_error "Config" "User configuration does not exist at $user_config" + if [ ! -f "$USER_CONFIG" ]; then + log_error "Config" "User configuration does not exist at $USER_CONFIG" return 1 fi - log_info "Config" "Differences between default and current configuration:" - diff -u "$default_config" "$user_config" || true + log_info "Config" "Differences between built-in and user configuration:" + diff -u "$BUILT_IN_CONFIG" "$USER_CONFIG" || true +} + +# Description: Parse and apply the configuration +# Example: worker config apply +apply_config() { + log_info "Config" "Parsing and applying configuration..." + + # Load and parse the configuration + local config_json + if ! config_json=$(load_and_parse_config); then + log_error "Config" "Failed to load and parse configuration" + return 1 + fi + + # Export variables from the configuration + if ! export_variables_from_config "$config_json"; then + log_error "Config" "Failed to export variables from configuration" + return 1 + fi + + log_success "Config" "Configuration successfully parsed and applied" + return 0 } # Handle config commands @@ -196,6 +220,9 @@ config_handler() { locations) show_locations ;; + apply) + apply_config + ;; init) init_config ;; diff --git a/lib/cli/env.sh b/lib/cli/env.sh index 0518ad75..b258cfd3 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -15,7 +15,7 @@ Available Commands: show Show environment variables (excludes secrets) set Set an environment variable unset Unset an environment variable - reload Reload environment from config + reload Reload environment from configuration (same as 'config apply') status Show environment status Options: @@ -365,8 +365,19 @@ env_handler() { unset_environment "$1" ;; reload) - config=$(load_and_parse_config) - export_variables_from_config "$config" + log_info "Env" "Reloading environment from configuration..." + local config_json + if ! config_json=$(load_and_parse_config); then + log_error "Env" "Failed to load and parse configuration" + return 1 + fi + + if ! export_variables_from_config "$config_json"; then + log_error "Env" "Failed to export variables from configuration" + return 1 + fi + + log_success "Env" "Environment successfully reloaded from configuration" ;; status) show_status "$format" diff --git a/lib/cli/info.sh b/lib/cli/info.sh deleted file mode 100644 index b5c384c9..00000000 --- a/lib/cli/info.sh +++ /dev/null @@ -1,513 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Get command metadata by scanning function contents -get_command_metadata() { - local commands={} - declare -A commands - local current_file="${BASH_SOURCE[0]}" - - # Scan the current file for show_* functions and their metadata - while IFS= read -r line; do - # Match show_* function definitions - if [[ $line =~ ^show_([a-z_]+)[[:space:]]*\(\)[[:space:]]*\{[[:space:]]*$ ]]; then - local cmd=${BASH_REMATCH[1]} - local desc="" - local options="" - local example="" - - # Skip internal functions like show_help - [[ $cmd == "help" ]] && continue - - # Look for metadata in comments above function - local temp_file=$(mktemp) - grep -B 10 "^show_${cmd}()" "$current_file" > "$temp_file" - - # Get description - desc=$(grep -B 10 "^show_${cmd}()" "$temp_file" | grep '^# Description:' | tail -n 1 | sed 's/^# Description:[[:space:]]*//') - if [ -z "$desc" ]; then - desc=$(grep -B 10 "^show_${cmd}()" "$temp_file" | grep '^#[[:space:]]' | tail -n 1 | sed 's/^#[[:space:]]*//') - fi - if [ -z "$desc" ]; then - desc="Show ${cmd//_/ } information" - fi - - # Get options - options=$(grep -B 10 "^show_${cmd}()" "$temp_file" | grep '^# Options:' | tail -n 1 | sed 's/^# Options:[[:space:]]*//') - if [ -z "$options" ] && grep -q 'format=' "$temp_file"; then - options="--format text|json" - fi - if grep -q 'verbose=' "$temp_file"; then - [[ -n "$options" ]] && options="$options, " - options+="--verbose" - fi - - # Get example - example=$(grep -B 10 "^show_${cmd}()" "$temp_file" | grep '^# Example:' | tail -n 1 | sed 's/^# Example:[[:space:]]*//') - if [ -z "$example" ] && [ -n "$options" ]; then - if [[ $options == *"format"* ]]; then - example="worker info $cmd --format json" - elif [[ $options == *"verbose"* ]]; then - example="worker info $cmd --verbose" - fi - fi - - rm "$temp_file" - - # Store metadata - commands[$cmd]="$desc|$options|$example" - fi - done < "$current_file" - - echo "$(declare -p commands)" -} - -# Show help for info command -info_help() { - local -A commands - eval "$(get_command_metadata)" - - # Find the longest command name for proper padding - local max_length=0 - for cmd in "${!commands[@]}"; do - local len=${#cmd} - if ((len > max_length)); then - max_length=$len - fi - done - - # Add padding for alignment - max_length=$((max_length + 2)) - - cat << EOF -Display information about the UDX Worker image and runtime - -Usage: worker info [command] - -Available Commands: -EOF - - # Sort commands alphabetically and display - local sorted_commands=($(echo "${!commands[@]}" | tr ' ' '\n' | sort)) - for cmd in "${sorted_commands[@]}"; do - IFS='|' read -r desc options example <<< "${commands[$cmd]}" - printf " %-${max_length}s %s\n" "$cmd" "$desc" - done - - # Collect unique options from all commands - local all_options="" - for cmd in "${!commands[@]}"; do - IFS='|' read -r _ options _ <<< "${commands[$cmd]}" - if [ -n "$options" ]; then - all_options="$all_options $options" - fi - done - - # Display unique options - if [ -n "$all_options" ]; then - echo -e "\nOptions:" - echo "$all_options" | tr ',' '\n' | tr ' ' '\n' | sort -u | grep -v '^$' | while read -r opt; do - printf " %s\n" "$opt" - done - fi - - # Display examples for commands that have them - local has_examples=false - for cmd in "${sorted_commands[@]}"; do - IFS='|' read -r desc options example <<< "${commands[$cmd]}" - if [ -n "$example" ]; then - if ! $has_examples; then - echo -e "\nExamples:" - has_examples=true - fi - printf " %-45s # %s\n" "$example" "$desc" - fi - done - - echo -} - -# Description: Display general information about the worker -# Options: --format text|json -# Example: worker info --format json -show_info() { - local format=${1:-text} - - case $format in - json) - { - echo "{" - echo " \"version\": \"$VERSION\"," - echo " \"os\": \"$(uname -s)\"," - echo " \"architecture\": \"$(uname -m)\"," - echo " \"hostname\": \"$(hostname)\"," - echo " \"user\": \"$USER\"," - echo " \"paths\": {" - echo " \"base\": \"$WORKER_BASE_DIR\"," - echo " \"config\": \"$WORKER_CONFIG_DIR\"," - echo " \"apps\": \"$WORKER_APP_DIR\"," - echo " \"data\": \"$WORKER_DATA_DIR\"," - echo " \"lib\": \"$WORKER_LIB_DIR\"," - echo " \"bin\": \"$WORKER_BIN_DIR\"" - echo " }," - echo " \"cloud\": {" - echo " \"gcp_config\": \"$CLOUDSDK_CONFIG\"," - echo " \"aws_config\": \"$AWS_CONFIG_FILE\"," - echo " \"azure_config\": \"$AZURE_CONFIG_DIR\"" - echo " }" - echo "}" - } | jq '.' - ;; - text) - echo "UDX Worker Information" - echo "=====================" - echo - echo "Version: $VERSION" - echo - echo "System" - echo "------" - echo "OS: $(uname -s)" - echo "Architecture: $(uname -m)" - echo "Hostname: $(hostname)" - echo "User: $USER" - echo - echo "Paths" - echo "-----" - echo "Base Directory: $WORKER_BASE_DIR" - echo "Config Directory: $WORKER_CONFIG_DIR" - echo "Apps Directory: $WORKER_APP_DIR" - echo "Data Directory: $WORKER_DATA_DIR" - echo "Library Directory: $WORKER_LIB_DIR" - echo "Binary Directory: $WORKER_BIN_DIR" - echo - echo "Cloud Configuration" - echo "------------------" - echo "GCP Config: $CLOUDSDK_CONFIG" - echo "AWS Config: $AWS_CONFIG_FILE" - echo "Azure Config: $AZURE_CONFIG_DIR" - ;; - *) - log_error "Info" "Unknown format: $format" - return 1 - ;; - esac -} - -# Description: Display detailed system information and resource usage -# Options: --format text|json, --verbose -# Example: worker info system --verbose --format json -show_system_info() { - local format=${1:-text} - - case $format in - json) - { - echo "{" - echo " \"kernel\": \"$(uname -r)\"," - echo " \"os\": {" - echo " \"name\": \"$(uname -s)\"," - echo " \"version\": \"$(cat /etc/os-release | grep VERSION= | cut -d'\"' -f2)\"" - echo " }," - echo " \"cpu\": {" - echo " \"architecture\": \"$(uname -m)\"," - echo " \"cores\": $(nproc)," - echo " \"model\": \"$(grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2 | xargs)\"" - echo " }," - echo " \"memory\": {" - echo " \"total\": $(free -b | grep Mem | awk '{print $2}')," - echo " \"free\": $(free -b | grep Mem | awk '{print $4}')" - echo " }," - echo " \"disk\": {" - echo " \"total\": $(df -B1 / | tail -1 | awk '{print $2}')," - echo " \"free\": $(df -B1 / | tail -1 | awk '{print $4}')" - echo " }" - echo "}" - } | jq '.' - ;; - text) - echo "System Information" - echo "==================" - echo - echo "Kernel: $(uname -r)" - echo - echo "Operating System" - echo "----------------" - echo "Name: $(uname -s)" - echo "Version: $(cat /etc/os-release | grep VERSION= | cut -d'\"' -f2)" - echo - echo "CPU" - echo "---" - echo "Architecture: $(uname -m)" - echo "Cores: $(nproc)" - echo "Model: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2 | xargs)" - echo - echo "Memory" - echo "------" - echo "Total: $(free -h | grep Mem | awk '{print $2}')" - echo "Free: $(free -h | grep Mem | awk '{print $4}')" - echo - echo "Disk" - echo "----" - echo "Total: $(df -h / | tail -1 | awk '{print $2}')" - echo "Free: $(df -h / | tail -1 | awk '{print $4}')" - ;; - *) - log_error "Info" "Unknown format: $format" - return 1 - ;; - esac -} - -# Get available CLI modules and their help info -get_cli_modules() { - local modules_dir="${WORKER_LIB_DIR}/cli" - local modules={} - - # Initialize modules array - declare -A modules - - # Scan through CLI modules - for module in "$modules_dir"/*.sh; do - local name=$(basename "$module" .sh) - local description="" - local commands=() - - # Extract help function content - if grep -q "${name}_help()" "$module"; then - # Get description from help function - description=$(grep -A 1 "${name}_help()" "$module" | grep -v "${name}_help()" | grep -v "^{" | head -n 1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - - # Get available commands - while IFS= read -r line; do - if [[ $line =~ ^[[:space:]]+([a-zA-Z0-9_-]+)[[:space:]]+(.+)$ ]]; then - commands+=("${BASH_REMATCH[1]}:${BASH_REMATCH[2]}") - fi - done < <(grep -A 20 "Available Commands:" "$module" | grep -B 20 "Examples:" | grep "^[[:space:]]*[a-zA-Z]") - fi - - modules[$name]="$description|${commands[*]}" - done - - echo "$(declare -p modules)" -} - -# Description: Display a high-level overview of the worker image and its capabilities -# Options: --format text|json -# Example: worker info overview --format text -show_overview() { - local format=${1:-text} - local -A modules - eval "$(get_cli_modules)" - - case $format in - json) - { - echo "{" - echo " \"name\": \"UDX Worker\"," - echo " \"description\": \"Universal Data Exchange Worker for managing cloud services and data pipelines\"," - echo " \"version\": \"$VERSION\"," - echo " \"modules\": {" - - local first=true - for module in "${!modules[@]}"; do - IFS='|' read -r description commands <<< "${modules[$module]}" - - if [ "$first" = true ]; then - first=false - else - echo "," - fi - - echo " \"$module\": {" - echo " \"description\": \"$description\"," - echo " \"commands\": [" - - IFS=' ' read -ra cmd_array <<< "$commands" - local first_cmd=true - for cmd in "${cmd_array[@]}"; do - IFS=':' read -r cmd_name cmd_desc <<< "$cmd" - if [ "$first_cmd" = true ]; then - first_cmd=false - else - echo "," - fi - echo " {" - echo " \"name\": \"$cmd_name\"," - echo " \"description\": \"$cmd_desc\"" - echo -n " }" - done - echo "" - echo " ]" - echo -n " }" - done - echo "" - echo " }," - echo " \"dependencies\": {" - echo " \"runtime\": \"$(cat /etc/os-release | grep PRETTY_NAME | cut -d '"' -f 2)\"," - echo " \"python\": \"$(python3 --version | cut -d ' ' -f 2)\"," - echo " \"supervisor\": \"$(supervisord -v)\"" - echo " }" - echo "}" - } | jq '.' - ;; - text) - cat << EOF -UDX Worker Overview -================== - -Description: Universal Data Exchange Worker for managing cloud services and data pipelines -Version: ${VERSION} - -Available Modules ----------------- -EOF - for module in "${!modules[@]}"; do - IFS='|' read -r description commands <<< "${modules[$module]}" - echo -e "\nβ€’ $module - $description" - - if [ -n "$commands" ]; then - echo " Commands:" - IFS=' ' read -ra cmd_array <<< "$commands" - for cmd in "${cmd_array[@]}"; do - IFS=':' read -r cmd_name cmd_desc <<< "$cmd" - echo " βœ“ $cmd_name - $cmd_desc" - done - fi - done - - echo -e "\nSystem Information -------------------" - echo "β€’ Runtime: $(cat /etc/os-release | grep PRETTY_NAME | cut -d '"' -f 2)" - echo "β€’ Python: $(python3 --version | cut -d ' ' -f 2)" - echo "β€’ Supervisor: $(supervisord -v)" - - echo -e "\nFor more information:" - echo "β€’ Run 'worker help' for usage instructions" - echo "β€’ Run 'worker info ' for detailed module information" - echo "β€’ Run 'worker help' for module-specific help" - ;; - *) - log_error "Info" "Unknown format: $format" - return 1 - ;; - esac -} - -# Description: Display available features and their capabilities -# Options: --format text|json -# Example: worker info features --format json -show_features() { - local format=${1:-text} - local -A modules - eval "$(get_cli_modules)" - - case $format in - json) - { - echo "{" - echo " \"features\": {" - - local first=true - for module in "${!modules[@]}"; do - IFS='|' read -r description commands <<< "${modules[$module]}" - - if [ "$first" = true ]; then - first=false - else - echo "," - fi - - # Convert module name to feature key - local feature_key=$(echo "$module" | tr '-' '_') - - echo " \"$feature_key\": {" - echo " \"description\": \"$description\"," - echo " \"commands\": [" - - IFS=' ' read -ra cmd_array <<< "$commands" - local first_cmd=true - for cmd in "${cmd_array[@]}"; do - IFS=':' read -r cmd_name cmd_desc <<< "$cmd" - if [ "$first_cmd" = true ]; then - first_cmd=false - else - echo "," - fi - echo " {" - echo " \"name\": \"$cmd_name\"," - echo " \"description\": \"$cmd_desc\"" - echo -n " }" - done - echo "" - echo " ]" - echo -n " }" - done - echo "" - echo " }" - echo "}" - } | jq '.' - ;; - text) - cat << EOF -UDX Worker Features -================== - -EOF - local count=1 - for module in "${!modules[@]}"; do - IFS='|' read -r description commands <<< "${modules[$module]}" - echo -e "\n$count. ${module^}" - echo " Description: $description" - - if [ -n "$commands" ]; then - echo " Commands:" - IFS=' ' read -ra cmd_array <<< "$commands" - for cmd in "${cmd_array[@]}"; do - IFS=':' read -r cmd_name cmd_desc <<< "$cmd" - echo " β€’ $cmd_name - $cmd_desc" - done - fi - ((count++)) - done - echo -e "\nUse 'worker help' for detailed feature documentation" - ;; - *) - log_error "Info" "Unknown format: $format" - return 1 - ;; - esac -} - -# Handle info commands -info_handler() { - local command=$1 - shift - - # If no command provided or help requested, show help - if [ -z "$command" ] || [ "$command" = "help" ]; then - info_help - return - fi - - # Get available commands - local -A commands - eval "$(get_command_metadata)" - - # Check if command exists - if [ -z "${commands[$command]+x}" ]; then - log_error "Info" "Unknown command: $command" - info_help - return 1 - fi - - # Try to execute the command function - local func_name="show_${command}" - if declare -F "$func_name" > /dev/null; then - "$func_name" "$@" - else - log_error "Info" "Command handler not implemented: $command" - return 1 - fi -} diff --git a/lib/cli/logs.sh b/lib/cli/logs.sh deleted file mode 100644 index 7d8d28f2..00000000 --- a/lib/cli/logs.sh +++ /dev/null @@ -1,281 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh - -# Show help for logs command -logs_help() { - cat << EOF -View and manage logs - -Usage: worker logs [command] - -Available Commands: - show Show logs for a service - follow Follow logs in real-time - search Search logs for pattern - export Export logs to file - clean Clean old logs - -Options: - --service Service name (required for show/follow) - --type Log type (out/err/all, default: all) - --lines Number of lines (default: 100) - --since Show logs since timestamp - --until Show logs until timestamp - --pattern Search pattern - --format Output format (text/json) - -Examples: - worker logs show --service myapp - worker logs follow --service myapp --type err - worker logs search --pattern "error" - worker logs export --service myapp --since "2024-01-01" - worker logs clean --older-than 7d -EOF -} - -# Get log file path -get_log_file() { - local service=$1 - local type=${2:-all} - - case $type in - out) - echo "/var/log/supervisor/${service}-stdout.log" - ;; - err) - echo "/var/log/supervisor/${service}-stderr.log" - ;; - all) - echo "/var/log/supervisor/${service}-*.log" - ;; - *) - log_error "Logs" "Invalid log type: $type" - return 1 - ;; - esac -} - -# Description: Display logs for a specific service -# Options: --service NAME, --type out|err|all, --lines N, --since TIME, --until TIME -# Example: worker logs show --service myapp --type err --lines 100 --since "2024-01-01" -show_logs() { - local service=$1 - local type=${2:-all} - local lines=${3:-100} - local since=$4 - local until=$5 - - if [ -z "$service" ]; then - log_error "Logs" "Service name is required" - return 1 - fi - - local log_file - log_file=$(get_log_file "$service" "$type") - - if [ ! -f "$log_file" ]; then - log_error "Logs" "Log file not found: $log_file" - return 1 - fi - - local cmd="tail -n $lines" - - if [ -n "$since" ]; then - cmd="$cmd | awk -v since=\"\$since\" '\$0 >= since'" - fi - - if [ -n "$until" ]; then - cmd="$cmd | awk -v until=\"\$until\" '\$0 <= until'" - fi - - eval "$cmd $log_file" -} - -# Description: Follow logs in real-time for a service -# Options: --service NAME, --type out|err|all -# Example: worker logs follow --service myapp --type err -follow_logs() { - local service=$1 - local type=${2:-all} - - if [ -z "$service" ]; then - log_error "Logs" "Service name is required" - return 1 - fi - - local log_file - log_file=$(get_log_file "$service" "$type") - - if [ ! -f "$log_file" ]; then - log_error "Logs" "Log file not found: $log_file" - return 1 - fi - - tail -f "$log_file" -} - -# Description: Search logs for a specific pattern -# Options: --pattern PATTERN, --service NAME, --type out|err|all -# Example: worker logs search --pattern "error" --service myapp -search_logs() { - local pattern=$1 - local service=$2 - local type=${3:-all} - - if [ -z "$pattern" ]; then - log_error "Logs" "Search pattern is required" - return 1 - fi - - local log_file - if [ -n "$service" ]; then - log_file=$(get_log_file "$service" "$type") - else - log_file="/var/log/supervisor/*.log" - fi - - grep -n "$pattern" $log_file -} - -# Description: Export logs to a file -# Options: --service NAME, --type out|err|all, --since TIME, --until TIME, --format text|json -# Example: worker logs export --service myapp --since "2024-01-01" --format json -export_logs() { - local service=$1 - local type=${2:-all} - local since=$3 - local until=$4 - local format=${5:-text} - - if [ -z "$service" ]; then - log_error "Logs" "Service name is required" - return 1 - fi - - local log_file - log_file=$(get_log_file "$service" "$type") - - if [ ! -f "$log_file" ]; then - log_error "Logs" "Log file not found: $log_file" - return 1 - fi - - local output_file="${service}_logs_$(date +%Y%m%d_%H%M%S)" - - case $format in - json) - output_file="$output_file.json" - awk '{printf "{\\"timestamp\\":\\"%s\\",\\"message\\":\\"%s\\"}\\n", $1, substr($0,index($0,$2))}' "$log_file" > "$output_file" - ;; - text) - output_file="$output_file.log" - if [ -n "$since" ] || [ -n "$until" ]; then - awk -v since="$since" -v until="$until" ' - ($0 >= since) && ($0 <= until || until=="") - ' "$log_file" > "$output_file" - else - cp "$log_file" "$output_file" - fi - ;; - *) - log_error "Logs" "Unknown format: $format" - return 1 - ;; - esac - - log_success "Logs" "Logs exported to $output_file" -} - -# Description: Clean old log files -# Options: --older-than DURATION -# Example: worker logs clean --older-than 7d -clean_logs() { - local older_than=${1:-7d} # Default: 7 days - - find /var/log/supervisor -name "*.log" -type f -mtime +"${older_than%d}" -delete - - log_success "Logs" "Cleaned logs older than $older_than" -} - -# Parse command line arguments -parse_args() { - local args=() - while [[ $# -gt 0 ]]; do - case $1 in - --service) - service=$2 - shift 2 - ;; - --type) - type=$2 - shift 2 - ;; - --lines) - lines=$2 - shift 2 - ;; - --since) - since=$2 - shift 2 - ;; - --until) - until=$2 - shift 2 - ;; - --pattern) - pattern=$2 - shift 2 - ;; - --format) - format=$2 - shift 2 - ;; - --older-than) - older_than=$2 - shift 2 - ;; - *) - args+=("$1") - shift - ;; - esac - done - set -- "${args[@]}" -} - -# Handle logs commands -logs_handler() { - local cmd=$1 - shift - - # Parse command line arguments - parse_args "$@" - - case $cmd in - show) - show_logs "$service" "$type" "$lines" "$since" "$until" - ;; - follow) - follow_logs "$service" "$type" - ;; - search) - search_logs "$pattern" "$service" "$type" - ;; - export) - export_logs "$service" "$type" "$since" "$until" "$format" - ;; - clean) - clean_logs "$older_than" - ;; - help) - logs_help - ;; - *) - log_error "Logs" "Unknown command: $cmd" - logs_help - exit 1 - ;; - esac -} diff --git a/lib/env_handler.sh b/lib/env_handler.sh index 54a4bbc6..3ce455fb 100644 --- a/lib/env_handler.sh +++ b/lib/env_handler.sh @@ -26,12 +26,6 @@ generate_env_file() { # Extract environment variables echo "$config" | yq eval '.config.env | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' - - - # Add computed variables - echo - echo "# Computed variables" - echo "export WORKER_ENV_FILE=\"$WORKER_ENV_FILE\"" - echo "export WORKER_SECRETS_FILE=\"$WORKER_SECRETS_FILE\"" } > "$WORKER_ENV_FILE" # File permissions are set during build diff --git a/lib/process_manager.sh b/lib/process_manager.sh index 4a1b19ba..a807713c 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -9,7 +9,7 @@ CONFIG_FILE="${USER_CONFIG_PATH}" # Check if user config exists if [[ ! -f "${USER_CONFIG_PATH}" ]]; then - log_info "No services configuration found at ${USER_CONFIG_PATH}. Services will not be started." + log_info "No services configuration found at ${USER_CONFIG_PATH}." log_info "Run 'worker help service' for information about service configuration" exit 0 fi diff --git a/lib/worker_config.sh b/lib/worker_config.sh index c4e67ff3..844f86b5 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -55,7 +55,6 @@ merge_worker_configs() { # Check if merge was successful if [ -s "${MERGED_CONFIG}.tmp" ]; then mv "${MERGED_CONFIG}.tmp" "$MERGED_CONFIG" - log_success "Worker configuration" "Using user configuration at $USER_CONFIG" return 0 else rm -f "${MERGED_CONFIG}.tmp" @@ -63,9 +62,6 @@ merge_worker_configs() { return 1 fi fi - - # No user config - log_info "Worker configuration" "No user configuration found at $USER_CONFIG, using built-in defaults" } # Load and parse the merged configuration From 55ed660323331595b83d5a4cb12453d535867b4c Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 20 Feb 2025 21:34:23 +0300 Subject: [PATCH 10/25] service cli improvements --- Dockerfile | 4 +- bin/entrypoint.sh | 13 +- .../.config/worker/services.yaml | 17 + examples/simple-service/index.sh | 13 + lib/cli.sh | 50 +- lib/cli/service.sh | 533 +++++++++++++----- lib/process_manager.sh | 50 +- 7 files changed, 483 insertions(+), 197 deletions(-) create mode 100644 examples/simple-service/.config/worker/services.yaml create mode 100755 examples/simple-service/index.sh diff --git a/Dockerfile b/Dockerfile index 279cc33f..c311e0cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -189,7 +189,9 @@ RUN \ # Set runtime directories permissions chmod 775 ${WORKER_APP_DIR} ${WORKER_DATA_DIR} && \ # Set sensitive file permissions - chmod 600 ${WORKER_CONFIG_DIR}/secrets + chmod 600 ${WORKER_CONFIG_DIR}/secrets && \ + # Set home directory executable + chmod 755 ${HOME} # Set up supervisor configuration RUN ln -sf ${WORKER_CONFIG_DIR}/supervisor/supervisord.conf /etc/supervisord.conf diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 8e28b787..9460b456 100644 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -8,10 +8,21 @@ log_info "Welcome to UDX Worker Container. Initializing environment..." # shellcheck disable=SC1091 source ${WORKER_LIB_DIR}/environment.sh -# Start the process manager in the background +# Start the process manager log_info "Starting process manager..." ${WORKER_LIB_DIR}/process_manager.sh & +# Wait for supervisor to be ready +max_attempts=10 +attempt=1 +while [ $attempt -le $max_attempts ]; do + if supervisorctl status >/dev/null 2>&1; then + break + fi + sleep 1 + attempt=$((attempt + 1)) +done + # Main execution logic if [ "$#" -gt 0 ]; then # Log the command being executed diff --git a/examples/simple-service/.config/worker/services.yaml b/examples/simple-service/.config/worker/services.yaml new file mode 100644 index 00000000..b537ab8f --- /dev/null +++ b/examples/simple-service/.config/worker/services.yaml @@ -0,0 +1,17 @@ +--- +kind: workerService +version: udx.io/worker-v1/service + +services: + - name: "index" + command: "/home/udx/index.sh" + autostart: true + autorestart: true + - name: "index2" + command: "/home/udx/index.sh" + autostart: true + autorestart: true + - name: "index3" + command: "/home/udx/index.sh" + autostart: true + autorestart: true diff --git a/examples/simple-service/index.sh b/examples/simple-service/index.sh new file mode 100755 index 00000000..42d92659 --- /dev/null +++ b/examples/simple-service/index.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Simple service that prints a message every 5 seconds +echo "Starting simple service..." + +# Trap SIGTERM for graceful shutdown +trap 'echo "Received shutdown signal, exiting..."; exit 0' SIGTERM + +# Main loop +while true; do + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Simple service is running..." + sleep 5 +done diff --git a/lib/cli.sh b/lib/cli.sh index d93f370f..133db6e4 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -68,24 +68,6 @@ show_help() { cat << EOF πŸš€ Welcome to UDX Worker Container! -This container helps you run and manage cloud services and applications. -Here's how to get started: - -1. Configure Services: - - Create service config: ${HOME}/.config/worker/services.yaml - - View service commands: worker help service - - Check service status: worker service list - -2. Configure Environment: - - Set environment vars: /etc/worker/environment - - View current settings: worker env show - - Configure cloud auth: worker auth setup - -3. Monitor & Manage: - - View container status: worker info overview - - Check service logs: worker service logs - - View system health: worker health check - Available Commands: EOF @@ -97,33 +79,15 @@ EOF cat << EOF -Tip: Run 'worker help [command]' for detailed information about any command +Run any command without arguments to see its detailed help and usage information. +For example: 'worker auth' will show auth command help. EOF } -# Show command-specific help -show_command_help() { - local cmd=$1 - local help_function="${cmd}_help" - - # Check if the help function exists - if [[ $(type -t "$help_function") == function ]]; then - "$help_function" - else - log_error "CLI" "No help available for command: $cmd" - show_help - return 1 - fi -} - # Main CLI interface if [ -z "$1" ] || [ "$1" = "help" ]; then - if [ -z "$2" ]; then - show_help - log_info "Container is ready. Run with a command to start services." - else - show_command_help "$2" - fi + show_help + log_info "Container is ready. Run a command to start services." exit 0 fi @@ -132,9 +96,9 @@ if [ "$1" = "version" ]; then exit 0 fi -# Handle app and service commands -if [ "$1" = "app" ] || [ "$1" = "service" ]; then - # Source and load configuration for app/service commands +# Handle app commands +if [ "$1" = "app" ]; then + # Source and load configuration for app commands source "${WORKER_LIB_DIR}/worker_config.sh" config=$(load_and_parse_config) export_variables_from_config "$config" diff --git a/lib/cli/service.sh b/lib/cli/service.sh index 419499f5..bfd00050 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -3,210 +3,461 @@ # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 source ${WORKER_LIB_DIR}/utils.sh +# Constants +SERVICES_CONFIG_DIR="${HOME}/.config/worker" +SERVICES_CONFIG_FILE="${SERVICES_CONFIG_DIR}/services.yaml" + +# Show help for service command +service_help() { + cat << 'EOF' +Usage: worker service [options] + +Commands: + list List all configured and running services + status Show detailed status of a service + start Start a service + stop Stop a service + restart Restart a service + logs View service logs + errors View service error logs + config Show current service configuration + init Initialize a new service configuration + help Show this help message + +Options: + --format json Output in JSON format (for list, status, config) + --tail N Show last N lines of logs (default: 100) + --follow Follow log output in real time + --error-only Show only error logs + +Examples: + worker service list + worker service start my-app + worker service logs my-app --tail 50 --follow + worker service status my-app --format json +EOF +} + service_handler() { local cmd=$1 shift # Remove the command from args + # Parse format option if present + local format="" + while [[ $# -gt 0 ]]; do + case "$1" in + --format) + format="$2" + shift 2 + ;; + --format=*) + format="${1#*=}" + shift + ;; + *) + break + ;; + esac + done + + # Show help if no command or help requested + if [ -z "$cmd" ] || [ "$cmd" = "help" ]; then + service_help + return 0 + fi + + # Check if services config exists before most commands + if [ "$cmd" != "init" ]; then + if [ ! -f "$SERVICES_CONFIG_FILE" ]; then + log_warn "Service" "No services configuration found" + log_info "Service" "Run 'worker help service' for information about service configuration" + return 1 + fi + fi + case $cmd in list) - list_services - ;; + list_services "$format" + ;; status) - check_status "$1" - ;; + check_status "$1" "$format" + ;; logs) follow_logs "$@" - ;; + ;; errors) follow_logs "$1" "err" - ;; + ;; config) - service_show_config - ;; + service_show_config "$format" + ;; + init) + init_service_config + ;; start|stop|restart) manage_service "$cmd" "$1" - ;; + ;; *) - log_warn "CLI" "Usage: $0 {list|status|logs|config|start|stop|restart}" - exit 1 - ;; + log_error "Service" "Unknown command: $cmd" + service_help + return 1 + ;; esac } -# Show help for service command -service_help() { - cat << EOF -Manage UDX Worker services and applications - -Usage: worker service [command] [options] - -Commands: - list List all configured services and their status - status [name] Show status of all services or a specific service - logs [name] View logs for all services or a specific service - errors [name] View error logs for all services or a specific service - config Show current service configuration - start [name] Start a service - stop [name] Stop a service - restart [name] Restart a service - -Configuration: - Services are configured in: ${HOME}/.config/worker/services.yaml - -Example service configuration: - version: "1.0" - services: - my-app: - name: "my-app" - command: "python app.py" - working_dir: "/opt/worker/apps" - autostart: true - autorestart: true - environment: - APP_PORT: "8080" - -Examples: - worker service list # List all services - worker service logs my-app # View logs for my-app - worker service start my-app # Start my-app service -EOF -} - # Description: List all configured services and their status -# Example: worker service list +# Example: worker service list [--format json] list_services() { - # Capture the output of supervisorctl status - local services_status - services_status=$(supervisorctl status 2>&1) # Also capture stderr to handle error messages + local format="$1" - # Check if Supervisor is not running, not accessible, or if there are no managed services - if [[ -z "$services_status" ]] || echo "$services_status" | grep -Eq 'no such|ERROR'; then - log_info "No services are currently managed." - log_info "To configure services, create ${HOME}/.config/worker/services.yaml" - log_info "Run 'worker help service' for configuration examples." - return 0 + # Check if services config exists + if [ ! -f "$SERVICES_CONFIG_FILE" ]; then + if [ "$format" = "json" ]; then + echo '{"error":"No services configuration found","services":[]}' + else + log_warn "Service" "No services configuration found" + log_info "Service" "Run 'worker service init' to create a new configuration" + fi + return 1 + fi + + # Get supervisor status and running services + local supervisor_status + local supervisor_running=false + supervisor_status=$(supervisorctl status 2>&1) + if [ $? -eq 0 ]; then + supervisor_running=true + fi + + # Format and display services + if [ "$format" = "json" ]; then + # Build JSON output + local json_services="[]" + + # Add running services if supervisor is running + if [ "$supervisor_running" = true ] && [ -n "$supervisor_status" ]; then + json_services="[" + local first=true + while IFS= read -r line; do + if [ -n "$line" ]; then + if [ "$first" = true ]; then + first=false + else + json_services+="," + fi + # Parse supervisor status line + local name status pid uptime + read -r name status pid uptime <<< "$line" + json_services+="{\"name\":\"$name\",\"status\":\"$status\",\"pid\":\"$pid\",\"uptime\":\"$uptime\"}" + fi + done <<< "$supervisor_status" + json_services+="]" + fi + echo "{\"services\":$json_services}" + else + # Show running services if supervisor is running + if [ "$supervisor_running" = true ]; then + if [ -n "$supervisor_status" ]; then + log_info "Service" "Running services:" + # Print table header + printf "%-2s %-15s %-8s %-6s %-12s\n" "" "NAME" "STATUS" "PID" "UPTIME" + printf "%-2s %-15s %-8s %-6s %-12s\n" "" "----" "------" "---" "------" + while IFS= read -r line; do + if [ -n "$line" ]; then + # Parse supervisor status line + local name status pid uptime + read -r name status pid uptime <<< "$line" + + # Use different symbols based on status + local symbol="⚠️" + case "$status" in + RUNNING) symbol="βœ…";; + STOPPED) symbol="β›”";; + FATAL) symbol="πŸ’€";; + *) symbol="⚠️";; + esac + printf "%-2s %-15s %-8s %-6s %-12s\n" "$symbol" "$name" "$status" "$pid" "$uptime" + fi + done <<< "$supervisor_status" + else + log_info "Service" "No running services" + fi + else + log_info "Service" "Supervisor is not running" + log_info "Service" "Use 'worker service start ' to start a service" + fi + log_info "Service" "Use 'worker service config' to view service configuration" fi - - log_info "Managed services:" - local i=1 - echo "$services_status" | while read -r line; do - log_info "$i. $line" - ((i++)) - done } # Description: Show detailed status of a specific service -# Example: worker service status my-app +# Example: worker service status my-app [--format json] check_status() { - # Require a service name for this function - if [ -z "$1" ]; then - log_warn "Service" "Error: No service name provided." - log_warn "Service" "Usage: $0 status " - exit 1 + local service=$1 + local format=$2 + + # Check if service name is provided + if [ -z "$service" ]; then + log_error "Service" "Service name required" + log_info "Service" "Usage: worker service status " + return 1 fi - - # Attempt to capture the status of the specific service, including errors - local service_status - service_status=$(supervisorctl status "$1" 2>&1) - - # Check if Supervisor is not running, not accessible, or if the service does not exist - if [[ -z "$service_status" ]] || echo "$service_status" | grep -Eq 'no such|ERROR'; then - log_warn "Service" "The service '$1' does not exist." - exit 1 + + # Check if service exists in config + if ! yq e ".services[] | select(.name == \"$service\") | .name" "$SERVICES_CONFIG_FILE" 2>/dev/null | grep -q "$service"; then + log_error "Service" "Service '$service' not found in configuration" + return 1 fi + + # Get service status from supervisor + local status_output + status_output=$(supervisorctl status "$service" 2>&1) - # Directly output the captured service status - echo "$service_status" + # Check if supervisor is running + if echo "$status_output" | grep -q "unix:///var/run/supervisor.sock no such file"; then + log_error "Service" "Supervisor is not running" + log_info "Service" "Use 'worker service start $service' to start the service" + return 1 + fi + + # Parse status output + local name status pid uptime + read -r name status pid uptime <<< "$status_output" + + if [ "$format" = "json" ]; then + echo "{\"name\":\"$name\",\"status\":\"$status\",\"pid\":\"$pid\",\"uptime\":\"$uptime\"}" + else + # Use different symbols based on status + local symbol="⚠️" + case "$status" in + RUNNING) symbol="βœ…";; + STOPPED) symbol="β›”";; + FATAL) symbol="πŸ’€";; + *) symbol="⚠️";; + esac + log_info "Service" "$symbol $name ($status) - PID: $pid, Uptime: $uptime" + fi } # Description: View and follow logs for a service # Options: --tail N, --follow, --error-only # Example: worker service logs my-app --tail 100 --follow follow_logs() { - local service_name="" - local type="out" - local lines=20 - local nostream=false + local service=$1 + shift + + # Check if service name is provided + if [ -z "$service" ]; then + log_error "Service" "Service name required" + log_info "Service" "Usage: worker service logs [options]" + return 1 + fi + + # Check if service exists in config + if ! yq e ".services[] | select(.name == \"$service\") | .name" "$SERVICES_CONFIG_FILE" 2>/dev/null | grep -q "$service"; then + log_error "Service" "Service '$service' not found in configuration" + return 1 + fi + + # Parse options + local tail_lines=100 + local follow=false + local error_only=false - # Parse arguments while [[ $# -gt 0 ]]; do case "$1" in - --lines=*) - lines="${1#*=}" + --tail) + tail_lines=$2 + shift 2 ;; - --lines) + --follow) + follow=true shift - if [[ -n "$1" && "$1" =~ ^[0-9]+$ ]]; then - lines="$1" - fi - ;; - --nostream) - nostream=true ;; - err) - type="err" + --error-only) + error_only=true + shift ;; *) - if [[ -z "$service_name" ]]; then - service_name="$1" - fi + log_error "Service" "Unknown option: $1" + return 1 ;; esac - shift done - if [[ -z "$service_name" ]]; then - log_error "Service" "Error: No service name provided." - log_error "Service" "Usage: $0 logs [--lines N] [--nostream]" - exit 1 - fi - - local logfile="/var/log/supervisor/$service_name" - logfile="$logfile.$type.log" - - if [[ ! -f "$logfile" ]]; then - log_error "Service" "Log file does not exist: $logfile" - exit 1 + # Build the tail command + local cmd="tail" + if [ "$follow" = true ]; then + cmd+=" -f" fi + cmd+=" -n $tail_lines" + + # Get log file paths + local log_dir="/var/log/supervisor" + local out_log="$log_dir/$service.out.log" + local err_log="$log_dir/$service.err.log" - # Ensure lines is a valid number - if ! [[ "$lines" =~ ^[0-9]+$ ]]; then - log_error "Service" "Invalid line count: $lines" - exit 1 + # Check if log files exist + if [ ! -f "$out_log" ] && [ ! -f "$err_log" ]; then + log_error "Service" "No log files found for service '$service'" + return 1 fi - - if [ "$nostream" = true ]; then - # Just show the last N lines without following - tail -n "$lines" "$logfile" + + # Show logs based on options + if [ "$error_only" = true ]; then + if [ -f "$err_log" ]; then + $cmd "$err_log" + else + log_warn "Service" "No error log file found for service '$service'" + fi else - # Show the last N lines and follow - exec tail -n "$lines" -f "$logfile" + if [ -f "$out_log" ]; then + $cmd "$out_log" + fi + if [ -f "$err_log" ]; then + $cmd "$err_log" + fi fi } # Description: Display current service configuration -# Example: worker service config +# Example: worker service config [--format json] service_show_config() { - if [ ! -f "/etc/supervisord.conf" ]; then - log_error "Service" "Configuration file is not generated since no services are managed." - exit 1 + local format="$1" + + if [ ! -f "$SERVICES_CONFIG_FILE" ]; then + log_warn "Service" "No services configuration found" + log_info "Service" "Run 'worker service init' to create a new configuration" + return 1 + fi + + if [ "$format" = "json" ]; then + # Build JSON output with config file info and content + echo "{" + echo " \"config_file\": \"$SERVICES_CONFIG_FILE\"," + echo -n " \"content\": " + yq e -o=json '.' "$SERVICES_CONFIG_FILE" + echo "}" + else + log_info "Service" "Current service configuration:" + log_info "Service" "Location: $SERVICES_CONFIG_FILE" + echo "" + cat "$SERVICES_CONFIG_FILE" fi - cat /etc/supervisord.conf } # Description: Start, stop, or restart a service # Example: worker service restart my-app manage_service() { - if [ -z "$2" ]; then - log_error "Service" "Error: No service name provided." - log_warn "Service" "Usage: $0 $1 " - exit 1 + local action=$1 + local service=$2 + + # Check if service name is provided + if [ -z "$service" ]; then + log_error "Service" "Service name required for $action command" + log_info "Service" "Usage: worker service $action " + return 1 + fi + + # Check if service exists in config + if ! yq e ".services[] | select(.name == \"$service\") | .name" "$SERVICES_CONFIG_FILE" 2>/dev/null | grep -q "$service"; then + log_error "Service" "Service '$service' not found in configuration" + return 1 + fi + + # Source process manager functions but prevent main from running + WORKER_SERVICE_MODE=1 source "${WORKER_LIB_DIR}/process_manager.sh" || return 1 + + # Check if supervisor is running and config is up to date + local supervisor_status + supervisor_status=$(supervisorctl status 2>&1) + local need_reload=false + + if echo "$supervisor_status" | grep -q "unix:///var/run/supervisor.sock no such file"; then + log_warn "Service" "Supervisor is not running" + log_info "Service" "Starting supervisor..." + need_reload=true + fi + + # Configure services + if ! configure_services; then + log_error "Service" "Failed to configure services" + return 1 fi - if [ ! -e "/var/run/supervisor/supervisord.sock" ]; then - log_error "Service" "Error: Service doesn't exist." - exit 1 + if [ "$need_reload" = true ]; then + # Start supervisord directly + supervisord + + # Wait for supervisor to be ready + local max_attempts=10 + local attempt=1 + while [ $attempt -le $max_attempts ]; do + if supervisorctl status >/dev/null 2>&1; then + break + fi + sleep 1 + attempt=$((attempt + 1)) + done + + if [ $attempt -gt $max_attempts ]; then + log_error "Service" "Failed to start supervisor" + return 1 + fi + else + # Reload config if supervisor is already running + log_info "Service" "Reloading supervisor configuration..." + supervisorctl reread + supervisorctl update fi + + # Show action being taken + case "$action" in + start) + log_info "Service" "πŸš€ Starting service: $service" + ;; + stop) + log_info "Service" "πŸ›‘ Stopping service: $service" + ;; + restart) + log_info "Service" "πŸ”„ Restarting service: $service" + ;; + esac + + # Execute supervisorctl command and wait for result + local result + result=$(supervisorctl "$action" "$service" 2>&1) - supervisorctl "$1" "$2" -} \ No newline at end of file + # Check for common errors + if echo "$result" | grep -q "ERROR (no such process)"; then + log_error "Service" "Service '$service' not found in supervisor" + log_info "Service" "Run 'worker service config' to check your service configuration" + return 1 + elif echo "$result" | grep -q "ERROR (already started)"; then + log_warn "Service" "Service '$service' is already running" + return 0 + elif echo "$result" | grep -q "ERROR (not running)"; then + log_warn "Service" "Service '$service' is not running" + return 0 + fi + + # Show the result + echo "$result" + + # For restart/start, wait for service to be running + if [ "$action" = "restart" ] || [ "$action" = "start" ]; then + local max_attempts=10 + local attempt=1 + while [ $attempt -le $max_attempts ]; do + if supervisorctl status "$service" | grep -q "RUNNING"; then + break + fi + sleep 1 + attempt=$((attempt + 1)) + done + fi + + # Show current status after action + echo "" + check_status "$service" +} diff --git a/lib/process_manager.sh b/lib/process_manager.sh index a807713c..d511e3f4 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -7,13 +7,6 @@ source "${WORKER_LIB_DIR}/utils.sh" USER_CONFIG_PATH="${HOME}/.config/worker/services.yaml" CONFIG_FILE="${USER_CONFIG_PATH}" -# Check if user config exists -if [[ ! -f "${USER_CONFIG_PATH}" ]]; then - log_info "No services configuration found at ${USER_CONFIG_PATH}." - log_info "Run 'worker help service' for information about service configuration" - exit 0 -fi - # Supervisor configuration paths COMMON_TEMPLATE_FILE="${WORKER_CONFIG_DIR}/supervisor/common.conf" PROGRAM_TEMPLATE_FILE="${WORKER_CONFIG_DIR}/supervisor/program.conf" @@ -25,6 +18,13 @@ trap 'handle_supervisor_signals SIGINT' SIGINT # Main execution main() { + # Check if user config exists + if [[ ! -f "${USER_CONFIG_PATH}" ]]; then + log_info "No services configuration found at ${USER_CONFIG_PATH}." + log_info "Run 'worker help service' for information about service configuration" + exit 0 + fi + log_info "Process Manager" "Starting process manager..." if ! configure_and_execute_services; then @@ -162,8 +162,34 @@ handle_supervisor_signals() { start_supervisor() { log_info "Starting supervisord..." - # Start supervisord in non-daemon mode - exec supervisord -n + # Check if supervisor is already running + if pgrep -f "supervisord" >/dev/null; then + # Reload configuration + if supervisorctl reread && supervisorctl update; then + log_info "Supervisor configuration reloaded" + return 0 + fi + log_error "Failed to reload supervisor configuration" + return 1 + fi + + # Start supervisord in daemon mode + supervisord + + # Wait for supervisor to be ready + local max_attempts=10 + local attempt=1 + while [ $attempt -le $max_attempts ]; do + if supervisorctl status >/dev/null 2>&1; then + log_info "Supervisor is ready" + return 0 + fi + sleep 1 + attempt=$((attempt + 1)) + done + + log_error "Failed to start supervisord" + return 1 } # Function to check for service configurations @@ -220,5 +246,7 @@ configure_and_execute_services() { start_supervisor } -# Execute main function -main \ No newline at end of file +# Only run main if not in service mode +if [ -z "$WORKER_SERVICE_MODE" ]; then + main +fi \ No newline at end of file From 35f5ab33325f7d0c8326573011c5e8e2caf0c02f Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 20 Feb 2025 21:43:18 +0300 Subject: [PATCH 11/25] shell fixes --- Dockerfile | 2 +- bin/entrypoint.sh | 6 +++--- lib/auth.sh | 3 ++- lib/auth/aws.sh | 2 +- lib/auth/azure.sh | 2 +- lib/auth/bitwarden.sh | 2 +- lib/auth/gcp.sh | 2 +- lib/cleanup.sh | 4 ++-- lib/cli.sh | 9 ++++++--- lib/cli/auth.sh | 6 +++--- lib/cli/config.sh | 4 ++-- lib/cli/env.sh | 4 ++-- lib/cli/health.sh | 20 +++++++++++++------- lib/cli/sbom.sh | 7 +++---- lib/cli/service.sh | 5 ++--- lib/env_handler.sh | 2 +- lib/secrets.sh | 2 +- lib/secrets/aws.sh | 2 +- lib/secrets/azure.sh | 2 +- lib/secrets/bitwarden.sh | 2 +- lib/secrets/gcp.sh | 2 +- 21 files changed, 49 insertions(+), 41 deletions(-) diff --git a/Dockerfile b/Dockerfile index c311e0cf..99e3d170 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && \ tzdata=2025a-2ubuntu1 \ curl=8.12.0+git20250209.89ed161+ds-1ubuntu1 \ bash=5.2.37-1ubuntu1 \ - apt-utils=2.9.29 \ + apt-utils=2.9.30 \ gettext=0.23.1-1 \ gnupg=2.4.4-2ubuntu22 \ ca-certificates=20241223 \ diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 9460b456..64a94e64 100644 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -1,16 +1,16 @@ #!/bin/bash # shellcheck disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" log_info "Welcome to UDX Worker Container. Initializing environment..." # shellcheck disable=SC1091 -source ${WORKER_LIB_DIR}/environment.sh +source "${WORKER_LIB_DIR}/environment.sh" # Start the process manager log_info "Starting process manager..." -${WORKER_LIB_DIR}/process_manager.sh & +"${WORKER_LIB_DIR}/process_manager.sh" & # Wait for supervisor to be ready max_attempts=10 diff --git a/lib/auth.sh b/lib/auth.sh index e3bd4878..e49e71ce 100644 --- a/lib/auth.sh +++ b/lib/auth.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Array to track configured providers declare -a configured_providers=() @@ -12,6 +12,7 @@ get_provider_env_vars() { local actors_json=$2 # Get all env var names from actor creds that match ${VAR} pattern + # shellcheck disable=SC2016 echo "$actors_json" | jq -r ".[].creds" 2>/dev/null | \ grep -o '\${[^}]*}' | sed 's/[\${}]//g' || true } diff --git a/lib/auth/aws.sh b/lib/auth/aws.sh index a3234d57..e11c3271 100644 --- a/lib/auth/aws.sh +++ b/lib/auth/aws.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Example usage of the function # aws_authenticate "/path/to/your/aws_creds.json" diff --git a/lib/auth/azure.sh b/lib/auth/azure.sh index 180b2382..e913180d 100644 --- a/lib/auth/azure.sh +++ b/lib/auth/azure.sh @@ -6,7 +6,7 @@ # azure_authenticate "/path/to/your/azure_creds.json" # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Function to authenticate Azure accounts azure_authenticate() { diff --git a/lib/auth/bitwarden.sh b/lib/auth/bitwarden.sh index 55867cda..cd221b05 100644 --- a/lib/auth/bitwarden.sh +++ b/lib/auth/bitwarden.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Function to authenticate Bitwarden using API key or master password # diff --git a/lib/auth/gcp.sh b/lib/auth/gcp.sh index 1d8e3b88..27b74d7c 100644 --- a/lib/auth/gcp.sh +++ b/lib/auth/gcp.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Function to authenticate GCP service accounts # diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 9082ed73..405f4f21 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -2,10 +2,10 @@ # Include worker config utilities first # shellcheck source=/dev/null -source ${WORKER_LIB_DIR}/worker_config.sh +source "${WORKER_LIB_DIR}/worker_config.sh" # shellcheck source=/dev/null -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Generic function to clean up authentication for any provider cleanup_provider() { diff --git a/lib/cli.sh b/lib/cli.sh index 133db6e4..9eb1d1a3 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -29,7 +29,8 @@ get_available_commands() { # Scan through CLI modules to find commands and their descriptions for module in "${WORKER_LIB_DIR}/cli/"*.sh; do if [ -f "$module" ]; then - local name=$(basename "$module" .sh) + local name + name=$(basename "$module" .sh) local description="" # Extract description from help function @@ -45,7 +46,7 @@ get_available_commands() { fi done - echo "$(declare -p commands)" + declare -p commands } # Print help information @@ -72,7 +73,8 @@ Available Commands: EOF # Sort commands alphabetically and display - local sorted_commands=($(echo "${!commands[@]}" | tr ' ' '\n' | sort)) + local sorted_commands + mapfile -t sorted_commands < <(printf '%s\n' "${!commands[@]}" | sort) for cmd in "${sorted_commands[@]}"; do printf " %-${max_length}s %s\n" "$cmd" "${commands[$cmd]}" done @@ -99,6 +101,7 @@ fi # Handle app commands if [ "$1" = "app" ]; then # Source and load configuration for app commands + # shellcheck disable=SC1091 source "${WORKER_LIB_DIR}/worker_config.sh" config=$(load_and_parse_config) export_variables_from_config "$config" diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh index 82b089df..d1c0f981 100644 --- a/lib/cli/auth.sh +++ b/lib/cli/auth.sh @@ -1,8 +1,8 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh -source ${WORKER_LIB_DIR}/auth.sh +source "${WORKER_LIB_DIR}/utils.sh" +source "${WORKER_LIB_DIR}/auth.sh" # Show help for auth command auth_help() { @@ -155,7 +155,7 @@ logout_provider() { log_info "Auth" "Logging out providers..." # Source cleanup utilities - source ${WORKER_LIB_DIR}/cleanup.sh + source "${WORKER_LIB_DIR}/cleanup.sh" # Function to logout from a specific provider do_provider_logout() { diff --git a/lib/cli/config.sh b/lib/cli/config.sh index 2cc313ea..33ef451b 100644 --- a/lib/cli/config.sh +++ b/lib/cli/config.sh @@ -1,8 +1,8 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh -source ${WORKER_LIB_DIR}/worker_config.sh +source "${WORKER_LIB_DIR}/utils.sh" +source "${WORKER_LIB_DIR}/worker_config.sh" # Show help for config command config_help() { diff --git a/lib/cli/env.sh b/lib/cli/env.sh index b258cfd3..9b4d763a 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -1,8 +1,8 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh -source ${WORKER_LIB_DIR}/worker_config.sh +source "${WORKER_LIB_DIR}/utils.sh" +source "${WORKER_LIB_DIR}/worker_config.sh" # Show help for env command env_help() { diff --git a/lib/cli/health.sh b/lib/cli/health.sh index d52cc5e9..335171c0 100644 --- a/lib/cli/health.sh +++ b/lib/cli/health.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Show help for health command health_help() { @@ -49,12 +49,18 @@ check_health() { check_system_resources || failed=1 # Gather all data - local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - local disk_usage=$(df -h / | awk 'NR==2 {print $5}' | tr -d '%') - local mem_total=$(free -b | awk '/Mem:/ {printf "%.2f", $2/1024/1024/1024}') - local mem_used=$(free -b | awk '/Mem:/ {printf "%.2f", $3/1024/1024/1024}') - local mem_usage=$(free | awk '/Mem:/ {printf("%.0f", $3/$2 * 100)}') - local load_avg=$(uptime | awk -F'load average:' '{print $2}' | cut -d, -f1 | tr -d ' ') + local timestamp + local disk_usage + local mem_total + local mem_used + local mem_usage + local load_avg + timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + disk_usage=$(df -h / | awk 'NR==2 {print $5}' | tr -d '%') + mem_total=$(free -b | awk '/Mem:/ {printf "%.2f", $2/1024/1024/1024}') + mem_used=$(free -b | awk '/Mem:/ {printf "%.2f", $3/1024/1024/1024}') + mem_usage=$(free | awk '/Mem:/ {printf("%.0f", $3/$2 * 100)}') + load_avg=$(uptime | awk -F'load average:' '{print $2}' | cut -d, -f1 | tr -d ' ') if [ "$format" = "json" ]; then { diff --git a/lib/cli/sbom.sh b/lib/cli/sbom.sh index 67323f35..d1be0e37 100644 --- a/lib/cli/sbom.sh +++ b/lib/cli/sbom.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Show help for sbom command sbom_help() { @@ -80,7 +80,7 @@ generate_sbom() { "version": "%s", "architecture": "%s" }' "$name" "$version" "$arch" - if [ $count -lt $total ]; then + if [ $count -lt "$total" ]; then echo "," else echo "" @@ -107,7 +107,7 @@ generate_sbom() { printf ' "%s": { "version": "%s" }' "$name" "$version" - if [ $count -lt $total ]; then + if [ $count -lt "$total" ]; then echo "," else echo "" @@ -214,7 +214,6 @@ sbom_handler() { return 0 fi - local args read -r command format type filter <<< "$(parse_args "$@")" case $command in diff --git a/lib/cli/service.sh b/lib/cli/service.sh index bfd00050..5b6a2ba7 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Constants SERVICES_CONFIG_DIR="${HOME}/.config/worker" @@ -124,8 +124,7 @@ list_services() { # Get supervisor status and running services local supervisor_status local supervisor_running=false - supervisor_status=$(supervisorctl status 2>&1) - if [ $? -eq 0 ]; then + if supervisor_status=$(supervisorctl status 2>&1); then supervisor_running=true fi diff --git a/lib/env_handler.sh b/lib/env_handler.sh index 3ce455fb..33b4d187 100644 --- a/lib/env_handler.sh +++ b/lib/env_handler.sh @@ -105,7 +105,7 @@ format_env_vars() { echo "$json" | jq . ;; text) - echo "$vars" | sed 's/export \([^=]*\)=\"\([^\"]*\)"/\1=\2/' + echo "${vars#export }" | tr -d '\"' ;; *) log_error "Environment" "Unknown format: $format" diff --git a/lib/secrets.sh b/lib/secrets.sh index 4a40d428..de589b07 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Dynamically source the required provider-specific modules source_provider_module() { diff --git a/lib/secrets/aws.sh b/lib/secrets/aws.sh index 67ee643e..e8899621 100644 --- a/lib/secrets/aws.sh +++ b/lib/secrets/aws.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Function to resolve AWS secret resolve_aws_secret() { diff --git a/lib/secrets/azure.sh b/lib/secrets/azure.sh index 80c515bd..6359c13e 100644 --- a/lib/secrets/azure.sh +++ b/lib/secrets/azure.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Function to resolve Azure secret resolve_azure_secret() { diff --git a/lib/secrets/bitwarden.sh b/lib/secrets/bitwarden.sh index 962d0689..ae578b84 100644 --- a/lib/secrets/bitwarden.sh +++ b/lib/secrets/bitwarden.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Function to resolve Bitwarden secret resolve_bitwarden_secret() { diff --git a/lib/secrets/gcp.sh b/lib/secrets/gcp.sh index 09bd7c67..cf599b12 100644 --- a/lib/secrets/gcp.sh +++ b/lib/secrets/gcp.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source ${WORKER_LIB_DIR}/utils.sh +source "${WORKER_LIB_DIR}/utils.sh" # Function to resolve GCP secret resolve_gcp_secret() { From 0c6e12c690380c3f136b04dc6c6f8620687642f0 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Thu, 20 Feb 2025 21:50:08 +0300 Subject: [PATCH 12/25] added examples --- .../simple-service/.config/worker/services.yaml | 17 ----------------- .../simple-service/.config/worker/services.yaml | 9 +++++++++ .../examples}/simple-service/index.sh | 0 src/scripts/main.sh | 7 ------- 4 files changed, 9 insertions(+), 24 deletions(-) delete mode 100644 examples/simple-service/.config/worker/services.yaml create mode 100644 src/examples/simple-service/.config/worker/services.yaml rename {examples => src/examples}/simple-service/index.sh (100%) delete mode 100644 src/scripts/main.sh diff --git a/examples/simple-service/.config/worker/services.yaml b/examples/simple-service/.config/worker/services.yaml deleted file mode 100644 index b537ab8f..00000000 --- a/examples/simple-service/.config/worker/services.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -kind: workerService -version: udx.io/worker-v1/service - -services: - - name: "index" - command: "/home/udx/index.sh" - autostart: true - autorestart: true - - name: "index2" - command: "/home/udx/index.sh" - autostart: true - autorestart: true - - name: "index3" - command: "/home/udx/index.sh" - autostart: true - autorestart: true diff --git a/src/examples/simple-service/.config/worker/services.yaml b/src/examples/simple-service/.config/worker/services.yaml new file mode 100644 index 00000000..11cbf511 --- /dev/null +++ b/src/examples/simple-service/.config/worker/services.yaml @@ -0,0 +1,9 @@ +--- +kind: workerService +version: udx.io/worker-v1/service + +services: + - name: "index" + command: "/home/udx/index.sh" + autostart: true + autorestart: true diff --git a/examples/simple-service/index.sh b/src/examples/simple-service/index.sh similarity index 100% rename from examples/simple-service/index.sh rename to src/examples/simple-service/index.sh diff --git a/src/scripts/main.sh b/src/scripts/main.sh deleted file mode 100644 index d317945d..00000000 --- a/src/scripts/main.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# Script to run as a supervisor service in a loop -while true; do - echo "Service is running at $(date)" - sleep 5 -done \ No newline at end of file From 58a7f02ebe9460835dd885767f7f273f02ffdd7a Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 11:28:17 +0300 Subject: [PATCH 13/25] added simple-config example --- src/examples/simple-config/.config/worker/worker.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/examples/simple-config/.config/worker/worker.yaml diff --git a/src/examples/simple-config/.config/worker/worker.yaml b/src/examples/simple-config/.config/worker/worker.yaml new file mode 100644 index 00000000..85441ca3 --- /dev/null +++ b/src/examples/simple-config/.config/worker/worker.yaml @@ -0,0 +1,8 @@ +--- +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + CREATED: "2025-02-21T07:57:59Z" + secrets: + TEST_ENV: "azure/kv-udx-worker-tooling/test-secret" \ No newline at end of file From a964d42eb767285f621e0ed699bdd8a2fa284fc3 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 11:28:38 +0300 Subject: [PATCH 14/25] apt-utils upgrade --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 99e3d170..76e57dda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ RUN apt-get update && \ tzdata=2025a-2ubuntu1 \ curl=8.12.0+git20250209.89ed161+ds-1ubuntu1 \ bash=5.2.37-1ubuntu1 \ - apt-utils=2.9.30 \ + apt-utils=2.9.30ubuntu1 \ gettext=0.23.1-1 \ gnupg=2.4.4-2ubuntu22 \ ca-certificates=20241223 \ From 6d9d0e9e1abdc5a995c85ce6cf3d2d0572b5b86b Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 11:28:50 +0300 Subject: [PATCH 15/25] cli tweaks --- lib/cli.sh | 15 ++++++++++----- lib/cli/service.sh | 2 +- lib/process_manager.sh | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/cli.sh b/lib/cli.sh index 9eb1d1a3..4ebaf35a 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -35,12 +35,17 @@ get_available_commands() { # Extract description from help function if grep -q "${name}_help()" "$module"; then - description=$(grep -A 5 "${name}_help()" "$module" | - grep -v "${name}_help()" | - grep -v "^{" | - grep -v "cat << EOF" | + # Try to find description from comment before help function + description=$(grep -B 2 "${name}_help()" "$module" | + grep "^#" | + grep -v "Show help" | head -n 1 | - sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + sed 's/^#[[:space:]]*//;s/[[:space:]]*$//') + + # If no description found, use a generic one + if [ -z "$description" ]; then + description="Manage ${name} operations" + fi commands[$name]="$description" fi fi diff --git a/lib/cli/service.sh b/lib/cli/service.sh index 5b6a2ba7..8aeb9190 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -70,7 +70,7 @@ service_handler() { if [ "$cmd" != "init" ]; then if [ ! -f "$SERVICES_CONFIG_FILE" ]; then log_warn "Service" "No services configuration found" - log_info "Service" "Run 'worker help service' for information about service configuration" + log_info "Service" "Run 'worker service' for information about service configuration" return 1 fi fi diff --git a/lib/process_manager.sh b/lib/process_manager.sh index d511e3f4..96c17c33 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -21,7 +21,7 @@ main() { # Check if user config exists if [[ ! -f "${USER_CONFIG_PATH}" ]]; then log_info "No services configuration found at ${USER_CONFIG_PATH}." - log_info "Run 'worker help service' for information about service configuration" + log_info "Run 'worker service' for information about service configuration" exit 0 fi @@ -29,7 +29,7 @@ main() { if ! configure_and_execute_services; then log_error "Process Manager" "Failed to configure and start services" - log_info "Run 'worker help service' for information about service configuration" + log_info "Run 'worker service' for information about service configuration" exit 1 fi From a296583b297a644dc462b3d2701b54600895287c Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 12:22:24 +0300 Subject: [PATCH 16/25] added secrets resolution to cli --- lib/cli/config.sh | 27 ++++ lib/secrets.sh | 126 +++++++++++------- .../simple-config/.config/worker/worker.yaml | 3 +- 3 files changed, 107 insertions(+), 49 deletions(-) diff --git a/lib/cli/config.sh b/lib/cli/config.sh index 33ef451b..f45de37d 100644 --- a/lib/cli/config.sh +++ b/lib/cli/config.sh @@ -3,6 +3,7 @@ # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 source "${WORKER_LIB_DIR}/utils.sh" source "${WORKER_LIB_DIR}/worker_config.sh" +source "${WORKER_LIB_DIR}/secrets.sh" # Show help for config command config_help() { @@ -18,6 +19,7 @@ Available Commands: init Initialize a new configuration file diff Show differences between default and current config apply Parse and apply the configuration + resolve Resolve a secret value by name Examples: worker config show @@ -205,6 +207,28 @@ apply_config() { return 0 } +# Description: Resolve a secret value by name +# Example: worker config resolve SECRET_NAME +resolve_secret() { + local secret_name="$1" + + if [[ -z "$secret_name" ]]; then + log_error "Config" "Secret name is required" + return 1 + fi + + # Load and parse the configuration + local config_json + config_json=$(load_and_parse_config) + if [[ -z "$config_json" ]]; then + log_error "Config" "Failed to load configuration" + return 1 + fi + + # Use resolve_secret_by_name from secrets.sh + resolve_secret_by_name "$secret_name" "$config_json" +} + # Handle config commands config_handler() { local cmd=$1 @@ -229,6 +253,9 @@ config_handler() { diff) show_diff ;; + resolve) + resolve_secret "$@" + ;; help) config_help ;; diff --git a/lib/secrets.sh b/lib/secrets.sh index de589b07..27d8e9cc 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -40,63 +40,28 @@ fetch_secrets() { secrets_env_file=$(mktemp /tmp/secret_vars.XXXXXX) echo "# Secrets environment variables" > "$secrets_env_file" + # Create a JSON object with the secrets in the format expected by resolve_secret_by_name + local config_json + config_json=$(echo "{ \"config\": { \"secrets\": $secrets_json } }") + # Process each secret in the JSON object echo "$secrets_json" | jq -c 'to_entries[]' | while IFS= read -r secret; do - local name url value provider secret_name key_vault_name + local name value name=$(echo "$secret" | jq -r '.key') - url=$(resolve_env_vars "$(echo "$secret" | jq -r '.value')") - - # Check if the secret has a valid name and URL - if [[ -z "$name" || -z "$url" ]]; then - log_error "Secrets" "Secret name or URL is missing or empty." - continue - fi - # Extract provider from the URL (first part before '/') - provider=$(echo "$url" | cut -d '/' -f 1) - - # Handle secrets based on the provider - case "$provider" in - gcp) - key_vault_name=$(echo "$url" | cut -d '/' -f 2) - secret_name=$(echo "$url" | cut -d '/' -f 3) - if [[ -z "$secret_name" ]]; then - log_error "Secrets" "Invalid GCP secret name format: $url" - continue - fi - ;; - azure|bitwarden) - key_vault_name=$(echo "$url" | cut -d '/' -f 2) - secret_name=$(echo "$url" | cut -d '/' -f 3) - if [[ -z "$key_vault_name" || -z "$secret_name" ]]; then - log_error "Secrets" "Invalid secret format for $provider: $url" - continue - fi - ;; - *) - log_warn "Unsupported provider: $provider" - continue - ;; - esac - - # Source the provider module dynamically - source_provider_module "$provider" - - # Determine the resolve function for the provider - local resolve_function="resolve_${provider}_secret" - if command -v "$resolve_function" > /dev/null; then - value=$("$resolve_function" "$key_vault_name" "$secret_name") - else - log_warn "No resolve function found for provider: $provider" + # Check if the secret has a valid name + if [[ -z "$name" ]]; then + log_error "Secrets" "Secret name is missing or empty." continue fi - # Export the secret as an environment variable - if [[ -n "$value" ]]; then + # Resolve the secret value using resolve_secret_by_name + value=$(resolve_secret_by_name "$name" "$config_json") + if [[ $? -eq 0 && -n "$value" ]]; then echo "export $name=\"$value\"" >> "$secrets_env_file" - log_success "Secrets" "Resolved secret for $name from $provider." + log_success "Secrets" "Resolved secret for $name." else - log_error "Secrets" "Failed to resolve secret for $name from $provider." + log_error "Secrets" "Failed to resolve secret for $name." fi done @@ -127,5 +92,70 @@ clean_up_files() { done } +# Resolve a secret value by name from the configuration +resolve_secret_by_name() { + local secret_name="$1" + local config_json="$2" + + if [[ -z "$secret_name" ]]; then + log_error "Secrets" "Secret name is required" + return 1 + fi + + if [[ -z "$config_json" ]]; then + log_error "Secrets" "Configuration is required" + return 1 + fi + + # Extract secrets section + local secrets + secrets=$(echo "$config_json" | jq -r '.config.secrets // empty') + if [[ -z "$secrets" || "$secrets" == "null" ]]; then + log_error "Secrets" "No secrets found in configuration" + return 1 + fi + + # Find the secret URL + local secret_url + secret_url=$(echo "$secrets" | jq -r ".[\"$secret_name\"] // empty") + if [[ -z "$secret_url" ]]; then + log_error "Secrets" "Secret '$secret_name' not found in configuration" + return 1 + fi + + # Resolve any environment variables in the URL + secret_url=$(resolve_env_vars "$secret_url") + if [[ -z "$secret_url" ]]; then + log_error "Secrets" "Failed to resolve environment variables in URL" + return 1 + fi + + # Extract provider and parts from URL + local provider key_vault_name secret_value + provider=$(echo "$secret_url" | cut -d '/' -f 1) + key_vault_name=$(echo "$secret_url" | cut -d '/' -f 2) + secret_value=$(echo "$secret_url" | cut -d '/' -f 3) + + # Source the provider module + source_provider_module "$provider" + + # Resolve the secret + local resolve_function="resolve_${provider}_secret" + if command -v "$resolve_function" > /dev/null; then + local value + value=$("$resolve_function" "$key_vault_name" "$secret_value") + if [[ -n "$value" ]]; then + echo "$value" + return 0 + else + log_error "Secrets" "Failed to resolve secret value" + return 1 + fi + else + log_error "Secrets" "No resolver found for provider: $provider" + return 1 + fi +} + # Example usage: # fetch_secrets '{"TEST": "gcp/new_relic_api_key"}' diff --git a/src/examples/simple-config/.config/worker/worker.yaml b/src/examples/simple-config/.config/worker/worker.yaml index 85441ca3..a897f2ec 100644 --- a/src/examples/simple-config/.config/worker/worker.yaml +++ b/src/examples/simple-config/.config/worker/worker.yaml @@ -5,4 +5,5 @@ config: env: CREATED: "2025-02-21T07:57:59Z" secrets: - TEST_ENV: "azure/kv-udx-worker-tooling/test-secret" \ No newline at end of file + TEST_ENV: "azure/kv-udx-worker-tooling/test-secret" + ANOTHER_TEST_ENV: "azure/kv-udx-worker-tooling/test-secret" \ No newline at end of file From 12697bdcebd41f2a2fb34d90ea47c2606e324d92 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 14:16:58 +0300 Subject: [PATCH 17/25] tests implementation --- Makefile | 4 +- src/tests/env.sh | 29 ----------- src/tests/main.sh | 85 +++++++++++++++++++++++++++++++++ src/tests/modules/10_config.sh | 33 +++++++++++++ src/tests/modules/20_env.sh | 46 ++++++++++++++++++ src/tests/modules/30_auth.sh | 36 ++++++++++++++ src/tests/modules/40_service.sh | 42 ++++++++++++++++ src/tests/modules/50_sbom.sh | 44 +++++++++++++++++ src/tests/modules/60_health.sh | 42 ++++++++++++++++ src/tests/run_tests.sh | 43 +++++++++++++++++ src/tests/service.sh | 27 ----------- src/tests/test_helpers.sh | 35 ++++++++++++++ 12 files changed, 408 insertions(+), 58 deletions(-) delete mode 100755 src/tests/env.sh create mode 100755 src/tests/main.sh create mode 100755 src/tests/modules/10_config.sh create mode 100755 src/tests/modules/20_env.sh create mode 100755 src/tests/modules/30_auth.sh create mode 100755 src/tests/modules/40_service.sh create mode 100755 src/tests/modules/50_sbom.sh create mode 100755 src/tests/modules/60_health.sh create mode 100755 src/tests/run_tests.sh delete mode 100755 src/tests/service.sh create mode 100755 src/tests/test_helpers.sh diff --git a/Makefile b/Makefile index 1cc33b1f..0b4bda00 100644 --- a/Makefile +++ b/Makefile @@ -104,8 +104,8 @@ test: clean @chmod +x src/tests/*.sh @$(MAKE) run \ INTERACTIVE=true \ - VOLUMES="$(PWD)/src/tests:/tests" \ - COMMAND="/bin/bash -c '/tests/env.sh && /tests/service.sh'" || exit 1 + VOLUMES="$(PWD)/src/tests:/home/udx/tests $(PWD)/src/examples/simple-config/.config/worker/worker.yaml:/home/udx/.config/worker/worker.yaml $(PWD)/src/examples/simple-service/.config/worker/services.yaml:/home/udx/.config/worker/services.yaml $(PWD)/src/examples/simple-service/index.sh:/home/udx/index.sh" \ + COMMAND="/home/udx/tests/main.sh" || exit 1 @$(MAKE) clean || exit 1 @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Tests completed successfully$(COLOR_RESET)\n" diff --git a/src/tests/env.sh b/src/tests/env.sh deleted file mode 100755 index 3d0a1dbd..00000000 --- a/src/tests/env.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# Test environment commands - -# Test environment show -echo "Testing: env show" -worker env show -if ! worker env show | grep "Environment variables"; then - echo "FAIL: env show should display environment variables" - exit 1 -fi - -# Test environment set/get -echo "Testing: env set/get" -worker env set TEST_VAR "test value" -if ! worker env show | grep "TEST_VAR=test value"; then - echo "FAIL: env set/get not working" - exit 1 -fi - -# Test environment JSON output -echo "Testing: env show --format json" -if ! worker env show --format json | jq -e '.variables'; then - echo "FAIL: JSON output not working" - exit 1 -fi - -# All tests passed -echo "Environment tests passed" diff --git a/src/tests/main.sh b/src/tests/main.sh new file mode 100755 index 00000000..278096b4 --- /dev/null +++ b/src/tests/main.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# Main test entrypoint that runs inside the container + +# Source test helpers +source "/home/udx/tests/test_helpers.sh" + +# Exit on any error +set -e + +# Initialize test environment +print_header "Setting up test environment" + +# Setup test config and examples +CONFIG_DIR=/home/udx/.config/worker + +# Create necessary directories +mkdir -p "$CONFIG_DIR" + +# Config files are already mounted at the right locations by the Makefile + +# Export test environment variables +export TEST_CONFIG_DIR=/home/udx/.config/worker +export TEST_CONFIG_FILE="$TEST_CONFIG_DIR/worker.yaml" + +# Counter for test results +TOTAL=0 +PASSED=0 +FAILED=0 + +# Directory containing test files +TEST_DIR="/home/udx/tests" +cd "$TEST_DIR" + +# Make all test files executable +chmod +x modules/*.sh + +# Don't exit on test failures +set +e + +# Run test modules in numeric order +for test_file in modules/[0-9]*.sh; do + # Skip if no files found + [ -e "$test_file" ] || continue + + # Extract just the filename + test_name=$(basename "$test_file") + + print_header "Running $test_name" + TOTAL=$((TOTAL + 1)) + + # Run test and capture all output + TEST_OUTPUT=$($test_file 2>&1) + TEST_RESULT=$? + + if [ $TEST_RESULT -eq 0 ]; then + print_success "$test_name passed" + PASSED=$((PASSED + 1)) + # Show test output even on success + echo "$TEST_OUTPUT" + else + print_error "$test_name failed (exit code: $TEST_RESULT)" + print_error "Test output:" + echo "$TEST_OUTPUT" + FAILED=$((FAILED + 1)) + fi + + echo # Add blank line between tests +done + +# Exit with error if any test failed +[ $FAILED -eq 0 ] + +# Print summary +print_header "Test Summary" +if [ "$FAILED" -eq 0 ]; then + print_success "All $TOTAL tests passed successfully" +else + print_error "$FAILED of $TOTAL tests failed" + printf "${GREEN}Passed: %d${NC}\n" "$PASSED" + printf "${RED}Failed: %d${NC}\n" "$FAILED" +fi + +# Exit with failure if any tests failed +[ "$FAILED" -eq 0 ] || exit 1 diff --git a/src/tests/modules/10_config.sh b/src/tests/modules/10_config.sh new file mode 100755 index 00000000..b90168b0 --- /dev/null +++ b/src/tests/modules/10_config.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Source test helpers +source "/home/udx/tests/test_helpers.sh" + +# Test configuration commands +print_header "Configuration Tests" + +# Test config show +print_info "Testing: config show" +CONFIG_OUTPUT=$(worker config show) +if ! echo "$CONFIG_OUTPUT" | grep -q "kind: workerConfig"; then + print_error "config show should display configuration" + exit 1 +fi + +# Test config show with format json +print_info "Testing: config show --format json" +CONFIG_JSON=$(worker config show --format json) +if ! echo "$CONFIG_JSON" | jq -e '.config.env' > /dev/null; then + print_error "config show json format should include env section" + exit 1 +fi + +# Test config locations +print_info "Testing: config locations" +if ! worker config locations | grep -q "/home/udx/.config/worker"; then + print_error "config locations should show config paths" + exit 1 +fi + +# All tests passed +print_success "All configuration tests passed" diff --git a/src/tests/modules/20_env.sh b/src/tests/modules/20_env.sh new file mode 100755 index 00000000..3e00e503 --- /dev/null +++ b/src/tests/modules/20_env.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Source test helpers +source "/home/udx/tests/test_helpers.sh" + +# Test environment commands +print_header "Environment Tests" + +# Get environment variables from config +print_info "Getting environment variables from config" +CONFIG_ENV=$(worker config show --format json | jq -r '.config.env | keys[]') +if [ -z "$CONFIG_ENV" ]; then + print_error "No environment variables found in config" + exit 1 +fi + +# Test environment show +print_info "Testing: env show" +ENV_OUTPUT=$(worker env show) +for var in $CONFIG_ENV; do + if ! echo "$ENV_OUTPUT" | grep -q "$var="; then + print_error "env show missing variable: $var" + exit 1 + fi +done + +# Test environment set/get +print_info "Testing: env set/get" +worker env set TEST_VAR "test value" +if ! worker env show | grep -q "TEST_VAR=test value"; then + print_error "env set/get not working" + exit 1 +fi + +# Test environment JSON output +print_info "Testing: env show --format json" +JSON_OUTPUT=$(worker env show --format json) +for var in $CONFIG_ENV; do + if ! echo "$JSON_OUTPUT" | jq -e --arg var "$var" 'has($var)' > /dev/null; then + print_error "JSON output missing variable: $var" + exit 1 + fi +done + +# All tests passed +print_success "All environment tests passed" diff --git a/src/tests/modules/30_auth.sh b/src/tests/modules/30_auth.sh new file mode 100755 index 00000000..0e2f7924 --- /dev/null +++ b/src/tests/modules/30_auth.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Source test helpers +source "/home/udx/tests/test_helpers.sh" + +# Test authentication commands +print_header "Authentication Tests" + +# Test auth status +print_info "Testing: auth status" + +# Capture both stdout and stderr +AUTH_OUTPUT=$(worker auth status 2>&1) + +# Test that we got some output +if [ -z "$AUTH_OUTPUT" ]; then + print_error "auth status produced no output" + exit 1 +fi + +# Test that output contains provider status +if ! printf "%s" "$AUTH_OUTPUT" | grep -q "Auth:"; then + print_error "auth status should show provider status" + exit 1 +fi + +# Test auth status json format +print_info "Testing: auth status --format json" +AUTH_JSON=$(worker auth status --format json) +if ! echo "$AUTH_JSON" | jq -e '.[] | select(.provider == "aws") | .status' > /dev/null; then + print_error "auth status json should include provider status" + exit 1 +fi + +# All tests passed +print_success "All authentication tests passed" diff --git a/src/tests/modules/40_service.sh b/src/tests/modules/40_service.sh new file mode 100755 index 00000000..01f24cef --- /dev/null +++ b/src/tests/modules/40_service.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Source test helpers +source "/home/udx/tests/test_helpers.sh" + +# Test service commands +print_header "Service Tests" + +# Test service list +print_info "Testing: service list" +SERVICE_OUTPUT=$(worker service list 2>&1) + +# Check that we get some output +if [ -z "$SERVICE_OUTPUT" ]; then + print_error "service list produced no output" + exit 1 +fi + +# Test service list json format +print_info "Testing: service list --format json" +LIST_JSON=$(worker service list --format json 2>&1) +if ! echo "$LIST_JSON" | jq -e '.services' > /dev/null; then + print_error "service list json should include services array" + exit 1 +fi + +# Test service config +print_info "Testing: service config" +CONFIG_OUTPUT=$(worker service config 2>&1) + +# Check that we get some output +if [ -z "$CONFIG_OUTPUT" ]; then + print_error "service config produced no output" + exit 1 +fi +if echo "$STATUS_OUTPUT" | grep -q "Simple service is running"; then + print_error "service should be stopped" + exit 1 +fi + +# All tests passed +print_success "All service tests passed" diff --git a/src/tests/modules/50_sbom.sh b/src/tests/modules/50_sbom.sh new file mode 100755 index 00000000..772334c9 --- /dev/null +++ b/src/tests/modules/50_sbom.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Source test helpers +source "/home/udx/tests/test_helpers.sh" + +# Test SBOM commands +print_header "SBOM Tests" + +# Test SBOM generation +print_info "Testing: sbom generate" +if ! worker sbom generate > /tmp/test-sbom.json; then + print_error "sbom generate should create SBOM file" + exit 1 +fi + +# Test SBOM verify +print_info "Testing: sbom verify" +if ! worker sbom verify; then + print_error "sbom verify should check package integrity" + exit 1 +fi + +# Test SBOM format options +print_info "Testing: sbom generate with format" +if ! worker sbom generate --format json > /tmp/test-sbom.json; then + print_error "sbom generate with json format failed" + exit 1 +fi + +# Test SBOM type options +print_info "Testing: sbom generate with type" +for type in system python all; do + if ! worker sbom generate --type "$type" > "/tmp/test-sbom-$type.json"; then + print_error "sbom generate for type $type failed" + exit 1 + fi + rm -f "/tmp/test-sbom-$type.json" +done + +# Cleanup +rm -f /tmp/test-sbom.json + +# All tests passed +print_success "All SBOM tests passed" diff --git a/src/tests/modules/60_health.sh b/src/tests/modules/60_health.sh new file mode 100755 index 00000000..2806ad7d --- /dev/null +++ b/src/tests/modules/60_health.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Source test helpers +source "/home/udx/tests/test_helpers.sh" + +# Test health check commands +print_header "Health Check Tests" + +# Test health status +print_info "Testing: health status" +HEALTH_OUTPUT=$(worker health status 2>&1) + +# Check for health check running message +if ! echo "$HEALTH_OUTPUT" | grep -q "Running health check"; then + print_error "health status should show running message" + exit 1 +fi + +# Check for successful completion +if ! echo "$HEALTH_OUTPUT" | grep -q "All health checks passed"; then + print_error "health status should show successful completion" + exit 1 +fi + +# Test health status json format +print_info "Testing: health status --format json" +HEALTH_JSON=$(worker health status --format json 2>&1 | sed -n '/^{/,/^}/p') + +# Verify JSON structure +if ! echo "$HEALTH_JSON" | jq -e '.status' > /dev/null; then + print_error "health status json should include status field" + exit 1 +fi + +# Verify health metrics are present +if ! echo "$HEALTH_JSON" | jq -e '.disk.usage_percent' > /dev/null; then + print_error "health status json should include disk usage" + exit 1 +fi + +# All tests passed +print_success "All health check tests passed" diff --git a/src/tests/run_tests.sh b/src/tests/run_tests.sh new file mode 100755 index 00000000..e75a57a3 --- /dev/null +++ b/src/tests/run_tests.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Main test runner for CLI commands + +# Set error handling +set -e + +# Change to script directory +cd "$(dirname "$0")" + +# Make all test files executable +chmod +x ./*.sh + +# Counter for test results +TOTAL=0 +PASSED=0 +FAILED=0 + +# Run each test file +for test_file in auth.sh config.sh env.sh health.sh sbom.sh service.sh; do + if [ -f "$test_file" ]; then + echo "Running tests from $test_file..." + TOTAL=$((TOTAL + 1)) + + if ./"$test_file"; then + echo "βœ“ $test_file passed" + PASSED=$((PASSED + 1)) + else + echo "βœ— $test_file failed" + FAILED=$((FAILED + 1)) + fi + echo "----------------------------------------" + fi +done + +# Print summary +echo "Test Summary:" +echo "Total: $TOTAL" +echo "Passed: $PASSED" +echo "Failed: $FAILED" + +# Exit with failure if any tests failed +[ "$FAILED" -eq 0 ] || exit 1 diff --git a/src/tests/service.sh b/src/tests/service.sh deleted file mode 100755 index dd063901..00000000 --- a/src/tests/service.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Test service commands - -# Test service list -echo "Testing: service list" -if ! worker service list | grep "Available services"; then - echo "FAIL: service list should show available services" - exit 1 -fi - -# Test service status -echo "Testing: service status" -if ! worker service status | grep "Service status"; then - echo "FAIL: service status not working" - exit 1 -fi - -# Test service JSON output -echo "Testing: service list --format json" -if ! worker service list --format json | jq -e '.services'; then - echo "FAIL: JSON output not working" - exit 1 -fi - -# All tests passed -echo "Service tests passed" diff --git a/src/tests/test_helpers.sh b/src/tests/test_helpers.sh new file mode 100755 index 00000000..5929c39c --- /dev/null +++ b/src/tests/test_helpers.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Test helper functions and common utilities + +# Colors and symbols +GREEN='\033[0;32m' +RED='\033[0;31m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +NC='\033[0m' +CHECK="βœ“" +CROSS="βœ—" +INFO="ℹ️" +WARN="⚠️" + +# Helper functions +print_header() { + printf "\n${BLUE}=== %s ===${NC}\n" "$1" +} + +print_success() { + printf "${GREEN}%s %s${NC}\n" "$CHECK" "$1" +} + +print_error() { + printf "${RED}%s %s${NC}\n" "$CROSS" "$1" +} + +print_info() { + printf "${BLUE}%s %s${NC}\n" "$INFO" "$1" +} + +print_warning() { + printf "${YELLOW}%s %s${NC}\n" "$WARN" "$1" +} From b64490c841d339f88b4d7d90c318e3e48854b876 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 15:00:07 +0300 Subject: [PATCH 18/25] codeql fixes --- lib/secrets.sh | 2 +- src/examples/simple-config/.config/worker/worker.yaml | 2 +- src/tests/main.sh | 1 + src/tests/modules/10_config.sh | 1 + src/tests/modules/20_env.sh | 1 + src/tests/modules/30_auth.sh | 1 + src/tests/modules/40_service.sh | 1 + src/tests/modules/50_sbom.sh | 1 + src/tests/modules/60_health.sh | 1 + 9 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/secrets.sh b/lib/secrets.sh index 27d8e9cc..255fb77c 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -42,7 +42,7 @@ fetch_secrets() { # Create a JSON object with the secrets in the format expected by resolve_secret_by_name local config_json - config_json=$(echo "{ \"config\": { \"secrets\": $secrets_json } }") + config_json="{ \"config\": { \"secrets\": $secrets_json } }" # Process each secret in the JSON object echo "$secrets_json" | jq -c 'to_entries[]' | while IFS= read -r secret; do diff --git a/src/examples/simple-config/.config/worker/worker.yaml b/src/examples/simple-config/.config/worker/worker.yaml index a897f2ec..601c9ef7 100644 --- a/src/examples/simple-config/.config/worker/worker.yaml +++ b/src/examples/simple-config/.config/worker/worker.yaml @@ -6,4 +6,4 @@ config: CREATED: "2025-02-21T07:57:59Z" secrets: TEST_ENV: "azure/kv-udx-worker-tooling/test-secret" - ANOTHER_TEST_ENV: "azure/kv-udx-worker-tooling/test-secret" \ No newline at end of file + ANOTHER_TEST_ENV: "azure/kv-udx-worker-tooling/test-secret" diff --git a/src/tests/main.sh b/src/tests/main.sh index 278096b4..93cf42c4 100755 --- a/src/tests/main.sh +++ b/src/tests/main.sh @@ -3,6 +3,7 @@ # Main test entrypoint that runs inside the container # Source test helpers +# shellcheck source=./test_helpers.sh disable=SC1091 source "/home/udx/tests/test_helpers.sh" # Exit on any error diff --git a/src/tests/modules/10_config.sh b/src/tests/modules/10_config.sh index b90168b0..cb717822 100755 --- a/src/tests/modules/10_config.sh +++ b/src/tests/modules/10_config.sh @@ -1,6 +1,7 @@ #!/bin/bash # Source test helpers +# shellcheck source=../test_helpers.sh disable=SC1091 source "/home/udx/tests/test_helpers.sh" # Test configuration commands diff --git a/src/tests/modules/20_env.sh b/src/tests/modules/20_env.sh index 3e00e503..31ec27f6 100755 --- a/src/tests/modules/20_env.sh +++ b/src/tests/modules/20_env.sh @@ -1,6 +1,7 @@ #!/bin/bash # Source test helpers +# shellcheck source=../test_helpers.sh disable=SC1091 source "/home/udx/tests/test_helpers.sh" # Test environment commands diff --git a/src/tests/modules/30_auth.sh b/src/tests/modules/30_auth.sh index 0e2f7924..ef011cba 100755 --- a/src/tests/modules/30_auth.sh +++ b/src/tests/modules/30_auth.sh @@ -1,6 +1,7 @@ #!/bin/bash # Source test helpers +# shellcheck source=../test_helpers.sh disable=SC1091 source "/home/udx/tests/test_helpers.sh" # Test authentication commands diff --git a/src/tests/modules/40_service.sh b/src/tests/modules/40_service.sh index 01f24cef..e11faf05 100755 --- a/src/tests/modules/40_service.sh +++ b/src/tests/modules/40_service.sh @@ -1,6 +1,7 @@ #!/bin/bash # Source test helpers +# shellcheck source=../test_helpers.sh disable=SC1091 source "/home/udx/tests/test_helpers.sh" # Test service commands diff --git a/src/tests/modules/50_sbom.sh b/src/tests/modules/50_sbom.sh index 772334c9..bf4c141b 100755 --- a/src/tests/modules/50_sbom.sh +++ b/src/tests/modules/50_sbom.sh @@ -1,6 +1,7 @@ #!/bin/bash # Source test helpers +# shellcheck source=../test_helpers.sh disable=SC1091 source "/home/udx/tests/test_helpers.sh" # Test SBOM commands diff --git a/src/tests/modules/60_health.sh b/src/tests/modules/60_health.sh index 2806ad7d..2ba2e12d 100755 --- a/src/tests/modules/60_health.sh +++ b/src/tests/modules/60_health.sh @@ -1,6 +1,7 @@ #!/bin/bash # Source test helpers +# shellcheck source=../test_helpers.sh disable=SC1091 source "/home/udx/tests/test_helpers.sh" # Test health check commands From 9b1c27b8fce00c991f8338aa2518abe26909bc80 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 15:12:26 +0300 Subject: [PATCH 19/25] non-interactive make test --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0b4bda00..d2c61930 100644 --- a/Makefile +++ b/Makefile @@ -101,11 +101,14 @@ clean: test: clean @printf "$(COLOR_BLUE)$(SYM_ARROW) Running tests...$(COLOR_RESET)\n" - @chmod +x src/tests/*.sh @$(MAKE) run \ - INTERACTIVE=true \ VOLUMES="$(PWD)/src/tests:/home/udx/tests $(PWD)/src/examples/simple-config/.config/worker/worker.yaml:/home/udx/.config/worker/worker.yaml $(PWD)/src/examples/simple-service/.config/worker/services.yaml:/home/udx/.config/worker/services.yaml $(PWD)/src/examples/simple-service/index.sh:/home/udx/index.sh" \ - COMMAND="/home/udx/tests/main.sh" || exit 1 + COMMAND="/home/udx/tests/main.sh" + @printf "$(COLOR_BLUE)$(SYM_ARROW) Following test output...$(COLOR_RESET)\n" + @docker logs -f $(CONTAINER_NAME) & LOGS_PID=$$!; \ + docker wait $(CONTAINER_NAME) > /dev/null; EXIT_CODE=$$?; \ + kill $$LOGS_PID 2>/dev/null || true; \ + exit $$EXIT_CODE @$(MAKE) clean || exit 1 @printf "$(COLOR_GREEN)$(SYM_SUCCESS) Tests completed successfully$(COLOR_RESET)\n" From 9b7dc1185b33fe865a01ead203f8b64daaf7a384 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 15:19:55 +0300 Subject: [PATCH 20/25] cleanup --- src/tests/main.sh | 3 --- src/tests/run_tests.sh | 43 ------------------------------------------ 2 files changed, 46 deletions(-) delete mode 100755 src/tests/run_tests.sh diff --git a/src/tests/main.sh b/src/tests/main.sh index 93cf42c4..88459e5a 100755 --- a/src/tests/main.sh +++ b/src/tests/main.sh @@ -33,9 +33,6 @@ FAILED=0 TEST_DIR="/home/udx/tests" cd "$TEST_DIR" -# Make all test files executable -chmod +x modules/*.sh - # Don't exit on test failures set +e diff --git a/src/tests/run_tests.sh b/src/tests/run_tests.sh deleted file mode 100755 index e75a57a3..00000000 --- a/src/tests/run_tests.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# Main test runner for CLI commands - -# Set error handling -set -e - -# Change to script directory -cd "$(dirname "$0")" - -# Make all test files executable -chmod +x ./*.sh - -# Counter for test results -TOTAL=0 -PASSED=0 -FAILED=0 - -# Run each test file -for test_file in auth.sh config.sh env.sh health.sh sbom.sh service.sh; do - if [ -f "$test_file" ]; then - echo "Running tests from $test_file..." - TOTAL=$((TOTAL + 1)) - - if ./"$test_file"; then - echo "βœ“ $test_file passed" - PASSED=$((PASSED + 1)) - else - echo "βœ— $test_file failed" - FAILED=$((FAILED + 1)) - fi - echo "----------------------------------------" - fi -done - -# Print summary -echo "Test Summary:" -echo "Total: $TOTAL" -echo "Passed: $PASSED" -echo "Failed: $FAILED" - -# Exit with failure if any tests failed -[ "$FAILED" -eq 0 ] || exit 1 From 0460665e09f79a0491d81d46e438a2a7b4364c1a Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 16:19:37 +0300 Subject: [PATCH 21/25] added resolved secrets to environment --- Dockerfile | 13 +++----- lib/cli/config.sh | 12 +++++++ lib/cli/env.sh | 18 ++++++++-- lib/env_handler.sh | 79 ++++++++++++++++++++++++-------------------- lib/secrets.sh | 47 ++++---------------------- lib/worker_config.sh | 18 +++------- 6 files changed, 87 insertions(+), 100 deletions(-) diff --git a/Dockerfile b/Dockerfile index 76e57dda..ca11516e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -133,7 +133,7 @@ RUN mkdir -p \ ${WORKER_LIB_DIR} \ ${WORKER_BIN_DIR} \ ${WORKER_ETC_DIR} \ - # Environment and secrets files directory + # Environment files directory ${WORKER_CONFIG_DIR}/environment.d \ # User and config directories ${HOME}/.config/worker \ @@ -142,12 +142,9 @@ RUN mkdir -p \ ${AWS_CONFIG_FILE%/*} \ ${AZURE_CONFIG_DIR} && \ # Create and set permissions for environment files - touch ${WORKER_CONFIG_DIR}/environment ${WORKER_CONFIG_DIR}/secrets && \ - chown ${USER}:${USER} \ - ${WORKER_CONFIG_DIR}/environment \ - ${WORKER_CONFIG_DIR}/secrets && \ - chmod 644 ${WORKER_CONFIG_DIR}/environment && \ - chmod 600 ${WORKER_CONFIG_DIR}/secrets + touch ${WORKER_CONFIG_DIR}/environment && \ + chown ${USER}:${USER} ${WORKER_CONFIG_DIR}/environment && \ + chmod 644 ${WORKER_CONFIG_DIR}/environment # Copy worker files COPY bin/entrypoint.sh ${WORKER_BIN_DIR}/ @@ -188,8 +185,6 @@ RUN \ ${WORKER_LIB_DIR}/process_manager.sh && \ # Set runtime directories permissions chmod 775 ${WORKER_APP_DIR} ${WORKER_DATA_DIR} && \ - # Set sensitive file permissions - chmod 600 ${WORKER_CONFIG_DIR}/secrets && \ # Set home directory executable chmod 755 ${HOME} diff --git a/lib/cli/config.sh b/lib/cli/config.sh index f45de37d..13a31b41 100644 --- a/lib/cli/config.sh +++ b/lib/cli/config.sh @@ -203,6 +203,18 @@ apply_config() { return 1 fi + # Extract secrets section from config + local secrets_json + secrets_json=$(echo "$config_json" | jq -r '.config.secrets // {}') + + # Fetch and set secrets if any are defined + if [[ "$secrets_json" != "{}" ]]; then + if ! fetch_secrets "$secrets_json"; then + log_error "Config" "Failed to fetch and set secrets" + return 1 + fi + fi + log_success "Config" "Configuration successfully parsed and applied" return 0 } diff --git a/lib/cli/env.sh b/lib/cli/env.sh index 9b4d763a..5df0d67a 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -12,10 +12,10 @@ Manage environment variables and secrets Usage: worker env [command] Available Commands: - show Show environment variables (excludes secrets) + show Show environment variables set Set an environment variable unset Unset an environment variable - reload Reload environment from configuration (same as 'config apply') + reload Reload environment and secrets from configuration status Show environment status Options: @@ -29,7 +29,7 @@ Examples: worker env show --filter AWS_* # Show only AWS variables worker env set MY_VAR "my value" # Set a new variable worker env unset MY_VAR # Remove a variable - worker env reload # Reload from config + worker env reload # Reload environment and secrets from config EOF } @@ -377,6 +377,18 @@ env_handler() { return 1 fi + # Extract secrets section from config + local secrets_json + secrets_json=$(echo "$config_json" | jq -r '.config.secrets // {}') + + # Fetch and set secrets if any are defined + if [[ "$secrets_json" != "{}" ]]; then + if ! fetch_secrets "$secrets_json"; then + log_error "Env" "Failed to fetch and set secrets" + return 1 + fi + fi + log_success "Env" "Environment successfully reloaded from configuration" ;; status) diff --git a/lib/env_handler.sh b/lib/env_handler.sh index 33b4d187..28ef70db 100644 --- a/lib/env_handler.sh +++ b/lib/env_handler.sh @@ -5,9 +5,8 @@ source "${WORKER_LIB_DIR}/utils.sh" # Environment file location WORKER_ENV_FILE="/etc/worker/environment" -WORKER_SECRETS_FILE="/etc/worker/secrets" -# Generate environment file from worker.yaml +# Generate environment file with regular variables generate_env_file() { local config config=$(load_and_parse_config) @@ -19,44 +18,67 @@ generate_env_file() { # Extract and evaluate environment variables { - echo "# Worker Environment Variables" - echo "# Generated on $(date)" - echo "# DO NOT EDIT THIS FILE DIRECTLY" - echo - # Extract environment variables echo "$config" | yq eval '.config.env | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' - } > "$WORKER_ENV_FILE" - # File permissions are set during build } -# Generate secrets file from worker.yaml -generate_secrets_file() { - local config - config=$(load_and_parse_config) +# Append resolved secrets to environment file +append_resolved_secrets() { + local secrets_json="$1" + local has_failures=false - if [ -z "$config" ]; then - log_error "Environment" "Failed to load configuration" + if [ -z "$secrets_json" ]; then + log_error "Environment" "No secrets provided" return 1 fi - # Extract and evaluate secrets + # Create a temporary file for resolved secrets + local temp_file + temp_file=$(mktemp) + { - echo "# Worker Secrets" + echo "" + echo "# Resolved Secrets" echo "# Generated on $(date)" echo "# DO NOT EDIT THIS FILE DIRECTLY" - echo "# DO NOT COMMIT THIS FILE" echo + } > "$temp_file" + + # Process each secret and resolve it + while IFS= read -r secret; do + local name value + name=$(echo "$secret" | jq -r '.key') - # Extract secrets - echo "$config" | yq eval '.config.secrets | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' - - } > "$WORKER_SECRETS_FILE" + # Create config JSON for resolve_secret_by_name + local config_json + config_json="{ \"config\": { \"secrets\": { \"$name\": $(echo "$secret" | jq '.value') } } }" + + # Resolve the secret + if ! value=$(resolve_secret_by_name "$name" "$config_json") || [[ -z "$value" ]]; then + log_error "Environment" "Failed to resolve secret for $name" + has_failures=true + else + echo "export $name=\"$value\"" >> "$temp_file" + log_success "Environment" "Resolved secret for $name" + fi + done < <(echo "$secrets_json" | jq -c 'to_entries[]') - # File permissions are set during build + # If any secret failed to resolve, error out + if [[ "$has_failures" == "true" ]]; then + log_error "Environment" "Failed to resolve one or more secrets" + rm -f "$temp_file" + return 1 + fi + + # If we get here, all secrets resolved successfully + cat "$temp_file" >> "$WORKER_ENV_FILE" + log_success "Environment" "Added all resolved secrets to environment file" + rm -f "$temp_file" } -# Load environment variables +# Load environment variables and secrets load_environment() { if [ -f "$WORKER_ENV_FILE" ]; then # shellcheck source=/dev/null @@ -69,19 +91,6 @@ load_environment() { fi } -# Load secrets -load_secrets() { - if [ -f "$WORKER_SECRETS_FILE" ]; then - # shellcheck source=/dev/null - source "$WORKER_SECRETS_FILE" - else - log_warn "Environment" "Secrets file not found, generating..." - generate_secrets_file - # shellcheck source=/dev/null - source "$WORKER_SECRETS_FILE" - fi -} - # Format environment variables as JSON or text format_env_vars() { local vars=$1 diff --git a/lib/secrets.sh b/lib/secrets.sh index 255fb77c..7b96aed7 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -2,6 +2,8 @@ # shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 source "${WORKER_LIB_DIR}/utils.sh" +# shellcheck source=${WORKER_LIB_DIR}/env_handler.sh disable=SC1091 +source "${WORKER_LIB_DIR}/env_handler.sh" # Dynamically source the required provider-specific modules source_provider_module() { @@ -35,49 +37,14 @@ fetch_secrets() { return 1 fi - # Create a temporary file to store environment variables - local secrets_env_file - secrets_env_file=$(mktemp /tmp/secret_vars.XXXXXX) - echo "# Secrets environment variables" > "$secrets_env_file" - - # Create a JSON object with the secrets in the format expected by resolve_secret_by_name - local config_json - config_json="{ \"config\": { \"secrets\": $secrets_json } }" - - # Process each secret in the JSON object - echo "$secrets_json" | jq -c 'to_entries[]' | while IFS= read -r secret; do - local name value - name=$(echo "$secret" | jq -r '.key') - - # Check if the secret has a valid name - if [[ -z "$name" ]]; then - log_error "Secrets" "Secret name is missing or empty." - continue - fi - - # Resolve the secret value using resolve_secret_by_name - value=$(resolve_secret_by_name "$name" "$config_json") - if [[ $? -eq 0 && -n "$value" ]]; then - echo "export $name=\"$value\"" >> "$secrets_env_file" - log_success "Secrets" "Resolved secret for $name." - else - log_error "Secrets" "Failed to resolve secret for $name." - fi - done - - # Source the environment file if it exists - if [[ -s "$secrets_env_file" ]]; then - set -a - # shellcheck disable=SC1090 - source "$secrets_env_file" - set +a - log_info "Secrets environment variables sourced successfully." - else - log_error "Secrets" "No secrets were written to the environment file." + # Resolve secrets and append them to environment file + if ! append_resolved_secrets "$secrets_json"; then + log_error "Secrets" "Failed to resolve and append secrets" return 1 fi - clean_up_files "$secrets_env_file" + # Source the environment file to update current session + load_environment } # Clean up temporary files diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 844f86b5..274d96c4 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -82,13 +82,12 @@ load_and_parse_config() { export_variables_from_config() { local config_json="$1" - # Extract environment variables and secrets - local env_vars secrets + # Extract only environment variables + local env_vars env_vars=$(echo "$config_json" | jq -r '.config.env // empty') - secrets=$(echo "$config_json" | jq -r '.config.secrets // empty') - if [[ -z "$env_vars" && -z "$secrets" ]]; then - log_info "No variables or secrets found in the configuration." + if [[ -z "$env_vars" ]]; then + log_info "No environment variables found in the configuration." return 0 fi @@ -98,15 +97,8 @@ export_variables_from_config() { generate_env_file fi - # Generate secrets file - if [[ -n "$secrets" && "$secrets" != "null" ]]; then - log_success "Worker configuration" "Found secrets in the configuration." - generate_secrets_file - fi - - # Load both environment and secrets + # Load environment variables load_environment - load_secrets } # Function to extract a specific section from the JSON configuration From dd946f8b83338bcd79a37b886974a1f7558f8d76 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 17:02:19 +0300 Subject: [PATCH 22/25] update docs --- README.md | 195 +++++++++++++++++++++++++++--------- docs/CLI.md | 101 ++++--------------- docs/authorization.md | 74 +++++++++----- docs/config.md | 105 ++++++++++++------- docs/container-structure.md | 116 +++++++++++++++++++++ docs/git-help.md | 164 ++++++++++++++++++++++-------- docs/notes.md | 92 +++++++++++++++-- docs/services.md | 126 +++++++++++++++++++---- 8 files changed, 723 insertions(+), 250 deletions(-) create mode 100644 docs/container-structure.md diff --git a/README.md b/README.md index 5494649c..9ed3824c 100644 --- a/README.md +++ b/README.md @@ -1,86 +1,187 @@ -## UDX Worker +# UDX Worker -The UDX Worker simplifies DevSecOps by providing a secure, containerized environment for handling secrets and running automation tasks. This repository contains the UDX Worker Docker image, designed for secure and reliable automation tasks based on 12-factor methodology. UDX Worker environments are ephemeral and adhere to zero-trust principles and methodology, ensuring maximum security and reliability. +[![Docker Pulls](https://img.shields.io/docker/pulls/usabilitydynamics/udx-worker.svg)](https://hub.docker.com/r/usabilitydynamics/udx-worker) [![License](https://img.shields.io/github/license/udx/worker.svg)](LICENSE) [![Documentation](https://img.shields.io/badge/docs-udx.dev-blue.svg)](https://udx.dev/worker) -### Deployment +**Secure, containerized environment for DevSecOps automation** -1. Make sure Docker installed. +[Quick Start](#quick-start) β€’ [Documentation](#documentation) β€’ [Development](#development) β€’ [Contributing](#contributing) -2. Pull the Docker image: +## πŸš€ Overview -```shell -docker pull usabilitydynamics/udx-worker:latest -``` +UDX Worker is a containerized solution that simplifies DevSecOps by providing: + +- πŸ”’ **Secure Environment**: Built on zero-trust principles +- πŸ€– **Automation Support**: Streamlined task execution +- πŸ”‘ **Secret Management**: Secure handling of sensitive data +- πŸ“¦ **12-Factor Compliance**: Modern application practices +- ♾️ **CI/CD Ready**: Seamless pipeline integration + +## πŸƒ Quick Start + +### Prerequisites + +- Docker 20.10 or later +- Make (for development) + +### Example 1: Simple Service + +```bash +# Create project structure +mkdir -p my-worker/.config/worker +cd my-worker + +# Create a simple service script +cat > index.sh <<'EOF' +#!/bin/bash +echo "Starting service..." +trap 'echo "Shutting down..."; exit 0' SIGTERM +while true; do + echo "[$(date)] Service running..." + sleep 5 +done +EOF +chmod +x index.sh -3. Run the Docker container: +# Create service configuration +cat > .config/worker/services.yaml <<'EOF' +kind: workerService +version: udx.io/worker-v1/service +services: + - name: "index" + command: "/home/udx/index.sh" + autostart: true + autorestart: true +EOF -```shell +# Run the worker docker run -d \ - --name my-app \ - -v $(pwd):/home/udx \ + --name my-service \ + -v "$(pwd):/home/udx" \ usabilitydynamics/udx-worker:latest + +# View service logs +docker logs -f my-service ``` -_Make sure to mount the current directory to `/home/udx` in the container._ +### Example 2: Secrets Management with Authorization + +```bash +# Define secrets configuration +cat > .config/worker/worker.yaml <<'EOF' +kind: workerConfig +version: udx.io/worker-v1/config +config: + secrets: + API_KEY: "azure/key-vault/api-key" + DB_PASS: "aws/secrets/database" +EOF + +# Create base64-encoded Azure credentials +AZURE_CREDS=$(echo '{ + "client_id": "your-client-id", + "client_secret": "your-client-secret", + "tenant_id": "your-tenant-id" +}' | base64) + +# Run with cloud provider credentials +docker run -d \ + --name my-secrets \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker" \ + -e AZURE_CREDS="${AZURE_CREDS}" \ + usabilitydynamics/udx-worker:latest -### Development +# Verify authorization and secrets +docker exec my-secrets worker auth verify +docker exec my-secrets worker env get API_KEY +``` -1. Clone the Repository +See [Authorization Guide](docs/authorization.md) for supported providers and credential formats (JSON, Base64, File Path). -```shell +### Development Setup + +```bash +# Clone and build git clone https://github.com/udx/worker.git cd worker -``` +make build -2. Build Image +# Run example service +make run VOLUMES="$(pwd)/src/examples/simple-service:/home/udx" -```shell -make build +# View logs +make log FOLLOW_LOGS=true + +# Run tests +make test ``` -3. Start the container +More examples available in [src/examples](src/examples). -```shell -make run -``` +## πŸ“š Documentation -_Interactively_ +### Core Concepts +- [Authorization](docs/authorization.md) - Credential management +- [Configuration](docs/config.md) - Worker setup +- [Services](docs/services.md) - Service management +- [CLI Reference](docs/cli.md) - Command line usage -```shell -make run-it -``` +### Additional Resources +- [Container Structure](docs/container-structure.md) - Directory layout +- [Development Notes](docs/notes.md) - Best practices +- [Git Tips](docs/git-help.md) - Version control helpers + +## πŸ› οΈ Development + +```bash +# Clone repository +git clone https://github.com/udx/worker.git +cd worker + +# Build image +make build -4. Run tests +# Run container +make run # Detached mode +make run-it # Interactive mode -```shell +# Run tests make test + +# View all commands +make help ``` -_For more details on available commands_ +## 🀝 Contributing -```shell -make -``` +We welcome contributions! Here's how you can help: -### Docs +1. Fork the repository +2. Create a feature branch +3. Commit your changes +4. Push to your branch +5. Open a Pull Request -- [Authorization](/docs/authorization.md) -- [CLI](/docs/cli.md) -- [Config](/docs/config.md) -- [Git Help](/docs/git.md) -- [Notes](/docs/notes.md) -- [Services](/docs/services.md) +Please ensure your PR: +- Follows our coding standards +- Includes appropriate tests +- Updates relevant documentation -### Resources +## πŸ”— Resources - [Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker) - [Documentation](https://udx.dev/worker) -- [Marketing Page](https://udx.io/products/udx-worker) +- [Product Page](https://udx.io/products/udx-worker) + +## 🎯 Custom Development -### Contributing +Need specific features or customizations? +[Contact our team](https://udx.io/) for professional development services. -Contributions are welcome! If you find any issues or have suggestions for improvements, please fork the repository and submit a pull request. +## πŸ“„ License -### Custom Development +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. -Looking for a unique feature for your next project? [Hire us!](https://udx.io/) +--- +
+Built with ❀️ by UDX +
diff --git a/docs/CLI.md b/docs/CLI.md index 10f07d0b..b2d16ebb 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -1,91 +1,28 @@ # UDX Worker CLI Documentation -The UDX Worker provides a command-line interface for managing services, environment variables, and generating Software Bill of Materials (SBOM). +The UDX Worker provides a command-line interface for managing services, environment variables, generating Software Bill of Materials (SBOM), and more. -## Commands +## Available Commands -### Service Management +| Command | Description | +|----------|-------------| +| `auth` | Manage authentication operations | +| `config` | Manage configuration operations | +| `env` | Manage environment variables | +| `health` | Monitor system health status | +| `help` | Show help for any command | +| `sbom` | Manage Software Bill of Materials | +| `service`| Manage service operations | +| `version`| Show version information | -* `worker service list` - - Lists of configured services. - - Shows service name, status, PID, and uptime +## Usage -* `worker service status ` - - Displays detailed status of a service - - Shows service state, uptime, and process information - -* `worker service logs [options]` - - Views service output logs - - Options: - - `--lines N`: Show last N lines (default: 20) - - `--nostream`: Show logs without following - -* `worker service errors [options]` - - Views service error logs - - Options: - - `--lines N`: Show last N lines (default: 20) - - `--nostream`: Show logs without following - -* `worker service config` - - Shows the services configuration settings - - Displays supervisor configuration for all services - -* `worker service start ` - - Starts a specified service - - Creates necessary log files and directories - -* `worker service stop ` - - Stops a specified service - - Service can be restarted later - -* `worker service restart ` - - Restarts a specified service - - Equivalent to stop followed by start - -### Environment Variables - -* `worker env set ` - - Sets an environment variable - - Variable will be available to all services - -* `worker env get [key]` - - Retrieves environment variable(s) - - If key is provided, shows specific variable - - If no key, shows all environment variables - -### Software Bill of Materials - -* `worker sbom generate` - - Generates container Software Bill of Materials - - Lists all installed packages with: - - Package name - - Version - - Architecture - -## Examples +To see detailed help and usage information for any command, run it without arguments: ```bash -# View service status -worker service status my_service - -# Get last 100 lines of logs without following -worker service logs my_service --lines=100 --nostream - -# Set and verify environment variable -worker env set MY_VAR "my value" -worker env get MY_VAR - -# Generate SBOM -worker sbom generate -``` - -## Exit Codes - -- 0: Command completed successfully -- 1: Command failed or invalid usage - -## Notes +# Show auth command help +worker auth -- All commands use logging levels: INFO, DEBUG, WARN, ERROR -- Service logs are stored in `/var/log/supervisor/` -- Configuration files are in `/etc/supervisord.conf` \ No newline at end of file +# Show service command help +worker service +``` \ No newline at end of file diff --git a/docs/authorization.md b/docs/authorization.md index 588a70d7..5d569cf1 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -1,46 +1,74 @@ -## Worker Authorization +# Worker Authorization -### Supported Environment Variables -- Azure: `AZURE_CREDS` -- AWS: `AWS_CREDS` -- GCP: `GCP_CREDS` -- Bitwarden: `BITWARDEN_CREDS` +## Overview -### Credential Formats +The UDX Worker supports multiple cloud providers and services through environment-based credential management. -Credentials can be provided in three ways: +## Supported Providers -- **JSON**: Stringified JSON. +| Provider | Environment Variable | Description | +|------------|---------------------|-------------| +| Azure | `AZURE_CREDS` | Azure cloud credentials | +| AWS | `AWS_CREDS` | Amazon Web Services credentials | +| GCP | `GCP_CREDS` | Google Cloud Platform credentials | +| Bitwarden | `BITWARDEN_CREDS` | Bitwarden secrets management credentials | -- **Base64 Encoded JSON**: Base64 encoded JSON strings. +## Credential Formats -- **File Path**: File path to JSON files with the credentials. +Credentials can be provided in three formats: +| Format | Description | Use Case | +|--------|-------------|----------| +| JSON | Plain JSON string | Direct configuration | +| Base64 | Base64-encoded JSON | Secure environment variables | +| File Path | Path to JSON file | Local development | -### Examples +## Format Examples -1. JSON +### 1. JSON Format ```json -{"client_id":"CLIENT_ID","client_secret":"CLIENT_SECRET","tenant_id":"TENANT_ID","subscription_id":"SUBSCRIPTION_ID"} +{ + "client_id": "CLIENT_ID", + "client_secret": "CLIENT_SECRET", + "tenant_id": "TENANT_ID", + "subscription_id": "SUBSCRIPTION_ID" +} ``` -2. Base64 Encoded JSON +### 2. Base64 Encoded Format -```base64 +```bash +# Original JSON +{ + "client_id": "CLIENT_ID", + "client_secret": "CLIENT_SECRET", + "tenant_id": "TENANT_ID", + "subscription_id": "SUBSCRIPTION_ID" +} + +# Base64 encoded value ewogICAgImNsaWVudF9pZCI6ICJDTElFTlRfSUQiLAogICAgImNsaWVudF9zZWNyZXQiOiAiQ0xJRU5UX1NFQ1JFVCIsCiAgICAidGVuYW50X2lkIjogIlRFTkFOVF9JRCIsCiAgICAic3Vic2NyaXB0aW9uX2lkIjogIlNVQlNDUklQVElPTl9JRCIKfQ== ``` -Here is how you can base64 encode a JSON string: - +**Generate Base64 Format:** ```bash -echo -n '{"client_id": "CLIENT_ID", "client_secret": "CLIENT_SECRET", "tenant_id": "TENANT_ID", "subscription_id": "SUBSCRIPTION_ID"}' | base64 +echo -n '{"client_id":"CLIENT_ID","client_secret":"CLIENT_SECRET","tenant_id":"TENANT_ID","subscription_id":"SUBSCRIPTION_ID"}' | base64 ``` -3. File Path +### 3. File Path Format -```txt -./creds/azure_creds.json +```bash +# Environment variable value +AZURE_CREDS="/path/to/azure_credentials.json" + +# Credential file content (azure_credentials.json) +{ + "client_id": "CLIENT_ID", + "client_secret": "CLIENT_SECRET", + "tenant_id": "TENANT_ID", + "subscription_id": "SUBSCRIPTION_ID" +} ``` -_Make sure to replace `./creds/azure_creds.json` with the actual file path to your credentials file_ \ No newline at end of file +> **Note**: Always use absolute paths in production environments to avoid path resolution issues. \ No newline at end of file diff --git a/docs/config.md b/docs/config.md index 063d0e18..2b29e1e1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1,69 +1,104 @@ -## Worker Configuration +# Worker Configuration -The `worker.yaml` configuration file is a crucial component for customizing the environment of the UDX Worker. It allows users to specify both environment variables and secrets that are essential for the worker's operations. +## Overview -### Structure +The UDX Worker uses `worker.yaml` as its primary configuration file, allowing you to: +- Define environment variables +- Reference secrets from various providers +- Configure worker behavior -- **env**: This section define various environment variables that your worker needs to function. +## File Location -```yaml -env: - AZURE_CLIENT_ID: "your-azure-client-id" - AZURE_TENANT_ID: "your-azure-tenant-id" - AZURE_SUBSCRIPTION_ID: "your-azure-subscription-id" - ... +```bash +/home/udx/.config/worker/worker.yaml ``` -- **secrets**: This section allows to reference secrets stored in secure locations. +## Configuration Structure + +| Section | Purpose | Required | +|---------|----------|----------| +| `kind` | Configuration type identifier | Yes | +| `version` | Schema version | Yes | +| `config.env` | Environment variables | No | +| `config.secrets` | Secret references | No | + +## Basic Example ```yaml -secrets: - DB_PASSWORD: "aws/secrets-manager/db_password" - API_KEY: "gcp/my-project/api_key" - ... +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + AZURE_CLIENT_ID: "12345678-1234-1234-1234-1234567890ab" + AWS_REGION: "us-west-2" + secrets: + DB_PASSWORD: "aws/prod/db_password" + API_KEY: "azure/kv-prod/api-key" ``` -Supported Providers +## Secret Provider References -1. Azure Key Vault +### Azure Key Vault ```yaml - AZURE_CLIENT_ID: "azure/{key_vault_name}/{secret_name}" +secrets: + CLIENT_SECRET: "azure/{vault_name}/{secret_name}" + API_KEY: "azure/kv-prod/api-key" ``` -- AWS Secrets Manager +### AWS Secrets Manager ```yaml - AWS_ACCESS_KEY_ID: "aws/{directory}/{secret_name}" +secrets: + DB_PASSWORD: "aws/{path}/{secret_name}" + ACCESS_KEY: "aws/prod/access-key" ``` -- GCP Secret Manager +### Google Cloud Secret Manager ```yaml - GCP_CREDS: "gcp/{project_id}/{secret_name}" +secrets: + SERVICE_KEY: "gcp/{project_id}/{secret_name}" + AUTH_TOKEN: "gcp/my-project/auth-token" ``` -- Bitwarden Vault +### Bitwarden Vault ```yaml - BITWARDEN_TOKEN: "bitwarden/{vault_name}/{secret_name}" +secrets: + MASTER_KEY: "bitwarden/{vault_name}/{secret_name}" + LICENSE_KEY: "bitwarden/prod/license-key" ``` - -### Config Example +## Environment Variables ```yaml ---- -kind: workerConfig -version: udx.io/worker-v1/config config: env: - AZURE_CLIENT_ID: "12345678-1234-1234-1234-1234567890ab" - - secrets: - APP_CLIENT_SECRET: "azure/kv-example/clientSecret" + # Cloud Provider Settings + AZURE_TENANT_ID: "tenant-id" + AWS_REGION: "us-west-2" + GCP_PROJECT: "my-project" + + # Application Settings + LOG_LEVEL: "info" + MAX_WORKERS: "5" + ENABLE_METRICS: "true" ``` -### Usage +## Best Practices + +1. **Secret Management** + - Never store sensitive values directly in `env` + - Use `secrets` section for sensitive data + - Reference secrets from appropriate providers + +2. **Environment Variables** + - Use `env` for non-sensitive configuration + - Keep values consistent across environments + - Document any required variables -To use this configuration file, make sure to mount it with your application under `/home/udx/`. It doesn't matter where you mount it, it could be autodetected in any subdirectory if it's mounted correctly and named `worker.yaml`. \ No newline at end of file +3. **File Handling** + - Keep configuration in version control (without sensitive data) + - Use different files for different environments + - Validate configuration before deployment \ No newline at end of file diff --git a/docs/container-structure.md b/docs/container-structure.md new file mode 100644 index 00000000..34c0cf79 --- /dev/null +++ b/docs/container-structure.md @@ -0,0 +1,116 @@ +# Container Directory Structure + +This document outlines the directory structure of the UDX Worker container and provides guidance for creating child images. + +## Base Directory Structure + +The worker container uses the following directory structure: + +``` +/ +β”œβ”€β”€ opt/worker/ # Base directory for worker-specific files +β”‚ β”œβ”€β”€ apps/ # Worker applications and plugins +β”‚ └── data/ # Worker data storage and processing +β”œβ”€β”€ etc/worker/ # Worker configuration files +β”œβ”€β”€ usr/local/worker/ +β”‚ β”œβ”€β”€ bin/ # Worker executable files +β”‚ β”œβ”€β”€ lib/ # Worker library files +β”‚ └── etc/ # Additional worker configuration +└── usr/local/configs/ # Cloud provider configurations + β”œβ”€β”€ gcloud/ # Google Cloud SDK config + β”œβ”€β”€ aws/ # AWS CLI config + └── azure/ # Azure CLI config +``` + +### Directory Purposes + +#### Worker-Specific Directories + +- `WORKER_BASE_DIR=/opt/worker` + - Main directory for worker-specific files + +- `WORKER_APP_DIR=/opt/worker/apps` + - Contains worker-specific applications and plugins + - Used for extending worker functionality + - **Not intended** for application code in child images + +- `WORKER_DATA_DIR=/opt/worker/data` + - Used for worker data storage and processing + - Temporary and persistent data used by the worker + +- `WORKER_CONFIG_DIR=/etc/worker` + - Contains worker configuration files + +- `WORKER_LIB_DIR=/usr/local/worker/lib` + - Worker library files and shared code + +- `WORKER_BIN_DIR=/usr/local/worker/bin` + - Worker executable files + - Added to system PATH + +- `WORKER_ETC_DIR=/usr/local/worker/etc` + - Additional worker configuration files + +#### Cloud Configuration Directories + +- `CLOUDSDK_CONFIG=/usr/local/configs/gcloud` + - Google Cloud SDK configuration + +- `AWS_CONFIG_FILE=/usr/local/configs/aws` + - AWS CLI configuration + +- `AZURE_CONFIG_DIR=/usr/local/configs/azure` + - Azure CLI configuration + +## Child Image Development + +When developing child images (e.g., PHP, Node.js), follow these guidelines: + +### Directory Structure Guidelines + +1. **Preserve Worker Directories** + - Maintain all worker directories as they are + - Do not modify or remove any worker-specific paths + - These directories are essential for worker functionality + +2. **Application Code Placement** + - Use framework/language-specific conventional directories for your application code + - Do not place application code in worker directories + +### Examples by Language + +#### PHP Applications +``` +/ +β”œβ”€β”€ var/www/ # PHP application code (standard location) +└── opt/worker/ # Worker directories (preserved) +``` + +#### Node.js Applications +``` +/ +β”œβ”€β”€ usr/src/app/ # Node.js application code (standard location) +└── opt/worker/ # Worker directories (preserved) +``` + +#### Python Applications +``` +/ +β”œβ”€β”€ usr/src/app/ # Python application code (standard location) +└── opt/worker/ # Worker directories (preserved) +``` + +### Best Practices + +1. **Separation of Concerns** + - Keep application code separate from worker functionality + - Use standard language/framework conventions for your application + - Don't mix application data with worker data + +2. **Configuration** + - Use appropriate configuration directories for your application + - Don't modify worker configurations unless specifically required + +3. **Data Storage** + - Use appropriate data directories for your application + - Don't use `WORKER_DATA_DIR` for application data storage diff --git a/docs/git-help.md b/docs/git-help.md index d207a592..6f34af09 100644 --- a/docs/git-help.md +++ b/docs/git-help.md @@ -1,73 +1,159 @@ -## Git Commands +# Git Quick Reference -### 1. Clean Ignored Files from Git +## 🧹 Cleanup Operations -Sometimes you may need to remove all files that are listed in your .gitignore from the repository's index but keep them in your working directory. This command helps you do that: +### Clean Ignored Files +Remove files from Git index while keeping them in working directory: -```shell +```bash +# Remove from index git rm -r --cached . + +# Commit the cleanup +git add . +git commit -m "chore: clean ignored files" ``` -- `rm -r --cached .`: This removes all files from the Git index, including ignored files. -- `.`: Refers to the current directory, so it applies the command recursively to all files and directories. +> πŸ’‘ Useful when `.gitignore` is updated but files are still tracked + +## πŸ”„ Commit Management -After running this command, you need to commit the changes to update the repository: +### Amend Last Commit -```shell +```bash +# Change commit message only +git commit --amend -m "new message" + +# Add staged changes to last commit git add . -git commit -m "Cleaned up ignored files" +git commit --amend --no-edit ``` -### 2. Override the Last Commit +⚠️ **Warning**: Don't amend pushed commits unless working alone + +### Force Push Changes -If you want to modify the last commit (e.g., change the commit message or add new changes), you can amend it: +```bash +# Force push with lease (safer than -f) +git push --force-with-lease -```shell -git commit --amend +# Force push (use with extreme caution) +git push -f ``` -- `--amend`: This option allows you to modify the most recent commit. -- You will be prompted to edit the commit message in your default text editor. You can either update the message or keep it as is. +> πŸ›‘οΈ Always prefer `--force-with-lease` over `-f` to prevent overwriting others' work -> Note: Use --amend with caution, especially if the commit has already been pushed to a shared repository, as it rewrites history. +## 🌿 Branch Operations -### 3. Force Push Amended Commit +### Move Commits Between Branches -After amending a commit, if the changes have already been pushed to a remote repository, you'll need to force push the updated commit: +```bash +# 1. Find unpushed commits +git log branch-name --not --remotes --oneline -```shell -git push -f +# 2. Save current work +git stash + +# 3. Switch and apply +git checkout target-branch +git cherry-pick + +# 4. Restore work +git stash pop ``` -- `-f` or `--force`: This option forces Git to push the amended commit to the remote repository, rewriting history. +## πŸš€ Common Workflows -> Note: Force pushing can overwrite changes in the remote repository, so use it carefully, especially when working in a shared environment. +### Feature Branch Workflow -### 4. Moving Unpushed Commits Between Branches +```bash +# 1. Create feature branch +git checkout -b feature/name -If you need to move an unpushed commit from one branch to another, you can use git cherry-pick. Here's an example of moving a commit from branch `UAT-69` to branch `1629`: +# 2. Make changes and commit +git add . +git commit -m "feat: add new feature" -1. First, identify the unpushed commit on the source branch: -```shell -git log UAT-69 --not --remotes --oneline +# 3. Update with main +git fetch origin +git rebase origin/main + +# 4. Push changes +git push -u origin feature/name ``` -2. Note the commit hash from the output (e.g., `4cff367`) +### Commit Message Format + +```bash +# Format +(): -3. Switch to the target branch and stash any current changes: -```shell -git checkout 1629 -git stash # if you have uncommitted changes +# Types +feat: New feature +fix: Bug fix +docs: Documentation +style: Formatting +refactor: Code restructure +test: Tests +chore: Maintenance ``` -4. Cherry-pick the commit: -```shell -git cherry-pick 4cff367 +## πŸ” Inspection Commands + +### View Changes + +```bash +# Show staged changes +git diff --staged + +# Show changes in last commit +git show HEAD + +# Show file history +git log -p filename ``` -5. Restore your stashed changes if any: -```shell -git stash pop +### Branch Information + +```bash +# List all branches +git branch -vv + +# Show merged branches +git branch --merged + +# Show unmerged branches +git branch --no-merged ``` -> Note: The cherry-pick command creates a new commit on the target branch with the same changes but a different commit hash. +## ⚑ Tips and Tricks + +1. **Stash Management**: + ```bash + # Named stash + git stash save "feature work in progress" + + # List stashes + git stash list + + # Apply specific stash + git stash apply stash@{n} + ``` + +2. **Quick Fixes**: + ```bash + # Undo last commit but keep changes + git reset --soft HEAD^ + + # Discard all local changes + git reset --hard HEAD + ``` + +3. **Search History**: + ```bash + # Search commit messages + git log --grep="keyword" + + # Search code changes + git log -S"code string" + ``` diff --git a/docs/notes.md b/docs/notes.md index 323c2b13..f5c061dc 100644 --- a/docs/notes.md +++ b/docs/notes.md @@ -1,11 +1,91 @@ -Creating a Non-Root User: Creating a non-root user inside the container helps to improve security by avoiding running processes as the root user. This can limit the potential damage in case of a security breach. +# Container Development Best Practices -Consistency: Using ARG for these values allows you to parameterize the Dockerfile, making it easier to build images with different users or permissions as needed. You can override these defaults during the build process if necessary. +## User Management -Customization: By defining these as arguments, you provide flexibility. For example, if you want to build the image in different environments where different user IDs are required, you can pass different values for UID and GID when building the Docker image. +### Non-Root User Benefits +- πŸ›‘οΈ **Enhanced Security**: Limits potential damage from security breaches +- πŸ”’ **Reduced Privileges**: Prevents unauthorized system modifications +- πŸ”„ **Best Practice**: Follows container security principles -Security: Running containers as a non-root user enhances security by limiting the potential damage that a compromised application can do. This is a good practice for reducing risks. +### User Configuration -Permissions: The UID and GID are important for managing file permissions. If files created or modified by the container need to be accessed by the host or other containers, consistent UIDs and GIDs help avoid permission issues. +```dockerfile +ARG USER=udx +ARG UID=500 +ARG GID=500 -Container Environment: Within a container, the application user is not a system user but rather a user that the containerized application runs as. This user doesn't perform system administration tasks but operates with limited privileges within the container. \ No newline at end of file +RUN groupadd -g $GID $USER && \ + useradd -u $UID -g $GID -m $USER +``` + +## Dockerfile Arguments + +### Benefits of Using ARGs +- πŸ”§ **Parameterization**: Easy to customize builds +- 🎯 **Flexibility**: Adapt to different environments +- πŸ—οΈ **Reusability**: Same Dockerfile, different configurations + +### Common ARGs +```dockerfile +ARG USER=udx +ARG UID=500 +ARG GID=500 +ARG HOME=/home/udx +``` + +## Permission Management + +### UID/GID Importance +- πŸ“ **File Access**: Consistent access across host and containers +- 🀝 **Shared Resources**: Proper permissions for mounted volumes +- πŸ” **Security**: Controlled access to resources + +### Best Practices +1. Use consistent UID/GID across environments +2. Document required permissions +3. Verify file ownership after operations + +## Container Security + +### Running as Non-Root +- Prevents privileged access +- Limits system modification capabilities +- Follows principle of least privilege + +### Security Checklist +- [ ] Use non-root user +- [ ] Set appropriate file permissions +- [ ] Limit mounted volumes +- [ ] Use read-only filesystems where possible + +## Environment Setup + +### Container User Context +- Application-specific user +- Limited privileges +- No system administration capabilities + +### Directory Permissions +```bash +chown -R $USER:$USER /app +chmod -R 755 /app +``` + +## Tips and Tricks + +1. **Testing User Setup**: + ```bash + docker run --rm -it myimage whoami + docker run --rm -it myimage id + ``` + +2. **Debugging Permissions**: + ```bash + docker run --rm -it myimage ls -la /app + docker run --rm -it myimage stat /app + ``` + +3. **Volume Mounting**: + ```bash + docker run -v $(pwd):/app:ro myimage # Read-only mount + ``` \ No newline at end of file diff --git a/docs/services.md b/docs/services.md index 1cf96f80..42a6265b 100644 --- a/docs/services.md +++ b/docs/services.md @@ -1,35 +1,125 @@ -## Services Configuration +# Service Configuration -The `services.yaml` file contains a list of services to be managed by the worker. Each service is defined using the following structure: +## Overview + +The UDX Worker uses `services.yaml` to define and manage multiple services. Each service can be configured with its own runtime settings, environment variables, and behavior policies. + +## File Location + +```bash +/home/udx/.config/worker/services.yaml +``` + +## Configuration Structure + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `kind` | string | Yes | - | Must be `workerService` | +| `version` | string | Yes | - | Must be `udx.io/worker-v1/service` | +| `services` | array | Yes | - | List of service definitions | + +### Service Definition Fields + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `name` | string | Yes | - | Unique service identifier | +| `command` | string | Yes | - | Command to execute | +| `ignore` | boolean | No | `false` | Skip service management | +| `autostart` | boolean | No | `true` | Start on worker launch | +| `autorestart` | boolean | No | `false` | Restart on failure | +| `envs` | array | No | `[]` | Environment variables | + +## Basic Example ```yaml ---- kind: workerService version: udx.io/worker-v1/service services: - - name: "" - ignore: "" - command: "" - autostart: "" - autorestart: "" + - name: "web-server" + command: "python app.py" + autostart: true + autorestart: true envs: - - "=" + - "PORT=8080" + - "DEBUG=true" ``` -### Service Fields Explanation +## Advanced Examples + +### Multiple Services + +```yaml +kind: workerService +version: udx.io/worker-v1/service +services: + - name: "api-server" + command: "node api/server.js" + autostart: true + autorestart: true + envs: + - "PORT=3000" + - "NODE_ENV=production" + + - name: "worker-queue" + command: "python worker.py" + autostart: true + envs: + - "QUEUE_URL=redis://localhost:6379" + + - name: "monitoring" + command: "./monitor.sh" + ignore: true # Temporarily disabled +``` + +### Service with Complex Command + +```yaml +services: + - name: "data-processor" + command: "bash -c 'source .env && python -m processor.main --config=prod.json'" + autostart: true + autorestart: true + envs: + - "PYTHONPATH=/app" + - "LOG_LEVEL=info" +``` + +## Best Practices + +1. **Service Naming** + - Use descriptive, lowercase names + - Separate words with hyphens + - Keep names concise but meaningful + +2. **Command Definition** + - Use absolute paths when possible + - Quote commands with spaces or special characters + - Consider using shell scripts for complex commands -* `name`: Unique identifier for the service. This is used to reference and manage the service within the system. +3. **Environment Variables** + - Use uppercase for variable names + - Group related variables together + - Document required variables -* `ignore`: Determines whether the service should be ignored. If set to "true", the service will not be managed by the worker. The default value is "false", which means the service is considered for management. +4. **Restart Policies** + - Enable `autorestart` for critical services + - Use `ignore` for maintenance or debugging + - Consider dependencies between services -* `command`: The command that the service will execute. This could be a shell script or any executable along with its arguments. +## Monitoring and Management -* `autostart`: Indicates whether the service should start automatically when the worker starts. The default is "true", meaning the service will start automatically. +Use the following CLI commands to manage services: -* `autorestart`: Specifies whether the service should automatically restart if it stops. If "true", the service will restart according to the policy defined by the worker management system. The default value is "false", indicating the service will not restart automatically. +```bash +# List all services +worker service list -* `envs`: An array of environment variables passed to the service in the format `KEY=value`. These variables are made available to the service at runtime. +# Check specific service status +worker service status web-server -## Usage +# View service logs +worker service logs web-server -To use this configuration file, make sure to mount it with your application under `/home/udx/`. It doesn't matter where you mount it, it could be autodetected in any subdirectory if it's mounted correctly and named `services.yaml`. \ No newline at end of file +# Restart a service +worker service restart web-server +``` \ No newline at end of file From e58c8fbfda2d8ba64d8caacd7f10ca26df4ece76 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 17:06:37 +0300 Subject: [PATCH 23/25] fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9ed3824c..5ee2357c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ **Secure, containerized environment for DevSecOps automation** -[Quick Start](#quick-start) β€’ [Documentation](#documentation) β€’ [Development](#development) β€’ [Contributing](#contributing) +[Quick Start](#-quick-start) β€’ [Documentation](#-documentation) β€’ [Development](#️-development) β€’ [Contributing](#-contributing) ## πŸš€ Overview From 94237f5d25e89b5cb71ea23e4f7a5982cb7f01ba Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 17:18:39 +0300 Subject: [PATCH 24/25] improved cleanup logic --- lib/cleanup.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 405f4f21..4c724111 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -7,6 +7,9 @@ source "${WORKER_LIB_DIR}/worker_config.sh" # shellcheck source=/dev/null source "${WORKER_LIB_DIR}/utils.sh" +# Enable actors cleanup by default +ACTORS_CLEANUP=${ACTORS_CLEANUP:-true} + # Generic function to clean up authentication for any provider cleanup_provider() { local provider=$1 @@ -63,8 +66,41 @@ cleanup_provider() { fi } +# Function to clean up credential files +cleanup_cred_files() { + local provider=$1 + local env_var_name="${provider^^}_CREDS" # Convert to uppercase + local creds_value="${!env_var_name}" + + # Skip if no credentials value found + if [[ -z "$creds_value" ]]; then + return 0 + fi + + # If the value is a file path and exists, remove it + if [[ -f "$creds_value" ]]; then + log_info "Removing credential file for $provider: $creds_value" + rm -f "$creds_value" + if [[ $? -eq 0 ]]; then + log_success "Cleanup" "Removed credential file for $provider" + return 0 + else + log_error "Cleanup" "Failed to remove credential file for $provider" + return 1 + fi + fi + + return 0 +} + # Function to clean up actors based on the providers configured during authentication cleanup_actors() { + # Check if cleanup is enabled + if [[ "${ACTORS_CLEANUP,,}" != "true" ]]; then + log_info "Actors cleanup is disabled via ACTORS_CLEANUP environment variable" + return 0 + fi + # Skip cleanup if no providers were configured during authentication if [[ ${#configured_providers[@]} -eq 0 ]]; then return 0 @@ -77,6 +113,12 @@ cleanup_actors() { # Only clean up providers that were actually configured for provider in "${configured_providers[@]}"; do + # First cleanup any credential files + if cleanup_cred_files "$provider"; then + any_cleanup=true + fi + + # Then cleanup provider sessions case "$provider" in azure) if cleanup_provider "az" "az logout" "az account show" "Azure"; then From 1a129b14646413ce64e6f492d26c98f82f0f2208 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 21 Feb 2025 17:46:12 +0300 Subject: [PATCH 25/25] fix --- lib/cleanup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 4c724111..6994ec09 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -80,8 +80,7 @@ cleanup_cred_files() { # If the value is a file path and exists, remove it if [[ -f "$creds_value" ]]; then log_info "Removing credential file for $provider: $creds_value" - rm -f "$creds_value" - if [[ $? -eq 0 ]]; then + if rm -f "$creds_value"; then log_success "Cleanup" "Removed credential file for $provider" return 0 else