Merge pull request #70 from udx/UAT-60

Services Management Improvements, CLI [UAT-60, UAT-61]

CLI.md

@@ -0,0 +1,25 @@
+### Commands
+
+* `worker service list` 
+Lists all available services.
+
+* `worker service status <service_name>`
+Displays the status of a specified service.
+
+* `worker service logs <service_name>`
+Tails the output logs for a specific service.
+
+* `worker service errors <service_name>`
+Tails the error logs for a specific service.
+
+* `worker service config`
+Shows the services configuration settings.
+
+* `worker service start <service_name>`
+Starts a specified service.
+
+* `worker service stop <service_name>`
+Stops a specified service.
+
+* `worker service restart <service_name>`
+Restarts a specified service.

Dockerfile

@@ -103,11 +103,8 @@ RUN groupadd -g ${GID} ${USER} && \
     useradd -l -m -u ${UID} -g ${GID} -s /bin/bash ${USER}
 
 # Create the Supervisor log directory and set permissions
-RUN mkdir -p /var/log/supervisor && \
-    chown -R ${USER}:${USER} /var/log/supervisor    
-
-# Create a directory for Supervisor runtime files
-RUN mkdir -p /var/run/supervisor && chown -R ${USER}:${USER} /var/run/supervisor
+RUN mkdir -p /var/log/supervisor /var/run/supervisor /home/${USER}/etc && \
+    chown -R ${USER}:${USER} /var/log/supervisor /var/run/supervisor /home/${USER}/etc
 
 # Prepare directories for the user and worker configuration
 RUN mkdir -p /etc/worker /home/${USER}/.cd/bin /home/${USER}/.cd/configs && \
@@ -120,6 +117,11 @@ RUN mkdir -p /etc/worker /home/${USER}/.cd/bin /home/${USER}/.cd/configs && \
 # Switch to the user directory
 WORKDIR /home/${USER}
 
+# Copy the CLI tool into the image
+COPY lib/cli.sh /usr/local/bin/udx_worker_mgmt
+RUN chmod +x /usr/local/bin/udx_worker_mgmt && \
+    ln -s /usr/local/bin/udx_worker_mgmt /usr/local/bin/worker
+
 # Copy built-in worker.yml to the container
 COPY ./src/configs /etc/worker
 COPY ./src/scripts /usr/local/scripts
@@ -137,6 +139,9 @@ COPY ./tests/tasks /usr/local/tests/tasks
 RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/tests/main.sh && \
     chown -R ${UID}:${GID} /usr/local/lib /etc/worker /home/${USER}/etc /home/${USER}/.cd /usr/local/tests
 
+# Create a symbolic link for the supervisord configuration file
+RUN ln -sf /home/${USER}/etc/supervisord.conf /etc/supervisord.conf    
+
 # Switch to non-root user
 USER ${USER}
 

bin/entrypoint.sh

@@ -18,9 +18,8 @@ handle_services() {
         log_info "Tailing Supervisor logs to keep the container alive."
         tail -f /var/log/supervisor/supervisord.log
     else
-        log_warn "No services are active. Exiting after a short delay."
-        sleep 10
-        exit "${cmd_exit_status:-0}"  # Use 0 if cmd_exit_status is unset
+        log_warn "No services are active."
+        tail -f /dev/null
     fi
 }
 
@@ -51,9 +50,13 @@ wait_for_services() {
     return 1
 }
 
+# Main execution path
 # Main execution path
 if [ "$#" -gt 0 ]; then
-    log_info "Executing command:" "$*"
+    log_info "Executing command: $*"
+    if [ "$1" == "exit" ]; then
+        exit 0
+    fi
     "$@"  # Execute the provided command
     cmd_exit_status=$?
     handle_services $cmd_exit_status

lib/cli.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+
+# Function to list all services
+list_services() {
+    # Capture the output of supervisorctl status
+    local services_status
+    services_status=$(supervisorctl status 2>&1) # Also capture stderr to handle error messages
+
+    # Check if Supervisor is not running, not accessible, or if there are no managed services
+    if [[ -z "$services_status" ]] || echo "$services_status" | grep -Eq 'no such|ERROR'; then
+        echo "No services are currently managed."
+        exit 1
+    fi
+
+    echo "Listing all managed services:"
+    local i=1
+    echo "$services_status" | while read -r line; do
+        echo "$i. $line"
+        ((i++))
+    done
+}
+
+# Function to check the status of one or all services
+check_status() {
+    # Require a service name for this function
+    if [ -z "$1" ]; then
+        echo "Error: No service name provided."
+        echo "Usage: $0 status <service_name>"
+        exit 1
+    fi
+
+    # Attempt to capture the status of the specific service, including errors
+    local service_status
+    service_status=$(supervisorctl status "$1" 2>&1)
+
+    # Check if Supervisor is not running, not accessible, or if the service does not exist
+    if [[ -z "$service_status" ]] || echo "$service_status" | grep -Eq 'no such|ERROR'; then
+        echo "The service '$1' does not exist."
+        exit 1
+    fi
+
+    # Directly output the captured service status
+    echo "$service_status"
+}
+
+# Function to follow logs for a specific service
+follow_logs() {
+    if [ -z "$1" ]; then
+        echo "Error: No service name provided."
+        echo "Usage: $0 logs <service_name>"
+        exit 1
+    fi
+
+    local logfile="/var/log/supervisor/$1"
+    local type=${2:-out}  # Default to 'out' if not specified
+    logfile="$logfile.$type.log"
+
+    if [ ! -f "$logfile" ]; then
+        echo "Log file does not exist: $logfile"
+        exit 1
+    fi
+
+    tail -f "$logfile"
+}
+
+# Function to show supervisor configuration
+show_config() {
+    if [ ! -f "/etc/supervisord.conf" ]; then
+        echo "Configuration file is not generated since no services are managed."
+        exit 1
+    fi
+    cat /etc/supervisord.conf
+}
+
+# Function to start, stop, or restart a service
+manage_service() {
+    if [ -z "$2" ]; then
+        echo "Error: No service name provided."
+        echo "Usage: $0 $1 <service_name>"
+        exit 1
+    fi
+
+    if [ ! -e "/var/run/supervisor/supervisord.sock" ]; then
+        echo "Error: Service doesn't exist."
+        exit 1
+    fi
+
+    supervisorctl "$1" "$2"
+}
+
+# CLI Interface
+case $1 in
+    service)
+        shift # Shift the arguments to the left, so $2 becomes $1, $3 becomes $2, etc.
+        case $1 in
+            list)
+                list_services
+            ;;
+            status)
+                check_status "$2"
+            ;;
+            logs)
+                follow_logs "$2"
+            ;;
+            errors)
+                follow_logs "$2" "err"
+            ;;            
+            config)
+                show_config
+            ;;
+            start)
+                manage_service start "$2"
+            ;;
+            stop)
+                manage_service stop "$2"
+            ;;
+            restart)
+                manage_service restart "$2"
+            ;;
+            *)
+                echo "Usage: $0 service {list|status [service_name]|logs <service_name>|config|start <service_name>|stop <service_name>|restart <service_name>}"
+                exit 1
+            ;;
+        esac
+    ;;
+    *)
+        echo "Usage: $0 {service {list|status [service_name]|logs <service_name>|config|start <service_name>|stop <service_name>|restart <service_name>}}"
+        exit 1
+    ;;
+esac

lib/process_manager.sh

@@ -9,8 +9,8 @@ FINAL_CONFIG="/home/${USER}/etc/supervisord.conf"
 # Function to check for service configurations
 should_generate_config() {
     local enabled_services_count
-    # Extract enabled services into JSON format
-    services_yaml=$(yq e -o=json '.services[] | select(.enabled == true)' "$CONFIG_FILE")
+    # Extract services into JSON format
+    services_yaml=$(yq e -o=json '.services[] | select(.ignore != true)' "$CONFIG_FILE")
     # Count the number of items in the JSON array, trimming any newlines or spaces
     enabled_services_count=$(echo "$services_yaml" | jq -c '. | length' | tr -d '\n')
 
@@ -25,28 +25,50 @@ should_generate_config() {
 # Helper function to parse and process each service configuration
 parse_service_info() {
     local service_json="$1"
-    local name command autostart autorestart environment
+    local name command autostart autorestart envs
 
     name=$(echo "$service_json" | jq -r '.name')
     command=$(echo "$service_json" | jq -r '.command')
-    autostart=$(echo "$service_json" | jq -r '.autostart // "false"')
+    # Ensure 'ignore' is considered. If not present, default to "false"
+    ignore=$(echo "$service_json" | jq -r '.ignore // "false"')
+    # Use 'true' as default for 'autostart' if not specified
+    autostart=$(echo "$service_json" | jq -r '.autostart // "true"')
+    # Use 'false' as default for 'autorestart' if not specified
     autorestart=$(echo "$service_json" | jq -r '.autorestart // "false"')
-    environment=$(echo "$service_json" | jq -r '.environment // [] | join(",")')
+    # Ensure 'envs' defaults to an empty array if not specified
+    envs=$(echo "$service_json" | jq -r '.envs // [] | join(",")')
 
+    # Ignore the service if 'ignore' is set to "true"
+    if [[ "$ignore" == "true" ]]; then
+        return
+    fi
+
+    # Check if 'name' is set, log error and return if not
+    if [ -z "$name" ]; then
+        echo "Error: 'name' not set for a service. Skipping..."
+        return
+    fi
+
+    # Check if 'command' is set, log error and return if not
+    if [ -z "$command" ]; then
+        echo "Error: 'command' not set for service $name. Skipping..."
+        return
+    fi
+
     # Add an additional newline for better separation and readability
-    echo -e "\n" >> "$FINAL_CONFIG"  # Adds two newlines to the end of the file
+    echo -e "\n" >> "$FINAL_CONFIG"
 
     sed "s|\${process_name}|$name|g; \
         s|\${command}|$command|g; \
         s|\${autostart}|$autostart|g; \
         s|\${autorestart}|$autorestart|g; \
-        s|\${envs}|$environment|g" "$PROGRAM_TEMPLATE_FILE" >> "$FINAL_CONFIG"
+        s|\${envs}|$envs|g" "$PROGRAM_TEMPLATE_FILE" >> "$FINAL_CONFIG"
 }
 
 # Function to start Supervisor with the generated configuration
 start_supervisor() {
     echo "Starting Supervisor with the generated configuration..."
-    supervisord -c "$FINAL_CONFIG"
+    supervisord
 }
 
 # Function to configure and start the Supervisor
@@ -61,7 +83,7 @@ configure_and_execute_services() {
 
     # Convert enabled services to JSON and process each.
     local services_yaml
-    services_yaml=$(yq e -o=json '.services[] | select(.enabled == true)' "$CONFIG_FILE" | jq -c .)
+    services_yaml=$(yq e -o=json '.services[] | select(.ignore != true)' "$CONFIG_FILE" | jq -c .)
 
     if [ -z "$services_yaml" ]; then
         echo "Failed to parse services from $CONFIG_FILE or no services defined."

src/configs/_worker_services.md

@@ -0,0 +1,47 @@
+## Services 
+
+The `services.yml` file, located in the `src/configs` directory, is a critical component for configuring service behavior in the UDX Worker environment. This file follows the `workerService` kind specification, designed to manage and orchestrate services dynamically, based on the `udx.io/worker-v1/service` version schema. Below, we break down the structure and explain the significance of each field, including default values as defined in the `lib/process_manager.sh` script.
+
+```yaml
+---
+kind: workerService
+version: udx.io/worker-v1/service
+services:
+  - name: "<service_name>"
+    ignore: "<true/false>"
+    command: "<command_to_run>"
+    autostart: "<true/false>"
+    autorestart: "<true/false>"
+    envs:
+      - "<ENV_VAR_NAME>=<value>"
+```
+
+### Service Fields Explanation
+
+* `name`: Unique identifier for the service. This is used to reference and manage the service within the system.
+
+* `ignore`: Determines whether the service should be ignored. If set to "true", the service will not be managed by the worker. The default value is "false", which means the service is considered for management.
+
+* `command`: The command that the service will execute. This could be a shell script or any executable along with its arguments.
+
+* `autostart`: Indicates whether the service should start automatically when the worker starts. The default is "true", meaning the service will start automatically.
+
+* `autorestart`: Specifies whether the service should automatically restart if it stops. If "true", the service will restart according to the policy defined by the worker management system. The default value is "false", indicating the service will not restart automatically.
+
+* `envs`: An array of environment variables passed to the service in the format `KEY=value`. These variables are made available to the service at runtime.
+
+## Usage
+
+To use these configuration file, ensure that the `services.yml` file is correctly configured and placed in the appropriate directory (`/etc/worker`) within the container.
+
+### Volume Mount
+
+If you have a worker configuration outside of the worker image, you can mount it as a volume into the container:
+
+```shell
+docker run -d --name udx-worker \
+  --env-file .env \
+  -v $(pwd)/my-tasks:/usr/src/app \
+  -v $(pwd)/src/configs/services.yml:/etc/worker \
+  usabilitydynamics/udx-worker:latest
+```

src/configs/services.yml

@@ -3,16 +3,16 @@ kind: workerService
 version: udx.io/worker-v1/service
 services:
   - name: "app_service"
-    enabled: "false"
+    ignore: "true"
     command: "sh /usr/local/scripts/process_example.sh"
     autostart: "true"
     autorestart: "false"
-    environment:
+    envs:
       - "KEY1=value1"
       - "KEY2=value2"
   - name: "another_service"
-    enabled: "false"
+    ignore: "true"
     command: "sh /usr/local/scripts/process_example.sh"
     autostart: "true"
     autorestart: "true"
-    environment: []
+    envs: []