From 8f6b209d63a2a0e71a31d2b5f4685499669d0fe8 Mon Sep 17 00:00:00 2001 From: Dmitry Smirnov Date: Fri, 27 Dec 2024 14:29:44 +0200 Subject: [PATCH] migrate to supervisor --- Dockerfile | 45 +++++----------- etc/home/default.service | 10 ---- etc/home/supervisor.default | 26 +++++++++ lib/environment.sh | 10 ++-- lib/process_manager.sh | 97 ++++++++++++++++++++-------------- src/configs/services.yml | 19 +++++-- src/scripts/process_example.sh | 14 +++-- 7 files changed, 127 insertions(+), 94 deletions(-) delete mode 100644 etc/home/default.service create mode 100644 etc/home/supervisor.default diff --git a/Dockerfile b/Dockerfile index 487f8a19..6083e605 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,12 +36,16 @@ RUN apt-get update && \ vim=2:9.1.0016-1ubuntu7.5 \ python3.12=3.12.3-1ubuntu0.3 \ python3-pip=24.0+dfsg-1ubuntu1.1 \ - systemd=255.4-1ubuntu8.4 && \ - ln -fs /usr/share/zoneinfo/$TZ /etc/localtime && \ - dpkg-reconfigure --frontend noninteractive tzdata && \ + supervisor=4.2.5-1ubuntu0.1 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# Configure the timezone +RUN echo $TZ > /etc/timezone && \ + rm /etc/localtime && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata + # Install yq (architecture-aware) RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "x86_64" ]; then ARCH="amd64"; elif [ "$ARCH" = "aarch64" ]; then ARCH="arm64"; fi && \ @@ -94,38 +98,17 @@ RUN ARCH=$(uname -m) && \ chmod +x /usr/local/bin/bw && \ rm -rf /tmp/* /var/tmp/* -# Prepare the system for systemd usage -RUN find /etc/systemd/system \ - /lib/systemd/system \ - -path '*.wants/*' \ - -not -name '*journald*' \ - -delete; \ - systemctl set-default multi-user.target; \ - systemctl mask \ - tmp.mount \ - etc-hostname.mount \ - etc-hosts.mount \ - etc-resolv.conf.mount \ - -- -.mount \ - swap.target \ - getty.target \ - getty-static.service \ - dev-mqueue.mount \ - cgproxy.service \ - systemd-remount-fs.service \ - sys-kernel-config.mount \ - sys-kernel-debug.mount \ - sys-fs-fuse-connections.mount \ - systemd-logind.service \ - systemd-random-seed.service \ - systemd-tmpfiles-setup-dev.service \ - systemd-tmpfiles-setup.service \ - systemd-update-utmp.service - # Create a new user and group with specific UID and GID, and set permissions RUN groupadd -g ${GID} ${USER} && \ useradd -l -m -u ${UID} -g ${GID} -s /bin/bash ${USER} +# Create the Supervisor log directory and set permissions +RUN mkdir -p /var/log/supervisor && \ + chown -R ${USER}:${USER} /var/log/supervisor + +# Create a directory for Supervisor runtime files +RUN mkdir -p /var/run/supervisor && chown -R ${USER}:${USER} /var/run/supervisor + # Prepare directories for the user and worker configuration RUN mkdir -p /etc/worker /home/${USER}/.cd/bin /home/${USER}/.cd/configs && \ touch /home/${USER}/.cd/configs/merged_worker.yml && \ diff --git a/etc/home/default.service b/etc/home/default.service deleted file mode 100644 index 808fdb4d..00000000 --- a/etc/home/default.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=${name} -After=${after} - -[Service] -ExecStart=${exec_start} -Restart=always - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/etc/home/supervisor.default b/etc/home/supervisor.default new file mode 100644 index 00000000..cb4fbe0c --- /dev/null +++ b/etc/home/supervisor.default @@ -0,0 +1,26 @@ +[supervisord] +logfile=/var/log/supervisor/supervisord.log ; Logfile path +logfile_maxbytes=50MB ; Maximum log file bytes before rotating +logfile_backups=10 ; Number of backups to keep +loglevel=info ; Log level (info, debug, warn, trace) +pidfile=/var/run/supervisor/supervisord.pid ; Location of the PID file +nodaemon=false ; Run in the foreground (Docker best practice) +minfds=1024 ; Minimum number of file descriptors +minprocs=200 ; Minimum number of processes + +[supervisorctl] +serverurl=unix:///var/run/supervisor/supervisord.sock ; Path to the UNIX socket for supervisorctl to connect to supervisord + +[unix_http_server] +file=/var/run/supervisor/supervisord.sock ; path to your socket file + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[program:${process_name}] +command=${command} ; Command to start the process +autostart=${autostart} ; Whether to start the process at supervisord start +autorestart=${autorestart} ; Whether to restart the process on exit +stderr_logfile=${stderr_logfile} ; Path for stderr logfile +stdout_logfile=${stdout_logfile} ; Path for stdout logfile +environment=${envs} ; Environment variables \ No newline at end of file diff --git a/lib/environment.sh b/lib/environment.sh index 44b60d4a..313ac293 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -90,12 +90,14 @@ configure_environment() { # Perform process manager setup log_info "Setting up process manager..." - if ! generate_and_activate_services; then - log_error "Failed to activate services." - return 1 + if should_generate_config; then + echo "Generating Supervisor configurations..." + configure_and_execute_services + else + echo "No services found in $CONFIG_FILE. Skipping Supervisor configuration." fi - log_info "Environment setup completed successfully." + log_info "Secure environment setup completed successfully." } # Call the main function diff --git a/lib/process_manager.sh b/lib/process_manager.sh index 25f3e9fe..919a2d96 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -1,66 +1,81 @@ #!/bin/bash -# Function to check if systemd should be enabled -should_enable_systemd() { - if [ -f "$CONFIG_FILE" ]; then +# Define paths +CONFIG_FILE="/etc/worker/services.yml" +TEMPLATE_FILE="/home/${USER}/etc/supervisor.default" +FINAL_CONFIG="/home/${USER}/etc/supervisord.conf" + +# Function to check for service configurations +should_generate_config() { + if [ -f "$CONFIG_FILE" ] && [ "$(yq e '.services | length' "$CONFIG_FILE")" -gt 0 ]; then return 0 else return 1 fi } -# Function to parse service information from YAML configuration +# Helper function to parse and process each service configuration parse_service_info() { - local service_yaml="$1" - name=$(echo "$service_yaml" | jq -r '.name' -) - exec_start=$(echo "$service_yaml" | jq -r '.exec_start' -) - after=$(echo "$service_yaml" | jq -r '.after' -) + local service_json="$1" + local name=$(echo "$service_json" | jq -r '.name') + local command=$(echo "$service_json" | jq -r '.command') + local autostart=$(echo "$service_json" | jq -r '.autostart // "false"') + local autorestart=$(echo "$service_json" | jq -r '.autorestart // "false"') + local stderr_logfile=$(echo "$service_json" | jq -r '.stderr_logfile // ""') + local stdout_logfile=$(echo "$service_json" | jq -r '.stdout_logfile // ""') + local environment=$(echo "$service_json" | jq -r '.environment // [] | join(",")') + + # For each service, replace placeholders and append to FINAL_CONFIG + sed "s|\${process_name}|$name|g; \ + s|\${command}|$command|g; \ + s|\${autostart}|$autostart|g; \ + s|\${autorestart}|$autorestart|g; \ + s|\${stderr_logfile}|$stderr_logfile|g; \ + s|\${stdout_logfile}|$stdout_logfile|g; \ + s|\${envs}|$environment|g" "$TEMPLATE_FILE" >> "$FINAL_CONFIG" } -# Function to create a systemd service file from a template -create_service_file() { - local template_file="$SERVICE_DIR/default.service" - sed -e "s|\${name}|$name|g" \ - -e "s|\${exec_start}|$exec_start|g" \ - -e "s|\${after}|$after|g" \ - "$template_file" > "${SERVICE_DIR}/${name}.service" +# Function to start Supervisor with the generated configuration +start_supervisor() { + echo "Starting Supervisor with the generated configuration..." + supervisord -c "$FINAL_CONFIG" } -# Main function to generate systemd service unit files from template based on services.yml -generate_and_activate_services() { - if ! should_enable_systemd; then - echo "Systemd is not enabled. services.yml not found." +# Function to configure and start the Supervisor +configure_and_execute_services() { + if ! should_generate_config; then + echo "No services found in $CONFIG_FILE. Skipping Supervisor configuration." return 1 fi - echo "services.yml found. Generating and managing systemd service files..." + # Copy the base Supervisor configuration. + cp "$TEMPLATE_FILE" "$FINAL_CONFIG" + # Remove the template [program:x] section from FINAL_CONFIG + sed -i '/\[program:\${process_name}\]/,/^$/d' "$FINAL_CONFIG" + + # Convert services to JSON and process each. local services_yaml - services_yaml=$(yq eval -o=json '.services[]' "$CONFIG_FILE" | jq -c .) - - # Use a temporary file to avoid a subshell + services_yaml=$(yq e -o=json '.services[]' "$CONFIG_FILE" | jq -c .) + + if [ -z "$services_yaml" ]; then + echo "Failed to parse services from $CONFIG_FILE or no services defined." + return 1 + fi + + # Use a temporary file to avoid subshell issues local services_file services_file=$(mktemp) + echo "$services_yaml" > "$services_file" - - # Read all lines into an array, then delete the file mapfile -t services_array < "$services_file" rm -f "$services_file" - + # Process each service in the array - for service_yaml in "${services_array[@]}"; do - parse_service_info "$service_yaml" - - if [[ -n "$name" && -n "$exec_start" && -n "$after" ]]; then - create_service_file || { echo "Failed to create service file for $name"; return 1; } - echo "Service file for $name created." - else - echo "Missing required service fields for a service in services.yml" - return 1 - fi + for service_json in "${services_array[@]}"; do + parse_service_info "$service_json" done -} - -# Variables (these should be defined or passed to the script) -CONFIG_FILE="/etc/worker/services.yml" -SERVICE_DIR="/home/${USER}/etc" \ No newline at end of file + + # After generating the config and processing services, start Supervisor + start_supervisor +} \ No newline at end of file diff --git a/src/configs/services.yml b/src/configs/services.yml index 7a8e7a9a..39b861c7 100644 --- a/src/configs/services.yml +++ b/src/configs/services.yml @@ -2,6 +2,19 @@ kind: workerService version: udx.io/worker-v1/service services: - - name: "myservice1" - exec_start: "/usr/local/scripts/process_example.sh" - after: "network.target" + - name: "webapp" + command: "sh /usr/local/scripts/process_example.sh" + autostart: "true" + autorestart: "false" + stderr_logfile: "/var/log/supervisor/webapp.err.log" + stdout_logfile: "/var/log/supervisor/webapp.out.log" + environment: + - "KEY1=value1" + - "KEY2=value2" + # - name: "another_service" + # command: "python another_service.py" + # autostart: "true" + # autorestart: "true" + # stderr_logfile: "/var/log/another_service.err.log" + # stdout_logfile: "/var/log/another_service.out.log" + # environment: [] # Even if empty, define it as an empty array diff --git a/src/scripts/process_example.sh b/src/scripts/process_example.sh index ffe8534b..175897b9 100644 --- a/src/scripts/process_example.sh +++ b/src/scripts/process_example.sh @@ -2,8 +2,12 @@ # Script to run as a systemd service in a loop -while true; do -# echo "Service is running at $(date)" >> /tmp/service_example.log - echo "Service is running at $(date)" - sleep 5 -done \ No newline at end of file +end_time=$(($(date +%s) + 30)) # Calculate end_time as current epoch time plus 30 seconds. + +while [ $(date +%s) -lt $end_time ]; do + echo "Service is running at $(date)" + sleep 5 +done + +echo "30 seconds have elapsed. Exiting now." +exit 0 # Clean exit with status 0, indicating success. \ No newline at end of file