Logging Improvements [UAT-81]

.yamllint

@@ -2,7 +2,7 @@ extends: default
 
 rules:
   line-length:
-    max: 150 # Set a longer line length if 80 is too restrictive
+    max: 170 # Set a longer line length if 80 is too restrictive
     level: warning # Level should be either "error" or "warning"
 
   truthy:

Dockerfile

@@ -23,7 +23,7 @@ USER root
 RUN apt-get update && \
     apt-get install -y \
     tzdata=2024b-6ubuntu1 \
-    curl=8.11.1-1ubuntu1 \
+    curl=8.12.0+git20250209.89ed161+ds-1ubuntu1 \
     bash=5.2.37-1ubuntu1 \
     apt-utils=2.9.28 \
     gettext=0.23.1-1 \
@@ -58,9 +58,9 @@ RUN ARCH=$(uname -m) && \
 ENV CLOUDSDK_CONFIG=/usr/local/configs/gcloud
 RUN ARCH=$(uname -m) && \
     if [ "$ARCH" = "x86_64" ]; then \
-    curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-508.0.0-linux-x86_64.tar.gz" -o google-cloud-sdk.tar.gz; \
+    curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-510.0.0-linux-x86_64.tar.gz" -o google-cloud-sdk.tar.gz; \
     elif [ "$ARCH" = "aarch64" ]; then \
-    curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-508.0.0-linux-arm.tar.gz" -o google-cloud-sdk.tar.gz; \
+    curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-510.0.0-linux-arm.tar.gz" -o google-cloud-sdk.tar.gz; \
     fi && \
     tar -xzf google-cloud-sdk.tar.gz && \
     ./google-cloud-sdk/install.sh -q && \
@@ -86,7 +86,7 @@ RUN mkdir -p $GNUPGHOME && \
     gpg --export EB3E94ADBE1229CF | tee /usr/share/keyrings/microsoft-archive-keyring.gpg && \
     echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/microsoft-archive-keyring.gpg] https://packages.microsoft.com/repos/azure-cli/ jammy main" | tee /etc/apt/sources.list.d/azure-cli.list && \
     apt-get update && \
-    apt-get install -y --no-install-recommends azure-cli=2.68.0-1~jammy && \
+    apt-get install -y --no-install-recommends azure-cli=2.69.0-1~jammy && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
@@ -116,17 +116,21 @@ RUN chmod +x /usr/local/bin/worker_mgmt && \
     ln -s /usr/local/bin/worker_mgmt /usr/local/bin/worker    
 
 # Copy the bin, etc, and lib directories
-COPY etc/home /etc
 COPY etc/configs /usr/local/configs
 COPY lib /usr/local/lib
 COPY bin/entrypoint.sh /usr/local/bin/entrypoint.sh
 
 # Set permissions during build
-RUN chmod +x /usr/local/bin/entrypoint.sh && \
-    chown -R ${UID}:${GID} /usr/local/configs && \
-    chown -R ${UID}:${GID} /usr/local/bin && \
-    chown -R ${UID}:${GID} /usr/local/lib && \
-    chmod -R g-w,o-w /usr/local/configs /usr/local/bin /usr/local/lib
+# Set ownership
+RUN chown -R ${UID}:${GID} /usr/local/configs /usr/local/bin /usr/local/lib && \
+    # Make specific scripts executable
+    chmod 755 /usr/local/bin/entrypoint.sh /usr/local/lib/process_manager.sh && \
+    # Set read-only permissions for config files
+    find /usr/local/configs -type f -exec chmod 644 {} + && \
+    # Set read-only permissions for library files
+    find /usr/local/lib -type f ! -name process_manager.sh -exec chmod 644 {} + && \
+    # Ensure directories are accessible
+    find /usr/local/configs /usr/local/bin /usr/local/lib -type d -exec chmod 755 {} +
 
 # Create a symbolic link for the supervisord configuration file
 RUN ln -sf /usr/local/configs/supervisor/supervisord.conf /etc/supervisord.conf    

Makefile

@@ -7,68 +7,93 @@ include Makefile.help
 
 .PHONY: run run-it clean build exec log test dev-pipeline
 
-# Build the Docker image
-MULTIPLATFORM ?= false
+# Use BuildKit for better output
+export DOCKER_BUILDKIT=1
+
+
+.SILENT: build run run-it clean exec log test dev-pipeline
+
+# Common shell function for error handling and output
+define exec_with_status
+@bash -c 'set -eo pipefail; \
+	printf "$(COLOR_BLUE)$(SYM_ARROW) %s$(COLOR_RESET)\n" "$(1)"; \
+	{ $(2); } && \
+		printf "$(COLOR_GREEN)$(SYM_SUCCESS) %s$(COLOR_RESET)\n" "$(3)" || \
+		{ printf "$(COLOR_RED)$(SYM_ERROR) %s$(COLOR_RESET)\n" "$(4)"; exit 1; }'
+endef
 
 build:
-	@echo "Building Docker image..."
-	@if [ "$(MULTIPLATFORM)" = "true" ]; then \
-		echo "Multiple platforms: [linux/amd64, linux/arm64]..."; \
-		docker buildx build --platform linux/amd64,linux/arm64 -t $(DOCKER_IMAGE) --load .; \
-	else \
-		echo "Only local platform..."; \
-		docker build -t $(DOCKER_IMAGE) .; \
-	fi
-	@echo "Docker image build completed."
+	@bash -c 'set -eo pipefail; \
+		filter="(error|Error|ERROR|failed|Failed|FAILED|\\[.*[0-9]+/[0-9]+\\]|^#[0-9]+ DONE|sha256|CACHED)"; \
+		printf "$(COLOR_BLUE)$(SYM_ARROW) Starting Docker build...$(COLOR_RESET)\n"; \
+		if [ "$(MULTIPLATFORM)" = "true" ]; then \
+			printf "$(COLOR_BLUE)$(SYM_ARROW) Building for multiple platforms: [linux/amd64, linux/arm64]$(COLOR_RESET)\n"; \
+			docker buildx build --progress=plain \
+				--platform linux/amd64,linux/arm64 \
+				-t $(DOCKER_IMAGE) \
+				--load . 2>&1 | grep -E "$$filter" || exit 1; \
+		else \
+			printf "$(COLOR_BLUE)$(SYM_ARROW) Building for local platform$(COLOR_RESET)\n"; \
+			DOCKER_BUILDKIT=1 docker build \
+				--progress=plain \
+				-t $(DOCKER_IMAGE) . 2>&1 | grep -E "$$filter" || exit 1; \
+		fi && \
+		printf "$(COLOR_GREEN)$(SYM_SUCCESS) Docker image build completed$(COLOR_RESET)\n" || \
+		{ printf "$(COLOR_RED)$(SYM_ERROR) Docker build failed$(COLOR_RESET)\n"; exit 1; }'
 
-# Run Docker container (supports interactive mode)
 run: clean
-	@echo "Running Docker container..."
-
+	@printf "$(COLOR_BLUE)$(SYM_ARROW) Starting container...$(COLOR_RESET)\n"
 	@if [ ! -f $(ENV_FILE) ]; then \
-		echo "Creating environment file..."; \
+		printf "$(COLOR_YELLOW)$(SYM_WARNING) Creating environment file...$(COLOR_RESET)\n" && \
 		touch $(ENV_FILE); \
-	else \
-		echo "Environment file exists..."; \
 	fi
-
 	@docker run $(if $(INTERACTIVE),-it,-d) --rm --name $(CONTAINER_NAME) \
 		--env-file $(ENV_FILE) \
 		$(foreach vol,$(VOLUMES),-v $(vol)) \
 		$(DOCKER_IMAGE) $(COMMAND)
-	$(if $(filter false,$(INTERACTIVE)),docker logs -f $(CONTAINER_NAME);)
+	@if [ "$(INTERACTIVE)" = "true" ]; then \
+		:; \
+	else \
+		printf "$(COLOR_BLUE)$(SYM_ARROW) Container started in detached mode. Use 'make log' to view logs$(COLOR_RESET)\n"; \
+	fi
+	@printf "$(COLOR_GREEN)$(SYM_SUCCESS) Container started successfully$(COLOR_RESET)\n"
 
-# Run Docker container in interactive mode
 run-it:
+	@printf "$(COLOR_BLUE)$(SYM_ARROW) Starting interactive container...$(COLOR_RESET)\n"
 	@$(MAKE) run INTERACTIVE=true COMMAND=/bin/bash
+	@printf "$(COLOR_GREEN)$(SYM_SUCCESS) Interactive container started$(COLOR_RESET)\n"
 
-# Exec into the running container
 exec:
-	@echo "Executing into Docker container..."
-	@docker exec -it $(CONTAINER_NAME) /bin/bash
+	@printf "$(COLOR_BLUE)$(SYM_ARROW) Executing into container...$(COLOR_RESET)\n"
+	@docker exec -it $(CONTAINER_NAME) /bin/bash || \
+		{ printf "$(COLOR_RED)$(SYM_ERROR) Failed to execute into container$(COLOR_RESET)\n"; exit 1; }
+	@printf "$(COLOR_GREEN)$(SYM_SUCCESS) Successfully executed into container$(COLOR_RESET)\n"
 
-# View the container logs
 log:
-	@echo "Viewing Docker container logs..."
+	@printf "$(COLOR_BLUE)$(SYM_ARROW) Fetching container logs...$(COLOR_RESET)\n"
 	@if [ "$(FOLLOW_LOGS)" = "true" ]; then \
-		docker logs -f $(CONTAINER_NAME); \
+		docker logs -f $(CONTAINER_NAME) || \
+			{ printf "$(COLOR_RED)$(SYM_ERROR) Failed to retrieve logs$(COLOR_RESET)\n"; exit 1; }; \
 	else \
-		docker logs $(CONTAINER_NAME); \
+		docker logs $(CONTAINER_NAME) || \
+			{ printf "$(COLOR_RED)$(SYM_ERROR) Failed to retrieve logs$(COLOR_RESET)\n"; exit 1; }; \
 	fi
+	@printf "$(COLOR_GREEN)$(SYM_SUCCESS) Logs retrieved successfully$(COLOR_RESET)\n"
 
-# Delete the running container
 clean:
-	@echo "Deleting Docker container if exists..."
+	@printf "$(COLOR_BLUE)$(SYM_ARROW) Cleaning up containers...$(COLOR_RESET)\n"
 	@docker stop $(CONTAINER_NAME) 2>/dev/null || true
 	@docker rm -f $(CONTAINER_NAME) 2>/dev/null || true
+	@printf "$(COLOR_GREEN)$(SYM_SUCCESS) Cleanup completed$(COLOR_RESET)\n"
 
-# Test Docker container
 test: clean
-	@echo "Setting up test environment..."
-	@$(MAKE) run VOLUMES="$(TEST_WORKER_CONFIG):/home/$(USER)/worker.yaml:ro $(TESTS_TASKS_DIR):/home/$(USER)/tasks:ro $(TESTS_MAIN_SCRIPT):/home/$(USER)/main.sh:ro" COMMAND="/home/$(USER)/main.sh"
-	@$(MAKE) log FOLLOW_LOGS=true
-	@$(MAKE) clean
+	@printf "$(COLOR_BLUE)$(SYM_ARROW) Running tests...$(COLOR_RESET)\n"
+	@$(MAKE) run \
+		VOLUMES="$(TEST_WORKER_CONFIG):/home/$(USER)/worker.yaml:ro $(TEST_SERVICES_CONFIG):/home/$(USER)/services.yaml:ro $(TESTS_TASKS_DIR):/home/$(USER)/tasks:ro $(TESTS_MAIN_SCRIPT):/home/$(USER)/main.sh:ro" \
+		COMMAND="/home/$(USER)/main.sh" || exit 1
+	@$(MAKE) log FOLLOW_LOGS=true || exit 1
+	@$(MAKE) clean || exit 1
+	@printf "$(COLOR_GREEN)$(SYM_SUCCESS) Tests completed successfully$(COLOR_RESET)\n"
 
-# Development pipeline
 dev-pipeline: build test
-	@echo "Development pipeline completed successfully."
+	@printf "$(COLOR_GREEN)$(SYM_SUCCESS) Development pipeline completed successfully$(COLOR_RESET)\n"

Makefile.variables

@@ -7,9 +7,23 @@ ENV_FILE ?= .env
 TESTS_MAIN_SCRIPT ?= ./src/tests/main.sh
 TESTS_TASKS_DIR ?= ./src/tests/tasks
 TEST_WORKER_CONFIG ?= ./src/tests/configs/worker.yaml
+TEST_SERVICES_CONFIG ?= ./src/tests/configs/services.yaml
 USER = udx
 VOLUMES ?= ./src/scripts:/home/$(USER)
 DEBUG ?= false
 COMMAND ?= 
 MULTIPLATFORM ?= false
-FOLLOW_LOGS ?= false
+FOLLOW_LOGS ?= false
+
+# Colors for better visibility
+COLOR_RESET=\033[0m
+COLOR_BLUE=\033[34m
+COLOR_GREEN=\033[32m
+COLOR_RED=\033[31m
+COLOR_YELLOW=\033[33m
+
+# Symbols
+SYM_ARROW=➜
+SYM_SUCCESS=✔
+SYM_ERROR=✖
+SYM_WARNING=⚠

bin/entrypoint.sh

@@ -3,62 +3,22 @@
 # shellcheck disable=SC1091
 source /usr/local/lib/utils.sh
 
-udx_logo
-
 log_info "Welcome to UDX Worker Container. Initializing environment..."
 
 # shellcheck disable=SC1091
 source /usr/local/lib/environment.sh
 
-handle_services() {
-
-    if check_active_services; then
-        wait_for_services
-        log_info "Services are fully running."
-    else
-        log_warn "No services are active."
-    fi
-}
-
-check_active_services() {
-    log_info "Checking for active or starting services..."
-    if supervisorctl status | grep -Eq 'RUNNING|STARTING'; then
-        log_info "Active or starting services found."
-        return 0
-    else
-        log_warn "No active or starting services detected."
-        return 1
-    fi
-}
+# Start the process manager in the background
+log_info "Starting process manager..."
+/usr/local/lib/process_manager.sh &
 
-wait_for_services() {
-    local attempts=0 max_attempts=10
-    log_info "Waiting for services to be fully running..."
-    while [ $attempts -lt $max_attempts ]; do
-        if supervisorctl status | grep -q "RUNNING"; then
-            log_info "All services are now running."
-            return 0
-        fi
-        log_info "Waiting for services to be fully running... (Attempt: $attempts)"
-        attempts=$((attempts + 1))
-        sleep 5
-    done
-    log_warn "Services are not fully running after $max_attempts attempts."
-    return 1
-}
-
-# Main execution path
+# Main execution logic
 if [ "$#" -gt 0 ]; then
+    # Log the command being executed
     log_info "Executing command: $*"
-
-    if [[ "$1" =~ \.sh$ ]]; then
-        "$@"  # Execute the provided command
-        log_info "Shell script execution completed. Exiting."
-        exit 0
-    else
-        handle_services
-        "$@"  # Execute the provided command
-    fi
-else
-    handle_services
-fi
+    # Execute the command passed to the container
+    exec "$@"
+fi
+
+# Keep the container running
+wait