[MRG] Optimize Dockerfiles

development-helps/Makefile

@@ -8,7 +8,7 @@ REGISTRY_HOST = localhost:5001
 API_POD = $$(kubectl get pod -l app=estela-django-api -o jsonpath="{.items[0].metadata.name}")
 API_DOC = docs/api.yaml
 PLATFORM ?= linux/$(shell uname -m)
-APPLE_SILICON ?= $$(uname -m | grep -q 'arm64' && echo "true" || echo "false")
+
 
 .PHONY: start
 start:
@@ -23,37 +23,37 @@ stop:
 .PHONY: update-api-image
 update-api-image:
 	-cd $(API_DIR) && \
-	 docker build ..  --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-django-api --tag  $(REGISTRY_HOST)/estela-django-api:latest  --platform $(PLATFORM)
+	 docker build .. --file docker-conf/Dockerfile-django-api --tag  $(REGISTRY_HOST)/estela-django-api:latest  --platform $(PLATFORM)
 	-docker push $(REGISTRY_HOST)/estela-django-api:latest
 
 
 .PHONY: update-celery-image
 update-celery-image:
 	-cd $(API_DIR) && \
-	 docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-celery-beat --tag $(REGISTRY_HOST)/estela-celery-beat:latest  --platform $(PLATFORM) && \
-	 docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-celery-worker --tag $(REGISTRY_HOST)/estela-celery-worker:latest  --platform $(PLATFORM)
+	 docker build .. --file docker-conf/Dockerfile-celery-beat --tag $(REGISTRY_HOST)/estela-celery-beat:latest  --platform $(PLATFORM) && \
+	 docker build .. --file docker-conf/Dockerfile-celery-worker --tag $(REGISTRY_HOST)/estela-celery-worker:latest  --platform $(PLATFORM)
 	-docker push $(REGISTRY_HOST)/estela-celery-beat:latest
 	-docker push $(REGISTRY_HOST)/estela-celery-worker:latest
 
 
 .PHONY: update-redis-image
 update-redis-image:
 	-cd $(API_DIR) && \
-	 docker build .  --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-redis --tag $(REGISTRY_HOST)/estela-redis:latest  --platform $(PLATFORM)
+	 docker build . --file docker-conf/Dockerfile-redis --tag $(REGISTRY_HOST)/estela-redis:latest  --platform $(PLATFORM)
 	-docker push $(REGISTRY_HOST)/estela-redis:latest
 
 
 .PHONY: update-build-project-image
 update-build-project-image:
 	-cd $(API_DIR) && \
-	 docker build ..  --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-build-project --tag $(REGISTRY_HOST)/estela-build-project:latest  --platform $(PLATFORM)
+	 docker build .. --file docker-conf/Dockerfile-build-project --tag $(REGISTRY_HOST)/estela-build-project:latest  --platform $(PLATFORM)
 	-docker push $(REGISTRY_HOST)/estela-build-project:latest
 
 
 .PHONY: update-consumer-image
 update-consumer-image:
 	-cd $(QUEUING_DIR) && \
-	 docker build ..  --build-arg APPLE_SILICON=${APPLE_SILICON} --file Dockerfile --tag $(REGISTRY_HOST)/estela-consumer:latest  --platform $(PLATFORM)
+	 docker build .. --file Dockerfile --tag $(REGISTRY_HOST)/estela-consumer:latest  --platform $(PLATFORM)
 	-docker push $(REGISTRY_HOST)/estela-consumer:latest
 
 

estela-api/docker-conf/Dockerfile-build-project

@@ -1,40 +1,32 @@
-FROM python:3.9
+FROM python:3.9-slim
 
 WORKDIR /home/estela
 
-ARG APPLE_SILICON=false
-COPY estela-api/requirements ./requirements
-
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
 
-RUN apt-get update
-
-RUN apt-get install \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
+    build-essential \
     curl \
+    git \
     gnupg \
-    lsb-release -y
-
-RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+    lsb-release \
+    unixodbc-dev \
+    default-libmysqlclient-dev \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \
+    && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list \
+    && apt-get update \
+    && apt-get install -y docker-ce docker-ce-cli containerd.io \
+    && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
-RUN echo \
-  "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
-  focal stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
-
-RUN apt-get update
-
-RUN apt-get install docker-ce docker-ce-cli containerd.io -y
-
-# If an Apple Silicon chip is used, install pyodbc and unixodbc-dev
-RUN if [ "$APPLE_SILICON" = "true" ]; then \
-        apt-get update && apt-get install -y unixodbc-dev && \
-        pip install --no-binary :all: pyodbc && \
-        sed -i '/pyodbc/d' requirements/deploy.txt; \
-    fi
+COPY estela-api/requirements ./requirements
 
-RUN pip install -r requirements/deploy.txt
-RUN if test -f "requirements/externalApps.txt"; then pip install -r requirements/externalApps.txt; fi
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements/deploy.txt \
+    && { [ -f requirements/externalApps.txt ] && pip install --no-cache-dir -r requirements/externalApps.txt || true; }
 
 COPY estela-api/ estela-api/
 COPY database_adapters/ estela-api/database_adapters/

estela-api/docker-conf/Dockerfile-celery-beat

@@ -1,23 +1,44 @@
-FROM python:3.9
+# Build Stage
+FROM python:3.9-slim as builder
 
-ARG APPLE_SILICON=false
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
 
-WORKDIR /home/estela
+WORKDIR /build
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    build-essential \
+    unixodbc-dev \
+    default-libmysqlclient-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY estela-api/requirements ./requirements
 
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
+RUN pip install --no-cache-dir --target=/build/deps -r requirements/test.txt \
+    && if [ -f requirements/externalApps.txt ]; then pip install --no-cache-dir --target=/build/deps -r requirements/externalApps.txt; fi
+
+
+# Final Stage
+FROM python:3.9-slim
 
-# If an Apple Silicon chip is used, install pyodbc and unixodbc-dev
-RUN if [ "$APPLE_SILICON" = "true" ]; then \
-        apt-get update && apt-get install -y unixodbc-dev && \
-        pip install --no-binary :all: pyodbc && \
-        sed -i '/pyodbc/d' requirements/test.txt; \
-    fi
-
-RUN pip install -r requirements/test.txt
-RUN if test -f "requirements/externalApps.txt"; then pip install -r requirements/externalApps.txt; fi
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /home/estela
+
+# Copy installed dependencies from the builder stage
+COPY --from=builder /build/deps /usr/local/lib/python3.9/site-packages
+COPY --from=builder /build/deps/bin /usr/local/bin
 
 COPY estela-api/ .
 COPY database_adapters/ ./database_adapters
+
+# Runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    unixodbc-dev \
+    default-libmysqlclient-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*

estela-api/docker-conf/Dockerfile-celery-worker

@@ -1,23 +1,44 @@
-FROM python:3.9
+# Build Stage
+FROM python:3.9-slim as builder
 
-ARG APPLE_SILICON=false
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
 
-WORKDIR /home/estela
+WORKDIR /build
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    build-essential \
+    unixodbc-dev \
+    default-libmysqlclient-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY estela-api/requirements ./requirements
 
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
+RUN pip install --no-cache-dir --target=/build/deps -r requirements/test.txt \
+    && if [ -f requirements/externalApps.txt ]; then pip install --no-cache-dir --target=/build/deps -r requirements/externalApps.txt; fi
 
-# If an Apple Silicon chip is used, install pyodbc and unixodbc-dev
-RUN if [ "$APPLE_SILICON" = "true" ]; then \
-        apt-get update && apt-get install -y unixodbc-dev && \
-        pip install --no-binary :all: pyodbc && \
-        sed -i '/pyodbc/d' requirements/test.txt; \
-    fi
 
-RUN pip install -r requirements/test.txt
-RUN if test -f "requirements/externalApps.txt"; then pip install -r requirements/externalApps.txt; fi
+# Final Stage
+FROM python:3.9-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /home/estela
+
+# Copy installed dependencies from the builder stage
+COPY --from=builder /build/deps /usr/local/lib/python3.9/site-packages
+COPY --from=builder /build/deps/bin /usr/local/bin
 
 COPY estela-api/ .
 COPY database_adapters/ ./database_adapters
+
+# Runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    unixodbc-dev \
+    default-libmysqlclient-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*

estela-api/docker-conf/Dockerfile-django-api

@@ -1,24 +1,45 @@
-FROM python:3.9
+# Build Stage
+FROM python:3.9-slim as builder
 
-WORKDIR /home/estela
-ARG APPLE_SILICON=false
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /build
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    build-essential \
+    unixodbc-dev \
+    default-libmysqlclient-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY estela-api/requirements ./requirements
 
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
+RUN pip install --no-cache-dir --target=/build/deps -r requirements/test.txt \
+    && if [ -f requirements/externalApps.txt ]; then pip install --no-cache-dir --target=/build/deps -r requirements/externalApps.txt; fi
+
 
-# If an Apple Silicon chip is used, install pyodbc and unixodbc-dev
-RUN if [ "$APPLE_SILICON" = "true" ]; then \
-        apt-get update && apt-get install -y unixodbc-dev && \
-        pip install --no-binary :all: pyodbc && \
-        sed -i '/pyodbc/d' requirements/test.txt; \
-    fi
+# Final Stage
+FROM python:3.9-slim
 
-RUN pip install -r requirements/test.txt
-RUN if test -f "requirements/externalApps.txt"; then pip install -r requirements/externalApps.txt; fi
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /home/estela
+
+# Copy installed dependencies from the builder stage
+COPY --from=builder /build/deps /usr/local/lib/python3.9/site-packages
 
 COPY estela-api/ .
 COPY database_adapters/ ./database_adapters
 
+# Runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    unixodbc-dev \
+    default-libmysqlclient-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
 EXPOSE 8000

installation/Makefile

@@ -7,7 +7,6 @@ LOCAL_API_IP = $$(kubectl get services -n $${NAMESPACE} estela-django-api-servic
 RESOURCES = db registry minio zookeeper kafka
 SERVICES ?= django-api celery-worker celery-beat redis build-project
 PLATFORM ?= linux/$(shell uname -m)
-APPLE_SILICON ?= $$(uname -m | grep -q 'arm64' && echo "true" || echo "false")
 
 
 .PHONY: resources
@@ -34,9 +33,9 @@ delete-resources:
 .PHONY: build-all-images
 build-all-images:
 	-. ./local/.env && for service in $(SERVICES); do \
-		cd $(API_DIR) && docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-$$service --tag $${LOCAL_REGISTRY}/estela-$$service:latest; \
+		cd $(API_DIR) && docker build .. --file docker-conf/Dockerfile-$$service --tag $${LOCAL_REGISTRY}/estela-$$service:latest; \
 	done
-	-. ./local/.env && cd $(QUEUING_DIR) && docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file Dockerfile --tag $${LOCAL_REGISTRY}/estela-consumer:latest
+	-. ./local/.env && cd $(QUEUING_DIR) && docker build .. --file Dockerfile --tag $${LOCAL_REGISTRY}/estela-consumer:latest
 
 .PHONY: upload-all-images
 upload-all-images:

queueing/Dockerfile

@@ -1,21 +1,45 @@
-FROM python:3.9
+# Build Stage
+FROM python:3.9-slim as builder
 
-ARG APPLE_SILICON=false
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
 
-WORKDIR /home/estela
+WORKDIR /build
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    build-essential \
+    unixodbc-dev \
+    default-libmysqlclient-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY queueing/requirements requirements
-RUN if [ "$APPLE_SILICON" = "true" ]; then \
-        apt-get update && apt-get install -y unixodbc-dev && \
-        pip install --no-binary :all: pyodbc && \
-        sed -i '/pyodbc/d' requirements/consumer.txt; \
-    fi
-RUN pip install -r requirements/consumer.txt
+
+RUN pip install --no-cache-dir --target=/build/deps -r requirements/consumer.txt
+
+
+# Final Stage
+FROM python:3.9-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /home/estela
+
+# Copy installed dependencies from the builder stage
+COPY --from=builder /build/deps /usr/local/lib/python3.9/site-packages
 
 COPY queueing/consumer.py .
 COPY queueing/inserter.py .
 COPY queueing/utils.py .
 COPY queueing/config config
 COPY database_adapters/ ./database_adapters
+
+# Runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    unixodbc-dev \
+    default-libmysqlclient-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*