feat(ingest): use uv for python package installs (datahub-project#9885

)
fjmacagno · Feb 26, 2024 · 02f41b7 · 02f41b7
1 parent a1f2216
commit 02f41b7
Show file tree

Hide file tree

Showing 16 changed files with 245 additions and 237 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,7 +1,11 @@
 **/node_modules/
 */build/
 */*/build/
-*/venv/
+**/venv/
+**/.tox/
+**/.mypy_cache/
+**/.pytest_cache/
+**/__pycache__/
 out
 **/*.class
 # Have to copy gradle/wrapper/gradle-wrapper.jar, can't exclude ALL jars

diff --git a/.github/scripts/docker_helpers.sh b/.github/scripts/docker_helpers.sh
@@ -24,7 +24,7 @@ function get_tag_full {
 }
 
 function get_python_docker_release_v {
-    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,0.0.0+docker.pr\1,g')
+    echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},1!0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),1!\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,1!0.0.0+docker.pr\1,g')
 }
 
 function get_unique_tag {
@@ -37,4 +37,4 @@ function get_unique_tag_slim {
 
 function get_unique_tag_full {
     echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
-}
+}
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -38,11 +38,17 @@ jobs:
         with:
           python-version: "3.10"
           cache: "pip"
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - name: Set up JDK 17
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
           java-version: 17
+      - uses: gradle/gradle-build-action@v2
       - name: Ensure packages are correct
         run: |
           python ./.github/scripts/check_python_package.py
@@ -978,14 +984,14 @@ jobs:
         if: failure()
         run: |
           docker ps -a
-          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
-          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
-          docker logs datahub-mae-consumer >& mae-${{ matrix.test_strategy }}.log || true
-          docker logs datahub-mce-consumer >& mce-${{ matrix.test_strategy }}.log || true
-          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
-          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
-          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
-          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-gms-1 >& gms-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-actions-1 >& actions-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-mae-consumer-1 >& mae-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-mce-consumer-1 >& mce-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-broker-1 >& broker-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-mysql-1 >& mysql-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-elasticsearch-1 >& elasticsearch-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-datahub-frontend-react-1 >& frontend-${{ matrix.test_strategy }}.log || true
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -51,6 +51,15 @@ jobs:
           java-version: 17
       - uses: gradle/gradle-build-action@v2
       - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}

diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile
@@ -50,7 +50,7 @@ RUN apt-get update && apt-get install -y -qq \
     ldap-utils \
     unixodbc \
     libodbc2 \
-    && python -m pip install --no-cache --upgrade pip wheel setuptools \
+    && python -m pip install --no-cache --upgrade pip uv>=0.1.10 wheel setuptools \
     && rm -rf /var/lib/apt/lists/* /var/cache/apk/*
 
 # compiled against newer golang for security fixes
@@ -59,16 +59,22 @@ COPY --from=dockerize-binary /go/bin/dockerize /usr/local/bin
 COPY ./docker/datahub-ingestion-base/base-requirements.txt requirements.txt
 COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh
 
-RUN pip install --no-cache -r requirements.txt && \
-    pip uninstall -y acryl-datahub && \
-    chmod +x /entrypoint.sh && \
-    addgroup --gid 1000 datahub && \
-    adduser --disabled-password --uid 1000 --gid 1000 --home /datahub-ingestion datahub
+RUN addgroup --gid 1000 datahub && \
+    adduser --disabled-password --uid 1000 --gid 1000 --home /datahub-ingestion datahub && \
+    chmod +x /entrypoint.sh
+
+USER datahub
+ENV VIRTUAL_ENV=/datahub-ingestion/.venv
+ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
+RUN python3 -m venv $VIRTUAL_ENV && \
+    uv pip install --no-cache -r requirements.txt && \
+    pip uninstall -y acryl-datahub
 
 ENTRYPOINT [ "/entrypoint.sh" ]
 
 FROM ${BASE_IMAGE} as full-install
 
+USER 0
 RUN apt-get update && apt-get install -y -qq \
     default-jre-headless \
     && rm -rf /var/lib/apt/lists/* /var/cache/apk/*
@@ -91,10 +97,11 @@ RUN if [ $(arch) = "x86_64" ]; then \
     ldconfig; \
     fi;
 
+USER datahub
+
 FROM ${BASE_IMAGE} as slim-install
 # Do nothing else on top of base
 
 FROM ${APP_ENV}-install
 
-USER datahub
 ENV PATH="/datahub-ingestion/.local/bin:$PATH"