feat: use argilla server as external python package (#4537)

# Description This is a work in progress PR where I will commit changes related with using `argilla_server` Python package as dependency at `argilla` repository. Task to do: - [x] Remove `src/argilla/server` code. - [x] Remove `tests/unit/server` code. - [x] Replace `argilla.server` code references to `argilla_server`. - [x] Make test suite to pass. - [x] Change GitHub workflows to adapt to this change. - [x] Build and deploy a quickstart image combining argilla_server backend and UI changes on this repo - #4574 Closes #4536 **Type of change** (Please delete options that are not relevant. Remember to title the PR according to the type of change) - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [x] Refactor (change restructuring the codebase without changing functionality) - [ ] Improvement (change adding some improvement to an existing functionality) - [ ] Documentation update **How Has This Been Tested** - [ ] Test suite should pass. - [ ] GitHub workflows should execute correctly and do what it's expected once we have the server code in a different repository. **Checklist** - [ ] I added relevant documentation - [ ] follows the style guidelines of this project - [ ] I did a self-review of my code - [ ] I made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] I filled out [the contributor form](https://tally.so/r/n9XrxK) (see text above) - [x] I have added relevant notes to the CHANGELOG.md file (See https://keepachangelog.com/) --------- Co-authored-by: Paco Aranda <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
argilla-io · Feb 7, 2024 · 487a359 · 487a359
1 parent 4e53300
commit 487a359
Show file tree

Hide file tree

Showing 398 changed files with 189 additions and 48,652 deletions.
diff --git a/.github/workflows/build-push-docker.yml → ...kflows/build-push-dev-frontend-docker.yml b/.github/workflows/build-push-docker.yml → ...kflows/build-push-dev-frontend-docker.yml
@@ -52,15 +52,16 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Download python package
-        uses: actions/download-artifact@v3
-        if: ${{ inputs.download-python-package }}
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
         with:
-          name: python-package
-          path: docker/dist
+          node-version: "18"
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+      - name: Build Frontend
+        working-directory: frontend
+        run: |
+            npm install
+            npm run build
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
@@ -72,14 +73,9 @@ jobs:
       - name: Generate Docker tags
         id: generate-docker-tags
         run: |
-          DOCKER_HUB_TAGS="$IMAGE_NAME:$DOCKER_IMAGE_TAG"
           GCP_REGISTRY_TAG="europe-docker.pkg.dev/argilla-ci/$IMAGE_NAME:$DOCKER_IMAGE_TAG"
 
-          if [[ $GITHUB_REF == refs/tags/* ]]; then
-            DOCKER_HUB_TAGS="$DOCKER_HUB_TAGS,$IMAGE_NAME:latest"
-          fi
-
-          echo "tags=$DOCKER_HUB_TAGS,$GCP_REGISTRY_TAG" >> $GITHUB_OUTPUT
+          echo "tags=$GCP_REGISTRY_TAG" >> $GITHUB_OUTPUT
         env:
           IMAGE_NAME: ${{ inputs.image-name }}
           DOCKER_IMAGE_TAG: ${{ steps.docker-image-tag-from-ref.outputs.docker-image-tag }}
@@ -109,19 +105,10 @@ jobs:
       - name: Build and push
         uses: docker/build-push-action@v4
         with:
-          context: docker
+          context: frontend
           file: ${{ inputs.dockerfile }}
           platforms: ${{ inputs.platforms }}
           tags: ${{ steps.generate-docker-tags.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: ${{ inputs.build-args }}
           push: true
-
-      - name: Docker Hub Description
-        uses: peter-evans/dockerhub-description@v3
-        if: github.event_name == 'release'
-        with:
-          username: ${{ secrets.AR_DOCKER_USERNAME }}
-          password: ${{ secrets.AR_DOCKER_PASSWORD }}
-          repository: ${{ inputs.image-name }}
-          readme-filepath: ${{ inputs.readme }}
diff --git a/.github/workflows/build-python-package.yml b/.github/workflows/build-python-package.yml
@@ -10,6 +10,7 @@ jobs:
     steps:
       - name: Checkout Code 🛎
         uses: actions/checkout@v3
+
       - name: Cache pip 👜
         uses: actions/cache@v3
         env:
@@ -18,14 +19,12 @@ jobs:
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ env.CACHE_NUMBER }}-${{ hashFiles('pyproject.toml') }}
-      - name: Setup Node.js
-        uses: actions/setup-node@v3
-        with:
-          node-version: "18"
+
       - name: Build Package 🍟
         run: |
           pip install -U build
-          scripts/build_distribution.sh
+          rm -rf dist && python -m build
+
       - name: Upload package artifact
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/check-repo-files.yml b/.github/workflows/check-repo-files.yml
@@ -12,6 +12,9 @@ on:
       buildChanges:
         description: "True if some files affecting the build have changed"
         value: ${{ jobs.check-repo-files.outputs.buildChanges }}
+      frontendChanges:
+        description: "True if some files affecting the frontend folder changed"
+        value: ${{ jobs.check-repo-files.outputs.frontendChanges }}
 
 jobs:
   check-repo-files:
@@ -21,6 +24,7 @@ jobs:
       pythonChanges: ${{ steps.path_filter.outputs.pythonChanges }}
       buildChanges: ${{ steps.path_filter.outputs.buildChanges }}
       end2endChanges: ${{ steps.path_filter.outputs.end2endChanges }}
+      frontendChanges: ${{ steps.path_filter.outputs.frontendChanges }}
     steps:
       - name: Checkout Code 🛎
         uses: actions/checkout@v3
@@ -40,6 +44,8 @@ jobs:
               - 'setup.py'
               - 'scripts/end2end_examples.py'
               - 'docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/**'
+            frontendChanges:
+              - 'frontend/**'
             buildChanges:
               - 'src/**'
               - 'frontend/**'

diff --git a/.github/workflows/deploy-environment.yml b/.github/workflows/deploy-environment.yml
@@ -44,7 +44,7 @@ jobs:
           service: argilla-quickstart-${{ inputs.image-version }}
           image: europe-docker.pkg.dev/argilla-ci/${{ inputs.image-name}}:${{ inputs.image-version }}
           region: europe-southwest1
-          flags: "--min-instances=1 --max-instances=1 --port=6900 --cpu=2000m --memory=4096Mi --no-cpu-throttling --allow-unauthenticated"
+          flags: "--min-instances=1 --max-instances=1 --port=3000 --cpu=2000m --memory=4096Mi --no-cpu-throttling --allow-unauthenticated"
           env_vars: |
             OWNER_PASSWORD=${{ steps.credentials.outputs.owner }}
             OWNER_API_KEY=${{ steps.credentials.outputs.owner }}

diff --git a/.github/workflows/end2end-examples.yml b/.github/workflows/end2end-examples.yml
@@ -88,7 +88,7 @@ jobs:
           pip install -e ".[server]"
           python -m argilla server database migrate
           python -m argilla server database users create_default
-          uvicorn argilla.server.app:app --port 6900 --host 0.0.0.0 &
+          python -m argilla server start &
       - name: Run end2end examples 📈
         env:
           ARGILLA_ENABLE_TELEMETRY: 0

diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml
@@ -51,26 +51,13 @@ jobs:
           echo "is_deployable=$IS_DEPLOYABLE" >> $GITHUB_OUTPUT
 
   run_unit_tests:
-    strategy:
-      matrix:
-        include:
-          - searchEngineDockerImage: docker.elastic.co/elasticsearch/elasticsearch:8.8.2
-            searchEngineDockerEnv: '{"discovery.type": "single-node", "xpack.security.enabled": "false"}'
-            coverageReport: coverage-elasticsearch-8.8.2
-            runsOn: extended-runner
-          - searchEngineDockerImage: opensearchproject/opensearch:2.4.1
-            searchEngineDockerEnv: '{"discovery.type": "single-node", "plugins.security.disabled": "true"}'
-            coverageReport: coverage-opensearch-2.4.1
-            runsOn: ubuntu-latest
     name: Run unit tests
     uses: ./.github/workflows/run-python-tests.yml
     needs: check_repo_files
     if: needs.check_repo_files.outputs.pythonChanges == 'true'
     with:
-      searchEngineDockerImage: ${{ matrix.searchEngineDockerImage }}
-      searchEngineDockerEnv: ${{ matrix.searchEngineDockerEnv }}
       coverageReport: coverage
-      runsOn: ${{ matrix.runsOn }}
+      runsOn: extended-runner
       pytestArgs: tests/unit
     secrets: inherit
 
@@ -82,6 +69,7 @@ jobs:
             searchEngineDockerEnv: '{"discovery.type": "single-node", "xpack.security.enabled": "false"}'
             coverageReport: coverage-elasticsearch-8.8.2
             runsOn: extended-runner
+
     name: Run end2end tests
     uses: ./.github/workflows/end2end-examples.yml
     needs: check_repo_files
@@ -92,36 +80,6 @@ jobs:
       searchEngineDockerEnv: ${{ matrix.searchEngineDockerEnv }}
     secrets: inherit
 
-  run_unit_test_with_extra_engines:
-    strategy:
-      matrix:
-        include:
-          - searchEngineDockerImage: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
-            searchEngineDockerEnv: '{"discovery.type": "single-node", "xpack.security.enabled": "false"}'
-            coverageReport: coverage-elasticsearch-8.8.0
-            runsOn: extended-runner
-          - searchEngineDockerImage: docker.elastic.co/elasticsearch/elasticsearch:8.6.0
-            searchEngineDockerEnv: '{"discovery.type": "single-node", "xpack.security.enabled": "false"}'
-            coverageReport: coverage-elasticsearch-8.6.0
-            runsOn: extended-runner
-          - searchEngineDockerImage: opensearchproject/opensearch:2.8.0
-            searchEngineDockerEnv: '{"discovery.type": "single-node", "plugins.security.disabled": "true"}'
-            coverageReport: coverage-opensearch-2.8.0
-            runsOn: extended-runner
-    name: Run unit tests with extra engines
-    uses: ./.github/workflows/run-python-tests.yml
-    needs: check_repo_files
-    if: |
-      needs.check_repo_files.outputs.pythonChanges == 'true' &&
-      startsWith(github.event.ref, 'refs/tags/v')
-    with:
-      searchEngineDockerImage: ${{ matrix.searchEngineDockerImage }}
-      searchEngineDockerEnv: ${{ matrix.searchEngineDockerEnv }}
-      coverageReport: coverage
-      runsOn: ${{ matrix.runsOn }}
-      pytestArgs: tests/unit
-    secrets: inherit
-
   run_integration_tests:
     name: Run integration tests
     uses: ./.github/workflows/run-python-tests.yml
@@ -144,68 +102,74 @@ jobs:
     steps:
       - name: Checkout Code 🛎
         uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
+
+      - name: Download coverage reports
+        uses: actions/download-artifact@v3
+
       - name: Copy all reports
         run: find coverage-report*/ -name "*.xml" -exec mv '{}' . \;
+
       - name: Display current files structure
         run: ls -R
+
       - name: Upload Coverage reports to Codecov 📦
         uses: codecov/codecov-action@v2
 
+  run_frontend_test:
+    name: Run Argilla Frontend tests
+    runs-on: ubuntu-latest
+    needs: check_repo_files
+    if: needs.check_repo_files.outputs.frontendChanges == 'true'
+    steps:
+      - name: Checkout Code 🛎
+        uses: actions/checkout@v3
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+
+      - name: Run Frontend tests
+        working-directory: frontend
+        run: |
+          npm install
+          npm run lint
+          npm run test
+
   build_python_package:
     name: Build Argilla python package
     uses: ./.github/workflows/build-python-package.yml
-    needs: deployable_check
-    if: needs.deployable_check.outputs.isDeployable == 'true'
-    secrets: inherit
-
-  build_server_docker_image:
-    name: Build Argilla server docker image
-    uses: ./.github/workflows/build-push-docker.yml
     needs:
-      - build_python_package
       - run_unit_tests
-    if: |
-      !cancelled() &&
-      needs.build_python_package.result == 'success' &&
-      !contains(needs.*.result, 'failure') &&
-      !contains(needs.*.result, 'cancelled')
-    with:
-      download-python-package: true
-      image-name: argilla/argilla-server
-      dockerfile: docker/Dockerfile
-      readme: README.md
-      platforms: linux/amd64,linux/arm64
+      - run_integration_tests
+      - check_repo_files
+    if: needs.check_repo_files.outputs.pythonChanges == 'true'
     secrets: inherit
 
-  build_quickstart_docker_image:
-    name: Build Argilla quickstart docker image
-    uses: ./.github/workflows/build-push-docker.yml
-    needs: build_server_docker_image
-    if: |
-      !cancelled() &&
-      needs.build_server_docker_image.result == 'success'
+  build_quickstart_for_develop_docker_image:
+    name: Build Quickstart For Develop docker image
+    uses: ./.github/workflows/build-push-dev-frontend-docker.yml
+    needs: deployable_check
+    if: needs.deployable_check.outputs.isDeployable == 'true'
     with:
-      download-python-package: false
-      image-name: argilla/argilla-quickstart
-      dockerfile: docker/quickstart.Dockerfile
-      readme: docker/quickstart.README.md
-      platforms: linux/amd64,linux/arm64
+      image-name: argilla/argilla-quickstart-for-dev
+      dockerfile: frontend/dev.frontend.Dockerfile
+      platforms: linux/amd64
       build-args: |
-        ARGILLA_VERSION=${{ needs.build_server_docker_image.outputs.version }}
+        ARGILLA_SERVER_TAG=main
     secrets: inherit
 
   deploy_environment:
     name: Deploy branch environment
     uses: ./.github/workflows/deploy-environment.yml
-    needs: build_quickstart_docker_image
+    needs: build_quickstart_for_develop_docker_image
     if: |
       !cancelled() &&
-      needs.build_quickstart_docker_image.result == 'success' &&
+      needs.build_quickstart_for_develop_docker_image.result == 'success' &&
       github.event_name == 'pull_request' && github.event.pull_request.draft == false
     with:
-      image-name: argilla/argilla-quickstart
-      image-version: ${{ needs.build_quickstart_docker_image.outputs.version }}
+      image-name: argilla/argilla-quickstart-for-dev
+      image-version: ${{ needs.build_quickstart_for_develop_docker_image.outputs.version }}
     secrets: inherit
 
   # This job will upload a Python Package using Twine when a release is created
@@ -217,26 +181,29 @@ jobs:
     if: ${{ github.event_name == 'release' }}
     needs:
       - run_integration_tests
-      - build_quickstart_docker_image
     defaults:
       run:
         shell: bash -l {0}
     steps:
       - name: Checkout Code 🛎
         uses: actions/checkout@v3
+
       - name: Download python package
         uses: actions/download-artifact@v2
         with:
           name: python-package
           path: dist
+
       - name: Publish Package to TestPyPI 🥪
         uses: pypa/gh-action-pypi-publish@master
         with:
           user: __token__
           password: ${{ secrets.AR_TEST_PYPI_API_TOKEN }}
           repository_url: https://test.pypi.org/legacy/
+
       - name: Test Installing 🍿
         run: pip install --index-url https://test.pypi.org/simple --no-deps argilla==${GITHUB_REF#refs/*/v}
+
       - name: Publish Package to PyPI 🥩
         uses: pypa/gh-action-pypi-publish@master
         with:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,6 +16,12 @@ These are the section headers that we use:
 
 ## [Unreleased]()
 
+## [1.24.0](https://github.com/argilla-io/argilla/compare/v1.23.0...v1.24.0)
+
+### Changed
+
+- The package is using the `argilla-server` dependency defined [here](https://github.com/argilla-io/argilla-server). ([#4537](https://github.com/argilla-io/argilla/pull/4537))
+
 ## [1.23.0](https://github.com/argilla-io/argilla/compare/v1.22.0...v1.23.0)
 
 ### Added