Merge remote-tracking branch 'upstream/master' into pr-osparc-dynamic…

…-scheduler-scheduling-options
ITISFoundation · Dec 5, 2024 · 509eb21 · 509eb21
2 parents c219c8e + e317f6b
commit 509eb21
Show file tree

Hide file tree

Showing 295 changed files with 8,514 additions and 18,027 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -94,9 +94,6 @@ component_management:
     - component_id: invitations
       paths:
         - services/invitations/**
-    - component_id: osparc_gateway_server
-      paths:
-        - services/osparc-gateway-server/**
     - component_id: payments
       paths:
         - services/payments/**

diff --git a/.env-devel b/.env-devel
@@ -129,6 +129,7 @@ DYNAMIC_SCHEDULER_PROFILING=1
 DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
 DYNAMIC_SCHEDULER_TRACING={}
+DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin
 
 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'
 
@@ -211,7 +212,7 @@ RESOURCE_USAGE_TRACKER_S3=null
 RESOURCE_USAGE_TRACKER_TRACING={}
 
 # NOTE: 172.17.0.1 is the docker0 interface, which redirect from inside a container onto the host network interface.
-R_CLONE_OPTION_BUFFER_SIZE=0M
+R_CLONE_OPTION_BUFFER_SIZE=16M
 R_CLONE_OPTION_RETRIES=3
 R_CLONE_OPTION_TRANSFERS=5
 R_CLONE_PROVIDER=MINIO
@@ -260,7 +261,6 @@ WB_API_WEBSERVER_PORT=8080
 WB_GC_ACTIVITY=null
 WB_GC_ANNOUNCEMENTS=0
 WB_GC_CATALOG=null
-WB_GC_CLUSTERS=0
 WB_GC_DB_LISTENER=0
 WB_GC_DIAGNOSTICS=null
 WB_GC_EMAIL=null
@@ -293,7 +293,6 @@ WB_GC_WALLETS=0
 WB_DB_EL_ACTIVITY=null
 WB_DB_EL_ANNOUNCEMENTS=0
 WB_DB_EL_CATALOG=null
-WB_DB_EL_CLUSTERS=0
 WB_DB_EL_DB_LISTENER=1
 WB_DB_EL_DIAGNOSTICS=null
 WB_DB_EL_EMAIL=null
@@ -360,7 +359,6 @@ TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT=["41"]
 WEBSERVER_ACTIVITY=null
 WEBSERVER_ANNOUNCEMENTS=1
 WEBSERVER_CATALOG={}
-WEBSERVER_CLUSTERS=0
 WEBSERVER_CREDIT_COMPUTATION_ENABLED=1
 WEBSERVER_DB_LISTENER=0
 WEBSERVER_DEV_FEATURES_ENABLED=0

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -33,7 +33,6 @@ Makefile                                  @pcrespov @sanderegg
 /services/efs-guardian/                   @matusdrobuliak66
 /services/invitations/                    @pcrespov
 /services/migration/                      @pcrespov
-/services/osparc-gateway-server/          @sanderegg
 /services/payments/                       @pcrespov @matusdrobuliak66
 /services/resource-usage-tracker/         @matusdrobuliak66
 /services/static-webserver/               @GitHK

diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml
@@ -75,7 +75,6 @@ jobs:
       efs-guardian: ${{ steps.filter.outputs.efs-guardian }}
       invitations: ${{ steps.filter.outputs.invitations }}
       migration: ${{ steps.filter.outputs.migration }}
-      osparc-gateway-server: ${{ steps.filter.outputs.osparc-gateway-server }}
       payments: ${{ steps.filter.outputs.payments }}
       dynamic-scheduler: ${{ steps.filter.outputs.dynamic-scheduler }}
       resource-usage-tracker: ${{ steps.filter.outputs.resource-usage-tracker }}
@@ -222,12 +221,6 @@ jobs:
               - 'packages/**'
               - 'services/migration/**'
               - 'services/docker-compose*'
-            osparc-gateway-server:
-              - 'packages/**'
-              - 'services/osparc-gateway-server/**'
-              - 'services/docker-compose*'
-              - 'scripts/mypy/*'
-              - 'mypy.ini'
             payments:
                 - 'packages/**'
                 - 'services/payments/**'
@@ -1161,64 +1154,6 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  unit-test-osparc-gateway-server:
-    needs: changes
-    if: ${{ needs.changes.outputs.osparc-gateway-server == 'true' || github.event_name == 'push' }}
-    timeout-minutes: 18 # if this timeout gets too small, then split the tests
-    name: "[unit] osparc-gateway-server"
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python: ["3.11"]
-        os: [ubuntu-22.04]
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v4
-      - name: setup docker buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          driver: docker-container
-      - name: setup python environment
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python }}
-      - name: install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: "0.4.x"
-          enable-cache: false
-          cache-dependency-glob: "**/osparc-gateway-server/requirements/ci.txt"
-      - name: show system version
-        run: ./ci/helpers/show_system_versions.bash
-      - name: install
-        run: |
-          make devenv
-          source .venv/bin/activate && \
-          pushd services/osparc-gateway-server && \
-          make install-ci
-      - name: typecheck
-        run: |
-          source .venv/bin/activate && \
-          pushd services/osparc-gateway-server && \
-          make mypy
-      - name: test
-        if: ${{ !cancelled() }}
-        run: |
-          source .venv/bin/activate && \
-          pushd services/osparc-gateway-server && \
-          make test-ci-unit
-      - uses: codecov/[email protected]
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-        with:
-          flags: unittests #optional
-      - name: Upload test results to Codecov
-        if: ${{ !cancelled() }}
-        uses: codecov/test-results-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-
   unit-test-payments:
     needs: changes
     if: ${{ needs.changes.outputs.payments == 'true' || github.event_name == 'push' }}
@@ -1965,7 +1900,6 @@ jobs:
         unit-test-models-library,
         unit-test-common-library,
         unit-test-notifications-library,
-        unit-test-osparc-gateway-server,
         unit-test-payments,
         unit-test-dynamic-scheduler,
         unit-test-postgres-database,
@@ -2317,84 +2251,6 @@ jobs:
         with:
           flags: integrationtests #optional
 
-  integration-test-osparc-gateway-server:
-    needs: [changes, build-test-images]
-    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.osparc-gateway-server == 'true' || github.event_name == 'push' }}
-    timeout-minutes: 30 # if this timeout gets too small, then split the tests
-    name: "[int] osparc-gateway-server"
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python: ["3.11"]
-        os: [ubuntu-22.04]
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v4
-      - name: setup docker buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          driver: docker-container
-
-      - name: setup python environment
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python }}
-      - name: expose github runtime for buildx
-        uses: crazy-max/ghaction-github-runtime@v3
-      # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249
-      - name: download docker images with retry
-        uses: Wandalen/wretry.action@master
-        with:
-          action: actions/download-artifact@v4
-          with: |
-            name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}
-            path: /${{ runner.temp }}/build
-          attempt_limit: 5
-          attempt_delay: 1000
-      - name: load docker images
-        run: make load-images local-src=/${{ runner.temp }}/build
-      - name: install uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          version: "0.4.x"
-          enable-cache: false
-          cache-dependency-glob: "**/osparc-gateway-server/requirements/ci.txt"
-      - name: show system version
-        run: ./ci/helpers/show_system_versions.bash
-      - name: install
-        run: |
-          make devenv && \
-          source .venv/bin/activate && \
-          pushd services/osparc-gateway-server && \
-          make install-ci
-      - name: integration-test
-        run: |
-          source .venv/bin/activate && \
-          pushd services/osparc-gateway-server && \
-          make test-ci-integration
-      - name: system-test
-        run: |
-          source .venv/bin/activate && \
-          pushd services/osparc-gateway-server && \
-          make test-system
-      - name: upload failed tests logs
-        if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ github.job }}_docker_logs
-          path: ./services/director-v2/test_failures
-      - name: cleanup
-        if: ${{ !cancelled() }}
-        run: |
-          pushd services/osparc-gateway-server && \
-          make down
-      - uses: codecov/[email protected]
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-        with:
-          flags: integrationtests #optional
-
   integration-test-simcore-sdk:
     needs: [changes, build-test-images]
     if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }}
@@ -2460,13 +2316,12 @@ jobs:
 
   integration-tests:
     # NOTE: this is a github required status check!
-    if: ${{ !cancelled() }}
+    if: ${{ always() }}
     needs:
       [
         integration-test-director-v2-01,
         integration-test-director-v2-02,
         integration-test-dynamic-sidecar,
-        integration-test-osparc-gateway-server,
         integration-test-simcore-sdk,
         integration-test-webserver-01,
         integration-test-webserver-02,
@@ -2792,7 +2647,7 @@ jobs:
 
   system-tests:
     # NOTE: this is a github required status check!
-    if: ${{ !cancelled() }}
+    if: ${{ always() }}
     needs:
       [
         system-test-e2e,

diff --git a/Makefile b/Makefile
@@ -44,7 +44,6 @@ SERVICES_NAMES_TO_BUILD := \
 	efs-guardian \
 	invitations \
   migration \
-	osparc-gateway-server \
 	payments \
 	resource-usage-tracker \
 	dynamic-scheduler \
@@ -323,6 +322,7 @@ printf "$$rows" "oSparc platform" "http://$(get_my_ip).nip.io:9081";\
 printf "$$rows" "oSparc public API doc" "http://$(get_my_ip).nip.io:8006/dev/doc";\
 printf "$$rows" "oSparc web API doc" "http://$(get_my_ip).nip.io:9081/dev/doc";\
 printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\
+printf "$$rows" "Dy-scheduler Dashboard" "http://$(get_my_ip).nip.io:8012";\
 printf "$$rows" "Docker Registry" "http://$${REGISTRY_URL}/v2/_catalog" $${REGISTRY_USER} $${REGISTRY_PW};\
 printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVITATIONS_USERNAME} $${INVITATIONS_PASSWORD};\
 printf "$$rows" "Jaeger" "http://$(get_my_ip).nip.io:16686";\

diff --git a/api/specs/web-server/_cluster.py b/api/specs/web-server/_cluster.py