Merge pull request #836 from linchiahuisage/SYNPY-1064-speed_up_integ…

…ration_tests

Synpy 1064 speed up integration tests

.github/workflows/build.yml

@@ -45,7 +45,28 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
 
+      - name: get-dependencies-location
+        shell: bash
+        run: |
+          SITE_PACKAGES_LOCATION=$(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
+          SITE_BIN_DIR=$(python3 -c "import os; import platform; import distutils.sysconfig; pre = distutils.sysconfig.get_config_var('prefix'); bindir = os.path.join(pre, 'Scripts' if platform.system() == 'Windows' else 'bin'); print(bindir)")
+          echo "::set-output name=site_packages_loc::$SITE_PACKAGES_LOCATION"
+          echo "::set-output name=site_bin_dir::$SITE_BIN_DIR"
+        id: get-dependencies
+
+      - name: Cache py-dependencies
+        id: cache-dependencies
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-py-dependencies
+        with:
+          path: |
+            ${{ steps.get-dependencies.outputs.site_packages_loc }}
+            ${{ steps.get-dependencies.outputs.site_bin_dir }}
+          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v7
+
       - name: install-py-dependencies
+        if: steps.cache-dependencies.outputs.cache-hit != 'true'
         shell: bash
         run: |
           pip install -e ".[boto3,pandas,pysftp,tests]"
@@ -59,7 +80,6 @@ jobs:
             pip install numpy
           fi
 
-
       - name: lint
         shell: bash
         run: |
@@ -112,7 +132,8 @@ jobs:
             export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}"
             export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}"
 
-            pytest -sv tests/integration
+             # use loadscope to avoid issues running tests concurrently that share scoped fixtures
+            pytest -sv tests/integration -n auto --dist loadscope
           fi
 
   # on a GitHub release, build the pip package and upload it as a GitHub release asset

setup.py

@@ -30,7 +30,8 @@
 test_deps = [
     "pytest>=5.0.0,<7.0",
     "pytest-mock>=3.0,<4.0",
-    "flake8>=3.7.0,<4.0"
+    "flake8>=3.7.0,<4.0",
+    "pytest-xdist[psutil]>=2.2,<3.0.0",
 ]
 
 install_requires = [

tests/integration/synapseclient/core/test_download.py

@@ -2,7 +2,6 @@
 import os
 import tempfile
 import shutil
-import time
 
 import pytest
 
@@ -59,7 +58,7 @@ def test_http_download__range_request_error(syn, project):
     file_path = utils.make_bogus_data_file()
     file_entity = syn.store(File(file_path, parent=project))
 
-    syn.cache.purge(time.time())
+    syn.cache.remove(file_entity['_file_handle']['id'])
     # download once and rename to temp file to simulate range exceed
     file_entity = syn.get(file_entity)
     shutil.move(file_entity.path, utils.temp_download_filename(file_entity.path, file_entity.dataFileHandleId))

tests/integration/synapseclient/test_command_line_client.py

@@ -1077,7 +1077,11 @@ def test_create__same_project_name(test_state):
     assert entity_id_first == entity_id_second
 
 
-def test_storeTable__csv(test_state):
+@patch.object(utils.sys.stdin, 'isatty')
+def test_storeTable__csv(mock_sys, test_state):
+    # when running on windows os with multiple CPU, the sys.stdin.isatty will return True
+    # Thus we mock the utils.sys.stdin.
+    mock_sys.return_value = False
     output = run(test_state,
                  'synapse',
                  'store-table',
@@ -1088,5 +1092,6 @@ def test_storeTable__csv(test_state):
                  '--parentid',
                  test_state.project.id
                  )
+
     mapping = json.loads(output)
     test_state.schedule_for_cleanup(mapping['tableId'])