chore: upgrade backend dependencies (#2641)

chore: upgrade backend dependencies (#2641)

.github/workflows/compatibility_tests.yml

@@ -2,7 +2,7 @@ name: Compatibility Tests
 
 on:
   schedule:
-    - cron: '0 8 7 * 2'
+    - cron: "0 8 7 * 2"
   push:
     branches:
       - main
@@ -30,94 +30,83 @@ jobs:
       matrix:
         # note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
         os: [ubuntu-latest, macos-latest, macos-11]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9, 3.10, 3.11]
         cellxgene_build: [main, latest]
-        exclude:
-          # 3.6 no longer avail on Big Sur (`macos-11`)
-          - os: macos-11
-            python-version: 3.6
-          # no pypi build exists for macos+py3.9 and source install fails to
-          # install `tables` py pkg (a `scanpy` dependency), so we test py3.9
-          # only on ubuntu
-          - os: macos-11
-            python-version: 3.9
-          - os: macos-latest
-            python-version: 3.9
         # add anndata pinned version test for subset of matrix configurations,
         # in order to reduce matrix cross-product explosion
         include:
-          - python-version: 3.8
+          - python-version: 3.9
             cellxgene_build: latest
             # TODO: dynamically use the literal version in requirements.txt,
             # to avoid having to update this in manually in the future
             # TODO: Do not bother running this if anndata latest version
             # matches this pinned version, to avoid a redundant test
-            anndata_version: '==0.7.6'
+            anndata_version: "==0.10.3"
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Cache env vars
-      run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
-    - name: Cache env vars (MacOS)
-      if: startsWith(matrix.os, 'macos')
-      run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
-    # FIXME: Only working for Linux
-    - name: Python cache
-      uses: actions/cache@v1
-      with:
-        path: ${{ env.PIP_CACHE }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: Node cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-    - name: Brew cache (MacOS)
-      if: startsWith(matrix.os, 'macos')
-      uses: actions/cache@v1
-      with:
-        path: ${{ env.BREW_CACHE }}
-        key: ${{ runner.os }}-brew-
-    - name: Install dependencies (Ubuntu Linux)
-      if: startsWith(matrix.os, 'ubuntu')
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libhdf5-serial-dev
-    - name: Install dependencies (MacOS)
-      if: startsWith(matrix.os, 'macos')
-      run: brew install hdf5
-    - name: Install cellxgene from `main` branch
-      if: matrix.cellxgene_build == 'main'
-      run: |
-        pip install -r server/requirements-dev.txt
-        make pydist install-dist
-    - name: Install cellxgene from latest release (pypi.org)
-      if: matrix.cellxgene_build == 'latest'
-      run: |
-        pip install --upgrade cellxgene
-        # install the additional dev requirements on top of what is in the
-        # cellxgene pip package, which are needed for testing, but otherwise
-        # keep same pip pkg versions as in the cxg release
-        sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
-        pip install -r server/requirements-dev.txt
-    - name: Install anndata version per matrix variable
-      run: pip install anndata${{ matrix.anndata_version }}
-    - name: Install node
-      run: make dev-env-client
-    # Run different types of test separately, to facilitate troubleshooting
-    - name: Unit Tests - client
-      run: make unit-test-client
-    - name: Unit Tests - server
-      run: make unit-test-server
-    - name: Smoke Tests
-      run: make smoke-test
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache env vars
+        run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
+      - name: Cache env vars (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
+      # FIXME: Only working for Linux
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ${{ env.PIP_CACHE }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Brew cache (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        uses: actions/cache@v1
+        with:
+          path: ${{ env.BREW_CACHE }}
+          key: ${{ runner.os }}-brew-
+      - name: Install dependencies (Ubuntu Linux)
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libhdf5-serial-dev
+      - name: Install dependencies (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        run: brew install hdf5
+      - name: Install cellxgene from `main` branch
+        if: matrix.cellxgene_build == 'main'
+        run: |
+          pip install -r server/requirements-dev.txt
+          make pydist install-dist
+      - name: Install cellxgene from latest release (pypi.org)
+        if: matrix.cellxgene_build == 'latest'
+        run: |
+          pip install --upgrade cellxgene
+          # install the additional dev requirements on top of what is in the
+          # cellxgene pip package, which are needed for testing, but otherwise
+          # keep same pip pkg versions as in the cxg release
+          sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
+      - name: Install anndata version per matrix variable
+        run: pip install anndata${{ matrix.anndata_version }}
+      - name: Install node
+        run: make dev-env-client
+      # Run different types of test separately, to facilitate troubleshooting
+      - name: Unit Tests - client
+        run: make unit-test-client
+      - name: Unit Tests - server
+        run: make unit-test-server
+      - name: Smoke Tests
+        run: make smoke-test
     # FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
     # - name: Smoke Tests with Annotations
     #   run: make smoke-test-annotations

.github/workflows/push_tests.yml

@@ -17,10 +17,10 @@ jobs:
       - uses: actions/checkout@v2
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
-      - name: Set up Python 3.7
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - name: Node cache
         uses: actions/cache@v1
         with:
@@ -46,12 +46,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.7 (pyenv)  # pyenv needed for mlflow in cli annotate tests
+      - name: Set up Python 3.9 (pyenv) # pyenv needed for mlflow in cli annotate tests
         uses: gabrielfalcao/pyenv-action@v9
         with:
-          default: 3.7
-          command: pip install -U pip  # upgrade pip after installing python
-      - run: pip install virtualenv  # virtualenv needed for mlflow in cli annotate tests
+          default: 3.9
+          command: pip install -U pip # upgrade pip after installing python
+      - run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
       - name: Python cache
         uses: actions/cache@v1
         with:
@@ -79,10 +79,10 @@ jobs:
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.7
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
       - name: Python cache
         uses: actions/cache@v1
         with:
@@ -110,10 +110,10 @@ jobs:
   #   timeout-minutes: 20
   #   steps:
   #     - uses: actions/checkout@v2
-  #     - name: Set up Python 3.7
+  #     - name: Set up Python 3.9
   #       uses: actions/setup-python@v4
   #       with:
-  #         python-version: 3.7
+  #         python-version: 3.9
   #     - name: Python cache
   #       uses: actions/cache@v1
   #       with:

server/app/app.py

@@ -48,12 +48,12 @@ def wrapper(*args, **kwargs):
 
 
 def cache_control(**cache_kwargs):
-    """ config driven """
+    """config driven"""
     return _cache_control(False, **cache_kwargs)
 
 
 def cache_control_always(**cache_kwargs):
-    """ always generate headers, regardless of the config """
+    """always generate headers, regardless of the config"""
     return _cache_control(True, **cache_kwargs)
 
 
@@ -228,7 +228,7 @@ def add_resource(resource, url):
 class Server:
     @staticmethod
     def _before_adding_routes(app, app_config):
-        """ will be called before routes are added, during __init__.  Subclass protocol """
+        """will be called before routes are added, during __init__.  Subclass protocol"""
         pass
 
     def __init__(self, app_config):

server/app/session.py

@@ -6,7 +6,7 @@
 
 
 def get_user_id(session: SessionMixin) -> str:
-    """ Gets a session-persistent user id. Creates one in the Flask session if non-extant """
+    """Gets a session-persistent user id. Creates one in the Flask session if non-extant"""
     if CXGUID not in session:
         session[CXGUID] = uuid4().hex
         session.permanent = True

server/cli/annotate.py

@@ -26,9 +26,7 @@ def wrapper(*args, **kwargs):
 
 
 @sort_options
-@click.command(
-    options_metavar="<options>"
-)
+@click.command(options_metavar="<options>")
 @click.argument(
     "input_h5ad_file",
     type=click.Path(exists=True, dir_okay=False, readable=True),
@@ -51,16 +49,16 @@ def wrapper(*args, **kwargs):
     "--output-h5ad-file",
     default="",
     help="The output H5AD file that will contain the generated annotation values. If this option is not provided, "
-         "the input file will be overwritten to include the new annotations; in this case you must specify "
-         "--overwrite.",
+    "the input file will be overwritten to include the new annotations; in this case you must specify "
+    "--overwrite.",
     metavar="<filename>",
 )
 @click.option(
     "--overwrite",
     default=False,
     is_flag=True,
     help="Allow overwriting of the specified H5AD output file, if it exists. For safety, you must specify this "
-         "flag if the specified output file already exists or if the --output-h5ad-file option is not provided.",
+    "flag if the specified output file already exists or if the --output-h5ad-file option is not provided.",
     show_default=True,
 )
 @click.option(

server/common/annotations/local_file_csv.py

@@ -145,7 +145,7 @@ def read_gene_sets(self, data_adaptor, context=None):
     def write_gene_sets(self, gene_sets, tid, data_adaptor):
         self.check_gene_sets_save_enabled()  # raises
 
-        if type(tid) != int or tid < 0:
+        if type(tid) is not int or tid < 0:
             raise ValueError("tid must be a positive integer")
 
         # may raise
@@ -175,7 +175,7 @@ def write_gene_sets(self, gene_sets, tid, data_adaptor):
 
             # update the cache
             self.last_geneset_fname = fname
-            self.last_geneset = gene_sets if type(gene_sets) == dict else {g["geneset_name"]: g for g in gene_sets}
+            self.last_geneset = gene_sets if isinstance(gene_sets, dict) else {g["geneset_name"]: g for g in gene_sets}
 
     def _get_userdata_idhash(self, data_adaptor):
         """

server/common/compute/diffexp_generic.py

@@ -56,7 +56,7 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
 
     # degrees of freedom for Welch's t-test
     with np.errstate(divide="ignore", invalid="ignore"):
-        dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
+        dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1))
     dof[np.isnan(dof)] = 1
 
     # Welch's t-test score calculation

server/common/compute/estimate_distribution.py

@@ -97,7 +97,7 @@ def estimate_approximate_distribution(X) -> XApproximateDistribution:
     if Xdata.size > CHUNKSIZE:
         min_val = max_val = Xdata[0]
         with concurrent.futures.ThreadPoolExecutor() as tp:
-            for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
+            for _min, _max in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
                 min_val = min(_min, min_val)
                 max_val = max(_max, max_val)
 

server/common/config/__init__.py

@@ -1,2 +1,2 @@
 DEFAULT_SERVER_PORT = 5005
-BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB
+BIG_FILE_SIZE_THRESHOLD = 100 * 2**20  # 100MB

server/common/config/app_config.py

@@ -19,7 +19,6 @@ class AppConfig(object):
     """
 
     def __init__(self):
-
         # the default configuration (see default_config.py)
         # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
         #  defaults within the config class?

server/common/config/base_config.py

@@ -50,7 +50,7 @@ def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
                     f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
                 )
         else:
-            if type(val) != vtype:
+            if type(val) is not vtype:
                 raise ConfigurationError(
                     f"Invalid type for attribute: {attrname}, "
                     f"expected type {vtype.__name__}, got {type(val).__name__}"
@@ -70,7 +70,7 @@ def update(self, **kw):
             if not hasattr(self, key):
                 raise ConfigurationError(f"unknown config parameter {key}.")
             try:
-                if type(value) == tuple:
+                if type(value) is tuple:
                     # convert tuple values to list values
                     value = list(value)
                 setattr(self, key, value)

server/common/config/external_config.py

@@ -29,7 +29,7 @@ def handle_environment(self, context):
             if name is None:
                 raise ConfigurationError("environment: 'name' is missing")
             required = envdict.get("required", False)
-            if type(required) != bool:
+            if type(required) is not bool:
                 raise ConfigurationError("environment: 'required' must be a bool")
             path = envdict.get("path")
             if path is None:

server/common/fbs/matrix.py

@@ -19,7 +19,7 @@
 
 # Serialization helper
 def serialize_column(builder, typed_arr):
-    """ Serialize NetEncoding.Column """
+    """Serialize NetEncoding.Column"""
 
     (u_type, u_value) = typed_arr
     Column.ColumnStart(builder)
@@ -30,7 +30,7 @@ def serialize_column(builder, typed_arr):
 
 # Serialization helper
 def serialize_matrix(builder, n_rows, n_cols, columns, col_idx):
-    """ Serialize NetEncoding.Matrix """
+    """Serialize NetEncoding.Matrix"""
 
     Matrix.MatrixStart(builder)
     Matrix.MatrixAddNRows(builder, n_rows)