diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 2db698d6..240de659 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -99,6 +99,7 @@ ENV PROJ_LIB=/opt/conda/share/proj # the remaining pip commands: https://www.anaconda.com/using-pip-in-a-conda-environment/ RUN conda config --add channels nvidia && \ conda config --add channels rapidsai && \ + conda config --set solver libmamba && \ # b/299991198 remove curl/libcurl install once DLVM base image includes version >= 7.86 conda install -c conda-forge mamba curl libcurl && \ # Base image channel order: conda-forge (highest priority), defaults. @@ -107,24 +108,17 @@ RUN conda config --add channels nvidia && \ /tmp/clean-layer.sh # Install spacy +# b/232247930: uninstall pyarrow to avoid double installation with the GPU specific version. +# b/341938540: unistall grpc-cpp to allow >=v24.4 cudf and cuml to be installed. {{ if eq .Accelerator "gpu" }} -RUN mamba install -y -c conda-forge spacy cupy cuda-version=$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION && \ +RUN pip uninstall -y pyarrow && \ + mamba remove -y --force grpc-cpp && \ + mamba install -y -c conda-forge spacy cudf>=24.4 cuml>=24.4 cupy cuda-version=$CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION && \ /tmp/clean-layer.sh {{ else }} RUN pip install spacy && \ /tmp/clean-layer.sh {{ end}} -{{ if eq .Accelerator "gpu" }} - -# b/232247930: uninstall pyarrow to avoid double installation with the GPU specific version. -RUN pip uninstall -y pyarrow && \ - mamba install -y cudf cuml && \ - /tmp/clean-layer.sh - -# TODO: b/296444923 - Resolve pandas dependency another way -RUN sed -i 's/^is_extension_type/# is_extension_type/g' /opt/conda/lib/python3.10/site-packages/cudf/api/types.py \ - && sed -i 's/^is_categorical/# is_categorical/g' /opt/conda/lib/python3.10/site-packages/cudf/api/types.py -{{ end }} # Install PyTorch {{ if eq .Accelerator "gpu" }} diff --git a/tests/common.py b/tests/common.py index e53e6a7c..30a7bb0f 100644 --- a/tests/common.py +++ b/tests/common.py @@ -2,6 +2,16 @@ import os import unittest +import subprocess + +def getAcceleratorName(): + try: + deviceName = subprocess.check_output(['nvidia-smi', '--query-gpu=name', '--format=csv,noheader']) + return deviceName.decode('utf-8').strip() + except FileNotFoundError: + return("nvidia-smi not found.") gpu_test = unittest.skipIf(len(os.environ.get('CUDA_VERSION', '')) == 0, 'Not running GPU tests') +# b/342143152 P100s are slowly being unsupported in new release of popular ml tools such as RAPIDS. +p100_exempt = unittest.skipIf(getAcceleratorName() == "Tesla P100-PCIE-16GB", 'Not running p100 exempt tests') tpu_test = unittest.skipIf(len(os.environ.get('ISTPUVM', '')) == 0, 'Not running TPU tests') diff --git a/tests/test_datashader.py b/tests/test_datashader.py deleted file mode 100644 index c8ff9295..00000000 --- a/tests/test_datashader.py +++ /dev/null @@ -1,38 +0,0 @@ -import unittest - -import numpy as np -import pandas as pd -import datashader as ds -import datashader.transfer_functions as tf - -class TestDatashader(unittest.TestCase): - # based on https://github.com/pyviz/datashader/blob/master/datashader/tests/test_pipeline.py - def test_pipeline(self): - df = pd.DataFrame({ - 'x': np.array(([0.] * 10 + [1] * 10)), - 'y': np.array(([0.] * 5 + [1] * 5 + [0] * 5 + [1] * 5)), - 'f64': np.arange(20, dtype='f8') - }) - df.f64.iloc[2] = np.nan - - cvs = ds.Canvas(plot_width=2, plot_height=2, x_range=(0, 1), y_range=(0, 1)) - - pipeline = ds.Pipeline(df, ds.Point('x', 'y')) - img = pipeline((0, 1), (0, 1), 2, 2) - agg = cvs.points(df, 'x', 'y', ds.count()) - self.assertTrue(img.equals(tf.shade(agg))) - - color_fn = lambda agg: tf.shade(agg, 'pink', 'red') - pipeline.color_fn = color_fn - img = pipeline((0, 1), (0, 1), 2, 2) - self.assertTrue(img.equals(color_fn(agg))) - - transform_fn = lambda agg: agg + 1 - pipeline.transform_fn = transform_fn - img = pipeline((0, 1), (0, 1), 2, 2) - self.assertTrue(img.equals(color_fn(transform_fn(agg)))) - - pipeline = ds.Pipeline(df, ds.Point('x', 'y'), ds.sum('f64')) - img = pipeline((0, 1), (0, 1), 2, 2) - agg = cvs.points(df, 'x', 'y', ds.sum('f64')) - self.assertTrue(img.equals(tf.shade(agg))) diff --git a/tests/test_geoviews.py b/tests/test_geoviews.py deleted file mode 100644 index 869fc0f1..00000000 --- a/tests/test_geoviews.py +++ /dev/null @@ -1,12 +0,0 @@ -import unittest - -import geoviews.feature as gf -import holoviews as hv -from cartopy import crs - -class TestGeoviews(unittest.TestCase): - def test_viz(self): - hv.extension('matplotlib') - (gf.ocean + gf.land + gf.ocean * gf.land * gf.coastline * gf.borders).options( - 'Feature', projection=crs.Geostationary(), global_extent=True - ).cols(3)