From 04037988f9fe84b507755cdfcb8b375f3d936581 Mon Sep 17 00:00:00 2001 From: Richard Postelnik Date: Mon, 19 Dec 2016 12:56:56 -0500 Subject: [PATCH 1/9] #515 switch to Numeric type --- odo/backends/sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/odo/backends/sql.py b/odo/backends/sql.py index 9df4a5af..cc7c5b7d 100644 --- a/odo/backends/sql.py +++ b/odo/backends/sql.py @@ -184,7 +184,7 @@ def discover_typeengine(typ): return dshape(revtypes[typ])[0] if type(typ) in revtypes: return revtypes[type(typ)] - if isinstance(typ, (sa.NUMERIC, sa.DECIMAL)): + if isinstance(typ, sa.Numeric): return datashape.Decimal(precision=typ.precision, scale=typ.scale) if isinstance(typ, (sa.String, sa.Unicode)): return datashape.String(typ.length, 'U8') From 4992061a3f53f29d8abe44c04d65d830d09924f6 Mon Sep 17 00:00:00 2001 From: Richard Postelnik Date: Wed, 14 Feb 2018 23:33:54 -0500 Subject: [PATCH 2/9] add 36 to builds, update tests --- .travis.yml | 16 ++++++++-------- conda.recipe/meta.yaml | 10 +++++----- odo/backends/pandas.py | 2 +- odo/backends/tests/test_bokeh.py | 8 ++++++-- odo/backends/tests/test_dask_array.py | 12 ++++++------ odo/backends/tests/test_dask_bag.py | 1 - odo/backends/tests/test_dask_dataframe.py | 2 +- odo/backends/tests/test_sqlite.py | 16 ++++++++-------- odo/tests/test_convert.py | 2 +- 9 files changed, 36 insertions(+), 33 deletions(-) diff --git a/.travis.yml b/.travis.yml index 841e6aab..1cad27c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,26 +7,26 @@ matrix: include: - python: 2.7 env: - - python: 3.4 - env: - python: 3.5 env: + - python: 3.6 + env: - python: 2.7 env: PANDAS_VERSION="git+https://github.com/pydata/pandas" - - python: 3.4 - env: PANDAS_VERSION="git+https://github.com/pydata/pandas" - python: 3.5 env: PANDAS_VERSION="git+https://github.com/pydata/pandas" + - python: 3.6 + env: PANDAS_VERSION="git+https://github.com/pydata/pandas" allow_failures: - python: 2.7 env: PANDAS_VERSION="git+https://github.com/pydata/pandas" - - python: 3.4 - env: PANDAS_VERSION="git+https://github.com/pydata/pandas" - python: 3.5 env: PANDAS_VERSION="git+https://github.com/pydata/pandas" + - python: 3.6 + env: PANDAS_VERSION="git+https://github.com/pydata/pandas" addons: - postgresql: "9.3" + postgresql: "9.5" services: - mongodb @@ -47,7 +47,7 @@ install: - source activate odo # update setuptools and pip - conda update setuptools pip - - if [ -n "$PANDAS_VERSION" ]; then pip install cython==0.24.1; pip install $PANDAS_VERSION; else conda install pandas=0.19.0; fi + - if [ -n "$PANDAS_VERSION" ]; then pip install cython==0.27.3; pip install $PANDAS_VERSION; else conda install pandas=0.22.0; fi # install the frozen ci dependencies - pip install -e .[ci] diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index e6898683..2d150050 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -8,7 +8,7 @@ build: {% else %}string: py{{ environ.get('PY_VER').replace('.', '') }}_{{ environ.get('GIT_BUILD_STR', 'GIT_STUB') }}{% endif %} source: - git_url: ../ + path: ../ requirements: build: @@ -18,7 +18,7 @@ requirements: - pandas >=0.15.0 - toolz >=0.7.3 - multipledispatch >=0.4.7 - - networkx + - networkx >1.0,<2.0 - dask >=0.11.1 run: @@ -28,7 +28,7 @@ requirements: - pandas >=0.15.0 - toolz >=0.7.3 - multipledispatch >=0.4.7 - - networkx + - networkx >1.0,<2.0 - dask >=0.11.1 test: @@ -38,7 +38,7 @@ test: - pytables >=3.0.0 - sqlalchemy >=0.8.0 - bcolz - - pymongo >=2.8,<3 # [not (py35 or win)] + - pymongo # [not (py35 or win)] - dask - paramiko # [not win] - sqlite3 # [win] @@ -49,7 +49,7 @@ test: - psycopg2 # [unix] commands: - - pip install sas7bdat && py.test -vx --pyargs odo -rsxX + - pip install sas7bdat && py.test -v --pyargs odo -rsxX about: home: https://github.com/blaze/odo/ diff --git a/odo/backends/pandas.py b/odo/backends/pandas.py index 634d6023..bfcc0264 100644 --- a/odo/backends/pandas.py +++ b/odo/backends/pandas.py @@ -18,7 +18,7 @@ def dshape_from_pandas(col): - if isinstance(col.dtype, categorical): + if col.dtype.name == 'category': return Categorical(col.cat.categories.tolist()) elif col.dtype.kind == 'M': tz = getattr(col.dtype, 'tz', None) diff --git a/odo/backends/tests/test_bokeh.py b/odo/backends/tests/test_bokeh.py index 2c3d4983..e7a6264e 100644 --- a/odo/backends/tests/test_bokeh.py +++ b/odo/backends/tests/test_bokeh.py @@ -4,6 +4,8 @@ bokeh = pytest.importorskip('bokeh') from odo.backends.bokeh import convert, pd, ColumnDataSource +import numpy as np +import numpy.testing as nt import pandas.util.testing as tm @@ -15,8 +17,10 @@ def test_convert_dataframe_to_cds(): cds = convert(ColumnDataSource, df) - assert cds.data == {'name': ['Alice', 'Bob', 'Charlie'], - 'balance': [100, 200, 300]} + nt.assert_equal(cds.data['name'], + np.array(['Alice', 'Bob', 'Charlie'])) + nt.assert_equal(cds.data['balance'], + np.array([100, 200, 300])) df2 = convert(pd.DataFrame, cds) assert isinstance(df2, pd.DataFrame) diff --git a/odo/backends/tests/test_dask_array.py b/odo/backends/tests/test_dask_array.py index c16808b7..dc622e4c 100644 --- a/odo/backends/tests/test_dask_array.py +++ b/odo/backends/tests/test_dask_array.py @@ -59,14 +59,14 @@ def test_into_inplace(): def test_insert_to_ooc(): - x = np.arange(600).reshape((20, 30)) - y = np.empty(shape=x.shape, dtype=x.dtype) - a = convert(Array, x, chunks=(4, 5)) + in_arr = np.arange(600).reshape((20, 30)) + out_arr = np.empty(shape=in_arr.shape, dtype=in_arr.dtype) + d_arr = convert(Array, in_arr, chunks=(4, 5)) - dsk = insert_to_ooc(y, a) - core.get(merge(dsk, a.dask), list(dsk.keys())) + dsk = insert_to_ooc(d_arr, out_arr) + core.get(merge(dsk, d_arr.dask), list(dsk.keys())) - assert eq(y, x) + assert eq(out_arr, in_arr) def test_array_interface(): diff --git a/odo/backends/tests/test_dask_bag.py b/odo/backends/tests/test_dask_bag.py index ad7621e4..c89916f4 100644 --- a/odo/backends/tests/test_dask_bag.py +++ b/odo/backends/tests/test_dask_bag.py @@ -31,7 +31,6 @@ def test_convert_logfiles_to_bag(): logs = chunks(TextFile)(list(map(TextFile, fns))) b = odo(logs, Bag) assert isinstance(b, Bag) - assert 'a1.log' in str(b.dask.values()) assert (list(map(methodcaller('strip'), odo(b, list))) == list(map(methodcaller('strip'), odo(logs, list)))) diff --git a/odo/backends/tests/test_dask_dataframe.py b/odo/backends/tests/test_dask_dataframe.py index 3c2daf93..80a2b6fb 100644 --- a/odo/backends/tests/test_dask_dataframe.py +++ b/odo/backends/tests/test_dask_dataframe.py @@ -24,7 +24,7 @@ def test_discover(): ddf = dd.from_pandas(df, npartitions=2) assert_dshape_equal(discover(ddf), var * Record([('x', Categorical(['a', 'b', 'c'])), - ('y', int64), ('z', float64)])) + ('y', int64), ('z', float64)])) assert_dshape_equal(discover(ddf.x), var * Categorical(['a', 'b', 'c'])) diff --git a/odo/backends/tests/test_sqlite.py b/odo/backends/tests/test_sqlite.py index 4ae5e92a..4e69f7fa 100644 --- a/odo/backends/tests/test_sqlite.py +++ b/odo/backends/tests/test_sqlite.py @@ -86,14 +86,14 @@ def test_different_encoding(): NULL = u'' - expected = [(u'1958.001.500131-1A', 1, NULL, NULL, 899), - (u'1958.001.500156-6', 1, NULL, NULL, 899), - (u'1958.001.500162-1', 1, NULL, NULL, 899), - (u'1958.001.500204-2', 1, NULL, NULL, 899), - (u'1958.001.500204-2A', 1, NULL, NULL, 899), - (u'1958.001.500204-2B', 1, NULL, NULL, 899), - (u'1958.001.500223-6', 1, NULL, NULL, 9610), - (u'1958.001.500233-9', 1, NULL, NULL, 4703), + expected = [(u'1958.001.500131-1A', 1, None, None, 899), + (u'1958.001.500156-6', 1, None, None, 899), + (u'1958.001.500162-1', 1, None, None, 899), + (u'1958.001.500204-2', 1, None, None, 899), + (u'1958.001.500204-2A', 1, None, None, 899), + (u'1958.001.500204-2B', 1, None, None, 899), + (u'1958.001.500223-6', 1, None, None, 9610), + (u'1958.001.500233-9', 1, None, None, 4703), (u'1909.017.000018-3', 1, 30.0, u'sumaria', 899)] assert result == expected diff --git a/odo/tests/test_convert.py b/odo/tests/test_convert.py index d75857d3..ac7e2834 100644 --- a/odo/tests/test_convert.py +++ b/odo/tests/test_convert.py @@ -165,7 +165,7 @@ def test_numpy_launders_python_types(): def test_numpy_asserts_type_after_dataframe(): df = pd.DataFrame({'name': ['Alice'], 'amount': [100]}) - ds = datashape.dshape('1 * {name: string[10, "ascii"], amount: int32}') + ds = datashape.dshape('1 * {amount: int32, name: string[10, "ascii"]}') x = convert(np.ndarray, df, dshape=ds) assert discover(x) == ds From f3bdeca77c90b25560746246668562eeecb56e40 Mon Sep 17 00:00:00 2001 From: Richard Postelnik Date: Wed, 14 Feb 2018 23:52:53 -0500 Subject: [PATCH 3/9] add osx support, generic language --- .travis.yml | 5 ++++- etc/ci-install.sh | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index abca9397..f8798a0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,9 @@ sudo: False +os: + - linux + - osx -language: python +language: generic matrix: fast_finish: true diff --git a/etc/ci-install.sh b/etc/ci-install.sh index c53abc39..51502699 100644 --- a/etc/ci-install.sh +++ b/etc/ci-install.sh @@ -10,7 +10,7 @@ if [ -n "$PANDAS_VERSION" ];then pip install cython==0.24.1 pip install $PANDAS_VERSION else - conda install pandas=0.19.0 + conda install pandas=0.22.1 fi conda install pytables=3.3.0 From e4d83176a59ff00cbb7f01bd17f876ec023f2c4f Mon Sep 17 00:00:00 2001 From: Richard Postelnik Date: Wed, 14 Feb 2018 23:57:40 -0500 Subject: [PATCH 4/9] remove marker --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f8798a0a..a5139e07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,4 @@ sudo: False -os: - - linux - - osx language: generic @@ -43,7 +40,6 @@ install: - conda config --set always_yes yes --set changeps1 no - conda update conda -<<<<<<< HEAD # Install dependencies # Use conda **ONLY** for numpy and pandas (if not pulling from master), this # speeds up the builds a lot. Use the normal pip install for the rest. From 6b17f5394f80b15b0823950cad6f6b92360ec5cd Mon Sep 17 00:00:00 2001 From: Richard Postelnik Date: Thu, 15 Feb 2018 00:18:20 -0500 Subject: [PATCH 5/9] update psycopg version --- etc/requirements_ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/requirements_ci.txt b/etc/requirements_ci.txt index 1371bb72..021aeac6 100644 --- a/etc/requirements_ci.txt +++ b/etc/requirements_ci.txt @@ -27,7 +27,7 @@ pandas==0.19.0 # # please update the .travis file! paramiko==2.0.2 partd==0.3.6 psutil==4.4.2 -psycopg2==2.6.2 +psycopg2==2.7.4 py==1.4.34 pyasn1==0.1.9 pycparser==2.17 From 864848419de621b64d2b62e0ed5e1d35c66c55ca Mon Sep 17 00:00:00 2001 From: Richard Postelnik Date: Thu, 15 Feb 2018 00:35:11 -0500 Subject: [PATCH 6/9] update dependencies --- .travis.yml | 4 ++-- etc/requirements_ci.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a5139e07..3359298c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,8 @@ addons: postgresql: "9.5" services: - - mongodb - - mysql + - mongodb + - mysql install: # Install conda diff --git a/etc/requirements_ci.txt b/etc/requirements_ci.txt index 021aeac6..c3797953 100644 --- a/etc/requirements_ci.txt +++ b/etc/requirements_ci.txt @@ -6,7 +6,7 @@ cffi==1.8.3 click==6.6 cloudpickle==0.2.1 cryptography==1.5.2 -dask==0.11.1 +dask==0.16.1 decorator==4.0.10 distributed==1.13.3 docutils==0.12 @@ -32,7 +32,7 @@ py==1.4.34 pyasn1==0.1.9 pycparser==2.17 pymongo==2.9.4 -PyMySQL==0.7.9 +PyMySQL==0.7.11 pytest==3.2.3 python-dateutil==2.5.3 pytz==2016.7 @@ -43,7 +43,7 @@ s3fs==0.0.7 s3transfer==0.1.9 sas7bdat==2.0.7 six==1.10.0 -SQLAlchemy==1.1.3 +SQLAlchemy==1.2.1 tables==3.3.0 tblib==1.3.0 toolz==0.8.0 From cc485a010c5b4b636eec24eda690f34c3e4bbe41 Mon Sep 17 00:00:00 2001 From: Richard Postelnik Date: Thu, 15 Feb 2018 10:05:38 -0500 Subject: [PATCH 7/9] Fix sqlite test --- odo/backends/tests/test_sqlite.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/odo/backends/tests/test_sqlite.py b/odo/backends/tests/test_sqlite.py index 4e69f7fa..4ae5e92a 100644 --- a/odo/backends/tests/test_sqlite.py +++ b/odo/backends/tests/test_sqlite.py @@ -86,14 +86,14 @@ def test_different_encoding(): NULL = u'' - expected = [(u'1958.001.500131-1A', 1, None, None, 899), - (u'1958.001.500156-6', 1, None, None, 899), - (u'1958.001.500162-1', 1, None, None, 899), - (u'1958.001.500204-2', 1, None, None, 899), - (u'1958.001.500204-2A', 1, None, None, 899), - (u'1958.001.500204-2B', 1, None, None, 899), - (u'1958.001.500223-6', 1, None, None, 9610), - (u'1958.001.500233-9', 1, None, None, 4703), + expected = [(u'1958.001.500131-1A', 1, NULL, NULL, 899), + (u'1958.001.500156-6', 1, NULL, NULL, 899), + (u'1958.001.500162-1', 1, NULL, NULL, 899), + (u'1958.001.500204-2', 1, NULL, NULL, 899), + (u'1958.001.500204-2A', 1, NULL, NULL, 899), + (u'1958.001.500204-2B', 1, NULL, NULL, 899), + (u'1958.001.500223-6', 1, NULL, NULL, 9610), + (u'1958.001.500233-9', 1, NULL, NULL, 4703), (u'1909.017.000018-3', 1, 30.0, u'sumaria', 899)] assert result == expected From d882c8059a8a34e604997ce6edc4bf6f5b8d5472 Mon Sep 17 00:00:00 2001 From: Richard Postelnik Date: Thu, 15 Feb 2018 10:26:35 -0500 Subject: [PATCH 8/9] handle mysql stat error --- odo/backends/tests/test_mysql.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/odo/backends/tests/test_mysql.py b/odo/backends/tests/test_mysql.py index 14860cac..5c716c18 100644 --- a/odo/backends/tests/test_mysql.py +++ b/odo/backends/tests/test_mysql.py @@ -1,23 +1,25 @@ from __future__ import absolute_import, division, print_function -import pytest - -pymysql = pytest.importorskip('pymysql') - -from datashape import var, DataShape, Record, dshape +import csv as csv_module +import getpass import itertools -from odo.backends.csv import CSV -from odo import resource, odo -import sqlalchemy -import sqlalchemy as sa import os import sys -import csv as csv_module -import getpass -from odo import drop, discover + +import pytest +import sqlalchemy +import sqlalchemy as sa +from datashape import var, Record, dshape + +from odo import resource, odo, drop, discover +from odo.backends.csv import CSV from odo.utils import tmpfile +pymysql = pytest.importorskip('pymysql') + + + pytestmark = pytest.mark.skipif(sys.platform == 'win32', reason='not well tested on win32 mysql') @@ -232,7 +234,7 @@ def test_different_encoding(name): sql = odo(os.path.join(os.path.dirname(__file__), 'encoding.csv'), url + '::%s' % name, encoding=encoding) - except sa.exc.OperationalError as e: + except (sa.exc.OperationalError, pymysql.err.InternalError) as e: pytest.skip(str(e)) else: try: From d7fde6f8ddf98eeda179aad51395d2fa56a6de27 Mon Sep 17 00:00:00 2001 From: Richard Postelnik Date: Thu, 15 Feb 2018 21:01:21 -0500 Subject: [PATCH 9/9] try with sudo required --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3359298c..7b665e49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: False +sudo: required language: generic