Merge pull request #1664 from CartoDB/release/1.0.4

Release/1.0.4

CHANGELOG.md

@@ -5,6 +5,25 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.0.4] - 2020-07-06
+
+## Added
+- Add list_tables function (#1649)
+- Add catalog public filter to providers, countries and categories (#1658)
+- Add set_default_do_credentials function for DO authentication (#1655)
+
+## Changed
+- Open publication link in another window (#1647)
+- Show a warning when uploading a GeoDataFrame without geometry (#1650)
+- Improve GeoDataFrame CRS check, docs and examples (#1656)
+
+## Fixed
+- Fix empty geometries issue (#1652)
+- Fix Layout publication API key issue (#1654)
+- Fix ColumnInfo comparison when replacing a table (#1660)
+- Fix formula count (#1662)
+- Fix catalog empty output (#1663)
+
 ## [1.0.3.1] - 2020-05-19
 
 ### Fixed

README.rst

@@ -4,8 +4,8 @@ CARTOframes
 
 .. image:: https://travis-ci.org/CartoDB/cartoframes.svg?branch=develop
     :target: https://travis-ci.org/CartoDB/cartoframes
-.. image:: https://img.shields.io/badge/pypi-v1.0.3.1-orange
-    :target: https://pypi.org/project/cartoframes/1.0.3.1
+.. image:: https://img.shields.io/badge/pypi-v1.0.4-orange
+    :target: https://pypi.org/project/cartoframes/1.0.4
 
 A Python package for integrating `CARTO <https://carto.com/>`__ maps, analysis, and data services into data science workflows.
 
@@ -14,11 +14,11 @@ Python data analysis workflows often rely on the de facto standards `pandas <htt
 Try it Out
 ==========
 
-* Stable (1.0.3.1): |stable|
+* Stable (1.0.4): |stable|
 * Latest (develop branch): |develop|
 
 .. |stable| image:: https://mybinder.org/badge_logo.svg
-    :target: https://mybinder.org/v2/gh/cartodb/cartoframes/v1.0.3.1?filepath=examples
+    :target: https://mybinder.org/v2/gh/cartodb/cartoframes/v1.0.4?filepath=examples
 
 .. |develop| image:: https://mybinder.org/badge_logo.svg
     :target: https://mybinder.org/v2/gh/cartodb/cartoframes/develop?filepath=examples

binder/requirements.txt

@@ -1,4 +1,4 @@
-cartoframes==1.0.3.1
+cartoframes==1.0.4
 # Additional dependencies from examples
 matplotlib
 dask

cartoframes/__init__.py

@@ -1,6 +1,6 @@
 from ._version import __version__
 from .utils.utils import check_package
-from .io.carto import read_carto, to_carto, has_table, delete_table, rename_table, \
+from .io.carto import read_carto, to_carto, list_tables, has_table, delete_table, rename_table, \
                       copy_table, create_table_from_query, describe_table, update_privacy_table
 
 
@@ -14,6 +14,7 @@
     '__version__',
     'read_carto',
     'to_carto',
+    'list_tables',
     'has_table',
     'delete_table',
     'rename_table',

cartoframes/_version.py

@@ -1 +1 @@
-__version__ = '1.0.3.1'
+__version__ = '1.0.4'

cartoframes/assets/templates/viz/footer.html.j2

@@ -1,3 +1,3 @@
 <div class="map-footer">
-  <span>Crafted with ♥ and CARTOframes | <a href="https://carto.com/developers/cartoframes/">Learn more</a></span>
+  <span>Crafted with ♥ and CARTOframes | <a href="https://carto.com/developers/cartoframes/" target="_blank">Learn more</a></span>
 </div>

cartoframes/auth/defaults.py

@@ -2,6 +2,7 @@
 from ..utils.utils import is_url, is_json_filepath
 
 _default_credentials = None
+_default_do_credentials = None
 
 
 def set_default_credentials(
@@ -85,30 +86,10 @@ def set_default_credentials(
 
     """
     global _default_credentials
-
-    _base_url = base_url if first is None else first
-    _username = username if first is None else first
-    _filepath = filepath if first is None else first
-    _api_key = (api_key if second is None else second) or 'default_public'
-    _credentials = credentials if first is None else first
-
-    if isinstance(_credentials, Credentials):
-        _default_credentials = _credentials
-
-    elif isinstance(_filepath, str) and is_json_filepath(_filepath):
-        _default_credentials = Credentials.from_file(_filepath)
-
-    elif isinstance(_base_url or _username, str) and isinstance(_api_key, str):
-        if _base_url and is_url(_base_url):
-            _default_credentials = Credentials(base_url=_base_url, api_key=_api_key, allow_non_secure=allow_non_secure)
-        else:
-            _default_credentials = Credentials(username=_username, api_key=_api_key, allow_non_secure=allow_non_secure)
-
-    else:
-        _default_credentials = Credentials.from_file()
-
-    if session:
-        _default_credentials.session = session
+    _default_credentials = _set_credentials(
+        first, second, credentials, filepath, username,
+        base_url, api_key, session, allow_non_secure
+    )
 
 
 def get_default_credentials():
@@ -139,3 +120,54 @@ def unset_default_credentials():
     """
     global _default_credentials
     _default_credentials = None
+
+
+def set_default_do_credentials(
+        first=None, second=None, credentials=None, filepath=None,
+        username=None, base_url=None, api_key=None, session=None, allow_non_secure=False):
+
+    global _default_do_credentials
+    _default_do_credentials = _set_credentials(
+        first, second, credentials, filepath, username,
+        base_url, api_key, session, allow_non_secure
+    )
+
+
+def get_default_do_credentials():
+    return _default_do_credentials
+
+
+def unset_default_do_credentials():
+    global _default_do_credentials
+    _default_do_credentials = None
+
+
+def _set_credentials(
+        first=None, second=None, credentials=None, filepath=None,
+        username=None, base_url=None, api_key=None, session=None, allow_non_secure=False):
+
+    _base_url = base_url if first is None else first
+    _username = username if first is None else first
+    _filepath = filepath if first is None else first
+    _api_key = (api_key if second is None else second) or 'default_public'
+    _credentials = credentials if first is None else first
+
+    if isinstance(_credentials, Credentials):
+        pass
+
+    elif isinstance(_filepath, str) and is_json_filepath(_filepath):
+        _credentials = Credentials.from_file(_filepath)
+
+    elif isinstance(_base_url or _username, str) and isinstance(_api_key, str):
+        if _base_url and is_url(_base_url):
+            _credentials = Credentials(base_url=_base_url, api_key=_api_key, allow_non_secure=allow_non_secure)
+        else:
+            _credentials = Credentials(username=_username, api_key=_api_key, allow_non_secure=allow_non_secure)
+
+    else:
+        _credentials = Credentials.from_file()
+
+    if session:
+        _credentials.session = session
+
+    return _credentials

cartoframes/data/observatory/catalog/repository/category_repo.py

@@ -1,9 +1,9 @@
-from .constants import COUNTRY_FILTER, PROVIDER_FILTER
+from .constants import COUNTRY_FILTER, PROVIDER_FILTER, PUBLIC_FILTER
 from .entity_repo import EntityRepository
 
 
 _CATEGORY_ID_FIELD = 'id'
-_ALLOWED_FILTERS = [COUNTRY_FILTER, PROVIDER_FILTER]
+_ALLOWED_FILTERS = [COUNTRY_FILTER, PROVIDER_FILTER, PUBLIC_FILTER]
 
 
 def get_category_repo():

cartoframes/data/observatory/catalog/repository/country_repo.py

@@ -1,9 +1,9 @@
-from .constants import CATEGORY_FILTER, PROVIDER_FILTER
+from .constants import CATEGORY_FILTER, PROVIDER_FILTER, PUBLIC_FILTER
 from .entity_repo import EntityRepository
 
 
 _COUNTRY_ID_FIELD = 'id'
-_ALLOWED_FILTERS = [CATEGORY_FILTER, PROVIDER_FILTER]
+_ALLOWED_FILTERS = [CATEGORY_FILTER, PROVIDER_FILTER, PUBLIC_FILTER]
 
 
 def get_country_repo():

cartoframes/data/observatory/catalog/repository/entity_repo.py

@@ -54,7 +54,7 @@ def _get_filtered_entities(self, filters):
         rows = self._get_rows(cleaned_filters)
 
         if len(rows) == 0:
-            return None
+            return CatalogList([])
 
         normalized_data = [self._get_entity_class()(self._map_row(row)) for row in rows]
         return CatalogList(normalized_data)

cartoframes/data/observatory/catalog/repository/provider_repo.py

@@ -1,9 +1,9 @@
-from .constants import CATEGORY_FILTER, COUNTRY_FILTER
+from .constants import CATEGORY_FILTER, COUNTRY_FILTER, PUBLIC_FILTER
 from .entity_repo import EntityRepository
 
 
 _PROVIDER_ID_FIELD = 'id'
-_ALLOWED_FILTERS = [CATEGORY_FILTER, COUNTRY_FILTER]
+_ALLOWED_FILTERS = [CATEGORY_FILTER, COUNTRY_FILTER, PUBLIC_FILTER]
 
 
 def get_provider_repo():

cartoframes/data/observatory/catalog/repository/repo_client.py

@@ -8,37 +8,55 @@
 class RepoClient:
 
     def __init__(self):
-        self._do_dataset = None
-        default_credentials = defaults.get_default_credentials() or Credentials(DEFAULT_USER)
+        default_credentials = Credentials(DEFAULT_USER)
         default_auth_client = default_credentials.get_api_key_auth_client()
         self._default_do_dataset = DODataset(auth_client=default_auth_client)
+        self._user_do_dataset = None
+        self._external_do_dataset = None
 
     def set_user_credentials(self, credentials):
         if credentials is not None:
             auth_client = credentials.get_api_key_auth_client()
-            self._do_dataset = DODataset(auth_client=auth_client)
+            self._user_do_dataset = DODataset(auth_client=auth_client)
         else:
-            self._do_dataset = None
+            self._user_do_dataset = None
 
     def reset_user_credentials(self):
-        self._do_dataset = None
+        self._user_do_dataset = None
+
+    def set_external_credentials(self):
+        # This must be checked every time to allow the definition of
+        # "default_do_credentials" at any point in the code because
+        # every repo uses a singleton instance of this client
+        external_credentials = defaults.get_default_do_credentials()
+        if external_credentials is not None:
+            external_auth_client = external_credentials.get_api_key_auth_client()
+            self._external_do_dataset = DODataset(auth_client=external_auth_client)
+        else:
+            self._external_do_dataset = None
 
     def get_countries(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('countries', filters)
 
     def get_categories(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('categories', filters)
 
     def get_providers(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('providers', filters)
 
     def get_datasets(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('datasets', filters, use_slug=True)
 
     def get_geographies(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('geographies', filters, use_slug=True)
 
     def get_variables(self, filters=None):
+        self.set_external_credentials()
         filter_id = self._get_filter_id(filters, use_slug=True)
         if filter_id:
             return self._fetch_entity_id('variables', filter_id)
@@ -47,6 +65,7 @@ def get_variables(self, filters=None):
             return self._fetch_entity(entity, filters)
 
     def get_variables_groups(self, filters=None):
+        self.set_external_credentials()
         filter_id = self._get_filter_id(filters, use_slug=True)
         if filter_id:
             return self._fetch_entity_id('variables_groups', filter_id)
@@ -75,7 +94,5 @@ def _fetch_entity_id(self, entity, filter_id):
             return self._fetch_entity('{0}/{1}'.format(entity, filter_id))
 
     def _fetch_entity(self, entity, filters=None):
-        if self._do_dataset:
-            return self._do_dataset.metadata(entity, filters)
-        else:
-            return self._default_do_dataset.metadata(entity, filters)
+        do_dataset = self._user_do_dataset or self._external_do_dataset or self._default_do_dataset
+        return do_dataset.metadata(entity, filters)

cartoframes/io/carto.py

@@ -6,7 +6,7 @@
 from carto.exceptions import CartoException
 
 from .managers.context_manager import ContextManager
-from ..utils.geom_utils import set_geometry, has_geometry
+from ..utils.geom_utils import check_crs, has_geometry, set_geometry
 from ..utils.logger import log
 from ..utils.utils import is_valid_str, is_sql_query
 from ..utils.metrics import send_metrics
@@ -65,7 +65,7 @@ def read_carto(source, credentials=None, limit=None, retry_times=3, schema=None,
 @send_metrics('data_uploaded')
 def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col=None, index=False, index_label=None,
              cartodbfy=True, log_enabled=True):
-    """Upload a DataFrame to CARTO.
+    """Upload a DataFrame to CARTO. The geometry's CRS must be WGS 84 (EPSG:4326) so you can use it on CARTO.
 
     Args:
         dataframe (pandas.DataFrame, geopandas.GeoDataFrame`): data to be uploaded.
@@ -90,6 +90,9 @@ def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col
     if not isinstance(dataframe, DataFrame):
         raise ValueError('Wrong dataframe. You should provide a valid DataFrame instance.')
 
+    if isinstance(dataframe, GeoDataFrame):
+        check_crs(dataframe)
+
     if not is_valid_str(table_name):
         raise ValueError('Wrong table name. You should provide a valid table name.')
 
@@ -118,6 +121,9 @@ def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col
         # Prepare geometry column for the upload
         gdf.rename_geometry(GEOM_COLUMN_NAME, inplace=True)
 
+    elif isinstance(dataframe, GeoDataFrame):
+        log.warning('Geometry column not found in the GeoDataFrame.')
+
     table_name = context_manager.copy_from(gdf, table_name, if_exists, cartodbfy)
 
     if log_enabled:
@@ -126,6 +132,23 @@ def to_carto(dataframe, table_name, credentials=None, if_exists='fail', geom_col
     return table_name
 
 
+def list_tables(credentials=None):
+    """List all of the tables in the CARTO account.
+
+    Args:
+        credentials (:py:class:`Credentials <cartoframes.auth.Credentials>`, optional):
+            instance of Credentials (username, api_key, etc).
+        schema (str, optional): prefix of the table. By default, it gets the
+            `current_schema()` using the credentials.
+
+    Returns:
+        DataFrame: A DataFrame with all the table names for the given credentials and schema.
+
+    """
+    context_manager = ContextManager(credentials)
+    return context_manager.list_tables()
+
+
 def has_table(table_name, credentials=None, schema=None):
     """Check if the table exists in the CARTO account.