release: 2024-05-21 (#510)

.github/workflows/snowflake.yml

@@ -16,7 +16,7 @@ on:
 
 env:
   NODE_VERSION: 18.16
-  PYTHON3_VERSION: 3.8.10
+  PYTHON3_VERSION: 3.8.18
   VIRTUALENV_VERSION: 20.21.1
   GCLOUD_VERSION: 290.0.1
 
@@ -154,7 +154,6 @@ jobs:
     timeout-minutes: 10
     env:
       PACKAGE_BUCKET: gs://carto-analytics-toolbox-core/snowflake
-      SF_DATABASE: 'carto'
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2

.gitignore

@@ -22,6 +22,7 @@ package-lock.json
 MANIFEST
 *.ipynb
 .pgpass
+.~lock.*
 
 # IntelliJ
 .idea/

CHANGELOG.md

@@ -4,6 +4,13 @@ CARTO Analytics Toolbox Core.
 
 All notable commits to this project will be documented in this file.
 
+## 2024-05-21
+
+- refactor(sf|h3): avoid memory limit exceeded in H3_POLYFILL_TABLE (#501)
+- docs(bq,sf,rs|constructors): fix ST_MAKEELLIPSE angle parameter explanation (#505)
+- chore(sf): update create-package and licenses year (#507)
+- chore(pg): DB Connection Error in tests when % character in PG_PASSWORD (#509)
+
 ## 2024-04-18
 
 - chore(sf|h3): reimplement basic h3 functions (#489)

LICENSE

@@ -1,6 +1,6 @@
 SPDX short identifier: BSD-3-Clause
 
-Copyright (c) 2021-2022, CARTO
+Copyright (c) 2021-2024, CARTO
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

clouds/bigquery/common/DROP_FUNCTIONS.sql

@@ -1,5 +1,5 @@
 ---------------------------------
--- Copyright (C) 2021-2023 CARTO
+-- Copyright (C) 2021-2024 CARTO
 ---------------------------------
 
 CREATE OR REPLACE PROCEDURE `@@BQ_DATASET@@.DROP_FUNCTIONS`()

clouds/bigquery/modules/doc/constructors/ST_MAKEELLIPSE.md

@@ -11,7 +11,7 @@ Takes a Point and calculates the ellipse polygon given two semi-axes expressed i
 * `center`: `GEOGRAPHY` center point.
 * `xSemiAxis`: `FLOAT64` semi (major) axis of the ellipse along the x-axis.
 * `ySemiAxis`: `FLOAT64` semi (minor) axis of the ellipse along the y-axis.
-* `angle`: `FLOAT64`|`NULL` angle of rotation (along the vertical axis), from North in decimal degrees, positive clockwise. If `NULL` the default value `0` is used.
+* `angle`: `FLOAT64`|`NULL` angle of rotation (along the horizontal axis), from East in decimal degrees, positive clockwise. If `NULL` the default value `0` is used.
 * `units`: `STRING`|`NULL` units of length, the supported options are: miles, kilometers, and degrees. If `NULL`the default value `kilometers` is used.
 * `steps`: `INT64`|`NULL` number of steps. If `NULL` the default value `64` is used.
 

clouds/postgres/common/DROP_FUNCTIONS.sql

@@ -1,5 +1,5 @@
 ---------------------------------
--- Copyright (C) 2021-2023 CARTO
+-- Copyright (C) 2021-2024 CARTO
 ---------------------------------
 
 CREATE OR REPLACE PROCEDURE @@PG_SCHEMA@@.__DROP_FUNCTIONS

clouds/postgres/common/test_utils/__init__.py

@@ -5,6 +5,7 @@
 import pandas as pd
 import psycopg2
 
+from urllib.parse import quote_plus
 from shapely import wkt
 from sqlalchemy import create_engine
 
@@ -15,14 +16,21 @@
     'password': os.environ['PG_PASSWORD'],
     'port': 5432,
 }
+URL_PG_CONFIG = {
+    'host': os.environ['PG_HOST'],
+    'database': os.environ['PG_DATABASE'],
+    'user': os.environ['PG_USER'],
+    'password': quote_plus(os.environ['PG_PASSWORD']),
+    'port': 5432,
+}
 
 conn = psycopg2.connect(**PG_CONFIG)
 conn.autocommit = (
     True  # Needed for running procedures since they must be out of a transaction
 )
 
 connection_string = 'postgresql://{user}:{password}@{host}:{port}/{database}'.format(
-    **PG_CONFIG
+    **URL_PG_CONFIG
 )
 engine = create_engine(connection_string)
 

clouds/redshift/common/DROP_FUNCTIONS.sql

@@ -1,5 +1,5 @@
 ---------------------------------
--- Copyright (C) 2021-2023 CARTO
+-- Copyright (C) 2021-2024 CARTO
 ---------------------------------
 
 CREATE OR REPLACE PROCEDURE @@RS_SCHEMA@@.__CREATE_DROP_TABLE

clouds/redshift/modules/doc/constructors/ST_MAKEELLIPSE.md

@@ -11,7 +11,7 @@ Takes a Point as input and calculates the ellipse polygon given two semi-axes ex
 * `center`: `GEOMETRY` center point.
 * `xSemiAxis`: `FLOAT8` semi (major) axis of the ellipse along the x-axis.
 * `ySemiAxis`: `FLOAT8` semi (minor) axis of the ellipse along the y-axis.
-* `angle` (optional): `FLOAT8` angle of rotation (along the vertical axis), from North in decimal degrees, positive clockwise. If not specified, the default value of `0` will be used.
+* `angle` (optional): `FLOAT8` angle of rotation (along the horizontal axis), from East in decimal degrees, positive clockwise. If not specified, the default value of `0` will be used.
 * `units` (optional): `VARCHAR(10)` units of length. The supported options are: miles, kilometers, meters, and degrees. If not specified, `kilometers` will be used.
 * `steps` (optional): `INT` number of steps. If not specified, the default value of `64` will be used.
 

clouds/snowflake/CHANGELOG.md

@@ -4,6 +4,10 @@ CARTO Analytics Toolbox Core for Snowflake.
 
 All notable commits to this project will be documented in this file.
 
+## [1.2.3] - 2024-05-21
+
+- refactor(h3): avoid memory limit exceeded in H3_POLYFILL_TABLE (#501)
+
 ## [1.2.2] - 2024-04-18
 
 - chore(h3): reimplement basic h3 functions (#489)

clouds/snowflake/Makefile

@@ -96,7 +96,7 @@ clean-modules:
 	$(MAKE) -C modules clean
 
 create-package:
-	$(MAKE) build production=1 dropfirst=1
+	$(MAKE) build production=1 dropfirst=1 database=CARTO
 
 	rm -rf $(DIST_DIR)
 	mkdir -p $(DIST_DIR)/$(PACKAGE_NAME)

clouds/snowflake/common/DROP_FUNCTIONS.sql

@@ -1,5 +1,5 @@
 ---------------------------------
--- Copyright (C) 2021-2023 CARTO
+-- Copyright (C) 2021-2024 CARTO
 ---------------------------------
 
 CREATE OR REPLACE PROCEDURE @@SF_SCHEMA@@._DROP_FUNCTIONS()

clouds/snowflake/common/Makefile

@@ -4,13 +4,17 @@ PYTHON3_VERSION = 3
 VENV3_DIR ?= $(COMMON_DIR)/../venv3
 VENV3_BIN = $(VENV3_DIR)/bin
 NODE_MODULES_DEV = $(COMMON_DIR)/node_modules
-export SF_SCHEMA_DEFAULT = carto
+export SF_SCHEMA_DEFAULT = CARTO
 
 ifneq (,$(wildcard $(ENV_DIR)/.env))
     include $(ENV_DIR)/.env
 	export $(shell sed 's/=.*//' $(ENV_DIR)/.env)
 endif
 
+ifdef database
+export SF_DATABASE = $(database)
+endif
+
 ifeq ($(production),1)
 export SF_SCHEMA = $(SF_DATABASE).$(SF_SCHEMA_DEFAULT)
 export SF_UNQUALIFIED_SCHEMA = $(SF_SCHEMA_DEFAULT)

clouds/snowflake/modules/doc/constructors/ST_MAKEELLIPSE.md

@@ -11,7 +11,7 @@ Takes a Point and calculates the ellipse polygon given two semi-axes expressed i
 * `center`: `GEOGRAPHY` center point.
 * `xSemiAxis`: `DOUBLE` semi (major) axis of the ellipse along the x-axis.
 * `ySemiAxis`: `DOUBLE` semi (minor) axis of the ellipse along the y-axis.
-* `angle` (optional): `DOUBLE` angle of rotation (along the vertical axis), from North in decimal degrees, positive clockwise. By default `angle` is `0`.
+* `angle` (optional): `DOUBLE` angle of rotation (along the horizontal axis), from East in decimal degrees, positive clockwise. By default `angle` is `0`.
 * `units` (optional): `STRING` units of length, the supported options are: miles, kilometers, and degrees. By default `units` is `kilometers`.
 * `steps` (optional): `INT` number of steps. By default `steps` is `64`.
 

clouds/snowflake/modules/sql/h3/H3_POLYFILL_TABLE.sql

@@ -13,23 +13,14 @@ RETURNS STRING
 LANGUAGE SQL
 EXECUTE AS CALLER
 AS $$
-DECLARE
-    column_names_csv STRING;
-    polyfill_query STRING;
 BEGIN
+    EXECUTE IMMEDIATE '
+        CREATE OR REPLACE TABLE ' || output_table || ' CLUSTER BY (H3) AS
+        WITH __input AS ( ' || input_query || ' )
+        SELECT CAST(cell.value AS STRING) AS h3, i.* EXCLUDE(geom)
+        FROM __input AS i, TABLE(FLATTEN(@@SF_SCHEMA@@.H3_POLYFILL(geom, ' || resolution || ', ''' || mode || '''))) AS cell;
+    ';
 
-    -- Validate
-    EXECUTE IMMEDIATE 'SELECT * FROM (' || input_query || ') WHERE FALSE';
-
-    -- New table with correct columns
-    EXECUTE IMMEDIATE 'CREATE OR REPLACE TABLE ' || output_table || ' CLUSTER BY (H3) AS SELECT * EXCLUDE geom, NULL as H3 FROM (' || input_query || ') WHERE FALSE';
-
-    column_names_csv := (SELECT LISTAGG(COLUMN_NAME, ',') FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME ILIKE :output_table AND TABLE_CATALOG=@@SF_SCHEMA@@._GET_DATABASE(:output_table) AND TABLE_SCHEMA=@@SF_SCHEMA@@._GET_SCHEMA(:output_table));
-
-    polyfill_query := 'INSERT INTO ' || output_table || ' (' || column_names_csv || ') SELECT ' || column_names_csv || ' FROM (WITH virtual_table AS (' || input_query || ') SELECT *, value as H3 FROM virtual_table, LATERAL FLATTEN(input => @@SF_SCHEMA@@.H3_POLYFILL(virtual_table.geom, ' || resolution || ',\'' || mode || '\')))';
-
-    EXECUTE IMMEDIATE polyfill_query;
-
-    RETURN 'Finished!';
+    RETURN 'H3 Polyfill result added in table ' || output_table;
 END;
 $$;

clouds/snowflake/version

@@ -1 +1 @@
-1.2.2
+1.2.3