From 77566fd849d50a67a79def44827292c0455aca92 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 22 Apr 2024 23:53:18 -0700 Subject: [PATCH 1/5] Use random db name --- .../sedona/snowflake/snowsql/TestBase.java | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java index 15bac69148..53c87b8793 100644 --- a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java +++ b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java @@ -32,11 +32,13 @@ import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.junit.Assert.assertArrayEquals; @@ -52,6 +54,8 @@ public class TestBase extends TestCase { private static boolean jarUploaded = false; + String snowflake_db_name = generateRandomString(8); + public void registerUDF(String functionName, Class ... paramTypes) { try { String ddl = UDFDDLGenerator.buildUDFDDL(UDFs.class.getMethod( @@ -102,12 +106,13 @@ public void init() throws SQLException { buildDDLConfigs = new HashMap<>(); buildDDLConfigs.put(Constants.SEDONA_VERSION, sedonaVersion); buildDDLConfigs.put(Constants.GEOTOOLS_VERSION, geotoolsVersion); + System.out.println("Using Snowflake DB: " + snowflake_db_name); // upload libraries if (!jarUploaded) { // drop then create db to make sure test env fresh - snowClient.executeQuery("drop database if exists " + System.getenv("SNOWFLAKE_DB")); - snowClient.executeQuery("create database " + System.getenv("SNOWFLAKE_DB")); - snowClient.executeQuery("use database " + System.getenv("SNOWFLAKE_DB")); + snowClient.executeQuery("drop database if exists " + snowflake_db_name); + snowClient.executeQuery("create database " + snowflake_db_name); + snowClient.executeQuery("use database " + snowflake_db_name); snowClient.executeQuery("create schema " + System.getenv("SNOWFLAKE_SCHEMA")); snowClient.executeQuery("use schema " + System.getenv("SNOWFLAKE_SCHEMA")); snowClient.executeQuery("CREATE STAGE ApacheSedona FILE_FORMAT = (COMPRESSION = NONE)"); @@ -118,7 +123,15 @@ public void init() throws SQLException { registerDependantUDFs(); } - public void tearDown() { + public void tearDown() + { + try { + System.out.println("Dropping Snowflake DB: " + snowflake_db_name); + snowClient.executeQuery("drop database if exists " + snowflake_db_name); + } + catch (SQLException e) { + throw new RuntimeException(e); + } if (snowClient != null) { try { snowClient.close(); @@ -174,4 +187,24 @@ public ResultSet sqlSingleRes(String sql) { throw new RuntimeException(e); } } + + /** + * Generates a random string of a specified length. + * + * @param length The length of the random string. + * @return A random string consisting of alphabet letters only. + */ + private static String generateRandomString(int length) { + // Generate a string of all letters + String chars = IntStream.concat(IntStream.rangeClosed('A', 'Z'), IntStream.rangeClosed('a', 'z')) + .mapToObj(c -> "" + (char) c) + .collect(Collectors.joining()); + + Random random = new Random(); + + // Build a random string from the full set of characters + return random.ints(length, 0, chars.length()) + .mapToObj(i -> "" + chars.charAt(i)) // Correctly refer to character position + .collect(Collectors.joining()); + } } From 9d7063597c3640597b54a3241ba82bb299a9964b Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Tue, 23 Apr 2024 00:10:02 -0700 Subject: [PATCH 2/5] Fix the silly bug --- .../java/org/apache/sedona/snowflake/snowsql/SnowClient.java | 4 ++-- .../java/org/apache/sedona/snowflake/snowsql/TestBase.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java index e0b792cefb..808181e3f3 100644 --- a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java +++ b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java @@ -88,7 +88,7 @@ public void uploadFile(String path, String stage) { } } - public static SnowClient newFromEnv() throws SQLException { + public static SnowClient newFromEnv(String snowflake_db_name) throws SQLException { Properties prop = new Properties(); // check auth method String authMethod = System.getenv("SNOWFLAKE_AUTH_METHOD"); @@ -96,7 +96,7 @@ public static SnowClient newFromEnv() throws SQLException { prop.put("user", System.getenv("SNOWFLAKE_USER")); prop.put("password", System.getenv("SNOWFLAKE_PASSWORD")); } - prop.put("db", System.getenv("SNOWFLAKE_DB")); + prop.put("db", snowflake_db_name); prop.put("schema", System.getenv("SNOWFLAKE_SCHEMA")); prop.put("warehouse", System.getenv("SNOWFLAKE_WAREHOUSE")); prop.put("role", System.getenv("SNOWFLAKE_ROLE")); diff --git a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java index 53c87b8793..7b2a9f474e 100644 --- a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java +++ b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java @@ -99,7 +99,7 @@ public void registerUDTF(Class clz) { } public void init() throws SQLException { - snowClient = SnowClient.newFromEnv(); + snowClient = SnowClient.newFromEnv(snowflake_db_name); String sedonaVersion = System.getenv("SEDONA_VERSION"); String geotoolsVersion = System.getenv("SNOWFLAKE_GEOTOOLS_VERSION"); // init configs From 31caa24b2609d3116cea7ec27d7b8ef49753b7c6 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 24 Apr 2024 18:25:25 -0700 Subject: [PATCH 3/5] Make sure the same database is shared among all test cases --- .../org/apache/sedona/snowflake/snowsql/SnowClient.java | 1 - .../org/apache/sedona/snowflake/snowsql/TestBase.java | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java index 808181e3f3..ac2004ef24 100644 --- a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java +++ b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java @@ -96,7 +96,6 @@ public static SnowClient newFromEnv(String snowflake_db_name) throws SQLExceptio prop.put("user", System.getenv("SNOWFLAKE_USER")); prop.put("password", System.getenv("SNOWFLAKE_PASSWORD")); } - prop.put("db", snowflake_db_name); prop.put("schema", System.getenv("SNOWFLAKE_SCHEMA")); prop.put("warehouse", System.getenv("SNOWFLAKE_WAREHOUSE")); prop.put("role", System.getenv("SNOWFLAKE_ROLE")); diff --git a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java index 7b2a9f474e..990ef2b469 100644 --- a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java +++ b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java @@ -54,7 +54,7 @@ public class TestBase extends TestCase { private static boolean jarUploaded = false; - String snowflake_db_name = generateRandomString(8); + private static String snowflake_db_name = null; public void registerUDF(String functionName, Class ... paramTypes) { try { @@ -106,9 +106,9 @@ public void init() throws SQLException { buildDDLConfigs = new HashMap<>(); buildDDLConfigs.put(Constants.SEDONA_VERSION, sedonaVersion); buildDDLConfigs.put(Constants.GEOTOOLS_VERSION, geotoolsVersion); - System.out.println("Using Snowflake DB: " + snowflake_db_name); // upload libraries if (!jarUploaded) { + snowflake_db_name = generateRandomString(8); // drop then create db to make sure test env fresh snowClient.executeQuery("drop database if exists " + snowflake_db_name); snowClient.executeQuery("create database " + snowflake_db_name); @@ -120,6 +120,11 @@ public void init() throws SQLException { snowClient.uploadFile(String.format("tmp/geotools-wrapper-%s.jar", geotoolsVersion), "ApacheSedona"); jarUploaded = true; } + else { + snowClient.executeQuery("use database " + snowflake_db_name); + snowClient.executeQuery("use schema " + System.getenv("SNOWFLAKE_SCHEMA")); + } + System.out.println("Using Snowflake DB: " + snowflake_db_name); registerDependantUDFs(); } From 30701722ccc6bcd0c5b4a020e0b22e41aa199459 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 24 Apr 2024 19:32:41 -0700 Subject: [PATCH 4/5] Fix --- .../org/apache/sedona/snowflake/snowsql/TestBase.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java index 990ef2b469..cf6c526ceb 100644 --- a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java +++ b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java @@ -62,7 +62,6 @@ public void registerUDF(String functionName, Class ... paramTypes) { functionName, paramTypes ), buildDDLConfigs, "@ApacheSedona", false, ""); - System.out.println(ddl); ResultSet res = snowClient.executeQuery(ddl); res.next(); assert res.getString(1).contains("successfully created"); @@ -77,7 +76,6 @@ public void registerUDFV2(String functionName, Class ... paramTypes) { functionName, paramTypes ), buildDDLConfigs, "@ApacheSedona", false, ""); - System.out.println(ddl); ResultSet res = snowClient.executeQuery(ddl); res.next(); assert res.getString(1).contains("successfully created"); @@ -89,7 +87,6 @@ public void registerUDFV2(String functionName, Class ... paramTypes) { public void registerUDTF(Class clz) { try { String ddl = UDTFDDLGenerator.buildUDTFDDL(clz, buildDDLConfigs, "@ApacheSedona", false, ""); - System.out.println(ddl); ResultSet res = snowClient.executeQuery(ddl); res.next(); assert res.getString(1).contains("successfully created"); @@ -108,7 +105,8 @@ public void init() throws SQLException { buildDDLConfigs.put(Constants.GEOTOOLS_VERSION, geotoolsVersion); // upload libraries if (!jarUploaded) { - snowflake_db_name = generateRandomString(8); + snowflake_db_name = "TMP_TESTDB_" + generateRandomString(8); + System.out.println("Creating Snowflake DB: " + snowflake_db_name); // drop then create db to make sure test env fresh snowClient.executeQuery("drop database if exists " + snowflake_db_name); snowClient.executeQuery("create database " + snowflake_db_name); @@ -121,10 +119,10 @@ public void init() throws SQLException { jarUploaded = true; } else { + System.out.println("Using Snowflake DB: " + snowflake_db_name); snowClient.executeQuery("use database " + snowflake_db_name); snowClient.executeQuery("use schema " + System.getenv("SNOWFLAKE_SCHEMA")); } - System.out.println("Using Snowflake DB: " + snowflake_db_name); registerDependantUDFs(); } @@ -133,6 +131,7 @@ public void tearDown() try { System.out.println("Dropping Snowflake DB: " + snowflake_db_name); snowClient.executeQuery("drop database if exists " + snowflake_db_name); + jarUploaded = false; } catch (SQLException e) { throw new RuntimeException(e); From 2b10b6aff23848039880d5ef33b8fdc44f787249 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Wed, 24 Apr 2024 20:21:58 -0700 Subject: [PATCH 5/5] Fix again --- .../java/org/apache/sedona/snowflake/snowsql/SnowClient.java | 2 +- .../test/java/org/apache/sedona/snowflake/snowsql/TestBase.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java index ac2004ef24..df5f6a7f31 100644 --- a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java +++ b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/SnowClient.java @@ -88,7 +88,7 @@ public void uploadFile(String path, String stage) { } } - public static SnowClient newFromEnv(String snowflake_db_name) throws SQLException { + public static SnowClient newFromEnv() throws SQLException { Properties prop = new Properties(); // check auth method String authMethod = System.getenv("SNOWFLAKE_AUTH_METHOD"); diff --git a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java index 053b238b36..5c830c4d63 100644 --- a/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java +++ b/snowflake-tester/src/test/java/org/apache/sedona/snowflake/snowsql/TestBase.java @@ -96,7 +96,7 @@ public void registerUDTF(Class clz) { } public void init() throws SQLException { - snowClient = SnowClient.newFromEnv(snowflake_db_name); + snowClient = SnowClient.newFromEnv(); String sedonaVersion = System.getenv("SEDONA_VERSION"); String geotoolsVersion = System.getenv("SNOWFLAKE_GEOTOOLS_VERSION"); // init configs