Merge branch 'zendesk:master' into master

zendesk · Jan 30, 2023 · 4304248 · 4304248
2 parents 0baa746 + 88b36ed
commit 4304248
Show file tree

Hide file tree

Showing 12 changed files with 130 additions and 82 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,22 @@
 # Maxwell changelog
 
+### [v1.39.4](https://github.com/zendesk/maxwell/releases/tag/v1.39.4)
+
+- Fix bugs with older versions of mariadb (<10.4)
+
+
+
+_Released 2022-12-07_
+
+### [v1.39.3](https://github.com/zendesk/maxwell/releases/tag/v1.39.3)
+
+- some bugfixes for 1.39.2 and google pubsub
+- couple of security upgrades, including in the docker image
+
+
+
+_Released 2022-12-04_
+
 ### [v1.39.2](https://github.com/zendesk/maxwell/releases/tag/v1.39.2)
 
 this is a bug-fix release.  some upgrades broke maxwell's http interface and there's

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 FROM maven:3.8-jdk-11 as builder
-ENV MAXWELL_VERSION=1.39.2 KAFKA_VERSION=1.0.0
+ENV MAXWELL_VERSION=1.39.4 KAFKA_VERSION=1.0.0
 
 RUN apt-get update \
     && apt-get -y upgrade \

diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@ search indexing and inter-service communication.  Maxwell gives you some of the
 benefits of event sourcing without having to re-architect your entire platform.
 
 <b>Download:</b><br>
-[https://github.com/zendesk/maxwell/releases/download/v1.39.2/maxwell-1.39.2.tar.gz](https://github.com/zendesk/maxwell/releases/download/v1.39.2/maxwell-1.39.2.tar.gz)
+[https://github.com/zendesk/maxwell/releases/download/v1.39.4/maxwell-1.39.4.tar.gz](https://github.com/zendesk/maxwell/releases/download/v1.39.4/maxwell-1.39.4.tar.gz)
 <br/>
 <b>Source:</b><br>
 [https://github.com/zendesk/maxwell](https://github.com/zendesk/maxwell)

diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md
@@ -1,13 +1,13 @@
 # Download
 ***
-- Download binary distro: [https://github.com/zendesk/maxwell/releases/download/v1.39.2/maxwell-1.39.2.tar.gz](https://github.com/zendesk/maxwell/releases/download/v1.39.2/maxwell-1.39.2.tar.gz)
+- Download binary distro: [https://github.com/zendesk/maxwell/releases/download/v1.39.4/maxwell-1.39.4.tar.gz](https://github.com/zendesk/maxwell/releases/download/v1.39.4/maxwell-1.39.4.tar.gz)
 - Sources and bug tracking is available on github: [https://github.com/zendesk/maxwell](https://github.com/zendesk/maxwell)
 
 **curl**:
 ```
-curl -sLo - https://github.com/zendesk/maxwell/releases/download/v1.39.2/maxwell-1.39.2.tar.gz \
+curl -sLo - https://github.com/zendesk/maxwell/releases/download/v1.39.4/maxwell-1.39.4.tar.gz \
        | tar zxvf -
-cd maxwell-1.39.2
+cd maxwell-1.39.4
 ```
 
 **docker**:

diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 
   <groupId>com.zendesk</groupId>
   <artifactId>maxwell</artifactId>
-  <version>1.39.2</version>
+  <version>1.39.4</version>
   <packaging>jar</packaging>
 
   <name>maxwell</name>
@@ -225,70 +225,6 @@
       <version>0.6.0</version>
     </dependency>
 
-    <!-- metrics -->
-    <dependency>
-      <groupId>io.dropwizard.metrics</groupId>
-      <artifactId>metrics-core</artifactId>
-      <version>4.1.17</version>
-    </dependency>
-    <dependency>
-      <groupId>io.dropwizard.metrics</groupId>
-      <artifactId>metrics-servlets</artifactId>
-      <version>4.1.17</version>
-    </dependency>
-    <dependency>
-      <groupId>io.dropwizard.metrics</groupId>
-      <artifactId>metrics-jmx</artifactId>
-      <version>4.1.17</version>
-    </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-server</artifactId>
-      <version>10.0.12</version>
-    </dependency>
-    <dependency>
-      <groupId>org.eclipse.jetty</groupId>
-      <artifactId>jetty-servlet</artifactId>
-      <version>10.0.12</version>
-    </dependency>
-    <dependency>
-      <groupId>com.viafoura</groupId>
-      <artifactId>metrics-datadog</artifactId>
-      <version>2.0.0-RC3</version>
-    </dependency>
-    <dependency>
-      <groupId>io.prometheus</groupId>
-      <artifactId>simpleclient_dropwizard</artifactId>
-      <version>0.9.0</version>
-    </dependency>
-    <dependency>
-      <groupId>io.prometheus</groupId>
-      <artifactId>simpleclient_servlet</artifactId>
-      <version>0.9.0</version>
-    </dependency>
-    <!-- OpenCensus used for pushing metrics to stackdriver -->
-    <dependency>
-        <groupId>io.opencensus</groupId>
-        <artifactId>opencensus-contrib-dropwizard</artifactId>
-        <version>${opencensus.version}</version>
-    </dependency>
-    <dependency>
-        <groupId>io.opencensus</groupId>
-        <artifactId>opencensus-api</artifactId>
-        <version>${opencensus.version}</version>
-    </dependency>
-    <dependency>
-        <groupId>io.opencensus</groupId>
-        <artifactId>opencensus-impl</artifactId>
-        <version>${opencensus.version}</version>
-        <scope>runtime</scope>
-    </dependency>
-    <dependency>
-        <groupId>io.opencensus</groupId>
-        <artifactId>opencensus-exporter-stats-stackdriver</artifactId>
-        <version>${opencensus.version}</version>
-    </dependency>
-
     <!-- producer libs -->
     <dependency>
       <groupId>com.google.cloud</groupId>
@@ -338,7 +274,7 @@
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-pubsub</artifactId>
-      <version>1.115.1</version>
+      <version>1.120.24</version>
     </dependency>
     <dependency>
       <groupId>io.nats</groupId>
@@ -355,6 +291,70 @@
       <artifactId>jedis</artifactId>
       <version>3.5.1</version>
     </dependency>
+
+    <!-- metrics -->
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>4.1.17</version>
+    </dependency>
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-servlets</artifactId>
+      <version>4.1.17</version>
+    </dependency>
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-jmx</artifactId>
+      <version>4.1.17</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-server</artifactId>
+      <version>10.0.12</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+      <version>10.0.12</version>
+    </dependency>
+    <dependency>
+      <groupId>com.viafoura</groupId>
+      <artifactId>metrics-datadog</artifactId>
+      <version>2.0.0-RC3</version>
+    </dependency>
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_dropwizard</artifactId>
+      <version>0.9.0</version>
+    </dependency>
+    <dependency>
+      <groupId>io.prometheus</groupId>
+      <artifactId>simpleclient_servlet</artifactId>
+      <version>0.9.0</version>
+    </dependency>
+    <!-- OpenCensus used for pushing metrics to stackdriver -->
+    <dependency>
+        <groupId>io.opencensus</groupId>
+        <artifactId>opencensus-contrib-dropwizard</artifactId>
+        <version>${opencensus.version}</version>
+    </dependency>
+    <dependency>
+        <groupId>io.opencensus</groupId>
+        <artifactId>opencensus-api</artifactId>
+        <version>${opencensus.version}</version>
+    </dependency>
+    <dependency>
+        <groupId>io.opencensus</groupId>
+        <artifactId>opencensus-impl</artifactId>
+        <version>${opencensus.version}</version>
+        <scope>runtime</scope>
+    </dependency>
+    <dependency>
+        <groupId>io.opencensus</groupId>
+        <artifactId>opencensus-exporter-stats-stackdriver</artifactId>
+        <version>${opencensus.version}</version>
+    </dependency>
 
     <!-- test -->
     <dependency>

diff --git a/src/main/antlr4/imports/mysql_alter_database.g4 b/src/main/antlr4/imports/mysql_alter_database.g4
@@ -5,5 +5,7 @@ import mysql_literal_tokens, mysql_idents;
 alter_database: ALTER (DATABASE | SCHEMA) name? alter_database_definition;
 alter_database_definition:
     (default_character_set | default_collation)+
-  | UPGRADE DATA DIRECTORY NAME;
+  | UPGRADE DATA DIRECTORY NAME
+  | alter_encryption;
 
+alter_encryption: ENCRYPTION '='? string_literal;
diff --git a/src/main/java/com/zendesk/maxwell/Maxwell.java b/src/main/java/com/zendesk/maxwell/Maxwell.java
@@ -353,13 +353,14 @@ public void run() {
 		} catch ( URISyntaxException e ) {
 			// catch URISyntaxException explicitly as well to provide more information to the user
 			LOGGER.error("Syntax issue with URI, check for misconfigured host, port, database, or JDBC options (see RFC 2396)");
-			LOGGER.error("URISyntaxException: " + e.getLocalizedMessage());
+			LOGGER.error("URISyntaxException: " + e.getLocalizedMessage(), e);
 			System.exit(1);
 		} catch ( ServerException e ) {
-			LOGGER.error("Maxwell couldn't find the requested binlog, exiting...");
+			LOGGER.error("Maxwell couldn't find the requested binlog, exiting...", e);
 			System.exit(2);
 		} catch ( Exception e ) {
 			e.printStackTrace();
+			LOGGER.error("Maxwell saw an exception and is exiting...", e);
 			System.exit(1);
 		}
 	}

diff --git a/src/main/java/com/zendesk/maxwell/MaxwellConfig.java b/src/main/java/com/zendesk/maxwell/MaxwellConfig.java
@@ -1110,7 +1110,7 @@ private void setup(OptionSet options, Properties properties) {
 		this.pubsubMessageOrderingKey			= fetchStringOption("pubsub_message_ordering_key", options, properties, null);
 		this.pubsubPublishDelayThreshold		= Duration.ofMillis(fetchLongOption("pubsub_publish_delay_threshold", options, properties, 1L));
 		this.pubsubRetryDelay 					= Duration.ofMillis(fetchLongOption("pubsub_retry_delay", options, properties, 100L));
-		this.pubsubRetryDelayMultiplier 		= fetchFloatOption("pubsub_retry_delay_multiplier", options, properties, 1.0f);
+		this.pubsubRetryDelayMultiplier 		= fetchFloatOption("pubsub_retry_delay_multiplier", options, properties, 1.3f);
 		this.pubsubMaxRetryDelay 		 		= Duration.ofSeconds(fetchLongOption("pubsub_max_retry_delay", options, properties, 60L));
 		this.pubsubInitialRpcTimeout 		 	= Duration.ofSeconds(fetchLongOption("pubsub_initial_rpc_timeout", options, properties, 5L));
 		this.pubsubRpcTimeoutMultiplier 		= fetchFloatOption("pubsub_rpc_timeout_multiplier", options, properties, 1.0f);
@@ -1379,6 +1379,9 @@ public void validate() {
 		} else if (this.producerType.equals("sns") && this.snsTopic == null) {
 			usageForOptions("please specify a topic ARN for SNS", "sns_topic");
 		} else if (this.producerType.equals("pubsub")) {
+			if (this.pubsubProjectId == null)
+				usageForOptions("please specify --pubsub_project_id.", "--pubsub_project_id");
+
 			if (this.pubsubRequestBytesThreshold <= 0L)
 				usage("--pubsub_request_bytes_threshold must be > 0");
 			if (this.pubsubMessageCountBatchSize <= 0L)

diff --git a/src/main/java/com/zendesk/maxwell/schema/SchemaCapturer.java b/src/main/java/com/zendesk/maxwell/schema/SchemaCapturer.java
@@ -10,6 +10,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.sql.Connection;
+import java.sql.DatabaseMetaData;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.Statement;
@@ -137,8 +138,9 @@ public Schema capture() throws SQLException {
 
 		Schema s = new Schema(databases, captureDefaultCharset(), this.sensitivity);
 		try {
-			if ( isMariaDB() )
+			if ( isMariaDB() && mariaSupportsJSON()) {
 				detectMariaDBJSON(s);
+			}
 		} catch ( InvalidSchemaError e ) {
 			e.printStackTrace();
 		}
@@ -176,17 +178,34 @@ private boolean isMySQLAtLeast56() throws SQLException {
 		if ( isMariaDB() )
 			return true;
 
-		java.sql.DatabaseMetaData meta = connection.getMetaData();
+		DatabaseMetaData meta = connection.getMetaData();
 		int major = meta.getDatabaseMajorVersion();
 		int minor = meta.getDatabaseMinorVersion();
 		return ((major == 5 && minor >= 6) || major > 5);
 	}
 
 	private boolean isMariaDB() throws SQLException {
-		java.sql.DatabaseMetaData meta = connection.getMetaData();
+		DatabaseMetaData meta = connection.getMetaData();
 		return meta.getDatabaseProductVersion().toLowerCase().contains("maria");
 	}
 
+	static final String MARIA_VERSION_REGEX = "[\\d\\.]+-(\\d+)\\.(\\d+)";
+	private boolean mariaSupportsJSON() throws SQLException {
+		DatabaseMetaData meta = connection.getMetaData();
+		String versionString = meta.getDatabaseProductVersion();
+		Pattern pattern = Pattern.compile(MARIA_VERSION_REGEX);
+		Matcher m = pattern.matcher(versionString);
+
+		if ( m.find() ) {
+			int major = Integer.parseInt(m.group(1));
+			int minor = Integer.parseInt(m.group(2));
+
+			return major >= 10 && minor > 1;
+		} else { // shrugging purple lady
+			return false;
+		}
+	}
+
 	private void captureTables(Database db, HashMap<String, Table> tables) throws SQLException {
 		columnPreparedStatement.setString(1, db.getName());
 
@@ -299,7 +318,7 @@ public void close() throws SQLException {
 	private void detectMariaDBJSON(Schema schema) throws SQLException, InvalidSchemaError {
 		String checkConstraintSQL = "SELECT CONSTRAINT_SCHEMA, TABLE_NAME, CONSTRAINT_NAME, CHECK_CLAUSE " +
 			"from INFORMATION_SCHEMA.CHECK_CONSTRAINTS " +
-			"where LEVEL='column' and CHECK_CLAUSE LIKE 'json_valid(%)'";
+			"where CHECK_CLAUSE LIKE 'json_valid(%)'";
 
 		String regex = "json_valid\\(`(.*)`\\)";
 		Pattern pattern = Pattern.compile(regex);

diff --git a/src/main/java/com/zendesk/maxwell/util/AbstractConfig.java b/src/main/java/com/zendesk/maxwell/util/AbstractConfig.java
@@ -114,7 +114,9 @@ protected Properties readPropertiesEnv(String envConfig) {
 				Properties properties = new Properties();
 				for (Map.Entry<String, Object> entry : stringMap.entrySet()) {
 					LOGGER.debug("Got env_config key: {}", entry.getKey());
-					properties.put(entry.getKey(), entry.getValue().toString());
+					if (entry.getKey() != null && entry.getValue() != null) {
+						properties.put(entry.getKey(), entry.getValue().toString());
+					}
 				}
 				return properties;
 			} catch (JsonProcessingException e) {

diff --git a/src/test/java/com/zendesk/maxwell/MaxwellConfigTest.java b/src/test/java/com/zendesk/maxwell/MaxwellConfigTest.java
@@ -12,6 +12,7 @@
 import org.junit.contrib.java.lang.system.EnvironmentVariables;
 
 import java.nio.file.Paths;
+import java.util.HashMap;
 import java.util.Map;
 
 import static org.junit.Assert.*;
@@ -88,12 +89,14 @@ public void testEnvVarConfigViaConfigFile() {
 
 	@Test
 	public void testEnvJsonConfig() throws JsonProcessingException {
-		Map<String, String> configMap = ImmutableMap.<String, String>builder()
+		Map<String, String> nonNullconfigMap = ImmutableMap.<String, String>builder()
 				.put("user", "foo")
 				.put("password", "bar")
 				.put("host", "remotehost")
 				.put("kafka.retries", "100")
 				.build();
+		HashMap<String, String> configMap = new HashMap<>(nonNullconfigMap);
+		configMap.put("ignore.me", null);
 		ObjectMapper mapper = new ObjectMapper();
 		String jsonConfig = mapper.writeValueAsString(configMap);
 		environmentVariables.set("MAXWELL_JSON", "    " + jsonConfig);

diff --git a/src/test/java/com/zendesk/maxwell/schema/ddl/DDLParserTest.java b/src/test/java/com/zendesk/maxwell/schema/ddl/DDLParserTest.java
@@ -260,7 +260,8 @@ public void testParsingSomeAlters() {
 			"ALTER TABLE t1 DROP PARTITION IF EXISTS p3", // some mariada-fu
 			"ALTER TABLE t1 DROP CONSTRAINT ck",
 			"ALTER TABLE t1 DROP CHECK ck",
-			"create table test ( i float default -1. )"
+			"create table test ( i float default -1. )",
+			"alter database d ENCRYPTION='Y'"
 		};
 
 		for ( String s : testSQL ) {