From 7e784de6cdd6f04a5dd4168e5b613b807b71651c Mon Sep 17 00:00:00 2001 From: Dylan Hercher Date: Fri, 2 Aug 2024 23:33:35 -0700 Subject: [PATCH] DatastreamToSQL - log schema map details (#1747) * adding logging to test schemmap * open schemmap prtection * Handling for String Primary Keys in Bulk Reader. (#1743) Limitation: For the current PR complete support only upto 3 byte code points. * Add support for ALO in SpannerChangeStreamsToBigQuery template (#1750) * Fixing exception in String .isSplittable (#1755) * Remove PIP_NO_DEPS from YamlTemplate Dockerfile (#1748) Signed-off-by: Jeffrey Kinard * Terraform template updates for custom transformation (#1746) * terraform changes for end-to-end template * terraform updates * formatting fix * Add string mapper case for IT (#1757) * Fixing Index Discovery for 5.7 and removing innodb_parallel_read_threads from init sequences for MySql5.7 compliance. (#1758) * adding schema map to dml handler * removing logs for testing * ut for reverse replication shadow tables (#1759) * ut for reverse replication shadow tables * incorporated review comments * unit test for source writer (#1749) * source writer unit test * added git configs for new template * added spanner pr workflow * added spanner pr workflow * added spanner pr workflow * Adding autoReconnect parameters to URL (#1760) * map should not be static * merging * adding logging to test schemmap * open schemmap prtection * adding schema map to dml handler * removing logs for testing * map should not be static * linebreaks --------- Signed-off-by: Jeffrey Kinard Co-authored-by: Vardhan Vinay Thigle <39047439+VardhanThigle@users.noreply.github.com> Co-authored-by: Dip Patel <37777672+Dippatel98@users.noreply.github.com> Co-authored-by: Jeff Kinard Co-authored-by: shreyakhajanchi <92910380+shreyakhajanchi@users.noreply.github.com> Co-authored-by: Deep1998 Co-authored-by: aksharauke <126752897+aksharauke@users.noreply.github.com> --- .../google/cloud/teleport/v2/templates/DataStreamToSQL.java | 1 + .../com/google/cloud/teleport/v2/transforms/CreateDml.java | 6 ++++-- .../com/google/cloud/teleport/v2/utils/DatastreamToDML.java | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSQL.java b/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSQL.java index 9199a31ff9..2e7c029871 100644 --- a/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSQL.java +++ b/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/templates/DataStreamToSQL.java @@ -356,6 +356,7 @@ public static PipelineResult run(Options options) { CdcJdbcIO.DataSourceConfiguration dataSourceConfiguration = getDataSourceConfiguration(options); validateOptions(options, dataSourceConfiguration); Map schemaMap = parseSchemaMap(options.getSchemaMap()); + LOG.info("Parsed schema map: {}", schemaMap); /* * Stage 1: Ingest and Normalize Data to FailsafeElement with JSON Strings diff --git a/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/transforms/CreateDml.java b/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/transforms/CreateDml.java index 8a5a8965a3..7c01d528be 100644 --- a/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/transforms/CreateDml.java +++ b/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/transforms/CreateDml.java @@ -63,10 +63,12 @@ public DatastreamToDML getDatastreamToDML() { String driverName = this.dataSourceConfiguration.getDriverClassName().get(); switch (driverName) { case "org.postgresql.Driver": - datastreamToDML = DatastreamToPostgresDML.of(dataSourceConfiguration); + datastreamToDML = + DatastreamToPostgresDML.of(dataSourceConfiguration).withSchemaMap(this.schemaMap); break; case "com.mysql.cj.jdbc.Driver": - datastreamToDML = DatastreamToMySQLDML.of(dataSourceConfiguration); + datastreamToDML = + DatastreamToMySQLDML.of(dataSourceConfiguration).withSchemaMap(this.schemaMap); break; default: throw new IllegalArgumentException( diff --git a/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/utils/DatastreamToDML.java b/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/utils/DatastreamToDML.java index dd39be5779..6a208e3579 100644 --- a/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/utils/DatastreamToDML.java +++ b/v2/datastream-to-sql/src/main/java/com/google/cloud/teleport/v2/utils/DatastreamToDML.java @@ -55,7 +55,7 @@ public abstract class DatastreamToDML private CdcJdbcIO.DataSourceConfiguration dataSourceConfiguration; private DataSource dataSource; public String quoteCharacter; - private static Map schemaMap = new HashMap(); + protected Map schemaMap = new HashMap(); public abstract String getDefaultQuoteCharacter();