From b737ef57760b47b6b8b15fedc1675f319f260fab Mon Sep 17 00:00:00 2001 From: Akshara Uke Date: Fri, 26 Jul 2024 10:24:32 +0000 Subject: [PATCH 1/5] source writer unit test --- .../templates/transforms/SourceWriterFn.java | 15 + .../transforms/SourceWriterFnTest.java | 204 ++++ .../test/resources/sourceWriterUTSession.json | 995 ++++++++++++++++++ 3 files changed, 1214 insertions(+) create mode 100644 v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java create mode 100644 v2/spanner-to-sourcedb/src/test/resources/sourceWriterUTSession.json diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index 7c177e090b..495f61937a 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -83,6 +83,21 @@ public SourceWriterFn( this.shadowTablePrefix = shadowTablePrefix; } + // for unit testing purposes + public void setSpannerDao(SpannerDao spannerDao) { + this.spannerDao = spannerDao; + } + + // for unit testing purposes + public void setMySqlDaoMap(Map mySqlDaoMap) { + this.mySqlDaoMap = mySqlDaoMap; + } + + // for unit testing purposes + public void setObjectMapper(ObjectMapper mapper) { + this.mapper = mapper; + } + /** Setup function connects to Cloud Spanner. */ @Setup public void setup() { diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java new file mode 100644 index 0000000000..7903398fc8 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.transforms; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.cloud.Timestamp; +import com.google.cloud.teleport.v2.spanner.ddl.Ddl; +import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; +import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; +import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; +import com.google.cloud.teleport.v2.templates.utils.MySqlDao; +import com.google.cloud.teleport.v2.templates.utils.SpannerDao; +import com.google.common.collect.ImmutableList; +import java.util.HashMap; +import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; +import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.Mod; +import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.ModType; +import org.apache.beam.sdk.testing.TestPipeline; +import org.apache.beam.sdk.transforms.DoFn; +import org.apache.beam.sdk.values.KV; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runners.MethodSorters; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class SourceWriterFnTest { + @Rule public final transient TestPipeline pipeline = TestPipeline.create(); + @Rule public final MockitoRule mocktio = MockitoJUnit.rule(); + @Mock private MySqlDao mockMySqlDao; + @Mock private SpannerDao mockSpannerDao; + @Mock HashMap mockMySqlDaoMap; + @Mock private SpannerConfig mockSpannerConfig; + @Mock private DoFn.ProcessContext processContext; + + private Shard testShard; + private Schema testSchema; + private Ddl testDdl; + private String testSourceDbTimezoneOffset; + + @Before + public void doBeforeEachTest() throws Exception { + when(mockMySqlDaoMap.get(any())).thenReturn(mockMySqlDao); + when(mockSpannerDao.getProcessedCommitTimestamp(eq("shadow_parent1"), any())).thenReturn(null); + when(mockSpannerDao.getProcessedCommitTimestamp(eq("shadow_child11"), any())) + .thenReturn(Timestamp.parseTimestamp("2025-02-02T00:00:00Z")); + doNothing().when(mockSpannerDao).updateProcessedCommitTimestamp(any()); + doNothing().when(mockMySqlDao).write(any()); + testShard = new Shard(); + testShard.setLogicalShardId("shardA"); + testShard.setUser("test"); + testShard.setHost("test"); + testShard.setPassword("test"); + testShard.setPort("1234"); + testShard.setDbName("test"); + + testSchema = SessionFileReader.read("src/test/resources/sourceWriterUTSession.json"); + testSourceDbTimezoneOffset = "+00:00"; + testDdl = getTestDdl(); + } + + @Test + public void testSourceIsAhead() throws Exception { + TrimmedShardedDataChangeRecord record = getChild11TrimmedDataChangeRecord("shardA"); + record.setShard("shardA"); + when(processContext.element()).thenReturn(KV.of(1L, record)); + SourceWriterFn sourceWriterFn = + new SourceWriterFn( + ImmutableList.of(testShard), + testSchema, + mockSpannerConfig, + testSourceDbTimezoneOffset, + testDdl, + "shadow_"); + ObjectMapper mapper = new ObjectMapper(); + mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + sourceWriterFn.setObjectMapper(mapper); + sourceWriterFn.setSpannerDao(mockSpannerDao); + sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.processElement(processContext); + verify(mockSpannerDao, atLeast(1)).getProcessedCommitTimestamp(any(), any()); + verify(mockMySqlDao, never()).write(any()); + verify(mockSpannerDao, never()).updateProcessedCommitTimestamp(any()); + } + + @Test + public void testSourceIsBehind() throws Exception { + TrimmedShardedDataChangeRecord record = getParent1TrimmedDataChangeRecord("shardA"); + record.setShard("shardA"); + when(processContext.element()).thenReturn(KV.of(1L, record)); + SourceWriterFn sourceWriterFn = + new SourceWriterFn( + ImmutableList.of(testShard), + testSchema, + mockSpannerConfig, + testSourceDbTimezoneOffset, + testDdl, + "shadow_"); + ObjectMapper mapper = new ObjectMapper(); + mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); + sourceWriterFn.setObjectMapper(mapper); + sourceWriterFn.setSpannerDao(mockSpannerDao); + sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.processElement(processContext); + verify(mockSpannerDao, atLeast(1)).getProcessedCommitTimestamp(any(), any()); + verify(mockMySqlDao, atLeast(1)).write(any()); + verify(mockSpannerDao, atLeast(1)).updateProcessedCommitTimestamp(any()); + } + + static Ddl getTestDdl() { + Ddl ddl = + Ddl.builder() + .createTable("parent1") + .column("id") + .int64() + .endColumn() + .column("update_ts") + .timestamp() + .endColumn() + .column("in_ts") + .timestamp() + .endColumn() + .column("migration_shard_id") + .string() + .max() + .endColumn() + // .primaryKeys(ImmutableList.of(IndexColumn.create("id", IndexColumn.Order.ASC))) + .endTable() + .createTable("child11") + .column("child_id") + .int64() + .endColumn() + .column("parent_id") + .int64() + .endColumn() + .column("update_ts") + .timestamp() + .endColumn() + .column("in_ts") + .timestamp() + .endColumn() + .column("migration_shard_id") + .string() + .max() + .endColumn() + .endTable() + .build(); + return ddl; + } + + private TrimmedShardedDataChangeRecord getChild11TrimmedDataChangeRecord(String shardId) { + return new TrimmedShardedDataChangeRecord( + Timestamp.parseTimestamp("2024-12-01T10:15:30.000Z"), + "serverTxnId", + "recordSeq", + "child11", + new Mod( + "{\"child_id\": \"42\" , \"parent_id\": \"42\"}", + "{}", + "{ \"migration_shard_id\": \"" + shardId + "\"}"), + ModType.valueOf("INSERT"), + 1, + ""); + } + + private TrimmedShardedDataChangeRecord getParent1TrimmedDataChangeRecord(String shardId) { + return new TrimmedShardedDataChangeRecord( + Timestamp.parseTimestamp("2020-12-01T10:15:30.000Z"), + "serverTxnId", + "recordSeq", + "parent1", + new Mod("{\"id\": \"42\"}", "{}", "{ \"migration_shard_id\": \"" + shardId + "\"}"), + ModType.valueOf("INSERT"), + 1, + ""); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/resources/sourceWriterUTSession.json b/v2/spanner-to-sourcedb/src/test/resources/sourceWriterUTSession.json new file mode 100644 index 0000000000..f9a22c0a1d --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/resources/sourceWriterUTSession.json @@ -0,0 +1,995 @@ +{ + "SpSchema": { + "t1": { + "Name": "child21", + "ColIds": [ + "c6", + "c7", + "c8", + "c9", + "c23" + ], + "ShardIdColumn": "c23", + "ColDefs": { + "c23": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c23", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c6": { + "Name": "child_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: child_id int(10)", + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c7": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: parent_id int(10)", + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c8": { + "Name": "update_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: update_ts timestamp", + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c9": { + "Name": "in_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: in_ts timestamp", + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c6", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c7", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [], + "Indexes": [ + { + "Name": "par_ind_5", + "TableId": "t1", + "Unique": false, + "Keys": [ + { + "ColId": "c7", + "Desc": false, + "Order": 1 + } + ], + "Id": "i11", + "StoredColumnIds": null + } + ], + "ParentId": "", + "Comment": "Spanner schema for source table child21", + "Id": "t1" + }, + "t2": { + "Name": "parent1", + "ColIds": [ + "c16", + "c17", + "c18", + "c24" + ], + "ShardIdColumn": "c24", + "ColDefs": { + "c16": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c17": { + "Name": "update_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: update_ts timestamp", + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c18": { + "Name": "in_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: in_ts timestamp", + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c24": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c24", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table parent1", + "Id": "t2" + }, + "t3": { + "Name": "child11", + "ColIds": [ + "c12", + "c13", + "c14", + "c15", + "c26" + ], + "ShardIdColumn": "c26", + "ColDefs": { + "c12": { + "Name": "child_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: child_id int(10)", + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "parent_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: parent_id int(10)", + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c14": { + "Name": "update_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: update_ts timestamp", + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15": { + "Name": "in_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: in_ts timestamp", + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c26": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c26", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c12", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "child11_ibfk_1", + "ColIds": [ + "c26", + "c13" + ], + "ReferTableId": "t2", + "ReferColumnIds": [ + "c24", + "c16" + ], + "Id": "f10" + } + ], + "Indexes": [ + { + "Name": "par_ind", + "TableId": "t3", + "Unique": false, + "Keys": [ + { + "ColId": "c13", + "Desc": false, + "Order": 1 + } + ], + "Id": "i22", + "StoredColumnIds": null + } + ], + "ParentId": "", + "Comment": "Spanner schema for source table child11", + "Id": "t3" + }, + "t4": { + "Name": "parent2", + "ColIds": [ + "c19", + "c20", + "c21", + "c25" + ], + "ShardIdColumn": "c25", + "ColDefs": { + "c19": { + "Name": "id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: id int(10)", + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c20": { + "Name": "update_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: update_ts timestamp", + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c21": { + "Name": "in_ts", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: in_ts timestamp", + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c25": { + "Name": "migration_shard_id", + "T": { + "Name": "STRING", + "Len": 50, + "IsArray": false + }, + "NotNull": false, + "Comment": "", + "Id": "c25", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c19", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table parent2", + "Id": "t4" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t1": { + "Name": "child21", + "Schema": "itfk", + "ColIds": [ + "c6", + "c7", + "c8", + "c9" + ], + "ColDefs": { + "c6": { + "Name": "child_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c6", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c7": { + "Name": "parent_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c7", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c8": { + "Name": "update_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c8", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c9": { + "Name": "in_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": true, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c9", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c6", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "child21_ibfk_1", + "ColIds": [ + "c7" + ], + "ReferTableId": "t4", + "ReferColumnIds": [ + "c19" + ], + "OnDelete": "", + "OnUpdate": "", + "Id": "f5" + } + ], + "Indexes": [ + { + "Name": "par_ind", + "Unique": false, + "Keys": [ + { + "ColId": "c7", + "Desc": false, + "Order": 1 + } + ], + "Id": "i11", + "StoredColumnIds": null + } + ], + "Id": "t1" + }, + "t2": { + "Name": "parent1", + "Schema": "itfk", + "ColIds": [ + "c16", + "c17", + "c18" + ], + "ColDefs": { + "c16": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c16", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c17": { + "Name": "update_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c17", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c18": { + "Name": "in_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": true, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c18", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c16", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t2" + }, + "t3": { + "Name": "child11", + "Schema": "itfk", + "ColIds": [ + "c12", + "c13", + "c14", + "c15" + ], + "ColDefs": { + "c12": { + "Name": "child_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c12", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c13": { + "Name": "parent_id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c13", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c14": { + "Name": "update_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c14", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c15": { + "Name": "in_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": true, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c15", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c12", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": [ + { + "Name": "child11_ibfk_1", + "ColIds": [ + "c13" + ], + "ReferTableId": "t2", + "ReferColumnIds": [ + "c16" + ], + "OnDelete": "", + "OnUpdate": "", + "Id": "f10" + } + ], + "Indexes": [ + { + "Name": "par_ind", + "Unique": false, + "Keys": [ + { + "ColId": "c13", + "Desc": false, + "Order": 1 + } + ], + "Id": "i22", + "StoredColumnIds": null + } + ], + "Id": "t3" + }, + "t4": { + "Name": "parent2", + "Schema": "itfk", + "ColIds": [ + "c19", + "c20", + "c21" + ], + "ColDefs": { + "c19": { + "Name": "id", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c19", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c20": { + "Name": "update_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c20", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + }, + "c21": { + "Name": "in_ts", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": true, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c21", + "AutoGen": { + "Name": "", + "GenerationType": "" + } + } + }, + "PrimaryKeys": [ + { + "ColId": "c19", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t4" + } + }, + "SchemaIssues": { + "t1": { + "ColumnLevelIssues": { + "c23": [ + 29, + 30 + ], + "c6": [ + 14 + ], + "c7": [ + 14, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19, + 19 + ], + "c8": [], + "c9": [ + 0 + ] + }, + "TableLevelIssues": null + }, + "t2": { + "ColumnLevelIssues": { + "c16": [ + 14 + ], + "c17": [], + "c18": [ + 0 + ], + "c24": [ + 29, + 30 + ] + }, + "TableLevelIssues": null + }, + "t3": { + "ColumnLevelIssues": { + "c12": [ + 14 + ], + "c13": [ + 14, + 24 + ], + "c14": [], + "c15": [ + 0 + ], + "c26": [ + 29, + 30 + ] + }, + "TableLevelIssues": null + }, + "t4": { + "ColumnLevelIssues": { + "c19": [ + 14 + ], + "c20": [], + "c21": [ + 0 + ], + "c25": [ + 29, + 30 + ] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [ + { + "Id": "r27", + "Name": "r27", + "Type": "add_shard_id_primary_key", + "ObjectType": "", + "AssociatedObjects": "All Tables", + "Enabled": true, + "Data": { + "AddedAtTheStart": true + }, + "AddedOn": { + "TimeOffset": null + } + } + ], + "IsSharded": true, + "SpRegion": "", + "ResourceValidation": false, + "UI": true, + "SpSequences": {}, + "SrcSequences": {} + } \ No newline at end of file From ed9ff6d90b514b964d1559d99cd728920e518dc0 Mon Sep 17 00:00:00 2001 From: Akshara Uke Date: Fri, 26 Jul 2024 11:41:37 +0000 Subject: [PATCH 2/5] added git configs for new template --- .github/CODEOWNERS | 2 ++ .github/codecov.yml | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 481cbb8f83..8ede2e4f3d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -19,6 +19,8 @@ v2/spanner-change-streams-to-sharded-file-sink/ @GoogleCloudPlatform/spanner-mig v2/gcs-to-sourcedb/ @GoogleCloudPlatform/spanner-migrations-team v2/spanner-migrations-sdk/ @GoogleCloudPlatform/spanner-migrations-team v2/spanner-custom-shard/ @GoogleCloudPlatform/spanner-migrations-team +v2/spanner-to-sourcedb/ @GoogleCloudPlatform/spanner-migrations-team + # Spanner Bulk migration template v2/sourcedb-to-spanner/ @GoogleCloudPlatform/spanner-migrations-team diff --git a/.github/codecov.yml b/.github/codecov.yml index 1b926bfb99..2a08bcbaac 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -26,6 +26,7 @@ component_management: - "v2/spanner-migrations-sdk/**" - "v2/spanner-custom-shard/**" - "v2/sourcedb-to-spanner/**" + - "v2/spanner-to-sourcedb/**" statuses: - type: project informational: true @@ -42,11 +43,11 @@ component_management: paths: - "v2/datastream-to-spanner/**" - "v2/spanner-common/**" + - "v2/spanner-migrations-sdk/**" - component_id: spanner-live-reverse-replication name: spanner-live-reverse-replication paths: - - "v2/spanner-change-streams-to-sharded-file-sink/**" - - "v2/gcs-to-sourcedb/**" + - "v2/spanner-to-sourcedb/**" - "v2/spanner-migrations-sdk/**" - "v2/spanner-custom-shard/**" - "v2/spanner-common/**" From 794ef30ed51446a8b1254913e1c3f330d3299945 Mon Sep 17 00:00:00 2001 From: Akshara Uke Date: Fri, 26 Jul 2024 11:46:36 +0000 Subject: [PATCH 3/5] added spanner pr workflow --- .github/workflows/spanner-pr.yml | 1 + cicd/internal/flags/common-flags.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spanner-pr.yml b/.github/workflows/spanner-pr.yml index feb5182d0f..8bc34916a9 100644 --- a/.github/workflows/spanner-pr.yml +++ b/.github/workflows/spanner-pr.yml @@ -30,6 +30,7 @@ on: - 'v2/spanner-migrations-sdk/**' - 'v2/spanner-custom-shard/**' - 'v2/sourcedb-to-spanner/**' + - 'v2/spanner-to-sourcedb/**' schedule: - cron: "0 */12 * * *" workflow_dispatch: diff --git a/cicd/internal/flags/common-flags.go b/cicd/internal/flags/common-flags.go index ce69275a38..01362ca5d8 100644 --- a/cicd/internal/flags/common-flags.go +++ b/cicd/internal/flags/common-flags.go @@ -29,7 +29,7 @@ const ( // Avoid making these vars public. var ( modulesToBuild string - moduleMap = map[string]string{ALL: "", SPANNER: "v2/datastream-to-spanner/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/sourcedb-to-spanner/,v2/spanner-custom-shard,plugins/templates-maven-plugin"} + moduleMap = map[string]string{ALL: "", SPANNER: "v2/datastream-to-spanner/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/sourcedb-to-spanner/,v2/spanner-to-sourcedb,v2/spanner-custom-shard,plugins/templates-maven-plugin"} ) // Registers all common flags. Must be called before flag.Parse(). From dc45bb5d9b5a2b83f603c2d896d8a03dfa6116ed Mon Sep 17 00:00:00 2001 From: Akshara Uke Date: Fri, 26 Jul 2024 11:52:37 +0000 Subject: [PATCH 4/5] added spanner pr workflow --- cicd/internal/flags/common-flags_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cicd/internal/flags/common-flags_test.go b/cicd/internal/flags/common-flags_test.go index 63b7aaec40..444b258a98 100644 --- a/cicd/internal/flags/common-flags_test.go +++ b/cicd/internal/flags/common-flags_test.go @@ -40,7 +40,7 @@ func TestModulesToBuild(t *testing.T) { }, { input: "SPANNER", - expected: []string{"v2/datastream-to-spanner/", "v2/spanner-change-streams-to-sharded-file-sink/", "v2/gcs-to-sourcedb/", "v2/sourcedb-to-spanner/", "v2/spanner-custom-shard", "plugins/templates-maven-plugin"}, + expected: []string{"v2/datastream-to-spanner/", "v2/spanner-change-streams-to-sharded-file-sink/", "v2/gcs-to-sourcedb/", "v2/sourcedb-to-spanner/", "v2/spanner-to-sourcedb/", "v2/spanner-custom-shard", "plugins/templates-maven-plugin"}, }, } From 2608dc5a306af9d17d63323d0d17bb8e2a722b52 Mon Sep 17 00:00:00 2001 From: Akshara Uke Date: Fri, 26 Jul 2024 11:54:35 +0000 Subject: [PATCH 5/5] added spanner pr workflow --- cicd/internal/flags/common-flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cicd/internal/flags/common-flags.go b/cicd/internal/flags/common-flags.go index 01362ca5d8..56a6f2568e 100644 --- a/cicd/internal/flags/common-flags.go +++ b/cicd/internal/flags/common-flags.go @@ -29,7 +29,7 @@ const ( // Avoid making these vars public. var ( modulesToBuild string - moduleMap = map[string]string{ALL: "", SPANNER: "v2/datastream-to-spanner/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/sourcedb-to-spanner/,v2/spanner-to-sourcedb,v2/spanner-custom-shard,plugins/templates-maven-plugin"} + moduleMap = map[string]string{ALL: "", SPANNER: "v2/datastream-to-spanner/,v2/spanner-change-streams-to-sharded-file-sink/,v2/gcs-to-sourcedb/,v2/sourcedb-to-spanner/,v2/spanner-to-sourcedb/,v2/spanner-custom-shard,plugins/templates-maven-plugin"} ) // Registers all common flags. Must be called before flag.Parse().