Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: BigQuery_To_JDBC #1000

Merged
merged 21 commits into from
Oct 26, 2024
+326 −1
Merged

feat: BigQuery_To_JDBC #1000

Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
1daa4a5
addition for BigQuery to JDBC
hhasija Oct 19, 2024
4c997b0
Bogquery to JDBC Java file
hhasija Oct 19, 2024
df6ffc1
Merge branch 'main' of https://github.com/GoogleCloudPlatform/datapro…
hhasija Oct 21, 2024
5ef89e2
Merge branch 'main' into BigQuery-To-JDBC
rajc242 Oct 23, 2024
ec3d642
Merge branch 'main' of https://github.com/GoogleCloudPlatform/datapro…
hhasija Oct 23, 2024
d6ed2ff
fix: enabling two properties for performance optimization
hhasija Oct 23, 2024
385c7a8
fix: adding sparkconf variable and defining properties for optimization
hhasija Oct 23, 2024
2c0921a
fix: enhancing read me file for better understanding
hhasija Oct 23, 2024
6b469ac
Merge remote-tracking branch 'origin/BigQuery-To-JDBC' into BigQuery-…
hhasija Oct 23, 2024
fcf0b0e
fix: providing validation through a separate class BigQueryToJDBCConfig
hhasija Oct 23, 2024
4b00f14
fix: added a new entry in main Read me file
hhasija Oct 23, 2024
3bc6e5d
fix: taking dataset name via parameter
hhasija Oct 24, 2024
6092e32
Merge branch 'main' of https://github.com/GoogleCloudPlatform/datapro…
hhasija Oct 24, 2024
386d6cc
Merge branch 'main' of https://github.com/GoogleCloudPlatform/datapro…
hhasija Oct 25, 2024
503ea96
fix: adding test cases towards jenkins file for integration
hhasija Oct 25, 2024
bc8afe8
chnaging the databasename to avoid any conflicts
hhasija Oct 25, 2024
a87d4c5
correcting the primary key as the database got overwrite in previous run
hhasija Oct 25, 2024
4ab6421
fix: removing not required test cases for performance
hhasija Oct 25, 2024
8c1384b
fix: reverting back all the changes
hhasija Oct 26, 2024
dabad3e
fix: as it is copying files to remove all space and tab related issues
hhasija Oct 26, 2024
5845ea0
fix: adding one integration test for new template BIGQUEYTOJDBC
hhasija Oct 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Loading
Loading status checks…
fix: reverting back all the changes
@hhasija
hhasija committed Oct 26, 2024
commit 8c1384b805eeb06b9fe757acc791613e23795c0d
132 changes: 66 additions & 66 deletions java/.ci/Jenkinsfile
View file
Original file line number Diff line number Diff line change
@@ -457,17 +457,42 @@ EOF
}
stage('Parallel Execution 4'){
parallel{
stage('JDBC TO SPANNER GOOGLESQL DIALECT'){
steps {
retry(count: stageRetryCount) {
sh '''

export JARS="gs://dataproc-templates/jars/mysql-connector-java.jar"
export SKIP_BUILD=true

cd java

bin/start.sh \
-- --template JDBCTOSPANNER \
--templateProperty project.id=$GCP_PROJECT \
--templateProperty jdbctospanner.jdbc.url="$TEST_JDBC_URL" \
--templateProperty jdbctospanner.jdbc.driver.class.name=com.mysql.jdbc.Driver \
--templateProperty jdbctospanner.sql="select * from test.employee" \
--templateProperty jdbctospanner.output.instance=$ENV_TEST_SPANNER_ID \
--templateProperty jdbctospanner.output.database=spark-ci-db \
--templateProperty jdbctospanner.output.table=employee \
--templateProperty jdbctospanner.output.saveMode=Overwrite \
--templateProperty jdbctospanner.output.primaryKey='empno'
'''
}
}
}
stage('CASSANDRA TO GCS'){
steps {
retry(count: stageRetryCount) {
sh '''

export SKIP_BUILD=true



cd java

bin/start.sh \
-- --template CASSANDRATOGCS \
--templateProperty project.id=$GCP_PROJECT \
@@ -485,13 +510,13 @@ EOF
steps {
retry(count: stageRetryCount) {
sh '''

export SKIP_BUILD=true



cd java

bin/start.sh \
-- --template CASSANDRATOBQ \
--templateProperty project.id=$GCP_PROJECT \
@@ -509,13 +534,13 @@ EOF
steps {
retry(count: stageRetryCount) {
sh '''

export SKIP_BUILD=true



cd java

bin/start.sh \
-- --template DATAPLEXGCSTOBQ \
--templateProperty=project.id=$GCP_PROJECT \
@@ -538,8 +563,8 @@ EOF
sh '''
export SKIP_BUILD=true



cd java

bin/start.sh \
@@ -561,13 +586,13 @@ EOF
steps {
retry(count: stageRetryCount) {
sh '''

export SKIP_BUILD=true



cd java

bin/start.sh \
-- --template GCSTOGCS \
--templateProperty project.id=$GCP_PROJECT \
@@ -622,10 +647,10 @@ EOF
stage('GCS TO BIGTABLE'){
steps {
retry(count: stageRetryCount) {
sh '''
sh '''
export SKIP_BUILD=true



cd java

@@ -647,8 +672,8 @@ EOF
sh '''
export SKIP_BUILD=true



cd java

bin/start.sh \
@@ -670,8 +695,8 @@ EOF
sh '''
export SKIP_BUILD=true



cd java

bin/start.sh \
@@ -693,10 +718,10 @@ EOF
sh '''
export SKIP_BUILD=true



cd java

bin/start.sh \
-- --template GCSTOBIGQUERY \
--templateProperty project.id=$GCP_PROJECT \
@@ -719,8 +744,8 @@ EOF
sh '''
export SKIP_BUILD=true



cd java

bin/start.sh \
@@ -742,8 +767,8 @@ EOF
sh '''
export SKIP_BUILD=true



cd java

bin/start.sh \
@@ -766,8 +791,8 @@ EOF
sh '''
export SKIP_BUILD=true



cd java

bin/start.sh \
@@ -792,8 +817,8 @@ EOF
sh '''
export SKIP_BUILD=true



cd java

bin/start.sh \
@@ -813,7 +838,7 @@ EOF
sh '''
export JARS="gs://dataproc-templates/jars/mysql-connector-java.jar"
export SKIP_BUILD=true

cd java

bin/start.sh \
@@ -824,7 +849,7 @@ EOF
--templateProperty jdbctojdbc.output.url="$TEST_JDBC_URL" \
--templateProperty jdbctojdbc.output.driver="com.mysql.cj.jdbc.Driver" \
--templateProperty jdbctojdbc.output.table="employee_output" \
--templateProperty jdbctojdbc.output.mode="Overwrite"
--templateProperty jdbctojdbc.output.mode="Overwrite"
'''
}
}
@@ -888,7 +913,7 @@ EOF
sh '''

export SKIP_BUILD=true

cd java
bin/start.sh \
-- \
@@ -914,7 +939,7 @@ EOF
sh '''

export SKIP_BUILD=true

cd java
bin/start.sh \
-- \
@@ -940,7 +965,7 @@ EOF
sh '''

export SKIP_BUILD=true

cd java
bin/start.sh \
-- \
@@ -996,7 +1021,7 @@ EOF

gcloud spanner databases execute-sql pgsqltest --instance=$ENV_TEST_SPANNER_ID --sql='DELETE FROM badges;'
export SKIP_BUILD=true


cd java

@@ -1015,31 +1040,6 @@ EOF
}
}
}
stage('BIGQUERY TO JDBC') {
steps{
retry(count: stageRetryCount) {
sh '''
export JARS="gs://dataproc-templates/jars/mysql-connector-java.jar"
export SKIP_BUILD=true

cd java

bin/start.sh \
-- --template BIGQUERYTOJDBC \
--templateProperty project.id=$GCP_PROJECT \
--templateProperty bigquery.jdbc.input.table=$GCP_PROJECT:dataproc_templates.emp_table \
--templateProperty bigquery.jdbc.dataset.name='dataproc_templates' \
--templateProperty bigquery.jdbc.output.table='bq_to_jdbc_employee' \
--templateProperty bigquery.jdbc.url=$TEST_JDBC_URL \
--templateProperty bigquery.jdbc.batch.size=10 \
--templateProperty bigquery.jdbc.output.driver='com.mysql.jdbc.Driver' \
--templateProperty bigquery.jdbc.output.mode='Overwrite' \
--templateProperty bigquery.jdbc.temp.table='temporary_view_name' \
--templateProperty bigquery.jdbc.temp.query='select * from global_temp.temporary_view_name where emp_id>=5'
'''
}
}
}
stage('JDBC TO SPANNER POSTGRESQL DIALECT'){
steps {
retry(count: stageRetryCount) {
27 changes: 27 additions & 0 deletions java/.ci/UiJenkinsfile
View file
Original file line number Diff line number Diff line change
@@ -410,6 +410,33 @@ pipeline {
}
}

stage('JDBC TO SPANNER'){
steps {
retry(count: stageRetryCount) {
sh '''

gcloud dataproc batches submit spark \
--class=com.google.cloud.dataproc.templates.main.DataProcTemplate \
--version="1.1" \
--project="${GCP_PROJECT}" \
--region="us-west1" \
--jars="${jarPath},gs://dataproc-templates/jars/mysql-connector-java.jar" \
--subnet="projects/yadavaja-sandbox/regions/us-west1/subnetworks/test-subnet1" \
-- --template=JDBCTOSPANNER \
--templateProperty project.id="${GCP_PROJECT}" \
--templateProperty jdbctospanner.jdbc.url="${TEST_JDBC_URL}" \
--templateProperty jdbctospanner.jdbc.driver.class.name="com.mysql.jdbc.Driver" \
--templateProperty jdbctospanner.sql="select * from test.employee" \
--templateProperty jdbctospanner.output.instance="${ENV_TEST_SPANNER_ID}" \
--templateProperty jdbctospanner.output.database="spark-ci-db" \
--templateProperty jdbctospanner.output.table="employee" \
--templateProperty jdbctospanner.output.saveMode="Overwrite" \
--templateProperty jdbctospanner.output.primaryKey="empno"
'''
}
}
}

stage('JDBC TO GCS (csv)'){
steps {
retry(count: stageRetryCount) {