Skip to content

Commit

Permalink
Merge branch 'main' into BigQuery-To-JDBC
Browse files Browse the repository at this point in the history
  • Loading branch information
@rajc242
rajc242 authored Oct 23, 2024
2 parents df6ffc1 + 0d617bd commit 5ef89e2
Show file tree
Hide file tree
Showing 3 changed files with 200 additions and 203 deletions.
242 changes: 121 additions & 121 deletions java/.ci/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -605,41 +605,41 @@ EOF
}
}
}
stage('HBASE TO GCS (Automated)(avro)') {
when {
// Run this stage only if JOB_TYPE is not set to CLUSTER
expression { env.JOB_TYPE != "CLUSTER" }
}
steps{
retry(count: stageRetryCount) {
sh '''
gsutil cp gs://python-dataproc-templates/surjitsh/hbase-site.xml .
gcloud auth configure-docker us-docker.pkg.dev
export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}'
export IMAGE_NAME_VERSION=hbase-to-gcs:1.0.3
export HBASE_SITE_PATH=../hbase-site.xml
export IMAGE=us-docker.pkg.dev/${GCP_PROJECT}/dataproc-cicd/${IMAGE_NAME_VERSION}
export JARS="gs://deps-dataproc-template/hbase-spark-protocol-shaded-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/hbase-spark-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/protobuf-java-2.5.0.jar,gs://deps-dataproc-template/htrace-core4-4.2.0-incubating.jar"
export SKIP_IMAGE_BUILD=TRUE
export SKIP_BUILD=true
cd java
bin/start.sh \
--container-image=$IMAGE \
--properties='spark.dataproc.driverEnv.SPARK_EXTRA_CLASSPATH=/etc/hbase/conf/' \
-- --template HBASETOGCS \
--templateProperty hbasetogcs.output.fileformat=avro \
--templateProperty hbasetogcs.output.savemode=overwrite \
--templateProperty hbasetogcs.output.path=gs://dataproc-templates/integration-testing/output/HBASETOGCS_automated/avro \
--templateProperty hbasetogcs.table.catalog=$CATALOG
'''
}
}
}
// stage('HBASE TO GCS (Automated)(avro)') {
// when {
// // Run this stage only if JOB_TYPE is not set to CLUSTER
// expression { env.JOB_TYPE != "CLUSTER" }
// }
// steps{
// retry(count: stageRetryCount) {
// sh '''

// gsutil cp gs://python-dataproc-templates/surjitsh/hbase-site.xml .
// gcloud auth configure-docker us-docker.pkg.dev
// export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}'
// export IMAGE_NAME_VERSION=hbase-to-gcs:1.0.3
// export HBASE_SITE_PATH=../hbase-site.xml
// export IMAGE=us-docker.pkg.dev/${GCP_PROJECT}/dataproc-cicd/${IMAGE_NAME_VERSION}
// export JARS="gs://deps-dataproc-template/hbase-spark-protocol-shaded-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/hbase-spark-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/protobuf-java-2.5.0.jar,gs://deps-dataproc-template/htrace-core4-4.2.0-incubating.jar"

// export SKIP_IMAGE_BUILD=TRUE
// export SKIP_BUILD=true

// cd java

// bin/start.sh \
// --container-image=$IMAGE \
// --properties='spark.dataproc.driverEnv.SPARK_EXTRA_CLASSPATH=/etc/hbase/conf/' \
// -- --template HBASETOGCS \
// --templateProperty hbasetogcs.output.fileformat=avro \
// --templateProperty hbasetogcs.output.savemode=overwrite \
// --templateProperty hbasetogcs.output.path=gs://dataproc-templates/integration-testing/output/HBASETOGCS_automated/avro \
// --templateProperty hbasetogcs.table.catalog=$CATALOG

// '''
// }
// }
// }
}
}
stage('Parallel Execution 6'){
Expand Down Expand Up @@ -877,32 +877,32 @@ EOF
}
}
}
stage('HBASE TO GCS (Manual)(avro)') {
when {
// Run this stage only if JOB_TYPE is not set to CLUSTER
expression { env.JOB_TYPE != "CLUSTER" }
}
steps{
retry(count: stageRetryCount) {
sh '''
export SKIP_BUILD=true
export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}'
export JARS="gs://deps-dataproc-template/hbase-spark-protocol-shaded-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/hbase-spark-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/protobuf-java-2.5.0.jar,gs://deps-dataproc-template/htrace-core4-4.2.0-incubating.jar"
cd java
bin/start.sh \
--container-image=us-docker.pkg.dev/${GCP_PROJECT}/dataproc-cicd/hbase-to-gcs:1.0.3 \
--properties='spark.dataproc.driverEnv.SPARK_EXTRA_CLASSPATH=/etc/hbase/conf/' \
-- --template=HBASETOGCS \
--templateProperty hbasetogcs.output.fileformat=avro \
--templateProperty hbasetogcs.output.savemode=overwrite \
--templateProperty hbasetogcs.output.path=gs://dataproc-templates/integration-testing/output/HBASETOGCS_manual/avro \
--templateProperty hbasetogcs.table.catalog=$CATALOG
'''
}
}
}
// stage('HBASE TO GCS (Manual)(avro)') {
// when {
// // Run this stage only if JOB_TYPE is not set to CLUSTER
// expression { env.JOB_TYPE != "CLUSTER" }
// }
// steps{
// retry(count: stageRetryCount) {
// sh '''
// export SKIP_BUILD=true
// export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}'
// export JARS="gs://deps-dataproc-template/hbase-spark-protocol-shaded-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/hbase-spark-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/protobuf-java-2.5.0.jar,gs://deps-dataproc-template/htrace-core4-4.2.0-incubating.jar"

// cd java

// bin/start.sh \
// --container-image=us-docker.pkg.dev/${GCP_PROJECT}/dataproc-cicd/hbase-to-gcs:1.0.3 \
// --properties='spark.dataproc.driverEnv.SPARK_EXTRA_CLASSPATH=/etc/hbase/conf/' \
// -- --template=HBASETOGCS \
// --templateProperty hbasetogcs.output.fileformat=avro \
// --templateProperty hbasetogcs.output.savemode=overwrite \
// --templateProperty hbasetogcs.output.path=gs://dataproc-templates/integration-testing/output/HBASETOGCS_manual/avro \
// --templateProperty hbasetogcs.table.catalog=$CATALOG
// '''
// }
// }
// }
}
}
stage('Parallel Execution 9'){
Expand Down Expand Up @@ -1067,66 +1067,66 @@ EOF
}
}
}
stage('HBASE TO GCS (Manual)') {
when {
// Run this stage only if JOB_TYPE is not set to CLUSTER
expression { env.JOB_TYPE != "CLUSTER" }
}
steps{
retry(count: stageRetryCount) {
sh '''
export SKIP_BUILD=true
export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}'
export JARS="gs://deps-dataproc-template/hbase-spark-protocol-shaded-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/hbase-spark-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/protobuf-java-2.5.0.jar,gs://deps-dataproc-template/htrace-core4-4.2.0-incubating.jar"
cd java
bin/start.sh \
--container-image=us-docker.pkg.dev/${GCP_PROJECT}/dataproc-cicd/hbase-to-gcs:1.0.3 \
--properties='spark.dataproc.driverEnv.SPARK_EXTRA_CLASSPATH=/etc/hbase/conf/' \
-- --template=HBASETOGCS \
--templateProperty hbasetogcs.output.fileformat=csv \
--templateProperty hbasetogcs.output.savemode=overwrite \
--templateProperty hbasetogcs.output.path=gs://dataproc-templates/integration-testing/output/HBASETOGCS_manual/csv \
--templateProperty hbasetogcs.table.catalog=$CATALOG
'''
}
}
}
stage('HBASE TO GCS (Automated)') {
when {
// Run this stage only if JOB_TYPE is not set to CLUSTER
expression { env.JOB_TYPE != "CLUSTER" }
}
steps{
retry(count: stageRetryCount) {
sh '''
gsutil cp gs://python-dataproc-templates/surjitsh/hbase-site.xml .
gcloud auth configure-docker us-docker.pkg.dev
export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}'
export IMAGE_NAME_VERSION=hbase-to-gcs:1.0.3
export HBASE_SITE_PATH=../hbase-site.xml
export IMAGE=us-docker.pkg.dev/${GCP_PROJECT}/dataproc-cicd/${IMAGE_NAME_VERSION}
export JARS="gs://deps-dataproc-template/hbase-spark-protocol-shaded-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/hbase-spark-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/protobuf-java-2.5.0.jar,gs://deps-dataproc-template/htrace-core4-4.2.0-incubating.jar"
export SKIP_IMAGE_BUILD=TRUE
export SKIP_BUILD=true
cd java
bin/start.sh \
--container-image=$IMAGE \
--properties='spark.dataproc.driverEnv.SPARK_EXTRA_CLASSPATH=/etc/hbase/conf/' \
-- --template HBASETOGCS \
--templateProperty hbasetogcs.output.fileformat=csv \
--templateProperty hbasetogcs.output.savemode=overwrite \
--templateProperty hbasetogcs.output.path=gs://dataproc-templates/integration-testing/output/HBASETOGCS_automated/csv \
--templateProperty hbasetogcs.table.catalog=$CATALOG
'''
}
}
}
// stage('HBASE TO GCS (Manual)') {
// when {
// // Run this stage only if JOB_TYPE is not set to CLUSTER
// expression { env.JOB_TYPE != "CLUSTER" }
// }
// steps{
// retry(count: stageRetryCount) {
// sh '''
// export SKIP_BUILD=true
// export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}'
// export JARS="gs://deps-dataproc-template/hbase-spark-protocol-shaded-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/hbase-spark-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/protobuf-java-2.5.0.jar,gs://deps-dataproc-template/htrace-core4-4.2.0-incubating.jar"

// cd java

// bin/start.sh \
// --container-image=us-docker.pkg.dev/${GCP_PROJECT}/dataproc-cicd/hbase-to-gcs:1.0.3 \
// --properties='spark.dataproc.driverEnv.SPARK_EXTRA_CLASSPATH=/etc/hbase/conf/' \
// -- --template=HBASETOGCS \
// --templateProperty hbasetogcs.output.fileformat=csv \
// --templateProperty hbasetogcs.output.savemode=overwrite \
// --templateProperty hbasetogcs.output.path=gs://dataproc-templates/integration-testing/output/HBASETOGCS_manual/csv \
// --templateProperty hbasetogcs.table.catalog=$CATALOG
// '''
// }
// }
// }
// stage('HBASE TO GCS (Automated)') {
// when {
// // Run this stage only if JOB_TYPE is not set to CLUSTER
// expression { env.JOB_TYPE != "CLUSTER" }
// }
// steps{
// retry(count: stageRetryCount) {
// sh '''
// gsutil cp gs://python-dataproc-templates/surjitsh/hbase-site.xml .
// gcloud auth configure-docker us-docker.pkg.dev
// export CATALOG='{"table":{"namespace":"default","name":"my_table"},"rowkey":"key","columns":{"key":{"cf":"rowkey","col":"key","type":"string"},"name":{"cf":"cf","col":"name","type":"string"}}}'
// export IMAGE_NAME_VERSION=hbase-to-gcs:1.0.3
// export HBASE_SITE_PATH=../hbase-site.xml
// export IMAGE=us-docker.pkg.dev/${GCP_PROJECT}/dataproc-cicd/${IMAGE_NAME_VERSION}
// export JARS="gs://deps-dataproc-template/hbase-spark-protocol-shaded-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/hbase-spark-1.0.1-spark_3.2-scala_2.12.jar,gs://deps-dataproc-template/protobuf-java-2.5.0.jar,gs://deps-dataproc-template/htrace-core4-4.2.0-incubating.jar"

// export SKIP_IMAGE_BUILD=TRUE
// export SKIP_BUILD=true

// cd java

// bin/start.sh \
// --container-image=$IMAGE \
// --properties='spark.dataproc.driverEnv.SPARK_EXTRA_CLASSPATH=/etc/hbase/conf/' \
// -- --template HBASETOGCS \
// --templateProperty hbasetogcs.output.fileformat=csv \
// --templateProperty hbasetogcs.output.savemode=overwrite \
// --templateProperty hbasetogcs.output.path=gs://dataproc-templates/integration-testing/output/HBASETOGCS_automated/csv \
// --templateProperty hbasetogcs.table.catalog=$CATALOG

// '''
// }
// }
// }
}
}
}
Expand Down
Loading

0 comments on commit 5ef89e2

Please sign in to comment.