Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: added log level #889

Closed
wants to merge 1 commit into from
Closed

feat: added log level #889

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 30 additions & 16 deletions java/.ci/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ pipeline {
script: "echo $branchName | sed -e 's|origin/||g' | sed -e 's|^null\$|main|'", // Remove "origin/" and set the default branch to main
returnStdout: true
).trim()

LOG_LEVEL = "TRACE"
MAVEN_HOME = "/var/lib/jenkins/tools/hudson.tasks.Maven_MavenInstallation/maven"
PATH = "$PATH:$MAVEN_HOME/bin"

Expand Down Expand Up @@ -95,7 +95,8 @@ pipeline {
--templateProperty jdbctobq.jdbc.driver.class.name=com.mysql.jdbc.Driver \
--templateProperty jdbctobq.sql="select * from test.employee" \
--templateProperty jdbctobq.write.mode=overwrite \
--templateProperty jdbctobq.temp.gcs.bucket=dataproc-templates/integration-testing/output/JDBCTOBIGQUERY
--templateProperty jdbctobq.temp.gcs.bucket=dataproc-templates/integration-testing/output/JDBCTOBIGQUERY \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand All @@ -116,7 +117,8 @@ pipeline {
--templateProperty gcs.bigquery.output.dataset="dataproc_templates" \
--templateProperty gcs.bigquery.output.table="gcs_to_bq_avro" \
--templateProperty gcs.bigquery.output.mode="Overwrite" \
--templateProperty gcs.bigquery.temp.bucket.name=dataproc-templates
--templateProperty gcs.bigquery.temp.bucket.name=dataproc-templates \
--templateProperty log.level=$LOG_LEVEL
'''


Expand All @@ -141,7 +143,8 @@ pipeline {
--templateProperty gcs.spanner.output.database=spark-ci-db \
--templateProperty gcs.spanner.output.table=badges \
--templateProperty gcs.spanner.output.saveMode=Overwrite \
--templateProperty gcs.spanner.output.primaryKey=empno
--templateProperty gcs.spanner.output.primaryKey=empno \
--templateProperty log.level=$LOG_LEVEL
'''


Expand All @@ -163,7 +166,8 @@ pipeline {
--templateProperty "spanner.gcs.input.table.id=(select id,name from badges2 where name = 'Teacher' limit 10000)" \
--templateProperty spanner.gcs.output.gcs.path=dataproc-templates/integration-testing/output/SPANNERTOGCS/parquet \
--templateProperty spanner.gcs.output.gcs.saveMode=Overwrite \
--templateProperty spanner.gcs.output.gcs.format=parquet
--templateProperty spanner.gcs.output.gcs.format=parquet \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand All @@ -181,7 +185,8 @@ pipeline {
--templateProperty hivetobq.bigquery.location=$GCP_PROJECT.dataproc_templates.test \
--templateProperty hivetobq.sql="select * from default.employee" \
--templateProperty hivetobq.write.mode=Overwrite \
--templateProperty hivetobq.temp.gcs.bucket=dataproc-templates/integration-testing/output/HIVETOBIGQUERY
--templateProperty hivetobq.temp.gcs.bucket=dataproc-templates/integration-testing/output/HIVETOBIGQUERY \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand All @@ -207,7 +212,8 @@ pipeline {
--templateProperty gcs.jdbc.output.table=avrodemo \
--templateProperty gcs.jdbc.output.saveMode=Overwrite \
--templateProperty gcs.jdbc.output.url="$TEST_JDBC_URL" \
--templateProperty gcs.jdbc.output.driver='com.mysql.jdbc.Driver'
--templateProperty gcs.jdbc.output.driver='com.mysql.jdbc.Driver' \
--templateProperty log.level=$LOG_LEVEL
'''

}
Expand All @@ -226,7 +232,8 @@ pipeline {
--templateProperty pubsubtogcs.input.subscription=pubsubtogcsv3-sub \
--templateProperty pubsubtogcs.gcs.bucket.name=pubsubtogcs_dev \
--templateProperty pubsubtogcs.gcs.output.data.format=JSON \
--templateProperty pubsubtogcs.batch.size=50
--templateProperty pubsubtogcs.batch.size=50 \
--templateProperty log.level=$LOG_LEVEL
'''

}
Expand All @@ -248,7 +255,8 @@ pipeline {
--templateProperty jdbctogcs.output.location=gs://dataproc-templates/integration-testing/output/JDBCTOGCS/csv \
--templateProperty jdbctogcs.output.format=csv \
--templateProperty jdbctogcs.write.mode=overwrite \
--templateProperty jdbctogcs.sql="select * from test.employee"
--templateProperty jdbctogcs.sql="select * from test.employee" \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand All @@ -269,7 +277,8 @@ pipeline {
--templateProperty jdbctogcs.output.location=gs://dataproc-templates/integration-testing/output/JDBCTOGCS/avro \
--templateProperty jdbctogcs.output.format=avro \
--templateProperty jdbctogcs.write.mode=overwrite \
--templateProperty jdbctogcs.sql="select * from test.employee"
--templateProperty jdbctogcs.sql="select * from test.employee" \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand All @@ -291,7 +300,8 @@ pipeline {
-- --template HIVETOGCS \
--templateProperty hive.input.table=employee \
--templateProperty hive.input.db=default \
--templateProperty hive.gcs.output.path=gs://dataproc-templates/integration-testing/output/HIVETOGCS
--templateProperty hive.gcs.output.path=gs://dataproc-templates/integration-testing/output/HIVETOGCS \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand All @@ -313,7 +323,8 @@ pipeline {
--templateProperty kafka.bq.dataset=kafkatobq \
--templateProperty kafka.bq.table=integration-test \
--templateProperty kafka.bq.temp.gcs.bucket=dataproc-templates-kafkatobq \
--templateProperty kafka.bq.await.termination.timeout=60000
--templateProperty kafka.bq.await.termination.timeout=60000 \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand All @@ -335,7 +346,8 @@ pipeline {
--templateProperty gcs.gcs.output.format=csv \
--templateProperty gcs.gcs.write.mode=overwrite \
--templateProperty gcs.gcs.temp.table=dataset \
--templateProperty gcs.gcs.temp.query='select * from global_temp.dataset where sal>1500'
--templateProperty gcs.gcs.temp.query='select * from global_temp.dataset where sal>1500' \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand All @@ -352,7 +364,8 @@ pipeline {
--templateProperty pubsub.input.project.id=$GCP_PROJECT \
--templateProperty pubsub.bq.output.project.id=$GCP_PROJECT \
--templateProperty pubsub.bq.output.dataset=dataproc_templates \
--templateProperty pubsub.bq.output.table=pubsubtobq
--templateProperty pubsub.bq.output.table=pubsubtobq \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand Down Expand Up @@ -380,7 +393,8 @@ pipeline {
--templateProperty jdbctospanner.output.database=spark-ci-db \
--templateProperty jdbctospanner.output.table=employee \
--templateProperty jdbctospanner.output.saveMode=Overwrite \
--templateProperty jdbctospanner.output.primaryKey='empno'
--templateProperty jdbctospanner.output.primaryKey='empno' \
--templateProperty log.level=$LOG_LEVEL
'''
}
}
Expand Down Expand Up @@ -870,4 +884,4 @@ pipeline {
}
}
}
}
}