Skip to content

Commit

Permalink
fix: Jenkins optimisation update (#1024)
Browse files Browse the repository at this point in the history 
* fix: adding blogpost link for BigQuery to JDBC

* fix: including pom.xml and jenkins file for running test cases

* fix: making changes in UI Jenkins file as well

* fix: changes made for python jenkins file as well

* fix: including requirements.txt file of python as well
  • Loading branch information
@hhasija
hhasija authored Nov 13, 2024
1 parent 3c5dc82 commit 7bea95b
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 88 deletions.
44 changes: 22 additions & 22 deletions java/.ci/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,9 @@ EOF
stage('JDBC TO BQ'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand All @@ -130,9 +130,9 @@ EOF
stage('Parallel Execution 1'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -262,9 +262,9 @@ EOF
stage('Parallel Execution 2'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -391,9 +391,9 @@ EOF
stage('Parallel Execution 3'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -494,9 +494,9 @@ EOF
stage('Parallel Execution 4'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -603,9 +603,9 @@ EOF
stage('Parallel Execution 5'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -699,9 +699,9 @@ EOF
stage('Parallel Execution 6'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -802,9 +802,9 @@ EOF
stage('Parallel Execution 7'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -884,9 +884,9 @@ EOF
stage('Parallel Execution 8'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -989,9 +989,9 @@ EOF
stage('Parallel Execution 9'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -1105,9 +1105,9 @@ EOF
stage('Parallel Execution 10'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down
20 changes: 10 additions & 10 deletions java/.ci/UiJenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,9 @@ pipeline {
stage('Parallel Execution 1'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -184,9 +184,9 @@ pipeline {
stage('Parallel Execution 2'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -286,9 +286,9 @@ pipeline {
stage('Parallel Execution 3'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -366,9 +366,9 @@ pipeline {
stage('Parallel Execution 4'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down Expand Up @@ -424,9 +424,9 @@ pipeline {
stage('Parallel Execution 5'){
when {
expression {
// Check if there are any changes in Java files in the last commit
// Check for changes in .java files, the pom.xml file, or the two Jenkinsfiles in their specific paths
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".java"',
script: 'git diff --name-only HEAD~1 | grep -E ".java$|java/pom.xml|java/.ci/Jenkinsfile|java/.ci/UiJenkinsfile"',
returnStatus: true
) == 0
}
Expand Down
112 changes: 56 additions & 56 deletions python/.ci/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,14 @@ pipeline {
}
stage('Parallel Execution 1'){
when {
expression {
// Check if there are any changes in Python files in the last commit
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
returnStatus: true
) == 0
expression {
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
return sh(
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
returnStatus: true
) == 0
}
}
}
parallel{
stage('Hive TO GCS') {
steps{
Expand Down Expand Up @@ -165,14 +165,14 @@ pipeline {
}
stage('Parallel Execution 2'){
when {
expression {
// Check if there are any changes in Python files in the last commit
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
returnStatus: true
) == 0
expression {
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
return sh(
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
returnStatus: true
) == 0
}
}
}
parallel{
stage('JDBC To JDBC') {
steps{
Expand Down Expand Up @@ -257,14 +257,14 @@ pipeline {
}
stage('Parallel Execution 3'){
when {
expression {
// Check if there are any changes in Python files in the last commit
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
returnStatus: true
) == 0
expression {
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
return sh(
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
returnStatus: true
) == 0
}
}
}
parallel{
stage('GCS TO BIGQUERY') {
steps{
Expand Down Expand Up @@ -353,14 +353,14 @@ pipeline {
}
stage('Parallel Execution 4'){
when {
expression {
// Check if there are any changes in Python files in the last commit
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
returnStatus: true
) == 0
expression {
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
return sh(
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
returnStatus: true
) == 0
}
}
}
parallel{
stage('MONGO TO GCS') {
steps{
Expand Down Expand Up @@ -514,14 +514,14 @@ pipeline {
}
stage('Parallel Execution 5'){
when {
expression {
// Check if there are any changes in Python files in the last commit
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
returnStatus: true
) == 0
expression {
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
return sh(
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
returnStatus: true
) == 0
}
}
}
parallel{
stage('JDBC TO GCS') {
steps{
Expand Down Expand Up @@ -614,14 +614,14 @@ pipeline {
}
stage('Parallel Execution 6'){
when {
expression {
// Check if there are any changes in Python files in the last commit
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
returnStatus: true
) == 0
expression {
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
return sh(
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
returnStatus: true
) == 0
}
}
}
parallel{
stage('S3 TO BigQuery (avro)') {
steps{
Expand Down Expand Up @@ -659,14 +659,14 @@ pipeline {
}
stage('Parallel Execution 7'){
when {
expression {
// Check if there are any changes in Python files in the last commit
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
returnStatus: true
) == 0
expression {
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
return sh(
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
returnStatus: true
) == 0
}
}
}
parallel{
// stage('Hbase TO GCS (Manual)') {
// when {
Expand Down Expand Up @@ -790,14 +790,14 @@ pipeline {
}
stage('Parallel Execution 8'){
when {
expression {
// Check if there are any changes in Python files in the last commit
return sh(
script: 'git diff --name-only HEAD~1 | grep -q ".py"',
returnStatus: true
) == 0
expression {
// Check for changes in .py files, python/.ci/Jenkinsfile, or python/requirements.txt
return sh(
script: 'git diff --name-only HEAD~1 | grep -E ".py$|python/.ci/Jenkinsfile|python/requirements.txt"',
returnStatus: true
) == 0
}
}
}
parallel{
stage('Elasticsearch to GCS'){
steps{
Expand Down

0 comments on commit 7bea95b

Please sign in to comment.