From 267e2d28402c6be97b974d16c82a115b80fb1719 Mon Sep 17 00:00:00 2001 From: Chakravarthy Racharla Date: Mon, 24 Feb 2025 10:49:41 +0530 Subject: [PATCH 1/4] fix clean to run during evaluate instead of config stage --- datahub-web-react/build.gradle | 3 +-- metadata-events/mxe-schemas/build.gradle | 6 +++--- metadata-events/mxe-utils-avro/build.gradle | 2 +- metadata-io/build.gradle | 6 ++++-- metadata-jobs/mae-consumer/build.gradle | 2 +- metadata-jobs/mce-consumer/build.gradle | 2 +- metadata-jobs/pe-consumer/build.gradle | 2 +- metadata-models/build.gradle | 3 +-- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/datahub-web-react/build.gradle b/datahub-web-react/build.gradle index 227c89b0a4be6..d5b0891fca70a 100644 --- a/datahub-web-react/build.gradle +++ b/datahub-web-react/build.gradle @@ -111,14 +111,13 @@ task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) { outputs.dir('dist') } -task cleanExtraDirs { +clean { delete 'node_modules/.yarn-integrity' delete 'dist' delete 'tmp' delete 'just' delete fileTree(dir: 'src', include: '*.generated.ts') } -clean.finalizedBy(cleanExtraDirs) configurations { assets diff --git a/metadata-events/mxe-schemas/build.gradle b/metadata-events/mxe-schemas/build.gradle index 6dfe69a420242..d0292fa968baf 100644 --- a/metadata-events/mxe-schemas/build.gradle +++ b/metadata-events/mxe-schemas/build.gradle @@ -31,7 +31,7 @@ task renameNamespace(type: Exec, dependsOn: copyOriginalAvsc) { build.dependsOn renameNamespace clean { - project.delete('src/main/pegasus') - project.delete('src/mainGeneratedAvroSchema/avro') - project.delete('src/renamed/avro') + delete 'src/main/pegasus' + delete 'src/mainGeneratedAvroSchema/avro' + delete 'src/renamed/avro' } diff --git a/metadata-events/mxe-utils-avro/build.gradle b/metadata-events/mxe-utils-avro/build.gradle index f893fadb8e2df..a99511c19859e 100644 --- a/metadata-events/mxe-utils-avro/build.gradle +++ b/metadata-events/mxe-utils-avro/build.gradle @@ -36,5 +36,5 @@ compileJava.dependsOn copyOriginalMXESchemas processResources.dependsOn copyOriginalMXESchemas clean { - project.delete("src/main/resources/avro") + delete "src/main/resources/avro" } diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 665fba0038cad..b0d23659870c2 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -159,5 +159,7 @@ tasks.withType(Test) { } clean { - project.delete("$projectDir/generated") -} \ No newline at end of file + //TODO: Is this really needed? dont see generated folder in projectDir. It is in build, but that doest need explicit clean + delete "$projectDir/generated" +} + diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle index 08c712f6167b4..4f7d374129144 100644 --- a/metadata-jobs/mae-consumer/build.gradle +++ b/metadata-jobs/mae-consumer/build.gradle @@ -64,5 +64,5 @@ compileJava.dependsOn avroSchemaSources processResources.dependsOn avroSchemaSources clean { - project.delete("src/main/resources/avro") + delete "src/main/resources/avro" } \ No newline at end of file diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle index 17accb7ab7de3..b9431ae0dfddf 100644 --- a/metadata-jobs/mce-consumer/build.gradle +++ b/metadata-jobs/mce-consumer/build.gradle @@ -58,5 +58,5 @@ compileJava.dependsOn avroSchemaSources processResources.dependsOn avroSchemaSources clean { - project.delete("src/main/resources/avro") + delete "src/main/resources/avro" } diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle index ad17a44377085..8873715d3fb57 100644 --- a/metadata-jobs/pe-consumer/build.gradle +++ b/metadata-jobs/pe-consumer/build.gradle @@ -44,5 +44,5 @@ compileJava.dependsOn avroSchemaSources processResources.dependsOn avroSchemaSources clean { - project.delete("src/main/resources/avro") + delete "src/main/resources/avro" } \ No newline at end of file diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle index 2d0b433d69013..88d6b3097242d 100644 --- a/metadata-models/build.gradle +++ b/metadata-models/build.gradle @@ -74,7 +74,6 @@ task openApiGenerate(type: GenerateSwaggerCode, dependsOn: 'generateJsonSchema') } tasks.getByName("compileJava").dependsOn(openApiGenerate) -task cleanExtraDirs { +clean { delete "$projectDir/src/generatedJsonSchema" } -clean.finalizedBy(cleanExtraDirs) From 3dbc9df4f1bd072ae0d35b162959890cf7d1ebe2 Mon Sep 17 00:00:00 2001 From: Chakravarthy Racharla Date: Mon, 24 Feb 2025 18:54:29 +0530 Subject: [PATCH 2/4] combine copy and rename into single task HAving these as two separate tasks caused copy to always be out-of-date when rename is run, and rename to be out-of-date when rename is done. Gradle does not like inplace updates of any files. By combining it, the copy+rename now appear to produce a new artificat that will be up-to-date if no sources have changed. This caused lot of downstream jars to get rebuilt since this jar is always out-of-dat and rebuilt everytime. --- metadata-events/mxe-schemas/build.gradle | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/metadata-events/mxe-schemas/build.gradle b/metadata-events/mxe-schemas/build.gradle index d0292fa968baf..48554eb34bc2b 100644 --- a/metadata-events/mxe-schemas/build.gradle +++ b/metadata-events/mxe-schemas/build.gradle @@ -19,13 +19,15 @@ generateDataTemplate.dependsOn copyMetadataModels mainCopySchemas.dependsOn copyMetadataModels pegasus.main.generationModes = [PegasusGenerationMode.PEGASUS, PegasusGenerationMode.AVRO] -task copyOriginalAvsc(type: Copy, dependsOn: generateAvroSchema) { +task renameNamespace(type: Copy, dependsOn: generateAvroSchema) { from("src/mainGeneratedAvroSchema/avro") into file("src/renamed/avro") -} -task renameNamespace(type: Exec, dependsOn: copyOriginalAvsc) { - commandLine 'bash', './rename-namespace.sh' + doLast { + project.exec { + commandLine 'bash', './rename-namespace.sh' + } + } } build.dependsOn renameNamespace From cb3e970fa1c913596418ca412c8c13962e7dcefe Mon Sep 17 00:00:00 2001 From: Chakravarthy Racharla Date: Tue, 25 Feb 2025 13:25:25 +0530 Subject: [PATCH 3/4] reuse versioning and git properties --- build.gradle | 50 ++++++++-- gradle/versioning/versioning-global.gradle | 101 ++++++++++++++++++++ gradle/versioning/versioning.gradle | 86 ++++------------- metadata-integration/java/versioning.gradle | 74 +++++--------- 4 files changed, 183 insertions(+), 128 deletions(-) create mode 100644 gradle/versioning/versioning-global.gradle diff --git a/build.gradle b/build.gradle index 13ec15ec9220c..962a9f40bbae0 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,6 @@ +import org.apache.tools.ant.filters.ReplaceTokens + + buildscript { ext.jdkVersionDefault = 17 ext.javaClassVersionDefault = 11 @@ -396,25 +399,56 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) { } } +apply plugin: 'com.gorylenko.gradle-git-properties' +gitProperties { + keys = ['git.commit.id','git.commit.id.describe','git.commit.time'] + // using any tags (not limited to annotated tags) for "git.commit.id.describe" property + // see http://ajoberstar.org/grgit/grgit-describe.html for more info about the describe method and available parameters + // 'it' is an instance of org.ajoberstar.grgit.Grgit + customProperty 'git.commit.id.describe', { it.describe(tags: true) } + gitPropertiesResourceDir = rootProject.buildDir + failOnNoGitDirectory = false +} + +def gitPropertiesGenerated = false + +apply from: 'gradle/versioning/versioning-global.gradle' + +tasks.register("generateGitPropertiesGlobal", com.gorylenko.GenerateGitPropertiesTask) { + doFirst { + if (!gitPropertiesGenerated) { + println "Generating git.properties" + gitPropertiesGenerated = true + } else { + // Skip actual execution if already run + onlyIf { false } + } + } +} + subprojects { apply plugin: 'maven-publish' - apply plugin: 'com.gorylenko.gradle-git-properties' apply plugin: 'com.diffplug.spotless' - gitProperties { - keys = ['git.commit.id','git.commit.id.describe','git.commit.time'] - // using any tags (not limited to annotated tags) for "git.commit.id.describe" property - // see http://ajoberstar.org/grgit/grgit-describe.html for more info about the describe method and available parameters - // 'it' is an instance of org.ajoberstar.grgit.Grgit - customProperty 'git.commit.id.describe', { it.describe(tags: true) } - failOnNoGitDirectory = false + def gitPropertiesTask = tasks.register("copyGitProperties", Copy) { + dependsOn rootProject.tasks.named("generateGitPropertiesGlobal") + def sourceFile = file("${rootProject.buildDir}/git.properties") + from sourceFile + into "$project.buildDir/resources/main" } plugins.withType(JavaPlugin).configureEach { + project.tasks.named(JavaPlugin.CLASSES_TASK_NAME).configure{ + dependsOn gitPropertiesTask + } if (project.name == 'datahub-web-react') { return } + /* TODO: evaluate ignoring jar timestamps for increased caching (compares checksum instead) + jar { + preserveFileTimestamps = false + }*/ dependencies { implementation externalDependency.annotationApi diff --git a/gradle/versioning/versioning-global.gradle b/gradle/versioning/versioning-global.gradle new file mode 100644 index 0000000000000..d38703b2d20e7 --- /dev/null +++ b/gradle/versioning/versioning-global.gradle @@ -0,0 +1,101 @@ +/** + Applies a consistent versioning scheme to all projects using this script + +Uses git tags to mint versions by default. +git tags can be of a few forms: +- short sha (typical for a PR or a commit) (e.g. 38960ae) +- versioned tags (typical for a release) (e.g. v0.8.45, v0.8.45.1, v0.8.45rc1, v0.8.45.1rc4) + +Produces the following variables and supports token replacement +- version: server version amenable for creating jars +- fullVersion: full version string +- cliMajorVersion: cli version amenable for binding to server as a default + 0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories) + + All inference can be overridden by passing in the releaseVersion property + e.g. -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4 + + **/ + +import groovy.json.JsonBuilder + +def detailedVersionString = "0.0.0-unknown-SNAPSHOT" +def cliMajorVersion = "0.15.0" // base default cli major version +def snapshotVersion = false +def javaVersion = "" + +if (project.hasProperty("releaseVersion")) { + version = releaseVersion + detailedVersionString = releaseVersion +} else { + try { + // apply this plugin in a try-catch block so that we can handle cases without .git directory + apply plugin: "com.palantir.git-version" + def details = versionDetails() + detailedVersionString = gitVersion() + version = details.lastTag + version = version.startsWith("v")? version.substring(1): version + def suffix = details.isCleanTag? "": "-SNAPSHOT" + snapshotVersion = ! details.isCleanTag + } + catch (Exception e) { + e.printStackTrace() + // last fall back + version = detailedVersionString + } +} + +// trim version if it is of size 4 to size 3 +def versionParts = version.tokenize(".") +cliMajorVersion = version +if (versionParts.size() > 3) { + // at-least 4 part version + // we check if the 4th part is a .0 in which case we want to create a release + if ((versionParts.size() == 4) && (versionParts[3] == '0')) { + versionParts = versionParts[0..2] + } + version = versionParts[0..2].join('.') + if (versionParts.size() > 3) { + version = version + "-" + versionParts[3..versionParts.size()-1].join('-') + } +} + +if (snapshotVersion) { + if (versionParts[versionParts.size()-1].isInteger()) { + def base_version = versionParts[0..versionParts.size()-2].join('.') + version = base_version + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" + cliMajorVersion = base_version + "." + versionParts[versionParts.size()-1] + } else { + // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version + version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' + cliMajorVersion = versionParts[0..versionParts.size()-1].join('.') + } + + // differences from metadata-integration/java/versioning.gradle + if (versionParts[versionParts.size()-1].isInteger()) { + javaVersion = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" + } else { + // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version + javaVersion = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' + } +} + +// Note: No task, we want this executed during config phase, once for rootProject. +def data = [ + fullVersion: detailedVersionString, + cliMajorVersion: cliMajorVersion, + version: version, + javaVersion: javaVersion +] + +// Convert to JSON +def jsonBuilder = new JsonBuilder(data) +def outputFile = file("${rootProject.buildDir}/version.json") + +// Ensure buildDir exists +rootProject.buildDir.mkdirs() + +// Write to file +outputFile.text = jsonBuilder.toPrettyString() + +println "git.properties JSON data written to ${outputFile}" diff --git a/gradle/versioning/versioning.gradle b/gradle/versioning/versioning.gradle index 2ffb085869f7b..084aa9f2fb06b 100644 --- a/gradle/versioning/versioning.gradle +++ b/gradle/versioning/versioning.gradle @@ -1,74 +1,29 @@ -/** - Applies a consistent versioning scheme to all projects using this script - -Uses git tags to mint versions by default. -git tags can be of a few forms: -- short sha (typical for a PR or a commit) (e.g. 38960ae) -- versioned tags (typical for a release) (e.g. v0.8.45, v0.8.45.1, v0.8.45rc1, v0.8.45.1rc4) - -Produces the following variables and supports token replacement -- version: server version amenable for creating jars -- fullVersion: full version string -- cliMajorVersion: cli version amenable for binding to server as a default - 0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories) - - All inference can be overridden by passing in the releaseVersion property - e.g. -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4 - - **/ - - +import groovy.json.JsonSlurper import org.apache.tools.ant.filters.ReplaceTokens + def detailedVersionString = "0.0.0-unknown-SNAPSHOT" def cliMajorVersion = "0.15.0" // base default cli major version -def snapshotVersion = false -if (project.hasProperty("releaseVersion")) { - version = releaseVersion - detailedVersionString = releaseVersion -} else { - try { - // apply this plugin in a try-catch block so that we can handle cases without .git directory - apply plugin: "com.palantir.git-version" - def details = versionDetails() - detailedVersionString = gitVersion() - version = details.lastTag - version = version.startsWith("v")? version.substring(1): version - def suffix = details.isCleanTag? "": "-SNAPSHOT" - snapshotVersion = ! details.isCleanTag - } - catch (Exception e) { - e.printStackTrace() - // last fall back - version = detailedVersionString - } -} -// trim version if it is of size 4 to size 3 -def versionParts = version.tokenize(".") -cliMajorVersion = version -if (versionParts.size() > 3) { - // at-least 4 part version - // we check if the 4th part is a .0 in which case we want to create a release - if ((versionParts.size() == 4) && (versionParts[3] == '0')) { - versionParts = versionParts[0..2] - } - version = versionParts[0..2].join('.') - if (versionParts.size() > 3) { - version = version + "-" + versionParts[3..versionParts.size()-1].join('-') +def inputFile = file("${rootProject.buildDir}/version.json") + +task readJsonData { + if (inputFile.exists()) { + def jsonSlurper = new JsonSlurper() + def data = jsonSlurper.parse(inputFile) + + detailedVersionString = data.fullVersion + cliMajorVersion = data.cliMajorVersion + version = data.version + } else { + println "git.properties JSON file not found: ${inputFile.path}" } } -if (snapshotVersion) { - if (versionParts[versionParts.size()-1].isInteger()) { - def base_version = versionParts[0..versionParts.size()-2].join('.') - version = base_version + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" - cliMajorVersion = base_version + "." + versionParts[versionParts.size()-1] - } else { - // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version - version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' - cliMajorVersion = versionParts[0..versionParts.size()-1].join('.') - } +task printVersionDetails() { + println("fullVersion=" + detailedVersionString) + println("cliMajorVersion=" + cliMajorVersion) + println("version=" + version) } processResources { @@ -76,8 +31,3 @@ processResources { filter(ReplaceTokens, tokens:[cliMajorVersion: cliMajorVersion]) } -task printVersionDetails() { - println("fullVersion=" + detailedVersionString) - println("cliMajorVersion=" + cliMajorVersion) - println("version=" + version) -} diff --git a/metadata-integration/java/versioning.gradle b/metadata-integration/java/versioning.gradle index baca5641f64ef..1bb591cfda1cd 100644 --- a/metadata-integration/java/versioning.gradle +++ b/metadata-integration/java/versioning.gradle @@ -1,60 +1,30 @@ -/** - Applies a consistent versioning scheme to all projects using this script - -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4 +import groovy.json.JsonSlurper +import org.apache.tools.ant.filters.ReplaceTokens - Not providing a property will make it use git tags to mint either a version like: - 0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories) - **/ +def detailedVersionString = "0.0.0-unknown-SNAPSHOT" +def cliMajorVersion = "0.15.0" // base default cli major version +task readJsonData { + def inputFile = file("${rootProject.buildDir}/version.json") -import org.apache.tools.ant.filters.ReplaceTokens + if (inputFile.exists()) { + def jsonSlurper = new JsonSlurper() + def data = jsonSlurper.parse(inputFile) -def detailedVersionString = "0.0.0-unknown-SNAPSHOT" -def snapshotVersion = false -if (project.hasProperty("releaseVersion")) { - version = releaseVersion - detailedVersionString = releaseVersion -} else { - try { - // apply this plugin in a try-catch block so that we can handle cases without .git directory - apply plugin: "com.palantir.git-version" - def details = versionDetails() - detailedVersionString = gitVersion() - version = details.lastTag - version = version.startsWith("v")? version.substring(1): version - def suffix = details.isCleanTag? "": "-SNAPSHOT" - snapshotVersion = ! details.isCleanTag - } - catch (Exception e) { - e.printStackTrace() - // last fall back - version = detailedVersionString - } + detailedVersionString = data.fullVersion + cliMajorVersion = data.cliMajorVersion + version = data.javaVersion + } else { + println "JSON file not found: ${inputFile.path}" } - // trim version if it is of size 4 to size 3 - def versionParts = version.tokenize(".") - if (versionParts.size() > 3) { - // at-least 4 part version - // we check if the 4th part is a .0 in which case we want to create a release - if ((versionParts.size() == 4) && (versionParts[3] == '0')) { - versionParts = versionParts[0..2] - } - version = versionParts[0..2].join('.') - if (versionParts.size() > 3) { - version = version + "-" + versionParts[3..versionParts.size()-1].join('-') - } - } +} - if (snapshotVersion) { - if (versionParts[versionParts.size()-1].isInteger()) { - version = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT" - } else { - // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version - version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT' - } - } +processResources { + filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString]) +} - processResources { - filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString]) - } +task printVersionDetails() { + println("fullVersion=" + detailedVersionString) + println("version=" + version) +} From 55749716f69ab54596595bf918ce80c9cc4178b2 Mon Sep 17 00:00:00 2001 From: Chakravarthy Racharla Date: Mon, 10 Mar 2025 15:48:36 +0530 Subject: [PATCH 4/4] load-cache expt --- .github/workflows/build-jars.yml | 52 ++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/build-jars.yml diff --git a/.github/workflows/build-jars.yml b/.github/workflows/build-jars.yml new file mode 100644 index 0000000000000..edb161afaba24 --- /dev/null +++ b/.github/workflows/build-jars.yml @@ -0,0 +1,52 @@ +name: build jars +on: + push: + branches: + - load-cache-workflow + # master + paths-ignore: + - "docs/**" + - "**.md" + pull_request: + branches: + - "load-cache-workflow" + paths-ignore: + - "docs/**" + - "**.md" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build-jars: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Check out the repo + uses: acryldata/sane-checkout-action@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: 17 + + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-home-cache-key: build-jars-cache-key + cache-write-only: true + + - name: Build JARs + run: | + ./gradlew jar -x datahub-web-react:jar -x datahub-frontend:jar --parallel --info + + # - uses: actions/upload-artifact@v4 + # if: always() + # with: + # name: Build Artifacts + # path: | + # **/build/libs/*.jar + # !**/build/libs/*-sources.jar + # !**/build/libs/*-javadoc.jar \ No newline at end of file