[do not merge] gradle cache build expt

.github/workflows/build-jars.yml

@@ -0,0 +1,52 @@
+name: build jars
+on:
+  push:
+    branches:
+      - load-cache-workflow
+      # master      
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+  pull_request:
+    branches:
+      - "load-cache-workflow"
+    paths-ignore:
+      - "docs/**"
+      - "**.md"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-jars:
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - name: Check out the repo
+        uses: acryldata/sane-checkout-action@v3
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          distribution: "zulu"
+          java-version: 17
+
+      - uses: gradle/actions/setup-gradle@v4
+        with:
+          gradle-home-cache-key: build-jars-cache-key
+          cache-write-only: true
+
+      - name: Build JARs
+        run: |
+          ./gradlew jar -x datahub-web-react:jar -x datahub-frontend:jar --parallel --info
+
+      # - uses: actions/upload-artifact@v4
+      #   if: always()
+      #   with:
+      #     name: Build Artifacts
+      #     path: |
+      #       **/build/libs/*.jar
+      #       !**/build/libs/*-sources.jar
+      #       !**/build/libs/*-javadoc.jar 

build.gradle

@@ -1,3 +1,6 @@
+import org.apache.tools.ant.filters.ReplaceTokens
+
+
 buildscript {
   ext.jdkVersionDefault = 17
   ext.javaClassVersionDefault = 11
@@ -396,25 +399,56 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
   }
 }
 
+apply plugin: 'com.gorylenko.gradle-git-properties'
+gitProperties {
+  keys = ['git.commit.id','git.commit.id.describe','git.commit.time']
+  // using any tags (not limited to annotated tags) for "git.commit.id.describe" property
+  // see http://ajoberstar.org/grgit/grgit-describe.html for more info about the describe method and available parameters
+  // 'it' is an instance of org.ajoberstar.grgit.Grgit
+  customProperty 'git.commit.id.describe', { it.describe(tags: true) }
+  gitPropertiesResourceDir = rootProject.buildDir
+  failOnNoGitDirectory = false
+}
+
+def gitPropertiesGenerated = false
+
+apply from: 'gradle/versioning/versioning-global.gradle'
+
+tasks.register("generateGitPropertiesGlobal", com.gorylenko.GenerateGitPropertiesTask) {
+  doFirst {
+    if (!gitPropertiesGenerated) {
+      println "Generating git.properties"
+      gitPropertiesGenerated = true
+    } else {
+      // Skip actual execution if already run
+      onlyIf { false }
+    }
+  }
+}
+
 subprojects {
 
   apply plugin: 'maven-publish'
-  apply plugin: 'com.gorylenko.gradle-git-properties'
   apply plugin: 'com.diffplug.spotless'
 
-  gitProperties {
-    keys = ['git.commit.id','git.commit.id.describe','git.commit.time']
-    // using any tags (not limited to annotated tags) for "git.commit.id.describe" property
-    // see http://ajoberstar.org/grgit/grgit-describe.html for more info about the describe method and available parameters
-    // 'it' is an instance of org.ajoberstar.grgit.Grgit
-    customProperty 'git.commit.id.describe', { it.describe(tags: true) }
-    failOnNoGitDirectory = false
+  def gitPropertiesTask = tasks.register("copyGitProperties", Copy) {
+    dependsOn rootProject.tasks.named("generateGitPropertiesGlobal")
+    def sourceFile = file("${rootProject.buildDir}/git.properties")
+    from sourceFile
+    into "$project.buildDir/resources/main"
   }
 
   plugins.withType(JavaPlugin).configureEach {
+    project.tasks.named(JavaPlugin.CLASSES_TASK_NAME).configure{
+      dependsOn gitPropertiesTask
+    }
     if (project.name == 'datahub-web-react') {
       return
     }
+    /* TODO: evaluate ignoring jar timestamps for increased caching (compares checksum instead)
+    jar {
+      preserveFileTimestamps = false
+    }*/
 
     dependencies {
       implementation externalDependency.annotationApi

datahub-web-react/build.gradle

@@ -111,14 +111,13 @@ task yarnBuild(type: YarnTask, dependsOn: [yarnInstall, yarnGenerate]) {
   outputs.dir('dist')
 }
 
-task cleanExtraDirs {
+clean {
   delete 'node_modules/.yarn-integrity'
   delete 'dist'
   delete 'tmp'
   delete 'just'
   delete fileTree(dir: 'src', include: '*.generated.ts')
 }
-clean.finalizedBy(cleanExtraDirs)
 
 configurations {
   assets

gradle/versioning/versioning-global.gradle

@@ -0,0 +1,101 @@
+/**
+ Applies a consistent versioning scheme to all projects using this script
+
+Uses git tags to mint versions by default. 
+git tags can be of a few forms:
+- short sha (typical for a PR or a commit) (e.g. 38960ae)
+- versioned tags (typical for a release) (e.g. v0.8.45, v0.8.45.1, v0.8.45rc1, v0.8.45.1rc4)
+
+Produces the following variables and supports token replacement
+- version: server version amenable for creating jars
+- fullVersion: full version string 
+- cliMajorVersion: cli version amenable for binding to server as a default
+  0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories)
+
+ All inference can be overridden by passing in the releaseVersion property
+ e.g. -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4
+
+ **/
+
+import groovy.json.JsonBuilder
+
+def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
+def cliMajorVersion = "0.15.0" // base default cli major version
+def snapshotVersion = false
+def javaVersion = ""
+
+if (project.hasProperty("releaseVersion")) {
+  version = releaseVersion
+  detailedVersionString = releaseVersion
+} else {
+  try {
+      // apply this plugin in a try-catch block so that we can handle cases without .git directory
+      apply plugin: "com.palantir.git-version"
+      def details = versionDetails()
+      detailedVersionString = gitVersion()
+      version = details.lastTag
+      version = version.startsWith("v")? version.substring(1): version
+      def suffix = details.isCleanTag? "": "-SNAPSHOT"
+      snapshotVersion = ! details.isCleanTag
+    }
+    catch (Exception e) {
+          e.printStackTrace()
+          // last fall back
+          version = detailedVersionString
+    }
+}
+
+// trim version if it is of size 4 to size 3
+def versionParts = version.tokenize(".")
+cliMajorVersion = version
+if (versionParts.size() > 3) {
+  // at-least 4 part version
+  // we check if the 4th part is a .0 in which case we want to create a release
+  if ((versionParts.size() == 4) && (versionParts[3] == '0')) {
+    versionParts = versionParts[0..2]
+  }
+  version = versionParts[0..2].join('.')
+  if (versionParts.size() > 3) {
+      version = version + "-" + versionParts[3..versionParts.size()-1].join('-')
+  }
+}
+
+if (snapshotVersion) {
+    if (versionParts[versionParts.size()-1].isInteger()) {
+      def base_version = versionParts[0..versionParts.size()-2].join('.')
+      version = base_version + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
+      cliMajorVersion = base_version + "." + versionParts[versionParts.size()-1]
+    } else {
+      // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
+      version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
+      cliMajorVersion = versionParts[0..versionParts.size()-1].join('.')
+    }
+
+    // differences from metadata-integration/java/versioning.gradle
+    if (versionParts[versionParts.size()-1].isInteger()) {
+      javaVersion = versionParts[0..versionParts.size()-2].join('.') + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
+    } else {
+      // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
+      javaVersion = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
+    }
+}
+
+// Note: No task, we want this executed during config phase, once for rootProject.
+def data = [
+    fullVersion: detailedVersionString,
+    cliMajorVersion: cliMajorVersion,
+    version: version,
+    javaVersion: javaVersion
+]
+
+// Convert to JSON
+def jsonBuilder = new JsonBuilder(data)
+def outputFile = file("${rootProject.buildDir}/version.json")
+
+// Ensure buildDir exists
+rootProject.buildDir.mkdirs()
+
+// Write to file
+outputFile.text = jsonBuilder.toPrettyString()
+
+println "git.properties JSON data written to ${outputFile}"

gradle/versioning/versioning.gradle

@@ -1,83 +1,33 @@
-/**
- Applies a consistent versioning scheme to all projects using this script
-
-Uses git tags to mint versions by default. 
-git tags can be of a few forms:
-- short sha (typical for a PR or a commit) (e.g. 38960ae)
-- versioned tags (typical for a release) (e.g. v0.8.45, v0.8.45.1, v0.8.45rc1, v0.8.45.1rc4)
-
-Produces the following variables and supports token replacement
-- version: server version amenable for creating jars
-- fullVersion: full version string 
-- cliMajorVersion: cli version amenable for binding to server as a default
-  0.8.44 or 0.8.44-1 (for clean tags) or 0.8.45-SNAPSHOT (for unclean repositories)
-
- All inference can be overridden by passing in the releaseVersion property
- e.g. -PreleaseVersion=0.2.3.4 will set the jar version to 0.2.3-4
-
- **/
-
-
+import groovy.json.JsonSlurper
 import org.apache.tools.ant.filters.ReplaceTokens
 
+
 def detailedVersionString = "0.0.0-unknown-SNAPSHOT"
 def cliMajorVersion = "0.15.0" // base default cli major version
-def snapshotVersion = false
-if (project.hasProperty("releaseVersion")) {
-  version = releaseVersion
-  detailedVersionString = releaseVersion
-} else {
-  try {
-      // apply this plugin in a try-catch block so that we can handle cases without .git directory
-      apply plugin: "com.palantir.git-version"
-      def details = versionDetails()
-      detailedVersionString = gitVersion()
-      version = details.lastTag
-      version = version.startsWith("v")? version.substring(1): version
-      def suffix = details.isCleanTag? "": "-SNAPSHOT"
-      snapshotVersion = ! details.isCleanTag
-    }
-    catch (Exception e) {
-          e.printStackTrace()
-          // last fall back
-          version = detailedVersionString
-    }
-}
 
-// trim version if it is of size 4 to size 3
-def versionParts = version.tokenize(".")
-cliMajorVersion = version
-if (versionParts.size() > 3) {
-  // at-least 4 part version
-  // we check if the 4th part is a .0 in which case we want to create a release
-  if ((versionParts.size() == 4) && (versionParts[3] == '0')) {
-    versionParts = versionParts[0..2]
-  }
-  version = versionParts[0..2].join('.')
-  if (versionParts.size() > 3) {
-      version = version + "-" + versionParts[3..versionParts.size()-1].join('-')
+def inputFile = file("${rootProject.buildDir}/version.json")
+
+task readJsonData {
+  if (inputFile.exists()) {
+    def jsonSlurper = new JsonSlurper()
+    def data = jsonSlurper.parse(inputFile)
+
+    detailedVersionString = data.fullVersion
+    cliMajorVersion = data.cliMajorVersion
+    version = data.version
+  } else {
+    println "git.properties JSON file not found: ${inputFile.path}"
   }
 }
 
-if (snapshotVersion) {
-    if (versionParts[versionParts.size()-1].isInteger()) {
-      def base_version = versionParts[0..versionParts.size()-2].join('.')
-      version = base_version + '.' + (versionParts[versionParts.size()-1].toInteger()+1).toString() + "-SNAPSHOT"
-      cliMajorVersion = base_version + "." + versionParts[versionParts.size()-1]
-    } else {
-      // we are unable to part the last token as an integer, so we just append SNAPSHOT to this version
-      version = versionParts[0..versionParts.size()-1].join('.') + '-SNAPSHOT'
-      cliMajorVersion = versionParts[0..versionParts.size()-1].join('.')
-  }
+task printVersionDetails() {
+  println("fullVersion=" + detailedVersionString)
+  println("cliMajorVersion=" + cliMajorVersion)
+  println("version=" +  version)
 }
 
 processResources {
   filter(ReplaceTokens, tokens:[fullVersion: detailedVersionString])
   filter(ReplaceTokens, tokens:[cliMajorVersion: cliMajorVersion])
 }
 
-task printVersionDetails() {
-  println("fullVersion=" + detailedVersionString)
-  println("cliMajorVersion=" + cliMajorVersion)
-  println("version=" +  version)
-}

metadata-events/mxe-schemas/build.gradle

@@ -19,19 +19,21 @@ generateDataTemplate.dependsOn copyMetadataModels
 mainCopySchemas.dependsOn copyMetadataModels
 pegasus.main.generationModes = [PegasusGenerationMode.PEGASUS, PegasusGenerationMode.AVRO]
 
-task copyOriginalAvsc(type: Copy, dependsOn: generateAvroSchema) {
+task renameNamespace(type: Copy, dependsOn: generateAvroSchema) {
   from("src/mainGeneratedAvroSchema/avro")
   into file("src/renamed/avro")
-}
 
-task renameNamespace(type: Exec, dependsOn: copyOriginalAvsc) {
-  commandLine 'bash', './rename-namespace.sh'
+  doLast {
+    project.exec {
+      commandLine 'bash', './rename-namespace.sh'
+    }
+  }
 }
 
 build.dependsOn renameNamespace
 
 clean {
-  project.delete('src/main/pegasus')
-  project.delete('src/mainGeneratedAvroSchema/avro')
-  project.delete('src/renamed/avro')
+  delete 'src/main/pegasus'
+  delete 'src/mainGeneratedAvroSchema/avro'
+  delete 'src/renamed/avro'
 }

metadata-events/mxe-utils-avro/build.gradle

@@ -36,5 +36,5 @@ compileJava.dependsOn copyOriginalMXESchemas
 processResources.dependsOn copyOriginalMXESchemas
 
 clean {
-  project.delete("src/main/resources/avro")
+  delete "src/main/resources/avro"
 }