feat: integrate iceberg initial load support via jvm grpc

.dockerignore

@@ -13,3 +13,12 @@ ui/.next
 .gitignore
 .github
 .gitmodules
+
+
+build
+*.jar
+.idea
+.gradle
+.dockerignore
+.sdkmanrc
+

.github/actions/genprotos/action.yml

@@ -12,7 +12,7 @@ runs:
           ./flow/generated/protos
           ./nexus/pt/src/gen
           ./ui/grpc_generated
-        key: ${{ runner.os }}-build-genprotos-${{ hashFiles('./protos/peers.proto', './protos/flow.proto', './protos/route.proto') }}
+        key: ${{ runner.os }}-build-genprotos-${{ hashFiles('./protos/peers.proto', './protos/flow.proto', './protos/route.proto', './protos/flow-jvm.proto') }}
 
     - if: steps.cache.outputs.cache-hit != 'true'
       uses: actions/setup-go@v5

.github/workflows/flow-jvm-build.yml

@@ -0,0 +1,38 @@
+name: Build & Test Flow JVM
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    paths: [flow-jvm/**, protos/**, .sdkmanrc]
+
+jobs:
+  build-test:
+    name: Build & Test Flow JVM
+    strategy:
+      matrix:
+        runner: [ubuntu-latest]
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '21'
+          cache: 'gradle'
+
+      - name: Check dependency versions
+        run: |
+          java -version
+          gradle -version
+        working-directory: flow-jvm
+      - name: Download/Setup gradle wrapper
+        working-directory: flow-jvm
+        run: |
+          gradle wrapper
+      - name: Build
+        working-directory: flow-jvm
+        run: ./gradlew quarkusGenerateCode --stacktrace --info && ./gradlew build test --stacktrace --info

.sdkmanrc

@@ -0,0 +1,5 @@
+# Enable auto-env through the sdkman_auto_env config
+# Add key=value pairs of SDKs to use below
+java=21.0.3-tem
+quarkus=3.11.3
+gradle=8.6

docker-bake.hcl

@@ -16,7 +16,8 @@ group "default" {
     "flow-worker",
     "flow-api",
     "flow-snapshot-worker",
-    "peerdb-ui"
+    "peerdb-ui",
+    "flow-jvm",
   ]
 }
 
@@ -90,3 +91,17 @@ target "peerdb-ui" {
     "${REGISTRY}/peerdb-ui:${SHA_SHORT}",
   ]
 }
+
+target "flow-jvm" {
+  context    = "."
+  dockerfile = "stacks/jvm.Dockerfile"
+  target     = "runner"
+  platforms = [
+    "linux/amd64",
+    "linux/arm64",
+  ]
+  tags = [
+    "${REGISTRY}/flow-jvm:${TAG}",
+    "${REGISTRY}/flow-jvm:${SHA_SHORT}",
+  ]
+}

flow-jvm/.dockerignore

@@ -0,0 +1,13 @@
+#*
+#!build/*-runner
+#!build/*-runner.jar
+#!build/lib/*
+#!build/quarkus-app/*
+build
+*.jar
+.idea
+.gradle
+src/main/docker
+.dockerignore
+.git
+.sdkmanrc

flow-jvm/.editorconfig

@@ -0,0 +1,3 @@
+[*.java]
+ij_java_names_count_to_use_import_on_demand = 999
+ij_java_class_count_to_use_import_on_demand = 999

flow-jvm/.gitattributes

@@ -0,0 +1,3 @@
+gradlew linguist-generated=true
+gradlew.bat linguist-generated=true
+

flow-jvm/.gitignore

@@ -0,0 +1,41 @@
+# Gradle
+.gradle/
+build/
+
+# Eclipse
+.project
+.classpath
+.settings/
+bin/
+
+# IntelliJ
+.idea
+*.ipr
+*.iml
+*.iws
+
+# NetBeans
+nb-configuration.xml
+
+# Visual Studio Code
+.vscode
+.factorypath
+
+# OSX
+.DS_Store
+
+# Vim
+*.swp
+*.swo
+
+# patch
+*.orig
+*.rej
+
+# Local environment
+.env
+
+# Plugin directory
+/.quarkus/cli/plugins/
+
+*.jar

flow-jvm/.sdkmanrc

@@ -0,0 +1 @@
+../.sdkmanrc

flow-jvm/README.md

@@ -0,0 +1,73 @@
+# flow-jvm
+
+
+## Dependencies
+Install sdkman and run `sdk env install` to setup the environment and run `gradle wrapper` to install gradle wrapper.
+
+
+
+This project uses Quarkus, the Supersonic Subatomic Java Framework.
+
+If you want to learn more about Quarkus, please visit its website: https://quarkus.io/ .
+
+
+## Generate the java code from proto files
+```shell script
+./gradlew quarkusGenerateCode
+```
+
+or
+```shell
+./gradlew clean quarkusGenerateCode
+```
+
+## Running the application in dev mode (`quarkusGenerateCode` can be skipped)
+
+You can run your application in dev mode that enables live coding using:
+```shell script
+./gradlew quarkusDev
+```
+
+> **_NOTE:_**  Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:9801/q/dev/.
+
+## Packaging and running the application
+
+The application can be packaged using:
+```shell script
+./gradlew build
+```
+It produces the `quarkus-run.jar` file in the `build/quarkus-app/` directory.
+Be aware that it’s not an _über-jar_ as the dependencies are copied into the `build/quarkus-app/lib/` directory.
+
+The application is now runnable using `java -jar build/quarkus-app/quarkus-run.jar`.
+
+If you want to build an _über-jar_, execute the following command:
+```shell script
+./gradlew build -Dquarkus.package.jar.type=uber-jar
+```
+
+The application, packaged as an _über-jar_, is now runnable using `java -jar build/*-runner.jar`.
+
+## Creating a native executable
+
+You can create a native executable using:
+```shell script
+./gradlew build -Dquarkus.native.enabled=true
+```
+
+Or, if you don't have GraalVM installed, you can run the native executable build in a container using:
+```shell script
+./gradlew build -Dquarkus.native.enabled=true -Dquarkus.native.container-build=true
+```
+
+You can then execute your native executable with: `./build/flow-jvm-0.0.1-SNAPSHOT-runner`
+
+If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling.
+
+## Provided Code
+
+### REST
+
+Easily start your REST Web Services
+
+[Related guide section...](https://quarkus.io/guides/getting-started-reactive#reactive-jax-rs-resources)

flow-jvm/build.gradle

@@ -0,0 +1,112 @@
+plugins {
+    id 'java'
+    id 'io.quarkus'
+}
+
+repositories {
+    mavenCentral()
+    mavenLocal()
+}
+
+ext {
+    icebergLibVersion = '1.5.2'
+}
+
+
+dependencies {
+    implementation 'io.quarkus:quarkus-config-yaml'
+    implementation enforcedPlatform("${quarkusPlatformGroupId}:${quarkusPlatformArtifactId}:${quarkusPlatformVersion}")
+    implementation 'io.quarkus:quarkus-arc'
+    implementation 'io.quarkus:quarkus-grpc'
+    implementation 'io.quarkus:quarkus-rest'
+    implementation 'io.quarkus:quarkus-logging-json'
+
+    // Logging adapter for dependencies, also prevents duplicate slf4j binding warnings https://quarkus.io/guides/logging#logging-apis
+    implementation("org.jboss.logging:commons-logging-jboss-logging")
+    implementation("org.jboss.logmanager:log4j-jboss-logmanager")
+    implementation("org.jboss.logmanager:log4j2-jboss-logmanager")
+    implementation("org.jboss.slf4j:slf4j-jboss-logmanager")
+
+    implementation "org.apache.iceberg:iceberg-core:${icebergLibVersion}"
+    implementation "org.apache.iceberg:iceberg-common:${icebergLibVersion}"
+    implementation "org.apache.iceberg:iceberg-data:${icebergLibVersion}"
+    implementation "org.apache.iceberg:iceberg-parquet:${icebergLibVersion}"
+
+    // This is forced due to version conflicts
+    implementation 'io.grpc:grpc-protobuf:1.63.0'
+    implementation 'com.google.protobuf:protobuf-java:4.27.1'
+
+    implementation 'org.apache.avro:avro:1.11.3'
+
+    implementation 'org.apache.hadoop:hadoop-client:3.4.0'
+    implementation 'org.apache.hadoop:hadoop-common:3.4.0'
+
+    // Drivers for JDBC Catalogs
+    runtimeOnly 'org.postgresql:postgresql:42.7.3'
+
+    // AWS Dependencies
+    implementation "org.apache.iceberg:iceberg-aws:${icebergLibVersion}"
+    runtimeOnly "org.apache.iceberg:iceberg-aws-bundle:${icebergLibVersion}"
+//    runtimeOnly 'software.amazon.awssdk:bundle:2.25.60'
+
+//    // HIVE4 Dependencies
+//    implementation 'org.apache.hive:hive-iceberg-catalog:4.0.0'
+//    // DO NOT USE THE BELOW DEPENDENCIES https://github.com/apache/iceberg/issues/10429
+////    implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergLibVersion}"
+////    runtimeOnly "org.apache.hive:hive-metastore:4.0.0"
+
+    // HIVE 3 Dependencies
+    implementation "org.apache.iceberg:iceberg-hive-metastore:${icebergLibVersion}"
+    runtimeOnly "org.apache.hive:hive-metastore:3.1.3"
+
+
+
+
+    // GCP Dependencies
+    implementation "org.apache.iceberg:iceberg-gcp:${icebergLibVersion}"
+    // This is currently causing issues with GRPC versions mismatch
+    runtimeOnly "org.apache.iceberg:iceberg-gcp-bundle:${icebergLibVersion}"
+
+
+    testImplementation 'io.quarkus:quarkus-junit5'
+    testImplementation 'io.rest-assured:rest-assured'
+
+}
+
+
+configurations.configureEach {
+//    // ch.qos.logback:logback-core
+//    exclude group: 'ch.qos.logback', module: 'logback-core'
+    // ch.qos.logback:logback-classic
+    exclude group: 'ch.qos.logback', module: 'logback-classic'
+    // org.apache.logging.log4j:log4j-slf4j-impl
+    exclude group: 'org.apache.logging.log4j', module: 'log4j-slf4j-impl'
+    // org.slf4j:slf4j-reload4j
+    exclude group: 'org.slf4j', module: 'slf4j-reload4j'
+}
+
+
+group 'io.peerdb'
+version '0.0.1-SNAPSHOT'
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_21
+    targetCompatibility = JavaVersion.VERSION_21
+}
+
+test {
+    systemProperty "java.util.logging.manager", "org.jboss.logmanager.LogManager"
+}
+compileJava {
+    options.encoding = 'UTF-8'
+    options.compilerArgs << '-parameters'
+}
+
+compileTestJava {
+    options.encoding = 'UTF-8'
+}
+
+
+quarkus {
+    quarkusBuildProperties.put("quarkus.grpc.codegen.proto-directory", "${project.projectDir}/../protos")
+}

flow-jvm/gradle.properties

@@ -0,0 +1,7 @@
+#Gradle properties
+#Fri May 24 04:59:17 IST 2024
+quarkusPlatformArtifactId=quarkus-bom
+quarkusPlatformGroupId=io.quarkus.platform
+quarkusPlatformVersion=3.11.3
+quarkusPluginId=io.quarkus
+quarkusPluginVersion=3.11.3

flow-jvm/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists