Skip to content

Commit

Permalink
Generate API client with kotlin, okhttp4, moshi, failsafe (#8150)
Browse files Browse the repository at this point in the history
  • Loading branch information
cgardens committed Aug 14, 2023
1 parent 408329a commit f2d38bf
Show file tree
Hide file tree
Showing 5 changed files with 275 additions and 86 deletions.
122 changes: 115 additions & 7 deletions airbyte-api/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ plugins {
id "io.airbyte.gradle.jvm.lib"
id "io.airbyte.gradle.publish"
id "org.openapi.generator"
id "org.jetbrains.kotlin.jvm"
id "org.jetbrains.kotlin.kapt"
}

def specFile = "$projectDir/src/main/openapi/config.yaml"
Expand Down Expand Up @@ -92,6 +94,91 @@ def genApiClient = tasks.register("generateApiClient", GenerateTask) {
]
}

def genApiClient2 = tasks.register("genApiClient2", GenerateTask) {
def clientOutputDir = "$buildDir/generated/api/client2"

inputs.file specFile
outputs.dir clientOutputDir

generatorName = "kotlin"
inputSpec = specFile
outputDir = clientOutputDir

apiPackage = "io.airbyte.api.client2.generated"
invokerPackage = "io.airbyte.api.client2.invoker.generated"
modelPackage = "io.airbyte.api.client2.model.generated"

schemaMappings = [
'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode',
'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode',
'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode',
'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode',
'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode',
'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode',
'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode',
'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode',
]

generateApiDocumentation = false

configOptions = [
generatePom : "false",
interfaceOnly : "true"
]

doLast {
/*
* UPDATE ApiClient.kt to use Failsafe.
*/
def apiClientFile = file('build/generated/api/client2/src/main/kotlin/org/openapitools/client/infrastructure/ApiClient.kt')
def apiClientFileText = apiClientFile.text

// replace class declaration
apiClientFileText = apiClientFileText.replace(
'open class ApiClient(val baseUrl: String, val client: OkHttpClient = defaultClient) {',
'open class ApiClient(val baseUrl: String, val client: OkHttpClient = defaultClient, val policy : RetryPolicy<Response> = RetryPolicy.ofDefaults()) {')

// replace execute call
apiClientFileText = apiClientFileText.replace(
'val response = client.newCall(request).execute()',
'''val call = client.newCall(request)
val failsafeCall = FailsafeCall.with(policy).compose(call)
val response: Response = failsafeCall.execute()''')

// add imports if not exist
if(!apiClientFileText.contains("import dev.failsafe.RetryPolicy")) {
def newImports = '''import dev.failsafe.RetryPolicy
import dev.failsafe.okhttp.FailsafeCall'''
apiClientFileText = apiClientFileText.replaceFirst('import ', newImports + '\nimport ')

}
apiClientFile.write(apiClientFileText)

/*
* UPDATE domain clients to use Failsafe.
*/
def dir = file('build/generated/api/client2/src/main/kotlin/io/airbyte/api/client2/generated')
dir.eachFile { domainClient ->
if (domainClient.name.endsWith('.kt')) {
def domainClientFileText = domainClient.text

// replace class declaration
domainClientFileText = domainClientFileText.replaceAll(
/class (\S+)\(basePath: kotlin.String = defaultBasePath, client: OkHttpClient = ApiClient.defaultClient\) : ApiClient\(basePath, client\)/,
'class $1(basePath: kotlin.String = defaultBasePath, client: OkHttpClient = ApiClient.defaultClient, policy : RetryPolicy<okhttp3.Response> = RetryPolicy.ofDefaults()) : ApiClient(basePath, client, policy)'
)

// add imports if not exist
if(!domainClientFileText.contains("import dev.failsafe.RetryPolicy")) {
def newImports = "import dev.failsafe.RetryPolicy"
domainClientFileText = domainClientFileText.replaceFirst('import ', newImports + '\nimport ')
}
domainClient.write(domainClientFileText)
}
}
}
}

def genApiDocs = tasks.register("generateApiDocs", GenerateTask) {
def docsOutputDir = "$buildDir/generated/api/docs"

Expand Down Expand Up @@ -161,16 +248,35 @@ def genAirbyteApiServer = tasks.register('generateAirbyteApiServer', GenerateTas

compileJava.dependsOn genApiDocs, genApiClient, genApiServer, genAirbyteApiServer

kapt {
correctErrorTypes true
}

// uses afterEvaluate because at configuration time, the kaptGenerateStubsKotlin task does not exist.
afterEvaluate {
tasks.named('kaptGenerateStubsKotlin').configure {
mustRunAfter genApiDocs, genApiClient, genApiClient2, genApiServer, genAirbyteApiServer
}
}

tasks.named("compileKotlin") {
dependsOn tasks.named("genApiClient2")
}

dependencies {
implementation libs.guava
implementation libs.commons.io
implementation libs.slf4j.api
implementation libs.jackson.datatype
implementation libs.swagger.annotations
implementation libs.failsafe.okhttp
implementation libs.guava
implementation libs.javax.annotation.api
implementation libs.javax.ws.rs.api
implementation libs.javax.validation.api
implementation libs.jackson.datatype
implementation libs.moshi.kotlin
implementation libs.okhttp
implementation libs.openapi.jackson.databind.nullable
implementation libs.reactor.core
implementation libs.slf4j.api
implementation libs.swagger.annotations

testRuntimeOnly libs.junit.jupiter.engine
testImplementation libs.bundles.junit
Expand All @@ -179,14 +285,16 @@ dependencies {

implementation platform(libs.micronaut.bom)
implementation libs.bundles.micronaut
implementation 'io.swagger.core.v3:swagger-annotations'
implementation 'io.projectreactor:reactor-core'
}

sourceSets {
main {
java {
srcDirs "$buildDir/generated/api/server/src/gen/java", "$buildDir/generated/airbyte_api/server/src/gen/java", "$buildDir/generated/api/client/src/main/java", "$projectDir/src/main/java"
srcDirs "$buildDir/generated/api/server/src/gen/java",
"$buildDir/generated/airbyte_api/server/src/gen/java",
"$buildDir/generated/api/client/src/main/java",
"$buildDir/generated/api/client2/src/main/kotlin"
"$projectDir/src/main/java"
}
resources {
srcDir "$projectDir/src/main/openapi/"
Expand Down
90 changes: 90 additions & 0 deletions airbyte-api/src/main/kotlin/AirbyteApiClient2.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package io.airbyte.api.client2

import dev.failsafe.RetryPolicy
import io.airbyte.api.client2.generated.AttemptApi
import io.airbyte.api.client2.generated.ConnectionApi
import io.airbyte.api.client2.generated.ConnectorBuilderProjectApi
import io.airbyte.api.client2.generated.DestinationApi
import io.airbyte.api.client2.generated.DestinationDefinitionApi
import io.airbyte.api.client2.generated.DestinationDefinitionSpecificationApi
import io.airbyte.api.client2.generated.HealthApi
import io.airbyte.api.client2.generated.JobRetryStatesApi
import io.airbyte.api.client2.generated.JobsApi
import io.airbyte.api.client2.generated.OperationApi
import io.airbyte.api.client2.generated.SourceApi
import io.airbyte.api.client2.generated.SourceDefinitionApi
import io.airbyte.api.client2.generated.SourceDefinitionSpecificationApi
import io.airbyte.api.client2.generated.StateApi
import io.airbyte.api.client2.generated.StreamStatusesApi
import io.airbyte.api.client2.generated.WorkspaceApi
import okhttp3.OkHttpClient

/**
* This class wraps all the generated API clients and provides a single entry point. This class is meant
* to consolidate all our API endpoints into a fluent-ish client. Our open API generators create a separate
* class per API "root-route". For example, if our API has two routes "/v1/First/get" and "/v1/Second/get",
* OpenAPI generates (essentially) the following files:
* <p>
* ApiClient.java, FirstApi.java, SecondApi.java
* <p>
* To call the API type-safely, we'd do new FirstApi(new ApiClient()).get() or new SecondApi(new
* ApiClient()).get(), which can get cumbersome if we're interacting with many pieces of the API.
* <p>
* Our new JVM (kotlin) client is designed to do a few things:
* <ol>
* <li>1. Use kotlin!</li>
* <li>2. Use OkHttp3 instead of the native java client (The native one dies on any network blip. OkHttp
* is more robust and smooths over network blips).</li>
* <li>3. Integrate failsafe (https://failsafe.dev/) for circuit breaking / retry<li>
* policies.
* </ol>
* <p>
* todo (cgardens): The LogsApi is intentionally not included because in the java client we had to do some
* work to set the correct headers in the generated code. At some point we will need to test that that
* functionality works in the new client (and if necessary, patch it). Context: https://github.com/airbytehq/airbyte/pull/1799
*/
@Suppress("MemberVisibilityCanBePrivate")
class AirbyteApiClient2
@JvmOverloads
constructor(
basePath: String,
policy: RetryPolicy<okhttp3.Response> = RetryPolicy.ofDefaults(),
httpClient: OkHttpClient = OkHttpClient(),
) {

val connectionApi: ConnectionApi
val connectorBuilderProjectApi: ConnectorBuilderProjectApi
val destinationDefinitionApi: DestinationDefinitionApi
val destinationApi: DestinationApi
val destinationSpecificationApi: DestinationDefinitionSpecificationApi
val jobsApi: JobsApi
val jobRetryStatesApi: JobRetryStatesApi
val operationApi: OperationApi
val sourceDefinitionApi: SourceDefinitionApi
val sourceApi: SourceApi
val sourceDefinitionSpecificationApi: SourceDefinitionSpecificationApi
val workspaceApi: WorkspaceApi
val healthApi: HealthApi
val attemptApi: AttemptApi
val stateApi: StateApi
val streamStatusesApi: StreamStatusesApi

init {
connectionApi = ConnectionApi(basePath = basePath, client = httpClient, policy = policy)
connectorBuilderProjectApi = ConnectorBuilderProjectApi(basePath = basePath, client = httpClient, policy = policy)
destinationDefinitionApi = DestinationDefinitionApi(basePath = basePath, client = httpClient, policy = policy)
destinationApi = DestinationApi(basePath = basePath, client = httpClient, policy = policy)
destinationSpecificationApi = DestinationDefinitionSpecificationApi(basePath = basePath, client = httpClient, policy = policy)
jobsApi = JobsApi(basePath = basePath, client = httpClient, policy = policy)
jobRetryStatesApi = JobRetryStatesApi(basePath = basePath, client = httpClient, policy = policy)
operationApi = OperationApi(basePath = basePath, client = httpClient, policy = policy)
sourceDefinitionApi = SourceDefinitionApi(basePath = basePath, client = httpClient, policy = policy)
sourceApi = SourceApi(basePath = basePath, client = httpClient, policy = policy)
sourceDefinitionSpecificationApi = SourceDefinitionSpecificationApi(basePath = basePath, client = httpClient, policy = policy)
workspaceApi = WorkspaceApi(basePath = basePath, client = httpClient, policy = policy)
healthApi = HealthApi(basePath = basePath, client = httpClient, policy = policy)
attemptApi = AttemptApi(basePath = basePath, client = httpClient, policy = policy)
stateApi = StateApi(basePath = basePath, client = httpClient, policy = policy)
streamStatusesApi = StreamStatusesApi(basePath = basePath, client = httpClient, policy = policy)
}
}
27 changes: 9 additions & 18 deletions airbyte-tests/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,18 @@ testing {
implementation(project(":airbyte-test-utils"))
implementation(project(":airbyte-commons-worker"))

implementation("com.fasterxml.jackson.core:jackson-databind")
implementation("io.github.cdimascio:java-dotenv:3.0.0")


implementation(libs.failsafe)
implementation(libs.jackson.databind)
implementation(libs.okhttp)
implementation(libs.temporal.sdk)
implementation("org.apache.commons:commons-csv:1.4")
implementation(libs.platform.testcontainers.postgresql)
implementation(libs.postgresql)
implementation("org.bouncycastle:bcprov-jdk15on:1.66")
implementation("org.bouncycastle:bcpkix-jdk15on:1.66")

// needed for fabric to connect to k8s.
runtimeOnly("org.bouncycastle:bcprov-jdk15on:1.66")
runtimeOnly("org.bouncycastle:bcpkix-jdk15on:1.66")
}
}

Expand Down Expand Up @@ -86,19 +90,6 @@ dependencies {
testImplementation(libs.assertj.core)
testImplementation(libs.junit.pioneer)
}
// Cole - Leaving these commented out, if nothing breaks I'll delete them
// test should run using the current version of the docker compose configuration.
//val taskAcceptance = tasks.register<Copy>("copyComposeFileForAcceptanceTests") {
// from("${rootDir}/docker-compose.yaml")
// into("${sourceSets.acceptanceTests.output.resourcesDir}")
//}
//tasks.named("checkstyleAcceptanceTests") {
// dependsOn(taskAcceptance)
//}
//tasks.named("pmdAcceptanceTests") {
// dependsOn(taskAcceptance)
//}
//

tasks.withType<Copy>().configureEach {
duplicatesStrategy = DuplicatesStrategy.INCLUDE
Expand Down
Loading

0 comments on commit f2d38bf

Please sign in to comment.