CORE-4992: Create a ZIP artifact containing the YourKit profiler agen…

…t. (#1457)

* CORE-4992: Create a ZIP artifact containing the YourKit profiler agent.

* Optionally add profiler agent to Docker images.

* Refactor publishOSGiImage task into corda.docker-app convention plugin.

* Add profiling support to the chart.

* Add engineering-tools-maven repository for YourKit Java Profiler agent.

applications/p2p-gateway/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'corda.common-publishing'
     id 'corda.common-app'
+    id 'corda.docker-app'
 }
 
 description 'Gateway Application'

applications/p2p-link-manager/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'corda.common-publishing'
     id 'corda.common-app'
+    id 'corda.docker-app'
 }
 
 description 'P2P Link Manager Application'
@@ -53,5 +54,5 @@ dependencies {
     implementation project(":libs:messaging:messaging")
     implementation project(":components:configuration:configuration-read-service")
 
-    image "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
+    dockerImage "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
 }

applications/tools/p2p-test/app-simulator/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'corda.common-publishing'
     id 'corda.common-app'
+    id 'corda.docker-app'
 }
 
 description "P2P Testing tools - Application-level simulator"

applications/workers/release/combined-worker/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'corda.common-publishing'
     id 'corda.quasar-app'
+    id 'corda.docker-app'
 }
 
 description 'Combined Worker'

applications/workers/release/crypto-worker/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'corda.common-publishing'
     id 'corda.common-app'
+    id 'corda.docker-app'
 }
 
 description 'Crypto Worker'
@@ -30,5 +31,5 @@ dependencies {
     testRuntimeOnly "org.apache.felix:org.apache.felix.framework:$felixVersion"
     testImplementation "org.mockito.kotlin:mockito-kotlin:$mockitoKotlinVersion"
 
-    image "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
+    dockerImage "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
 }

applications/workers/release/db-worker/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'corda.common-publishing'
     id 'corda.common-app'
+    id 'corda.docker-app'
 }
 
 description 'DB Worker'
@@ -41,5 +42,5 @@ dependencies {
         exclude group: 'org.osgi'
     }
 
-    image "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
+    dockerImage "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
 }

applications/workers/release/flow-worker/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'corda.common-publishing'
     id 'corda.quasar-app'
+    id 'corda.docker-app'
 }
 
 description 'Flow Worker'
@@ -41,5 +42,5 @@ dependencies {
         exclude group: 'org.osgi'
     }
 
-    image "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
+    dockerImage "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
 }

applications/workers/release/member-worker/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'corda.common-publishing'
     id 'corda.common-app'
+    id 'corda.docker-app'
 }
 
 description 'Member Worker'
@@ -25,5 +26,5 @@ dependencies {
     runtimeOnly "org.apache.felix:org.apache.felix.configadmin:$felixConfigAdminVersion"
     runtimeOnly project(":libs:messaging:kafka-message-bus-impl")
 
-    image "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
+    dockerImage "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
 }

applications/workers/release/rpc-worker/build.gradle

@@ -1,6 +1,7 @@
 plugins {
     id 'corda.common-publishing'
     id 'corda.common-app'
+    id 'corda.docker-app'
 }
 
 description 'RPC Worker'
@@ -56,7 +57,7 @@ dependencies {
     e2eTestImplementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jacksonVersion"
     e2eTestImplementation "com.fasterxml.jackson.module:jackson-module-kotlin:$jacksonVersion"
 
-    image "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
+    dockerImage "io.opentelemetry.javaagent:opentelemetry-javaagent:$openTelemetryVersion"
 }
 
 tasks.register('e2eTest', Test) {

build.gradle

@@ -11,7 +11,7 @@ buildscript {
 }
 
 plugins {
-    id 'org.jetbrains.kotlin.jvm'
+    id 'org.jetbrains.kotlin.jvm' apply false
     id 'org.jetbrains.kotlin.plugin.allopen' apply false
     id 'org.jetbrains.kotlin.plugin.jpa' apply false
     id 'io.gitlab.arturbosch.detekt' apply false
@@ -137,19 +137,31 @@ allprojects {
         //needed for C5 binaries
         maven {
             url = "$artifactoryContextUrl/corda-os-maven"
+            authentication {
+                basic(BasicAuthentication)
+            }
             credentials {
                 username = findProperty('cordaArtifactoryUsername') ?: System.getenv('CORDA_ARTIFACTORY_USERNAME')
                 password = findProperty('cordaArtifactoryPassword') ?: System.getenv('CORDA_ARTIFACTORY_PASSWORD')
             }
         }
 
-        // needed for plugin-host dependencies
         maven {
-            url = "$artifactoryContextUrl/engineering-tools-maven-unstable-local"
+            url = "$artifactoryContextUrl/engineering-tools-maven"
+            authentication {
+                basic(BasicAuthentication)
+            }
             credentials {
                 username = findProperty('cordaArtifactoryUsername') ?: System.getenv('CORDA_ARTIFACTORY_USERNAME')
                 password = findProperty('cordaArtifactoryPassword') ?: System.getenv('CORDA_ARTIFACTORY_PASSWORD')
             }
+            content {
+                // YourKit Java Profiler agent artifact.
+                // To publish agent locally:
+                //   $ cd profiler
+                //   $ ../gradlew publishToMavenLocal
+                includeGroup 'com.yourkit.corda'
+            }
         }
 
         def cordaUseCache = System.getenv("CORDA_USE_CACHE")
@@ -346,8 +358,6 @@ subprojects {
     task allDependencies(type: DependencyReportTask) {}
 
     configurations {
-        // configuration used by DeployableContainerBuilder to add dependencies to Docker images
-        image { transitive = false }
         all {
             resolutionStrategy {
                 // FORCE Gradle to use latest dynamic versions.
@@ -371,43 +381,6 @@ subprojects {
             "javax.persistence.MappedSuperclass"
         )
     }
-
-    // apply docker publishing to workers and select testing applications
-    def nonWorkerImages = ['p2p-gateway', 'p2p-link-manager', 'app-simulator', 'configuration-publisher']
-    if (it.toString().contains(":applications:workers:release:") || nonWorkerImages.contains(it.name)) {
-
-        tasks.register('publishOSGiImage', DeployableContainerBuilder) {
-            // Bit of a hack to ensure we always get a flat list of tasks
-            def sourceTasks = (List<Task>)[project.tasks.named("appJar").get()].flatten()
-            dependsOn(sourceTasks)
-            it.sourceTasks = sourceTasks
-            it.useShortName = true
-            if (project.hasProperty('jibRemotePublish')) {
-                remotePublish = jibRemotePublish.toBoolean()
-            }
-
-            if (project.hasProperty('isReleaseCandidate')) {
-                releaseCandidate = isReleaseCandidate.toBoolean()
-            }
-
-            if (project.hasProperty('isNightly')) {
-                nightlyBuild = isNightly.toBoolean()
-            }
-
-            if (project.hasProperty('isPreTest')) {
-                preTest = isPreTest.toBoolean()
-            }
-
-            if (project.hasProperty('baseImage')) {
-                baseImageName = baseImage
-            }
-
-            if(project.hasProperty('useDockerDaemon')){
-                useDaemon = useDockerDaemon.toBoolean()
-            }
-        }
-
-    }
 }
 
 // report updatable dependencies: gradle dependencyUpdates
@@ -416,6 +389,7 @@ def isNonStable = { String version ->
     def regex = /^[0-9,.v-]+(-r)?$/
     return !stableKeyword && !(version ==~ regex)
 }
+
 tasks.named("dependencyUpdates").configure {
     rejectVersionIf {
         isNonStable(it.candidate.version) && !isNonStable(it.currentVersion)

buildSrc/src/main/groovy/DeployableContainerBuilder.groovy

@@ -2,13 +2,21 @@ import com.google.cloud.tools.jib.api.*
 import com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath
 import com.google.cloud.tools.jib.api.buildplan.Platform
 import org.gradle.api.DefaultTask
-import org.gradle.api.Task
+import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.ListProperty
 import org.gradle.api.provider.MapProperty
 import org.gradle.api.provider.Property
 import org.gradle.api.provider.ProviderFactory
-import org.gradle.api.tasks.*
+import org.gradle.api.tasks.Exec
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFiles
+import org.gradle.api.tasks.Internal
+import org.gradle.api.tasks.Optional
+import org.gradle.api.tasks.PathSensitive
+import org.gradle.api.tasks.SkipWhenEmpty
+import org.gradle.api.tasks.TaskAction
+import static org.gradle.api.tasks.PathSensitivity.RELATIVE
 
 import javax.inject.Inject
 import java.nio.file.Files
@@ -101,9 +109,16 @@ abstract class DeployableContainerBuilder extends DefaultTask {
     final ListProperty<String> arguments =
             getObjects().listProperty(String)
 
-    @Input
-    final ListProperty<Task> sourceTasks =
-            getObjects().listProperty(Task)
+    @PathSensitive(RELATIVE)
+    @SkipWhenEmpty
+    @InputFiles
+    final ConfigurableFileCollection sourceFiles =
+            getObjects().fileCollection()
+
+    @PathSensitive(RELATIVE)
+    @InputFiles
+    final ConfigurableFileCollection extraSourceFiles =
+            getObjects().fileCollection()
 
     @Input
     final Property<String> overrideEntryName =
@@ -135,7 +150,6 @@ abstract class DeployableContainerBuilder extends DefaultTask {
 
     @TaskAction
     def updateImage() {
-        def outputFiles = sourceTasks.get().collect{ it -> it.getOutputs().files.files }.flatten() as List<File>
         def buildBaseDir = temporaryDir.toPath()
         def containerizationDir = Paths.get("$buildBaseDir/containerization/")
 
@@ -148,7 +162,7 @@ abstract class DeployableContainerBuilder extends DefaultTask {
             Files.createDirectories(containerizationDir)
         }
 
-        outputFiles.forEach{
+        sourceFiles.forEach{
             def jarName = useShortName
                 ? it.name.replace("corda-", "").replace("-${project.version}", "")
                 : it.name
@@ -175,7 +189,7 @@ abstract class DeployableContainerBuilder extends DefaultTask {
             logger.info("Resolving base image ${baseImageName.get()}: ${baseImageTag.get()} from remote repo")
             builder = setCredentialsOnBaseImage(builder)
         }
-        List<Path> imageFiles = project.configurations.getByName("image").collect { it.toPath() }
+        List<Path> imageFiles = extraSourceFiles.collect { it.toPath() }
         if (!imageFiles.empty) {
             builder.addLayer(imageFiles, CONTAINER_LOCATION)
         }

buildSrc/src/main/groovy/corda.docker-app.gradle

@@ -0,0 +1,53 @@
+plugins {
+    id 'corda.common-app'
+}
+
+configurations {
+    dockerImage {
+        canBeConsumed = false
+        transitive = false
+    }
+
+    profilerAgent {
+        canBeConsumed = false
+        visible = false
+    }
+}
+
+if (enableProfiling.toBoolean()) {
+    dependencies {
+        // Unzip the Java profiler into the Docker image.
+        profilerAgent "com.yourkit.corda:yourkit-agent-linux-x86-64:$profilerVersion@zip"
+        dockerImage provider { zipTree(configurations.profilerAgent.singleFile) }
+    }
+}
+
+tasks.register('publishOSGiImage', DeployableContainerBuilder) {
+    it.sourceFiles.setFrom(tasks.named('appJar', Jar))
+    it.extraSourceFiles.setFrom(configurations.dockerImage)
+    it.useShortName = true
+
+    if (project.hasProperty('jibRemotePublish')) {
+        remotePublish = jibRemotePublish.toBoolean()
+    }
+
+    if (project.hasProperty('isReleaseCandidate')) {
+        releaseCandidate = isReleaseCandidate.toBoolean()
+    }
+
+    if (project.hasProperty('isNightly')) {
+        nightlyBuild = isNightly.toBoolean()
+    }
+
+    if (project.hasProperty('isPreTest')) {
+        preTest = isPreTest.toBoolean()
+    }
+
+    if (project.hasProperty('baseImage')) {
+        baseImageName = baseImage
+    }
+
+    if (project.hasProperty('useDockerDaemon')) {
+        useDaemon = useDockerDaemon.toBoolean()
+    }
+}

charts/corda/templates/_helpers.tpl

@@ -163,6 +163,9 @@ Worker environment variables
   value: {{- if ( get .Values.workers .worker ).debug.enabled }}
       -agentlib:jdwp=transport=dt_socket,server=y,address=5005,suspend={{ if ( get .Values.workers .worker ).debug.suspend }}y{{ else }}n{{ end }}
     {{- end -}}
+    {{- if ( get .Values.workers .worker ).profiling.enabled }}
+      -agentpath:/opt/override/libyjpagent.so=exceptions=disable,port=10045,listen=all
+    {{- end -}}
     {{- if ( get .Values.workers .worker ).verifyInstrumentation }}
       -Dco.paralleluniverse.fibers.verifyInstrumentation=true
     {{- end -}}