From c3e2afccd939586f49cf2752995ccc676bda2b7e Mon Sep 17 00:00:00 2001 From: Kavith Lokuhewage Date: Wed, 20 Nov 2019 05:57:57 +0530 Subject: [PATCH 1/6] Fix compiler plugin to check if services are there --- api/compiler/Ballerina.lock | 4 ++-- docker/compiler/Dockerfile | 5 +++-- .../playground/compiler/PlaygroundCompilerPlugin.java | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/compiler/Ballerina.lock b/api/compiler/Ballerina.lock index d76ed76..9cdeda3 100644 --- a/api/compiler/Ballerina.lock +++ b/api/compiler/Ballerina.lock @@ -1,4 +1,4 @@ -org_name = "ballerina.io" +org_name = "ballerina.playground" version = "1.0.0" lockfile_version = "1.0.0" -ballerina_version = "1.0.2" +ballerina_version = "1.0.3" diff --git a/docker/compiler/Dockerfile b/docker/compiler/Dockerfile index c985507..bee074b 100644 --- a/docker/compiler/Dockerfile +++ b/docker/compiler/Dockerfile @@ -4,11 +4,12 @@ FROM ballerina/ballerina:$BALLERINA_VERSION LABEL maintainer "ballerina.io" -COPY plugins/*.jar $BALLERINA_HOME/bre/lib - WORKDIR /api COPY api/*.jar /api +COPY plugins/*.jar $BALLERINA_HOME/bre/lib/ +RUN cp $BALLERINA_HOME/bre/lib/*.jar $(ballerina home)/bre/lib/ + USER root RUN chown -R ballerina:troupe /api && \ diff --git a/utils/compiler-plugin/src/main/java/org/ballerinalang/playground/compiler/PlaygroundCompilerPlugin.java b/utils/compiler-plugin/src/main/java/org/ballerinalang/playground/compiler/PlaygroundCompilerPlugin.java index 05eda29..5872e58 100644 --- a/utils/compiler-plugin/src/main/java/org/ballerinalang/playground/compiler/PlaygroundCompilerPlugin.java +++ b/utils/compiler-plugin/src/main/java/org/ballerinalang/playground/compiler/PlaygroundCompilerPlugin.java @@ -27,7 +27,7 @@ public void process(PackageNode packageNode) { .collect(Collectors.toList()) .forEach(serviceNode -> { diagnosticLog.logDiagnostic(Diagnostic.Kind.ERROR, serviceNode.getPosition(), - "Running Ballerina services is not allowed in playground."); + "Running Ballerina services is not allowed in playground yet."); }); }); } From 9bea998da9e6786e9c44085d28dee5b1c542cdfd Mon Sep 17 00:00:00 2001 From: Kavith Lokuhewage Date: Wed, 20 Nov 2019 05:59:10 +0530 Subject: [PATCH 2/6] Pass proper versions to image tags --- build.gradle | 2 +- docker/compiler/build.gradle | 12 ++++++------ docker/compiler/cloudbuild.yaml | 4 ++-- docker/controller/build.gradle | 12 +++++++----- docker/controller/cloudbuild.yaml | 4 ++-- docker/executor/build.gradle | 11 ++++++----- docker/executor/cloudbuild.yaml | 4 ++-- k8s/build.gradle | 7 +++---- 8 files changed, 29 insertions(+), 27 deletions(-) diff --git a/build.gradle b/build.gradle index ddada42..a19802f 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,6 @@ allprojects { } } subprojects { - version = '1.0.0-SNAPSHOT' + version = '1.0.0-m1' ext.setProperty("ballerinaVersion", "1.0.1") } diff --git a/docker/compiler/build.gradle b/docker/compiler/build.gradle index 9112bc3..24ff9ad 100644 --- a/docker/compiler/build.gradle +++ b/docker/compiler/build.gradle @@ -7,6 +7,10 @@ configurations { } } +def env = System.getenv() +def gcpProjectID = env["BPG_GCP_PROJECT_ID"] +def imageTag = 'gcr.io/' + gcpProjectID +'/playground-compiler:' + project.version + dependencies { compilerPlugins project(":utils:compiler-plugin") api project(path: ":api:compiler", configuration: "apiJar") @@ -35,14 +39,10 @@ task cloudBuild(type: Exec) { dependsOn copyCompilerPlugins inputs.files(inputFiles) environment("BALLERINA_VERSION", project.ext.ballerinaVersion) - commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", - '--substitutions=_BALLERINA_VERSION=' + project.ext.ballerinaVersion, "." + commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", + '--substitutions=_IMAGE_TAG=' + imageTag + ',_BALLERINA_VERSION=' + project.ext.ballerinaVersion, "." } -def env = System.getenv() -def gcpProjectID = env["BPG_GCP_PROJECT_ID"] -def imageTag = 'gcr.io/' + gcpProjectID +'/playground-compiler:' + project.version - task build(type: Exec) { dependsOn copyApi dependsOn copyCompilerPlugins diff --git a/docker/compiler/cloudbuild.yaml b/docker/compiler/cloudbuild.yaml index 85cd8cb..814f899 100644 --- a/docker/compiler/cloudbuild.yaml +++ b/docker/compiler/cloudbuild.yaml @@ -1,6 +1,6 @@ steps: - name: 'gcr.io/cloud-builders/docker' - args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/playground-compiler','--build-arg', + args: [ 'build', '-t', '$_IMAGE_TAG','--build-arg', 'BALLERINA_VERSION=$_BALLERINA_VERSION', '.' ] images: -- 'gcr.io/$PROJECT_ID/playground-compiler' \ No newline at end of file +- '$_IMAGE_TAG' \ No newline at end of file diff --git a/docker/controller/build.gradle b/docker/controller/build.gradle index d6fa207..5beb2b7 100644 --- a/docker/controller/build.gradle +++ b/docker/controller/build.gradle @@ -24,16 +24,18 @@ def inputFiles = [ project.file("Dockerfile") ] +def env = System.getenv() +def gcpProjectID = env["BPG_GCP_PROJECT_ID"] +def imageTag = 'gcr.io/' + gcpProjectID +'/playground-controller:' + project.version + + task cloudBuild(type: Exec) { dependsOn copyApi inputs.files(inputFiles) - commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", "." + commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", + '--substitutions=_IMAGE_TAG=' + imageTag, "." } -def env = System.getenv() -def gcpProjectID = env["BPG_GCP_PROJECT_ID"] -def imageTag = 'gcr.io/' + gcpProjectID +'/playground-controller:' + project.version - task build(type: Exec) { dependsOn copyApi inputs.files(inputFiles) diff --git a/docker/controller/cloudbuild.yaml b/docker/controller/cloudbuild.yaml index 9769e0a..dbe7790 100644 --- a/docker/controller/cloudbuild.yaml +++ b/docker/controller/cloudbuild.yaml @@ -1,5 +1,5 @@ steps: - name: 'gcr.io/cloud-builders/docker' - args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/playground-controller', '.' ] + args: [ 'build', '-t', '$_IMAGE_TAG', '.' ] images: -- 'gcr.io/$PROJECT_ID/playground-controller' \ No newline at end of file +- '$_IMAGE_TAG' \ No newline at end of file diff --git a/docker/executor/build.gradle b/docker/executor/build.gradle index 8f56ac8..e4b9222 100644 --- a/docker/executor/build.gradle +++ b/docker/executor/build.gradle @@ -24,16 +24,17 @@ def inputFiles = [ project.file("Dockerfile") ] +def env = System.getenv() +def gcpProjectID = env["BPG_GCP_PROJECT_ID"] +def imageTag = 'gcr.io/' + gcpProjectID +'/playground-executor:' + project.version + task cloudBuild(type: Exec) { dependsOn copyApi inputs.files(inputFiles) - commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", "." + commandLine "gcloud", "builds", "submit", "--config", "cloudbuild.yaml", + '--substitutions=_IMAGE_TAG=' + imageTag, "." } -def env = System.getenv() -def gcpProjectID = env["BPG_GCP_PROJECT_ID"] -def imageTag = 'gcr.io/' + gcpProjectID +'/playground-executor:' + project.version - task build(type: Exec) { dependsOn copyApi inputs.files(inputFiles) diff --git a/docker/executor/cloudbuild.yaml b/docker/executor/cloudbuild.yaml index 17819cc..a34bcd0 100644 --- a/docker/executor/cloudbuild.yaml +++ b/docker/executor/cloudbuild.yaml @@ -1,5 +1,5 @@ steps: - name: 'gcr.io/cloud-builders/docker' - args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/playground-executor', '.' ] + args: [ 'build', '-t', '$_IMAGE_TAG', '.' ] images: -- 'gcr.io/$PROJECT_ID/playground-executor' \ No newline at end of file +- '$_IMAGE_TAG' \ No newline at end of file diff --git a/k8s/build.gradle b/k8s/build.gradle index 1f652c0..61a15c9 100644 --- a/k8s/build.gradle +++ b/k8s/build.gradle @@ -2,7 +2,7 @@ def namespace = 'ballerina-playground-v2'; task deployWebServer(type: Exec) { dependsOn(":docker:nginx:cloudBuild") - environment("RELEASE_VERSION", "latest") // replace latest with project.version + environment("RELEASE_VERSION", project.version) commandLine './deploy-web-server.sh' } @@ -20,13 +20,13 @@ task undeployNfs(type: Exec) { task deployCompiler(type: Exec) { dependsOn(":docker:compiler:cloudBuild") - environment("RELEASE_VERSION", "latest") // replace latest with project.version + environment("RELEASE_VERSION", project.version) commandLine './deploy-compiler.sh' } task deployExecutor(type: Exec) { dependsOn(":docker:executor:cloudBuild") - environment("RELEASE_VERSION", "latest") // replace latest with project.version + environment("RELEASE_VERSION", project.version) commandLine './deploy-executor.sh' } @@ -73,7 +73,6 @@ task undeployController(type: Exec) { } task deployIngressResources(type: Exec) { - environment("RELEASE_VERSION", "latest") // replace latest with project.version commandLine './deploy-ingress-resources.sh' } From 6c0f48752d144fedf50b80b4ae80c313c48b1070 Mon Sep 17 00:00:00 2001 From: Kavith Lokuhewage Date: Wed, 20 Nov 2019 07:05:28 +0530 Subject: [PATCH 3/6] Invalidate cache if invalid --- api/controller/Ballerina.toml | 2 +- .../src/playground_commons/redis/natives.bal | 5 +++ .../src/playground_controller/run.bal | 36 ++++++++++++++++--- .../playground/cache/RedisCache.java | 8 +++-- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/api/controller/Ballerina.toml b/api/controller/Ballerina.toml index e6382bb..1c5a192 100644 --- a/api/controller/Ballerina.toml +++ b/api/controller/Ballerina.toml @@ -8,7 +8,7 @@ target = "java8" [[platform.libraries]] artifactId = "redis-client" version = "@project.version@" - path = "./libs/redis-client-1.0.0-SNAPSHOT.jar" + path = "./libs/redis-client-1.0.0-m1.jar" groupId = "ballerina" modules = ["playground_commons"] diff --git a/api/controller/src/playground_commons/redis/natives.bal b/api/controller/src/playground_commons/redis/natives.bal index bfbb1a7..a9ed30b 100644 --- a/api/controller/src/playground_commons/redis/natives.bal +++ b/api/controller/src/playground_commons/redis/natives.bal @@ -34,6 +34,11 @@ public function redisSet(string key, string value) = @java:Method { class:"org/ballerinalang/playground/cache/RedisCache" } external; +public function redisRemove(string key) = @java:Method { + name: "remove", + class:"org/ballerinalang/playground/cache/RedisCache" +} external; + public function redisPushToList(string key, string value) = @java:Method { name: "pushToList", class:"org/ballerinalang/playground/cache/RedisCache" diff --git a/api/controller/src/playground_controller/run.bal b/api/controller/src/playground_controller/run.bal index dd7c34c..d9008c9 100644 --- a/api/controller/src/playground_controller/run.bal +++ b/api/controller/src/playground_controller/run.bal @@ -146,13 +146,17 @@ function run(http:WebSocketCaller caller, RunData data) returns error? { } } }; - if (commons:redisContains(cacheId)) { - log:printDebug("Found cached responses. "); - string[] cachedResponses = commons:redisGetList(cacheId); - foreach string response in cachedResponses { + [boolean, boolean, string[]] checkCacheResult = checkCache(cacheId); + if (checkCacheResult[0] && checkCacheResult[1]) { + log:printDebug("Found valid cached responses. "); + foreach string response in checkCacheResult[2] { respHandler(response, false); } } else { + // invalidate cache entry if it exists & invalid + if (checkCacheResult[0] && !checkCacheResult[1]) { + commons:redisRemove(cacheId); + } log:printDebug("Cached responses not found. Compiling. "); error? compilerResult = invokeCompiler(respHandler, data, compilerCallBack); if (compilerResult is error) { @@ -161,6 +165,30 @@ function run(http:WebSocketCaller caller, RunData data) returns error? { } } + +# Check if a valid cache exists and return cache if it exists. +# +# + cacheId - cacheId Parameter +# + return - [isCacheAvailable, isCacheValid, cache] +function checkCache(string cacheId) returns [boolean, boolean, string[]] { + if (commons:redisContains(cacheId)) { + string[] cachedResponses = commons:redisGetList(cacheId); + string lastResponse = cachedResponses[cachedResponses.length() - 1]; + json|error jsonResponse = lastResponse.fromJsonString(); + if (jsonResponse is json) { + PlaygroundResponse|error response = PlaygroundResponse.constructFrom(jsonResponse); + if (response is PlaygroundResponse) { + boolean isValid = response.'type == ControlResponse && + (response.data == "Finished Executing." + || response.data == "Finished Compiling with errors."); + return [true, isValid, cachedResponses]; + } + } + return [true, false, cachedResponses]; + } + return [false, false, []]; +} + function createJSONResponse(PlaygroundResponse reponse) returns json|error { json jsonResp = check json.constructFrom(reponse); return jsonResp.toJsonString(); diff --git a/utils/redis-client/src/main/java/org/ballerinalang/playground/cache/RedisCache.java b/utils/redis-client/src/main/java/org/ballerinalang/playground/cache/RedisCache.java index 2082a7e..c3d4a4a 100644 --- a/utils/redis-client/src/main/java/org/ballerinalang/playground/cache/RedisCache.java +++ b/utils/redis-client/src/main/java/org/ballerinalang/playground/cache/RedisCache.java @@ -2,8 +2,6 @@ import redis.clients.jedis.Jedis; -import java.util.List; - /** * Cache Adaptor for Redis */ @@ -40,6 +38,12 @@ public static void set(String key, String value) { } } + public static void remove(String key) { + try (Jedis client = redisClient.getWriteClient()) { + client.del(key); + } + } + public static void pushToList(String key, String value) { try (Jedis client = redisClient.getWriteClient()) { client.lpush(key, value); From bbc3045443ea516008a86df3675e1071b7b481bd Mon Sep 17 00:00:00 2001 From: Kavith Lokuhewage Date: Wed, 20 Nov 2019 07:12:38 +0530 Subject: [PATCH 4/6] Avoid using in-mem cache in executor --- api/executor/Ballerina.lock | 2 +- .../src/playground_executor/execute.bal | 38 +++++-------------- .../playground_executor/utils/cache_utils.bal | 15 -------- 3 files changed, 11 insertions(+), 44 deletions(-) diff --git a/api/executor/Ballerina.lock b/api/executor/Ballerina.lock index df89a1f..9cdeda3 100644 --- a/api/executor/Ballerina.lock +++ b/api/executor/Ballerina.lock @@ -1,4 +1,4 @@ -org_name = "ballerina.io" +org_name = "ballerina.playground" version = "1.0.0" lockfile_version = "1.0.0" ballerina_version = "1.0.3" diff --git a/api/executor/src/playground_executor/execute.bal b/api/executor/src/playground_executor/execute.bal index bd96eba..5680389 100644 --- a/api/executor/src/playground_executor/execute.bal +++ b/api/executor/src/playground_executor/execute.bal @@ -19,35 +19,17 @@ function getAppJar(string cacheId) returns string|error { function execute(ExecuteData data) returns ExecutorResponse|error { log:printDebug("Executing request: " + data.toString()); - string? cacheId = getCacheId(data.sourceCode, data.balVersion); - if (cacheId is string) { - log:printDebug("Searching for cached output. Cache ID: " + cacheId); - boolean hasCachedOutputResult = hasCachedOutput(cacheId); - if (hasCachedOutputResult) { - string? cachedOutput = getCachedOutput(cacheId); - if (cachedOutput is string) { - log:printDebug("Found cached output. " + cachedOutput); - return createDataResponse(cachedOutput); - } else { - return createErrorResponse("Invalid cached output returned from cache."); - } - } else { - log:printDebug("Cached output not found."); - string appJar = check getAppJar(cacheId); - log:printDebug("Executing jar: " + appJar); - string cwd = check filepath:parent(appJar); - string|error execStatus = execJar(cwd, appJar); - if (execStatus is error) { - log:printError("Error while executing jar: " + execStatus.reason()); - return createErrorResponse(execStatus.reason()); - } else { - log:printDebug("Executed jar: " + execStatus); - setCachedOutput(cacheId, execStatus); - return createDataResponse(execStatus); - } - } + string cacheId = getCacheId(data.sourceCode, data.balVersion); + string appJar = check getAppJar(cacheId); + log:printDebug("Executing jar: " + appJar); + string cwd = check filepath:parent(appJar); + string|error execStatus = execJar(cwd, appJar); + if (execStatus is error) { + log:printError("Error while executing jar: " + execStatus.reason()); + return createErrorResponse(execStatus.reason()); } else { - return createErrorResponse("Cannot access cache"); + log:printDebug("Executed jar: " + execStatus); + return createDataResponse(execStatus); } } diff --git a/api/executor/src/playground_executor/utils/cache_utils.bal b/api/executor/src/playground_executor/utils/cache_utils.bal index 618173a..f1ffec5 100644 --- a/api/executor/src/playground_executor/utils/cache_utils.bal +++ b/api/executor/src/playground_executor/utils/cache_utils.bal @@ -1,21 +1,6 @@ import ballerina/crypto; -import ballerina/cache; - -cache:Cache inMemCache = new(); function getCacheId(string sourceCode, string balVersion) returns string { string cacheSource = sourceCode + balVersion; return crypto:hashMd5(cacheSource.toBytes()).toBase16(); -} - -function hasCachedOutput(string cacheId) returns boolean { - return inMemCache.hasKey(cacheId); -} - -function getCachedOutput(string cacheId) returns string? { - return inMemCache.get(cacheId); -} - -function setCachedOutput(string cacheId, string output) { - inMemCache.put(cacheId, output); } \ No newline at end of file From d4c04c75ad70b3a083485eab3e32d9fa03b4efa1 Mon Sep 17 00:00:00 2001 From: Kavith Lokuhewage Date: Wed, 20 Nov 2019 07:22:40 +0530 Subject: [PATCH 5/6] Remove inmem cache in compiler --- .../src/playground_compiler/compile.bal | 41 +++++-------------- .../playground_compiler/utils/cache_utils.bal | 17 +------- 2 files changed, 12 insertions(+), 46 deletions(-) diff --git a/api/compiler/src/playground_compiler/compile.bal b/api/compiler/src/playground_compiler/compile.bal index 866946d..c2f1e29 100644 --- a/api/compiler/src/playground_compiler/compile.bal +++ b/api/compiler/src/playground_compiler/compile.bal @@ -34,37 +34,18 @@ function createSourceFile(string cacheId, string sourceCode) returns string|erro function compile(CompileData data) returns CompilerResponse|error { log:printDebug("Compiling request: " + data.toString()); - string? cacheId = getCacheId(data.sourceCode, data.balVersion); - if (cacheId is string) { - boolean hasCachedOutputResult = hasCachedOutput(cacheId); - log:printDebug("Searching for cached output. Cache ID: " + cacheId); - if (hasCachedOutputResult) { - string? cachedOutput = getCachedOutput(cacheId); - if (cachedOutput is string) { - log:printDebug("Found cached output. " + cachedOutput); - return createDataResponse(cachedOutput); - } else { - log:printError("Empty cached output returned. " ); - return createErrorResponse("Invalid cached output returned from cache."); - } - } else { - log:printDebug("Cached output not found."); - string sourceFile = check createSourceFile(cacheId, data.sourceCode); - string buildDir = check filepath:parent(sourceFile); - log:printDebug("Using " + sourceFile + " for compilation."); - string|error execStatus = execBallerinaCmd(buildDir, "build", "app.bal"); - if (execStatus is error) { - log:printError("Error while executing compile command. ", execStatus); - return createErrorResponse(execStatus.reason()); - } else { - log:printDebug("Finished executing compile command. Output: " + execStatus); - string jarPath = check filepath:build(buildDir, "app.jar"); - setCachedOutput(cacheId, execStatus); - return createDataResponse(execStatus); - } - } + string cacheId = getCacheId(data.sourceCode, data.balVersion); + string sourceFile = check createSourceFile(cacheId, data.sourceCode); + string buildDir = check filepath:parent(sourceFile); + log:printDebug("Using " + sourceFile + " for compilation."); + string|error execStatus = execBallerinaCmd(buildDir, "build", "app.bal"); + if (execStatus is error) { + log:printError("Error while executing compile command. ", execStatus); + return createErrorResponse(execStatus.reason()); } else { - return createErrorResponse("Cannot access cache"); + log:printDebug("Finished executing compile command. Output: " + execStatus); + string jarPath = check filepath:build(buildDir, "app.jar"); + return createDataResponse(execStatus); } } diff --git a/api/compiler/src/playground_compiler/utils/cache_utils.bal b/api/compiler/src/playground_compiler/utils/cache_utils.bal index 2ad6890..f1ffec5 100644 --- a/api/compiler/src/playground_compiler/utils/cache_utils.bal +++ b/api/compiler/src/playground_compiler/utils/cache_utils.bal @@ -1,21 +1,6 @@ import ballerina/crypto; -import ballerina/cache; -cache:Cache inMemCache = new(); - -function getCacheId(string sourceCode, string balVersion) returns string? { +function getCacheId(string sourceCode, string balVersion) returns string { string cacheSource = sourceCode + balVersion; return crypto:hashMd5(cacheSource.toBytes()).toBase16(); -} - -function hasCachedOutput(string cacheId) returns boolean { - return inMemCache.hasKey(cacheId); -} - -function getCachedOutput(string cacheId) returns string? { - return inMemCache.get(cacheId); -} - -function setCachedOutput(string cacheId, string output) { - inMemCache.put(cacheId, output); } \ No newline at end of file From 215c6c163246d53ccd93d456e7466fc182e8dea4 Mon Sep 17 00:00:00 2001 From: Kavith Lokuhewage Date: Wed, 20 Nov 2019 07:23:01 +0530 Subject: [PATCH 6/6] Use proper version for controller dep --- k8s/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/build.gradle b/k8s/build.gradle index 61a15c9..ca30902 100644 --- a/k8s/build.gradle +++ b/k8s/build.gradle @@ -48,7 +48,7 @@ task undeployExecutor(type: Exec) { task deployController(type: Exec) { dependsOn(":docker:controller:cloudBuild") - environment("RELEASE_VERSION", "latest") // replace latest with project.version + environment("RELEASE_VERSION", project.version) commandLine './deploy-controller.sh' }