Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit b56a1d3

Browse files
miguelbranco80datYori
miguelbranco80
and
datYori
authoredMar 20, 2025··
Move DAS mock to main and publish Docker (#32)
Co-authored-by: Yann Bouzonie <[email protected]>
1 parent 3adf1db commit b56a1d3

35 files changed

+236
-752
lines changed
 

Diff for: ‎.github/scripts/dnd-sbt

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash -exu
2+
SCRIPT_HOME="$(cd "$(dirname "$0")"; pwd)"
3+
COMPONENT_HOME="$(cd "${SCRIPT_HOME}/../.."; pwd)"
4+
5+
cd "${COMPONENT_HOME}"
6+
7+
tmp_dir=$(mktemp -d -t dnd-sbt-XXXXXXXXXX)
8+
cp -R . ${tmp_dir}
9+
10+
docker run \
11+
-v /var/run/docker.sock:/var/run/docker.sock \
12+
-v /usr/bin/docker:/usr/bin/docker \
13+
-v ${HOME}/.docker/config.json:/root/.docker/config.json \
14+
-v ${tmp_dir}:/root/ \
15+
-e GITHUB_TOKEN=${GITHUB_TOKEN} \
16+
sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19 \
17+
/bin/bash -c "git config --global --add safe.directory /root; sbt ${1}"

Diff for: ‎.github/workflows/ci.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ jobs:
2121
runs-on: self-hosted
2222
container:
2323
image: sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19
24+
options: --user 1001:1001
2425
steps:
2526
- uses: actions/checkout@v4
2627
- run: sbt headerCheckAll javafmtCheckAll scalafmtCheckAll
2728
test:
2829
runs-on: self-hosted
2930
container:
3031
image: sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19
32+
options: --user 1001:1001
3133
steps:
3234
- uses: actions/checkout@v4
33-
- run: sbt clean test
35+
- run: sbt clean Test/compile

Diff for: ‎.github/workflows/docker-ci.yaml

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: Docker CI
2+
on:
3+
pull_request:
4+
paths:
5+
- .github/workflows/docker-ci.yaml
6+
- .github/scripts/**
7+
- build.sbt
8+
- src/**
9+
10+
env:
11+
GITHUB_TOKEN: ${{ secrets.READ_PACKAGES }}
12+
13+
jobs:
14+
build-and-test:
15+
name: Build & Test
16+
runs-on: self-hosted
17+
steps:
18+
- uses: actions/checkout@v4
19+
20+
- name: Build Docker image
21+
run: |
22+
.github/scripts/dnd-sbt Docker/publishLocal
23+
IMAGE_NAME=$(.github/scripts/dnd-sbt printDockerImageName | grep DOCKER_IMAGE | cut -d= -f2)
24+
echo "IMAGE=${IMAGE_NAME}" >> $GITHUB_ENV
25+
26+
- name: Test image - run container
27+
run: |
28+
CONTAINER_ID=$(docker run -d -p 50051 ${IMAGE})
29+
echo "CONTAINER_ID=${CONTAINER_ID}" >> $GITHUB_ENV
30+
HOST_PORT=$(docker port ${CONTAINER_ID} 50051 | cut -d':' -f2)
31+
echo "HOST_PORT=${HOST_PORT}" >> $GITHUB_ENV
32+
sleep 15
33+
34+
- name: Test image - verify service is running
35+
run: |
36+
nc -z localhost ${HOST_PORT}
37+
if [ $? -ne 0 ]; then
38+
echo "Service check failed!"
39+
exit 1
40+
fi
41+
42+
- name: Cleanup container
43+
if: always()
44+
run: |
45+
if [ ! -z "${CONTAINER_ID}" ]; then
46+
docker stop ${CONTAINER_ID}
47+
docker rm ${CONTAINER_ID}
48+
fi
49+
#
50+
# security-scan:
51+
# name: Security Scan
52+
# runs-on: self-hosted
53+
# steps:
54+
# - uses: actions/checkout@v4
55+
#
56+
# - name: Build Docker image
57+
# run: |
58+
# .github/scripts/dnd-sbt Docker/publishLocal
59+
# IMAGE_NAME=$(.github/scripts/dnd-sbt printDockerImageName | grep DOCKER_IMAGE | cut -d= -f2)
60+
# echo "IMAGE=${IMAGE_NAME}" >> $GITHUB_ENV
61+
#
62+
# - name: Run Trivy vulnerability scanner
63+
# uses: aquasecurity/trivy-action@master
64+
# with:
65+
# image-ref: ${{ env.IMAGE }}
66+
# format: 'table'
67+
# exit-code: '1'
68+
# ignore-unfixed: true
69+
# vuln-type: 'os,library'
70+
# severity: 'CRITICAL,HIGH'

Diff for: ‎.github/workflows/publish.yaml

+31-5
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,59 @@
11
name: publish
22
on:
3+
workflow_dispatch:
34
push:
45
tags:
56
- "v*.*.*"
7+
- "v*.*.*-*"
68

79
env:
810
GITHUB_TOKEN: ${{ secrets.WRITE_PACKAGES }}
911

1012
jobs:
11-
publish:
13+
publish-jars:
1214
runs-on: self-hosted
1315
container:
1416
image: sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19
17+
options: --user 1001:1001
1518
steps:
1619
- uses: actions/checkout@v4
1720
with:
1821
fetch-depth: 0
1922
- name: sbt publish
23+
run: sbt clean publish
24+
publish-docker-image:
25+
runs-on: self-hosted
26+
outputs:
27+
should_trigger_deploy: ${{ steps.should_trigger_deploy.outputs.should_trigger_deploy }}
28+
steps:
29+
- uses: actions/checkout@v4
30+
with:
31+
fetch-depth: 0
32+
- uses: docker/login-action@v3
33+
with:
34+
registry: ghcr.io
35+
username: ${{ github.actor }}
36+
password: ${{ secrets.WRITE_PACKAGES }}
37+
logout: false
38+
- name: publish docker images
39+
run: .github/scripts/dnd-sbt Docker/publish
40+
- name: set should_trigger_deploy
41+
id: should_trigger_deploy
42+
shell: bash
2043
run: |
21-
git config --global --add safe.directory $GITHUB_WORKSPACE
22-
sbt clean publish
44+
pattern='^refs/tags/v[0-9]+\.0\.0$'
45+
echo "should_trigger_deploy=$([[ "$GITHUB_REF" =~ $pattern ]] && echo false || echo true)" >> $GITHUB_OUTPUT
2346
gh-release:
24-
needs: [publish]
47+
needs: [publish-jars, publish-docker-image]
2548
runs-on: self-hosted
2649
steps:
50+
- uses: actions/checkout@v4
51+
with:
52+
fetch-depth: 0
2753
- uses: softprops/action-gh-release@v2
2854
with:
2955
token: ${{ secrets.RAW_CI_PAT }}
3056
generate_release_notes: true
3157
draft: false
3258
prerelease: ${{ contains(github.ref_name, '-') }}
33-
tag_name: ${{ github.ref_name }}
59+
tag_name: ${{ github.ref_name }}

Diff for: ‎build.sbt

+85-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import sbt.*
22
import sbt.Keys.*
33

4+
import com.typesafe.sbt.packager.docker.{Cmd, LayeredMapping}
5+
46
ThisBuild / credentials += Credentials(
57
"GitHub Package Registry",
68
"maven.pkg.github.com",
@@ -29,7 +31,8 @@ lazy val compileSettings = Seq(
2931
// Ensure Java-based DAS SDK code is compiled first, so it is accessible from Scala.
3032
compileOrder := CompileOrder.JavaThenScala,
3133
// Ensure we fork new JVM for run, so we can set JVM flags.
32-
Compile / run / fork := true)
34+
Compile / run / fork := true,
35+
Compile / mainClass := Some("com.rawlabs.das.server.DASServer"))
3336

3437
lazy val testSettings = Seq(
3538
// Ensure we fork new JVM for run, so we can set JVM flags.
@@ -50,8 +53,72 @@ lazy val publishSettings = Seq(
5053
lazy val strictBuildSettings =
5154
commonSettings ++ compileSettings ++ buildSettings ++ testSettings ++ Seq(scalacOptions ++= Seq("-Xfatal-warnings"))
5255

56+
lazy val dockerSettings = Seq(
57+
Docker / packageName := "das-mock-server",
58+
dockerBaseImage := "eclipse-temurin:21-jre",
59+
dockerLabels ++= Map(
60+
"vendor" -> "RAW Labs SA",
61+
"product" -> "das-mock-server",
62+
"image-type" -> "final",
63+
"org.opencontainers.image.source" -> "https://github.com/raw-labs/das-server-scala"),
64+
Docker / daemonUser := "raw",
65+
Docker / daemonUserUid := Some("1001"),
66+
Docker / daemonGroup := "raw",
67+
Docker / daemonGroupGid := Some("1001"),
68+
dockerExposedVolumes := Seq("/var/log/raw"),
69+
dockerExposedPorts := Seq(50051),
70+
dockerEnvVars := Map("PATH" -> s"${(Docker / defaultLinuxInstallLocation).value}/bin:$$PATH"),
71+
dockerEnvVars += "LANG" -> "C.UTF-8",
72+
updateOptions := updateOptions.value.withLatestSnapshots(true),
73+
Linux / linuxPackageMappings += packageTemplateMapping(s"/var/lib/${packageName.value}")(),
74+
bashScriptDefines := {
75+
val ClasspathPattern = "declare -r app_classpath=\"(.*)\"\n".r
76+
bashScriptDefines.value.map {
77+
case ClasspathPattern(classpath) => s"""
78+
|declare -r app_classpath="$${app_home}/../conf:$classpath"
79+
|""".stripMargin
80+
case _ @entry => entry
81+
}
82+
},
83+
Docker / dockerLayerMappings := (Docker / dockerLayerMappings).value.map {
84+
case lm @ LayeredMapping(Some(1), file, path) => {
85+
val fileName = java.nio.file.Paths.get(path).getFileName.toString
86+
if (!fileName.endsWith(".jar")) {
87+
// If it is not a jar, put it on the top layer. Configuration files and other small files.
88+
LayeredMapping(Some(2), file, path)
89+
} else if (fileName.startsWith("com.raw-labs") && fileName.endsWith(".jar")) {
90+
// If it is one of our jars, also top layer. These will change often.
91+
LayeredMapping(Some(2), file, path)
92+
} else {
93+
// Otherwise it is a 3rd party library, which only changes when we change dependencies, so leave it in layer 1
94+
lm
95+
}
96+
}
97+
case lm @ _ => lm
98+
},
99+
Docker / version := {
100+
val ver = version.value
101+
// Docker tags have their own restrictions - only allow [a-zA-Z0-9_.-]
102+
// Replace + with - and ensure no invalid characters
103+
ver.replaceAll("[+]", "-").replaceAll("[^\\w.-]", "-")
104+
},
105+
dockerAlias := {
106+
val devRegistry = sys.env.getOrElse("DEV_REGISTRY", "ghcr.io/raw-labs/das-server-scala")
107+
dockerAlias.value.withRegistryHost(Some(devRegistry))
108+
},
109+
dockerAliases := {
110+
val devRegistry = sys.env.getOrElse("DEV_REGISTRY", "ghcr.io/raw-labs/das-server-scala")
111+
val releaseRegistry = sys.env.get("RELEASE_DOCKER_REGISTRY")
112+
val baseAlias = dockerAlias.value.withRegistryHost(Some(devRegistry))
113+
114+
releaseRegistry match {
115+
case Some(releaseReg) => Seq(baseAlias, dockerAlias.value.withRegistryHost(Some(releaseReg)))
116+
case None => Seq(baseAlias)
117+
}
118+
})
119+
53120
lazy val root = (project in file("."))
54-
.enablePlugins(BuildInfoPlugin)
121+
.enablePlugins(BuildInfoPlugin, JavaAppPackaging, DockerPlugin)
55122
.settings(
56123
name := "das-server-scala",
57124
buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion),
@@ -71,19 +138,29 @@ lazy val root = (project in file("."))
71138
// Configuration
72139
"com.typesafe" % "config" % "1.4.3",
73140
// Protocol DAS
74-
"com.raw-labs" %% "protocol-das" % "1.0.0",
141+
"com.raw-labs" %% "protocol-das" % "1.0.2",
75142
// Akka Streams
76143
"org.apache.pekko" %% "pekko-actor-typed" % "1.1.3",
77144
"org.apache.pekko" %% "pekko-stream" % "1.1.3",
78145
"org.apache.pekko" %% "pekko-http" % "1.1.0",
79-
// Jackson databind
80-
"com.fasterxml.jackson.core" % "jackson-databind" % "2.18.2" % Test,
81-
// gRPC Testing
82-
"io.grpc" % "grpc-inprocess" % "1.62.2",
83146
// Web UI
84147
"com.typesafe.akka" %% "akka-http" % "10.5.3",
85148
"com.lihaoyi" %% "scalatags" % "0.13.1",
149+
// Jackson databind
150+
"com.fasterxml.jackson.core" % "jackson-databind" % "2.18.2" % Test,
151+
// gRPC Testing
152+
"io.grpc" % "grpc-inprocess" % "1.62.2" % Test,
86153
// Postgres
87154
"org.postgresql" % "postgresql" % "42.7.4" % Test,
88155
// Testing
89-
"org.scalatest" %% "scalatest" % "3.2.19" % Test))
156+
"org.scalatest" %% "scalatest" % "3.2.19" % Test),
157+
dockerSettings)
158+
159+
lazy val printDockerImageName = taskKey[Unit]("Prints the full Docker image name that will be produced")
160+
161+
printDockerImageName := {
162+
// Get the main Docker alias (the first one in the sequence)
163+
val alias = (Docker / dockerAliases).value.head
164+
// The toString method already returns the full image name with registry and tag
165+
println(s"DOCKER_IMAGE=${alias}")
166+
}

Diff for: ‎project/plugins.sbt

+6
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,14 @@ autoCompilerPlugins := true
44

55
addDependencyTreePlugin
66

7+
libraryDependencies += "commons-io" % "commons-io" % "2.11.0"
8+
79
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
810

11+
addSbtPlugin("nl.gn0s1s" % "sbt-dotenv" % "3.1.1")
12+
13+
addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1")
14+
915
addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.8.0")
1016

1117
addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0")

Diff for: ‎src/main/resources/META-INF/services/com.rawlabs.das.sdk.DASSdkBuilder

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
com.rawlabs.das.mock.DASMockBuilder

Diff for: ‎src/main/scala/com/rawlabs/das/server/grpc/FunctionServiceGrpcImpl.scala

+23-42
Original file line numberDiff line numberDiff line change
@@ -74,22 +74,20 @@ class FunctionServiceGrpcImpl(provider: DASSdkManager)
7474
responseObserver: StreamObserver[ExecuteFunctionResponse]): Unit = {
7575
logger.debug(s"Executing function for DAS ID: ${request.getDasId}, function ID: ${request.getFunctionId.getName}")
7676

77-
withDAS(request.getDasId, responseObserver) { das =>
78-
withFunction(das, request.getFunctionId, responseObserver) { function =>
79-
val sdkArgs = request.getArgsList.asScala
80-
.map { arg =>
81-
assert(arg.hasNamedArg)
82-
arg.getNamedArg.getName -> arg.getNamedArg.getValue
83-
}
84-
.toMap
85-
.asJava
86-
val resultValue = function.execute(sdkArgs)
77+
withFunction(request.getDasId, request.getFunctionId, responseObserver) { function =>
78+
val sdkArgs = request.getArgsList.asScala
79+
.map { arg =>
80+
assert(arg.hasNamedArg)
81+
arg.getNamedArg.getName -> arg.getNamedArg.getValue
82+
}
83+
.toMap
84+
.asJava
85+
val resultValue = function.execute(sdkArgs)
8786

88-
val response = ExecuteFunctionResponse.newBuilder().setOutput(resultValue).build()
89-
responseObserver.onNext(response)
90-
responseObserver.onCompleted()
91-
logger.debug("Function executed successfully.")
92-
}
87+
val response = ExecuteFunctionResponse.newBuilder().setOutput(resultValue).build()
88+
responseObserver.onNext(response)
89+
responseObserver.onCompleted()
90+
logger.debug("Function executed successfully.")
9391
}
9492
}
9593

@@ -127,34 +125,17 @@ class FunctionServiceGrpcImpl(provider: DASSdkManager)
127125
/**
128126
* Helper method to retrieve a function from the DAS. If the function is not found, respond with an error.
129127
*/
130-
private def withFunction(das: DASSdk, functionId: FunctionId, responseObserver: StreamObserver[_])(
128+
private def withFunction(DASId: DASId, functionId: FunctionId, responseObserver: StreamObserver[_])(
131129
f: DASFunction => Unit): Unit = {
132-
val maybeFunction = das.getFunction(functionId.getName).toScala
133-
maybeFunction match {
134-
case None =>
135-
logger.error(s"Function ${functionId.getName} not found.")
136-
responseObserver.onError(
137-
Status.INVALID_ARGUMENT.withDescription(s"Function ${functionId.getName} not found").asRuntimeException())
138-
case Some(fn) =>
139-
try {
140-
f(fn)
141-
} catch {
142-
case ex: DASSdkInvalidArgumentException =>
143-
logger.error("DASSdk invalid argument error", ex)
144-
responseObserver.onError(Status.INVALID_ARGUMENT.withDescription(ex.getMessage).asRuntimeException())
145-
case ex: DASSdkPermissionDeniedException =>
146-
logger.error("DASSdk permission denied error", ex)
147-
responseObserver.onError(Status.PERMISSION_DENIED.withDescription(ex.getMessage).asRuntimeException())
148-
case ex: DASSdkUnauthenticatedException =>
149-
logger.error("DASSdk unauthenticated error", ex)
150-
responseObserver.onError(Status.UNAUTHENTICATED.withDescription(ex.getMessage).asRuntimeException())
151-
case ex: DASSdkUnsupportedException =>
152-
logger.error("DASSdk unsupported feature", ex)
153-
responseObserver.onError(Status.UNIMPLEMENTED.withDescription(ex.getMessage).asRuntimeException())
154-
case t: Throwable =>
155-
logger.error("DASSdk unexpected error", t)
156-
responseObserver.onError(Status.INTERNAL.withCause(t).asRuntimeException())
157-
}
130+
withDAS(DASId, responseObserver) { das =>
131+
val maybeFunction = das.getFunction(functionId.getName).toScala
132+
maybeFunction match {
133+
case None =>
134+
logger.error(s"Function ${functionId.getName} not found.")
135+
responseObserver.onError(
136+
Status.INVALID_ARGUMENT.withDescription(s"Function ${functionId.getName} not found").asRuntimeException())
137+
case Some(fn) => f(fn)
138+
}
158139
}
159140
}
160141

Diff for: ‎src/test/resources/META-INF/services/com.rawlabs.das.sdk.DASSdkBuilder

-2
This file was deleted.

Diff for: ‎src/test/scala/com/rawlabs/das/postgresql/DASPosgresqlBuilder.scala

-25
This file was deleted.

Diff for: ‎src/test/scala/com/rawlabs/das/postgresql/DASPostgresql.scala

-58
This file was deleted.

Diff for: ‎src/test/scala/com/rawlabs/das/postgresql/DASPostgresqlTable.scala

-247
This file was deleted.

Diff for: ‎src/test/scala/com/rawlabs/das/postgresql/PostgresqlBackend.scala

-364
This file was deleted.

0 commit comments

Comments
 (0)
Please sign in to comment.