Skip to content

Commit 28a7a42

Browse files
bgaidiozmiguelbranco80datYori
authored
RD-15410: DAS function support (#31)
* Added the service, * Added functions to `DASMock`. --------- Co-authored-by: Miguel Branco <[email protected]> Co-authored-by: Yann Bouzonie <[email protected]>
1 parent 9c876dc commit 28a7a42

35 files changed

+1964
-700
lines changed

‎.github/scripts/dnd-sbt

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash -exu
2+
SCRIPT_HOME="$(cd "$(dirname "$0")"; pwd)"
3+
COMPONENT_HOME="$(cd "${SCRIPT_HOME}/../.."; pwd)"
4+
5+
cd "${COMPONENT_HOME}"
6+
7+
tmp_dir=$(mktemp -d -t dnd-sbt-XXXXXXXXXX)
8+
cp -R . ${tmp_dir}
9+
10+
docker run \
11+
-v /var/run/docker.sock:/var/run/docker.sock \
12+
-v /usr/bin/docker:/usr/bin/docker \
13+
-v ${HOME}/.docker/config.json:/root/.docker/config.json \
14+
-v ${tmp_dir}:/root/ \
15+
-e GITHUB_TOKEN=${GITHUB_TOKEN} \
16+
sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19 \
17+
/bin/bash -c "git config --global --add safe.directory /root; sbt ${1}"

‎.github/workflows/ci.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@ jobs:
2121
runs-on: self-hosted
2222
container:
2323
image: sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19
24+
options: --user 1001:1001
2425
steps:
2526
- uses: actions/checkout@v4
2627
- run: sbt headerCheckAll javafmtCheckAll scalafmtCheckAll
2728
test:
2829
runs-on: self-hosted
2930
container:
3031
image: sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19
32+
options: --user 1001:1001
3133
steps:
3234
- uses: actions/checkout@v4
33-
- run: sbt clean test
35+
- run: sbt clean Test/compile

‎.github/workflows/docker-ci.yaml

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: Docker CI
2+
on:
3+
pull_request:
4+
paths:
5+
- .github/workflows/docker-ci.yaml
6+
- .github/scripts/**
7+
- build.sbt
8+
- src/**
9+
10+
env:
11+
GITHUB_TOKEN: ${{ secrets.READ_PACKAGES }}
12+
13+
jobs:
14+
build-and-test:
15+
name: Build & Test
16+
runs-on: self-hosted
17+
steps:
18+
- uses: actions/checkout@v4
19+
20+
- name: Build Docker image
21+
run: |
22+
.github/scripts/dnd-sbt Docker/publishLocal
23+
IMAGE_NAME=$(.github/scripts/dnd-sbt printDockerImageName | grep DOCKER_IMAGE | cut -d= -f2)
24+
echo "IMAGE=${IMAGE_NAME}" >> $GITHUB_ENV
25+
26+
- name: Test image - run container
27+
run: |
28+
CONTAINER_ID=$(docker run -d -p 50051 ${IMAGE})
29+
echo "CONTAINER_ID=${CONTAINER_ID}" >> $GITHUB_ENV
30+
HOST_PORT=$(docker port ${CONTAINER_ID} 50051 | cut -d':' -f2)
31+
echo "HOST_PORT=${HOST_PORT}" >> $GITHUB_ENV
32+
sleep 15
33+
34+
- name: Test image - verify service is running
35+
run: |
36+
nc -z localhost ${HOST_PORT}
37+
if [ $? -ne 0 ]; then
38+
echo "Service check failed!"
39+
exit 1
40+
fi
41+
42+
- name: Cleanup container
43+
if: always()
44+
run: |
45+
if [ ! -z "${CONTAINER_ID}" ]; then
46+
docker stop ${CONTAINER_ID}
47+
docker rm ${CONTAINER_ID}
48+
fi
49+
#
50+
# security-scan:
51+
# name: Security Scan
52+
# runs-on: self-hosted
53+
# steps:
54+
# - uses: actions/checkout@v4
55+
#
56+
# - name: Build Docker image
57+
# run: |
58+
# .github/scripts/dnd-sbt Docker/publishLocal
59+
# IMAGE_NAME=$(.github/scripts/dnd-sbt printDockerImageName | grep DOCKER_IMAGE | cut -d= -f2)
60+
# echo "IMAGE=${IMAGE_NAME}" >> $GITHUB_ENV
61+
#
62+
# - name: Run Trivy vulnerability scanner
63+
# uses: aquasecurity/trivy-action@master
64+
# with:
65+
# image-ref: ${{ env.IMAGE }}
66+
# format: 'table'
67+
# exit-code: '1'
68+
# ignore-unfixed: true
69+
# vuln-type: 'os,library'
70+
# severity: 'CRITICAL,HIGH'

‎.github/workflows/publish.yaml

+30-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
name: publish
22
on:
3+
workflow_dispatch:
34
push:
45
tags:
56
- "v*.*.*"
@@ -9,26 +10,50 @@ env:
910
GITHUB_TOKEN: ${{ secrets.WRITE_PACKAGES }}
1011

1112
jobs:
12-
publish:
13+
publish-jars:
1314
runs-on: self-hosted
1415
container:
1516
image: sbtscala/scala-sbt:eclipse-temurin-jammy-21.0.2_13_1.9.9_2.12.19
17+
options: --user 1001:1001
1618
steps:
1719
- uses: actions/checkout@v4
1820
with:
1921
fetch-depth: 0
2022
- name: sbt publish
23+
run: sbt clean publish
24+
publish-docker-image:
25+
runs-on: self-hosted
26+
outputs:
27+
should_trigger_deploy: ${{ steps.should_trigger_deploy.outputs.should_trigger_deploy }}
28+
steps:
29+
- uses: actions/checkout@v4
30+
with:
31+
fetch-depth: 0
32+
- uses: docker/login-action@v3
33+
with:
34+
registry: ghcr.io
35+
username: ${{ github.actor }}
36+
password: ${{ secrets.WRITE_PACKAGES }}
37+
logout: false
38+
- name: publish docker images
39+
run: .github/scripts/dnd-sbt Docker/publish
40+
- name: set should_trigger_deploy
41+
id: should_trigger_deploy
42+
shell: bash
2143
run: |
22-
git config --global --add safe.directory $GITHUB_WORKSPACE
23-
sbt clean publish
44+
pattern='^refs/tags/v[0-9]+\.0\.0$'
45+
echo "should_trigger_deploy=$([[ "$GITHUB_REF" =~ $pattern ]] && echo false || echo true)" >> $GITHUB_OUTPUT
2446
gh-release:
25-
needs: [publish]
47+
needs: [publish-jars, publish-docker-image]
2648
runs-on: self-hosted
2749
steps:
50+
- uses: actions/checkout@v4
51+
with:
52+
fetch-depth: 0
2853
- uses: softprops/action-gh-release@v2
2954
with:
3055
token: ${{ secrets.RAW_CI_PAT }}
3156
generate_release_notes: true
3257
draft: false
3358
prerelease: ${{ contains(github.ref_name, '-') }}
34-
tag_name: ${{ github.ref_name }}
59+
tag_name: ${{ github.ref_name }}

‎build.sbt

+85-8
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import sbt.*
22
import sbt.Keys.*
33

4+
import com.typesafe.sbt.packager.docker.{Cmd, LayeredMapping}
5+
46
ThisBuild / credentials += Credentials(
57
"GitHub Package Registry",
68
"maven.pkg.github.com",
@@ -29,7 +31,8 @@ lazy val compileSettings = Seq(
2931
// Ensure Java-based DAS SDK code is compiled first, so it is accessible from Scala.
3032
compileOrder := CompileOrder.JavaThenScala,
3133
// Ensure we fork new JVM for run, so we can set JVM flags.
32-
Compile / run / fork := true)
34+
Compile / run / fork := true,
35+
Compile / mainClass := Some("com.rawlabs.das.server.DASServer"))
3336

3437
lazy val testSettings = Seq(
3538
// Ensure we fork new JVM for run, so we can set JVM flags.
@@ -50,8 +53,72 @@ lazy val publishSettings = Seq(
5053
lazy val strictBuildSettings =
5154
commonSettings ++ compileSettings ++ buildSettings ++ testSettings ++ Seq(scalacOptions ++= Seq("-Xfatal-warnings"))
5255

56+
lazy val dockerSettings = Seq(
57+
Docker / packageName := "das-mock-server",
58+
dockerBaseImage := "eclipse-temurin:21-jre",
59+
dockerLabels ++= Map(
60+
"vendor" -> "RAW Labs SA",
61+
"product" -> "das-mock-server",
62+
"image-type" -> "final",
63+
"org.opencontainers.image.source" -> "https://github.com/raw-labs/das-server-scala"),
64+
Docker / daemonUser := "raw",
65+
Docker / daemonUserUid := Some("1001"),
66+
Docker / daemonGroup := "raw",
67+
Docker / daemonGroupGid := Some("1001"),
68+
dockerExposedVolumes := Seq("/var/log/raw"),
69+
dockerExposedPorts := Seq(50051),
70+
dockerEnvVars := Map("PATH" -> s"${(Docker / defaultLinuxInstallLocation).value}/bin:$$PATH"),
71+
dockerEnvVars += "LANG" -> "C.UTF-8",
72+
updateOptions := updateOptions.value.withLatestSnapshots(true),
73+
Linux / linuxPackageMappings += packageTemplateMapping(s"/var/lib/${packageName.value}")(),
74+
bashScriptDefines := {
75+
val ClasspathPattern = "declare -r app_classpath=\"(.*)\"\n".r
76+
bashScriptDefines.value.map {
77+
case ClasspathPattern(classpath) => s"""
78+
|declare -r app_classpath="$${app_home}/../conf:$classpath"
79+
|""".stripMargin
80+
case _ @entry => entry
81+
}
82+
},
83+
Docker / dockerLayerMappings := (Docker / dockerLayerMappings).value.map {
84+
case lm @ LayeredMapping(Some(1), file, path) => {
85+
val fileName = java.nio.file.Paths.get(path).getFileName.toString
86+
if (!fileName.endsWith(".jar")) {
87+
// If it is not a jar, put it on the top layer. Configuration files and other small files.
88+
LayeredMapping(Some(2), file, path)
89+
} else if (fileName.startsWith("com.raw-labs") && fileName.endsWith(".jar")) {
90+
// If it is one of our jars, also top layer. These will change often.
91+
LayeredMapping(Some(2), file, path)
92+
} else {
93+
// Otherwise it is a 3rd party library, which only changes when we change dependencies, so leave it in layer 1
94+
lm
95+
}
96+
}
97+
case lm @ _ => lm
98+
},
99+
Docker / version := {
100+
val ver = version.value
101+
// Docker tags have their own restrictions - only allow [a-zA-Z0-9_.-]
102+
// Replace + with - and ensure no invalid characters
103+
ver.replaceAll("[+]", "-").replaceAll("[^\\w.-]", "-")
104+
},
105+
dockerAlias := {
106+
val devRegistry = sys.env.getOrElse("DEV_REGISTRY", "ghcr.io/raw-labs/das-server-scala")
107+
dockerAlias.value.withRegistryHost(Some(devRegistry))
108+
},
109+
dockerAliases := {
110+
val devRegistry = sys.env.getOrElse("DEV_REGISTRY", "ghcr.io/raw-labs/das-server-scala")
111+
val releaseRegistry = sys.env.get("RELEASE_DOCKER_REGISTRY")
112+
val baseAlias = dockerAlias.value.withRegistryHost(Some(devRegistry))
113+
114+
releaseRegistry match {
115+
case Some(releaseReg) => Seq(baseAlias, dockerAlias.value.withRegistryHost(Some(releaseReg)))
116+
case None => Seq(baseAlias)
117+
}
118+
})
119+
53120
lazy val root = (project in file("."))
54-
.enablePlugins(BuildInfoPlugin)
121+
.enablePlugins(BuildInfoPlugin, JavaAppPackaging, DockerPlugin)
55122
.settings(
56123
name := "das-server-scala",
57124
buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion),
@@ -71,19 +138,29 @@ lazy val root = (project in file("."))
71138
// Configuration
72139
"com.typesafe" % "config" % "1.4.3",
73140
// Protocol DAS
74-
"com.raw-labs" %% "protocol-das" % "1.0.0",
141+
"com.raw-labs" %% "protocol-das" % "1.0.2",
75142
// Akka Streams
76143
"org.apache.pekko" %% "pekko-actor-typed" % "1.1.3",
77144
"org.apache.pekko" %% "pekko-stream" % "1.1.3",
78145
"org.apache.pekko" %% "pekko-http" % "1.1.0",
79-
// Jackson databind
80-
"com.fasterxml.jackson.core" % "jackson-databind" % "2.18.2" % Test,
81-
// gRPC Testing
82-
"io.grpc" % "grpc-inprocess" % "1.62.2",
83146
// Web UI
84147
"com.typesafe.akka" %% "akka-http" % "10.5.3",
85148
"com.lihaoyi" %% "scalatags" % "0.13.1",
149+
// Jackson databind
150+
"com.fasterxml.jackson.core" % "jackson-databind" % "2.18.2" % Test,
151+
// gRPC Testing
152+
"io.grpc" % "grpc-inprocess" % "1.62.2" % Test,
86153
// Postgres
87154
"org.postgresql" % "postgresql" % "42.7.4" % Test,
88155
// Testing
89-
"org.scalatest" %% "scalatest" % "3.2.19" % Test))
156+
"org.scalatest" %% "scalatest" % "3.2.19" % Test),
157+
dockerSettings)
158+
159+
lazy val printDockerImageName = taskKey[Unit]("Prints the full Docker image name that will be produced")
160+
161+
printDockerImageName := {
162+
// Get the main Docker alias (the first one in the sequence)
163+
val alias = (Docker / dockerAliases).value.head
164+
// The toString method already returns the full image name with registry and tag
165+
println(s"DOCKER_IMAGE=${alias}")
166+
}

‎project/plugins.sbt

+6
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,14 @@ autoCompilerPlugins := true
44

55
addDependencyTreePlugin
66

7+
libraryDependencies += "commons-io" % "commons-io" % "2.11.0"
8+
79
addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
810

11+
addSbtPlugin("nl.gn0s1s" % "sbt-dotenv" % "3.1.1")
12+
13+
addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.1")
14+
915
addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.8.0")
1016

1117
addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0")

‎src/main/resources/META-INF/services/com.rawlabs.das.sdk.DASSdkBuilder

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
com.rawlabs.das.mock.DASMockBuilder

‎src/test/scala/com/rawlabs/das/mock/DASMock.scala ‎src/main/scala/com/rawlabs/das/mock/DASMock.scala

+32-4
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package com.rawlabs.das.mock
1111

12+
import com.rawlabs.das.mock.functions._
1213
import com.rawlabs.das.sdk.scala._
1314
import com.rawlabs.protocol.das.v1.functions._
1415
import com.rawlabs.protocol.das.v1.tables._
@@ -354,6 +355,20 @@ class DASMock(options: Map[String, String]) extends DASSdk with StrictLogging {
354355
.build())
355356
.build())
356357
.build())))
358+
.addColumns(
359+
ColumnDefinition
360+
.newBuilder()
361+
.setName("list_of_lists")
362+
.setType(
363+
Type
364+
.newBuilder()
365+
.setList(
366+
ListType
367+
.newBuilder()
368+
.setInnerType(Type
369+
.newBuilder()
370+
.setList(ListType.newBuilder().setInnerType(Type.newBuilder().setInt(IntType.newBuilder())))))
371+
.build()))
357372
.setStartupCost(1000)
358373
.build()
359374
val eventTable = TableDefinition
@@ -379,8 +394,23 @@ class DASMock(options: Map[String, String]) extends DASSdk with StrictLogging {
379394
Seq(bigTable, smallTable, slowTable, brokenTable, inMemoryTable, allTypesTable, eventTable)
380395
}
381396

397+
private val functions = Seq(
398+
new RecordConcatFunction,
399+
new MultiplyIntFunction,
400+
new NoArgFunction,
401+
new RecipeListOfTypedRecordsFunction,
402+
new RecipeListOfUntypedRecordsFunction,
403+
new MultiplyStringFunction,
404+
new UnspecifiedRowsFunction,
405+
new RangeFunction,
406+
new SingleRowPlayerInfoTypedFunction,
407+
new SingleRowPlayerInfoUntypedFunction,
408+
new RockAlbumsListOfRecordsWithNestedList,
409+
new AllTypesFunction,
410+
new AllTypesNullsFunction).map(f => f.definition.getFunctionId.getName -> f).toMap
411+
382412
override def functionDefinitions: Seq[FunctionDefinition] = {
383-
Seq.empty
413+
functions.values.map(_.definition).toSeq
384414
}
385415

386416
private val tables = Map(
@@ -394,8 +424,6 @@ class DASMock(options: Map[String, String]) extends DASSdk with StrictLogging {
394424

395425
override def getTable(name: String): Option[DASTable] = tables.get(name)
396426

397-
override def getFunction(name: String): Option[DASFunction] = {
398-
None
399-
}
427+
override def getFunction(name: String): Option[DASFunction] = functions.get(name)
400428

401429
}

‎src/test/scala/com/rawlabs/das/mock/DASMockAllTypesTable.scala ‎src/main/scala/com/rawlabs/das/mock/DASMockAllTypesTable.scala

+11
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,17 @@ class DASMockAllTypesTable(maxRows: Int) extends DASTable {
302302
val listBuilder = ValueList.newBuilder()
303303
records.foreach(r => listBuilder.addValues(r))
304304
Value.newBuilder().setList(listBuilder.build()).build()
305+
},
306+
"list_of_lists" -> {
307+
val items = (i to i + 3).map(j => {
308+
val listBuilder = ValueList.newBuilder()
309+
(j to j + 3).foreach(k =>
310+
listBuilder.addValues(Value.newBuilder().setInt(ValueInt.newBuilder().setV(k).build())))
311+
Value.newBuilder().setList(listBuilder.build()).build()
312+
})
313+
val listBuilder = ValueList.newBuilder()
314+
items.foreach(listBuilder.addValues)
315+
Value.newBuilder().setList(listBuilder.build()).build()
305316
})
306317
val row = Row.newBuilder()
307318
columns.foreach(col => row.addColumns(Column.newBuilder().setName(col).setData(values(col))))

‎src/test/scala/com/rawlabs/das/mock/DASMockBuilder.scala ‎src/main/scala/com/rawlabs/das/mock/DASMockBuilder.scala

File renamed without changes.

‎src/test/scala/com/rawlabs/das/mock/DASMockEventTable.scala ‎src/main/scala/com/rawlabs/das/mock/DASMockEventTable.scala

File renamed without changes.

‎src/test/scala/com/rawlabs/das/mock/DASMockInMemoryTable.scala ‎src/main/scala/com/rawlabs/das/mock/DASMockInMemoryTable.scala

File renamed without changes.

‎src/test/scala/com/rawlabs/das/mock/DASMockStorage.scala ‎src/main/scala/com/rawlabs/das/mock/DASMockStorage.scala

File renamed without changes.

‎src/test/scala/com/rawlabs/das/mock/DASMockTable.scala ‎src/main/scala/com/rawlabs/das/mock/DASMockTable.scala

File renamed without changes.

‎src/test/scala/com/rawlabs/das/mock/DASMockTestApp.scala ‎src/main/scala/com/rawlabs/das/mock/DASMockTestApp.scala

File renamed without changes.

0 commit comments

Comments
 (0)