chore: linux static image compilation (#13)

vinted · May 25, 2021 · 5fa38f2 · 5fa38f2
1 parent d7edb0a
commit 5fa38f2
Show file tree

Hide file tree

Showing 9 changed files with 196 additions and 23 deletions.
diff --git a/Makefile b/Makefile
@@ -1,5 +1,13 @@
 include dockerfiles/docker.mk
 
+.PHONY: pom.xml
+pom.xml:
+	clojure -Spom
+
+.PHONY: uberjar
+uberjar: pom.xml
+	clojure -X:uberjar :jar target/ket-uber.jar :main-class core
+
 .PHONY: lint
 lint:
 	clojure -M:clj-kondo
@@ -27,8 +35,13 @@ run-integration-tests:
 	docker-compose $(ES_TEST) build
 	docker-compose $(ES_TEST) up --remove-orphans --abort-on-container-exit --exit-code-from tools-test
 
-build-ket:
-	docker build -f dockerfiles/Dockerfile.executable-builder -t ket-native-image .
-	docker rm ket-native-image-build || true
-	docker create --name ket-native-image-build ket-native-image
-	docker cp ket-native-image-build:/usr/src/app/ket ket
+docker_build = (docker build --build-arg $1 --build-arg $2 -f dockerfiles/Dockerfile.executable-builder -t ket-native-image .; \
+				docker rm ket-native-image-build || true; \
+				docker create --name ket-native-image-build ket-native-image; \
+				docker cp ket-native-image-build:/usr/src/app/ket ket)
+
+build:
+	clojure -M:native-image
+
+build-linux-static:
+	$(call docker_build, KET_STATIC=true, KET_MUSL=true)
diff --git a/README.md b/README.md
@@ -223,7 +223,9 @@ Also, in the same fashion the script can be provided:
 
 ## Logging
 
-Logging is controlled by the [logback](http://logback.qos.ch/) library. The output layout is JSON (you can query it with `jq` or collect logs with logstash or beats).
+Logging is controlled by the [logback](http://logback.qos.ch/) library.
+Logger logs to `System.err`.
+The output layout is JSON (you can query it with `jq` or collect logs with logstash).
 Default logging level is `INFO`.
 When executed as a binary, i.e. `./ket OPERATION`, then logging levels are controlled by an environment variable called: `ROOT_LOGGER_LEVEL`, e.g. `ROOT_LOGGER_LEVEL=WARN ./ket operation -f config.json`
 Acceptable values of the `ROOT_LOGGER_LEVEL` are: `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `FATAL`.

diff --git a/deps.edn b/deps.edn
@@ -42,8 +42,15 @@
   {:main-opts  ["-m" "clj-kondo.main" "--lint" "src" "test"]
    :extra-deps {clj-kondo/clj-kondo {:mvn/version "2021.03.03"}}
    :jvm-opts   ["-Dclojure.main.report=stderr"]}
-  :native-ket
+  :uberjar
+  {:replace-deps {com.github.seancorfield/depstar {:mvn/version "2.0.216"}}
+   :exec-fn      hf.depstar/uberjar
+   :exec-args    {:aot true}}
+  :native-image-linux-static-musl
   {:main-opts  ["-m" "clj.native-image" "core"
+                "--static"
+                "--libc=musl"
+                "--verbose"
                 "--enable-https"
                 "--no-fallback"
                 "--language:js"
@@ -60,6 +67,35 @@
                 "-H:IncludeResources=logback.xml"
                 "-H:Name=ket"]
    :jvm-opts   ["-Dclojure.compiler.direct-linking=true"]
+   :extra-deps {org.jboss.logging/commons-logging-jboss-logging
+                {:mvn/version "1.0.0.Final"}
+                borkdude/clj-reflector-graal-java11-fix
+                {:mvn/version "0.0.1-graalvm-20.3.0"
+                 :exclusions  [org.graalvm.nativeimage/svm]}
+                clj.native-image/clj.native-image
+                {:git/url    "https://github.com/taylorwood/clj.native-image.git"
+                 :exclusions [commons-logging/commons-logging
+                              org.slf4j/slf4j-nop]
+                 :sha        "f3e40672d5c543b80a2019c1f07b2d3fe785962c"}}}
+  :native-image
+  {:main-opts  ["-m" "clj.native-image" "core"
+                "--enable-https"
+                "--no-fallback"
+                "--language:js"
+                "--verbose"
+                "--allow-incomplete-classpath"
+                "--initialize-at-build-time"
+                "--enable-all-security-services"
+                "--initialize-at-run-time=org.httpkit.client.HttpClient"
+                "--initialize-at-run-time=org.httpkit.client.SslContextFactory"
+                "--report-unsupported-elements-at-runtime"
+                "-J-Dclojure.compiler.direct-linking=true"
+                "-H:ReflectionConfigurationFiles=graalvm/reflect-config.json"
+                ;; optional native image name override
+                "-H:+ReportExceptionStackTraces"
+                "-H:IncludeResources=logback.xml"
+                "-H:Name=ket"]
+   :jvm-opts   ["-Dclojure.compiler.direct-linking=true"]
    :extra-deps {org.jboss.logging/commons-logging-jboss-logging
                 {:mvn/version "1.0.0.Final"}
                 borkdude/clj-reflector-graal-java11-fix

diff --git a/dockerfiles/Dockerfile.executable-builder b/dockerfiles/Dockerfile.executable-builder
@@ -1,11 +1,12 @@
-FROM findepi/graalvm:java11-native as BUILDER
+FROM findepi/graalvm:21.0.0.2-java11-all as BUILDER
 
 ENV GRAALVM_HOME=/graalvm
 ENV JAVA_HOME=/graalvm
 ENV CLOJURE_VERSION=1.10.3.814
 
-RUN apt-get install -y curl \
-    && gu install native-image \
+RUN apt-get update && apt-get install -y curl git make
+
+RUN gu install native-image \
     && gu install ruby \
     && gu install python \
     && gu install r \
@@ -14,13 +15,44 @@ RUN apt-get install -y curl \
     && ./linux-install-$CLOJURE_VERSION.sh \
     && rm linux-install-$CLOJURE_VERSION.sh
 
+ENV MUSL_DIR=${HOME}/.musl
+ENV MUSL_VERSION=1.2.2
+ENV ZLIB_VERSION=1.2.11
+
+RUN mkdir $MUSL_DIR \
+    && curl https://musl.libc.org/releases/musl-${MUSL_VERSION}.tar.gz -o musl-${MUSL_VERSION}.tar.gz \
+    && tar zxvf musl-${MUSL_VERSION}.tar.gz \
+    && cd musl-${MUSL_VERSION} \
+    && ./configure --disable-shared --prefix=${MUSL_DIR} \
+    && make \
+    && make install \
+    && curl https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz -o zlib-${ZLIB_VERSION}.tar.gz \
+    && tar zxvf zlib-${ZLIB_VERSION}.tar.gz \
+    && cd zlib-${ZLIB_VERSION} \
+    && ./configure --static --prefix=${MUSL_DIR} \
+    && make \
+    && make install
+
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
+ENV PATH=$PATH:${MUSL_DIR}/bin
+
 COPY deps.edn /usr/src/app/
-RUN clojure -P -M:native-ket
+RUN clojure -P && clojure -P -M:uberjar
 COPY resources/ /usr/src/app/resources
 COPY graalvm/ /usr/src/app/graalvm/
 COPY src/ /usr/src/app/src
+COPY script/ /usr/src/app/script
+
+RUN clojure -Spom
+RUN clojure -X:uberjar :jar target/ket-uber.jar :main-class core
+
+ARG KET_STATIC
+ENV KET_STATIC=$KET_STATIC
+
+ARG KET_MUSL
+ENV KET_MUSL=$KET_MUSL
 
-RUN clojure -M:native-ket
+# RUN script/compile
+RUN clojure -M:native-image-linux-static-musl
diff --git a/resources/KET_VERSION b/resources/KET_VERSION
@@ -0,0 +1 @@
+v2021.05.24
diff --git a/resources/logback.xml b/resources/logback.xml
@@ -1,6 +1,7 @@
 <configuration scan="true" scanPeriod="30 seconds">
     <statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
     <appender name="A1" class="ch.qos.logback.core.ConsoleAppender">
+        <target>${LOGGER_TARGET:-System.err}</target>
         <layout class="ch.qos.logback.contrib.json.classic.JsonLayout">
             <jsonFormatter
                     class="ch.qos.logback.contrib.jackson.JacksonJsonFormatter">

diff --git a/script/compile b/script/compile
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+KET_XMX=${KET_XMX:-"-J-Xmx48500m"}
+
+if [ -z "$GRAALVM_HOME" ]; then
+    echo "Please set GRAALVM_HOME"
+    exit 1
+fi
+
+KET_JAR=${KET_JAR:-"target/ket-uber.jar"}
+
+if [[ ! -f "$KET_JAR" ]]
+then
+    echo "Please run make uberjar first."
+    exit 1
+fi
+
+args=( "-jar"
+       "$KET_JAR"
+       "-H:Name=ket"
+       "-H:IncludeResources=KET_VERSION"
+       "--verbose"
+       "--no-server"
+       "--enable-https"
+       "--no-fallback"
+       "--language:js"
+       "--allow-incomplete-classpath"
+       "--initialize-at-build-time"
+       "--enable-all-security-services"
+       "--initialize-at-run-time=org.httpkit.client.HttpClient"
+       "--initialize-at-run-time=org.httpkit.client.SslContextFactory"
+       "--report-unsupported-elements-at-runtime"
+       "-J-Dclojure.compiler.direct-linking=true"
+       "-H:ReflectionConfigurationFiles=graalvm/reflect-config.json"
+       "-H:+ReportExceptionStackTraces"
+       "-H:IncludeResources=logback.xml"
+       "$KET_XMX")
+
+KET_STATIC=${KET_STATIC:-}
+KET_MUSL=${KET_MUSL:-}
+
+if [ "$KET_STATIC" = "true" ]; then
+    args+=("--static")
+    if [ "$KET_MUSL" = "true" ]; then
+        args+=("--libc=musl")
+    fi
+fi
+
+"$GRAALVM_HOME/bin/native-image" "${args[@]}"
diff --git a/script/setup-musl b/script/setup-musl
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+MUSL_DIR=${HOME}/.musl
+
+MUSL_GCC_COMPILER=${HOME}/.musl/bin/musl-gcc
+
+if [ -f "$MUSL_GCC_COMPILER" ]; then
+    echo "MUSL is already setup at ${MUSL_DIR}"
+    exit 0
+fi
+
+MUSL_VERSION=1.2.2
+ZLIB_VERSION=1.2.11
+
+mkdir $MUSL_DIR || true
+cd $MUSL_DIR
+curl https://musl.libc.org/releases/musl-${MUSL_VERSION}.tar.gz -o musl-${MUSL_VERSION}.tar.gz \
+    && tar zxvf musl-${MUSL_VERSION}.tar.gz \
+    && cd musl-${MUSL_VERSION} \
+    && ./configure --disable-shared --prefix=${MUSL_DIR} \
+    && make \
+    && make install \
+    && curl https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz -o zlib-${ZLIB_VERSION}.tar.gz \
+    && tar zxvf zlib-${ZLIB_VERSION}.tar.gz \
+    && cd zlib-${ZLIB_VERSION} \
+    && ./configure --static --prefix=${MUSL_DIR} \
+    && make \
+    && make install
+
+export PATH=$PATH:${MUSL_DIR}/bin
diff --git a/src/core.clj b/src/core.clj
@@ -12,6 +12,8 @@
   (:import (org.slf4j LoggerFactory)
            (ch.qos.logback.classic Logger Level)))
 
+(def version (str/trim (slurp (io/resource "KET_VERSION"))))
+
 (defn find-operation [operation-name cli-operations]
   (first (filter (fn [op] (= (name operation-name) (:name op))) cli-operations)))
 
@@ -35,16 +37,14 @@
           resp (cond
                  (true? (:docs options)) (:docs operation)
                  (true? (:defaults options)) (:defaults operation))]
-      (println
-        (json/encode
-          (if-let [msg (if (empty? resp)
-                         (if (empty? options)
-                           (throw (Exception. (format "Configuration for the operation '%s' is bad: '%s'"
-                                                      (name operation-name) options)))
-                           ((:handler-fn operation) options))
-                         resp)]
-            msg
-            (format "Operation '%s' is finished" (name operation-name))))))
+      (if-let [msg (if (empty? resp)
+                     (if (empty? options)
+                       (throw (Exception. (format "Configuration for the operation '%s' is bad: '%s'"
+                                                  (name operation-name) options)))
+                       ((:handler-fn operation) options))
+                     resp)]
+        (println (json/encode msg))
+        (log/infof "Operation '%s' is finished" (name operation-name))))
     (throw (Exception. "Operation name was not provided"))))
 
 (defn handle-subcommand [{:keys [options] :as cli-opts} cli-operations]
@@ -78,6 +78,11 @@
 (def cli-operations
   (concat ops/operations ops-overrides/cli))
 
+(defn print-summary-msg [summary]
+  (println (format "ket %s" version))
+  (println "Supported options:")
+  (println summary))
+
 (defn handle-cli [args]
   (let [{:keys [options summary errors arguments] :as cli-opts} (cli/recursive-parse args cli-operations)]
     (if errors
@@ -87,7 +92,7 @@
       (if (or (get options :help)
               (and (empty? options) (empty? arguments))
               (empty? args))
-        (println summary)
+        (print-summary-msg summary)
         (handle-subcommand cli-opts cli-operations)))))
 
 (comment