From 34ebe4e8693def6e6d3baf2553b380bc795533f6 Mon Sep 17 00:00:00 2001 From: Casey Link Date: Thu, 13 Jun 2024 20:36:58 +0200 Subject: [PATCH] Add container image and slim down the JDK used for Datomic --- .gitignore | 3 + CHANGELOG.md | 5 + README.md | 236 ++++++++++++++++++++++++++- flake.nix | 92 ++++------- nixos-modules/datomic-console.nix | 4 +- nixos-modules/datomic-pro.nix | 2 +- pkgs/datomic-generate-properties.nix | 39 +++++ pkgs/datomic-pro-container-image.nix | 71 ++++++++ pkgs/datomic-pro.nix | 7 +- pkgs/generate-properties.clj | 134 +++++++++++++++ tests/container-image.nix | 50 ++++++ tests/nixos-module.nix | 67 ++++++++ 12 files changed, 641 insertions(+), 69 deletions(-) create mode 100644 pkgs/datomic-generate-properties.nix create mode 100644 pkgs/datomic-pro-container-image.nix create mode 100755 pkgs/generate-properties.clj create mode 100644 tests/container-image.nix create mode 100644 tests/nixos-module.nix diff --git a/.gitignore b/.gitignore index b2be92b..1c045ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ result +.lsp +.clj-kondo +data diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bab5e1..27f0ec9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [UNRELEASED] + +- Added Docker container image +- Slimmed down the JDK used in the nix package + ## v0.1.0 (2024-06-12) diff --git a/README.md b/README.md index 8e90d06..4634675 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,37 @@ [![GitHub License](https://img.shields.io/github/license/ramblurr/datomic-pro-flake)](./LICENSE) [![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/ramblurr/datomic-pro-flake/ci.yml)](https://github.com/Ramblurr/datomic-pro-flake/actions/workflows/ci.yml) -❄️ This flake exposes a `datomic-pro` nix package and several NixOS modules for running Datomic Pro on NixOS. +️ This flake exposes: -> 🐋 Looking for a container / docker image instead? Check out my other repo [ramblurr/containers](https://github.com/Ramblurr/containers/tree/main/apps/datomic-pro). +* A `datomic-pro` nix package +* ❄ Several NixOS modules for running Datomic Pro on NixOS +* 🐋 A container image that you can use to run Datomic (no nix required!) -This repo also contains a working NixOS module test that runs the flake inside a virtual machine in CI to verify that it's working. +All of the above are tested automatically with a virtual machine! -## Usage + +**Table of Contents** + +- [datomic-pro-flake](#datomic-pro-flake) + - [Usage - NixOS Module](#usage---nixos-module) + - [`flake.nix`](#flakenix) + - [`/etc/datomic-pro/secrets.properties`](#etcdatomic-prosecretsproperties) + - [`configuration.nix`](#configurationnix) + - [🐋 Usage - Docker (OCI) Container Image](#🐋-usage---docker-oci-container-image) + - [Transactor Mode](#transactor-mode) + - [Env vars](#env-vars) + - [Console Mode](#console-mode) + - [Env vars](#env-vars-1) + - [Example Compose](#example-compose) + - [Datomic Pro with Local Storage](#datomic-pro-with-local-storage) + - [Datomic Pro with Postgres Storage and memcached](#datomic-pro-with-postgres-storage-and-memcached) + - [Discussion](#discussion) + - [License](#license) + + + + +## Usage - NixOS Module ### `flake.nix` @@ -94,6 +118,210 @@ A basic dev-mode datomic that stores its state in `/var/lib/datomic-pro`: } ``` +## 🐋 Usage - Docker (OCI) Container Image + +This flake also provides a container image for Datomic Pro that can be driven entirely with environment variables or `_FILE` style secrets. + +If you don't want to build the container image yourself with nix, you can get the latest image with: + +``` shell +docker/podman pull ghcr.io/ramblurr/datomic-pro: +``` + +The available tags you can find here: https://github.com/users/Ramblurr/packages/datomic-pro-flake/package/datomic-pro + +### Transactor Mode + +Runs a Datomic Transactor. This is the default mode when the container is run with no command. + +* The default port is `4334`. +* A rw volume of `/config` is required. +* Configure with env vars (see below) or add `/config/transactor.properties` to supply a config to the transactor. +* A rw volume of `/data` is optional (for use in H2 mode). + +#### Env vars + +> [!IMPORTANT] +> All env vars can be passed with `_FILE` to read the value from a file +> (e.g, when using secrets). Example: `DATOMIC_STORAGE_ADMIN_PASSWORD` can be +> passed as `DATOMIC_STORAGE_ADMIN_PASSWORD_FILE=/run/secrets/admin-password` and +> the value from that file will be used as the admin password. + +* `DATOMIC_TRANSACTOR_PROPERTIES_PATH` - The path to the properties file for the transactor. Defaults to `/config/transactor.properties` + +The following environment vars configure the properties, refer to the Datomic documentation for more information: + +* `DATOMIC_ALT_HOST` - `alt-host` +* `DATOMIC_DATA_DIR` - `data-dir` (default: /data) +* `DATOMIC_ENCRYPT_CHANNEL` - `encrypt-channel` +* `DATOMIC_HEARTBEAT_INTERVAL_MSEC` - `heartbeat-interval-msec` +* `DATOMIC_HOST` - `host` (default: 0.0.0.0) +* `DATOMIC_MEMCACHED` - `memcached` +* `DATOMIC_MEMCACHED_AUTO_DISCOVERY` - `memcached-auto-discovery` +* `DATOMIC_MEMCACHED_CONFIG_TIMEOUT_MSEC` - `memcached-config-timeout-msec` +* `DATOMIC_MEMCACHED_PASSWORD` - `memcached-password` +* `DATOMIC_MEMCACHED_USERNAME` - `memcached-username` +* `DATOMIC_MEMORY_INDEX_MAX` - `memory-index-max` (default: 256m) +* `DATOMIC_MEMORY_INDEX_THRESHOLD` - `memory-index-threshold` (default: 32m) +* `DATOMIC_OBJECT_CACHE_MAX` - `object-cache-max` (default: 128m) +* `DATOMIC_PID_FILE` - `pid-file` +* `DATOMIC_HEALTHCHECK_CONCURRENCY` - `ping-concurrency` +* `DATOMIC_HEALTHCHECK_HOST` - `ping-host` +* `DATOMIC_HEALTHCHECK_PORT` - `ping-port` +* `DATOMIC_PORT` - `port` (default: 4334) +* `DATOMIC_PROTOCOL` - `protocol` (default: dev) +* `DATOMIC_READ_CONCURRENCY` - `read-concurrency` +* `DATOMIC_SQL_DRIVER_CLASS` - `sql-driver-class` +* `DATOMIC_SQL_URL` - `sql-url` +* `DATOMIC_STORAGE_ACCESS` - `storage-access` (default: remote) +* `DATOMIC_STORAGE_ADMIN_PASSWORD` - `storage-admin-password` +* `DATOMIC_STORAGE_DATOMIC_PASSWORD` - `storage-datomic-password` +* `DATOMIC_VALCACHE_MAX_GB` - `valcache-max-gb` +* `DATOMIC_VALCACHE_PATH` - `valcache-path` +* `DATOMIC_WRITE_CONCURRENCY` - `write-concurrency` + +### Console Mode + +Runs the Datomic Console. + +* Run this mode by passing the `console` as the first and only argument to the container. +* The default port is `8080`. + +#### Env vars + +* `DB_URI` - the database connection URI that console uses to connect to datomic +* `DB_URI_FILE` - will read the connection URI from the file specified by this env var + +### Example Compose + +#### Datomic Pro with Local Storage + +Please note the tag below may not be up to date. + +``` yaml +--- +services: + datomic-transactor: + image: ghcr.io/ramblurr/datomic-pro:1.0.7075 + environment: + DATOMIC_STORAGE_ADMIN_PASSWORD: unsafe + DATOMIC_STORAGE_DATOMIC_PASSWORD: unsafe + volumes: + - ./data:/data + ports: + - 127.0.0.1:4334:4334 + #user: 1000:1000 # if using rootful containers uncomment this + + datomic-console: + image: ghcr.io/ramblurr/datomic-pro:1.0.7075 + command: console + environment: + DB_URI: datomic:dev://datomic-transactor:4334/?password=unsafe + ports: + - 127.0.0.1:8081:8080 + #user: 1000:1000 # if using rootful containers uncomment this +``` + +#### Datomic Pro with Postgres Storage and memcached + +This compose file is near-production ready. But you shouldn't manage the +lifecycle of the postgres schema this way. How you do it depends on your +environment. + +Please note the tag below may not be up to date. + +``` yaml +--- +services: + datomic-memcached: + image: docker.io/memcached:latest + command: memcached -m 1024 + ports: + - 127.0.0.1:11211:11211 + restart: always + healthcheck: + test: + [ + "CMD-SHELL", + 'bash -c ''echo "version" | (exec 3<>/dev/tcp/localhost/11211; cat >&3; timeout 0.1 cat <&3; exec 3<&-)''', + ] + interval: 5s + retries: 60 + + datomic-storage: + image: docker.io/library/postgres:latest + environment: + POSTGRES_PASSWORD: unsafe + command: postgres -c 'max_connections=1024' + volumes: + - ./data:/var/lib/postgresql/data + ports: + - 127.0.0.1:5432:5432 + restart: always + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 3s + retries: 30 + + datomic-storage-migrator: + image: ghcr.io/ramblurr/datomic-pro:1.0.7075 + environment: + PGUSER: postgres + PGPASSWORD: unsafe + volumes: + - "./postgres-migrations:/migrations" + entrypoint: /bin/sh + command: > + -c '(psql -h datomic-storage -lqt | cut -d \| -f 1 | grep -qw "datomic" || psql -h datomic-storage -f /opt/datomic-pro/bin/sql/postgres-db.sql) && + (psql -h datomic-storage -d datomic -c "\dt" | grep -q "datomic_kvs" || psql -h datomic-storage -d datomic -f /opt/datomic-pro/bin/sql/postgres-table.sql) && + (psql -h datomic-storage -d datomic -c "\du" | cut -d \| -f 1 | grep -qw "datomic" || psql -h datomic-storage -d datomic -f /opt/datomic-pro/bin/sql/postgres-user.sql)' + depends_on: + datomic-storage: + condition: service_healthy + + datomic-transactor: + image: ghcr.io/ramblurr/datomic-pro:1.0.7075 + environment: + DATOMIC_STORAGE_ADMIN_PASSWORD: unsafe + DATOMIC_STORAGE_DATOMIC_PASSWORD: unsafe + DATOMIC_PROTOCOL: sql + DATOMIC_SQL_URL: jdbc:postgresql://datomic-storage:5432/datomic?user=datomic&password=datomic + DATOMIC_HEALTHCHECK_HOST: 127.0.0.1 + DATOMIC_HEALTHCHECK_PORT: 9999 + DATOMIC_MEMCACHED: datomic-memcached:11211 + ports: + - 127.0.0.1:4334:4334 + #user: 1000:1000 # if using rootful containers uncomment this + restart: always + healthcheck: + test: + [ + "CMD-SHELL", + 'if [[ $(curl -s -o /dev/null -w "%{http_code}" -X GET http://127.0.0.1:9999/health) = "200" ]]; then echo 0; else echo 1; fi', + ] + interval: 10s + timeout: 3s + retries: 30 + depends_on: + datomic-storage: + condition: service_healthy + datomic-memcached: + condition: service_healthy + datomic-storage-migrator: + condition: service_completed_successfully + + datomic-console: + image: ghcr.io/ramblurr/datomic-pro:1.0.7075 + command: console + environment: + DB_URI: datomic:sql://?jdbc:postgresql://datomic-storage:5432/datomic?user=datomic&password=datomic + ports: + - 127.0.0.1:8081:8080 + #user: 1000:1000 # if using rootful containers uncomment this +``` + + ### Discussion Feel free to [open an issue](https://github.com/Ramblurr/datomic-pro-flake/issues/new) or reach out to me on the Clojurians slack ([@Ramblurr](https://clojurians.slack.com/team/U70QFSCG2)). diff --git a/flake.nix b/flake.nix index b70b5af..3e7b4be 100644 --- a/flake.nix +++ b/flake.nix @@ -17,78 +17,50 @@ inherit system; overlays = [ self.overlays."${system}" ]; }; + + jdk-minimal = pkgs.jdk21.override { + headless = true; + enableJavaFX = false; + enableGnome2 = false; + }; in { - overlays = final: prev: { datomic-pro = pkgs.callPackage ./pkgs/datomic-pro.nix { }; }; + overlays = final: prev: { + jdk-minimal = jdk-minimal; + datomic-pro = pkgs.callPackage ./pkgs/datomic-pro.nix { }; + datomic-pro-container = pkgs.callPackage ./pkgs/datomic-pro-container-image.nix { }; + datomic-generate-properties = pkgs.callPackage ./pkgs/datomic-generate-properties.nix { }; + }; packages = { default = self.packages.${system}.datomic-pro; datomic-pro = pkgs.datomic-pro; + datomic-pro-container = pkgs.datomic-pro-container; + datomic-generate-properties = pkgs.datomic-generate-properties; }; nixosModules = { datomic-pro = import ./nixos-modules/datomic-pro.nix; datomic-console = import ./nixos-modules/datomic-console.nix; }; checks = { - # A VM test of the NixOS module. - vmTest = - with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system; }; - - makeTest { - name = "datomic-pro module test"; - nodes = { - client = - { ... }: - { - nixpkgs.overlays = [ self.overlays."${system}" ]; - virtualisation.memorySize = 2048; - imports = [ - self.nixosModules.${system}.datomic-pro - self.nixosModules.${system}.datomic-console - ]; - environment.etc."datomic-pro/do-not-do-this.properties" = { - text = '' - storage-admin-password=do-not-do-it-this-way-in-prod - storage-datomic-password=do-not-do-it-this-way-in-prod - ''; - mode = "0600"; - }; - services.datomic-pro = { - enable = true; - secretsFile = "/etc/datomic-pro/do-not-do-this.properties"; - settings = { - enable = true; - host = "localhost"; - port = 4334; - memory-index-max = "256m"; - memory-index-threshold = "32m"; - object-cache-max = "128m"; - protocol = "dev"; - storage-access = "remote"; - }; - }; - - environment.etc."datomic-console/do-not-do-this" = { - text = "datomic:dev://localhost:4334/?password=do-not-do-it-this-way-in-prod"; - mode = "0600"; - }; - services.datomic-console = { - enable = true; - alias = "dev"; - port = 8080; - dbUriFile = "/etc/datomic-console/do-not-do-this"; - }; - }; - }; + # A test of the NixOS module that runs in a VM + moduleTest = import ./tests/nixos-module.nix { + inherit + system + pkgs + nixpkgs + self + ; + }; - testScript = '' - start_all() - machine.wait_for_unit("datomic-pro.service") - machine.wait_for_open_port(4334) - machine.wait_for_unit("datomic-console.service") - machine.wait_for_open_port(8080) - machine.succeed("curl --fail http://localhost:8080/browse") - ''; - }; + # A test of the container image that runs in a VM + containerImageTest = import ./tests/container-image.nix { + inherit + system + pkgs + nixpkgs + self + ; + }; }; } ); diff --git a/nixos-modules/datomic-console.nix b/nixos-modules/datomic-console.nix index c9f23c6..debc29f 100644 --- a/nixos-modules/datomic-console.nix +++ b/nixos-modules/datomic-console.nix @@ -13,13 +13,13 @@ in services.datomic-console = { enable = lib.mkEnableOption "Datomic Pro Console"; package = lib.mkPackageOption pkgs "datomic-pro" { }; - javaPackage = lib.mkPackageOption pkgs "temurin-bin" { }; + javaPackage = lib.mkPackageOption pkgs "jdk-minimal" { }; port = lib.mkOption { type = lib.types.port; description = "The port the console will bind to"; }; alias = lib.mkOption { - type = lib.types.string; + type = lib.types.str; default = "dev"; description = "A text-based name for a transactor that is associated with a transactor URI that does not include a database name"; }; diff --git a/nixos-modules/datomic-pro.nix b/nixos-modules/datomic-pro.nix index 6860f6b..97547df 100644 --- a/nixos-modules/datomic-pro.nix +++ b/nixos-modules/datomic-pro.nix @@ -28,7 +28,7 @@ in services.datomic-pro = { enable = lib.mkEnableOption "Datomic Pro"; package = lib.mkPackageOption pkgs "datomic-pro" { }; - javaPackage = lib.mkPackageOption pkgs "temurin-bin" { }; + javaPackage = lib.mkPackageOption pkgs "jdk-minimal" { }; secretsFile = lib.mkOption { type = lib.types.nullOr lib.types.path; default = null; diff --git a/pkgs/datomic-generate-properties.nix b/pkgs/datomic-generate-properties.nix new file mode 100644 index 0000000..e1a6475 --- /dev/null +++ b/pkgs/datomic-generate-properties.nix @@ -0,0 +1,39 @@ +{ + lib, + stdenv, + fetchzip, + babashka, + dockerTools, + datomic-pro, + hostname, + ... +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "datomic-generate-properties"; + version = "0.1"; + + src = ./generate-properties.clj; + + dontUnpack = true; + propagatedBuildInputs = [ + babashka + hostname + ]; + installPhase = '' + runHook preInstall + ls -al $src + mkdir -p $out/bin + cp -r $src $out/bin/datomic-generate-properties + chmod +x $out/bin/datomic-generate-properties + find $out + runHook postInstall + ''; + meta = { + description = "Generates a java properties file for Datomic Pro based off of environment variables and sane defaults."; + homepage = ""; + changelog = ""; + license = lib.licenses.asl20; + mainProgram = "datomic-generate-properties"; + }; +}) diff --git a/pkgs/datomic-pro-container-image.nix b/pkgs/datomic-pro-container-image.nix new file mode 100644 index 0000000..dfedb91 --- /dev/null +++ b/pkgs/datomic-pro-container-image.nix @@ -0,0 +1,71 @@ +{ + lib, + stdenv, + fetchzip, + dockerTools, + datomic-pro, + datomic-generate-properties, + writeShellScriptBin, + coreutils, + jdk-minimal, + babashka, + hostname, + bash, + ... +}: + +let + entrypoint = writeShellScriptBin "datomic-entrypoint" '' + set -e + export PATH=${bash}/bin:${hostname}/bin:${jdk-minimal}/bin:${coreutils}/bin:${babashka}:$PATH + if [ "$(id -u)" = "0" ]; then + echo "WARNING: Running Datomic as root is not recommended. Please run as a non-root user." + echo " This can be ignored if you are using rootless mode." + fi + if [ "$1" = "console" ]; then + echo "Starting Datomic Console..." + if [ -n "$DB_URI_FILE" ] && [ -f "$DB_URI_FILE" ]; then + DB_URI=$(cat "$DB_URI_FILE") + fi + if [ -z "$DB_URI" ]; then + echo "DB_URI is not set. Please set DB_URI environment variable or provide a file path with DB_URI_FILE." + exit 1 + fi + ${datomic-pro}/bin/console -p 8080 dev "$DB_URI" + else + echo "Generating Datomic Properties" + ${datomic-generate-properties}/bin/datomic-generate-properties + echo "Starting Datomic Transactor..." + ${datomic-pro}/bin/transactor "$DATOMIC_TRANSACTOR_PROPERTIES_PATH" + fi + ''; +in +dockerTools.buildLayeredImage { + name = "ghcr.io/ramblurr/datomic-pro"; + tag = datomic-pro.version; + fromImage = null; + config = { + WorkingDir = datomic-pro; + Entrypoint = [ "${entrypoint}/bin/datomic-entrypoint" ]; + + Env = [ + "DATOMIC_TRANSACTOR_PROPERTIES_PATH=/config/transactor.properties" + "LC_ALL=C.UTF-8" + "LANG=C.UTF-8" + "UMASK=0002" + "TZ=Etc/UTC" + ]; + Labels = { + "org.opencontainers.image.authors" = "github.com/ramblurr"; + "org.opencontainers.image.url" = "https://github.com/Ramblurr/datomic-pro-flake/tree/main/pkgs/datomic-pro-container-image.nix"; + "org.opencontainers.image.source" = "https://github.com/Ramblurr/datomic-pro-flake"; + "org.opencontainers.image.description" = "Datomic Pro"; + "org.opencontainers.image.version" = datomic-pro.version; + "org.opencontainers.image.licenses" = "Apache-2.0"; + }; + Volumes = { + "/config" = { }; + "/data" = { }; + }; + }; +} diff --git a/pkgs/datomic-pro.nix b/pkgs/datomic-pro.nix index aae3a28..e1c5de2 100644 --- a/pkgs/datomic-pro.nix +++ b/pkgs/datomic-pro.nix @@ -2,10 +2,13 @@ lib, stdenv, fetchzip, - temurin-bin, + jdk-minimal, ... }: +let +in + stdenv.mkDerivation (finalAttrs: { pname = "datomic-pro"; version = "1.0.7075"; @@ -14,7 +17,7 @@ stdenv.mkDerivation (finalAttrs: { url = "https://datomic-pro-downloads.s3.amazonaws.com/${finalAttrs.version}/datomic-pro-${finalAttrs.version}.zip"; sha256 = "sha256-5WBtENqvH7n2iuNxXwjPlR7tDiKSYoF7EXgBE6BflGg="; }; - propagatedBuildInputs = [ temurin-bin ]; + propagatedBuildInputs = [ jdk-minimal ]; installPhase = '' runHook preInstall ls -al $src diff --git a/pkgs/generate-properties.clj b/pkgs/generate-properties.clj new file mode 100755 index 0000000..bda3c58 --- /dev/null +++ b/pkgs/generate-properties.clj @@ -0,0 +1,134 @@ +#!/usr/bin/env bb +(require '[babashka.fs :as fs]) +(require '[babashka.process :as p]) + +(defn write-properties [props file-path] + (with-open [wrtr (io/writer file-path)] + (doseq [[k v] (sort-by first props)] + (.write wrtr (str k "=" v "\n"))))) + +(defn exit-msg [msg] + (println msg) + (System/exit 1)) + +(defn assert-msg + ;; instead of using (assert) which throws an exception and looks messy, we do this for better DX + [condition msg] + (when (not condition) + (exit-msg (str "Error: " msg)))) + +(defn read-env-or-file [env-name] + (let [ef (System/getenv (str env-name "_FILE"))] + (if (fs/exists? ef) + (str/trim (slurp ef)) + (System/getenv env-name)))) + +(def config-opts + {"host" "DATOMIC_HOST" + "protocol" "DATOMIC_PROTOCOL" + "storage-access" "DATOMIC_STORAGE_ACCESS" + "storage-admin-password" "DATOMIC_STORAGE_ADMIN_PASSWORD" + "storage-datomic-password" "DATOMIC_STORAGE_DATOMIC_PASSWORD" + "data-dir" "DATOMIC_DATA_DIR" + "sql-driver-class" "DATOMIC_SQL_DRIVER_CLASS" + "pid-file" "DATOMIC_PID_FILE" + "alt-host" "DATOMIC_ALT_HOST" + "ping-host" "DATOMIC_HEALTHCHECK_HOST" + "ping-port" "DATOMIC_HEALTHCHECK_PORT" + "ping-concurrency" "DATOMIC_HEALTHCHECK_CONCURRENCY" + "heartbeat-interval-msec" "DATOMIC_HEARTBEAT_INTERVAL_MSEC" + "encrypt-channel" "DATOMIC_ENCRYPT_CHANNEL" + "write-concurrency" "DATOMIC_WRITE_CONCURRENCY" + "read-concurrency" "DATOMIC_READ_CONCURRENCY" + "port" "DATOMIC_PORT" + "sql-url" "DATOMIC_SQL_URL" + "memory-index-threshold" "DATOMIC_MEMORY_INDEX_THRESHOLD" + "memory-index-max" "DATOMIC_MEMORY_INDEX_MAX" + "object-cache-max" "DATOMIC_OBJECT_CACHE_MAX" + "memcached" "DATOMIC_MEMCACHED" + "memcached-config-timeout-msec" "DATOMIC_MEMCACHED_CONFIG_TIMEOUT_MSEC" + "memcached-username" "DATOMIC_MEMCACHED_USERNAME" + "memcached-password" "DATOMIC_MEMCACHED_PASSWORD" + "memcached-auto-discovery" "DATOMIC_MEMCACHED_AUTO_DISCOVERY" + "valcache-path" "DATOMIC_VALCACHE_PATH" + "valcache-max-gb" "DATOMIC_VALCACHE_MAX_GB"}) + +(def config-defaults {"host" "0.0.0.0" + "port" "4334" + "storage-access" "remote" + "protocol" "dev" + "data-dir" "/data" + "memory-index-max" "256m" + "memory-index-threshold" "32m" + "object-cache-max" "128m"}) + +(def config-sql-defaults {"host" "0.0.0.0" + "port" "4334" + "sql-driver-class" "org.postgresql.Driver" + "memory-index-max" "256m" + "memory-index-threshold" "32m" + "object-cache-max" "128m"}) + +(defn get-config [protocol prop env-var] + (or + (read-env-or-file env-var) + (get (condp = protocol "dev" + config-defaults + config-sql-defaults) prop))) + +(defn load-config [] + (let [protocol (or (read-env-or-file "DATOMIC_PROTOCOL") "dev")] + (reduce (fn [props [prop env-var]] + (if-let [val (get-config protocol prop env-var)] + (assoc props prop val) + props)) {} config-opts))) + +(defn validate-dev-mode [props] + (assert-msg (contains? props "storage-access") "storage-access is required") + (if (= "remote" (get props "storage-access")) + (do + (assert-msg (contains? props "storage-admin-password") "storage-admin-password is required in dev mode") + (assert-msg (contains? props "storage-datomic-password") "storage-datomic-password is required in dev mode") + props) + (do + (println "WARNING: Running in dev mode with storage-access != remote.. you won't be able to connect to this transactor outside the container") + props))) + +(defn validate-sql-mode [props] + (assert-msg (contains? props "sql-url") "sql-url is required") + props) + +(defn validate-config [props] + (condp = (get props "protocol") + "dev" (validate-dev-mode props) + "sql" (validate-sql-mode props) + props)) + +(defn get-ip [] + (str/trim (p/shell {:out :string} "hostname -i"))) + +(defn update-alt-host [props] + (if (contains? props "alt-host") + (assoc props "alt-host" (get-ip)) + props)) + +(defn conf-file [] + (or + (System/getenv "DATOMIC_TRANSACTOR_PROPERTIES_PATH") + "/config/transactor.properties")) + +(defn format-readme [] + (doseq [[prop env-var] (sort-by first config-opts)] + (let [default (get config-defaults prop nil)] + (println (format "* `%s` - `%s`%s" env-var prop (if default (format " (default: %s)" default) "")))))) + +(defn -main [& args] + #_(format-readme) + (-> + (load-config) + (validate-config) + (update-alt-host) + (write-properties (conf-file)))) + +(when (= *file* (System/getProperty "babashka.file")) + (apply -main *command-line-args*)) diff --git a/tests/container-image.nix b/tests/container-image.nix new file mode 100644 index 0000000..e11abb9 --- /dev/null +++ b/tests/container-image.nix @@ -0,0 +1,50 @@ +{ + self, + system, + pkgs, + nixpkgs, +}: + +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system pkgs; }; +with pkgs.lib; + +makeTest { + name = "datomic-pro dev-mode container test"; + nodes = { + docker = + { ... }: + { + nixpkgs.overlays = [ self.overlays."${system}" ]; + virtualisation = { + diskSize = 8192; + memorySize = 2048; + docker.enable = true; + }; + environment.systemPackages = with pkgs; [ jq ]; + }; + }; + + testScript = '' + start_all() + docker.wait_for_unit("sockets.target") + docker.succeed( + "docker load --input='${pkgs.datomic-pro-container}'" + ) + + docker.succeed("rm -rf ./data && mkdir ./data") + docker.succeed( + """ + docker run -d --name datomic -v ./data:/data -p 4335:4334 -e DATOMIC_STORAGE_ADMIN_PASSWORD=unsafe -e DATOMIC_STORAGE_DATOMIC_PASSWORD=unsafe ghcr.io/ramblurr/datomic-pro:${pkgs.datomic-pro.version} + """ + ) + docker.wait_for_open_port(4335) + def try_logs(_) -> bool: + status, _ = docker.execute("docker logs datomic | grep -q 'System started'") + return status == 0 + with docker.nested("waiting for datomic to start"): + retry(try_logs) + docker.wait_for_file("./data/db/datomic.trace.db") + docker.succeed("docker rm -f datomic") + docker.wait_for_closed_port(4335) + ''; +} diff --git a/tests/nixos-module.nix b/tests/nixos-module.nix new file mode 100644 index 0000000..120384d --- /dev/null +++ b/tests/nixos-module.nix @@ -0,0 +1,67 @@ +{ + self, + system, + pkgs, + nixpkgs, +}: + +with import (nixpkgs + "/nixos/lib/testing-python.nix") { inherit system pkgs; }; +with pkgs.lib; + +makeTest { + name = "datomic-pro module test"; + nodes = { + client = + { ... }: + { + nixpkgs.overlays = [ self.overlays."${system}" ]; + virtualisation.memorySize = 2048; + imports = [ + self.nixosModules.${system}.datomic-pro + self.nixosModules.${system}.datomic-console + ]; + environment.etc."datomic-pro/do-not-do-this.properties" = { + text = '' + storage-admin-password=do-not-do-it-this-way-in-prod + storage-datomic-password=do-not-do-it-this-way-in-prod + ''; + mode = "0600"; + }; + services.datomic-pro = { + enable = true; + secretsFile = "/etc/datomic-pro/do-not-do-this.properties"; + settings = { + enable = true; + host = "localhost"; + port = 4334; + memory-index-max = "256m"; + memory-index-threshold = "32m"; + object-cache-max = "128m"; + protocol = "dev"; + storage-access = "remote"; + }; + }; + + environment.etc."datomic-console/do-not-do-this" = { + text = "datomic:dev://localhost:4334/?password=do-not-do-it-this-way-in-prod"; + mode = "0600"; + }; + services.datomic-console = { + enable = true; + alias = "dev"; + port = 8080; + dbUriFile = "/etc/datomic-console/do-not-do-this"; + }; + }; + }; + + testScript = '' + start_all() + machine.wait_for_unit("datomic-pro.service") + machine.wait_for_open_port(4334) + + machine.wait_for_unit("datomic-console.service") + machine.wait_for_open_port(8080) + machine.succeed("curl --fail http://localhost:8080/browse") + ''; +}