diff --git a/.tool-versions b/.tool-versions index 4f16989..0327b7d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ erlang 27.0 -elixir 1.17.0 +elixir 1.17.1 kind 0.23.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 131414e..66256d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +- Support for FLAME >= 0.2.0 and livebook integraion (requires livebook >= 0.13.0) - [#32](https://github.com/mruoss/flame_k8s_backend/pull/32) + ## [0.3.3] - 2024-04-29 ### Changed diff --git a/lib/flame_k8s_backend.ex b/lib/flame_k8s_backend.ex index 8b1f86d..3a3dcab 100644 --- a/lib/flame_k8s_backend.ex +++ b/lib/flame_k8s_backend.ex @@ -156,12 +156,8 @@ defmodule FLAMEK8sBackend do require Logger - defstruct env: %{}, - runner_pod_manifest: nil, + defstruct runner_pod_manifest: nil, parent_ref: nil, - runner_node_basename: nil, - runner_pod_ip: nil, - runner_pod_name: nil, runner_node_name: nil, runner_pod_tpl: nil, boot_timeout: nil, @@ -175,11 +171,10 @@ defmodule FLAMEK8sBackend do @impl true def init(opts) do conf = Application.get_env(:flame, __MODULE__) || [] - [node_base | _ip] = node() |> to_string() |> String.split("@") + [_node_base | _ip] = node() |> to_string() |> String.split("@") default = %FLAMEK8sBackend{ - boot_timeout: 30_000, - runner_node_basename: node_base + boot_timeout: 30_000 } provided_opts = @@ -197,11 +192,6 @@ defmodule FLAMEK8sBackend do parent_ref = make_ref() - encoded_parent = - parent_ref - |> FLAME.Parent.new(self(), __MODULE__) - |> FLAME.Parent.encode() - req = K8sClient.connect() case K8sClient.get_pod(req, System.get_env("POD_NAMESPACE"), System.get_env("POD_NAME")) do @@ -214,7 +204,7 @@ defmodule FLAMEK8sBackend do RunnerPodTemplate.manifest( base_pod, opts[:runner_pod_tpl], - encoded_parent, + parent_ref, Keyword.take(provided_opts, [:app_container_name, :omit_owner_reference]) ) ) @@ -266,11 +256,7 @@ defmodule FLAMEK8sBackend do case created_pod do {:ok, pod} -> log(state, "Runner pod created and scheduled", pod_ip: pod["status"]["podIP"]) - - struct!(state, - runner_pod_ip: pod["status"]["podIP"], - runner_pod_name: pod["metadata"]["name"] - ) + state :error -> Logger.error("failed to schedule runner pod within #{state.boot_timeout}ms") @@ -279,7 +265,6 @@ defmodule FLAMEK8sBackend do end) remaining_connect_window = state.boot_timeout - req_connect_time - runner_node_name = :"#{state.runner_node_basename}@#{new_state.runner_pod_ip}" log(state, "Waiting for Remote UP.", remaining_connect_window: remaining_connect_window) @@ -297,7 +282,7 @@ defmodule FLAMEK8sBackend do new_state = struct!(new_state, remote_terminator_pid: remote_terminator_pid, - runner_node_name: runner_node_name + runner_node_name: node(remote_terminator_pid) ) {:ok, remote_terminator_pid, new_state} diff --git a/lib/flame_k8s_backend/runner_pod_template.ex b/lib/flame_k8s_backend/runner_pod_template.ex index 64b166e..2b87186 100644 --- a/lib/flame_k8s_backend/runner_pod_template.ex +++ b/lib/flame_k8s_backend/runner_pod_template.ex @@ -121,22 +121,22 @@ defmodule FLAMEK8sBackend.RunnerPodTemplate do """ @spec manifest(parent_pod_manifest(), t() | callback(), Keyword.t()) :: runner_pod_template :: map() - def manifest(parent_pod_manifest, template_args_or_callback, encoded_parent, opts \\ []) + def manifest(parent_pod_manifest, template_args_or_callback, parent_ref, opts \\ []) - def manifest(parent_pod_manifest, template_callback, encoded_parent, opts) + def manifest(parent_pod_manifest, template_callback, parent_ref, opts) when is_function(template_callback) do app_container = app_container(parent_pod_manifest, opts) parent_pod_manifest |> template_callback.() - |> apply_defaults(parent_pod_manifest, app_container, encoded_parent, opts) + |> apply_defaults(parent_pod_manifest, app_container, parent_ref, opts) end - def manifest(parent_pod_manifest, nil, encoded_parent, opts) do - manifest(parent_pod_manifest, %RunnerPodTemplate{}, encoded_parent, opts) + def manifest(parent_pod_manifest, nil, parent_ref, opts) do + manifest(parent_pod_manifest, %RunnerPodTemplate{}, parent_ref, opts) end - def manifest(parent_pod_manifest, %RunnerPodTemplate{} = template_opts, encoded_parent, opts) do + def manifest(parent_pod_manifest, %RunnerPodTemplate{} = template_opts, parent_ref, opts) do app_container = app_container(parent_pod_manifest, opts) env = template_opts.env || [] @@ -159,25 +159,35 @@ defmodule FLAMEK8sBackend.RunnerPodTemplate do } } - apply_defaults(runner_pod_template, parent_pod_manifest, app_container, encoded_parent, opts) + apply_defaults(runner_pod_template, parent_pod_manifest, app_container, parent_ref, opts) end defp apply_defaults( runner_pod_template, parent_pod_manifest, app_container, - encoded_parent, + parent_ref, opts ) do parent_pod_manifest_name = parent_pod_manifest["metadata"]["name"] pod_name_sliced = String.slice(parent_pod_manifest_name, 0..40) - runner_pod_name = pod_name_sliced <> rand_id(20) + runner_pod_name = "#{pod_name_sliced}-#{rand_id(20)}" object_references = if opts[:omit_owner_reference], do: [], else: object_references(parent_pod_manifest) + parent = FLAME.Parent.new(parent_ref, self(), FLAMEK8sBackend, runner_pod_name, "POD_IP") + + parent = + if case(System.get_env("FLAME_K8S_BACKEND_GIT_REF")) do + nil -> parent + git_ref -> struct(parent, backend_vsn: [github: "mruoss/flame_k8s_backend", ref: git_ref]) + end + + encoded_parent = FLAME.Parent.encode(parent) + runner_pod_template |> Map.merge(%{"apiVersion" => "v1", "kind" => "Pod"}) |> put_in(~w(metadata name), runner_pod_name) diff --git a/test/flame_k8s_backend/runner_pod_template_test.exs b/test/flame_k8s_backend/runner_pod_template_test.exs index 8d4e946..fdca0bd 100644 --- a/test/flame_k8s_backend/runner_pod_template_test.exs +++ b/test/flame_k8s_backend/runner_pod_template_test.exs @@ -6,6 +6,14 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do import YamlElixir.Sigil + defp flame_parent(pod_manifest) do + pod_manifest + |> get_in(env_var_access("FLAME_PARENT")) + |> List.first() + |> Base.decode64!() + |> :erlang.binary_to_term() + end + defp env_var_access(name) do app_container_access(["env", Access.filter(&(&1["name"] == name)), "value"]) end @@ -40,7 +48,7 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do ) end - MUT.manifest(parent_pod_manifest, callback, "ENCODED_PARNET_REF") + MUT.manifest(parent_pod_manifest, callback, make_ref()) end test "should return pod manifest with data form callback", %{ @@ -63,7 +71,7 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do ) end - pod_manifest = MUT.manifest(parent_pod_manifest, callback, "ENCODED_PARNET_REF") + pod_manifest = MUT.manifest(parent_pod_manifest, callback, make_ref()) assert get_in(pod_manifest, app_container_access(~w(resources requests memory))) == "100Mi" assert get_in(pod_manifest, app_container_access(~w(resources limits memory))) == "500Mi" @@ -85,7 +93,7 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do """ end - pod_manifest = MUT.manifest(parent_pod_manifest, callback, "ENCODED_PARNET_REF") + pod_manifest = MUT.manifest(parent_pod_manifest, callback, make_ref()) assert get_in(pod_manifest, app_container_access() ++ ["image"]) == "flame-test-image:0.1.0" owner_references = get_in(pod_manifest, ~w(metadata ownerReferences)) @@ -112,7 +120,7 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do end pod_manifest = - MUT.manifest(parent_pod_manifest, callback, "ENCODED_PARNET_REF", + MUT.manifest(parent_pod_manifest, callback, make_ref(), app_container_name: "other-container" ) @@ -136,9 +144,7 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do end pod_manifest = - MUT.manifest(parent_pod_manifest, callback, "ENCODED_PARNET_REF", - omit_owner_reference: true - ) + MUT.manifest(parent_pod_manifest, callback, make_ref(), omit_owner_reference: true) assert [] == get_in(pod_manifest, ~w(metadata ownerReferences)) end @@ -149,17 +155,17 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do parent_pod_manifest_full: parent_pod_manifest } do template_opts = %MUT{} - pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, "ENCODED_PARNET_REF") + pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, make_ref()) assert get_in(pod_manifest, env_var_access("RELEASE_NODE")) == ["flame_test@$(POD_IP)"] - assert get_in(pod_manifest, env_var_access("FLAME_PARENT")) == ["ENCODED_PARNET_REF"] end test "Only default envs if add_parent_env is set to false", %{ parent_pod_manifest_full: parent_pod_manifest } do + ref = make_ref() template_opts = %MUT{add_parent_env: false} - pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, "ENCODED_PARNET_REF") + pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, ref) assert get_in(pod_manifest, app_container_access(~w(resources requests memory))) == "100Mi" assert get_in(pod_manifest, env_var_access("POD_NAMESPACE")) == ["test-namespace"] @@ -168,7 +174,8 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do get_in(pod_manifest, env_var_access("POD_NAME")) assert get_in(pod_manifest, env_var_access("PHX_SERVER")) == ["false"] - assert get_in(pod_manifest, env_var_access("FLAME_PARENT")) == ["ENCODED_PARNET_REF"] + parent = flame_parent(pod_manifest) + assert parent.ref == ref end end @@ -177,22 +184,20 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do parent_pod_manifest_full: parent_pod_manifest } do template_opts = %MUT{env: [%{"name" => "FOO", "value" => "bar"}]} - pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, "ENCODED_PARNET_REF") + pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, make_ref()) assert get_in(pod_manifest, env_var_access("RELEASE_NODE")) == ["flame_test@$(POD_IP)"] assert get_in(pod_manifest, env_var_access("FOO")) == ["bar"] - assert get_in(pod_manifest, env_var_access("FLAME_PARENT")) == ["ENCODED_PARNET_REF"] end test "No parent envs if add_parent_env is set to false", %{ parent_pod_manifest_full: parent_pod_manifest } do template_opts = %MUT{env: [%{"name" => "FOO", "value" => "bar"}], add_parent_env: false} - pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, "ENCODED_PARNET_REF") + pod_manifest = MUT.manifest(parent_pod_manifest, template_opts, make_ref()) assert get_in(pod_manifest, env_var_access("RELEASE_NODE")) == [] assert get_in(pod_manifest, env_var_access("FOO")) == ["bar"] - assert get_in(pod_manifest, env_var_access("FLAME_PARENT")) == ["ENCODED_PARNET_REF"] end end @@ -200,12 +205,11 @@ defmodule FLAMEK8sBackend.RunnerPodTemplateTest do test "Uses parent pod's values for empty template opts", %{ parent_pod_manifest_full: parent_pod_manifest } do - pod_manifest = MUT.manifest(parent_pod_manifest, nil, "ENCODED_PARNET_REF") + pod_manifest = MUT.manifest(parent_pod_manifest, nil, make_ref()) assert get_in(pod_manifest, app_container_access(~w(resources requests memory))) == "100Mi" assert get_in(pod_manifest, env_var_access("RELEASE_NODE")) == ["flame_test@$(POD_IP)"] - assert get_in(pod_manifest, env_var_access("FLAME_PARENT")) == ["ENCODED_PARNET_REF"] end end end diff --git a/test/integration/Dockerfile b/test/integration/Dockerfile index f9deda8..e8d92ee 100755 --- a/test/integration/Dockerfile +++ b/test/integration/Dockerfile @@ -1,5 +1,5 @@ -FROM hexpm/elixir:1.15.7-erlang-26.1.2-debian-buster-20231009 +FROM hexpm/elixir:1.17.1-erlang-27.0-debian-bullseye-20240612-slim ENV MIX_ENV=test \ MIX_HOME=/opt/mix \