[PROTOTYPE] Add logical types decoding

.github/workflows/ci.yaml

@@ -184,4 +184,4 @@ jobs:
           mix local.rebar --force
           mix local.hex --force
           mix deps.get
-      - run: mix test
+      - run: mix do cmd mix test, testi

config/config.exs

@@ -1,4 +1,4 @@
 use Mix.Config
 
 config :avrora,
   schemas_path: Path.expand("./test/fixtures/schemas"),
@@ -9,4 +9,5 @@
   convert_null_values: true,
   names_cache_ttl: :infinity
 
 config :logger, :console, format: "$time $metadata[$level] $levelpad$message\n"
+config :elixir, :time_zone_database, Tz.TimeZoneDatabase

lib/avrora/avro_decoder_options.ex

@@ -4,6 +4,7 @@ defmodule Avrora.AvroDecoderOptions do
   `:avro_ocf` decoder options.
   """
 
+  alias Avrora.AvroTypeConverter
   alias Avrora.Config
 
   @options %{
@@ -13,7 +14,8 @@ defmodule Avrora.AvroDecoderOptions do
     map_type: :map,
     record_type: :map
   }
-  @null_type_name "null"
+  # TODO Rename avro_type_converter into something better
+  @type_converters [AvroTypeConverter.NullIntoNil, AvroTypeConverter.PrimitiveIntoLogical]
 
   @doc """
   A unified erlavro decoder options compatible for both binary and OCF decoders.
@@ -25,17 +27,17 @@ defmodule Avrora.AvroDecoderOptions do
   # NOTE: This is internal module function and should never be used directly
   @doc false
   def __hook__(type, sub_name_or_idx, data, decode_fun) do
-    convert = convert_null_values()
-    decoder_hook = decoder_hook()
+    result = decoder_hook().(type, sub_name_or_idx, data, decode_fun)
 
-    result = decoder_hook.(type, sub_name_or_idx, data, decode_fun)
-
-    if convert == true && :avro.get_type_name(type) == @null_type_name,
-      do: {nil, data},
-      else: result
+    @type_converters
+    |> List.foldl(result, fn type_converter, value ->
+      case type_converter.convert(value, type) do
+        {:ok, result} -> result
+        {:error, reason} -> raise(reason)
+      end
+    end)
   end
 
-  defp convert_null_values, do: Config.self().convert_null_values()
   defp convert_map_to_proplist, do: Config.self().convert_map_to_proplist()
   defp decoder_hook, do: Config.self().decoder_hook()
 end

lib/avrora/avro_logical_type_caster.ex

@@ -0,0 +1,14 @@
+defmodule Avrora.AvroLogicalTypeCaster do
+  @moduledoc """
+  TODO Write AvroLogicalTypeCaster moduledoc
+  """
+
+  @doc """
+  TODO Write convert callback doc
+
+  NOTE that type is an erlavro type
+       and we are converting erlang/avro types into Elixir
+  """
+  @callback cast(value :: term(), type :: term()) ::
+              {:ok, result :: term()} | {:error, reason :: Exception.t() | term()}
+end

lib/avrora/avro_logical_type_caster/date.ex

@@ -0,0 +1,15 @@
+defmodule Avrora.AvroLogicalTypeCaster.Date do
+  @moduledoc """
+  The `date` logical type represents a date within the calendar,
+  with no reference to a particular time zone or time of day.
+
+  The `date` logical type annotates an Avro `int`, where the `int` stores
+  the number of days from the unix epoch, 1 January 1970 (ISO calendar).
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+  @unix_epoch ~D[1970-01-01]
+
+  @impl true
+  def cast(value, _type), do: {:ok, Date.add(@unix_epoch, value)}
+end

lib/avrora/avro_logical_type_caster/decimal.ex

@@ -0,0 +1,40 @@
+defmodule Avrora.AvroLogicalTypeCaster.Decimal do
+  @moduledoc """
+  The `decimal` logical type represents an arbitrary-precision signed decimal
+  number of the form `unscaled × 10-scale`.
+
+  The `decimal` logical type annotates Avro bytes or fixed types.
+  The byte array must contain the two’s-complement representation
+  of the unscaled integer value in big-endian byte order.
+
+  NOTE: This module is NOT INCLUDED into defaults of `Avrora.Config` and must
+        be added manually, like this
+
+        config :avrora, logical_types_casting: %{
+          "decimal" => Avrora.AvroLogicalTypeCaster.Decimal
+          ...
+        }
+
+  NOTE: This module REQUIRES presence of the Decimal library, for details see
+        https://hex.pm/packages/decimal
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+  @default_scale_prop {"scale", 0}
+
+  @impl true
+  def cast(value, type) do
+    <<value::signed-integer-64-big>> = value
+
+    scale =
+      :avro.get_custom_props(type)
+      |> List.keyfind("scale", 0, @default_scale_prop)
+      |> elem(1)
+
+    {:ok, decimal(value, scale)}
+  end
+
+  defp decimal(value, 0), do: Decimal.new(value)
+  defp decimal(value, scale) when value > 0, do: Decimal.new(1, value, -scale)
+  defp decimal(value, scale) when value < 0, do: Decimal.new(-1, -value, -scale)
+end

lib/avrora/avro_logical_type_caster/local_timestamp_micros.ex

@@ -0,0 +1,21 @@
+defmodule Avrora.AvroLogicalTypeCaster.LocalTimestampMicros do
+  @moduledoc """
+  TODO Write AvroLogicalTypeCaster.LocalTimestampMicros moduledoc
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+  # @timezone "Etc/UTC"
+  @timezone "Japan"
+
+  alias Avrora.Errors
+
+  @impl true
+  def cast(value, _type) do
+    with {:ok, date_time} <- DateTime.from_unix(value, :microsecond),
+         {:ok, local_date_time} <- DateTime.shift_zone(date_time, @timezone) do
+      {:ok, local_date_time}
+    else
+      {:error, reason} -> {:error, %Errors.LogicalTypeDecodingError{code: reason}}
+    end
+  end
+end

lib/avrora/avro_logical_type_caster/local_timestamp_millis.ex

@@ -0,0 +1,21 @@
+defmodule Avrora.AvroLogicalTypeCaster.LocalTimestampMillis do
+  @moduledoc """
+  TODO Write AvroLogicalTypeCaster.LocalTimestampMillis moduledoc
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+  # @timezone "Etc/UTC"
+  @timezone "Japan"
+
+  alias Avrora.Errors
+
+  @impl true
+  def cast(value, _type) do
+    with {:ok, date_time} <- DateTime.from_unix(value, :millisecond),
+         {:ok, local_date_time} <- DateTime.shift_zone(date_time, @timezone) do
+      {:ok, local_date_time}
+    else
+      {:error, reason} -> {:error, %Errors.LogicalTypeDecodingError{code: reason}}
+    end
+  end
+end

lib/avrora/avro_logical_type_caster/noop.ex

@@ -0,0 +1,11 @@
+defmodule Avrora.AvroLogicalTypeCaster.Noop do
+  @moduledoc """
+  This is no-op module used for unsupported logical types.
+  It keeps the original value untouched and does not generate any warning.
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+
+  @impl true
+  def cast(value, _type), do: {:ok, value}
+end

lib/avrora/avro_logical_type_caster/noop_warning.ex

@@ -0,0 +1,18 @@
+defmodule Avrora.AvroLogicalTypeCaster.NoopWarning do
+  @moduledoc """
+  This is no-op module used for unsupported logical types.
+  It keeps the original value untouched, but generated the warning.
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+
+  require Logger
+
+  @impl true
+  def cast(value, type) do
+    {_, logical_type} = :avro.get_custom_props(type) |> List.keyfind("logicalType", 0)
+    Logger.warning("unsupported logical type `#{logical_type}', its value was not type casted")
+
+    {:ok, value}
+  end
+end

lib/avrora/avro_logical_type_caster/time_micros.ex

@@ -0,0 +1,22 @@
+defmodule Avrora.AvroLogicalTypeCaster.TimeMicros do
+  @moduledoc """
+  The `time-micros` logical type represents a time of day, with no reference to
+  a particular calendar, time zone or date, with a precision of one microsecond.
+
+  The `time-micros` logical type annotates an Avro `long`, where the `long`
+  stores the number of microseconds after midnight, 00:00:00.000000.
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+  @microseconds 1_000_000
+  @precision 6
+
+  @impl true
+  def cast(value, _type) do
+    time =
+      div(value, @microseconds)
+      |> Time.from_seconds_after_midnight({rem(value, @microseconds), @precision})
+
+    {:ok, time}
+  end
+end

lib/avrora/avro_logical_type_caster/time_millis.ex

@@ -0,0 +1,23 @@
+defmodule Avrora.AvroLogicalTypeCaster.TimeMillis do
+  @moduledoc """
+  The `time-millis` logical type represents a time of day, with no reference to
+  a particular calendar, time zone or date, with a precision of one millisecond.
+
+  The `time-millis` logical type annotates an Avro `int`, where the `int` stores
+  the number of milliseconds after midnight, 00:00:00.000.
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+  @milliseconds 1_000
+  @precision 3
+
+  @impl true
+  def cast(value, _type) do
+    time =
+      div(value, @milliseconds)
+      |> Time.from_seconds_after_midnight({rem(value, @milliseconds) * 1_000, @precision})
+      |> Time.truncate(:millisecond)
+
+    {:ok, time}
+  end
+end

lib/avrora/avro_logical_type_caster/timestamp_micros.ex

@@ -0,0 +1,15 @@
+defmodule Avrora.AvroLogicalTypeCaster.TimestampMicros do
+  @moduledoc """
+  TODO Write AvroLogicalTypeCaster.TimestampMicros moduledoc
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+
+  alias Avrora.Errors
+
+  @impl true
+  def cast(value, _type) do
+    with {:error, reason} <- DateTime.from_unix(value, :microsecond),
+         do: {:error, %Errors.LogicalTypeDecodingError{code: reason}}
+  end
+end

lib/avrora/avro_logical_type_caster/timestamp_millis.ex

@@ -0,0 +1,15 @@
+defmodule Avrora.AvroLogicalTypeCaster.TimestampMillis do
+  @moduledoc """
+  TODO Write AvroLogicalTypeCaster.TimestampMillis moduledoc
+  """
+
+  @behaviour Avrora.AvroLogicalTypeCaster
+
+  alias Avrora.Errors
+
+  @impl true
+  def cast(value, _type) do
+    with {:error, reason} <- DateTime.from_unix(value, :millisecond),
+         do: {:error, %Errors.LogicalTypeDecodingError{code: reason}}
+  end
+end

lib/avrora/avro_type_converter.ex

@@ -0,0 +1,14 @@
+defmodule Avrora.AvroTypeConverter do
+  @moduledoc """
+  TODO Write AvroTypeConverter moduledoc
+  """
+
+  @doc """
+  TODO Write convert callback doc
+
+  NOTE that type is an erlavro type
+       and we are converting erlang/avro types into Elixir
+  """
+  @callback convert(value :: term(), type :: term()) ::
+              {:ok, result :: {term(), binary()}} | {:error, reason :: Exception.t() | term()}
+end

lib/avrora/avro_type_converter/null_into_nil.ex

@@ -0,0 +1,21 @@
+defmodule Avrora.AvroTypeConverter.NullIntoNil do
+  @moduledoc """
+  TODO Write NullIntoNil moduledoc
+  """
+
+  @behaviour Avrora.AvroTypeConverter
+  @null_type_name "null"
+
+  alias Avrora.Config
+
+  @impl true
+  def convert(value, type) do
+    if enabled() && :avro.get_type_name(type) == @null_type_name do
+      {:ok, {nil, elem(value, 1)}}
+    else
+      {:ok, value}
+    end
+  end
+
+  defp enabled, do: Config.self().convert_null_values() == true
+end

lib/avrora/avro_type_converter/primitive_into_logical.ex

@@ -0,0 +1,48 @@
+# TODO Merge into type caster and remove list handling in decoder options
+defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do
+  @moduledoc """
+  TODO Write PrimitiveIntoLogical moduledoc
+  """
+
+  @behaviour Avrora.AvroTypeConverter
+
+  @logical_type "logicalType"
+
+  alias Avrora.AvroLogicalTypeCaster
+  alias Avrora.Config
+
+  @impl true
+  def convert(value, type) do
+    with true <- enabled(),
+         {@logical_type, logical_type} <- :avro.get_custom_props(type) |> List.keyfind(@logical_type, 0),
+         {value, rest} <- value,
+         {:ok, converted} <- do_convert(value, type, logical_type) do
+      {:ok, {converted, rest}}
+    else
+      {:error, reason} -> {:error, reason}
+      _ -> {:ok, value}
+    end
+  end
+
+  # TODO Remove and replace with config
+  # TODO Add support module to test Japan timezone in local timestamp
+  @config %{
+    "uuid" => AvroLogicalTypeCaster.Noop,
+    "date" => AvroLogicalTypeCaster.Date,
+    "decimal" => AvroLogicalTypeCaster.Decimal,
+    "time-millis" => AvroLogicalTypeCaster.TimeMillis,
+    "time-micros" => AvroLogicalTypeCaster.TimeMicros,
+    "timestamp-millis" => AvroLogicalTypeCaster.TimestampMillis,
+    "timestamp-micros" => AvroLogicalTypeCaster.TimestampMicros,
+    "local-timestamp-millis" => AvroLogicalTypeCaster.LocalTimestampMillis,
+    "local-timestamp-micros" => AvroLogicalTypeCaster.LocalTimestampMicros,
+    "_" => AvroLogicalTypeCaster.NoopWarning
+  }
+
+  # TODO Replace fetch! with fetch and raise generic error of Avrora
+  defp do_convert(value, type, logical_type) do
+    Map.get(@config, logical_type, Map.fetch!(@config, "_")).cast(value, type)
+  end
+
+  defp enabled, do: Config.self().cast_logical_types() == true
+end