From c8cbe38caa8eac44e83f262806294067d613c6e6 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Mon, 13 Nov 2023 22:26:40 +0100 Subject: [PATCH 01/31] Add config option for logical type conversion --- lib/avrora/config.ex | 5 +++++ test/avrora/codec/plain_test.exs | 18 ++++++++++++++++++ test/support/config.ex | 2 ++ 3 files changed, 25 insertions(+) diff --git a/lib/avrora/config.ex b/lib/avrora/config.ex index 8c120ed1..1314bde7 100644 --- a/lib/avrora/config.ex +++ b/lib/avrora/config.ex @@ -12,6 +12,7 @@ defmodule Avrora.Config do * `registry_schemas_autoreg` automatically register schemas in Schema Registry, default `true` * `convert_null_values` convert `:null` values in the decoded message into `nil`, default `true` * `convert_map_to_proplist` bring back old behavior and configure decoding AVRO map-type as proplist, default `false` + * `convert_logical_types` convert AVRO primitive or complex type into correcponding Elixir representation, default `true` * `names_cache_ttl` duration to cache global schema names millisecods, default `:infinity` * `decoder_hook` function to amend decoded payload, default `fn _, _, data, fun -> fun.(data) end` @@ -31,6 +32,7 @@ defmodule Avrora.Config do @callback registry_schemas_autoreg :: boolean() @callback convert_null_values :: boolean() @callback convert_map_to_proplist :: boolean() + @callback convert_logical_types :: boolean() @callback names_cache_ttl :: integer() | atom() @callback decoder_hook :: (any(), any(), any(), any() -> any()) @callback file_storage :: module() @@ -65,6 +67,9 @@ defmodule Avrora.Config do @doc false def convert_map_to_proplist, do: get_env(:convert_map_to_proplist, false) + @doc false + def convert_logical_types, do: get_env(:convert_logical_types, true) + @doc false def names_cache_ttl, do: get_env(:names_cache_ttl, :infinity) diff --git a/test/avrora/codec/plain_test.exs b/test/avrora/codec/plain_test.exs index c35549d6..b43b610d 100644 --- a/test/avrora/codec/plain_test.exs +++ b/test/avrora/codec/plain_test.exs @@ -135,6 +135,14 @@ defmodule Avrora.Codec.PlainTest do assert decoded_int == %{"union_field" => {"io.confluent.as_int", %{"value" => 42}}} assert decoded_str == %{"union_field" => {"io.confluent.as_str", %{"value" => "42"}}} end + + test "when decoding message and logical types must be as is" do + stub(Avrora.ConfigMock, :convert_logical_types, fn -> false end) + + {:ok, decoded} = Codec.Plain.decode(logical_type_message(), schema: logical_type_schema()) + + assert decoded == %{"birthday" => 17100} + end end describe "encode/2" do @@ -273,6 +281,7 @@ defmodule Avrora.Codec.PlainTest do end defp null_value_message, do: <<12, 117, 115, 101, 114, 45, 49, 0>> + defp logical_type_message, do: <<152, 139, 2>> defp map_message, do: <<1, 20, 6, 107, 101, 121, 10, 118, 97, 108, 117, 101, 0>> defp payment_payload, do: %{"id" => "00000000-0000-0000-0000-000000000000", "amount" => 15.99} @@ -306,6 +315,11 @@ defmodule Avrora.Codec.PlainTest do %{schema | id: nil, version: nil} end + defp logical_type_schema do + {:ok, schema} = Schema.Encoder.from_json(logical_type_json_schema()) + %{schema | id: nil, version: nil} + end + defp payment_json_schema do ~s({"namespace":"io.confluent","name":"Payment","type":"record","fields":[{"name":"id","type":"string"},{"name":"amount","type":"double"}]}) end @@ -329,4 +343,8 @@ defmodule Avrora.Codec.PlainTest do defp union_json_schema do ~s({"namespace":"io.confluent","name":"Union_Value","type":"record","fields":[{"name":"union_field","type":[{"type":"record","name":"as_str","fields":[{"name":"value","type":"string"}]},{"type":"record","name":"as_int","fields":[{"name":"value","type":"int"}]}]}]}) end + + defp logical_type_json_schema do + ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"birthday","type":{"type":"int","logicalType":"date"}}]}) + end end diff --git a/test/support/config.ex b/test/support/config.ex index 184431f3..92110f77 100644 --- a/test/support/config.ex +++ b/test/support/config.ex @@ -52,6 +52,8 @@ defmodule Support.Config do @impl true def convert_map_to_proplist, do: false @impl true + def convert_logical_types, do: true + @impl true def file_storage, do: Avrora.Storage.FileMock @impl true def memory_storage, do: Avrora.Storage.MemoryMock From a0df51ae26fb8f02486c07b9e6ad92488e4fe210 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 21 Nov 2023 09:41:17 +0100 Subject: [PATCH 02/31] Add Hook module with null value conversion --- lib/avrora/avro_decoder_options.ex | 19 ++++----- lib/avrora/hook.ex | 11 ++++++ lib/avrora/hook/null_value_conversion.ex | 19 +++++++++ .../hook/null_value_conversion_test.exs | 39 +++++++++++++++++++ 4 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 lib/avrora/hook.ex create mode 100644 lib/avrora/hook/null_value_conversion.ex create mode 100644 test/avrora/hook/null_value_conversion_test.exs diff --git a/lib/avrora/avro_decoder_options.ex b/lib/avrora/avro_decoder_options.ex index f141222a..3b749f56 100644 --- a/lib/avrora/avro_decoder_options.ex +++ b/lib/avrora/avro_decoder_options.ex @@ -5,6 +5,7 @@ defmodule Avrora.AvroDecoderOptions do """ alias Avrora.Config + alias Avrora.Hook @options %{ encoding: :avro_binary, @@ -13,7 +14,7 @@ defmodule Avrora.AvroDecoderOptions do map_type: :map, record_type: :map } - @null_type_name "null" + @hooks [Hook.NullValueConversion] @doc """ A unified erlavro decoder options compatible for both binary and OCF decoders. @@ -25,17 +26,17 @@ defmodule Avrora.AvroDecoderOptions do # NOTE: This is internal module function and should never be used directly @doc false def __hook__(type, sub_name_or_idx, data, decode_fun) do - convert = convert_null_values() - decoder_hook = decoder_hook() + result = decoder_hook().(type, sub_name_or_idx, data, decode_fun) - result = decoder_hook.(type, sub_name_or_idx, data, decode_fun) - - if convert == true && :avro.get_type_name(type) == @null_type_name, - do: {nil, data}, - else: result + @hooks + |> List.foldl(result, fn hook, result -> + case hook.process(result, type, sub_name_or_idx, data) do + {:ok, res} -> res + {:error, reason} -> raise(reason) + end + end) end - defp convert_null_values, do: Config.self().convert_null_values() defp convert_map_to_proplist, do: Config.self().convert_map_to_proplist() defp decoder_hook, do: Config.self().decoder_hook() end diff --git a/lib/avrora/hook.ex b/lib/avrora/hook.ex new file mode 100644 index 00000000..1b3ff41f --- /dev/null +++ b/lib/avrora/hook.ex @@ -0,0 +1,11 @@ +defmodule Avrora.Hook do + @moduledoc """ + TODO + """ + + @doc """ + TODO + """ + @callback process(value :: term(), type :: term(), sub_name_or_idx :: term(), data :: binary()) :: + {:ok, result :: {term(), binary()}} | {:error, reason :: term()} +end diff --git a/lib/avrora/hook/null_value_conversion.ex b/lib/avrora/hook/null_value_conversion.ex new file mode 100644 index 00000000..e3ceb1b0 --- /dev/null +++ b/lib/avrora/hook/null_value_conversion.ex @@ -0,0 +1,19 @@ +defmodule Avrora.Hook.NullValueConversion do + @moduledoc """ + TODO + """ + + @behaviour Avrora.Hook + @null_type_name "null" + + alias Avrora.Config + + @impl true + def process(value, type, _sub_name_or_idx, data) do + result = if convert() == true && :avro.get_type_name(type) == @null_type_name, do: {nil, data}, else: value + + {:ok, result} + end + + defp convert, do: Config.self().convert_null_values() +end diff --git a/test/avrora/hook/null_value_conversion_test.exs b/test/avrora/hook/null_value_conversion_test.exs new file mode 100644 index 00000000..fc53f9ae --- /dev/null +++ b/test/avrora/hook/null_value_conversion_test.exs @@ -0,0 +1,39 @@ +defmodule Avrora.Hook.NullValueConversionTest do + use ExUnit.Case, async: true + doctest Avrora.Hook.NullValueConversion + + import Mox + import Support.Config + + alias Avrora.{Codec, Schema} + + setup :verify_on_exit! + setup :support_config + + describe "process/4" do + test "when null values must be kept as is" do + stub(Avrora.ConfigMock, :convert_null_values, fn -> false end) + + {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) + + assert decoded == %{"key" => "user-1", "value" => :null} + end + + test "when null values must be converted" do + {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) + + assert decoded == %{"key" => "user-1", "value" => nil} + end + end + + defp message, do: <<12, 117, 115, 101, 114, 45, 49, 0>> + + defp schema do + {:ok, schema} = Schema.Encoder.from_json(json_schema()) + %{schema | id: nil, version: nil} + end + + defp json_schema do + ~s({"namespace":"io.confluent","name":"Null_Value","type":"record","fields":[{"name":"key","type":"string"},{"name":"value","type":["null","int"]}]}) + end +end From 8ca0270535c9143e113f78135748b58d469ffb06 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 21 Nov 2023 09:42:17 +0100 Subject: [PATCH 03/31] Fix mistype --- lib/avrora/codec.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/avrora/codec.ex b/lib/avrora/codec.ex index e78c9d09..4d11b9f4 100644 --- a/lib/avrora/codec.ex +++ b/lib/avrora/codec.ex @@ -45,7 +45,7 @@ defmodule Avrora.Codec do {:ok, %{"id" => "00000000-0000-0000-0000-000000000000", "amount" => 15.99}} """ - @callback decode(payloadd :: binary()) :: {:ok, result :: map() | list(map())} | {:error, reason :: term()} + @callback decode(payload :: binary()) :: {:ok, result :: map() | list(map())} | {:error, reason :: term()} @doc """ Decode a binary Avro message into the Elixir data with given schema. From ea342f361474ed859e2f0e2c08221590a16510c9 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 22 Nov 2023 11:22:36 +0100 Subject: [PATCH 04/31] Update null value conversion and its tests --- lib/avrora/hook/null_value_conversion.ex | 19 ----------------- lib/avrora/hook/null_values_conversion.ex | 21 +++++++++++++++++++ ...st.exs => null_values_conversion_test.exs} | 4 ++-- 3 files changed, 23 insertions(+), 21 deletions(-) delete mode 100644 lib/avrora/hook/null_value_conversion.ex create mode 100644 lib/avrora/hook/null_values_conversion.ex rename test/avrora/hook/{null_value_conversion_test.exs => null_values_conversion_test.exs} (91%) diff --git a/lib/avrora/hook/null_value_conversion.ex b/lib/avrora/hook/null_value_conversion.ex deleted file mode 100644 index e3ceb1b0..00000000 --- a/lib/avrora/hook/null_value_conversion.ex +++ /dev/null @@ -1,19 +0,0 @@ -defmodule Avrora.Hook.NullValueConversion do - @moduledoc """ - TODO - """ - - @behaviour Avrora.Hook - @null_type_name "null" - - alias Avrora.Config - - @impl true - def process(value, type, _sub_name_or_idx, data) do - result = if convert() == true && :avro.get_type_name(type) == @null_type_name, do: {nil, data}, else: value - - {:ok, result} - end - - defp convert, do: Config.self().convert_null_values() -end diff --git a/lib/avrora/hook/null_values_conversion.ex b/lib/avrora/hook/null_values_conversion.ex new file mode 100644 index 00000000..40db3be1 --- /dev/null +++ b/lib/avrora/hook/null_values_conversion.ex @@ -0,0 +1,21 @@ +defmodule Avrora.Hook.NullValuesConversion do + @moduledoc """ + TODO + """ + + @behaviour Avrora.Hook + @null_type_name "null" + + alias Avrora.Config + + @impl true + def process(value, type, _sub_name_or_idx, data) do + if enabled() && :avro.get_type_name(type) == @null_type_name do + {:ok, {nil, elem(value, 1)}} + else + {:ok, value} + end + end + + defp enabled, do: Config.self().convert_null_values() == true +end diff --git a/test/avrora/hook/null_value_conversion_test.exs b/test/avrora/hook/null_values_conversion_test.exs similarity index 91% rename from test/avrora/hook/null_value_conversion_test.exs rename to test/avrora/hook/null_values_conversion_test.exs index fc53f9ae..8c1b2991 100644 --- a/test/avrora/hook/null_value_conversion_test.exs +++ b/test/avrora/hook/null_values_conversion_test.exs @@ -1,6 +1,6 @@ -defmodule Avrora.Hook.NullValueConversionTest do +defmodule Avrora.Hook.NullValuesConversionTest do use ExUnit.Case, async: true - doctest Avrora.Hook.NullValueConversion + doctest Avrora.Hook.NullValuesConversion import Mox import Support.Config From 59251364768495913773b6707a46afd1305aedad Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 22 Nov 2023 11:23:14 +0100 Subject: [PATCH 05/31] Add initial implementation for Date conversion --- lib/avrora/avro_decoder_options.ex | 2 +- lib/avrora/hook/logical_types_conversion.ex | 43 +++++++++++++++++++ test/avrora/codec/plain_test.exs | 2 +- .../hook/logical_types_conversion_test.exs | 39 +++++++++++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 lib/avrora/hook/logical_types_conversion.ex create mode 100644 test/avrora/hook/logical_types_conversion_test.exs diff --git a/lib/avrora/avro_decoder_options.ex b/lib/avrora/avro_decoder_options.ex index 3b749f56..4af3eba2 100644 --- a/lib/avrora/avro_decoder_options.ex +++ b/lib/avrora/avro_decoder_options.ex @@ -14,7 +14,7 @@ defmodule Avrora.AvroDecoderOptions do map_type: :map, record_type: :map } - @hooks [Hook.NullValueConversion] + @hooks [Hook.NullValuesConversion, Hook.LogicalTypesConversion] @doc """ A unified erlavro decoder options compatible for both binary and OCF decoders. diff --git a/lib/avrora/hook/logical_types_conversion.ex b/lib/avrora/hook/logical_types_conversion.ex new file mode 100644 index 00000000..9f69b41a --- /dev/null +++ b/lib/avrora/hook/logical_types_conversion.ex @@ -0,0 +1,43 @@ +defmodule Avrora.Hook.LogicalTypesConversion do + @moduledoc """ + TODO + """ + + @behaviour Avrora.Hook + @unix_epoch ~D[1970-01-01] + @logical_type "logicalType" + + alias Avrora.Config + + @impl true + def process(value, type, _sub_name_or_idx, data) do + if enabled() do + case :avro.get_custom_props(type) |> List.keyfind(@logical_type, 0) do + {@logical_type, logical_type} -> + with {:ok, val} <- convert(elem(value, 0), logical_type), + rest <- elem(value, 1) do + {:ok, {val, rest}} + end + + _ -> + {:ok, value} + end + else + {:ok, value} + end + end + + # Supported logical types: + # 1. Date + # 2 ... + # TODO: make conversion into some logical types with dates first + defp convert(value, type) do + case type do + "Date" -> {:ok, to_date(value)} + _ -> {:error, "unknown logical type `#{type}'"} + end + end + + defp to_date(value), do: Date.add(@unix_epoch, value) + defp enabled, do: Config.self().convert_logical_types() == true +end diff --git a/test/avrora/codec/plain_test.exs b/test/avrora/codec/plain_test.exs index b43b610d..954001d7 100644 --- a/test/avrora/codec/plain_test.exs +++ b/test/avrora/codec/plain_test.exs @@ -345,6 +345,6 @@ defmodule Avrora.Codec.PlainTest do end defp logical_type_json_schema do - ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"birthday","type":{"type":"int","logicalType":"date"}}]}) + ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"birthday","type":{"type":"int","logicalType":"Date"}}]}) end end diff --git a/test/avrora/hook/logical_types_conversion_test.exs b/test/avrora/hook/logical_types_conversion_test.exs new file mode 100644 index 00000000..f9a5dac2 --- /dev/null +++ b/test/avrora/hook/logical_types_conversion_test.exs @@ -0,0 +1,39 @@ +defmodule Avrora.Hook.LogicalTypesConversionTest do + use ExUnit.Case, async: true + doctest Avrora.Hook.LogicalTypesConversion + + import Mox + import Support.Config + + alias Avrora.{Codec, Schema} + + setup :verify_on_exit! + setup :support_config + + describe "process/4" do + test "when logical types must be kept as is" do + stub(Avrora.ConfigMock, :convert_logical_types, fn -> false end) + + {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) + + assert decoded == %{"birthday" => 17100} + end + + test "when logical types must be converted" do + {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) + + assert decoded == %{"birthday" => ~D[2016-10-26]} + end + end + + defp message, do: <<152, 139, 2>> + + defp schema do + {:ok, schema} = Schema.Encoder.from_json(json_schema()) + %{schema | id: nil, version: nil} + end + + defp json_schema do + ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"birthday","type":{"type": "int","logicalType":"Date"}}]}) + end +end From 007e834bdd7101c8668d8cd685263775bfee4e2c Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 23 Nov 2023 09:46:50 +0100 Subject: [PATCH 06/31] Rewrite first conversion and add extra tests --- lib/avrora/hook/logical_types_conversion.ex | 23 ++++++++----------- .../hook/logical_types_conversion_test.exs | 23 +++++++++++++++---- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/lib/avrora/hook/logical_types_conversion.ex b/lib/avrora/hook/logical_types_conversion.ex index 9f69b41a..38e7bafc 100644 --- a/lib/avrora/hook/logical_types_conversion.ex +++ b/lib/avrora/hook/logical_types_conversion.ex @@ -10,27 +10,22 @@ defmodule Avrora.Hook.LogicalTypesConversion do alias Avrora.Config @impl true - def process(value, type, _sub_name_or_idx, data) do - if enabled() do - case :avro.get_custom_props(type) |> List.keyfind(@logical_type, 0) do - {@logical_type, logical_type} -> - with {:ok, val} <- convert(elem(value, 0), logical_type), - rest <- elem(value, 1) do - {:ok, {val, rest}} - end - - _ -> - {:ok, value} - end + def process(value, type, _sub_name_or_idx, _data) do + with true <- enabled(), + {@logical_type, logical_type} <- :avro.get_custom_props(type) |> List.keyfind(@logical_type, 0), + {value, rest} <- value, + {:ok, converted} <- convert(value, logical_type) do + {:ok, {converted, rest}} else - {:ok, value} + {:error, reason} -> {:error, reason} + _ -> {:ok, value} end end # Supported logical types: # 1. Date # 2 ... - # TODO: make conversion into some logical types with dates first + # TODO: Introduce error class and wrap this message into it! defp convert(value, type) do case type do "Date" -> {:ok, to_date(value)} diff --git a/test/avrora/hook/logical_types_conversion_test.exs b/test/avrora/hook/logical_types_conversion_test.exs index f9a5dac2..8accf843 100644 --- a/test/avrora/hook/logical_types_conversion_test.exs +++ b/test/avrora/hook/logical_types_conversion_test.exs @@ -16,24 +16,39 @@ defmodule Avrora.Hook.LogicalTypesConversionTest do {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) - assert decoded == %{"birthday" => 17100} + assert decoded == %{"birthday" => 17100, "number" => 17100} end test "when logical types must be converted" do {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) - assert decoded == %{"birthday" => ~D[2016-10-26]} + assert decoded == %{"birthday" => ~D[2016-10-26], "number" => 17100} + end + + test "when logical types must be converted, but it is unknown logical type" do + result = Codec.Plain.decode(message(), schema: malformed_schema()) + + assert result == {:error, %RuntimeError{message: "unknown logical type `Something'"}} end end - defp message, do: <<152, 139, 2>> + defp message, do: <<152, 139, 2, 152, 139, 2>> defp schema do {:ok, schema} = Schema.Encoder.from_json(json_schema()) %{schema | id: nil, version: nil} end + defp malformed_schema do + {:ok, schema} = Schema.Encoder.from_json(malformed_json_schema()) + %{schema | id: nil, version: nil} + end + defp json_schema do - ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"birthday","type":{"type": "int","logicalType":"Date"}}]}) + ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"Date"}}]}) + end + + defp malformed_json_schema do + ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"Something"}}]}) end end From f06266c160f7b838f2564dcc8102764adc0f0de2 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 23 Nov 2023 09:51:15 +0100 Subject: [PATCH 07/31] Rename decoder option --- lib/avrora/avro_decoder_options.ex | 12 +++++------ lib/avrora/avro_type_converter.ex | 13 ++++++++++++ .../null_into_nil.ex} | 6 +++--- .../primitive_into_logical.ex} | 12 +++++------ lib/avrora/client.ex | 21 +++++++++++++++---- lib/avrora/config.ex | 6 +++--- lib/avrora/hook.ex | 11 ---------- .../null_into_nil_test.exs} | 6 +++--- .../primitive_into_logical_test.exs} | 8 +++---- test/avrora/codec/plain_test.exs | 2 +- test/support/config.ex | 2 +- 11 files changed, 57 insertions(+), 42 deletions(-) create mode 100644 lib/avrora/avro_type_converter.ex rename lib/avrora/{hook/null_values_conversion.ex => avro_type_converter/null_into_nil.ex} (71%) rename lib/avrora/{hook/logical_types_conversion.ex => avro_type_converter/primitive_into_logical.ex} (71%) delete mode 100644 lib/avrora/hook.ex rename test/avrora/{hook/null_values_conversion_test.exs => avro_type_converter/null_into_nil_test.exs} (88%) rename test/avrora/{hook/logical_types_conversion_test.exs => avro_type_converter/primitive_into_logical_test.exs} (87%) diff --git a/lib/avrora/avro_decoder_options.ex b/lib/avrora/avro_decoder_options.ex index 4af3eba2..b060ca1a 100644 --- a/lib/avrora/avro_decoder_options.ex +++ b/lib/avrora/avro_decoder_options.ex @@ -4,8 +4,8 @@ defmodule Avrora.AvroDecoderOptions do `:avro_ocf` decoder options. """ + alias Avrora.AvroTypeConverter alias Avrora.Config - alias Avrora.Hook @options %{ encoding: :avro_binary, @@ -14,7 +14,7 @@ defmodule Avrora.AvroDecoderOptions do map_type: :map, record_type: :map } - @hooks [Hook.NullValuesConversion, Hook.LogicalTypesConversion] + @type_converters [AvroTypeConverter.NullIntoNil, AvroTypeConverter.PrimitiveIntoLogical] @doc """ A unified erlavro decoder options compatible for both binary and OCF decoders. @@ -28,10 +28,10 @@ defmodule Avrora.AvroDecoderOptions do def __hook__(type, sub_name_or_idx, data, decode_fun) do result = decoder_hook().(type, sub_name_or_idx, data, decode_fun) - @hooks - |> List.foldl(result, fn hook, result -> - case hook.process(result, type, sub_name_or_idx, data) do - {:ok, res} -> res + @type_converters + |> List.foldl(result, fn type_converter, value -> + case type_converter.convert(value, type) do + {:ok, result} -> result {:error, reason} -> raise(reason) end end) diff --git a/lib/avrora/avro_type_converter.ex b/lib/avrora/avro_type_converter.ex new file mode 100644 index 00000000..63b483a7 --- /dev/null +++ b/lib/avrora/avro_type_converter.ex @@ -0,0 +1,13 @@ +defmodule Avrora.AvroTypeConverter do + @moduledoc """ + TODO + """ + + @doc """ + TODO + + NOTE that type is an erlavro type + and we are converting erlang/avro types into Elixir + """ + @callback convert(value :: term(), type :: term()) :: {:ok, result :: {term(), binary()}} | {:error, reason :: term()} +end diff --git a/lib/avrora/hook/null_values_conversion.ex b/lib/avrora/avro_type_converter/null_into_nil.ex similarity index 71% rename from lib/avrora/hook/null_values_conversion.ex rename to lib/avrora/avro_type_converter/null_into_nil.ex index 40db3be1..96767fc8 100644 --- a/lib/avrora/hook/null_values_conversion.ex +++ b/lib/avrora/avro_type_converter/null_into_nil.ex @@ -1,15 +1,15 @@ -defmodule Avrora.Hook.NullValuesConversion do +defmodule Avrora.AvroTypeConverter.NullIntoNil do @moduledoc """ TODO """ - @behaviour Avrora.Hook + @behaviour Avrora.AvroTypeConverter @null_type_name "null" alias Avrora.Config @impl true - def process(value, type, _sub_name_or_idx, data) do + def convert(value, type) do if enabled() && :avro.get_type_name(type) == @null_type_name do {:ok, {nil, elem(value, 1)}} else diff --git a/lib/avrora/hook/logical_types_conversion.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex similarity index 71% rename from lib/avrora/hook/logical_types_conversion.ex rename to lib/avrora/avro_type_converter/primitive_into_logical.ex index 38e7bafc..cd9d3ce8 100644 --- a/lib/avrora/hook/logical_types_conversion.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -1,20 +1,20 @@ -defmodule Avrora.Hook.LogicalTypesConversion do +defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do @moduledoc """ TODO """ - @behaviour Avrora.Hook + @behaviour Avrora.AvroTypeConverter @unix_epoch ~D[1970-01-01] @logical_type "logicalType" alias Avrora.Config @impl true - def process(value, type, _sub_name_or_idx, _data) do + def convert(value, type) do with true <- enabled(), {@logical_type, logical_type} <- :avro.get_custom_props(type) |> List.keyfind(@logical_type, 0), {value, rest} <- value, - {:ok, converted} <- convert(value, logical_type) do + {:ok, converted} <- do_convert(value, logical_type) do {:ok, {converted, rest}} else {:error, reason} -> {:error, reason} @@ -26,7 +26,7 @@ defmodule Avrora.Hook.LogicalTypesConversion do # 1. Date # 2 ... # TODO: Introduce error class and wrap this message into it! - defp convert(value, type) do + defp do_convert(value, type) do case type do "Date" -> {:ok, to_date(value)} _ -> {:error, "unknown logical type `#{type}'"} @@ -34,5 +34,5 @@ defmodule Avrora.Hook.LogicalTypesConversion do end defp to_date(value), do: Date.add(@unix_epoch, value) - defp enabled, do: Config.self().convert_logical_types() == true + defp enabled, do: Config.self().decode_logical_types() == true end diff --git a/lib/avrora/client.ex b/lib/avrora/client.ex index 5a45f307..b807a683 100644 --- a/lib/avrora/client.ex +++ b/lib/avrora/client.ex @@ -27,15 +27,20 @@ defmodule Avrora.Client do {:ok, pid} = MyClient.start_link() """ - # NOTE: Modules below contain usage of some other modules which should be defined - # under the private client module, for instance, `Avrora.Config` could be - # defined as `MyClient.Config`. Hence they are listed together with some - # aliases. + # NOTE: Modules below contain usage of other Avrora modules which will not be + # able to resolve in private client module until we define them specificly + # for the private client. + # + # For instance, `Avrora.Config` could be defined as `MyClient.Config` + # and because `Avrora.Resolver` as `MyClient.Resolver` in private client + # is using it we list it here. @modules ~w( encoder resolver avro_schema_store avro_decoder_options + avro_type_converter/null_into_nil + avro_type_converter/primitive_into_logical schema/encoder codec/plain codec/schema_registry @@ -46,12 +51,19 @@ defmodule Avrora.Client do utils/registrar ) + # NOTE: Aliases used in Avrora modules should target correct private client + # module when generated. Because of that we list every module which was + # declared in `alias` statement. + # + # As a tradeoff, we don't use grouping in alias declarations, in other + # words you will not find constructions like `alias Avrora.{Codec, Config}` @aliases ~w( Codec Config Resolver Schema.Encoder AvroDecoderOptions + AvroTypeConverter Codec.Plain Codec.SchemaRegistry Codec.ObjectContainerFile @@ -115,6 +127,7 @@ defmodule Avrora.Client do def registry_schemas_autoreg, do: get(@opts, :registry_schemas_autoreg, true) def convert_null_values, do: get(@opts, :convert_null_values, true) def convert_map_to_proplist, do: get(@opts, :convert_map_to_proplist, false) + def decode_logical_types, do: get(@opts, :decode_logical_types, true) def names_cache_ttl, do: get(@opts, :names_cache_ttl, :infinity) def decoder_hook, do: get(@opts, :decoder_hook, fn _, _, data, fun -> fun.(data) end) def file_storage, do: unquote(:"Elixir.#{module}.Storage.File") diff --git a/lib/avrora/config.ex b/lib/avrora/config.ex index 1314bde7..0e3020d2 100644 --- a/lib/avrora/config.ex +++ b/lib/avrora/config.ex @@ -12,7 +12,7 @@ defmodule Avrora.Config do * `registry_schemas_autoreg` automatically register schemas in Schema Registry, default `true` * `convert_null_values` convert `:null` values in the decoded message into `nil`, default `true` * `convert_map_to_proplist` bring back old behavior and configure decoding AVRO map-type as proplist, default `false` - * `convert_logical_types` convert AVRO primitive or complex type into correcponding Elixir representation, default `true` + * `decode_logical_types` convert decvoded AVRO primitive or complex type into corresponding Elixir representation, default `true` * `names_cache_ttl` duration to cache global schema names millisecods, default `:infinity` * `decoder_hook` function to amend decoded payload, default `fn _, _, data, fun -> fun.(data) end` @@ -32,7 +32,7 @@ defmodule Avrora.Config do @callback registry_schemas_autoreg :: boolean() @callback convert_null_values :: boolean() @callback convert_map_to_proplist :: boolean() - @callback convert_logical_types :: boolean() + @callback decode_logical_types :: boolean() @callback names_cache_ttl :: integer() | atom() @callback decoder_hook :: (any(), any(), any(), any() -> any()) @callback file_storage :: module() @@ -68,7 +68,7 @@ defmodule Avrora.Config do def convert_map_to_proplist, do: get_env(:convert_map_to_proplist, false) @doc false - def convert_logical_types, do: get_env(:convert_logical_types, true) + def decode_logical_types, do: get_env(:decode_logical_types, true) @doc false def names_cache_ttl, do: get_env(:names_cache_ttl, :infinity) diff --git a/lib/avrora/hook.ex b/lib/avrora/hook.ex deleted file mode 100644 index 1b3ff41f..00000000 --- a/lib/avrora/hook.ex +++ /dev/null @@ -1,11 +0,0 @@ -defmodule Avrora.Hook do - @moduledoc """ - TODO - """ - - @doc """ - TODO - """ - @callback process(value :: term(), type :: term(), sub_name_or_idx :: term(), data :: binary()) :: - {:ok, result :: {term(), binary()}} | {:error, reason :: term()} -end diff --git a/test/avrora/hook/null_values_conversion_test.exs b/test/avrora/avro_type_converter/null_into_nil_test.exs similarity index 88% rename from test/avrora/hook/null_values_conversion_test.exs rename to test/avrora/avro_type_converter/null_into_nil_test.exs index 8c1b2991..09f2443c 100644 --- a/test/avrora/hook/null_values_conversion_test.exs +++ b/test/avrora/avro_type_converter/null_into_nil_test.exs @@ -1,6 +1,6 @@ -defmodule Avrora.Hook.NullValuesConversionTest do +defmodule Avrora.AvroTypeConverter.NullIntoNilTest do use ExUnit.Case, async: true - doctest Avrora.Hook.NullValuesConversion + doctest Avrora.AvroTypeConverter.NullIntoNil import Mox import Support.Config @@ -10,7 +10,7 @@ defmodule Avrora.Hook.NullValuesConversionTest do setup :verify_on_exit! setup :support_config - describe "process/4" do + describe "convert/2" do test "when null values must be kept as is" do stub(Avrora.ConfigMock, :convert_null_values, fn -> false end) diff --git a/test/avrora/hook/logical_types_conversion_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs similarity index 87% rename from test/avrora/hook/logical_types_conversion_test.exs rename to test/avrora/avro_type_converter/primitive_into_logical_test.exs index 8accf843..4ad604d5 100644 --- a/test/avrora/hook/logical_types_conversion_test.exs +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -1,6 +1,6 @@ -defmodule Avrora.Hook.LogicalTypesConversionTest do +defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do use ExUnit.Case, async: true - doctest Avrora.Hook.LogicalTypesConversion + doctest Avrora.AvroTypeConverter.PrimitiveIntoLogical import Mox import Support.Config @@ -10,9 +10,9 @@ defmodule Avrora.Hook.LogicalTypesConversionTest do setup :verify_on_exit! setup :support_config - describe "process/4" do + describe "convert/2" do test "when logical types must be kept as is" do - stub(Avrora.ConfigMock, :convert_logical_types, fn -> false end) + stub(Avrora.ConfigMock, :decode_logical_types, fn -> false end) {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) diff --git a/test/avrora/codec/plain_test.exs b/test/avrora/codec/plain_test.exs index 954001d7..18325155 100644 --- a/test/avrora/codec/plain_test.exs +++ b/test/avrora/codec/plain_test.exs @@ -137,7 +137,7 @@ defmodule Avrora.Codec.PlainTest do end test "when decoding message and logical types must be as is" do - stub(Avrora.ConfigMock, :convert_logical_types, fn -> false end) + stub(Avrora.ConfigMock, :decode_logical_types, fn -> false end) {:ok, decoded} = Codec.Plain.decode(logical_type_message(), schema: logical_type_schema()) diff --git a/test/support/config.ex b/test/support/config.ex index 92110f77..d40ba988 100644 --- a/test/support/config.ex +++ b/test/support/config.ex @@ -52,7 +52,7 @@ defmodule Support.Config do @impl true def convert_map_to_proplist, do: false @impl true - def convert_logical_types, do: true + def decode_logical_types, do: true @impl true def file_storage, do: Avrora.Storage.FileMock @impl true From a076caa649ca572747c7f800b05b0bd63d74b8c8 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Sun, 26 Nov 2023 22:10:49 +0100 Subject: [PATCH 08/31] Add more integration tests for conversions --- test/avrora/codec/plain_test.exs | 41 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/test/avrora/codec/plain_test.exs b/test/avrora/codec/plain_test.exs index 18325155..501f6c91 100644 --- a/test/avrora/codec/plain_test.exs +++ b/test/avrora/codec/plain_test.exs @@ -80,15 +80,15 @@ defmodule Avrora.Codec.PlainTest do test "when payload is a valid binary and null values must be as is" do stub(Avrora.ConfigMock, :convert_null_values, fn -> false end) - {:ok, decoded} = Codec.Plain.decode(null_value_message(), schema: null_value_schema()) + {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) - assert decoded == %{"key" => "user-1", "value" => :null} + assert decoded == %{"birthday" => ~D[2016-10-26], "guests" => :null} end test "when payload is a valid binary and null values must be converted" do - {:ok, decoded} = Codec.Plain.decode(null_value_message(), schema: null_value_schema()) + {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) - assert decoded == %{"key" => "user-1", "value" => nil} + assert decoded == %{"birthday" => ~D[2016-10-26], "guests" => nil} end test "when payload is a valid binary and map type must be decoded as proplist" do @@ -139,9 +139,18 @@ defmodule Avrora.Codec.PlainTest do test "when decoding message and logical types must be as is" do stub(Avrora.ConfigMock, :decode_logical_types, fn -> false end) - {:ok, decoded} = Codec.Plain.decode(logical_type_message(), schema: logical_type_schema()) + {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) - assert decoded == %{"birthday" => 17100} + assert decoded == %{"birthday" => 17100, "guests" => nil} + end + + test "when decoding message and all types must be as is" do + stub(Avrora.ConfigMock, :convert_null_values, fn -> false end) + stub(Avrora.ConfigMock, :decode_logical_types, fn -> false end) + + {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) + + assert decoded == %{"birthday" => 17100, "guests" => :null} end end @@ -280,8 +289,7 @@ defmodule Avrora.Codec.PlainTest do 48, 48, 48, 48, 48, 48, 48, 48, 48, 123, 20, 174, 71, 225, 250, 47, 64>> end - defp null_value_message, do: <<12, 117, 115, 101, 114, 45, 49, 0>> - defp logical_type_message, do: <<152, 139, 2>> + defp convertable_message, do: <<152, 139, 2, 0>> defp map_message, do: <<1, 20, 6, 107, 101, 121, 10, 118, 97, 108, 117, 101, 0>> defp payment_payload, do: %{"id" => "00000000-0000-0000-0000-000000000000", "amount" => 15.99} @@ -290,11 +298,6 @@ defmodule Avrora.Codec.PlainTest do %{schema | id: nil, version: nil} end - defp null_value_schema do - {:ok, schema} = Schema.Encoder.from_json(null_value_json_schema()) - %{schema | id: nil, version: nil} - end - defp map_schema do {:ok, schema} = Schema.Encoder.from_json(map_json_schema()) %{schema | id: nil, version: nil} @@ -315,8 +318,8 @@ defmodule Avrora.Codec.PlainTest do %{schema | id: nil, version: nil} end - defp logical_type_schema do - {:ok, schema} = Schema.Encoder.from_json(logical_type_json_schema()) + defp convertable_schema do + {:ok, schema} = Schema.Encoder.from_json(converterable_json_schema()) %{schema | id: nil, version: nil} end @@ -324,10 +327,6 @@ defmodule Avrora.Codec.PlainTest do ~s({"namespace":"io.confluent","name":"Payment","type":"record","fields":[{"name":"id","type":"string"},{"name":"amount","type":"double"}]}) end - defp null_value_json_schema do - ~s({"namespace":"io.confluent","name":"Null_Value","type":"record","fields":[{"name":"key","type":"string"},{"name":"value","type":["null","int"]}]}) - end - defp map_json_schema do ~s({"namespace":"io.confluent","name":"Map_Value","type":"record","fields":[{"name":"map_field", "type": {"type": "map", "values": "string"}}]}) end @@ -344,7 +343,7 @@ defmodule Avrora.Codec.PlainTest do ~s({"namespace":"io.confluent","name":"Union_Value","type":"record","fields":[{"name":"union_field","type":[{"type":"record","name":"as_str","fields":[{"name":"value","type":"string"}]},{"type":"record","name":"as_int","fields":[{"name":"value","type":"int"}]}]}]}) end - defp logical_type_json_schema do - ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"birthday","type":{"type":"int","logicalType":"Date"}}]}) + defp converterable_json_schema do + ~s({"namespace":"io.confluent","name":"Converter","type":"record","fields":[{"name":"birthday","type":{"type":"int","logicalType":"Date"}},{"name":"guests","type":["null","int"]}]}) end end From e3866a0a85b9565b1089c63cb3d988baff353720 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Sun, 26 Nov 2023 22:17:44 +0100 Subject: [PATCH 09/31] Correct spelling --- lib/avrora/config.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/avrora/config.ex b/lib/avrora/config.ex index 0e3020d2..171da093 100644 --- a/lib/avrora/config.ex +++ b/lib/avrora/config.ex @@ -12,7 +12,7 @@ defmodule Avrora.Config do * `registry_schemas_autoreg` automatically register schemas in Schema Registry, default `true` * `convert_null_values` convert `:null` values in the decoded message into `nil`, default `true` * `convert_map_to_proplist` bring back old behavior and configure decoding AVRO map-type as proplist, default `false` - * `decode_logical_types` convert decvoded AVRO primitive or complex type into corresponding Elixir representation, default `true` + * `decode_logical_types` convert decoded AVRO primitive or complex type into corresponding Elixir representation, default `true` * `names_cache_ttl` duration to cache global schema names millisecods, default `:infinity` * `decoder_hook` function to amend decoded payload, default `fn _, _, data, fun -> fun.(data) end` From 5ec9576aa6818a17aa8492477d9ad6e3b4961935 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Mon, 4 Dec 2023 15:27:08 +0100 Subject: [PATCH 10/31] Draft Decimal logical type --- .../primitive_into_logical.ex | 41 ++++++++++-- lib/avrora/utils/decimal.ex | 15 +++++ mix.exs | 1 + mix.lock | 1 + .../primitive_into_logical_test.exs | 64 +++++++++++++++++-- 5 files changed, 109 insertions(+), 13 deletions(-) create mode 100644 lib/avrora/utils/decimal.ex diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index cd9d3ce8..090d85bd 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -6,15 +6,17 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do @behaviour Avrora.AvroTypeConverter @unix_epoch ~D[1970-01-01] @logical_type "logicalType" + @default_decimal_scale_prop {"scale", 0} alias Avrora.Config + alias Avrora.Utils @impl true def convert(value, type) do with true <- enabled(), {@logical_type, logical_type} <- :avro.get_custom_props(type) |> List.keyfind(@logical_type, 0), {value, rest} <- value, - {:ok, converted} <- do_convert(value, logical_type) do + {:ok, converted} <- do_convert(value, type, logical_type) do {:ok, {converted, rest}} else {:error, reason} -> {:error, reason} @@ -23,16 +25,43 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do end # Supported logical types: + # Unsupported logical types: Decimal, Duration # 1. Date # 2 ... # TODO: Introduce error class and wrap this message into it! - defp do_convert(value, type) do - case type do - "Date" -> {:ok, to_date(value)} - _ -> {:error, "unknown logical type `#{type}'"} + # + # Fixed = value * 10^-scale + # https://hexdocs.pm/decimal/Decimal.html#new/3 = sign * coefficient * 10 ^ exponent + # + # {:avro_fixed_type, "money", "", [], 5, "io.confluent.money", + # [{"logicalType", "Decimal"}]} + # + # {:avro_primitive_type, "bytes", + # [{"precision", 3}, {"logicalType", "Decimal"}, {"scale", 2}]} + defp do_convert(value, type, logical_type) do + case logical_type do + "Date" -> + to_date(value) + + "Decimal" -> + <> = value + + scale = + :avro.get_custom_props(type) + |> List.keyfind("scale", 0, @default_decimal_scale_prop) + |> elem(1) + + Utils.Decimal.new(value, scale) + + "UUID" -> + {:ok, value} + + _ -> + # TODO: Maybe I should warn and return as is? + {:error, "unknown logical type `#{logical_type}'"} end end - defp to_date(value), do: Date.add(@unix_epoch, value) + defp to_date(value), do: {:ok, Date.add(@unix_epoch, value)} defp enabled, do: Config.self().decode_logical_types() == true end diff --git a/lib/avrora/utils/decimal.ex b/lib/avrora/utils/decimal.ex new file mode 100644 index 00000000..dcc7b732 --- /dev/null +++ b/lib/avrora/utils/decimal.ex @@ -0,0 +1,15 @@ +defmodule Avrora.Utils.Decimal do + @moduledoc """ + TODO + """ + + @doc """ + TODO + """ + if Code.ensure_loaded?(Decimal) do + def new(value, 0), do: {:ok, Decimal.new(value)} + def new(value, scale), do: {:ok, %{Decimal.new(value) | exp: -scale}} + else + def cast(_value, _scale), do: {:error, :missing_decimal_module} + end +end diff --git a/mix.exs b/mix.exs index 5700be89..af107e02 100644 --- a/mix.exs +++ b/mix.exs @@ -115,6 +115,7 @@ defmodule Avrora.MixProject do {:jason, "~> 1.0"}, {:erlavro, "~> 2.9.3"}, {:credo, "~> 1.5", only: :dev, runtime: false}, + {:decimal, "~> 2.0", only: [:dev, :test], runtime: false}, {:ex_doc, "~> 0.24", only: :dev, runtime: false}, {:dialyxir, "~> 1.1", only: :dev, runtime: false}, {:mox, "~> 1.0", only: :test}, diff --git a/mix.lock b/mix.lock index e5c64d0c..5a56d2bd 100644 --- a/mix.lock +++ b/mix.lock @@ -2,6 +2,7 @@ "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, "credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"}, + "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "dialyxir": {:hex, :dialyxir, "1.3.0", "fd1672f0922b7648ff9ce7b1b26fcf0ef56dda964a459892ad15f6b4410b5284", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "00b2a4bcd6aa8db9dcb0b38c1225b7277dca9bc370b6438715667071a304696f"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"}, diff --git a/test/avrora/avro_type_converter/primitive_into_logical_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs index 4ad604d5..aed74ae5 100644 --- a/test/avrora/avro_type_converter/primitive_into_logical_test.exs +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -26,29 +26,79 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do end test "when logical types must be converted, but it is unknown logical type" do - result = Codec.Plain.decode(message(), schema: malformed_schema()) + result = Codec.Plain.decode(message(), schema: unknown_schema()) - assert result == {:error, %RuntimeError{message: "unknown logical type `Something'"}} + assert result == {:error, %RuntimeError{message: "unknown logical type `Unknown'"}} + end + + test "when logical type is UUID" do + {:ok, decoded} = Codec.Plain.decode(uuid_message(), schema: uuid_schema()) + + assert decoded == %{"uuid" => "016c25fd-70e0-56fe-9d1a-56e80fa20b82"} + end + + test "when logical type is Decimal without scale" do + {:ok, decoded} = Codec.Plain.decode(decimal_fixed_message(), schema: decimal_without_scale_schema()) + + assert decoded == %{"decimal" => Decimal.new("123456")} + end + + test "when logical type is Decimal with scale" do + {:ok, decoded} = Codec.Plain.decode(decimal_bytes_message(), schema: decimal_with_scale_schema()) + + assert decoded == %{"decimal" => Decimal.new("-1234.56")} end end defp message, do: <<152, 139, 2, 152, 139, 2>> + defp uuid_message, do: "H016c25fd-70e0-56fe-9d1a-56e80fa20b82" + defp decimal_fixed_message, do: <<0, 0, 0, 0, 0, 1, 226, 64>> + defp decimal_bytes_message, do: <<16, 255, 255, 255, 255, 255, 254, 29, 192>> defp schema do {:ok, schema} = Schema.Encoder.from_json(json_schema()) %{schema | id: nil, version: nil} end - defp malformed_schema do - {:ok, schema} = Schema.Encoder.from_json(malformed_json_schema()) + # TODO: Replace with unknown instead + defp unknown_schema do + {:ok, schema} = Schema.Encoder.from_json(unknown_json_schema()) + %{schema | id: nil, version: nil} + end + + defp uuid_schema do + {:ok, schema} = Schema.Encoder.from_json(uuid_json_schema()) %{schema | id: nil, version: nil} end + defp decimal_without_scale_schema do + {:ok, schema} = Schema.Encoder.from_json(decimal_without_scale_json_schema()) + %{schema | id: nil, version: nil} + end + + defp decimal_with_scale_schema do + {:ok, schema} = Schema.Encoder.from_json(decimal_with_scale_json_schema()) + %{schema | id: nil, version: nil} + end + + # TODO: Rename all the things defp json_schema do - ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"Date"}}]}) + ~s({"namespace":"io.confluent","name":"Date_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"Date"}}]}) + end + + defp unknown_json_schema do + ~s({"namespace":"io.confluent","name":"Unknown_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type":"int","logicalType":"Unknown"}}]}) + end + + defp uuid_json_schema do + ~s({"namespace":"io.confluent","name":"Uuid_Type","type":"record","fields":[{"name":"uuid","type":{"type":"string","logicalType":"UUID"}}]}) + end + + defp decimal_without_scale_json_schema do + ~s({"namespace":"io.confluent","name":"Decimal_Without_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"fixed","size":8,"precision":3,"name":"money","logicalType":"Decimal"}}]}) end - defp malformed_json_schema do - ~s({"namespace":"io.confluent","name":"Logical_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"Something"}}]}) + defp decimal_with_scale_json_schema do + ~s({"namespace":"io.confluent","name":"Decimal_With_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"bytes","precision":3,"logicalType":"Decimal","scale":2}}]}) end end From f5c26d93197e158e5c97f3a1bdc6cbf8f3516c35 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Mon, 4 Dec 2023 20:03:09 +0100 Subject: [PATCH 11/31] Change behavior for unsupported logical types --- .../primitive_into_logical.ex | 6 +- .../primitive_into_logical_test.exs | 59 ++++++++++--------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index 090d85bd..835b5aa3 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -8,6 +8,7 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do @logical_type "logicalType" @default_decimal_scale_prop {"scale", 0} + require Logger alias Avrora.Config alias Avrora.Utils @@ -57,8 +58,9 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do {:ok, value} _ -> - # TODO: Maybe I should warn and return as is? - {:error, "unknown logical type `#{logical_type}'"} + Logger.warning("unsupported logical type `#{logical_type}' was not converted") + + {:ok, value} end end diff --git a/test/avrora/avro_type_converter/primitive_into_logical_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs index aed74ae5..09585f13 100644 --- a/test/avrora/avro_type_converter/primitive_into_logical_test.exs +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -4,6 +4,7 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do import Mox import Support.Config + import ExUnit.CaptureLog alias Avrora.{Codec, Schema} @@ -14,91 +15,93 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do test "when logical types must be kept as is" do stub(Avrora.ConfigMock, :decode_logical_types, fn -> false end) - {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) + {:ok, decoded} = Codec.Plain.decode(date_type_message(), schema: date_type_schema()) assert decoded == %{"birthday" => 17100, "number" => 17100} end test "when logical types must be converted" do - {:ok, decoded} = Codec.Plain.decode(message(), schema: schema()) + {:ok, decoded} = Codec.Plain.decode(date_type_message(), schema: date_type_schema()) assert decoded == %{"birthday" => ~D[2016-10-26], "number" => 17100} end test "when logical types must be converted, but it is unknown logical type" do - result = Codec.Plain.decode(message(), schema: unknown_schema()) + output = + capture_log(fn -> + decoded = %{"birthday" => 17100, "number" => 17100} + assert {:ok, decoded} == Codec.Plain.decode(date_type_message(), schema: unknown_type_schema()) + end) - assert result == {:error, %RuntimeError{message: "unknown logical type `Unknown'"}} + assert output =~ "unsupported logical type `Unknown' was not converted" end test "when logical type is UUID" do - {:ok, decoded} = Codec.Plain.decode(uuid_message(), schema: uuid_schema()) + {:ok, decoded} = Codec.Plain.decode(uuid_type_message(), schema: uuid_type_schema()) assert decoded == %{"uuid" => "016c25fd-70e0-56fe-9d1a-56e80fa20b82"} end test "when logical type is Decimal without scale" do - {:ok, decoded} = Codec.Plain.decode(decimal_fixed_message(), schema: decimal_without_scale_schema()) + {:ok, decoded} = Codec.Plain.decode(decimal_fixed_type_message(), schema: decimal_fixed_type_schema()) assert decoded == %{"decimal" => Decimal.new("123456")} end test "when logical type is Decimal with scale" do - {:ok, decoded} = Codec.Plain.decode(decimal_bytes_message(), schema: decimal_with_scale_schema()) + {:ok, decoded} = Codec.Plain.decode(decimal_bytes_type_message(), schema: decimal_bytes_type_schema()) assert decoded == %{"decimal" => Decimal.new("-1234.56")} end end - defp message, do: <<152, 139, 2, 152, 139, 2>> - defp uuid_message, do: "H016c25fd-70e0-56fe-9d1a-56e80fa20b82" - defp decimal_fixed_message, do: <<0, 0, 0, 0, 0, 1, 226, 64>> - defp decimal_bytes_message, do: <<16, 255, 255, 255, 255, 255, 254, 29, 192>> + defp date_type_message, do: <<152, 139, 2, 152, 139, 2>> + defp uuid_type_message, do: "H016c25fd-70e0-56fe-9d1a-56e80fa20b82" + defp decimal_fixed_type_message, do: <<0, 0, 0, 0, 0, 1, 226, 64>> + defp decimal_bytes_type_message, do: <<16, 255, 255, 255, 255, 255, 254, 29, 192>> - defp schema do - {:ok, schema} = Schema.Encoder.from_json(json_schema()) + defp date_type_schema do + {:ok, schema} = Schema.Encoder.from_json(date_type_json_schema()) %{schema | id: nil, version: nil} end - # TODO: Replace with unknown instead - defp unknown_schema do - {:ok, schema} = Schema.Encoder.from_json(unknown_json_schema()) + defp unknown_type_schema do + {:ok, schema} = Schema.Encoder.from_json(unknown_type_json_schema()) %{schema | id: nil, version: nil} end - defp uuid_schema do - {:ok, schema} = Schema.Encoder.from_json(uuid_json_schema()) + defp uuid_type_schema do + {:ok, schema} = Schema.Encoder.from_json(uuid_type_json_schema()) %{schema | id: nil, version: nil} end - defp decimal_without_scale_schema do - {:ok, schema} = Schema.Encoder.from_json(decimal_without_scale_json_schema()) + defp decimal_fixed_type_schema do + {:ok, schema} = Schema.Encoder.from_json(decimal_fixed_type_json_schema()) %{schema | id: nil, version: nil} end - defp decimal_with_scale_schema do - {:ok, schema} = Schema.Encoder.from_json(decimal_with_scale_json_schema()) + defp decimal_bytes_type_schema do + {:ok, schema} = Schema.Encoder.from_json(decimal_bytes_type_json_schema()) %{schema | id: nil, version: nil} end - # TODO: Rename all the things - defp json_schema do + defp date_type_json_schema do ~s({"namespace":"io.confluent","name":"Date_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"Date"}}]}) end - defp unknown_json_schema do + defp unknown_type_json_schema do ~s({"namespace":"io.confluent","name":"Unknown_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type":"int","logicalType":"Unknown"}}]}) end - defp uuid_json_schema do + defp uuid_type_json_schema do ~s({"namespace":"io.confluent","name":"Uuid_Type","type":"record","fields":[{"name":"uuid","type":{"type":"string","logicalType":"UUID"}}]}) end - defp decimal_without_scale_json_schema do + defp decimal_fixed_type_json_schema do ~s({"namespace":"io.confluent","name":"Decimal_Without_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"fixed","size":8,"precision":3,"name":"money","logicalType":"Decimal"}}]}) end - defp decimal_with_scale_json_schema do + defp decimal_bytes_type_json_schema do ~s({"namespace":"io.confluent","name":"Decimal_With_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"bytes","precision":3,"logicalType":"Decimal","scale":2}}]}) end end From f51e9278345a8abfa18093c95de8b0720a54720f Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Mon, 4 Dec 2023 20:03:28 +0100 Subject: [PATCH 12/31] Adjust vertical formatting in resolver test --- test/avrora/resolver_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/avrora/resolver_test.exs b/test/avrora/resolver_test.exs index 375e22a4..bb1fbed6 100644 --- a/test/avrora/resolver_test.exs +++ b/test/avrora/resolver_test.exs @@ -5,6 +5,7 @@ defmodule Avrora.ResolverTest do import Mox import Support.Config import ExUnit.CaptureLog + alias Avrora.{Resolver, Schema} setup :verify_on_exit! From e0e2b9472855b3a5141bd30cbabb8c6d55d99db3 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 5 Dec 2023 19:14:31 +0100 Subject: [PATCH 13/31] Finalize decimal logical type handling --- .../primitive_into_logical.ex | 28 ++++++++----------- lib/avrora/utils/decimal.ex | 15 ---------- 2 files changed, 12 insertions(+), 31 deletions(-) delete mode 100644 lib/avrora/utils/decimal.ex diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index 835b5aa3..782a24c7 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -10,7 +10,6 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do require Logger alias Avrora.Config - alias Avrora.Utils @impl true def convert(value, type) do @@ -25,20 +24,8 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do end end - # Supported logical types: - # Unsupported logical types: Decimal, Duration - # 1. Date - # 2 ... - # TODO: Introduce error class and wrap this message into it! - # - # Fixed = value * 10^-scale - # https://hexdocs.pm/decimal/Decimal.html#new/3 = sign * coefficient * 10 ^ exponent - # - # {:avro_fixed_type, "money", "", [], 5, "io.confluent.money", - # [{"logicalType", "Decimal"}]} - # - # {:avro_primitive_type, "bytes", - # [{"precision", 3}, {"logicalType", "Decimal"}, {"scale", 2}]} + # TODO: Introduce error class and wrap this message into it! + # FIXME: Refactor this shit defp do_convert(value, type, logical_type) do case logical_type do "Date" -> @@ -52,7 +39,7 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do |> List.keyfind("scale", 0, @default_decimal_scale_prop) |> elem(1) - Utils.Decimal.new(value, scale) + to_decimal(value, scale) "UUID" -> {:ok, value} @@ -65,5 +52,14 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do end defp to_date(value), do: {:ok, Date.add(@unix_epoch, value)} + + if Code.ensure_loaded?(Decimal) do + def to_decimal(value, 0), do: {:ok, Decimal.new(value)} + def to_decimal(value, scale) when is_integer(value) and value > 0, do: {:ok, Decimal.new(1, value, -scale)} + def to_decimal(value, scale) when is_integer(value) and value < 0, do: {:ok, Decimal.new(-1, -value, -scale)} + else + def to_decimal(_value, _scale), do: {:error, :missing_decimal_module} + end + defp enabled, do: Config.self().decode_logical_types() == true end diff --git a/lib/avrora/utils/decimal.ex b/lib/avrora/utils/decimal.ex deleted file mode 100644 index dcc7b732..00000000 --- a/lib/avrora/utils/decimal.ex +++ /dev/null @@ -1,15 +0,0 @@ -defmodule Avrora.Utils.Decimal do - @moduledoc """ - TODO - """ - - @doc """ - TODO - """ - if Code.ensure_loaded?(Decimal) do - def new(value, 0), do: {:ok, Decimal.new(value)} - def new(value, scale), do: {:ok, %{Decimal.new(value) | exp: -scale}} - else - def cast(_value, _scale), do: {:error, :missing_decimal_module} - end -end From 114000e556ce92c04d0b2d59635a91745e2f65b7 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 5 Dec 2023 19:15:39 +0100 Subject: [PATCH 14/31] Add integration tests for missing Decimal library * Update integration test dependencies --- test/integration/mix.exs | 2 +- .../test/decimal_logical_type_test.exs | 34 +++++++++++++++++++ test/integration/test/test_helper.exs | 2 ++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 test/integration/test/decimal_logical_type_test.exs create mode 100644 test/integration/test/test_helper.exs diff --git a/test/integration/mix.exs b/test/integration/mix.exs index 3cca9d8e..0b947e51 100644 --- a/test/integration/mix.exs +++ b/test/integration/mix.exs @@ -20,7 +20,7 @@ defmodule Integration.MixProject do defp deps do [ {:avrora, path: "../../"}, - {:dialyxir, "~> 1.0.0", runtime: false} + {:dialyxir, "~> 1.4.0", runtime: false} ] end end diff --git a/test/integration/test/decimal_logical_type_test.exs b/test/integration/test/decimal_logical_type_test.exs new file mode 100644 index 00000000..7568c4f1 --- /dev/null +++ b/test/integration/test/decimal_logical_type_test.exs @@ -0,0 +1,34 @@ +defmodule Integration.DecimalLogicalTypeTest do + use ExUnit.Case + + describe "decimal logical type" do + test "when decimal library is not installed" do + json = ~s( + { + "namespace": "io.confluent", + "name": "Decimal_Test", + "type": "record", + "fields": [ + { + "name": "decimal", + "type": { + "type": "bytes", + "precision": 3, + "logicalType": "Decimal", + "scale": 2 + } + } + ] + } + ) + + {:ok, _} = Avrora.start_link() + {:ok, schema} = Avrora.Schema.Encoder.from_json(json) + + schema = %{schema | id: nil, version: nil} + message = <<16, 255, 255, 255, 255, 255, 254, 29, 192>> + + assert {:error, :missing_decimal_module} == Avrora.Codec.Plain.decode(message, schema: schema) + end + end +end diff --git a/test/integration/test/test_helper.exs b/test/integration/test/test_helper.exs new file mode 100644 index 00000000..112dbe4c --- /dev/null +++ b/test/integration/test/test_helper.exs @@ -0,0 +1,2 @@ +Mix.shell(Mix.Shell.Process) +ExUnit.start(capture_log: true) From 89f9d48a72899329b323dd1fe2f497a9c47a8bb2 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 6 Dec 2023 11:01:41 +0100 Subject: [PATCH 15/31] Separate integration test from the rest --- mix.exs | 4 +++- test/integration/test/decimal_logical_type_test.exs | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index af107e02..6cbcc6f4 100644 --- a/mix.exs +++ b/mix.exs @@ -125,7 +125,9 @@ defmodule Avrora.MixProject do defp aliases do [ - docso: ["docs", "cmd open doc/index.html"], + test: ["test --exclude integration"], + testi: ["cmd --cd test/integration mix test --color"], + showdocs: ["docs", "cmd open doc/index.html"], check: ["cmd mix coveralls", "dialyzer", "credo"], release: [ "check", diff --git a/test/integration/test/decimal_logical_type_test.exs b/test/integration/test/decimal_logical_type_test.exs index 7568c4f1..afc16bdf 100644 --- a/test/integration/test/decimal_logical_type_test.exs +++ b/test/integration/test/decimal_logical_type_test.exs @@ -1,6 +1,7 @@ defmodule Integration.DecimalLogicalTypeTest do use ExUnit.Case + @tag :integration describe "decimal logical type" do test "when decimal library is not installed" do json = ~s( From 9bad73d4e4eaff9102a1752f4ec09dedb5e1dcd2 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 6 Dec 2023 11:02:50 +0100 Subject: [PATCH 16/31] Apply code recommendations --- test/integration/test/decimal_logical_type_test.exs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/test/decimal_logical_type_test.exs b/test/integration/test/decimal_logical_type_test.exs index afc16bdf..196e749b 100644 --- a/test/integration/test/decimal_logical_type_test.exs +++ b/test/integration/test/decimal_logical_type_test.exs @@ -1,6 +1,8 @@ defmodule Integration.DecimalLogicalTypeTest do use ExUnit.Case + alias Avrora.{Codec, Schema} + @tag :integration describe "decimal logical type" do test "when decimal library is not installed" do @@ -24,12 +26,12 @@ defmodule Integration.DecimalLogicalTypeTest do ) {:ok, _} = Avrora.start_link() - {:ok, schema} = Avrora.Schema.Encoder.from_json(json) + {:ok, schema} = Schema.Encoder.from_json(json) schema = %{schema | id: nil, version: nil} message = <<16, 255, 255, 255, 255, 255, 254, 29, 192>> - assert {:error, :missing_decimal_module} == Avrora.Codec.Plain.decode(message, schema: schema) + assert {:error, :missing_decimal_module} == Codec.Plain.decode(message, schema: schema) end end end From a56149cff93573ba465adfdd8aeadbff689a1e34 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 6 Dec 2023 13:59:57 +0100 Subject: [PATCH 17/31] Add integration test for missing decimal library --- .../primitive_into_logical.ex | 2 +- lib/avrora/errors.ex | 25 +++++++++++++++++++ mix.exs | 2 +- .../test/decimal_logical_type_test.exs | 6 ++++- 4 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 lib/avrora/errors.ex diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index 782a24c7..2c01ca66 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -58,7 +58,7 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do def to_decimal(value, scale) when is_integer(value) and value > 0, do: {:ok, Decimal.new(1, value, -scale)} def to_decimal(value, scale) when is_integer(value) and value < 0, do: {:ok, Decimal.new(-1, -value, -scale)} else - def to_decimal(_value, _scale), do: {:error, :missing_decimal_module} + def to_decimal(_value, _scale), do: {:error, %Avrora.Errors.ConfigurationError{code: :missing_decimal_lib}} end defp enabled, do: Config.self().decode_logical_types() == true diff --git a/lib/avrora/errors.ex b/lib/avrora/errors.ex new file mode 100644 index 00000000..78f171d8 --- /dev/null +++ b/lib/avrora/errors.ex @@ -0,0 +1,25 @@ +defmodule Avrora.Errors do + @moduledoc """ + TODO + """ + + defmodule ConfigurationError do + defexception [:code] + + @type t :: %__MODULE__{code: atom()} + + @messages %{ + missing_decimal_lib: "missing `Decimal' library, see https://hex.pm/packages/decimal" + } + + @impl true + def exception(code) when is_atom(code), do: %__MODULE__{code: code} + def exception(_), do: %__MODULE__{} + + @impl true + def message(%{code: code}) when is_atom(code) and code != nil, + do: "incorrect configuration, #{Map.get(@messages, code, inspect(code))}" + + def message(_), do: "incorrect configuration" + end +end diff --git a/mix.exs b/mix.exs index 6cbcc6f4..7f4bc6bb 100644 --- a/mix.exs +++ b/mix.exs @@ -125,7 +125,7 @@ defmodule Avrora.MixProject do defp aliases do [ - test: ["test --exclude integration"], + test: ["test --exclude integration --color"], testi: ["cmd --cd test/integration mix test --color"], showdocs: ["docs", "cmd open doc/index.html"], check: ["cmd mix coveralls", "dialyzer", "credo"], diff --git a/test/integration/test/decimal_logical_type_test.exs b/test/integration/test/decimal_logical_type_test.exs index 196e749b..e5cc0592 100644 --- a/test/integration/test/decimal_logical_type_test.exs +++ b/test/integration/test/decimal_logical_type_test.exs @@ -31,7 +31,11 @@ defmodule Integration.DecimalLogicalTypeTest do schema = %{schema | id: nil, version: nil} message = <<16, 255, 255, 255, 255, 255, 254, 29, 192>> - assert {:error, :missing_decimal_module} == Codec.Plain.decode(message, schema: schema) + {status, result} = Codec.Plain.decode(message, schema: schema) + + assert status == :error + assert result.code == :missing_decimal_lib + assert Exception.message(result) =~ "missing `Decimal' library" end end end From d05200d8e22b6b4aa7973606e0fae25e47c1781d Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 6 Dec 2023 20:15:53 +0100 Subject: [PATCH 18/31] Adjust mix testing commands --- .github/workflows/ci.yaml | 2 +- mix.exs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8a96addf..942a967f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -184,4 +184,4 @@ jobs: mix local.rebar --force mix local.hex --force mix deps.get - - run: mix test + - run: mix do cmd mix test, testi diff --git a/mix.exs b/mix.exs index 7f4bc6bb..8cf4d6e9 100644 --- a/mix.exs +++ b/mix.exs @@ -127,6 +127,7 @@ defmodule Avrora.MixProject do [ test: ["test --exclude integration --color"], testi: ["cmd --cd test/integration mix test --color"], + testall: ["do cmd mix test, testi"], showdocs: ["docs", "cmd open doc/index.html"], check: ["cmd mix coveralls", "dialyzer", "credo"], release: [ From fb1c9c21d92afba0ee10cb2cc937d54d38803198 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 6 Dec 2023 20:16:22 +0100 Subject: [PATCH 19/31] Change logical type values to downcase --- .../primitive_into_logical.ex | 6 +++--- .../primitive_into_logical_test.exs | 18 +++++++++--------- test/avrora/codec/plain_test.exs | 2 +- .../test/decimal_logical_type_test.exs | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index 2c01ca66..457c93aa 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -28,10 +28,10 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do # FIXME: Refactor this shit defp do_convert(value, type, logical_type) do case logical_type do - "Date" -> + "date" -> to_date(value) - "Decimal" -> + "decimal" -> <> = value scale = @@ -41,7 +41,7 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do to_decimal(value, scale) - "UUID" -> + "uuid" -> {:ok, value} _ -> diff --git a/test/avrora/avro_type_converter/primitive_into_logical_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs index 09585f13..1979791e 100644 --- a/test/avrora/avro_type_converter/primitive_into_logical_test.exs +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -33,22 +33,22 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do assert {:ok, decoded} == Codec.Plain.decode(date_type_message(), schema: unknown_type_schema()) end) - assert output =~ "unsupported logical type `Unknown' was not converted" + assert output =~ "unsupported logical type `unknown' was not converted" end - test "when logical type is UUID" do + test "when logical type is uuid" do {:ok, decoded} = Codec.Plain.decode(uuid_type_message(), schema: uuid_type_schema()) assert decoded == %{"uuid" => "016c25fd-70e0-56fe-9d1a-56e80fa20b82"} end - test "when logical type is Decimal without scale" do + test "when logical type is decimal without scale" do {:ok, decoded} = Codec.Plain.decode(decimal_fixed_type_message(), schema: decimal_fixed_type_schema()) assert decoded == %{"decimal" => Decimal.new("123456")} end - test "when logical type is Decimal with scale" do + test "when logical type is decimal with scale" do {:ok, decoded} = Codec.Plain.decode(decimal_bytes_type_message(), schema: decimal_bytes_type_schema()) assert decoded == %{"decimal" => Decimal.new("-1234.56")} @@ -86,22 +86,22 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do end defp date_type_json_schema do - ~s({"namespace":"io.confluent","name":"Date_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"Date"}}]}) + ~s({"namespace":"io.confluent","name":"Date_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"date"}}]}) end defp unknown_type_json_schema do - ~s({"namespace":"io.confluent","name":"Unknown_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type":"int","logicalType":"Unknown"}}]}) + ~s({"namespace":"io.confluent","name":"Unknown_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type":"int","logicalType":"unknown"}}]}) end defp uuid_type_json_schema do - ~s({"namespace":"io.confluent","name":"Uuid_Type","type":"record","fields":[{"name":"uuid","type":{"type":"string","logicalType":"UUID"}}]}) + ~s({"namespace":"io.confluent","name":"Uuid_Type","type":"record","fields":[{"name":"uuid","type":{"type":"string","logicalType":"uuid"}}]}) end defp decimal_fixed_type_json_schema do - ~s({"namespace":"io.confluent","name":"Decimal_Without_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"fixed","size":8,"precision":3,"name":"money","logicalType":"Decimal"}}]}) + ~s({"namespace":"io.confluent","name":"Decimal_Without_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"fixed","size":8,"precision":3,"name":"money","logicalType":"decimal"}}]}) end defp decimal_bytes_type_json_schema do - ~s({"namespace":"io.confluent","name":"Decimal_With_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"bytes","precision":3,"logicalType":"Decimal","scale":2}}]}) + ~s({"namespace":"io.confluent","name":"Decimal_With_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"bytes","precision":3,"logicalType":"decimal","scale":2}}]}) end end diff --git a/test/avrora/codec/plain_test.exs b/test/avrora/codec/plain_test.exs index 501f6c91..894d2950 100644 --- a/test/avrora/codec/plain_test.exs +++ b/test/avrora/codec/plain_test.exs @@ -344,6 +344,6 @@ defmodule Avrora.Codec.PlainTest do end defp converterable_json_schema do - ~s({"namespace":"io.confluent","name":"Converter","type":"record","fields":[{"name":"birthday","type":{"type":"int","logicalType":"Date"}},{"name":"guests","type":["null","int"]}]}) + ~s({"namespace":"io.confluent","name":"Converter","type":"record","fields":[{"name":"birthday","type":{"type":"int","logicalType":"date"}},{"name":"guests","type":["null","int"]}]}) end end diff --git a/test/integration/test/decimal_logical_type_test.exs b/test/integration/test/decimal_logical_type_test.exs index e5cc0592..36a8fdc0 100644 --- a/test/integration/test/decimal_logical_type_test.exs +++ b/test/integration/test/decimal_logical_type_test.exs @@ -17,7 +17,7 @@ defmodule Integration.DecimalLogicalTypeTest do "type": { "type": "bytes", "precision": 3, - "logicalType": "Decimal", + "logicalType": "decimal", "scale": 2 } } From d1e0d01fe8ad702618b844aa19591a176e6670c3 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 6 Dec 2023 20:40:13 +0100 Subject: [PATCH 20/31] Update dependencies --- mix.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mix.lock b/mix.lock index 5a56d2bd..a63de65f 100644 --- a/mix.lock +++ b/mix.lock @@ -1,27 +1,27 @@ %{ "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, - "credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"}, + "credo": {:hex, :credo, "1.7.1", "6e26bbcc9e22eefbff7e43188e69924e78818e2fe6282487d0703652bc20fd62", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e9871c6095a4c0381c89b6aa98bc6260a8ba6addccf7f6a53da8849c748a58a2"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, - "dialyxir": {:hex, :dialyxir, "1.3.0", "fd1672f0922b7648ff9ce7b1b26fcf0ef56dda964a459892ad15f6b4410b5284", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "00b2a4bcd6aa8db9dcb0b38c1225b7277dca9bc370b6438715667071a304696f"}, + "dialyxir": {:hex, :dialyxir, "1.4.2", "764a6e8e7a354f0ba95d58418178d486065ead1f69ad89782817c296d0d746a5", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "516603d8067b2fd585319e4b13d3674ad4f314a5902ba8130cd97dc902ce6bbd"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "erlavro": {:hex, :erlavro, "2.9.8", "9b9c0eff6dc1c708a277b4143c0020659c42bcd634d0d7237c6435fb0c2f3266", [:make, :rebar3], [{:jsone, "1.4.6", [hex: :jsone, repo: "hexpm", optional: false]}, {:snappyer, "1.2.8", [hex: :snappyer, repo: "hexpm", optional: false]}], "hexpm", "7182c539f408633927b30380aa6123ea3e4b9a04c2bc752f0fe227ef5e9c3a70"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "ex_doc": {:hex, :ex_doc, "0.30.2", "7a3e63ddb387746925bbbbcf6e9cb00e43c757cc60359a2b40059aea573e3e57", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "5ba8cb61d069012f16b50e575b0e3e6cf4083935f7444fab0d92c9314ce86bb6"}, - "excoveralls": {:hex, :excoveralls, "0.16.1", "0bd42ed05c7d2f4d180331a20113ec537be509da31fed5c8f7047ce59ee5a7c5", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "dae763468e2008cf7075a64cb1249c97cb4bc71e236c5c2b5e5cdf1cfa2bf138"}, + "ex_doc": {:hex, :ex_doc, "0.30.9", "d691453495c47434c0f2052b08dd91cc32bc4e1a218f86884563448ee2502dd2", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "d7aaaf21e95dc5cddabf89063327e96867d00013963eadf2c6ad135506a8bc10"}, + "excoveralls": {:hex, :excoveralls, "0.18.0", "b92497e69465dc51bc37a6422226ee690ab437e4c06877e836f1c18daeb35da9", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "1109bb911f3cb583401760be49c02cbbd16aed66ea9509fc5479335d284da60b"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~> 2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, "jsone": {:hex, :jsone, "1.4.6", "644d6d57befb22c8e19b324dee19d73b1c004565009861a8f64c68b7b9e64dbf", [:rebar3], [], "hexpm", "78eee8bb38f0bee2e73673d71bc75fc6fb01f56f0d23e769a26eee3655487a38"}, - "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, + "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, - "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, + "makeup_erlang": {:hex, :makeup_erlang, "0.1.3", "d684f4bac8690e70b06eb52dad65d26de2eefa44cd19d64a8095e1417df7c8fd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "b78dc853d2e670ff6390b605d807263bf606da3c82be37f9d7f68635bd886fc9"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, + "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, "snappyer": {:hex, :snappyer, "1.2.8", "201ce9067a33c71a6a5087c0c3a49a010b17112d461e6df696c722dcb6d0934a", [:rebar3], [], "hexpm", "35518e79a28548b56d8fd6aee2f565f12f51c2d3d053f9cfa817c83be88c4f3d"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, From a1f6efd3180233ed30d72bf6c377bd10c544eccb Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Wed, 6 Dec 2023 21:49:14 +0100 Subject: [PATCH 21/31] Add time-millis and time-micros logical types --- .../primitive_into_logical.ex | 38 +++++++++++++++++-- .../primitive_into_logical_test.exs | 32 ++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index 457c93aa..41fa05a4 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -7,6 +7,10 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do @unix_epoch ~D[1970-01-01] @logical_type "logicalType" @default_decimal_scale_prop {"scale", 0} + @millisecond 1_000 + @microsecond 1_000_000 + @millisecond_precision 3 + @microsecond_precision 6 require Logger alias Avrora.Config @@ -31,6 +35,12 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do "date" -> to_date(value) + "time-millis" -> + to_time_millis(value) + + "time-micros" -> + to_time_micros(value) + "decimal" -> <> = value @@ -53,12 +63,34 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do defp to_date(value), do: {:ok, Date.add(@unix_epoch, value)} + defp to_time_millis(value) do + time = + div(value, @millisecond) + |> Time.from_seconds_after_midnight({rem(value, @millisecond) * 1_000, @millisecond_precision}) + |> Time.truncate(:millisecond) + + {:ok, time} + end + + defp to_time_micros(value) do + time = + div(value, @microsecond) + |> Time.from_seconds_after_midnight({rem(value, @microsecond), @microsecond_precision}) + + {:ok, time} + end + if Code.ensure_loaded?(Decimal) do def to_decimal(value, 0), do: {:ok, Decimal.new(value)} - def to_decimal(value, scale) when is_integer(value) and value > 0, do: {:ok, Decimal.new(1, value, -scale)} - def to_decimal(value, scale) when is_integer(value) and value < 0, do: {:ok, Decimal.new(-1, -value, -scale)} + + def to_decimal(value, scale) when is_integer(value) and value > 0, + do: {:ok, Decimal.new(1, value, -scale)} + + def to_decimal(value, scale) when is_integer(value) and value < 0, + do: {:ok, Decimal.new(-1, -value, -scale)} else - def to_decimal(_value, _scale), do: {:error, %Avrora.Errors.ConfigurationError{code: :missing_decimal_lib}} + def to_decimal(_value, _scale), + do: {:error, %Avrora.Errors.ConfigurationError{code: :missing_decimal_lib}} end defp enabled, do: Config.self().decode_logical_types() == true diff --git a/test/avrora/avro_type_converter/primitive_into_logical_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs index 1979791e..5a35df76 100644 --- a/test/avrora/avro_type_converter/primitive_into_logical_test.exs +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -53,12 +53,26 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do assert decoded == %{"decimal" => Decimal.new("-1234.56")} end + + test "when logical type is time with millisecond precision" do + {:ok, decoded} = Codec.Plain.decode(time_millis_type_message(), schema: time_millis_type_schema()) + + assert decoded == %{"time" => ~T[04:28:07.123]} + end + + test "when logical type is time with microsecond precision" do + {:ok, decoded} = Codec.Plain.decode(time_micros_type_message(), schema: time_micros_type_schema()) + + assert decoded == %{"time" => ~T[04:28:07.000000]} + end end defp date_type_message, do: <<152, 139, 2, 152, 139, 2>> defp uuid_type_message, do: "H016c25fd-70e0-56fe-9d1a-56e80fa20b82" defp decimal_fixed_type_message, do: <<0, 0, 0, 0, 0, 1, 226, 64>> defp decimal_bytes_type_message, do: <<16, 255, 255, 255, 255, 255, 254, 29, 192>> + defp time_millis_type_message, do: <<166, 225, 171, 15>> + defp time_micros_type_message, do: <<128, 143, 225, 237, 119>> defp date_type_schema do {:ok, schema} = Schema.Encoder.from_json(date_type_json_schema()) @@ -85,6 +99,16 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do %{schema | id: nil, version: nil} end + defp time_millis_type_schema do + {:ok, schema} = Schema.Encoder.from_json(time_millis_type_json_schema()) + %{schema | id: nil, version: nil} + end + + defp time_micros_type_schema do + {:ok, schema} = Schema.Encoder.from_json(time_micros_type_json_schema()) + %{schema | id: nil, version: nil} + end + defp date_type_json_schema do ~s({"namespace":"io.confluent","name":"Date_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"date"}}]}) end @@ -104,4 +128,12 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do defp decimal_bytes_type_json_schema do ~s({"namespace":"io.confluent","name":"Decimal_With_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"bytes","precision":3,"logicalType":"decimal","scale":2}}]}) end + + defp time_millis_type_json_schema do + ~s({"namespace":"io.confluent","name":"Time_Millis","type":"record","fields":[{"name":"time","type":{"type":"int","logicalType":"time-millis"}}]}) + end + + defp time_micros_type_json_schema do + ~s({"namespace":"io.confluent","name":"Time_Micros","type":"record","fields":[{"name":"time","type":{"type":"long","logicalType":"time-micros"}}]}) + end end From 6f4b5fcbc5c6928cddb8a2c9b366437d3f4ddba0 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 7 Dec 2023 10:23:08 +0100 Subject: [PATCH 22/31] Add timestamp-millis and timestamp-micros --- .../primitive_into_logical.ex | 34 +++++++++++----- lib/avrora/errors.ex | 28 ++++++++++++- .../primitive_into_logical_test.exs | 40 +++++++++++++++++++ .../test/decimal_logical_type_test.exs | 7 ++-- 4 files changed, 95 insertions(+), 14 deletions(-) diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index 41fa05a4..ef9659ab 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -32,15 +32,6 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do # FIXME: Refactor this shit defp do_convert(value, type, logical_type) do case logical_type do - "date" -> - to_date(value) - - "time-millis" -> - to_time_millis(value) - - "time-micros" -> - to_time_micros(value) - "decimal" -> <> = value @@ -54,6 +45,21 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do "uuid" -> {:ok, value} + "date" -> + to_date(value) + + "time-millis" -> + to_time_millis(value) + + "time-micros" -> + to_time_micros(value) + + "timestamp-millis" -> + to_timestamp_millis(value) + + "timestamp-micros" -> + to_timestamp_micros(value) + _ -> Logger.warning("unsupported logical type `#{logical_type}' was not converted") @@ -80,6 +86,16 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do {:ok, time} end + defp to_timestamp_millis(value) do + with {:error, reason} <- DateTime.from_unix(value, :millisecond), + do: {:error, %Avrora.Errors.LogicalTypeDecodingError{code: reason}} + end + + defp to_timestamp_micros(value) do + with {:error, reason} <- DateTime.from_unix(value, :microsecond), + do: {:error, %Avrora.Errors.LogicalTypeDecodingError{code: reason}} + end + if Code.ensure_loaded?(Decimal) do def to_decimal(value, 0), do: {:ok, Decimal.new(value)} diff --git a/lib/avrora/errors.ex b/lib/avrora/errors.ex index 78f171d8..e749f536 100644 --- a/lib/avrora/errors.ex +++ b/lib/avrora/errors.ex @@ -4,10 +4,13 @@ defmodule Avrora.Errors do """ defmodule ConfigurationError do + @moduledoc """ + TODO + """ + defexception [:code] @type t :: %__MODULE__{code: atom()} - @messages %{ missing_decimal_lib: "missing `Decimal' library, see https://hex.pm/packages/decimal" } @@ -22,4 +25,27 @@ defmodule Avrora.Errors do def message(_), do: "incorrect configuration" end + + defmodule LogicalTypeDecodingError do + @moduledoc """ + TODO + """ + + defexception [:code] + + @type t :: %__MODULE__{code: atom()} + @messages %{ + invalid_unix_time: "given value is an invalid Unix time" + } + + @impl true + def exception(code) when is_atom(code), do: %__MODULE__{code: code} + def exception(_), do: %__MODULE__{} + + @impl true + def message(%{code: code}) when is_atom(code) and code != nil, + do: "logical type decoding error, #{Map.get(@messages, code, inspect(code))}" + + def message(_), do: "logical type decoding error" + end end diff --git a/test/avrora/avro_type_converter/primitive_into_logical_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs index 5a35df76..2b4e0f70 100644 --- a/test/avrora/avro_type_converter/primitive_into_logical_test.exs +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -65,6 +65,25 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do assert decoded == %{"time" => ~T[04:28:07.000000]} end + + test "when logical type is timestamp with millisecond precision" do + {:ok, decoded} = Codec.Plain.decode(timestamp_millis_type_message(), schema: timestamp_millis_type_schema()) + + assert decoded == %{"timestamp" => ~U[2016-10-26 04:28:07.123Z]} + end + + test "when logical type is timestamp with microsecond precision" do + {:ok, decoded} = Codec.Plain.decode(timestamp_micros_type_message(), schema: timestamp_micros_type_schema()) + + assert decoded == %{"timestamp" => ~U[2016-10-26 04:28:07.000000Z]} + end + + test "when logical type is timestamp and its value is incorrect" do + {:error, error} = Codec.Plain.decode(timestamp_incorrect_type_message(), schema: timestamp_millis_type_schema()) + + assert error.code == :invalid_unix_time + assert Exception.message(error) =~ "invalid Unix time" + end end defp date_type_message, do: <<152, 139, 2, 152, 139, 2>> @@ -73,6 +92,9 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do defp decimal_bytes_type_message, do: <<16, 255, 255, 255, 255, 255, 254, 29, 192>> defp time_millis_type_message, do: <<166, 225, 171, 15>> defp time_micros_type_message, do: <<128, 143, 225, 237, 119>> + defp timestamp_millis_type_message, do: <<166, 161, 246, 243, 255, 85>> + defp timestamp_micros_type_message, do: <<128, 143, 229, 211, 161, 239, 159, 5>> + defp timestamp_incorrect_type_message, do: <<128, 128, 155, 199, 153, 131, 162, 132, 7>> defp date_type_schema do {:ok, schema} = Schema.Encoder.from_json(date_type_json_schema()) @@ -109,6 +131,16 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do %{schema | id: nil, version: nil} end + defp timestamp_millis_type_schema do + {:ok, schema} = Schema.Encoder.from_json(timestamp_millis_type_json_schema()) + %{schema | id: nil, version: nil} + end + + defp timestamp_micros_type_schema do + {:ok, schema} = Schema.Encoder.from_json(timestamp_micros_type_json_schema()) + %{schema | id: nil, version: nil} + end + defp date_type_json_schema do ~s({"namespace":"io.confluent","name":"Date_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"date"}}]}) end @@ -136,4 +168,12 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do defp time_micros_type_json_schema do ~s({"namespace":"io.confluent","name":"Time_Micros","type":"record","fields":[{"name":"time","type":{"type":"long","logicalType":"time-micros"}}]}) end + + defp timestamp_millis_type_json_schema do + ~s({"namespace":"io.confluent","name":"Timestamp_Millis","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"timestamp-millis"}}]}) + end + + defp timestamp_micros_type_json_schema do + ~s({"namespace":"io.confluent","name":"Timestamp_Micros","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"timestamp-micros"}}]}) + end end diff --git a/test/integration/test/decimal_logical_type_test.exs b/test/integration/test/decimal_logical_type_test.exs index 36a8fdc0..a728d337 100644 --- a/test/integration/test/decimal_logical_type_test.exs +++ b/test/integration/test/decimal_logical_type_test.exs @@ -31,11 +31,10 @@ defmodule Integration.DecimalLogicalTypeTest do schema = %{schema | id: nil, version: nil} message = <<16, 255, 255, 255, 255, 255, 254, 29, 192>> - {status, result} = Codec.Plain.decode(message, schema: schema) + {:error, error} = Codec.Plain.decode(message, schema: schema) - assert status == :error - assert result.code == :missing_decimal_lib - assert Exception.message(result) =~ "missing `Decimal' library" + assert error.code == :missing_decimal_lib + assert Exception.message(error) =~ "missing `Decimal' library" end end end From 87af9fc3621c848099014e0c0b52d05d3d383c12 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 7 Dec 2023 12:13:53 +0100 Subject: [PATCH 23/31] Add local-timestamp-{millis,micros} logical type --- config/config.exs | 1 + .../primitive_into_logical.ex | 25 +++- lib/avrora/codec.ex | 2 +- lib/avrora/config.ex | 3 +- lib/avrora/errors.ex | 4 +- mix.exs | 1 + mix.lock | 2 + .../primitive_into_logical_test.exs | 129 ++++++++---------- 8 files changed, 92 insertions(+), 75 deletions(-) diff --git a/config/config.exs b/config/config.exs index 8e1cc952..85dbc972 100644 --- a/config/config.exs +++ b/config/config.exs @@ -10,3 +10,4 @@ config :avrora, names_cache_ttl: :infinity config :logger, :console, format: "$time $metadata[$level] $levelpad$message\n" +config :elixir, :time_zone_database, Tz.TimeZoneDatabase diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index ef9659ab..e6d57471 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -28,7 +28,6 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do end end - # TODO: Introduce error class and wrap this message into it! # FIXME: Refactor this shit defp do_convert(value, type, logical_type) do case logical_type do @@ -60,6 +59,12 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do "timestamp-micros" -> to_timestamp_micros(value) + "local-timestamp-millis" -> + to_local_timestamp_millis(value) + + "local-timestamp-micros" -> + to_local_timestamp_micros(value) + _ -> Logger.warning("unsupported logical type `#{logical_type}' was not converted") @@ -96,6 +101,24 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do do: {:error, %Avrora.Errors.LogicalTypeDecodingError{code: reason}} end + defp to_local_timestamp_millis(value) do + with {:ok, date_time} <- DateTime.from_unix(value, :millisecond), + {:ok, local_date_time} <- DateTime.shift_zone(date_time, "Japan") do + {:ok, local_date_time} + else + {:error, reason} -> {:error, %Avrora.Errors.LogicalTypeDecodingError{code: reason}} + end + end + + defp to_local_timestamp_micros(value) do + with {:ok, date_time} <- DateTime.from_unix(value, :microsecond), + {:ok, local_date_time} <- DateTime.shift_zone(date_time, "Japan") do + {:ok, local_date_time} + else + {:error, reason} -> {:error, %Avrora.Errors.LogicalTypeDecodingError{code: reason}} + end + end + if Code.ensure_loaded?(Decimal) do def to_decimal(value, 0), do: {:ok, Decimal.new(value)} diff --git a/lib/avrora/codec.ex b/lib/avrora/codec.ex index 4d11b9f4..04e57bc7 100644 --- a/lib/avrora/codec.ex +++ b/lib/avrora/codec.ex @@ -61,7 +61,7 @@ defmodule Avrora.Codec do """ @callback decode(payload :: binary(), options :: keyword(Avrora.Schema.t())) :: - {:ok, result :: map() | list(map())} | {:error, reason :: term()} + {:ok, result :: map() | list(map())} | {:error, reason :: Exception.t() | term()} @doc """ Encode the Elixir data into a binary Avro message. diff --git a/lib/avrora/config.ex b/lib/avrora/config.ex index 171da093..36f1acf5 100644 --- a/lib/avrora/config.ex +++ b/lib/avrora/config.ex @@ -12,7 +12,8 @@ defmodule Avrora.Config do * `registry_schemas_autoreg` automatically register schemas in Schema Registry, default `true` * `convert_null_values` convert `:null` values in the decoded message into `nil`, default `true` * `convert_map_to_proplist` bring back old behavior and configure decoding AVRO map-type as proplist, default `false` - * `decode_logical_types` convert decoded AVRO primitive or complex type into corresponding Elixir representation, default `true` + TODO Rename into cast_logical_types + * `decode_logical_types` convert logical AVRO primitive or complex type into corresponding Elixir representation, default `true` * `names_cache_ttl` duration to cache global schema names millisecods, default `:infinity` * `decoder_hook` function to amend decoded payload, default `fn _, _, data, fun -> fun.(data) end` diff --git a/lib/avrora/errors.ex b/lib/avrora/errors.ex index e749f536..31c8296e 100644 --- a/lib/avrora/errors.ex +++ b/lib/avrora/errors.ex @@ -35,7 +35,9 @@ defmodule Avrora.Errors do @type t :: %__MODULE__{code: atom()} @messages %{ - invalid_unix_time: "given value is an invalid Unix time" + invalid_unix_time: "given value is an invalid UNIX time", + time_zone_not_found: "configured local timezone not found in timezone database", + utc_only_time_zone_database: "default timezone database does not support configured local timezone" } @impl true diff --git a/mix.exs b/mix.exs index 8cf4d6e9..4064c7b3 100644 --- a/mix.exs +++ b/mix.exs @@ -115,6 +115,7 @@ defmodule Avrora.MixProject do {:jason, "~> 1.0"}, {:erlavro, "~> 2.9.3"}, {:credo, "~> 1.5", only: :dev, runtime: false}, + {:tz, "~> 0.26.2", only: [:dev, :test], runtime: false}, {:decimal, "~> 2.0", only: [:dev, :test], runtime: false}, {:ex_doc, "~> 0.24", only: :dev, runtime: false}, {:dialyxir, "~> 1.1", only: :dev, runtime: false}, diff --git a/mix.lock b/mix.lock index a63de65f..fab8d34e 100644 --- a/mix.lock +++ b/mix.lock @@ -25,5 +25,7 @@ "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, "snappyer": {:hex, :snappyer, "1.2.8", "201ce9067a33c71a6a5087c0c3a49a010b17112d461e6df696c722dcb6d0934a", [:rebar3], [], "hexpm", "35518e79a28548b56d8fd6aee2f565f12f51c2d3d053f9cfa817c83be88c4f3d"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, + "tz": {:hex, :tz, "0.26.2", "a40e4bb223344c6fc7b74dda25df1f26b88a30db23fa6e55de843bd79148ccdb", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:mint, "~> 1.5", [hex: :mint, repo: "hexpm", optional: true]}], "hexpm", "224b0618dd1e032778a094040bc710ef9aff6e2fa8fffc2716299486f27b9e68"}, + "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, } diff --git a/test/avrora/avro_type_converter/primitive_into_logical_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs index 2b4e0f70..e58579f4 100644 --- a/test/avrora/avro_type_converter/primitive_into_logical_test.exs +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -15,13 +15,13 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do test "when logical types must be kept as is" do stub(Avrora.ConfigMock, :decode_logical_types, fn -> false end) - {:ok, decoded} = Codec.Plain.decode(date_type_message(), schema: date_type_schema()) + {:ok, decoded} = Codec.Plain.decode(date_message(), schema: schema(date_json())) assert decoded == %{"birthday" => 17100, "number" => 17100} end test "when logical types must be converted" do - {:ok, decoded} = Codec.Plain.decode(date_type_message(), schema: date_type_schema()) + {:ok, decoded} = Codec.Plain.decode(date_message(), schema: schema(date_json())) assert decoded == %{"birthday" => ~D[2016-10-26], "number" => 17100} end @@ -30,150 +30,137 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do output = capture_log(fn -> decoded = %{"birthday" => 17100, "number" => 17100} - assert {:ok, decoded} == Codec.Plain.decode(date_type_message(), schema: unknown_type_schema()) + assert {:ok, decoded} == Codec.Plain.decode(date_message(), schema: schema(unknown_json())) end) assert output =~ "unsupported logical type `unknown' was not converted" end test "when logical type is uuid" do - {:ok, decoded} = Codec.Plain.decode(uuid_type_message(), schema: uuid_type_schema()) + {:ok, decoded} = Codec.Plain.decode(uuid_message(), schema: schema(uuid_json())) assert decoded == %{"uuid" => "016c25fd-70e0-56fe-9d1a-56e80fa20b82"} end test "when logical type is decimal without scale" do - {:ok, decoded} = Codec.Plain.decode(decimal_fixed_type_message(), schema: decimal_fixed_type_schema()) + {:ok, decoded} = Codec.Plain.decode(decimal_fixed_message(), schema: schema(decimal_fixed_json())) assert decoded == %{"decimal" => Decimal.new("123456")} end test "when logical type is decimal with scale" do - {:ok, decoded} = Codec.Plain.decode(decimal_bytes_type_message(), schema: decimal_bytes_type_schema()) + {:ok, decoded} = Codec.Plain.decode(decimal_bytes_message(), schema: schema(decimal_bytes_json())) assert decoded == %{"decimal" => Decimal.new("-1234.56")} end test "when logical type is time with millisecond precision" do - {:ok, decoded} = Codec.Plain.decode(time_millis_type_message(), schema: time_millis_type_schema()) + {:ok, decoded} = Codec.Plain.decode(time_millis_message(), schema: schema(time_millis_json())) assert decoded == %{"time" => ~T[04:28:07.123]} end test "when logical type is time with microsecond precision" do - {:ok, decoded} = Codec.Plain.decode(time_micros_type_message(), schema: time_micros_type_schema()) + {:ok, decoded} = Codec.Plain.decode(time_micros_message(), schema: schema(time_micros_json())) assert decoded == %{"time" => ~T[04:28:07.000000]} end test "when logical type is timestamp with millisecond precision" do - {:ok, decoded} = Codec.Plain.decode(timestamp_millis_type_message(), schema: timestamp_millis_type_schema()) + {:ok, decoded} = Codec.Plain.decode(timestamp_millis_message(), schema: schema(timestamp_millis_json())) assert decoded == %{"timestamp" => ~U[2016-10-26 04:28:07.123Z]} end test "when logical type is timestamp with microsecond precision" do - {:ok, decoded} = Codec.Plain.decode(timestamp_micros_type_message(), schema: timestamp_micros_type_schema()) + {:ok, decoded} = Codec.Plain.decode(timestamp_micros_message(), schema: schema(timestamp_micros_json())) assert decoded == %{"timestamp" => ~U[2016-10-26 04:28:07.000000Z]} end test "when logical type is timestamp and its value is incorrect" do - {:error, error} = Codec.Plain.decode(timestamp_incorrect_type_message(), schema: timestamp_millis_type_schema()) + {:error, error} = Codec.Plain.decode(timestamp_invalid_message(), schema: schema(timestamp_millis_json())) assert error.code == :invalid_unix_time - assert Exception.message(error) =~ "invalid Unix time" + assert Exception.message(error) =~ "invalid UNIX time" end - end - defp date_type_message, do: <<152, 139, 2, 152, 139, 2>> - defp uuid_type_message, do: "H016c25fd-70e0-56fe-9d1a-56e80fa20b82" - defp decimal_fixed_type_message, do: <<0, 0, 0, 0, 0, 1, 226, 64>> - defp decimal_bytes_type_message, do: <<16, 255, 255, 255, 255, 255, 254, 29, 192>> - defp time_millis_type_message, do: <<166, 225, 171, 15>> - defp time_micros_type_message, do: <<128, 143, 225, 237, 119>> - defp timestamp_millis_type_message, do: <<166, 161, 246, 243, 255, 85>> - defp timestamp_micros_type_message, do: <<128, 143, 229, 211, 161, 239, 159, 5>> - defp timestamp_incorrect_type_message, do: <<128, 128, 155, 199, 153, 131, 162, 132, 7>> - - defp date_type_schema do - {:ok, schema} = Schema.Encoder.from_json(date_type_json_schema()) - %{schema | id: nil, version: nil} - end + test "when logical type is local timestamp with millisecond precision" do + {:ok, decoded} = Codec.Plain.decode(timestamp_millis_message(), schema: schema(local_timestamp_millis_json())) - defp unknown_type_schema do - {:ok, schema} = Schema.Encoder.from_json(unknown_type_json_schema()) - %{schema | id: nil, version: nil} - end - - defp uuid_type_schema do - {:ok, schema} = Schema.Encoder.from_json(uuid_type_json_schema()) - %{schema | id: nil, version: nil} - end + assert DateTime.to_string(decoded["timestamp"]) == "2016-10-26 13:28:07.123+09:00 JST Japan" + end - defp decimal_fixed_type_schema do - {:ok, schema} = Schema.Encoder.from_json(decimal_fixed_type_json_schema()) - %{schema | id: nil, version: nil} - end + test "when logical type is local timestamp with microsecond precision" do + {:ok, decoded} = Codec.Plain.decode(timestamp_micros_message(), schema: schema(local_timestamp_micros_json())) - defp decimal_bytes_type_schema do - {:ok, schema} = Schema.Encoder.from_json(decimal_bytes_type_json_schema()) - %{schema | id: nil, version: nil} - end + assert DateTime.to_string(decoded["timestamp"]) == "2016-10-26 13:28:07.000000+09:00 JST Japan" + end - defp time_millis_type_schema do - {:ok, schema} = Schema.Encoder.from_json(time_millis_type_json_schema()) - %{schema | id: nil, version: nil} - end + test "when logical type is local timestamp and its value is incorrect" do + {:error, error} = Codec.Plain.decode(timestamp_invalid_message(), schema: schema(local_timestamp_millis_json())) - defp time_micros_type_schema do - {:ok, schema} = Schema.Encoder.from_json(time_micros_type_json_schema()) - %{schema | id: nil, version: nil} + assert error.code == :invalid_unix_time + assert Exception.message(error) =~ "invalid UNIX time" + end end - defp timestamp_millis_type_schema do - {:ok, schema} = Schema.Encoder.from_json(timestamp_millis_type_json_schema()) - %{schema | id: nil, version: nil} - end + defp date_message, do: <<152, 139, 2, 152, 139, 2>> + defp uuid_message, do: "H016c25fd-70e0-56fe-9d1a-56e80fa20b82" + defp decimal_fixed_message, do: <<0, 0, 0, 0, 0, 1, 226, 64>> + defp decimal_bytes_message, do: <<16, 255, 255, 255, 255, 255, 254, 29, 192>> + defp time_millis_message, do: <<166, 225, 171, 15>> + defp time_micros_message, do: <<128, 143, 225, 237, 119>> + defp timestamp_millis_message, do: <<166, 161, 246, 243, 255, 85>> + defp timestamp_micros_message, do: <<128, 143, 229, 211, 161, 239, 159, 5>> + defp timestamp_invalid_message, do: <<128, 128, 155, 199, 153, 131, 162, 132, 7>> - defp timestamp_micros_type_schema do - {:ok, schema} = Schema.Encoder.from_json(timestamp_micros_type_json_schema()) + defp schema(json) do + {:ok, schema} = Schema.Encoder.from_json(json) %{schema | id: nil, version: nil} end - defp date_type_json_schema do - ~s({"namespace":"io.confluent","name":"Date_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"date"}}]}) + defp date_json do + ~s({"namespace":"io.confluent","name":"Date","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type": "int","logicalType":"date"}}]}) end - defp unknown_type_json_schema do - ~s({"namespace":"io.confluent","name":"Unknown_Type","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type":"int","logicalType":"unknown"}}]}) + defp unknown_json do + ~s({"namespace":"io.confluent","name":"Unknown","type":"record","fields":[{"name":"number","type":"int"},{"name":"birthday","type":{"type":"int","logicalType":"unknown"}}]}) end - defp uuid_type_json_schema do - ~s({"namespace":"io.confluent","name":"Uuid_Type","type":"record","fields":[{"name":"uuid","type":{"type":"string","logicalType":"uuid"}}]}) + defp uuid_json do + ~s({"namespace":"io.confluent","name":"Uuid","type":"record","fields":[{"name":"uuid","type":{"type":"string","logicalType":"uuid"}}]}) end - defp decimal_fixed_type_json_schema do - ~s({"namespace":"io.confluent","name":"Decimal_Without_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"fixed","size":8,"precision":3,"name":"money","logicalType":"decimal"}}]}) + defp decimal_fixed_json do + ~s({"namespace":"io.confluent","name":"Decimal_Without_Scale","type":"record","fields":[{"name":"decimal","type":{"type":"fixed","size":8,"precision":3,"name":"money","logicalType":"decimal"}}]}) end - defp decimal_bytes_type_json_schema do - ~s({"namespace":"io.confluent","name":"Decimal_With_Scale_Type","type":"record","fields":[{"name":"decimal","type":{"type":"bytes","precision":3,"logicalType":"decimal","scale":2}}]}) + defp decimal_bytes_json do + ~s({"namespace":"io.confluent","name":"Decimal_With_Scale","type":"record","fields":[{"name":"decimal","type":{"type":"bytes","precision":3,"logicalType":"decimal","scale":2}}]}) end - defp time_millis_type_json_schema do + defp time_millis_json do ~s({"namespace":"io.confluent","name":"Time_Millis","type":"record","fields":[{"name":"time","type":{"type":"int","logicalType":"time-millis"}}]}) end - defp time_micros_type_json_schema do + defp time_micros_json do ~s({"namespace":"io.confluent","name":"Time_Micros","type":"record","fields":[{"name":"time","type":{"type":"long","logicalType":"time-micros"}}]}) end - defp timestamp_millis_type_json_schema do + defp timestamp_millis_json do ~s({"namespace":"io.confluent","name":"Timestamp_Millis","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"timestamp-millis"}}]}) end - defp timestamp_micros_type_json_schema do + defp timestamp_micros_json do ~s({"namespace":"io.confluent","name":"Timestamp_Micros","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"timestamp-micros"}}]}) end + + defp local_timestamp_millis_json do + ~s({"namespace":"io.confluent","name":"Local_Timestamp_Millis","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"local-timestamp-millis"}}]}) + end + + defp local_timestamp_micros_json do + ~s({"namespace":"io.confluent","name":"Local_Timestamp_Micros","type":"record","fields":[{"name":"timestamp","type":{"type":"long","logicalType":"local-timestamp-micros"}}]}) + end end From 281019abb7d54e70649b17c4544f960180a32e7b Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 7 Dec 2023 12:19:07 +0100 Subject: [PATCH 24/31] Add todo for future work --- lib/avrora/avro_type_converter.ex | 4 ++-- lib/avrora/avro_type_converter/null_into_nil.ex | 2 +- lib/avrora/avro_type_converter/primitive_into_logical.ex | 2 +- lib/avrora/config.ex | 1 + lib/avrora/errors.ex | 6 +++--- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/avrora/avro_type_converter.ex b/lib/avrora/avro_type_converter.ex index 63b483a7..5a53f3a5 100644 --- a/lib/avrora/avro_type_converter.ex +++ b/lib/avrora/avro_type_converter.ex @@ -1,10 +1,10 @@ defmodule Avrora.AvroTypeConverter do @moduledoc """ - TODO + TODO Write AvroTypeConverter moduledoc """ @doc """ - TODO + TODO Write convert callback doc NOTE that type is an erlavro type and we are converting erlang/avro types into Elixir diff --git a/lib/avrora/avro_type_converter/null_into_nil.ex b/lib/avrora/avro_type_converter/null_into_nil.ex index 96767fc8..b94be189 100644 --- a/lib/avrora/avro_type_converter/null_into_nil.ex +++ b/lib/avrora/avro_type_converter/null_into_nil.ex @@ -1,6 +1,6 @@ defmodule Avrora.AvroTypeConverter.NullIntoNil do @moduledoc """ - TODO + TODO Write NullIntoNil moduledoc """ @behaviour Avrora.AvroTypeConverter diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index e6d57471..bca9af70 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -1,6 +1,6 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do @moduledoc """ - TODO + TODO Write PrimitiveIntoLogical moduledoc """ @behaviour Avrora.AvroTypeConverter diff --git a/lib/avrora/config.ex b/lib/avrora/config.ex index 36f1acf5..4da775e5 100644 --- a/lib/avrora/config.ex +++ b/lib/avrora/config.ex @@ -13,6 +13,7 @@ defmodule Avrora.Config do * `convert_null_values` convert `:null` values in the decoded message into `nil`, default `true` * `convert_map_to_proplist` bring back old behavior and configure decoding AVRO map-type as proplist, default `false` TODO Rename into cast_logical_types + TODO Introduce configurable list of logical type casting * `decode_logical_types` convert logical AVRO primitive or complex type into corresponding Elixir representation, default `true` * `names_cache_ttl` duration to cache global schema names millisecods, default `:infinity` * `decoder_hook` function to amend decoded payload, default `fn _, _, data, fun -> fun.(data) end` diff --git a/lib/avrora/errors.ex b/lib/avrora/errors.ex index 31c8296e..5b37f7e8 100644 --- a/lib/avrora/errors.ex +++ b/lib/avrora/errors.ex @@ -1,11 +1,11 @@ defmodule Avrora.Errors do @moduledoc """ - TODO + TODO Write Errors moduledoc """ defmodule ConfigurationError do @moduledoc """ - TODO + TODO Write ConfigurationError moduledoc """ defexception [:code] @@ -28,7 +28,7 @@ defmodule Avrora.Errors do defmodule LogicalTypeDecodingError do @moduledoc """ - TODO + TODO Write LogicalTypeDecodingError moduledoc """ defexception [:code] From bb232202724a68e29a3993e3edeccfaf4f385969 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 7 Dec 2023 13:32:02 +0100 Subject: [PATCH 25/31] Rename logical type configuration option --- lib/avrora/avro_type_converter/primitive_into_logical.ex | 2 +- lib/avrora/client.ex | 2 +- lib/avrora/config.ex | 6 +++--- .../avro_type_converter/primitive_into_logical_test.exs | 2 +- test/avrora/codec/plain_test.exs | 4 ++-- test/support/config.ex | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index bca9af70..dd8055d3 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -132,5 +132,5 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do do: {:error, %Avrora.Errors.ConfigurationError{code: :missing_decimal_lib}} end - defp enabled, do: Config.self().decode_logical_types() == true + defp enabled, do: Config.self().cast_logical_types() == true end diff --git a/lib/avrora/client.ex b/lib/avrora/client.ex index b807a683..89431c64 100644 --- a/lib/avrora/client.ex +++ b/lib/avrora/client.ex @@ -127,7 +127,7 @@ defmodule Avrora.Client do def registry_schemas_autoreg, do: get(@opts, :registry_schemas_autoreg, true) def convert_null_values, do: get(@opts, :convert_null_values, true) def convert_map_to_proplist, do: get(@opts, :convert_map_to_proplist, false) - def decode_logical_types, do: get(@opts, :decode_logical_types, true) + def cast_logical_types, do: get(@opts, :cast_logical_types, true) def names_cache_ttl, do: get(@opts, :names_cache_ttl, :infinity) def decoder_hook, do: get(@opts, :decoder_hook, fn _, _, data, fun -> fun.(data) end) def file_storage, do: unquote(:"Elixir.#{module}.Storage.File") diff --git a/lib/avrora/config.ex b/lib/avrora/config.ex index 4da775e5..071b74b1 100644 --- a/lib/avrora/config.ex +++ b/lib/avrora/config.ex @@ -14,7 +14,7 @@ defmodule Avrora.Config do * `convert_map_to_proplist` bring back old behavior and configure decoding AVRO map-type as proplist, default `false` TODO Rename into cast_logical_types TODO Introduce configurable list of logical type casting - * `decode_logical_types` convert logical AVRO primitive or complex type into corresponding Elixir representation, default `true` + * `cast_logical_types` convert logical AVRO primitive or complex type into corresponding Elixir representation, default `true` * `names_cache_ttl` duration to cache global schema names millisecods, default `:infinity` * `decoder_hook` function to amend decoded payload, default `fn _, _, data, fun -> fun.(data) end` @@ -34,7 +34,7 @@ defmodule Avrora.Config do @callback registry_schemas_autoreg :: boolean() @callback convert_null_values :: boolean() @callback convert_map_to_proplist :: boolean() - @callback decode_logical_types :: boolean() + @callback cast_logical_types :: boolean() @callback names_cache_ttl :: integer() | atom() @callback decoder_hook :: (any(), any(), any(), any() -> any()) @callback file_storage :: module() @@ -70,7 +70,7 @@ defmodule Avrora.Config do def convert_map_to_proplist, do: get_env(:convert_map_to_proplist, false) @doc false - def decode_logical_types, do: get_env(:decode_logical_types, true) + def cast_logical_types, do: get_env(:cast_logical_types, true) @doc false def names_cache_ttl, do: get_env(:names_cache_ttl, :infinity) diff --git a/test/avrora/avro_type_converter/primitive_into_logical_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs index e58579f4..3c17ab96 100644 --- a/test/avrora/avro_type_converter/primitive_into_logical_test.exs +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -13,7 +13,7 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do describe "convert/2" do test "when logical types must be kept as is" do - stub(Avrora.ConfigMock, :decode_logical_types, fn -> false end) + stub(Avrora.ConfigMock, :cast_logical_types, fn -> false end) {:ok, decoded} = Codec.Plain.decode(date_message(), schema: schema(date_json())) diff --git a/test/avrora/codec/plain_test.exs b/test/avrora/codec/plain_test.exs index 894d2950..1199b4dc 100644 --- a/test/avrora/codec/plain_test.exs +++ b/test/avrora/codec/plain_test.exs @@ -137,7 +137,7 @@ defmodule Avrora.Codec.PlainTest do end test "when decoding message and logical types must be as is" do - stub(Avrora.ConfigMock, :decode_logical_types, fn -> false end) + stub(Avrora.ConfigMock, :cast_logical_types, fn -> false end) {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) @@ -146,7 +146,7 @@ defmodule Avrora.Codec.PlainTest do test "when decoding message and all types must be as is" do stub(Avrora.ConfigMock, :convert_null_values, fn -> false end) - stub(Avrora.ConfigMock, :decode_logical_types, fn -> false end) + stub(Avrora.ConfigMock, :cast_logical_types, fn -> false end) {:ok, decoded} = Codec.Plain.decode(convertable_message(), schema: convertable_schema()) diff --git a/test/support/config.ex b/test/support/config.ex index d40ba988..8f9d3815 100644 --- a/test/support/config.ex +++ b/test/support/config.ex @@ -52,7 +52,7 @@ defmodule Support.Config do @impl true def convert_map_to_proplist, do: false @impl true - def decode_logical_types, do: true + def cast_logical_types, do: true @impl true def file_storage, do: Avrora.Storage.FileMock @impl true From 7161297413be9e3842c36a931a295cb7e7068ec8 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 7 Dec 2023 15:04:14 +0100 Subject: [PATCH 26/31] Add type caster and its types --- lib/avrora/avro_decoder_options.ex | 1 + lib/avrora/avro_logical_type_caster.ex | 14 ++++++ lib/avrora/avro_logical_type_caster/date.ex | 11 +++++ .../avro_logical_type_caster/decimal.ex | 24 +++++++++ lib/avrora/avro_logical_type_caster/noop.ex | 10 ++++ .../avro_logical_type_caster/noop_warning.ex | 17 +++++++ lib/avrora/avro_type_converter.ex | 3 +- .../primitive_into_logical.ex | 49 +++++++------------ lib/avrora/errors.ex | 24 +-------- .../primitive_into_logical_test.exs | 2 +- .../test/decimal_logical_type_test.exs | 3 +- 11 files changed, 100 insertions(+), 58 deletions(-) create mode 100644 lib/avrora/avro_logical_type_caster.ex create mode 100644 lib/avrora/avro_logical_type_caster/date.ex create mode 100644 lib/avrora/avro_logical_type_caster/decimal.ex create mode 100644 lib/avrora/avro_logical_type_caster/noop.ex create mode 100644 lib/avrora/avro_logical_type_caster/noop_warning.ex diff --git a/lib/avrora/avro_decoder_options.ex b/lib/avrora/avro_decoder_options.ex index b060ca1a..607abe0f 100644 --- a/lib/avrora/avro_decoder_options.ex +++ b/lib/avrora/avro_decoder_options.ex @@ -14,6 +14,7 @@ defmodule Avrora.AvroDecoderOptions do map_type: :map, record_type: :map } + # TODO Rename avro_type_converter into something better @type_converters [AvroTypeConverter.NullIntoNil, AvroTypeConverter.PrimitiveIntoLogical] @doc """ diff --git a/lib/avrora/avro_logical_type_caster.ex b/lib/avrora/avro_logical_type_caster.ex new file mode 100644 index 00000000..ec633c02 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster.ex @@ -0,0 +1,14 @@ +defmodule Avrora.AvroLogicalTypeCaster do + @moduledoc """ + TODO Write AvroLogicalTypeCaster moduledoc + """ + + @doc """ + TODO Write convert callback doc + + NOTE that type is an erlavro type + and we are converting erlang/avro types into Elixir + """ + @callback cast(value :: term(), type :: term()) :: + {:ok, result :: term()} | {:error, reason :: Exception.t() | term()} +end diff --git a/lib/avrora/avro_logical_type_caster/date.ex b/lib/avrora/avro_logical_type_caster/date.ex new file mode 100644 index 00000000..19455e75 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/date.ex @@ -0,0 +1,11 @@ +defmodule Avrora.AvroLogicalTypeCaster.Date do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.Date moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + @unix_epoch ~D[1970-01-01] + + @impl true + def cast(value, _type), do: {:ok, Date.add(@unix_epoch, value)} +end diff --git a/lib/avrora/avro_logical_type_caster/decimal.ex b/lib/avrora/avro_logical_type_caster/decimal.ex new file mode 100644 index 00000000..700c494a --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/decimal.ex @@ -0,0 +1,24 @@ +defmodule Avrora.AvroLogicalTypeCaster.Decimal do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.Decimal moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + @default_scale_prop {"scale", 0} + + @impl true + def cast(value, type) do + <> = value + + scale = + :avro.get_custom_props(type) + |> List.keyfind("scale", 0, @default_scale_prop) + |> elem(1) + + {:ok, decimal(value, scale)} + end + + defp decimal(value, 0), do: Decimal.new(value) + defp decimal(value, scale) when value > 0, do: Decimal.new(1, value, -scale) + defp decimal(value, scale) when value < 0, do: Decimal.new(-1, -value, -scale) +end diff --git a/lib/avrora/avro_logical_type_caster/noop.ex b/lib/avrora/avro_logical_type_caster/noop.ex new file mode 100644 index 00000000..4bd04f43 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/noop.ex @@ -0,0 +1,10 @@ +defmodule Avrora.AvroLogicalTypeCaster.Noop do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.Noop moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + + @impl true + def cast(value, _type), do: {:ok, value} +end diff --git a/lib/avrora/avro_logical_type_caster/noop_warning.ex b/lib/avrora/avro_logical_type_caster/noop_warning.ex new file mode 100644 index 00000000..083ea185 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/noop_warning.ex @@ -0,0 +1,17 @@ +defmodule Avrora.AvroLogicalTypeCaster.NoopWarning do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.NoopWarning moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + + require Logger + + @impl true + def cast(value, type) do + {_, logical_type} = :avro.get_custom_props(type) |> List.keyfind("logicalType", 0) + Logger.warning("unsupported logical type `#{logical_type}', its value was not type casted") + + {:ok, value} + end +end diff --git a/lib/avrora/avro_type_converter.ex b/lib/avrora/avro_type_converter.ex index 5a53f3a5..bef3e45f 100644 --- a/lib/avrora/avro_type_converter.ex +++ b/lib/avrora/avro_type_converter.ex @@ -9,5 +9,6 @@ defmodule Avrora.AvroTypeConverter do NOTE that type is an erlavro type and we are converting erlang/avro types into Elixir """ - @callback convert(value :: term(), type :: term()) :: {:ok, result :: {term(), binary()}} | {:error, reason :: term()} + @callback convert(value :: term(), type :: term()) :: + {:ok, result :: {term(), binary()}} | {:error, reason :: Exception.t() | term()} end diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index dd8055d3..e3c01bef 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -1,18 +1,18 @@ +# TODO Merge into type caster and remove list handling in decoder options defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do @moduledoc """ TODO Write PrimitiveIntoLogical moduledoc """ @behaviour Avrora.AvroTypeConverter - @unix_epoch ~D[1970-01-01] + @logical_type "logicalType" - @default_decimal_scale_prop {"scale", 0} @millisecond 1_000 @microsecond 1_000_000 @millisecond_precision 3 @microsecond_precision 6 - require Logger + alias Avrora.AvroLogicalTypeCaster alias Avrora.Config @impl true @@ -28,24 +28,28 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do end end + @config %{ + "decimal" => AvroLogicalTypeCaster.Decimal, + "uuid" => AvroLogicalTypeCaster.Noop, + "date" => AvroLogicalTypeCaster.Date, + "_" => AvroLogicalTypeCaster.NoopWarning + } + + defp do_convert2(value, type, logical_type) do + Map.get(@config, logical_type, Map.fetch!(@config, "_")).cast(value, type) + end + # FIXME: Refactor this shit defp do_convert(value, type, logical_type) do case logical_type do "decimal" -> - <> = value - - scale = - :avro.get_custom_props(type) - |> List.keyfind("scale", 0, @default_decimal_scale_prop) - |> elem(1) - - to_decimal(value, scale) + do_convert2(value, type, logical_type) "uuid" -> - {:ok, value} + do_convert2(value, type, logical_type) "date" -> - to_date(value) + do_convert2(value, type, logical_type) "time-millis" -> to_time_millis(value) @@ -66,14 +70,10 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do to_local_timestamp_micros(value) _ -> - Logger.warning("unsupported logical type `#{logical_type}' was not converted") - - {:ok, value} + do_convert2(value, type, logical_type) end end - defp to_date(value), do: {:ok, Date.add(@unix_epoch, value)} - defp to_time_millis(value) do time = div(value, @millisecond) @@ -119,18 +119,5 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do end end - if Code.ensure_loaded?(Decimal) do - def to_decimal(value, 0), do: {:ok, Decimal.new(value)} - - def to_decimal(value, scale) when is_integer(value) and value > 0, - do: {:ok, Decimal.new(1, value, -scale)} - - def to_decimal(value, scale) when is_integer(value) and value < 0, - do: {:ok, Decimal.new(-1, -value, -scale)} - else - def to_decimal(_value, _scale), - do: {:error, %Avrora.Errors.ConfigurationError{code: :missing_decimal_lib}} - end - defp enabled, do: Config.self().cast_logical_types() == true end diff --git a/lib/avrora/errors.ex b/lib/avrora/errors.ex index 5b37f7e8..90066650 100644 --- a/lib/avrora/errors.ex +++ b/lib/avrora/errors.ex @@ -3,29 +3,6 @@ defmodule Avrora.Errors do TODO Write Errors moduledoc """ - defmodule ConfigurationError do - @moduledoc """ - TODO Write ConfigurationError moduledoc - """ - - defexception [:code] - - @type t :: %__MODULE__{code: atom()} - @messages %{ - missing_decimal_lib: "missing `Decimal' library, see https://hex.pm/packages/decimal" - } - - @impl true - def exception(code) when is_atom(code), do: %__MODULE__{code: code} - def exception(_), do: %__MODULE__{} - - @impl true - def message(%{code: code}) when is_atom(code) and code != nil, - do: "incorrect configuration, #{Map.get(@messages, code, inspect(code))}" - - def message(_), do: "incorrect configuration" - end - defmodule LogicalTypeDecodingError do @moduledoc """ TODO Write LogicalTypeDecodingError moduledoc @@ -36,6 +13,7 @@ defmodule Avrora.Errors do @type t :: %__MODULE__{code: atom()} @messages %{ invalid_unix_time: "given value is an invalid UNIX time", + missing_decimal_lib: "missing `Decimal' library, see https://hex.pm/packages/decimal", time_zone_not_found: "configured local timezone not found in timezone database", utc_only_time_zone_database: "default timezone database does not support configured local timezone" } diff --git a/test/avrora/avro_type_converter/primitive_into_logical_test.exs b/test/avrora/avro_type_converter/primitive_into_logical_test.exs index 3c17ab96..8480aecb 100644 --- a/test/avrora/avro_type_converter/primitive_into_logical_test.exs +++ b/test/avrora/avro_type_converter/primitive_into_logical_test.exs @@ -33,7 +33,7 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogicalTest do assert {:ok, decoded} == Codec.Plain.decode(date_message(), schema: schema(unknown_json())) end) - assert output =~ "unsupported logical type `unknown' was not converted" + assert output =~ "unsupported logical type `unknown', its value was not type casted" end test "when logical type is uuid" do diff --git a/test/integration/test/decimal_logical_type_test.exs b/test/integration/test/decimal_logical_type_test.exs index a728d337..b1f02e76 100644 --- a/test/integration/test/decimal_logical_type_test.exs +++ b/test/integration/test/decimal_logical_type_test.exs @@ -33,8 +33,7 @@ defmodule Integration.DecimalLogicalTypeTest do {:error, error} = Codec.Plain.decode(message, schema: schema) - assert error.code == :missing_decimal_lib - assert Exception.message(error) =~ "missing `Decimal' library" + assert error == %UndefinedFunctionError{module: Decimal, function: :new, arity: 3} end end end From 1d4698e7c2631ee1539f5c25cafd3223f3ac1325 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 7 Dec 2023 15:12:59 +0100 Subject: [PATCH 27/31] Add time-millis --- .../avro_logical_type_caster/time_millis.ex | 19 ++++++ .../primitive_into_logical.ex | 58 ++++++------------- 2 files changed, 36 insertions(+), 41 deletions(-) create mode 100644 lib/avrora/avro_logical_type_caster/time_millis.ex diff --git a/lib/avrora/avro_logical_type_caster/time_millis.ex b/lib/avrora/avro_logical_type_caster/time_millis.ex new file mode 100644 index 00000000..35b0c053 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/time_millis.ex @@ -0,0 +1,19 @@ +defmodule Avrora.AvroLogicalTypeCaster.TimeMillis do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.TimeMillis moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + @milliseconds 1_000 + @precision 3 + + @impl true + def cast(value, _type) do + time = + div(value, @milliseconds) + |> Time.from_seconds_after_midnight({rem(value, @milliseconds) * 1_000, @precision}) + |> Time.truncate(:millisecond) + + {:ok, time} + end +end diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index e3c01bef..b9446727 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -7,9 +7,7 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do @behaviour Avrora.AvroTypeConverter @logical_type "logicalType" - @millisecond 1_000 @microsecond 1_000_000 - @millisecond_precision 3 @microsecond_precision 6 alias Avrora.AvroLogicalTypeCaster @@ -29,9 +27,15 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do end @config %{ - "decimal" => AvroLogicalTypeCaster.Decimal, "uuid" => AvroLogicalTypeCaster.Noop, "date" => AvroLogicalTypeCaster.Date, + "decimal" => AvroLogicalTypeCaster.Decimal, + "time-millis" => AvroLogicalTypeCaster.TimeMillis, + "time-micros" => AvroLogicalTypeCaster.TimeMicros, + "timestamp-millis" => AvroLogicalTypeCaster.TimestampMillis, + "timestamp-micros" => AvroLogicalTypeCaster.TimestampMicros, + "local-timestamp-millis" => AvroLogicalTypeCaster.LocalTimestampMillis, + "local-timestamp-micros" => AvroLogicalTypeCaster.LocalTimestampMicros, "_" => AvroLogicalTypeCaster.NoopWarning } @@ -42,47 +46,19 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do # FIXME: Refactor this shit defp do_convert(value, type, logical_type) do case logical_type do - "decimal" -> - do_convert2(value, type, logical_type) - - "uuid" -> - do_convert2(value, type, logical_type) - - "date" -> - do_convert2(value, type, logical_type) - - "time-millis" -> - to_time_millis(value) - - "time-micros" -> - to_time_micros(value) - - "timestamp-millis" -> - to_timestamp_millis(value) - - "timestamp-micros" -> - to_timestamp_micros(value) - - "local-timestamp-millis" -> - to_local_timestamp_millis(value) - - "local-timestamp-micros" -> - to_local_timestamp_micros(value) - - _ -> - do_convert2(value, type, logical_type) + "decimal" -> do_convert2(value, type, logical_type) + "uuid" -> do_convert2(value, type, logical_type) + "date" -> do_convert2(value, type, logical_type) + "time-millis" -> do_convert2(value, type, logical_type) + "time-micros" -> to_time_micros(value) + "timestamp-millis" -> to_timestamp_millis(value) + "timestamp-micros" -> to_timestamp_micros(value) + "local-timestamp-millis" -> to_local_timestamp_millis(value) + "local-timestamp-micros" -> to_local_timestamp_micros(value) + _ -> do_convert2(value, type, logical_type) end end - defp to_time_millis(value) do - time = - div(value, @millisecond) - |> Time.from_seconds_after_midnight({rem(value, @millisecond) * 1_000, @millisecond_precision}) - |> Time.truncate(:millisecond) - - {:ok, time} - end - defp to_time_micros(value) do time = div(value, @microsecond) From 9781178e35f2928b81cdfd71f3ee14f769302d0f Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 7 Dec 2023 21:54:31 +0100 Subject: [PATCH 28/31] Add time and timestamp logical types --- .../avro_logical_type_caster/time_micros.ex | 18 +++++++++++++ .../timestamp_micros.ex | 15 +++++++++++ .../timestamp_millis.ex | 15 +++++++++++ .../primitive_into_logical.ex | 27 ------------------- 4 files changed, 48 insertions(+), 27 deletions(-) create mode 100644 lib/avrora/avro_logical_type_caster/time_micros.ex create mode 100644 lib/avrora/avro_logical_type_caster/timestamp_micros.ex create mode 100644 lib/avrora/avro_logical_type_caster/timestamp_millis.ex diff --git a/lib/avrora/avro_logical_type_caster/time_micros.ex b/lib/avrora/avro_logical_type_caster/time_micros.ex new file mode 100644 index 00000000..50ad6851 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/time_micros.ex @@ -0,0 +1,18 @@ +defmodule Avrora.AvroLogicalTypeCaster.TimeMicros do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.TimeMicros moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + @microseconds 1_000_000 + @precision 6 + + @impl true + def cast(value, _type) do + time = + div(value, @microseconds) + |> Time.from_seconds_after_midnight({rem(value, @microseconds), @precision}) + + {:ok, time} + end +end diff --git a/lib/avrora/avro_logical_type_caster/timestamp_micros.ex b/lib/avrora/avro_logical_type_caster/timestamp_micros.ex new file mode 100644 index 00000000..85e36c2c --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/timestamp_micros.ex @@ -0,0 +1,15 @@ +defmodule Avrora.AvroLogicalTypeCaster.TimestampMicros do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.TimestampMicros moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + + alias Avrora.Errors + + @impl true + def cast(value, _type) do + with {:error, reason} <- DateTime.from_unix(value, :microsecond), + do: {:error, %Errors.LogicalTypeDecodingError{code: reason}} + end +end diff --git a/lib/avrora/avro_logical_type_caster/timestamp_millis.ex b/lib/avrora/avro_logical_type_caster/timestamp_millis.ex new file mode 100644 index 00000000..7eab8d74 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/timestamp_millis.ex @@ -0,0 +1,15 @@ +defmodule Avrora.AvroLogicalTypeCaster.TimestampMillis do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.TimestampMillis moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + + alias Avrora.Errors + + @impl true + def cast(value, _type) do + with {:error, reason} <- DateTime.from_unix(value, :millisecond), + do: {:error, %Errors.LogicalTypeDecodingError{code: reason}} + end +end diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index b9446727..22371fe5 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -7,8 +7,6 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do @behaviour Avrora.AvroTypeConverter @logical_type "logicalType" - @microsecond 1_000_000 - @microsecond_precision 6 alias Avrora.AvroLogicalTypeCaster alias Avrora.Config @@ -46,37 +44,12 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do # FIXME: Refactor this shit defp do_convert(value, type, logical_type) do case logical_type do - "decimal" -> do_convert2(value, type, logical_type) - "uuid" -> do_convert2(value, type, logical_type) - "date" -> do_convert2(value, type, logical_type) - "time-millis" -> do_convert2(value, type, logical_type) - "time-micros" -> to_time_micros(value) - "timestamp-millis" -> to_timestamp_millis(value) - "timestamp-micros" -> to_timestamp_micros(value) "local-timestamp-millis" -> to_local_timestamp_millis(value) "local-timestamp-micros" -> to_local_timestamp_micros(value) _ -> do_convert2(value, type, logical_type) end end - defp to_time_micros(value) do - time = - div(value, @microsecond) - |> Time.from_seconds_after_midnight({rem(value, @microsecond), @microsecond_precision}) - - {:ok, time} - end - - defp to_timestamp_millis(value) do - with {:error, reason} <- DateTime.from_unix(value, :millisecond), - do: {:error, %Avrora.Errors.LogicalTypeDecodingError{code: reason}} - end - - defp to_timestamp_micros(value) do - with {:error, reason} <- DateTime.from_unix(value, :microsecond), - do: {:error, %Avrora.Errors.LogicalTypeDecodingError{code: reason}} - end - defp to_local_timestamp_millis(value) do with {:ok, date_time} <- DateTime.from_unix(value, :millisecond), {:ok, local_date_time} <- DateTime.shift_zone(date_time, "Japan") do From be2aba542d8d1392315901ed3544b63ce59df42a Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 7 Dec 2023 22:25:34 +0100 Subject: [PATCH 29/31] Add local timestamp --- .../local_timestamp_micros.ex | 21 ++++++++++++++ .../local_timestamp_millis.ex | 21 ++++++++++++++ .../primitive_into_logical.ex | 29 +------------------ 3 files changed, 43 insertions(+), 28 deletions(-) create mode 100644 lib/avrora/avro_logical_type_caster/local_timestamp_micros.ex create mode 100644 lib/avrora/avro_logical_type_caster/local_timestamp_millis.ex diff --git a/lib/avrora/avro_logical_type_caster/local_timestamp_micros.ex b/lib/avrora/avro_logical_type_caster/local_timestamp_micros.ex new file mode 100644 index 00000000..6145a552 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/local_timestamp_micros.ex @@ -0,0 +1,21 @@ +defmodule Avrora.AvroLogicalTypeCaster.LocalTimestampMicros do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.LocalTimestampMicros moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + # @timezone "Etc/UTC" + @timezone "Japan" + + alias Avrora.Errors + + @impl true + def cast(value, _type) do + with {:ok, date_time} <- DateTime.from_unix(value, :microsecond), + {:ok, local_date_time} <- DateTime.shift_zone(date_time, @timezone) do + {:ok, local_date_time} + else + {:error, reason} -> {:error, %Errors.LogicalTypeDecodingError{code: reason}} + end + end +end diff --git a/lib/avrora/avro_logical_type_caster/local_timestamp_millis.ex b/lib/avrora/avro_logical_type_caster/local_timestamp_millis.ex new file mode 100644 index 00000000..2531e082 --- /dev/null +++ b/lib/avrora/avro_logical_type_caster/local_timestamp_millis.ex @@ -0,0 +1,21 @@ +defmodule Avrora.AvroLogicalTypeCaster.LocalTimestampMillis do + @moduledoc """ + TODO Write AvroLogicalTypeCaster.LocalTimestampMillis moduledoc + """ + + @behaviour Avrora.AvroLogicalTypeCaster + # @timezone "Etc/UTC" + @timezone "Japan" + + alias Avrora.Errors + + @impl true + def cast(value, _type) do + with {:ok, date_time} <- DateTime.from_unix(value, :millisecond), + {:ok, local_date_time} <- DateTime.shift_zone(date_time, @timezone) do + {:ok, local_date_time} + else + {:error, reason} -> {:error, %Errors.LogicalTypeDecodingError{code: reason}} + end + end +end diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index 22371fe5..2163ea36 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -37,35 +37,8 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do "_" => AvroLogicalTypeCaster.NoopWarning } - defp do_convert2(value, type, logical_type) do - Map.get(@config, logical_type, Map.fetch!(@config, "_")).cast(value, type) - end - - # FIXME: Refactor this shit defp do_convert(value, type, logical_type) do - case logical_type do - "local-timestamp-millis" -> to_local_timestamp_millis(value) - "local-timestamp-micros" -> to_local_timestamp_micros(value) - _ -> do_convert2(value, type, logical_type) - end - end - - defp to_local_timestamp_millis(value) do - with {:ok, date_time} <- DateTime.from_unix(value, :millisecond), - {:ok, local_date_time} <- DateTime.shift_zone(date_time, "Japan") do - {:ok, local_date_time} - else - {:error, reason} -> {:error, %Avrora.Errors.LogicalTypeDecodingError{code: reason}} - end - end - - defp to_local_timestamp_micros(value) do - with {:ok, date_time} <- DateTime.from_unix(value, :microsecond), - {:ok, local_date_time} <- DateTime.shift_zone(date_time, "Japan") do - {:ok, local_date_time} - else - {:error, reason} -> {:error, %Avrora.Errors.LogicalTypeDecodingError{code: reason}} - end + Map.get(@config, logical_type, Map.fetch!(@config, "_")).cast(value, type) end defp enabled, do: Config.self().cast_logical_types() == true From 55211e2a38f054ccd02b6a78e61632fdbff11a38 Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Thu, 7 Dec 2023 22:51:34 +0100 Subject: [PATCH 30/31] Finish with logical types base structure --- .../primitive_into_logical.ex | 3 +++ lib/avrora/client.ex | 11 +++++++++++ lib/avrora/config.ex | 16 ++++++++++++++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/lib/avrora/avro_type_converter/primitive_into_logical.ex b/lib/avrora/avro_type_converter/primitive_into_logical.ex index 2163ea36..6b0a74b9 100644 --- a/lib/avrora/avro_type_converter/primitive_into_logical.ex +++ b/lib/avrora/avro_type_converter/primitive_into_logical.ex @@ -24,6 +24,8 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do end end + # TODO Remove and replace with config + # TODO Add support module to test Japan timezone in local timestamp @config %{ "uuid" => AvroLogicalTypeCaster.Noop, "date" => AvroLogicalTypeCaster.Date, @@ -37,6 +39,7 @@ defmodule Avrora.AvroTypeConverter.PrimitiveIntoLogical do "_" => AvroLogicalTypeCaster.NoopWarning } + # TODO Replace fetch! with fetch and raise generic error of Avrora defp do_convert(value, type, logical_type) do Map.get(@config, logical_type, Map.fetch!(@config, "_")).cast(value, type) end diff --git a/lib/avrora/client.ex b/lib/avrora/client.ex index 89431c64..0e789945 100644 --- a/lib/avrora/client.ex +++ b/lib/avrora/client.ex @@ -114,6 +114,16 @@ defmodule Avrora.Client do @opts unquote(opts) @otp_app Keyword.get(@opts, :otp_app) + # TODO Add tests to check logical types resolution + @logical_types_casting %{ + "_" => Avrora.AvroLogicalTypeCaster.NoopWarning, + "uuid" => Avrora.AvroLogicalTypeCaster.Noop, + "date" => Avrora.AvroLogicalTypeCaster.Date, + "time-millis" => Avrora.AvroLogicalTypeCaster.TimeMillis, + "time-micros" => Avrora.AvroLogicalTypeCaster.TimeMicros, + "timestamp-millis" => Avrora.AvroLogicalTypeCaster.TimestampMillis, + "timestamp-micros" => Avrora.AvroLogicalTypeCaster.TimestampMicros + } def schemas_path do path = get(@opts, :schemas_path, "./priv/schemas") @@ -130,6 +140,7 @@ defmodule Avrora.Client do def cast_logical_types, do: get(@opts, :cast_logical_types, true) def names_cache_ttl, do: get(@opts, :names_cache_ttl, :infinity) def decoder_hook, do: get(@opts, :decoder_hook, fn _, _, data, fun -> fun.(data) end) + def logical_types_casting, do: get(@opts, :logical_types_casting, @logical_types_casting) def file_storage, do: unquote(:"Elixir.#{module}.Storage.File") def memory_storage, do: unquote(:"Elixir.#{module}.Storage.Memory") def registry_storage, do: unquote(:"Elixir.#{module}.Storage.Registry") diff --git a/lib/avrora/config.ex b/lib/avrora/config.ex index 071b74b1..9e9b2dcd 100644 --- a/lib/avrora/config.ex +++ b/lib/avrora/config.ex @@ -12,11 +12,10 @@ defmodule Avrora.Config do * `registry_schemas_autoreg` automatically register schemas in Schema Registry, default `true` * `convert_null_values` convert `:null` values in the decoded message into `nil`, default `true` * `convert_map_to_proplist` bring back old behavior and configure decoding AVRO map-type as proplist, default `false` - TODO Rename into cast_logical_types - TODO Introduce configurable list of logical type casting * `cast_logical_types` convert logical AVRO primitive or complex type into corresponding Elixir representation, default `true` * `names_cache_ttl` duration to cache global schema names millisecods, default `:infinity` * `decoder_hook` function to amend decoded payload, default `fn _, _, data, fun -> fun.(data) end` + * `logical_types_casting` mapping between logical type and casting logic, default `uuid, date, time-millis, time-micros, timestamp-millis, timestamp-micros` ## Internal use interface: @@ -43,6 +42,16 @@ defmodule Avrora.Config do @callback http_client :: module() @callback ets_lib :: module() | atom() + @logical_types_casting %{ + "_" => Avrora.AvroLogicalTypeCaster.NoopWarning, + "uuid" => Avrora.AvroLogicalTypeCaster.Noop, + "date" => Avrora.AvroLogicalTypeCaster.Date, + "time-millis" => Avrora.AvroLogicalTypeCaster.TimeMillis, + "time-micros" => Avrora.AvroLogicalTypeCaster.TimeMicros, + "timestamp-millis" => Avrora.AvroLogicalTypeCaster.TimestampMillis, + "timestamp-micros" => Avrora.AvroLogicalTypeCaster.TimestampMicros + } + @doc false def schemas_path do path = get_env(:schemas_path, "./priv/schemas") @@ -78,6 +87,9 @@ defmodule Avrora.Config do @doc false def decoder_hook, do: get_env(:decoder_hook, fn _, _, data, fun -> fun.(data) end) + @doc false + def logical_types_casting, do: get_env(:logical_types_casting, @logical_types_casting) + @doc false def file_storage, do: Avrora.Storage.File From ff3e5ff774a6779c58c4562006079489c9b8832c Mon Sep 17 00:00:00 2001 From: Sergey Fedorov Date: Tue, 9 Apr 2024 10:08:16 +0200 Subject: [PATCH 31/31] Add type casting descriptions --- lib/avrora/avro_logical_type_caster/date.ex | 6 +++++- lib/avrora/avro_logical_type_caster/decimal.ex | 18 +++++++++++++++++- lib/avrora/avro_logical_type_caster/noop.ex | 3 ++- .../avro_logical_type_caster/noop_warning.ex | 3 ++- .../avro_logical_type_caster/time_micros.ex | 6 +++++- .../avro_logical_type_caster/time_millis.ex | 6 +++++- 6 files changed, 36 insertions(+), 6 deletions(-) diff --git a/lib/avrora/avro_logical_type_caster/date.ex b/lib/avrora/avro_logical_type_caster/date.ex index 19455e75..449872f6 100644 --- a/lib/avrora/avro_logical_type_caster/date.ex +++ b/lib/avrora/avro_logical_type_caster/date.ex @@ -1,6 +1,10 @@ defmodule Avrora.AvroLogicalTypeCaster.Date do @moduledoc """ - TODO Write AvroLogicalTypeCaster.Date moduledoc + The `date` logical type represents a date within the calendar, + with no reference to a particular time zone or time of day. + + The `date` logical type annotates an Avro `int`, where the `int` stores + the number of days from the unix epoch, 1 January 1970 (ISO calendar). """ @behaviour Avrora.AvroLogicalTypeCaster diff --git a/lib/avrora/avro_logical_type_caster/decimal.ex b/lib/avrora/avro_logical_type_caster/decimal.ex index 700c494a..a81b59ed 100644 --- a/lib/avrora/avro_logical_type_caster/decimal.ex +++ b/lib/avrora/avro_logical_type_caster/decimal.ex @@ -1,6 +1,22 @@ defmodule Avrora.AvroLogicalTypeCaster.Decimal do @moduledoc """ - TODO Write AvroLogicalTypeCaster.Decimal moduledoc + The `decimal` logical type represents an arbitrary-precision signed decimal + number of the form `unscaled × 10-scale`. + + The `decimal` logical type annotates Avro bytes or fixed types. + The byte array must contain the two’s-complement representation + of the unscaled integer value in big-endian byte order. + + NOTE: This module is NOT INCLUDED into defaults of `Avrora.Config` and must + be added manually, like this + + config :avrora, logical_types_casting: %{ + "decimal" => Avrora.AvroLogicalTypeCaster.Decimal + ... + } + + NOTE: This module REQUIRES presence of the Decimal library, for details see + https://hex.pm/packages/decimal """ @behaviour Avrora.AvroLogicalTypeCaster diff --git a/lib/avrora/avro_logical_type_caster/noop.ex b/lib/avrora/avro_logical_type_caster/noop.ex index 4bd04f43..9927cb54 100644 --- a/lib/avrora/avro_logical_type_caster/noop.ex +++ b/lib/avrora/avro_logical_type_caster/noop.ex @@ -1,6 +1,7 @@ defmodule Avrora.AvroLogicalTypeCaster.Noop do @moduledoc """ - TODO Write AvroLogicalTypeCaster.Noop moduledoc + This is no-op module used for unsupported logical types. + It keeps the original value untouched and does not generate any warning. """ @behaviour Avrora.AvroLogicalTypeCaster diff --git a/lib/avrora/avro_logical_type_caster/noop_warning.ex b/lib/avrora/avro_logical_type_caster/noop_warning.ex index 083ea185..ea0f95d1 100644 --- a/lib/avrora/avro_logical_type_caster/noop_warning.ex +++ b/lib/avrora/avro_logical_type_caster/noop_warning.ex @@ -1,6 +1,7 @@ defmodule Avrora.AvroLogicalTypeCaster.NoopWarning do @moduledoc """ - TODO Write AvroLogicalTypeCaster.NoopWarning moduledoc + This is no-op module used for unsupported logical types. + It keeps the original value untouched, but generated the warning. """ @behaviour Avrora.AvroLogicalTypeCaster diff --git a/lib/avrora/avro_logical_type_caster/time_micros.ex b/lib/avrora/avro_logical_type_caster/time_micros.ex index 50ad6851..9408d85c 100644 --- a/lib/avrora/avro_logical_type_caster/time_micros.ex +++ b/lib/avrora/avro_logical_type_caster/time_micros.ex @@ -1,6 +1,10 @@ defmodule Avrora.AvroLogicalTypeCaster.TimeMicros do @moduledoc """ - TODO Write AvroLogicalTypeCaster.TimeMicros moduledoc + The `time-micros` logical type represents a time of day, with no reference to + a particular calendar, time zone or date, with a precision of one microsecond. + + The `time-micros` logical type annotates an Avro `long`, where the `long` + stores the number of microseconds after midnight, 00:00:00.000000. """ @behaviour Avrora.AvroLogicalTypeCaster diff --git a/lib/avrora/avro_logical_type_caster/time_millis.ex b/lib/avrora/avro_logical_type_caster/time_millis.ex index 35b0c053..70d46549 100644 --- a/lib/avrora/avro_logical_type_caster/time_millis.ex +++ b/lib/avrora/avro_logical_type_caster/time_millis.ex @@ -1,6 +1,10 @@ defmodule Avrora.AvroLogicalTypeCaster.TimeMillis do @moduledoc """ - TODO Write AvroLogicalTypeCaster.TimeMillis moduledoc + The `time-millis` logical type represents a time of day, with no reference to + a particular calendar, time zone or date, with a precision of one millisecond. + + The `time-millis` logical type annotates an Avro `int`, where the `int` stores + the number of milliseconds after midnight, 00:00:00.000. """ @behaviour Avrora.AvroLogicalTypeCaster