From 93aab5f0f032659bf7b8a9f5aa5205393d7f76bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20S=C3=A1nchez=20Fraile?= Date: Wed, 12 Feb 2025 09:11:34 +0100 Subject: [PATCH 1/5] feat: TD-7106 Add data_structures to implementation cache --- CHANGELOG.md | 8 +++++++- lib/td_cache/implementation_cache.ex | 20 +++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d5ace4..691ccca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ # Changelog -## [7.0.1] 2025-01-13 +## [Unreleased] + +### Added + +- [TD-7106] Add `data_structures` to implementation cache + +## [7.0.1] 2025-01-13 ### Changed diff --git a/lib/td_cache/implementation_cache.ex b/lib/td_cache/implementation_cache.ex index ec9a34d..3e131f4 100644 --- a/lib/td_cache/implementation_cache.ex +++ b/lib/td_cache/implementation_cache.ex @@ -128,7 +128,8 @@ defmodule TdCache.ImplementationCache do {:minimum, :float}, {:result_type, :string}, {:updated_at, :datetime}, - {:status, :string} + {:status, :string}, + {:df_content, :map} ] @rule_props [ @@ -172,6 +173,7 @@ defmodule TdCache.ImplementationCache do implementation |> MapHelpers.parse_fields(@props) + |> decode_df_content() |> put_optional(:execution_result_info, execution_result_info) |> put_optional(:rule, rule) |> put_optional(:concepts_links, concepts) @@ -269,6 +271,7 @@ defmodule TdCache.ImplementationCache do implementation_props = implementation |> Map.take(props_keys) + |> encode_df_content() result_props_keys = Enum.map(@result_props, fn {key, _} -> key end) @@ -307,4 +310,19 @@ defmodule TdCache.ImplementationCache do _ -> ["SADD", "implementation:deleted_ids", implementation_ref] end end + + defp encode_df_content(%{df_content: content} = props) when is_map(content) do + %{props | df_content: Jason.encode!(content)} + end + + defp encode_df_content(props), do: props + + defp decode_df_content(%{df_content: content} = implementation) when is_binary(content) do + case Jason.decode(content) do + {:ok, decoded} -> %{implementation | df_content: decoded} + _ -> implementation + end + end + + defp decode_df_content(implementation), do: implementation end From 84c546a68fcde20f0fe64f0c88cd64c1e8d8cac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20S=C3=A1nchez=20Fraile?= Date: Wed, 12 Feb 2025 11:01:09 +0100 Subject: [PATCH 2/5] test: TS-7106 Updated ImplementationCache test --- test/td_cache/implementation_cache_test.exs | 24 +++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/test/td_cache/implementation_cache_test.exs b/test/td_cache/implementation_cache_test.exs index 92131a8..3b52bc1 100644 --- a/test/td_cache/implementation_cache_test.exs +++ b/test/td_cache/implementation_cache_test.exs @@ -11,7 +11,12 @@ defmodule TdCache.ImplementationCacheTest do alias TdCache.Redix setup do - implementation = build(:implementation) + dfcontent = %{ + "Impact" => %{"origin" => "user", "value" => "High"}, + "Quality Principles" => %{"origin" => "user", "value" => "Completeness"} + } + + implementation = build(:implementation, df_content: dfcontent) on_exit(fn -> ImplementationCache.delete(implementation.id) @@ -19,15 +24,14 @@ defmodule TdCache.ImplementationCacheTest do Redix.command(["DEL", "relation_impl_id_to_impl_ref"]) end) - {:ok, implementation: implementation} + {:ok, implementation: implementation, dfcontent: dfcontent} end describe "ImplementationCache" do test "writes an implementation entry in redis and reads it back", %{ implementation: implementation } do - assert {:ok, [10, 0, 1, 0]} = ImplementationCache.put(implementation) - + assert {:ok, [11, 0, 1, 0]} = ImplementationCache.put(implementation) {:ok, impl} = ImplementationCache.get(implementation.id) assert_maps_equal(impl, implementation, [ @@ -38,7 +42,8 @@ defmodule TdCache.ImplementationCacheTest do :goal, :minimum, :rule_id, - :status + :status, + :df_content ]) refute Map.has_key?(impl, :execution_result_info) @@ -85,12 +90,13 @@ defmodule TdCache.ImplementationCacheTest do end test "writes implementations entries in redis and list implementations keys", %{ - implementation: %{id: impl_id} = implementation + implementation: %{id: impl_id} = implementation, + dfcontent: dfcontent } do - %{id: impl2_id} = impl2 = build(:implementation) + %{id: impl2_id} = impl2 = build(:implementation, df_content: dfcontent) %{id: impl3_id} = impl3 = build(:implementation) - assert {:ok, [10, 0, 1, 0]} = ImplementationCache.put(implementation) - assert {:ok, [10, 0, 1, 0]} = ImplementationCache.put(impl2) + assert {:ok, [11, 0, 1, 0]} = ImplementationCache.put(implementation) + assert {:ok, [11, 0, 1, 0]} = ImplementationCache.put(impl2) assert {:ok, [10, 0, 1, 0]} = ImplementationCache.put(impl3) assert [ From 24c0550de7a62c275cbf42aad0d1e6f5e351e7c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20S=C3=A1nchez=20Fraile?= Date: Tue, 18 Feb 2025 12:58:10 +0100 Subject: [PATCH 3/5] feat: TD-7106 Added data_structures in implementation fields on redis --- lib/td_cache/implementation_cache.ex | 54 +++++++++++++++- lib/td_cache/utils/map_helpers.ex | 54 ++++++++++++++++ test/td_cache/implementation_cache_test.exs | 68 +++++++++++++++++++-- 3 files changed, 171 insertions(+), 5 deletions(-) diff --git a/lib/td_cache/implementation_cache.ex b/lib/td_cache/implementation_cache.ex index 3e131f4..1afb346 100644 --- a/lib/td_cache/implementation_cache.ex +++ b/lib/td_cache/implementation_cache.ex @@ -129,7 +129,8 @@ defmodule TdCache.ImplementationCache do {:result_type, :string}, {:updated_at, :datetime}, {:status, :string}, - {:df_content, :map} + {:df_content, :map}, + {:data_structures, :list} ] @rule_props [ @@ -174,6 +175,7 @@ defmodule TdCache.ImplementationCache do implementation |> MapHelpers.parse_fields(@props) |> decode_df_content() + |> decode_data_structures() |> put_optional(:execution_result_info, execution_result_info) |> put_optional(:rule, rule) |> put_optional(:concepts_links, concepts) @@ -272,6 +274,7 @@ defmodule TdCache.ImplementationCache do implementation |> Map.take(props_keys) |> encode_df_content() + |> encode_data_structures() result_props_keys = Enum.map(@result_props, fn {key, _} -> key end) @@ -325,4 +328,53 @@ defmodule TdCache.ImplementationCache do end defp decode_df_content(implementation), do: implementation + + defp encode_data_structures(%{data_structures: data_structures} = props) do + encoded_data_structures = + data_structures + |> Enum.map(fn %{ + data_structure: ds, + type: link_type + } -> + current_version = Map.get(ds, :current_version) + + cv = + Map.new() + |> Map.put(:name, Map.get(current_version, :name)) + |> Map.put(:domains, Map.get(current_version, :domains)) + |> Map.put(:path, Map.get(current_version, :path)) + + data_structure = + Map.new() + |> Map.put(:id, Map.get(ds, :id)) + |> Map.put(:external_id, Map.get(ds, :external_id)) + |> Map.put(:current_version, cv) + + %{data_structure: data_structure, type: link_type} + end) + |> Jason.encode!() + + Map.put(props, :data_structures, encoded_data_structures) + end + + defp encode_data_structures(props), do: props + + defp decode_data_structures(%{data_structures: data_structures} = implementation) do + case Jason.decode(data_structures) do + {:ok, decoded_data_structures} -> + decoded_data_structures = + decoded_data_structures + |> MapHelpers.atomize_keys() + |> Enum.map(fn %{type: link_type} = ds -> + %{ds | type: String.to_atom(link_type)} + end) + + %{implementation | data_structures: MapHelpers.atomize_keys(decoded_data_structures)} + + _ -> + [] + end + end + + defp decode_data_structures(implementation), do: implementation end diff --git a/lib/td_cache/utils/map_helpers.ex b/lib/td_cache/utils/map_helpers.ex index d197691..d7d9910 100644 --- a/lib/td_cache/utils/map_helpers.ex +++ b/lib/td_cache/utils/map_helpers.ex @@ -17,4 +17,58 @@ defmodule TdCache.Utils.MapHelpers do def parse_string(:string, value) when is_atom(value), do: Atom.to_string(value) # def parse_string(:datetime, value) when is_binary(value), do: DateTime.from_iso8601(value) def parse_string(_, value), do: value + + def atomize_keys(nil), do: nil + + # Structs don't do enumerable and anyway the keys are already + # atoms + def atomize_keys(%{__struct__: _} = struct) do + struct + end + + def atomize_keys(%{} = map) do + map + |> Enum.into(%{}, fn {k, v} -> {to_atom_key(k), atomize_keys(v)} end) + end + + # Walk the list and atomize the keys of + # of any map members + def atomize_keys([head | rest]) do + [atomize_keys(head) | atomize_keys(rest)] + end + + def atomize_keys(not_a_map) do + not_a_map + end + + @doc """ + Convert map atom keys to strings + """ + def stringify_keys(nil), do: nil + + def stringify_keys(%{} = map) do + Enum.into(map, %{}, fn {k, v} -> {to_string_key(k), stringify_keys(v)} end) + end + + # Walk the list and stringify the keys of + # of any map members + def stringify_keys([head | rest]) do + [stringify_keys(head) | stringify_keys(rest)] + end + + def stringify_keys(not_a_map) do + not_a_map + end + + defp to_atom_key(key) when is_binary(key) do + String.to_atom(key) + end + + defp to_atom_key(key), do: key + + defp to_string_key(key) when is_atom(key) do + Atom.to_string(key) + end + + defp to_string_key(key), do: Atom.to_string(key) end diff --git a/test/td_cache/implementation_cache_test.exs b/test/td_cache/implementation_cache_test.exs index 3b52bc1..f951a7a 100644 --- a/test/td_cache/implementation_cache_test.exs +++ b/test/td_cache/implementation_cache_test.exs @@ -16,7 +16,65 @@ defmodule TdCache.ImplementationCacheTest do "Quality Principles" => %{"origin" => "user", "value" => "Completeness"} } - implementation = build(:implementation, df_content: dfcontent) + domains = [ + %{ + id: 111, + name: "domain_name_111", + external_id: "domain_external_id_111" + }, + %{ + id: 222, + name: "domain_name_222", + external_id: "domain_external_id_222" + }, + %{ + id: 333, + name: "domain_name_333", + external_id: "domain_external_id_333" + } + ] + + data_structures = [ + %{ + data_structure: %{ + id: 111, + external_id: "data_tructure_111", + current_version: %{ + path: ["this", "is", "a", "path"], + name: "data_structure_name_111", + domains: domains + } + }, + type: :dataset + }, + %{ + data_structure: %{ + id: 222, + external_id: "data_tructure_222", + current_version: %{ + path: ["this", "is", "a", "path"], + name: "data_structure_name_222", + domains: domains + } + }, + type: :population + }, + %{ + data_structure: %{ + id: 333, + external_id: "data_tructure_333", + current_version: %{ + path: ["this", "is", "a", "path"], + name: "data_structure_name_333", + domains: domains + } + }, + type: :validation + } + ] + + implementation = + build(:implementation, df_content: dfcontent, data_structures: data_structures) on_exit(fn -> ImplementationCache.delete(implementation.id) @@ -31,7 +89,8 @@ defmodule TdCache.ImplementationCacheTest do test "writes an implementation entry in redis and reads it back", %{ implementation: implementation } do - assert {:ok, [11, 0, 1, 0]} = ImplementationCache.put(implementation) + assert {:ok, [12, 0, 1, 0]} = ImplementationCache.put(implementation) + {:ok, impl} = ImplementationCache.get(implementation.id) assert_maps_equal(impl, implementation, [ @@ -43,7 +102,8 @@ defmodule TdCache.ImplementationCacheTest do :minimum, :rule_id, :status, - :df_content + :df_content, + :data_structures ]) refute Map.has_key?(impl, :execution_result_info) @@ -95,7 +155,7 @@ defmodule TdCache.ImplementationCacheTest do } do %{id: impl2_id} = impl2 = build(:implementation, df_content: dfcontent) %{id: impl3_id} = impl3 = build(:implementation) - assert {:ok, [11, 0, 1, 0]} = ImplementationCache.put(implementation) + assert {:ok, [12, 0, 1, 0]} = ImplementationCache.put(implementation) assert {:ok, [11, 0, 1, 0]} = ImplementationCache.put(impl2) assert {:ok, [10, 0, 1, 0]} = ImplementationCache.put(impl3) From 503e871b3b50d156c5c4328c588a57690d765fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20S=C3=A1nchez=20Fraile?= Date: Tue, 18 Feb 2025 13:40:32 +0100 Subject: [PATCH 4/5] chore TD-7106 Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 691ccca..7df3a9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- [TD-7106] Add `data_structures` to implementation cache +- [TD-7106] Issues with customisation and sorting in implementations tables ## [7.0.1] 2025-01-13 From fff462f1161fd8323a24e99952d3dd9fcf70cf7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20S=C3=A1nchez=20Fraile?= Date: Tue, 18 Feb 2025 17:34:02 +0100 Subject: [PATCH 5/5] test: TD-7106 fix bug when encode data_structures domains --- lib/td_cache/implementation_cache.ex | 29 +++++++++++++++------ test/td_cache/implementation_cache_test.exs | 12 ++++----- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/lib/td_cache/implementation_cache.ex b/lib/td_cache/implementation_cache.ex index 1afb346..088a512 100644 --- a/lib/td_cache/implementation_cache.ex +++ b/lib/td_cache/implementation_cache.ex @@ -333,21 +333,19 @@ defmodule TdCache.ImplementationCache do encoded_data_structures = data_structures |> Enum.map(fn %{ - data_structure: ds, + data_structure: %{current_version: current_version, domains: domains} = ds, type: link_type } -> - current_version = Map.get(ds, :current_version) - cv = Map.new() |> Map.put(:name, Map.get(current_version, :name)) - |> Map.put(:domains, Map.get(current_version, :domains)) |> Map.put(:path, Map.get(current_version, :path)) data_structure = Map.new() |> Map.put(:id, Map.get(ds, :id)) |> Map.put(:external_id, Map.get(ds, :external_id)) + |> Map.put(:domains, domains) |> Map.put(:current_version, cv) %{data_structure: data_structure, type: link_type} @@ -360,16 +358,31 @@ defmodule TdCache.ImplementationCache do defp encode_data_structures(props), do: props defp decode_data_structures(%{data_structures: data_structures} = implementation) do + implementation + |> Map.delete(:data_structures) + |> Map.put("data_structures", data_structures) + |> decode_data_structures() + end + + defp decode_data_structures(%{"data_structures" => data_structures} = implementation) do case Jason.decode(data_structures) do {:ok, decoded_data_structures} -> decoded_data_structures = decoded_data_structures - |> MapHelpers.atomize_keys() - |> Enum.map(fn %{type: link_type} = ds -> - %{ds | type: String.to_atom(link_type)} + |> Enum.map(fn %{ + "type" => link_type, + "data_structure" => + %{"domains" => domains} = + data_structure + } -> + ds = %{data_structure | "domains" => domains} + + %{"type" => String.to_atom(link_type), "data_structure" => ds} end) - %{implementation | data_structures: MapHelpers.atomize_keys(decoded_data_structures)} + implementation + |> Map.delete("data_structures") + |> Map.put(:data_structures, MapHelpers.atomize_keys(decoded_data_structures)) _ -> [] diff --git a/test/td_cache/implementation_cache_test.exs b/test/td_cache/implementation_cache_test.exs index f951a7a..03af12a 100644 --- a/test/td_cache/implementation_cache_test.exs +++ b/test/td_cache/implementation_cache_test.exs @@ -39,10 +39,10 @@ defmodule TdCache.ImplementationCacheTest do data_structure: %{ id: 111, external_id: "data_tructure_111", + domains: domains, current_version: %{ path: ["this", "is", "a", "path"], - name: "data_structure_name_111", - domains: domains + name: "data_structure_name_111" } }, type: :dataset @@ -51,10 +51,10 @@ defmodule TdCache.ImplementationCacheTest do data_structure: %{ id: 222, external_id: "data_tructure_222", + domains: domains, current_version: %{ path: ["this", "is", "a", "path"], - name: "data_structure_name_222", - domains: domains + name: "data_structure_name_222" } }, type: :population @@ -63,10 +63,10 @@ defmodule TdCache.ImplementationCacheTest do data_structure: %{ id: 333, external_id: "data_tructure_333", + domains: domains, current_version: %{ path: ["this", "is", "a", "path"], - name: "data_structure_name_333", - domains: domains + name: "data_structure_name_333" } }, type: :validation