From f90f24b201c833598baee79f69f82720fd17b33d Mon Sep 17 00:00:00 2001 From: Cristine Guadelupe Date: Thu, 18 Jan 2024 22:23:24 -0300 Subject: [PATCH 1/4] Supported operations for lists --- lib/assets/data_transform_cell/main.js | 13 ++++++++++--- lib/kino_explorer/data_transform_cell.ex | 21 +++++++++++++-------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/lib/assets/data_transform_cell/main.js b/lib/assets/data_transform_cell/main.js index de83b30..e7cd7ab 100644 --- a/lib/assets/data_transform_cell/main.js +++ b/lib/assets/data_transform_cell/main.js @@ -568,7 +568,7 @@ export async function init(ctx, payload) { operation_type="sorting" label="Sort by" v-model="operation.sort_by" - :options="dataFrameColumns(operation.data_options)" + :options="dataFrameColumnsByTypes(sortTypes, operation.data_options)" :index="index" :disabled="noDataFrame" /> @@ -720,6 +720,7 @@ export async function init(ctx, payload) { dataFrames: payload.data_frame_variables, dataFrameAlias: payload.data_frame_alias.slice(7), pivotWiderTypes: payload.operation_types.pivot_wider, + sortTypes: payload.operation_types.sort, summariseTypes: payload.operation_types.summarise, queriedFilterTypes: payload.operation_types.queried_filter, fillMissingOptions: payload.operation_options.fill_missing, @@ -752,8 +753,14 @@ export async function init(ctx, payload) { return data_options ? Object.keys(data_options) : []; }, dataFrameColumnsWithTypes(data_options) { - const dataFrameColumns = data_options - ? Object.entries(data_options) + const notAllowedTypes = ["list"] + const dataOptions = Object.fromEntries( + Object.entries(data_options).filter(([col, type]) => + !notAllowedTypes.includes(type) + ) + ); + const dataFrameColumns = dataOptions + ? Object.entries(dataOptions) : {}; const columns = Array.from(dataFrameColumns, ([name, type]) => { return { label: `${name} (${type})`, value: name }; diff --git a/lib/kino_explorer/data_transform_cell.ex b/lib/kino_explorer/data_transform_cell.ex index e7bc487..a03e97c 100644 --- a/lib/kino_explorer/data_transform_cell.ex +++ b/lib/kino_explorer/data_transform_cell.ex @@ -21,6 +21,7 @@ defmodule KinoExplorer.DataTransformCell do "string", "time" ] + @special_column_types ["list"] @filter_options %{ "binary" => ["equal", "contains", "not contains", "not equal"], "boolean" => ["equal", "not equal"], @@ -32,7 +33,8 @@ defmodule KinoExplorer.DataTransformCell do "float" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], "integer" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], "string" => ["equal", "contains", "not contains", "not equal"], - "time" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"] + "time" => ["less", "less equal", "equal", "not equal", "greater equal", "greater"], + "list" => [] } @fill_missing_options %{ "binary" => ["forward", "backward", "max", "min", "scalar"], @@ -45,10 +47,11 @@ defmodule KinoExplorer.DataTransformCell do "float" => ["forward", "backward", "max", "min", "mean", "scalar", "nan"], "integer" => ["forward", "backward", "max", "min", "mean", "scalar"], "string" => ["forward", "backward", "max", "min", "scalar"], - "time" => ["forward", "backward", "max", "min", "mean", "scalar"] + "time" => ["forward", "backward", "max", "min", "mean", "scalar"], + "list" => ["forward", "backward"] } @summarise_options %{ - count: @column_types, + count: @column_types ++ @special_column_types, max: [ "date", "datetime[ms]", @@ -69,16 +72,17 @@ defmodule KinoExplorer.DataTransformCell do "integer", "time" ], - n_distinct: @column_types, - nil_count: @column_types, + n_distinct: @column_types ++ @special_column_types, + nil_count: @column_types ++ @special_column_types, standard_deviation: ["float", "integer"], sum: ["boolean", "float", "integer"], variance: ["float", "integer"] } @pivot_wider_types %{ names_from: @column_types, - values_from: @column_types + values_from: @column_types ++ @special_column_types } + @sort_types @column_types @queried_filter_options [ "mean", "median", @@ -139,7 +143,8 @@ defmodule KinoExplorer.DataTransformCell do }, operation_types: %{ pivot_wider: @pivot_wider_types, - queried_filter: @queried_filter_types + queried_filter: @queried_filter_types, + sort: @sort_types }, missing_require: nil ) @@ -980,6 +985,6 @@ defmodule KinoExplorer.DataTransformCell do end defp build_data_options(df) do - df |> DataFrame.dtypes() |> normalize_dtypes() |> Map.reject(fn {_k, v} -> v == "list" end) + df |> DataFrame.dtypes() |> normalize_dtypes() end end From be4bdea2205a406d729c6cef17a26ea7ed1cd7a7 Mon Sep 17 00:00:00 2001 From: Cristine Guadelupe Date: Thu, 18 Jan 2024 22:24:56 -0300 Subject: [PATCH 2/4] Format main.js --- lib/assets/data_transform_cell/main.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/assets/data_transform_cell/main.js b/lib/assets/data_transform_cell/main.js index e7cd7ab..8116d6c 100644 --- a/lib/assets/data_transform_cell/main.js +++ b/lib/assets/data_transform_cell/main.js @@ -753,15 +753,13 @@ export async function init(ctx, payload) { return data_options ? Object.keys(data_options) : []; }, dataFrameColumnsWithTypes(data_options) { - const notAllowedTypes = ["list"] + const notAllowedTypes = ["list"]; const dataOptions = Object.fromEntries( - Object.entries(data_options).filter(([col, type]) => - !notAllowedTypes.includes(type) + Object.entries(data_options).filter( + ([col, type]) => !notAllowedTypes.includes(type) ) ); - const dataFrameColumns = dataOptions - ? Object.entries(dataOptions) - : {}; + const dataFrameColumns = dataOptions ? Object.entries(dataOptions) : {}; const columns = Array.from(dataFrameColumns, ([name, type]) => { return { label: `${name} (${type})`, value: name }; }); From 7766895c5f4672f020c3065ec92ac3958759a230 Mon Sep 17 00:00:00 2001 From: Cristine Guadelupe Date: Fri, 19 Jan 2024 10:58:16 -0300 Subject: [PATCH 3/4] Applying suggestions --- lib/kino_explorer/data_transform_cell.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/kino_explorer/data_transform_cell.ex b/lib/kino_explorer/data_transform_cell.ex index a03e97c..e2ec2a9 100644 --- a/lib/kino_explorer/data_transform_cell.ex +++ b/lib/kino_explorer/data_transform_cell.ex @@ -21,7 +21,7 @@ defmodule KinoExplorer.DataTransformCell do "string", "time" ] - @special_column_types ["list"] + @composite_column_types ["list"] @filter_options %{ "binary" => ["equal", "contains", "not contains", "not equal"], "boolean" => ["equal", "not equal"], @@ -51,7 +51,7 @@ defmodule KinoExplorer.DataTransformCell do "list" => ["forward", "backward"] } @summarise_options %{ - count: @column_types ++ @special_column_types, + count: @column_types ++ @composite_column_types, max: [ "date", "datetime[ms]", @@ -72,15 +72,15 @@ defmodule KinoExplorer.DataTransformCell do "integer", "time" ], - n_distinct: @column_types ++ @special_column_types, - nil_count: @column_types ++ @special_column_types, + n_distinct: @column_types ++ @composite_column_types, + nil_count: @column_types ++ @composite_column_types, standard_deviation: ["float", "integer"], sum: ["boolean", "float", "integer"], variance: ["float", "integer"] } @pivot_wider_types %{ names_from: @column_types, - values_from: @column_types ++ @special_column_types + values_from: @column_types ++ @composite_column_types } @sort_types @column_types @queried_filter_options [ From 6e7397953880a97fe6f9b043bd9590f2cf335125 Mon Sep 17 00:00:00 2001 From: Cristine Guadelupe Date: Fri, 19 Jan 2024 11:02:30 -0300 Subject: [PATCH 4/4] Remove old test --- .../data_transform_cell_test.exs | 45 ------------------- 1 file changed, 45 deletions(-) diff --git a/test/kino_explorer/data_transform_cell_test.exs b/test/kino_explorer/data_transform_cell_test.exs index f9e9509..1ee89b9 100644 --- a/test/kino_explorer/data_transform_cell_test.exs +++ b/test/kino_explorer/data_transform_cell_test.exs @@ -150,51 +150,6 @@ defmodule KinoExplorer.DataTransformCellTest do }) end - test "removes list-type columns from data options" do - {kino, _source} = start_smart_cell!(DataTransformCell, %{}) - - df = - Explorer.DataFrame.new( - [ - a: ["a", "b"], - b: [1, 2], - c: ["https://elixir-lang.org", "https://www.erlang.org"], - d: [<<110, 120>>, <<200, 210>>], - e: [[1, 2], [3, 4]] - ], - dtypes: [d: :binary] - ) - - env = Code.env_for_eval([]) - DataTransformCell.scan_binding(kino.pid, binding(), env) - - data_frame_variables = %{"df" => true} - - assert_broadcast_event(kino, "set_available_data", %{ - "data_frame_variables" => ^data_frame_variables, - "fields" => %{ - operations: [ - %{ - "active" => true, - "column" => nil, - "data_options" => %{ - "a" => "string", - "b" => "integer", - "c" => "string", - "d" => "binary" - }, - "datalist" => [], - "filter" => nil, - "operation_type" => "filters", - "type" => "string", - "value" => nil - } - ], - root_fields: %{"assign_to" => nil, "data_frame" => "df"} - } - }) - end - describe "code generation" do test "source for a data frame without operations" do attrs = build_attrs(%{})