From eb8f5d8b19591f7f4b36795f2607b37d9d9938d3 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Tue, 7 May 2024 02:35:32 +0800 Subject: [PATCH 01/14] renamed `Adbc.Buffer` to `Adbc.Column` --- c_src/adbc_nif.cpp | 8 +-- lib/adbc_buffer.ex | 149 ------------------------------------------- lib/adbc_column.ex | 155 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 153 deletions(-) delete mode 100644 lib/adbc_buffer.ex create mode 100644 lib/adbc_column.ex diff --git a/c_src/adbc_nif.cpp b/c_src/adbc_nif.cpp index 435db07..68ee9e8 100644 --- a/c_src/adbc_nif.cpp +++ b/c_src/adbc_nif.cpp @@ -2128,14 +2128,14 @@ int adbc_buffer_to_arrow_type_struct(ErlNifEnv *env, ERL_NIF_TERM values, struct { case kErrorBufferIsNotAMap: case kErrorBufferWrongStruct: - snprintf(error_out->message, sizeof(error_out->message), "Expected `%%Adbc.Buffer{}` or primitive data types."); + snprintf(error_out->message, sizeof(error_out->message), "Expected `%%Adbc.Column{}` or primitive data types."); return 1; case kErrorBufferGetMapValue: - snprintf(error_out->message, sizeof(error_out->message), "Invalid `%%Adbc.Buffer{}`."); + snprintf(error_out->message, sizeof(error_out->message), "Invalid `%%Adbc.Column{}`."); return 1; case kErrorBufferGetDataListLength: case kErrorBufferDataIsNotAList: - snprintf(error_out->message, sizeof(error_out->message), "Expected the `data` field of `Adbc.Buffer` to be a list of values."); + snprintf(error_out->message, sizeof(error_out->message), "Expected the `data` field of `Adbc.Column` to be a list of values."); return 1; case kErrorBufferUnknownType: case kErrorBufferGetMetadataKey: @@ -2275,7 +2275,7 @@ static int on_load(ErlNifEnv *env, void **, ERL_NIF_TERM) { kAtomSecondKey = erlang::nif::atom(env, "second"); kAtomMicrosecondKey = erlang::nif::atom(env, "microsecond"); - kAdbcBufferStructValue = enif_make_atom(env, "Elixir.Adbc.Buffer"); + kAdbcBufferStructValue = enif_make_atom(env, "Elixir.Adbc.Column"); kAdbcBufferNameKey = enif_make_atom(env, "name"); kAdbcBufferTypeKey = enif_make_atom(env, "type"); kAdbcBufferNullableKey = enif_make_atom(env, "nullable"); diff --git a/lib/adbc_buffer.ex b/lib/adbc_buffer.ex deleted file mode 100644 index a1232df..0000000 --- a/lib/adbc_buffer.ex +++ /dev/null @@ -1,149 +0,0 @@ -defmodule Adbc.Buffer do - @moduledoc """ - Documentation for `Adbc.Buffer`. - - One `Adbc.Buffer` corresponds to a column in the table. It contains the column's name, type, and - data. The data is a list of values of the column's type. The type can be one of the following: - - * `:boolean` - * `:u8` - * `:u16` - * `:u32` - * `:u64` - * `:i8` - * `:i16` - * `:i32` - * `:i64` - * `:f32` - * `:f64` - * `:string` - * `:large_string`, when the size of the string is larger than 4GB - * `:binary` - * `:large_binary`, when the size of the binary is larger than 4GB - * `:fixed_size_binary` - """ - defstruct name: nil, - type: nil, - nullable: false, - metadata: %{}, - data: nil - - @spec buffer(atom, list, Keyword.t()) :: %Adbc.Buffer{} - def buffer(type, data, opts \\ []) when is_atom(type) and is_list(data) and is_list(opts) do - name = opts[:name] - nullable = opts[:nullable] || false - metadata = opts[:metadata] - - %Adbc.Buffer{ - name: name, - type: type, - nullable: nullable, - metadata: metadata, - data: data - } - end - - @spec get_metadata(%Adbc.Buffer{}, String.t(), String.t()) :: String.t() | nil - def get_metadata(%Adbc.Buffer{metadata: metadata}, key, default \\ nil) - when is_binary(key) or is_atom(key) do - metadata[to_string(key)] || default - end - - @spec set_metadata(%Adbc.Buffer{}, String.t(), String.t()) :: %Adbc.Buffer{} - def set_metadata(buffer = %Adbc.Buffer{metadata: metadata}, key, value) - when (is_binary(key) or is_atom(key)) and (is_binary(value) or is_atom(value)) do - %Adbc.Buffer{buffer | metadata: Map.put(metadata, to_string(key), to_string(value))} - end - - @spec delete_metadata(%Adbc.Buffer{}, String.t()) :: %Adbc.Buffer{} - def delete_metadata(buffer = %Adbc.Buffer{metadata: metadata}, key) - when is_binary(key) or is_atom(key) do - %Adbc.Buffer{buffer | metadata: Map.delete(metadata, to_string(key))} - end - - @spec delete_all_metadata(%Adbc.Buffer{}) :: %Adbc.Buffer{} - def delete_all_metadata(buffer = %Adbc.Buffer{}) do - %Adbc.Buffer{buffer | metadata: %{}} - end - - @spec boolean([boolean()], Keyword.t()) :: %Adbc.Buffer{} - def boolean(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:boolean, data, opts) - end - - @spec u8([0..255], Keyword.t()) :: %Adbc.Buffer{} - def u8(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:u8, data, opts) - end - - @spec u16([0..65535], Keyword.t()) :: %Adbc.Buffer{} - def u16(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:u16, data, opts) - end - - @spec u32([0..4_294_967_295], Keyword.t()) :: %Adbc.Buffer{} - def u32(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:u32, data, opts) - end - - @spec u64([0..18_446_744_073_709_551_615], Keyword.t()) :: %Adbc.Buffer{} - def u64(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:u64, data, opts) - end - - @spec i8([-128..127], Keyword.t()) :: %Adbc.Buffer{} - def i8(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:i8, data, opts) - end - - @spec i16([-32768..32767], Keyword.t()) :: %Adbc.Buffer{} - def i16(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:i16, data, opts) - end - - @spec i32([-2_147_483_648..2_147_483_647], Keyword.t()) :: %Adbc.Buffer{} - def i32(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:i32, data, opts) - end - - @spec i64([-9_223_372_036_854_775_808..9_223_372_036_854_775_807], Keyword.t()) :: - %Adbc.Buffer{} - def i64(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:i64, data, opts) - end - - @spec f32([float], Keyword.t()) :: %Adbc.Buffer{} - def f32(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:f32, data, opts) - end - - @spec f64([float], Keyword.t()) :: %Adbc.Buffer{} - def f64(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:f64, data, opts) - end - - @spec string([String.t()], Keyword.t()) :: %Adbc.Buffer{} - def string(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:string, data, opts) - end - - @spec large_string([String.t()], Keyword.t()) :: %Adbc.Buffer{} - def large_string(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:large_string, data, opts) - end - - @spec binary([binary()], Keyword.t()) :: %Adbc.Buffer{} - def binary(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:binary, data, opts) - end - - @spec large_binary([binary()], Keyword.t()) :: %Adbc.Buffer{} - def large_binary(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:large_binary, data, opts) - end - - @spec fixed_size_binary([binary()], Keyword.t()) :: %Adbc.Buffer{} - def fixed_size_binary(data, opts \\ []) when is_list(data) and is_list(opts) do - buffer(:fixed_size_binary, data, opts) - end -end diff --git a/lib/adbc_column.ex b/lib/adbc_column.ex new file mode 100644 index 0000000..fb483b5 --- /dev/null +++ b/lib/adbc_column.ex @@ -0,0 +1,155 @@ +defmodule Adbc.Column do + @moduledoc """ + Documentation for `Adbc.Column`. + + `Adbc.Column` corresponds to a column in the table. It contains the column's name, type, and + data. The data is a list of values of the column's type. The type can be one of the following: + + * `:boolean` + * `:u8` + * `:u16` + * `:u32` + * `:u64` + * `:i8` + * `:i16` + * `:i32` + * `:i64` + * `:f32` + * `:f64` + * `:string` + * `:large_string`, when the size of the string is larger than 4GB + * `:binary` + * `:large_binary`, when the size of the binary is larger than 4GB + * `:fixed_size_binary` + """ + defstruct name: nil, + type: nil, + nullable: false, + metadata: %{}, + data: nil + + @spec column(atom, list, Keyword.t()) :: %Adbc.Column{} + def column(type, data, opts \\ []) when is_atom(type) and is_list(data) and is_list(opts) do + name = opts[:name] + nullable = opts[:nullable] || false + metadata = opts[:metadata] + + %Adbc.Column{ + name: name, + type: type, + nullable: nullable, + metadata: metadata, + data: data + } + end + + @spec get_metadata(%Adbc.Column{}, String.t(), String.t() | nil) :: String.t() | nil + def get_metadata(%Adbc.Column{metadata: metadata}, key, default \\ nil) + when is_binary(key) or is_atom(key) do + metadata[to_string(key)] || default + end + + @spec set_metadata(%Adbc.Column{}, String.t(), String.t()) :: %Adbc.Column{} + def set_metadata(buffer = %Adbc.Column{metadata: metadata}, key, value) + when (is_binary(key) or is_atom(key)) and (is_binary(value) or is_atom(value)) do + %Adbc.Column{buffer | metadata: Map.put(metadata, to_string(key), to_string(value))} + end + + @spec delete_metadata(%Adbc.Column{}, String.t()) :: %Adbc.Column{} + def delete_metadata(buffer = %Adbc.Column{metadata: metadata}, key) + when is_binary(key) or is_atom(key) do + %Adbc.Column{buffer | metadata: Map.delete(metadata, to_string(key))} + end + + @spec delete_all_metadata(%Adbc.Column{}) :: %Adbc.Column{} + def delete_all_metadata(buffer = %Adbc.Column{}) do + %Adbc.Column{buffer | metadata: %{}} + end + + @spec boolean([boolean()], Keyword.t()) :: %Adbc.Column{} + def boolean(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:boolean, data, opts) + end + + @spec u8([0..255] | binary(), Keyword.t()) :: %Adbc.Column{} + def u8(u8, opts \\ []) + + def u8(data, opts) when is_list(data) and is_list(opts) do + column(:u8, data, opts) + end + + def u8(data, opts) when is_binary(data) and is_list(opts) do + column(:u8, data, opts) + end + + @spec u16([0..65535], Keyword.t()) :: %Adbc.Column{} + def u16(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:u16, data, opts) + end + + @spec u32([0..4_294_967_295], Keyword.t()) :: %Adbc.Column{} + def u32(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:u32, data, opts) + end + + @spec u64([0..18_446_744_073_709_551_615], Keyword.t()) :: %Adbc.Column{} + def u64(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:u64, data, opts) + end + + @spec i8([-128..127], Keyword.t()) :: %Adbc.Column{} + def i8(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:i8, data, opts) + end + + @spec i16([-32768..32767], Keyword.t()) :: %Adbc.Column{} + def i16(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:i16, data, opts) + end + + @spec i32([-2_147_483_648..2_147_483_647], Keyword.t()) :: %Adbc.Column{} + def i32(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:i32, data, opts) + end + + @spec i64([-9_223_372_036_854_775_808..9_223_372_036_854_775_807], Keyword.t()) :: + %Adbc.Column{} + def i64(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:i64, data, opts) + end + + @spec f32([float], Keyword.t()) :: %Adbc.Column{} + def f32(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:f32, data, opts) + end + + @spec f64([float], Keyword.t()) :: %Adbc.Column{} + def f64(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:f64, data, opts) + end + + @spec string([String.t()], Keyword.t()) :: %Adbc.Column{} + def string(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:string, data, opts) + end + + @spec large_string([String.t()], Keyword.t()) :: %Adbc.Column{} + def large_string(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:large_string, data, opts) + end + + @spec binary([binary()], Keyword.t()) :: %Adbc.Column{} + def binary(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:binary, data, opts) + end + + @spec large_binary([binary()], Keyword.t()) :: %Adbc.Column{} + def large_binary(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:large_binary, data, opts) + end + + @spec fixed_size_binary([binary()], Keyword.t()) :: %Adbc.Column{} + def fixed_size_binary(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:fixed_size_binary, data, opts) + end +end From e2d320e96f1161aa9c625ac1e1dd9133dbe2a33e Mon Sep 17 00:00:00 2001 From: Cocoa Date: Wed, 8 May 2024 20:33:06 +0800 Subject: [PATCH 02/14] wrap primivitive data types in `Adbc.Column` --- c_src/adbc_nif.cpp | 307 ++++++++++++++++++++++---------- lib/adbc_column.ex | 36 ++++ lib/adbc_connection.ex | 21 ++- lib/adbc_result.ex | 4 +- test/adbc_connection_test.exs | 320 +++++++++++++++++++++++++--------- test/adbc_sqlite_test.exs | 210 ++++++++++++++-------- test/adbc_test.exs | 95 ++++++++-- 7 files changed, 731 insertions(+), 262 deletions(-) diff --git a/c_src/adbc_nif.cpp b/c_src/adbc_nif.cpp index 68ee9e8..087e176 100644 --- a/c_src/adbc_nif.cpp +++ b/c_src/adbc_nif.cpp @@ -21,45 +21,59 @@ static ERL_NIF_TERM kAtomNil; static ERL_NIF_TERM kAtomTrue; static ERL_NIF_TERM kAtomFalse; static ERL_NIF_TERM kAtomEndOfSeries; - -static ERL_NIF_TERM kAtomDateModule; -static ERL_NIF_TERM kAtomTimeModule; -static ERL_NIF_TERM kAtomNaiveDateTimeModule; -static ERL_NIF_TERM kAtomCalendarISO; static ERL_NIF_TERM kAtomStructKey; + static ERL_NIF_TERM kAtomCalendarKey; +static ERL_NIF_TERM kAtomCalendarISO; + +static ERL_NIF_TERM kAtomDateModule; static ERL_NIF_TERM kAtomYearKey; static ERL_NIF_TERM kAtomMonthKey; static ERL_NIF_TERM kAtomDayKey; + +static ERL_NIF_TERM kAtomNaiveDateTimeModule; +static ERL_NIF_TERM kAtomTimeModule; static ERL_NIF_TERM kAtomHourKey; static ERL_NIF_TERM kAtomMinuteKey; static ERL_NIF_TERM kAtomSecondKey; static ERL_NIF_TERM kAtomMicrosecondKey; -static ERL_NIF_TERM kAdbcBufferStructValue; -static ERL_NIF_TERM kAdbcBufferNameKey; -static ERL_NIF_TERM kAdbcBufferTypeKey; -static ERL_NIF_TERM kAdbcBufferNullableKey; -static ERL_NIF_TERM kAdbcBufferMetadataKey; -static ERL_NIF_TERM kAdbcBufferDataKey; -// static ERL_NIF_TERM kAdbcBufferPrivateKey; - -static ERL_NIF_TERM kAdbcBufferTypeU8; -static ERL_NIF_TERM kAdbcBufferTypeU16; -static ERL_NIF_TERM kAdbcBufferTypeU32; -static ERL_NIF_TERM kAdbcBufferTypeU64; -static ERL_NIF_TERM kAdbcBufferTypeI8; -static ERL_NIF_TERM kAdbcBufferTypeI16; -static ERL_NIF_TERM kAdbcBufferTypeI32; -static ERL_NIF_TERM kAdbcBufferTypeI64; -static ERL_NIF_TERM kAdbcBufferTypeF32; -static ERL_NIF_TERM kAdbcBufferTypeF64; -static ERL_NIF_TERM kAdbcBufferTypeString; -static ERL_NIF_TERM kAdbcBufferTypeLargeString; -static ERL_NIF_TERM kAdbcBufferTypeBinary; -static ERL_NIF_TERM kAdbcBufferTypeLargeBinary; -static ERL_NIF_TERM kAdbcBufferTypeFixedSizeBinary; -static ERL_NIF_TERM kAdbcBufferTypeBool; +static ERL_NIF_TERM kAtomAdbcColumnModule; +static ERL_NIF_TERM kAtomNameKey; +static ERL_NIF_TERM kAtomTypeKey; +static ERL_NIF_TERM kAtomNullableKey; +static ERL_NIF_TERM kAtomMetadataKey; +static ERL_NIF_TERM kAtomDataKey; +// static ERL_NIF_TERM kAtomPrivateKey; + +static ERL_NIF_TERM kAdbcColumnTypeU8; +static ERL_NIF_TERM kAdbcColumnTypeU16; +static ERL_NIF_TERM kAdbcColumnTypeU32; +static ERL_NIF_TERM kAdbcColumnTypeU64; +static ERL_NIF_TERM kAdbcColumnTypeI8; +static ERL_NIF_TERM kAdbcColumnTypeI16; +static ERL_NIF_TERM kAdbcColumnTypeI32; +static ERL_NIF_TERM kAdbcColumnTypeI64; +static ERL_NIF_TERM kAdbcColumnTypeF32; +static ERL_NIF_TERM kAdbcColumnTypeF64; +static ERL_NIF_TERM kAdbcColumnTypeStruct; +static ERL_NIF_TERM kAdbcColumnTypeMap; +static ERL_NIF_TERM kAdbcColumnTypeList; +static ERL_NIF_TERM kAdbcColumnTypeLargeList; +static ERL_NIF_TERM kAdbcColumnTypeFixedSizeList; +static ERL_NIF_TERM kAdbcColumnTypeString; +static ERL_NIF_TERM kAdbcColumnTypeLargeString; +static ERL_NIF_TERM kAdbcColumnTypeBinary; +static ERL_NIF_TERM kAdbcColumnTypeLargeBinary; +static ERL_NIF_TERM kAdbcColumnTypeFixedSizeBinary; +static ERL_NIF_TERM kAdbcColumnTypeDenseUnion; +static ERL_NIF_TERM kAdbcColumnTypeSparseUnion; +static ERL_NIF_TERM kAdbcColumnTypeDate32; +static ERL_NIF_TERM kAdbcColumnTypeDate64; +static ERL_NIF_TERM kAdbcColumnTypeTime32; +static ERL_NIF_TERM kAdbcColumnTypeTime64; +static ERL_NIF_TERM kAdbcColumnTypeTimestamp; +static ERL_NIF_TERM kAdbcColumnTypeBool; constexpr int kErrorBufferIsNotAMap = 1; constexpr int kErrorBufferGetDataListLength = 2; @@ -86,6 +100,42 @@ static ERL_NIF_TERM nif_error_from_adbc_error(ErlNifEnv *env, struct AdbcError * return nif_error; } + +ERL_NIF_TERM make_adbc_column(ErlNifEnv *env, ERL_NIF_TERM name_term, ERL_NIF_TERM type_term, bool nullable, ERL_NIF_TERM metadata, ERL_NIF_TERM data) { + ERL_NIF_TERM nullable_term = nullable ? kAtomTrue : kAtomFalse; + + ERL_NIF_TERM keys[] = { + kAtomStructKey, + kAtomNameKey, + kAtomTypeKey, + kAtomNullableKey, + kAtomMetadataKey, + kAtomDataKey, + }; + ERL_NIF_TERM values[] = { + kAtomAdbcColumnModule, + name_term, + type_term, + nullable_term, + metadata, + data, + }; + + ERL_NIF_TERM adbc_column; + enif_make_map_from_arrays(env, keys, values, sizeof(keys)/sizeof(keys[0]), &adbc_column); + return adbc_column; +} + +ERL_NIF_TERM make_adbc_column(ErlNifEnv *env, ERL_NIF_TERM name_term, const char * type, bool nullable, ERL_NIF_TERM metadata, ERL_NIF_TERM data) { + ERL_NIF_TERM type_term = erlang::nif::make_binary(env, type); + return make_adbc_column(env, name_term, type_term, nullable, metadata, data); +} + +ERL_NIF_TERM make_adbc_column(ErlNifEnv *env, const char * name, const char * type, bool nullable, ERL_NIF_TERM metadata, ERL_NIF_TERM data) { + ERL_NIF_TERM name_term = erlang::nif::make_binary(env, name == nullptr ? "" : name); + return make_adbc_column(env, name_term, type, nullable, metadata, data); +} + template static ERL_NIF_TERM values_from_buffer(ErlNifEnv *env, int64_t offset, int64_t count, const uint8_t * validity_bitmap, const T * value_buffer, const M& value_to_nif) { std::vector values(count); if (validity_bitmap == nullptr) { @@ -158,8 +208,8 @@ template static ERL_NIF_TERM strings_from_buffer( return strings_from_buffer(env, 0, length, validity_bitmap, offsets_buffer, value_buffer, value_to_nif); } -static int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &out_terms, ERL_NIF_TERM &error, bool *end_of_series = nullptr); -static int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, int64_t level, std::vector &out_terms, ERL_NIF_TERM &error, bool *end_of_series = nullptr); +static int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &out_terms, ERL_NIF_TERM &value_type, ERL_NIF_TERM &error, bool *end_of_series = nullptr); +static int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, int64_t level, std::vector &out_terms, ERL_NIF_TERM &value_type, ERL_NIF_TERM &error, bool *end_of_series = nullptr); static int get_arrow_array_children_as_list(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &children, ERL_NIF_TERM &error); static int get_arrow_array_children_as_list(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, uint64_t level, std::vector &children, ERL_NIF_TERM &error); static ERL_NIF_TERM get_arrow_array_map_children(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level); @@ -207,14 +257,15 @@ int get_arrow_array_children_as_list(ErlNifEnv *env, struct ArrowSchema * schema struct ArrowSchema * child_schema = schema->children[child_i]; struct ArrowArray * child_values = values->children[child_i]; std::vector childrens; - if (arrow_array_to_nif_term(env, child_schema, child_values, level + 1, childrens, error) == 1) { + ERL_NIF_TERM child_type; + if (arrow_array_to_nif_term(env, child_schema, child_values, level + 1, childrens, child_type, error) == 1) { return 1; } - if (childrens.size() == 0) { + if (childrens.size() == 1) { children[child_i - offset] = childrens[0]; } else { - children[child_i - offset] = enif_make_tuple2(env, childrens[0], childrens[1]); + children[child_i - offset] = make_adbc_column(env, childrens[0], child_type, child_values->null_count > 0, kAtomNil, childrens[1]); } } @@ -331,7 +382,8 @@ ERL_NIF_TERM get_arrow_array_dense_union_children(ErlNifEnv *env, struct ArrowSc std::vector field_values; union_element_name[0] = erlang::nif::make_binary(env, field_schema->name); - if (arrow_array_to_nif_term(env, field_schema, field_array, child_offset, 1, level + 1, field_values, error) == 1) { + ERL_NIF_TERM field_type; + if (arrow_array_to_nif_term(env, field_schema, field_array, child_offset, 1, level + 1, field_values, field_type, error) == 1) { return error; } @@ -392,7 +444,8 @@ ERL_NIF_TERM get_arrow_array_sparse_union_children(ErlNifEnv *env, struct ArrowS std::vector field_values; union_element_name[0] = erlang::nif::make_binary(env, field_schema->name); - if (arrow_array_to_nif_term(env, field_schema, field_array, child_i, 1, level + 1, field_values, error) == 1) { + ERL_NIF_TERM field_type; + if (arrow_array_to_nif_term(env, field_schema, field_array, child_i, 1, level + 1, field_values, field_type, error) == 1) { return error; } @@ -450,6 +503,7 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * if ((offset + count) > items_values->n_children) { return erlang::nif::error(env, "invalid offset for ArrowArray (list), (offset + count) > items_values->n_children"); } + printf("quq list\r\n"); children.resize(count); for (int64_t child_i = offset; child_i < offset + count; child_i++) { if (bitmap_buffer && values->null_count > 0) { @@ -463,20 +517,25 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * struct ArrowArray * item_values = items_values->children[child_i]; std::vector childrens; - if (arrow_array_to_nif_term(env, item_schema, item_values, level + 1, childrens, error) == 1) { + ERL_NIF_TERM item_type; + if (arrow_array_to_nif_term(env, item_schema, item_values, level + 1, childrens, item_type, error) == 1) { return error; } + // todo: convert to Adbc.Column if (childrens.size() == 1) { children[child_i - offset] = childrens[0]; } else { children[child_i - offset] = enif_make_tuple2(env, childrens[0], childrens[1]); } } + return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); } else { children.resize(1); + printf("simple list\r\n"); std::vector childrens; - if (arrow_array_to_nif_term(env, items_schema, items_values, offset, count, level + 1, childrens, error) == 1) { + ERL_NIF_TERM children_type; + if (arrow_array_to_nif_term(env, items_schema, items_values, offset, count, level + 1, childrens, children_type, error) == 1) { return error; } @@ -485,15 +544,15 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * } else { children[0] = childrens[1]; } + return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); } - return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); } ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level) { return get_arrow_array_list_children(env, schema, values, 0, -1, level); } -int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, int64_t level, std::vector &out_terms, ERL_NIF_TERM &error, bool *end_of_series) { +int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, int64_t level, std::vector &out_terms, ERL_NIF_TERM &term_type, ERL_NIF_TERM &error, bool *end_of_series) { if (schema == nullptr) { error = erlang::nif::error(env, "invalid ArrowSchema (nullptr) when invoking next"); return 1; @@ -505,6 +564,9 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct const char* format = schema->format ? schema->format : ""; const char* name = schema->name ? schema->name : ""; + term_type = kAtomNil; + + // printf("arrow_array_to_nif_term: level=%ld, name=%s, format=%s, offset=%ld, count=%ld\n", level, name, format, offset, count); ERL_NIF_TERM current_term{}, children_term{}; std::vector children; @@ -520,6 +582,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct if (format[0] == 'l') { // NANOARROW_TYPE_INT64 using value_type = int64_t; + term_type = kAdbcColumnTypeI64; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=l), values->n_buffers != 2"); @@ -536,6 +599,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'c') { // NANOARROW_TYPE_INT8 using value_type = int8_t; + term_type = kAdbcColumnTypeI8; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=c), values->n_buffers != 2"); @@ -552,6 +616,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 's') { // NANOARROW_TYPE_INT16 using value_type = int16_t; + term_type = kAdbcColumnTypeI16; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=s), values->n_buffers != 2"); @@ -568,6 +633,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'i') { // NANOARROW_TYPE_INT32 using value_type = int32_t; + term_type = kAdbcColumnTypeI32; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=i), values->n_buffers != 2"); @@ -584,6 +650,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'L') { // NANOARROW_TYPE_UINT64 using value_type = uint64_t; + term_type = kAdbcColumnTypeU64; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=L), values->n_buffers != 2"); @@ -600,6 +667,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'C') { // NANOARROW_TYPE_UINT8 using value_type = uint8_t; + term_type = kAdbcColumnTypeU8; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=C), values->n_buffers != 2"); @@ -616,6 +684,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'S') { // NANOARROW_TYPE_UINT16 using value_type = uint16_t; + term_type = kAdbcColumnTypeU16; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=S), values->n_buffers != 2"); @@ -632,6 +701,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'I') { // NANOARROW_TYPE_UINT32 using value_type = uint32_t; + term_type = kAdbcColumnTypeU32; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=I), values->n_buffers != 2"); @@ -648,6 +718,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'f') { // NANOARROW_TYPE_FLOAT using value_type = float; + term_type = kAdbcColumnTypeF32; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=f), values->n_buffers != 2"); @@ -664,6 +735,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'g') { // NANOARROW_TYPE_DOUBLE using value_type = double; + term_type = kAdbcColumnTypeF64; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=g), values->n_buffers != 2"); @@ -680,6 +752,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'b') { // NANOARROW_TYPE_BOOL using value_type = bool; + term_type = kAdbcColumnTypeBool; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=b), values->n_buffers != 2"); @@ -701,6 +774,11 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'u' || format[0] == 'z') { // NANOARROW_TYPE_BINARY // NANOARROW_TYPE_STRING + if (format[0] == 'z') { + term_type = kAdbcColumnTypeBinary; + } else { + term_type = kAdbcColumnTypeString; + } offset_buffer_index = 1; data_buffer_index = 2; if (count == -1) count = values->length; @@ -722,6 +800,11 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format[0] == 'U' || format[0] == 'Z') { // NANOARROW_TYPE_LARGE_STRING // NANOARROW_TYPE_LARGE_BINARY + if (format[0] == 'Z') { + term_type = kAdbcColumnTypeLargeBinary; + } else { + term_type = kAdbcColumnTypeLargeString; + } offset_buffer_index = 1; data_buffer_index = 2; if (count == -1) count = values->length; @@ -748,6 +831,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct // NANOARROW_TYPE_STRUCT // only handle and return children if this is a struct is_struct = true; + term_type = kAdbcColumnTypeStruct; if (values->length > 0 || values->release != nullptr) { if (count == -1) count = values->n_children; @@ -763,12 +847,15 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } } else if (strncmp("+m", format, 2) == 0) { // NANOARROW_TYPE_MAP + term_type = kAdbcColumnTypeMap; children_term = get_arrow_array_map_children(env, schema, values, offset, count, level); } else if (strncmp("+l", format, 2) == 0) { // NANOARROW_TYPE_LIST + term_type = kAdbcColumnTypeList; children_term = get_arrow_array_list_children(env, schema, values, offset, count, level); } else if (strncmp("+L", format, 2) == 0) { // NANOARROW_TYPE_LARGE_LIST + term_type = kAdbcColumnTypeLargeList; children_term = get_arrow_array_list_children(env, schema, values, offset, count, level); } else { format_processed = false; @@ -776,18 +863,22 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct } else if (format_len >= 3) { if (strncmp("+w:", format, 3) == 0) { // NANOARROW_TYPE_FIXED_SIZE_LIST + term_type = kAdbcColumnTypeFixedSizeList; children_term = get_arrow_array_list_children(env, schema, values, offset, count, level); } else if (format_len > 3 && strncmp("w:", format, 2) == 0) { // NANOARROW_TYPE_FIXED_SIZE_BINARY if (get_arrow_array_children_as_list(env, schema, values, offset, count, level, children, error) == 1) { return 1; } + term_type = kAdbcColumnTypeFixedSizeBinary; children_term = enif_make_list_from_array(env, children.data(), (unsigned)count); } else if (format_len > 4 && (strncmp("+ud:", format, 4) == 0)) { // NANOARROW_TYPE_DENSE_UNION + term_type = kAdbcColumnTypeDenseUnion; children_term = get_arrow_array_dense_union_children(env, schema, values, offset, count, level); } else if (format_len > 4 && (strncmp("+us:", format, 4) == 0)) { // NANOARROW_TYPE_SPARSE_UNION + term_type = kAdbcColumnTypeSparseUnion; children_term = get_arrow_array_sparse_union_children(env, schema, values, offset, count, level); } else if (strncmp("td", format, 2) == 0) { char unit = format[2]; @@ -827,6 +918,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct }; if (unit == 'D') { using value_type = uint32_t; + term_type = kAdbcColumnTypeDate32; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=tdD), values->n_buffers != 2"); @@ -842,6 +934,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ); } else { using value_type = uint64_t; + term_type = kAdbcColumnTypeDate64; if (count == -1) count = values->length; if (values->n_buffers != 2) { error = erlang::nif::error(env, "invalid n_buffers value for ArrowArray (format=tdm), values->n_buffers != 2"); @@ -868,21 +961,25 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct // NANOARROW_TYPE_TIME32 unit = 1000000000; us_precision = 0; + term_type = kAdbcColumnTypeTime32; break; case 'm': // milliseconds // NANOARROW_TYPE_TIME32 unit = 1000000; us_precision = 3; + term_type = kAdbcColumnTypeTime32; break; case 'u': // microseconds // NANOARROW_TYPE_TIME64 unit = 1000; us_precision = 6; + term_type = kAdbcColumnTypeTime64; break; case 'n': // nanoseconds // NANOARROW_TYPE_TIME64 unit = 1; us_precision = 6; + term_type = kAdbcColumnTypeTime64; break; default: format_processed = false; @@ -938,6 +1035,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct // timestamp } else if (strncmp("ts", format, 2) == 0) { // NANOARROW_TYPE_TIMESTAMP + term_type = kAdbcColumnTypeTimestamp; uint64_t unit; uint8_t us_precision; switch (format[2]) { @@ -1045,6 +1143,12 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct out_terms.clear(); if (is_struct) { + // if (level == 0) { + // ERL_NIF_TERM adbc_column = make_adbc_column(env, name, schema->format, values->null_count > 0, kAtomNil, children_term); + // out_terms.emplace_back(adbc_column); + // } else { + // out_terms.emplace_back(children_term); + // } out_terms.emplace_back(children_term); } else { if (schema->children) { @@ -1054,13 +1158,24 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct out_terms.emplace_back(erlang::nif::make_binary(env, name)); out_terms.emplace_back(current_term); } + // if (level == 0) { + // if (schema->children) { + // ERL_NIF_TERM adbc_column = make_adbc_column(env, name, schema->format, values->null_count > 0, kAtomNil, children_term); + // out_terms.emplace_back(adbc_column); + // } else { + // ERL_NIF_TERM adbc_column = make_adbc_column(env, name, schema->format, values->null_count > 0, kAtomNil, current_term); + // out_terms.emplace_back(adbc_column); + // } + // } else { + + // } } return 0; } -int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &out_terms, ERL_NIF_TERM &error, bool *end_of_series) { - return arrow_array_to_nif_term(env, schema, values, 0, -1, level, out_terms, error, end_of_series); +int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &out_terms, ERL_NIF_TERM &type_term, ERL_NIF_TERM &error, bool *end_of_series) { + return arrow_array_to_nif_term(env, schema, values, 0, -1, level, out_terms, type_term, error, end_of_series); } static ERL_NIF_TERM adbc_database_new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1544,7 +1659,8 @@ static ERL_NIF_TERM adbc_arrow_array_stream_next(ErlNifEnv *env, int argc, const auto schema = (struct ArrowSchema*)res->private_data; bool end_of_series = false; - if (arrow_array_to_nif_term(env, schema, &out, 0, out_terms, error, &end_of_series) == 1) { + ERL_NIF_TERM out_type; + if (arrow_array_to_nif_term(env, schema, &out, 0, out_terms, out_type, error, &end_of_series) == 1) { if (out.release) out.release(&out); return error; } @@ -1555,9 +1671,11 @@ static ERL_NIF_TERM adbc_arrow_array_stream_next(ErlNifEnv *env, int argc, const if (out.release) { out.release(&out); } - return ret; + return kAtomEndOfSeries; } + // ret = make_adbc_column(env, schema->name == nullptr ? "" : schema->name, out_type, out.null_count > 0, kAtomNil, out_terms[0]); } else { + // ret = make_adbc_column(env, out_terms[0], out_type, out.null_count > 0, kAtomNil, out_terms[1]); ret = enif_make_tuple2(env, out_terms[0], out_terms[1]); } @@ -1925,23 +2043,23 @@ int adbc_buffer_to_adbc_field(ErlNifEnv *env, ERL_NIF_TERM adbc_buffer, struct A if (!enif_get_map_value(env, adbc_buffer, kAtomStructKey, &struct_name_term)) { return kErrorBufferGetMapValue; } - if (!enif_is_identical(struct_name_term, kAdbcBufferStructValue)) { + if (!enif_is_identical(struct_name_term, kAtomAdbcColumnModule)) { return kErrorBufferWrongStruct; } - if (!enif_get_map_value(env, adbc_buffer, kAdbcBufferNameKey, &name_term)) { + if (!enif_get_map_value(env, adbc_buffer, kAtomNameKey, &name_term)) { return kErrorBufferGetMapValue; } - if (!enif_get_map_value(env, adbc_buffer, kAdbcBufferTypeKey, &type_term)) { + if (!enif_get_map_value(env, adbc_buffer, kAtomTypeKey, &type_term)) { return kErrorBufferGetMapValue; } - if (!enif_get_map_value(env, adbc_buffer, kAdbcBufferNullableKey, &nullable_term)) { + if (!enif_get_map_value(env, adbc_buffer, kAtomNullableKey, &nullable_term)) { return kErrorBufferGetMapValue; } - if (!enif_get_map_value(env, adbc_buffer, kAdbcBufferMetadataKey, &metadata_term)) { + if (!enif_get_map_value(env, adbc_buffer, kAtomMetadataKey, &metadata_term)) { return kErrorBufferGetMapValue; } - if (!enif_get_map_value(env, adbc_buffer, kAdbcBufferDataKey, &data_term)) { + if (!enif_get_map_value(env, adbc_buffer, kAtomDataKey, &data_term)) { return kErrorBufferGetMapValue; } if (!enif_is_list(env, data_term)) { @@ -2001,37 +2119,37 @@ int adbc_buffer_to_adbc_field(ErlNifEnv *env, ERL_NIF_TERM adbc_buffer, struct A NANOARROW_RETURN_NOT_OK(ArrowSchemaSetMetadata(schema_out, (const char*)metadata_buffer.data)); ArrowBufferReset(&metadata_buffer); - if (enif_is_identical(type_term, kAdbcBufferTypeI8)) { + if (enif_is_identical(type_term, kAdbcColumnTypeI8)) { do_get_list_integer(env, data_term, nullable, NANOARROW_TYPE_INT8, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeI16)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeI16)) { do_get_list_integer(env, data_term, nullable, NANOARROW_TYPE_INT16, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeI32)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeI32)) { do_get_list_integer(env, data_term, nullable, NANOARROW_TYPE_INT32, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeI64)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeI64)) { do_get_list_integer(env, data_term, nullable, NANOARROW_TYPE_INT64, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeU8)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeU8)) { do_get_list_integer(env, data_term, nullable, NANOARROW_TYPE_UINT8, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeU16)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeU16)) { do_get_list_integer(env, data_term, nullable, NANOARROW_TYPE_UINT16, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeU32)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeU32)) { do_get_list_integer(env, data_term, nullable, NANOARROW_TYPE_UINT32, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeU64)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeU64)) { do_get_list_integer(env, data_term, nullable, NANOARROW_TYPE_UINT64, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeF32)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeF32)) { do_get_list_float(env, data_term, nullable, NANOARROW_TYPE_FLOAT, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeF64)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeF64)) { do_get_list_float(env, data_term, nullable, NANOARROW_TYPE_DOUBLE, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeString)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeString)) { do_get_list_string(env, data_term, nullable, NANOARROW_TYPE_STRING, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeLargeString)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeLargeString)) { do_get_list_string(env, data_term, nullable, NANOARROW_TYPE_LARGE_STRING, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeBinary)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeBinary)) { do_get_list_string(env, data_term, nullable, NANOARROW_TYPE_BINARY, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeLargeBinary)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeLargeBinary)) { do_get_list_string(env, data_term, nullable, NANOARROW_TYPE_LARGE_BINARY, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeFixedSizeBinary)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeFixedSizeBinary)) { do_get_list_fixed_size_binary(env, data_term, nullable, NANOARROW_TYPE_FIXED_SIZE_BINARY, array_out, schema_out, error_out); - } else if (enif_is_identical(type_term, kAdbcBufferTypeBool)) { + } else if (enif_is_identical(type_term, kAdbcColumnTypeBool)) { do_get_list_boolean(env, data_term, nullable, NANOARROW_TYPE_BOOL, array_out, schema_out, error_out); } else { if (schema_out->release) schema_out->release(schema_out); @@ -2265,6 +2383,7 @@ static int on_load(ErlNifEnv *env, void **, ERL_NIF_TERM) { kAtomCalendarISO = erlang::nif::atom(env, "Elixir.Calendar.ISO"); kAtomTimeModule = erlang::nif::atom(env, "Elixir.Time"); kAtomNaiveDateTimeModule = erlang::nif::atom(env, "Elixir.NaiveDateTime"); + kAtomAdbcColumnModule = erlang::nif::atom(env, "Elixir.Adbc.Column"); kAtomStructKey = erlang::nif::atom(env, "__struct__"); kAtomCalendarKey = erlang::nif::atom(env, "calendar"); kAtomYearKey = erlang::nif::atom(env, "year"); @@ -2275,30 +2394,42 @@ static int on_load(ErlNifEnv *env, void **, ERL_NIF_TERM) { kAtomSecondKey = erlang::nif::atom(env, "second"); kAtomMicrosecondKey = erlang::nif::atom(env, "microsecond"); - kAdbcBufferStructValue = enif_make_atom(env, "Elixir.Adbc.Column"); - kAdbcBufferNameKey = enif_make_atom(env, "name"); - kAdbcBufferTypeKey = enif_make_atom(env, "type"); - kAdbcBufferNullableKey = enif_make_atom(env, "nullable"); - kAdbcBufferMetadataKey = enif_make_atom(env, "metadata"); - kAdbcBufferDataKey = enif_make_atom(env, "data"); + kAtomAdbcColumnModule = erlang::nif::atom(env, "Elixir.Adbc.Column"); + kAtomNameKey = erlang::nif::atom(env, "name"); + kAtomTypeKey = erlang::nif::atom(env, "type"); + kAtomNullableKey = erlang::nif::atom(env, "nullable"); + kAtomMetadataKey = erlang::nif::atom(env, "metadata"); + kAtomDataKey = erlang::nif::atom(env, "data"); // kAdbcBufferPrivateKey = enif_make_atom(env, "__private__"); - kAdbcBufferTypeU8 = enif_make_atom(env, "u8"); - kAdbcBufferTypeU16 = enif_make_atom(env, "u16"); - kAdbcBufferTypeU32 = enif_make_atom(env, "u32"); - kAdbcBufferTypeU64 = enif_make_atom(env, "u64"); - kAdbcBufferTypeI8 = enif_make_atom(env, "i8"); - kAdbcBufferTypeI16 = enif_make_atom(env, "i16"); - kAdbcBufferTypeI32 = enif_make_atom(env, "i32"); - kAdbcBufferTypeI64 = enif_make_atom(env, "i64"); - kAdbcBufferTypeF32 = enif_make_atom(env, "f32"); - kAdbcBufferTypeF64 = enif_make_atom(env, "f64"); - kAdbcBufferTypeString = enif_make_atom(env, "string"); - kAdbcBufferTypeLargeString = enif_make_atom(env, "large_string"); - kAdbcBufferTypeBinary = enif_make_atom(env, "binary"); - kAdbcBufferTypeLargeBinary = enif_make_atom(env, "large_binary"); - kAdbcBufferTypeFixedSizeBinary = enif_make_atom(env, "fixed_size_binary"); - kAdbcBufferTypeBool = enif_make_atom(env, "boolean"); + kAdbcColumnTypeU8 = erlang::nif::atom(env, "u8"); + kAdbcColumnTypeU16 = erlang::nif::atom(env, "u16"); + kAdbcColumnTypeU32 = erlang::nif::atom(env, "u32"); + kAdbcColumnTypeU64 = erlang::nif::atom(env, "u64"); + kAdbcColumnTypeI8 = erlang::nif::atom(env, "i8"); + kAdbcColumnTypeI16 = erlang::nif::atom(env, "i16"); + kAdbcColumnTypeI32 = erlang::nif::atom(env, "i32"); + kAdbcColumnTypeI64 = erlang::nif::atom(env, "i64"); + kAdbcColumnTypeF32 = erlang::nif::atom(env, "f32"); + kAdbcColumnTypeF64 = erlang::nif::atom(env, "f64"); + kAdbcColumnTypeStruct = erlang::nif::atom(env, "struct"); + kAdbcColumnTypeMap = erlang::nif::atom(env, "map"); + kAdbcColumnTypeList = erlang::nif::atom(env, "list"); + kAdbcColumnTypeLargeList = erlang::nif::atom(env, "large_list"); + kAdbcColumnTypeFixedSizeList = erlang::nif::atom(env, "fixed_size_list"); + kAdbcColumnTypeString = erlang::nif::atom(env, "string"); + kAdbcColumnTypeLargeString = erlang::nif::atom(env, "large_string"); + kAdbcColumnTypeBinary = erlang::nif::atom(env, "binary"); + kAdbcColumnTypeLargeBinary = erlang::nif::atom(env, "large_binary"); + kAdbcColumnTypeFixedSizeBinary = erlang::nif::atom(env, "fixed_size_binary"); + kAdbcColumnTypeDenseUnion = erlang::nif::atom(env, "dense_union"); + kAdbcColumnTypeSparseUnion = erlang::nif::atom(env, "sparse_union"); + kAdbcColumnTypeDate32 = erlang::nif::atom(env, "date32"); + kAdbcColumnTypeDate64 = erlang::nif::atom(env, "date64"); + kAdbcColumnTypeTime32 = erlang::nif::atom(env, "time32"); + kAdbcColumnTypeTime64 = erlang::nif::atom(env, "time64"); + kAdbcColumnTypeTimestamp = erlang::nif::atom(env, "timestamp"); + kAdbcColumnTypeBool = erlang::nif::atom(env, "boolean"); return 0; } diff --git a/lib/adbc_column.ex b/lib/adbc_column.ex index fb483b5..88a39dc 100644 --- a/lib/adbc_column.ex +++ b/lib/adbc_column.ex @@ -71,6 +71,42 @@ defmodule Adbc.Column do column(:boolean, data, opts) end + @doc """ + A buffer for an unsigned 8-bit integer column. + + ## Arguments + + * `data`: + * A list of unsigned 8-bit integer values + * A single binary type value where each byte represents an unsigned 8-bit integer + * `opts` - A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:null_count` - The number of null values in the column, defaults to 0, only used when + `:nullable` is `true` + * `:null`: only used when `:nullable` is `true`, can be one of the following: + * A list of booleans with the same length as indicating whether each value is null + * A list of non-negative integers where each integer represents the corresponding index of a + null value + * A single binary type value where each bit indicates whether each value is null + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.u8([1, 2, 3]) + %Adbc.Column{ + name: nil, + type: :u8, + nullable: false, + null_count: 0, + null: nil, + metadata: %{}, + data: [1, 2, 3] + } + """ @spec u8([0..255] | binary(), Keyword.t()) :: %Adbc.Column{} def u8(u8, opts \\ []) diff --git a/lib/adbc_connection.ex b/lib/adbc_connection.ex index e37b0f7..75bce27 100644 --- a/lib/adbc_connection.ex +++ b/lib/adbc_connection.ex @@ -392,22 +392,35 @@ defmodule Adbc.Connection do defp normalize_rows(-1), do: nil defp normalize_rows(rows) when is_integer(rows) and rows >= 0, do: rows - defp stream_results(reference, num_rows), do: stream_results(reference, %{}, num_rows) + defp stream_results(reference, num_rows), do: stream_results(reference, [], num_rows) defp stream_results(reference, acc, num_rows) do case Adbc.Nif.adbc_arrow_array_stream_next(reference) do {:ok, results, _done} -> - acc = Map.merge(acc, Map.new(results), fn _k, v1, v2 -> v1 ++ v2 end) - stream_results(reference, acc, num_rows) + stream_results(reference, acc ++ results, num_rows) :end_of_series -> - {:ok, %Adbc.Result{data: acc, num_rows: num_rows}} + {:ok, %Adbc.Result{data: merge_columns(acc), num_rows: num_rows}} {:error, reason} -> {:error, error_to_exception(reason)} end end + defp merge_columns(columns) do + Enum.reduce(columns, [], fn column, merged_columns -> + case Enum.find_index(merged_columns, &(&1.name == column.name)) do + nil -> + merged_columns ++ [column] + + index -> + merged_column = Enum.at(merged_columns, index) + merged_column = %{merged_column | data: merged_column.data ++ column.data} + List.replace_at(merged_columns, index, merged_column) + end + end) + end + ## Callbacks @impl true diff --git a/lib/adbc_result.ex b/lib/adbc_result.ex index 1959999..9eb8a9f 100644 --- a/lib/adbc_result.ex +++ b/lib/adbc_result.ex @@ -4,7 +4,7 @@ defmodule Adbc.Result do It has two fields: - * `:data` - a map of field names to a list of values + * `:data` - a list of `Adbc.Column` * `:num_rows` - the number of rows returned, if returned by the database @@ -13,6 +13,6 @@ defmodule Adbc.Result do @type t :: %Adbc.Result{ num_rows: non_neg_integer() | nil, - data: %{optional(binary) => list(term)} + data: [%Adbc.Column{}] } end diff --git a/test/adbc_connection_test.exs b/test/adbc_connection_test.exs index 74a765a..1eb4962 100644 --- a/test/adbc_connection_test.exs +++ b/test/adbc_connection_test.exs @@ -47,33 +47,58 @@ defmodule Adbc.Connection.Test do {:ok, %Adbc.Result{ num_rows: nil, - data: %{ - "info_name" => [0, 1, 100, 101, 102], - "info_value" => [ - %{"string_value" => ["SQLite"]}, - # %{"string_value" => ["3.43.2"]}, - %{"string_value" => [_]}, - %{"string_value" => ["ADBC SQLite Driver"]}, - # %{"string_value" => ["(unknown)"]}, - %{"string_value" => [_]}, - # %{"string_value" => ["0.4.0"]} - %{"string_value" => [_]} - ] - } + data: [ + %Adbc.Column{ + name: "info_name", + type: :u32, + nullable: false, + metadata: nil, + data: [0, 1, 100, 101, 102] + }, + %Adbc.Column{ + name: "info_value", + type: :dense_union, + nullable: false, + metadata: nil, + data: [ + %{"string_value" => ["SQLite"]}, + # "3.43.2" + %{"string_value" => [_]}, + %{"string_value" => ["ADBC SQLite Driver"]}, + # "(unknown)" + %{"string_value" => [_]}, + # "0.4.0" + %{"string_value" => [_]} + ] + } + ] }} = Connection.get_info(conn) end test "get some info from a connection", %{db: db} do conn = start_supervised!({Connection, database: db}) - assert {:ok, - %Adbc.Result{ - num_rows: nil, - data: %{ - "info_name" => [0], - "info_value" => [%{"string_value" => ["SQLite"]}] - } - }} == Connection.get_info(conn, [0]) + assert { + :ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "info_name", + type: :u32, + nullable: false, + metadata: nil, + data: [0] + }, + %Adbc.Column{ + name: "info_value", + type: :dense_union, + nullable: false, + metadata: nil, + data: [%{"string_value" => ["SQLite"]}] + } + ] + } + } == Connection.get_info(conn, [0]) end end @@ -98,53 +123,64 @@ defmodule Adbc.Connection.Test do {:ok, %Adbc.Result{ - num_rows: nil, - data: %{ - "catalog_db_schemas" => [ - {"db_schema_name", []}, - {"db_schema_tables", - [ - {"table_name", []}, - {"table_type", []}, - {"table_columns", - [ - {"column_name", []}, - {"ordinal_position", []}, - {"remarks", []}, - {"xdbc_data_type", []}, - {"xdbc_type_name", []}, - {"xdbc_column_size", []}, - {"xdbc_decimal_digits", []}, - {"xdbc_num_prec_radix", []}, - {"xdbc_nullable", []}, - {"xdbc_column_def", []}, - {"xdbc_sql_data_type", []}, - {"xdbc_datetime_sub", []}, - {"xdbc_char_octet_length", []}, - {"xdbc_is_nullable", []}, - {"xdbc_scope_catalog", []}, - {"xdbc_scope_schema", []}, - {"xdbc_scope_table", []}, - {"xdbc_is_autoincrement", []}, - {"xdbc_is_generatedcolumn", []} - ]}, - {"table_constraints", - [ - {"constraint_name", []}, - {"constraint_type", []}, - {"constraint_column_names", [[]]}, - {"constraint_column_usage", - [ - {"fk_catalog", []}, - {"fk_db_schema", []}, - {"fk_table", []}, - {"fk_column_name", []} - ]} - ]} - ]} - ], - "catalog_name" => [] - } + data: [ + %Adbc.Column{ + name: "catalog_name", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "catalog_db_schemas", + type: :list, + nullable: false, + metadata: nil, + data: [ + {"db_schema_name", []}, + {"db_schema_tables", + [ + {"table_name", []}, + {"table_type", []}, + {"table_columns", + [ + {"column_name", []}, + {"ordinal_position", []}, + {"remarks", []}, + {"xdbc_data_type", []}, + {"xdbc_type_name", []}, + {"xdbc_column_size", []}, + {"xdbc_decimal_digits", []}, + {"xdbc_num_prec_radix", []}, + {"xdbc_nullable", []}, + {"xdbc_column_def", []}, + {"xdbc_sql_data_type", []}, + {"xdbc_datetime_sub", []}, + {"xdbc_char_octet_length", []}, + {"xdbc_is_nullable", []}, + {"xdbc_scope_catalog", []}, + {"xdbc_scope_schema", []}, + {"xdbc_scope_table", []}, + {"xdbc_is_autoincrement", []}, + {"xdbc_is_generatedcolumn", []} + ]}, + {"table_constraints", + [ + {"constraint_name", []}, + {"constraint_type", []}, + {"constraint_column_names", [[]]}, + {"constraint_column_usage", + [ + {"fk_catalog", []}, + {"fk_db_schema", []}, + {"fk_table", []}, + {"fk_column_name", []} + ]} + ]} + ]} + ] + } + ] }} = Connection.get_objects(conn, 0) end end @@ -162,17 +198,56 @@ defmodule Adbc.Connection.Test do test "select", %{db: db} do conn = start_supervised!({Connection, database: db}) - assert {:ok, %Adbc.Result{data: %{"num" => [123]}}} = + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [123] + } + ] + }} = Connection.query(conn, "SELECT 123 as num") - assert {:ok, %Adbc.Result{data: %{"num" => [123], "bool" => [1]}}} = - Connection.query(conn, "SELECT 123 as num, true as bool") + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [123] + }, + %Adbc.Column{ + name: "bool", + type: :i64, + nullable: false, + metadata: nil, + data: [1] + } + ] + }} = Connection.query(conn, "SELECT 123 as num, true as bool") end test "select with parameters", %{db: db} do conn = start_supervised!({Connection, database: db}) - assert {:ok, %Adbc.Result{data: %{"num" => [579]}}} = + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [579] + } + ] + }} = Connection.query(conn, "SELECT 123 + ? as num", [456]) end @@ -186,7 +261,18 @@ defmodule Adbc.Connection.Test do conn = start_supervised!({Connection, database: db}) assert {:ok, ref} = Connection.prepare(conn, "SELECT 123 + ? as num") - assert {:ok, %Adbc.Result{data: %{"num" => [579]}}} = + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [579] + } + ] + }} = Connection.query(conn, ref, [456]) end @@ -195,10 +281,32 @@ defmodule Adbc.Connection.Test do assert {:ok, ref_a} = Connection.prepare(conn, "SELECT 123 + ? as num") assert {:ok, ref_b} = Connection.prepare(conn, "SELECT 1000 + ? as num") - assert {:ok, %Adbc.Result{data: %{"num" => [579]}}} = + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [579] + } + ] + }} = Connection.query(conn, ref_a, [456]) - assert {:ok, %Adbc.Result{data: %{"num" => [1456]}}} = + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [1456] + } + ] + }} = Connection.query(conn, ref_b, [456]) end end @@ -207,17 +315,48 @@ defmodule Adbc.Connection.Test do test "select", %{db: db} do conn = start_supervised!({Connection, database: db}) - assert %Adbc.Result{data: %{"num" => [123]}} = + assert %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [123] + } + ] + } = Connection.query!(conn, "SELECT 123 as num") - assert %Adbc.Result{data: %{"num" => [123], "bool" => [1]}} = + assert %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [123] + }, + %Adbc.Column{name: "bool", type: :i64, nullable: false, metadata: nil, data: [1]} + ] + } = Connection.query!(conn, "SELECT 123 as num, true as bool") end test "select with parameters", %{db: db} do conn = start_supervised!({Connection, database: db}) - assert %Adbc.Result{data: %{"num" => [579]}} = + assert %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [579] + } + ] + } = Connection.query!(conn, "SELECT 123 + ? as num", [456]) end @@ -234,7 +373,18 @@ defmodule Adbc.Connection.Test do test "without parameters", %{db: db} do conn = start_supervised!({Connection, database: db}) - assert %Adbc.Result{data: %{"num" => [123], "bool" => [1]}} == + assert %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [123] + }, + %Adbc.Column{name: "bool", type: :i64, nullable: false, metadata: nil, data: [1]} + ] + } == Connection.query!(conn, "SELECT 123 as num, true as bool", [], "adbc.sqlite.query.batch_rows": 1 ) @@ -243,7 +393,17 @@ defmodule Adbc.Connection.Test do test "with parameters", %{db: db} do conn = start_supervised!({Connection, database: db}) - assert %Adbc.Result{data: %{"num" => [579]}} == + assert %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i64, + nullable: false, + metadata: nil, + data: [579] + } + ] + } == Connection.query!(conn, "SELECT 123 + ? as num", [456], "adbc.sqlite.query.batch_rows": 10 ) diff --git a/test/adbc_sqlite_test.exs b/test/adbc_sqlite_test.exs index 5858acf..4f43e60 100644 --- a/test/adbc_sqlite_test.exs +++ b/test/adbc_sqlite_test.exs @@ -79,33 +79,57 @@ defmodule Adbc.SQLite.Test do {:ok, %Adbc.Result{ num_rows: nil, - data: %{ - "b1" => [<<100, 97, 116, 97, 1, 2>>], - "i1" => [1], - "i2" => [2], - "i3" => [3], - "i4" => [4], - "i5" => [5], - "i6" => [6], - "i7" => ~c"\a", - "i8" => ~c"\b", - "i9" => ~c"\t", - "n1" => [1.1], - "n2" => [2.2], - "n3" => [1], - "n4" => ["2021-01-01"], - "n5" => ["2021-01-01 00:00:00"], - "r1" => [1.1], - "r2" => [2.2], - "r3" => [3.3], - "r4" => [4.4], - "t1" => ["hello"], - "t2" => ["world"], - "t3" => ["goodbye"], - "t4" => ["world"], - "t5" => ["foo"], - "t6" => ["bar"] - } + data: [ + %Adbc.Column{name: "i1", type: :i64, nullable: false, metadata: nil, data: [1]}, + %Adbc.Column{name: "i2", type: :i64, nullable: false, metadata: nil, data: [2]}, + %Adbc.Column{name: "i3", type: :i64, nullable: false, metadata: nil, data: [3]}, + %Adbc.Column{name: "i4", type: :i64, nullable: false, metadata: nil, data: [4]}, + %Adbc.Column{name: "i5", type: :i64, nullable: false, metadata: nil, data: [5]}, + %Adbc.Column{name: "i6", type: :i64, nullable: false, metadata: nil, data: [6]}, + %Adbc.Column{name: "i7", type: :i64, nullable: false, metadata: nil, data: ~c"\a"}, + %Adbc.Column{name: "i8", type: :i64, nullable: false, metadata: nil, data: ~c"\b"}, + %Adbc.Column{name: "i9", type: :i64, nullable: false, metadata: nil, data: ~c"\t"}, + %Adbc.Column{name: "t1", type: :string, nullable: false, metadata: nil, data: ["hello"]}, + %Adbc.Column{name: "t2", type: :string, nullable: false, metadata: nil, data: ["world"]}, + %Adbc.Column{ + name: "t3", + type: :string, + nullable: false, + metadata: nil, + data: ["goodbye"] + }, + %Adbc.Column{name: "t4", type: :string, nullable: false, metadata: nil, data: ["world"]}, + %Adbc.Column{name: "t5", type: :string, nullable: false, metadata: nil, data: ["foo"]}, + %Adbc.Column{name: "t6", type: :string, nullable: false, metadata: nil, data: ["bar"]}, + %Adbc.Column{ + name: "b1", + type: :string, + nullable: false, + metadata: nil, + data: [<<100, 97, 116, 97, 1, 2>>] + }, + %Adbc.Column{name: "r1", type: :f64, nullable: false, metadata: nil, data: [1.1]}, + %Adbc.Column{name: "r2", type: :f64, nullable: false, metadata: nil, data: [2.2]}, + %Adbc.Column{name: "r3", type: :f64, nullable: false, metadata: nil, data: [3.3]}, + %Adbc.Column{name: "r4", type: :f64, nullable: false, metadata: nil, data: [4.4]}, + %Adbc.Column{name: "n1", type: :f64, nullable: false, metadata: nil, data: [1.1]}, + %Adbc.Column{name: "n2", type: :f64, nullable: false, metadata: nil, data: [2.2]}, + %Adbc.Column{name: "n3", type: :i64, nullable: false, metadata: nil, data: [1]}, + %Adbc.Column{ + name: "n4", + type: :string, + nullable: false, + metadata: nil, + data: ["2021-01-01"] + }, + %Adbc.Column{ + name: "n5", + type: :string, + nullable: false, + metadata: nil, + data: ["2021-01-01 00:00:00"] + } + ] }} = Adbc.Connection.query(conn, "SELECT * FROM test") end @@ -118,29 +142,29 @@ defmodule Adbc.SQLite.Test do VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, [ - Adbc.Buffer.i8([1]), - Adbc.Buffer.i16([2]), - Adbc.Buffer.i32([3]), - Adbc.Buffer.i64([4]), - Adbc.Buffer.u8([5]), - Adbc.Buffer.u16([6]), - Adbc.Buffer.u32([7]), - Adbc.Buffer.u64([8]), - Adbc.Buffer.u64([9]), - Adbc.Buffer.string(["hello"]), - Adbc.Buffer.string(["world"]), - Adbc.Buffer.string(["goodbye"]), - Adbc.Buffer.string(["world"]), - Adbc.Buffer.string(["foo"]), - Adbc.Buffer.string(["bar"]), - Adbc.Buffer.binary([<<"data", 0x01, 0x02>>]), - Adbc.Buffer.f32([1.1]), - Adbc.Buffer.f64([2.2]), - Adbc.Buffer.f32([3.3]), - Adbc.Buffer.f64([4.4]), + Adbc.Column.i8([1]), + Adbc.Column.i16([2]), + Adbc.Column.i32([3]), + Adbc.Column.i64([4]), + Adbc.Column.u8([5]), + Adbc.Column.u16([6]), + Adbc.Column.u32([7]), + Adbc.Column.u64([8]), + Adbc.Column.u64([9]), + Adbc.Column.string(["hello"]), + Adbc.Column.string(["world"]), + Adbc.Column.string(["goodbye"]), + Adbc.Column.string(["world"]), + Adbc.Column.string(["foo"]), + Adbc.Column.string(["bar"]), + Adbc.Column.binary([<<"data", 0x01, 0x02>>]), + Adbc.Column.f32([1.1]), + Adbc.Column.f64([2.2]), + Adbc.Column.f32([3.3]), + Adbc.Column.f64([4.4]), 1.1, 2.2, - Adbc.Buffer.boolean([true]), + Adbc.Column.boolean([true]), "2021-01-01", "2021-01-01 00:00:00" ] @@ -149,33 +173,69 @@ defmodule Adbc.SQLite.Test do {:ok, %Adbc.Result{ num_rows: nil, - data: %{ - "b1" => [<<100, 97, 116, 97, 1, 2>>], - "i1" => [1], - "i2" => [2], - "i3" => [3], - "i4" => [4], - "i5" => [5], - "i6" => [6], - "i7" => ~c"\a", - "i8" => ~c"\b", - "i9" => ~c"\t", - "n1" => [1.1], - "n2" => [2.2], - "n3" => [1], - "n4" => ["2021-01-01"], - "n5" => ["2021-01-01 00:00:00"], - "r1" => [r1], - "r2" => [2.2], - "r3" => [r3], - "r4" => [4.4], - "t1" => ["hello"], - "t2" => ["world"], - "t3" => ["goodbye"], - "t4" => ["world"], - "t5" => ["foo"], - "t6" => ["bar"] - } + data: [ + %Adbc.Column{name: "i1", type: :i64, nullable: false, metadata: nil, data: [1]}, + %Adbc.Column{name: "i2", type: :i64, nullable: false, metadata: nil, data: [2]}, + %Adbc.Column{name: "i3", type: :i64, nullable: false, metadata: nil, data: [3]}, + %Adbc.Column{name: "i4", type: :i64, nullable: false, metadata: nil, data: [4]}, + %Adbc.Column{name: "i5", type: :i64, nullable: false, metadata: nil, data: [5]}, + %Adbc.Column{name: "i6", type: :i64, nullable: false, metadata: nil, data: [6]}, + %Adbc.Column{name: "i7", type: :i64, nullable: false, metadata: nil, data: ~c"\a"}, + %Adbc.Column{name: "i8", type: :i64, nullable: false, metadata: nil, data: ~c"\b"}, + %Adbc.Column{name: "i9", type: :i64, nullable: false, metadata: nil, data: ~c"\t"}, + %Adbc.Column{name: "t1", type: :string, nullable: false, metadata: nil, data: ["hello"]}, + %Adbc.Column{name: "t2", type: :string, nullable: false, metadata: nil, data: ["world"]}, + %Adbc.Column{ + name: "t3", + type: :string, + nullable: false, + metadata: nil, + data: ["goodbye"] + }, + %Adbc.Column{name: "t4", type: :string, nullable: false, metadata: nil, data: ["world"]}, + %Adbc.Column{name: "t5", type: :string, nullable: false, metadata: nil, data: ["foo"]}, + %Adbc.Column{name: "t6", type: :string, nullable: false, metadata: nil, data: ["bar"]}, + %Adbc.Column{ + name: "b1", + type: :binary, + nullable: false, + metadata: nil, + data: [<<100, 97, 116, 97, 1, 2>>] + }, + %Adbc.Column{ + name: "r1", + type: :f64, + nullable: false, + metadata: nil, + data: [r1] + }, + %Adbc.Column{name: "r2", type: :f64, nullable: false, metadata: nil, data: [2.2]}, + %Adbc.Column{ + name: "r3", + type: :f64, + nullable: false, + metadata: nil, + data: [r3] + }, + %Adbc.Column{name: "r4", type: :f64, nullable: false, metadata: nil, data: [4.4]}, + %Adbc.Column{name: "n1", type: :f64, nullable: false, metadata: nil, data: [1.1]}, + %Adbc.Column{name: "n2", type: :f64, nullable: false, metadata: nil, data: [2.2]}, + %Adbc.Column{name: "n3", type: :i64, nullable: false, metadata: nil, data: [1]}, + %Adbc.Column{ + name: "n4", + type: :string, + nullable: false, + metadata: nil, + data: ["2021-01-01"] + }, + %Adbc.Column{ + name: "n5", + type: :string, + nullable: false, + metadata: nil, + data: ["2021-01-01 00:00:00"] + } + ] }} = Connection.query(conn, "SELECT * FROM test") assert is_float(r1) and is_float(r3) diff --git a/test/adbc_test.exs b/test/adbc_test.exs index 14358d2..71fdc20 100644 --- a/test/adbc_test.exs +++ b/test/adbc_test.exs @@ -32,17 +32,50 @@ defmodule AdbcTest do end test "runs queries", %{conn: conn} do - assert {:ok, %Adbc.Result{data: %{"num" => [123]}}} = + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :i32, + nullable: false, + metadata: nil, + data: [123] + } + ] + }} = Connection.query(conn, "SELECT 123 as num") end test "list responses", %{conn: conn} do - assert {:ok, %Adbc.Result{data: %{"num" => [[1, 2, 3]]}}} = + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :list, + nullable: false, + metadata: nil, + data: [[1, 2, 3]] + } + ] + }} = Connection.query(conn, "SELECT ARRAY[1, 2, 3] as num") end test "list responses with null", %{conn: conn} do - assert {:ok, %Adbc.Result{data: %{"num" => [[1, 2, 3, nil, 5]]}}} = + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :list, + nullable: false, + metadata: nil, + data: [[1, 2, 3, nil, 5]] + } + ] + }} = Connection.query(conn, "SELECT ARRAY[1, 2, 3, null, 5] as num") end @@ -52,9 +85,15 @@ defmodule AdbcTest do """ %Adbc.Result{ - data: %{ - "generate_series" => generate_series - } + data: [ + %Adbc.Column{ + name: "generate_series", + type: :timestamp, + nullable: false, + metadata: nil, + data: generate_series + } + ] } = Connection.query!(conn, query) assert Enum.count(generate_series) == 3_506_641 @@ -73,13 +112,43 @@ defmodule AdbcTest do """ assert %Adbc.Result{ - data: %{ - "date" => [~D[2023-03-01]], - "datetime" => [~N[2023-03-01 10:23:45.000000]], - "datetime_usec" => [~N[2023-03-01 10:23:45.123456]], - "time" => [~T[10:23:45.000000]], - "time_usec" => [~T[10:23:45.123456]] - } + data: [ + %Adbc.Column{ + name: "datetime", + type: :timestamp, + nullable: false, + metadata: nil, + data: [~N[2023-03-01 10:23:45.000000]] + }, + %Adbc.Column{ + name: "datetime_usec", + type: :timestamp, + nullable: false, + metadata: nil, + data: [~N[2023-03-01 10:23:45.123456]] + }, + %Adbc.Column{ + name: "date", + type: :date32, + nullable: false, + metadata: nil, + data: [~D[2023-03-01]] + }, + %Adbc.Column{ + name: "time", + type: :time64, + nullable: false, + metadata: nil, + data: [~T[10:23:45.000000]] + }, + %Adbc.Column{ + name: "time_usec", + type: :time64, + nullable: false, + metadata: nil, + data: [~T[10:23:45.123456]] + } + ] } = Connection.query!(conn, query) end end From 8cebed746e5ee838261e6bef748d3a072d5a7a13 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Wed, 8 May 2024 21:52:29 +0800 Subject: [PATCH 03/14] wrap list type in `Adbc.Column` --- c_src/adbc_nif.cpp | 21 ++- test/adbc_connection_test.exs | 289 +++++++++++++++++++++++++++++----- test/adbc_test.exs | 53 ++++++- 3 files changed, 309 insertions(+), 54 deletions(-) diff --git a/c_src/adbc_nif.cpp b/c_src/adbc_nif.cpp index 087e176..ef20cb9 100644 --- a/c_src/adbc_nif.cpp +++ b/c_src/adbc_nif.cpp @@ -145,7 +145,7 @@ template static ERL_NIF_TERM values_from_buffer(ErlNifE } else { for (int64_t i = offset; i < offset + count; i++) { uint8_t vbyte = validity_bitmap[i/8]; - if (vbyte & (1 << (i & 0b11111111))) { + if (vbyte & (1 << (i % 8))) { values[i - offset] = value_to_nif(env, value_buffer[i]); } else { values[i - offset] = kAtomNil; @@ -186,7 +186,7 @@ template static ERL_NIF_TERM strings_from_buffer( uint8_t vbyte = validity_bitmap[i / 8]; OffsetT end_index = offsets_buffer[i + 1]; size_t nbytes = end_index - offset; - if (nbytes > 0 && vbyte & (1 << (i & 0b11111111))) { + if (nbytes > 0 && vbyte & (1 << (i % 8))) { values[i - element_offset] = value_to_nif(env, value_buffer, offset, nbytes); } else { values[i - element_offset] = kAtomNil; @@ -249,7 +249,7 @@ int get_arrow_array_children_as_list(ErlNifEnv *env, struct ArrowSchema * schema for (int64_t child_i = offset; child_i < offset + count; child_i++) { if (bitmap_buffer && values->null_count > 0) { uint8_t vbyte = bitmap_buffer[child_i / 8]; - if (!(vbyte & (1 << (child_i & 0b11111111)))) { + if (!(vbyte & (1 << (child_i % 8)))) { children[child_i - offset] = kAtomNil; continue; } @@ -503,12 +503,11 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * if ((offset + count) > items_values->n_children) { return erlang::nif::error(env, "invalid offset for ArrowArray (list), (offset + count) > items_values->n_children"); } - printf("quq list\r\n"); children.resize(count); for (int64_t child_i = offset; child_i < offset + count; child_i++) { if (bitmap_buffer && values->null_count > 0) { uint8_t vbyte = bitmap_buffer[child_i / 8]; - if (!(vbyte & (1 << (child_i & 0b11111111)))) { + if (!(vbyte & (1 << (child_i % 8)))) { children[child_i - offset] = kAtomNil; continue; } @@ -522,17 +521,15 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * return error; } - // todo: convert to Adbc.Column if (childrens.size() == 1) { children[child_i - offset] = childrens[0]; } else { - children[child_i - offset] = enif_make_tuple2(env, childrens[0], childrens[1]); + children[child_i - offset] = make_adbc_column(env, childrens[0], item_type, item_values->null_count > 0, kAtomNil, childrens[1]); } } return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); } else { children.resize(1); - printf("simple list\r\n"); std::vector childrens; ERL_NIF_TERM children_type; if (arrow_array_to_nif_term(env, items_schema, items_values, offset, count, level + 1, childrens, children_type, error) == 1) { @@ -541,10 +538,12 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * if (childrens.size() == 1) { children[0] = childrens[0]; + return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); } else { - children[0] = childrens[1]; + ERL_NIF_TERM column[1]; + column[0] = make_adbc_column(env, childrens[0], children_type, values->null_count > 0, kAtomNil, childrens[1]); + return enif_make_list_from_array(env, column, 1); } - return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); } } @@ -566,7 +565,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct const char* name = schema->name ? schema->name : ""; term_type = kAtomNil; - // printf("arrow_array_to_nif_term: level=%ld, name=%s, format=%s, offset=%ld, count=%ld\n", level, name, format, offset, count); + printf("arrow_array_to_nif_term: level=%lld, name=%s, format=%s, offset=%lld, count=%lld, length=%lld\n", level, name, format, offset, count, values->length); ERL_NIF_TERM current_term{}, children_term{}; std::vector children; diff --git a/test/adbc_connection_test.exs b/test/adbc_connection_test.exs index 1eb4962..c3a4be5 100644 --- a/test/adbc_connection_test.exs +++ b/test/adbc_connection_test.exs @@ -137,47 +137,241 @@ defmodule Adbc.Connection.Test do nullable: false, metadata: nil, data: [ - {"db_schema_name", []}, - {"db_schema_tables", - [ - {"table_name", []}, - {"table_type", []}, - {"table_columns", - [ - {"column_name", []}, - {"ordinal_position", []}, - {"remarks", []}, - {"xdbc_data_type", []}, - {"xdbc_type_name", []}, - {"xdbc_column_size", []}, - {"xdbc_decimal_digits", []}, - {"xdbc_num_prec_radix", []}, - {"xdbc_nullable", []}, - {"xdbc_column_def", []}, - {"xdbc_sql_data_type", []}, - {"xdbc_datetime_sub", []}, - {"xdbc_char_octet_length", []}, - {"xdbc_is_nullable", []}, - {"xdbc_scope_catalog", []}, - {"xdbc_scope_schema", []}, - {"xdbc_scope_table", []}, - {"xdbc_is_autoincrement", []}, - {"xdbc_is_generatedcolumn", []} - ]}, - {"table_constraints", - [ - {"constraint_name", []}, - {"constraint_type", []}, - {"constraint_column_names", [[]]}, - {"constraint_column_usage", - [ - {"fk_catalog", []}, - {"fk_db_schema", []}, - {"fk_table", []}, - {"fk_column_name", []} - ]} - ]} - ]} + %Adbc.Column{ + name: "db_schema_name", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "db_schema_tables", + type: :list, + nullable: false, + metadata: nil, + data: [ + %Adbc.Column{ + name: "table_name", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "table_type", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "table_columns", + type: :list, + nullable: false, + metadata: nil, + data: [ + %Adbc.Column{ + name: "column_name", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "ordinal_position", + type: :i32, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "remarks", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_data_type", + type: :i16, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_type_name", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_column_size", + type: :i32, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_decimal_digits", + type: :i16, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_num_prec_radix", + type: :i16, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_nullable", + type: :i16, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_column_def", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_sql_data_type", + type: :i16, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_datetime_sub", + type: :i16, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_char_octet_length", + type: :i32, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_is_nullable", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_scope_catalog", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_scope_schema", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_scope_table", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_is_autoincrement", + type: :boolean, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "xdbc_is_generatedcolumn", + type: :boolean, + nullable: false, + metadata: nil, + data: [] + } + ] + }, + %Adbc.Column{ + name: "table_constraints", + type: :list, + nullable: false, + metadata: nil, + data: [ + %Adbc.Column{ + name: "constraint_name", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "constraint_type", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "constraint_column_names", + type: :list, + nullable: false, + metadata: nil, + data: [[]] + }, + %Adbc.Column{ + name: "constraint_column_usage", + type: :list, + nullable: false, + metadata: nil, + data: [ + %Adbc.Column{ + name: "fk_catalog", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "fk_db_schema", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "fk_table", + type: :string, + nullable: false, + metadata: nil, + data: [] + }, + %Adbc.Column{ + name: "fk_column_name", + type: :string, + nullable: false, + metadata: nil, + data: [] + } + ] + } + ] + } + ] + } ] } ] @@ -189,7 +383,18 @@ defmodule Adbc.Connection.Test do test "get table types from a connection", %{db: db} do conn = start_supervised!({Connection, database: db}) - assert {:ok, %Adbc.Result{data: %{"table_type" => ["table", "view"]}}} = + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "table_type", + type: :string, + nullable: false, + metadata: nil, + data: ["table", "view"] + } + ] + }} = Connection.get_table_types(conn) end end diff --git a/test/adbc_test.exs b/test/adbc_test.exs index 71fdc20..b69f2f7 100644 --- a/test/adbc_test.exs +++ b/test/adbc_test.exs @@ -72,13 +72,64 @@ defmodule AdbcTest do type: :list, nullable: false, metadata: nil, - data: [[1, 2, 3, nil, 5]] + data: [ + %Adbc.Column{ + name: "item", + type: :i32, + nullable: false, + metadata: nil, + data: [1, 2, 3, nil, 5] + } + ] } ] }} = Connection.query(conn, "SELECT ARRAY[1, 2, 3, null, 5] as num") end + test "nested list responses with null", %{conn: conn} do + # import adbc_driver_postgresql + # import adbc_driver_manager + # import pyarrow + # db = adbc_driver_postgresql.connect(uri="postgres://postgres:postgres@localhost"): + # conn = adbc_driver_manager.AdbcConnection(db) + # stmt = adbc_driver_manager.AdbcStatement(conn) + # stmt.set_sql_query("SELECT ARRAY[ARRAY[1, 2, 3, null, 5], ARRAY[6, null, 7, null, 9]] as num") + # stream, _ = stmt.execute_query() + # reader = pyarrow.RecordBatchReader._import_from_c(stream.address) + # print(reader.read_all()) + # + # pyarrow.Table + # num: list + # child 0, item: int32 + # ---- + # num: [[[1,2,3,null,5,6,null,7,null,9]]] + assert {:ok, + %Adbc.Result{ + data: [ + %Adbc.Column{ + name: "num", + type: :list, + nullable: false, + metadata: nil, + data: [ + %Adbc.Column{ + name: "item", + type: :i32, + nullable: false, + metadata: nil, + data: [1, 2, 3, nil, 5, 6, nil, 7, nil, 9] + } + ] + } + ] + }} = + Connection.query( + conn, + "SELECT ARRAY[ARRAY[1, 2, 3, null, 5], ARRAY[6, null, 7, null, 9]] as num" + ) + end + test "getting all chunks", %{conn: conn} do query = """ SELECT * FROM generate_series('2000-03-01 00:00'::timestamp, '2100-03-04 12:00'::timestamp, '15 minutes') From 23c51d0c26e6ea399c2d0cc3a5fef09076c16942 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Wed, 8 May 2024 22:51:32 +0800 Subject: [PATCH 04/14] return metadata if any --- c_src/adbc_nif.cpp | 55 +++++++++++++++++++++++++---------- test/adbc_connection_test.exs | 10 ++++++- test/adbc_test.exs | 10 ++++++- 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/c_src/adbc_nif.cpp b/c_src/adbc_nif.cpp index ef20cb9..c12f457 100644 --- a/c_src/adbc_nif.cpp +++ b/c_src/adbc_nif.cpp @@ -208,8 +208,8 @@ template static ERL_NIF_TERM strings_from_buffer( return strings_from_buffer(env, 0, length, validity_bitmap, offsets_buffer, value_buffer, value_to_nif); } -static int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &out_terms, ERL_NIF_TERM &value_type, ERL_NIF_TERM &error, bool *end_of_series = nullptr); -static int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, int64_t level, std::vector &out_terms, ERL_NIF_TERM &value_type, ERL_NIF_TERM &error, bool *end_of_series = nullptr); +static int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &out_terms, ERL_NIF_TERM &value_type, ERL_NIF_TERM &metadata, ERL_NIF_TERM &error, bool *end_of_series = nullptr); +static int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, int64_t level, std::vector &out_terms, ERL_NIF_TERM &value_type, ERL_NIF_TERM &metadata, ERL_NIF_TERM &error, bool *end_of_series = nullptr); static int get_arrow_array_children_as_list(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &children, ERL_NIF_TERM &error); static int get_arrow_array_children_as_list(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, uint64_t level, std::vector &children, ERL_NIF_TERM &error); static ERL_NIF_TERM get_arrow_array_map_children(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level); @@ -258,14 +258,15 @@ int get_arrow_array_children_as_list(ErlNifEnv *env, struct ArrowSchema * schema struct ArrowArray * child_values = values->children[child_i]; std::vector childrens; ERL_NIF_TERM child_type; - if (arrow_array_to_nif_term(env, child_schema, child_values, level + 1, childrens, child_type, error) == 1) { + ERL_NIF_TERM child_metadata; + if (arrow_array_to_nif_term(env, child_schema, child_values, level + 1, childrens, child_type, child_metadata, error) == 1) { return 1; } if (childrens.size() == 1) { children[child_i - offset] = childrens[0]; } else { - children[child_i - offset] = make_adbc_column(env, childrens[0], child_type, child_values->null_count > 0, kAtomNil, childrens[1]); + children[child_i - offset] = make_adbc_column(env, childrens[0], child_type, child_values->null_count > 0, child_metadata, childrens[1]); } } @@ -383,7 +384,8 @@ ERL_NIF_TERM get_arrow_array_dense_union_children(ErlNifEnv *env, struct ArrowSc union_element_name[0] = erlang::nif::make_binary(env, field_schema->name); ERL_NIF_TERM field_type; - if (arrow_array_to_nif_term(env, field_schema, field_array, child_offset, 1, level + 1, field_values, field_type, error) == 1) { + ERL_NIF_TERM field_metadata; + if (arrow_array_to_nif_term(env, field_schema, field_array, child_offset, 1, level + 1, field_values, field_type, field_metadata, error) == 1) { return error; } @@ -445,7 +447,9 @@ ERL_NIF_TERM get_arrow_array_sparse_union_children(ErlNifEnv *env, struct ArrowS union_element_name[0] = erlang::nif::make_binary(env, field_schema->name); ERL_NIF_TERM field_type; - if (arrow_array_to_nif_term(env, field_schema, field_array, child_i, 1, level + 1, field_values, field_type, error) == 1) { + // todo: use field_metadata + ERL_NIF_TERM field_metadata; + if (arrow_array_to_nif_term(env, field_schema, field_array, child_i, 1, level + 1, field_values, field_type, field_metadata, error) == 1) { return error; } @@ -517,14 +521,15 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * std::vector childrens; ERL_NIF_TERM item_type; - if (arrow_array_to_nif_term(env, item_schema, item_values, level + 1, childrens, item_type, error) == 1) { + ERL_NIF_TERM item_metadata; + if (arrow_array_to_nif_term(env, item_schema, item_values, level + 1, childrens, item_type, item_metadata, error) == 1) { return error; } if (childrens.size() == 1) { children[child_i - offset] = childrens[0]; } else { - children[child_i - offset] = make_adbc_column(env, childrens[0], item_type, item_values->null_count > 0, kAtomNil, childrens[1]); + children[child_i - offset] = make_adbc_column(env, childrens[0], item_type, item_values->null_count > 0, item_metadata, childrens[1]); } } return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); @@ -532,7 +537,8 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * children.resize(1); std::vector childrens; ERL_NIF_TERM children_type; - if (arrow_array_to_nif_term(env, items_schema, items_values, offset, count, level + 1, childrens, children_type, error) == 1) { + ERL_NIF_TERM children_metadata; + if (arrow_array_to_nif_term(env, items_schema, items_values, offset, count, level + 1, childrens, children_type, children_metadata, error) == 1) { return error; } @@ -541,7 +547,7 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); } else { ERL_NIF_TERM column[1]; - column[0] = make_adbc_column(env, childrens[0], children_type, values->null_count > 0, kAtomNil, childrens[1]); + column[0] = make_adbc_column(env, childrens[0], children_type, values->null_count > 0, children_metadata, childrens[1]); return enif_make_list_from_array(env, column, 1); } } @@ -551,7 +557,7 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * return get_arrow_array_list_children(env, schema, values, 0, -1, level); } -int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, int64_t level, std::vector &out_terms, ERL_NIF_TERM &term_type, ERL_NIF_TERM &error, bool *end_of_series) { +int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, int64_t offset, int64_t count, int64_t level, std::vector &out_terms, ERL_NIF_TERM &term_type, ERL_NIF_TERM &arrow_metadata, ERL_NIF_TERM &error, bool *end_of_series) { if (schema == nullptr) { error = erlang::nif::error(env, "invalid ArrowSchema (nullptr) when invoking next"); return 1; @@ -564,8 +570,7 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct const char* format = schema->format ? schema->format : ""; const char* name = schema->name ? schema->name : ""; term_type = kAtomNil; - - printf("arrow_array_to_nif_term: level=%lld, name=%s, format=%s, offset=%lld, count=%lld, length=%lld\n", level, name, format, offset, count, values->length); + arrow_metadata = kAtomNil; ERL_NIF_TERM current_term{}, children_term{}; std::vector children; @@ -574,6 +579,23 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct int64_t data_buffer_index = 1; int64_t offset_buffer_index = 2; + std::vector metadata_keys, metadata_values; + if (schema->metadata) { + struct ArrowMetadataReader metadata_reader{}; + struct ArrowStringView key; + struct ArrowStringView value; + if (ArrowMetadataReaderInit(&metadata_reader, schema->metadata) == NANOARROW_OK) { + while (ArrowMetadataReaderRead(&metadata_reader, &key, &value) == NANOARROW_OK) { + // printf("key: %.*s, value: %.*s\n", (int)key.size_bytes, key.data, (int)value.size_bytes, value.data); + metadata_keys.push_back(erlang::nif::make_binary(env, key.data, (size_t)key.size_bytes)); + metadata_values.push_back(erlang::nif::make_binary(env, value.data, (size_t)key.size_bytes)); + } + if (metadata_keys.size() > 0) { + enif_make_map_from_arrays(env, metadata_keys.data(), metadata_values.data(), (unsigned)metadata_keys.size(), &arrow_metadata); + } + } + } + bool is_struct = false; size_t format_len = strlen(format); bool format_processed = true; @@ -1173,8 +1195,8 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct return 0; } -int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &out_terms, ERL_NIF_TERM &type_term, ERL_NIF_TERM &error, bool *end_of_series) { - return arrow_array_to_nif_term(env, schema, values, 0, -1, level, out_terms, type_term, error, end_of_series); +int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector &out_terms, ERL_NIF_TERM &out_type, ERL_NIF_TERM &metadata, ERL_NIF_TERM &error, bool *end_of_series) { + return arrow_array_to_nif_term(env, schema, values, 0, -1, level, out_terms, out_type, metadata, error, end_of_series); } static ERL_NIF_TERM adbc_database_new(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1659,7 +1681,8 @@ static ERL_NIF_TERM adbc_arrow_array_stream_next(ErlNifEnv *env, int argc, const auto schema = (struct ArrowSchema*)res->private_data; bool end_of_series = false; ERL_NIF_TERM out_type; - if (arrow_array_to_nif_term(env, schema, &out, 0, out_terms, out_type, error, &end_of_series) == 1) { + ERL_NIF_TERM out_metadata; + if (arrow_array_to_nif_term(env, schema, &out, 0, out_terms, out_type, out_metadata, error, &end_of_series) == 1) { if (out.release) out.release(&out); return error; } diff --git a/test/adbc_connection_test.exs b/test/adbc_connection_test.exs index c3a4be5..f5a5222 100644 --- a/test/adbc_connection_test.exs +++ b/test/adbc_connection_test.exs @@ -330,7 +330,15 @@ defmodule Adbc.Connection.Test do type: :list, nullable: false, metadata: nil, - data: [[]] + data: [ + %Adbc.Column{ + name: "item", + type: :string, + nullable: false, + metadata: nil, + data: [] + } + ] }, %Adbc.Column{ name: "constraint_column_usage", diff --git a/test/adbc_test.exs b/test/adbc_test.exs index b69f2f7..8799316 100644 --- a/test/adbc_test.exs +++ b/test/adbc_test.exs @@ -56,7 +56,15 @@ defmodule AdbcTest do type: :list, nullable: false, metadata: nil, - data: [[1, 2, 3]] + data: [ + %Adbc.Column{ + name: "item", + type: :i32, + nullable: false, + metadata: nil, + data: [1, 2, 3] + } + ] } ] }} = From f2357b2f03cf1c65c0108aca6c82f0d3ed29749c Mon Sep 17 00:00:00 2001 From: Cocoa Date: Wed, 8 May 2024 23:05:08 +0800 Subject: [PATCH 05/14] return nullable value in `Adbc.Column` --- c_src/adbc_nif.cpp | 14 +++-- test/adbc_connection_test.exs | 89 ++++++++++++++--------------- test/adbc_sqlite_test.exs | 103 +++++++++++++++++----------------- test/adbc_test.exs | 26 ++++----- 4 files changed, 118 insertions(+), 114 deletions(-) diff --git a/c_src/adbc_nif.cpp b/c_src/adbc_nif.cpp index c12f457..c509d28 100644 --- a/c_src/adbc_nif.cpp +++ b/c_src/adbc_nif.cpp @@ -247,7 +247,7 @@ int get_arrow_array_children_as_list(ErlNifEnv *env, struct ArrowSchema * schema const uint8_t * bitmap_buffer = (const uint8_t *)values->buffers[bitmap_buffer_index]; children.resize(count); for (int64_t child_i = offset; child_i < offset + count; child_i++) { - if (bitmap_buffer && values->null_count > 0) { + if (bitmap_buffer && ((schema->flags & ARROW_FLAG_NULLABLE) || (values->null_count > 0))) { uint8_t vbyte = bitmap_buffer[child_i / 8]; if (!(vbyte & (1 << (child_i % 8)))) { children[child_i - offset] = kAtomNil; @@ -266,7 +266,8 @@ int get_arrow_array_children_as_list(ErlNifEnv *env, struct ArrowSchema * schema if (childrens.size() == 1) { children[child_i - offset] = childrens[0]; } else { - children[child_i - offset] = make_adbc_column(env, childrens[0], child_type, child_values->null_count > 0, child_metadata, childrens[1]); + bool nullable = (child_schema->flags & ARROW_FLAG_NULLABLE) || (child_values->null_count > 0); + children[child_i - offset] = make_adbc_column(env, childrens[0], child_type, nullable, child_metadata, childrens[1]); } } @@ -508,8 +509,9 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * return erlang::nif::error(env, "invalid offset for ArrowArray (list), (offset + count) > items_values->n_children"); } children.resize(count); + bool items_nullable = (items_schema->flags & ARROW_FLAG_NULLABLE) || (items_values->null_count > 0); for (int64_t child_i = offset; child_i < offset + count; child_i++) { - if (bitmap_buffer && values->null_count > 0) { + if (bitmap_buffer && items_nullable) { uint8_t vbyte = bitmap_buffer[child_i / 8]; if (!(vbyte & (1 << (child_i % 8)))) { children[child_i - offset] = kAtomNil; @@ -529,7 +531,8 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * if (childrens.size() == 1) { children[child_i - offset] = childrens[0]; } else { - children[child_i - offset] = make_adbc_column(env, childrens[0], item_type, item_values->null_count > 0, item_metadata, childrens[1]); + bool children_nullable = (item_schema->flags & ARROW_FLAG_NULLABLE) || (item_values->null_count > 0); + children[child_i - offset] = make_adbc_column(env, childrens[0], item_type, children_nullable, item_metadata, childrens[1]); } } return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); @@ -546,8 +549,9 @@ ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSchema * children[0] = childrens[0]; return enif_make_list_from_array(env, children.data(), (unsigned)children.size()); } else { + bool children_nullable = (schema->flags & ARROW_FLAG_NULLABLE) || (values->null_count > 0); ERL_NIF_TERM column[1]; - column[0] = make_adbc_column(env, childrens[0], children_type, values->null_count > 0, children_metadata, childrens[1]); + column[0] = make_adbc_column(env, childrens[0], children_type, children_nullable, children_metadata, childrens[1]); return enif_make_list_from_array(env, column, 1); } } diff --git a/test/adbc_connection_test.exs b/test/adbc_connection_test.exs index f5a5222..207ac50 100644 --- a/test/adbc_connection_test.exs +++ b/test/adbc_connection_test.exs @@ -58,7 +58,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "info_value", type: :dense_union, - nullable: false, + nullable: true, metadata: nil, data: [ %{"string_value" => ["SQLite"]}, @@ -92,7 +92,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "info_value", type: :dense_union, - nullable: false, + nullable: true, metadata: nil, data: [%{"string_value" => ["SQLite"]}] } @@ -123,31 +123,32 @@ defmodule Adbc.Connection.Test do {:ok, %Adbc.Result{ + num_rows: nil, data: [ %Adbc.Column{ name: "catalog_name", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "catalog_db_schemas", type: :list, - nullable: false, + nullable: true, metadata: nil, data: [ %Adbc.Column{ name: "db_schema_name", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "db_schema_tables", type: :list, - nullable: false, + nullable: true, metadata: nil, data: [ %Adbc.Column{ @@ -167,7 +168,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "table_columns", type: :list, - nullable: false, + nullable: true, metadata: nil, data: [ %Adbc.Column{ @@ -180,126 +181,126 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "ordinal_position", type: :i32, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "remarks", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_data_type", type: :i16, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_type_name", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_column_size", type: :i32, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_decimal_digits", type: :i16, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_num_prec_radix", type: :i16, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_nullable", type: :i16, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_column_def", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_sql_data_type", type: :i16, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_datetime_sub", type: :i16, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_char_octet_length", type: :i32, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_is_nullable", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_scope_catalog", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_scope_schema", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_scope_table", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_is_autoincrement", type: :boolean, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "xdbc_is_generatedcolumn", type: :boolean, - nullable: false, + nullable: true, metadata: nil, data: [] } @@ -308,13 +309,13 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "table_constraints", type: :list, - nullable: false, + nullable: true, metadata: nil, data: [ %Adbc.Column{ name: "constraint_name", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, @@ -343,20 +344,20 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "constraint_column_usage", type: :list, - nullable: false, + nullable: true, metadata: nil, data: [ %Adbc.Column{ name: "fk_catalog", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, %Adbc.Column{ name: "fk_db_schema", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [] }, @@ -417,7 +418,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [123] } @@ -431,14 +432,14 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [123] }, %Adbc.Column{ name: "bool", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [1] } @@ -455,7 +456,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [579] } @@ -480,7 +481,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [579] } @@ -500,7 +501,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [579] } @@ -514,7 +515,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [1456] } @@ -533,7 +534,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [123] } @@ -546,11 +547,11 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [123] }, - %Adbc.Column{name: "bool", type: :i64, nullable: false, metadata: nil, data: [1]} + %Adbc.Column{name: "bool", type: :i64, nullable: true, metadata: nil, data: [1]} ] } = Connection.query!(conn, "SELECT 123 as num, true as bool") @@ -564,7 +565,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [579] } @@ -591,11 +592,11 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [123] }, - %Adbc.Column{name: "bool", type: :i64, nullable: false, metadata: nil, data: [1]} + %Adbc.Column{name: "bool", type: :i64, nullable: true, metadata: nil, data: [1]} ] } == Connection.query!(conn, "SELECT 123 as num, true as bool", [], @@ -611,7 +612,7 @@ defmodule Adbc.Connection.Test do %Adbc.Column{ name: "num", type: :i64, - nullable: false, + nullable: true, metadata: nil, data: [579] } diff --git a/test/adbc_sqlite_test.exs b/test/adbc_sqlite_test.exs index 4f43e60..b94785f 100644 --- a/test/adbc_sqlite_test.exs +++ b/test/adbc_sqlite_test.exs @@ -80,58 +80,57 @@ defmodule Adbc.SQLite.Test do %Adbc.Result{ num_rows: nil, data: [ - %Adbc.Column{name: "i1", type: :i64, nullable: false, metadata: nil, data: [1]}, - %Adbc.Column{name: "i2", type: :i64, nullable: false, metadata: nil, data: [2]}, - %Adbc.Column{name: "i3", type: :i64, nullable: false, metadata: nil, data: [3]}, - %Adbc.Column{name: "i4", type: :i64, nullable: false, metadata: nil, data: [4]}, - %Adbc.Column{name: "i5", type: :i64, nullable: false, metadata: nil, data: [5]}, - %Adbc.Column{name: "i6", type: :i64, nullable: false, metadata: nil, data: [6]}, - %Adbc.Column{name: "i7", type: :i64, nullable: false, metadata: nil, data: ~c"\a"}, - %Adbc.Column{name: "i8", type: :i64, nullable: false, metadata: nil, data: ~c"\b"}, - %Adbc.Column{name: "i9", type: :i64, nullable: false, metadata: nil, data: ~c"\t"}, - %Adbc.Column{name: "t1", type: :string, nullable: false, metadata: nil, data: ["hello"]}, - %Adbc.Column{name: "t2", type: :string, nullable: false, metadata: nil, data: ["world"]}, + %Adbc.Column{name: "i1", type: :i64, nullable: true, metadata: nil, data: [1]}, + %Adbc.Column{name: "i2", type: :i64, nullable: true, metadata: nil, data: [2]}, + %Adbc.Column{name: "i3", type: :i64, nullable: true, metadata: nil, data: [3]}, + %Adbc.Column{name: "i4", type: :i64, nullable: true, metadata: nil, data: [4]}, + %Adbc.Column{name: "i5", type: :i64, nullable: true, metadata: nil, data: [5]}, + %Adbc.Column{name: "i6", type: :i64, nullable: true, metadata: nil, data: [6]}, + %Adbc.Column{name: "i7", type: :i64, nullable: true, metadata: nil, data: ~c"\a"}, + %Adbc.Column{name: "i8", type: :i64, nullable: true, metadata: nil, data: ~c"\b"}, + %Adbc.Column{name: "i9", type: :i64, nullable: true, metadata: nil, data: ~c"\t"}, + %Adbc.Column{name: "t1", type: :string, nullable: true, metadata: nil, data: ["hello"]}, + %Adbc.Column{name: "t2", type: :string, nullable: true, metadata: nil, data: ["world"]}, %Adbc.Column{ name: "t3", type: :string, - nullable: false, + nullable: true, metadata: nil, data: ["goodbye"] }, - %Adbc.Column{name: "t4", type: :string, nullable: false, metadata: nil, data: ["world"]}, - %Adbc.Column{name: "t5", type: :string, nullable: false, metadata: nil, data: ["foo"]}, - %Adbc.Column{name: "t6", type: :string, nullable: false, metadata: nil, data: ["bar"]}, + %Adbc.Column{name: "t4", type: :string, nullable: true, metadata: nil, data: ["world"]}, + %Adbc.Column{name: "t5", type: :string, nullable: true, metadata: nil, data: ["foo"]}, + %Adbc.Column{name: "t6", type: :string, nullable: true, metadata: nil, data: ["bar"]}, %Adbc.Column{ name: "b1", type: :string, - nullable: false, + nullable: true, metadata: nil, data: [<<100, 97, 116, 97, 1, 2>>] }, - %Adbc.Column{name: "r1", type: :f64, nullable: false, metadata: nil, data: [1.1]}, - %Adbc.Column{name: "r2", type: :f64, nullable: false, metadata: nil, data: [2.2]}, - %Adbc.Column{name: "r3", type: :f64, nullable: false, metadata: nil, data: [3.3]}, - %Adbc.Column{name: "r4", type: :f64, nullable: false, metadata: nil, data: [4.4]}, - %Adbc.Column{name: "n1", type: :f64, nullable: false, metadata: nil, data: [1.1]}, - %Adbc.Column{name: "n2", type: :f64, nullable: false, metadata: nil, data: [2.2]}, - %Adbc.Column{name: "n3", type: :i64, nullable: false, metadata: nil, data: [1]}, + %Adbc.Column{name: "r1", type: :f64, nullable: true, metadata: nil, data: [1.1]}, + %Adbc.Column{name: "r2", type: :f64, nullable: true, metadata: nil, data: [2.2]}, + %Adbc.Column{name: "r3", type: :f64, nullable: true, metadata: nil, data: [3.3]}, + %Adbc.Column{name: "r4", type: :f64, nullable: true, metadata: nil, data: [4.4]}, + %Adbc.Column{name: "n1", type: :f64, nullable: true, metadata: nil, data: [1.1]}, + %Adbc.Column{name: "n2", type: :f64, nullable: true, metadata: nil, data: [2.2]}, + %Adbc.Column{name: "n3", type: :i64, nullable: true, metadata: nil, data: [1]}, %Adbc.Column{ name: "n4", type: :string, - nullable: false, + nullable: true, metadata: nil, data: ["2021-01-01"] }, %Adbc.Column{ name: "n5", type: :string, - nullable: false, + nullable: true, metadata: nil, data: ["2021-01-01 00:00:00"] } ] - }} = - Adbc.Connection.query(conn, "SELECT * FROM test") + }} = Adbc.Connection.query(conn, "SELECT * FROM test") end test "insert with Adbc.Buffer", %{db: _, conn: conn} do @@ -174,64 +173,64 @@ defmodule Adbc.SQLite.Test do %Adbc.Result{ num_rows: nil, data: [ - %Adbc.Column{name: "i1", type: :i64, nullable: false, metadata: nil, data: [1]}, - %Adbc.Column{name: "i2", type: :i64, nullable: false, metadata: nil, data: [2]}, - %Adbc.Column{name: "i3", type: :i64, nullable: false, metadata: nil, data: [3]}, - %Adbc.Column{name: "i4", type: :i64, nullable: false, metadata: nil, data: [4]}, - %Adbc.Column{name: "i5", type: :i64, nullable: false, metadata: nil, data: [5]}, - %Adbc.Column{name: "i6", type: :i64, nullable: false, metadata: nil, data: [6]}, - %Adbc.Column{name: "i7", type: :i64, nullable: false, metadata: nil, data: ~c"\a"}, - %Adbc.Column{name: "i8", type: :i64, nullable: false, metadata: nil, data: ~c"\b"}, - %Adbc.Column{name: "i9", type: :i64, nullable: false, metadata: nil, data: ~c"\t"}, - %Adbc.Column{name: "t1", type: :string, nullable: false, metadata: nil, data: ["hello"]}, - %Adbc.Column{name: "t2", type: :string, nullable: false, metadata: nil, data: ["world"]}, + %Adbc.Column{name: "i1", type: :i64, nullable: true, metadata: nil, data: [1]}, + %Adbc.Column{name: "i2", type: :i64, nullable: true, metadata: nil, data: [2]}, + %Adbc.Column{name: "i3", type: :i64, nullable: true, metadata: nil, data: [3]}, + %Adbc.Column{name: "i4", type: :i64, nullable: true, metadata: nil, data: [4]}, + %Adbc.Column{name: "i5", type: :i64, nullable: true, metadata: nil, data: [5]}, + %Adbc.Column{name: "i6", type: :i64, nullable: true, metadata: nil, data: [6]}, + %Adbc.Column{name: "i7", type: :i64, nullable: true, metadata: nil, data: ~c"\a"}, + %Adbc.Column{name: "i8", type: :i64, nullable: true, metadata: nil, data: ~c"\b"}, + %Adbc.Column{name: "i9", type: :i64, nullable: true, metadata: nil, data: ~c"\t"}, + %Adbc.Column{name: "t1", type: :string, nullable: true, metadata: nil, data: ["hello"]}, + %Adbc.Column{name: "t2", type: :string, nullable: true, metadata: nil, data: ["world"]}, %Adbc.Column{ name: "t3", type: :string, - nullable: false, + nullable: true, metadata: nil, data: ["goodbye"] }, - %Adbc.Column{name: "t4", type: :string, nullable: false, metadata: nil, data: ["world"]}, - %Adbc.Column{name: "t5", type: :string, nullable: false, metadata: nil, data: ["foo"]}, - %Adbc.Column{name: "t6", type: :string, nullable: false, metadata: nil, data: ["bar"]}, + %Adbc.Column{name: "t4", type: :string, nullable: true, metadata: nil, data: ["world"]}, + %Adbc.Column{name: "t5", type: :string, nullable: true, metadata: nil, data: ["foo"]}, + %Adbc.Column{name: "t6", type: :string, nullable: true, metadata: nil, data: ["bar"]}, %Adbc.Column{ name: "b1", type: :binary, - nullable: false, + nullable: true, metadata: nil, data: [<<100, 97, 116, 97, 1, 2>>] }, %Adbc.Column{ name: "r1", type: :f64, - nullable: false, + nullable: true, metadata: nil, data: [r1] }, - %Adbc.Column{name: "r2", type: :f64, nullable: false, metadata: nil, data: [2.2]}, + %Adbc.Column{name: "r2", type: :f64, nullable: true, metadata: nil, data: [2.2]}, %Adbc.Column{ name: "r3", type: :f64, - nullable: false, + nullable: true, metadata: nil, data: [r3] }, - %Adbc.Column{name: "r4", type: :f64, nullable: false, metadata: nil, data: [4.4]}, - %Adbc.Column{name: "n1", type: :f64, nullable: false, metadata: nil, data: [1.1]}, - %Adbc.Column{name: "n2", type: :f64, nullable: false, metadata: nil, data: [2.2]}, - %Adbc.Column{name: "n3", type: :i64, nullable: false, metadata: nil, data: [1]}, + %Adbc.Column{name: "r4", type: :f64, nullable: true, metadata: nil, data: [4.4]}, + %Adbc.Column{name: "n1", type: :f64, nullable: true, metadata: nil, data: [1.1]}, + %Adbc.Column{name: "n2", type: :f64, nullable: true, metadata: nil, data: [2.2]}, + %Adbc.Column{name: "n3", type: :i64, nullable: true, metadata: nil, data: [1]}, %Adbc.Column{ name: "n4", type: :string, - nullable: false, + nullable: true, metadata: nil, data: ["2021-01-01"] }, %Adbc.Column{ name: "n5", type: :string, - nullable: false, + nullable: true, metadata: nil, data: ["2021-01-01 00:00:00"] } diff --git a/test/adbc_test.exs b/test/adbc_test.exs index 8799316..3d2c3c8 100644 --- a/test/adbc_test.exs +++ b/test/adbc_test.exs @@ -38,7 +38,7 @@ defmodule AdbcTest do %Adbc.Column{ name: "num", type: :i32, - nullable: false, + nullable: true, metadata: nil, data: [123] } @@ -54,13 +54,13 @@ defmodule AdbcTest do %Adbc.Column{ name: "num", type: :list, - nullable: false, + nullable: true, metadata: nil, data: [ %Adbc.Column{ name: "item", type: :i32, - nullable: false, + nullable: true, metadata: nil, data: [1, 2, 3] } @@ -78,13 +78,13 @@ defmodule AdbcTest do %Adbc.Column{ name: "num", type: :list, - nullable: false, + nullable: true, metadata: nil, data: [ %Adbc.Column{ name: "item", type: :i32, - nullable: false, + nullable: true, metadata: nil, data: [1, 2, 3, nil, 5] } @@ -118,13 +118,13 @@ defmodule AdbcTest do %Adbc.Column{ name: "num", type: :list, - nullable: false, + nullable: true, metadata: nil, data: [ %Adbc.Column{ name: "item", type: :i32, - nullable: false, + nullable: true, metadata: nil, data: [1, 2, 3, nil, 5, 6, nil, 7, nil, 9] } @@ -148,7 +148,7 @@ defmodule AdbcTest do %Adbc.Column{ name: "generate_series", type: :timestamp, - nullable: false, + nullable: true, metadata: nil, data: generate_series } @@ -175,35 +175,35 @@ defmodule AdbcTest do %Adbc.Column{ name: "datetime", type: :timestamp, - nullable: false, + nullable: true, metadata: nil, data: [~N[2023-03-01 10:23:45.000000]] }, %Adbc.Column{ name: "datetime_usec", type: :timestamp, - nullable: false, + nullable: true, metadata: nil, data: [~N[2023-03-01 10:23:45.123456]] }, %Adbc.Column{ name: "date", type: :date32, - nullable: false, + nullable: true, metadata: nil, data: [~D[2023-03-01]] }, %Adbc.Column{ name: "time", type: :time64, - nullable: false, + nullable: true, metadata: nil, data: [~T[10:23:45.000000]] }, %Adbc.Column{ name: "time_usec", type: :time64, - nullable: false, + nullable: true, metadata: nil, data: [~T[10:23:45.123456]] } From ccd78afc674f81f8539562158c7933bc294ea5d8 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Thu, 9 May 2024 00:06:06 +0800 Subject: [PATCH 06/14] implemented `Adbc.Result.to_map/1` and added some tests for it --- lib/adbc_result.ex | 37 +++++++++++++++++++++++ test/adbc_connection_test.exs | 57 +++++++++++++++++++++++++++++++++-- test/adbc_sqlite_test.exs | 33 ++++++++++++++++++-- 3 files changed, 123 insertions(+), 4 deletions(-) diff --git a/lib/adbc_result.ex b/lib/adbc_result.ex index 9eb8a9f..09ff0a7 100644 --- a/lib/adbc_result.ex +++ b/lib/adbc_result.ex @@ -15,4 +15,41 @@ defmodule Adbc.Result do num_rows: non_neg_integer() | nil, data: [%Adbc.Column{}] } + def to_map(r = %Adbc.Result{data: data}) do + %Adbc.Result{ + r + | data: + Map.new( + Enum.map(data, fn %Adbc.Column{name: name, type: type, data: data} -> + case type do + :list -> + {name, Enum.map(data, &list_to_map/1)} + + _ -> + {name, data} + end + end) + ) + } + end + + defp list_to_map(%Adbc.Column{name: name, type: type, data: data}) do + case type do + :list -> + list = Enum.map(data, &list_to_map/1) + + if name == "item" do + list + else + {name, list} + end + + _ -> + if name == "item" do + data + else + {name, data} + end + end + end end diff --git a/test/adbc_connection_test.exs b/test/adbc_connection_test.exs index 207ac50..2a3aacb 100644 --- a/test/adbc_connection_test.exs +++ b/test/adbc_connection_test.exs @@ -122,7 +122,7 @@ defmodule Adbc.Connection.Test do conn = start_supervised!({Connection, database: db}) {:ok, - %Adbc.Result{ + results = %Adbc.Result{ num_rows: nil, data: [ %Adbc.Column{ @@ -385,6 +385,56 @@ defmodule Adbc.Connection.Test do } ] }} = Connection.get_objects(conn, 0) + + assert %Adbc.Result{ + num_rows: nil, + data: %{ + "catalog_db_schemas" => [ + {"db_schema_name", []}, + {"db_schema_tables", + [ + {"table_name", []}, + {"table_type", []}, + {"table_columns", + [ + {"column_name", []}, + {"ordinal_position", []}, + {"remarks", []}, + {"xdbc_data_type", []}, + {"xdbc_type_name", []}, + {"xdbc_column_size", []}, + {"xdbc_decimal_digits", []}, + {"xdbc_num_prec_radix", []}, + {"xdbc_nullable", []}, + {"xdbc_column_def", []}, + {"xdbc_sql_data_type", []}, + {"xdbc_datetime_sub", []}, + {"xdbc_char_octet_length", []}, + {"xdbc_is_nullable", []}, + {"xdbc_scope_catalog", []}, + {"xdbc_scope_schema", []}, + {"xdbc_scope_table", []}, + {"xdbc_is_autoincrement", []}, + {"xdbc_is_generatedcolumn", []} + ]}, + {"table_constraints", + [ + {"constraint_name", []}, + {"constraint_type", []}, + {"constraint_column_names", [[]]}, + {"constraint_column_usage", + [ + {"fk_catalog", []}, + {"fk_db_schema", []}, + {"fk_table", []}, + {"fk_column_name", []} + ]} + ]} + ]} + ], + "catalog_name" => [] + } + } = Adbc.Result.to_map(results) end end @@ -393,7 +443,7 @@ defmodule Adbc.Connection.Test do conn = start_supervised!({Connection, database: db}) assert {:ok, - %Adbc.Result{ + result = %Adbc.Result{ data: [ %Adbc.Column{ name: "table_type", @@ -405,6 +455,9 @@ defmodule Adbc.Connection.Test do ] }} = Connection.get_table_types(conn) + + assert %Adbc.Result{num_rows: nil, data: %{"table_type" => ["table", "view"]}} = + Adbc.Result.to_map(result) end end diff --git a/test/adbc_sqlite_test.exs b/test/adbc_sqlite_test.exs index b94785f..54fb9a8 100644 --- a/test/adbc_sqlite_test.exs +++ b/test/adbc_sqlite_test.exs @@ -77,8 +77,7 @@ defmodule Adbc.SQLite.Test do ) {:ok, - %Adbc.Result{ - num_rows: nil, + list_result = %Adbc.Result{ data: [ %Adbc.Column{name: "i1", type: :i64, nullable: true, metadata: nil, data: [1]}, %Adbc.Column{name: "i2", type: :i64, nullable: true, metadata: nil, data: [2]}, @@ -131,6 +130,36 @@ defmodule Adbc.SQLite.Test do } ] }} = Adbc.Connection.query(conn, "SELECT * FROM test") + + %Adbc.Result{ + data: %{ + "b1" => [<<100, 97, 116, 97, 1, 2>>], + "i1" => [1], + "i2" => [2], + "i3" => [3], + "i4" => [4], + "i5" => [5], + "i6" => [6], + "i7" => ~c"\a", + "i8" => ~c"\b", + "i9" => ~c"\t", + "n1" => [1.1], + "n2" => [2.2], + "n3" => [1], + "n4" => ["2021-01-01"], + "n5" => ["2021-01-01 00:00:00"], + "r1" => [1.1], + "r2" => [2.2], + "r3" => [3.3], + "r4" => [4.4], + "t1" => ["hello"], + "t2" => ["world"], + "t3" => ["goodbye"], + "t4" => ["world"], + "t5" => ["foo"], + "t6" => ["bar"] + } + } = Adbc.Result.to_map(list_result) end test "insert with Adbc.Buffer", %{db: _, conn: conn} do From 2f1c71b6a795b4876cc56be5eedcecbf7e6becb9 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Thu, 9 May 2024 00:23:06 +0800 Subject: [PATCH 07/14] updated inline docs for some functions in `Adbc.Column` --- lib/adbc_column.ex | 420 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 400 insertions(+), 20 deletions(-) diff --git a/lib/adbc_column.ex b/lib/adbc_column.ex index 88a39dc..223018d 100644 --- a/lib/adbc_column.ex +++ b/lib/adbc_column.ex @@ -66,32 +66,49 @@ defmodule Adbc.Column do %Adbc.Column{buffer | metadata: %{}} end + @doc """ + A column that contains booleans. + + ## Arguments + + * `data`: A list of booleans + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.boolean([true, false, true]) + %Adbc.Column{ + name: nil, + type: :boolean, + nullable: false, + metadata: %{}, + data: [true, false, true] + } + + """ @spec boolean([boolean()], Keyword.t()) :: %Adbc.Column{} def boolean(data, opts \\ []) when is_list(data) and is_list(opts) do column(:boolean, data, opts) end @doc """ - A buffer for an unsigned 8-bit integer column. + A column that contains unsigned 8-bit integers. ## Arguments - * `data`: - * A list of unsigned 8-bit integer values - * A single binary type value where each byte represents an unsigned 8-bit integer - * `opts` - A keyword list of options + * `data`: A list of unsigned 8-bit integer values + * `opts`: A keyword list of options ## Options * `:name` - The name of the column * `:nullable` - A boolean value indicating whether the column is nullable - * `:null_count` - The number of null values in the column, defaults to 0, only used when - `:nullable` is `true` - * `:null`: only used when `:nullable` is `true`, can be one of the following: - * A list of booleans with the same length as indicating whether each value is null - * A list of non-negative integers where each integer represents the corresponding index of a - null value - * A single binary type value where each bit indicates whether each value is null * `:metadata` - A map of metadata ## Examples @@ -101,89 +118,452 @@ defmodule Adbc.Column do name: nil, type: :u8, nullable: false, - null_count: 0, - null: nil, metadata: %{}, data: [1, 2, 3] } + """ @spec u8([0..255] | binary(), Keyword.t()) :: %Adbc.Column{} - def u8(u8, opts \\ []) - - def u8(data, opts) when is_list(data) and is_list(opts) do + def u8(data, opts \\ []) when is_list(data) and is_list(opts) do column(:u8, data, opts) end - def u8(data, opts) when is_binary(data) and is_list(opts) do - column(:u8, data, opts) - end + @doc """ + A column that contains unsigned 16-bit integers. + + ## Arguments + + * `data`: A list of unsigned 16-bit integer values + * `opts`: A keyword list of options + + ## Options + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.u16([1, 2, 3]) + %Adbc.Column{ + name: nil, + type: :u16, + nullable: false, + metadata: %{}, + data: [1, 2, 3] + } + + """ @spec u16([0..65535], Keyword.t()) :: %Adbc.Column{} def u16(data, opts \\ []) when is_list(data) and is_list(opts) do column(:u16, data, opts) end + @doc """ + A column that contains unsigned 32-bit integers. + + ## Arguments + + * `data`: A list of unsigned 32-bit integer values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.u32([1, 2, 3]) + %Adbc.Column{ + name: nil, + type: :u32, + nullable: false, + metadata: %{}, + data: [1, 2, 3] + } + + """ @spec u32([0..4_294_967_295], Keyword.t()) :: %Adbc.Column{} def u32(data, opts \\ []) when is_list(data) and is_list(opts) do column(:u32, data, opts) end + @doc """ + A column that contains unsigned 64-bit integers. + + ## Arguments + + * `data`: A list of unsigned 64-bit integer values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.u32([1, 2, 3]) + %Adbc.Column{ + name: nil, + type: :u32, + nullable: false, + metadata: %{}, + data: [1, 2, 3] + } + + """ @spec u64([0..18_446_744_073_709_551_615], Keyword.t()) :: %Adbc.Column{} def u64(data, opts \\ []) when is_list(data) and is_list(opts) do column(:u64, data, opts) end + @doc """ + A column that contains signed 8-bit integers. + + ## Arguments + + * `data`: A list of signed 8-bit integer values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.i8([1, 2, 3]) + %Adbc.Column{ + name: nil, + type: :i8, + nullable: false, + metadata: %{}, + data: [1, 2, 3] + } + + """ @spec i8([-128..127], Keyword.t()) :: %Adbc.Column{} def i8(data, opts \\ []) when is_list(data) and is_list(opts) do column(:i8, data, opts) end + @doc """ + A column that contains signed 16-bit integers. + + ## Arguments + + * `data`: A list of signed 16-bit integer values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.i16([1, 2, 3]) + %Adbc.Column{ + name: nil, + type: :i16, + nullable: false, + metadata: %{}, + data: [1, 2, 3] + } + + """ @spec i16([-32768..32767], Keyword.t()) :: %Adbc.Column{} def i16(data, opts \\ []) when is_list(data) and is_list(opts) do column(:i16, data, opts) end + @doc """ + A column that contains signed 32-bit integers. + + ## Arguments + + * `data`: A list of signed 32-bit integer values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.i32([1, 2, 3]) + %Adbc.Column{ + name: nil, + type: :i32, + nullable: false, + metadata: %{}, + data: [1, 2, 3] + } + + """ @spec i32([-2_147_483_648..2_147_483_647], Keyword.t()) :: %Adbc.Column{} def i32(data, opts \\ []) when is_list(data) and is_list(opts) do column(:i32, data, opts) end + @doc """ + A column that contains signed 64-bit integers. + + ## Arguments + + * `data`: A list of signed 64-bit integer values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.i64([1, 2, 3]) + %Adbc.Column{ + name: nil, + type: :i64, + nullable: false, + metadata: %{}, + data: [1, 2, 3] + } + + """ @spec i64([-9_223_372_036_854_775_808..9_223_372_036_854_775_807], Keyword.t()) :: %Adbc.Column{} def i64(data, opts \\ []) when is_list(data) and is_list(opts) do column(:i64, data, opts) end + @doc """ + A column that contains 32-bit single-precision floats. + + ## Arguments + + * `data`: A list of 32-bit single-precision float values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.f32([1.0, 2.0, 3.0]) + %Adbc.Column{ + name: nil, + type: :f32, + nullable: false, + metadata: %{}, + data: [1.0, 2.0, 3.0] + } + + """ @spec f32([float], Keyword.t()) :: %Adbc.Column{} def f32(data, opts \\ []) when is_list(data) and is_list(opts) do column(:f32, data, opts) end + @doc """ + A column that contains 64-bit double-precision floats. + + ## Arguments + + * `data`: A list of 64-bit double-precision float values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.f64([1.0, 2.0, 3.0]) + %Adbc.Column{ + name: nil, + type: :f64, + nullable: false, + metadata: %{}, + data: [1.0, 2.0, 3.0] + } + + """ @spec f64([float], Keyword.t()) :: %Adbc.Column{} def f64(data, opts \\ []) when is_list(data) and is_list(opts) do column(:f64, data, opts) end + @doc """ + A column that contains UTF-8 encoded strings. + + ## Arguments + + * `data`: A list of UTF-8 encoded string values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.string(["a", "ab", "abc"]) + %Adbc.Column{ + name: nil, + type: :string, + nullable: false, + metadata: %{}, + data: ["a", "ab", "abc"] + } + + """ @spec string([String.t()], Keyword.t()) :: %Adbc.Column{} def string(data, opts \\ []) when is_list(data) and is_list(opts) do column(:string, data, opts) end + @doc """ + A column that contains UTF-8 encoded large strings. + + Similar to `string/2`, but for strings larger than 2GB. + + ## Arguments + + * `data`: A list of UTF-8 encoded string values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.large_string(["a", "ab", "abc"]) + %Adbc.Column{ + name: nil, + type: :large_string, + nullable: false, + metadata: %{}, + data: ["a", "ab", "abc"] + } + + """ @spec large_string([String.t()], Keyword.t()) :: %Adbc.Column{} def large_string(data, opts \\ []) when is_list(data) and is_list(opts) do column(:large_string, data, opts) end + @doc """ + A column that contains binary values. + + ## Arguments + + * `data`: A list of binary values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.binary([<<0>>, <<1>>, <<2>>]) + %Adbc.Column{ + name: nil, + type: :binary, + nullable: false, + metadata: %{}, + data: [<<0>>, <<1>>, <<2>>] + } + + """ @spec binary([binary()], Keyword.t()) :: %Adbc.Column{} def binary(data, opts \\ []) when is_list(data) and is_list(opts) do column(:binary, data, opts) end + @doc """ + A column that contains large binary values. + + Similar to `binary/2`, but for binary values larger than 2GB. + + ## Arguments + + * `data`: A list of binary values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.large_binary([<<0>>, <<1>>, <<2>>]) + %Adbc.Column{ + name: nil, + type: :large_binary, + nullable: false, + metadata: %{}, + data: [<<0>>, <<1>>, <<2>>] + } + + """ @spec large_binary([binary()], Keyword.t()) :: %Adbc.Column{} def large_binary(data, opts \\ []) when is_list(data) and is_list(opts) do column(:large_binary, data, opts) end + @doc """ + A column that contains fixed size binaries. + + Similar to `binary/2`, but each binary value has the same fixed size in bytes. + + ## Arguments + + * `data`: A list of binary values + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + + ## Examples + + iex> Adbc.Buffer.fixed_size_binary([<<0>>, <<1>>, <<2>>]) + %Adbc.Column{ + name: nil, + type: :fixed_size_binary, + nullable: false, + metadata: %{}, + data: [<<0>>, <<1>>, <<2>>] + } + + """ @spec fixed_size_binary([binary()], Keyword.t()) :: %Adbc.Column{} def fixed_size_binary(data, opts \\ []) when is_list(data) and is_list(opts) do column(:fixed_size_binary, data, opts) From 5da6c8601cf4d53f592417edefa665ee23b53f7a Mon Sep 17 00:00:00 2001 From: Cocoa Date: Thu, 9 May 2024 12:01:10 +0800 Subject: [PATCH 08/14] mix format --- test/adbc_connection_test.exs | 95 +++++++++++++++++------------------ 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/test/adbc_connection_test.exs b/test/adbc_connection_test.exs index 2a3aacb..a5f7596 100644 --- a/test/adbc_connection_test.exs +++ b/test/adbc_connection_test.exs @@ -387,54 +387,53 @@ defmodule Adbc.Connection.Test do }} = Connection.get_objects(conn, 0) assert %Adbc.Result{ - num_rows: nil, - data: %{ - "catalog_db_schemas" => [ - {"db_schema_name", []}, - {"db_schema_tables", - [ - {"table_name", []}, - {"table_type", []}, - {"table_columns", - [ - {"column_name", []}, - {"ordinal_position", []}, - {"remarks", []}, - {"xdbc_data_type", []}, - {"xdbc_type_name", []}, - {"xdbc_column_size", []}, - {"xdbc_decimal_digits", []}, - {"xdbc_num_prec_radix", []}, - {"xdbc_nullable", []}, - {"xdbc_column_def", []}, - {"xdbc_sql_data_type", []}, - {"xdbc_datetime_sub", []}, - {"xdbc_char_octet_length", []}, - {"xdbc_is_nullable", []}, - {"xdbc_scope_catalog", []}, - {"xdbc_scope_schema", []}, - {"xdbc_scope_table", []}, - {"xdbc_is_autoincrement", []}, - {"xdbc_is_generatedcolumn", []} - ]}, - {"table_constraints", - [ - {"constraint_name", []}, - {"constraint_type", []}, - {"constraint_column_names", [[]]}, - {"constraint_column_usage", - [ - {"fk_catalog", []}, - {"fk_db_schema", []}, - {"fk_table", []}, - {"fk_column_name", []} - ]} - ]} - ]} - ], - "catalog_name" => [] - } - } = Adbc.Result.to_map(results) + data: %{ + "catalog_db_schemas" => [ + {"db_schema_name", []}, + {"db_schema_tables", + [ + {"table_name", []}, + {"table_type", []}, + {"table_columns", + [ + {"column_name", []}, + {"ordinal_position", []}, + {"remarks", []}, + {"xdbc_data_type", []}, + {"xdbc_type_name", []}, + {"xdbc_column_size", []}, + {"xdbc_decimal_digits", []}, + {"xdbc_num_prec_radix", []}, + {"xdbc_nullable", []}, + {"xdbc_column_def", []}, + {"xdbc_sql_data_type", []}, + {"xdbc_datetime_sub", []}, + {"xdbc_char_octet_length", []}, + {"xdbc_is_nullable", []}, + {"xdbc_scope_catalog", []}, + {"xdbc_scope_schema", []}, + {"xdbc_scope_table", []}, + {"xdbc_is_autoincrement", []}, + {"xdbc_is_generatedcolumn", []} + ]}, + {"table_constraints", + [ + {"constraint_name", []}, + {"constraint_type", []}, + {"constraint_column_names", [[]]}, + {"constraint_column_usage", + [ + {"fk_catalog", []}, + {"fk_db_schema", []}, + {"fk_table", []}, + {"fk_column_name", []} + ]} + ]} + ]} + ], + "catalog_name" => [] + } + } = Adbc.Result.to_map(results) end end From 20d6943bdc529d65a0f55a7cc8936ae44ccfb5db Mon Sep 17 00:00:00 2001 From: Cocoa Date: Thu, 9 May 2024 15:04:46 +0800 Subject: [PATCH 09/14] use `Enum.zip_with/2` --- lib/adbc_connection.ex | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/adbc_connection.ex b/lib/adbc_connection.ex index 75bce27..e3cbf62 100644 --- a/lib/adbc_connection.ex +++ b/lib/adbc_connection.ex @@ -397,28 +397,27 @@ defmodule Adbc.Connection do defp stream_results(reference, acc, num_rows) do case Adbc.Nif.adbc_arrow_array_stream_next(reference) do {:ok, results, _done} -> - stream_results(reference, acc ++ results, num_rows) + stream_results(reference, [results | acc], num_rows) :end_of_series -> - {:ok, %Adbc.Result{data: merge_columns(acc), num_rows: num_rows}} + {:ok, %Adbc.Result{data: merge_columns(Enum.reverse(acc)), num_rows: num_rows}} {:error, reason} -> {:error, error_to_exception(reason)} end end - defp merge_columns(columns) do - Enum.reduce(columns, [], fn column, merged_columns -> - case Enum.find_index(merged_columns, &(&1.name == column.name)) do - nil -> - merged_columns ++ [column] - - index -> - merged_column = Enum.at(merged_columns, index) - merged_column = %{merged_column | data: merged_column.data ++ column.data} - List.replace_at(merged_columns, index, merged_column) - end - end) + defp merge_columns(chucked_results) do + if Enum.count(chucked_results) == 1 do + [chucked_results] = chucked_results + chucked_results + else + Enum.zip_with(chucked_results, fn columns -> + Enum.reduce(columns, fn column, merged_column -> + %{merged_column | data: merged_column.data ++ column.data} + end) + end) + end end ## Callbacks From 7c63d7c3454b6ac1de0d6f038cd7a789e1768399 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Thu, 9 May 2024 15:08:32 +0800 Subject: [PATCH 10/14] added support for `date{32,64}` when binding parameters --- c_src/adbc_nif.cpp | 116 ++++++++++++++++++++++++++++++++++++++++++++- lib/adbc_column.ex | 12 +++++ 2 files changed, 126 insertions(+), 2 deletions(-) diff --git a/c_src/adbc_nif.cpp b/c_src/adbc_nif.cpp index c509d28..da72b7e 100644 --- a/c_src/adbc_nif.cpp +++ b/c_src/adbc_nif.cpp @@ -83,6 +83,7 @@ constexpr int kErrorBufferDataIsNotAList = 5; constexpr int kErrorBufferUnknownType = 6; constexpr int kErrorBufferGetMetadataKey = 7; constexpr int kErrorBufferGetMetadataValue = 8; +constexpr int kErrorExpectedCalendarISO = 9; static ERL_NIF_TERM nif_error_from_adbc_error(ErlNifEnv *env, struct AdbcError * adbc_error) { char const* message = (adbc_error->message == nullptr) ? "unknown error" : adbc_error->message; @@ -1699,9 +1700,7 @@ static ERL_NIF_TERM adbc_arrow_array_stream_next(ErlNifEnv *env, int argc, const } return kAtomEndOfSeries; } - // ret = make_adbc_column(env, schema->name == nullptr ? "" : schema->name, out_type, out.null_count > 0, kAtomNil, out_terms[0]); } else { - // ret = make_adbc_column(env, out_terms[0], out_type, out.null_count > 0, kAtomNil, out_terms[1]); ret = enif_make_tuple2(env, out_terms[0], out_terms[1]); } @@ -2056,6 +2055,112 @@ int do_get_list_fixed_size_binary(ErlNifEnv *env, ERL_NIF_TERM list, bool nullab } } +int get_utc_offset() { + time_t zero = 24*60*60L; + struct tm * timeptr; + int gmtime_hours; + timeptr = localtime( &zero ); + gmtime_hours = timeptr->tm_hour; + if( timeptr->tm_mday < 2 ) { + gmtime_hours -= 24; + } + return gmtime_hours; +} + +int get_list_date(ErlNifEnv *env, ERL_NIF_TERM list, bool nullable, const std::function &callback) { + ERL_NIF_TERM head, tail; + tail = list; + while (enif_get_list_cell(env, tail, &head, &tail)) { + if (enif_is_identical(head, kAtomNil)) { + callback(0, true); + } else { + int64_t val; + if (erlang::nif::get(env, head, &val)) { + callback(val, false); + } else if (enif_is_map(env, head)) { + ERL_NIF_TERM struct_name_term, calendar_term, year_term, month_term, day_term; + if (!enif_get_map_value(env, head, kAtomStructKey, &struct_name_term)) { + return kErrorBufferGetMapValue; + } + if (!enif_is_identical(struct_name_term, kAtomDateModule)) { + return kErrorBufferWrongStruct; + } + + if (!enif_get_map_value(env, head, kAtomCalendarKey, &calendar_term)) { + return kErrorBufferGetMapValue; + } + if (!enif_is_identical(calendar_term, kAtomCalendarISO)) { + return kErrorExpectedCalendarISO; + } + + if (!enif_get_map_value(env, head, kAtomYearKey, &year_term)) { + return kErrorBufferGetMapValue; + } + if (!enif_get_map_value(env, head, kAtomMonthKey, &month_term)) { + return kErrorBufferGetMapValue; + } + if (!enif_get_map_value(env, head, kAtomDayKey, &day_term)) { + return kErrorBufferGetMapValue; + } + + tm time{}; + if (!erlang::nif::get(env, year_term, &time.tm_year) || !erlang::nif::get(env, month_term, &time.tm_mon) || !erlang::nif::get(env, day_term, &time.tm_mday)) { + return kErrorBufferGetMapValue; + } + time.tm_year -= 1900; + time.tm_mon -= 1; + // mktime always gives local time + // so we need to adjust it to UTC + val = mktime(&time) + get_utc_offset() * 3600; + callback(val, false); + } else { + return 1; + } + } + } + return 0; +} + +int do_get_list_date32(ErlNifEnv *env, ERL_NIF_TERM list, bool nullable, ArrowType nanoarrow_type, struct ArrowArray* array_out, struct ArrowSchema* schema_out, struct ArrowError* error_out) { + NANOARROW_RETURN_NOT_OK(ArrowSchemaSetType(schema_out, nanoarrow_type)); + NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromSchema(array_out, schema_out, error_out)); + NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(array_out)); + if (nullable) { + return get_list_date(env, list, nullable, [&array_out](int64_t val, bool is_nil) -> void { + val /= 24 * 60 * 60; + ArrowArrayAppendInt(array_out, (int32_t)val); + if (is_nil) { + ArrowArrayAppendNull(array_out, 1); + } + }); + } else { + return get_list_date(env, list, nullable, [&array_out](int64_t val, bool) -> void { + val /= 24 * 60 * 60; + ArrowArrayAppendInt(array_out, (int32_t)val); + }); + } +} + +int do_get_list_date64(ErlNifEnv *env, ERL_NIF_TERM list, bool nullable, ArrowType nanoarrow_type, struct ArrowArray* array_out, struct ArrowSchema* schema_out, struct ArrowError* error_out) { + NANOARROW_RETURN_NOT_OK(ArrowSchemaSetType(schema_out, nanoarrow_type)); + NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromSchema(array_out, schema_out, error_out)); + NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(array_out)); + if (nullable) { + return get_list_date(env, list, nullable, [&array_out](int64_t val, bool is_nil) -> void { + val *= 1000; + ArrowArrayAppendInt(array_out, val); + if (is_nil) { + ArrowArrayAppendNull(array_out, 1); + } + }); + } else { + return get_list_date(env, list, nullable, [&array_out](int64_t val, bool) -> void { + val *= 1000; + ArrowArrayAppendInt(array_out, val); + }); + } +} + // non-zero return value indicating errors int adbc_buffer_to_adbc_field(ErlNifEnv *env, ERL_NIF_TERM adbc_buffer, struct ArrowArray* array_out, struct ArrowSchema* schema_out, struct ArrowError* error_out) { array_out->release = NULL; @@ -2175,6 +2280,10 @@ int adbc_buffer_to_adbc_field(ErlNifEnv *env, ERL_NIF_TERM adbc_buffer, struct A do_get_list_string(env, data_term, nullable, NANOARROW_TYPE_LARGE_BINARY, array_out, schema_out, error_out); } else if (enif_is_identical(type_term, kAdbcColumnTypeFixedSizeBinary)) { do_get_list_fixed_size_binary(env, data_term, nullable, NANOARROW_TYPE_FIXED_SIZE_BINARY, array_out, schema_out, error_out); + } else if (enif_is_identical(type_term, kAdbcColumnTypeDate32)) { + do_get_list_date32(env, data_term, nullable, NANOARROW_TYPE_DATE32, array_out, schema_out, error_out); + } else if (enif_is_identical(type_term, kAdbcColumnTypeDate64)) { + do_get_list_date64(env, data_term, nullable, NANOARROW_TYPE_DATE64, array_out, schema_out, error_out); } else if (enif_is_identical(type_term, kAdbcColumnTypeBool)) { do_get_list_boolean(env, data_term, nullable, NANOARROW_TYPE_BOOL, array_out, schema_out, error_out); } else { @@ -2286,6 +2395,9 @@ int adbc_buffer_to_arrow_type_struct(ErlNifEnv *env, ERL_NIF_TERM values, struct case kErrorBufferGetMetadataValue: // error message is already set return 1; + case kErrorExpectedCalendarISO: + snprintf(error_out->message, sizeof(error_out->message), "Expected `Calendar.ISO`."); + return 1; default: break; } diff --git a/lib/adbc_column.ex b/lib/adbc_column.ex index 223018d..f519e6b 100644 --- a/lib/adbc_column.ex +++ b/lib/adbc_column.ex @@ -21,6 +21,8 @@ defmodule Adbc.Column do * `:binary` * `:large_binary`, when the size of the binary is larger than 4GB * `:fixed_size_binary` + * `:date32` + * `:date64` """ defstruct name: nil, type: nil, @@ -568,4 +570,14 @@ defmodule Adbc.Column do def fixed_size_binary(data, opts \\ []) when is_list(data) and is_list(opts) do column(:fixed_size_binary, data, opts) end + + @spec date32([Date.t()], Keyword.t()) :: %Adbc.Column{} + def date32(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:date32, data, opts) + end + + @spec date64([Date.t()], Keyword.t()) :: %Adbc.Column{} + def date64(data, opts \\ []) when is_list(data) and is_list(opts) do + column(:date64, data, opts) + end end From 9c65d608e58aefaf688bc9db6aafb5ecec89bd21 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Thu, 9 May 2024 15:49:48 +0800 Subject: [PATCH 11/14] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Valim --- lib/adbc_connection.ex | 17 +++++++---------- lib/adbc_result.ex | 26 ++++++++++---------------- 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/lib/adbc_connection.ex b/lib/adbc_connection.ex index e3cbf62..401ec3d 100644 --- a/lib/adbc_connection.ex +++ b/lib/adbc_connection.ex @@ -407,17 +407,14 @@ defmodule Adbc.Connection do end end - defp merge_columns(chucked_results) do - if Enum.count(chucked_results) == 1 do - [chucked_results] = chucked_results - chucked_results - else - Enum.zip_with(chucked_results, fn columns -> - Enum.reduce(columns, fn column, merged_column -> - %{merged_column | data: merged_column.data ++ column.data} - end) + defp merge_columns([result]), do: result + + defp merge_columns(chucked_results) do + Enum.zip_with(chucked_results, fn columns -> + Enum.reduce(columns, fn column, merged_column -> + %{merged_column | data: merged_column.data ++ column.data} end) - end + end) end ## Callbacks diff --git a/lib/adbc_result.ex b/lib/adbc_result.ex index 09ff0a7..3f01d7d 100644 --- a/lib/adbc_result.ex +++ b/lib/adbc_result.ex @@ -15,22 +15,16 @@ defmodule Adbc.Result do num_rows: non_neg_integer() | nil, data: [%Adbc.Column{}] } - def to_map(r = %Adbc.Result{data: data}) do - %Adbc.Result{ - r - | data: - Map.new( - Enum.map(data, fn %Adbc.Column{name: name, type: type, data: data} -> - case type do - :list -> - {name, Enum.map(data, &list_to_map/1)} - - _ -> - {name, data} - end - end) - ) - } + @doc """ + Returns a map of columns as a result. + """ + def to_map(%Adbc.Result{data: data}) do + Map.new(data, fn %Adbc.Column{name: name, type: type, data: data} -> + case type do + :list -> {name, Enum.map(data, &list_to_map/1)} + _ -> {name, data} + end + end) end defp list_to_map(%Adbc.Column{name: name, type: type, data: data}) do From b3ed9d8b4657b611fb6cb9a850a6655b9d564daa Mon Sep 17 00:00:00 2001 From: Cocoa Date: Thu, 9 May 2024 15:18:30 +0800 Subject: [PATCH 12/14] remove unused commented code --- c_src/adbc_nif.cpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/c_src/adbc_nif.cpp b/c_src/adbc_nif.cpp index da72b7e..6e304ab 100644 --- a/c_src/adbc_nif.cpp +++ b/c_src/adbc_nif.cpp @@ -1169,12 +1169,6 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct out_terms.clear(); if (is_struct) { - // if (level == 0) { - // ERL_NIF_TERM adbc_column = make_adbc_column(env, name, schema->format, values->null_count > 0, kAtomNil, children_term); - // out_terms.emplace_back(adbc_column); - // } else { - // out_terms.emplace_back(children_term); - // } out_terms.emplace_back(children_term); } else { if (schema->children) { @@ -1184,17 +1178,6 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct out_terms.emplace_back(erlang::nif::make_binary(env, name)); out_terms.emplace_back(current_term); } - // if (level == 0) { - // if (schema->children) { - // ERL_NIF_TERM adbc_column = make_adbc_column(env, name, schema->format, values->null_count > 0, kAtomNil, children_term); - // out_terms.emplace_back(adbc_column); - // } else { - // ERL_NIF_TERM adbc_column = make_adbc_column(env, name, schema->format, values->null_count > 0, kAtomNil, current_term); - // out_terms.emplace_back(adbc_column); - // } - // } else { - - // } } return 0; From 3929a65b97de7c0e353850d03e3e9405bc34d90c Mon Sep 17 00:00:00 2001 From: Cocoa Date: Thu, 9 May 2024 15:52:35 +0800 Subject: [PATCH 13/14] allow to use seconds and milliseconds in `Adbc.Buffer.date{32,64}` respectively --- c_src/adbc_nif.cpp | 22 ++++++++++++---------- lib/adbc_column.ex | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/c_src/adbc_nif.cpp b/c_src/adbc_nif.cpp index 6e304ab..ba3f037 100644 --- a/c_src/adbc_nif.cpp +++ b/c_src/adbc_nif.cpp @@ -2050,7 +2050,7 @@ int get_utc_offset() { return gmtime_hours; } -int get_list_date(ErlNifEnv *env, ERL_NIF_TERM list, bool nullable, const std::function &callback) { +int get_list_date(ErlNifEnv *env, ERL_NIF_TERM list, bool nullable, const std::function &normalize_ex_value, const std::function &callback) { ERL_NIF_TERM head, tail; tail = list; while (enif_get_list_cell(env, tail, &head, &tail)) { @@ -2095,7 +2095,7 @@ int get_list_date(ErlNifEnv *env, ERL_NIF_TERM list, bool nullable, const std::f // mktime always gives local time // so we need to adjust it to UTC val = mktime(&time) + get_utc_offset() * 3600; - callback(val, false); + callback(normalize_ex_value(val), false); } else { return 1; } @@ -2108,17 +2108,18 @@ int do_get_list_date32(ErlNifEnv *env, ERL_NIF_TERM list, bool nullable, ArrowTy NANOARROW_RETURN_NOT_OK(ArrowSchemaSetType(schema_out, nanoarrow_type)); NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromSchema(array_out, schema_out, error_out)); NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(array_out)); + auto second_to_day = [](int64_t val) -> int64_t { + return val / (24 * 60 * 60); + }; if (nullable) { - return get_list_date(env, list, nullable, [&array_out](int64_t val, bool is_nil) -> void { - val /= 24 * 60 * 60; + return get_list_date(env, list, nullable, second_to_day, [&array_out](int64_t val, bool is_nil) -> void { ArrowArrayAppendInt(array_out, (int32_t)val); if (is_nil) { ArrowArrayAppendNull(array_out, 1); } }); } else { - return get_list_date(env, list, nullable, [&array_out](int64_t val, bool) -> void { - val /= 24 * 60 * 60; + return get_list_date(env, list, nullable, second_to_day, [&array_out](int64_t val, bool) -> void { ArrowArrayAppendInt(array_out, (int32_t)val); }); } @@ -2128,17 +2129,18 @@ int do_get_list_date64(ErlNifEnv *env, ERL_NIF_TERM list, bool nullable, ArrowTy NANOARROW_RETURN_NOT_OK(ArrowSchemaSetType(schema_out, nanoarrow_type)); NANOARROW_RETURN_NOT_OK(ArrowArrayInitFromSchema(array_out, schema_out, error_out)); NANOARROW_RETURN_NOT_OK(ArrowArrayStartAppending(array_out)); + auto second_to_millisecond = [](int64_t val) -> int64_t { + return val * 1000; + }; if (nullable) { - return get_list_date(env, list, nullable, [&array_out](int64_t val, bool is_nil) -> void { - val *= 1000; + return get_list_date(env, list, nullable, second_to_millisecond, [&array_out](int64_t val, bool is_nil) -> void { ArrowArrayAppendInt(array_out, val); if (is_nil) { ArrowArrayAppendNull(array_out, 1); } }); } else { - return get_list_date(env, list, nullable, [&array_out](int64_t val, bool) -> void { - val *= 1000; + return get_list_date(env, list, nullable, second_to_millisecond, [&array_out](int64_t val, bool) -> void { ArrowArrayAppendInt(array_out, val); }); } diff --git a/lib/adbc_column.ex b/lib/adbc_column.ex index f519e6b..c643fd7 100644 --- a/lib/adbc_column.ex +++ b/lib/adbc_column.ex @@ -571,12 +571,42 @@ defmodule Adbc.Column do column(:fixed_size_binary, data, opts) end - @spec date32([Date.t()], Keyword.t()) :: %Adbc.Column{} + @doc """ + A column that contains date represented as 32-bit integers in UTC. + ## Arguments + + * `data`: a list, each element of which can be one of the following: + * a `Date.t()` + * a 32-bit integer representing the number of days since the Unix epoch. + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + """ + @spec date32([Date.t() | integer()], Keyword.t()) :: %Adbc.Column{} def date32(data, opts \\ []) when is_list(data) and is_list(opts) do column(:date32, data, opts) end - @spec date64([Date.t()], Keyword.t()) :: %Adbc.Column{} + @doc """ + A column that contains date represented as 64-bit integers in UTC. + ## Arguments + + * `data`: a list, each element of which can be one of the following: + * a `Date.t()` + * a 64-bit integer representing the number of milliseconds since the Unix epoch. + * `opts`: A keyword list of options + + ## Options + + * `:name` - The name of the column + * `:nullable` - A boolean value indicating whether the column is nullable + * `:metadata` - A map of metadata + """ + @spec date64([Date.t() | integer()], Keyword.t()) :: %Adbc.Column{} def date64(data, opts \\ []) when is_list(data) and is_list(opts) do column(:date64, data, opts) end From f1d3916745129431a6d613c52aee0297665ab4a7 Mon Sep 17 00:00:00 2001 From: Cocoa Date: Thu, 9 May 2024 15:58:06 +0800 Subject: [PATCH 14/14] fixed tests related to `Adbc.Result.to_map/1` --- lib/adbc_connection.ex | 2 +- test/adbc_connection_test.exs | 95 +++++++++++++++++------------------ test/adbc_sqlite_test.exs | 54 ++++++++++---------- 3 files changed, 73 insertions(+), 78 deletions(-) diff --git a/lib/adbc_connection.ex b/lib/adbc_connection.ex index 401ec3d..f2a324b 100644 --- a/lib/adbc_connection.ex +++ b/lib/adbc_connection.ex @@ -409,7 +409,7 @@ defmodule Adbc.Connection do defp merge_columns([result]), do: result - defp merge_columns(chucked_results) do + defp merge_columns(chucked_results) do Enum.zip_with(chucked_results, fn columns -> Enum.reduce(columns, fn column, merged_column -> %{merged_column | data: merged_column.data ++ column.data} diff --git a/test/adbc_connection_test.exs b/test/adbc_connection_test.exs index a5f7596..a7f3a19 100644 --- a/test/adbc_connection_test.exs +++ b/test/adbc_connection_test.exs @@ -386,53 +386,51 @@ defmodule Adbc.Connection.Test do ] }} = Connection.get_objects(conn, 0) - assert %Adbc.Result{ - data: %{ - "catalog_db_schemas" => [ - {"db_schema_name", []}, - {"db_schema_tables", - [ - {"table_name", []}, - {"table_type", []}, - {"table_columns", - [ - {"column_name", []}, - {"ordinal_position", []}, - {"remarks", []}, - {"xdbc_data_type", []}, - {"xdbc_type_name", []}, - {"xdbc_column_size", []}, - {"xdbc_decimal_digits", []}, - {"xdbc_num_prec_radix", []}, - {"xdbc_nullable", []}, - {"xdbc_column_def", []}, - {"xdbc_sql_data_type", []}, - {"xdbc_datetime_sub", []}, - {"xdbc_char_octet_length", []}, - {"xdbc_is_nullable", []}, - {"xdbc_scope_catalog", []}, - {"xdbc_scope_schema", []}, - {"xdbc_scope_table", []}, - {"xdbc_is_autoincrement", []}, - {"xdbc_is_generatedcolumn", []} - ]}, - {"table_constraints", - [ - {"constraint_name", []}, - {"constraint_type", []}, - {"constraint_column_names", [[]]}, - {"constraint_column_usage", - [ - {"fk_catalog", []}, - {"fk_db_schema", []}, - {"fk_table", []}, - {"fk_column_name", []} - ]} - ]} - ]} - ], - "catalog_name" => [] - } + assert %{ + "catalog_db_schemas" => [ + {"db_schema_name", []}, + {"db_schema_tables", + [ + {"table_name", []}, + {"table_type", []}, + {"table_columns", + [ + {"column_name", []}, + {"ordinal_position", []}, + {"remarks", []}, + {"xdbc_data_type", []}, + {"xdbc_type_name", []}, + {"xdbc_column_size", []}, + {"xdbc_decimal_digits", []}, + {"xdbc_num_prec_radix", []}, + {"xdbc_nullable", []}, + {"xdbc_column_def", []}, + {"xdbc_sql_data_type", []}, + {"xdbc_datetime_sub", []}, + {"xdbc_char_octet_length", []}, + {"xdbc_is_nullable", []}, + {"xdbc_scope_catalog", []}, + {"xdbc_scope_schema", []}, + {"xdbc_scope_table", []}, + {"xdbc_is_autoincrement", []}, + {"xdbc_is_generatedcolumn", []} + ]}, + {"table_constraints", + [ + {"constraint_name", []}, + {"constraint_type", []}, + {"constraint_column_names", [[]]}, + {"constraint_column_usage", + [ + {"fk_catalog", []}, + {"fk_db_schema", []}, + {"fk_table", []}, + {"fk_column_name", []} + ]} + ]} + ]} + ], + "catalog_name" => [] } = Adbc.Result.to_map(results) end end @@ -455,8 +453,7 @@ defmodule Adbc.Connection.Test do }} = Connection.get_table_types(conn) - assert %Adbc.Result{num_rows: nil, data: %{"table_type" => ["table", "view"]}} = - Adbc.Result.to_map(result) + assert %{"table_type" => ["table", "view"]} = Adbc.Result.to_map(result) end end diff --git a/test/adbc_sqlite_test.exs b/test/adbc_sqlite_test.exs index 54fb9a8..1fe4205 100644 --- a/test/adbc_sqlite_test.exs +++ b/test/adbc_sqlite_test.exs @@ -131,34 +131,32 @@ defmodule Adbc.SQLite.Test do ] }} = Adbc.Connection.query(conn, "SELECT * FROM test") - %Adbc.Result{ - data: %{ - "b1" => [<<100, 97, 116, 97, 1, 2>>], - "i1" => [1], - "i2" => [2], - "i3" => [3], - "i4" => [4], - "i5" => [5], - "i6" => [6], - "i7" => ~c"\a", - "i8" => ~c"\b", - "i9" => ~c"\t", - "n1" => [1.1], - "n2" => [2.2], - "n3" => [1], - "n4" => ["2021-01-01"], - "n5" => ["2021-01-01 00:00:00"], - "r1" => [1.1], - "r2" => [2.2], - "r3" => [3.3], - "r4" => [4.4], - "t1" => ["hello"], - "t2" => ["world"], - "t3" => ["goodbye"], - "t4" => ["world"], - "t5" => ["foo"], - "t6" => ["bar"] - } + %{ + "b1" => [<<100, 97, 116, 97, 1, 2>>], + "i1" => [1], + "i2" => [2], + "i3" => [3], + "i4" => [4], + "i5" => [5], + "i6" => [6], + "i7" => ~c"\a", + "i8" => ~c"\b", + "i9" => ~c"\t", + "n1" => [1.1], + "n2" => [2.2], + "n3" => [1], + "n4" => ["2021-01-01"], + "n5" => ["2021-01-01 00:00:00"], + "r1" => [1.1], + "r2" => [2.2], + "r3" => [3.3], + "r4" => [4.4], + "t1" => ["hello"], + "t2" => ["world"], + "t3" => ["goodbye"], + "t4" => ["world"], + "t5" => ["foo"], + "t6" => ["bar"] } = Adbc.Result.to_map(list_result) end