Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: parse dates, times, and timestamps #30

Merged
merged 4 commits into from
Oct 17, 2023
Merged

feat: parse dates, times, and timestamps #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8a3a5b3
parse dates, times, and timestamps
superhawk610 Oct 13, 2023
213ccb3
reject timestamps with timezones
superhawk610 Oct 13, 2023
ffcdcc6
add tests for temporal types
superhawk610 Oct 16, 2023
a0b14fe
Update c_src/adbc_nif.cpp
josevalim Oct 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 192 additions & 0 deletions c_src/adbc_nif.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,93 @@ static ERL_NIF_TERM get_arrow_array_list_children(ErlNifEnv *env, struct ArrowSc
return enif_make_list_from_array(env, children.data(), (unsigned)items_values->n_children);
}

static ERL_NIF_TERM unix_second_offset_to_date(ErlNifEnv *env, uint64_t seconds) {
time_t t = (time_t)seconds;
tm* time = gmtime(&t);

ERL_NIF_TERM date;
ERL_NIF_TERM keys[] = {
erlang::nif::atom(env, "__struct__"),
erlang::nif::atom(env, "calendar"),
erlang::nif::atom(env, "year"),
erlang::nif::atom(env, "month"),
erlang::nif::atom(env, "day")
};
ERL_NIF_TERM values[] = {
erlang::nif::atom(env, "Elixir.Date"),
erlang::nif::atom(env, "Elixir.Calendar.ISO"),
enif_make_int(env, time->tm_year + 1900),
enif_make_int(env, time->tm_mon + 1),
enif_make_int(env, time->tm_mday)
};
enif_make_map_from_arrays(env, keys, values, sizeof(keys)/sizeof(keys[0]), &date);
return date;
}

static ERL_NIF_TERM nanoseconds_to_time(ErlNifEnv *env, uint64_t ns, uint8_t us_precision) {
// Elixir only supports microsecond precision
uint64_t us = ns / 1000;
time_t s = (time_t)(us / 1000000);
tm* time = gmtime(&s);

us = us % 1000000;

ERL_NIF_TERM t;
ERL_NIF_TERM keys[] = {
erlang::nif::atom(env, "__struct__"),
erlang::nif::atom(env, "calendar"),
erlang::nif::atom(env, "hour"),
erlang::nif::atom(env, "minute"),
erlang::nif::atom(env, "second"),
erlang::nif::atom(env, "microsecond")
};
ERL_NIF_TERM values[] = {
erlang::nif::atom(env, "Elixir.Time"),
erlang::nif::atom(env, "Elixir.Calendar.ISO"),
enif_make_int(env, time->tm_hour),
enif_make_int(env, time->tm_min),
enif_make_int(env, time->tm_sec),
enif_make_tuple2(env, enif_make_int(env, us), enif_make_int(env, us_precision))
};
enif_make_map_from_arrays(env, keys, values, sizeof(keys)/sizeof(keys[0]), &t);
return t;
}

static ERL_NIF_TERM unix_timestamp_to_naive_datetime(ErlNifEnv *env, uint64_t timestamp, uint8_t us_precision) {
// Elixir only supports microsecond precision
uint64_t us = timestamp / 1000;
time_t t = (time_t)(us / 1000000);
tm* time = gmtime(&t);

us = us % 1000000;

ERL_NIF_TERM date_time;
ERL_NIF_TERM keys[] = {
erlang::nif::atom(env, "__struct__"),
erlang::nif::atom(env, "calendar"),
erlang::nif::atom(env, "year"),
erlang::nif::atom(env, "month"),
erlang::nif::atom(env, "day"),
erlang::nif::atom(env, "hour"),
erlang::nif::atom(env, "minute"),
erlang::nif::atom(env, "second"),
erlang::nif::atom(env, "microsecond"),
};
ERL_NIF_TERM values[] = {
erlang::nif::atom(env, "Elixir.NaiveDateTime"),
erlang::nif::atom(env, "Elixir.Calendar.ISO"),
enif_make_int(env, time->tm_year + 1900),
enif_make_int(env, time->tm_mon + 1),
enif_make_int(env, time->tm_mday),
enif_make_int(env, time->tm_hour),
enif_make_int(env, time->tm_min),
enif_make_int(env, time->tm_sec),
enif_make_tuple2(env, enif_make_int(env, us), enif_make_int(env, us_precision))
};
enif_make_map_from_arrays(env, keys, values, sizeof(keys)/sizeof(keys[0]), &date_time);
return date_time;
}

int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct ArrowArray * values, uint64_t level, std::vector<ERL_NIF_TERM> &out_terms, ERL_NIF_TERM &error) {
if (schema == nullptr) {
error = erlang::nif::error(env, "invalid ArrowSchema (nullptr) when invoking next");
Expand Down Expand Up @@ -451,6 +538,111 @@ int arrow_array_to_nif_term(ErlNifEnv *env, struct ArrowSchema * schema, struct
children_term = enif_make_list_from_array(env, children.data(), (unsigned)schema->n_children);
} else if (format_len > 4 && (strncmp("+ud:", format, 4) == 0 || strncmp("+us:", format, 4) == 0)) {
children_term = get_arrow_array_union_children(env, schema, values, level);
// date
} else if (strncmp("td", format, 2) == 0) {
char unit = format[2];

if (unit == 'D' || unit == 'm') {
using value_type = uint64_t;
current_term = values_from_buffer(
env,
values->length,
bitmap_buffer,
(const value_type *)values->buffers[1],
[unit](ErlNifEnv *env, uint64_t val) -> ERL_NIF_TERM {
switch (unit) {
case 'D': // days
return unix_second_offset_to_date(env, val * 24 * 60 * 60);
case 'm': // milliseconds
return unix_second_offset_to_date(env, val / 1000);
default:
__builtin_unreachable();
}
}
);
} else {
format_processed = false;
}
// time
} else if (strncmp("tt", format, 2) == 0) {
uint64_t unit;
uint8_t us_precision;
switch (format[2]) {
case 's': // seconds
unit = 1000000000;
us_precision = 0;
break;
case 'm': // milliseconds
unit = 1000000;
us_precision = 3;
break;
case 'u': // microseconds
unit = 1000;
us_precision = 6;
break;
case 'n': // nanoseconds
unit = 1;
us_precision = 6;
break;
default:
format_processed = false;
}

if (format_processed) {
using value_type = uint64_t;
current_term = values_from_buffer(
env,
values->length,
bitmap_buffer,
(const value_type *)values->buffers[1],
[unit, us_precision](ErlNifEnv *env, uint64_t val) -> ERL_NIF_TERM {
return nanoseconds_to_time(env, val * unit, us_precision);
}
);
}
// timestamp
} else if (strncmp("ts", format, 2) == 0) {
uint64_t unit;
uint8_t us_precision;
switch (format[2]) {
case 's': // seconds
unit = 1000000000;
us_precision = 0;
break;
case 'm': // milliseconds
unit = 1000000;
us_precision = 3;
break;
case 'u': // microseconds
unit = 1000;
us_precision = 6;
break;
case 'n': // nanoseconds
unit = 1;
us_precision = 6;
break;
default:
format_processed = false;
}

if (format_len > 4) {
// TODO: handle timezones (Snowflake always returns naive datetimes)
// std::string timezone (&format[4]);
format_processed = false;
}

if (format_processed) {
using value_type = uint64_t;
current_term = values_from_buffer(
env,
values->length,
bitmap_buffer,
(const value_type *)values->buffers[1],
[unit, us_precision](ErlNifEnv *env, uint64_t val) -> ERL_NIF_TERM {
return unix_timestamp_to_naive_datetime(env, val * unit, us_precision);
}
);
}
} else {
format_processed = false;
}
Expand Down
37 changes: 35 additions & 2 deletions test/adbc_connection_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,14 @@ defmodule Adbc.Connection.Test do

alias Adbc.Connection

setup do
db = start_supervised!({Adbc.Database, driver: :sqlite, uri: ":memory:"})
setup ctx do
opts =
case Map.get(ctx, :driver, :sqlite) do
:postgresql -> [driver: :postgresql, uri: "postgres://postgres:postgres@localhost"]
:sqlite -> [driver: :sqlite, uri: ":memory:"]
end

db = start_supervised!({Adbc.Database, opts})
%{db: db}
end

Expand Down Expand Up @@ -207,6 +213,33 @@ defmodule Adbc.Connection.Test do
Connection.query!(conn, "SELECT 123 + ? as num", [456])
end

@tag :postgresql
@tag driver: :postgresql
test "select with temporal types", %{db: db} do
conn = start_supervised!({Connection, database: db})

query = """
select
'2023-03-01T10:23:45'::timestamp as datetime,
'2023-03-01T10:23:45.123456'::timestamp as datetime_usec,
-- timestamp support is not yet implemented
-- '2023-03-01T10:23:45 PST'::timestamptz as datetime_tz,
'2023-03-01'::date as date,
'10:23:45'::time as time,
'10:23:45.123456'::time as time_usec
"""

assert %Adbc.Result{
data: %{
"date" => [~D[2023-03-01]],
"datetime" => [~N[2023-03-01 10:23:45.000000]],
"datetime_usec" => [~N[2023-03-01 10:23:45.123456]],
"time" => [~T[10:23:45.000000]],
"time_usec" => [~T[10:23:45.123456]]
}
} = Connection.query!(conn, query)
end

test "fails on invalid query", %{db: db} do
conn = start_supervised!({Connection, database: db})

Expand Down
Loading