Skip to content
This repository was archived by the owner on May 5, 2019. It is now read-only.

Stack should use similar_nullable, not NullableArray #54

Merged
merged 3 commits into from
Apr 23, 2017
Merged

Stack should use similar_nullable, not NullableArray #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0ac8e5c
Add tests to trigger duplicate entries in unstack warning
cjprybol Apr 16, 2017
71edd51
Update stack functions to use similar_nullable
cjprybol Apr 16, 2017
7b59074
Update similar_nullable conventions (depwarn and compat)
cjprybol Apr 16, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions src/abstractdatatable/join.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,21 @@
##

# Like similar, but returns a nullable array
similar_nullable{T}(dv::AbstractArray{T}, dims::@compat(Union{Int, Tuple{Vararg{Int}}})) =
NullableArray(T, dims)
similar_nullable{T}(dv::AbstractArray{T}, dims::Union{Int, Tuple{Vararg{Int}}}) =
NullableArray{T}(dims)

similar_nullable{T<:Nullable}(dv::AbstractArray{T}, dims::@compat(Union{Int, Tuple{Vararg{Int}}})) =
NullableArray(eltype(T), dims)
similar_nullable{T<:Nullable}(dv::AbstractArray{T}, dims::Union{Int, Tuple{Vararg{Int}}}) =
NullableArray{eltype(T)}(dims)

similar_nullable{T,R}(dv::CategoricalArray{T,R}, dims::@compat(Union{Int, Tuple{Vararg{Int}}})) =
NullableCategoricalArray(T, dims)
similar_nullable{T,R}(dv::CategoricalArray{T,R}, dims::Union{Int, Tuple{Vararg{Int}}}) =
NullableCategoricalArray{T}(dims)

similar_nullable(dt::AbstractDataTable, dims::Int) =
DataTable(Any[similar_nullable(x, dims) for x in columns(dt)], copy(index(dt)))

similar_nullable{T,R}(dv::NullableCategoricalArray{T,R}, dims::Union{Int, Tuple{Vararg{Int}}}) =
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would be more logical to add this definition right after that for CategoricalArray.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The AbstractDataTable function that separates the two will disappear in #44. I thought adding the new function to the end of the set like this would minimize the risk of a merge conflict, although that's purely speculative. I can reverse them if you'd like, but I'm not sure it will matter.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a big deal, but it's always better to have commits be correct when taken in isolation.

NullableCategoricalArray{T}(dims)

# helper structure for DataTables joining
immutable DataTableJoiner{DT1<:AbstractDataTable, DT2<:AbstractDataTable}
dtl::DT1
Expand Down
20 changes: 6 additions & 14 deletions src/abstractdatatable/reshape.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,12 +198,7 @@ function unstack(dt::AbstractDataTable, rowkey::Int, colkey::Int, value::Int)
keycol = NullableCategoricalArray(dt[colkey])
Nrow = length(refkeycol.pool)
Ncol = length(keycol.pool)
T = eltype(valuecol)
if T <: Nullable
T = eltype(T)
end
payload = DataTable(Any[NullableArray(T, Nrow) for i in 1:Ncol],
map(Symbol, levels(keycol)))
payload = DataTable(Any[similar_nullable(valuecol, Nrow) for i in 1:Ncol], map(Symbol, levels(keycol)))
nowarning = true
for k in 1:nrow(dt)
j = Int(CategoricalArrays.order(keycol.pool)[keycol.refs[k]])
Expand All @@ -216,7 +211,9 @@ function unstack(dt::AbstractDataTable, rowkey::Int, colkey::Int, value::Int)
payload[j][i] = valuecol[k]
end
end
insert!(payload, 1, NullableArray(levels(refkeycol)), _names(dt)[rowkey])
levs = levels(refkeycol)
col = similar_nullable(dt[rowkey], length(levs))
insert!(payload, 1, copy!(col, levs), _names(dt)[rowkey])
end
unstack(dt::AbstractDataTable, rowkey, colkey, value) =
unstack(dt, index(dt)[rowkey], index(dt)[colkey], index(dt)[value])
Expand All @@ -235,15 +232,10 @@ function unstack(dt::AbstractDataTable, colkey::Int, value::Int)
end
keycol = NullableCategoricalArray(dt[colkey])
valuecol = dt[value]
dt1 = dt[g.idx[g.starts], g.cols]
dt1 = nullable!(dt[g.idx[g.starts], g.cols], g.cols)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need to specify g.cols the second time AFAICT.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The nullify! function I had written for the pre-refactor PR didn't need any specification of columns when you want to nullify all of the columns, but this existing nullable! function does require columns to be specified.

julia> nullable!(DataTable(a = 1:4))
ERROR: MethodError: no method matching nullable!(::DataTables.DataTable)
Closest candidates are:
  nullable!(::DataTables.DataTable, ::Union{Real, Symbol}) at /Users/Cameron/.julia/v0.6/DataTables/src/datatable/datatable.jl:779
  nullable!(::DataTables.DataTable, ::Array{T<:Union{Real, Symbol},1}) where T<:Union{Real, Symbol} at /Users/Cameron/.julia/v0.6/DataTables/src/datatable/datatable.jl:783
  nullable!(::Array{Symbol,1}, ::DataTables.AbstractDataTable) at deprecated.jl:50
  ...

julia> nullable!(DataTable(a = 1:4), :a)
4×1 DataTables.DataTable
│ Row │ a │
├─────┼───┤
│ 1   │ 1 │
│ 2   │ 2 │
│ 3   │ 3 │
│ 4   │ 4 │

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, so that's fine for now.

Nrow = length(g)
Ncol = length(levels(keycol))
T = eltype(valuecol)
if T <: Nullable
T = eltype(T)
end
dt2 = DataTable(Any[NullableArray(T, Nrow) for i in 1:Ncol],
map(@compat(Symbol), levels(keycol)))
dt2 = DataTable(Any[similar_nullable(valuecol, Nrow) for i in 1:Ncol], map(Symbol, levels(keycol)))
nowarning = true
for k in 1:nrow(dt)
j = Int(CategoricalArrays.order(keycol.pool)[keycol.refs[k]])
Expand Down
38 changes: 38 additions & 0 deletions test/datatable.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,4 +341,42 @@ module TestDataTable
@test find(c -> isa(c, NullableCategoricalArray), categorical!(DataTable(A=1:3, B=4:6), :A).columns) == [1]
@test find(c -> isa(c, NullableCategoricalArray), categorical!(DataTable(A=1:3, B=4:6), [1]).columns) == [1]
@test find(c -> isa(c, NullableCategoricalArray), categorical!(DataTable(A=1:3, B=4:6), 1).columns) == [1]

@testset "unstack nullable promotion" begin
dt = DataTable(Any[repeat(1:2, inner=4), repeat('a':'d', outer=2), collect(1:8)],
[:id, :variable, :value])
udt = unstack(dt)
@test udt == unstack(dt, :variable, :value) == unstack(dt, :id, :variable, :value)
@test udt == DataTable(Any[Nullable[1, 2], Nullable[1, 5], Nullable[2, 6],
Nullable[3, 7], Nullable[4, 8]], [:id, :a, :b, :c, :d])
@test all(typeof.(udt.columns) .== NullableVector{Int})
dt = DataTable(Any[categorical(repeat(1:2, inner=4)),
categorical(repeat('a':'d', outer=2)), categorical(1:8)],
[:id, :variable, :value])
udt = unstack(dt)
@test udt == unstack(dt, :variable, :value) == unstack(dt, :id, :variable, :value)
@test udt == DataTable(Any[Nullable[1, 2], Nullable[1, 5], Nullable[2, 6],
Nullable[3, 7], Nullable[4, 8]], [:id, :a, :b, :c, :d])
@test all(typeof.(udt.columns) .== NullableCategoricalVector{Int, UInt32})
end

@testset "duplicate entries in unstack warnings" begin
dt = DataTable(id=NullableArray([1, 2, 1, 2]), variable=["a", "b", "a", "b"], value=[3, 4, 5, 6])
@static if VERSION >= v"0.6.0-dev.1980"
@test_warn "Duplicate entries in unstack." unstack(dt, :id, :variable, :value)
@test_warn "Duplicate entries in unstack at row 3." unstack(dt, :variable, :value)
end
a = unstack(dt, :id, :variable, :value)
b = unstack(dt, :variable, :value)
@test a == b == DataTable(id = Nullable[1, 2], a = [5, Nullable()], b = [Nullable(), 6])

dt = DataTable(id=NullableArray(1:2), variable=["a", "b"], value=3:4)
@static if VERSION >= v"0.6.0-dev.1980"
@test_nowarn unstack(dt, :id, :variable, :value)
@test_nowarn unstack(dt, :variable, :value)
end
a = unstack(dt, :id, :variable, :value)
b = unstack(dt, :variable, :value)
@test a == b == DataTable(id = Nullable[1, 2], a = [3, Nullable()], b = [Nullable(), 4])
end
end