update to DataFrames 0.8+

* drop Julia 0.4 support (since DataFrames require Julia 0.5)
* convert from using DataArrays to NullableArrays

Author: alyst

NEWS.md

@@ -1,3 +1,12 @@
+## RData v0.1.0 Release Notes
+
+Updated to DataFrames 0.9, dropped Julia v0.4
+
+##### Changes
+* using NullableArrays.jl and CategoricalArrays.jl
+instead of DataArrays.jl ([#19], see [JuliaStats/DataFrames.jl#1008])
+* Julia v0.4 not supported (DataFrames.jl v0.9 requirements)
+
 ## RData v0.0.4 Release Notes
 
 Now the recommended way to load `.RData`/`.rda` files is by `FileIO.load()`.

REQUIRE

@@ -1,6 +1,5 @@
-julia 0.4
-DataFrames 0.7
-DataArrays 0.3
+julia 0.5
+DataFrames 0.8+
 FileIO 0.1.2
 GZip 0.2
 Compat 0.8

src/RData.jl

@@ -3,7 +3,6 @@ __precompile__()
 module RData
 
 using Compat, DataFrames, GZip, FileIO
-import DataArrays: data
 import DataFrames: identifier
 import Compat: UTF8String, unsafe_string
 import FileIO: load

src/convert.jl

@@ -11,71 +11,81 @@ end
 
 ##############################################################################
 ##
-## Conversion of intermediate R objects into DataArray and DataFrame objects
+## Conversion of intermediate R objects into NullableArray and DataFrame objects
 ##
 ##############################################################################
 
-namask(rl::RLogicalVector) = BitArray(rl.data .== R_NA_INT32)
-namask(ri::RIntegerVector) = BitArray(ri.data .== R_NA_INT32)
-namask(rn::RNumericVector) = BitArray(map(isna_float64, reinterpret(UInt64, rn.data)))
+namask(rl::RLogicalVector) = [flag == R_NA_INT32 for flag in rl.data]
+namask(ri::RIntegerVector) = [flag == R_NA_INT32 for flag in ri.data]
+namask(rn::RNumericVector) = map(isna_float64, reinterpret(UInt64, rn.data))
 # if re or im is NA, the whole complex number is NA
 # FIXME avoid temporary Vector{Bool}
-namask(rc::RComplexVector) = BitArray([isna_float64(v.re) || isna_float64(v.im) for v in reinterpret(Complex{UInt64}, rc.data)])
+namask(rc::RComplexVector) = [isna_float64(v.re) || isna_float64(v.im) for v in reinterpret(Complex{UInt64}, rc.data)]
 namask(rv::RNullableVector) = rv.na
 
-DataArrays.data(rv::RVEC) = DataArray(rv.data, namask(rv))
+# convert R vector into NullableArray
+_nullable(rv::RVEC) = NullableArray(rv.data, namask(rv))
 
-function DataArrays.data(ri::RIntegerVector)
-    if !isfactor(ri) return DataArray(ri.data, namask(ri)) end
-    # convert factor into PooledDataArray
+# converts Vector{Int32} into Vector{R} replacing R_NA_INT32 with 0
+_zero_nas{R}(::Type{R}, v::Vector{Int32}) = [x != R_NA_INT32 ? R(x) : zero(R) for x in v]
+
+# convert R factor into NullableCategoricalArray{String}
+# TODO option to convert into Symbol etc?
+function _nullable(ri::RIntegerVector)
+    isfactor(ri) || return NullableArray(ri.data, namask(ri)) # not a factor
+    # convert factor into NullableCategoricalArray
     pool = getattr(ri, "levels", emptystrvec)
     sz = length(pool)
     REFTYPE = sz <= typemax(UInt8)  ? UInt8 :
               sz <= typemax(UInt16) ? UInt16 :
               sz <= typemax(UInt32) ? UInt32 :
                                       UInt64
-    dd = ri.data
-    dd[namask(ri)] = 0
-    refs = convert(Vector{REFTYPE}, dd)
-    return PooledDataArray(DataArrays.RefArray(refs), pool)
+    return NullableCategoricalArray{String, 1, REFTYPE}(_zero_nas(REFTYPE, ri.data),
+                                                        CategoricalPool{String, REFTYPE}(pool))
 end
 
 function sexp2julia(rex::RSEXPREC)
     warn("Conversion of $(typeof(rex)) to Julia is not implemented")
     return nothing
 end
 
+# FIXME remove when anynull(NullableCategoricalArray) would be available
+_anynull{T,N,R}(A::NullableCategoricalArray{T,N,R}) = any(A.refs == zero(R))
+_anynull(A::NullableArray) = anynull(A)
+
+# convert nullable array without nulls into non-nullable array
+# `A` is expected to contain no nulls
+_dropnonulls(A::NullableArray) = A.values
+_dropnonulls{T,N,R}(A::NullableCategoricalArray{T,N,R}) = CategoricalArray{T,N,R}(A.refs, A.pool)
+
 function sexp2julia(rv::RVEC)
     # FIXME dimnames
-    # FIXME forceDataArrays option to always convert to DataArray
-    nas = namask(rv)
-    hasna = any(nas)
+    # FIXME forceNullable option to always convert to NullableArray
+    jv = _nullable(rv)
+    hasna = _anynull(jv)
     if hasnames(rv)
         # if data has no NA, convert to simple Vector
-        return DictoVec(hasna ? DataArray(rv.data, nas) : rv.data, names(rv))
+        return DictoVec(hasna ? jv : _dropnonulls(jv), names(rv))
     else
         hasdims = hasdim(rv)
         if !hasdims && length(rv.data)==1
             # scalar
-            # FIXME handle NAs
-            # if hasna
-            return rv.data[1]
+            return hasna ? jv[1] : get(jv[1])
         elseif !hasdims
             # vectors
-            return hasna ? DataArray(rv.data, nas) : rv.data
+            return hasna ? jv : _dropnonulls(jv)
         else
             # matrices and so on
-            dims = tuple(convert(Vector{Int64}, getattr(rv, "dim"))...)
-            return hasna ? DataArray(reshape(rv.data, dims), reshape(nas, dims)) :
-                         reshape(rv.data, dims)
+            dims = tuple(convert(Vector{Int}, getattr(rv, "dim"))...)
+            return reshape(jv, dims)
         end
     end
 end
 
 function sexp2julia(rl::RList)
     if isdataframe(rl)
         # FIXME remove Any type assertion workaround
-        DataFrame(Any[data(col) for col in rl.data], map(identifier, names(rl)))
+        DataFrame(Any[_nullable(col) for col in rl.data], map(identifier, names(rl)))
     elseif hasnames(rl)
         DictoVec(Any[sexp2julia(item) for item in rl.data], names(rl))
     else

src/sxtypes.jl

@@ -125,7 +125,7 @@ typealias RComplexVector RVector{Complex128, CPLXSXP}
 """
 immutable RNullableVector{T, S} <: RVEC{T, S}
     data::Vector{T}
-    na::BitVector                # mask of NA elements
+    na::Vector{Bool}             # mask of NA elements
     attr::Hash                   # collection of R object attributes
 end
 

test/RDA.jl

@@ -23,19 +23,21 @@ module TestRDA
     df[:int] = Int32[1, 2]
     df[:logi] = [true, false]
     df[:chr] = ["ab", "c"]
-    df[:factor] = pool(df[:chr])
+    df[:factor] = categorical(df[:chr])
     df[:cplx] = Complex128[1.1+0.5im, 1.0im]
     @test isequal(sexp2julia(load("$testdir/data/types.rda",convert=false)["df"]), df)
     @test isequal(sexp2julia(load("$testdir/data/types_ascii.rda",convert=false)["df"]), df)
 
-    df[2, :] = NA
+    for col in DataFrames.columns(df)
+        col[2] = Nullable{eltype(col)}() # FIXME nullify!() is not supported by CategoricalArrays
+    end
     append!(df, df[2, :])
     df[3, :num] = NaN
-    df[:, :cplx] = @data [NA, @compat(Complex128(1,NaN)), NaN]
+    df[:, :cplx] = NullableArray{Complex128}(Nullable{Complex128}[Nullable{Complex128}(), Complex128(1,NaN), NaN])
     @test isequal(sexp2julia(load("$testdir/data/NAs.rda",convert=false)["df"]), df)
     # ASCII format saves NaN as NA
-    df[3, :num] = NA
-    df[:, :cplx] = @data [NA, NA, NA]
+    df[3, :num] = Nullable{Complex128}()
+    df[:, :cplx] = NullableArray{Complex128}(3)
     @test isequal(sexp2julia(load("$testdir/data/NAs_ascii.rda",convert=false)["df"]), df)
 
     rda_names = names(sexp2julia(load("$testdir/data/names.rda",convert=false)["df"]))