Merge pull request #11060 from JuliaLang/jcb/modserialize

Modularize serialization code

Author: jakebolewski

base/Enums.jl

@@ -91,7 +91,7 @@ macro enum(T,syms...)
         immutable $(esc(T)) <: Enum
             val::$enumT
             function $(esc(typename))(x::Integer)
-                $(membershiptest(:x, values)) || enum_argument_error($(Meta.quot(typename)), x)
+                $(membershiptest(:x, values)) || enum_argument_error($(Expr(:quote, typename)), x)
                 new(x)
             end
         end
@@ -100,7 +100,7 @@ macro enum(T,syms...)
         Base.length{E<:$(esc(typename))}(x::Type{E}) = $(length(vals))
         Base.next{E<:$(esc(typename))}(x::Type{E},s) = (E($values[s]),s+1)
         Base.done{E<:$(esc(typename))}(x::Type{E},s) = s > $(length(values))
-        Base.names{E<:$(esc(typename))}(x::Type{E}) = [$(map(x->Meta.quot(x[1]), vals)...)]
+        Base.names{E<:$(esc(typename))}(x::Type{E}) = [$(map(x->Expr(:quote, (x[1])), vals)...)]
         Base.isless{E<:$(esc(typename))}(x::E, y::E) = isless(x.val, y.val)
         function Base.print{E<:$(esc(typename))}(io::IO,x::E)
             for (sym, i) in $vals

base/dict.jl

@@ -404,26 +404,6 @@ function convert{K,V}(::Type{Dict{K,V}},d::Associative)
 end
 convert{K,V}(::Type{Dict{K,V}},d::Dict{K,V}) = d
 
-function serialize(s, t::Dict)
-    serialize_type(s, typeof(t))
-    write(s, Int32(length(t)))
-    for (k,v) in t
-        serialize(s, k)
-        serialize(s, v)
-    end
-end
-
-function deserialize{K,V}(s, T::Type{Dict{K,V}})
-    n = read(s, Int32)
-    t = T(); sizehint!(t, n)
-    for i = 1:n
-        k = deserialize(s)
-        v = deserialize(s)
-        t[k] = v
-    end
-    return t
-end
-
 hashindex(key, sz) = ((hash(key)%Int) & (sz-1)) + 1
 
 isslotempty(h::Dict, i::Int) = h.slots[i] == 0x0

base/exports.jl

@@ -14,6 +14,7 @@ export
     LinAlg,
     BLAS,
     LAPACK,
+    Serializer,
     SparseMatrix,
     Docs,
     Markdown,

base/gmp.jl

@@ -6,8 +6,7 @@ import Base: *, +, -, /, <, <<, >>, >>>, <=, ==, >, >=, ^, (~), (&), (|), ($),
              binomial, cmp, convert, div, divrem, factorial, fld, gcd, gcdx, lcm, mod,
              ndigits, promote_rule, rem, show, isqrt, string, isprime, powermod,
              sum, trailing_zeros, trailing_ones, count_ones, base, tryparse_internal,
-             serialize, deserialize, bin, oct, dec, hex, isequal, invmod,
-             prevpow2, nextpow2, ndigits0z, widen, signed
+             bin, oct, dec, hex, isequal, invmod, prevpow2, nextpow2, ndigits0z, widen, signed
 
 if Clong == Int32
     typealias ClongMax Union(Int8, Int16, Int32)
@@ -224,15 +223,6 @@ convert(::Type{Float16}, n::BigInt) = Float16(n,RoundNearest)
 
 promote_rule{T<:Integer}(::Type{BigInt}, ::Type{T}) = BigInt
 
-# serialization
-
-function serialize(s, n::BigInt)
-    Base.serialize_type(s, BigInt)
-    serialize(s, base(62,n))
-end
-
-deserialize(s, ::Type{BigInt}) = get(tryparse_internal(BigInt, deserialize(s), 62, true))
-
 # Binary ops
 for (fJ, fC) in ((:+, :add), (:-,:sub), (:*, :mul),
                  (:fld, :fdiv_q), (:div, :tdiv_q), (:mod, :fdiv_r), (:rem, :tdiv_r),

base/mpfr.jl

@@ -18,9 +18,8 @@ import
         gamma, lgamma, digamma, erf, erfc, zeta, eta, log1p, airyai,
         eps, signbit, sin, cos, tan, sec, csc, cot, acos, asin, atan,
         cosh, sinh, tanh, sech, csch, coth, acosh, asinh, atanh, atan2,
-        serialize, deserialize, cbrt, typemax, typemin, unsafe_trunc,
-        realmin, realmax, get_rounding, set_rounding, maxintfloat, widen,
-        significand, frexp, tryparse
+        cbrt, typemax, typemin, unsafe_trunc, realmin, realmax, get_rounding,
+        set_rounding, maxintfloat, widen, significand, frexp, tryparse
 
 import Base.Rounding: get_rounding_raw, set_rounding_raw
 
@@ -197,15 +196,6 @@ function convert(::Type{Rational{BigInt}}, x::FloatingPoint)
     BigInt(ldexp(x,s)) // (BigInt(1) << s)
 end
 
-# serialization
-
-function serialize(s, n::BigFloat)
-    Base.serialize_type(s, BigFloat)
-    serialize(s, string(n))
-end
-
-deserialize(s, ::Type{BigFloat}) = BigFloat(deserialize(s))
-
 # Basic arithmetic without promotion
 for (fJ, fC) in ((:+,:add), (:*,:mul))
     @eval begin

base/regex.jl

@@ -244,16 +244,3 @@ function eachmatch(re::Regex, str::AbstractString, ovr::Bool=false)
 end
 
 eachmatch(re::Regex, str::AbstractString) = RegexMatchIterator(re,str)
-
-# Don't serialize the pointers
-function serialize(s, r::Regex)
-    serialize_type(s, typeof(r))
-    serialize(s, r.pattern)
-    serialize(s, r.options)
-end
-
-function deserialize(s, t::Type{Regex})
-    pattern = deserialize(s)
-    options = deserialize(s)
-    Regex(pattern, options)
-end

base/serialize.jl

@@ -1,3 +1,9 @@
+module Serializer
+
+import Base: GMP, Bottom, svec, unsafe_convert, uncompressed_ast
+
+export serialize, deserialize
+
 ## serializing values ##
 
 # dummy types to tell number of bytes used to store length (4 or 1)
@@ -158,6 +164,29 @@ function serialize{T,N,A<:Array}(s, a::SubArray{T,N,A})
     serialize_array_data(s, a)
 end
 
+function serialize{T<:AbstractString}(s, ss::SubString{T})
+    # avoid saving a copy of the parent string, keeping the type of ss
+    invoke(serialize, (Any,Any), s, convert(SubString{T}, convert(T,ss)))
+end
+
+# Don't serialize the pointers
+function serialize(s, r::Regex)
+    Serializer.serialize_type(s, typeof(r))
+    serialize(s, r.pattern)
+    serialize(s, r.options)
+end
+
+function serialize(s, n::BigInt)
+    Serializer.serialize_type(s, BigInt)
+    serialize(s, base(62,n))
+end
+
+
+function serialize(s, n::BigFloat)
+    Serializer.serialize_type(s, BigFloat)
+    serialize(s, string(n))
+end
+
 function serialize(s, e::Expr)
     l = length(e.args)
     if l <= 255
@@ -174,6 +203,26 @@ function serialize(s, e::Expr)
     end
 end
 
+function serialize(s, t::Dict)
+    serialize_type(s, typeof(t))
+    write(s, Int32(length(t)))
+    for (k,v) in t
+        serialize(s, k)
+        serialize(s, v)
+    end
+end
+
+function deserialize{K,V}(s, T::Type{Dict{K,V}})
+    n = read(s, Int32)
+    t = T(); sizehint!(t, n)
+    for i = 1:n
+        k = deserialize(s)
+        v = deserialize(s)
+        t[k] = v
+    end
+    return t
+end
+
 function serialize_mod_names(s, m::Module)
     p = module_parent(m)
     if m !== p
@@ -572,3 +621,18 @@ function deserialize(s, t::DataType)
         return x
     end
 end
+
+deserialize(s, ::Type{BigFloat}) = BigFloat(deserialize(s))
+
+deserialize(s, ::Type{BigInt}) = get(GMP.tryparse_internal(BigInt, deserialize(s), 62, true))
+
+deserialize(s, ::Type{BigInt}) = get(GMP.tryparse_internal(BigInt, deserialize(s), 62, true))
+
+function deserialize(s, t::Type{Regex})
+    pattern = deserialize(s)
+    options = deserialize(s)
+    Regex(pattern, options)
+end
+
+
+end

base/sharedarray.jl

@@ -188,10 +188,10 @@ end
 # Don't serialize s (it is the complete array) and
 # pidx, which is relevant to the current process only
 function serialize(s, S::SharedArray)
-    serialize_type(s, typeof(S))
+    Serializer.serialize_type(s, typeof(S))
     for n in SharedArray.name.names
         if n in [:s, :pidx, :loc_subarr_1d]
-            writetag(s, UndefRefTag)
+            Serializer.writetag(s, Serializer.UndefRefTag)
         else
             serialize(s, getfield(S, n))
         end

base/string.jl

@@ -634,11 +634,6 @@ convert{T<:AbstractString}(::Type{SubString{T}}, s::T) = SubString(s, 1, endof(s
 
 bytestring{T <: ByteString}(p::SubString{T}) = bytestring(pointer(p.string.data)+p.offset, nextind(p, p.endof)-1)
 
-function serialize{T}(s, ss::SubString{T})
-    # avoid saving a copy of the parent string, keeping the type of ss
-    invoke(serialize, (Any,Any), s, convert(SubString{T}, convert(T,ss)))
-end
-
 function getindex(s::AbstractString, r::UnitRange{Int})
     if first(r) < 1 || endof(s) < last(r)
         throw(BoundsError(s, r))

base/sysimg.jl

@@ -202,6 +202,7 @@ importall .Enums
 
 # concurrency and parallelism
 include("serialize.jl")
+importall .Serializer
 include("multi.jl")
 include("managers.jl")
 

test/remote.jl

@@ -1,7 +1,7 @@
 # Check that serializer hasn't gone out-of-frame
-@test Base.ser_tag[Symbol] == 2
-@test Base.ser_tag[()] == 47
-@test Base.ser_tag[false] == 123
+@test Serializer.ser_tag[Symbol] == 2
+@test Serializer.ser_tag[()] == 47
+@test Serializer.ser_tag[false] == 123
 
 # issue #1770
 let

test/serialize.jl

@@ -8,20 +8,20 @@ end
 
 # Tags
 create_serialization_stream() do s
-    Base.writetag(s, Bool)
-    @test takebuf_array(s)[end] == UInt8(Base.ser_tag[Bool])
+    Serializer.writetag(s, Bool)
+    @test takebuf_array(s)[end] == UInt8(Serializer.ser_tag[Bool])
 end
 
 create_serialization_stream() do s
-    Base.write_as_tag(s, Bool)
-    @test takebuf_array(s)[end] == UInt8(Base.ser_tag[Bool])
+    Serializer.write_as_tag(s, Bool)
+    @test takebuf_array(s)[end] == UInt8(Serializer.ser_tag[Bool])
 end
 
 create_serialization_stream() do s
-    Base.write_as_tag(s, Symbol)
+    Serializer.write_as_tag(s, Symbol)
     data = takebuf_array(s)
     @test data[end-1] == 0x00
-    @test data[end] == UInt8(Base.ser_tag[Symbol])
+    @test data[end] == UInt8(Serializer.ser_tag[Symbol])
 end
 
 # Boolean & Empty & Nothing