Rename utf->unicode, merge cstring into utf32, remove RepStrings & RopeStrings

Author: ScottPJones

base/RepStrings.jl

@@ -4,6 +4,5 @@ include("strings/basic.jl")
 include("strings/search.jl")
 include("strings/parse.jl")
 include("strings/strutil.jl")
-include("strings/cstring.jl")
 include("strings/strio.jl")
 

base/RopeStrings.jl

@@ -51,6 +51,16 @@ eltype{T<:AbstractString}(::Type{T}) = Char
 (*)(s1::AbstractString, ss::AbstractString...) = string(s1, ss...)
 (^)(s::AbstractString, r::Integer) = repeat(s,r)
 
+function repeat(s::ByteString, r::Integer)
+    r < 0 && throw(ArgumentError("can't repeat a string $r times"))
+    d = s.data; n = length(d)
+    out = Array(UInt8, n*r)
+    for i=1:r
+        copy!(out, 1+(i-1)*n, d, 1, n)
+    end
+    convert(typeof(s), out)
+end
+
 length(s::DirectIndexString) = endof(s)
 function length(s::AbstractString)
     i = start(s)

base/string.jl

@@ -90,11 +90,9 @@ include("osutils.jl")
 # strings & printing
 include("SubStrings.jl")
 include("RevStrings.jl")
-include("RepStrings.jl")
-include("RopeStrings.jl")
 include("char.jl")
 include("ascii.jl")
-include("utf.jl")
+include("unicode.jl")
 include("iobuffer.jl")
 include("string.jl")
 include("shell.jl")

base/strings/basic.jl

@@ -0,0 +1,10 @@
+# This file is a part of Julia. License is MIT: http://julialang.org/license
+
+include("unicode/UnicodeError.jl")
+include("unicode/types.jl")
+include("unicode/checkstring.jl")
+include("unicode/utf8.jl")
+include("unicode/utf16.jl")
+include("unicode/utf32.jl")
+include("unicode/utf8proc.jl")
+importall .UTF8proc

base/sysimg.jl

@@ -100,3 +100,37 @@ function map(f, s::UTF32String)
     end
     UTF32String(out)
 end
+
+# Definitions for C compatible strings, that don't allow embedded
+# '\0', and which are terminated by a '\0'
+
+containsnul(s::AbstractString) = '\0' in s
+containsnul(s::ByteString) = containsnul(unsafe_convert(Ptr{Cchar}, s), sizeof(s))
+containsnul(s::Union{UTF16String,UTF32String}) = findfirst(s.data, 0) != length(s.data)
+
+if sizeof(Cwchar_t) == 2
+    const WString = UTF16String
+    const wstring = utf16
+elseif sizeof(Cwchar_t) == 4
+    const WString = UTF32String
+    const wstring = utf32
+end
+wstring(s::Cwstring) = wstring(box(Ptr{Cwchar_t}, unbox(Cwstring,s)))
+
+# Cwstring is defined in c.jl, but conversion needs to be defined here
+# to have WString
+function unsafe_convert(::Type{Cwstring}, s::WString)
+    if containsnul(s)
+        throw(ArgumentError("embedded NUL chars are not allowed in C strings: $(repr(s))"))
+    end
+    return Cwstring(unsafe_convert(Ptr{Cwchar_t}, s))
+end
+
+# pointer conversions of ASCII/UTF8/UTF16/UTF32 strings:
+pointer(x::Union{ByteString,UTF16String,UTF32String}) = pointer(x.data)
+pointer{T<:ByteString}(x::SubString{T}) = pointer(x.string.data) + x.offset
+pointer(x::ByteString, i::Integer) = pointer(x.data)+(i-1)
+pointer{T<:ByteString}(x::SubString{T}, i::Integer) = pointer(x.string.data) + x.offset + (i-1)
+pointer(x::Union{UTF16String,UTF32String}, i::Integer) = pointer(x)+(i-1)*sizeof(eltype(x.data))
+pointer{T<:Union{UTF16String,UTF32String}}(x::SubString{T}) = pointer(x.string.data) + x.offset*sizeof(eltype(x.data))
+pointer{T<:Union{UTF16String,UTF32String}}(x::SubString{T}, i::Integer) = pointer(x.string.data) + (x.offset + (i-1))*sizeof(eltype(x.data))

base/unicode.jl

@@ -15,9 +15,8 @@ Upon return, `tests` is a vector of fully-expanded test names, and
 """ ->
 function choosetests(choices = [])
     testnames = [
-        "linalg", "core", "keywordargs", "numbers",
-        "string", "utf", "triplequote", "printf",
-        "RepStrings", "RopeStrings", "SubStrings", "RevStrings",
+        "linalg", "core", "keywordargs", "numbers", "string", "unicode", "char",
+        "SubStrings", "RevStrings", "triplequote", "printf",
         "dates", "dict", "hashing", "remote", "iobuffer", "staged",
         "arrayops", "tuple", "subarray", "reduce", "reducedim", "random",
         "abstractarray", "intfuncs", "simdloop", "blas", "sparse",
@@ -31,7 +30,7 @@ function choosetests(choices = [])
         "euler", "show", "lineedit", "replcompletions", "repl",
         "replutil", "sets", "test", "goto", "llvmcall", "grisu",
         "nullable", "meta", "profile", "libgit2", "docs", "markdown",
-        "base64", "parser", "serialize", "functors", "char", "misc",
+        "base64", "parser", "serialize", "functors", "misc",
         "enums", "cmdlineargs", "i18n"
     ]
 

base/utf/UnicodeError.jl renamed to base/unicode/UnicodeError.jl

@@ -79,7 +79,6 @@ for a in vals, b in vals
     @test isequal(a,b) == (hash(a)==hash(b))
 end
 
-@test hash(RopeString("1","2")) == hash("12")
 @test hash(SubString("--hello--",3,7)) == hash("hello")
 @test hash(:(X.x)) == hash(:(X.x))
 @test hash(:(X.x)) != hash(:(X.y))

base/utf/checkstring.jl renamed to base/unicode/checkstring.jl

@@ -4,5 +4,4 @@ include("strings/basic.jl")
 include("strings/search.jl")
 include("strings/parse.jl")
 include("strings/strutil.jl")
-include("strings/cstring.jl")
 include("strings/strio.jl")

base/utf/types.jl renamed to base/unicode/types.jl

@@ -0,0 +1,6 @@
+# This file is a part of Julia. License is MIT: http://julialang.org/license
+
+include("unicode/checkstring.jl")
+include("unicode/utf16.jl")
+include("unicode/utf32.jl")
+include("unicode/utf8proc.jl")

base/utf/utf16.jl renamed to base/unicode/utf16.jl

@@ -11,3 +11,9 @@ u32 = utf32(u8)
 @test u8 == utf32(u32.data[1:end-1]) == utf32(copy!(Array(UInt8, 20), 1, reinterpret(UInt8, u32.data), 1, 20))
 @test u8 == utf32(pointer(u32)) == utf32(convert(Ptr{Int32}, pointer(u32)))
 @test_throws UnicodeError utf32(UInt8[1,2,3])
+
+# Wstring
+u8 = "\U10ffff\U1d565\U1d7f6\U00066\U2008a"
+w = wstring(u8)
+@test length(w) == 5 && utf8(w) == u8 && collect(u8) == collect(w)
+@test u8 == WString(w.data)