Skip to content

Commit

Permalink
Add package extension to support InlineStrings in Arrow.jl
Browse files Browse the repository at this point in the history
Fixes apache/arrow-julia#196.

This utilizes the new package extension feature of Julia 1.9 to
add a conditional dependency on the ArrowTypes.jl package. With
ArrowTypes.jl, it adds the necessary overloads to allow round-
tripping of inline strings through the arrow format. Other language
implementations will read them as normal strings, but in the Julia
implementation, the additional type metadata signal that these strings
were originally inline strings and can be deserialized as such.

I'm explicitly not using the Requires.jl hack for backwards compat w/
older Julia versions because I like the idea of this being sort of a
"beta" feature for users already using 1.9 to see if there are any
unexpected issues that pop up for inline strings in the arrow format.
  • Loading branch information
quinnj committed Jun 14, 2023
1 parent b7923a2 commit d8094b8
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 1 deletion.
9 changes: 8 additions & 1 deletion Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ version = "1.4.0"
Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"

[extras]
Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
Expand All @@ -15,5 +16,11 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
Parsers = "2"
julia = "1.6"

[weakdeps]
ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"

[extensions]
ArrowTypesExt = "ArrowTypes"

[targets]
test = ["Test", "Random", "Serialization"]
test = ["Arrow", "Test", "Random", "Serialization"]
15 changes: 15 additions & 0 deletions ext/ArrowTypesExt.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
module ArrowTypesExt

using ArrowTypes, InlineStrings

for sz in (1, 4, 8, 16, 32, 64, 128, 256)
nm = Symbol(:InlineString, max(1, sz - 1))
arrow_nm = Symbol("JuliaLang.InlineStrings.", nm)
@eval begin
ArrowTypes.arrowname(::Type{$nm}) = $(Meta.quot(arrow_nm))
ArrowTypes.JuliaType(::Val{$(Meta.quot(arrow_nm))}) = $nm
ArrowTypes.fromarrow(::Type{$nm}, ptr::Ptr{UInt8}, len::Int) = $nm(ptr, len)
end
end

end
8 changes: 8 additions & 0 deletions ext/tests.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
using Test, Arrow, InlineStrings

@testset "basic Arrow.jl interop" begin
t = (x = inlinestrings(["a", "b", "sailor"]),)
t2 = Arrow.Table(Arrow.tobuffer(t))
@test isequal(t.x, t2.x)
@test t2.x[1] isa InlineString
end
2 changes: 2 additions & 0 deletions test/runtests.jl
Original file line number Diff line number Diff line change
Expand Up @@ -562,3 +562,5 @@ end
@test repr(String31["foo", "bar"]) == "String31[\"foo\", \"bar\"]"
@test repr(InlineString[inline1"a", inline15"a"]) == "InlineString[String1(\"a\"), String15(\"a\")]"
end

include(joinpath(dirname(pathof(InlineStrings)), "../ext/tests.jl"))

0 comments on commit d8094b8

Please sign in to comment.