Skip to content

Commit aa07471

Browse files
alexrenzandrebsguedes
alexrenz
and
andrebsguedes
authoredNov 26, 2024··
Tests for SPCS Azure (#46)
Add a few tests for SPCS Azure, as introduced by [object_store_ffi#28](RelationalAI/object_store_ffi#28) --------- Co-authored-by: André Guedes <[email protected]>
·
v0.12.3v0.11.0
1 parent ff207d6 commit aa07471

File tree

4 files changed

+365
-182
lines changed

4 files changed

+365
-182
lines changed
 

‎Project.toml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name = "RustyObjectStore"
22
uuid = "1b5eed3d-1f46-4baa-87f3-a4a892b23610"
3-
version = "0.10.0"
3+
version = "0.11.0"
44

55
[deps]
66
Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -20,7 +20,7 @@ ReTestItems = "1"
2020
Sockets = "1"
2121
Test = "1"
2222
julia = "1.8"
23-
object_store_ffi_jll = "0.10.0"
23+
object_store_ffi_jll = "0.11.0"
2424

2525
[extras]
2626
CloudBase = "85eb1798-d7c4-4918-bb13-c944d38e27ed"

‎src/mock_server.jl

+50-8
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
using CloudBase: CloudCredentials, AWSCredentials, AbstractStore, AWS
1+
using CloudBase: CloudBase, CloudCredentials, AbstractStore
2+
using CloudBase: AWSCredentials, AWS
3+
using CloudBase: AzureCredentials, Azure
24
using JSON3, HTTP, Sockets, Base64
35
using RustyObjectStore: SnowflakeConfig, ClientOptions
46
using Base: UUID
@@ -118,6 +120,31 @@ function construct_stage_info(credentials::AWSCredentials, store::AWS.Bucket, pa
118120
)
119121
end
120122

123+
function construct_stage_info(credentials::AzureCredentials, store::Azure.Container, encrypted::Bool)
124+
m = match(r"(https?://.*?)/(.*)", store.baseurl)
125+
@assert !isnothing(m)
126+
test_endpoint = m.captures[1]
127+
rest = split(HTTP.unescapeuri(m.captures[2]), "/")
128+
account = rest[1]
129+
container = rest[2]
130+
131+
Dict(
132+
"locationType" => "AZURE",
133+
"location" => container * "/",
134+
"path" => container * "/",
135+
"region" => "westus2",
136+
"storageAccount" => account,
137+
"isClientSideEncrypted" => encrypted,
138+
"ciphers" => encrypted ? "AES_CBC" : nothing,
139+
"creds" => Dict(
140+
"AZURE_SAS_TOKEN" => "dummy-token",
141+
),
142+
"useS3RegionalUrl" => false,
143+
"endPoint" => "blob.core.windows.net",
144+
"testEndpoint" => test_endpoint,
145+
)
146+
end
147+
121148
function next_id_and_key(gw::SFGatewayMock)
122149
@lock gw.keys_lock begin
123150
key_id = gw.next_key_id
@@ -216,6 +243,8 @@ function start(gw::SFGatewayMock)
216243

217244
stage_info = if isa(gw.credentials, AWSCredentials) && isa(gw.store, AWS.Bucket)
218245
construct_stage_info(gw.credentials, gw.store, stage_path(stage), gw.encrypted)
246+
elseif isa(gw.credentials, AzureCredentials) && isa(gw.store, Azure.Container)
247+
construct_stage_info(gw.credentials, gw.store, gw.encrypted)
219248
else
220249
error("unimplemented")
221250
end
@@ -251,18 +280,31 @@ function start(gw::SFGatewayMock)
251280

252281
stage_info = if isa(gw.credentials, AWSCredentials) && isa(gw.store, AWS.Bucket)
253282
construct_stage_info(gw.credentials, gw.store, stage_path(stage), gw.encrypted)
283+
elseif isa(gw.credentials, AzureCredentials) && isa(gw.store, Azure.Container)
284+
construct_stage_info(gw.credentials, gw.store, gw.encrypted)
254285
else
255286
error("unimplemented")
256287
end
257288

258289
encryption_material = if gw.encrypted
259-
# fetch key id from s3 meta and return key
260-
response = AWS.head(
261-
stage_info["testEndpoint"] * "/" * stage_info["location"] * path;
262-
service="s3", region="us-east-1", credentials=gw.credentials
263-
)
264-
pos = findfirst(x -> x[1] == "x-amz-meta-x-amz-matdesc", response.headers)
265-
matdesc = JSON3.read(response.headers[pos][2])
290+
# fetch key id from blob meta and return key
291+
headers, metadata_key = if isa(gw.credentials, AWSCredentials)
292+
response = AWS.head(
293+
stage_info["testEndpoint"] * "/" * stage_info["location"] * path;
294+
service="s3", region="us-east-1", credentials=gw.credentials
295+
)
296+
response.headers, "x-amz-meta-x-amz-matdesc"
297+
elseif isa(gw.credentials, AzureCredentials)
298+
response = Azure.head(
299+
stage_info["testEndpoint"] * "/" * stage_info["storageAccount"] * "/" * stage_info["location"] * path;
300+
service="blob", region="westus2", credentials=gw.credentials
301+
)
302+
response.headers, "x-ms-meta-matdesc"
303+
else
304+
error("unknown credentials type: $(typeof(gw.credentials))")
305+
end
306+
pos = findfirst(x -> x[1] == metadata_key, headers)
307+
matdesc = JSON3.read(headers[pos][2])
266308
key_id = matdesc["queryId"]
267309
key = find_key_by_id(gw, key_id)
268310
Dict(

‎test/basic_unified_tests.jl

+36-2
Original file line numberDiff line numberDiff line change
@@ -776,7 +776,7 @@ Minio.with(; debug=true, public=true) do conf
776776
end # Minio.with
777777
end # @testitem
778778

779-
@testitem "Basic Snowflake Stage usage" setup=[InitializeObjectStore, SnowflakeMock, ReadWriteCases] begin
779+
@testitem "Basic Snowflake Stage usage: AWS, non-encrypted" setup=[InitializeObjectStore, SnowflakeMock, ReadWriteCases] begin
780780
using CloudBase.CloudTest: Minio
781781
using RustyObjectStore: SnowflakeConfig, ClientOptions
782782

@@ -793,7 +793,7 @@ Minio.with(; debug=true, public=false) do conf
793793
end # Minio.with
794794
end # @testitem
795795

796-
@testitem "Basic Snowflake Stage usage (encrypted)" setup=[InitializeObjectStore, SnowflakeMock, ReadWriteCases] begin
796+
@testitem "Basic Snowflake Stage usage: AWS, encrypted" setup=[InitializeObjectStore, SnowflakeMock, ReadWriteCases] begin
797797
using CloudBase.CloudTest: Minio
798798
using RustyObjectStore: SnowflakeConfig, ClientOptions
799799

@@ -809,3 +809,37 @@ Minio.with(; debug=true, public=false) do conf
809809
end
810810
end # Minio.with
811811
end # @testitem
812+
813+
@testitem "Basic Snowflake Stage usage: Azure, non-encrypted" setup=[InitializeObjectStore, SnowflakeMock, ReadWriteCases] begin
814+
using CloudBase.CloudTest: Azurite
815+
using RustyObjectStore: SnowflakeConfig, ClientOptions
816+
817+
# For interactive testing, use Azurite.run() instead of Azurite.with()
818+
# conf, p = Azurite.run(; debug=true, public=false); atexit(() -> kill(p))
819+
Azurite.with(; debug=true, public=false) do conf
820+
credentials, container = conf
821+
with(SFGatewayMock(credentials, container, false)) do config::SnowflakeConfig
822+
run_read_write_test_cases(config)
823+
run_stream_test_cases(config)
824+
run_list_test_cases(config)
825+
run_sanity_test_cases(config)
826+
end
827+
end # Azurite.with
828+
end # @testitem
829+
830+
@testitem "Basic Snowflake Stage usage: Azure, encrypted" setup=[InitializeObjectStore, SnowflakeMock, ReadWriteCases] begin
831+
using CloudBase.CloudTest: Azurite
832+
using RustyObjectStore: SnowflakeConfig, ClientOptions
833+
834+
# For interactive testing, use Azurite.run() instead of Azurite.with()
835+
# conf, p = Azurite.run(; debug=true, public=false); atexit(() -> kill(p))
836+
Azurite.with(; debug=true, public=false) do conf
837+
credentials, container = conf
838+
with(SFGatewayMock(credentials, container, true)) do config::SnowflakeConfig
839+
run_read_write_test_cases(config)
840+
run_stream_test_cases(config)
841+
run_list_test_cases(config; strict_entry_size=false)
842+
run_sanity_test_cases(config)
843+
end
844+
end # Azurite.with
845+
end # @testitem

‎test/snowflake_stage_exception_tests.jl

+277-170
Large diffs are not rendered by default.

2 commit comments

Comments
 (2)

alexrenz commented on Nov 26, 2024

@alexrenz
alexrenz
on Nov 26, 2024
MemberAuthor

@JuliaRegistrator register

JuliaRegistrator commented on Nov 26, 2024

@JuliaRegistrator
JuliaRegistrator
on Nov 26, 2024

Registration pull request created: JuliaRegistries/General/120189

Tip: Release Notes

Did you know you can add release notes too? Just add markdown formatted text underneath the comment after the text
"Release notes:" and it will be added to the registry PR, and if TagBot is installed it will also be added to the
release that TagBot creates. i.e.

@JuliaRegistrator register

Release notes:

## Breaking changes

- blah

To add them here just re-invoke and the PR will be updated.

Tagging

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.11.0 -m "<description of version>" aa07471106f4722d66742be8ff0d9b4e1f2be2e2
git push origin v0.11.0
Please sign in to comment.