Fix tests, use new its-live catalog

.gitignore

@@ -1,5 +1,4 @@
-/Manifest.toml
-/docs/Manifest.toml
+Manifest.toml
 /docs/build/
 
 /test/ref.parquet/

Project.toml

@@ -24,14 +24,16 @@ Zarr = "0.9"
 julia = "1.10"
 
 [extras]
+ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3"
+AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc"
+AWSS3 = "1c724243-ef5b-51ab-93f4-b0a88ac62a95"
 CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
 PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
-NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
 Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c"
 
 [targets]
-test = ["CondaPkg", "PythonCall", "Dates", "JSON3", "NCDatasets", "Rasters", "YAXArrays", "Test"]
+test = ["ArchGDAL", "AWS", "AWSS3", "CondaPkg", "PythonCall", "Dates", "JSON3", "Rasters", "YAXArrays", "Test"]

docs/CondaPkg.toml

@@ -1,7 +1,6 @@
 [deps]
 # netcdf4 = ""
-virtualizarr = ""
-xarray = ""
+# virtualizarr = ""
 zarr = ""
 certifi = ""
 s3fs = ""

test/its_live_catalog.jl

@@ -1,10 +1,23 @@
 using JSON3, Kerchunk, Zarr, YAXArrays
 using Test
 
+# We need to set the AWS config to say we're coming from the "us-west-2" region,
+# otherwise AWS will actually refuse to recognize the s3 URL as a bucket, 
+# and instead will require us to use the HTTP endpoint, which AWSS3.jl won't support.
+import AWS
+AWS.global_aws_config(AWS.AWSConfig(; region="us-west-2"))
+
 @testset "ITS_LIVE catalog" begin
-    catalog_json = JSON3.read(open(joinpath(dirname(dirname(pathof(Kerchunk))), "test", "data", "its_live_catalog.json"))) 
+    # We make the catalog path relative to the module Kerchunk.jl, so that it can be run from the REPL.
+    catalog_path = joinpath(dirname(dirname(pathof(Kerchunk))), "test", "data", "its_live_catalog.json")
+    catalog_json = JSON3.read(open(catalog_path)) 
+    # This catalog is actually a collection of catalogs, since the data is stored in different projections.
+    # We simply access the first in the list.
     arbitrary_choice_dictionary = catalog_json[first(keys(catalog_json))]
+    # We create a ReferenceStore, which is a dictionary of references to the data.
     st = Kerchunk.ReferenceStore(arbitrary_choice_dictionary)
+    # We open the Zarr array from the ReferenceStore.
     za = Zarr.zopen(st)
+    # We test that reading works.
     @test_nowarn za["vx"][1, 1] # test that reading works
 end

test/python_local_kerchunk.jl

@@ -4,18 +4,17 @@ using Zarr, JSON3
 
 # You can't import kerchunk.hdf because importing h5py introduces a version of libhdf5 that is incompatible with any extant netcdf4_jll.
 
-
-using Rasters, NCDatasets, Dates, YAXArrays
+using Rasters, ArchGDAL, Dates, YAXArrays
 
 using Test
 
 @testset "Reading a Kerchunked NetCDF file" begin
 
     # First, we create a NetCDF dataset:
 
-    ras = Raster(rand(LinRange(0, 10, 100), X(1:100), Y(5:150), Ti(DateTime("2000-01-31"):Month(1):DateTime("2001-01-31"))))
+    ras = Raster(rand(LinRange(0, 10, 100), X(1:100), Y(5:150), Band(1:12)))
 
-    write("test.nc", ras; source = :netcdf, force = true)
+    write("test.nc", ras; source = Rasters.GDALsource(), force = true)
     @test Raster("test.nc") == ras # test read-write roundtrip
 
     # Create a Kerchunk catalog.

test/runtests.jl

@@ -3,7 +3,7 @@ using CondaPkg, PythonCall
 using Kerchunk
 
 # Raster creation and analysis packages
-using Rasters, NCDatasets # to save a raster
+using Rasters, ArchGDAL # to save a raster
 using YAXArrays # to open a raster
 
 using Test
@@ -12,16 +12,7 @@ using Test
     @static if !(Sys.iswindows())
         include("python_local_kerchunk.jl")
     end
-    if ( false ) # an ode to GEMB :P
-        # In all seriousness, this will only be possible to test when:
-        # - HTTPPaths are a thing
-        # - we update this catalog to switch all URLs from the s3 protocol to the new ITS_LIVE HTTP
-        # protocol
-        # The bucket you are attempting to access must be addressed using the specified endpoint. 
-        # Please send all future requests to this endpoint.
-        # `its-live-data.s3-us-west-2.amazonaws.com`
-        include("its_live_catalog.jl")
-    end
+    include("its_live_catalog.jl")
     include("corrections.jl")
     include("real_zarr.jl")
     end