fix for NASA earth data

JuliaClimate · May 15, 2024 · e163edb · e163edb
1 parent 9cc9f60
commit e163edb
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 19 deletions.
diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ Pkg.add("STAC")
 
 ## Example
 
-Accessing a catalog and sub-catalogs are indexed with their identitiers. To find all subcatalog identifiers, one can simply display the catalog structure in a julia session. 
+Accessing a catalog and sub-catalogs are indexed with their identitiers. To find all subcatalog identifiers, one can simply display the catalog structure in a julia session.
 
 ``` julia
 using STAC
@@ -56,28 +56,28 @@ search_results = collect(search(catalog, collections, lon_range, lat_range, time
 
 ### NASA EarthData
 
-Retrieve a list of OPeNDAP URLs from the NASA [Common Metadata Repository (CMR)](https://www.earthdata.nasa.gov/eosdis/science-system-description/eosdis-components/cmr) of the collection [C1996881146-POCLOUD](https://cmr.earthdata.nasa.gov/search/concepts/C1996881146-POCLOUD.html). A token is obtained from [https://urs.earthdata.nasa.gov/home](https://urs.earthdata.nasa.gov/home) (after registration and login) and clicking on `Generate Token`:
+Retrieve a list of OPeNDAP URLs from the NASA [Common Metadata Repository (CMR)](https://www.earthdata.nasa.gov/eosdis/science-system-description/eosdis-components/cmr) of the collection [C1996881146-POCLOUD](https://cmr.earthdata.nasa.gov/search/concepts/C1996881146-POCLOUD.html). If asked, a token can be obtained from [https://urs.earthdata.nasa.gov/home](https://urs.earthdata.nasa.gov/home) (after registration and login) and clicking on `Generate Token`:
 
 
 ```julia
-using STAC, URIs, Dates
+using STAC, Dates
 
-token = "put_your_user_token_here"
 timerange = (DateTime(2019,1,1),DateTime(2019,12,31))
 collection_concept_id = "C1996881146-POCLOUD"
 baseurl = "https://cmr.earthdata.nasa.gov/search/granules.stac"
 
-url = string(URI(URI(baseurl), query = Dict(
+query = Dict(
     "collection_concept_id" => collection_concept_id,
     "temporal" => join(string.(timerange),','),
-    "pageSize" => 1000, # default is 100
-    "token" => token)))
+    "pageSize" => 1000,
+)
 
-collection = STAC.FeatureCollection(url)
-opendap_url = [href(item.assets["opendap"]) for item in collection]
+url = baseurl
+collection = STAC.FeatureCollection(url,query)
 
+opendap_url = [href(item.assets["opendap"]) for item in collection]
 @show length(opendap_url)
 # output 365, one URL per day
 ```
 
-To load the dataset, the NetCDF library need to be made aware of your EarthData username and password as explained [here](https://alexander-barth.github.io/NCDatasets.jl/latest/tutorials/#NASA-EarthData). 
+To load the dataset, the NetCDF library need to be made aware of your EarthData username and password as explained [here](https://alexander-barth.github.io/NCDatasets.jl/latest/tutorials/#NASA-EarthData).
diff --git a/src/search.jl b/src/search.jl
@@ -1,27 +1,58 @@
 
+function FeatureCollection(url,query; method=:get, _enctype = :form_urlencoded)
+    next_request =
+        if method == :get
+            Dict(
+                :method => "GET",
+                :href => string(URI(URI(url), query = query)))
+        else
+            Dict(
+                :method => "POST",
+                :href => url,
+                :body => query
+            )
+        end
 
-function FeatureCollection(url,query)
     ch = Channel{STAC.Item}() do c
         while true
-            @debug "post $url" query
-            r = HTTP.post(url,[],JSON3.write(query))
+            url = next_request[:href]
+            if next_request[:method] == "POST"
+                @debug "post $url" query
+                 if _enctype == :form_urlencoded
+                     r = HTTP.post(url,[],body=next_request[:body])
+                 elseif _enctype == :json
+                     b = JSON3.write(next_request[:body])
+                     r = HTTP.post(url,[],b)
+                 else
+                     error("unknown encoding for POST $_enctype")
+                 end
+            else
+                @debug "get $url"
+                r = HTTP.get(url)
+            end
             data = JSON3.read(String(r.body))
             for d in data[:features]
                 put!(c,STAC.Item("",d,STAC._assets(d)))
             end
 
             # check if there is a next page
             next = filter(d -> get(d,"rel",nothing) == "next",data[:links])
-            # no more next page
+
             if length(next) == 0
+                # no more next page
                 break
             else
-                url = next[1][:href]
+                next_request = next[1]
+                @debug "next " next_request
             end
         end
     end
 end
 
+format_datetime(x::Union{Dates.Date,Dates.DateTime}) = Dates.format(x,Dates.dateformat"yyyy-mm-ddTHH:MM:SS.sssZ")
+format_datetime(x::Union{AbstractVector,Tuple}) = join(format_datetime.(x),"/")
+format_bbox(bbox) = join(string.(bbox),',')
+
 
 """
     search(cat::Catalog, collections, lon_range, lat_range, datetime;
@@ -75,15 +106,13 @@ search_results = collect(
 ))
 ```
 
-
+Currently only POST search requests are supported.
 """
 function search(cat::Catalog, collections, lon_range, lat_range, time_range;
                 query = nothing,
                 filter = nothing,
                 extra_query = nothing,
                 limit = 200)
-    format_datetime(x::Union{Dates.Date,Dates.DateTime}) = Dates.format(x,Dates.dateformat"yyyy-mm-ddTHH:MM:SS.sssZ")
-    format_datetime(x::Union{AbstractVector,Tuple}) = join(format_datetime.(x),"/")
 
     if collections isa AbstractString
         collections = [collections]
@@ -92,6 +121,9 @@ function search(cat::Catalog, collections, lon_range, lat_range, time_range;
     west, east = lon_range
     south, north = lat_range
 
+    # for POST request in JSON format
+    # e.g. do not join bbox elements as a string
+
     full_query = Dict(
         "collections" => collections,
         "bbox" => [west, south, east, north],
@@ -113,7 +145,11 @@ function search(cat::Catalog, collections, lon_range, lat_range, time_range;
     end
 
     @debug "full query:" full_query
-    return FeatureCollection(cat.url * "/search",full_query)
+    return FeatureCollection(
+        cat.url * "/search",full_query,
+        method = :post,
+        _enctype = :json
+    )
 end
 
 export search
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -148,3 +148,7 @@ end
     )
     @test length(search_results) == 1
 end
+
+@testset "providers" begin
+    include("test_earth_data.jl")
+end
diff --git a/test/test_earth_data.jl b/test/test_earth_data.jl
@@ -0,0 +1,19 @@
+using Test
+using Dates
+using STAC
+
+timerange = (DateTime(2019,1,1),DateTime(2019,12,31))
+collection_concept_id = "C1996881146-POCLOUD"
+baseurl = "https://cmr.earthdata.nasa.gov/search/granules.stac"
+
+query = Dict(
+    "collection_concept_id" => collection_concept_id,
+    "temporal" => join(string.(timerange),','),
+    "pageSize" => 1000,
+)
+
+url = baseurl
+collection = STAC.FeatureCollection(url,query)
+
+opendap_url = [href(item.assets["opendap"]) for item in collection]
+@test length(opendap_url) == 365