Merge pull request #851 from OpenGeoscience/cmake-external-data

Refactor how we fetch external data.
OpenGeoscience · Jul 12, 2018 · 318c985 · 318c985
2 parents a56a346 + 5803c2e
commit 318c985
Show file tree

Hide file tree

Showing 72 changed files with 51 additions and 206 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -41,13 +41,6 @@ function(add_geojs_test test_name)
   set_property(TEST "notes-report" APPEND PROPERTY DEPENDS "${test_name}")
 endfunction()
 
-# set variables for girder.cmake
-set(Girder_KEY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/testing/test-data")
-set(Girder_DATA_DIR "${GEOJS_DEPLOY_DIR}/data")
-include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/girder.cmake)
-
-add_download_target()
-
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testing/test-runners/geojs_test_runner.py.in
                ${CMAKE_CURRENT_BINARY_DIR}/test/geojs_test_runner.py
 )
@@ -73,12 +66,37 @@ add_test(
 )
 set_property(TEST "notes-report" APPEND PROPERTY DEPENDS "notes-reset")
 
-add_custom_target(
-  data_files
-  ALL
-  DEPENDS ${Girder_DOWNLOAD_FILES}
-)
-add_test(NAME get_data_files COMMAND "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target data_files)
+# Use ExternalData to download all referenced files in tests/external-data
+include(ExternalData)
+set(ExternalData_URL_TEMPLATES "https://data.kitware.com/api/v1/file/hashsum/%(algo)/%(hash)/download")
+set(ExternalData_BINARY_ROOT "${GEOJS_DEPLOY_DIR}/data")
+set(ExternalData_SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/tests/external-data")
+# Get a list of files to download based on the name *.sha512
+file(GLOB data_files_list "${ExternalData_SOURCE_ROOT}/*.sha512")
+# Obtain the files using ExternalData
+foreach(data_file ${data_files_list})
+  string(REGEX REPLACE "^(.*)\\.sha512$" "\\1" base_file "${data_file}")
+  ExternalData_Expand_Arguments(data_files tmp DATA{${base_file}})
+endforeach()
+# Add a target to get the files
+ExternalData_Add_Target(data_files)
+# For any file that is a .tgz file, expand it after we get it
+foreach(data_file ${data_files_list})
+  string(REGEX REPLACE "^(.*)\\.sha512$" "\\1" base_file "${data_file}")
+  get_filename_component(base_file "${base_file}" NAME)
+  get_filename_component(base_fileext "${base_file}" EXT)
+  if("${base_fileext}" STREQUAL ".tgz")
+    get_filename_component(base_filename "${base_file}" NAME_WE)
+    add_custom_command(
+      TARGET data_files POST_BUILD
+      DEPENDS "${ExternalData_BINARY_ROOT}/${base_file}"
+      COMMAND ${CMAKE_COMMAND} -E make_directory "${ExternalData_BINARY_ROOT}/${base_filename}"
+      COMMAND ${CMAKE_COMMAND} -E chdir "${ExternalData_BINARY_ROOT}/${base_filename}" tar zxf "${ExternalData_BINARY_ROOT}/${base_file}"
+    )
+  endif()
+endforeach()
+# Create a test that gets this target
+add_test(NAME get_data_files COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" --target data_files)
 
 if(HEADLESS_TESTS)
   add_test(

diff --git a/cmake/configure-js-coverage-test.cmake b/cmake/configure-js-coverage-test.cmake
diff --git a/cmake/configure-js-unit-test.cmake b/cmake/configure-js-unit-test.cmake
diff --git a/cmake/girder.cmake b/cmake/girder.cmake
diff --git a/testing/test-data/AdderallCities2015.csv.md5 b/testing/test-data/AdderallCities2015.csv.md5
diff --git a/testing/test-data/AdderallCities2015.csv.url b/testing/test-data/AdderallCities2015.csv.url
diff --git a/testing/test-data/base-images.tgz.md5 b/testing/test-data/base-images.tgz.md5
diff --git a/testing/test-data/base-images.tgz.url b/testing/test-data/base-images.tgz.url
diff --git a/testing/test-data/blue.jpg.md5 b/testing/test-data/blue.jpg.md5
diff --git a/testing/test-data/blue.jpg.url b/testing/test-data/blue.jpg.url
diff --git a/testing/test-data/cities.csv.md5 b/testing/test-data/cities.csv.md5
diff --git a/testing/test-data/cities.csv.url b/testing/test-data/cities.csv.url
diff --git a/testing/test-data/countries.json.md5 b/testing/test-data/countries.json.md5
diff --git a/testing/test-data/countries.json.url b/testing/test-data/countries.json.url
diff --git a/testing/test-data/earthquakes-video.webm.md5 b/testing/test-data/earthquakes-video.webm.md5
diff --git a/testing/test-data/earthquakes-video.webm.url b/testing/test-data/earthquakes-video.webm.url
diff --git a/testing/test-data/earthquakes.json.md5 b/testing/test-data/earthquakes.json.md5
diff --git a/testing/test-data/earthquakes.json.url b/testing/test-data/earthquakes.json.url
diff --git a/testing/test-data/grid.jpg.md5 b/testing/test-data/grid.jpg.md5
diff --git a/testing/test-data/grid.jpg.url b/testing/test-data/grid.jpg.url
diff --git a/testing/test-data/hurricanes.json.md5 b/testing/test-data/hurricanes.json.md5
diff --git a/testing/test-data/hurricanes.json.url b/testing/test-data/hurricanes.json.url
diff --git a/testing/test-data/land_polygons.json.md5 b/testing/test-data/land_polygons.json.md5
diff --git a/testing/test-data/land_polygons.json.url b/testing/test-data/land_polygons.json.url
diff --git a/testing/test-data/land_shallow_topo_2048.png.md5 b/testing/test-data/land_shallow_topo_2048.png.md5
diff --git a/testing/test-data/land_shallow_topo_2048.png.url b/testing/test-data/land_shallow_topo_2048.png.url
diff --git a/testing/test-data/oahu-dense.json.md5 b/testing/test-data/oahu-dense.json.md5
diff --git a/testing/test-data/oahu-dense.json.url b/testing/test-data/oahu-dense.json.url
diff --git a/testing/test-data/oahu-medium.json.md5 b/testing/test-data/oahu-medium.json.md5
diff --git a/testing/test-data/oahu-medium.json.url b/testing/test-data/oahu-medium.json.url
diff --git a/testing/test-data/oahu.json.md5 b/testing/test-data/oahu.json.md5
diff --git a/testing/test-data/oahu.json.url b/testing/test-data/oahu.json.url
diff --git a/testing/test-data/red.jpg.md5 b/testing/test-data/red.jpg.md5
diff --git a/testing/test-data/red.jpg.url b/testing/test-data/red.jpg.url
diff --git a/testing/test-data/roads.json.md5 b/testing/test-data/roads.json.md5
diff --git a/testing/test-data/roads.json.url b/testing/test-data/roads.json.url
diff --git a/testing/test-data/sample.json.md5 b/testing/test-data/sample.json.md5
diff --git a/testing/test-data/sample.json.url b/testing/test-data/sample.json.url
diff --git a/testing/test-data/spark.png.md5 b/testing/test-data/spark.png.md5
diff --git a/testing/test-data/spark.png.url b/testing/test-data/spark.png.url
diff --git a/testing/test-data/temperature_data.tsv.md5 b/testing/test-data/temperature_data.tsv.md5
diff --git a/testing/test-data/temperature_data.tsv.url b/testing/test-data/temperature_data.tsv.url
diff --git a/testing/test-data/tilefancy.png.md5 b/testing/test-data/tilefancy.png.md5
diff --git a/testing/test-data/tilefancy.png.url b/testing/test-data/tilefancy.png.url
diff --git a/testing/test-data/tiles.tgz.md5 b/testing/test-data/tiles.tgz.md5
diff --git a/testing/test-data/tiles.tgz.url b/testing/test-data/tiles.tgz.url
diff --git a/testing/test-data/vtkCube.dat.md5 b/testing/test-data/vtkCube.dat.md5
diff --git a/testing/test-data/vtkCube.dat.url b/testing/test-data/vtkCube.dat.url
diff --git a/testing/test-data/vtkSceneMetadata.json.md5 b/testing/test-data/vtkSceneMetadata.json.md5
diff --git a/testing/test-data/vtkSceneMetadata.json.url b/testing/test-data/vtkSceneMetadata.json.url
diff --git a/testing/test-data/white.jpg.md5 b/testing/test-data/white.jpg.md5
diff --git a/testing/test-data/white.jpg.url b/testing/test-data/white.jpg.url
diff --git a/tests/external-data/AdderallCities2015.csv.sha512 b/tests/external-data/AdderallCities2015.csv.sha512
@@ -0,0 +1 @@
+c3e984482cc6db1193a6dca2a55396a2baad8541a5c8c679f33750b76f766f40a119ec3e63abbabcd095d752c3df8ce21bf24cbe629502121f24ba90b4b0674c
diff --git a/tests/external-data/base-images.tgz.sha512 b/tests/external-data/base-images.tgz.sha512
@@ -0,0 +1 @@
+d7c9ac0d2324d047ee687fa0c1f857b24b03beae3cb84060d7507c53aca2b8a184abdcb6161ceb2cb238219b8235cfab75f61d9a1a2ff9fcf521b3f96c5f6c84
diff --git a/tests/external-data/blue.jpg.sha512 b/tests/external-data/blue.jpg.sha512
@@ -0,0 +1 @@
+867b1f3c568289efc7d0dba97d827a2bc4d83a7465cebcb3b5aec7bac6a38cf70d037d1814402bc97ad1f2f6737cfb5ce97db0a4fb53a716e77fd3ba57a7ab3b
diff --git a/tests/external-data/cities.csv.sha512 b/tests/external-data/cities.csv.sha512
@@ -0,0 +1 @@
+5a665e5feda24f28e5cf4ed0801b67e73bbcf3ea781b2e50d11284214e67b25b68e6a1c48da46e5e4d4d0c54c2ec18f88d292224b4541fb279396cf7b94beac9
diff --git a/tests/external-data/earthquakes-video.webm.sha512 b/tests/external-data/earthquakes-video.webm.sha512
@@ -0,0 +1 @@
+834a9d05f5fb00145e529fa4b217398db0df188c69d751f3869128b6e9c92d3000f85378752c56d9d9b5fa0870437dd9bdfeb5d62f6c87c2c03a7f1a20ee8523
diff --git a/tests/external-data/earthquakes.json.sha512 b/tests/external-data/earthquakes.json.sha512
@@ -0,0 +1 @@
+f098b6437411384b552419b4a36264c1bb3fed816ccfe9545145175e0b92a0b7ad5ebdcb9dddd0a12a90499143ffa471c02f6e049be5b973db607ff066892500
diff --git a/tests/external-data/grid.jpg.sha512 b/tests/external-data/grid.jpg.sha512
@@ -0,0 +1 @@
+60d201a14c7d31e7881301e6784e0372ddf27f26e5e4eafba9ba39158dfd050e3683faaa660fcde47e6c994dd3ee64c5a5231474ca75090053ef9207fedd9029
diff --git a/tests/external-data/hurricanes.json.sha512 b/tests/external-data/hurricanes.json.sha512
@@ -0,0 +1 @@
+012f15036bfc9ac3abb81a2a61e2c7b602ef3d8f7bd3b3f0fb66972ee69034730655f69febb2df5d657f22cd2b1e69170f6568bcbb03d7ebdecdfbafb80cc3b7
diff --git a/tests/external-data/land_polygons.json.sha512 b/tests/external-data/land_polygons.json.sha512
@@ -0,0 +1 @@
+30a828392d58678599130e0dca6d7f27e7e07e4e5b5d7f7a37871eb395d53d97b76134c0a07e805fbdfac0f42e6d3ca6e287c9727815a9dc711d541b1c8f68a1
diff --git a/tests/external-data/land_shallow_topo_2048.png.sha512 b/tests/external-data/land_shallow_topo_2048.png.sha512
@@ -0,0 +1 @@
+8a8330dba5bacdb511038ad0f6ee5a764a40aa7a8868a445749f653ae5d85d8317684ac706e7a9f049590170df6bc3fefc2912d52124d1b3b17aa43c529ff2a8
diff --git a/tests/external-data/oahu-dense.json.sha512 b/tests/external-data/oahu-dense.json.sha512
@@ -0,0 +1 @@
+692a44ec4a18b16d1530403a9a2bf286ff2ef7d45fa58f555c278f91a0fa708b5626a3f38955e06a11cdc06a1009e859328687c1f32a2169e0c8ef1b518418c1
diff --git a/tests/external-data/oahu-medium.json.sha512 b/tests/external-data/oahu-medium.json.sha512
@@ -0,0 +1 @@
+83375c5c2678b11d8de7f59fd8e2f9b889d3ec20f4a81279d996711bfc0942bd9dcdce149cbca995930459c26b3bf58be60569687085adf361ff805436b75aab
diff --git a/tests/external-data/oahu.json.sha512 b/tests/external-data/oahu.json.sha512
@@ -0,0 +1 @@
+e44282c44fa95f0b40c2135ee94ff40a755771f3b4bf9acda0eef0048cd0fb29b3a71352e2eecdca6cfc35fafde96ffb1a9658c9fee2346071808a9123c26cb5
diff --git a/tests/external-data/red.jpg.sha512 b/tests/external-data/red.jpg.sha512
@@ -0,0 +1 @@
+70884f5c3d1747633412dcf64d4886d985b49f508b68eb9f3874b2fdcfad38e7623f4a7c7355baa1656503f46e2e091576a19ded9fad7c7c65387c313206d2e6
diff --git a/tests/external-data/roads.json.sha512 b/tests/external-data/roads.json.sha512
@@ -0,0 +1 @@
+ae8d8b99c3fab73798ddcc246ea53b1bc8d598414d00df4ce697373a343fbf7ffed4dbae5f07a997ac7731b5f86140686b8425b99bd5818b8c9ec68e7c4d3315
diff --git a/tests/external-data/sample.json.sha512 b/tests/external-data/sample.json.sha512
@@ -0,0 +1 @@
+5d4b00559f17fac607eafe4a9bb933386ebd572fbe545e6138b8eafbbf510074a8ae398c7df0420a017218af83b18cc322633b77007b63a27dfea7a50db70244
diff --git a/tests/external-data/temperature_data.tsv.sha512 b/tests/external-data/temperature_data.tsv.sha512
@@ -0,0 +1 @@
+bde5cdad7c4518694393de83da938bcf5d485b41116b51f0fce4b462737992d2973f076c577040fa816873b196c6397244aa03736fc7750a0eea0ba5cbcc48e4
diff --git a/tests/external-data/tilefancy.png.sha512 b/tests/external-data/tilefancy.png.sha512
@@ -0,0 +1 @@
+455effa59d421cbb73c6def815813063f1c649363de4007fa0de00cc2e0f24cba745c046266e5a83fc43b121a648017d509d7bf03c30fbee1448817e3f849683
diff --git a/tests/external-data/tiles.tgz.sha512 b/tests/external-data/tiles.tgz.sha512
@@ -0,0 +1 @@
+30defdfd28f6773c78f31b64b728d42630583b4e2b74c7098d75e6887bf5d8a6bea08a85669ad969058ac2c98dc45d5512684c62227403e7fcf57ea4dc1675f5
diff --git a/tests/external-data/white.jpg.sha512 b/tests/external-data/white.jpg.sha512
@@ -0,0 +1 @@
+ea7a9d7ff76775e742572f89e90cce1248ec99c33b2f486e0fa1d19ab461b87dff324533ecb186a3db14e40a3826da97b5d66566360a201228f60140b0e89942
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		c3e984482cc6db1193a6dca2a55396a2baad8541a5c8c679f33750b76f766f40a119ec3e63abbabcd095d752c3df8ce21bf24cbe629502121f24ba90b4b0674c
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		d7c9ac0d2324d047ee687fa0c1f857b24b03beae3cb84060d7507c53aca2b8a184abdcb6161ceb2cb238219b8235cfab75f61d9a1a2ff9fcf521b3f96c5f6c84
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		867b1f3c568289efc7d0dba97d827a2bc4d83a7465cebcb3b5aec7bac6a38cf70d037d1814402bc97ad1f2f6737cfb5ce97db0a4fb53a716e77fd3ba57a7ab3b
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		5a665e5feda24f28e5cf4ed0801b67e73bbcf3ea781b2e50d11284214e67b25b68e6a1c48da46e5e4d4d0c54c2ec18f88d292224b4541fb279396cf7b94beac9
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		834a9d05f5fb00145e529fa4b217398db0df188c69d751f3869128b6e9c92d3000f85378752c56d9d9b5fa0870437dd9bdfeb5d62f6c87c2c03a7f1a20ee8523
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		f098b6437411384b552419b4a36264c1bb3fed816ccfe9545145175e0b92a0b7ad5ebdcb9dddd0a12a90499143ffa471c02f6e049be5b973db607ff066892500
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		60d201a14c7d31e7881301e6784e0372ddf27f26e5e4eafba9ba39158dfd050e3683faaa660fcde47e6c994dd3ee64c5a5231474ca75090053ef9207fedd9029
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		012f15036bfc9ac3abb81a2a61e2c7b602ef3d8f7bd3b3f0fb66972ee69034730655f69febb2df5d657f22cd2b1e69170f6568bcbb03d7ebdecdfbafb80cc3b7
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		30a828392d58678599130e0dca6d7f27e7e07e4e5b5d7f7a37871eb395d53d97b76134c0a07e805fbdfac0f42e6d3ca6e287c9727815a9dc711d541b1c8f68a1
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		8a8330dba5bacdb511038ad0f6ee5a764a40aa7a8868a445749f653ae5d85d8317684ac706e7a9f049590170df6bc3fefc2912d52124d1b3b17aa43c529ff2a8