From df7eade867a16295b4b2ccaa97c2589bd102f6df Mon Sep 17 00:00:00 2001 From: "James W. McClain" Date: Tue, 23 Jul 2019 22:14:17 -0400 Subject: [PATCH 1/4] Add Metadata Tight-Loop Experiment --- .gitignore | 1 + src/experiments/thread/Makefile | 7 ++- src/experiments/thread/metadata.cpp | 90 +++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 src/experiments/thread/metadata.cpp diff --git a/.gitignore b/.gitignore index b8f5e2a..82a7546 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ src/experiments/thread/rawthread src/experiments/thread/wrapthread src/experiments/thread/pattern src/experiments/thread/oversubscribe +src/experiments/thread/metadata src/unit_tests/dataset_tests src/unit_tests/token_tests src/unit_tests/cache_tests diff --git a/src/experiments/thread/Makefile b/src/experiments/thread/Makefile index 1952368..49e9b62 100644 --- a/src/experiments/thread/Makefile +++ b/src/experiments/thread/Makefile @@ -5,7 +5,7 @@ LDFLAGS ?= $(shell pkg-config gdal --libs) -L$(shell pwd) -lgdalwarp_bindings -l BOOST_ROOT ?= /usr/include SO ?= so -all: rawthread wrapthread pattern oversubscribe +all: rawthread wrapthread pattern oversubscribe metadata ../../libgdalwarp_bindings.$(SO): $(MAKE) -C ../.. libgdalwarp_bindings.$(SO) @@ -25,6 +25,9 @@ pattern: pattern.o libgdalwarp_bindings.$(SO) oversubscribe: oversubscribe.o libgdalwarp_bindings.$(SO) $(CC) $< $(LDFLAGS) -o $@ +metadata: metadata.o libgdalwarp_bindings.$(SO) + $(CC) $< $(LDFLAGS) -lboost_timer -o $@ + %.o: %.cpp $(CXX) $(CFLAGS) $(CXXFLAGS) $(GDALCFLAGS) -I$(BOOST_ROOT) $< -c -o $@ @@ -32,6 +35,6 @@ clean: rm -f *.o libgdalwarp_bindings.$(SO) cleaner: clean - rm -f rawthread wrapthread pattern oversubscribe + rm -f rawthread wrapthread pattern oversubscribe metadata cleanest: cleaner diff --git a/src/experiments/thread/metadata.cpp b/src/experiments/thread/metadata.cpp new file mode 100644 index 0000000..a0ec517 --- /dev/null +++ b/src/experiments/thread/metadata.cpp @@ -0,0 +1,90 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include + +#include "../../bindings.h" +#include "../../locked_dataset.hpp" + +char const *options[] = { + "-dstnodata", "107", + "-tap", "-tr", "33", "42", + "-r", "bilinear", + "-t_srs", "epsg:3857", + "-co", "BLOCKXSIZE=512", "-co", "BLOCKYSIZE=512", + nullptr}; + +namespace t = boost::timer; + +void CSLDestroy(char **papszStrList) +{ + if (!papszStrList) + return; + + for (char **papszPtr = papszStrList; *papszPtr != nullptr; ++papszPtr) + { + free(*papszPtr); + } + + free(papszStrList); +} + +int main(int argc, char **argv) +{ + init(33); + + uint64_t token = get_token(argv[1], options); + auto handle = GDALOpen(argv[1], GA_ReadOnly); + + char **domain_list = NULL; + + { + t::auto_cpu_timer timer; + + for (int i = 0; i < (1 << 18); ++i) + { + assert(noop(token, locked_dataset::SOURCE, 0, 1) > 0); + } + } + + { + t::auto_cpu_timer timer; + + for (int i = 0; i < (1 << 18); ++i) + { + assert(get_metadata_domain_list(token, 0, 10, 1, 0, &domain_list) > 0); + CSLDestroy(domain_list); + } + } + { + t::auto_cpu_timer timer; + + for (int i = 0; i < (1 << 18); ++i) + { + domain_list = GDALGetMetadataDomainList(handle); + CSLDestroy(domain_list); + } + } + + GDALClose(handle); + deinit(); +} From 2605dc42cc2866627ab9213c46de030305dcd213 Mon Sep 17 00:00:00 2001 From: "James W. McClain" Date: Thu, 25 Jul 2019 19:30:56 -0400 Subject: [PATCH 2/4] Add Java Metadata Tight-Loop Test Program --- src/main/GDALWarpMetadataTest.java | 58 ++++++++++++++++++++++++++++++ src/main/Makefile | 3 ++ 2 files changed, 61 insertions(+) create mode 100644 src/main/GDALWarpMetadataTest.java diff --git a/src/main/GDALWarpMetadataTest.java b/src/main/GDALWarpMetadataTest.java new file mode 100644 index 0000000..e4d8c9c --- /dev/null +++ b/src/main/GDALWarpMetadataTest.java @@ -0,0 +1,58 @@ + +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.azavea.gdal.GDALWarp; +import java.util.Arrays; +import java.util.Random; + +class GDALWarpMetadataTest { + + private static String options[] = { /* */ + "-tap", "-tr", "5", "7", /* */ + "-r", "bilinear", /* */ + "-t_srs", "epsg:3857", /* */ + "-co", "BLOCKXSIZE=512", "-co", "BLOCKYSIZE=512" /* */ + }; + + public static void main(String[] args) throws Exception { + long token = GDALWarp.get_token(args[0], options); + long before, after; + int len = 0; + //byte[][] domain_list1 = new byte[1 << 10][1 << 10]; + + // Domain List Metadata + before = System.currentTimeMillis(); + for (int j = 0; j < (1 << 18); ++j) { + byte[][] domain_list1 = new byte[1 << 4][1 << 10]; + GDALWarp.get_metadata_domain_list(token, GDALWarp.SOURCE, 0, 0, domain_list1); + for (int i = 0; i < domain_list1.length; ++i) { + String str = new String(domain_list1[i], "UTF-8").trim(); + if (str.length() > 0) { + len = len + str.length(); + } else { + break; + } + } + } + after = System.currentTimeMillis(); + System.out.println(after - before); + + + GDALWarp.deinit(); + return; + } +} diff --git a/src/main/Makefile b/src/main/Makefile index 4879a5c..8dc433c 100644 --- a/src/main/Makefile +++ b/src/main/Makefile @@ -19,6 +19,9 @@ java/com/azavea/gdal/GDALWarp.class java/cz/adamh/utils/NativeUtils.class: java/ GDALWarpThreadTest.class: GDALWarpThreadTest.java gdalwarp.jar $(JAVAC) -cp gdalwarp.jar $< +GDALWarpMetadataTest.class: GDALWarpMetadataTest.java gdalwarp.jar + $(JAVAC) -cp gdalwarp.jar $< + # MIT-Licensed code (that license is compatible with Apache 2.0) java/cz/adamh/utils/NativeUtils.java: wget -k "https://raw.githubusercontent.com/adamheinrich/native-utils/e6a39489662846a77504634b6fafa4995ede3b1d/src/main/$@" -O $@ From 00976fc4036012a23c721a996da530424f74eab4 Mon Sep 17 00:00:00 2001 From: "James W. McClain" Date: Fri, 26 Jul 2019 18:25:57 -0400 Subject: [PATCH 3/4] ThreadLocal Arrays --- src/bindings.cpp | 2 +- src/com_azavea_gdal_GDALWarp.c | 149 +++++++++++++--- src/main/GDALWarpThreadTest.java | 57 +++---- src/main/java/com/azavea/gdal/GDALWarp.java | 179 +++++++++++++++++++- 4 files changed, 321 insertions(+), 66 deletions(-) diff --git a/src/bindings.cpp b/src/bindings.cpp index 76f24ea..e6e35eb 100644 --- a/src/bindings.cpp +++ b/src/bindings.cpp @@ -118,7 +118,7 @@ inline void pthread_yield() #define DOIT(fn) \ bool done = false; \ auto query_result = query_token(token); \ - int code = -CPLE_AppDefined; \ + int code = CPLE_None; \ uint64_t then, now; \ if (query_result) \ { \ diff --git a/src/com_azavea_gdal_GDALWarp.c b/src/com_azavea_gdal_GDALWarp.c index 36210a0..d8688ea 100644 --- a/src/com_azavea_gdal_GDALWarp.c +++ b/src/com_azavea_gdal_GDALWarp.c @@ -28,7 +28,7 @@ #define htobe16(x) htons(x) #define htobe32(x) htonl(x) -uint64_t htobe64(uint64_t x) +uint64_t htobe64(uint64_t x) // Never tested { uint64_t retval; uint32_t *p1 = (uint32_t *)&retval; @@ -215,19 +215,59 @@ JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata_1domain_1list { char **domain_list = NULL; jint retval = get_metadata_domain_list(token, dataset, attempts, copies, band_number, &domain_list); - jsize max_size = (*env)->GetArrayLength(env, _domain_list); - - for (int i = 0; i < max_size && domain_list != NULL && domain_list[i] != NULL; ++i) + if (retval < 0) { - jbyteArray array = (*env)->GetObjectArrayElement(env, _domain_list, i); - jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL); - jsize array_size = (*env)->GetArrayLength(env, array); - strncpy((char *)bytes, domain_list[i], array_size); - (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + return retval; } - CSLDestroy(domain_list); + else + { + int i = 0; + jsize max_size = (*env)->GetArrayLength(env, _domain_list); - return retval; + for (i = 0; (i < max_size) && (domain_list != NULL) && (domain_list[i] != NULL); ++i) + { + jbyteArray array = (*env)->GetObjectArrayElement(env, _domain_list, i); + jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL); + jsize array_size = (*env)->GetArrayLength(env, array); + int len = strlen(domain_list[i]); + + if (len == 0) + { + bytes[0] = ' '; + bytes[1] = 0; + (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + } + else if (len < (array_size - 1)) + { + strncpy((char *)bytes, domain_list[i], len); + bytes[len] = 0; + (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + } + else + { + (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + CSLDestroy(domain_list); + return -CPLE_AppDefined; + } + } + if (i < (max_size - 1)) + { + jbyteArray array = (*env)->GetObjectArrayElement(env, _domain_list, i); + jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL); + bytes[0] = 0; + (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + } + + CSLDestroy(domain_list); + if (i == max_size) + { + return -CPLE_AppDefined; + } + else + { + return retval; + } + } } JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata(JNIEnv *env, jclass obj, @@ -236,19 +276,57 @@ JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata(JNIEnv *env, const char *domain = (*env)->GetStringUTFChars(env, _domain, NULL); char **list = NULL; jint retval = get_metadata(token, dataset, attempts, copies, band_number, domain, &list); - jsize max_size = (*env)->GetArrayLength(env, _list); - - for (int i = 0; i < max_size && list != NULL && list[i] != NULL; ++i) + if (retval < 0) { - jbyteArray array = (*env)->GetObjectArrayElement(env, _list, i); - jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL); - jsize array_size = (*env)->GetArrayLength(env, array); - strncpy((char *)bytes, list[i], array_size); - (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + return retval; } - (*env)->ReleaseStringUTFChars(env, _domain, domain); + else + { + int i = 0; + jsize max_size = (*env)->GetArrayLength(env, _list); - return retval; + for (i = 0; (i < max_size) && (list != NULL) && (list[i] != NULL); ++i) + { + jbyteArray array = (*env)->GetObjectArrayElement(env, _list, i); + jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL); + jsize array_size = (*env)->GetArrayLength(env, array); + int len = strlen(list[i]); + + if (len == 0) + { + bytes[0] = ' '; + bytes[1] = 0; + (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + } + else if (len < (array_size - 1)) + { + strncpy((char *)bytes, list[i], len); + bytes[len] = 0; + (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + } + else + { + (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + return -CPLE_AppDefined; + } + } + if (i < (max_size - 1)) + { + jbyteArray array = (*env)->GetObjectArrayElement(env, _list, i); + jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL); + bytes[0] = 0; + (*env)->ReleaseByteArrayElements(env, array, bytes, 0); + } + + if (i == max_size) + { + return -CPLE_AppDefined; + } + else + { + return retval; + } + } } JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata_1item(JNIEnv *env, jclass obj, @@ -258,15 +336,32 @@ JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata_1item(JNIEnv const char *domain = (*env)->GetStringUTFChars(env, _domain, NULL); jsize max_size = (*env)->GetArrayLength(env, _value); const char *value_src = NULL; - jbyte *value = (*env)->GetByteArrayElements(env, _value, NULL); + jbyte *bytes = (*env)->GetByteArrayElements(env, _value, NULL); jint retval = get_metadata_item(token, dataset, attempts, copies, band_number, key, domain, &value_src); - strncpy((char *)value, value_src, max_size); - (*env)->ReleaseStringUTFChars(env, _key, key); - (*env)->ReleaseStringUTFChars(env, _domain, domain); - (*env)->ReleaseByteArrayElements(env, _value, value, 0); + if (retval < 0) + { + return retval; + } + else + { + int len = strlen(value_src); + if (len < (max_size - 1)) + { + strncpy((char *)bytes, value_src, len); + bytes[len] = 0; + } + else + { + retval = -CPLE_AppDefined; + } - return retval; + (*env)->ReleaseStringUTFChars(env, _key, key); + (*env)->ReleaseStringUTFChars(env, _domain, domain); + (*env)->ReleaseByteArrayElements(env, _value, bytes, 0); + + return retval; + } } JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1overview_1widths_1heights(JNIEnv *env, jclass obj, diff --git a/src/main/GDALWarpThreadTest.java b/src/main/GDALWarpThreadTest.java index a047ea4..a5a1160 100644 --- a/src/main/GDALWarpThreadTest.java +++ b/src/main/GDALWarpThreadTest.java @@ -268,61 +268,60 @@ public static void main(String[] args) throws Exception { // Domain List Metadata { - byte[][] domain_list1 = new byte[1 << 10][1 << 10]; + String[][] domain_list1 = new String[1][0]; System.out.println(ANSI_BLUE + "Source domain list:" + ANSI_RESET); GDALWarp.get_metadata_domain_list(token, GDALWarp.SOURCE, 0, 0, domain_list1); - for (int i = 0; i < domain_list1.length; ++i) { - String str = new String(domain_list1[i], "UTF-8").trim(); - if (str.length() > 0) { - System.out.println(ANSI_GREEN + str + ANSI_RESET); + for (int i = 0; i < domain_list1[0].length; ++i) { + if (domain_list1[0][i].length() > 0) { + System.out.println(ANSI_GREEN + domain_list1[0][i] + ANSI_RESET); } } - - byte[][] domain_list2 = new byte[1 << 10][1 << 10]; + } + { + String[][] domain_list2 = new String[1][0]; System.out.println(ANSI_BLUE + "Warped domain list:" + ANSI_RESET); GDALWarp.get_metadata_domain_list(token, GDALWarp.WARPED, 0, 0, domain_list2); - for (int i = 0; i < domain_list2.length; ++i) { - String str = new String(domain_list2[i], "UTF-8").trim(); - if (str.length() > 0) { - System.out.println(ANSI_GREEN + str + ANSI_RESET); + for (int i = 0; i < domain_list2[0].length; ++i) { + if (domain_list2[0][i].length() > 0) { + System.out.println(ANSI_GREEN + domain_list2[0][i] + ANSI_RESET); } } } // Metadata { - byte[][] list1 = new byte[1 << 10][1 << 10]; + String[][] list1 = new String[1][0]; System.out.println(ANSI_BLUE + "Source metadata:" + ANSI_RESET); GDALWarp.get_metadata(token, GDALWarp.SOURCE, 0, 0, "", list1); - for (int i = 0; i < list1.length; ++i) { - String str = new String(list1[i], "UTF-8").trim(); - if (str.length() > 0) { - System.out.println(ANSI_GREEN + str + ANSI_RESET); + for (int i = 0; i < list1[0].length; ++i) { + if (list1[0][i].length() > 0) { + System.out.println(ANSI_GREEN + list1[0][i] + ANSI_RESET); } } - - byte[][] list2 = new byte[1 << 10][1 << 10]; + } + { + String[][] list2 = new String[1][0]; System.out.println(ANSI_BLUE + "Warped metadata:" + ANSI_RESET); GDALWarp.get_metadata(token, GDALWarp.WARPED, 0, 0, "", list2); - for (int i = 0; i < list2.length; ++i) { - String str = new String(list2[i], "UTF-8").trim(); - if (str.length() > 0) { - System.out.println(ANSI_GREEN + str + ANSI_RESET); + for (int i = 0; i < list2[0].length; ++i) { + if (list2[0][i].length() > 0) { + System.out.println(ANSI_GREEN + list2[0][i] + ANSI_RESET); } } } // Metadata item { - byte[] bytes1 = new byte[1 << 10]; + String[] item = new String[1]; System.out.print(ANSI_BLUE + "Source AREA_OR_POINT:" + ANSI_RESET); - GDALWarp.get_metadata_item(token, GDALWarp.SOURCE, 0, 0, "AREA_OR_POINT", "", bytes1); - System.out.println(ANSI_GREEN + new String(bytes1, "UTF-8").trim() + ANSI_RESET); - - byte[] bytes2 = new byte[1 << 10]; + GDALWarp.get_metadata_item(token, GDALWarp.SOURCE, 0, 0, "AREA_OR_POINT", "", item); + System.out.println(ANSI_GREEN + item[0] + ANSI_RESET); + } + { + String[] item = new String[1]; System.out.print(ANSI_BLUE + "Warped AREA_OR_POINT:" + ANSI_RESET); - GDALWarp.get_metadata_item(token, GDALWarp.WARPED, 0, 0, "AREA_OR_POINT", "", bytes2); - System.out.println(ANSI_GREEN + new String(bytes1, "UTF-8").trim() + ANSI_RESET); + GDALWarp.get_metadata_item(token, GDALWarp.WARPED, 0, 0, "AREA_OR_POINT", "", item); + System.out.println(ANSI_GREEN + item[0] + ANSI_RESET); } // Overviews diff --git a/src/main/java/com/azavea/gdal/GDALWarp.java b/src/main/java/com/azavea/gdal/GDALWarp.java index 14008b1..35592c2 100644 --- a/src/main/java/com/azavea/gdal/GDALWarp.java +++ b/src/main/java/com/azavea/gdal/GDALWarp.java @@ -16,6 +16,8 @@ package com.azavea.gdal; +import java.io.UnsupportedEncodingException; + import cz.adamh.utils.NativeUtils; public class GDALWarp { @@ -62,8 +64,57 @@ public class GDALWarp { private static final String ANSI_RESET = "\u001B[0m"; private static final String ANSI_RED = "\u001B[31m"; + private static ThreadLocal scratch_1d = new ThreadLocal<>(); + private static ThreadLocal scratch_2d = new ThreadLocal<>(); + private static native void _init(int size); + private static int ensure_scratch_1d() { + int len; + + if (scratch_1d.get() == null) { + len = 1 << 8; + scratch_1d.set(new byte[len]); + } else { + len = scratch_1d.get().length; + } + return len; + } + + private static int ensure_scratch_2d() { + int len; + + if (scratch_2d.get() == null) { + len = 1 << 8; + scratch_2d.set(new byte[len][len]); + } else { + len = scratch_2d.get().length; + } + return len; + } + + private static void grow_scratch_1d(int _len) { + int len; + + if (_len <= 0) { + len = (scratch_1d.get().length << 1); + } else { + len = _len; + } + scratch_1d.set(new byte[len]); + } + + private static void grow_scratch_2d(int _len) { + int len; + + if (_len <= 0) { + len = (scratch_2d.get().length << 1); + } else { + len = _len; + } + scratch_2d.set(new byte[len][len]); + } + public static void init(int size) throws Exception { String os = System.getProperty("os.name").toLowerCase(); @@ -101,17 +152,16 @@ else if (os.contains("win")) { * https://gdal.org/api/raster_c_api.html?highlight=rasterio#_CPPv415GDALVersionInfoPKc) * @return The requested information */ - public static String get_version_info(String key) throws Exception { - final int first_size = 1 << 8; - byte[] value = new byte[first_size]; - int len = _get_version_info(key, value); + public static String get_version_info(String key) throws UnsupportedEncodingException { + int array_len = ensure_scratch_1d(); + int result_len = _get_version_info(key, scratch_1d.get()); - if (len < first_size) { - return new String(value, "UTF-8").trim(); + if (result_len < array_len) { + return new String(scratch_1d.get(), 0, result_len, "UTF-8").trim(); } else { - byte[] value2 = new byte[len + 1]; - _get_version_info(key, value2); - return new String(value2, "UTF-8").trim(); + grow_scratch_1d(result_len + 1); + _get_version_info(key, scratch_1d.get()); + return new String(scratch_1d.get(), 0, result_len, "UTF-8").trim(); } } @@ -253,6 +303,44 @@ public static native int get_color_interpretation(long token, int dataset, int a public static native int get_metadata_domain_list(long token, int dataset, int attempts, /* */ int band_number, byte[][] domain_list); + /** + * Get the list of metadata domain lists. + * + * @param token A token associated with some uri, options pair + * @param dataset 0 (or GDALWarp::SOURCE) for the source dataset, 1 (or + * GDALWarp::WARPED) for the warped dataset + * @param attempts The number of attempts to make before giving up + * @param band_number The band to query (zero for the file itself) + * @param domain_list The return-location for the list of strings + * @return The number of attempts made (upon success) or a negative error code + * (upon failure) + */ + public static int get_metadata_domain_list(long token, int dataset, int attempts, int band_number, + String[][] domain_list) throws UnsupportedEncodingException { + int array_len = ensure_scratch_2d(); + int retval = get_metadata_domain_list(token, dataset, attempts, band_number, scratch_2d.get()); + + if (retval >= 0) { + int n = 0; + for (n = 0; scratch_2d.get()[n][0] != 0; ++n) { + } + domain_list[0] = new String[n]; + for (--n; n >= 0; --n) { + int m = 0; + byte[] bytes = scratch_2d.get()[n]; + for (m = 0; (m < array_len) && (bytes[m] != 0); ++m) { + } + domain_list[0][n] = new String(bytes, 0, m, "UTF-8").trim(); + } + return retval; + } else if (retval == -1) { // CPLE_AppDefined means array too small + grow_scratch_2d(0); + return get_metadata_domain_list(token, dataset, attempts, band_number, domain_list); + } else { // Return other error code + return retval; + } + } + /** * Get the metadata found in a particular metadata domain. * @@ -269,6 +357,45 @@ public static native int get_metadata_domain_list(long token, int dataset, int a public static native int get_metadata(long token, int dataset, int attempts, /* */ int band_number, String domain, byte[][] list); + /** + * Get the metadata found in a particular metadata domain. + * + * @param token A token associated with some uri, options pair + * @param dataset 0 (or GDALWarp::SOURCE) for the source dataset, 1 (or + * GDALWarp::WARPED) for the warped dataset + * @param attempts The number of attempts to make before giving up + * @param band_number The band to query (zero for the file itself) + * @param domain The metadata domain to query + * @param list The return-location for the list of strings + * @return The number of attempts made (upon success) or a negative error code + * (upon failure) + */ + public static int get_metadata(long token, int dataset, int attempts, int band_number, String domain, + String[][] list) throws UnsupportedEncodingException { + int array_len = ensure_scratch_2d(); + int retval = get_metadata(token, dataset, attempts, band_number, domain, scratch_2d.get()); + + if (retval >= 0) { + int n = 0; + for (n = 0; scratch_2d.get()[n][0] != 0; ++n) { + } + list[0] = new String[n]; + for (--n; n >= 0; --n) { + int m = 0; + byte[] bytes = scratch_2d.get()[n]; + for (m = 0; (m < array_len) && (bytes[m] != 0); ++m) { + } + list[0][n] = new String(bytes, 0, m, "UTF-8").trim(); + } + return retval; + } else if (retval == -1) { // CPLE_AppDefined means array too small + grow_scratch_2d(0); + return get_metadata(token, dataset, attempts, band_number, domain, list); + } else { // Return other error code + return retval; + } + } + /** * Get a particular metadata value associated with a key. * @@ -286,6 +413,40 @@ public static native int get_metadata(long token, int dataset, int attempts, /* public static native int get_metadata_item(long token, int dataset, int attempts, /* */ int band_number, String key, String domain, byte[] value); + /** + * Get a particular metadata value associated with a key. + * + * @param token A token associated with some uri, options pair + * @param dataset 0 (or GDALWarp::SOURCE) for the source dataset, 1 (or + * GDALWarp::WARPED) for the warped dataset + * @param attempts The number of attempts to make before giving up + * @param band_number The band to query (zero for the file itself) + * @param key The key of the key, value metadata pair + * @param domain The metadata domain to query + * @param value The return-location for the value of the key, value pair + * @return The number of attempts made (upon success) or a negative error code + * (upon failure) + */ + public static int get_metadata_item(long token, int dataset, int attempts, int band_number, String key, + String domain, String[] value) throws UnsupportedEncodingException { + int array_len = ensure_scratch_1d(); + int retval = get_metadata_item(token, dataset, attempts, band_number, key, domain, scratch_1d.get()); + + if (retval >= 0) { + int m = 0; + byte[] bytes = scratch_1d.get(); + for (m = 0; (m < array_len) && (bytes[m] != 0); ++m) { + } + value[0] = new String(bytes, 0, m, "UTF-8").trim(); + return retval; + } else if (retval == -1) { // CPLE_AppDefined means array too small + grow_scratch_1d(0); + return get_metadata_item(token, dataset, attempts, band_number, key, domain, value); + } else { // Return other error code + return retval; + } + } + /** * Get the widths and heights of all overviews. * From fcfe40987b76866c91cc755b9fbdfd6d76353720 Mon Sep 17 00:00:00 2001 From: "James W. McClain" Date: Fri, 26 Jul 2019 20:34:34 -0400 Subject: [PATCH 4/4] Update Metadata Test Program --- src/main/GDALWarpMetadataTest.java | 97 +++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 15 deletions(-) diff --git a/src/main/GDALWarpMetadataTest.java b/src/main/GDALWarpMetadataTest.java index e4d8c9c..59ac06a 100644 --- a/src/main/GDALWarpMetadataTest.java +++ b/src/main/GDALWarpMetadataTest.java @@ -32,25 +32,92 @@ public static void main(String[] args) throws Exception { long token = GDALWarp.get_token(args[0], options); long before, after; int len = 0; - //byte[][] domain_list1 = new byte[1 << 10][1 << 10]; - - // Domain List Metadata - before = System.currentTimeMillis(); - for (int j = 0; j < (1 << 18); ++j) { - byte[][] domain_list1 = new byte[1 << 4][1 << 10]; - GDALWarp.get_metadata_domain_list(token, GDALWarp.SOURCE, 0, 0, domain_list1); - for (int i = 0; i < domain_list1.length; ++i) { - String str = new String(domain_list1[i], "UTF-8").trim(); - if (str.length() > 0) { - len = len + str.length(); - } else { - break; + + // Noop + { + len = 0; + before = System.currentTimeMillis(); + for (int j = 0; j < (1 << 18); ++j) { + len += GDALWarp.noop(token, GDALWarp.SOURCE, 0); + } + after = System.currentTimeMillis(); + System.out.println(after - before + " " + len); + } + + // Domain List Metadata ThreadLocal + { + len = 0; + before = System.currentTimeMillis(); + String[][] domain_list = new String[1][0]; + for (int j = 0; j < (1 << 18); ++j) { + GDALWarp.get_metadata_domain_list(token, GDALWarp.SOURCE, 0, 0, domain_list); + for (int i = 0; i < domain_list[0].length; ++i) { + len = len + domain_list[0][i].length(); + } + } + after = System.currentTimeMillis(); + System.out.println(after - before + " " + len); + } + + // Domain List Metadata Reallocate 16⨯1024 + { + // Domain List Metadata + before = System.currentTimeMillis(); + for (int j = 0; j < (1 << 18); ++j) { + byte[][] domain_list1 = new byte[1 << 4][1 << 10]; + GDALWarp.get_metadata_domain_list(token, GDALWarp.SOURCE, 0, 0, domain_list1); + for (int i = 0; i < domain_list1.length; ++i) { + String str = new String(domain_list1[i], "UTF-8").trim(); + if (str.length() > 0) { + len = len + str.length(); + } else { + break; + } } } + after = System.currentTimeMillis(); + System.out.println(after - before + " " + len); } - after = System.currentTimeMillis(); - System.out.println(after - before); + // Domain List Metadata Reallocate 1024⨯1024 + { + // Domain List Metadata + before = System.currentTimeMillis(); + for (int j = 0; j < (1 << 18); ++j) { + byte[][] domain_list1 = new byte[1 << 10][1 << 10]; + GDALWarp.get_metadata_domain_list(token, GDALWarp.SOURCE, 0, 0, domain_list1); + for (int i = 0; i < domain_list1.length; ++i) { + String str = new String(domain_list1[i], "UTF-8").trim(); + if (str.length() > 0) { + len = len + str.length(); + } else { + break; + } + } + } + after = System.currentTimeMillis(); + System.out.println(after - before + " " + len); + } + + // Domain List Metadata Preallocate + { + // Domain List Metadata + byte[][] domain_list1 = new byte[1 << 10][1 << 10]; + before = System.currentTimeMillis(); + for (int j = 0; j < (1 << 18); ++j) { + GDALWarp.get_metadata_domain_list(token, GDALWarp.SOURCE, 0, 0, domain_list1); + for (int i = 0; i < domain_list1.length; ++i) { + String str = new String(domain_list1[i], "UTF-8").trim(); + if (str.length() > 0) { + len = len + str.length(); + } else { + break; + } + } + } + after = System.currentTimeMillis(); + System.out.println(after - before + " " + len); + } GDALWarp.deinit(); return;