Skip to content

Commit

Permalink
Merge pull request #59 from jamesmcclain/metadata
Browse files Browse the repository at this point in the history 
Add Optional In-Built ThreadLocal Arrays for Metadata
  • Loading branch information
James McClain authored Jan 2, 2020
2 parents ac3bc4e + fcfe409 commit 1d7ddea
Show file tree
Hide file tree
Showing 9 changed files with 545 additions and 68 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ src/experiments/thread/rawthread
src/experiments/thread/wrapthread
src/experiments/thread/pattern
src/experiments/thread/oversubscribe
src/experiments/thread/metadata
src/unit_tests/dataset_tests
src/unit_tests/token_tests
src/unit_tests/cache_tests
Expand Down
2 changes: 1 addition & 1 deletion src/bindings.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ inline void pthread_yield()
#define DOIT(fn) \
bool done = false; \
auto query_result = query_token(token); \
int code = -CPLE_AppDefined; \
int code = CPLE_None; \
uint64_t then, now; \
if (query_result) \
{ \
Expand Down
149 changes: 122 additions & 27 deletions src/com_azavea_gdal_GDALWarp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
#define htobe16(x) htons(x)
#define htobe32(x) htonl(x)

uint64_t htobe64(uint64_t x)
uint64_t htobe64(uint64_t x) // Never tested
{
uint64_t retval;
uint32_t *p1 = (uint32_t *)&retval;
Expand Down Expand Up @@ -215,19 +215,59 @@ JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata_1domain_1list
{
char **domain_list = NULL;
jint retval = get_metadata_domain_list(token, dataset, attempts, copies, band_number, &domain_list);
jsize max_size = (*env)->GetArrayLength(env, _domain_list);

for (int i = 0; i < max_size && domain_list != NULL && domain_list[i] != NULL; ++i)
if (retval < 0)
{
jbyteArray array = (*env)->GetObjectArrayElement(env, _domain_list, i);
jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL);
jsize array_size = (*env)->GetArrayLength(env, array);
strncpy((char *)bytes, domain_list[i], array_size);
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
return retval;
}
CSLDestroy(domain_list);
else
{
int i = 0;
jsize max_size = (*env)->GetArrayLength(env, _domain_list);

return retval;
for (i = 0; (i < max_size) && (domain_list != NULL) && (domain_list[i] != NULL); ++i)
{
jbyteArray array = (*env)->GetObjectArrayElement(env, _domain_list, i);
jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL);
jsize array_size = (*env)->GetArrayLength(env, array);
int len = strlen(domain_list[i]);

if (len == 0)
{
bytes[0] = ' ';
bytes[1] = 0;
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
}
else if (len < (array_size - 1))
{
strncpy((char *)bytes, domain_list[i], len);
bytes[len] = 0;
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
}
else
{
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
CSLDestroy(domain_list);
return -CPLE_AppDefined;
}
}
if (i < (max_size - 1))
{
jbyteArray array = (*env)->GetObjectArrayElement(env, _domain_list, i);
jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL);
bytes[0] = 0;
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
}

CSLDestroy(domain_list);
if (i == max_size)
{
return -CPLE_AppDefined;
}
else
{
return retval;
}
}
}

JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata(JNIEnv *env, jclass obj,
Expand All @@ -236,19 +276,57 @@ JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata(JNIEnv *env,
const char *domain = (*env)->GetStringUTFChars(env, _domain, NULL);
char **list = NULL;
jint retval = get_metadata(token, dataset, attempts, copies, band_number, domain, &list);
jsize max_size = (*env)->GetArrayLength(env, _list);

for (int i = 0; i < max_size && list != NULL && list[i] != NULL; ++i)
if (retval < 0)
{
jbyteArray array = (*env)->GetObjectArrayElement(env, _list, i);
jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL);
jsize array_size = (*env)->GetArrayLength(env, array);
strncpy((char *)bytes, list[i], array_size);
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
return retval;
}
(*env)->ReleaseStringUTFChars(env, _domain, domain);
else
{
int i = 0;
jsize max_size = (*env)->GetArrayLength(env, _list);

return retval;
for (i = 0; (i < max_size) && (list != NULL) && (list[i] != NULL); ++i)
{
jbyteArray array = (*env)->GetObjectArrayElement(env, _list, i);
jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL);
jsize array_size = (*env)->GetArrayLength(env, array);
int len = strlen(list[i]);

if (len == 0)
{
bytes[0] = ' ';
bytes[1] = 0;
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
}
else if (len < (array_size - 1))
{
strncpy((char *)bytes, list[i], len);
bytes[len] = 0;
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
}
else
{
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
return -CPLE_AppDefined;
}
}
if (i < (max_size - 1))
{
jbyteArray array = (*env)->GetObjectArrayElement(env, _list, i);
jbyte *bytes = (*env)->GetByteArrayElements(env, array, NULL);
bytes[0] = 0;
(*env)->ReleaseByteArrayElements(env, array, bytes, 0);
}

if (i == max_size)
{
return -CPLE_AppDefined;
}
else
{
return retval;
}
}
}

JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata_1item(JNIEnv *env, jclass obj,
Expand All @@ -258,15 +336,32 @@ JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1metadata_1item(JNIEnv
const char *domain = (*env)->GetStringUTFChars(env, _domain, NULL);
jsize max_size = (*env)->GetArrayLength(env, _value);
const char *value_src = NULL;
jbyte *value = (*env)->GetByteArrayElements(env, _value, NULL);
jbyte *bytes = (*env)->GetByteArrayElements(env, _value, NULL);
jint retval = get_metadata_item(token, dataset, attempts, copies, band_number, key, domain, &value_src);

strncpy((char *)value, value_src, max_size);
(*env)->ReleaseStringUTFChars(env, _key, key);
(*env)->ReleaseStringUTFChars(env, _domain, domain);
(*env)->ReleaseByteArrayElements(env, _value, value, 0);
if (retval < 0)
{
return retval;
}
else
{
int len = strlen(value_src);
if (len < (max_size - 1))
{
strncpy((char *)bytes, value_src, len);
bytes[len] = 0;
}
else
{
retval = -CPLE_AppDefined;
}

return retval;
(*env)->ReleaseStringUTFChars(env, _key, key);
(*env)->ReleaseStringUTFChars(env, _domain, domain);
(*env)->ReleaseByteArrayElements(env, _value, bytes, 0);

return retval;
}
}

JNIEXPORT jint JNICALL Java_com_azavea_gdal_GDALWarp_get_1overview_1widths_1heights(JNIEnv *env, jclass obj,
Expand Down
7 changes: 5 additions & 2 deletions src/experiments/thread/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ LDFLAGS ?= $(shell pkg-config gdal --libs) -L$(shell pwd) -lgdalwarp_bindings -l
BOOST_ROOT ?= /usr/include
SO ?= so

all: rawthread wrapthread pattern oversubscribe
all: rawthread wrapthread pattern oversubscribe metadata

../../libgdalwarp_bindings.$(SO):
$(MAKE) -C ../.. libgdalwarp_bindings.$(SO)
Expand All @@ -25,13 +25,16 @@ pattern: pattern.o libgdalwarp_bindings.$(SO)
oversubscribe: oversubscribe.o libgdalwarp_bindings.$(SO)
$(CC) $< $(LDFLAGS) -o $@

metadata: metadata.o libgdalwarp_bindings.$(SO)
$(CC) $< $(LDFLAGS) -lboost_timer -o $@

%.o: %.cpp
$(CXX) $(CFLAGS) $(CXXFLAGS) $(GDALCFLAGS) -I$(BOOST_ROOT) $< -c -o $@

clean:
rm -f *.o libgdalwarp_bindings.$(SO)

cleaner: clean
rm -f rawthread wrapthread pattern oversubscribe
rm -f rawthread wrapthread pattern oversubscribe metadata

cleanest: cleaner
90 changes: 90 additions & 0 deletions src/experiments/thread/metadata.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright 2019 Azavea
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include <cstdint>
#include <cstdlib>
#include <cassert>

#include <boost/timer/timer.hpp>

#include <gdal.h>

#include "../../bindings.h"
#include "../../locked_dataset.hpp"

char const *options[] = {
"-dstnodata", "107",
"-tap", "-tr", "33", "42",
"-r", "bilinear",
"-t_srs", "epsg:3857",
"-co", "BLOCKXSIZE=512", "-co", "BLOCKYSIZE=512",
nullptr};

namespace t = boost::timer;

void CSLDestroy(char **papszStrList)
{
if (!papszStrList)
return;

for (char **papszPtr = papszStrList; *papszPtr != nullptr; ++papszPtr)
{
free(*papszPtr);
}

free(papszStrList);
}

int main(int argc, char **argv)
{
init(33);

uint64_t token = get_token(argv[1], options);
auto handle = GDALOpen(argv[1], GA_ReadOnly);

char **domain_list = NULL;

{
t::auto_cpu_timer timer;

for (int i = 0; i < (1 << 18); ++i)
{
assert(noop(token, locked_dataset::SOURCE, 0, 1) > 0);
}
}

{
t::auto_cpu_timer timer;

for (int i = 0; i < (1 << 18); ++i)
{
assert(get_metadata_domain_list(token, 0, 10, 1, 0, &domain_list) > 0);
CSLDestroy(domain_list);
}
}
{
t::auto_cpu_timer timer;

for (int i = 0; i < (1 << 18); ++i)
{
domain_list = GDALGetMetadataDomainList(handle);
CSLDestroy(domain_list);
}
}

GDALClose(handle);
deinit();
}
Loading

0 comments on commit 1d7ddea

Please sign in to comment.