diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c3354b..e0c1368 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,3 +6,12 @@ find_package(Threads) add_executable(memsetVsMadvise memsetVsMadvise.cpp) set_property(TARGET memsetVsMadvise PROPERTY CXX_STANDARD 11) target_link_libraries(memsetVsMadvise PRIVATE ${CMAKE_THREAD_LIBS_INIT} gflags) + +SET(JEMALLOC_COMPILE_FLAGS "-L/usr/local/lib") +SET(JEMALLOC_LINK_FLAGS "-ljemalloc -lm -lpthread") + +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${JEMALLOC_COMPILE_FLAGS}") +SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${JEMALLOC_LINK_FLAGS}") +add_executable(stress stress_test/Main.cpp stress_test/Producers.cpp stress_test/Mixer.cpp stress_test/ToFreeQueue.cpp stress_test/Distribution.cpp) +set_property(TARGET stress PROPERTY CXX_STANDARD 14) +target_link_libraries(stress PRIVATE gflags) diff --git a/stress_test/Distribution.cpp b/stress_test/Distribution.cpp new file mode 100644 index 0000000..fa845ce --- /dev/null +++ b/stress_test/Distribution.cpp @@ -0,0 +1,44 @@ +#include "Distribution.h" + +#include +#include +#include +#include +#include + +#include + +SizeClass parseSizeClass(const std::string &ln) { + std::istringstream strStream(ln); + size_t sizeClass; + double freq; + if (!(strStream >> sizeClass >> freq)) { + std::cout << "File format invalid. Failed to following line:\n\e[0;31m" + << ln << "\e[0m" << std::endl; + exit(1); + } + if (freq > 1.0) { + std::cout << "Warning: this looks off; frequency greater than 1.0" << std::endl; + freq = 1.0; + } + return {sizeClass, freq}; +} + +Distribution parseDistribution(const char *fileName) { + std::string line; + std::ifstream f(fileName); + + if (!f) { + std::cout << "Specified file '" << fileName << "' not found." << std::endl; + exit(1); + } + + Distribution d; + + while (std::getline(f, line)) { + d.push_back(parseSizeClass(line)); + } + + std::sort(begin(d), end(d)); + return d; +} diff --git a/stress_test/Distribution.h b/stress_test/Distribution.h new file mode 100644 index 0000000..3c37487 --- /dev/null +++ b/stress_test/Distribution.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +using std::size_t; + +struct SizeClass { + size_t size; + double freq; + bool operator<(const SizeClass &that) const { + return this->freq < that.freq; + } +}; + +typedef std::vector Distribution; + +Distribution parseDistribution(const char *fileName); diff --git a/stress_test/Main.cpp b/stress_test/Main.cpp new file mode 100644 index 0000000..26ac427 --- /dev/null +++ b/stress_test/Main.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include "Mixer.h" +#include "Distribution.h" + +DEFINE_int32(num_producers, 1000, "number of producers to run on each thread"); +DEFINE_int32(num_threads, 1, "number of threads to run"); +DEFINE_bool(print_malloc_stats, false, "print out malloc stats after running"); +DEFINE_string(distribution_file, "", "path to distribution file"); +static bool validateDistributionFile(const char *flagName, const std::string &val) { + return val.length() != 0; +} +DEFINE_validator(distribution_file, &validateDistributionFile); + +using std::shared_ptr; +using std::make_shared; +using std::vector; + +void createAndRunMixer(const Distribution &distr, int me, + vector> toFreeQueues) { + Mixer m(FLAGS_num_producers, distr, me, toFreeQueues); + m.run(); +} + +int main(int argc, char **argv) { + gflags::ParseCommandLineFlags(&argc, &argv, true); + Distribution distr = parseDistribution(FLAGS_distribution_file.c_str()); + + // Set up a work queue for each thread + vector threads; + vector> toFreeQueues; + for (int i = 0; i < FLAGS_num_threads; i++) { + shared_ptr toFreeQ = make_shared(); + toFreeQueues.push_back(toFreeQ); + } + + for (int i = 0; i < FLAGS_num_threads; i++) { + // each thread gets an arbitrary id given by [i] + threads.push_back(std::thread(createAndRunMixer, distr, i, toFreeQueues)); + } + + using namespace std::chrono; + + high_resolution_clock::time_point beginTime = high_resolution_clock::now(); + for (auto it = begin(threads); it != end(threads); ++it) { + it->join(); + } + // Cleanup any remaining memory + for (int i = 0; i < FLAGS_num_threads; i++) { + toFreeQueues[i]->freeIgnoreLifetime(); + } + high_resolution_clock::time_point endTime = high_resolution_clock::now(); + duration span = duration_cast>(endTime - beginTime); + + if (FLAGS_print_malloc_stats) { + je_malloc_stats_print(NULL, NULL, NULL); + } + + std::cout << "Elapsed time: " << span.count() << std::endl; +} diff --git a/stress_test/Mixer.cpp b/stress_test/Mixer.cpp new file mode 100644 index 0000000..2b90260 --- /dev/null +++ b/stress_test/Mixer.cpp @@ -0,0 +1,77 @@ +#include "Mixer.h" + +#include +#include + +DEFINE_int32(producer_duration, 10000, "scales the length of producers. Making" + "this number higher means each producer runs for a long time."); + +using std::make_unique; +using std::unique_ptr; +using std::shared_ptr; +using std::vector; + +Mixer::Mixer(int numProducers, const Distribution &distr, int me, + vector> toFreeQueues) + : producersRemaining_(numProducers), + toFreeQueues_(toFreeQueues), me_(me), + consumerIdPicker_(0, toFreeQueues.size() - 1) { + this->totalWeight_ = 0.0; + this->initProducers(distr); + this->producerWeightPicker_ = std::uniform_real_distribution(0.0, this->totalWeight_); +} + +void Mixer::addProducer(double weight, unique_ptr p) { + this->totalWeight_ += weight; + this->producers_.push_back(std::move(p)); + this->weightArray_.push_back(this->totalWeight_); +} + +void Mixer::initProducers(const Distribution &distr) { + auto oneSecond = std::chrono::duration(1.0); + + std::uniform_int_distribution vectorInitFuzzer(1, 100); + + for (auto it = begin(distr); it != end(distr); ++it) { + addProducer(it->freq / 3.0, + std::move(make_unique(it->size, FLAGS_producer_duration))); + // provide a bit of fuzziness to vector initial value + int vectorInit = vectorInitFuzzer(this->generator_); + addProducer(it->freq / 3.0, + std::move(make_unique(FLAGS_producer_duration, oneSecond, vectorInit))); + addProducer(it->freq / 3.0, + std::move(make_unique(it->size, FLAGS_producer_duration, oneSecond))); + } +} + +const Producer &Mixer::pickProducer() { + double r = this->producerWeightPicker_(this->generator_); + int producerIndex; + for (producerIndex = 0; producerIndex < this->weightArray_.size(); ++producerIndex) { + if (r <= weightArray_[producerIndex]) { + break; + } + } + assert(producerIndex != this->weightArray_.size()); + return *(this->producers_[producerIndex]); +} + +// Picks next producer for the mixer to run. Currently uniform random choice +ToFreeQueue& Mixer::pickConsumer() { + int consumerIndex = this->consumerIdPicker_(this->generator_); + return *(this->toFreeQueues_[consumerIndex]); +} + +void Mixer::run() { + while (this->producersRemaining_ > 0) { + this->toFreeQueues_[this->me_]->free(); + // otherwise run a random producer + Allocation a = this->pickProducer().run(); + if (!a.isEmpty()) { + this->pickConsumer().addToFree(std::move(a)); + } + producersRemaining_--; + } + je_mallctl("thread.tcache.flush", NULL, NULL, NULL, 0); + // Main loop will eventually cleanup memory +} diff --git a/stress_test/Mixer.h b/stress_test/Mixer.h new file mode 100644 index 0000000..526ae6d --- /dev/null +++ b/stress_test/Mixer.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +#include "Producers.h" +#include "ToFreeQueue.h" +#include "Distribution.h" + +class Mixer { +public: + void run(); + Mixer(int numProducers, const Distribution &distr, int me, + std::vector> toFreeQueues); + +private: + int producersRemaining_; + // the thread id that this mixer is running on + int me_; + // work queues for each thread indexed by thread number + std::vector> toFreeQueues_; + // Picks next producer for the mixer to run + const Producer &pickProducer(); + /* Picks a consumer to free memory allocated by a producer. Currently uniform + * random choice */ + ToFreeQueue &pickConsumer(); + + std::uniform_int_distribution consumerIdPicker_; + std::default_random_engine generator_; + + // for picking producer with weighted random choice + std::vector weightArray_; + double totalWeight_; + std::vector> producers_; + std::uniform_real_distribution producerWeightPicker_; + // initializes [producers_], [totalWeight_], [weightArray_], and [producerWeightPicker_] + void initProducers(const Distribution &distr); + void addProducer(double weight, std::unique_ptr p); +}; diff --git a/stress_test/Producers.cpp b/stress_test/Producers.cpp new file mode 100644 index 0000000..b1f54cb --- /dev/null +++ b/stress_test/Producers.cpp @@ -0,0 +1,96 @@ +#include "Producers.h" + +#include + +// Allocation + +bool Allocation::operator<(const Allocation &that) const { + return this->toFree_ < that.toFree_; +} + +bool Allocation::operator>(const Allocation &that) const { + return !(*this < that); +} + +bool Allocation::isEmpty() const { return this->toFree_.size() == 0; } + +std::chrono::high_resolution_clock::time_point Allocation::freeAfter() const { + return this->freeAfter_; +} + +Allocation::Allocation(std::vector toFree, + std::chrono::high_resolution_clock::time_point freeAfter) + : toFree_(toFree), freeAfter_(freeAfter) {} + +Allocation::~Allocation() { + for (auto it = begin(this->toFree_); it != end(this->toFree_); ++it) { + free(*it); + } +} + +// Simple Producer + +SimpleProducer::SimpleProducer(int allocSize, int numAllocs) + : allocSize_(allocSize), numAllocs_(numAllocs) {} + +Allocation SimpleProducer::run() const { + for (int i = 0; i < this->numAllocs_; i++) { + char *ptr = (char *)calloc(this->allocSize_, sizeof(char)); + if (ptr == NULL) { + std::cout << "allocation failed" << std::endl; + } + free(ptr); + } + return std::move(Allocation()); +} + +void swap(Allocation &a1, Allocation &a2) { + a1.toFree_.swap(a2.toFree_); + std::swap(a1.freeAfter_, a2.freeAfter_); +} + +// Vector Producer + +VectorProducer::VectorProducer(size_t vectorSize, + std::chrono::duration lifetime, + size_t initialSize) + : vectorSize_(vectorSize), lifetime_(lifetime), initialSize_(initialSize) {} + +std::chrono::high_resolution_clock::time_point addToNow(std::chrono::duration d) { + using namespace std::chrono; + high_resolution_clock::time_point t = high_resolution_clock::now(); + high_resolution_clock::duration dHighResolution = + duration_cast(d); + t += dHighResolution;; + return t; +} + +Allocation VectorProducer::run() const { + + void *ptr = malloc(this->initialSize_); + size_t currSize = this->initialSize_; + while (currSize < this->vectorSize_) { + free(ptr); + currSize *= 2; + ptr = malloc(currSize); + } + + return std::move(Allocation(std::vector({ptr}), addToNow(this->lifetime_))); +} + +// LinkedList Producer + +Allocation LinkedListProducer::run() const { + + std::vector toFree; + toFree.reserve(this->numNodes_); + + for (int i = 0; i < this->numNodes_; i++) { + toFree.push_back(malloc(this->nodeSize_)); + } + + return std::move(Allocation(toFree, addToNow(this->lifetime_))); +} +// allocate [numNodes] blocks of size [nodeSize] with lifetime [lifetime] +LinkedListProducer::LinkedListProducer(size_t nodeSize, int numNodes, std::chrono::duration lifetime) : + nodeSize_(nodeSize), numNodes_(numNodes), lifetime_(lifetime) {} diff --git a/stress_test/Producers.h b/stress_test/Producers.h new file mode 100644 index 0000000..c4c4ace --- /dev/null +++ b/stress_test/Producers.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include + +class Allocation { +public: + // sorts based on [freeAfter] field + bool operator<(const Allocation &that) const; + bool operator>(const Allocation &that) const; + // true iff [this->toFree_] is empty + bool isEmpty() const; + std::chrono::high_resolution_clock::time_point freeAfter() const; + Allocation(std::vector toFree, + std::chrono::high_resolution_clock::time_point freeAfter); + // makes an allocation such that [isEmpty()] is true + Allocation() = default; + + // Disable copy constructor: whoever owns the Allocation should deallocate it + Allocation(Allocation const &) = delete; + Allocation &operator=(Allocation const &) = delete; + + // must define a move constructor since we deleted the copy constructor + Allocation(Allocation&&) = default; + Allocation& operator=(Allocation&&) = default; + + // The destructor deallocates the memory in [toFree_] + ~Allocation(); + + // needed to sort + friend void swap(Allocation &a1, Allocation &a2); + +private: + std::vector toFree_; + // absolute time after which this should be freed + std::chrono::high_resolution_clock::time_point freeAfter_; +}; + +class Producer { +public: + virtual Allocation run() const = 0; +}; + +// allocates a vector of size [sz] +class VectorProducer : public Producer { +public: + Allocation run() const; + // allocate, and then free after [lifetime] has elapsed + VectorProducer(size_t vectorSize, std::chrono::duration lifetime, size_t initialSize); + +private: + size_t vectorSize_; + size_t initialSize_; + std::chrono::duration lifetime_; +}; + +/* allocates a block of size [alloc_sz], and then immediately frees it. Repeats + * this [n_allocs] times. */ +class SimpleProducer : public Producer { +public: + Allocation run() const; + SimpleProducer(int allocSize, int numAllocs); +private: + int allocSize_; + int numAllocs_; +}; + +// Allocates many similarly sized blocks, and then frees them all at once later. +class LinkedListProducer : public Producer { +public: + Allocation run() const; + // allocate [numNodes] blocks of size [nodeSize] with lifetime [lifetime] + LinkedListProducer(size_t nodeSize, int numNodes, std::chrono::duration lifetime); +private: + size_t nodeSize_; + int numNodes_; + std::chrono::duration lifetime_; +}; diff --git a/stress_test/ToFreeQueue.cpp b/stress_test/ToFreeQueue.cpp new file mode 100644 index 0000000..58cdf67 --- /dev/null +++ b/stress_test/ToFreeQueue.cpp @@ -0,0 +1,27 @@ +#include "ToFreeQueue.h" + +#include +#include +#include + +void ToFreeQueue::free() { + std::lock_guard guard(this->lock_); + + while (!this->q_.empty() && + this->q_.top().freeAfter() < std::chrono::high_resolution_clock::now()) { + this->q_.pop(); + } +} + +void ToFreeQueue::freeIgnoreLifetime() { + std::lock_guard guard(this->lock_); + + while (!this->q_.empty()) { + this->q_.pop(); + } +} + +void ToFreeQueue::addToFree(Allocation a) { + std::lock_guard guard(this->lock_); + this->q_.push(std::move(a)); +} diff --git a/stress_test/ToFreeQueue.h b/stress_test/ToFreeQueue.h new file mode 100644 index 0000000..ade5c8a --- /dev/null +++ b/stress_test/ToFreeQueue.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include +#include + +#include "Producers.h" + +class ToFreeQueue { +public: + // frees all allocations whose lifetime has elapsed + void free(); + // free all allocations, even if the lifetime hasn't expired + void freeIgnoreLifetime(); + // Add an allocation to be freed after a particular time + void addToFree(Allocation a); + +private: + std::mutex lock_; + std::priority_queue, std::greater> q_; +}; diff --git a/stress_test/distributions/adfinder.txt b/stress_test/distributions/adfinder.txt new file mode 100644 index 0000000..cc32525 --- /dev/null +++ b/stress_test/distributions/adfinder.txt @@ -0,0 +1,91 @@ +8 0.0090475869034 +16 0.0371816165473 +32 0.249494030275 +48 0.153471785857 +64 0.0833917342719 +80 0.0641320417645 +96 0.0990601323096 +112 0.0180347594562 +128 0.0328361596507 +160 0.0333902865652 +192 0.0201117690268 +224 0.058487415762 +256 0.015514937901 +320 0.00665660276036 +384 0.00159740677832 +448 0.000535682381663 +512 0.0131783396516 +640 0.0162427708289 +768 0.00117251634497 +896 0.00778532765506 +1024 0.00863362190594 +1280 0.000847846220479 +1536 0.000696548098703 +1792 0.000598641216258 +2048 0.000544905067297 +2560 0.0076125356438 +3072 6.08895444935e-05 +3584 0.0070287766828 +4096 0.0464055654527 +5120 0.000121436526822 +6144 9.04721265046e-05 +7168 7.72089511569e-05 +8192 0.000207314161953 +10240 0.000168671037045 +12288 3.20615046956e-05 +14336 6.19056613406e-05 +16384 0.00440350366137 +20480 0.000591156492855 +24576 5.35877556583e-05 +28672 1.96489499362e-05 +32768 4.83389923471e-05 +40960 1.6742788683e-05 +49152 1.00914152355e-05 +57344 8.35697420142e-06 +65536 0.000132731185374 +81920 8.65979714299e-06 +98304 2.06259982643e-05 +114688 8.53253102303e-06 +131072 1.06447461919e-05 +163840 7.04806144037e-05 +196608 1.27425412207e-05 +229376 8.70993228116e-06 +262144 4.50411398472e-06 +327680 8.57964799234e-06 +393216 2.33304452325e-06 +458752 1.8239800434e-06 +524288 1.64054244757e-06 +655360 2.64223915462e-05 +786432 9.47537343745e-07 +917504 1.43111506439e-06 +1048576 2.09075263478e-06 +1310720 1.13030423539e-06 +1572864 1.17004345862e-06 +1835008 1.46397956968e-06 +2097152 1.67910793846e-06 +2621440 2.84244435519e-06 +3145728 1.63635054638e-06 +3670016 2.23411565529e-06 +4194304 6.9568792058e-07 +5242880 1.13181331982e-07 +6291456 5.8686616583e-08 +7340032 3.13554208601e-08 +8388608 3.32836954049e-07 +10485760 4.34280962714e-08 +12582912 1.8947393354e-08 +14680064 5.19795746878e-09 +16777216 2.01211256856e-09 +20971520 5.0302814214e-10 +25165824 1.1904999364e-08 +29360128 3.60168149772e-07 +33554432 3.3535209476e-10 +41943040 5.12418000794e-07 +50331648 1.1904999364e-08 +58720256 6.7070418952e-10 +67108864 6.7070418952e-10 +100663296 1.1904999364e-08 +134217728 3.3535209476e-10 +167772160 5.0302814214e-10 +268435456 3.3535209476e-10 +335544320 3.3535209476e-10 +536870912 3.3535209476e-10 diff --git a/stress_test/distributions/adindexer.txt b/stress_test/distributions/adindexer.txt new file mode 100644 index 0000000..c1ecbdd --- /dev/null +++ b/stress_test/distributions/adindexer.txt @@ -0,0 +1,104 @@ +8 0.209085823124 +16 0.029586297063 +32 0.178274773997 +48 0.241066858147 +64 0.0177491726159 +80 0.00316715233976 +96 0.00172374180903 +112 0.0242949424404 +128 0.042802062118 +160 0.017090771598 +192 0.000478063296447 +224 0.000410013192658 +256 0.0151998573151 +320 0.0839238412055 +384 0.00558095305553 +448 0.000161538552304 +512 0.0249588592745 +640 0.0142109771606 +768 0.000120852257388 +896 0.000232773650461 +1024 0.0837722607808 +1280 0.00493321069493 +1536 6.13240332779e-05 +1792 0.000189652124801 +2048 0.000228215920232 +2560 6.82940023995e-05 +3072 8.79778177058e-05 +3584 2.65205712767e-05 +4096 0.000206021089145 +5120 1.30274847089e-05 +6144 9.15062452871e-06 +7168 7.98717709207e-06 +8192 3.07972326457e-05 +10240 6.86707906904e-06 +12288 3.15464911243e-05 +14336 3.19897225942e-06 +16384 2.24374225231e-05 +20480 6.18179653499e-05 +24576 4.39363122955e-06 +28672 2.21741245744e-06 +32768 8.47108491682e-06 +40960 1.73420518899e-05 +49152 1.68619271135e-06 +57344 6.30948211536e-07 +65536 5.22531806653e-06 +81920 1.55959850997e-06 +98304 4.56552308675e-05 +114688 5.5086722641e-07 +131072 3.75851667986e-06 +163840 2.80804434068e-06 +196608 7.63295083494e-07 +229376 5.44202857804e-07 +262144 2.64950858272e-06 +327680 1.04873189625e-06 +393216 7.00811088945e-07 +458752 1.19235480013e-06 +524288 9.51394007181e-07 +655360 4.59595507965e-07 +786432 3.07117058434e-07 +917504 1.508720664e-06 +1048576 3.31441560735e-07 +1310720 5.43033048422e-07 +1572864 8.03828535503e-07 +1835008 4.84449526256e-07 +2097152 5.57797460785e-07 +2621440 8.81598703988e-07 +3145728 4.43335600651e-07 +3670016 2.0883202256e-07 +4194304 3.99407928993e-07 +5242880 5.07401496526e-07 +6291456 8.35394556684e-08 +7340032 2.007491493e-06 +8388608 4.8193155587e-07 +10485760 1.79530291511e-07 +12582912 4.34723765289e-08 +14680064 1.64165908729e-06 +16777216 1.51244389254e-08 +20971520 2.98810968705e-08 +25165824 2.87068563914e-08 +29360128 1.63484849252e-06 +33554432 1.2679581626e-08 +41943040 2.08128586047e-08 +50331648 1.92376039243e-08 +58720256 1.62509232657e-06 +67108864 1.57868877994e-08 +83886080 1.5264018454e-08 +100663296 1.15518676565e-08 +117440512 1.61000001309e-06 +134217728 4.67702198366e-09 +167772160 7.21160898002e-10 +201326592 1.48884830555e-09 +234881024 4.34730411933e-07 +268435456 5.50563696324e-10 +335544320 3.93259783089e-10 +402653184 5.0957605696e-10 +469762048 1.80811986072e-07 +536870912 4.29816326305e-10 +671088640 3.48948821614e-10 +805306368 2.20447033337e-10 +939524096 1.08836583574e-07 +1073741824 2.52023024484e-07 +1342177280 2.65145715725e-07 +1610612736 6.42508941384e-11 +1879048192 5.49876876421e-08 diff --git a/stress_test/distributions/multifeed.txt b/stress_test/distributions/multifeed.txt new file mode 100644 index 0000000..84a7b9b --- /dev/null +++ b/stress_test/distributions/multifeed.txt @@ -0,0 +1,142 @@ +8 0.152825841152 +16 0.0624298739386 +32 0.32177883898 +48 0.140391266756 +64 0.0892268301422 +80 0.0176315647257 +96 0.0155377318314 +112 0.0427224149156 +128 0.0184088020922 +160 0.0206062740843 +192 0.0291457125272 +224 0.00786826861338 +256 0.00732744710948 +320 0.0117055888817 +384 0.00516131905986 +448 0.00477161311241 +512 0.00656374951823 +640 0.00290697088989 +768 0.00171603025145 +896 0.00366906821009 +1024 0.00210134987502 +1280 0.00295046170314 +1536 0.00293633060676 +1792 0.00689489817897 +2048 0.00122428584786 +2560 0.00265963158262 +3072 0.00298039380778 +3584 0.000856269037326 +4096 0.00143912181768 +5120 0.000835374998611 +6144 0.00258876081079 +7168 0.00154101997597 +8192 0.000295868506902 +10240 0.00163702411359 +12288 0.000278552085022 +14336 0.000362782132825 +16384 0.00174070010633 +20480 0.000662920620457 +24576 0.000116024674674 +28672 3.85176590106e-05 +32768 5.46356467897e-05 +40960 8.18061226398e-05 +49152 3.48428326573e-05 +57344 1.41138300222e-05 +65536 7.18129801279e-05 +81920 1.53603621636e-05 +98304 1.42579182199e-05 +114688 7.0013001868e-06 +131072 8.86293956764e-06 +163840 3.52361848708e-05 +196608 2.87794766288e-05 +229376 4.47638074602e-06 +262144 4.40796598777e-06 +327680 3.83673060359e-06 +393216 2.98595200603e-06 +458752 4.52403903561e-06 +524288 1.31922777485e-06 +655360 4.17891539287e-05 +786432 2.52347685673e-06 +917504 1.38350954372e-06 +1048576 7.23194621828e-06 +1310720 2.80836460687e-06 +1572864 3.0161586559e-06 +1835008 9.1152561694e-07 +2097152 3.15808716293e-07 +2621440 3.23979769108e-06 +3145728 3.79440345733e-07 +3670016 2.79862423466e-07 +4194304 2.65001296395e-07 +5242880 2.3036700168e-07 +6291456 9.68065983079e-08 +7340032 1.34697728831e-07 +8388608 7.00364794353e-10 +10485760 5.18995425297e-10 +12582912 8.66275912569e-08 +14680064 7.81283435931e-11 +16777216 1.35608482094e-10 +20971520 2.33268911585e-10 +25165824 8.43596370543e-08 +29360128 3.23674566314e-11 +33554432 1.31702064914e-10 +41943040 1.1719251539e-10 +50331648 1.31997278441e-07 +58720256 3.34835758256e-12 +67108864 5.58059597094e-12 +83886080 8.59411779525e-11 +100663296 8.25928203699e-11 +117440512 1.11611919419e-12 +16384 0.00174070010633 +20480 0.000662920620457 +24576 0.000116024674674 +28672 3.85176590106e-05 +32768 5.46356467897e-05 +40960 8.18061226398e-05 +49152 3.48428326573e-05 +57344 1.41138300222e-05 +65536 7.18129801279e-05 +81920 1.53603621636e-05 +98304 1.42579182199e-05 +114688 7.0013001868e-06 +131072 8.86293956764e-06 +163840 3.52361848708e-05 +196608 2.87794766288e-05 +229376 4.47638074602e-06 +262144 4.40796598777e-06 +327680 3.83673060359e-06 +393216 2.98595200603e-06 +458752 4.52403903561e-06 +524288 1.31922777485e-06 +655360 4.17891539287e-05 +786432 2.52347685673e-06 +917504 1.38350954372e-06 +1048576 7.23194621828e-06 +1310720 2.80836460687e-06 +1572864 3.0161586559e-06 +1835008 9.1152561694e-07 +2097152 3.15808716293e-07 +2621440 3.23979769108e-06 +3145728 3.79440345733e-07 +3670016 2.79862423466e-07 +4194304 2.65001296395e-07 +5242880 2.3036700168e-07 +6291456 9.68065983079e-08 +7340032 1.34697728831e-07 +8388608 7.00364794353e-10 +10485760 5.18995425297e-10 +12582912 8.66275912569e-08 +14680064 7.81283435931e-11 +16777216 1.35608482094e-10 +20971520 2.33268911585e-10 +25165824 8.43596370543e-08 +29360128 3.23674566314e-11 +33554432 1.31702064914e-10 +41943040 1.1719251539e-10 +50331648 1.31997278441e-07 +58720256 3.34835758256e-12 +67108864 5.58059597094e-12 +83886080 8.59411779525e-11 +100663296 8.25928203699e-11 +117440512 1.11611919419e-12 +167772160 8.48250587583e-11 +335544320 8.48250587583e-11