From 5dc0f2654f8596b7846692399a00a1fa17525840 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Mon, 14 Nov 2016 14:38:36 -0500 Subject: [PATCH] Merge develop-2.2 into develop (using existing rebar.conf from develop) --- docs/BATCHING.md | 254 +---- docs/yz-batching-overview.graffle | 926 +++++++++--------- docs/yz-batching-overview.png | Bin 40754 -> 43649 bytes include/yokozuna.hrl | 5 +- priv/yokozuna.schema | 5 +- .../yz_solrq_drain_fsm_intercepts.erl | 27 +- riak_test/yokozuna_essential.erl | 5 +- riak_test/yz_aae_test.erl | 70 +- riak_test/yz_rt.erl | 18 +- riak_test/yz_solrq_test.erl | 31 +- riak_test/yz_stat_test.erl | 100 +- src/yz_doc.erl | 2 + src/yz_entropy_mgr.erl | 2 + src/yz_exchange_fsm.erl | 106 +- src/yz_fuse.erl | 12 + src/yz_index_hashtree.erl | 73 +- src/yz_kv.erl | 107 +- src/yz_misc.erl | 2 +- src/yz_solr.erl | 43 +- src/yz_solr_proc.erl | 8 +- src/yz_solrq.erl | 6 +- src/yz_solrq_drain_fsm.erl | 114 ++- src/yz_solrq_drain_mgr.erl | 53 +- src/yz_solrq_helper.erl | 136 ++- src/yz_solrq_sup.erl | 27 +- src/yz_solrq_worker.erl | 187 ++-- test/yokozuna_schema_tests.erl | 5 +- test/yz_component_tests.erl | 2 +- test/yz_solrq_eqc.erl | 18 +- test/yz_solrq_eqc_ibrowse.erl | 14 +- 30 files changed, 1273 insertions(+), 1085 deletions(-) diff --git a/docs/BATCHING.md b/docs/BATCHING.md index f118902c..8e8960ad 100644 --- a/docs/BATCHING.md +++ b/docs/BATCHING.md @@ -1,7 +1,7 @@ # Introduction -In Yokozuna versions prior to 2.0.4, update operations on Solr (notably, `add` and `delete` operations) are synchronous blocking operations and are performed once-at-a-time in Solr. In particular, calls to the `yz_kv:index/3` Erlang function block until the associated data is written to Solr, and each such call results in an HTTP POST with a single Solr operation. +In Yokozuna versions prior to 2.0.4 (e.g., Riak 2.0.7), update operations on Solr (notably, `add` and `delete` operations) are synchronous blocking operations and are performed once-at-a-time in Solr. In particular, calls to the `yz_kv:index/3` Erlang function block until the associated data is written to Solr, and each such call results in an HTTP POST with a single Solr operation. Yokozuna version 2.0.4 introduces batching and asynchronous delivery of Solr operations. The primary objective of this work is to decouple update operations in Solr from the Riak vnodes that are responsible for managment of replicas of Riak objects across the cluster. Without batching and asynchronous delivery, Riak vnodes have to wait for Solr operations to complete, sometimes an inordinate amount of time, which can have an impact on read and write operations in a Riak cluster, even for Riak objects that aren't being indexed through Yokozuna! This feature is intended to free up Riak vnodes to do other work, thus allowing Riak vnodes to service requests from more clients concurrently, and increasing operational throughput throughout the cluster. @@ -14,31 +14,21 @@ This document describes the batching and asynchronous delivery subsystem introdu The Yokozuna batching subsystem introduces two sets of additional `gen_server` processes into each Riak node, a set of "workers", and a set of "helpers". The job of the worker processes is to enqueue updates from components inside of Riak -- typically Riak vnodes, but also sometimes parts of the Yokozuna AAE subsystem -- and to make enqueued objects available to helper processes, which, in turn, are responsible for dispatching batches of operations to Solr. -For each Riak node, there is a fixed, but configurable, number of helper and worker processes, and by default, 10 of each are created. A change to the number of workers or helpers requires a restart of the Yokozuna OTP application. +For each vnode and Solr index on a Riak node, there is a pair of helper and worker processes which are responsible for enqueing and dispatching batches of operations into Solr. For example, on a Riak node with 12 vnodes and 5 indices, there will be 60 such pairs. Each pair of helper and worker processes has an associated supervisor, which oversees the lifecycle of each pair of `gen_server` processes. Each such supervisor is itself supervised by a supervisor for the entire batching subsytem, which in turn is supervised by the supervision hierarchy for the Yokozuna application. -When enqueuing an object, a worker process is selected by hashing the Solr index (core) with which the operation is associated, along with the Riak bucket and key (bkey). This hash, modulo the number of workers, determines the worker process on which the datum is enqueued. When a batch of objects is dequeued from a worker process, a helper is randomly selected from among the set of configured helpers using a uniform PRNG. These algorithms for selecting workers and helpers is designed to provide as even a distribution of load across all workers and helpers as possible. - -Once a batch is dequeued from a worker, the Riak objects are transformed into Solr operations, and a single HTTP request is formed, containing a batch of Solr operations (encoded in a JSON payload). The batch is then delivered to Solr. +When enqueuing an object, a worker and helper process pair is selected based on the Solr index (core) with which the operation is associated, along with the Riak partition on which the Riak object is stored. The associated helper process will periodically dequeued batches of objects from a worker process, translate the objects into a set of Solr "operations", and dispatch those operations in batches of HTTP requests to the Solr server running on the same node. The following diagram illustrates the relationship between Riak vnodes, Yokozuna workers and helpers, and Apache Solr: -![YZ Batching Overview](https://raw.githubusercontent.com/basho/yokozuna/feature-solr-batching-rebased/docs/yz-batching-overview.png) +![YZ Batching Overview](https://raw.githubusercontent.com/basho/yokozuna/docs/yz-batching-overview.png) Each helper process is stateless; a helper's only role is to dequeue batches from the workers, to transform those batches into something that Solr can understand, and to dispatch the transformed batches to Solr (and report the results back to the worker process, on whose behalf it is doing the work). The worker processes, on the other hand, maintain the queues of objects that are ready for update in Solr, in addition to managing the run-time behavior between the batching subsystem and the collection of vnodes and other processes that are communicating with the workers. -In order to batch operations into a single HTTP POST into Solr, all operations must be organized under the same Solr core, i.e. Riak search index. As a consequence, each Yokozuna worker process maintains a table (`dict`) of "indexq" structures, keyed off the Riak search index. These indexq structures include, most critically, the enqueued object - -Indexq structures are created on-demand in each worker process, as data is added to the system. In most cases, all worker processes will contain entries for each Riak serch index, but it is sometimes possible for a worker process to be missing an entry for given index because, for example, it has not yet seen an object that needs to be updated for a given search index. This is expected behavior. - -The relationship between Riak search indices and indexq structures within each worker process is illustrated in the following diagram: - -![YZ Batching Worker](https://raw.githubusercontent.com/basho/yokozuna/feature-solr-batching-rebased/docs/yz-batching-worker.png) - ## Batching Parameters -When an update message is sent to a worker process, it is immediately enqueued in the indexq structure associated with the index for which the operation is destined. +When an update message is sent to a worker process, it is immediately enqueued in the worker associated with the index and Riak partition for which the operation is destined. The Yokozuna batching subsystem provides configuration parameters that drive batching benavior, including configuration of: @@ -52,7 +42,7 @@ If when enqueing an update operation the number of batched messages is smaller t ## Backpressure -Each worker process is configured with a high water mark (10000, by default), which represents the total number of messages that may be enqueued across all indexq structures in a given worker process before calls into the batching subsystem (update/index) will block calling vnodes. If the total number of enqueued messages exceeds this threshold, calling vnodes (and parts of the AAE subsystem) will block until data is successfully written to Solr, or it is purged, in a manner described below. +Each worker process is configured with a high water mark (1000, by default), which represents the total number of messages that may be enqueued in a given worker process before calls into the batching subsystem (update/index) will block calling vnodes. If the total number of enqueued messages exceeds this threshold, calling vnodes (and parts of the AAE subsystem) will block until data is successfully written to Solr, or it is purged, in a manner described below. This way, the batching subsystem exerts back-pressure on the vnode and AAE systems, in the case where Solr is being driven beyond its operational capacity. @@ -64,13 +54,12 @@ In the undesirable cases where Solr becomes unresponsive (e.g., data corruption, ## Purge Strategies -If a Solr core has become unresponsive and the specified error threshold has been traversed, and if, in addition, the high water mark has been exceeded, then the yokozuna batching system has a mechanism for automatically purging enqueued entries, so as to allow vnodes to continue servicing requests, as well as to allow update operations to occur for indices that are not in a pathological state. +If a Solr core has become unresponsive and the specified error threshold has been traversed, and if, in addition, the high water mark has been exceeded for a particular `yz_solrq_worker` process, then the yokozuna batching system has a mechanism for automatically purging enqueued entries, so as to allow vnodes to continue servicing requests, as well as to allow update operations to occur for indices that are not in a pathological state. -The yokozuna batching subsystem supports 4 different purge strategies: +The yokozuna batching subsystem supports 3 different purge strategies: -* `purge_one` (default behavior): Purge the oldest entry from a randomly selected indexq structure among the set of search indexes which have crossed the error threshold; -* `purge_index`: Purge all entries from a randomly selected indexq structure among the set of search indexes which have crossed the error threshold; -* `purge_all`: Purge all entries from all indexq structure among the set of search indexes which have crossed the error threshold; +* `purge_one` (default behavior): Purge the oldest entry from the `yz_solrq_worker`; +* `purge_index`: Purge all entries from the `yz_solrq_worker`; * `off`: Perform no purges. This has the effect of blocking vnodes indefinitely, and is not recommended for production servers. Note that purging enqueued messages should be considered safe, as long as Active Anti-Entropy (AAE) is enabled, as the AAE subsystem will eventually detect missing entries in Solr, and will correct for the difference. @@ -93,29 +82,39 @@ Recall that a Riak object is indexed in Solr if it is in a bucket which is assoc In order for Riak/KV and YZ AAE trees to be comparable, they must represent the same replica sets, where a replica set is determined by its initial Riak partition and replication factor (`n_val`). Because AAE trees are ignorant of buckets -- they are based entirely on the ring topology and replication factor, YZ AAE trees need to contain entries not only for Riak objects that are indexed in Solr, but also Riak object that are not. If YZ AAE trees did not contain hashes for entries that are not indexed in Solr, the comparison with Riak/KV AAE trees would always show data missing in Solr, and thus repairs would always be attempted. -# Configuration and Statistics +# Commands, Configuration, and Statistics The batching subsystem is designed to be primarily invisible to the user, except perhaps for improved throughput under high load. However, some parameters of the batching subsystem are tunable via Cuttlefish configuration properties in the `riak.conf` configuration file (or, alternatively, via the `advanced.config` file). Changes to this file require a restart of Riak (or, alternatively, of just the Yokozuna application, via the Riak console). The batching subsystem also introduces a set of statistics, which provides operators visibility into such measurements as available queue capacity, averages and histograms for batch sizes, AAE activity, and so forth. -This section describes the configuration parameters and statistics of the batching subsystem, from the user's point of view. +This section describes the commands, configuration parameters, and statistics of the batching subsystem, from the user's point of view. -## Configuration +## Commands -The behavior of the batching subsystem may be controlled via the following Cuttlefish configuration parameters, as defined in `riak.conf`. Consult the Cuttlefish schema (TODO add link) for the associated configuration settings in the Riak `advanced.config` file. +The `riak-admin` command may be used to control the participation of a node in distributed query. This command can be useful, for example, if a node is down for repair or reindexing. The node can be temporarily removed from coverage plans, so that it is not consulted as part of a distributed query. -* `search.queue.batch.minimum` (default: 1) The minimum batch size, in number of Riak objects. Any batches that are smaller than this amount will not be immediately flushed to Solr, but are guaranteed to be flushed within the value specified in `search.queue.batch.flush_interval`. +Here are sample usages of this command: -* `search.queue.batch.maximum` (default: 100) The maximum batch size, in number of Riak objects. Any batches that are larger than this amount will be split, where the first `search.queue.batch.maximum` objects will be flushed to Solr, and the remaining objects enqueued for that index will be retained until the next batch is delivered. This parameter ensures that at most `search.queue.batch.maximum` objects will be delivered into Solr in any given request. + shell$ riak-admin set search.dist_query=off # disable distributed query for this node + shell$ riak-admin set search.dist_query=on # enable distributed query for this node + shell$ riak-admin show search.dist_query # get the status of distributed query for this node -* `search.queue.batch.flush_interval` (default: 1 second) The maximum delay between notification to flush batches to Solr. This setting is used to increase or decrease the frequency of batch delivery into Solr, specifically for relatively low-volume input into Riak. This setting ensures that data will be delivered into Solr in accordance with the `search.queue.batch.maximum` and `search.queue.batch.maximum` settings within the specified interval. Batches that are smaller than `search.queue.batch.maximum` will be delivered to Solr within this interval. This setting will generally have no effect on heavily loaded systems. +> Note. that even if a node is removed from a distributed query, it's search endpoint may still be consulted for query. If distributed query is disabled on the node and the search endpoint is used for query, only the other available nodes in the cluster will be consulted as part of a distributed query. -* `search.queue.high_watermark` (default: 10000) The queue high water mark. If the total number of queued messages in a Solrq worker instance exceeds this limit, then the calling vnode will be blocked until the total number falls below this limit. This parameter exercises flow control between Riak and the Yokozuna batching subsystem, if writes into Solr start to fall behind. +Using this command will only temporarily enable or disable distributed query until explicitly disabling or re-enabling via the same command, or after restart. See the `search.dist_query` configuration setting to control a node's participation in drributed queries across restarts of a Riak server. -* `search.queue.worker_count` (default: 10) The number of solr queue workers to instantiate in the Yokozuna application. Solr queue workers are responsible for enqueing objects for insertion or update into Solr. Increasing the number of solr queue workers distributes the queuing of objects, and can lead to greater throughput under high load, potentially at the expense of smaller batch sizes. +## Configuration + +The behavior of the batching subsystem may be controlled via the following Cuttlefish configuration parameters, as defined in `riak.conf`. Consult the Cuttlefish schema (TODO add link) for the associated configuration settings in the Riak `advanced.config` file. + +* `search.queue.batch.minimum` (default: 10) The minimum batch size, in number of Riak objects. Any batches that are smaller than this amount will not be immediately flushed to Solr, but are guaranteed to be flushed within the value specified in `search.queue.batch.flush_interval`. -* `search.queue.helper_count` (default: 10) The number of solr queue helpers to instantiate in the Yokozuna application. Solr queue helpers are responsible for delivering batches of data into Solr. Increasing the number of solr queue helpers may increase concurrent writes into Solr. +* `search.queue.batch.maximum` (default: 500) The maximum batch size, in number of Riak objects. Any batches that are larger than this amount will be split, where the first `search.queue.batch.maximum` objects will be flushed to Solr, and the remaining objects enqueued for that index will be retained until the next batch is delivered. This parameter ensures that at most `search.queue.batch.maximum` objects will be delivered into Solr in any given request. + +* `search.queue.batch.flush_interval` (default: 500ms) The maximum delay between notification to flush batches to Solr. This setting is used to increase or decrease the frequency of batch delivery into Solr, specifically for relatively low-volume input into Riak. This setting ensures that data will be delivered into Solr in accordance with the `search.queue.batch.maximum` and `search.queue.batch.maximum` settings within the specified interval. Batches that are smaller than `search.queue.batch.maximum` will be delivered to Solr within this interval. This setting will generally have no effect on heavily loaded systems. + +* `search.queue.high_watermark` (default: 1000) The queue high water mark. If the total number of queued messages in a Solrq worker instance exceeds this limit, then the calling vnode will be blocked until the total number falls below this limit. This parameter exercises flow control between Riak and the Yokozuna batching subsystem, if writes into Solr start to fall behind. * `search.index.error_threshold.failure_count` (default: 3) The number of failures within the specified `search.index.error_threshold.failure_interval` before writes into Solr will be short-circuited. Once the error threshold is crossed for a given Riak index (i.e., Solr core), Yokozuna will make no further attempts to write to Solr for objects destined for that index until the error is reset. @@ -123,8 +122,22 @@ The behavior of the batching subsystem may be controlled via the following Cuttl * `search.index.error_threshold.reset_interval` (default: 30 seconds) The amount of time it takes for a an error error threashold traversal associated with a Solr core to reset. If `search.index.error_threshold.failure_count` failures occur within `search.index.error_threshold.failure_interval`, requests to Solr for that core are short-circuited for this interval of time. -* `search.queue.high_watermark.purge_strategy` (default: `purge_one`) The high watermrk purge strategy. If a Solr core threshold is traversed, and if the number of enqueued messages in a solr worker exceeds `search.queue.high_watermark`, then Yokozuna will use the defined purge strategy to purge enqueued messages. Valid values are `purge_one`, `purge_index`, `purge_all`, and `off`. +* `search.queue.high_watermark.purge_strategy` (default: `purge_one`) The high watermrk purge strategy. If a Solr core threshold is traversed, and if the number of enqueued messages in a solr worker exceeds `search.queue.high_watermark`, then Yokozuna will use the defined purge strategy to purge enqueued messages. Valid values are `purge_one`, `purge_index`, and `off`. + +* `search.anti_entropy.throttle` (default: on) Whether the throttle for Yokozuna active anti-entropy is enabled. +* `search.anti_entropy.throttle.$tier.solrq_queue_length` Sets the throttling tiers for active anti-entropy. Each tier is a minimum solrq queue size and a time-delay that the throttle should observe at that size and above. For example: + + search.anti_entropy.throttle.tier1.solrq_queue_length = 0 + search.anti_entropy.throttle.tier1.delay = 0ms + search.anti_entropy.throttle.tier2.solrq_queue_length = 40 + search.anti_entropy.throttle.tier2.delay = 5ms + + will introduce a 5 millisecond sleep for any queues of length 40 or higher. If configured, there must be a tier which includes a mailbox size of 0. Both `.solrq_queue_length` and `.delay` must be set for each tier. There is no limit to the number of tiers that may be specified. + +* `search.anti_entropy.throttle.$tier.delay` See above. + +* `search.dist_query` (Default: on) Enable or disable this node in distributed query plans. If enabled, this node will participate in distributed Solr queries. If disabled, the node will be excluded from yokozuna cover plans, and will therefore never be consulted in a distributed query. Note that this node may still be used to execute a query. Use this flag if you have a long running administrative operation (e.g., reindexing) which requires that the node be removed from query plans, and which would otherwise result in inconsistent search results. The following options are hidden from the default `riak.conf` file: @@ -171,6 +184,10 @@ The Yokozuna batching subsystem maintains a set of statistics that provide visib * `search_detected_repairs_count` The total number of AAE repairs that have been detected when comparing YZ and Riak/KV AAE trees. Note that this statistic is a measurement of the differences found in the AAE trees; there may be some latency between the time the trees are compared and the time that the repair is written to Solr. +* `search_index_bad_entry_(count|one)` The number of writes to Solr that have resulted in an error due to the format of the data (e.g., non-unicode data) since the last restart of Riak. + +* `search_index_extract_fail_(count|one)` The number of failures that have occurred extracting data into a format suitable to insert into Solr (e.g., badly formatted JSON) since the last start of Riak. + The following statistics refer to query operations, and are not impacted by Yokozuna batching. They are included here for completeness: * `search_query_throughput_(count|one)` The total count of queries, per Riak node, and the count of queries within the metric measurement window. @@ -178,176 +195,3 @@ The following statistics refer to query operations, and are not impacted by Yoko * `search_query_latency_(min|mean|max|median|95|99|999)` The minimum, mean, maximum, median, 95th percentile, 99th percentile, and 99.9th percentile measurements of querying latency, as measured from the time it takes to send a request to Solr to the time the response is received from Solr. * `search_query_fail_(count|one)` The total count of failed queries, per Riak node, and the count of query failures within the metric measurement window. - - -# Implementation Notes - -This section describes the internal components that form the batching subsystem. It is targeted primarily at a developer audience. By convention, the Erlang modules that form this subsystem contain the `solrq` moniker in their names. - -In the remainder of this document, we generalize update and delete operations that are posted to solr as simply "Solr operations", without considering whether the operations originated as the result of a Riak put operations, delete operation, or AAE and/or read repair. - -## Supervision Tree - -The Solrq subsystem contains a supervision hierarchy that branches off the `yz_general_sup` supervisor in the Yokozuna supervision tree. The `yz_solrq_sup` is the top-level supervisor in this hierarchy. - -The `yz_solrq_sup` supervisor monitors a pool of Solrq workers and a pool of Solrq helpers. The objects in this pool are OTP registered gen_server processes, taking on the names `yz_solrq_i` and `yz_solrq_helper_j`, respectively, where i and j are padded integer values in the range {1..n} and {1..m}, respectively, where n and m are defined in config via the `num_solrq` and `num_solrq_helpers` Yokozuna configuration properties, respectively. These values are configurable, but both default to 10. - -The following diagram illustrates this supervision hierarchy: - -![YZ Batching Supervision Tree](https://github.com/basho/internal_wiki/blob/master/images/yokozuna/yz-solrq-supervision-tree.png) - -There will generally be a `yz_solrq_sup` per `yokozuna` application instance. - -> **Note.** The `yz_solr_sup` module also provides an entrypoint API for components in the batching subsystem. Beware that these APIs are simply convenience functions through a common module, and that all calls through these APIs are sequential Erlang functions, and are not executed in the context of the `yz_solrq_sup` supervisor OTP process. - -## Solrq Component Overview - -The `yz_solrq_sup`, `yz_solrq`, and `yz_solrq_helper` processes form the major components in the Yokozuna batching subsystem. - -When a `riak_kv_vnode` services a put or delete request, an API call is made into the `yz_solrq_sup` module to locate a `yz_solrq` instance from the pool of Solrq workers, by taking the (portable) hash of the index and the bucket and key (bkey) associated with the Riak object, and "dividing" that hash space by the number of workers in the pool, n: - - hash({Index, BKey}) mod n -> yz_solrq_i - -This way, we get a roughly even distribution of load to all workers in the pool, assuming an even distribution of writes/deletes to indices and partitions. - -Once a worker is located, the Riak Object and associated operational data is enqueued onto the queue associated with the index on the worker. (The internal structure of the Solrq worker process is discussed below in more detail.) - -It is the job of the `yz_solrq_helper` process to periodically pull batches of data that have been enqueued on the worker queues, to prepare the data for Solr, including extracting fields via Yokozuna extractors, as well as translation to Solr operations, and to dispatch Solr operations to Solr via HTTP POST operations. - -The `yz_solrq_helper` instances form a pool, as well, and an instance of a helper is selected by using a uniform random distribution. - -The following diagram illustrates the relationship between these components: - -![YZ Batching Overview](https://github.com/basho/internal_wiki/blob/master/images/yokozuna/yz-batching-overview.png) - -The following subsections describe these components in more detail. - -### The Solrq Worker Process(es) - -The Yokozuna Solrq batching subsystem maintains a fixed-size (but configurable) pool of worker processes, which are responsible for enqueuing messages to be delivered to Solr. - -The Solrq worker processes provide caching of (add/delete) operations written to Solr. As gen_server processes, they maintain state about what data is cached for batching, as well as book keeping information about how the queues are configured. The Solrq helper processes (see below) coordinate with the worker processes, and do the actual writes to Solr. But it is the worker processes that are the first line of caching, between Yokozuna and Solr. - -Solr workers are gen_server processes that form a pool of batching/caching resources. As a pool of resources, they provide increased throughput under high load, as indexing/delete operations can be distributed evenly across workers in the pool. However, when posting an operation (add/delete) to Solr, all operations (or any batched operations thereof) need to take place under the umbrella of a single Solr index (or "core"). Specifically, the HTTP POST operation to Solr to update an index contains the Solr index as part of its URL, even if that POST operation contains multiple documents. - -As a consequence, the Solrq workers must partition the operations by index, so that when batches are delivered to Solr, each batch is POSTed under the URL of a single core. - -Each Solrq process manages this partitioning by maintaining a dictionary of index -> `indexq` mappings, where the index key is a Solr index name, and an `indexq` is a data structure that holds the queue of operations to be POSTed to Solr under that index. - -> **Note.** This mapping is encpasulated in an Erlang `dict` structure, but the population of entries in this dictionary are done lazily. Initially, the dictionary on a `yz_solrq` instance is empty, and as operations are performed on an index, entries are added. Eventually, a given worker instance may contain an entry for each index configured by the user, but due to the luck of the draw, there may be some solrq instances that do not contain entries for some indices (e.g., if objects map to a strict subset of partitons on a given node, for example). - -The `indexq` structure contains book keeping information, such as the queue of data for that index, its length, cached configuration information for that queue, such as its minimum and maximum batch size, and other state information described in more detail below. - -The Yokozuna Solrq worker process is illustrated in the following diagram: - -![YZ Solrq Worker](https://github.com/basho/internal_wiki/blob/master/images/yokozuna/yz-solrq-worker.png) - -### Solrq Helper Process(es) and Batching Protocol - -The `yz_solrq_helper` process is a stateless `gen_server` OTP process, whose only role is to dequeue data from `yz_solrq` instances, to dispatch the data to Solr, and to reply back to the worker process when a batch has completed. As a stateless object, there is not much to describe about a helper process, except for the protocol of messages that are sent between the `riak_kv_vnode`, `yz_solrq` worker, and `yz_solrq_helper` processes. - -When an indexing operation (add/delete) is requested from a `riak_kv_vnode`, a `yz_solrq` worker process is located (as described above), and a syncronous index message is delivered to the worker, via a `gen_server:call`: - - {index, Index, {BKey, Docs, Reason, P}} - -If the total number of queued messages for this worker is not above the configured high water mark (See the Backpressure section, below), an `ok` is immediately delivered back to the vnode, and the requested data is enqueued onto the `yz_solrq` worker, keyed off the supplied index. - -If the number of queued messages is above the configured minimum (default: 1), and if there are currently no pending helpers who have been told to request a batch, the `yz_solrq` worker will locate a `yz_solrq_helper` as described above, and send it an asynchronous message (via `gen_server:cast`), telling it that the worker is ready to deliver a batch: - - {ready, Index, QPid} - -> **Note.** This asynchronous message is delivered via a cast to the worker. No further work is required on the part of the worker, and it does not synchronously wait for a response from the helper. - -Upon receipt of the `ready` message, the selected helper will send an asynchronous message back to the worker, requesting a batch for the specified index: - - {request_batch, Index, HPid} - -Upon receipt of this message, the worker will select a batch of messages to deliver, typically less than or equal to the size of the configured maximum batch size, and send a batch message back to the helper: - - {batch, Index, BatchMax, QPid, Entries} - -> **Note.** The number of entries in the batch may exceed the configured batch maximum in the case where the queues are being drained. See the "Draining" section below for more information. - -> **Comment.** I'd clear up what async means here. We're still "synchronously" waiting for the solr http update to finish, via an ibrowse call, before we return the cast message, right? - -Upon receipt of the `batch` message, the helper dispatches the batch (or batches, in the case of draining) to Solr, and then sends an asynchronous message back to the worker, indicating that the batch (or batches) have completed, the number of messages that were delivered into Solr, and a return status, indicating whether the writes to Solr were successful or not: - - {batch_complete, NumDelivered, Result} - -> *Note.* The `batch_complete` message is new, and has not been shipped with any of the patches to date. - -Upon receipt of the `batch_complete` message, the worker will: - -* Decrement the number of queued messages by the number of delivered messages -* Unblock any waiting vnodes if the total number of queued messages is below the high water mark -* Mark the pending state of the batch back to false, so that new batches for the same index can proceed -* Request a helper, if there is still data left in the queues to be flushed and if the number of queued messages is above the configured minimum. -* Re-queue (but pre-pend) any undelivered messages, in case of any failures in delivery - -The Yokozuna batching protocol between vnodes, workers, and helpers is illustrated in the following diagram. - -![YZ Solrq Batching Protocol](https://github.com/basho/internal_wiki/blob/master/images/yokozuna/yz-batching-sequence.png) - - - -### Flushing - -In the above scenario, data enqueued for a specific index is written to Solr as soon as the number of queued messages exceeds the configured minimum, which defaults to 1. Hence, in many cases, enqueued messages get sent almost immediately after being enqueued, but asynchronously from the perspective of vnodes. - -> *Note*. This does not entail that batch sizes are always 1; because the messaging protocol between workers and helpers is asynchronous, more messages may arrive in the queue between the time that a worker is notified that a batch is ready and the time that the batch is actually retrieved from the queues. In heavily loaded systems, the queues typically grow in size in this time. - -What happens, however, if the user configures a relatively high minimum batch size, but data drips into the queues at a relatively slow rate? - -In this case, the batching subsystem will set a timer, specified by the `solrq_delayms_max` configuration setting (default: 1000ms). If no data has been flushed within this time interval (which would happen, for example, as the result of a write into the queue), then a batch will automatically be initiated, along the lines of the protocol described above. - -## Backpressure - -Each `yz_solrq` worker maintains a record of how many messages are enqueued in all of the `indexq` structures held in the worker. When data is sent to a worker from a vnode, the worker will increment the count of queued messages. If the total count exceeds the configured high water mark (`solrq_queue_hwm`; default: 10000), a reply to the calling vnode is deferred until the total count of queued messages again falls below this limit. - -The number of enqueued messages is decremented once the `batch_complete` message is received from the associated helper process, where the message received back from the helper contains the number of messages that have been delivered to Solr. - -> **Note.** In previous implementations of the Yokozuna batching patch, the total number of enqueued messages was decremented when a batch was delivered to the helper for dispatch to the worker, but this behavior has changed with the introduction of the `batch_complete` message. - -> **Note.** Setting the `solrq_batch_min` and `solrq_batch_max` values to 1 results in immediate backpressure on the vnode, until the message has been successfully delivered to Solr. This special case of the Yokozuna batching patch simulates the `dw` semantics of the pre-batching Yokozuna implementation. - -### Fuses - -Yokozuna batching makes use of the [Fuse](https://github.com/jlouis/fuse) library, an OTP application which supports a circuit-breaker pattern. Every time some condition applies (at the discretion of the application), a fuse is "melted" slightly. If some configured number of melts occur within a configured time interval, then the fuse "blows", and applications can then modify their behavior, with the knowledge that some operation is likely to be fruitless. Fuses eventually "heal", after a configured amount of time, presumably while not under load. - -Yokozuna uses this pattern in its handling of communication with Solr. For each Solr index, a fuse is created, which represents the connection to Solr. If requests to Solr continuously fail (for a given index), the fuse for that index is blown, and any subsequent Solr requests for that index are short-circuited. Once a fuse heals, batching resumes. - -The Fuse library makes use of Erlang alarm handling, whereby fuse events (fuse "trips" and "heals") are delivered through Erlang alarm handlers. The `yz_events` event handler subscribes to these events, and will notify the batching subsystem when a fuse blows or heals. - -If a fuse trips, this has the effect of effectively pausing the delivery of any batches to Solr, for a given Solr index (the index associated with the tripped fuse). Once a fuse heals, delivery of batches is resumed. - -If delivery of batches to Solr is paused, then it is possible that enqueued messages will pile up, potentially reaching the configured high water mark (and thus causing back-pressure on the calling vnodes). If the `purge_blown_indices` yokozuna configuration variable is set to true (its default value), then when the high water mark is reached, then entries will be discarded on indices that are blocked. A warning will be logged to the system logs, indicating which index has had data discarded (but not the data contents). - -> Note. The rationale for defaulting this value to true is that AAE is also enabled by default on Riak systems, and that any divergence between Riak and Yokozuna can be eventually repaired by AAE. - -If the `purge_blown_indices` yokozuna configuration variable is set to false, then no data will be automatically purged from indices that have blown fuses, and the only way to resume batching for *all* indices (and therefore to relieve back pressure on the vnodes on the Riak node) is for the fuse to heal. For pathological indices, this may take a long time! This variable should only be set to false if Yokozuna AAE is not enabled, and if Solr indices are known to be well-behaved. Otherwise, there is a non-trivial risk that all of Riak can wedge while fuses heal. - -## Draining - -Some applications have the need to not only to periodically flush idle queues, but also to completely drain the contents of all queues on demand, or at least all of the messages associated with a given Riak partition. These applications include stopping the Yokozuna application (to ensure everything in memory is flushed to Solr), as well as the YZ AAE subsystem, which, when exchanging hash trees between Riak K/V and Yokozuna, snapshots the Riak K/V hash tree, drains the queues, and then snapshots the Yokozuna hash tree, in order to minimize divergence between the two hash trees. - -Draining is provided as a `drain` function in the `yz_solrq_mgr` module, which, when invoked, will spawn and monitor a `gen_fsm` process, `yz_solrq_drain_fsm`. The role of the FSM is to trigger a drain on all of the `yz_solrq` workers in the pool, and then to wait for all of the drains to complete. Once all of the queues have been drained, the FSM terminates, and the calling process will receive a `'DOWN'` message from the FSM it is monitoring, and the drain function can then return to the caller. If not all of the queues have drained within a configured timeout, the drain function will return `{error, timeout}`. - -In the current implementation, there may only be one Drain FSM active at time. An attempt to call the `drain` funtion while a drain is active will result in a return value of `{error, in_progress}` - -The `yz_solrq_drain_fsm` has two states: - -* *prepare* In this state, the Drain FSM will iterate over all `yz_solrq` instances, generate a token (via `erlang:make_ref()`) for each instance, and send the `{drain, DPid, Token, Partition}` message to each worker, where `DPid` is the PID of the drain FSM, and `Token` is the generated token, and `Partition` is the desired Riak partition to drain (or `undefined`, if all messages should be drained). It will then enter the *wait* state. - -* *wait* In this state, the Drain FSM will wait for `{drain_complete, Token}` messages back from each `yz_solrq` worker instance. Once all delivered tokens have been received, the Drain FSM will terminate normally. - -When a `yz_solrq` worker receives a `{drain, DPid, Token, Partition}` message, it will iterate over all of it `indexq` structures, set the `draining` flag on each structure, and initiate the batching protocol with an associated helper, as described above. However, unlike the normal batching case, when the worker is in the draining state, it will deliver *all* of its enqueued messages that match the specified partition (or all messages, if `Partition` is `undefined`) to the helper, along with the configured maximum batch size. In this case, the helper will iterate over all of the messages it has been sent, and form batches of messages for delivery to Solr. For example, if the batch size is 100 and 1027 messages have been enqueued, the helper will end up sequentially delivering 10 batches of 100, and then an 11th of size 27. - -While in the draining state, and messages that get enqueued get put onto a special "auxiliary queue" (`aux_queue`), which prevents them from getting sent in any batches to Solr. This prevents messages new messages delivered during the drain phase from getting written to Solr. When the drain for an `indexq` is completed, the `draining` flag is reset to false, and any messages on the `aux_queue` are moved back to the normal queue, for subsequent dispatch into Solr. - -Once the batch has completed and a `batch_complete` message is sent back to the worker, the worker records which index has been drained. Once all of the indices have been drained in the worker, it sends the `{drain_complete, Token}` message back to the Drain FSM, thus completing the drain for that `yz_solrq` worker instance. It will not, however, continue with batching until it receives a `batch_complete` message back from the `yz_solrq_drain_fsm`. This blocks writes into Solr until all queues have been drained. - -The `yz_solrq_drain_fsm` will stay in the waiting state until all of the tokens it has delivered to the `yz_solrq` instances have been returned. Once all of the tokens are returned, it will update the YZ index hashtree before firing off a `drain_complete` message to all of the `solrqs`, indicating that they can proceed with normal batching operations. Any messages that have been cached on the auxiliary queues are then moved to the normal queue, and batching proceeds as usual. The `yz_solrq_drain_fsm` then terminates, indicating to the caller that all queues have drained. - -The relationship between the caller of the `drain` function (in this example, `yz_exchange_fsm`), the `yz_solrq_drain_fsm`, and the Solrq workers and helpers is illustrated in the following sequence diagram: - -![YZ Solrq Draining](https://github.com/basho/internal_wiki/blob/master/images/yokozuna/yz-solrq-draining.png) diff --git a/docs/yz-batching-overview.graffle b/docs/yz-batching-overview.graffle index cceb03c7..ea6a2726 100644 --- a/docs/yz-batching-overview.graffle +++ b/docs/yz-batching-overview.graffle @@ -14,7 +14,7 @@ BackgroundGraphic Bounds - {{0, 0}, {1466, 576}} + {{0, 0}, {733, 576}} Class SolidGraphic ID @@ -42,111 +42,35 @@ ColumnSpacing 36 CreationDate - 2015-12-23 01:06:49 +0000 + 2016-11-07 18:20:22 +0000 Creator Fred Dushin DisplayScale - 1 0/72 in = 1.0000 in + 1 0/72 in = 1 0/72 in GraphDocumentVersion 8 GraphicsList Bounds - {{393.00000184774399, 238.50006103515625}, {98, 24}} + {{508, 454}, {55, 14}} Class ShapedGraphic FitText YES Flow Resize - FontInfo - - Color - - w - 0 - - Font - Helvetica - Size - 12 - ID - 313 - Line - - ID - 307 - Position - 0.49700599908828735 - RotationType - 0 - + 37 Shape Rectangle Style - shadow - - Draws - NO - - stroke + fill Draws NO - - Text - - Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 dequeue (batch)} - - Wrap - NO - - - Bounds - {{99.087750594127698, 219.72650554174598}, {101, 24}} - Class - ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Color - - w - 0 - - Font - Helvetica - Size - 12 - - ID - 312 - Line - - ID - 302 - Position - 0.44889700412750244 - RotationType - 0 - - Shape - Rectangle - Style - shadow Draws @@ -160,20 +84,24 @@ Text + Pad + 0 Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 enqueue (object)} +\f0\fs24 \cf0 Riak node} + VerticalPad + 0 Wrap NO Bounds - {{587, 315.00003051757812}, {11, 14}} + {{413, 415}, {11, 14}} Class ShapedGraphic FitText @@ -181,7 +109,7 @@ Flow Resize ID - 310 + 36 Shape Rectangle Style @@ -207,7 +135,7 @@ Pad 0 Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc @@ -220,100 +148,74 @@ NO - Bounds - {{587, 180.50003051757812}, {11, 14}} Class - ShapedGraphic - FitText - YES - Flow - Resize + LineGraphic + Head + + ID + 23 + Info + 4 + ID - 309 - Shape - Rectangle + 35 + Points + + {411, 387} + {423, 387} + Style - fill - - Draws - NO - - shadow - - Draws - NO - stroke - Draws - NO + HeadArrow + 0 + Legacy + + TailArrow + 0 - Text + Tail - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 ...} - VerticalPad - 0 + ID + 22 + Info + 3 - Wrap - NO - Bounds - {{279, 180.50003051757812}, {11, 14}} Class - ShapedGraphic - FitText - YES - Flow - Resize + LineGraphic + Head + + ID + 21 + ID - 308 - Shape - Rectangle + 34 + Points + + {411, 351} + {423, 351} + Style - fill - - Draws - NO - - shadow - - Draws - NO - stroke - Draws - NO + HeadArrow + 0 + Legacy + + TailArrow + 0 - Text + Tail - Pad - 0 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 ...} - VerticalPad - 0 + ID + 20 - Wrap - NO Class @@ -321,31 +223,31 @@ Head ID - 294 + 19 ID - 307 + 33 Points - {359, 250.50006103515625} - {526, 250.50006103515625} + {411, 298} + {423, 298} Style stroke HeadArrow - FilledArrow + 0 Legacy TailArrow - FilledArrow + 0 Tail ID - 293 + 18 Info 3 @@ -353,12 +255,17 @@ Class LineGraphic + Head + + ID + 14 + ID - 306 + 32 Points - {451, 423} - {517, 423} + {411, 262} + {423, 262} Style @@ -370,80 +277,69 @@ TailArrow 0 - Width - 0.25 + Tail + + ID + 13 + Info + 3 + - Bounds - {{428.50005722045898, 327.50005185604095}, {110.99988555908203, 24}} Class - ShapedGraphic - FitText - Vertical - Flow - Resize - FontInfo - - Color - - w - 0 - - Font - Helvetica - Size - 12 - - ID - 305 - Line + LineGraphic + Head ID - 303 - Position - 0.74924027919769287 - RotationType + 17 + Info 4 - Rotation - 90 - Shape - Rectangle + ID + 31 + Points + + {411, 211} + {423, 211} + Style - shadow - - Draws - NO - stroke - Draws - NO + HeadArrow + 0 + Legacy + + TailArrow + 0 - Text + Tail - Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 m-many helpers} + ID + 16 + Info + 3 Class LineGraphic + Head + + ID + 12 + Info + 4 + ID - 304 + 30 Points - {455, 94} - {521, 94} + {411, 173} + {423, 173} Style @@ -455,20 +351,30 @@ TailArrow 0 - Width - 0.25 + Tail + + ID + 11 + Info + 3 + Class LineGraphic + Head + + ID + 22 + ID - 303 + 29 Points - {484, 93} - {484, 422} + {208, 305} + {288, 387} Style @@ -479,44 +385,40 @@ Legacy TailArrow - FilledArrow - Width - 0.5 + 0 + Tail + + ID + 6 + Info + 3 + Class LineGraphic - ControlPoints - - {-0.5, -87.0001220703125} - {-22, -7.50006103515625} - Head ID - 293 + 20 ID - 302 + 28 Points - {106.5, 284.0001220703125} - {226, 250.50006103515625} + {208, 305} + {288, 351} Style stroke - Bezier - HeadArrow FilledArrow Legacy - LineType - 1 TailArrow 0 @@ -524,153 +426,243 @@ Tail ID - 254 + 6 Info - 2 + 3 - Bounds - {{349.50011825561523, 156.0000018030405}, {95.999763488769531, 24}} Class - ShapedGraphic - FitText - Vertical - Flow - Resize - FontInfo + LineGraphic + Head + + ID + 18 + + ID + 27 + Points + + {208, 271} + {288, 298} + + Style - Color + stroke - w + HeadArrow + FilledArrow + Legacy + + TailArrow 0 - Font - Helvetica - Size - 12 - ID - 300 - Line + Tail ID - 297 - Position - 0.22188450396060944 - RotationType - 4 + 5 + Info + 3 - Rotation - 90 - Shape - Rectangle + + + Class + LineGraphic + Head + + ID + 13 + + ID + 26 + Points + + {208, 271} + {288, 262} + Style - shadow - - Draws - NO - stroke - Draws - NO + HeadArrow + FilledArrow + Legacy + + TailArrow + 0 - Text + Tail - Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc - -\f0\fs24 \cf0 n-many workers} + ID + 5 + Info + 3 + + + + Class + LineGraphic + Head + + ID + 16 + Info + 4 - - - Class - LineGraphic ID - 299 + 25 Points - {365, 423} - {431, 423} + {208, 237} + {288, 211} Style stroke HeadArrow - 0 + FilledArrow Legacy TailArrow 0 - Width - 0.25 + Tail + + ID + 4 + Info + 3 + Class LineGraphic + Head + + ID + 11 + Info + 4 + ID - 298 + 24 Points - {365, 94} - {431, 94} + {208, 237} + {288, 173} Style stroke HeadArrow - 0 + FilledArrow Legacy TailArrow 0 - Width - 0.25 + Tail + + ID + 4 + Info + 3 + + Bounds + {{423, 374}, {123, 26}} Class - LineGraphic + ShapedGraphic ID - 297 - Points + 23 + Magnets - {397.5, 95} - {397.5, 424} + {0, 1} + {0, -1} + {1, 0} + {-1, 0} - Style + Shape + Rectangle + Text - stroke - - HeadArrow - FilledArrow - Legacy - - TailArrow - FilledArrow - Width - 0.5 - + Text + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 helper 228359, B} + + + + Bounds + {{288, 374}, {123, 26}} + Class + ShapedGraphic + ID + 22 + Magnets + + {0, 1} + {0, -1} + {1, 0} + {-1, 0} + + Shape + Rectangle + Text + + Text + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 worker 228359, B} + + + + Bounds + {{423, 338}, {123, 26}} + Class + ShapedGraphic + ID + 21 + Magnets + + {0, 1} + {0, -1} + {1, 0} + {-1, 0} + + Shape + Rectangle + Text + + Text + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 helper 228359, A} Bounds - {{526, 363}, {133, 61}} + {{288, 338}, {123, 26}} Class ShapedGraphic ID - 296 + 20 Magnets {0, 1} @@ -683,21 +675,21 @@ Text Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 Solrq Helper m} +\f0\fs24 \cf0 worker 228359, A} Bounds - {{226, 363}, {133, 61}} + {{423, 285}, {123, 26}} Class ShapedGraphic ID - 295 + 19 Magnets {0, 1} @@ -710,21 +702,21 @@ Text Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 Solrq Worker n} +\f0\fs24 \cf0 helper 114179, B} Bounds - {{526, 220.00006103515625}, {133, 61}} + {{288, 285}, {123, 26}} Class ShapedGraphic ID - 294 + 18 Magnets {0, 1} @@ -737,21 +729,21 @@ Text Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 Solrq Helper j} +\f0\fs24 \cf0 worker 114179, B} Bounds - {{226, 220.00006103515625}, {133, 61}} + {{423, 198}, {123, 26}} Class ShapedGraphic ID - 293 + 17 Magnets {0, 1} @@ -764,21 +756,21 @@ Text Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 Solrq Worker i} +\f0\fs24 \cf0 helper 0, B} Bounds - {{526, 92}, {133, 61}} + {{288, 198}, {123, 26}} Class ShapedGraphic ID - 292 + 16 Magnets {0, 1} @@ -791,17 +783,17 @@ Text Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 Solrq Helper 1} +\f0\fs24 \cf0 worker 0, B} Bounds - {{287, 315.00003051757812}, {11, 14}} + {{141, 326}, {11, 14}} Class ShapedGraphic FitText @@ -809,7 +801,7 @@ Flow Resize ID - 291 + 15 Shape Rectangle Style @@ -835,7 +827,7 @@ Pad 0 Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc @@ -849,117 +841,178 @@ Bounds - {{707.02464470537973, 219.07510029874931}, {48, 38}} + {{423, 249}, {123, 26}} Class ShapedGraphic - FitText - YES - Flow - Resize - FontInfo - - Color - - w - 0 - - Font - Helvetica - Size - 12 - ID - 261 - Line + 14 + Magnets + + {0, 1} + {0, -1} + {1, 0} + {-1, 0} + + Shape + Rectangle + Text - ID - 260 - Position - 0.51475578546524048 - RotationType - 0 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 helper 114179, A} + + + Bounds + {{288, 249}, {123, 26}} + Class + ShapedGraphic + ID + 13 + Magnets + + {0, 1} + {0, -1} + {1, 0} + {-1, 0} + Shape Rectangle - Style + Text - shadow - - Draws - NO - - stroke - - Draws - NO - + Text + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 worker 114179, A} + + + Bounds + {{423, 160}, {123, 26}} + Class + ShapedGraphic + ID + 12 + Magnets + + {0, 1} + {0, -1} + {1, 0} + {-1, 0} + + Shape + Rectangle Text Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 POST\ -(batch)} +\f0\fs24 \cf0 helper 0, A} - Wrap - NO + Bounds + {{288, 160}, {123, 26}} Class - LineGraphic - ControlPoints + ShapedGraphic + ID + 11 + Magnets - {60, -46.50006103515625} - {-81, 2.82421875} + {0, 1} + {0, -1} + {1, 0} + {-1, 0} - Head + Shape + Rectangle + Text - ID - 259 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 worker 0, A} + + + Bounds + {{124, 130}, {54, 41}} + Class + ShapedGraphic ID - 260 - Points + 10 + Magnets - {659, 250.50006103515625} - {815, 257.00006103515625} + {0, 1} + {0, -1} + {1, 0} + {-1, 0} - Style + Shape + NoteShape + Text - stroke - - Bezier - - HeadArrow - FilledArrow - Legacy - - LineType - 1 - TailArrow - 0 - + Text + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 index\ +'B'} - Tail + + + Bounds + {{56, 130}, {54, 41}} + Class + ShapedGraphic + ID + 9 + Magnets + + {0, 1} + {0, -1} + {1, 0} + {-1, 0} + + Shape + NoteShape + Text - ID - 294 + Text + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 +\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} +{\colortbl;\red255\green255\blue255;} +\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc + +\f0\fs24 \cf0 index\ +'A'} Bounds - {{815, 185.00009155273438}, {143.99993896484375, 143.99993896484375}} + {{85, 292}, {123, 26}} Class ShapedGraphic ID - 259 + 6 Magnets - {-0.5, 0} + {0, 1} {0, -1} {1, 0} {-1, 0} @@ -969,22 +1022,21 @@ Text Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 Apache\ -Solr} +\f0\fs24 \cf0 vnode 228359} Bounds - {{61, 284.0001220703125}, {91, 61}} + {{85, 258}, {123, 26}} Class ShapedGraphic ID - 254 + 5 Magnets {0, 1} @@ -997,21 +1049,21 @@ Solr} Text Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 riak_kv_vnode} +\f0\fs24 \cf0 vnode 114179} Bounds - {{226, 94}, {133, 61}} + {{85, 224}, {123, 26}} Class ShapedGraphic ID - 253 + 4 Magnets {0, 1} @@ -1024,23 +1076,21 @@ Solr} Text Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 + {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf470 \cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} {\colortbl;\red255\green255\blue255;} \pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qc -\f0\fs24 \cf0 Solrq Worker 1} +\f0\fs24 \cf0 vnode 0} Bounds - {{50, 64}, {639, 423}} + {{35, 111}, {538, 367}} Class ShapedGraphic - HFlip - YES ID - 311 + 3 Magnets {0, 1} @@ -1065,23 +1115,9 @@ Solr} stroke Pattern - 2 + 1 - Text - - Align - 2 - Text - {\rtf1\ansi\ansicpg1252\cocoartf1404\cocoasubrtf340 -\cocoascreenfonts1{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\pardirnatural\qr - -\f0\fs24 \cf0 Riak Node Process Boundary} - - TextPlacement - 2 GridInfo @@ -1091,7 +1127,7 @@ Solr} GuidesVisible YES HPages - 2 + 1 ImageCounter 1 KeepToScale @@ -1131,7 +1167,7 @@ Solr} MasterSheets ModificationDate - 2016-04-28 19:05:52 +0000 + 2016-11-07 19:32:26 +0000 Modifier Fred Dushin NotesVisible @@ -1212,7 +1248,7 @@ Solr} Frame - {{161, 201}, {1233, 882}} + {{123, 100}, {1215, 1018}} ListView OutlineWidth @@ -1226,7 +1262,7 @@ Solr} SidebarWidth 120 VisibleRegion - {{0, -74}, {1098, 725}} + {{-174, -150}, {1080, 876}} Zoom 1 ZoomValues diff --git a/docs/yz-batching-overview.png b/docs/yz-batching-overview.png index ddfeb9f21848cc7fabb787108c1bdf299e38d07a..c65d20244af547d26683ed8ac4b2e10cab3cf9b0 100644 GIT binary patch literal 43649 zcmZ^L1yogC7cC+ZB5)DuZjkQomJ&s}8&UEi-7VdqQc@y~AadzW>6Qyp0@B^^_Vx4s zzyG~4o?|#J_i)ZWd#@d9tvTl+;;EVf)_s!uNJvOnN{X`2kdTl$k&y0$pre8-%M`u5 z;NyS;0uPMqOJ=P5+)wv{~e@vABeyW6SkTIdcUWM69Ejg=aItf8{I_lJs3_v9u#CfNM`ueH7tp>qpXi@E|F3KRzUQC&pV~M; z9e@>_ZOxVJp_b0zVW=shcjDZCpZ@>f@&C4^?rdubM*Zh$uD?(J@3p`8i*g_a{y&EJ z$I5@M0yB%>7v=cxl!@Q3SmuT!AxR=B$x3Ou-`UDQnJqv;pY#n&CrREL;a~u5Fe5OkTczY(U(emC=c$)v_Lrje9)Sp zbUkYQ`vrX55A(-OYr{{cpZ@n+R7)E6|Fp9R;Y5D^jphpq>3`ey$0Zv3w@nBW8M3^| zeQk>rlmEU4JYz2Wryu`xCq$+NU1jm@z{6Mnc2V9$1?}IiWjmtF;-+O?Z&}+1VlXf{ z++Hu-!dcp{YBxh;p)E)4^RX3y6&J0)6(+E@N_{IXs7rnQt;ZkZb(DkzkIsxL)kxbd z{@WTAY5-BDhR6hSzDY+oF4smy#eeLA=+AhRUz|GtwRC8|YJx@+)X^Ge#|hSP9zEq`%&Z?2s7KQDa<28FwNH?e}x-FN%pW zEp30MS9ao10*A#oajhYIF+Me+0{V9hym!IN_YbY;vh_aLVpR^>8neY{8Q~~J#ncSZ zklT7p3T=OJSJQF|_L!L=A63ge6KG&XlS|d^ejK;lOdbCYs&PN{dev+&}Ff5)nY$h%;eSF zLB7j%c5}H~yfD`Vfg~7TEfLx`o&P$d3uVu9TJEYVDtOn&)_iravlvEs;z32Zf+78} zwA_M}!Y9Z4U;F(=Y%9dXDva3_{|N88p}ai{yWFYYO=xw#^a~}na?J8RhE^;t-OPRPa<_T++LXZ9+sUnq*!paz0)z3xup3M7 zqIMIPDeOV)>{YPzz}DaMC;Ra4Hj&TH9noqp-O0T@&$_jx>q}X{6l$%aNq6cZhM(;( zlV85jHk=SUopZ^|j&H6|j__ve+*716yT$r@E(Ofu+F%4|n5!;xFdCI`K^q?_DSADl z;v+P03~1k~+3zB)7uori`xq>liyb3nvseEmkDKUGP4P4oZ^h62CrremeeK{d`?u z+T`x(K|e<-W8E?x3OX62eOW$-ubquB&gQeFNc;NjuP%%k(1iD|zs_96{US7SZ&i?*3nOz5 zfA2CV1fR7l-);|cGIn1Kv7h%?J0?AZQAN=NdVBA+qTSrCrW)_0;_Ug+_=aMU{*vJt z!lF&6m^N`T!!xCpbf5d~E-Ps#^>=*;6CsXLITN!FcB6avrb;_wZ6~ff8--aXaQ*>W zmx&kJ9&JuTVyEB0o|qx`JzKn)>}RhZ5E;t^`#G~EiiGbo*(%uMztsd6Q3RY<6y3IK zreKGIB148g#}oGPas>M_A5VWcNGy<7?>1e+$lpxocpI0Zopn;ZJCsL}52jDQUAfiE z%5}^L3_CFQ`g>yS$Wnz`S2tiXZ_=E9cko@l)^m)5``uo{zy7e_q`Z?ehDw@yamo&(czk9 zr9>lAHpJ$=^Sg7)y;#$59+XvZdU8lu_ry`ppN6FS&p4tG=xWrCvDMBkSx-+?tRH;G zMS~FXvvO25@g_DLCG)oDazMA<{XOO=Ai!gXoCXx3obms7wG6uY8}$LvPld~phJ(J< z>@e~-Y36wlo;7LKG{=}#q`mj84-df#6@r=V70uWdW{{~>C0qSuQ+Vg>KOwh5zv`^4 zPE5fT@`K{9;fF0yK5FHb39#|5KnqQAhrznQ|@i;DBnHoG^= zn|C7K%A;;juu?lT9;e)5_V((f?XHHB+V6?;4~#0;PH_00D@(X=V0S}E+n3Qqd7nE? zTAJ(={}Zu6*^xC26+t|4h(xS~C;y1P7t=P#MAG1$lu?qKVgf9xdaCBl7FO(=Z2KYJ zf4uf5{QvHcm#VKA!;}YMKYBe%=^a89*qC&Q{!A*2@Sl5NgwOix{wGSoxI@e7@fb@v z^51LY!IF;xBDl84kB%7b!N5+3pZstzj`6uV+xM-iUu+Lc9dg<`o-(dCzP*@g9CTfx zTx@+ZCpd7jIW?rp)%nPNREg2GxTNZdN7D!U)?B>)=>rSI>Xl@qLb!Nz6>zA$LGBHw zu9y_$ z`ok$)e};5F1`!z2aLTpb>qmYcjIpgIwE9S3L>5mdg(&W)rttOA*d}Nx(VZ!`WDKP; z5sFFCSHAssA-;{2R8X_1{TYaPajwmjl4|7Ht52GuZ!gp=xV-Ck>s34Yge6&dtA{q0 zmD25Zz@C3YG^xN~P1imySlW#z)X_Lisw<+^SS;R1C({L`uzNDwOJa^}_yU7HBnmRw+_>p^2nG8^a>}G?dxf=$5h&Q9> z@rgH_e!vk}m0V1;d<~-x7Pc<0!gkS&v=3hVA{YarWfX?UjvDTmFbdC}SaH}vA4`91 z<4S_E_uctYl!6Bxe%NV6XSuQB&Xj&vPBSl)ADG|Pquq&r#P_sUOwkcriT~wI-t{Bo zJ3b|!0?a7wd9Ci}hJJ_ULgaop(!AK&%14`k<}QX!5=$xI5y4{l=iSW(Y=*PBMIWJv zbC&dW%*w(UN|ZZERC?Y!G;PF)>Z1Bq|u4qT&c|m3M1yp$A{67}oawj7z z$5cpZPNXSGdcB3(zL6R?Z;WB$!xJ#P@! z#NnV&alXtx>no_@Mox^avbB6X-l(#S;-qmm#5bF;fV7k~+FOjJh%`?PjF_1Zd5nE! zyDn5R29h1nM)n}0_2S_AbRpFz-#JM|`AH&fc6?yJqKJE8($39DR~55QxW!Ha{HY*WEF)l0yoXj^ru@G%8l=h{b6r z?JoUv!x6n}(8Wh$6ox;K{*p510+G06J#IGl$7j!Ip2`!C-Jk}&#c6y<IC(<(>zALLmgiDOB zW&}0Ay*@p4(d{%j-1;&E6Z(}#`zz&Bg^9~j1S1R_U-0e9ZVG#Fi1IBAz4Kw^WZklB zs=Rk+44Rv<62htgj4`H0lKJw}^jLT?wL0bxjI|z0kBO9l#Km~oj+uoT+4 zosRggEI;4wjwsSs6#>QsL73KAK3^9Qk&fF|W%otJg~zXS>$#t;L2zKsO?5*M9>!Sm zO-r+2DaXutAAYxwpikM0%>3AmKr+5A)^5>zEIUl6@rX`lDR z)53ul+QpE0*~0%JsfM2QGOlAqPC(j5C+SV&@#Nu`2nUdXS#CnUl9d)UU@yw|QFd+d|b=8NCO*cl*m_|kwK5+Aq#yw7e~#iOiactRETDqB(R>=>S@DI_pu;F4?|;R}|@941G@ z&#oxq<;3==QJw+LATO|_dk z1jQyr*UAZcH^Z?FBSz51U8gj&3&rDnH5G0eM*d4@E2$!9uOWoc=$rT3@@;{6_uQ-I z(K}9GI(kQyIkp)x-Its*ABQ-sp69U-+fe6-2X~jM=;vD-`kt>gEjBNPlFp5(OKdAq zo4U<=SR>rB2s(Rrap~80mrIcxo^pp5o8|Q~IAY_QZ6FWwnXcb~^@kNfCE6n@Dky1t*BHh4R-m1;twl(41%Xn`SYZn&GaSfK8ydTTXhWJH0<;~Acf3QjyrUH1& zJRp7IrB{h-M-n);^ESEBeU|rPAeys}I17|fmxP8y4+TqJ#o?8_%8^v7%%(=8^sWdZ zTaFx^x~|sH8M-*$B$Que^UP6m#Qhk}SI80I&ZR{=pu}rruNGF2}J|w9t%=GNb@IL6hdE|=Yq@3WiF1mXT4uFE@ruZehdX519 zcq?z~PiPg8;ptJTXV%sFXBQmrBq4MqhoK=W+qDZP+X|mZ8$py(JQeiv zROW5MN(k?8rbanBq7O^>8oa)I=YOwu zJZr8xy4-LEXJO-7V%2_F{_cc zLfJV>bvbfq$~Y-^O1QQOV5aM#@6FyaUwy0{9~n0u@BS%Gt1R#Ug^R-?jGvxaM@mt^ z0@p{9>g{;1j*WC{AEU=cekvKA;4fczKSc(WI4819FmHBYa&RVUiP)Kuk^}Wq(3$tO zeNR#0bX=meZ(Deg&8DYwb(kI+3+p9hxOfNby~y36J+9epVYm3Y0id+WD3Z8I^8_qbqD{BZV+c?_A$6sWWR#4S_71&og!R&*Y3^e3iUVNWe`jEvQ za9~E~dhaEZ02pJJ;>oXV1jZ9Z7MQ*0Rkc;s4@`(%&8qgSw z+XeY4%$u}HAT9|YM4!Fw(=Ai>n}li?eCtnB9~)N$8KGByXc?Y1w+#4DQAADX5r2G7 z>z(yxAIeV?uzS&?OQUX=9JF2>IfqZ|JaK)W-~CO`0WWYi~czR&M&M#qB1 ztZm`z!FF1+PU$ug+1x-$6CxpGB`F&sL58`Hk?g<4FV<<@`q^sF?0l{vXohIKMuHpI zr2GQ{dgDr%Y}Ik~h?E2dHx5EVUzuG?zX$M82+@^3fvyE^w;oyrtzkgVR<(6XEP%N>vr`{R4PJ|vgrq;`z^Tyb# zs#-+%<5h!00`YB4)deFC$^t=v6CAm%AA}^MKrGj~r}kJM1OMT!5JAVVfaKI6tXpq9 zyfgfiE9DzH<`qRk;PU=gdSk-G6Pgx$c#biNaBV|=t_`v^&`K$lf%^Nv?P=<4EXtL)fYj}CLJ^s$4OJQKW_)-4Wc@`O285@mY+ zE-F>>7wd%d+(G^&m=PpxLxlm-xB`dAY^pD-ZF{I$e;eU?jEG%vBm|>`F0aTNNgsS{ zAf}LRJ1Ld*r+(iahI1;D?*!?{i!MjKK^o8w;P8+5gClH90%%NDX#aET_s7qFt3C7X zy%jdD789hNoqBg;ag%s!7tQFf7#wusp=?~67gk0?kY=ulLhDMT7WLDw3!P4!EDeo8 zXiA}Q-Ly8zrSW{_dW)BF-m|je(vvo>fb_fb(yDIFq!3_e=5=3Q8COXu-#A=hbiSoV z0Ci@oUC-V#qU_lMC_&Xbfr!6U>8oIXE2PjcH2xhCvb6t&LW=pe>-Q6QG!iW5MCtMk z)j{rad6CRNfJ#6Ct$#D-AZ#GfdAITOgob0W{RM7iVQZBw zu~V~BkYGRV?4XkN531Q^-Ay;$Mgnq~ujs)y>>yE>oLR^peQuK$R}s+GYqWM1a19kj zPO2+$7{wrAKk9|%E=M*GJt)|a=P9ZFbF?3&i_{~yu{z0Y4!_hV7<`@1nhL#|`d z-YE{WMV}P}NIZ-PPDuiKk>w)WAw3w#?-oK9M;JdiN~`*^QhVgYe{_0ww|k&LrEpM@ zKG=@e8$u+{HX*D>#ki-#{RCZ7y6wP{7lWr`sYi9jIhvC?BIkpufUvQT8C2$2UPJ>x zBLL<4k5>Qp;y# zAyYv z4s=yp#LShJ4mS5b8KPa90%yWh(ZZ;92S7U&Da!ACHzhCb$IGOYI{ z9FR;smh|einYt5dNugqq@G7gIl21`WS^^Ij#|5Q4RRm_W?MvOA8(|}FtMkR$(N#zu z1IV2T3HcZd-!c;QiL9LQud1?7Ie1QsEM$H}@r2uBOEoU%vQ(^gu9$s{@`j4k0k{!4 zB|;9c_#x--G%vVdITc$!=rvIOtimUSVpCuhBBPKTyJ37h2>TpG#fcuBJUC(_rANh4 zHQ)&Ei46N3Cvh=a&>*MBY|Y9VNAjA617INcb;m~{Vn(9Zd+feZgUx&1GQRf;6G2j| z%4TbTMWuvCOzj*0NHLN>?>0My;13-R2{e}DdXeg=E)k*Tq#VCeF}+A?NQSceN#q+ z{%O%vT_)doa!SB5W{;8!zQ+meR4=iw#(PA|xh=bp`5JvZTvF?XeS$(kMlLtRRW6#a z`-8SQV}B%W`b(Q%yTX>?!Cs2RVDpUzaiR%Z!*D;23U%kG#j)H>eJ0(OrhePf5jAA5 z9l8Yok3#CWnr3A_Z<|lkHlM2ZDOnzvp%jn=o;R~Q_Zh7|`D|OwWGCyz_X;$#xVSN$ z<>^YxBKZ$v{Ao^?eiA{N20&44bg+e1AP7JZ=K0G_DVFb=H9CPmvpCvAN(F`%P~Ln4 z+iFqvQ8zE}q`ZzZst|y$Q#4W)+QO)-66qg~*6J-q0do6cP6(p8F5H#{%Q01f7*+0;3ApQaGc|I!Xd$0w;**|$8lRXBpscDYf%Z-2NvB>hiuMa(Hcq3Cx zS&_=yX-q6z!*)F^bo{+NG*&!VNcA`B)Vq<8M+lcUprJsH|EoI&G4He}R5%B29zO%n z=8!R2<1_|pBv&vpX8bZq%HN=mYn^zEEOh8ZU) zBDgl#QEc)OR^&Y672qv;)AlTqwKj_M@E(#|aLiihJdwW ziwuaJ{S2}PYbPd}=5X(#Ps?16Rz74xiTil`d3~(-1=APSfFZy*t|QR!+HMN3Z~M#L z+R>NQV`_(nJl7Y;hq?U>8PI37eUFR3oi6w_|86WrN3z8akYa@BnjU;2MPw*m#}nGQ z*{xGs!!kJ0Fh+(w1V?c)YsXRBMPm2uxtdU?-y_X>zoPsU=jMxvax%{VH5CCk)E5WJzL*+=(%KzJ8~vF9Tl}XdWHQ;$!O{vr-35WY%fRmFOKW(O zf>X{^1YXJ<0|?g%aAaTo{18Cy>0UcdeGrcLw}SWJQSWj>6(TnTZ3?o({%B&Jf&wpn z+)p6$U7AVGG({(nAy=dRDL5{}3yggvBR7yS`9&s&>YTR00Qke*rqCYCmI{ z%I5d|AVhgpjRXXyj<=l%zM$SbR_yCse#M~h<~PuZ8MexfhdTgv`kg$maJ}PZ-%ajz zg!LU{44bh47m!z%&h$P|qDsqCEW&nI;Rk^`|rKBP- z*Ps^uMqeWF)%lViHq%sb8LV%pju$YA7rN#61*6vpWX)VgV7T^aTCHtWFT>#!A&C1y z?=I_<+3WI3YE7q`V%E~FDG+jpi^VOU5WCfyCTMQ*26`6$AA)aD25|Z7Z%__??LJIV zs0vsj!ActpGu8JHozQ$<2ZRSh5(kL1ON!o|X zn=W3u0et3?lcV{nYPTQ=edL=G6$PQOccsW;Fy0l@doZXOzw-mu7;XL6d=BKM(@X>e ziU2Jn&--!hJNM0LenowmiI~&GKld{Go#QSd(`0WvdgitguW%vnl4|&-=#9Q|S@D3J zmQ8isQ_8mG6uQrSGQ%M}gM#hkP93=RXM2k=BiuS%2-=UTQb&qNqvm*0|CX)*?3jVn zPAq}BpA~76FZYW#Hhp3DnFY|h7~a1cWf>G$@E%K~&gkF{%ijjX=2+2Nuh9r21IN~T znPlHN+aF{;)o#e?1S~0aLh+$4JtOdoSh+F8Ss5>^SSe((d(elGYGatx|}k?74T$U3n-nfH*mKF z*JeOFw>^(RMYB`mJ3ksz?@ytRX(h$fpkU?7qyGamHR~DaJ7v7FMP$09lY)wL@GocG zPj7nmyD4`|p|-0sr{pV%+izLQ91)C%&-sa~gqE1%^Jl}D-a2$T_tI(Ux9jfbiV;wT z<8}N5v*_e^d$|DDcW7MwmFZ=F&vvsZ!QLX{)1NrDO6^mpH9C|V*&}U%9?ST#?iHUS zQHN8U+_~?0zwyQE1SN)jlz?)TNwb%Q!+b&btIM$xC(*wFaI(fm!;2nA8r<-JOt9jE z3JJMlKrlVAv}rY=@%mY&r02Zv`5P)%YcTsoQF(8L~i+IqGvw3yI-3%WGEOs-w0Cpb!Imw#xW{YOB` zdl=)9(b$FqUYaNC==|6Fq3h#vzhjFayz<7Aclw?i`Ta9b1|d&UA(Qwdm?gqG8ZAK_ zMkEp_#RIZSd`SJg7x%cL`Se*4ee3QW3I?qBFB&$VfnSwtkXb=_!lv3D1z| z7q6T1wf;i)?!D4G?K)B%73k@j@>G3j3>=vU$zRO19|WAxQuz_W9v^TvD?h}A34hWY z3r58}6wIYMpwiK!;uPE)tvKL|JY*kC@6od>T`g-kc(U-#Y^iBEMr5v!rQCpP!@HCx zIHT*;_$LkfNnLB#wDa=1UnG7w?djCikesGq)3TqkWIYH$#5LJ^sANTj$*K}w5A`Go ze>l6ZfS~u+_YHaZHJ{chI7clpH58bP$=)Oer`RQtOQ(+=quhz9TUfZt$}(*a6=Hn} zHP>kJZWwM1i6giwdW_M2P=|&|wJ0A$wh?BgeSe{7%^VR$5ObNy7MfX|bEA;R`-d`r zSfJndIkk6}JNzd=>0AJa`MI2_k~_F9Jk8JzJ`r1{WQ5sXb3lX~be9eVHAf%gfZncG zOaJxiwJsI*S-E73mZ9r}c%mW^15U@g_9+266bbq!_nC@8Nv6*j*r`!gAAkI=ZN4sb zpgYcQ1Ph?P`D&G;NMjCAzv9;`0aqsb8no{`gEFGz{XJeduIGD&bLV>#Vvy{`LCsy%;CDN4x+pjtH7 zj6-@i#Q|8OH%_|vPFFWiBGkIWZNM^K8swSlR451QyMR6}KkEudjd()Ds6q0g>8wU_ zfvcNrc|Er~>uxhd{$!k6kH$ex!Iy-*NJ^$F)vZL+lmA9Uj)tT1oc)P=j}2S!qosEY zHS<{wgx&66pJoE~rfOHqeLiMWfeIu6X>We3C%|pdAEDhTVS@^N0D!(kI^M^Ly6nKD z{E!hGL0#@b6rT(43A%gDNMzB}CTA(bxAqBG$m_vMH@GI#d=_8upV_WFW63nkvqtDK z|420z{dM)Q0(4N;g|m&x+lI@|4LOy@%GIVIq=rOs1Pm_4mTI`LNb##Z8HuGH0{f^5 zB+La2bk6e(r6z=Y1z<16Gu7in$lFp>M{XV$8-@D6ciSLl46_eyTc<*Ia=^h`Q^p>5 zLS>`=M<|@_1mvi6oCEdG^=Rr&YJ(;9P8GlS+lDn`x;S#Gisyy!1?=etR7bIu)<3ru zgnl`2*at8Jpoj@kKB7NZ{$f5L;?YaDp-9UhT_|YfDN#CGT4#{Vj&S{oVKp^FS6<1O~)7U2K#nm4? z;-|rLUgShN#t^DyZ?z7Ql(LSn$s6>Z$`|p&ArIm8<$$K7efS4Wo5Xni!kdPf zhND+(16@hAc+_&Jm*5WHH_wsju=gZZnV;|1>D_;~z1!xEJ3xVHiwLl?!tN>e1J#&O z=6r)5$rl*AclDIxX30{m`#~&{~+G{NOD%En5xdgmRf{~#qhJllYee3+~ zWC~dp<=wOpHVWi{uYKs`{NbM;ue}tqO=-sDDIaA+v2BeFvE8R;D+!z{!`Mr}!cR_S zuo|m*={|G3L@Svdc~m?%a_{$GTtk}uqvjHG7XV|eKM|pS+hQ`y{4TuVCwi}PxWk<` zkHkR*|0k)`ZI2xun-50xXi6jbrHY+Qx0j>7?@T}%=4W|bs&(KG|0cnx*mBh<)eMUgxLPz909#iq(ptOPXjB7n>DGS&j8ig*QhM zC|mR#0$TIREW%sv0Wo(6UckwU@J^BGYX1EwH+i`Jhh@wR4%fj$wS9ZUWb~!0TA)-( z*S&3d+C)Kaz9gMbKr6Dq5G*;yf%ddxAZ8blw}qO>MwWZ|=zv2@MN zOo5!lL%NG%`8~8o)pCG~4%LwA1A5ek{C=M>f9LUU`7M`=`v9C~V0t0VHzm#FEW6h< zI(=lP*_2jS>h5TIFapgMzPhu%1xVhNj7LEC`Aor20-w(~v!Mh? zem*hM)Xq68zwjeuO9{`v$2|a(iK6&itz#CSz3LN?v^Q80FELLJGjJX7DB;+8fn#Hw z*YL(mPwojd9Ss+Rz?E{}x95Jl>pb2>AxOPT#VTO|zK-fC(%vj6PN%UCz+DZh?Q8@e(kosVs6YN1wG%1Wwb z;4eMz8si<=;7?9EHKX!2!oR7Sn(Gw}tB(06(rR2aPf5WJXl|ONk=BRj=aTX;s{q|3cQh=Ll|j)%m0BRJocORZW$Xvf01gA&FBV_hU__+gs0UdzN1z$G_L9b$HG%+r=N2#s7;(Xf@Ma1=S730-W zfM%6u)5S$-M_LE<-gkk8vk~4uIR{W-Tsc1+uyY0+c(>em!Ellgsi&m&2(ALzqBRUG z<^<>H&W9T$W@Rb%2CVV-Bi%x}l#^3WyfY$O!W*iKn6@j4sU>G<9-E&eiR}iFu$Q?+ z0i_WUY>81zpbGUL%?=QV=vse{Khxa!{Lr4ROlz)wLQB8SbyBz6O?yw2M2lqF@XbtT zG?~yS$CeuqnV&@z$^#;fK+lGHp@qz2B|hes=_g2Oeq)PlarzB-(~0ahd;wc`>I$8> z=R@#_0H77z9Gb7ck_tmV)eih*dDsVnvQ3b%aZ41Dt`QS7++1wd_TDsM!9$UNQ-?QAy)=NUCR*yofE5Uh51u<*XLtQ7QB1;SmI&i z@f!-?Gt)4R9yvn86(t-2DMi4?3QIhT^q{8GaY@6GHIz3|pqGDVmEo#&uYWZ&6+7}F z>1y-($4JkruJs(?k>}TFQ!#2~|0-aQA|#Ut8J54OzKd-!aQVK9(D71u?@2k>0)WI) z^!#V<_2nst0PF#VRSX_VAh+XBNxy-iS6{yW>YW8S<&M<{r>=u*NIZzHr^13DjYp%( z^KS#`Lk@L{h;s1mC+df+xik3UcIcx7w7drC-3~~`T@ZSnA6D;Q_qji!zD|+Bix zzq(r3y)CQN#Cs?}NQAwR(f-lzVfx{*@#TM#xe4Rl-L@+_2(h=x%4ypT$dvaNby}jC zcO-z%J?K)YQZnN{9N_T^%yadVK6&A!;uUT zc62h@XkZ2R0nk=8mWNC1XHmjR*ULT@zhxKt;8uAb$O-=zJ#0&7n`gNHxM!m^)afkk zhC3Oz@+rI8n$sD>?!KWql&!LRpRjc#brYeHs+DF17}$;IVGAcRMxiO0OL{4h!a|;; z|DK=e0sm4LqxIHwWv|J}0;2bbat<3}WDT_jF1tt7tZ4WYnih=n_jNyez7ypcAqz&c zo_p$O)^$!;m2kw+?coN*aQ2xVzxFpj2A_s>>)1O6h8p0@^{})LJDGHO5Wk`_Ejeu~ z$rYr1|F`ceh`J_F32+eKIQwiQ8l`~W0>+Yro3``Vp}!QM!cZJ!ARiR7d@c0fLONko zpl~LCo|J<7XHG#Mv%MhyRBY9wN%t2~4`-lr&VK)DW;*;>m9LR0#ptP#M?84T>fIm- zsZ8!!!k?f?LSeYecW;xWe&KEMDWUCU^BR$ZP^4XXCWH$2L88_T2%w4PVif*h2jz!fhd7XJ zg@;zsJUObhu^)-+P8N_#e{wl_yveChvo256G|^42Y=V+^KSV4dJot#;L8`g9HQmoSa+kFN#ekJshfaY}gt z5qV0dL&iq$6@O#+?2iV@beRvA=*%wsFD5_7xn%7TFgyiXS(tH6<(m|xQ>vD}ZBQ;X zbd9RawV7z}8YHp11=cn`xjxq;r{SoVWIouWzr6ATZFiw5!_F}+QZaPOG zqT%Xt;r0aOSJv%S;Slqhsxal9`DnlEO=HV`lDsUJ$9{^D8}e(LQhL}d8}f?IV>`ub z27k14$8-f5r%1x2(fAy>O{_YE>i|=Q4yg5SRh8_vlSnr-G~_XKN=sS}+@HZy<&D2w z*ps`Z9oES*mMEI#(1!cyc_(7^5Bu-Yx4?O_rVPS3{wRZuKrA5 z80Yu+n~8RWur9Nw1Cb%_Huwz$>%$thjJf2;95T}K0aMtrEA z+Xv;}sC5K&uL+S6p!x(}&i=qe9euWq$1S!f=! zJV+tp2l5JX!=L#)569{aJP3z@}(E7B83DS#Xu9X!axje)_0I27(wqe0$xXUZeNzK_{YaV6-WEwh} z&vE#y95kGQq-N2J5-9w1Ji1NpozmOg;|FDu^B!nzrznMDTC8g@r8U}?`aR$#E_bx& ziRTH$cifwY@31=anvLgx=&c;b6nhLBP&afWS{@N4eN{!!W zu{@@3*i#>f9ZU&bN4r%OSmN4F5=nqb`HbwGRGDKuz)Irhd&M16r7*sly+ey1dXn+c z9Uh0%_s%)Az162#Y1hK69x$y|M^4vPw{Ou=0GqUOz8q4I0`s@;%Yq!lY;k$ zg)~NV^z!N=giDtIsD&GOZBMYw zKUu>#0(eMWH2-YR^?t11k%g{645h^UHbO1cMe1-=IPboUmHPE-KSC}GsE~Dp8*~GN z%H>Y_3oYHw1^r0hgAWLFa2rGzJ0)KRN0&+GW%P4^-Acm^pJLJ0IeO>HL+;;jQ+u?vW3K8P$w@p!@$$MX@a z0~}mxY};GQ4#IW@aY`1ckaJ$M_;&Re47_)+^xM)~35@q|mIio+jy$7`*g+;>EA?AE z6=b+_lmDdapyZ>3=Y_hrzp`caeJQ%o)UG@)2%x@j8n7jeR36h(9Y`anqsiWbHHq69 zIM4w%RZat*reC)dYIMh@!wn>uh!QxDew4NSqFfC`eE}RWG)39=ZcM_fb$Ez>V2;SD` z5g_Ce_20}`_G5`p#&DT;o!#4+vA09`w5Nc?i(J9j^8Bl`>Ca>>FE z&TsWiL-4PReaNItjQp-$1wiz|lD2RPN__NRR!mmJPa;Y>1)_9p6{irvEvZ;I7M41j zw2Yi^1-Dse?~@iD-IH~cgnawpjZS14;^)|F8E!`B4=4E@3k+J(yBG8PDRqXw4GFD% zu86jKqRVE+hn&Ew+64VEb*^rnh_Et)y4gePRrYUv)8C`mKYUn~KGW(x76( zg`_*ZMB5S>h22vqL-+MesF(xIdqRrfE2tyB%}0d0=izN`s8QpN6Q_^XKxmt}Tk`ck ze+%HT8{lcqe{iKr?04YZL?+>Bj{Xk^HhULG^k>R}bX;&UlXnJ=tee)ObVmRO0sP z@DJ|;Y6#uR*KORxBSQ-lhQE;kwreNNJZ4{NSi#lW1=+b>+q)(-EHvqAKcul0m2_Oi z^Km|lXgPN(``$biLdLz=cubLt39a-0zPH$Ztti;jH=HBWc>Eh|h!eq_awQL-%J;7Q zj-bd8WUIQxACjzZ5mY`w0X3YDs1r@l@Kh$Uj5ua}#A;!?hH)2H+MyyZWHgnXc@9C2 zN__6&2s9erW-}e1GQK$=ye|E=2A=`3s~EA>9x^4m?vrN%nz6sVQ-(hCm4Qn9$yEK# zpjljO6OTJJIYdmgz$rt7vmyqn5A*E=E*fwU)bQ@k&tfM8SpaCcdPL!5;|3YGKkwU- zCoH8aHYSi0UV>QBgYfdGOF*po=u**~IqfB@uBwwrv!Dc<#EP9T`C++4cZG3ip+B^E z;39&B>H7$JZ=VmlSCUAnb7~zh&pWpQ2QmgAEpH5xoERgI=tSX`hv7Dr*;*)4P9k3s z4iCM&q1PenVVukY))_X1_kA|X9eA-bo2W(3cxqs$xz4Qv5|TNP$NsErO>hP4R0wv# z?AOBrXGvnF<6y2clln2q`CvrwCa((sTLt%<+ zixoX_UxC1L7VgwdK2M1H@VV-6Bq(kl*t`eBZ&H}pDP0w|T!=bQ7Z>>>%|U{y+-h>0 zmfqk@SI8!z97uPzh!O&*PO_N6g}A#U`l)%&7|UTf&>rT|DL;y4N6mJ!!(wU29fl2tEcKHp@ff6QEq!s)`)KffFVN zJ3{&PLSj1K-WvNHC)Wc)*>B12Ide?YpiHyDs8I> zdPF}`q!=d=-VW*+wNF76>glW?8n&$=zDYY1B`bCG{Y13Ur^BkeHp5hCIYPi`iQRZJ z(?orv5{ttM_PrX$;MR4L@jrXW=%eGRv-{RL|3IrS78gidTl z59gw>abR*dD%-1)4b(sz(|a;hbilK^8IOEJRL$x8ITEB8IQP{I6}lyQE1ARQ8wprLCXvPjR1(6zWTdAU-$A_ugdCzbn=93jdd@b1d9{5%TB z4?_KbikAgxy4o+zHvJgCTuhnh#l(99NljD}?!C;DgkpF{8)9_TLm3xJ0M> zOyWUJ=Fvq-^9Y^5kQnlce<@YH3!twJyJ|$4<|L1=$f$y6iEu+!Sxdd>J^VComQ#aW zxCN<_II!!Lr2h|9=N(V=8~^=q;@Ep{$0mEr<{*+CA~UNf9A#%5d+)un_ZC8C_AW&! zWRp#{`#ODpzx#gN|2B@}oX_X_T-W>cdOcqwqHU2x!&&i`M2l&c*JJKCc@yAaf^c8T z)W`MAf7qq9+NEkolHrB3no=gj+#eZ}5 zk)J-GYcd=lwhJlk?KI)ZPQWNAz+I+Y$lwa#Ks%=B#l{V5| z@E4AGDiQS+Cexk8kYuo$(ImA)QxXdea6b~#&ZBtss3z^LP{|2 zfXw=vw*C1;WSQktPv!hIPU?WnjhPYUP{GPV!UUDp_Y(;`o*1uFhqi7kEJo5K4H>OJ zCxtN2N8otl=|+5W6sTZZ%K9Z)ABnI_htoXle43E>!a#~qHnY=+yJ@o0+(D6}GfOa7 z!((bCZfFC4ZE*2<;5k2&lE>c6$Dc2{FM2{41}~=f@ztEqbs(325Q!pEngPnB=%vaYtIcZ9QTvptL6O`TjcN%DTsRk;+g-t}UBxz)nfUYye z`ecbWKY4uH&o;gt7;mn9@T3M9R3$qR3ARI%iWmfECQTSME^R!9RX;X^CJ zZ{~_3c8B0q@pDDR=YrH_f1+K;0xW%A>xAf31dw<#e;>QLpr}l{vfoK-iFBq^L)TMM zl7F>hHO&IYpIqq4BEmB$io{%(>FldZm>d{`7}T^MMu<=3H!f;1+DQaZte_`!aY8%g zU+orDu&TSZguJ~&;B0y8S#-2=H<7pZui73O7G2)XxlFGtaT_x?c1&<#vNKxy78jQB zle2j&SQgb21&f3>CX$MyLM+_`xF#aTncRKdo{+e=Rm+C0VyP`zpy>z<;wEX)*gpzy z@}d|-WC8iad^iC*@#XOkcHKLuy$7KSyhK#@2VG`o&=GNXgV;T(lF<^^w;rk14*A`7 z9xP_!XU>C?xVgkOxxS|d4T-L!8>Dm?tH*=KJ01lb*m#!K!9zW2<}$^;W9t4d@AeR> z4pO=(zF>MK!K%1x!i=yM{LX83|7=4*75%o3L_zzEFr8E|AWoo+hM-hRm7&)Ef2V~a zg#S(4GE!H@BdF1PS4_{$1Y#BJPYd#sCh?UD9(*NHMmuE5za<^H>$_xR%eoZzuq~*z zT%eUQkS(8~C%>hMLu;+dAMe1Yl2C&1w?#xOT+|Y-L+7J}W!%boD`Cd@ zape1-+eRM%T#V54px?3-X6D|I#NHWb8sbp(SMqi4zhps({5HT<%|I%tFI0O6`}7fW z0)!!~Rr!UqcNwYL#1QniL~UtcrbxZ#_U^K7R;_ki;BFHyNM|r9N8?c7(%~MIDeQ-Y zc3A|J+Sd1P=T_f4UwtK~Oz;+$R~UMZ6tPig_CXr>4Pv=*2#6#j|2V66=fV{JL0ON^J^@zjJBlWM>if*+lO<_WIsNYh7A&Bg?c` zk?P6zoR=h5wzgcv4DfX(38bQat*$_}2`L_4-+?Q7e0m5gNInh7}nW}l* zW6xcwVPwS#^NH(;*Q35)U8_^dV)ZE9xWM4syGrS$B+1p9hs5d5(D2=b$J5c3#Q*Jz z>~HvuF=$B{qhZ^Y$OTp3g;;T!-rs4kVD*T7N=qLZjXpxYXt)&vql&ASOc0+G`~8CFQA3|=d*qAL z-eSCE3-MKgd@xWpZG8Fl2f;$bTZY4j!15VQzL}t956tYL(~eFw+?(*(GT?-umyXX~ zYtGXsqEB@DqE}i-s)0Kjuwu%jY0+1xuw?W2KE}Gut^$*UCq-MrpeoOddT?e|7LOzv zX&>P>Wgq4(V@S~N7h}9BORTz{pPJ(vr;0*v$yV^`ZTzUNa=L<>ynr(s4U=HfH_AQ9 zfh27?U@GWyi4iv-0jN?YpghdIUmX%RQbPBem7=;DG2FdvM~F&;yO81Cw^%(G>6TT@ z#a}~5Evhy@d+IR>#A!hBkhxQqBKkyBw&BlKD3BpXrW|z0Q21D(5p*yxkWXn`G?(-S z5l2ysZZYAbLgWiJqCb;FSDo8*Di34SohqH>Ehh6{lcaHS`~>=%V3)Cx5)(hPc9E%| z=$p~a9Y$)nryy(A+;e8(vnhKm;D`DUkI;50J6zU?XfF%+jT?n(Laiu!vFF=k#A!QwO2aal`OBB zpH)FAei8cq{n8fa*d4B8W%O)Cu9qb+EJWw&JKUBhUS#v%7PQwPs;KP2YUUnY5C1OMuqQGdCK5-t7fUQXyhlrvlvSm;_e$3Rh z(_V==@@d5{E*&M(O(esF8i0QOeypx4M=|g4U8FxevW9yQJ99j;eefbMxPf!_o7euv ziZ`d7JC@Gu3j}$2)M{EdACVRW7|IiymP6aWRDdJ_`+sVz*=Ceq^;4-o1+ZFLDH60} z1*v>r81*w-s}xTQYxuK4yuLN?WA`-_Ez*rq!<&g0E5>7OkJGe0r_>Hk?rlfVND)(3 z@*@}(o;Hf|m{Ghug|d#&MB+f;vVsU)q@RG}WJ~O!*ur3^<{^o7r85!3wMA~xe*icj zitn_P`hO|mpYVk12@FGS?_PSnc#XXR^Lxs-_9h)@qSyP{CwoUt>MCQn-|#Hcu?F!- z@#|S3H$rjLYlA}ZMutqBM9GcO+eqQVgu7dRA~8BbD;K}@PX(9UR+s~fg4X~)DHT+}i z4VNK$K5d@W0m}#+;tbz?_6ykwJPd!et(rnphk#@ z9Y3;}iN4sMI5Oj2r&_SL^OMTxLh|Ph;_`F7d;SOVD`$ZQL-e|%EW8p%O+ZPg(W~2l83DGdZqNK#`YTY&^Bc;s+ zVjcm0HuwISq9(P;Rx2VA)0_ft0#Zo3C_@x2oN>i$X`gK&*2I|TlEwN;I6X&ATq5Wg zldGV31)OPF8mu8@Q=$XGwv@pegF=a+KB&3SGa;oSCZPsk32bKdw8ZH5#t2`^wy=`( zW;D@-IK73QNH_2wJ>BO!(As>S)i;C%uG>EvB3n_(6J_V#S5^08SOj43 zDEnvgG=7F>B<0=;kl^P&4wI7yMO+P%{N?HgM2jj`1zDnWWY06jh5PzC}W~$_a3Ocrex#rI-x+SiB>?w}#(N2*x`t@3pOqJ_Qa*V}vWH>c7cw zCTsAaRd>&Dw%CNr%^Rn}jV)-<^63AT>-@g^8S^;=f!uyUc86SopDR;cka#k+fcIXz z1q*MI+_Q&Nrz#ZF6FZ$RP}$?lncSO|P+KK2l)f&Mo(?!36}F^@F*gN&FNP`7>-Y-G z_uWnJ$nG#*ZQPlaFs_^;i~%9K>cQvZ+PdPU{PPP>UyB5wnR0uJ8nFnT_!P~W4pRuc zfeKW~N!{_>6j3NKeiQx&Xc(@9GRJWo!e^6t>qgHu90R|xuA+vOf9oWQ;Iu=I;i4g9 zg&&CW;&Fy>FZ^TbCQQUjb3G{-soBp}nI4LBDm69y8C3Yp<3YZ?Xl_^lIfU zuvq`R5CZp{x1Hrr{CyoImEdH`fHW}XcYUZnJ}$ktgEHvn_L*(oN8|snBlJ9>U@2(^F-qs6Z5lQ5Nysh(U^5fSi8>ct0V~_kbkn#W_n~obHQTdO4 z*FNHa0Wtd9`Q!EJgZIkWp^Xt;rPY0h(|8Mc9~`GtD5D=R27t`R>w!j$F|wbe6t)i^ z6dfUdFXUIE_$Dh(d}SKIJ98VZC7>tX^Wg(v;I%L-g|q-bYY3|;a7y|U{IZYla84T!4zf|e?#s3?%Tb?Pk`ZPkiZ zHBLX+1)ID>a+W>R_<92+hx|I}y9ERCui}QI;mWWJp!NO~*KgKGw${ec(X2P1|Tfy9RPmgA;Q-KMbF|s0;qyEYcwC^|O=+i*_$e4cA3C_3-12 zJI4J9>Am~!r{J`byh#ZU*4;ridJF;^A5Rwn@uI1cbOWoUUW>6F1<8xX!0M_Y&iECg?2mn>QQ4o=s#`h0J6T^!8Lz<{*$HIOU?csOm7BTKf-|Lenic$ zGh{unr|!moca}$Iq~speDS&$l^! z!~F#tkrhqt@GRhI_Jies)oL~u_ib&U{p^~_I4S0J!0`_FsFv>K;Q0Zm&>3K1Hr@U5 zgDDQ9neka@SAr3bfR>aa5GZLVe5gdd0ItuD-I`OZitl*}LFIQV&ZYK31|yxPKlJ3> z^E6x9nj>Yzr)a6m`UJFh(-=%E$-VZYi$6f?A-5tFvo&-LSHYtF9Wq>w@f^KY#iC2rC(EO+dtNp{xg40I|8ofh4t3msI3cDg)nMyQ=g~ZICnwgSD@+w%72R z=}NS@GFu@oOZ{|>%NGJCA14CrtY^0T#N;K)j(1h1cc&I<%A*3;#3e7hWh#XSa`%926At_eB>$`FkQ21& z*FcH>ytA$(Roj{O-6(6rOR+}sYgp+xYfP=>w2SMn!G@MpS1gR_=;WAPQixD3&=mFMtd;2GmMfV6pTSAnSK4f=I+E=!&W#m9>#mL-N>;p7tNPcGX zX5BJz8EJc9Ds|Q&Pb z|A;&c=46igG^`lrhSEcLB7W~j_HVp=dzty>rYN&C!@n1(*?S*`$S5?3h3^ppp2!>H zeey?2rje$AXtcV?bjDe5iEpL$*aU_i{-Q`;Or>pOJTW-S)px6`#OE>2X8x`g3<~5b zPU}zCeZfud=C6%6?>h9wVzXQWnweL&2Y?~a&BlQ{tBgAd5xum?gV%y)=iSCxSHJXC zky3AZIh8`tvnTf9A!>B^oGqnTz|U-ATlSwvsNJ@u2BAZ(cOKTY&q8Kg+7hotNAjc6 zHK1$o@7*h9i^*sc+9=ziD*6K<8!0+2H1>8%5u_2HeVX9JNLlSqk!b_Pzl4yGT?RGG z6f#~xpJzhz=!{guO60u0*ZM6EJyE>~b-{NLHKzbv_v><8V4md|gbc61vo4cDWnsN5 z%xAL0zm}7iYRVUbxq|)7nzagxNVH8pjM8#NU4|!KNtW50r~;0SXDEKG1T$uOZ^Y;M z_UsJT9fn)4SH{rGfs?>9>hU_`3KZjJXK{5DvI{vo9EL7&0QC8KUE}~nM$#%lv zqT3E9k`AeYK5j%+H_FsbyTTFOzM1pD`zn;_$|B+9QhPJKO7>S;N8OjQXdi~)nokij z!wS}hHu)2m{ul&d!|2f@$uieRKWJkRc?ydo2|Aqm#g+x`K@4ETzKH)QjC2tWu&(x+=ecrTte+PO35D>v;+Jm!W_jS+&-4i3o0Tk z7n!aVnGo{E$05E8VW zb;_H50}nZA^S7)G2O!QPAqJ@+QyOL_1EHKK{!_VU+}RcrZs1wIQN9wqF29!E!^BES z4Cc!zp z0%Ux&_{Q(C%^NvI_$k($J1s0%eI6>9YQJj#DcIkj&7{aEUEGceQRMcvkKsmG9AbV& zjm^ZBw?3}Y;46Y}Z3EpbB2h(RtQMG{<*-Mc}aY_USei%dE^XIwFEWGGvcE}!kb~T*n zpFm*k2&!eLLl&ADx=vTIicBNY{qMPr5vA-VX%T0=*u;bg!yztoQf@rCL>BCheTv;! zR5O@NO+RZq=IiYG=-pj_YAK83vh{_=ewcKr*2d8FcnwCHZSmV0jfdIniAo}8j!yaL z^Ar*y(NL#1T_nbvTqh-`NjP;S$Irg%9p2yL@+~?}s>Jc}62IqJ6j4~x=QDC}bTv(B z_sBuh@6nK8eaUCapL$yrJ6s(eGDVctbk!?9U-Zfsy&v4%ktt$MmSLO8{b=a;%9}v; zY%xdo?t&f*flSeG8`zGX9(;XJK;iYE?_G(0_(@Wyl=lOzkkvCp*LBfwLy5@ua;=jj zHI+w06(37hM84@=%-_r9Pe1a0P&|>7Kwez3vNEfeG4sx+YDIWTZ>#3yoYNplW-*ee z*#>i$%RoTpeud30?=Mn@q)u+H2j&B-dPV%@IV1|)w=ZYWo;^%Y?M&(*$iq!i-{dLE z3bmE_z@H7YUQW0b3R}Di#Ls5HXqL5Z{cp~Y{l}j75l~j zWHSq2RMMRjhi5gv6BJWpGn0us3(sh#IiW^^wl6!oYF`P65D80}NqNj#@Ae|b1s=6M zp<-5K6k*pxwrmV9MZQ2m{@gq{|D-|Bg<1ipti3fZ9(Ftus`xV(jJsb}uiy9mgkZL$ z4!9+~(#?pR2LuILDfA+CVtMbl@mw+w>C3=c=dox&>-o|(XXHWiqgVaizvO3RCz!4wsX%xs!mA-o+bZq1fvZxfG9A)t6Oz2JgC zoV;KQ-D6a8pCwirdFZDMLV_S|u{_UH15ZaO!Flg8$%b8A+Ek z+TZz6ZyjYpxfFCM?N`uAly=FMeOt*Fcb6CYIPpR4g}{BWhiVu%Xj|q6RCIWokNc%q zS^7mo4iw6nRL%$lb*S843JBflUPku}>{|5h;(d3#YK=CXN|eINt6TUUyycq!{3`*B zd$W9`z`wjf2iUGnc>KFrbZ=C(fO4cyNurg_{)M<}7>&B?uq!O$6(24yi+VRrPed1q zEm41EjB!#lA3HV$Es~FGG%AjCnZ@IBKkYn9D_qNB2aXXYCDCb3+^c zo-~DQTH`C>faXfSr~6A{@mcy^7=08Rhkmpc&+^cQlYO&5I>MQK?a05Ssr)cM5??pm zeYo#G_czl=R5wL#bMg7uV@qB5FSY^<&M*_rZV;Ya?YlyQ>Xi!XVrXD~rF!OHgBLAF zK@ZvUX>;GWP+*cI%3sN#Hgce<3SbMKg7@`e0#esNgrYdHA%$5%=3Q5Xzm%OkrUeIf z$HWsBsTe@eL{w#ly~sha^9m-GJoz)kt}&nun?Jv2=z5zXX1L@=|6G1}PrR zMVs1rt&<9P_A#NWw^%@0GLC8ze}RWa^emHqgT3An|-3wnU!bz3*D|sbnfv{u!`Gu(+smDXQAU%>>37-MYI7{ov3$# zalr(`#;BE5Uc?w99J`lOny%4>WB@DDAmQk&n(Xd(mZHQr!8+leVeNZ9-?V@26BJ;4 zE-0o|;hx%=&M2mBk<@;vo650$4$q+jgR!yQ~2|f(y_@1kZJyO+rN*ut>{SiGBRHt zx|rA-8TOL+Mwy86%JV_usV9fEkGm07&3D#0?EvCF>ZvB@-a^_VjdZ1_$`)OB%(~vu zCs58r@6N=CEXY5a=Ym_zhj+?+xCdQKRelyFN4#=(1o*Vul9(hkslq;Lw)D6>A(`(9 z4Fph)cRf~r$lDS(0*k!E`Hb5#R(>q>u+A3<$phc|pdxQR0>eVE=tEd*ro^LGB@;!K zf=lHW55_J_N($+VEf@ z5VYtd?}`;!c_+r8cqSM}ot!G#NbRY87ZZP+B%L3#2#9Ky5VMrCysPiA2)Fz#NBdR9 z0uhS}9!j(K|379I$hrIzF(BoqV}3dP=6K-B--LM2CIike1t{1HD)G zft-qr&0jSM%%^Itl1aA6=YL^fv8ebLz2~1cwB8;v^z{aUn~nu3yk&vKkq;X{LJou5 zLky`p_&5AF;S3MzBpvLP#WND$ZPmTz4bYiAIjnn}TU|ZZDNU63u&2QK4Jki1o9S@m!)ZJ)m}tG zipC9=PI}aCyyyjQLbFgaSCHy7s{yG_*#Ap)zF_4u=$bzw$v{w|CrM^8=?|+R#a|`YIg`KcP|uvtV*1Z{^=UoF(3C z`S&D|uPMwtYCP>~D{BvSk+g-uw@6EvkIq)$yVQrm)3HAr!ha}feyjZ?@A&PE5p|!~ zCMn-6M@`{xs;0D}^xTIQr3hvLyz9?+1w>San$7w7#^t8ozo%30kn@?x4&~<)5f6?4 zr|Fu=vZ8rBaPtTM@}*Y%>2Zvp`;2slWnhN;Dfhpj%(l7ReUOt zSsrV?cF@LPE8cV$&7g-nL_XAXZ?PUt-LwI=Nk82mXh2P3D)-X>bXm=^4Eo8zV9DZy}FK3|#@ z1UuoVV5d9tv+PORPpT?WL!=cr|3yX2)GNE99^sii$g=MBOnfC2FD7h8@#Yj-ioS+R ztcVtX9U3yccUK4_U{wmgp_B9S0deB@I-@nfrupM*cY!k^3g@lk?FMIrHKUX}C5!KH z_Ov;NZkI%*`NYsI3k)>0658Uf%r)pJim9OBbyxt4Lm=yuMr4PrB8BLfry{$=-6Q|Bqs?|7JM8){Vf2MpXhBh^WJPuz3Og#V|8 zBL>x*L7i)MJvBy%Mob*8^M=GsWnTFyv={CH;~(QswUtpYq*BAnglf!BPMTEmTS`sk zq7LTV6e-j$+1Gd@R?mBDhzLyg1+Ig#HnE5_>0CV_{|MSqmDmu zvD$qLrotW80%mD;IiAI+NP)HU`~Mo(0A=#?NKbZ_$EY zVNS)KrxX-W4f}L7^qa3dMtfkSDM$%7{KxIbaiG92M?X+K+iB*7(L)`TX)RUO|tQikXtsaVg(ccBiTPcs$ zRV@?$=*ma}1O>w8Q1J@+uexIc4#|L905t6(PWxcgbm|od(DBQnGv^OUi4(<%&KmWi zatynuIi5{fj$IVU)6gR(6?VI^7!jzpBJ-E(o7Eh$9|r!#Em>fwPqQs90xzOyfoPBh zEv3|gz7Ulhtub7H9(aJcar;|5mqkY5SBqVhiO>IQX_?f4e%`~I)DbYU#S8F&7?CB! zs{p9nyWDm@*$~D(|5|w~qZ2QnfWLX-+*;HtW&OWbVGJnnLzwPVGHa8f^9sH~MTb!k z7wNTGgkG?`kvRwj@cc!g)Q#=3zo5e6G0(86nr!~(zfaPFN;suoNhrqZF#uzNnw0|8 z*w-lRoAmLdlDB!~Z$@D(Mz>MHU66u)gL1Z+uV%z+u>bc3NTCB6dKfE&U)jy;58|Zm z>Puu7fOIXP==$$LcA$lrn$PE=MDBX%jn34w;J2ed=P`LnBHS^t6{xxNLft0%%%Puk z5&^@k@So8z%*( zIg8-i-roW)4y!e8^Px+BTGD_&Lhg!@K^*p`&%A<WY#0)W|vJ1|O2oP?5 zH1=GW?e~M5M}{;@rs}3M#g_?7?mth@RhX&{rV3+Cam8Kp_lN3%ALAwU*G!>gcaOLC z+K59YmeUZdCsn%b|M44&u;x9L@@W6%o33=xh@wDej$O_wND+Mph;zSP;KkmrFcX%3 zRlpM1^Ar`|6qX46Y%+A#b(+Wn5y>y0;XGY}!`?l2)GlOH&z^iMWZxEFVO7l|m?Zt~ zknn`%e(2NMXaAWBY_hc@Cx|v|AV1&U`T|*U@d{iz%&hPiaQy2{DKi zbpW+p*3?98q%}W}6_|9MrrW&%2t>ceX_TQHgKuO9AyKhSS=0Ht7FR zOlllvFLXVhFX{b%l*-zcro_I+RM{ftXX|(Y1d9IWrnL$GVlCwr)s7Fw?d?I}=ds+| zQ54)O)GQaEMLGq`R|EbPp(s2x;_F6{T; z+WlW{B@~KBCD4_ahN2OQy2+-39*mDnsaT=A5h^<|Y&U7sOl zs&}nC@rbF6pD5d$f#ZWtKYemK2m}5cGy$jWBtksfLR(ge=XZEzm6&3o0$lV9_fa(! zds_~#X~2a1=^mr7hC{b{*ccFU6ED=j3%8+Cq#u3%w{oxXjVyW7xU~9pjGlzwBcsD_ z?0~2HOXW^@z&BJwO7`Q^FRn?&w1~(8#x2K63e~Ew09ciDw|Ev}glgwH<3dLT@PhOw z7u>*0-ipY7vIFob15_#82UN)js$Mxu4{yxyF`BoizK=^f*mUwf=?;0C zhYMP?;!VL>W;#^hy+hlSb5ICzS}hWg0PTg)Q3!JLJFcKFYyUZ?=(&rZGIl}IQ{NMV zUrb{ltmWs-M8mNH!h>%B)WQw4P}t0^2RNhC2!Mmt>&+E@KVIIXj<#4ulHR&a_|0&o z0oQ}AkT9o;oZQnako;La-f#%7a|@9dXQ~BR6Hi<`T=C_KX`FyUfwCpgtg!$Dr=8Fz zC_*EOvUwOi4Hm*x=k^2mBP&!54(gW>(FG^P$IZNg##@*N88PIlA5pGB!~bB)C{<=T z&xuPE;u_Tj=B^8<`Y^Etgx+7kzHH(xLRbT0!kiYAAXIs-#YA&W6-n%hBF=FXH?)pnr*e)bG1}xpzO0 zm8bmHoO=LvekY+|B@4n@#;Rk9oe-o7&)uUx-E zcT%ZJ7;^Dz;>-7s?>l6K&4d?bWM+POk8h`zZyESjTzq-I+hMQ=;0Q6C&I~^sE2^de zRbk7Iv5AU~!Y6kg{PB77f1PyKQdmRA;UDiPOJiVade9`c?VxCeB zm&7EENhmJOuo1s=Q8sPQ>t5reVkbhr;Tbpn*T)OLFWmr&u?UE<(o1m@1BOaxzg{^3 zgx-GJ0B@u}^R4J;D~Jtyw}B22mIrNMIpugx+8BE#2Ri+-Zyq+GqEX+UG_d|tb90kk z^XANzI*DY~nB-?la4u1+K#Ifdc*2N?&>ynY0^DdJ$dhtA0VFM#R3m!B|j8uPv5(}-| zi_ay~U=MGKu`GUh?g@gL(;$Phu9Z~9+7(CIJlUd*MZOaJS}@}~{H^;(z}%4RmIFpy z(&9ycIT^d!QrV~G>`wdS)BPpMB`*iYZ77gsYqrDUMdc8;uOXzcvxFVy;H|vmzN7X8N3JA(m#0=LNFs{cCI_;;SC4hNycf4m7&l80+v_b zo!&ot;=v~eVn)3wxR(qRM~o%kKpL_9#Y|^IdUJh;$Mubc(6bCAsI}v<^KSJA0cBI}NXAYjjum;Rjqu_PTnrmL#J%I{*ACU$8y^N-_ z(@mrg*sEQ%j%5w46=nfaXwHjlO)=NyuMaB{=G4|vIm71%ZUDc(0+tpTE#dQ~Il{R{ zfw`;1`my=)1sGv5^MP9qs(SIPu1YhbJF;b^AKuX<0q!8@W@XM*pZ;Bj2B5E%qC zC~;csVKI}x8jFH6n!?Xu(!_-mGOL2F$pCoX^&+EDwm11lo{aW-G4=laZx5!=0rX|0 z*!@sueI@#ku*gX{C#unI25Jinn}eE(%{Yp&P`^I{mtd;XTGq?rlX5e2K<#XBn7Gdf z$D_m8>KB2{g_+$mH9v?(c->SLOcYC&1+VATw23RG{`u~D4t}AkU~zmSS)icT-+L0b z4tBAcR4bs&X8eazRFaL`kH6M__N0yYeVT)b3rCTs*1sf2iu)sHULZNYCqSEyZU|n? zT8JrTu7ecvsCSXUbg2{7)UX(I7@lNulD3S95D{QIYWUvMoK#9g6%4HeewR(urgZ8r z*oDlWlGKChjsj2;)%#`4o&*I1t9HQI*+wlveNpp(c~Uo8;+ z`A2ISS6UjX<|q9ayiay`W|ZzZ75hB1tYQTFZ=gPvtzw9ZS)L`d5hp4er)6}4#~tgo zk#D~>ycN2I$VxO$PFvRxo)U)rCR!F6%c_m4?dCvwTXrs|r{R?H=k#)qBqeXd5ee>R zvh}h68SALGO6`Jj9OiriRw3P)Q2|9vTA0EO=kOD=zj*<-1hzWmY<;SnO!?r{F{X(mCnzoI;pjLCw3;RU@LygxHXqQrK*OOO+j>Xj;^?YxaUPD-0vlbB9GacxK^d4Cm#YE z&Q8G#=Xat4iYrD`I&SwX3ywT|uZ>jKX>j3JFKh`$*Pcv0-Y>4vTZfDILSorhKwx+H z^se{Uu39XNYT+J^AKI4~;Ejq}L1Jy~P*tc`8J_&Pz=PS;Ua%?e7Kp?Qxz#W@QplbF zy;wOvr}vj1$ncPxAheaEm&c8?Iu>OWbPD z0hRTBTioZK>$m}uJGH$m;asO{M~P!!yQCi;>XB=*iAhbY<*u~6BGe;X;gsT-*|#}* zlHP-dR@MaxHjcAqJ*As*P%tGL}!m5#$3trDvr`Ap{r@zDrabXV8PUG5Pmx@f8c3oK3%#C<6UDM ztoAK0Ff2I@C*shxZ0kcbfi06t(maDMl8dB1NdG%zAZ?1t5$Z5`!&PxUC82&o>w6{t zPM|=?c|O>|y`9hLojjx9Fy9Q{){v7p5tZ-glp~x4(smb3|06F&ND$XA?RrW})%s6c z8A{}u(rIr69$8v-?*#Rhoxg9HrzcH}hMSSoGa`GbDZco@>ZV#V3k7ZkNW-X^XFWee zCZ8aQVP68!tvH(v{$4Le^?Y_mtPB@ecO+DWclN+jcXo(%#<@lPj&MeF%5|H&RSqL<{{XDyoAtEd zz1Yl(O1Gm+3GDCCpOkslCqMGT+;wlBxy_KU9cdhX@ep(gpw#*)A!l7|@rh!5T`9S9 zp8CYmPgYb9Z{ky>5x?N~(CuP2*9JCsDQ%yv()eS8zsUahOoK+j69H0gK$vDwPYdYA z6`bA`9x3&n4~UPqS?L+my!t2@9SCFRvdv{)u|v~+gY)Lj9nO2|$_o0zc6C3{&V(+* zj5VNp0XOpy%QN$URggrLaBto3!Ci<#db&40h$qw>y4sn-NgHN^%tznp#Q%27eCOMc zYWZaTj00%r9UFpAbkYd;-8sXXZY`8SMME8Wfng;?Sj3#b{I82r*=ZHN&H~5qSucv(mo5 zMH_vLUK`AWSm}9g!W@qt1;rq>q7Q{(NMp{h^uDWt9E44r*cB{oHfB*SKu(#6-iK-< zslqBIU5ndDDUABj3(b$B`2Dq&yQX6r(y^Q04t&wxGy}35+tOowexIBmd^ESKG$!-0 zPc@GSZo=?S%BUYTP5_Ip6(>3%-7x^AEFFqt~eJqWVv z6|+D8AV{ERnwO4N+7C3)rpKAAuKTwl=NO=lu2)KF)MQpG{l1vmp0o-F#;QMhv5J2p z$r|EXCU{wGJ($+wHgE2x!P0!Yd1(seNE0Nz#Y9r6EDae(SmI%}SRckXu2UF*Xn& zUlP{iA5|+KH!b`4YgBTgLY%bmI~Jz-ySQ>}kni$X=1~05Rl=qzVgP>dO&fK>MHhrL zO`KFN=gC5%Rzcgq=f`ie-JbmJx0^{{8@o2#53m#?Z#W|wwQ2Mmw(J|Q>l5_Gw`?Y1 zXPm!VJhU|`u9@11J)eee@3_Veb9Xj8g53NKI!`tBIfEMn-_lN0{ryuO;&9rc^W8ob zyKL@0arK6Ej3G1}?81A)?lkZlX#}d&M2$AC4%suMop(Xwr%XE1yMB^7c{;NW3r~sU z_jmmep;Bn*u7trCzfjW!saK>+toiSpM^h{Jp>f#iS;q9V>+?tD7;oQwsB1vgPdWj8 za#EjJYeZXUb|O31y8{d@W!PoePC<=I7{;;qo&#h-Vg&dUXVlbbnnz;VR~KN6LHCf@ zq+8&RikE5ZsoBeArG{QS7>v6*Ib32#knP~3$j6-dXTy!RHSBIbyB;?KKVOvM zu@EOYjKMCCx4@~OOfl~2<2~m`SCJJ^Icf;51)tJ;xI#y{zUfZ0WA2Mq?=r*vy_vv zf6@5)E4uBz#TPup=wR1!%#99?TFOl`75?Qv6!R9KS)d2D%Tc`GB?dK!s9YZn^jn2G(!whE zo@9TAI|aR-R_8JaPd%Sw+sf9W(43$^?+Cn8;dwYEk>zz>YPtEKu`f3Eq>hH)Pqg`m zXMItRujnM9>{-{$n$%xo<0Nv$&Cjeb+yEwKALRt{AYja3vQbxMyFTyr<_<# zHO~Qf`G!W2B4z8ep<~apZRn4LJ+(l*LO92f58^8aA5ZS?J783YcZo#4_=}9x5p!2x z?+u|~Z*KYIJFl3Q@U6lL@kpi_#u4O1+qa#hhB3pna$?Fni3wq%JT&IvY$Z{@}5t%JFKTxYM9&?VzTq`YzaWr5^jzL}4@^|oukl3WC^Et{e zoB@pR-;;bZQsGhc>OlmN_o`=xrs{%E@1oD_XcXi3>nAA5`+M2qM$??&8;47+QQ?~! zVD$c$c>J9TeILfOW$_)(6KU8Z49;RzqlMEX=!$v1PyyYWt;ZiTP!)N(Wg?SM?0R&r zZV@t2K1fa{E(p8pVoAop&IXNA)4bfOrRSO<2uxB`3+zKa(zLgcu=_2WP4h>|seoQ| zTE>B(*lAK0)9Za>1Cj;w6Fqjo1xI-QUbc!8K7c^k1jgB2aSoz*?(l$8k@+43>)|wi zmf2fUUG#u({QAJx!O>b=P`ss$ywW|UmgxDrUq6erb%H=R068_vGNa z_FfY;Max?q?`nL9BlpNy!0KS*_qqh9a2*GYSwIrKk`VN#Y!`yGKK}uIQ72@Y1m+}F zLFItuf;~gLL`0m{8Mkw@9YRx{}hY z29L$;Q)m<&#AxOjNKiZgc5@f0wKVJEL5bVvVp8Qy{5!^zn$yzINv*&^BqsaMOGny{ z83LN*MxF1ajvF-n)T2%IFUR`sWdwiL48(y!kn(?-ySBW#=Mp8olcv^HW8s!f!pM5Kqg~_`>%@7PHhb?a*ur68fb1klv{Vxf7?L zQ{uH^tpHtRijrHw{C(jmxnImtVx-`V!q)D zS<7&6>{^?M^s#BG+FK|J zmo!f)PO7TuM$g0Tou3#Z!8%kJ;P8w+IR)HI*-9252Vdr3g^DO4Kq;w*Eud!oLVPuQ zRi)Z2%!S6X7FlN zDtFw7X{ba56%2oZ-78WoB)(Zwt*YOfwUA<-a#V$g0({exJ+I81tXV(XY#$l@5`)tP zSo$@i=2}w$U3-3iKQw{~+Igwe14w+pO7xBZFhi|eq}TOzU<9b$V=(c4^M&`{Od#9uV)Mk9 ze)=2`Xp_X|?F$;gWnE$p>UKnXK5oCi3KMW&Gorr1}iKJi;!~45A8p3Vi19UM8wjj5!`Dm{0RTu-(na9@se28?L-R-AABOU?oL}#~ z{YAg}drh4qTf{h10&H%sC|ds2UjCoft~?&f?tN>rj7XMbi@_KoGS;GOqb6CCr8nz{ zCcE}hgJvj^oicVJS+Zn%B}q~w%34T?WDD6#$oGCEy`SITKmYk~&U4oLoO9jxeO-xw zc(%OPs~vG$fe7pJ&vOVKFitB7k_Xb(k=Q3oC+tmw3wFB-72<3gYn8H1$%LhJ|=a~p7QGhk~JZ-%fM?L$v>hag2*)ld=?F0PUE3vZTFdN zaRI(u1IR$I0@;>-E8BiG9Ic8H^_6A51o7#2LGa`C=S0BPgN)t5RWzeKatXwP)Sd*I zaIv90A%U?pv`a{qeH9lCQb%yn{VMO%Qb1IA1i(H6gb=h1aR68qBV5@jyJIl*tU?}F z<>R_xUAoRXu} zUbmr4J^$75!a(wY)%lc$5*PT(+?YMvj>oyG-$x<#QjxPT(y2_vVH&uSqhZgxR!$fB+}dLgsXxB^ z=tnvY_V^9$?VZY+pMI0FJa)3U>D2UDAzmc+yEEKxF4z7GWNFGINQzmXOde+%d*>05 z(1j#`xjTs=3R+#tTYbO5OY|kOWQ)}>fJ-C5X32?LtnsqFO{dwv%v3uWyZYZ#(fDd)ebH& zM|-C4!HwsDdAZa5z)%k^y!;!`h5GU<;}8?^!ieiFb#eI0j?p1VNS3vQ1RA@`Un#BM zK>TQ{Aj*8KkwzE~U108501m(uV6}ehB<~hJ1Cn1jOE&V9MomDW<>nM!5?2+7AB#dD z$N3R(n0;4n&cP=%+=ylQ_q8g*6`pS@=5EoPPnk*Ry>DA&tfqh}<8v)CR&0W-B;W8z z*>wAfJOKG$>eU5dU-Kb(Y3eu5TPB3zCVb|~4S4(!Vi4jQ%9MC+lQ*kamgoIi(PM`B6nk4{H1bbw^;1Zg4;nT+mVhJ=HbhOoW9Kj zcksrHK+dBv(Xb6+HP3dqpc8!x_ICv#6UOr!~^CPi7-ApNG(F%&OA(x)JJh{>>Fy0URo<@ihzDOrP>t3&((Z~B!Z?#~Y;pEn@d9`I>z0^QZ=G!* z`xSCKT}z=A@Y`45Wwp;7vrs6C0r{~m_psN#agmY^E#`7nKl-ZpF?WdEc_I zd>PWpMeH6BZespIhjYmGk&d^3m{7|r;-pQ&vn-KuT|XN0izSlHSKW`zo^ln#cfwr! zrkE>O=7|#1UcNI4?vhbUtp@U|;Q=c6v8RDwd!)Aa1&TY95avrO?zeqpY~rtd%Dia~ z*sGCB@6;_+@I(=*HcBd&(n|V<&k52}PrxfzI;5(eRj}yFuN+PM>1M@^eV?zE?je}= zl6;|YaPi1Z+BP@H0X7M^ux=`N+}B= z7a22}i1)OI!n8@3Jnd;}0kiH~#|H8QIcByb#e_P!Nv0$A+cSqs5~JrEwwZ?x)Q9PjjKvGelGRXJ6e3)EKf-oLYLRdm5W@_|%=x8#Y{xAOQuA+MJ zuG@lH2mJAl6Eu3Kx|(rA)4l8qS{%;rBCHoBaB4R_c^&?r~4*5?RqmU$M!UQLa1TTS&r4u(%33M7mB{b|S-7kiRa5LMu2Q<(sq^;C8K+ z;N4M@WwzbUXro!~(Zc2Vf1lWH9;{Pf3*p)BhNPr>@gUeqA-LC~z8P?k6#LcaF$BC? zzxz!`)l5v7g4h`i!#e>gav^o;HgrNRS*)9J`~{`CvXP$d{OZZ_xEBvwy+f27Z!6Ze z@!xcdVDw+~G(h_-y1DCfcoFzR_OgCjG>F)ok9B+za=MoZu9Cfs6HE@1H*aCgS{^-N zP&)9B0>6S$;3B5>yZ11jPE(J-zh~(c%+&qN_mn8uIX5M=2mDz+FAnvjZ%pS=sH^Xu zJw^#I?D(b>BB~M-I`X_W%|Gx{vzWr1YNmy5@Nj42RQq0qCD-B9&Pk?Cewcccub|JN zeJoD|YPe#HLYNiu;ul5Zu(^CJ_R{qeZ6;b82c{+s(mW@5;9BTbsj1(8lxM79Vyb{T zOSm+}7JR{Bl+BB;eM8mGo@ZMEyABS-jVy3#GTvzzlR$G6UlX0?wCQiYFd0WXr`_!0 zk9G>2z2o`q(KpTS_ofmy`#NtK*j(dX!zv{~IWO+%|H;-n`^HJ-yIMc(MR$4}I(3Z~ zMIbSXHt*u$K2RrLZI=;7Q=6)Y-$xca&4od?kJ5eIOXy7wZ=W4E&WaDtDUP5wQM2pS zWp^**_cES;okiM{JWys{8HvHho9XvDoQTe3iGmlV}x0Cxm8C;8#rnu8} zP6qV&ketJb`EDk)@*$NxfnApHl$A=pB;K}dYVj$BxX7CJSXq(xI;-?Fac?nIw``=7 z>%88T);uqv@o8$`v8IMPv@=&cH<8HnG(q2tilAY#s1ODL4aGB#jCEO@#Z)e)6o;0H zm3=v98u5cStQO~uS;Yso@US2MC4n<#;oWAmnJuOxp`zSavpj6@u1v^;0M+`W%5#lE zX1_*^Sjbdi>5k234p;TL3pvL%R}>z`WF}m8Fyh5> z1gTuC=j+Q+JiO%EX}LY!!ol)b$M!=NjhO^&&HcNVpEADCC?U?-Umz2OiXlqR-%7bL z$~g&tkK>-QtB`D_-ePKFs3FzA$)5EM-L-Iw>AqDva49^TgDiius^Ue{>24G)J?=Wa(%P2#h zA^g^x3uC*38)Oh@6J8_142(p@pPWd*NO1%aV=fbk6xK$=fsg0r;-m@$m1gc^+kiA3 zrsFgt7>JgoF%cHa=X@wE(}qM+WM_B`WoOFx0YEpE9FO;Yl#-c?q%nA4Kk_~BviN9# zAe7Rl^Ia?)Vi2V1nZI`Bjm|cQ>^4D>o<}3n?P_xv>j5JVY{c*fAA67VjvK4!M%!qJ zj0SuZbeu(ZJ@ER{&GErZ|4W9}8jktHndiaL_rqJBw|Ex-5c;xpTN7c_i8$vDMY;nS zgn5pPr~Xqdo4-KyC2FAB)dp~AcBahF#~KuF#tx*L%^|T8jZY z+@X8*b#|41|1}l3>&-&g>JR;U_XQhVAK$MR?Ddhp5=^)#NX>H8RIev^$uiUtTq)Wu zRaN5{Y{o}jHlF>G{il-vU`fh<`BqD7v^>M3M3YZi_pG?{(kCK}VZ8oj>iu#YH4LSskKtg=ek*TKGsC50$(`x> z@E~0#C@r}|mFVK1wKX@xqxMQV_hVcjJk@Vhmq=XHM_&8+fJh#xtRAj~yj~9W>6(is zkmHM$QWnSC9t&M2%iM6~Y-z5kJ+qWpd|C@_SZn%zn6I15PigdFi3ewanK)_AvFbb?Fnto+FMrXws_nh}%%@O4bpe**+)8c>U+KO3PAwi%tu3yDHJrCwAb% z4v+K!wLEs{EIq~c&PSIM#8o(CT=!}$pCn3MwL})ee9A=za3%74&NoontA0A4Y-m4A z_s?=uE%7j|+s;B2v*F|~4ZZMI|6+zlXIV=_51=;TT?RXLFP;)Sqccg4w%7wRtRBMP z#B_vKz$WA?57^;_&-Aa(_MLTCMCXNHSBcnP)s8rVW)KLEe3vrHWtjQ2ZV|m4!Hu^tQV^)RA+WhZhK`2ArRs^JYlY8-%p_mj{gK zF=iUbcyM;5dOETl=iblU&t2Dp-F~M`0iAk~6ywY{mf#_jBunO^s*Pr)tH&MHu!ad# z6{}`rJ3<)Zl`rU2Qs08Y_P1tk;vaO%ww4&BYn(T^$1BYDU5(hTy8U|c4`~mfJm$Nw zrx&VA)YJ7%pgfQ(hi#%31~7VOC)Yig$MR5JrJkQ&CT7zr_p^4WqJml;V+`L9Zxt8E zClT!4@S<$Pw9ow$+4OU!Jb$_VU~0FEidp>6vcrBK4FxwTDRS=c5PEh@hKt^}-}((; zxT{tHE}k^)mRLGHMQ+vq0fs^pE?9e4O^6`B`Jp4cW$6tund@fc@G1;UvqVe^{3l{{ zK3u(uHr(A#n_l}J3H}f}yt+yFz7VLe7k}FzYVyI<)eg??D?+V(9Wg_j7y($ejdVOg zSg&;0n{hwsauq9@;m=2~Td3kFGexZ{13GEj|5z|U=l1`uGN$p(gapGX#I|Pcrhj*V z9?5d;jO_1*!e2s&P%+UlFUJu1H>#NEB!nzyKmcZ~*(JyfZUxlbH5+-VjzEZUe$rQM z^tnyHKm5pysrBsdzL1>JiNJ*60+Kz+_3$ol6S*>GYH$jIJ@rg97o5gzSO}@N(1_*3 z&@IVFTv*qKia;tvZ*?ItFLFNTLH%wh44U>%FwHKwczuKQ0pQ<;*k*8jJG^`>BL27e zf{frK`t1}q_0Qb&_b=>hs8yg(N%rA8-PLJX8wUoIIDR8T#{0it`@S|g7wt`NVgHOT zAvJ*uIW{!$A8eez7_B#CSQwd|#~W|zubnYa&p9Qi^MlRahJ^KH(agCvN_}nTOxFu@ zqI^V=1w%m$=J&|mw%h*GY2DSXFlYXs63;@eq8srW^XvYOn(o2W8`UtwBA4IBAfVgt zt~4904Y3`{^aNbl)>c#nAL8KqzD-x{?+9OJ-M~e+%kj8nvDUQe9Xe}O;(lcD|0xK5^uWw;{qw21Xuuq!7P^FG!bO0Y ze{D;J9R9l;xPp_4z`?)vmvd<9szDq%>^=L>-)aoFegKB59;L)Z539}(1V~O4DMII_ z0%|JfGaa2`h=^f$fg-r0mCM~*t%#a16?5qmwFKl>OavZGG=L1sZ3Mcm$j#I8fw0F` z0-Id>&wvEI2di`B+28e_RoO092=C$yaBwZm!x5YT{5z6}R+w)U5K?6#DM!0r46Df@ z9@9YL9W0QX=kz*L_dTz8Lsf)*Tt{6!<6=yxD|Ek+=0L^Ot#$B7JAbG^?mLJbgCPB)Bw z8_;g_4dy{!qpFUwC>{Y2qY{dWs4i5%0R{5if$ySExYf>+3KgdQy!6wSwfW|f{*#)-jjkT;MkHLqbA z1hp@d@ogizQKE|UWWuEjXu&kl2yZ*YmJy`f+#RC_ZwSR9oXL^@Y|T7$YLhlS&YZVV z!~SP$Y)|9Hr%%Vsa59Ku{obW25NjNPM4}wx_;-G@t-n+e4WvN-LTz(Pkc5D|G@*wk zjR=nV3pCO1B9G16Z(-Ix5A8t}r>PNi+jl`^0bh~+(X~EmvvF3(_!9>~v?f`$6x;@T zg+US}U%(-=6%k01Ox&WiRJe^p3CSpmK%{@(y>0hoKc!uPp!-&!X+wF@0!aG^N9E03 z70#enUfRlSAlp~9`~FcLntUS!m&a<(#J$OY1ghTf7LiheU`^arj~u69vqy?~=l&Q= zG<868C-9&d$AfohJBQ)!v}IcgbWx@6AHGgb3d`Qbv@mkf*e3{<5Fb8d>36*7~(L(Y&bhjpZ?B);Ft4GznJE@p#S?Ywx}GTKBr|wI@_j{s|5iDHaL}3XYVdxH1aLl|mF0)FzCp z;F}xp6BY0Ws*~~)QIz6-@-^@mOnXT!ClnNHLg-&ql%$l~;DRv=RZVA2Iaxjq{H#ax98zy*}~428hWnbGdmY&0a{wA!EXb^0?bFhF+b7j_Q#J@&uP2DkD<-|~ss+1NXpI5~mm3v%;cUim-Y{rjHZ z*DIRaIop9*aI`RzvUN6b1UEYyLe>f1|8w{M{}umzmV%>&2^jU|-5h`J{=eV-xt^aL z8uAzEDDM+ij=sBsyphBRP;uQj-!@&;g>m>M6lLC zD|@#Pm`K>8O2rKGK_cI?Z(E94Hxrd-vdw=kwla|tq(q^?XdKU19_%|F#OBr^{qVE3tjNgwFE>*tzKaU1qQ%9rW7lwMX(qa91-QPFk7smbl{LfvI zbf{=nv{cxkf&VcQ6^+H@AA|qiMIeRJn)JC#H1nU^|C-t=P4f5iKf@F!$ki@AZ@D$LgY=1b6iuo6ycY+4 z7vn=C5Apsx2x>*{fPA(>wfSgln*Tegmo`#RD4@T+ycZ$s!0yK%#XGD{Za%&&_XC19?fag^N1<#27 zXQqTXG200WW$Q=3Jn3lqYo@Rf=2yLO&AAyI82+0iVb)eYvJ5Ok{s|JC*Xof1?Gu2}RR5hx|KC>Jy#4owC<>VSxXFrEtO^EQJoq(fR5l9!_t!tGD@ldQpeH@gN&ncOnK|v@+kZWlg4W+nFXWyB z0UEX96PVt{?FUL?f962Z2v(b#>$<&SE0_VDbaK zSqE|;j!8u}EHL6|@rX@u{-DCv2bR5FVDaH?XQfcRySz;5c9$k(j3jn|W*|4)TL!90 z8pV9W74R+!(xu8;2jba?^Wd9kq;kT@Xd>c*4y$ZN=4WK7f?MO7FoHZRxO!CtJip|W ze7(3+kZkKJSfex)cx$*>KLwd^hURO}ZwfX0!G?P&tJ_!UB6YZ;g!4L69>82!>+fo1 z_uD;ntW)d65Z`#;9m|-^?sKx=cN?>O`UcFXE264cuOaxcOMKWwgs#r!L-%ntnTQRr z&$MrRY}00mX^-WF&EY$jX*a3|mTA&$m5s59*oNDet>Xw7czsjA6s|;Ua^O2)!tx8~ zR2k)xZX$#(PMvteA}R?=?>EXIhucF5#c~Sn$v)bwd{;c|^<|^x`Zt;le1G9rm`@Gm zetzn+MCj&v-oX!htS%$kwX4_j?B|@<_T);s-&`4nvNz&mz!jTTbmBrC+i9qU9*MTj6(Y!r7f|RSBA7qW;26uy0+F zcnxZ9b&^PzykC>ofZI~zp!t%Fr3a@(UBg(ytnHv>)*QpnLY4e^_ldBHn2C4-n;RLs zo9+Akmh>4RSTb$<>Gzg>d3#EfvsNix!^iCXNu4MBzXEG z*u39G)2z?IEKQr&R$W(R>Wi;|VvAzS_^nAVa$;tB`AU6t5}tvz>6ilg#eL^3N`+5@+VFXZV`t?}B7w`8Vq~iVd4~+OWH7 z>zi{t8I!TqL)Bqu(l?@>x|HR+rC_Vazw<32kf=G>it$ksKf=D ztVu{T?d@i%3QcqG?DjG1iavFFLmPqotlIZzsv32xgWAu;O=3_2l+i}tE(EGAKYo|? ztV)%10$S3HA!t4%7$Z(ZGPGfutoihbO&dvuWx3)|I#z0 zcDa!Ar-Kb2W-Xb0g6p!$_CNwc$aziu{Gbsvv!Z<7b34*;VFsg(54xK>RK z?G&BTC(o!X7t6#H&J5?At6MnXyb}2OZ`KPygQlMFjc*KEKn$9$_01BQv@iX#go-5Z zQjF^{qCEb|6a=!=2~RuN=vozGvPAwwZIl2~!jBEbox-8GF#MNt;DT_9N5p?2*x#&> zPv`%%h=4rt-}@@Teeg|%H>V=hN5Acjxe%73`QJPjP>Bz08;P;l{ZHmWfps&21hYsK zb9zl0>)+I01hTCL2#HrvF;r!MHAVXjx{ycwWA5*dFJ(|#NfU3c{F8f70$$?-YwqAB zS&LG){i{~Rt_HjzmLR(U;(h^E2Gr#pB&Cjy|Y;}KlO&g$P%AW zDJuYF&c(w0)ezKF}2O+lPwlE zPs0nPCln@6sC=9a8U_F|v zR6T9m5adJ1AS>xSwUJueuvOpdd(bdHnVqO_Iq!3z*JPJid-$_hX66o!2>xes5Y_-R z_!5tIUza|hr1`kVb<%_wMB*dsaBc*~hEb$I_3VVc=Nf&o-}%wBK*Dr--w?5e38f=XiqVVQkfJkQ?Ldrvtt} zJBU?Go>uhnRDgl?CFs~bBop+m1+mtwdLfu>ivQOa!opH8N~;N>u^@_xpHo&PLC4M! z55HqP^ZYSUrlME%J;`dSYFyt+240uNR{9a<+RIjRv!BWB44qi_skZBk!;?n~F9@+# zPVkoByKT=QYU~W=J4i3_G^r%C(ogGaZ=q=?`Rk6>yKY)u91ZyW07jOAS(=2moH{lj z5jq=5?A-S?v@PhpzP~lYIyhPpy0EC|=PR(-6zLi(wKBc^z&7Pk zg11r&bNHY614ztLI9|5O@l3V{8M6&6*udI;L{*;U8$R7!bsZ5wubSj1Aj7DE}Q^u0E3ZRxCex+LX^Ied6) zisiu?p|aKAb8wyWi=y{Sn@dkOM}e{+<;7M08@AC{Fe=}}cR1v{_g<=-NH(2m82GrD zf3(n1zs7ytByX^+i_%NBVfw|Q{soiDhlv|c$PJzPxD##4I*8sW+G=O6u9MxkNPo8Z5S`1?k-Vi!oeP60m>=?#cE~%NRKJm;-h6@)H z;WqE{V2QCleI8&OCk#nd>7{Ohy5$6&DJaX7^6G;(z2mfQLL#u+U!S8SO|Bml#h$dtNj05qaAQw?SVz+WW+Uivf*jt1S!b2fVP;#*g;;q$ zrk7up7d0$m-NN`jap4|>GNzU87biZ}gw=6U!qX6yMPM&Y6VeOen+w47blAJIZiC21 zO{`HjD?MmaIV4fNKV@}o!#_ti;gy=YR>vrY_u)dwbFW!4yT*l(PuflAiMo!Vn~lHv z-;EpK6gO-)PZu|A@r|&iJ^E29c3WfYd**2{T7eFOzWV|?8Na*z?GOfy8eO;plSNnqz!&%*IS?e*f7cB113^w;_lBU zF2RKjluMaE^qF?mqy3S!&Jlb+n~%Pe2{n~Qypj){&PnrR)(m6O+qQst z9|O#F*4iXbbOlpFOG^g5Zzr$wLt$cheoweOLcgvdua#k=fHcFBzuJsDbk5*ln0Ep> zZXl%jfUf-(ym6;33f{L{2j*rv;2Py^G<)7R7vB&>oqi8*An+wAQC+mxE@q#%7npYF zf#@ivSH7AVHkFg^yB;DIlF;d5X>I8+_@$Zh6UK-DWbUfF)V4y<#cGzw5>v-&P1@M-)l z^>+dVQhVoo21G-^uMu+|D-<@t)V(1y7_%|7G?aZZ4&=TRU}>ozqKyvGvq!F)S*x1m zJ+}#^Prwknt?K7}`m>8>2!!pNA`nmwMQa@EqCStkkW&Dj!~1x=V5B&y6rpVu#8E8h zF$3n$F25^tapGJ2p}L{~JxfU^va4?Ho8fQHpqtuX#{PcVl|0ECsB)~d9QmE;%_g}S z!SM_iQTA$g(KF)Cs&Ktq>lIe@Y{3j(tf2YYDD^Y*`#s4uaOrR zpJ~s5eHV6JetzA^;oc$DYm=w}q&Eg9+;K#rbcz{>*N5JR$ffXdTqm^eEkMgZUrE&W zv`{&70*Onbv0+`rWvcs|asAo{Y~*Z)$a7|Wm932R6rs}vJj2&{=Tuy-YJ|M;SV?hG zbCgMUr77WJ!PhZ!T=tHY9m;%pT@r&<#yau8EoFH zo`!RY$K#^QozfNL6*5?s0$+|kNhJ*=7{5kt@82z_C3?8+ci~0Dmh@1S$Vf{MlY)`D zjQW8ix{K**EKO22@{LghX82Y_SNmK1+z9{9-VL3?{K(^jZK~k=;zq-=E2P*Z3ST8_ zd!|K2j3xxkuP&Gm4hHo-m~T|<%8e~l9V~3`5D{bG+M0X5RkYonZ)CF&M2yRczM;8y zK7~-c=0cv&DMw*X$>-UWZqSHGLDRavTm1ekv4c?|jV6j5R)%Kf7$YZ59c2wm)N|E( zb63i?oFFIZjX2!xoK0QMHmmcUDPV%$OtHVyVUk3#`Ywmr$zjx~T3NrQe_R+b<1di= zlR@Trr&FDYhHZyK5yPwP+HVoq3Al1EGZi#flZ%sPq(o)5o)Z|na7XJ0z<`HXHi@fy zf5Is8-Y>`GE7rmp5ok+Q3q;+@HA9T$mF<|j<%$BDVHstI+Eds`qeZH#tH|tP6R36o zwSS0kqWAXlMLm0R?A0x-fTSCPX1YtA3}GL1JL!IKnMwyLB|MiPa+{Sg5?}5tkcyun zU3jZ@Y#biNU9ONW*xWE%RfuirEcJtV3jzYOR#T5e;5{AcvHQiLlp6Z8<;u3!HhZU8 zr|M$b>Q+7~%C=FChNBgV@AW7!ItB`d))KJmn$-%p2rR{@r4X5PV;WWOrhQqHvgFbm zk^uaf7OH#2xuoR?_C686O#VySvP8P3W@%Pl0zh0;@#jh{qP0F@gq-IuXO zl31WkFB6=u(OB$yUDJVoXw$NgVQ3?MT%90)sFHEHW~O}1;;RS+siRBWauN!_?h|$` zXX{C~>IMlfF)_6sRF*lLq|OLuRAHJUdNBzZDqR^QeF8o*<)jcaS8P0*;f|f0OvM0E z<~L$5m;9Q@B>m`()74ow{F^x<8K`Jnp)t)q(vsIu>&`70er^j~>*waKEj8 z***dplHEK~Uq_>8}A-PzOxH8-6HHbp6h| zm*oUEv_TPHqkc2Yw1ZVv8W5qg;7a<5&-I7qG{EavCx`!*ape-1A&$Nno!H*{%f7LfQVI|EAt1gAhA_kO38kZP;|K}BG zI^IKUvH9W*h`e`yS8>X(BQF7K6^%WF`T(HPekuD+ix@2D##n)dA(ha-Q;UEWQR}&w zFI0naz54jI^|{UA&H_&#*T@7zvh}kh`dYJrwD5X?gMS(5P&&YZv)IyvdIv-5fg~#G zSc%{Huk=j_s|cN~DeQmY9GC!3GxA;K0n=v&Eif|e*Zz$gml{Ws6jW50iq=6ic>ovT z-~$crU?ix|%9CyDyAk?cHq)S(+w5T}k#(x>Ci^`n5CEsBzthpXf#77|aU`_%AZ%O> zTQ^D6Gd(*wr~yxyjd--ht7DNJe_xzNM{9F9Tjq0VNkzPuhs=1GSO;`+`>BiAWdO|`IXPOzsZ0PfalOx}#c-?_iy65lR4Ufj}{ zbfzg?F3n>p4A?`LgmIl$F$X$09|-`l14?cTgt!nTfd(6@-RTi&*yAd?5kEaZq-E#o zJ9g9WgIEzPrt(3y4DfYrzNbH*Jo^5|s4d4?sB)bj&X}oe zO>AChb~F?(y?@4tMm;DD_AT&59Pj%x`Ia!x1(w4ws<6jncR3vlm05`ZV{1w_7t0`5 zF0eoGiAc*hn&)Iv5)XAuNnb(AJ(sm!ac#09M-I@Rs<=0uWnmJ5Y80e(x^MyN&r)PC zPqW9pBI$xYg&HOLi1$w-6|Xvfc{J5{y(-in2a51acK~bhfpy!MF&ofS#aA=J)VTLc zHM8~tYGT=l(dJNO13NNP=|Vm+SCEb$C$^k+1mjXnMa%H!JjTp@T`#HhF#MGorb;|| zxD6nj)1Ui4F0EdDz9n4V)Vzy}Ra2fnH=Jq1##em*{Xt60FFt&M94M`rOZ^@)+tF*z zcym984f==BuxduF zWb`dH+YGEJ#@-}W>n&W?R%uB_RKJkN8>|9;#D&b))<^GFmm^}Y&bc77WM+WVKY_vo zMQD^0H4rpC*G{m(922BT{u;)=g=U+&(`gXHN|cl0_zEI^(G>|0010DZ{h zovSqv^I*^T?Z;=)$l;w{P1AVH?T=CBHq{-OqbkuGB}AgG7`eKhKFiA^MLAnEi=t`> zwQw_+vh|H0gFW){?@dIv?_sKAs&3nlOraN~7q~RRz;I|~8lmApd$=~p`6kN>cdUQQ z$1+W>Urclxb5d2B=2s^mel@?KnD5rTIt&9qmW!BEIkewUp)h;G2Luk5Wl7P|1F1O4 ztYdK*f(pI}O}Va$k~r*=iap9-{nz8Kg=%cS9@AdAak;@$A-&4=_EHHFGxyy}xPV`` zQJMMK%G4yec&|7CD^XfHD3peAg--3gj@NQN!F@WHs4fme^t?8!_5lSjb(PHeso&uz zs#9kK6Fv3N0&Qh&VgC8>V(~`|L>y6|Wc8^45gRsKSIzIpE}kq^3`!zVuD>0`f*p75 zv%wyH33*YxeA2Y-8%zcT&5o_F6m3$?w8P@^uitgo8q_wS(~exLt=^#6DK2CAUl^+%1GsHUOUFN0L=BCM`lQa9q?`$AK(*$pWaISfyQ2t_HEH-e&9E)MjeF?fej-6sIEQy9f>qnGuwCeB8$^&-E0p- zWj^+Xlzh`S*#315htxQP!d0`$PVDqAIG~240d=5s?p4q^+`_mS7PGjCmFS4^8_h*& zK}zZVP^<$6T`eZs%G??Og$R%Q9|ALZh z7{3K<)XWqZ2nZQiFi@Z7%{q_BA)370rUA=k{W*%JdfxYBQbj>P?Xk7)Gq0;>RaEr@)QJsx-ak7Qrb`P7%cc9a#Kix# zHXeF;wQ{E<3xrS9Y3;|a_%^*x{sLNMBT(j<{vOO;-eSo3K>g5;yne7cq zyjbtkZN!NDqdtAVGZ!eLJnyJFcl(YD4Nn?UI=G1UA+&nuW0p*~QC})F=c;r&pvs+~ zaGNn45IiaD1B7ZdK$}xoEQN|cNpb%bTDS;bni$G)V(fwv3)1Z9R3S*|1<@}oJFyCa zBn(f7+hx$8%-9DBuacpa@%f&* zPrFXrZ0>x*+kSdI1)32kSVZ`T4FkUSChg$%;%1wOFI4B~zOmG^9I_aZ9eJ2D2S0*M zRg1R758 zJZB$QY#(3|gU(JbH!pCOSN_-v#2>U307Y=Tzo}E8gRdZg`I_tj{H0nv2XT?8Qn-0I zy(VW*vNfbKs1w9Eo^KV1eBp*}n_{#FWZfii z1`SO}?Zsv-^=E52-Q6_!LIVB1F7B^DiomoM&5XYr|1@4R-UJD@9AB60DX%eX>zm60;~xXU>bI9ec|)y1v(n^_wft+d>Ab18WQi+!3uEh4CT*{y#Ts_RAN3f z-dy$o07@BN-_QhPs?E~p=zS>t5$Rapu=N~~doT%7lf{***K_S?Up9@P**5-^@Y)-C zvI+E$8V!Aq4v70xZ1rj<9U!Inwtk`|MlFYyJ21S*)h)wlvUZB=7C%ePEG4!VAL7Ig8z%BOtf4 z^y)L>->Us4nqtW>SqKR%uA5ck)o-;*zCWNB#h`nQg4AlG#OUruLv3=S)AVV}rt&-G zAUtA-Mbc;Vk5fOmPVZvt&qXj=^gtK;E#1YOjKl z=6XxwB;0vkF~&uvxOX9P>QMjLLCY8);+@7ylL%X#9LK!l^XOGUamxPU13EsEJ=K zyB#Q?Cy^2WlAOJ5Ng7CKx-^K5l9stGAUd`2To%+^Nq|S;PI5QeW-QBBWYs21#@b4d zfSoguxuL657ThW-V=eR!V+rpFVW8^MeF~xZ+L{E1ibYbrgiN?uxlrY z*$Wb6+-FFCmFHWd(*YC*VX`U2ykQ#5;Oe{P#Gr$Pj|!` zF3wLrJ@{z~6xDmd5QAhs#u{z=Pj;Xlw19adK8#?5Ue4DF=4%p8hSwTn6A{?#je{We zSpi9`Qt7?KWQtS&Cl7{c^PY9OjjP^3=QG=$N6(EWyV}KqUt7+p{Icu=kb`|ND zRpez%ilT>=b(Wf9UiY{}U9vDJWN(6S>pHII#?47GLk0s4MW}@ZB}4Q*x&x!v`Ik&fQBobX#XXf6L5%z z3v{-7Ys!(70zZKKTm!Pj9M6>borZ%suWwH}Z(Z63%?367@3sQv-O=u=pv2&XCD}_= zTp0kGtm|5K;uN^iRzuezXx9B(c&H2uCU|AJ81R=L{8o-w0a*|xicP{CMu*bx80K|y z-odNaZ$I5`xo8Fr7d*1nS#JZoBB|!rweFn#eP@Ox>6LXq%6l!{2%kZt#E%93!V%!W2X1hJ7H@tr|Dhi(^Z z)gNKl69Ayw|6hzRY`^s9Nd|K*(v+t9RLe z$Rc*@f07-T7UO8)LJqKvjLTj|nf>E%<PLjrA;y6}c% z=0EgYMmA=968-I(s03fcTX}DIqE+n0e)V+97YP}{!5jl z?(Us2w8nQiE_sjNrkj$c$Sdn%5Tra@FF+GYVW0~?&c__AqX%B12?!&Ig+`FTEKeUN zLal*Bn@&UPU1x$*3|X=M*s9C&E%dTgX|7WipX_WlYI-F0W}JtqHKgu*b%`#?D~Z{E zL7JRqP7(Lw7;C$}Q_v6UOO%BgP&(L*E&YE;QKAy3!NCInF^b%re&7*G|5%Rkp6yI6 zwYr)uowh*kJ4t`xCQL=<7!5vllt3M`r4cH`(t4!NVTTVA2{{b3!DWt9Kohmg_vjch*$5WXm!LS4Qap-W+EU3yUJG_si)CY<*QaNXqYwCf-&&T(NNb5BoyVl z1aeD!Jt6>cl?s{{bPPW$)G-1{a-OFMM;R{n1I%|m#4s!pDN`&-7$6~*8MI>R|EvZL z5j8eIucRM%2=s?dpo_716gH(=x~Bs0-})@O^WuB5o&L|DVSR6h$c=M?Z_Sb~K6`#h z}=;<#^Ff!e{L6cKxI6Mow?y1|VsD?*-jA78mCSE!cRw zKwhl@GDRC``B*7 z2SQ){uLKJJ7%ND$DQ&aXLgh2Wwln!Bj9L)vy$j-T##hFi)DKjXI4QzKyz&WJ|K zEg_X^sM?O-KbD650ss=PcXjB!P5;A{%ICOzclj#;aA;QtO*_+8B~3qi*;^{d^}YH) zHx@~AdJ|~d2#Jtr`c*UMxnbW2pt;$5iSY7S*J-vNu@in?(R=kwJh5O%HKnk1Bg zrWtN>l~3wz_QiGIu>`ycOo7be%3HlvzcB?$n8W#rzRIFkY9bEBWu2rSyz*!%9=Tdi zlx(m!{3zei@soeJe6wLazo5;vZts@-`F12!PnsrKFp%AQDm?LP(ky)v(l7wjt)NfT z9R@vHkA7ruusaWw_1+RSH1rcE#MM=g_7kTpCQrEGmx;GTP zCC zqH)9cERHw+^XSAZAlTS7WiEC?gl?L;CerVNt|cRL4M-+q&@jZD1vU!slV~Z7;ezta z=}m8Dt4aK{NJavrG)mM3joyDw5pD%~+i>CZ`8#`FpKu;d#y)#?Fi+LK zmVUmMgTD!y&x{?5n!wZr<_dte02|OLx9~>X!a0*FR2Ne`RHZK8x_Z(S>b_#B%>+3U zzCenklz!?tSRMJ8Em2Tte7t@DhRymlxz)rJw4K!eO6YUkR#Ej-KK{!4NI}x}O^|QN z5unRjrB+^a`j^sHGr;{0FLjE$o-rg+G|AG*-=eGZ+W$e)Uh2tj0Qi;C7A3#NpDR|` z0l$@x%G#$cs7mX)`yv^XXpL#R+(}Dtku7IF)~+up)3elv!Bo@h!Ti%X|etZ(!N#T_-a+mt5*m_5w25 z3r+}ng4UqPzl4IC?p4Zu{dvHFYDZYX$D~ua=wNXYe0CU|mVJ6C9a{0u`hnxm7zAF~uB8 z`!fW_dXYzDt|HQLB<59-je~p^NM@bq84%YSRSwxAG|5J zPbui_-fE&TzJ`7?^`7v_&C?|5wUgy97iT^I#9D(X!hB@R6yICio_`HfD*P6j+I zkDIc{7N;Y9g4(25_K9?g^n@2w;L3&NOdeRVC0e+7MQoLHZJU28q5Mf`R*#u`su(^n#4wSwh}VFaU*k6s7h-T;SS)jnAdkm-_zI9?v;@hRm-e+5=jZ@zk{zI zSPt~7t}JDOE(!NzI+=QgDPN^Wk5hLX2_vqCKN;h0IhoyVv(+*((Hx}JSOip1Cuks~ zf{Kby1WgH0&zuz`iR;{v`KiJr^ZrFLn*R1&lNee#JU=r&Y?Esw?jVKofvxUBaJhUB z4&wO(3FnAKmBA*2fuBzU5P8okzuH3n6pW?M4#RWAO!!zB2koAo5FIIE(73;|tC*5@ zt;yJR)@6KWQPD2@8m3xkALzCmbW>~GhKTOY*N5HZ-`AUMXOU;xen8Ltkcj_}9v^_9 zrDn-9og3Rphk64WPnJm-G-iN2%IB=#D<%`D945Y2IU+yJ5KB--kgNWS)#Bt=^Dk@z7y7XKl`;LrpPlY0(>$#xHD^W!i=xXLn9oMwhCa6g!I>+y?>n zCk^Qd^Y~ND*(-SbgUz6lj=~EyLRxw%>bJ-K18+egYQ9S=2V+v;l+(-#4Id5lgQ~-R ziz<3LT#!{INBGGtr&m=t*DE8d@9^i8S4-&m`1qwDL2Dxw-!zi`3Z^Q$OI*R(yU_U2 z;Fw&WZ{ZKH(#wV(k}_xS<@<;O{Hz@dNiqnjMAf?UO*_Ae_Yr(OkT6(kZk_Lm$AaFs z&f<_O70yLbU-*s^NbmDI9>n1RY6Dqlg>4WGoU|Z+t6?kKOW*fsdEpI%cuYg7_=nC< zmPxKbylW%w+QfzR>P9@Sy$v0A*&Qc=>}Z4ye>ioJJb%hiE3iSIl_<=mT3^E88e<;c zTeh1?sg|wvOf9{9=ykhe0YY5G(JkSS9%AvJq995ZRvwuDTxVWTEqVD}2qo55&H4|- z!R**oU6XjVwl|}uOTuDr)4X@3K%eZiDCubNV5ZdD%a%U6?2{N7co|@IIj)Z=Ps`3f z*U&KsO`=v3<#j+eRlEYLMY|x*^t^_gB+*h^UeElQf z9oMs1P)6nXQ&PAYl;$FeR@aN$0I%_hb|YW+-5BTEH?7%ttraye3!H;las<_mI{VZz z6`p3x`ez_Leu8?qV+4wQ1^>3uB#8;z+`ts{(9R3LQ|1O*0INxare4N*G-M4o8Pmsi zsLpGQd6Tan9`M8&)PRf|BE?{$wmIM8XLw&yXlr9tgT$K(>ih%eOtduLKi`Udo7$f( zy#>!N)GKu5;;EZkpIW4DRcBQ{BcBz!W#{mi1>2-d6KTa4BRKmDP`nSqIglAXJ&9pz z^CeZf&c-CJWy}vuk@-D68}EjFxQz@{3y--yrS9hDa26$a46yt$T)5!IAnyQ$>jaH^ zX!jlI`Sfw)n05O|dDpmhx7sPG-6dvQu_MkXtMVy<_1BDG(TfyNu&r>?-xWqX2H3a` zqFoWR<+j!4x>DX*uo}yqic?-L%`XFWBL%plEq#~B6+o>;q%m<+6QTMHs2VAAdrN*3 z9c!hym0J`~rVO+M;{cSvs$d1JSR#6w&zAK7Q)|S>frf(I{0(V6I+g-caR`9^#H7m> z3{BrPHb5OW7NSzt5P}_3)^|Ggl;^(_JLZ2KyjMv)yrnm;fBq1V;ZWaZ;#a!*ATBBz zrd3wdi((tIZwr6V0!$GM-X4Q!1ca6ug-S?xa~WKrFyiG{HOH7+uTXbRozVIXE}@VC zxGs51v!gKZ@)+o#gPbjoq!tU|AggauhP2Ak9etf6sTkMGl3EfA6|+S}{vrMA<VWPLD%=rYSZ2KpnJ4^}iZltq3@_wa%j^wy2YaX3O}u zCL{G-%o?fMEg;27V&Kqm%djmk)pd%=TTE`=?eCVukMw6yP&F%UUMY6_0njU<=~g6h zFT2_Jbc1bA921ZA#Jxw0FYn|_>DPV-v7F1NE@LXgt9PCJOj@+c0pbZ2y&!dO({BEl z!*a^mhgf!={f_DrVNRTl_wxs!mFEqbXgUEOgwVhg;mnu&05qK+_NL$u>qJ7Eq0qE? zvQA)-B&?*~wfbN&_N8cg+ejiIU`!Mx)A&k}Re_+&7_bH4wavGRa0yX3={HsfaDl<} zv|S4YLv#%9xZBpPy>1a7MqlVCz$V7QTnzOYz((YYh?+I-Vhxe`*hCYJ3OZwr-PGndKNQji#PIQ*`$IAkaJlTj4@=`= z%*A9%d9{BIYT%o5!;ZI(07XjqBB@#Jc*EeLo_&U*d2uF)UW!%(* zd2b_Nh(>LWZh!zQ=(qat3lo2pH9-1X_9HSk^^!<>&5cc>VQIVtK=a;Hd+o_2?uD+y zgeRp?;2E^snKh?^ERCRegizNsLe?3|)(t<$fs-VR%9T_-THXI$> z{DLrd+?X!BhbK0l>y@+3)-T}Mmq%g|cn0p$4j=9-ZvAofmg^dJ%d5J>T zQP4#--R=m^!~`E4(S(o_yvSw!P#)8>=@u);!{HMojL@ii1lmsl3=0EiNm$%XA-#WL z#2fSv<*|W=Y*)|@G7UVLY;kvSTk>VkKR5avEsV?t&;^o|QbKsZYu>Y$ogcpqa<) z&26dTW=pR;9Cc=IP(X|19&=PtovQeOGc?I^oPF$a^Pr=s=IyGIL%vZ;lr3z1P!nYG z>Dr{6M_p^5^+5lWT}_m_aK5w?bn0fKxS{xmwNp`4q`DkqQ8wv6|1SU|Ahp`84!(mr z@!sgZ*mFc^Qlw`{SqZ!cLAIp0$&6w-=W5(=Azo!~suo;6(ZIKE)^nqXB0Mwy*XXyx zvX7R=9)lC);n)!oLqiVX*DOt=G}YKWm7nwBaq+xT6hLKAJED3NPamI`b#?5W<%|`N zR^QM)%h9{jHZ`+t&7kv?j(gHYAyHiAVSwsEa$;=&A3ah|4JSUDrPTCFaiUt)+?;1{mEFYR`!st&I0HH%J$UIs+6m=w@xdJWa`qrYQnde<@*Qv-33ihH6 zc+;Np$d_4dOAC}8u51Si&=_??c!s|l=?owkZpBn*Q^GGR8R1xNiJyj zG2~S{aH3`by||}_H+R>iHD6-&Tci+WAr)M6?9dxj5!iuhMs8dMS;OUH>N;=z5{nPd zhfd5*C1#%pOkpiE8#b-lfhaTBde8fC$G-St2%Gl!+0I#?uGJ!emd9R-v7R;nT1heF zS_;M#ie0Y{AB#OiFsqQ+)*9X(&o;y+y#oL4sqRR&m%;Hgsj^+H?r>DN5!BYFm4o*X zhd$^Y+&{qCtJZ15qAEl=5G|YS_4lsD=AT9e_r#>-^d$7W0p(8{AcoA!&Fed4QVo)B zDi9tXE(I>HzcLqmb+OVL-@D39adrmIb-m?RUoyGFq1VIw!81?yTg41Aex1*SkD-H5 z{FBU{o|;R4zJ6j=Zu}<#8P*$oF7G{R!O^d_?<^&a&&tVr7xNcH_%zi0FvD9e&bEWJ zeCR!Wp3yg{0G4IpUV_@TAf}FA`r~b-Q=M%q%$JTLUrY*bbD1mVKB9lNC&*BViz*=G zBcVh=T|VLECHYX%YYjALUJni;RN&`moF3&+wf#|9oM+lt8GIAK(QXm0g6q9~dTX?* z4B+tImz+MOy0rDjoAX#!PEp|>@D)Evvxh5-1j@!ouqO3k5%@wP*kAg+r5XKs>blx4F8=9R*lH!%S%J3Kb)XO74)KbgY()`SF zdM^@=rgq$idz{huB7QduQ`lHdv3?X|xHd=P8n?ul@RV)CW?-p9bgr}NGqyyV@B$6} zy>hkpdTq)^&D;wwui0#nFsr^&fG;Uo1ywh_?r9Jqe3Pp`YKLi(-N1VpYLl7axU+a33S{qOVc9Q=j6U?$@;7q-w>36`u_q@HEq0XNoLl#O!YRXS<_`xN|6je>&R2ON$F?taXDDzYi$lAEjd#5_c}d0}cH z%1vNxr_SOL$3;x^%31Utf=0|)b8c3{-xdU5#8qKbd{;9)b!D;s6vpn1P(wxvC-%DW-i%;sI*NGPR(slpK#nDw`IQ!tKJ($DZ z<;PNR_ClwltD+MnyUBKUS9aNvrKxTmz02qhhef+7#H>LZzX(f~+F7=3$#zS;S(eb{ z;PX$}QWA-Uf_SkWk)4VSFltwy*7T(HWZdSdXePiF2sx#5eHhOgDp- z_qI4M`>~Kgo^EjL9Y59aWdS8e4+llfU7X&Jmu=ce$+=9;T3R6m;janX{+KTD^9ClY z7u|Jj9v-NKbMyHj%z9KsR@lTVe!13Awy@(z+=HU4w-1?No-HZe4yp1z= z=Tze4g;bUV2hrv$?1|gF7~2YGcq<(;@a^!Z-mrPzA+Al`c(R{_EX$9Vndg~?_xN%t zg;l9m{qD|7rT3EIIC`*GrOa*W`>L89HmDpxhwYq(1&(k#b$xN;b$yA}beo^k>TYaj zH-;$Xy1g1HIHqNPirw(*lQ7n{T|DwM%doD|keca=h(~^YKI3Pc9Tir(Zxt+~<;``ZR1sKsOZ(<-W-D}*S*w|b z(sOC_3VGF>h3S$qJaAoUpPHB`oc&kEobYl5Z?1+`)5mhIRnqVrwI~`yMfuK_z12NMB1{OZo99{1qG6~;RKUon3h8EWQWr+(X z()&u}s`5+0w{YSa)S|f+OtcBJK2I5t8=ENERbQ)Ay^(BiL>D`$I5O{AAy@n23 zzRWU3Ip{spY!)k;t*1>Ax=4>yfx9#X3w^HN5J}?_+%hmxgKz0~x7Ri-pY&}cs!qegiAHD zDDo+okJQ<&PnNahJ+FxE+L3%?>8ju3Ifh+-zO}5_K#e#C?zk!qG1Odx#i*jQy7J(B z`H_1>c==XKv$R$PfpbXxn%uM~ZbX-S!MWn(NGbe+r*uw?^x0RDZ`sQMbi_zU@TW=a zHp+EmO7Vyyc07fs39L*vaO`vcv#761CPZiY8blAHRzLwCk+=P;rvvdEEt9#o(S^H~ zvU^N9oDI)%LdLk7IJ5Z4R+`p2>8!#twXGIMJ0(p}B7ADjeHu@uI_IhMQnc8?Qe09v z5i5yKiYhCyAZ<4@Lt4}5BVBFPT{}lWN-rh4G!u`3!{v2fn~Q>j+I{z?do2Gy!rnS6 zs;+$>7i5%BK?G4m1f>x$2ti^frF-ZWq(KlwN=j5f3{pZ$a%MjCtaf? zqv~%TD$i)FY6o%XKvfs>k7ezfGa*yE-svvWtAqyqtv}kWQn8x7ntIW9IF7Y>VSjV3 ze!bR1_c`X_oToBgi_^&^0`IPNntQc6d$o$TX5!(C>&{jIJ34~H-iyC_#kwjz6z@~D zZrphHXsKb@WOp*ol}A7G-bA8d|7Bv^=h==C*Q*EyCSzRlqnek??Q;9f3D%lk`1bEM zZZ|$cPAd57Q?5dmZQC#Iqh~K@v0YLg|G+>?r1umuIZ7TP;yva4gs5j(Wu^v zU9sYm6t}SvPS26nq8xL&8y`!~Ra4>HNnkuSwp-Mc#+5kanQJMmZS`bll~rX~B5zkH zsw&GZE72y4rc~T=5iCxMV$Yy&9gI>D`&{N0G>^$P)K*Z;zU2{P6S3+qgAHg(A{1%3 zVX{A2CCn%eCiyHgCr?f$C1$sr<@$`h9Gc#wHjZmLhcWcrZ4S?HE}<6NIFdxTn`61% zbpGsCgr&98DTU=9=UE>J6c)>qu-I|HA|RWh7en_+wuhDs`fxLUXvJc2DBi^xK;i z=cPBiAFe1)X_Hget{2dzXkdN{5g0Mvujg53q7iXqyEf}ZyTFgH0tP}@)iCP{16cR9 zm42sH#im?V!q`1_cQMR+YCY^j)+e21Rb%0bO~&zK2R)4o&)XGJ3J9`v)1O_s*hv-B z{Bpq1cYDT!E?n(d@x?kqmkwXy$LEQ8GU@P&9_EgAIF)Btp0O-nbA8Q59KmYT(;%zr z%+WA74OJ=_zj8BV`kPC^jw3AM>)2Kn=mS2 zw&|GpZ7l9x0bOZsmKHB}dV1&O@=wj?>7F;;lA3}Iw-r!M-k!Z$!s>{YbM(t-HQuwQ z>yMtDbs#~AhuoGT>z2APt!HL)TxQC(!l~>8TMbUPrK}J6-IBkXa}0;kJ=fVNBwruY z4+(rEEHmf~>07sMam1Kq#wCPg_e|GKFzbOyNiB(?1+1s+#N$8k&xc1J#%qBzj(ieCbH}W$p$ zwy{ArwKiWFzROl=iZ%67aiV{vt@elH3kjnM7jPNvZ$2sUYnkWVD$F2^V{besNqp|# zHF-9&$K1|1eXWxyMdb+Dv^-L#sE>94d6g znf-V#A4cXRH|2Z1Pwv-@VvYcJA^lN$CmQzz{p>Z{0>0qos7b}Qx0q#Ht)ISINin`(>nZAh~Z67lbiPE&@)L7RJ?jNJ*uOv{0Fq(gsDk%XSs2A>V%oU zEZLm$mDbiJ)9H#Jv_@nGmY&h_#a`8&BgrX_#5wifdgGoT6W5KE5NIyAjgEezEES(! zwbV0H)^obcB}b=1UtmbTM@LteJCa18oxe=~;1n5ERw)GW}N4m{dNM_t9+Xhv_f+Ul>X+#8^dao zq0v)TabsK;e`8g{voEM7WoaHRJL&eRp#yz37uz#_dMtEDqN_KpLg#&F4}mFCE((25 zNY{r^et4u?pzuI5;Z;4;!6?etP3AihiIae8*@3R-oTjYWbo%S1!%7?Ns_0vV7USp9 z$uvYhyI08GhBS$c|14Hae>=2k;ovxNs@!ujuWK~Vy&P}(QmlmR5%=yE};}ttFC34-)u=#f4!_p)E_@|OKv4_L+A1a(; zVQvU$RjQl1^*OvYfd5rjrD1pB?`-*rCE+W?B;hjX58V7Z52K$Q@;GKI_XPizQA?<~ zkV6qaG0>VoM@Vb*d~$V+Wd=|Ga@XXxq0PtF`m>BKA6jQ|arXoMsgnV_PL9bA9C;pm zq90=R$1?e2%ZWN;*+a3PS_o&pyRd$9K`eO!tb5X;;=U_K9I}lbSqTu=q1j%!>os{- zlo4Te9UmSIiP_qCo%23Bmg?=!a|4O@9}&sv-{MoN9I?yQZWfiD!6l%taCrx0zO<1e zRPi5A6!izoo#ac?3DraS1`kR+=Wb15?k3Um)>*De0I@+bKU(<9fB2Ws<|62-i~^_O zYUZ!vnjVUl$_^xrjc7HU`1nyGVIZrhYh-2Eo}Ipp!} z?5pUJSgS;`6E-$VVkvi#0?ym9lc|)#PQZeB{iYe%t3V54`32QVj0bBB=KO&k1=nkb z{Ikg{8Y>+M1DQ_GL%Wv7@r%N70A_y>YFd81VD{lrY>}V%Z&rUol1axk6TK{nCSMm7 zDXn3yo{u_h$*Q(zfhuCgcK5aF6iCSfiWf`m!cOrAl^sgf*`SfP8+)h)s)!tgA7C2h z)Ro3AGI&RM=_j)@(v&y_^kPrK{n6l&_(32nTCSz_ld&gIwWm30N`M#5y-xL;v4*&D zu6N(fLwk`Y$Hc{etIu_ynVjwsp^YWn`$26O5I^AGW8%9_ee*lQnB4D>LTM(&A-7%+ z>{t;4(P=!tBc^%n_s=;Xj2D0)p4~f1+p&&dk;-!#Pxcj~$V; z+^*-55&JchQ)eZaOH^tg{y z{?nth|0s96?P*mwj|PX{1EFx;$E31{FPZ*n<6aFTd4gn#GhS%u{+q8>_P`&?xpXVP z+Woq9D^omVqjNx`B3nhQ+pd*>AbgchTqMgZ$!d@O8=v?(%WW37$ry``PKyQ+)^0`v(h!IMZz3-5o(7;FL6$v6;8)$PB?A z?~Q8re(BABBB8fzCg9*&7(~hzLp6t9Oq+zSm2=7ha%%@+yXL zF$_kxB#Lp{`UBRiBWld)_zVM<7^g2&I5LxV6UeD`{(ZFl?#&WFhFQ0es<}5}uw?&I z_N2UdyWENY>ZSw!N_$SCKTci&AUNJKTJH*mwBh2v-{FvDq8M^Z|MOKGN}a+8gJ_JU z@&V`i?AsQ5C^h8_AANHAFWL0pK_wysR$_HaK+o#`UU16+tdBxw$Duy_ccE!u^S=m+ zvc<-5JJOwAf$snns#@o14tV6JH zXib*PS=!oDSN(nVGyX+hJzYl`jsxRX03rBw&*qYaS^w3Hu=2aHa3AyMHL7^n5!&NC z`eVSJYR}v4ryjdNO#OIEX+)fHdzUD2r_R?~Ew6$3G64mx8{dKFyf7qI_&6!&M(r8T zwwMht4bY@w$%#JKZs1E?Z+T-}wr;gY0yV&lBYF2!-Ndq(@rCh{XF=;Fb2{wh5ZtaPi{vv0?YK@(O<^Gy=c26`42s zjXWP^r@>(kQ$n0K*>rFg;#c9kQqX^bDu#0ri&iV{T~bkn!?_5^Y`9!vnbGgZ&c z4A;~kawW5xrY8oxky1v|2$%G}*ti`+;8dL#3)clx$6Sp~zH)wfP+z<_iPS>Nm2fHN zj4V<*Q+;^#x63}K%%ueZ^;?|uH6k3bk*An$3k7RaA;+!piEd_rih5B`$1Z3^ghH?*_~5eul2k$7dP%B7LNxS(y>jg8-_&{ zZsLP7h4GavdLhs&2FJL^MQxTb;X77b&OcAV;~OK7FI-ARhbzQlf9J{8)!TdiX1!o# zyVT&4b))o^-Sy@$M<)_OI-fe%%k%!Dw<3)2wnQHvR76?m}m%#P4W!v=1gxS^K$~iRE%*1UO(M@3FPDSZt>h^A(O{jpY4Kl4m@q z&qZnTJ(KK>XciHR`ovM@;iRbIKRj*Bhq&UYIe6u|sP0h2H)S{B-ODV3RJ;FV|4B}^ z;#0@Uo;_`Rj71xBBLsyri4ia!D=k3iq|Mi--4{I{Z*WbkhAhDn7KTt)PgVw%k>07} zvWt@OV|&ZYAMHnitC?cyUMgIi{JD81K6PSHc^$KZ;bqllRVq>ql?iUjGSKd7OI`dJ z0x!sOWyd$=%1CNgP7ZZ`U*;`c-426e;jgX|a4h0c6B(TFK7|)5CNxIpVK#vi2sJ*v zHY+9XPz`9U5~%R$a$W98SKXUaAHI5E#WN+F_ah7)r8>;cNX<%(8~qvj?Ph+6R7D_# zdawo$q4R}W*XVie$u*DZ49WR$bnOfq+(tCaBE4>gnSiqpqtgIuMDV`(N`j-`rOu&O zJng3Hpi)xku0y6xcgxn#YS4D-C_0K{d&Qw@%MzpM;dbLmm;F{v){*-c)a>YTsB?!n zmI_k^h`oU~XqzdQXxV(tmz<#gf)k~8*3S(nW1Po(CV2$;FMR|L8 zXUqGvb${#|v|{!rSiB;SMW=&TsEuGAlPGInvh*B>;-)BBRQ-AI10IXcdw z|KLG+cnww7Ie%-9H`Um4liH!N8`!SIQ`4X#As~q=kIEDJ^`W&M{gc`thG&<$MfSv_ zvkTM0)_nfU8Gl}>cL{{-Yw;HsZS1F&h6W9bu--I$jfy<1c=zMOjC!-#vElCB38+Y8RHE|QG*P79P7T(t`GJxgq z%1=CVD+jDyVdn=$RylJcQv2J6m%ph)r+N;@tRX4}Fv<<+;tR>GYRhxeD!S1!;=WBL zhp(V*OCn2Mj`?j`zjS6jm9JU|_?`zynvQc|grfR%<~>*b`d_xoC9`!jROl`jsJ;x4?0pa?YYMb<2S=XLA3!h}MC!3%qXB)fcYh`iBq;|Ilc9qwvU?X+ zi$%ji{Urus_IHH(0gsS~&>Eun#yCirOz-!olR{{{eFE+=%U-)D978APk^liWEqm1D zR+zwSc4lHB_yt;GZ_WOD8^~z%bR#0{H4w2kBym~K5JONrcvh}7vL|WL5$Q-WNrMEu zx%C4Zj!KRLx(&QNABxL{@8{5#WM2e&3{M)}22T(#B%cK`3y;^)!~gW5+#w>vfQa7= za)^g5=p=({fl}ScHnFq4*$;_}Q-{(CTuX^EgZ;l7q*KHNL)~RT{1Yj6==Pg*)*mrI zE&2jv?iJ|z{s0QzC%QRjjm;H=K}WB#)+g~5o}MDYwyO2dKB6)|8vU&{w&AA=~Z4eun5f*kEC3la|-m5Yfzn73mOd7SjthVjyu=wymjjoM44HfqaA_4u0h- zZ;m;4m;LYkY?Z+RQS2ggof!kJ1mS(m^yDR-GylgJA5DyS33OwvHsC=o@c_d4=JlQT zWQbg}kEe8i`qxu>fhN%Dd!6C8eu32ER)9>W8sxCG_PYe}s|1>XL|1{!bz|?}vIx0! z4(Ra3;L6htE}xa}vk})tkQ_J!&5fSZF>;HVW}vJ*FGZ6xZfvG>r9@rX!<{wFGK0?LsFrs;|hb?`6akqYR zBB(^(Uw*Ln&}eUD4wkMLeiiKh&V|F9V$3pRs#Xk8UzJKg=l`4G}bZPqL?J6!`fm>@7^7K2T|-7dl(t zYlLF{l_&DAT|w!mn`c}VKy?<}U=fYxy?R<5n4=$NYY$TLli3EVjVqtW61e92v6J@`0TbTLEqua={cIP}&6j+}DQ#y--Va^ACt?`atCB zh)`#Eidv<9pC7f~U#Y-TNhO&x08?)0SUx@rg!`kz??Wd%f$CB6^V#)pEOM5wBfoC} zDfa_s+$(FSFBMK_#DC+hm`a_pwDk3btyGkd4Q(Hx+nsyuznZ_bHL<379ijmKH!1a z(9gh9{R&fh;haxQkmm3ZvaXP;%jY-QV!aw)3vr24u+whedC47`NiPicX!0XRmOH(s zKs%4P+}7D%EH$ItBZ18i_Jg}9_ny=n@wGiiu5F@+0WjrY{1r=uJ3=dgF2|#v>`zOy z8_C!<^-%(Y)*zqiIYfE7w;ndKwenNgab7G;Zu+W|TT6Wp%yiN59f;NZk0w?M9RBTD zvd0WAPY|K^$QEKDYnXZ;(LQ=1K+#&~{qHLQpTrtzb#20B5lQYvLk)#%xWgv7R*YkZ zcC{jia27xn^JkFt=O)I=^kkG6+2R|Jpi8*FMj-<9+8R7A3Tj?`Qm;oKv7=6 zqoyGuXuo~`#NFb;+-T`kV<$a&pl$tY~%{E4OwUw~ooW^uXe5 zGzVH^U$eN#E{nq(T_pcX(!=l`51#(eu1ZK&?TE8#6o0)cm)t0 z6dYrrg|34*n=HS#wq~kFTmoXXV@aomVi3@djmd5qg$|bUPiR*Ce9Y)+{Wv1cxd7&( z;j39WIw*O>R9%1qL8_fadQ`KEmn5+cG3I9)e@X{;G5=?xe^mh%C#J%a{8xQ7e|=KI8QvGL z%expvZlw6Y@*d^E{ zEX>$U{FusgO#RKtSg)h3W=xa!5J!@ISl|`Q1K>-86V}2pyL4G?Ojuad6DPig`Z;*j z$F3f4fgPWJ{HJy<pi;Vfh~>h~uZs1CcfWQ`OvG)exTR0XwjP<^ z^8nFfQOmpMeNr!cvV_DoM>$S#TBF~-c(C2dK~&B@>I6!%ZqYvWTArNuC+w!J}W1j*21!;k2bV z1Cna1BABqROW9xTc=A0G{sHN$b;B^tkbu>X(J`XE9oO@8sYPPfd(KftY5#tZWds33 zn^!lOhPaXW7YG_3w%q5C-C+C$EI)0zgikBeUNt13s`sld&ygp}REjlk=B>n6feM8g z7nd?9aXq^GU8#9>R$lmO=dWDFp{vZYR1Ng|nGc?p@=yFBVmjmdDMr9{X6D{VXoscK zsGI3OD)EsrMH63K=|=JW1yWxM<&p z76@>L{z_J~m8K}bF7=Q~?zBHd(UKQ$QluZ36l4)jC_qzV{lv{K954Dvt@vt(j{o2$ zP_-Noo0v7cZrtKf2y9bq>oae}8NBurmrPZA=eX?sa%X~C%>#>x4MBV=ZRP_fVW$UE z6`0u7vOb8o95^)e3sT=69UvhlswR6q6hK9LV#nZW*5<;}L_&fC657ZN!zEOk7-sx)#TXPv>$H9c@(C z3yIFO{fMOwGR>KO!+d2g&dJgHPnFHV3fe2aK6V086|M=+{AuP(DjExCBxMPK>^j(! zK|obWD@(!Ddb)3!&Oqsc@_?D9{)D*Ija1B_~h{~)Wc~f4j}fAZXPdu6tZ>l zxLU0G;m=@=WcBAZ0cP;>bjN4J?&#{5OVU938H?67GR{G)TteF$+5Wzjandg$Gx!oH zg7zVjzS~re0sfWsV5-w zKa7L~B8q06e=||wD4u5(E16>^D_J!@T{gdFY$mzTf$k{1HRy!xynEDEfmEFxno zxF6=2O%`od^Md@_Igybqc@)}Yny_2P>03B=rQQ;utkX}taC7gXOo1c_Wc}+;0wPic z4JC@$)>;g>&Li=7#2yLBP=anDX;KE8pZj2^QhU+O0F0E>dsk2gqzt)Aa=jeU2JsSj zfZQ#jVoEN2lk9RH0F~E_#{`>BThKoy0?xZsfGg_n6Lt28ugN^jSGi0 zF5F+Y*?uf1cT?w3r&gUDDkIm|4SQSWuX6OO2q^!8lvtgJ*?(kIx(4(3#%KJ-cZ`hP zc}631YUY<%D%zZlxzLzwYrW9;`y_UT<_X}912cf0XA|4Ym`6UAp zEOlYf-$rkOg?1kBvKZ>h$YMWc^M$FNcDF~y8YVg<=Gz$Sp6m71tn!n4u_hdC3TZh% z&)#dLS_|^~7-XO_cqQXPbxY5$065KuOQMYmAd+W^Cnnu5*3q+@e?#0q{>hg>s}_jn zG*h=Vu^W4&l@VOar}Fvi7j=M)_`cN79J|qMBf!M8qQ0|Jc}V@l{-zZk;56%honsby zf{@C=6+PfyzGhf5qnNy=CsR7(Wo5pX4DD~Ek;PW+9I?P%vU_dx%U*e^;laf;L|-Sg zzYNLR{d39?H%TG@*CBdmFPZv*jKiC@WrT^OP$4MF4$!Uj@G1^Joot= z$#InE3uzk4Cr?NjpZGBa(36Ewlak+0)xw_=AX8xFGL~n}dKp438hqcMNR%pYG?*@f z)S9MK*Tv(`4VrPS`JW4U>#MgxyCOE>YQR02+zETQOq+^_uCO-(#Vq>cCJR#0(t>(ld&62TD5UD zx=)C;eHREI=`my)Kd6CHsb;^^9L)m?0FgMk5t1=HizX^PZVBbTipj?^35uC6_XjnH z6(2_*@PB_?x572M3e;4BJ{zv9K{fzc4T#}`G^K5k8|*#gLvr1#mpPa;sM49Vs43@jT@h#XtW*Pd;pho@j5M%=v-u)_KC zS%~*5v0lhZ)nDfPGvuJ3{`tK(^}^I&jmT~zp9-a>;E8F)qC5FCYtw!-O3>=oVb8HC z)#@)_{d72$TtT4G$REr0wb^iH4R|&u43M8U(yc*?zIX7={om%Asxyf9Bv8kT5moE$ zFR#qlU%P7j;g&y2)sTYDXmb<(^33Q0{4SkD3WZNE+1W*jn|9%cHa$>k^cnQCdaSxU ziUlssmNS9sRU63|f83DWpU>z#77{9S@+(v{pNV6RezA*$$s0e6{G7$(V`uH#Qe{c- z_1-RO?(iK@os()^H$8etmORcIoxVz#qZoQiSTJI>&ejgwJaf@8zAc@Px7vdGXpL+|ukTyi3v;2zj zX3XN&%XJ!;nN`ct(>!OFT+*Un20Nk%;ET(l~9wd_iQQ&)^H$D zK4PDS6uiF13-P_s*Eou_mxzR12L_QoV<0ix@Lht}W>K}tgh_M%;UtU-y1R}hfwG!1 z_ru<;bx{@W3yzBuU++>9b>^Ec7uo7wA|u+r$S5Bbf$yH|2dOJ4#n|Zjq#)|OzNHDN zo$Yk{>;Yj;e` zE9qgSMKnX>xgQ??GpoW55ZUq1kI`QxdW8Qx>t|7L?dhL}^S7nki|>rQe;6%#x4`mEYUeu?xLCIu zq`zy9?b%vi9Q47Bca_`quyG0`(U!-aXL2uL>$*Sh6m`HF3nX(s9!EIuu$@;7FW4l+Ae#bHR>%F~Qo_P@W z5$5LI4$%F~JfQQQA1Ke*)J_6gfG!DsdX91qsQ>m(rCR@`2*QTY~@*<{F3z9E|BwsqXqUc=U_z2p|GN1TpRgc>IHsP(nMzmwj28G7ZTu>K8T4 z4IReCsV~^V#Fz~e8p@~OEvxlm?2;(bQEz0wYSM?ZU=&gaJ3!oZg2^-|@tnbDt=hx5 z{t}0~L^IJm28Qy*2@n0~)LQCcPH2kF0`tkW>a_Ts<(c1chW-;@KZo531it?a}0zKgvp-JnVlh+f;efvwQ3>E@hcj17+aDR%H5Op!mu2Uvpw&JCi7Pl z!ylpGH2bWigmNXyGQKicgEf8&6E=SL!nuy~ns`!_tK$?yTY2hTw(m2vW(5r*p?hmA zg3&__W9ktxC8Ox~Lr3W&>lv`s+nCoIyjDGcC z<@Tqir(f!XuW@ef}MM zi_?1n`;FG8QL>i)^Bb+@D?BFE1Nz0bm{E57B^hfm*6$LMWdyIhq39x=u0`%oId80C zBj$&}iGM7VFv8h=pwrH&3b zoxBoI8VX}#W3O8EfpKV#VhnG#rFwNrRfvX!o=1l<>&$(1eQ4}6SVsk}>lM;3lQF(ydl!v44IyDYRVF`9o-4N5wkOvCv0eoJZW*}}_@r+vP2U1%QNiso<<8(0 zF2aQu_$`~faJP3HJjA3HH<>y$15Td|KkK>@?nN|wD~0i@GP#@_(%AA8&QsSc^q*Tc z(oPLkRYIq>40cqzPHWf=loiy_T-X1H6wJCOX4l%^2vy<<*S zc*QML22JO`Bh|W^Age)jC9K%*+#v?ho32AI$Zwji%=WT!y)~~V7aA_KRK&qd4Q@$| zgU417*IRG@&tpR$-DBugR3h3x`;aez(XlS{<1mCkCvmp-_~lo_@1k)b1KlH z4jp57)u&Ngba0AyuJ{5 z(qD>()Qysx(unJ$X7Z&Onyojv^c(B%@jS)YXXs11KSZ;iS$LNkmEB2MmzS_v1TUUu z+7jh>!2*sViG_BPTk6-yk8>0C;dIX)BZt`FK@Dr)M7ELz5+@)V`9>n*K5q{uNJUWj zxNhZV)`)GZr1LRL; z_v$PD34i_vBdO$A8 z`}c=xmisG9cHz}Cd%u-ox#ULBnrTlDMUoA~+7EHBM+~>R!Dso2f=mf1%T0Jkjl$ZI zH2zYw$&ZWGo?l#+g2qzaAqKk1TkhPl^XbkYp3WRvJoHZ*z(a+7EHH#gas7K=Xl5^s zsjHEoKrbCJsg^7R`lkFBSgN;2_e1?ZUqfS%d<@g*=dhQP{GWfaLSUvs$6}aopZVL@ zU}Hi2G*@xL`AsJ&O*oPez2H(YqH1_tTjJ1fKi!_b-Vh>+}pqZ%R_YjQi^Qu+~p`v`D zxCyWoCjr~ed|3a|qK%+T*Intp(CKYCgjfdL14{#)+n8?21O6mrGF`^OQ;3DA%4?CR zb`ju=BM!Ie(TC}@N~?~{hpt*uT&l>mKO+v`-Xev}8kUNrKq=lvI>Z&c3yWzM-(J#0 z4B;Kx)8yyhE7cr~ZPu#1=P)x+{`PBv@YSfVfH>ZU2_sOok=C+o5Q$`_OxP82kNua! zbpeJ}&X;3)hlxnOfX58Qb#P5HY~^~6RHZ(Jow0&~+pC}J0O8F6E3O21lb?N7pe|NE zUgrx|uErg%pj+$%i&%%x&(0zSKE2?&Vk@!P+cFDg0uA2}rl30yh7x zkn0LL4cXtF_#%(OgMlfw*rC5WY8^eNp=l?@#jJ$%l)6{$!H zp}fpCVJSr*CAX>8I>fk&#N7;h4Pvra!8v~CUaREcUS!MIk)h1mZ3aWkg^$Y83lrPt zv%9?tW&W66Fh;NkUn*=B6id?uH-Dbno$8A)uJY`KW8AF4TCLdT$y*J*{uWqii}QeA zs)`jmGQS2Ov+G=mr{DE^v(~nm;BTd12XYc*Cl1~gREDot_XedfvMfN#c=M+J;K>9( z;u=6HI}kfqhYb5KKc`|!|xe!zxY5NRu< zps)LlE!$cv;T3&>^&NY!^klTMWTK++Y(&#%7m@}iYXav$;#bRLy)KfVEV`OW7ZvIqP{$z*zsvqLtBRZVJ9 znf0M`nYPF_q#xE+yA2oRg_dhSj~GChCfu#k%QyA6Q(a0QDzfPqsDR^5X1Yg$4;)|^ zl9!RxN;SyD&(Yoyhi1|TRR6}6ltmLg{Jzwt;}f{kU9N;9nvE`D8N1+xb{gIP`s(rrqa_3};eFL9 zw8i)CS3o5PdAF`u1T3^Jp~#6d9X@GBmG|5h36@_YJI^FjTsEq;Ih9XrxxB*ls|)qY z)1(sCHc+_7Q)CD{ECUNIN1cjGZ542|(cQ(mtzz_Q-Od=hL;*PNeC^y!b_MIja+l@V z-j1)7r>}gZF~nKv+Cber*Zt$ywC$^=wFl_%cFtf-rd)^Yeg1I6h)j>e9$tdlJBRu- z45G?)K!y836FlOW*GgxRV^DfeGA%$oj6llfb2-E65J&bPCf*b3zcH0nH{7N|3t@4Q zD8qGbBFBji-cD)bvTfi7EvOA(K8t=+21|yI*iv>^~ZTK z2CU{`FVYhxG~r+vJd`Ul9_MqQ7JJ@#n!{9>_N3#_OWSm%RT6R>_^w<5Rsu0gKC^wc zJvH zyC=L&U*CwH3atw%(V;Tb&c_bguH}XIbx9u+{)ocG^;{({aU8^!qLLuv`}O!0`FScY z$WXaINSXVJa+)3?SxOrcaU+y5msDS)&%k-*FNMAHG!`IHp)(V=xgYB(!mUuKLlK)SsIO zn0s?0Vv3^dQh!X}2qlH}n=9G*MonU0-G^Jx|pt^qX+Bbz4`Qql?N2^#VB` z$*cB7b(b%AWF|bCA2%$!V-`Hrp`=7G>gsqD#c$CAq(mDd6uzQHVf72a$}zG?iSnqr z-VzlDE6koeT#boRUDkXD7k5Law5sbu&ZD92385P2O*JO`JN_b-i21y(?Mes)PioK% zeYii7otoIkg~*vcITo8BY76Blc4B z!yi}wtjAo_{u~vTI!`=Z7VkKy)HHDV9AtQD#OkFw;~3*oCQ(4MGYpPyMKrka)iBb> zB)U3f_g#_T*PT<_YyK85(GAu+Z19^-Vu1(uZKu9vQAqBPS}39^KehVE5@b6AMZ0s2 zW)j)5y{;^$4R|NcHz|Jj&ab3Ql);-2AV1y_;X!+!AcZTG|8Q^l0IS%V+-7&n2Q49+ zQwwndb_4asp2FO3ow_h?>RdBJeND6M!!udzdR5(?A})aL5BSuJob;s5szzsp$d(Y2 zLuIO2zC=&#uAE@k>BH@wjR_f=o-FIcVK;}mcG5FJO3;89EFwf8yVbze&vt_$C91O&vRB^&JnNQOYO2AA?XzTdOj!YW%SUL>gSmug_Eah zaPvy{i-Hsuu%%bTqORGOo^u!P{Q^l}u{_CbZhMJ{0yX}-h0>o4XiO?0ED{#UXZ3rUC)@cSwj1b2!WviMO|oo)#zWJOTPsFYuj?>+GymO!Jt zV&k8Y)_J3@{G&$E9DUp{%r+B}T}f#I4~XIG??i^?KNDA5&I7>eq7({ic$(<-nYaMD z0Q&Gs+b(QF_0$w>Be?-N=-}22O31leOLVx5op75waD8#IKZKUMu($nm zI|+$rU=FFtmk@pPCAk|%{sZkBC#HRgcM{%IZ6uK`G@WdK>s7oqcQxw$9H-pw| zBs5PCwH(||YlLIUE!_y6&Xa!Pi_&vj!jX7-jXoCgwY$m=v~w8h=cbNLN?uWCbXG#3 zG(Za(e0ycwz(FH>)}El`-8XrrET`~XmoWMIu%qfPv<#&%CDWx?JPro zs{`A}N3l}4flj@H$wl^55ENnZt#>k>&bkrCHPtkoH79X*I? zzp{+ae*Bg4uy(BimlDxA?+w07G|7auO+U<}e|4waf+dgAQCnIe05;0bzO-(Z_M**^ zD8Jx`7Dd@^wf29!)^MNki~p$;g=ecns^YglE5vU)*%u_{wPV+eCDiz^vpIunT5jT@vcD7{9yW_HY<-(fi`WUk#AAjpGjFdSM^c(Nn43ZOWU`-AD7hZbVv_M)TRuEo zgT49nFrEZk?^d{WpficWvrCwIw(-T233;ePpeLE3MMV)22oC>@mKF9$0~DCIpSV zedOY|*o#QKH<9nL-*Zg_W&U-%nQ{%}$eZs!bxC;R%vGai4J zPgp}`IR5_wGQf0p&TVQ2{}Y=yP*TgtzPCwQcIdyOk24trjL^_WeR}}Xe*%*XCxp%M z6h(DLu0J>S<2|x7ff@Od@4toZ+YABn&oQ`GEGquqZs zYN8O+x%Z;{6C9F0x49>XTni?3CO?yP$M~t zvP>8U%Rl$|8#5*M8=xV{aF@&fpXO>Qzzc{nH)nOWDtPLeW`_2=G3huYT<>xW|M{W$dFTvt7bcEF z|J^#Y{NJif@MngZnEduyxL4>pl3|Q8!8-_(_UAmf|J{0~ zI{;Pb0gjvl$dqFrRKfR7M+Qi`jL?!G+yQaBft-_G$kCo#8_(g6U|B7;ompm1aPn2~ zGE!|oc+A0;v|le8zx)1R>CHJzk$!LslnjLtD@3p?dA+$jV}+c{popjY91GAxTa*_- zWTgO`^dE!YZi36R9YA(1iq5LA13@U-f5Pcvk@v3tcht~4ej>W}Izli%z(%YXavpa_ z{VLDm*8m%J*ZBGh`2y^PXa=GM;*dRIWRmA>svNmXpqtp(3+Y(`F7H_?N6I3Dm}e2f zqlYcMh!-H)vtYl|NGKpPO1|g_3&KV7Lg>bxlexTXg1Vzi`GDe`Ef(D`#P}E^CdH21%JaC(^4FFEOfJ0v=oR>yBz_Pr99N&PP z(AL?36GixIomQHeDsELZXGL7|;hhT>8#*z93;cCL)|&xWc)FN1JU$k z+`+0)lPRagcfHx9?;+WV0jrHk5CRn;W(5K*2wKhryFk3N2z?}KZ#Me-`FUQawYF(j zLws@vQQ8O?oMTcmbh{;St~>Fn)w7x2c#zvrh||ojp}#JuRQXYgExqW!30s5hQR+N$ z4%%5R4h2>6IOKdrgCgsf6enARCmDn~?*nv=c(zs!nV`I&T&C=R9ckuzODutsr8gFR zu(j4XX<5}qbvr8EYjwT^kbu~LRxk&sM+I;_CaeO(r7R_%{uAD3!+W`$*uAU9vN?JM z0O0}4yMRYl8n*(N+qJaw^y;@uQATwG7oLlbt^hc7jV_;0H$VBycx=-axcGM<2;ye{ z@XiGBhudG{%9C|xqQ^6a#sC{-DZc&{cToYVfyIdIH&!L#S{ted&iyRfR9Xgb_SyFK zg0K7X?@2%6mErAqcgJgm0Vb@wtR@^0Dd5(7my3ZDTuE5YP zaop)Fgn^sxlo+R@ItAoLqt+9%{+dqna5R^aSayZ#B7z2wdjb}3_#Ee{?KYgz4$`e# zh^2s6p4Y9eyP1rt>BwPjz}+7Y5GiH))dJ#xI6_pX`7-sP=44Ut0~uy*eKDSjQIU@c z-{HKdj;bnpuAJm~O5e4>${H>IX;>%BnxyNhJteyqH-U&=VceHjv(txEuGo{j3;T`T zv|3|q%9}6EB_Ri496hYvvH){*q*`eWDhEm?Mp*yx!OS(}?>i4>qQ7OH`BCbp#J1v6)RVjkmP5rAQxlQy}n6fgKy znSL~fPZ&u{&=nX#p(!ZYB;44bL|!lD$T4}au^D88skk=?hfZif{!NU zAe_?nQxM~at<{pgvwR>Kn8kzqB-&LE36@LpRgrN$ceIoV%-X7V+#(ov(rPs}wZsIp z^p<-rnJ?$7zyYC_DIzoegH9-tz~D+H9Z{ogSzxdNnH@%U!8P$eD~p=yuC z+pBjtd|oDR%DA=Udg?Xb)dT=fQ9Cc6Sm&r-F30+Y&Dz`W`QzPHUlc?RciLf%&3zdH zCjAUJe?5QXW+zv~R>h8_EY7sBj>NOBSLmL!Ll>rA?+Po zj(NZ-HE=c!S)M;~Yj!+)&ctspeK@oHWqL?&I3g2Mw5!Nh)d`%L$p8^C68V}FxXdm> z^3){eK+JKElR=zY?l9UJ)s<0$ljhwk*=T<^eOk0>cU~iH>OV~6;&r5 z!Fx%A@10%waBgAVc=@!r{x->T&->i*{Dyk{)0_>{A0AldptVfwr`*k!(*tTUg?x4i zXM0+Pr{q8FjmcajvWi`ar1&?_^{)kgW5?aK&k_SI2fqU1rj?%%IGV2rGd9`KUricl zAsV@?=lo7gIjz=6n=ltNf88M#ek_7wX;5G4ojPm;FtA^q=X!_>cV)-u&nOWxoel zsmD%z5D#ipFg0qiuMwOwRR^sB-R04sc+jv09n-bcBx&t8Qp> Removes all items associated with one random %% erroring (references to fuses blown in the code) index in %% order to get below the search.queue.high_watermark. -%% - purge_all -> Removes all items associated with all -%% erroring (references to fuses blown in the code) indices in -%% order to get below the search.queue.high_watermark. %% - off -> purging is disabled {mapping, "search.queue.high_watermark.purge_strategy", "yokozuna.solrq_hwm_purge_strategy", [ {default, purge_one}, {commented, purge_one}, - {datatype, {enum, [purge_one, purge_index, purge_all, off]}} + {datatype, {enum, [purge_one, purge_index, off]}} ]}. %% @doc The amount of time to wait before a drain operation times out. diff --git a/riak_test/intercepts/yz_solrq_drain_fsm_intercepts.erl b/riak_test/intercepts/yz_solrq_drain_fsm_intercepts.erl index da63551d..ed28dc59 100644 --- a/riak_test/intercepts/yz_solrq_drain_fsm_intercepts.erl +++ b/riak_test/intercepts/yz_solrq_drain_fsm_intercepts.erl @@ -27,19 +27,24 @@ prepare_crash(start, State) -> {stop, {error, something_bad_happened}, State}. +%% Put a 1 second sleep in front of resume_workers. +resume_workers_sleep_1s(Pid) -> + timer:sleep(1000), + ?M:resume_workers_orig(Pid). -%% Put a 5 second sleep in front of prepare. -prepare_sleep_5s(start, State) -> - timer:sleep(5000), +%% restore the original prepare +prepare_orig(start, State) -> ?M:prepare_orig(start, State). +%% restore the original resume_workers +resume_workers_orig(Pid) -> + ?M:resume_workers_orig(Pid). -%% Put a 5 second sleep in front of prepare. -prepare_sleep_1s(start, State) -> - timer:sleep(1000), - ?M:prepare_orig(start, State). +%% Timeout on a cancel, full stop +cancel_timeout(_Pid, _CancelTimeout) -> + lager:log(info, self(), "Intercepting cancel/2 and returning timeout"), + timeout. - -%% restore the original -prepare_orig(start, State) -> - ?M:prepare_orig(start, State). +%% restore the original cancel +cancel_orig(Pid, CancelTimeout) -> + ?M:cancel_orig(Pid, CancelTimeout). diff --git a/riak_test/yokozuna_essential.erl b/riak_test/yokozuna_essential.erl index 6fecc39a..67668fac 100644 --- a/riak_test/yokozuna_essential.erl +++ b/riak_test/yokozuna_essential.erl @@ -418,7 +418,10 @@ verify_correct_solrqs(Cluster) -> ?assertEqual(ok, rt:wait_until(Cluster, fun check_queues_match/1)). check_queues_match(Node) -> - CurrentIndexes = rpc:call(Node, yz_index, get_indexes_from_meta, []), + %% Current Indexes includes ?YZ_INDEX_TOMBSTONE because we need to write the entries + %% for non-indexed data to the YZ AAE tree. Excluding them makes the solrq supervisor + %% constantly start and stop these queues. + CurrentIndexes = rpc:call(Node, yz_index, get_indexes_from_meta, []) ++ [?YZ_INDEX_TOMBSTONE], OwnedPartitions = rt:partitions_for_node(Node), ActiveQueues = rpc:call(Node, yz_solrq_sup, active_queues, []), ExpectedQueueus = [{Index, Partition} || Index <- CurrentIndexes, Partition <- OwnedPartitions], diff --git a/riak_test/yz_aae_test.erl b/riak_test/yz_aae_test.erl index cac0d634..0daeafe5 100644 --- a/riak_test/yz_aae_test.erl +++ b/riak_test/yz_aae_test.erl @@ -10,31 +10,34 @@ -define(BUCKET_TYPE, <<"data">>). -define(INDEX1, <<"fruit_aae">>). -define(INDEX2, <<"fruitpie_aae">>). --define(BUCKETWITHTYPE, - {?BUCKET_TYPE, ?INDEX2}). +-define(BUCKETWITHTYPE, {?BUCKET_TYPE, ?INDEX2}). -define(BUCKET, ?INDEX1). -define(REPAIR_MFA, {yz_exchange_fsm, repair, 2}). -define(SPACER, "testfor spaces "). -define(AAE_THROTTLE_LIMITS, [{-1, 0}, {10000, 10}]). --define(CFG, - [{riak_core, - [ - {ring_creation_size, 16}, - {default_bucket_props, [{n_val, ?N}]}, - {handoff_concurrency, 10}, - {vnode_management_timer, 1000} - ]}, - {yokozuna, - [ - {enabled, true}, - {?SOLRQ_DRAIN_ENABLE, true}, - {anti_entropy_tick, 1000}, - %% allow AAE to build trees and exchange rapidly - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8}, - {aae_throttle_limits, ?AAE_THROTTLE_LIMITS} - ]} - ]). +-define(CFG, [ + {riak_core, [ + {ring_creation_size, 16}, + {default_bucket_props, [{n_val, ?N}]}, + {handoff_concurrency, 10}, + {vnode_management_timer, 1000} + ]}, + {riak_kv, [ + {force_hashtree_upgrade, true}, + {anti_entropy_tick, 1000}, + {anti_entropy_build_limit, {100, 1000}}, + {anti_entropy_concurrency, 8} + ]}, + {yokozuna, [ + {enabled, true}, + {?SOLRQ_DRAIN_ENABLE, true}, + {anti_entropy_tick, 1000}, + %% allow AAE to build trees and exchange rapidly + {anti_entropy_build_limit, {100, 1000}}, + {anti_entropy_concurrency, 8}, + {aae_throttle_limits, ?AAE_THROTTLE_LIMITS} + ]} +]). confirm() -> Cluster = rt:build_cluster(5, ?CFG), @@ -101,15 +104,14 @@ aae_run(Cluster, Bucket, Index) -> RepairCountBefore = get_cluster_repair_count(Cluster), yz_rt:count_calls(Cluster, ?REPAIR_MFA), - NumKeys = [{Bucket, K} || K <- yz_rt:random_keys(?NUM_KEYS)], - NumKeysSpaces = [{Bucket, add_space_to_key(K)} || + RandomBKeys = [{Bucket, K} || K <- yz_rt:random_keys(?NUM_KEYS)], + RandomBKeysWithSpaces = [{Bucket, add_space_to_key(K)} || K <- yz_rt:random_keys(?NUM_KEYS_SPACES)], - {DelNumKeys, _ChangeKeys} = lists:split(length(NumKeys) div 2, - NumKeys), - {DelNumKeysSpaces, _ChangeKeysSpaces} = lists:split( - length(NumKeysSpaces) div 2, - NumKeysSpaces), - AllDelKeys = DelNumKeys ++ DelNumKeysSpaces, + {RandomBKeysToDelete, _} = lists:split(length(RandomBKeys) div 2, RandomBKeys), + {RandomBKeysWithSpacesToDelete, _} = lists:split( + length(RandomBKeysWithSpaces) div 2, + RandomBKeysWithSpaces), + AllDelKeys = RandomBKeysToDelete ++ RandomBKeysWithSpacesToDelete, lager:info("Deleting ~p keys", [length(AllDelKeys)]), [delete_key_in_solr(Cluster, Index, K) || K <- AllDelKeys], lager:info("Verify Solr indexes missing"), @@ -174,7 +176,7 @@ create_orphan_postings(Cluster, Index, Bucket, Keys) -> Keys2 = [{Bucket, ?INT_TO_BIN(K)} || K <- Keys], lager:info("Create orphan postings with keys ~p", [Keys]), ObjNodePs = [create_obj_node_partition_tuple(Cluster, Key) || Key <- Keys2], - [ok = rpc:call(Node, yz_kv, index, [Obj, put, P]) + [ok = rpc:call(Node, yz_kv, index, [{Obj, no_old_object}, put, P]) || {Obj, Node, P} <- ObjNodePs], yz_rt:commit(Cluster, Index), ok. @@ -330,7 +332,8 @@ verify_count_and_repair_after_error_value(Cluster, {BType, _Bucket}, Index, %% 1. write KV data to non-indexed bucket Conn = yz_rt:select_random(PBConns), lager:info("write 1 bad search field to bucket ~p", [Bucket]), - Obj = riakc_obj:new(Bucket, <<"akey_bad_data">>, <<"{\"date_register\":3333}">>, + Key = <<"akey_bad_data">>, + Obj = riakc_obj:new(Bucket, Key, <<"{\"date_register\":3333}">>, "application/json"), ok = riakc_pb_socket:put(Conn, Obj), @@ -349,6 +352,11 @@ verify_count_and_repair_after_error_value(Cluster, {BType, _Bucket}, Index, %% 5. verify count after expiration verify_exchange_after_expire(Cluster, Index), + %% 6. Because it's possible we'll try to repair this key again + %% after clearing trees, delete it from KV + ok = riakc_pb_socket:delete(Conn, Bucket, Key), + yz_rt:commit(Cluster, Index), + ok; verify_count_and_repair_after_error_value(_Cluster, _Bucket, _Index, _PBConns) -> ok. diff --git a/riak_test/yz_rt.erl b/riak_test/yz_rt.erl index 87c48caf..f8ab2c5a 100644 --- a/riak_test/yz_rt.erl +++ b/riak_test/yz_rt.erl @@ -27,6 +27,9 @@ -type search_type() :: solr | yokozuna. -type cluster() :: cluster(). +-type intercept() :: {{TargetFunctionName::atom(), TargetArity::non_neg_integer()}, InterceptFunctionName::atom()}. +-type intercepts() :: [intercept()]. + -export_type([prop/0, props/0, cluster/0]). %% @doc Get {Host, Port} from `Cluster'. @@ -552,6 +555,7 @@ remove_index(Node, BucketType) -> ok = rpc:call(Node, riak_core_bucket_type, update, [BucketType, Props]). really_remove_index(Cluster, {BucketType, Bucket}, Index, PBConn) -> + lager:info("Removing index ~p", [Index]), Node = hd(Cluster), F = fun(_) -> Props = [{?YZ_INDEX, ?YZ_INDEX_TOMBSTONE}], @@ -965,17 +969,17 @@ check_fuse_status(Node, Partition, Indices, FuseCheckFunction) -> -spec intercept_index_batch(node() | cluster(), module()) -> ok | [ok]. intercept_index_batch(Cluster, Intercept) -> - add_intercept( + add_intercepts( Cluster, - yz_solr, index_batch, 2, Intercept). + yz_solr, [{{index_batch, 2}, Intercept}]). --spec add_intercept(node() | cluster(), module(), atom(), non_neg_integer(), module()) -> ok | [ok]. -add_intercept(Cluster, Module, Function, Arity, Intercept) when is_list(Cluster) -> - [add_intercept(Node, Module, Function, Arity, Intercept) || Node <- Cluster]; -add_intercept(Node, Module, Function, Arity, Intercept) -> +-spec add_intercepts(node() | cluster(), module(), intercepts()) -> ok | [ok]. +add_intercepts(Cluster, Module, Intercepts) when is_list(Cluster) -> + [add_intercepts(Node, Module, Intercepts) || Node <- Cluster]; +add_intercepts(Node, Module, Intercepts) -> rt_intercept:add( Node, - {Module, [{{Function, Arity}, Intercept}]}). + {Module, Intercepts}). -spec set_yz_aae_mode(node() | cluster(), automatic | manual) -> ok | [ok]. set_yz_aae_mode(Cluster, Mode) when is_list(Cluster) -> diff --git a/riak_test/yz_solrq_test.erl b/riak_test/yz_solrq_test.erl index fe52125b..27db111a 100644 --- a/riak_test/yz_solrq_test.erl +++ b/riak_test/yz_solrq_test.erl @@ -111,17 +111,18 @@ confirm() -> pass. confirm_drain_fsm_failure(Cluster) -> + lager:info("Starting confirm_drain_fsm_failure"), yz_stat:reset(), try yz_rt:load_intercept_code(Cluster), - yz_rt:add_intercept(Cluster, yz_solrq_drain_fsm, prepare, 2, prepare_crash), + yz_rt:add_intercepts(Cluster, yz_solrq_drain_fsm, [{{prepare, 2}, prepare_crash}]), %% drain solrqs and wait until the drain failure stats are touched yz_rt:drain_solrqs(Cluster), yz_rt:wait_until(Cluster, fun check_drain_failure_stats/1), lager:info("confirm_drain_fsm_failure ok") after - yz_rt:add_intercept(Cluster, yz_solrq_drain_fsm, prepare, 2, prepare_orig) + yz_rt:add_intercepts(Cluster, yz_solrq_drain_fsm, [{{prepare, 2}, prepare_orig}]) end. check_drain_failure_stats(Node) -> @@ -138,19 +139,20 @@ check_drain_failure_stats(Node) -> yz_rt:check_stat_values(Stats, Pairs). confirm_drain_fsm_timeout(Cluster) -> + lager:info("Starting confirm_drain_fsm_timeout"), yz_stat:reset(), [rpc:call( - Node, application, set_env, [?YZ_APP_NAME, ?SOLRQ_DRAIN_TIMEOUT, 500]) + Node, application, set_env, [?YZ_APP_NAME, ?SOLRQ_DRAIN_TIMEOUT, 250]) || Node <- Cluster], try yz_rt:load_intercept_code(Cluster), - yz_rt:add_intercept(Cluster, yz_solrq_drain_fsm, prepare, 2, prepare_sleep_1s), + yz_rt:add_intercepts(Cluster, yz_solrq_drain_fsm, [{{resume_workers, 1}, resume_workers_sleep_1s}]), yz_rt:drain_solrqs(Cluster), yz_rt:wait_until(Cluster, fun check_drain_timeout_stats/1), lager:info("confirm_drain_fsm_timeout ok") after - yz_rt:add_intercept(Cluster, yz_solrq_drain_fsm, prepare, 2, prepare_orig), + yz_rt:add_intercepts(Cluster, yz_solrq_drain_fsm, [{{resume_workers, 1}, resume_workers_orig}]), [rpc:call( Node, application, set_env, [?YZ_APP_NAME, ?SOLRQ_DRAIN_TIMEOUT, 60000]) || Node <- Cluster] @@ -170,17 +172,21 @@ check_drain_timeout_stats(Node) -> yz_rt:check_stat_values(Stats, Pairs). confirm_drain_fsm_kill(Cluster) -> + lager:info("Starting confirm_drain_fsm_kill"), [rpc:call( Node, application, set_env, [?YZ_APP_NAME, ?SOLRQ_DRAIN_TIMEOUT, 10]) || Node <- Cluster], + %% technically not needed for this test (because the cancel intercept will + %% just return timeout), but added for completeness [rpc:call( Node, application, set_env, [?YZ_APP_NAME, ?SOLRQ_DRAIN_CANCEL_TIMEOUT, 10]) || Node <- Cluster], try yz_test_listener:start(), yz_rt:load_intercept_code(Cluster), - yz_rt:add_intercept(Cluster, yz_solrq_drain_fsm, prepare, 2, prepare_sleep_5s), - yz_rt:add_intercept(Cluster, yz_solrq_drain_mgr, unlink_and_kill, 2, count_unlink_and_kill), + yz_rt:add_intercepts(Cluster, yz_solrq_drain_fsm, [{{resume_workers, 1}, resume_workers_sleep_1s}, + {{cancel, 2}, cancel_timeout}]), + yz_rt:add_intercepts(Cluster, yz_solrq_drain_mgr, [{{unlink_and_kill, 2}, count_unlink_and_kill}]), yz_rt:drain_solrqs(Cluster), yz_rt:wait_until(Cluster, fun check_drain_cancel_timeout_stats/1), @@ -188,8 +194,9 @@ confirm_drain_fsm_kill(Cluster) -> lager:info("confirm_drain_fsm_kill ok") after - yz_rt:add_intercept(Cluster, yz_solrq_drain_fsm, prepare, 2, prepare_orig), - yz_rt:add_intercept(Cluster, yz_solrq_drain_mgr, unlink_and_kill, 2, unlink_and_kill_orig), + yz_rt:add_intercepts(Cluster, yz_solrq_drain_fsm, [{{resume_workers, 1}, resume_workers_orig}, + {{cancel, 2}, cancel_orig}]), + yz_rt:add_intercepts(Cluster, yz_solrq_drain_mgr, [{{unlink_and_kill, 2}, unlink_and_kill_orig}]), yz_test_listener:stop(), [rpc:call( Node, application, set_env, [?YZ_APP_NAME, ?SOLRQ_DRAIN_TIMEOUT, 60000]) @@ -214,6 +221,7 @@ check_drain_cancel_timeout_stats(Node) -> confirm_batch_size(Cluster, PBConn, BKey, Index) -> + lager:info("Starting confirm_batch_size"), %% First, put one less than the min batch size and expect that there are no %% search results (because the index operations are queued). Count = ?SOLRQ_BATCH_MIN_SETTING - 1, @@ -246,6 +254,7 @@ confirm_batch_size(Cluster, PBConn, BKey, Index) -> ok. confirm_hwm(Cluster, PBConn, Bucket, Index, HWM) -> + lager:info("Starting confirm_hwm"), yz_rt:drain_solrqs(Cluster), {OldMin, OldMax, OldDelay} = set_index(Cluster, Index, 1, 100, 100), try @@ -267,6 +276,7 @@ confirm_hwm(Cluster, PBConn, Bucket, Index, HWM) -> gteq(A, B) -> A >= B. confirm_draining(Cluster, PBConn, Bucket, Index) -> + lager:info("Starting confirm_draining"), Count = ?SOLRQ_BATCH_MIN_SETTING - 1, Count = put_objects(PBConn, Bucket, Count), yz_rt:commit(Cluster, Index), @@ -278,6 +288,7 @@ confirm_draining(Cluster, PBConn, Bucket, Index) -> ok. confirm_requeue_undelivered([Node|_] = Cluster, PBConn, BKey, Index) -> + lager:info("Starting confirm_requeue_undelivered"), yz_rt:load_intercept_code(Node), yz_rt:intercept_index_batch(Node, index_batch_returns_other_error), @@ -300,6 +311,7 @@ confirm_requeue_undelivered([Node|_] = Cluster, PBConn, BKey, Index) -> ok. confirm_no_contenttype_data(Cluster, PBConn, BKey, Index) -> + lager:info("Starting confirm_no_contenttype_data"), yz_rt:set_index(Cluster, Index, 1, 100, 100), Count = 1, Count = put_no_contenttype_objects(PBConn, BKey, Count), @@ -309,6 +321,7 @@ confirm_no_contenttype_data(Cluster, PBConn, BKey, Index) -> ok. confirm_purge_strategy(Cluster, PBConn) -> + lager:info("Starting confirm_purge_strategy"), confirm_purge_one_strategy(Cluster, PBConn, {?BUCKET5, ?INDEX5}), confirm_purge_idx_strategy(Cluster, PBConn, diff --git a/riak_test/yz_stat_test.erl b/riak_test/yz_stat_test.erl index 485e86f5..d626a514 100644 --- a/riak_test/yz_stat_test.erl +++ b/riak_test/yz_stat_test.erl @@ -15,6 +15,7 @@ -define(NUM_NODES, 1). -define(RING_SIZE, 8). -define(NUM_ENTRIES, 10). +-define(EVENT_TICK_INTERVAL, 1000). -define(CFG, [ {riak_core, [ @@ -38,7 +39,12 @@ %% allow AAE to build trees and exchange rapidly {anti_entropy_tick, 1000}, {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 8} + {anti_entropy_concurrency, 8}, + %% Force a complete check of indexes after every tick + %% to make yokozuna remove/add indexes faster + %% Needed for recreate test + {events_full_check_after, 1}, + {events_tick_interval, ?EVENT_TICK_INTERVAL} ]} ]). @@ -54,16 +60,16 @@ prepare_cluster(NumNodes) -> Cluster. confirm_stats(Cluster) -> - {Host, Port} = yz_rt:select_random( - [yz_rt:riak_pb(I) || {_,I} <- rt:connection_info(Cluster)] - ), + PBConn0 = get_pb_connection(Cluster), Index = <<"yz_stat_test">>, Bucket = {Index, <<"b1">>}, + create_indexed_bucket(Cluster, Bucket, Index, PBConn0), - {ok, PBConn} = riakc_pb_socket:start_link(Host, Port), - yz_rt:create_indexed_bucket(PBConn, Cluster, Bucket, Index, ?N_VAL), - {ok, BProps} = riakc_pb_socket:get_bucket(PBConn, Bucket), - ?assertEqual(?N_VAL, proplists:get_value(n_val, BProps)), + %% because the cluster will die w/o fix for this confirm step, + %% it closes the PB connection. Will get another once it's done + confirm_recreate_indexed_bucket(Cluster, Bucket, Index, PBConn0), + + PBConn = get_pb_connection(Cluster), %% %% Clear the yz and kv hashtrees, because we have created a new %% bucket type with a different n_vals. @@ -75,16 +81,41 @@ confirm_stats(Cluster) -> yz_rt:set_yz_aae_mode(Cluster, manual), + Values = confirm_index_stats(Cluster, PBConn, Bucket, Index), + + confirm_query_stats(Cluster, PBConn, Index, Values), + + confirm_aae_repair_and_stats(Cluster, Index, Bucket, Values), + + confirm_extract_fail_stats(Cluster, PBConn, Bucket), + + confirm_fuse_and_purge_stats(Cluster, PBConn, Index, Bucket), + + riakc_pb_socket:stop(PBConn). + +get_pb_connection(Cluster) -> + {Host, Port} = yz_rt:select_random( + [yz_rt:riak_pb(I) || {_, I} <- rt:connection_info(Cluster)] + ), + {ok, PBConn} = riakc_pb_socket:start_link(Host, Port), + PBConn. + +create_indexed_bucket(Cluster, Bucket, Index, PBConn) -> + yz_rt:create_indexed_bucket(PBConn, Cluster, Bucket, Index, ?N_VAL), + {ok, BProps} = riakc_pb_socket:get_bucket(PBConn, Bucket), + ?assertEqual(?N_VAL, proplists:get_value(n_val, BProps)). + +confirm_fuse_and_purge_stats(Cluster, PBConn, Index, Bucket) -> yz_rt:reset_stats(Cluster), - Values = populate_data_and_wait(PBConn, Cluster, Bucket, Index, ?NUM_ENTRIES), - yz_rt:verify_num_match(yokozuna, Cluster, Index, ?NUM_ENTRIES), - yz_rt:verify_num_match(solr, Cluster, Index, ?NUM_ENTRIES * ?N_VAL), - yz_rt:wait_until(Cluster, fun check_index_stats/1), + blow_fuses(Cluster, PBConn, Index, Bucket), + yz_rt:wait_until(Cluster, fun check_fuse_and_purge_stats/1). +confirm_extract_fail_stats(Cluster, PBConn, Bucket) -> yz_rt:reset_stats(Cluster), - search_values(PBConn, Index, Values), - yz_rt:wait_until(Cluster, fun check_query_stats/1), + write_bad_json(Cluster, PBConn, Bucket, 1), + yz_rt:wait_until(Cluster, fun check_index_extract_fail_stats/1). +confirm_aae_repair_and_stats(Cluster, Index, Bucket, Values) -> yz_rt:reset_stats(Cluster), [delete_key_in_solr(Cluster, Index, {Bucket, K}) || K <- Values], yz_rt:verify_num_match(yokozuna, Cluster, Index, 0), @@ -93,20 +124,22 @@ confirm_stats(Cluster) -> yz_rt:wait_for_full_exchange_round(Cluster), yz_rt:drain_solrqs(Cluster), yz_rt:wait_until(Cluster, fun check_aae_stats/1), - yz_rt:reset_stats(Cluster), yz_rt:verify_num_match(yokozuna, Cluster, Index, ?NUM_ENTRIES), - yz_rt:verify_num_match(solr, Cluster, Index, ?NUM_ENTRIES * ?N_VAL), + yz_rt:verify_num_match(solr, Cluster, Index, ?NUM_ENTRIES * ?N_VAL). +confirm_query_stats(Cluster, PBConn, Index, Values) -> yz_rt:reset_stats(Cluster), - write_bad_json(Cluster, PBConn, Bucket, 1), - yz_rt:wait_until(Cluster, fun check_index_extract_fail_stats/1), + search_values(PBConn, Index, Values), + yz_rt:wait_until(Cluster, fun check_query_stats/1). +confirm_index_stats(Cluster, PBConn, Bucket, Index) -> yz_rt:reset_stats(Cluster), - blow_fuses(Cluster, PBConn, Index, Bucket), - yz_rt:wait_until(Cluster, fun check_fuse_and_purge_stats/1), - - riakc_pb_socket:stop(PBConn). + Values = populate_data_and_wait(PBConn, Cluster, Bucket, Index, ?NUM_ENTRIES), + yz_rt:verify_num_match(yokozuna, Cluster, Index, ?NUM_ENTRIES), + yz_rt:verify_num_match(solr, Cluster, Index, ?NUM_ENTRIES * ?N_VAL), + yz_rt:wait_until(Cluster, fun check_index_stats/1), + Values. clear_hashtrees(Cluster) -> yz_rt:clear_kv_trees(Cluster), @@ -374,3 +407,26 @@ check_fuse_and_purge_stats(Node) -> {error_threshold_recovered_one, ErrorThresholdRecoveredOneValue, '>', 0} ], yz_rt:check_stat_values(Stats, Pairs). + +%% @doc This test exists to test the ability to remove and recreate an index. +%% Specifically, there were issues using the `fuse_stats_exometer' plugin +%% as in its `init' function it always calls `exometer:new' rather than +%% `re_register' which would have not crashed. We will be submitting a PR +%% to fuse to get this resolved, but in the mean time there's a workaround +%% in `yz_fuse' to remove the exometer statistics on fuse removal. +confirm_recreate_indexed_bucket(Cluster, Bucket, Index, PBConn) -> + rt:setup_log_capture(Cluster), + lager:info("Removing index ~p", [Index]), + yz_rt:really_remove_index(Cluster, Bucket, Index, PBConn), + ?assert(rt:expect_in_log(hd(Cluster), "Delta: Removed: \\[<<\"yz_stat_test\">>\\] Added: \\[\\] Same: \\[\\]")), + lager:info("Recreating index ~p", [Index]), + ok = riakc_pb_socket:create_search_index(PBConn, Index, <<>>, [{n_val, ?N_VAL}]), + %% Stop the PB Connection so it doesn't crash our test + riakc_pb_socket:stop(PBConn), + yz_rt:wait_for_index(Cluster, Index), + Props = [{?YZ_INDEX, Index}], + lager:info("Adding index ~p back to bucket ~p", [Index, Bucket]), + rpc:call(hd(Cluster), riak_core_bucket, set_bucket, [Bucket, Props]), + %% Allow yz_events to tick before continuing to ensure index is removed + timer:sleep(?EVENT_TICK_INTERVAL + 100), + ?assert(rt:expect_not_in_logs(hd(Cluster), "gen_server fuse_server terminated with reason: exists")). \ No newline at end of file diff --git a/src/yz_doc.erl b/src/yz_doc.erl index 5ddb0880..8ee99398 100644 --- a/src/yz_doc.erl +++ b/src/yz_doc.erl @@ -68,6 +68,8 @@ doc_id(O, Partition, Sibling) -> end. %% @doc grab all siblings' vtags from Object contents +sibling_vtags(Cs) when length(Cs) == 1 -> + [none]; sibling_vtags(Cs) -> [get_vtag(MD) || {MD, _V} <- Cs]. %% @doc count of Object contents that are siblings and not tombstones diff --git a/src/yz_entropy_mgr.erl b/src/yz_entropy_mgr.erl index 2ac18ca5..e7159f6c 100644 --- a/src/yz_entropy_mgr.erl +++ b/src/yz_entropy_mgr.erl @@ -217,11 +217,13 @@ handle_cast({exchange_status, Pid, Index, {StartIdx, N}, Status}, S) -> {noreply, S2}; handle_cast(clear_trees, S) -> + lager:info("Clearing YZ hashtrees and stopping all current exchanges."), clear_all_exchanges(S#state.exchanges), clear_all_trees(S#state.trees), {noreply, S}; handle_cast(expire_trees, S) -> + lager:info("Expiring YZ hashtrees."), ok = expire_all_trees(S#state.trees), {noreply, S}; diff --git a/src/yz_exchange_fsm.erl b/src/yz_exchange_fsm.erl index 922549ac..673e22c8 100644 --- a/src/yz_exchange_fsm.erl +++ b/src/yz_exchange_fsm.erl @@ -34,6 +34,8 @@ built :: integer(), timeout :: pos_integer()}). +-type repair_count() :: {DeleteCount::non_neg_integer(), RepairCount::non_neg_integer()}. + %%%=================================================================== %%% API %%%=================================================================== @@ -189,26 +191,37 @@ key_exchange(timeout, S=#state{index=Index, (_, _) -> ok end, + AccFun = fun(KeyDiffs, Accum) -> + hashtree_compare_accum_fun(Index, KeyDiffs, Accum) + end, + async_do_compare(IndexN, Remote, AccFun, YZTree), + {next_state, key_exchange, S}; - AccFun = fun(KeyDiff, Count) -> - lists:foldl(fun(Diff, InnerCount) -> - case repair(Index, Diff) of - full_repair -> InnerCount + 1; - _ -> InnerCount - end - end, Count, KeyDiff) - end, - case yz_index_hashtree:compare(IndexN, Remote, AccFun, 0, YZTree) of +key_exchange({compare_complete, CompareResult}, State=#state{index=Index, + index_n=IndexN}) -> + case CompareResult of {error, Reason} -> lager:error("An error occurred comparing hashtrees. Error: ~p", [Reason]); - 0 -> + {0, 0} -> yz_kv:update_aae_exchange_stats(Index, IndexN, 0); - Count -> - yz_stat:detected_repairs(Count), - lager:info("Will repair ~b keys of partition ~p for preflist ~p", - [Count, Index, IndexN]) + {YZDeleteCount, YZRepairCount} -> + yz_stat:detected_repairs(YZDeleteCount + YZRepairCount), + lager:info("Will delete ~p keys and repair ~b keys of partition ~p for preflist ~p", + [YZDeleteCount, YZRepairCount, Index, IndexN]) end, - {stop, normal, S}. + {stop, normal, State}. + +async_do_compare(IndexN, Remote, AccFun, YZTree) -> + ExchangePid = self(), + spawn_link( + fun() -> + CompareResult = yz_index_hashtree:compare(IndexN, Remote, AccFun, {0, 0}, YZTree), + compare_complete(ExchangePid, CompareResult) + end). + +compare_complete(ExchangePid, CompareResult) -> + gen_fsm:send_event(ExchangePid, {compare_complete, CompareResult}). + %%%=================================================================== %%% Internal functions @@ -222,6 +235,30 @@ exchange_bucket_kv(Tree, IndexN, Level, Bucket) -> exchange_segment_kv(Tree, IndexN, Segment) -> riak_kv_index_hashtree:exchange_segment(IndexN, Segment, Tree). +-spec hashtree_compare_accum_fun(p(), [keydiff()], repair_count()) -> + repair_count(). +hashtree_compare_accum_fun(Index, KeyDiffs, Accum) -> + lists:foldl( + fun(KeyDiff, InnerAccum) -> + repair_fold_func(Index, KeyDiff, InnerAccum) + end, + Accum, + KeyDiffs + ). + +-spec repair_fold_func(p(), keydiff(), repair_count()) -> + repair_count(). +repair_fold_func(Index, KeyDiff, Accum) -> + RepairResult = repair(Index, KeyDiff), + update_repair_func_accum(RepairResult, KeyDiff, Accum). + +update_repair_func_accum(full_repair, _KeyDiff={remote_missing, _KeyBin}, {YZDeleteCount, YZRepairCount}) -> + {YZDeleteCount + 1, YZRepairCount}; +update_repair_func_accum(full_repair, _KeyDiff, {YZDeleteCount, YZRepairCount}) -> + {YZDeleteCount, YZRepairCount + 1}; +update_repair_func_accum(_RepairType, _KeyDiff, Accum) -> + Accum. + %% @private %% %% @doc If Yokozuna gets {remote_missing, _} b/c yz has it, but kv doesn't. @@ -234,16 +271,11 @@ repair(Partition, {remote_missing, KeyBin}) -> Index = yz_kv:get_index(BKey), FakeObj = fake_kv_object(BKey), yz_entropy_mgr:throttle(), - case yz_kv:should_index(Index) of - true -> - Repair = full_repair, - yz_solrq:index(Index, BKey, FakeObj, {anti_entropy_delete, Repair}, Partition), - Repair; - false -> - Repair = tree_repair, - yz_solrq:index(Index, BKey, FakeObj, {anti_entropy_delete, Repair}, Partition), - Repair - end; + Repair = determine_repair_type(Index), + yz_solrq:index(Index, BKey, {FakeObj, no_old_object}, + {anti_entropy_delete, Repair}, Partition), + Repair; + repair(Partition, {_Reason, KeyBin}) -> %% Either Yokozuna is missing the key or the hash doesn't %% match. In either case the object must be re-indexed. @@ -254,18 +286,10 @@ repair(Partition, {_Reason, KeyBin}) -> case yz_kv:local_get(Partition, BKey) of {ok, Obj} -> yz_entropy_mgr:throttle(), - case yz_kv:should_index(Index) of - true -> - Repair = full_repair, - yz_solrq:index(Index, BKey, Obj, {anti_entropy, Repair}, - Partition), - Repair; - false -> - Repair = tree_repair, - yz_solrq:index(Index, BKey, Obj, {anti_entropy, Repair}, - Partition), - Repair - end; + Repair = determine_repair_type(Index), + yz_solrq:index(Index, BKey, {Obj, no_old_object}, + {anti_entropy, Repair}, Partition), + Repair; _Other -> %% In most cases Other will be `{error, notfound}' which %% is fine because hashtree updates are async and the @@ -277,6 +301,14 @@ repair(Partition, {_Reason, KeyBin}) -> failed_repair end. +determine_repair_type(Index) -> + case yz_kv:should_index(Index) of + true -> + full_repair; + false -> + tree_repair + end. + %% @private fake_kv_object({Bucket, Key}) -> riak_object:new(Bucket, Key, <<"fake object">>). diff --git a/src/yz_fuse.erl b/src/yz_fuse.erl index 412ad25e..4f08949a 100644 --- a/src/yz_fuse.erl +++ b/src/yz_fuse.erl @@ -93,6 +93,7 @@ remove(Index) -> FuseName = fuse_name_for_index(Index), fuse:remove(FuseName), yz_stat:delete_dynamic_stats(Index, ?DYNAMIC_STATS), + remove_fuse_stats(FuseName), ok. -spec reset(index_name()) -> ok. @@ -230,3 +231,14 @@ round_trip_through_fuse_name(IndexName) -> FuseName = fuse_name_for_index(IndexName), index_for_fuse_name(FuseName). -endif. + +%% @doc Remove exometer stats for fuse `Name'. +remove_fuse_stats(Name) -> + _ = exometer:delete(metric(Name, ok)), + _ = exometer:delete(metric(Name, blown)), + _ = exometer:delete(metric(Name, melt)), + ok. + +%% Internal. +metric(Name, Counter) -> + [fuse, Name, Counter]. \ No newline at end of file diff --git a/src/yz_index_hashtree.erl b/src/yz_index_hashtree.erl index 88ffa897..0ffc766c 100644 --- a/src/yz_index_hashtree.erl +++ b/src/yz_index_hashtree.erl @@ -50,26 +50,59 @@ start(Index, RPs) -> start_link(Index, RPs) -> gen_server:start_link(?MODULE, [Index, RPs], []). +%% @doc Insert the given `Key' and `Hash' pair on `Tree' for the given +%% `Id'. The result of this call is not useful, as it may fail +%% but is protected by a `try/catch' +-spec insert({p(),n()}, bkey(), binary(), tree(), list()) -> + ok | term(). +insert(Id, BKey, Hash, Tree, Options) -> + SyncOrAsync = hashtree_call_mode(), + insert(SyncOrAsync, Id, BKey, Hash, Tree, Options). + %% @doc Insert the given `Key' and `Hash' pair on `Tree' for the given %% `Id'. The result of a sync call should be ignored since it %% uses `catch'. +%% WARNING: In almost all cases, the caller should use `insert/5' +%% rather than calling `insert/6' directly to prevent mailbox +%% overload of the hashtree. -spec insert(async | sync, {p(),n()}, bkey(), binary(), tree(), list()) -> ok | term(). insert(async, Id, BKey, Hash, Tree, Options) -> gen_server:cast(Tree, {insert, Id, BKey, Hash, Options}); insert(sync, Id, BKey, Hash, Tree, Options) -> - catch gen_server:call(Tree, {insert, Id, BKey, Hash, Options}, infinity). + try + gen_server:call(Tree, {insert, Id, BKey, Hash, Options}, infinity) + catch + Error -> + lager:error("Synchronous insert into hashtree failed for reason ~p", [Error]) + end. + +%% @doc Delete the `BKey' from `Tree'. The id will be determined from +%% `BKey'. The result of the sync call should be ignored since +%% it uses catch. +-spec delete({p(),n()}, bkey(), tree()) -> ok. +delete(Id, BKey, Tree) -> + SyncOrAsync = hashtree_call_mode(), + delete(SyncOrAsync, Id, BKey, Tree). %% @doc Delete the `BKey' from `Tree'. The id will be determined from %% `BKey'. The result of the sync call should be ignored since %% it uses catch. +%% WARNING: In almost all cases, the caller should use `delete/3' +%% rather than calling `delete/4' directly to prevent mailbox +%% overload of the hashtree. -spec delete(async | sync, {p(),n()}, bkey(), tree()) -> ok. delete(async, Id, BKey, Tree) -> gen_server:cast(Tree, {delete, Id, BKey}); delete(sync, Id, BKey, Tree) -> - catch gen_server:call(Tree, {delete, Id, BKey}, infinity). + try + gen_server:call(Tree, {delete, Id, BKey}, infinity) + catch + Error -> + lager:error("Synchronous delete from hashtree failed for reason ~p", [Error]) + end. -spec update({p(), n()}, tree()) -> ok. update(Id, Tree) -> @@ -329,7 +362,7 @@ fold_keys(Partition, Tree, Indexes) -> %% TODO: return _yz_fp from iterator and use that for %% more efficient get_index_N IndexN = get_index_n(BKey), - insert(async, IndexN, BKey, Hash, Tree, [if_missing]) + insert(IndexN, BKey, Hash, Tree, [if_missing]) end, Filter = [{partition, LogicalPartition}], [yz_entropy:iterate_entropy_data(I, Filter, F) || I <- Indexes]. @@ -434,6 +467,40 @@ do_delete(Id, Key, S=#state{trees=Trees}) -> handle_unexpected_key(Id, Key, S) end. +%% @private +%% +%% @doc Determine the method used to make the hashtree update. Most +%% updates will be performed in async manner but want to occasionally +%% use a blocking call to avoid overloading the hashtree. +%% +%% NOTE: This uses the process dictionary and thus is another function +%% which relies running on a long-lived process. In this case that +%% process is the KV vnode. In the future this should probably use +%% cast only + sidejob for overload protection. +-spec hashtree_call_mode() -> async | sync. +hashtree_call_mode() -> + case get(yz_hashtree_tokens) of + undefined -> + put(yz_hashtree_tokens, max_hashtree_tokens() - 1), + async; + N when N > 0 -> + put(yz_hashtree_tokens, N - 1), + async; + _ -> + put(yz_hashtree_tokens, max_hashtree_tokens() - 1), + sync + end. + +%% @private +%% +%% @doc Return the max number of async hashtree calls that may be +%% performed before requiring a blocking call. +-spec max_hashtree_tokens() -> pos_integer(). +max_hashtree_tokens() -> + %% Use same max as riak_kv if no override provided + app_helper:get_env(yokozuna, anti_entropy_max_async, + app_helper:get_env(riak_kv, anti_entropy_max_async, 90)). + -spec handle_unexpected_key({p(),n()}, binary(), state()) -> state(). handle_unexpected_key(Id, Key, S=#state{index=Partition}) -> RP = riak_kv_util:responsible_preflists(Partition), diff --git a/src/yz_kv.erl b/src/yz_kv.erl index e238ad1d..21ce4cb2 100644 --- a/src/yz_kv.erl +++ b/src/yz_kv.erl @@ -30,8 +30,6 @@ -include_lib("eunit/include/eunit.hrl"). -endif. --type delops() :: []|[{id, _}]|[{siblings, _}]. - %%%=================================================================== %%% TODO: move to riak_core %%%=================================================================== @@ -214,20 +212,20 @@ index_binary(Bucket, Key, Bin, Reason, P) -> true -> RObj = riak_object:from_binary(Bucket, Key, Bin), index( - RObj, Reason, P + {RObj, no_old_object}, Reason, P ); _ -> ok end. %% @doc Index the data supplied in the Riak Object. --spec index(riak_object:riak_object(), write_reason(), p()) -> ok. -index(Obj, Reason, P) -> +-spec index(object_pair(), write_reason(), p()) -> ok. +index({Obj, _OldObj}=Objects, Reason, P) -> case yokozuna:is_enabled(index) andalso ?YZ_ENABLED of true -> BKey = {riak_object:bucket(Obj), riak_object:key(Obj)}, Index = yz_kv:get_index(BKey), - yz_solrq:index(Index, BKey, Obj, Reason, P); + yz_solrq:index(Index, BKey, Objects, Reason, P); false -> ok end. @@ -276,51 +274,17 @@ update_hashtree(Action, Partition, IdxN, BKey) -> not_registered -> ok; Tree -> - Method = get_method(), case Action of {insert, ObjHash} -> - yz_index_hashtree:insert(Method, IdxN, BKey, + yz_index_hashtree:insert(IdxN, BKey, ObjHash, Tree, []), ok; delete -> - yz_index_hashtree:delete(Method, IdxN, BKey, Tree), + yz_index_hashtree:delete(IdxN, BKey, Tree), ok end end. -%% @private -%% -%% @doc Determine the method used to make the hashtree update. Most -%% updates will be performed in async manner but want to occasionally -%% use a blocking call to avoid overloading the hashtree. -%% -%% NOTE: This uses the process dictionary and thus is another function -%% which relies running on a long-lived process. In this case that -%% process is the KV vnode. In the future this should probably use -%% cast only + sidejob for overload protection. --spec get_method() -> async | sync. -get_method() -> - case get(yz_hashtree_tokens) of - undefined -> - put(yz_hashtree_tokens, max_hashtree_tokens() - 1), - async; - N when N > 0 -> - put(yz_hashtree_tokens, N - 1), - async; - _ -> - put(yz_hashtree_tokens, max_hashtree_tokens() - 1), - sync - end. - -%% @private -%% -%% @doc Return the max number of async hashtree calls that may be -%% performed before requiring a blocking call. --spec max_hashtree_tokens() -> pos_integer(). -max_hashtree_tokens() -> - %% Use same max as riak_kv - app_helper:get_env(riak_kv, anti_entropy_max_async, 90). - %% @doc Write a value -spec put(any(), binary(), binary(), binary(), string()) -> ok. put(Client, Bucket, Key, Value, ContentType) -> @@ -340,50 +304,6 @@ put(Client, Bucket, Key, Value, ContentType) -> check_flag(Flag) -> true == erlang:get(Flag). -%% @private -%% -%% @doc General cleanup for non-sibling-permitted documents, -%% setting up a delete operation of the bkey if the field -%% contains a tombstone. -%% e.g. for allow_mult=false/lww=true/datatype/sc --spec cleanup([doc()], bkey()) -> [{bkey, bkey()}]. -cleanup([], _BKey) -> - []; -cleanup([{doc, Fields}|T], BKey) -> - case proplists:is_defined(tombstone, Fields) of - true -> [{bkey, BKey}]; - false -> cleanup(T, BKey) - end. - -%% @private -%% @doc Cleanup for siblings-permitted objects. -%% `Last-case' if single-document and has no tombstones is to cleanup -%% possible siblings on reconcilation. -%% -%% If there's a tombstone, remove w/ bkey Query. -%% -%% For Docs > 1, we start w/ & remove the original doc_id (ODocID) in order -%% to preserve vtag-based ids used for sibling writes, but just remove -%% via BKey if there's at least 1 tombstone. -%% -%% Tombstone tuples come through as {tombstone, <<>>}. --spec cleanup_for_sibs([doc()], bkey(), binary()|[{id, binary()}]) - -> [{id, binary()}|{siblings, bkey()}]. -cleanup_for_sibs(Docs, BKey, ODocID) when is_binary(ODocID) -> - case length(Docs) > 1 of - true -> cleanup_for_sibs(Docs, BKey, [{id, ODocID}]); - false -> cleanup_for_sibs(Docs, BKey, []) - end; -cleanup_for_sibs([{doc, Fields}|T], BKey, Ops) -> - case proplists:is_defined(tombstone, Fields) of - true -> [{bkey, BKey}]; - false -> cleanup_for_sibs(T, BKey, Ops) - end; -cleanup_for_sibs([], BKey, []) -> - [{siblings, BKey}]; -cleanup_for_sibs([], _BKey, Ops) -> - Ops. - %% @private %% %% @doc Get first partition from a preflist. @@ -509,21 +429,6 @@ siblings_permitted(Obj, BProps) when is_list(BProps) -> end; siblings_permitted(_, _) -> true. -%% @private -%% -%% @doc Set yz_solr:index delete operation(s). -%% If object relates to lww=true/allow_mult=false/datatype/sc -%% do cleanup of tombstones only. --spec delete_operation(riak_kv_bucket:props(), obj(), [doc()], - bkey(), lp()) -> delops(). -delete_operation(BProps, Obj, Docs, BKey, LP) -> - case siblings_permitted(Obj, BProps) of - true -> cleanup_for_sibs(Docs, - BKey, - yz_doc:doc_id(Obj, ?INT_TO_BIN(LP))); - _ -> cleanup(Docs, BKey) - end. - %% @private %% %% @doc Merge siblings for objects that shouldn't have them. diff --git a/src/yz_misc.erl b/src/yz_misc.erl index 90cb426f..4fab9093 100644 --- a/src/yz_misc.erl +++ b/src/yz_misc.erl @@ -230,7 +230,7 @@ owned_and_next_partitions(Node, Ring) -> %% @doc Filter out all entries for partitions that are not currently owned or %% this node is a future owner of. --spec filter_out_fallbacks(ordset(p), solr_entries()) -> [{bkey(), obj(), +-spec filter_out_fallbacks(ordset(p), solr_entries()) -> [{bkey(), object_pair(), write_reason(), p()}]. filter_out_fallbacks(OwnedAndNext, Entries) -> lists:filter(fun({_Bkey, _Obj, _Reason, P}) -> diff --git a/src/yz_solr.erl b/src/yz_solr.erl index 3bc0a696..91998106 100644 --- a/src/yz_solr.erl +++ b/src/yz_solr.erl @@ -62,7 +62,7 @@ -type delete_op() :: {id, binary()} | {bkey, bkey()} - | {siblings, bkey()} + | {bkey, bkey(), lp()} | {'query', binary()}. -type ibrowse_config_key() :: max_sessions | max_pipeline_size. @@ -438,24 +438,28 @@ encode_commit() -> %% %% @doc Encode a delete operation into a mochijson2 compatiable term. -spec encode_delete(delete_op()) -> {struct, [{atom(), binary()}]}. +encode_delete({bkey, {{Type, Bucket},Key}, LP}) -> + PNQ = encode_field_query(?YZ_PN_FIELD_B, escape_special_chars(?INT_TO_BIN(LP))), + TypeQ = encode_field_query(?YZ_RT_FIELD_B, escape_special_chars(Type)), + BucketQ = encode_field_query(?YZ_RB_FIELD_B, escape_special_chars(Bucket)), + KeyQ = encode_field_query(?YZ_RK_FIELD_B, escape_special_chars(Key)), + ?QUERY(<>); +%% NOTE: Used for testing only. This deletes _all_ documents for _all_ partitions +%% which may cause extra docs to be deleted if a fallback partition exists +%% on the same node as a primary, or if handoff is still ongoing in a newly-built +%% cluster encode_delete({bkey,{{Type, Bucket},Key}}) -> - TypeQ = encode_nested_query(?YZ_RT_FIELD_B, escape_special_chars(Type)), - BucketQ = encode_nested_query(?YZ_RB_FIELD_B, escape_special_chars(Bucket)), - KeyQ = encode_nested_query(?YZ_RK_FIELD_B, escape_special_chars(Key)), + TypeQ = encode_field_query(?YZ_RT_FIELD_B, escape_special_chars(Type)), + BucketQ = encode_field_query(?YZ_RB_FIELD_B, escape_special_chars(Bucket)), + KeyQ = encode_field_query(?YZ_RK_FIELD_B, escape_special_chars(Key)), ?QUERY(<>); +%% NOTE: Also only used for testing encode_delete({bkey,{Bucket,Key}}) -> %% Need to take legacy (pre 2.0.0) objects into account. - encode_delete({bkey,{{?DEFAULT_TYPE,Bucket},Key}}); -encode_delete({siblings,{{Type,Bucket},Key}}) -> - VTagQ = <>, - TypeQ = encode_nested_query(?YZ_RT_FIELD_B, escape_special_chars(Type)), - BucketQ = encode_nested_query(?YZ_RB_FIELD_B, escape_special_chars(Bucket)), - KeyQ = encode_nested_query(?YZ_RK_FIELD_B, escape_special_chars(Key)), - ?QUERY(<>); -encode_delete({siblings,{Bucket,Key}}) -> + encode_delete({bkey,{{?DEFAULT_TYPE, Bucket}, Key}}); +encode_delete({bkey, {Bucket,Key}, LP}) -> %% Need to take legacy (pre 2.0.0) objects into account. - encode_delete({siblings,{{?DEFAULT_TYPE,Bucket},Key}}); + encode_delete({bkey, {{?DEFAULT_TYPE, Bucket}, Key}, LP}); encode_delete({'query', Query}) -> ?QUERY(Query); encode_delete({id, Id}) -> @@ -474,18 +478,17 @@ encode_field({Name,Value}) -> %% @private %% -%% @doc Encode a field and query into a Solr nested query using the -%% term query parser. --spec encode_nested_query(binary(), binary()) -> binary(). -encode_nested_query(Field, Query) -> - <<"_query_:\"{!term f=",Field/binary,"}",Query/binary,"\"">>. +%% @doc Encode a field and query. +-spec encode_field_query(binary(), binary()) -> binary(). +encode_field_query(Field, Query) -> + <>. %% @private %% %% @doc Escape the backslash and double quote chars to prevent from %% being improperly interpreted by Solr's query parser. -spec escape_special_chars(binary()) ->binary(). -escape_special_chars(Bin) -> +escape_special_chars(Bin) when is_binary(Bin)-> Bin2 = binary:replace(Bin, <<"\\">>, <<"\\\\">>, [global]), binary:replace(Bin2, <<"\"">>, <<"\\\"">>, [global]). diff --git a/src/yz_solr_proc.erl b/src/yz_solr_proc.erl index bae878f5..a0616a8b 100644 --- a/src/yz_solr_proc.erl +++ b/src/yz_solr_proc.erl @@ -191,7 +191,13 @@ handle_info({'EXIT', _Port, Reason}, S=?S_MATCH) -> {stop, normal, S}; _ -> {stop, {port_exit, Reason}, S} - end. + end; + +%% ibrowse does not protect from late replies - handle them here +handle_info({Ref, _Msg} = Message, State) + when is_reference(Ref) -> + lager:info("Received late reply: ~p", [Message]), + {noreply, State}. code_change(_, S, _) -> {ok, S}. diff --git a/src/yz_solrq.erl b/src/yz_solrq.erl index 6081dd9f..3818f44c 100644 --- a/src/yz_solrq.erl +++ b/src/yz_solrq.erl @@ -54,11 +54,11 @@ %%% API functions %%%=================================================================== --spec index(index_name(), bkey(), obj(), write_reason(), p()) -> ok. -index(Index, BKey, Obj, Reason, P) -> +-spec index(index_name(), bkey(), object_pair(), write_reason(), p()) -> ok. +index(Index, BKey, ObjectPair, Reason, P) -> WorkerName = yz_solrq:worker_regname(Index, P), ok = ensure_worker(Index, P), - yz_solrq_worker:index(WorkerName, BKey, Obj, Reason, P). + yz_solrq_worker:index(WorkerName, BKey, ObjectPair, Reason, P). %% @doc From the hash, return the registered name of a queue -spec worker_regname(index_name(), p()) -> regname(). diff --git a/src/yz_solrq_drain_fsm.erl b/src/yz_solrq_drain_fsm.erl index 3e7a450a..c5dc4b73 100644 --- a/src/yz_solrq_drain_fsm.erl +++ b/src/yz_solrq_drain_fsm.erl @@ -31,7 +31,20 @@ terminate/3, code_change/4]). --export([start_prepare/1, prepare/2, wait/2, drain_complete/2]). +%% gen_fsm states +-export([ + prepare/2, + wait_for_drain_complete/2, + wait_for_snapshot_complete/2, + wait_for_yz_hashtree_updated/2 +]). + +%% API +-export([ + start_prepare/1, + drain_complete/2, + drain_already_in_progress/2, + resume_workers/1]). -include("yokozuna.hrl"). -define(SERVER, ?MODULE). @@ -41,7 +54,8 @@ exchange_fsm_pid, yz_index_hashtree_update_params, partition, - time_start + time_start, + owner_pid }). %%%=================================================================== @@ -79,6 +93,16 @@ start_link(Params) -> drain_complete(DPid, Token) -> gen_fsm:send_event(DPid, {drain_complete, Token}). +%% @doc Notify the drain FSM identified by DPid that the solrq associated +%% with the specified Token has an existing drain request in progress and +%% cannot perform another drain at this time. +%% +%% NB. This function is typically called from each solrq. +%% @end +%% +drain_already_in_progress(DPid, Token) -> + gen_fsm:send_event(DPid, {drain_already_in_progress, Token}). + %% @doc Start draining. This operation will send a start message to this %% FSM with a start message, which in turn will initiate drains on all of %% the solrqs. @@ -110,6 +134,15 @@ cancel(DPid, Timeout) -> timeout end. +%% +%% Resume workers, typically called from the callback supplied to +%% yz_index_hashtree. We are declaring this one-liner as a public +%% API function in order to have an effective intecept in riak_test +%% C.f., yz_solrq_test:confirm_drain_fsm_timeout +%% +resume_workers(Pid) -> + gen_fsm:send_event(Pid, resume_workers). + %%%=================================================================== %%% gen_fsm callbacks %%%=================================================================== @@ -118,7 +151,8 @@ init(Params) -> {ok, prepare, #state{ exchange_fsm_pid = proplists:get_value(?EXCHANGE_FSM_PID, Params), yz_index_hashtree_update_params = proplists:get_value(?YZ_INDEX_HASHTREE_PARAMS, Params), - partition = proplists:get_value(?DRAIN_PARTITION, Params) + partition = proplists:get_value(?DRAIN_PARTITION, Params), + owner_pid = proplists:get_value(owner_pid, Params) }}. @@ -137,7 +171,7 @@ prepare(start, #state{partition = P} = State) -> %% normally. Otherwise, we keep waiting. %% @end %% -wait({drain_complete, Token}, +wait_for_drain_complete({drain_complete, Token}, #state{ tokens = Tokens, exchange_fsm_pid = ExchangeFSMPid, @@ -151,22 +185,64 @@ wait({drain_complete, Token}, [] -> lager:debug("Solrq drain completed for all workers for partition ~p. Resuming batching.", [Partition]), yz_stat:drain_end(?YZ_TIME_ELAPSED(StartTS)), - CompleteCallback = fun() -> - [yz_solrq_worker:drain_complete(Name) || Name <- yz_solrq:solrq_worker_names()] + Self = self(), + %% + %% This callback function will be called from within the + %% yz_index_hashtree, after the hashtree we are exchanging + %% is snapshotted, but before it is updated. In this callback + %% we let the workers know they can resume normal operations, + %% and we inform ourself that workers have been resumed. + %% + SnapshotCompleteCallback = fun() -> + resume_workers(Self) end, - maybe_update_yz_index_hashtree( - ExchangeFSMPid, YZIndexHashtreeUpdateParams, CompleteCallback + spawn_link( + fun() -> + maybe_update_yz_index_hashtree( + ExchangeFSMPid, YZIndexHashtreeUpdateParams, SnapshotCompleteCallback + ), + gen_fsm:send_event(Self, yz_hashtree_updated) + end ), - {stop, normal, NewState}; + {next_state, wait_for_snapshot_complete, NewState}; _ -> - {next_state, wait, NewState} - end. + {next_state, wait_for_drain_complete, NewState} + end; +%% If a drain is already in progress, but we are draining all queues, +%% Try again until the previous drain completes. +wait_for_drain_complete({drain_already_in_progress, Token, QPid}, + #state{partition=undefined, + tokens = Tokens0}=State) -> + NewToken = yz_solrq_worker:drain(QPid, undefined), + Tokens1 = lists:delete(Token, Tokens0), + Tokens2 = [NewToken | Tokens1], + {next_state, wait_for_drain_complete, State#state{tokens = Tokens2}}; + +%% In the case of a single-partition drain, just stop and let +%% the calling process handle retries +wait_for_drain_complete({drain_already_in_progress, _Token}, State) -> + {stop, overlapping_drain_requested, State}. + +%% The workers have resumed normal operations. Draining is now "complete", +%% but we need to wait for the yz_index_hashtree to update its inner hashes, +%% which can take some time. In the meantime, notify the process waiting +%% for drains to complete +wait_for_snapshot_complete(resume_workers, #state{partition=Partition, owner_pid=OwnerPid} = State) -> + lists:foreach( + fun yz_solrq_worker:drain_complete/1, + get_solrq_ids(Partition) + ), + notify_workers_resumed(OwnerPid), + {next_state, wait_for_yz_hashtree_updated, State}. + +wait_for_yz_hashtree_updated(yz_hashtree_updated, State) -> + {stop, normal, State}. handle_event(_Event, StateName, State) -> {next_state, StateName, State}. -handle_sync_event(cancel, _From, _StateName, State) -> - [yz_solrq_worker:cancel_drain(Name) || Name <- yz_solrq:solrq_worker_names()], +handle_sync_event(cancel, _From, _StateName, #state{partition=Partition} = State) -> + [yz_solrq_worker:cancel_drain(Name) || Name <- get_solrq_ids(Partition)], {stop, normal, ok, State}; handle_sync_event(_Event, _From, StateName, State) -> @@ -185,8 +261,11 @@ code_change(_OldVsn, StateName, State, _Extra) -> %%% Internal functions %%%=================================================================== +notify_workers_resumed(OwnerPid) -> + OwnerPid ! workers_resumed. + maybe_update_yz_index_hashtree(undefined, undefined, Callback) -> - Callback(), + maybe_callback(Callback), ok; maybe_update_yz_index_hashtree(Pid, {YZTree, Index, IndexN}, Callback) -> yz_exchange_fsm:update_yz_index_hashtree(Pid, YZTree, Index, IndexN, Callback). @@ -209,7 +288,7 @@ maybe_send_drain_messages(_P, [], #state{ maybe_send_drain_messages(P, SolrqIds, State) -> TS = os:timestamp(), Tokens = [yz_solrq_worker:drain(SolrqId, P) || SolrqId <- SolrqIds], - {next_state, wait, State#state{tokens = Tokens, time_start = TS}}. + {next_state, wait_for_drain_complete, State#state{tokens = Tokens, time_start = TS}}. %% %% @doc if partition is `undefined` then drain all queues. @@ -222,3 +301,8 @@ get_solrq_ids(undefined) -> get_solrq_ids(P) -> yz_solrq:solrq_workers_for_partition(P). +maybe_callback(undefined) -> + ok; +maybe_callback(Callback) -> + Callback(). + diff --git a/src/yz_solrq_drain_mgr.erl b/src/yz_solrq_drain_mgr.erl index 63e4befb..33819520 100644 --- a/src/yz_solrq_drain_mgr.erl +++ b/src/yz_solrq_drain_mgr.erl @@ -141,30 +141,53 @@ maybe_drain(false, ExchangeFSMPid, Params) -> actual_drain(Params, ExchangeFSMPid) -> DrainTimeout = application:get_env(?YZ_APP_NAME, ?SOLRQ_DRAIN_TIMEOUT, ?SOLRQ_DRAIN_TIMEOUT_DEFAULT), - {ok, Pid} = yz_solrq_sup:start_drain_fsm(Params), + Params1 = [{owner_pid, self()} | Params], + {ok, Pid} = yz_solrq_sup:start_drain_fsm(Params1), Reference = erlang:monitor(process, Pid), yz_solrq_drain_fsm:start_prepare(Pid), try - receive - {'DOWN', Reference, process, Pid, normal} -> - lager:debug("Drain ~p completed normally.", [Pid]), - ok; - {'DOWN', Reference, process, Pid, Reason} -> - lager:debug("Drain ~p failed with reason ~p", [Pid, Reason]), - yz_stat:drain_fail(), - maybe_exchange_fsm_drain_error(ExchangeFSMPid, Reason), + WorkersResumed = wait_for_workers_resumed_or_crash(DrainTimeout, Reference, Pid, ExchangeFSMPid), + case WorkersResumed of + ok -> + wait_for_exit(Reference, Pid, ExchangeFSMPid); + {error, Reason} -> {error, Reason} - after DrainTimeout -> - lager:debug("Drain ~p timed out. Cancelling...", [Pid]), - yz_stat:drain_timeout(), - _ = cancel(Reference, Pid), - maybe_exchange_fsm_drain_error(ExchangeFSMPid, timeout), - {error, timeout} end after erlang:demonitor(Reference) end. +wait_for_workers_resumed_or_crash(DrainTimeout, Reference, Pid, ExchangeFSMPid) -> + receive + workers_resumed -> + lager:debug("Workers resumed."), + ok; + {'DOWN', Reference, process, Pid, Reason} -> + lager:error("Drain ~p exited prematurely.", [Pid]), + handle_drain_fsm_pid_crash(Reason, ExchangeFSMPid) + after DrainTimeout -> + lager:debug("Drain ~p timed out. Cancelling...", [Pid]), + yz_stat:drain_timeout(), + _ = cancel(Reference, Pid), + maybe_exchange_fsm_drain_error(ExchangeFSMPid, timeout), + {error, timeout} + end. + +wait_for_exit(Reference, Pid, ExchangeFSMPid) -> + receive + {'DOWN', Reference, process, Pid, normal} -> + lager:debug("Drain ~p completed normally.", [Pid]), + ok; + {'DOWN', Reference, process, Pid, Reason} -> + lager:error("Drain ~p crashed with reason ~p.", [Pid, Reason]), + handle_drain_fsm_pid_crash(Reason, ExchangeFSMPid) + end. + +handle_drain_fsm_pid_crash(Reason, ExchangeFSMPid) -> + yz_stat:drain_fail(), + maybe_exchange_fsm_drain_error(ExchangeFSMPid, Reason), + {error, Reason}. + enabled() -> application:get_env(?YZ_APP_NAME, ?SOLRQ_DRAIN_ENABLE, ?SOLRQ_DRAIN_ENABLE_DEFAULT). diff --git a/src/yz_solrq_helper.erl b/src/yz_solrq_helper.erl index be958edb..54425d49 100644 --- a/src/yz_solrq_helper.erl +++ b/src/yz_solrq_helper.erl @@ -34,8 +34,8 @@ %% TODO: Dynamically pulse_instrument. -ifdef(EQC). -%% -define(EQC_DEBUG(S, F), ok). --define(EQC_DEBUG(S, F), eqc:format(S, F)). +-define(EQC_DEBUG(S, F), _=element(1, {S, F}), ok). +%%-define(EQC_DEBUG(S, F), eqc:format(S, F)). %%-define(EQC_DEBUG(S, F), io:fwrite(user, S, F)). debug_entries(Entries) -> [erlang:element(1, Entry) || Entry <- Entries]. @@ -150,9 +150,9 @@ do_batch(Index, Entries0) -> LI = yz_cover:logical_index(Ring), OwnedAndNext = yz_misc:owned_and_next_partitions(node(), Ring), - Entries1 = [{BKey, Obj, Reason, P, - riak_kv_util:get_index_n(BKey), yz_kv:hash_object(Obj, P)} || - {BKey, Obj, Reason, P} <- + Entries1 = [{BKey, {Obj, _OldObj}, Reason, P, + riak_kv_util:get_index_n(BKey), yz_kv:hash_object(Obj, P)} || + {BKey, {Obj, _OldObj}, Reason, P} <- yz_misc:filter_out_fallbacks(OwnedAndNext, Entries0)], case update_solr(Index, LI, Entries1) of ok -> @@ -160,23 +160,20 @@ do_batch(Index, Entries0) -> ok; {ok, Entries2} -> update_aae_and_repair_stats(Entries2), - {ok, [{BKey, Obj, Reason, P} || {BKey, Obj, Reason, P, _, _} <- Entries2]}; + {ok, [{BKey, Objects, Reason, P} || {BKey, Objects, Reason, P, _, _} <- Entries2]}; {error, Reason} -> {error, Reason} end. -%% @doc Entries is [{Index, BKey, Obj, Reason, P, ShortPL, Hash}] +%% @doc Entries is [{BKey, Obj, Reason, P, ShortPL, Hash}] -spec update_solr(index_name(), logical_idx(), solr_entries()) -> ok | {ok, SuccessEntries :: solr_entries()} | {error, fuse_blown} | {error, tuple()}. update_solr(_Index, _LI, []) -> % nothing left after filtering fallbacks ok; -update_solr(Index, LI, Entries) when ?YZ_SHOULD_INDEX(Index) -> +update_solr(Index, LI, Entries0) when ?YZ_SHOULD_INDEX(Index) -> case yz_fuse:check(Index) of - ok -> - send_solr_ops_for_entries(Index, solr_ops(LI, Entries), - Entries); blown -> %% ?EQC_DEBUG( "Fuse Blown: can't currently send solr " %% "operations for index ~s", [Index]), @@ -186,8 +183,8 @@ update_solr(Index, LI, Entries) when ?YZ_SHOULD_INDEX(Index) -> %% yz_index:add_index/1 on 1st creation or diff-check. %% We send entries until we can ask again for %% ok | error, as we wait for the tick. - send_solr_ops_for_entries(Index, solr_ops(LI, Entries), - Entries) + Ops = solr_ops(LI, Entries0), + send_solr_ops_for_entries(Index, Ops, Entries0) end; update_solr(_Index, _LI, _Entries) -> ok. @@ -198,50 +195,83 @@ solr_ops(LI, Entries) -> [get_ops_for_entry(Entry, LI) || Entry <- Entries]. -spec get_ops_for_entry(solr_entry(), logical_idx()) -> solr_ops(). -get_ops_for_entry({BKey, Obj0, Reason, P, ShortPL, Hash}, LI) -> +get_ops_for_entry({BKey, {Obj0, _OldObj}=Objects, Reason, P, ShortPL, Hash}, LI) -> {Bucket, _} = BKey, BProps = riak_core_bucket:get_bucket(Bucket), Obj = yz_kv:maybe_merge_siblings(BProps, Obj0), ObjValues = riak_object:get_values(Obj), Action = get_reason_action(Reason), - get_ops_for_entry_action(Action, ObjValues, LI, P, Obj, BKey, ShortPL, + get_ops_for_entry_action(Action, ObjValues, LI, P, Objects, BKey, ShortPL, Hash, BProps). -spec get_ops_for_entry_action(write_action(), [riak_object:value()], - logical_idx(), p(), obj(), bkey(), short_preflist(), hash(), + logical_idx(), p(), object_pair(), bkey(), short_preflist(), hash(), riak_core_bucket:properties()) -> solr_ops(). -get_ops_for_entry_action(_Action, [notfound], _LI, _P, _Obj, BKey, - _ShortPL, _Hash, _BProps) -> - [{delete, yz_solr:encode_delete({bkey, BKey})}]; -get_ops_for_entry_action(anti_entropy_delete, _ObjValues, LI, P, Obj, _BKey, - _ShortPL, _Hash, _BProps) -> +get_ops_for_entry_action(_Action, [notfound], LI, P, _Objects, BKey, + _ShortPL, _Hash, _BProps) -> + LP = yz_cover:logical_partition(LI, P), + [{delete, yz_solr:encode_delete({bkey, BKey, LP})}]; +get_ops_for_entry_action(anti_entropy_delete, _ObjValues, LI, P, _FakeObjects, BKey, + _ShortPL, _Hash, _BProps) -> + %% anti-entropy is the "case of last resort" and at this point + %% we need to do a cleanup of _any_ documents that may be + %% floating around. + get_ops_for_object_cleanup(BKey, LI, P); +get_ops_for_entry_action(anti_entropy, _ObjValues, LI, P, {Obj, _OldObj}, BKey, + ShortPL, Hash, _BProps) -> + %% anti-entropy is the "case of last resort" and at this point + %% we need to do a cleanup of _any_ documents that may be + %% floating around. + DeleteOpsForEntry = get_ops_for_object_cleanup(BKey, LI, P), + AddOpsForEntry = get_ops_for_add(LI, ShortPL, P, Obj, Hash), + [DeleteOpsForEntry, AddOpsForEntry]; +get_ops_for_entry_action(delete, _ObjValues, LI, P, {Obj, _OldObj}, _BKey, + _ShortPL, _Hash, BProps) -> + [get_ops_for_deletes(LI, P, Obj, BProps)]; + +get_ops_for_entry_action(Action, _ObjValues, LI, P, {Obj, OldObj}, _BKey, + ShortPL, Hash, BProps) when + Action == handoff; + Action == put -> + DeleteOps = get_ops_for_deletes(LI, P, OldObj, BProps), + AddOps = get_ops_for_add(LI, ShortPL, P, Obj, Hash), + [DeleteOps, AddOps]. + +get_ops_for_object_cleanup(BKey, LI, P) -> + LP = yz_cover:logical_partition(LI, P), + [[{delete, yz_solr:encode_delete({bkey, BKey, LP})}]]. + +get_ops_for_add(LI, ShortPL, P, Obj, Hash) -> + LFPN = yz_cover:logical_partition(LI, element(1, ShortPL)), + LP = yz_cover:logical_partition(LI, P), + Docs = yz_doc:make_docs(Obj, Hash, ?INT_TO_BIN(LFPN), + ?INT_TO_BIN(LP)), + AddOps = yz_doc:adding_docs(Docs), + [{add, yz_solr:encode_doc(Doc)} || + Doc <- AddOps]. + + +get_ops_for_deletes(_LI, _P, no_old_object, _BProps) -> + []; +get_ops_for_deletes(LI, P, Obj, BProps) -> + case yz_kv:siblings_permitted(Obj, BProps) of + true -> + get_ops_for_sibling_deletes(LI, P, Obj); + _ -> + get_ops_for_no_sibling_deletes(LI, P, Obj) + end. + +get_ops_for_no_sibling_deletes(LI, P, Obj) -> + LP = yz_cover:logical_partition(LI, P), + DocId = yz_doc:doc_id(Obj, ?INT_TO_BIN(LP)), + [{delete, yz_solr:encode_delete({id, DocId})}]. + +get_ops_for_sibling_deletes(LI, P, Obj) -> LP = yz_cover:logical_partition(LI, P), DocIds = yz_doc:doc_ids(Obj, ?INT_TO_BIN(LP)), - DeleteOps = - [{delete, yz_solr:encode_delete({id, DocId})} - || DocId <- DocIds], - [DeleteOps]; -get_ops_for_entry_action(delete, _ObjValues, _LI, _P, _Obj, BKey, - _ShortPL, _Hash, _BProps) -> - [{delete, yz_solr:encode_delete({bkey, BKey})}]; -get_ops_for_entry_action(Action, _ObjValues, LI, P, Obj, BKey, - ShortPL, Hash, BProps) when Action == handoff; - Action == put; - Action == anti_entropy -> - LFPN = yz_cover:logical_partition(LI, element(1, ShortPL)), - LP = yz_cover:logical_partition(LI, P), - Docs = yz_doc:make_docs(Obj, Hash, ?INT_TO_BIN(LFPN), - ?INT_TO_BIN(LP)), - AddOps = yz_doc:adding_docs(Docs), - DeleteOps = yz_kv:delete_operation(BProps, Obj, Docs, BKey, - LP), - - OpsForEntry = [[{delete, yz_solr:encode_delete(DeleteOp)} || - DeleteOp <- DeleteOps], - [{add, yz_solr:encode_doc(Doc)} - || Doc <- AddOps] - ], - [OpsForEntry]. + DeleteOps = [{delete, yz_solr:encode_delete({id, DocId})} + || DocId <- DocIds], + [DeleteOps]. %% @doc A function that takes in an `Index', a list of `Ops' and the list %% of `Entries', and attempts to batch_index them into Solr. @@ -256,8 +286,9 @@ get_ops_for_entry_action(Action, _ObjValues, LI, P, Obj, BKey, {error, tuple()}. send_solr_ops_for_entries(Index, Ops, Entries) -> T1 = os:timestamp(), + PreparedOps = prepare_ops_for_batch(Index, Ops), %% ?EQC_DEBUG("send_solr_ops_for_entries: About to send entries. ~p", Entries), - case yz_solr:index_batch(Index, prepare_ops_for_batch(Ops)) of + case yz_solr:index_batch(Index, PreparedOps) of ok -> yz_stat:index_end(Index, length(Ops), ?YZ_TIME_ELAPSED(T1)), ok; @@ -295,9 +326,8 @@ send_solr_single_ops(Index, Ops) -> end, Ops). - single_op_batch(Index, Op) -> - Ops = prepare_ops_for_batch([Op]), + Ops = prepare_ops_for_batch(Index, [Op]), case yz_solr:index_batch(Index, Ops) of ok -> T1 = os:timestamp(), @@ -382,8 +412,10 @@ get_reason_action(Reason) when is_tuple(Reason) -> get_reason_action(Reason) -> Reason. --spec prepare_ops_for_batch(solr_ops()) -> solr_entries(). -prepare_ops_for_batch(Ops) -> +-define(QUERY(Bin), {struct, [{'query', Bin}]}). + +-spec prepare_ops_for_batch(index_name(), solr_ops()) -> solr_entries(). +prepare_ops_for_batch(_Index, Ops) -> %% Flatten combined operators for a batch. lists:flatten(Ops). @@ -394,4 +426,4 @@ terminate(_Reason, _State) -> ok. code_change(_OldVsn, State, _Extra) -> - {ok, State}. + {ok, State}. \ No newline at end of file diff --git a/src/yz_solrq_sup.erl b/src/yz_solrq_sup.erl index 7d5e7819..064ed12d 100644 --- a/src/yz_solrq_sup.erl +++ b/src/yz_solrq_sup.erl @@ -47,6 +47,10 @@ start_drain_fsm(Parameters) -> ). -spec start_queue_pair(Index::index_name(), Partition::p()) -> ok. start_queue_pair(Index, Partition) -> + lager:info( + "Starting solrq supervisor for index ~p and partition ~p", + [Index, Partition] + ), validate_child_started( supervisor:start_child(?MODULE, queue_pair_spec({Index, Partition}))). @@ -100,15 +104,18 @@ validate_child_started(Error) -> throw(Error). required_queues() -> - {ok, Ring} = riak_core_ring_manager:get_my_ring(), - Partitions = riak_core_ring:my_indices(Ring), - Indexes = yz_index:get_indexes_from_meta(), - [{Index, Partition} || + AllVnodes = riak_core_vnode_manager:all_vnodes(riak_kv_vnode), + Partitions = [Idx || {_Mod, Idx, _Pid} <- AllVnodes], + %% Indexes includes ?YZ_INDEX_TOMBSTONE because we need to write the entries + %% for non-indexed data to the YZ AAE tree. Excluding them makes this process + %% constantly start and stop these queues. + Indexes = yz_index:get_indexes_from_meta() ++ [?YZ_INDEX_TOMBSTONE], + CalculatedQueues = [{Index, Partition} || Partition <- Partitions, - Index <- Indexes]. -%% TODO: we shouldn't need ?YZ_INDEX_TOMBSTONE if we just update the YZ AAE tree -%% when we call index rather than pushing the value all the way to the solrq - %%Index =/= ?YZ_INDEX_TOMBSTONE]. + Index <- Indexes], + CalculatedQueues. + %% TODO: we shouldn't need ?YZ_INDEX_TOMBSTONE if we just update the YZ AAE tree + %% when we call index rather than pushing the value all the way to the solrq sync_active_queue_pairs() -> ActiveQueues = active_queues(), @@ -120,6 +127,10 @@ sync_active_queue_pairs() -> ok. stop_queue_pair(Index, Partition) -> + lager:info( + "Stopping solrq supervisor for index ~p and partition ~p", + [Index, Partition] + ), SupId = {Index, Partition}, case supervisor:terminate_child(?MODULE, SupId) of ok -> diff --git a/src/yz_solrq_worker.erl b/src/yz_solrq_worker.erl index b6e3df3b..9e418d61 100644 --- a/src/yz_solrq_worker.erl +++ b/src/yz_solrq_worker.erl @@ -47,12 +47,12 @@ -define(COUNT_PER_REPORT, 20). -type solrq_message() :: tuple(). % {BKey, Docs, Reason, P}. --type pending_vnode() :: {pid(), atom()}. --type drain_info() :: {pid(), reference()} | undefined. +-type pending_processes() :: {pid(), atom()}. +-type drain_info() :: {pid(), Token::reference(), DrainFSMMonitor::reference()} | undefined. -type worker_state_status() :: {all_queue_len, non_neg_integer()} | {queue_hwm, non_neg_integer()} - | {pending_vnode, pending_vnode()} + | {pending_processes, pending_processes()} | {drain_info, drain_info()} | {purge_strategy, purge_strategy() | {queue, yz_queue()} @@ -74,12 +74,14 @@ index :: index_name(), partition :: p(), queue_hwm = 1000 :: non_neg_integer(), - pending_vnode = none :: pending_vnode() | none, + %% Both the vnode and the yz_exchange_fsm can call into `index`. + %% Therefore, we need a list of processes, not just a single vnode PID + pending_processes = [] :: [pending_processes()], drain_info = undefined :: drain_info(), purge_strategy :: purge_strategy(), helper_pid = undefined :: pid() | undefined, queue = queue:new() :: yz_queue(), % solrq_message() - timer_ref = undefined :: reference()|undefined, + flush_timer_ref = undefined :: reference()|undefined, batch_min = yz_solrq:get_min_batch_size() :: solrq_batch_min(), batch_max = yz_solrq:get_max_batch_size() :: solrq_batch_max(), delayms_max = yz_solrq:get_flush_interval() :: solrq_batch_flush_interval(), @@ -113,9 +115,9 @@ status(QPid) -> status(QPid, Timeout) -> gen_server:call(QPid, status, Timeout). --spec index(solrq_id(), bkey(), obj(), write_reason(), p()) -> ok. -index(QPid, BKey, Obj, Reason, P) -> - gen_server:call(QPid, {index, {BKey, Obj, Reason, P}}, infinity). +-spec index(solrq_id(), bkey(), object_pair(), write_reason(), p()) -> ok. +index(QPid, BKey, Objects, Reason, P) -> + gen_server:call(QPid, {index, {BKey, Objects, Reason, P}}, infinity). -spec set_hwm(Index :: index_name(), Partition :: p(), HWM :: solrq_hwm()) -> {ok, OldHWM :: solrq_hwm()} | {error, bad_hwm_value}. @@ -215,13 +217,13 @@ handle_call({index, E}, From, State0) -> State2 = maybe_send_reply(From, State1), State3 = enqueue(E, State2), State4 = maybe_send_batch_to_helper(State3), - FinalState = maybe_start_timer(State4), + FinalState = maybe_start_flush_timer(State4), ?EQC_DEBUG("index. NewState: ~p~n", [debug_state(FinalState)]), {noreply, FinalState}; handle_call(status, _From, #state{} = State) -> {reply, internal_status(State), State}; handle_call({set_hwm, NewHWM}, _From, #state{queue_hwm = OldHWM} = State) -> - {reply, {ok, OldHWM}, maybe_unblock_vnodes(State#state{queue_hwm = NewHWM})}; + {reply, {ok, OldHWM}, maybe_unblock_processes(State#state{queue_hwm = NewHWM})}; handle_call(get_hwm, _From, #state{queue_hwm = HWM} = State) -> {reply, HWM, State}; handle_call({set_index, Min, Max, DelayMS}, _From, State0) -> @@ -238,7 +240,7 @@ handle_call({set_purge_strategy, NewPurgeStrategy}, #state{purge_strategy=OldPurgeStrategy} = State) -> {reply, {ok, OldPurgeStrategy}, State#state{purge_strategy=NewPurgeStrategy}}; handle_call(cancel_drain, _From, State) -> - NewState = handle_drain_complete(State), + NewState = stop_draining(State), {reply, ok, NewState}; handle_call(all_queue_len, _From, #state{queue = Queue} = State) -> Len = queue:len(Queue), @@ -257,12 +259,16 @@ handle_cast(stop, State) -> %% during its prepare state, typically as the result of %% a request to drain the queues. %% -%% This handler will iterate over all IndexQs in the -%% solrq, and initiate a drain on the queue, if it is currently +%% This handler initiate a drain on the queue, if it is currently %% non-empty. %% @end %% -%% TODO: +handle_cast({drain, DPid, Token, _TargetPartition}, + #state{draining = Draining}=State) when Draining =/= false -> + %% Drain already in progress - notify caller and continue + yz_solrq_drain_fsm:drain_already_in_progress(DPid, Token), + {noreply, State}; + handle_cast({drain, DPid, Token, TargetPartition}, #state{queue = Queue, in_flight_len = InFlightLen, @@ -270,23 +276,9 @@ handle_cast({drain, DPid, Token, TargetPartition}, when TargetPartition == undefined; TargetPartition == QueueParitition -> ?EQC_DEBUG("drain{~p=DPid, ~p=Token, ~p=Partition}. State: ~p~n", [DPid, Token, TargetPartition, internal_status(State)]), - NewState0 = case {queue:is_empty(Queue), InFlightLen} of - {true, 0} -> - State#state{draining = wait_for_drain_complete}; - {true, _InFlightLen} -> - State#state{draining = true}; - _ -> - drain_queue(State) - end, - NewState = - case NewState0#state.draining of - wait_for_drain_complete -> - yz_solrq_drain_fsm:drain_complete(DPid, Token), - NewState0; - _ -> - NewState0#state{drain_info = {DPid, Token}} - - end, + NewState0 = monitor_draining_process(DPid, State, Token), + NewState = maybe_drain_queue(Queue, InFlightLen, NewState0), + maybe_send_drain_complete(NewState#state.draining, DPid, Token), ?EQC_DEBUG("drain. NewState: ~p~n", [internal_status(NewState)]), {noreply, NewState}; @@ -306,7 +298,7 @@ handle_cast(blown_fuse, State) -> %% @end %% handle_cast(healed_fuse, #state{} = State) -> - State1 = maybe_start_timer(State#state{fuse_blown = false}), + State1 = maybe_start_flush_timer(State#state{fuse_blown = false}), State2 = maybe_send_batch_to_helper(State1), {noreply, State2}; @@ -324,7 +316,7 @@ handle_cast(healed_fuse, #state{} = State) -> %% that have not been delivered (a subset of what was sent for delivery) %% and should be retried. This handler will decrement the all_queues_len %% field on the solrq state record by the supplied NumDelievered value -%% thus potentially unblocking any vnodes waiting on this solrq instance, +%% thus potentially unblocking any processes waiting on this solrq instance, %% if the number of queued messages are above the high water mark. %% @end %% @@ -332,9 +324,9 @@ handle_cast({batch_complete, {_NumDelivered, Result}}, #state{} = State) -> ?EQC_DEBUG("batch_complete. State: ~p~n", [debug_state(State)]), State1 = handle_batch(Result, State#state{in_flight_len = 0}), - State2 = maybe_unblock_vnodes(State1), + State2 = maybe_unblock_processes(State1), State3 = maybe_send_batch_to_helper(State2), - State4 = maybe_start_timer(State3), + State4 = maybe_start_flush_timer(State3), ?EQC_DEBUG("batch_complete. NewState: ~p~n", [debug_state(State4)]), {noreply, State4}; @@ -348,27 +340,54 @@ handle_cast({batch_complete, {_NumDelivered, Result}}, %% @end %% handle_cast(drain_complete, State) -> - State1 = handle_drain_complete(State), + State1 = stop_draining(State), {noreply, State1}. -handle_drain_complete(#state{queue = Queue, - aux_queue = AuxQueue} = State) -> +monitor_draining_process(DPid, State, Token) -> + MonitorRef = erlang:monitor(process, DPid), + State#state{drain_info = {DPid, Token, MonitorRef}}. + +maybe_drain_queue(Queue, InFlightLen, State) -> + case {queue:is_empty(Queue), InFlightLen} of + {true, 0} -> + %% If our queue is empty, and we have no in-flight messages, + %% skip the `draining=true` state and go + %% straight to `wait_for_drain_complete` + State#state{draining = wait_for_drain_complete}; + _ -> + drain_queue(State) + end. + +stop_draining(#state{queue = Queue, + aux_queue = AuxQueue, + drain_info = DrainInfo} = State) -> + maybe_demonitor_draining_process(DrainInfo), State1 = State#state{ queue = queue:join(Queue, AuxQueue), aux_queue = queue:new(), - draining = false}, + draining = false, + drain_info=undefined}, State2 = maybe_send_batch_to_helper(State1), - State3 = maybe_start_timer(State2), - State3#state{drain_info = undefined}. + maybe_start_flush_timer(State2). %% @doc Timer has fired - request a helper. handle_info({timeout, TimerRef, flush}, - #state{timer_ref = TimerRef} = State) -> - {noreply, flush(State#state{timer_ref = undefined})}; + #state{flush_timer_ref = TimerRef} = State) -> + {noreply, flush(State#state{flush_timer_ref = undefined})}; handle_info({timeout, _TimerRef, flush}, State) -> - lager:info("Received timeout from stale Timer Reference"), + lager:debug("Received timeout from stale Timer Reference"), + {noreply, State}; + +handle_info({'DOWN', MonitorRef, process, DPid, Info}, + #state{draining=Draining, drain_info = {DPid, _Token, MonitorRef}} = State) + when Draining =:= true; Draining =:= wait_for_drain_complete -> + lager:info("Drain FSM terminated for reason ~p without notifying of drain_complete. Resuming normal operations.", [Info]), + NewState = stop_draining(State), + {noreply, NewState}; + +handle_info({'DOWN', _MonitorRef, process, _DPid, _Info}, State) -> {noreply, State}. terminate(_Reason, _State) -> @@ -413,7 +432,7 @@ handle_batch(ok, #state{draining = true, queue = Queue}=State) -> case queue:is_empty(Queue) of true -> - #state{drain_info={DPid, Token}, batch_start = T1} = State, + #state{drain_info={DPid, Token, _MonitorRef}, batch_start = T1} = State, yz_stat:batch_end(?YZ_TIME_ELAPSED(T1)), yz_solrq_drain_fsm:drain_complete(DPid, Token), State#state{draining = wait_for_drain_complete, @@ -453,19 +472,19 @@ internal_status(#state{queue = Queue, aux_queue = AuxQueue} = State) -> ++ [{queue_len, queue:len(Queue)}, {aux_queue_len, queue:len(AuxQueue)}]. %% @doc Check HWM, if we are not over it send a reply. -maybe_send_reply(From, #state{} = State) -> +maybe_send_reply(From, #state{pending_processes = PendingProcesses} = State) -> case over_hwm(State) of true -> - log_blocked_vnode(From, State), + log_blocked_process(From, State), yz_stat:blocked_vnode(From), - State#state{pending_vnode = From}; + State#state{pending_processes = [From | PendingProcesses] }; false -> gen_server:reply(From, ok), State end. -log_blocked_vnode(From, State) -> - lager:info("Blocking vnode ~p due to SolrQ ~p exceeding HWM of ~p", [ +log_blocked_process(From, State) -> + lager:info("Blocking process ~p due to SolrQ ~p exceeding HWM of ~p", [ From, self(), State#state.queue_hwm]), @@ -503,7 +522,7 @@ flush(State) -> handle_blown_fuse(#state{} = State) -> State1 = State#state{fuse_blown = true}, State2 = maybe_purge(State1), - State3 = maybe_unblock_vnodes(State2), + State3 = maybe_unblock_processes(State2), State3. %% @doc purge entries depending on purge strategy if we are over the HWM @@ -534,8 +553,6 @@ queue_has_items(#state{queue = Queue, %% to purge from the aux_queue, if it is not empty. %% purge_index: Purge all entries (both in the regular queue and in the %% aux_queue) from a randomly blown indexq -%% purge_all: Purge all entries (both in the regular queue and in the -%% aux_queue) from all blown indexqs %% -spec purge(state()) -> state(). purge(State) -> @@ -633,7 +650,7 @@ send_batch_to_helper(#state{index = Index, State2; false -> % may be another full batch - maybe_start_timer(State2) + maybe_start_flush_timer(State2) end. %% @doc Get up to batch_max entries. @@ -651,46 +668,51 @@ prepare_for_batching(#state{queue = Q, batch_max = Max} = State) -> %% If previous `TimerRef' was set, but not yet triggered, cancel it %% first as it's invalid for the next batch of this queue. -maybe_cancel_timer(#state{timer_ref=undefined} = State) -> +maybe_cancel_timer(#state{flush_timer_ref =undefined} = State) -> State; -maybe_cancel_timer(#state{timer_ref=TimerRef} = State) -> +maybe_cancel_timer(#state{flush_timer_ref =TimerRef} = State) -> erlang:cancel_timer(TimerRef), - State#state{timer_ref =undefined}. + State#state{flush_timer_ref =undefined}. -%% @doc Send replies to blocked vnodes if under the high water mark +%% @doc Send replies to blocked processes if under the high water mark %% and return updated state --spec maybe_unblock_vnodes(state()) -> state(). -maybe_unblock_vnodes(#state{pending_vnode = none} = State) -> +-spec maybe_unblock_processes(state()) -> state(). +maybe_unblock_processes(#state{pending_processes = []} = State) -> State; -maybe_unblock_vnodes(#state{pending_vnode = PendingVnode} = State) -> +maybe_unblock_processes(#state{pending_processes = PendingProcesses, queue_hwm = QueueHWM} = State) -> case over_hwm(State) of true -> State; _ -> - lager:debug("Unblocking vnode ~p due to SolrQ ~p going below HWM of ~p", [ - PendingVnode, - self(), - State#state.queue_hwm]), - gen_server:reply(PendingVnode, ok), - State#state{pending_vnode = none} + lists:foreach(fun(PendingProcess) -> + unblock_process(PendingProcess, QueueHWM) end, + PendingProcesses), + State#state{pending_processes = []} end. -maybe_start_timer(#state{delayms_max = infinity}=State) -> +unblock_process(PendingProcess, HWM) -> + lager:debug("Unblocking process ~p due to SolrQ ~p going below HWM of ~p", [ + PendingProcess, + self(), + HWM]), + gen_server:reply(PendingProcess, ok). + +maybe_start_flush_timer(#state{delayms_max = infinity} = State) -> lager:debug("Infinite delay, will not start timer and flush."), - State#state{timer_ref = undefined}; -maybe_start_timer(#state{timer_ref = undefined, - fuse_blown = false, - delayms_max = DelayMS, - queue = Queue} = State) -> + State#state{flush_timer_ref = undefined}; +maybe_start_flush_timer(#state{flush_timer_ref = undefined, + fuse_blown = false, + delayms_max = DelayMS, + queue = Queue} = State) -> case queue:is_empty(Queue) of true -> State; false -> TimerRef = erlang:start_timer(DelayMS, self(), flush), - State#state{timer_ref = TimerRef} + State#state{flush_timer_ref = TimerRef} end; %% timer already running or blown fuse, so don't start a new timer. -maybe_start_timer(State) -> +maybe_start_flush_timer(State) -> State. %% @doc Read settings from the application environment @@ -732,10 +754,23 @@ debug_state(State) -> {all_queue_len, Len}, {drain_info, State#state.drain_info}, {batch_start, State#state.batch_start}, - {timer_ref, State#state.timer_ref} + {timer_ref, State#state.flush_timer_ref} ] ++ debug_indexq(State). get_helper(Index, Partition, State0) -> HelperName = yz_solrq:helper_regname(Index, Partition), HPid = whereis(HelperName), State0#state{helper_pid = HPid}. + +maybe_send_drain_complete(wait_for_drain_complete, DPid, Token) -> + yz_solrq_drain_fsm:drain_complete(DPid, Token); +maybe_send_drain_complete(_Draining, _DPid, _Token) -> + ok. + +maybe_demonitor_draining_process({_DPid, _TokenRef, MonRef}) + when is_reference(MonRef) -> + %% Demonitor and remove a potential 'DOWN' + %% message from this MonRef in our mailbox + erlang:demonitor(MonRef, [flush]); +maybe_demonitor_draining_process(undefined) -> + ok. diff --git a/test/yokozuna_schema_tests.erl b/test/yokozuna_schema_tests.erl index a573c0c6..634c2ef0 100644 --- a/test/yokozuna_schema_tests.erl +++ b/test/yokozuna_schema_tests.erl @@ -104,8 +104,7 @@ override_schema_test() -> {["search", "queue", "batch", "maximum"], 10000}, {["search", "queue", "batch", "flush_interval"], infinity}, {["search", "queue", "high_watermark"], 100000}, - {["search", "queue", "high_watermark", "purge_strategy"], - "purge_all"}, + {["search", "queue", "high_watermark", "purge_strategy"], "purge_index"}, {["search", "queue", "drain", "enable"], "off"}, {["search", "queue", "drain", "timeout"], "2m"}, {["search", "queue", "drain", "cancel", "timeout"], "10ms"}, @@ -144,7 +143,7 @@ override_schema_test() -> cuttlefish_unit:assert_config(Config, "yokozuna.fuse_ctx", sync), cuttlefish_unit:assert_config(Config, "yokozuna.solrq_hwm_purge_strategy", - purge_all), + purge_index), cuttlefish_unit:assert_config(Config, "yokozuna.solrq_batch_min", 100), cuttlefish_unit:assert_config(Config, "yokozuna.solrq_batch_max", 10000), cuttlefish_unit:assert_config(Config, "yokozuna.solrq_batch_flush_interval", diff --git a/test/yz_component_tests.erl b/test/yz_component_tests.erl index 967886bd..7faf6871 100644 --- a/test/yz_component_tests.erl +++ b/test/yz_component_tests.erl @@ -6,7 +6,7 @@ disable_index_test()-> yokozuna:disable(index), - ?assertEqual(yz_kv:index(riak_object:new({<<"type">>, <<"bucket">>}, <<"key">>, <<"value">>), delete, {}), ok). + ?assertEqual(yz_kv:index({riak_object:new({<<"type">>, <<"bucket">>}, <<"key">>, <<"value">>), no_old_object}, delete, {}), ok). disable_search_test()-> yokozuna:disable(search), diff --git a/test/yz_solrq_eqc.erl b/test/yz_solrq_eqc.erl index 593c7a1d..dc4937ec 100644 --- a/test/yz_solrq_eqc.erl +++ b/test/yz_solrq_eqc.erl @@ -139,7 +139,6 @@ prop_ok() -> Pids = ?MODULE:send_entries(PE), yz_solrq_drain_mgr:drain(), wait_for_vnodes(Pids, timer:seconds(20)), - timer:sleep(500), catch yz_solrq_eqc_ibrowse:wait(expected_keys(Entries)), IBrowseKeys = yz_solrq_eqc_ibrowse:keys(), MeltsByIndex = melts_by_index(Entries), @@ -152,7 +151,7 @@ prop_ok() -> eqc:format("melts_by_index: ~p~n", [MeltsByIndex]), eqc:format("Entries: ~p\n", [Entries]), %debug_history([ibrowse, solr_responses, yz_kv]) - debug_history([solr_responses]) + eqc:format("debug history: ~p\n", [debug_history([solr_responses, ibrowse])]) end, begin %% For each vnode, spawn a process and start sending @@ -176,7 +175,7 @@ prop_ok() -> %% TODO Modify ordering test to center around key order, NOT partition order. %% requires a fairly significant change to the test structure, becuase currently %% all keys are unique. - {insert_order, ordered(expected_entry_keys(PE), IBrowseKeys)}, + %% {insert_order, ordered(expected_entry_keys(PE), IBrowseKeys)}, {melts, equals(MeltsByIndex, errors_by_index(Entries))} ]) % ) @@ -541,7 +540,7 @@ send_vnode_entries(Runner, P, Events) -> Runner ! {self(), done}. make_obj(B,K) -> - riak_object:new(B, K, K, "application/yz_solrq_eqc"). % Set Key as value + {riak_object:new(B, K, K, "application/yz_solrq_eqc"), no_old_object}. % Set Key as value %% Wait for send_entries - should probably set a global timeout and %% and look for that instead @@ -573,9 +572,11 @@ wait_for_vnodes_msgs([Pid | Pids], Ref) -> start_solrqs(Partitions, Indexes) -> %% Ring retrieval for required workers - meck:expect(riak_core_ring_manager, get_my_ring, fun() -> {ok, not_a_real_ring} end), - meck:expect(riak_core_ring, my_indices, fun(_) -> unique_entries(Partitions) end), - meck:expect(yz_index, get_indexes_from_meta, fun() -> unique_entries(Indexes) end), + UniquePartitions = unique_entries(Partitions), + meck:expect(riak_core_vnode_manager, all_vnodes, fun(riak_kv_vnode) -> + [{riak_kv_vnode, Idx, fake_pid} || Idx <- UniquePartitions] + end), + meck:expect(yz_index, get_indexes_from_meta, fun() -> unique_entries(Indexes) -- [?YZ_INDEX_TOMBSTONE] end), %% And start up supervisors to own the solrq/solrq helper _ = yz_solrq_sup:start_link(), _ = yz_solrq_sup:sync_active_queue_pairs(). @@ -587,8 +588,7 @@ start_solrqs(Partitions, Indexes) -> %% {parse_solr_url(Url), parse_solr_reqs(mochijson2:decode(JsonIolist))}. debug_history(Mods) -> - [io:format("~p\n====\n~p\n\n", [Mod, meck:history(Mod)]) || Mod <- Mods], - ok. + [{Mod, meck:history(Mod)} || Mod <- Mods]. -else. %% EQC is not defined diff --git a/test/yz_solrq_eqc_ibrowse.erl b/test/yz_solrq_eqc_ibrowse.erl index c7a62758..60e2c1e1 100644 --- a/test/yz_solrq_eqc_ibrowse.erl +++ b/test/yz_solrq_eqc_ibrowse.erl @@ -32,7 +32,7 @@ code_change/3]). -ifdef(EQC). -%% -define(EQC_DEBUG(S, F), eqc:format(S, F)). +%%-define(EQC_DEBUG(S, F), io:fwrite(user, S, F)). -define(EQC_DEBUG(S, F), ok). -else. -define(EQC_DEBUG(S, F), ok). @@ -83,7 +83,7 @@ handle_call( expected=Expected } = State ) -> - ?EQC_DEBUG("In get_response", []), + ?EQC_DEBUG("In get_response~n", []), SolrReqs = parse_solr_reqs(mochijson2:decode(B)), {Keys, Res, NewFailed} = get_response(SolrReqs, KeyRes, AlredyFailed), WrittenKeys = case Res of @@ -114,7 +114,7 @@ handle_call({wait, Keys}, From, #state{written=Written} = State) -> true -> {reply, ok, State}; _ -> - ?EQC_DEBUG("Process ~p waiting for keys...: ~p", [From, Keys]), + ?EQC_DEBUG("Process ~p waiting for keys...: ~p~n", [From, Keys]), {noreply, State#state{root=From, expected=lists:usort(Keys)}} end end; @@ -146,6 +146,8 @@ parse_solr_req({<<"add">>, {struct, [{<<"doc">>, Doc}]}}) -> {add, find_key_field(Doc)}; parse_solr_req({<<"delete">>, {struct, [{<<"query">>, Query}]}}) -> {delete, parse_delete_query(Query)}; +parse_solr_req({<<"delete">>, {struct, [{<<"id">>, Id}]}}) -> + {delete, parse_delete_id(Id)}; parse_solr_req({delete, _Query}) -> {delete, could_parse_bkey}; parse_solr_req({Other, Thing}) -> @@ -158,6 +160,10 @@ parse_delete_query(Query) -> {match, [Key]} = re:run(Query, "(XKEYX[0-9]+)",[{capture,[1],binary}]), Key. +parse_delete_id(Id) -> + {match, [Key]} = re:run(Id, "(XKEYX[0-9]+)",[{capture,[1],binary}]), + Key. + %% Decide what to return for the request... If any of the seq %% numbers had failures generated, apply to all of them, but @@ -197,4 +203,4 @@ maybe_reply(undefined) -> ok; maybe_reply(Root) -> timer:sleep(500), - gen_server:reply(Root, ok). \ No newline at end of file + gen_server:reply(Root, ok).