From 6628c464b8c7f6a06172b18d38b6cbdc9d765895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Wed, 27 Mar 2024 11:14:47 +0100 Subject: [PATCH 01/18] Prepare next-major branch --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eba307cbea..1acca9819b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +# NEXT MAJOR RELEASE + +### Enhancements +* (PR [#????](https://github.com/realm/realm-core/pull/????)) +* None. + +### Fixed +* ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) +* None. + +### Breaking changes +* None. + +### Compatibility +* Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. + +----------- + +### Internals +* None. + +---------------------------------------------- + # NEXT RELEASE ### Enhancements From ecdd6cac9a9a33b7e4e697918ade4f5931d91bb1 Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Thu, 11 Apr 2024 16:13:51 +0100 Subject: [PATCH 02/18] RCORE-2050 New Array Header support for compression (#7521) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * code review --------- Co-authored-by: Finn Schiermer Andersen Co-authored-by: Jørgen Edelbo --- src/realm/alloc.cpp | 7 +- src/realm/alloc.hpp | 23 +- src/realm/alloc_slab.cpp | 3 +- src/realm/array.cpp | 36 +- src/realm/array.hpp | 5 +- src/realm/array_basic_tpl.hpp | 16 +- src/realm/node.cpp | 48 ++- src/realm/node.hpp | 33 +- src/realm/node_header.hpp | 638 ++++++++++++++++++++++++++++++---- test/test_alloc.cpp | 42 ++- 10 files changed, 683 insertions(+), 168 deletions(-) diff --git a/src/realm/alloc.cpp b/src/realm/alloc.cpp index f15cddfdffd..c19043b117a 100644 --- a/src/realm/alloc.cpp +++ b/src/realm/alloc.cpp @@ -113,7 +113,8 @@ Allocator& Allocator::get_default() noexcept // * adding a cross-over mapping. (if the array crosses a mapping boundary) // * using an already established cross-over mapping. (ditto) // this can proceed concurrently with other calls to translate() -char* Allocator::translate_less_critical(RefTranslation* ref_translation_ptr, ref_type ref) const noexcept +char* Allocator::translate_less_critical(RefTranslation* ref_translation_ptr, ref_type ref, + bool known_in_slab) const noexcept { size_t idx = get_section_index(ref); RefTranslation& txl = ref_translation_ptr[idx]; @@ -122,7 +123,9 @@ char* Allocator::translate_less_critical(RefTranslation* ref_translation_ptr, re #if REALM_ENABLE_ENCRYPTION realm::util::encryption_read_barrier(addr, NodeHeader::header_size, txl.encrypted_mapping, nullptr); #endif - auto size = NodeHeader::get_byte_size_from_header(addr); + // if we know the translation is inside the slab area, we don't need to check + // for anything beyond the header, and we don't need to check if decryption is needed + auto size = known_in_slab ? 8 : NodeHeader::get_byte_size_from_header(addr); bool crosses_mapping = offset + size > (1 << section_shift); // Move the limit on use of the existing primary mapping. // Take into account that another thread may attempt to change / have changed it concurrently, diff --git a/src/realm/alloc.hpp b/src/realm/alloc.hpp index f5d23e229b4..788e5ea94cc 100644 --- a/src/realm/alloc.hpp +++ b/src/realm/alloc.hpp @@ -113,6 +113,9 @@ class Allocator { /// Calls do_translate(). char* translate(ref_type ref) const noexcept; + /// Simpler version if we know the ref points inside the slab area + char* translate_in_slab(ref_type ref) const noexcept; + /// Returns true if, and only if the object at the specified 'ref' /// is in the immutable part of the memory managed by this /// allocator. The method by which some objects become part of the @@ -249,8 +252,8 @@ class Allocator { /// then entirely the responsibility of the caller that the memory /// is not modified by way of the returned memory pointer. virtual char* do_translate(ref_type ref) const noexcept = 0; - char* translate_critical(RefTranslation*, ref_type ref) const noexcept; - char* translate_less_critical(RefTranslation*, ref_type ref) const noexcept; + char* translate_critical(RefTranslation*, ref_type ref, bool known_in_slab = false) const noexcept; + char* translate_less_critical(RefTranslation*, ref_type ref, bool known_in_slab = false) const noexcept; virtual void get_or_add_xover_mapping(RefTranslation&, size_t, size_t, size_t) = 0; Allocator() noexcept; size_t get_section_index(size_t pos) const noexcept; @@ -556,7 +559,8 @@ inline Allocator::Allocator() noexcept } // performance critical part of the translation process. Less critical code is in translate_less_critical. -inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr, ref_type ref) const noexcept +inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr, ref_type ref, + bool known_in_slab) const noexcept { size_t idx = get_section_index(ref); RefTranslation& txl = ref_translation_ptr[idx]; @@ -574,7 +578,7 @@ inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr, } else { // the lowest possible xover offset may grow concurrently, but that will be handled inside the call - return translate_less_critical(ref_translation_ptr, ref); + return translate_less_critical(ref_translation_ptr, ref, known_in_slab); } } realm::util::terminate("Invalid ref translation entry", __FILE__, __LINE__, txl.cookie, 0x1234567890, ref, idx); @@ -592,6 +596,17 @@ inline char* Allocator::translate(ref_type ref) const noexcept } } +inline char* Allocator::translate_in_slab(ref_type ref) const noexcept +{ + auto ref_translation_ptr = m_ref_translation_ptr.load(std::memory_order_acquire); + if (REALM_LIKELY(ref_translation_ptr)) { + return translate_critical(ref_translation_ptr, ref, true); + } + else { + return do_translate(ref); + } +} + } // namespace realm diff --git a/src/realm/alloc_slab.cpp b/src/realm/alloc_slab.cpp index 60b441a940b..9e2325dcf04 100644 --- a/src/realm/alloc_slab.cpp +++ b/src/realm/alloc_slab.cpp @@ -235,7 +235,7 @@ MemRef SlabAlloc::do_alloc(size_t size) #endif char* addr = reinterpret_cast(entry); - REALM_ASSERT_EX(addr == translate(ref), addr, ref, get_file_path_for_assertions()); + REALM_ASSERT_EX(addr == translate_in_slab(ref), addr, ref, get_file_path_for_assertions()); #if REALM_ENABLE_ALLOC_SET_ZERO std::fill(addr, addr + size, 0); @@ -300,6 +300,7 @@ SlabAlloc::FreeBlock* SlabAlloc::pop_freelist_entry(FreeList list) void SlabAlloc::FreeBlock::unlink() { + REALM_ASSERT_DEBUG(next != nullptr && prev != nullptr); auto _next = next; auto _prev = prev; _next->prev = prev; diff --git a/src/realm/array.cpp b/src/realm/array.cpp index 25b0d9f5538..d249f7736ed 100644 --- a/src/realm/array.cpp +++ b/src/realm/array.cpp @@ -989,42 +989,18 @@ MemRef Array::clone(MemRef mem, Allocator& alloc, Allocator& target_alloc) MemRef Array::create(Type type, bool context_flag, WidthType width_type, size_t size, int_fast64_t value, Allocator& alloc) { - REALM_ASSERT_7(value, ==, 0, ||, width_type, ==, wtype_Bits); - REALM_ASSERT_7(size, ==, 0, ||, width_type, !=, wtype_Ignore); - - bool is_inner_bptree_node = false, has_refs = false; - switch (type) { - case type_Normal: - break; - case type_InnerBptreeNode: - is_inner_bptree_node = true; - has_refs = true; - break; - case type_HasRefs: - has_refs = true; - break; - } - + REALM_ASSERT_DEBUG(value == 0 || width_type == wtype_Bits); + REALM_ASSERT_DEBUG(size == 0 || width_type != wtype_Ignore); int width = 0; - size_t byte_size_0 = header_size; - if (value != 0) { - width = int(bit_width(value)); - byte_size_0 = calc_aligned_byte_size(size, width); // Throws - } - // Adding zero to Array::initial_capacity to avoid taking the - // address of that member - size_t byte_size = std::max(byte_size_0, initial_capacity + 0); - MemRef mem = alloc.alloc(byte_size); // Throws - char* header = mem.get_addr(); - - init_header(header, is_inner_bptree_node, has_refs, context_flag, width_type, width, size, byte_size); - + if (value != 0) + width = static_cast(bit_width(value)); + auto mem = Node::create_node(size, alloc, context_flag, type, width_type, width); if (value != 0) { + const auto header = mem.get_addr(); char* data = get_data_from_header(header); size_t begin = 0, end = size; REALM_TEMPEX(fill_direct, width, (data, begin, end, value)); } - return mem; } diff --git a/src/realm/array.hpp b/src/realm/array.hpp index 26990e6cfd4..d1f8463a64a 100644 --- a/src/realm/array.hpp +++ b/src/realm/array.hpp @@ -485,7 +485,7 @@ class Array : public Node, public ArrayParent { /// It is an error to specify a non-zero value unless the width /// type is wtype_Bits. It is also an error to specify a non-zero /// size if the width type is wtype_Ignore. - static MemRef create(Type, bool context_flag, WidthType, size_t size, int_fast64_t value, Allocator&); + static MemRef create(Type, bool, WidthType, size_t, int_fast64_t, Allocator&); // Overriding method in ArrayParent void update_child_ref(size_t, ref_type) override; @@ -938,8 +938,7 @@ inline void Array::adjust(size_t begin, size_t end, int_fast64_t diff) inline size_t Array::get_byte_size() const noexcept { const char* header = get_header_from_data(m_data); - WidthType wtype = Node::get_wtype_from_header(header); - size_t num_bytes = NodeHeader::calc_byte_size(wtype, m_size, m_width); + size_t num_bytes = NodeHeader::get_byte_size_from_header(header); REALM_ASSERT_7(m_alloc.is_read_only(m_ref), ==, true, ||, num_bytes, <=, get_capacity_from_header(header)); diff --git a/src/realm/array_basic_tpl.hpp b/src/realm/array_basic_tpl.hpp index 4c8e0ae2319..fc1e1c04df1 100644 --- a/src/realm/array_basic_tpl.hpp +++ b/src/realm/array_basic_tpl.hpp @@ -40,16 +40,12 @@ inline MemRef BasicArray::create_array(size_t init_size, Allocator& allocator // Adding zero to Array::initial_capacity to avoid taking the // address of that member size_t byte_size = std::max(byte_size_0, Node::initial_capacity + 0); // Throws - - MemRef mem = allocator.alloc(byte_size); // Throws - - bool is_inner_bptree_node = false; - bool has_refs = false; - bool context_flag = false; - int width = sizeof(T); - init_header(mem.get_addr(), is_inner_bptree_node, has_refs, context_flag, wtype_Multiply, width, init_size, - byte_size); - + MemRef mem = allocator.alloc(byte_size); // Throws + uint8_t flags = 0; + const int width = sizeof(T) * 8; // element width is in bits now + const auto header = mem.get_addr(); + init_header(header, Encoding::WTypMult, flags, width, init_size); + set_capacity_in_header(byte_size, header); return mem; } diff --git a/src/realm/node.cpp b/src/realm/node.cpp index cc880689509..f23cff4316b 100644 --- a/src/realm/node.cpp +++ b/src/realm/node.cpp @@ -32,11 +32,24 @@ MemRef Node::create_node(size_t size, Allocator& alloc, bool context_flag, Type size_t byte_size = std::max(byte_size_0, size_t(initial_capacity)); MemRef mem = alloc.alloc(byte_size); // Throws - char* header = mem.get_addr(); - - init_header(header, type == type_InnerBptreeNode, type != type_Normal, context_flag, width_type, width, size, - byte_size); - + const auto header = mem.get_addr(); + REALM_ASSERT_DEBUG(width_type != WidthType::wtype_Extend); + Encoding encoding{static_cast(width_type)}; + + uint8_t flags = 0; + if (type == type_InnerBptreeNode) + flags |= static_cast(Flags::InnerBPTree) | static_cast(Flags::HasRefs); + if (type != type_Normal) + flags |= static_cast(Flags::HasRefs); + if (context_flag) + flags |= static_cast(Flags::Context); + // width must be passed to init_header in bits, but for wtype_Multiply and wtype_Ignore + // it is provided by the caller of this function in bytes, so convert to bits + if (width_type != wtype_Bits) + width = width * 8; + + init_header(header, encoding, flags, width, size); + set_capacity_in_header(byte_size, header); return mem; } @@ -69,17 +82,20 @@ size_t Node::calc_item_count(size_t bytes, size_t width) const noexcept void Node::alloc(size_t init_size, size_t new_width) { REALM_ASSERT(is_attached()); - + char* header = get_header_from_data(m_data); + REALM_ASSERT(!wtype_is_extended(header)); size_t needed_bytes = calc_byte_len(init_size, new_width); // this method is not public and callers must (and currently do) ensure that // needed_bytes are never larger than max_array_payload. REALM_ASSERT_RELEASE(init_size <= max_array_size); - if (is_read_only()) + if (is_read_only()) { do_copy_on_write(needed_bytes); + // header has changed after copy on write if the array was compressed + header = get_header_from_data(m_data); + } REALM_ASSERT(!m_alloc.is_read_only(m_ref)); - char* header = get_header_from_data(m_data); size_t orig_capacity_bytes = get_capacity_from_header(header); size_t orig_width = get_width_from_header(header); @@ -114,8 +130,7 @@ void Node::alloc(size_t init_size, size_t new_width) // this array instance in a corrupt state update_parent(); // Throws } - - // Update header + // update width (important when we convert from normal uncompressed array into compressed format) if (new_width != orig_width) { set_width_in_header(int(new_width), header); } @@ -123,9 +138,20 @@ void Node::alloc(size_t init_size, size_t new_width) m_size = init_size; } +void Node::destroy() noexcept +{ + if (!is_attached()) + return; + char* header = get_header_from_data(m_data); + m_alloc.free_(m_ref, header); + m_data = nullptr; +} + void Node::do_copy_on_write(size_t minimum_size) { const char* header = get_header_from_data(m_data); + // only type A arrays should be allowed during copy on write + REALM_ASSERT(!wtype_is_extended(header)); // Calculate size in bytes size_t array_size = calc_byte_size(get_wtype_from_header(header), m_size, get_width_from_header(header)); @@ -140,7 +166,6 @@ void Node::do_copy_on_write(size_t minimum_size) const char* old_end = header + array_size; char* new_begin = mref.get_addr(); realm::safe_copy_n(old_begin, old_end - old_begin, new_begin); - ref_type old_ref = m_ref; // Update internal data @@ -150,7 +175,6 @@ void Node::do_copy_on_write(size_t minimum_size) // Update capacity in header. Uses m_data to find header, so // m_data must be initialized correctly first. set_capacity_in_header(new_size, new_begin); - update_parent(); #if REALM_ENABLE_MEMDEBUG diff --git a/src/realm/node.hpp b/src/realm/node.hpp index 8bea1c9559d..ea4d55a0e8c 100644 --- a/src/realm/node.hpp +++ b/src/realm/node.hpp @@ -114,7 +114,7 @@ class Node : public NodeHeader { { } - virtual ~Node() {} + virtual ~Node() = default; /**************************** Initializers *******************************/ @@ -123,10 +123,10 @@ class Node : public NodeHeader { char* init_from_mem(MemRef mem) noexcept { char* header = mem.get_addr(); + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); m_ref = mem.get_ref(); m_data = get_data_from_header(header); m_size = get_size_from_header(header); - return header; } @@ -212,14 +212,7 @@ class Node : public NodeHeader { /// children of that array. See non-static destroy_deep() for an /// alternative. If this accessor is already in the detached state, this /// function has no effect (idempotency). - void destroy() noexcept - { - if (!is_attached()) - return; - char* header = get_header_from_data(m_data); - m_alloc.free_(m_ref, header); - m_data = nullptr; - } + void destroy() noexcept; /// Shorthand for `destroy(MemRef(ref, alloc), alloc)`. static void destroy(ref_type ref, Allocator& alloc) noexcept @@ -234,7 +227,6 @@ class Node : public NodeHeader { alloc.free_(mem); } - /// Setting a new parent affects ownership of the attached array node, if /// any. If a non-null parent is specified, and there was no parent /// originally, then the caller passes ownership to the parent, and vice @@ -245,6 +237,7 @@ class Node : public NodeHeader { m_parent = parent; m_ndx_in_parent = ndx_in_parent; } + void set_ndx_in_parent(size_t ndx) noexcept { m_ndx_in_parent = ndx; @@ -333,8 +326,6 @@ class Node : public NodeHeader { // Includes array header. Not necessarily 8-byte aligned. virtual size_t calc_byte_len(size_t num_items, size_t width) const; virtual size_t calc_item_count(size_t bytes, size_t width) const noexcept; - static void init_header(char* header, bool is_inner_bptree_node, bool has_refs, bool context_flag, - WidthType width_type, int width, size_t size, size_t capacity) noexcept; private: friend class NodeTree; @@ -362,22 +353,6 @@ class ArrayPayload { virtual void set_spec(Spec*, size_t) const {} }; - -inline void Node::init_header(char* header, bool is_inner_bptree_node, bool has_refs, bool context_flag, - WidthType width_type, int width, size_t size, size_t capacity) noexcept -{ - // Note: Since the header layout contains unallocated bit and/or - // bytes, it is important that we put the entire header into a - // well defined state initially. - std::fill(header, header + header_size, 0); - set_is_inner_bptree_node_in_header(is_inner_bptree_node, header); - set_hasrefs_in_header(has_refs, header); - set_context_flag_in_header(context_flag, header); - set_wtype_in_header(width_type, header); - set_width_in_header(width, header); - set_size_in_header(size, header); - set_capacity_in_header(capacity, header); -} } // namespace realm #endif /* REALM_NODE_HPP */ diff --git a/src/realm/node_header.hpp b/src/realm/node_header.hpp index b5805fc343a..46d67673d92 100644 --- a/src/realm/node_header.hpp +++ b/src/realm/node_header.hpp @@ -20,16 +20,35 @@ #define REALM_NODE_HEADER_HPP #include +#include + +namespace { +// helper converting a number of bits into bytes and aligning to 8 byte boundary +static inline size_t align_bits_to8(size_t n) +{ + n = (n + 7) >> 3; + return (n + 7) & ~size_t(7); +} +} // namespace namespace realm { +// The header holds metadata for all allocations. It is 8 bytes. +// A field in byte 5 indicates the type of the allocation. +// +// Up to and including Core v 13, this field would always hold values 0,1 or 2. +// when stored in the file. This value now indicates that the chunk of memory +// must be interpreted according to the methods in NodeHeader. +// const size_t max_array_size = 0x00ffffffL; // Maximum number of elements in an array const size_t max_array_payload_aligned = 0x07ffffc0L; // Maximum number of bytes that the payload of an array can be // Even though the encoding supports arrays with size up to max_array_payload_aligned, // the maximum allocation size is smaller as it must fit within a memory section // (a contiguous virtual address range). This limitation is enforced in SlabAlloc::do_alloc(). + class NodeHeader { + public: enum Type { type_Normal, @@ -49,10 +68,46 @@ class NodeHeader { }; enum WidthType { + // The first 3 encodings where the only one used as far as Core v13. wtype_Bits = 0, // width indicates how many bits every element occupies wtype_Multiply = 1, // width indicates how many bytes every element occupies wtype_Ignore = 2, // each element is 1 byte + wtype_Extend = 3 // the layouts are described in byte 4 of the header. + }; + // Accessing flags. + enum class Flags { // bit positions in flags "byte", used for masking + Context = 1, + HasRefs = 2, + InnerBPTree = 4, + // additional flags can be supported by new layouts, but old layout is full + }; + // Possible header encodings (and corresponding memory layouts): + enum class Encoding { + WTypBits = 0, // Corresponds to wtype_Bits + WTypMult = 1, // Corresponds to wtype_Multiply + WTypIgn = 2, // Corresponds to wtype_Ignore + Packed = 4, // wtype is wtype_Extend + Flex = 5 // wtype is wtype_Extend }; + // * Packed: tightly packed array (any element size <= 64) + // * WTypBits: less tightly packed. Correspond to wtype_Bits + // * WTypMult: less tightly packed. Correspond to wtype_Multiply + // * WTypIgn: single byte elements. Correspond to wtype_Ignore + // encodings with more flexibility but lower number of elements: + // * Flex: Pair of arrays (2 element sizes, 2 element count) + // + // Encodings: bytes: + // name: | b0 | b1 | b2 | b3 | b4:0-2 | b4:3-4 | b4:5-7 | b5 | b6 | b7 | + // oldies | cap/chksum | 'A' | width | wtype | flags | size | + // Packed | cap/chksum | - | width | flags2 | wtype | flags | enc | size | + // Flex | cap/chksum | w_A + size_A | flags2 | wtype | flags | enc | w_B + size_B | + // + // legend: cap = capacity, chksum = checksum, flags = 3 flag bits, flags2 = 3 additional flag bits + // size = number of elements, w_A = bits per A element, w_B = bits per B element + // size_A = number of A elements, size_B = number of B elements, + // enc = the encoding for the array, corresponding to different memory layouts + // For Flex: w + size is 6 bits for element width, 10 bits for number of elements + // static const int header_size = 8; // Number of bytes used by header @@ -74,98 +129,113 @@ class NodeHeader { return get_data_from_header(const_cast(header)); } - static bool get_is_inner_bptree_node_from_header(const char* header) noexcept + // Helpers for NodeHeader::Type + // handles all header formats + static inline bool get_is_inner_bptree_node_from_header(const char* header) noexcept { typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); return (int(h[4]) & 0x80) != 0; } - static bool get_hasrefs_from_header(const char* header) noexcept + static inline bool get_hasrefs_from_header(const char* header) noexcept { typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); return (int(h[4]) & 0x40) != 0; } - static bool get_context_flag_from_header(const char* header) noexcept + static Type get_type_from_header(const char* header) noexcept + { + if (get_is_inner_bptree_node_from_header(header)) + return type_InnerBptreeNode; + if (get_hasrefs_from_header(header)) + return type_HasRefs; + return type_Normal; + } + + static inline bool get_context_flag_from_header(const char* header) noexcept { typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); return (int(h[4]) & 0x20) != 0; } - - static WidthType get_wtype_from_header(const char* header) noexcept + static inline void set_is_inner_bptree_node_in_header(bool value, char* header) noexcept { typedef unsigned char uchar; - const uchar* h = reinterpret_cast(header); - return WidthType((int(h[4]) & 0x18) >> 3); + uchar* h = reinterpret_cast(header); + h[4] = uchar((int(h[4]) & ~0x80) | int(value) << 7); } - static uint_least8_t get_width_from_header(const char* header) noexcept + static inline void set_hasrefs_in_header(bool value, char* header) noexcept { typedef unsigned char uchar; - const uchar* h = reinterpret_cast(header); - return uint_least8_t((1 << (int(h[4]) & 0x07)) >> 1); + uchar* h = reinterpret_cast(header); + h[4] = uchar((int(h[4]) & ~0x40) | int(value) << 6); } - static size_t get_size_from_header(const char* header) noexcept + static inline void set_context_flag_in_header(bool value, char* header) noexcept { typedef unsigned char uchar; - const uchar* h = reinterpret_cast(header); - return (size_t(h[5]) << 16) + (size_t(h[6]) << 8) + h[7]; + uchar* h = reinterpret_cast(header); + h[4] = uchar((int(h[4]) & ~0x20) | int(value) << 5); } - static size_t get_capacity_from_header(const char* header) noexcept + // Helpers for NodeHeader::WidthType: + // handles all header formats + static inline WidthType get_wtype_from_header(const char* header) noexcept { typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); - return (size_t(h[0]) << 19) + (size_t(h[1]) << 11) + (h[2] << 3); + int h4 = h[4]; + return WidthType((h4 & 0x18) >> 3); } - static Type get_type_from_header(const char* header) noexcept + static inline bool wtype_is_extended(const char* header) noexcept { - if (get_is_inner_bptree_node_from_header(header)) - return type_InnerBptreeNode; - if (get_hasrefs_from_header(header)) - return type_HasRefs; - return type_Normal; + return get_wtype_from_header(header) == wtype_Extend; } - static void set_is_inner_bptree_node_in_header(bool value, char* header) noexcept + static inline void set_wtype_in_header(WidthType value, char* header) noexcept { typedef unsigned char uchar; uchar* h = reinterpret_cast(header); - h[4] = uchar((int(h[4]) & ~0x80) | int(value) << 7); + auto h4 = h[4]; + h4 = (h4 & ~0x18) | int(value) << 3; + h[4] = h4; } - static void set_hasrefs_in_header(bool value, char* header) noexcept + static size_t unsigned_to_num_bits(uint64_t value) { - typedef unsigned char uchar; - uchar* h = reinterpret_cast(header); - h[4] = uchar((int(h[4]) & ~0x40) | int(value) << 6); + if constexpr (sizeof(size_t) == sizeof(uint64_t)) + return 1 + log2(value); + uint32_t high = value >> 32; + if (high) + return 33 + log2(high); + uint32_t low = value & 0xFFFFFFFFUL; + if (low) + return 1 + log2(low); + return 0; } - static void set_context_flag_in_header(bool value, char* header) noexcept + static inline size_t signed_to_num_bits(int64_t value) { - typedef unsigned char uchar; - uchar* h = reinterpret_cast(header); - h[4] = uchar((int(h[4]) & ~0x20) | int(value) << 5); + if (value >= 0) + return 1 + unsigned_to_num_bits(value); + else + return 1 + unsigned_to_num_bits(~value); // <-- is this correct???? } - static void set_wtype_in_header(WidthType value, char* header) noexcept - { - // Indicates how to calculate size in bytes based on width - // 0: bits (width/8) * size - // 1: multiply width * size - // 2: ignore 1 * size - typedef unsigned char uchar; - uchar* h = reinterpret_cast(header); - h[4] = uchar((int(h[4]) & ~0x18) | int(value) << 3); - } - static void set_width_in_header(int value, char* header) noexcept + // Helper functions for old layouts only: + // Handling width and sizes: + static inline uint_least8_t get_width_from_header(const char* header) noexcept; + + static inline size_t get_size_from_header(const char* header) noexcept; + + static inline void set_width_in_header(size_t value, char* header) noexcept { + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); // Pack width in 3 bits (log2) int w = 0; while (value) { @@ -179,8 +249,9 @@ class NodeHeader { h[4] = uchar((int(h[4]) & ~0x7) | w); } - static void set_size_in_header(size_t value, char* header) noexcept + static inline void set_size_in_header(size_t value, char* header) noexcept { + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); REALM_ASSERT_3(value, <=, max_array_size); typedef unsigned char uchar; uchar* h = reinterpret_cast(header); @@ -189,29 +260,132 @@ class NodeHeader { h[7] = uchar(value & 0x000000FF); } - // Note: There is a copy of this function is test_alloc.cpp - static void set_capacity_in_header(size_t value, char* header) noexcept + + // Note: The wtype must have been set prior to calling this function + static size_t get_capacity_from_header(const char* header) noexcept { - REALM_ASSERT_3(value, <=, (0xffffff << 3)); - typedef unsigned char uchar; - uchar* h = reinterpret_cast(header); - h[0] = uchar((value >> 19) & 0x000000FF); - h[1] = uchar((value >> 11) & 0x000000FF); - h[2] = uchar(value >> 3 & 0x000000FF); + if (!wtype_is_extended(header)) { + typedef unsigned char uchar; + const uchar* h = reinterpret_cast(header); + return (size_t(h[0]) << 19) + (size_t(h[1]) << 11) + (h[2] << 3); + } + else { + return reinterpret_cast(header)[0] << 3; + } } - static size_t get_byte_size_from_header(const char* header) noexcept + // Note: There is a (no longer a correct) copy of this function is test_alloc.cpp + // Note 2: The wtype must have been set prior to calling this function + static void set_capacity_in_header(size_t value, char* header) noexcept { - size_t size = get_size_from_header(header); - uint_least8_t width = get_width_from_header(header); - WidthType wtype = get_wtype_from_header(header); - size_t num_bytes = calc_byte_size(wtype, size, width); + if (!wtype_is_extended(header)) { + REALM_ASSERT_3(value, <=, (0xffffff << 3)); + typedef unsigned char uchar; + uchar* h = reinterpret_cast(header); + h[0] = uchar((value >> 19) & 0x000000FF); + h[1] = uchar((value >> 11) & 0x000000FF); + h[2] = uchar(value >> 3 & 0x000000FF); + } + else { + REALM_ASSERT_DEBUG(value < (65536 << 3)); + REALM_ASSERT_DEBUG((value & 0x7) == 0); + (reinterpret_cast(header))[0] = static_cast(value >> 3); + } + } - return num_bytes; + static size_t get_byte_size_from_header(const char* header) noexcept; + + // ^ First 3 must overlap numerically with corresponding wtype_X enum. + static Encoding get_encoding(const char* header) + { + auto wtype = get_wtype_from_header(header); + if (wtype == wtype_Extend) { + const auto h = reinterpret_cast(header); + return static_cast(h[5] + 3); + } + return Encoding(int(wtype)); } + static void set_encoding(char* header, Encoding enc) + { + if (enc < Encoding::Packed) { + set_wtype_in_header(static_cast(enc), header); + } + else { + set_wtype_in_header(wtype_Extend, header); + auto h = reinterpret_cast(header); + h[5] = static_cast(enc) - 3; + } + } + static std::string enc_to_string(Encoding enc) + { + switch (enc) { + case Encoding::WTypMult: + return "Mult"; + case Encoding::WTypIgn: + return "Ign"; + case Encoding::WTypBits: + return "Bits"; + case Encoding::Packed: + return "Pack"; + case Encoding::Flex: + return "Flex"; + default: + return "Err"; + } + } + static std::string header_to_string(const char* header) + { + std::string retval = "{" + enc_to_string(get_encoding(header)) + "}"; + return retval; + } + +private: + friend class Node; + // Setting element size for encodings with a single element size: + static void inline set_element_size(char* header, size_t bits_per_element, Encoding); + // Getting element size for encodings with a single element size: + static inline size_t get_element_size(const char* header, Encoding); + // Used only by flex at this stage. + // Setting element sizes for encodings with two element sizes (called A and B) + static inline void set_elementA_size(char* header, size_t bits_per_element); + static inline void set_elementB_size(char* header, size_t bits_per_element); + // Getting element sizes for encodings with two element sizes (called A and B) + static inline size_t get_elementA_size(const char* header); + static inline size_t get_elementB_size(const char* header); + // Setting num of elements for encodings with two element sizes (called A and B) + static inline void set_arrayA_num_elements(char* header, size_t num_elements); + static inline void set_arrayB_num_elements(char* header, size_t num_elements); + // Getting number of elements for encodings with two element sizes (called A and B) + static inline size_t get_arrayA_num_elements(const char* header); + static inline size_t get_arrayB_num_elements(const char* header); + // Getting the number of elements in the array(s). All encodings except Flex have one number of elements. + static inline size_t get_num_elements(const char* header, Encoding); + // Setting the number of elements in the array(s). All encodings except Flex have one number of elements. + static inline void set_num_elements(char* header, size_t num_elements, Encoding); + + static inline size_t calc_size(size_t num_elements); + static inline size_t calc_size(size_t num_elements, size_t element_size, Encoding); + static inline size_t calc_size(size_t arrayA_num_elements, size_t arrayB_num_elements, size_t elementA_size, + size_t elementB_size); static size_t calc_byte_size(WidthType wtype, size_t size, uint_least8_t width) noexcept { + // the width need to be adjusted to nearest power of two: + if (width > 8) { + if (width > 32) + width = 64; + else if (width > 16) + width = 32; + else + width = 16; + } + else { // width <= 8 + if (width > 4) + width = 8; + else if (width > 2) + width = 4; + // else width is already a power of 2 + } size_t num_bytes = 0; switch (wtype) { case wtype_Bits: { @@ -229,16 +403,360 @@ class NodeHeader { case wtype_Ignore: num_bytes = size; break; + default: { + REALM_ASSERT(false); + break; + } } - + num_bytes += header_size; // Ensure 8-byte alignment num_bytes = (num_bytes + 7) & ~size_t(7); + return num_bytes; + } - num_bytes += header_size; + static inline void set_flags(char* header, uint8_t flags) + { + REALM_ASSERT_DEBUG(flags <= 7); + auto h = reinterpret_cast(header); + h[4] = (h[4] & 0b00011111) | flags << 5; + } + static inline uint8_t get_flags(char* header) + { + auto h = reinterpret_cast(header); + return h[4] >> 5; + } - return num_bytes; + static inline void set_flags2(char* header, uint8_t flags) + { + REALM_ASSERT_DEBUG(flags <= 7); + auto h = reinterpret_cast(header); + h[4] = (h[4] & 0b11111000) | flags; + } + static inline uint8_t get_flags2(char* header) + { + auto h = reinterpret_cast(header); + return h[4] & 0b0111; } }; + +inline void NodeHeader::set_element_size(char* header, size_t bits_per_element, Encoding encoding) +{ + switch (encoding) { + case NodeHeader::Encoding::Packed: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Packed); + REALM_ASSERT_DEBUG(bits_per_element <= 64); + (reinterpret_cast(header)[3] = static_cast(bits_per_element)); + } break; + case NodeHeader::Encoding::WTypBits: { + REALM_ASSERT_DEBUG(bits_per_element <= 64); + // TODO: Only powers of two allowed + // TODO: Optimize + NodeHeader::set_wtype_in_header(wtype_Bits, reinterpret_cast(header)); + NodeHeader::set_width_in_header(bits_per_element, reinterpret_cast(header)); + } break; + case NodeHeader::Encoding::WTypMult: { + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG((bits_per_element & 0x7) == 0); + // TODO: Only powers of two allowed + // TODO: Optimize + NodeHeader::set_wtype_in_header(wtype_Multiply, reinterpret_cast(header)); + NodeHeader::set_width_in_header(bits_per_element >> 3, reinterpret_cast(header)); + } break; + default: + REALM_UNREACHABLE(); + } +} + +inline size_t NodeHeader::get_element_size(const char* header, Encoding encoding) +{ + switch (encoding) { + case NodeHeader::Encoding::Packed: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Packed); + const auto bits_per_element = (reinterpret_cast(header))[3]; + REALM_ASSERT_DEBUG(bits_per_element <= 64); + return bits_per_element; + } break; + case NodeHeader::Encoding::WTypBits: { + REALM_ASSERT_DEBUG(get_wtype_from_header(header) == wtype_Bits); + const auto bits_per_element = NodeHeader::get_width_from_header(reinterpret_cast(header)); + REALM_ASSERT_DEBUG(bits_per_element <= 64); + return bits_per_element; + } break; + case NodeHeader::Encoding::WTypMult: { + REALM_ASSERT_DEBUG(get_wtype_from_header(header) == wtype_Multiply); + const auto bits_per_element = NodeHeader::get_width_from_header(reinterpret_cast(header)) + << 3; + REALM_ASSERT_DEBUG(bits_per_element <= 64); + return bits_per_element; + } break; + default: + REALM_UNREACHABLE(); + } +} + +inline void NodeHeader::set_elementA_size(char* header, size_t bits_per_element) +{ + // we're a bit low on bits for the Flex encoding, so we need to squeeze stuff + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG(bits_per_element > 0); + uint16_t word = (reinterpret_cast(header))[1]; + word &= ~(0b111111 << 10); + // we only have 6 bits, so store values in range 1-64 as 0-63 + word |= (bits_per_element - 1) << 10; + (reinterpret_cast(header))[1] = word; +} + +inline void NodeHeader::set_elementB_size(char* header, size_t bits_per_element) +{ + // we're a bit low on bits for the Flex encoding, so we need to squeeze stuff + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG(bits_per_element > 0); + uint16_t word = (reinterpret_cast(header))[3]; + word &= ~(0b111111 << 10); + // we only have 6 bits, so store values in range 1-64 as 0-63 + word |= (bits_per_element - 1) << 10; + (reinterpret_cast(header))[3] = word; +} + +inline size_t NodeHeader::get_elementA_size(const char* header) +{ + const auto encoding = get_encoding(header); + REALM_ASSERT_DEBUG(encoding == Encoding::Flex); + uint16_t word = (reinterpret_cast(header))[1]; + auto bits_per_element = (word >> 10) & 0b111111; + // we only have 6 bits, so store values in range 1-64 as 0-63 + // this means that Flex cannot support element sizes of 0 + bits_per_element++; + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG(bits_per_element > 0); + return bits_per_element; +} + +inline size_t NodeHeader::get_elementB_size(const char* header) +{ + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + uint16_t word = (reinterpret_cast(header))[3]; + auto bits_per_element = (word >> 10) & 0b111111; + // same as above + bits_per_element++; + REALM_ASSERT_DEBUG(bits_per_element <= 64); + REALM_ASSERT_DEBUG(bits_per_element > 0); + return bits_per_element; +} + +inline size_t NodeHeader::get_num_elements(const char* header, Encoding encoding) +{ + switch (encoding) { + case NodeHeader::Encoding::Packed: + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Packed); + return (reinterpret_cast(header))[3]; + break; + case NodeHeader::Encoding::WTypBits: + case NodeHeader::Encoding::WTypMult: + case NodeHeader::Encoding::WTypIgn: { + REALM_ASSERT_DEBUG(get_wtype_from_header(header) != wtype_Extend); + typedef unsigned char uchar; + const uchar* h = reinterpret_cast(header); + return (size_t(h[5]) << 16) + (size_t(h[6]) << 8) + h[7]; + break; + } + case NodeHeader::Encoding::Flex: + return get_arrayB_num_elements(header); + break; + default: + REALM_UNREACHABLE(); + } } +inline void NodeHeader::set_num_elements(char* header, size_t num_elements, Encoding encoding) +{ + switch (encoding) { + case NodeHeader::Encoding::Packed: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Packed); + REALM_ASSERT_DEBUG(num_elements < 0x10000); + (reinterpret_cast(header))[3] = static_cast(num_elements); + } break; + case NodeHeader::Encoding::WTypBits: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::WTypBits); + NodeHeader::set_wtype_in_header(wtype_Bits, header); + NodeHeader::set_size_in_header(num_elements, header); + } break; + case NodeHeader::Encoding::WTypMult: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::WTypMult); + NodeHeader::set_wtype_in_header(wtype_Multiply, header); + NodeHeader::set_size_in_header(num_elements, header); + } break; + case NodeHeader::Encoding::WTypIgn: { + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::WTypIgn); + NodeHeader::set_wtype_in_header(wtype_Ignore, header); + NodeHeader::set_size_in_header(num_elements, header); + } break; + default: + REALM_UNREACHABLE(); + } +} + +inline void NodeHeader::set_arrayA_num_elements(char* header, size_t num_elements) +{ + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + REALM_ASSERT_DEBUG(num_elements < 0b10000000000); // 10 bits + uint16_t word = (reinterpret_cast(header))[1]; + word &= ~(0b1111111111 << 10); + word |= num_elements << 10; + (reinterpret_cast(header))[1] = word; +} + +inline void NodeHeader::set_arrayB_num_elements(char* header, size_t num_elements) +{ + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + REALM_ASSERT_DEBUG(num_elements < 0b10000000000); // 10 bits + uint16_t word = (reinterpret_cast(header))[3]; + word &= ~(0b1111111111 << 10); + word |= num_elements << 10; + (reinterpret_cast(header))[3] = word; +} + +inline size_t NodeHeader::get_arrayA_num_elements(const char* header) +{ + const auto encoding = get_encoding(header); + REALM_ASSERT_DEBUG(encoding == Encoding::Flex); + const uint16_t word = (reinterpret_cast(header))[1]; + const auto num_elements = word & 0b1111111111; + return num_elements; +} + +inline size_t NodeHeader::get_arrayB_num_elements(const char* header) +{ + REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); + const uint16_t word = (reinterpret_cast(header))[3]; + const auto num_elements = word & 0b1111111111; + return num_elements; +} + +inline size_t NodeHeader::calc_size(size_t num_elements) +{ + return calc_byte_size(wtype_Ignore, num_elements, 0); +} + +inline size_t NodeHeader::calc_size(size_t num_elements, size_t element_size, Encoding encoding) +{ + using Encoding = NodeHeader::Encoding; + switch (encoding) { + case Encoding::Packed: + return NodeHeader::header_size + align_bits_to8(num_elements * element_size); + case Encoding::WTypBits: + return calc_byte_size(wtype_Bits, num_elements, static_cast(element_size)); + case Encoding::WTypMult: + return calc_byte_size(wtype_Multiply, num_elements, static_cast(element_size)); + case Encoding::WTypIgn: + return calc_byte_size(wtype_Ignore, num_elements, 0); + default: + REALM_UNREACHABLE(); + } +} + +inline size_t NodeHeader::calc_size(size_t arrayA_num_elements, size_t arrayB_num_elements, size_t elementA_size, + size_t elementB_size) +{ + return NodeHeader::header_size + + align_bits_to8(arrayA_num_elements * elementA_size + arrayB_num_elements * elementB_size); +} + +size_t inline NodeHeader::get_byte_size_from_header(const char* header) noexcept +{ + const auto h = header; + + const auto encoding = get_encoding(h); + REALM_ASSERT_DEBUG(encoding >= Encoding::WTypBits && encoding <= Encoding::Flex); + const auto size = get_num_elements(h, encoding); + switch (encoding) { + case Encoding::WTypBits: + case Encoding::WTypIgn: + case Encoding::WTypMult: { + const auto width = get_width_from_header(header); + return calc_byte_size(WidthType(int(encoding)), size, static_cast(width)); + } + case Encoding::Packed: + return NodeHeader::header_size + align_bits_to8(size * get_element_size(h, encoding)); + case Encoding::Flex: + return NodeHeader::header_size + align_bits_to8(get_arrayA_num_elements(h) * get_elementA_size(h) + + get_arrayB_num_elements(h) * get_elementB_size(h)); + default: + REALM_UNREACHABLE(); + } +} + + +uint_least8_t inline NodeHeader::get_width_from_header(const char* header) noexcept +{ + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); + typedef unsigned char uchar; + const uchar* h = reinterpret_cast(header); + return uint_least8_t((1 << (int(h[4]) & 0x07)) >> 1); +} + +// A little helper: +size_t inline NodeHeader::get_size_from_header(const char* header) noexcept +{ + return get_num_elements(header, get_encoding(header)); +} + +} // namespace realm + + +namespace { + +static inline void init_header(char* header, realm::NodeHeader::Encoding enc, uint8_t flags, uint8_t bits_pr_elem, + size_t num_elems) +{ + using Encoding = realm::NodeHeader::Encoding; + std::fill(header, header + realm::NodeHeader::header_size, 0); + const auto hb = reinterpret_cast(header); + REALM_ASSERT_DEBUG(enc <= Encoding::Packed); + if (enc < Encoding::Packed) { + // old layout + uint8_t wtype = static_cast(enc); + hb[4] = (flags << 5) | (wtype << 3); + if (enc == Encoding::WTypBits) + realm::NodeHeader::set_width_in_header(bits_pr_elem, reinterpret_cast(header)); + else + realm::NodeHeader::set_width_in_header(bits_pr_elem >> 3, reinterpret_cast(header)); + realm::NodeHeader::set_size_in_header(num_elems, reinterpret_cast(header)); + } + else if (enc == Encoding::Packed) { + hb[2] = 0; + hb[3] = static_cast(bits_pr_elem); + hb[4] = (flags << 5) | (realm::NodeHeader::wtype_Extend << 3); + hb[5] = static_cast(enc) - realm::NodeHeader::wtype_Extend; + const auto hw = reinterpret_cast(header); + hw[3] = static_cast(num_elems); + } +} + +// init the header for flex array. Passing A bit width and size (values) and B bit width and size (indices) +static inline void init_header(char* header, realm::NodeHeader::Encoding enc, uint8_t flags, uint8_t bits_pr_elemA, + uint8_t bits_pr_elemB, size_t num_elemsA, size_t num_elemsB) +{ + std::fill(header, header + realm::NodeHeader::header_size, 0); + const auto hb = reinterpret_cast(header); + REALM_ASSERT_DEBUG(enc == realm::NodeHeader::Encoding::Flex); + REALM_ASSERT_DEBUG(flags < 8); + hb[4] = (flags << 5) | (realm::NodeHeader::wtype_Extend << 3); + hb[5] = + static_cast(realm::NodeHeader::Encoding::Flex) - static_cast(realm::NodeHeader::Encoding::Packed); + const auto hw = reinterpret_cast(header); + REALM_ASSERT_DEBUG(bits_pr_elemA > 0); + REALM_ASSERT_DEBUG(bits_pr_elemB > 0); + REALM_ASSERT_DEBUG(bits_pr_elemA <= 64); + REALM_ASSERT_DEBUG(bits_pr_elemB <= 64); + REALM_ASSERT_DEBUG(num_elemsA < 1024); + REALM_ASSERT_DEBUG(num_elemsB < 1024); + hw[3] = static_cast(((bits_pr_elemB - 1) << 10) | num_elemsB); + hw[1] = static_cast(((bits_pr_elemA - 1) << 10) | num_elemsA); +} +} // namespace + + #endif /* REALM_NODE_HEADER_HPP */ diff --git a/test/test_alloc.cpp b/test/test_alloc.cpp index ad3c6c3e7df..28d80c47a66 100644 --- a/test/test_alloc.cpp +++ b/test/test_alloc.cpp @@ -70,17 +70,21 @@ using namespace realm::util; namespace { + void set_capacity(char* header, size_t value) { + NodeHeader::set_wtype_in_header(NodeHeader::wtype_Ignore, header); typedef unsigned char uchar; uchar* h = reinterpret_cast(header); h[0] = uchar((value >> 19) & 0x000000FF); h[1] = uchar((value >> 11) & 0x000000FF); h[2] = uchar((value >> 3) & 0x000000FF); + REALM_ASSERT(NodeHeader::get_capacity_from_header(header) == value); } size_t get_capacity(const char* header) { + REALM_ASSERT(NodeHeader::get_wtype_from_header(header) == NodeHeader::wtype_Ignore); typedef unsigned char uchar; const uchar* h = reinterpret_cast(header); return (size_t(h[0]) << 19) + (size_t(h[1]) << 11) + (h[2] << 3); @@ -286,8 +290,11 @@ TEST(Alloc_Fuzzy) refs.push_back(r); set_capacity(r.get_addr(), siz); - // write some data to the allcoated area so that we can verify it later - memset(r.get_addr() + 3, static_cast(reinterpret_cast(r.get_addr())), siz - 3); + // write some data to the allcoated area so that we can verify it later. + // We must keep byte 4 in the header unharmed, since it is needed later by the allocator + // to determine the encoding, and hence the size of any released object. + // We simply skip the header. + memset(r.get_addr() + 8, static_cast(reinterpret_cast(r.get_addr())), siz - 8); } else if (refs.size() > 0) { // free random entry @@ -303,7 +310,7 @@ TEST(Alloc_Fuzzy) size_t siz = get_capacity(r.get_addr()); // verify that all the data we wrote during allocation is intact - for (size_t c = 3; c < siz; c++) { + for (size_t c = 8; c < siz; c++) { if (r.get_addr()[c] != static_cast(reinterpret_cast(r.get_addr()))) { // faster than using 'CHECK' for each character, which is slow CHECK(false); @@ -365,7 +372,7 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab { // Extendind the first mapping const auto initial_baseline = alloc.get_baseline(); const auto initial_version = alloc.get_mapping_version(); - const char* initial_translated = alloc.translate(1000); + const char* initial_translated = alloc.translate_in_slab(1000); _impl::SimulatedFailure::prime_mmap([](size_t) { return true; @@ -376,7 +383,7 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab CHECK_THROW(alloc.update_reader_view(page_size * 2), std::bad_alloc); CHECK_EQUAL(initial_baseline, alloc.get_baseline()); CHECK_EQUAL(initial_version, alloc.get_mapping_version()); - CHECK_EQUAL(initial_translated, alloc.translate(1000)); + CHECK_EQUAL(initial_translated, alloc.translate_in_slab(1000)); _impl::SimulatedFailure::prime_mmap(nullptr); alloc.get_file().resize(page_size * 2); @@ -400,7 +407,7 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab { // Add a new complete section after a complete section const auto initial_baseline = alloc.get_baseline(); const auto initial_version = alloc.get_mapping_version(); - const char* initial_translated = alloc.translate(1000); + const char* initial_translated = alloc.translate_in_slab(1000); _impl::SimulatedFailure::prime_mmap([](size_t) { return true; @@ -409,14 +416,14 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab CHECK_THROW(alloc.update_reader_view(section_size * 2), std::bad_alloc); CHECK_EQUAL(initial_baseline, alloc.get_baseline()); CHECK_EQUAL(initial_version, alloc.get_mapping_version()); - CHECK_EQUAL(initial_translated, alloc.translate(1000)); + CHECK_EQUAL(initial_translated, alloc.translate_in_slab(1000)); _impl::SimulatedFailure::prime_mmap(nullptr); alloc.update_reader_view(section_size * 2); CHECK_EQUAL(alloc.get_baseline(), section_size * 2); - CHECK_EQUAL(initial_version, alloc.get_mapping_version()); // did not alter an existing mapping - CHECK_EQUAL(initial_translated, alloc.translate(1000)); // first section was not remapped - CHECK_EQUAL(0, *alloc.translate(section_size * 2 - page_size)); + CHECK_EQUAL(initial_version, alloc.get_mapping_version()); // did not alter an existing mapping + CHECK_EQUAL(initial_translated, alloc.translate_in_slab(1000)); // first section was not remapped + CHECK_EQUAL(0, *alloc.translate_in_slab(section_size * 2 - page_size)); alloc.purge_old_mappings(4, 4); } @@ -426,8 +433,8 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab { // Add complete section and a a partial section after that const auto initial_baseline = alloc.get_baseline(); const auto initial_version = alloc.get_mapping_version(); - const char* initial_translated_1 = alloc.translate(1000); - const char* initial_translated_2 = alloc.translate(section_size + 1000); + const char* initial_translated_1 = alloc.translate_in_slab(1000); + const char* initial_translated_2 = alloc.translate_in_slab(section_size + 1000); _impl::SimulatedFailure::prime_mmap([](size_t size) { // Let the first allocation succeed and only the second one fail @@ -437,21 +444,22 @@ NONCONCURRENT_TEST_IF(Alloc_MapFailureRecovery, _impl::SimulatedFailure::is_enab CHECK_THROW(alloc.update_reader_view(section_size * 3 + page_size), std::bad_alloc); CHECK_EQUAL(initial_baseline, alloc.get_baseline()); CHECK_EQUAL(initial_version, alloc.get_mapping_version()); - CHECK_EQUAL(initial_translated_1, alloc.translate(1000)); - CHECK_EQUAL(initial_translated_2, alloc.translate(section_size + 1000)); + CHECK_EQUAL(initial_translated_1, alloc.translate_in_slab(1000)); + CHECK_EQUAL(initial_translated_2, alloc.translate_in_slab(section_size + 1000)); _impl::SimulatedFailure::prime_mmap(nullptr); alloc.update_reader_view(section_size * 3 + page_size); CHECK_EQUAL(alloc.get_baseline(), section_size * 3 + page_size); CHECK_EQUAL(initial_version, alloc.get_mapping_version()); // did not alter an existing mapping - CHECK_EQUAL(initial_translated_1, alloc.translate(1000)); - CHECK_EQUAL(initial_translated_2, alloc.translate(section_size + 1000)); - CHECK_EQUAL(0, *alloc.translate(section_size * 2 + 1000)); + CHECK_EQUAL(initial_translated_1, alloc.translate_in_slab(1000)); + CHECK_EQUAL(initial_translated_2, alloc.translate_in_slab(section_size + 1000)); + CHECK_EQUAL(0, *alloc.translate_in_slab(section_size * 2 + 1000)); alloc.purge_old_mappings(5, 5); } } + // This test reproduces the sporadic issue that was seen for large refs (addresses) // on 32-bit iPhone 5 Simulator runs on certain host machines. TEST(Alloc_ToAndFromRef) From aabe7ccf17774057f0a7f62fed43b6e6ca19e4c0 Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Fri, 12 Apr 2024 10:11:56 +0100 Subject: [PATCH 03/18] fix encoding enum numbers (#7581) --- src/realm/node_header.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/realm/node_header.hpp b/src/realm/node_header.hpp index 46d67673d92..c475bffc1b2 100644 --- a/src/realm/node_header.hpp +++ b/src/realm/node_header.hpp @@ -86,8 +86,8 @@ class NodeHeader { WTypBits = 0, // Corresponds to wtype_Bits WTypMult = 1, // Corresponds to wtype_Multiply WTypIgn = 2, // Corresponds to wtype_Ignore - Packed = 4, // wtype is wtype_Extend - Flex = 5 // wtype is wtype_Extend + Packed = 3, // wtype is wtype_Extend + Flex = 4 // wtype is wtype_Extend }; // * Packed: tightly packed array (any element size <= 64) // * WTypBits: less tightly packed. Correspond to wtype_Bits @@ -753,8 +753,8 @@ static inline void init_header(char* header, realm::NodeHeader::Encoding enc, ui REALM_ASSERT_DEBUG(bits_pr_elemB <= 64); REALM_ASSERT_DEBUG(num_elemsA < 1024); REALM_ASSERT_DEBUG(num_elemsB < 1024); - hw[3] = static_cast(((bits_pr_elemB - 1) << 10) | num_elemsB); hw[1] = static_cast(((bits_pr_elemA - 1) << 10) | num_elemsA); + hw[3] = static_cast(((bits_pr_elemB - 1) << 10) | num_elemsB); } } // namespace From 549c7073fb3c9ae915b5e6db320082267dde9fc3 Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Tue, 23 Apr 2024 16:47:39 +0100 Subject: [PATCH 04/18] RCORE-2055 Array Classification for enabling compression (#7564) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update next-major * specified part of new layout (new width encoding) * new header format for compressed arrays * code review * code review * start of classifying arrays for compression * classification down to column types * first attempt to cut through the BPlusTree madness * [wip] start on 'type driven' write process * all tests passing (but no compression enabled) * enabled compression for signed integer leafs only * removed some dubious constructions in cluster tree * delete tmp array while classifying arrays * enabled compression of links and backlinks (excl collections) * also compress bplustree of integers/links (experimental) * pref for compressing dicts (not working) * wip * wip * finally: compressing collections (incl dicts) * compressing timestamps now * enabled compression on ObjectID, TypedLink and UUID * also compressing Mixed properties (not list/dicts of Mixed) * Array compression with collections in Mixed (#7412) --------- Co-authored-by: Finn Schiermer Andersen * merge next-major + collection in mixed * enable dynamic choice of compression method * moved typed_write/typed_print for bptree into class * Merge pull request #7432 from realm/fsa/clean_typed_write moved typed_write/typed_print for bptree into class * cleanup unrelated code changes * fix compilation * cleanup * code review * code review * swap byte 3&4 with byte 6&7 for flex formats for storing A and B sizes * Some modifications * Move Encoding definition * Testing * Perserve type information in typed_write (#7598) * call directly Array::destroy() * Fix issue * remove table from typed_print * lint * point fix avoid compressing history array --------- Co-authored-by: Finn Schiermer Andersen Co-authored-by: Jørgen Edelbo Co-authored-by: Finn Schiermer Andersen --- src/realm/array.cpp | 48 ++++++- src/realm/array.hpp | 46 +++++-- src/realm/array_integer.hpp | 16 +++ src/realm/array_mixed.cpp | 85 ++++++++++++ src/realm/array_mixed.hpp | 1 + src/realm/array_timestamp.cpp | 22 +++ src/realm/array_timestamp.hpp | 1 + src/realm/bplustree.cpp | 87 ++++++++++-- src/realm/bplustree.hpp | 12 ++ src/realm/cluster.cpp | 229 ++++++++++++++++++++++++++++++++ src/realm/cluster.hpp | 24 ++-- src/realm/cluster_tree.cpp | 82 ++++++++++++ src/realm/cluster_tree.hpp | 17 +++ src/realm/db.cpp | 6 +- src/realm/dictionary.cpp | 30 +++++ src/realm/dictionary.hpp | 2 + src/realm/group.cpp | 113 ++++++++++++++-- src/realm/group.hpp | 9 +- src/realm/group_writer.cpp | 14 +- src/realm/impl/array_writer.hpp | 4 + src/realm/node.hpp | 12 ++ src/realm/node_header.hpp | 8 +- src/realm/spec.hpp | 5 + src/realm/table.cpp | 57 ++++++++ src/realm/table.hpp | 20 +-- src/realm/transaction.hpp | 2 +- 26 files changed, 883 insertions(+), 69 deletions(-) diff --git a/src/realm/array.cpp b/src/realm/array.cpp index d249f7736ed..2f96b15877d 100644 --- a/src/realm/array.cpp +++ b/src/realm/array.cpp @@ -209,7 +209,6 @@ size_t Array::bit_width(int64_t v) return uint64_t(v) >> 31 ? 64 : uint64_t(v) >> 15 ? 32 : uint64_t(v) >> 7 ? 16 : 8; } - void Array::init_from_mem(MemRef mem) noexcept { char* header = Node::init_from_mem(mem); @@ -289,7 +288,7 @@ ref_type Array::do_write_shallow(_impl::ArrayWriterBase& out) const } -ref_type Array::do_write_deep(_impl::ArrayWriterBase& out, bool only_if_modified) const +ref_type Array::do_write_deep(_impl::ArrayWriterBase& out, bool only_if_modified, bool compress) const { // Temp array for updated refs Array new_array(Allocator::get_default()); @@ -304,7 +303,7 @@ ref_type Array::do_write_deep(_impl::ArrayWriterBase& out, bool only_if_modified bool is_ref = (value != 0 && (value & 1) == 0); if (is_ref) { ref_type subref = to_ref(value); - ref_type new_subref = write(subref, m_alloc, out, only_if_modified); // Throws + ref_type new_subref = write(subref, m_alloc, out, only_if_modified, compress); // Throws value = from_ref(new_subref); } new_array.add(value); // Throws @@ -1334,3 +1333,46 @@ bool QueryStateFindAll::match(size_t index) noexcept return (m_limit > m_match_count); } + +void Array::typed_print(std::string prefix) const +{ + std::cout << "Generic Array " << header_to_string(get_header()) << " @ " << m_ref; + if (!is_attached()) { + std::cout << " Unattached"; + return; + } + if (size() == 0) { + std::cout << " Empty" << std::endl; + return; + } + std::cout << " size = " << size() << " {"; + if (has_refs()) { + std::cout << std::endl; + for (unsigned n = 0; n < size(); ++n) { + auto pref = prefix + " " + to_string(n) + ":\t"; + RefOrTagged rot = get_as_ref_or_tagged(n); + if (rot.is_ref() && rot.get_as_ref()) { + Array a(m_alloc); + a.init_from_ref(rot.get_as_ref()); + std::cout << pref; + a.typed_print(pref); + } + else if (rot.is_tagged()) { + std::cout << pref << rot.get_as_int() << std::endl; + } + } + std::cout << prefix << "}" << std::endl; + } + else { + std::cout << " Leaf of unknown type }" << std::endl; + } +} + +ref_type ArrayPayload::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc) +{ + Array arr(alloc); + arr.init_from_ref(ref); + // By default we are not compressing + constexpr bool compress = false; + return arr.write(out, true, out.only_modified, compress); +} diff --git a/src/realm/array.hpp b/src/realm/array.hpp index d1f8463a64a..793e52df62b 100644 --- a/src/realm/array.hpp +++ b/src/realm/array.hpp @@ -379,11 +379,12 @@ class Array : public Node, public ArrayParent { /// /// \param only_if_modified Set to `false` to always write, or to `true` to /// only write the array if it has been modified. - ref_type write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified) const; + ref_type write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified, bool compress_in_flight) const; /// Same as non-static write() with `deep` set to true. This is for the /// cases where you do not already have an array accessor available. - static ref_type write(ref_type, Allocator&, _impl::ArrayWriterBase&, bool only_if_modified); + static ref_type write(ref_type, Allocator&, _impl::ArrayWriterBase&, bool only_if_modified, + bool compress_in_flight); size_t find_first(int64_t value, size_t begin = 0, size_t end = size_t(-1)) const; @@ -459,6 +460,13 @@ class Array : public Node, public ArrayParent { Array& operator=(const Array&) = delete; // not allowed Array(const Array&) = delete; // not allowed + /// Takes a 64-bit value and returns the minimum number of bits needed + /// to fit the value. For alignment this is rounded up to nearest + /// log2. Possible results {0, 1, 2, 4, 8, 16, 32, 64} + static size_t bit_width(int64_t value); + + void typed_print(std::string prefix) const; + protected: // This returns the minimum value ("lower bound") of the representable values // for the given bit width. Valid widths are 0, 1, 2, 4, 8, 16, 32, and 64. @@ -518,12 +526,6 @@ class Array : public Node, public ArrayParent { template int64_t get_universal(const char* const data, const size_t ndx) const; -protected: - /// Takes a 64-bit value and returns the minimum number of bits needed - /// to fit the value. For alignment this is rounded up to nearest - /// log2. Posssible results {0, 1, 2, 4, 8, 16, 32, 64} - static size_t bit_width(int64_t value); - protected: Getter m_getter = nullptr; // cached to avoid indirection const VTable* m_vtable = nullptr; @@ -538,7 +540,7 @@ class Array : public Node, public ArrayParent { private: ref_type do_write_shallow(_impl::ArrayWriterBase&) const; - ref_type do_write_deep(_impl::ArrayWriterBase&, bool only_if_modified) const; + ref_type do_write_deep(_impl::ArrayWriterBase&, bool only_if_modified, bool compress) const; void _mem_usage(size_t& mem) const noexcept; @@ -552,6 +554,23 @@ class Array : public Node, public ArrayParent { friend class ArrayWithFind; }; +class TempArray : public Array { +public: + TempArray(size_t sz, Type type = Type::type_HasRefs) + : Array(Allocator::get_default()) + { + create(type, false, sz); + } + ~TempArray() + { + destroy(); + } + ref_type write(_impl::ArrayWriterBase& out) + { + return Array::write(out, false, false, false); + } +}; + // Implementation: @@ -829,7 +848,7 @@ inline void Array::destroy_deep() noexcept m_data = nullptr; } -inline ref_type Array::write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified) const +inline ref_type Array::write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified, bool compress) const { REALM_ASSERT(is_attached()); @@ -839,10 +858,11 @@ inline ref_type Array::write(_impl::ArrayWriterBase& out, bool deep, bool only_i if (!deep || !m_has_refs) return do_write_shallow(out); // Throws - return do_write_deep(out, only_if_modified); // Throws + return do_write_deep(out, only_if_modified, compress); // Throws } -inline ref_type Array::write(ref_type ref, Allocator& alloc, _impl::ArrayWriterBase& out, bool only_if_modified) +inline ref_type Array::write(ref_type ref, Allocator& alloc, _impl::ArrayWriterBase& out, bool only_if_modified, + bool compress) { if (only_if_modified && alloc.is_read_only(ref)) return ref; @@ -853,7 +873,7 @@ inline ref_type Array::write(ref_type ref, Allocator& alloc, _impl::ArrayWriterB if (!array.m_has_refs) return array.do_write_shallow(out); // Throws - return array.do_write_deep(out, only_if_modified); // Throws + return array.do_write_deep(out, only_if_modified, compress); // Throws } inline void Array::add(int_fast64_t value) diff --git a/src/realm/array_integer.hpp b/src/realm/array_integer.hpp index bc2d4fca595..3b50d3757d1 100644 --- a/src/realm/array_integer.hpp +++ b/src/realm/array_integer.hpp @@ -70,6 +70,14 @@ class ArrayInteger : public Array, public ArrayPayload { } template bool find(value_type value, size_t start, size_t end, QueryStateBase* state) const; + + template + static ref_type typed_write(ref_type ref, T& out, Allocator& alloc) + { + Array arr(alloc); + arr.init_from_ref(ref); + return arr.write(out, false, out.only_modified, out.compress); + } }; class ArrayIntNull : public Array, public ArrayPayload { @@ -139,6 +147,14 @@ class ArrayIntNull : public Array, public ArrayPayload { size_t find_first(value_type value, size_t begin = 0, size_t end = npos) const; + template + static ref_type typed_write(ref_type ref, T& out, Allocator& alloc) + { + Array arr(alloc); + arr.init_from_ref(ref); + return arr.write(out, false, out.only_modified, out.compress); + } + protected: void avoid_null_collision(int64_t value); diff --git a/src/realm/array_mixed.cpp b/src/realm/array_mixed.cpp index ae580988b26..b0542da93b0 100644 --- a/src/realm/array_mixed.cpp +++ b/src/realm/array_mixed.cpp @@ -18,6 +18,8 @@ #include #include +#include +#include using namespace realm; @@ -328,6 +330,89 @@ void ArrayMixed::verify() const // TODO: Implement } +ref_type ArrayMixed::typed_write(ref_type top_ref, _impl::ArrayWriterBase& out, Allocator& alloc) +{ + if (out.only_modified && alloc.is_read_only(top_ref)) + return top_ref; + + ArrayRef top(alloc); + top.init_from_ref(top_ref); + size_t sz = top.size(); + TempArray written_leaf(sz); + + /* + Mixed stores things using different arrays. We need to take into account this in order to + understand what we need to compress and what we can instead leave not compressed. + + The main subarrays are: + + composite array : index 0 + int array : index 1 + pair_int array: index 2 + string array: index 3 + ref array: index 4 + key array: index 5 + + Description of each array: + 1. composite array: the data stored here is either a small int (< 32 bits) or an offset to one of + the other arrays where the actual data is. + 2. int and pair int arrays, they are used for storing integers, timestamps, floats, doubles, + decimals, links. In general we can compress them, but we need to be careful, controlling the col_type + should prevent compressing data that we want to leave in the current format. + 3. string array is for strings and binary data (no compression for now) + 4. ref array is actually storing refs to collections. they can only be BPlusTree or + BPlusTree. + 5. key array stores unique identifiers for collections in mixed (integers that can be compressed) + */ + Array composite(alloc); + composite.init_from_ref(top.get_as_ref(0)); + written_leaf.set_as_ref(0, composite.write(out, true, out.only_modified, false)); + for (size_t i = 1; i < sz; ++i) { + auto ref = top.get(i); + ref_type new_ref = ref; + if (ref && !(out.only_modified && alloc.is_read_only(ref))) { + if (i < 3) { // int, and pair_int + // integer arrays + new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress); + } + else if (i == 4) { // collection in mixed + ArrayRef arr_ref(alloc); + arr_ref.init_from_ref(ref); + auto ref_sz = arr_ref.size(); + TempArray written_ref_leaf(ref_sz); + + for (size_t k = 0; k < ref_sz; k++) { + ref_type new_sub_ref = 0; + if (auto sub_ref = arr_ref.get(k)) { + auto header = alloc.translate(sub_ref); + // Now we have to find out if the nested collection is a + // dictionary or a list. If the top array has a size of 2 + // and it is not a BplusTree inner node, then it is a dictionary + if (NodeHeader::get_size_from_header(header) == 2 && + !NodeHeader::get_is_inner_bptree_node_from_header(header)) { + new_sub_ref = Dictionary::typed_write(sub_ref, out, alloc); + } + else { + new_sub_ref = BPlusTree::typed_write(sub_ref, out, alloc); + } + } + written_ref_leaf.set_as_ref(k, new_sub_ref); + } + new_ref = written_ref_leaf.write(out); + } + else if (i == 5) { // unique keys associated to collections in mixed + new_ref = Array::write(ref, alloc, out, out.only_modified, out.compress); + } + else { + // all the rest we don't want to compress it, at least for now (strings will be needed) + new_ref = Array::write(ref, alloc, out, out.only_modified, false); + } + } + written_leaf.set(i, new_ref); + } + return written_leaf.write(out); +} + void ArrayMixed::ensure_array_accessor(Array& arr, size_t ndx_in_parent) const { if (!arr.is_attached()) { diff --git a/src/realm/array_mixed.hpp b/src/realm/array_mixed.hpp index ab71e254c63..a0de93b8339 100644 --- a/src/realm/array_mixed.hpp +++ b/src/realm/array_mixed.hpp @@ -100,6 +100,7 @@ class ArrayMixed : public ArrayPayload, private Array { int64_t get_key(size_t ndx) const; void verify() const; + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& outs, Allocator& alloc); private: enum { diff --git a/src/realm/array_timestamp.cpp b/src/realm/array_timestamp.cpp index 8efe9246e86..1e2549eb469 100644 --- a/src/realm/array_timestamp.cpp +++ b/src/realm/array_timestamp.cpp @@ -18,6 +18,7 @@ #include #include +#include using namespace realm; @@ -244,4 +245,25 @@ void ArrayTimestamp::verify() const REALM_ASSERT(m_seconds.size() == m_nanoseconds.size()); #endif } + +ref_type ArrayTimestamp::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc) +{ + // timestamps could be compressed, but the formats we support at the moment are not producing + // noticeable gains. + Array top(alloc); + top.init_from_ref(ref); + REALM_ASSERT_DEBUG(top.size() == 2); + + TempArray written_top(2); + + auto rot0 = top.get_as_ref_or_tagged(0); + auto rot1 = top.get_as_ref_or_tagged(1); + REALM_ASSERT_DEBUG(rot0.is_ref() && rot0.get_as_ref()); + REALM_ASSERT_DEBUG(rot1.is_ref() && rot1.get_as_ref()); + written_top.set_as_ref(0, Array::write(rot0.get_as_ref(), alloc, out, out.only_modified, false)); + written_top.set_as_ref(1, Array::write(rot1.get_as_ref(), alloc, out, out.only_modified, false)); + + return written_top.write(out); +} + } // namespace realm diff --git a/src/realm/array_timestamp.hpp b/src/realm/array_timestamp.hpp index 9ff8acfc1d8..621fc302a36 100644 --- a/src/realm/array_timestamp.hpp +++ b/src/realm/array_timestamp.hpp @@ -108,6 +108,7 @@ class ArrayTimestamp : public ArrayPayload, private Array { size_t find_first(Timestamp value, size_t begin, size_t end) const noexcept; void verify() const; + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc); private: ArrayIntNull m_seconds; diff --git a/src/realm/bplustree.cpp b/src/realm/bplustree.cpp index c545fa4440c..f993049159c 100644 --- a/src/realm/bplustree.cpp +++ b/src/realm/bplustree.cpp @@ -20,6 +20,7 @@ #include #include #include +#include using namespace realm; @@ -169,13 +170,11 @@ class BPlusTreeInner : public BPlusTreeNode, private Array { return (child_ndx) > 0 ? size_t(m_offsets.get(child_ndx - 1)) : 0; } }; -} +} // namespace realm /****************************** BPlusTreeNode ********************************/ -BPlusTreeNode::~BPlusTreeNode() -{ -} +BPlusTreeNode::~BPlusTreeNode() {} /****************************** BPlusTreeLeaf ********************************/ @@ -275,9 +274,7 @@ void BPlusTreeInner::create(size_t elems_per_child) Array::create(Array::type_InnerBptreeNode, false, 1, tagged); } -BPlusTreeInner::~BPlusTreeInner() -{ -} +BPlusTreeInner::~BPlusTreeInner() {} void BPlusTreeInner::init_from_mem(MemRef mem) { @@ -651,7 +648,7 @@ ref_type BPlusTreeInner::insert_bp_node(size_t child_ndx, ref_type new_sibling_r new_split_offset = size_t(elem_ndx_offset + state.split_offset); new_split_size = elem_ndx_offset + state.split_size; new_sibling.add_bp_node_ref(new_sibling_ref); // Throws - set_tree_size(new_split_offset); // Throws + set_tree_size(new_split_offset); // Throws } else { // Case 2/2: The split child was not the last child of the @@ -661,9 +658,9 @@ ref_type BPlusTreeInner::insert_bp_node(size_t child_ndx, ref_type new_sibling_r new_split_offset = size_t(elem_ndx_offset + state.split_size); new_split_size = get_tree_size() + 1; - move(&new_sibling, new_ref_ndx, (new_split_offset - 1)); // Strips off tree size + move(&new_sibling, new_ref_ndx, (new_split_offset - 1)); // Strips off tree size add_bp_node_ref(new_sibling_ref, elem_ndx_offset + state.split_offset); // Throws - append_tree_size(new_split_offset); // Throws + append_tree_size(new_split_offset); // Throws } new_sibling.append_tree_size(new_split_size - new_split_offset); // Throws @@ -737,9 +734,7 @@ void BPlusTreeInner::verify() const /****************************** BPlusTreeBase ********************************/ -BPlusTreeBase::~BPlusTreeBase() -{ -} +BPlusTreeBase::~BPlusTreeBase() {} void BPlusTreeBase::create() { @@ -837,6 +832,72 @@ std::unique_ptr BPlusTreeBase::create_root_from_ref(ref_type ref) } } +// this should only be called for a column_type which we can safely compress. +ref_type BPlusTreeBase::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc, + TypedWriteFunc leaf_write_func) +{ + if (out.only_modified && alloc.is_read_only(ref)) + return ref; + + if (!NodeHeader::get_is_inner_bptree_node_from_header(alloc.translate(ref))) { + // leaf + return leaf_write_func(ref, out, alloc); + } + + Array node(alloc); + node.init_from_ref(ref); + REALM_ASSERT_DEBUG(node.has_refs()); + TempArray written_node(node.size(), NodeHeader::type_InnerBptreeNode); + for (unsigned j = 0; j < node.size(); ++j) { + RefOrTagged rot = node.get_as_ref_or_tagged(j); + if (rot.is_ref() && rot.get_as_ref()) { + if (j == 0) { + // keys (ArrayUnsigned me thinks) + Array a(alloc); + a.init_from_ref(rot.get_as_ref()); + written_node.set_as_ref(j, a.write(out, false, out.only_modified, false)); + } + else { + written_node.set_as_ref(j, BPlusTreeBase::typed_write(rot.get_as_ref(), out, alloc, leaf_write_func)); + } + } + else + written_node.set(j, rot); + } + return written_node.write(out); +} + +void BPlusTreeBase::typed_print(std::string prefix, Allocator& alloc, ref_type root, ColumnType col_type) +{ + char* header = alloc.translate(root); + Array a(alloc); + a.init_from_ref(root); + if (NodeHeader::get_is_inner_bptree_node_from_header(header)) { + std::cout << "{" << std::endl; + REALM_ASSERT(a.has_refs()); + for (unsigned j = 0; j < a.size(); ++j) { + auto pref = prefix + " " + std::to_string(j) + ":\t"; + RefOrTagged rot = a.get_as_ref_or_tagged(j); + if (rot.is_ref() && rot.get_as_ref()) { + if (j == 0) { + std::cout << pref << "BPTree offsets as ArrayUnsigned as "; + Array a(alloc); + a.init_from_ref(rot.get_as_ref()); + a.typed_print(prefix); + } + else { + std::cout << pref << "Subtree beeing "; + BPlusTreeBase::typed_print(pref, alloc, rot.get_as_ref(), col_type); + } + } + } + } + else { + std::cout << "BPTree Leaf[" << col_type << "] as "; + a.typed_print(prefix); + } +} + size_t BPlusTreeBase::size_from_header(const char* header) { auto node_size = Array::get_size_from_header(header); diff --git a/src/realm/bplustree.hpp b/src/realm/bplustree.hpp index 6db1ffafc34..1f78d32ac26 100644 --- a/src/realm/bplustree.hpp +++ b/src/realm/bplustree.hpp @@ -126,6 +126,8 @@ class BPlusTreeLeaf : public BPlusTreeNode { /*****************************************************************************/ class BPlusTreeBase { public: + using TypedWriteFunc = ref_type (*)(ref_type, _impl::ArrayWriterBase&, Allocator&); + BPlusTreeBase(Allocator& alloc) : m_alloc(alloc) { @@ -216,6 +218,10 @@ class BPlusTreeBase { m_root->verify(); } + static ref_type typed_write(ref_type, _impl::ArrayWriterBase&, Allocator&, TypedWriteFunc); + static void typed_print(std::string prefix, Allocator& alloc, ref_type root, ColumnType col_type); + + protected: template struct LeafTypeTrait { @@ -556,6 +562,11 @@ class BPlusTree : public BPlusTreeBase { } } + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc) + { + return BPlusTreeBase::typed_write(ref, out, alloc, LeafArray::typed_write); + } + protected: LeafNode m_leaf_cache; @@ -685,6 +696,7 @@ ColumnAverageType bptree_average(const BPlusTree& tree, size_t* return_cnt *return_cnt = cnt; return avg; } + } // namespace realm #endif /* REALM_BPLUSTREE_HPP */ diff --git a/src/realm/cluster.cpp b/src/realm/cluster.cpp index 6dcf93887a2..f801bf98031 100644 --- a/src/realm/cluster.cpp +++ b/src/realm/cluster.cpp @@ -1531,4 +1531,233 @@ void Cluster::remove_backlinks(const Table* origin_table, ObjKey origin_key, Col } } +namespace { + +template +static void switch_on_type(ColKey ck, Fn&& fn) +{ + bool is_optional = ck.is_nullable(); + auto type = ck.get_type(); + switch (type) { + case col_type_Int: + return is_optional ? fn((util::Optional*)0) : fn((int64_t*)0); + case col_type_Bool: + return is_optional ? fn((util::Optional*)0) : fn((bool*)0); + case col_type_Float: + return is_optional ? fn((util::Optional*)0) : fn((float*)0); + case col_type_Double: + return is_optional ? fn((util::Optional*)0) : fn((double*)0); + case col_type_String: + return fn((StringData*)0); + case col_type_Binary: + return fn((BinaryData*)0); + case col_type_Timestamp: + return fn((Timestamp*)0); + case col_type_Link: + return fn((ObjKey*)0); + case col_type_ObjectId: + return is_optional ? fn((util::Optional*)0) : fn((ObjectId*)0); + case col_type_Decimal: + return fn((Decimal128*)0); + case col_type_UUID: + return is_optional ? fn((util::Optional*)0) : fn((UUID*)0); + case col_type_Mixed: + return fn((Mixed*)0); + default: + REALM_COMPILER_HINT_UNREACHABLE(); + } +} + +} // namespace + +ref_type Cluster::typed_write(ref_type ref, _impl::ArrayWriterBase& out) const +{ + REALM_ASSERT_DEBUG(ref == get_mem().get_ref()); + bool only_modified = out.only_modified; + if (only_modified && m_alloc.is_read_only(ref)) + return ref; + REALM_ASSERT_DEBUG(!get_is_inner_bptree_node_from_header(get_header())); + REALM_ASSERT_DEBUG(!get_context_flag_from_header(get_header())); + TempArray written_cluster(size()); + for (size_t j = 0; j < size(); ++j) { + RefOrTagged leaf_rot = get_as_ref_or_tagged(j); + // Handle nulls + if (!leaf_rot.is_ref() || !leaf_rot.get_as_ref()) { + written_cluster.set(j, leaf_rot); + continue; + } + // prune subtrees which should not be written: + if (only_modified && m_alloc.is_read_only(leaf_rot.get_as_ref())) { + written_cluster.set(j, leaf_rot); + continue; + } + // from here: this leaf exists and needs to be written. + ref = leaf_rot.get_as_ref(); + ref_type new_ref = ref; + if (j == 0) { + // Keys (ArrayUnsigned me thinks, so don't compress) + Array leaf(m_alloc); + leaf.init_from_ref(ref); + new_ref = leaf.write(out, false, only_modified, false); + } + else { + // Columns + auto col_key = out.table->m_leaf_ndx2colkey[j - 1]; + auto col_type = col_key.get_type(); + if (col_key.is_collection()) { + ArrayRef arr_ref(m_alloc); + arr_ref.init_from_ref(ref); + auto sz = arr_ref.size(); + TempArray written_ref_leaf(sz); + + for (size_t k = 0; k < sz; k++) { + ref_type new_sub_ref = 0; + // Now we have to find out if the nested collection is a + // dictionary or a list. If the top array has a size of 2 + // and it is not a BplusTree inner node, then it is a dictionary + if (auto sub_ref = arr_ref.get(k)) { + if (col_key.is_dictionary()) { + new_sub_ref = Dictionary::typed_write(sub_ref, out, m_alloc); + } + else { + // List or set - Can be handled the same way + // For some reason, switch_on_type() would not compile on Windows + // switch_on_type(col_key, [&](auto t) { + // using U = std::decay_t; + // new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + // }); + switch (col_type) { + case col_type_Int: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Bool: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Float: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Double: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_String: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Binary: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Timestamp: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Link: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_ObjectId: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Decimal: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_UUID: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + case col_type_Mixed: + new_sub_ref = BPlusTree::typed_write(sub_ref, out, m_alloc); + break; + default: + REALM_COMPILER_HINT_UNREACHABLE(); + } + } + } + written_ref_leaf.set_as_ref(k, new_sub_ref); + } + new_ref = written_ref_leaf.write(out); + } + else if (col_type == col_type_BackLink) { + Array leaf(m_alloc); + leaf.init_from_ref(ref); + new_ref = leaf.write(out, true, only_modified, false); + } + else { + switch_on_type(col_key, [&](auto t) { + using U = std::decay_t; + new_ref = ColumnTypeTraits::cluster_leaf_type::typed_write(ref, out, m_alloc); + }); + } + } + written_cluster.set_as_ref(j, new_ref); + } + return written_cluster.write(out); +} + +void Cluster::typed_print(std::string prefix) const +{ + REALM_ASSERT_DEBUG(!get_is_inner_bptree_node_from_header(get_header())); + std::cout << "Cluster of size " << size() << " " << header_to_string(get_header()) << std::endl; + const auto table = get_owning_table(); + for (unsigned j = 0; j < size(); ++j) { + RefOrTagged rot = get_as_ref_or_tagged(j); + auto pref = prefix + " " + std::to_string(j) + ":\t"; + if (rot.is_ref() && rot.get_as_ref()) { + if (j == 0) { + std::cout << pref << "Keys as ArrayUnsigned as "; + Array a(m_alloc); + a.init_from_ref(rot.get_as_ref()); + a.typed_print(pref); + } + else { + auto col_key = table->m_leaf_ndx2colkey[j - 1]; + auto col_type = col_key.get_type(); + auto col_attr = col_key.get_attrs(); + std::string attr_string; + if (col_attr.test(col_attr_Dictionary)) + attr_string = "Dict:"; + if (col_attr.test(col_attr_List)) + attr_string = "List:"; + if (col_attr.test(col_attr_Set)) + attr_string = "Set:"; + if (col_attr.test(col_attr_Nullable)) + attr_string += "Null:"; + std::cout << pref << "Column[" << attr_string << col_type << "] as "; + // special cases for the types we want to compress + if (col_attr.test(col_attr_List) || col_attr.test(col_attr_Set)) { + // That is a single bplustree + // propagation of nullable missing here? + // handling of mixed missing here? + BPlusTreeBase::typed_print(pref, m_alloc, rot.get_as_ref(), col_type); + } + else if (col_attr.test(col_attr_Dictionary)) { + Array dict_top(m_alloc); + dict_top.init_from_ref(rot.get_as_ref()); + if (dict_top.size() == 0) { + std::cout << "{ empty }" << std::endl; + continue; + } + std::cout << "{" << std::endl; + auto ref0 = dict_top.get_as_ref(0); + if (ref0) { + auto p = pref + " 0:\t"; + std::cout << p; + BPlusTreeBase::typed_print(p, m_alloc, ref0, col_type); + } + if (dict_top.size() == 1) { + continue; // is this really possible? or should all dicts have both trees? + } + auto ref1 = dict_top.get_as_ref(1); + if (ref1) { + auto p = pref + " 1:\t"; + std::cout << p; + BPlusTreeBase::typed_print(p, m_alloc, dict_top.get_as_ref(1), col_type); + } + } + else { + // handle all other cases as generic arrays + Array a(m_alloc); + a.init_from_ref(rot.get_as_ref()); + a.typed_print(pref); + } + } + } + } +} + } // namespace realm diff --git a/src/realm/cluster.hpp b/src/realm/cluster.hpp index f66fca9557f..68cd2f92298 100644 --- a/src/realm/cluster.hpp +++ b/src/realm/cluster.hpp @@ -80,9 +80,9 @@ class ClusterNode : public Array { // This structure is used to bring information back to the upper nodes when // inserting new objects or finding existing ones. struct State { - int64_t split_key; // When a node is split, this variable holds the value of the - // first key in the new node. (Relative to the key offset) - MemRef mem; // MemRef to the Cluster holding the new/found object + int64_t split_key; // When a node is split, this variable holds the value of the + // first key in the new node. (Relative to the key offset) + MemRef mem; // MemRef to the Cluster holding the new/found object size_t index = realm::npos; // The index within the Cluster at which the object is stored. operator bool() const @@ -113,9 +113,7 @@ class ClusterNode : public Array { { m_keys.set_parent(this, 0); } - virtual ~ClusterNode() - { - } + virtual ~ClusterNode() {} void init_from_parent() { ref_type ref = get_ref_from_parent(); @@ -199,6 +197,14 @@ class ClusterNode : public Array { { return m_offset; } + virtual ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const = 0; + + virtual void typed_print(std::string prefix) const + { + static_cast(get_owning_table()); + std::cout << "ClusterNode as "; + Array::typed_print(prefix); + } protected: #if REALM_MAX_BPNODE_SIZE > 256 @@ -215,7 +221,7 @@ class ClusterNode : public Array { uint64_t get(size_t ndx) const { - return (m_data != nullptr) ? ArrayUnsigned::get(ndx) : uint64_t(ndx); + return is_attached() ? ArrayUnsigned::get(ndx) : uint64_t(ndx); } }; @@ -313,6 +319,8 @@ class Cluster : public ClusterNode { void verify() const; void dump_objects(int64_t key_offset, std::string lead) const override; + virtual ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const override; + virtual void typed_print(std::string prefix) const override; static void remove_backlinks(const Table* origin_table, ObjKey origin_key, ColKey col, const std::vector& keys, CascadeState& state); static void remove_backlinks(const Table* origin_table, ObjKey origin_key, ColKey col, @@ -360,6 +368,6 @@ class Cluster : public ClusterNode { void verify(ref_type ref, size_t index, util::Optional& sz) const; }; -} +} // namespace realm #endif /* SRC_REALM_CLUSTER_HPP_ */ diff --git a/src/realm/cluster_tree.cpp b/src/realm/cluster_tree.cpp index 03102396fb0..29d5f52ce84 100644 --- a/src/realm/cluster_tree.cpp +++ b/src/realm/cluster_tree.cpp @@ -120,6 +120,88 @@ class ClusterNodeInner : public ClusterNode { void dump_objects(int64_t key_offset, std::string lead) const override; + virtual ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const override + { + REALM_ASSERT_DEBUG(ref == get_mem().get_ref()); + if (out.only_modified && m_alloc.is_read_only(ref)) { + return ref; + } + REALM_ASSERT_DEBUG(get_is_inner_bptree_node_from_header(get_header())); + REALM_ASSERT_DEBUG(!get_context_flag_from_header(get_header())); + REALM_ASSERT_DEBUG(has_refs()); + TempArray written_node(size(), type_InnerBptreeNode); + for (unsigned j = 0; j < size(); ++j) { + RefOrTagged rot = get_as_ref_or_tagged(j); + if (rot.is_ref() && rot.get_as_ref()) { + if (out.only_modified && m_alloc.is_read_only(rot.get_as_ref())) { + written_node.set(j, rot); + continue; + } + if (j == 0) { + // keys (ArrayUnsigned, me thinks) + Array array_unsigned(m_alloc); + array_unsigned.init_from_ref(rot.get_as_ref()); + written_node.set_as_ref(j, array_unsigned.write(out, false, out.only_modified, false)); + } + else { + auto header = m_alloc.translate(rot.get_as_ref()); + MemRef m(header, rot.get_as_ref(), m_alloc); + if (get_is_inner_bptree_node_from_header(header)) { + ClusterNodeInner inner_node(m_alloc, m_tree_top); + inner_node.init(m); + written_node.set_as_ref(j, inner_node.typed_write(rot.get_as_ref(), out)); + } + else { + Cluster cluster(j, m_alloc, m_tree_top); + cluster.init(m); + written_node.set_as_ref(j, cluster.typed_write(rot.get_as_ref(), out)); + } + } + } + else { // not a ref, just copy value over + written_node.set(j, rot); + } + } + return written_node.write(out); + } + + virtual void typed_print(std::string prefix) const override + { + REALM_ASSERT(get_is_inner_bptree_node_from_header(get_header())); + REALM_ASSERT(has_refs()); + std::cout << "ClusterNodeInner " << header_to_string(get_header()) << std::endl; + for (unsigned j = 0; j < size(); ++j) { + RefOrTagged rot = get_as_ref_or_tagged(j); + auto pref = prefix + " " + std::to_string(j) + ":\t"; + if (rot.is_ref() && rot.get_as_ref()) { + if (j == 0) { + std::cout << pref << "Keys as ArrayUnsigned as "; + Array a(m_alloc); + a.init_from_ref(rot.get_as_ref()); + a.typed_print(pref); + } + else { + auto header = m_alloc.translate(rot.get_as_ref()); + MemRef m(header, rot.get_as_ref(), m_alloc); + if (get_is_inner_bptree_node_from_header(header)) { + ClusterNodeInner a(m_alloc, m_tree_top); + a.init(m); + std::cout << pref; + a.typed_print(pref); + } + else { + Cluster a(j, m_alloc, m_tree_top); + a.init(m); + std::cout << pref; + a.typed_print(pref); + } + } + } + // just ignore entries, which are not refs. + } + Array::typed_print(prefix); + } + private: static constexpr size_t s_key_ref_index = 0; static constexpr size_t s_sub_tree_depth_index = 1; diff --git a/src/realm/cluster_tree.hpp b/src/realm/cluster_tree.hpp index 9ebaf243775..43d796c995e 100644 --- a/src/realm/cluster_tree.hpp +++ b/src/realm/cluster_tree.hpp @@ -190,6 +190,23 @@ class ClusterTree { } void verify() const; + ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const + { + REALM_ASSERT_DEBUG(m_root); + return m_root->typed_write(ref, out); + } + + void typed_print(std::string prefix) const + { + if (m_root) { + std::cout << prefix << "ClusterTree as "; + m_root->typed_print(prefix); + } + else { + std::cout << "Emtpy ClusterTree" << std::endl; + } + } + protected: friend class Obj; friend class Cluster; diff --git a/src/realm/db.cpp b/src/realm/db.cpp index ac2967bfdb9..2def85cc3e0 100644 --- a/src/realm/db.cpp +++ b/src/realm/db.cpp @@ -2557,6 +2557,10 @@ void DB::low_level_commit(uint_fast64_t new_version, Transaction& transaction, b // Add 4k to ensure progress on small commits size_t work_limit = commit_size / 2 + out.get_free_list_size() + 0x1000; transaction.cow_outliers(out.get_evacuation_progress(), limit, work_limit); + // moving blocks around may have left table accessors with stale data, + // and we need them to work later in the process when we determine which + // arrays to compress during writing, so make sure they're up to date: + transaction.update_table_accessors(); } ref_type new_top_ref; @@ -2755,7 +2759,7 @@ TransactionRef DB::start_write(bool nonblocking) end_write_on_correct_thread(); throw; } - + tr->update_allocator_wrappers(true); return tr; } diff --git a/src/realm/dictionary.cpp b/src/realm/dictionary.cpp index 2405afe5695..fea04af1629 100644 --- a/src/realm/dictionary.cpp +++ b/src/realm/dictionary.cpp @@ -1212,6 +1212,36 @@ LinkCollectionPtr Dictionary::clone_as_obj_list() const return nullptr; } +ref_type Dictionary::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc) +{ + if (out.only_modified && alloc.is_read_only(ref)) + return ref; + + ArrayRef dict_top(alloc); + dict_top.init_from_ref(ref); + REALM_ASSERT_DEBUG(dict_top.size() == 2); + TempArray written_dict_top(2); + + // We have to find out what kind of keys we are using - strings or ints + // Btw - ints is only used in tests. Can probably be removed at some point + auto key_ref = dict_top.get(0); + auto header = alloc.translate(key_ref); + if (!NodeHeader::get_hasrefs_from_header(header) && + NodeHeader::get_wtype_from_header(header) != Array::wtype_Multiply) { + // Key type int. + REALM_ASSERT(!NodeHeader::get_is_inner_bptree_node_from_header(header)); + written_dict_top.set_as_ref(0, BPlusTree::typed_write(key_ref, out, alloc)); + } + else { + written_dict_top.set_as_ref(0, BPlusTree::typed_write(key_ref, out, alloc)); + } + + auto values_ref = dict_top.get_as_ref(1); + written_dict_top.set_as_ref(1, BPlusTree::typed_write(values_ref, out, alloc)); + + return written_dict_top.write(out); +} + /************************* DictionaryLinkValues *************************/ DictionaryLinkValues::DictionaryLinkValues(const Obj& obj, ColKey col_key) diff --git a/src/realm/dictionary.hpp b/src/realm/dictionary.hpp index a01c7fd8b8e..be8765f5a8f 100644 --- a/src/realm/dictionary.hpp +++ b/src/realm/dictionary.hpp @@ -232,6 +232,8 @@ class Dictionary final : public CollectionBaseImpl, public Colle LinkCollectionPtr clone_as_obj_list() const final; + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc); + private: using Base::set_collection; diff --git a/src/realm/group.cpp b/src/realm/group.cpp index f3e2728794d..a2490f6b326 100644 --- a/src/realm/group.cpp +++ b/src/realm/group.cpp @@ -490,6 +490,19 @@ void Group::remap_and_update_refs(ref_type new_top_ref, size_t new_file_size, bo update_refs(new_top_ref); } +void Group::update_table_accessors() +{ + for (unsigned j = 0; j < m_table_accessors.size(); ++j) { + Table* table = m_table_accessors[j]; + // this should be filtered further as an optimization + if (table) { + table->refresh_allocator_wrapper(); + table->update_from_parent(); + } + } +} + + void Group::validate_top_array(const Array& arr, const SlabAlloc& alloc, std::optional read_lock_file_size, std::optional read_lock_version) { @@ -928,23 +941,92 @@ void Group::validate(ObjLink link) const } } +ref_type Group::typed_write_tables(_impl::ArrayWriterBase& out) const +{ + ref_type ref = m_top.get_as_ref(1); + if (out.only_modified && m_alloc.is_read_only(ref)) + return ref; + Array a(m_alloc); + a.init_from_ref(ref); + REALM_ASSERT_DEBUG(a.has_refs()); + TempArray dest(a.size()); + for (unsigned j = 0; j < a.size(); ++j) { + RefOrTagged rot = a.get_as_ref_or_tagged(j); + if (rot.is_tagged()) { + dest.set(j, rot); + } + else { + auto table = do_get_table(j); + REALM_ASSERT_DEBUG(table); + dest.set_as_ref(j, table->typed_write(rot.get_as_ref(), out)); + } + } + return dest.write(out); +} +void Group::table_typed_print(std::string prefix, ref_type ref) const +{ + REALM_ASSERT(m_top.get_as_ref(1) == ref); + Array a(m_alloc); + a.init_from_ref(ref); + REALM_ASSERT(a.has_refs()); + for (unsigned j = 0; j < a.size(); ++j) { + auto pref = prefix + " " + to_string(j) + ":\t"; + RefOrTagged rot = a.get_as_ref_or_tagged(j); + if (rot.is_tagged() || rot.get_as_ref() == 0) + continue; + auto table_accessor = do_get_table(j); + REALM_ASSERT(table_accessor); + table_accessor->typed_print(pref, rot.get_as_ref()); + } +} +void Group::typed_print(std::string prefix) const +{ + std::cout << "Group top array" << std::endl; + for (unsigned j = 0; j < m_top.size(); ++j) { + auto pref = prefix + " " + to_string(j) + ":\t"; + RefOrTagged rot = m_top.get_as_ref_or_tagged(j); + if (rot.is_ref() && rot.get_as_ref()) { + if (j == 1) { + // Tables + std::cout << pref << "All Tables" << std::endl; + table_typed_print(pref, rot.get_as_ref()); + } + else { + Array a(m_alloc); + a.init_from_ref(rot.get_as_ref()); + std::cout << pref; + a.typed_print(pref); + } + } + else { + std::cout << pref << rot.get_as_int() << std::endl; + } + } + std::cout << "}" << std::endl; +} + + ref_type Group::DefaultTableWriter::write_names(_impl::OutputStream& out) { - bool deep = true; // Deep - bool only_if_modified = false; // Always - return m_group->m_table_names.write(out, deep, only_if_modified); // Throws + bool deep = true; // Deep + bool only_if_modified = false; // Always + bool compress = false; // true; + return m_group->m_table_names.write(out, deep, only_if_modified, compress); // Throws } ref_type Group::DefaultTableWriter::write_tables(_impl::OutputStream& out) { - bool deep = true; // Deep - bool only_if_modified = false; // Always - return m_group->m_tables.write(out, deep, only_if_modified); // Throws + // bool deep = true; // Deep + // bool only_if_modified = false; // Always + // bool compress = false; // true; + // return m_group->m_tables.write(out, deep, only_if_modified, compress); // Throws + return m_group->typed_write_tables(out); } auto Group::DefaultTableWriter::write_history(_impl::OutputStream& out) -> HistoryInfo { bool deep = true; // Deep bool only_if_modified = false; // Always + bool compress = false; ref_type history_ref = _impl::GroupFriend::get_history_ref(*m_group); HistoryInfo info; if (history_ref) { @@ -962,7 +1044,7 @@ auto Group::DefaultTableWriter::write_history(_impl::OutputStream& out) -> Histo info.version = history_schema_version; Array history{const_cast(_impl::GroupFriend::get_alloc(*m_group))}; history.init_from_ref(history_ref); - info.ref = history.write(out, deep, only_if_modified); // Throws + info.ref = history.write(out, deep, only_if_modified, compress); // Throws } info.sync_file_id = m_group->get_sync_file_id(); return info; @@ -1041,6 +1123,7 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table bool pad_for_encryption, uint_fast64_t version_number) { _impl::OutputStream out_2(out); + out_2.only_modified = false; // Write the file header SlabAlloc::Header streaming_header; @@ -1062,6 +1145,7 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table REALM_ASSERT(version_number == 0 || version_number == 1); } else { + // table_writer.typed_print(""); // Because we need to include the total logical file size in the // top-array, we have to start by writing everything except the // top-array, and then finally compute and write a correct version of @@ -1098,9 +1182,10 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table _impl::DeepArrayDestroyGuard dg_3(&version_list); bool deep = true; // Deep bool only_if_modified = false; // Always - ref_type free_list_ref = free_list.write(out_2, deep, only_if_modified); - ref_type size_list_ref = size_list.write(out_2, deep, only_if_modified); - ref_type version_list_ref = version_list.write(out_2, deep, only_if_modified); + bool compress = false; + ref_type free_list_ref = free_list.write(out_2, deep, only_if_modified, compress); + ref_type size_list_ref = size_list.write(out_2, deep, only_if_modified, compress); + ref_type version_list_ref = version_list.write(out_2, deep, only_if_modified, compress); top.add(RefOrTagged::make_ref(free_list_ref)); // Throws top.add(RefOrTagged::make_ref(size_list_ref)); // Throws top.add(RefOrTagged::make_ref(version_list_ref)); // Throws @@ -1135,7 +1220,8 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table // Write the top array bool deep = false; // Shallow bool only_if_modified = false; // Always - top.write(out_2, deep, only_if_modified); // Throws + bool compress = false; + top.write(out_2, deep, only_if_modified, compress); // Throws REALM_ASSERT_3(size_t(out_2.get_ref_of_next_array()), ==, final_file_size); dg_top.reset(nullptr); // Destroy now @@ -1274,6 +1360,11 @@ class TransactAdvancer : public _impl::NullInstructionObserver { void Group::update_allocator_wrappers(bool writable) { m_is_writable = writable; + // This is tempting: + // m_alloc.set_read_only(!writable); + // - but m_alloc may refer to the "global" allocator in the DB object. + // Setting it here would cause different transactions to raze for + // changing the shared allocator setting. This is somewhat of a mess. for (size_t i = 0; i < m_table_accessors.size(); ++i) { auto table_accessor = m_table_accessors[i]; if (table_accessor) { diff --git a/src/realm/group.hpp b/src/realm/group.hpp index ff663b723e3..eba80721a6a 100644 --- a/src/realm/group.hpp +++ b/src/realm/group.hpp @@ -517,6 +517,9 @@ class Group : public ArrayParent { m_alloc.enable_debug(enable); } #endif + ref_type typed_write_tables(_impl::ArrayWriterBase& out) const; + void table_typed_print(std::string prefix, ref_type ref) const; + void typed_print(std::string prefix) const; protected: static constexpr size_t s_table_name_ndx = 0; @@ -638,7 +641,7 @@ class Group : public ArrayParent { void reset_free_space_tracking(); void remap_and_update_refs(ref_type new_top_ref, size_t new_file_size, bool writable); - + void update_table_accessors(); /// Recursively update refs stored in all cached array /// accessors. This includes cached array accessors in any /// currently attached table accessors. This ensures that the @@ -1128,6 +1131,10 @@ class Group::TableWriter { virtual ref_type write_names(_impl::OutputStream&) = 0; virtual ref_type write_tables(_impl::OutputStream&) = 0; virtual HistoryInfo write_history(_impl::OutputStream&) = 0; + void typed_print(std::string prefix) + { + m_group->typed_print(prefix); + } virtual ~TableWriter() noexcept {} void set_group(const Group* g) diff --git a/src/realm/group_writer.cpp b/src/realm/group_writer.cpp index 9243d23e0b6..dd112444f80 100644 --- a/src/realm/group_writer.cpp +++ b/src/realm/group_writer.cpp @@ -659,15 +659,17 @@ ref_type GroupWriter::write_group() // that has been release during the current transaction (or since the last // commit), as that would lead to clobbering of the previous database // version. - bool deep = true, only_if_modified = true; + constexpr bool deep = true; + _impl::ArrayWriterBase::compress = true; + _impl::ArrayWriterBase::only_modified = true; std::unique_ptr in_memory_writer; _impl::ArrayWriterBase* writer = this; if (m_alloc.is_in_memory()) { in_memory_writer = std::make_unique(*this); writer = in_memory_writer.get(); } - ref_type names_ref = m_group.m_table_names.write(*writer, deep, only_if_modified); // Throws - ref_type tables_ref = m_group.m_tables.write(*writer, deep, only_if_modified); // Throws + ref_type names_ref = m_group.m_table_names.write(*writer, deep, only_modified, compress); // Throws + ref_type tables_ref = m_group.typed_write_tables(*writer); int_fast64_t value_1 = from_ref(names_ref); int_fast64_t value_2 = from_ref(tables_ref); @@ -680,8 +682,8 @@ ref_type GroupWriter::write_group() if (top.size() > Group::s_hist_ref_ndx) { if (ref_type history_ref = top.get_as_ref(Group::s_hist_ref_ndx)) { Allocator& alloc = top.get_alloc(); - ref_type new_history_ref = Array::write(history_ref, alloc, *writer, only_if_modified); // Throws - top.set(Group::s_hist_ref_ndx, from_ref(new_history_ref)); // Throws + ref_type new_history_ref = Array::write(history_ref, alloc, *writer, only_modified, false); // Throws + top.set(Group::s_hist_ref_ndx, from_ref(new_history_ref)); // Throws } } if (top.size() > Group::s_evacuation_point_ndx) { @@ -703,7 +705,7 @@ ref_type GroupWriter::write_group() for (auto index : m_evacuation_progress) { arr.add(int64_t(index)); } - ref = arr.write(*writer, false, only_if_modified); + ref = arr.write(*writer, false, only_modified, compress); top.set_as_ref(Group::s_evacuation_point_ndx, ref); } else if (ref) { diff --git a/src/realm/impl/array_writer.hpp b/src/realm/impl/array_writer.hpp index f039ad505f9..5066800b5ba 100644 --- a/src/realm/impl/array_writer.hpp +++ b/src/realm/impl/array_writer.hpp @@ -22,10 +22,14 @@ #include namespace realm { +class Table; namespace _impl { class ArrayWriterBase { public: + bool only_modified = true; + bool compress = true; + const Table* table; virtual ~ArrayWriterBase() { } diff --git a/src/realm/node.hpp b/src/realm/node.hpp index ea4d55a0e8c..5cb637ab7d1 100644 --- a/src/realm/node.hpp +++ b/src/realm/node.hpp @@ -22,6 +22,8 @@ #include #include +#include + namespace realm { class Mixed; @@ -261,6 +263,11 @@ class Node : public NodeHeader { } } + void typed_print(int) const + { + std::cout << "Generic Node ERROR\n"; + } + protected: /// The total size in bytes (including the header) of a new empty /// array. Must be a multiple of 8 (i.e., 64-bit aligned). @@ -339,6 +346,10 @@ class Node : public NodeHeader { class Spec; class Mixed; +namespace _impl { +class ArrayWriterBase; +} + /// Base class for all nodes holding user data class ArrayPayload { public: @@ -351,6 +362,7 @@ class ArrayPayload { return false; } virtual void set_spec(Spec*, size_t) const {} + static ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc); }; } // namespace realm diff --git a/src/realm/node_header.hpp b/src/realm/node_header.hpp index c475bffc1b2..d8e6652106e 100644 --- a/src/realm/node_header.hpp +++ b/src/realm/node_header.hpp @@ -208,7 +208,7 @@ class NodeHeader { static size_t unsigned_to_num_bits(uint64_t value) { if constexpr (sizeof(size_t) == sizeof(uint64_t)) - return 1 + log2(value); + return static_cast(1) + log2(value); uint32_t high = value >> 32; if (high) return 33 + log2(high); @@ -301,7 +301,9 @@ class NodeHeader { auto wtype = get_wtype_from_header(header); if (wtype == wtype_Extend) { const auto h = reinterpret_cast(header); - return static_cast(h[5] + 3); + int encoding = h[5] + 3; + REALM_ASSERT_DEBUG_EX(encoding >= int(Encoding::Packed) && encoding <= int(Encoding::Flex), encoding); + return static_cast(encoding); } return Encoding(int(wtype)); } @@ -566,6 +568,7 @@ inline size_t NodeHeader::get_num_elements(const char* header, Encoding encoding return get_arrayB_num_elements(header); break; default: + printf("Encoding %d\n", int(encoding)); REALM_UNREACHABLE(); } } @@ -669,7 +672,6 @@ size_t inline NodeHeader::get_byte_size_from_header(const char* header) noexcept const auto h = header; const auto encoding = get_encoding(h); - REALM_ASSERT_DEBUG(encoding >= Encoding::WTypBits && encoding <= Encoding::Flex); const auto size = get_num_elements(h, encoding); switch (encoding) { case Encoding::WTypBits: diff --git a/src/realm/spec.hpp b/src/realm/spec.hpp index 9db898cb969..28e126653b4 100644 --- a/src/realm/spec.hpp +++ b/src/realm/spec.hpp @@ -84,6 +84,11 @@ class Spec { void set_ndx_in_parent(size_t) noexcept; void verify() const; + void typed_print(std::string prefix) const + { + std::cout << prefix << "Spec as "; + m_top.typed_print(prefix); + } private: // Underlying array structure. diff --git a/src/realm/table.cpp b/src/realm/table.cpp index 81926077d0d..f152cd5344b 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -3336,3 +3336,60 @@ ColKey Table::find_opposite_column(ColKey col_key) const } return ColKey(); } + +ref_type Table::typed_write(ref_type ref, _impl::ArrayWriterBase& out) const +{ + REALM_ASSERT(ref == m_top.get_mem().get_ref()); + if (out.only_modified && m_alloc.is_read_only(ref)) + return ref; + out.table = this; + // ignore ref from here, just use Tables own accessors + TempArray dest(m_top.size()); + for (unsigned j = 0; j < m_top.size(); ++j) { + RefOrTagged rot = m_top.get_as_ref_or_tagged(j); + if (rot.is_tagged() || (rot.is_ref() && rot.get_as_ref() == 0)) { + dest.set(j, rot); + } + else { + ref_type new_ref; + if (j == 2) { + // only do type driven write for clustertree + new_ref = m_clusters.typed_write(rot.get_as_ref(), out); + } + else { + // rest is handled using untyped approach + Array a(m_alloc); + a.init_from_ref(rot.get_as_ref()); + new_ref = a.write(out, true, out.only_modified, false); + } + dest.set_as_ref(j, new_ref); + } + } + return dest.write(out); +} + +void Table::typed_print(std::string prefix, ref_type ref) const +{ + REALM_ASSERT(ref == m_top.get_mem().get_ref()); + std::cout << prefix << "Table with key = " << m_key << " " << NodeHeader::header_to_string(m_top.get_header()) + << " {" << std::endl; + for (unsigned j = 0; j < m_top.size(); ++j) { + auto pref = prefix + " " + to_string(j) + ":\t"; + auto rot = m_top.get_as_ref_or_tagged(j); + if (rot.is_ref() && rot.get_as_ref()) { + if (j == 0) { + m_spec.typed_print(pref); + } + else if (j == 2) { + m_clusters.typed_print(pref); + } + else { + Array a(m_alloc); + a.init_from_ref(rot.get_as_ref()); + std::cout << pref; + a.typed_print(pref); + } + } + } + std::cout << prefix << "}" << std::endl; +} diff --git a/src/realm/table.hpp b/src/realm/table.hpp index 6bdf33bc047..46b0a6f9587 100644 --- a/src/realm/table.hpp +++ b/src/realm/table.hpp @@ -63,8 +63,7 @@ template class SubQuery; class TableView; -struct Link { -}; +struct Link {}; typedef Link BackLink; @@ -691,6 +690,9 @@ class Table { Replication* const* m_repl; }; + ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const; + void typed_print(std::string prefix, ref_type ref) const; + private: enum LifeCycleCookie { cookie_created = 0x1234, @@ -727,13 +729,13 @@ class Table { { m_alloc.refresh_ref_translation(); } - Spec m_spec; // 1st slot in m_top - ClusterTree m_clusters; // 3rd slot in m_top - std::unique_ptr m_tombstones; // 13th slot in m_top - TableKey m_key; // 4th slot in m_top - Array m_index_refs; // 5th slot in m_top - Array m_opposite_table; // 7th slot in m_top - Array m_opposite_column; // 8th slot in m_top + Spec m_spec; // 1st slot in m_top + ClusterTree m_clusters; // 3rd slot in m_top + std::unique_ptr m_tombstones; // 13th slot in m_top + TableKey m_key; // 4th slot in m_top + Array m_index_refs; // 5th slot in m_top + Array m_opposite_table; // 7th slot in m_top + Array m_opposite_column; // 8th slot in m_top std::vector> m_index_accessors; ColKey m_primary_key_col; Replication* const* m_repl; diff --git a/src/realm/transaction.hpp b/src/realm/transaction.hpp index abd95d7d3ff..4da316c0d2e 100644 --- a/src/realm/transaction.hpp +++ b/src/realm/transaction.hpp @@ -411,7 +411,7 @@ inline bool Transaction::promote_to_write(O* observer, bool nonblocking) db->m_logger->log(util::LogCategory::transaction, util::Logger::Level::trace, "Tr %1: Promote to write: %2 -> %3", m_log_id, old_version, m_read_lock.m_version); } - + update_allocator_wrappers(true); set_transact_stage(DB::transact_Writing); return true; } From 2569ca469a00c2a517750ee1cde25c32707e07d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Wed, 24 Apr 2024 09:07:34 +0200 Subject: [PATCH 05/18] Update test with compaction of nested collection --- test/test_list.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_list.cpp b/test/test_list.cpp index 26dbc0163a9..14e517141df 100644 --- a/test/test_list.cpp +++ b/test/test_list.cpp @@ -824,6 +824,10 @@ TEST(List_Nested_InMixed) "[{\"Seven\":7, \"Six\":6}, \"Hello\", {\"Points\": [1.25, 4.5, 6.75], \"Hello\": \"World\"}]"); CHECK_EQUAL(obj.get_list_ptr(col_any)->size(), 3); // tr->to_json(std::cout); + tr->commit(); + db->compact(); + tr = db->start_write(); + tr->verify(); } From c77f07ef64ebf72d2e35863ae2b0c29bb87661f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Tue, 14 May 2024 13:27:32 +0200 Subject: [PATCH 06/18] Preserve context flag when comitting BPlusTree --- src/realm/array.hpp | 4 ++-- src/realm/bplustree.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/realm/array.hpp b/src/realm/array.hpp index 793e52df62b..1df0aa2b992 100644 --- a/src/realm/array.hpp +++ b/src/realm/array.hpp @@ -556,10 +556,10 @@ class Array : public Node, public ArrayParent { class TempArray : public Array { public: - TempArray(size_t sz, Type type = Type::type_HasRefs) + TempArray(size_t sz, Type type = Type::type_HasRefs, bool cf = false) : Array(Allocator::get_default()) { - create(type, false, sz); + create(type, cf, sz); } ~TempArray() { diff --git a/src/realm/bplustree.cpp b/src/realm/bplustree.cpp index f993049159c..b07e68d0e2b 100644 --- a/src/realm/bplustree.cpp +++ b/src/realm/bplustree.cpp @@ -847,7 +847,7 @@ ref_type BPlusTreeBase::typed_write(ref_type ref, _impl::ArrayWriterBase& out, A Array node(alloc); node.init_from_ref(ref); REALM_ASSERT_DEBUG(node.has_refs()); - TempArray written_node(node.size(), NodeHeader::type_InnerBptreeNode); + TempArray written_node(node.size(), NodeHeader::type_InnerBptreeNode, node.get_context_flag()); for (unsigned j = 0; j < node.size(); ++j) { RefOrTagged rot = node.get_as_ref_or_tagged(j); if (rot.is_ref() && rot.get_as_ref()) { From bb518a4908ba31723feeb7d28f75b8f4fdab4c9a Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Tue, 14 May 2024 15:30:26 +0100 Subject: [PATCH 07/18] RCORE-2076 Introducing bitfield iterator for accessing unaligned values for compressed arrays (#7571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update next-major * specified part of new layout (new width encoding) * new header format for compressed arrays * code review * code review * start of classifying arrays for compression * classification down to column types * first attempt to cut through the BPlusTree madness * [wip] start on 'type driven' write process * all tests passing (but no compression enabled) * enabled compression for signed integer leafs only * removed some dubious constructions in cluster tree * delete tmp array while classifying arrays * enabled compression of links and backlinks (excl collections) * also compress bplustree of integers/links (experimental) * pref for compressing dicts (not working) * wip * wip * finally: compressing collections (incl dicts) * compressing timestamps now * enabled compression on ObjectID, TypedLink and UUID * also compressing Mixed properties (not list/dicts of Mixed) * Array compression with collections in Mixed (#7412) --------- Co-authored-by: Finn Schiermer Andersen * merge next-major + collection in mixed * enable dynamic choice of compression method * moved typed_write/typed_print for bptree into class * Merge pull request #7432 from realm/fsa/clean_typed_write moved typed_write/typed_print for bptree into class * cleanup unrelated code changes * fix compilation * cleanup * code review * code review * scaffolding for accessing unaligned memory inside compressed arrays * introducing parallel scan for scanning in one go multiple compressed arrays * Silence warning * tests + point fix for upper bound * fix ubsan * remove dead code in subword search * remove tests of 'unaligned iterator' for flexibility * Some fixes to conform with coding polycies * Enhance test * bugfix for serch with width=31 bits * bugfix for search with width=31 bits * bugfix for search with width=31 bits --------- Co-authored-by: Finn Schiermer Andersen Co-authored-by: Jørgen Edelbo Co-authored-by: Finn Schiermer Andersen --- src/realm/array_direct.hpp | 651 +++++++++++++++++++++++++++++++++++-- src/realm/group_writer.cpp | 4 +- test/test_array.cpp | 522 +++++++++++++++++++++++++++++ 3 files changed, 1153 insertions(+), 24 deletions(-) diff --git a/src/realm/array_direct.hpp b/src/realm/array_direct.hpp index d595363c762..97c5a2b0d33 100644 --- a/src/realm/array_direct.hpp +++ b/src/realm/array_direct.hpp @@ -19,6 +19,7 @@ #ifndef REALM_ARRAY_DIRECT_HPP #define REALM_ARRAY_DIRECT_HPP +#include #include #include @@ -71,11 +72,6 @@ namespace realm { -/// Takes a 64-bit value and returns the minimum number of bits needed -/// to fit the value. For alignment this is rounded up to nearest -/// log2. Posssible results {0, 1, 2, 4, 8, 16, 32, 64} -size_t bit_width(int64_t value); - // Direct access methods template @@ -168,15 +164,15 @@ int64_t get_direct(const char* data, size_t ndx) noexcept return *reinterpret_cast(data + ndx); } if (w == 16) { - size_t offset = ndx * 2; + size_t offset = ndx << 1; return *reinterpret_cast(data + offset); } if (w == 32) { - size_t offset = ndx * 4; + size_t offset = ndx << 2; return *reinterpret_cast(data + offset); } if (w == 64) { - size_t offset = ndx * 8; + size_t offset = ndx << 3; return *reinterpret_cast(data + offset); } REALM_ASSERT_DEBUG(false); @@ -188,6 +184,222 @@ inline int64_t get_direct(const char* data, size_t width, size_t ndx) noexcept REALM_TEMPEX(return get_direct, width, (data, ndx)); } +// An iterator for getting a 64 bit word from any (byte-address+bit-offset) address. +class UnalignedWordIter { +public: + UnalignedWordIter(const uint64_t* data, size_t bit_offset) + : m_word_ptr(data + (bit_offset >> 6)) + , m_in_word_offset(bit_offset & 0x3F) + { + } + // 'num_bits' number of bits which must be read + // WARNING returned word may be garbage above the first 'num_bits' bits. + uint64_t get(size_t num_bits) + { + auto first_word = m_word_ptr[0]; + uint64_t result = first_word >> m_in_word_offset; + // note: above shifts in zeroes + if (m_in_word_offset + num_bits <= 64) + return result; + // if we're here, in_word_offset > 0 + auto first_word_size = 64 - m_in_word_offset; + auto second_word = m_word_ptr[1]; + result |= second_word << first_word_size; + // note: above shifts in zeroes below the bits we want + return result; + } + uint64_t get_with_unsafe_prefetch(size_t num_bits) + { + auto first_word = m_word_ptr[0]; + uint64_t result = first_word >> m_in_word_offset; + // note: above shifts in zeroes + auto first_word_size = 64 - m_in_word_offset; + auto second_word = m_word_ptr[1]; + REALM_ASSERT_DEBUG(num_bits <= 64); + result |= (m_in_word_offset + num_bits > 64) ? (second_word << first_word_size) : 0; + // note: above shifts in zeroes below the bits we want + return result; + } + // bump the iterator the specified number of bits + void bump(size_t num_bits) + { + auto total_offset = m_in_word_offset + num_bits; + m_word_ptr += total_offset >> 6; + m_in_word_offset = total_offset & 0x3F; + } + +private: + const uint64_t* m_word_ptr; + unsigned m_in_word_offset; +}; + +// Read a bit field of up to 64 bits. +// - Any alignment and size is supported +// - The start of the 'data' area must be 64 bit aligned in all cases. +// - For fields of 64-bit or less, the first 64-bit word is filled with the zero-extended +// value of the bitfield. +// iterator useful for scanning arrays faster than by indexing each element +// supports arrays of pairs by differentiating field size and step size. +class BfIterator { +public: + BfIterator() = default; + BfIterator(const BfIterator&) = default; + BfIterator(BfIterator&&) = default; + BfIterator& operator=(const BfIterator&) = default; + BfIterator& operator=(BfIterator&&) = default; + BfIterator(uint64_t* data_area, size_t initial_offset, size_t field_size, size_t step_size, size_t index) + : data_area(data_area) + , field_size(static_cast(field_size)) + , step_size(static_cast(step_size)) + , offset(initial_offset) + { + if (field_size < 64) + mask = (1ULL << field_size) - 1; + move(index); + } + + inline uint64_t get_full_word_with_value() const + { + const auto in_word_position = field_position & 0x3F; + const auto first_word = first_word_ptr[0]; + uint64_t result = first_word >> in_word_position; + // note: above shifts in zeroes above the bitfield + if (in_word_position + field_size > 64) { + // if we're here, in_word_position > 0 + const auto first_word_size = 64 - in_word_position; + const auto second_word = first_word_ptr[1]; + return result | second_word << first_word_size; + // note: above shifts in zeroes below the bits we want + } + return result; + } + + inline uint64_t get_value() const + { + auto result = get_full_word_with_value(); + // discard any bits above the field we want + if (field_size < 64) + result &= mask; + return result; + } + + // get unaligned word - this should not be called if the next word extends beyond + // end of array. For that particular case, you must use get_last_unaligned_word instead. + inline uint64_t get_unaligned_word() const + { + const auto in_word_position = field_position & 0x3F; + const auto first_word = first_word_ptr[0]; + if (in_word_position == 0) + return first_word; + uint64_t result = first_word >> in_word_position; + // note: above shifts in zeroes above the bitfield + const auto first_word_size = 64 - in_word_position; + const auto second_word = first_word_ptr[1]; + result |= second_word << first_word_size; + // note: above shifts in zeroes below the bits we want + return result; + } + + inline uint64_t get_last_unaligned_word() const + { + const auto in_word_position = field_position & 0x3F; + const auto first_word = first_word_ptr[0]; + const uint64_t result = first_word >> in_word_position; + // note: above shifts in zeroes above the bitfield + return result; + } + + void set_value(uint64_t value) const + { + const auto in_word_position = field_position & 0x3F; + auto first_word = first_word_ptr[0]; + uint64_t mask = 0ULL - 1ULL; + if (field_size < 64) { + mask = (1ULL << field_size) - 1; + value &= mask; + } + // zero out field in first word: + const auto first_word_mask = ~(mask << in_word_position); + first_word &= first_word_mask; + // or in relevant part of value + first_word |= value << in_word_position; + first_word_ptr[0] = first_word; + if (in_word_position + field_size > 64) { + // bitfield crosses word boundary. + // discard the lowest bits of value (it has been written to the first word) + const auto bits_written_to_first_word = 64 - in_word_position; + // bit_written_to_first_word must be lower than 64, so shifts based on it are well defined + value >>= bits_written_to_first_word; + const auto second_word_mask = mask >> bits_written_to_first_word; + auto second_word = first_word_ptr[1]; + // zero out the field in second word, then or in the (high part of) value + second_word &= ~second_word_mask; + second_word |= value; + first_word_ptr[1] = second_word; + } + } + inline void operator++() + { + const auto next_field_position = field_position + step_size; + if ((next_field_position >> 6) > (field_position >> 6)) { + first_word_ptr = data_area + (next_field_position >> 6); + } + field_position = next_field_position; + } + + inline void move(size_t index) + { + field_position = offset + index * step_size; + first_word_ptr = data_area + (field_position >> 6); + } + + inline uint64_t operator*() const + { + return get_value(); + } + +private: + friend bool operator<(const BfIterator&, const BfIterator&); + uint64_t* data_area = nullptr; + uint64_t* first_word_ptr = nullptr; + size_t field_position = 0; + uint8_t field_size = 0; + uint8_t step_size = 0; // may be different than field_size if used for arrays of pairs + size_t offset = 0; + uint64_t mask = 0; +}; + + +inline bool operator<(const BfIterator& a, const BfIterator& b) +{ + REALM_ASSERT(a.data_area == b.data_area); + return a.field_position < b.field_position; +} + +inline uint64_t read_bitfield(uint64_t* data_area, size_t field_position, size_t width) +{ + BfIterator it(data_area, field_position, width, width, 0); + return *it; +} + +inline void write_bitfield(uint64_t* data_area, size_t field_position, size_t width, uint64_t value) +{ + BfIterator it(data_area, field_position, width, width, 0); + it.set_value(value); +} + +inline int64_t sign_extend_field_by_mask(uint64_t sign_mask, uint64_t value) +{ + uint64_t sign_extension = 0ULL - (value & sign_mask); + return value | sign_extension; +} + +inline int64_t sign_extend_value(size_t width, uint64_t value) +{ + uint64_t sign_mask = 1ULL << (width - 1); + uint64_t sign_extension = 0ULL - (value & sign_mask); + return value | sign_extension; +} template inline std::pair get_two(const char* data, size_t ndx) noexcept @@ -200,6 +412,364 @@ inline std::pair get_two(const char* data, size_t width, size_ REALM_TEMPEX(return get_two, width, (data, ndx)); } +/* Subword parallel search + + The following provides facilities for subword parallel search for bitfields of any size. + To simplify, the first bitfield must be aligned within the word: it must occupy the lowest + bits of the word. + + In general the metods here return a vector with the most significant bit in each field + marking that a condition was met when comparing the corresponding pair of fields in two + vectors. Checking if any field meets a condition is as simple as comparing the return + vector against 0. Finding the first to meet a condition is also supported. + + Vectors are "split" into fields according to a MSB vector, wich indicates the most + significant bit of each field. The MSB must be passed in as an argument to most + bit field comparison functions. It can be generated by the field_sign_bit template. + + The simplest condition to test is any_field_NE(A,B), where A and B are words. + This condition should be true if any bitfield in A is not equal to the corresponding + field in B. + + This is almost as simple as a direct word compare, but needs to take into account that + we may want to have part of the words undefined. +*/ +constexpr uint8_t num_fields_table[65] = {0, 64, 32, 21, 16, 12, 10, 9, // 0-7 + 8, 7, 6, 5, 5, 4, 4, 4, // 8-15 + 4, 3, 3, 3, 3, 3, 2, 2, // 16-23 + 2, 2, 2, 2, 2, 2, 2, 2, // 24-31 + 2, 1, 1, 1, 1, 1, 1, 1, // 32-39 + 1, 1, 1, 1, 1, 1, 1, 1, // 40-47 + 1, 1, 1, 1, 1, 1, 1, 1, // 48-55 + 1, 1, 1, 1, 1, 1, 1, 1, // 56-63 + 1}; + +constexpr uint8_t num_bits_table[65] = {64, 64, 64, 63, 64, 60, 60, 63, // 0-7 + 64, 63, 60, 55, 60, 52, 56, 60, // 8-15 + 64, 51, 54, 57, 60, 63, 44, 46, // 16-23 + 48, 50, 52, 54, 56, 58, 60, 62, // 24-31 + 64, 33, 34, 35, 36, 37, 38, 39, // 32-39 + 40, 41, 42, 43, 44, 45, 46, 47, // 40-47 + 48, 49, 50, 51, 52, 53, 54, 55, // 48-55 + 56, 57, 58, 59, 60, 61, 62, 63, // 56-63 + 64}; + +inline uint8_t num_fields_for_width(uint8_t width) +{ + REALM_ASSERT_DEBUG(width); + const auto retval = num_fields_table[width]; +#ifdef REALM_DEBUG + REALM_ASSERT_DEBUG(width == 0 || retval == int(64 / width)); +#endif + return retval; +} + +inline uint8_t num_bits_for_width(uint8_t width) +{ + REALM_ASSERT_DEBUG(width); + return num_bits_table[width]; +} + +inline uint64_t cares_about(uint8_t width) +{ + return 0xFFFFFFFFFFFFFFFFULL >> (64 - num_bits_table[width]); +} + +// true if any field in A differs from corresponding field in B. If you also want +// to find which fields, use find_all_fields_NE instead. +bool inline any_field_NE(int width, uint64_t A, uint64_t B) +{ + return (A ^ B) & cares_about(width); +} + +// Populate all fields in a vector with a given value of a give width. +// Bits outside of the given field are ignored. +constexpr uint64_t populate(size_t width, uint64_t value) +{ + value &= 0xFFFFFFFFFFFFFFFFULL >> (64 - width); + if (width < 8) { + value |= value << width; + width <<= 1; + value |= value << width; + width <<= 1; + value |= value << width; + width <<= 1; + } + // width now in range 8..64 + if (width < 32) { + value |= value << width; + width <<= 1; + value |= value << width; + width <<= 1; + } + // width now in range 32..128 + if (width < 64) { + value |= value << width; + } + return value; +} + +// provides a set bit in pos 0 of each field, remaining bits zero +constexpr uint64_t field_bit0(int width) +{ + return populate(width, 1); +} + +// provides a set sign-bit in each field, remaining bits zero +constexpr uint64_t field_sign_bit(int width) +{ + return populate(width, 1ULL << (width - 1)); +} + +/* Unsigned LT. + + This can be determined by trial subtaction. However, some care must be exercised + since simply subtracting one vector from another will allow carries from one + bitfield to flow into the next one. To avoid this, we isolate bitfields by clamping + the MSBs to 1 in A and 0 in B before subtraction. After the subtraction the MSBs in + the result indicate borrows from the MSB. We then compute overflow (borrow OUT of MSB) + using boolean logic as described below. + + Unsigned LT is also used to find all zero fields or all non-zero fields, so it is + the backbone of all comparisons returning vectors. +*/ + +// compute the overflows in unsigned trial subtraction A-B. The overflows +// will be marked by 1 in the sign bit of each field in the result. Other +// bits in the result are zero. +// Overflow are detected for each field pair where A is less than B. +inline uint64_t unsigned_LT_vector(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // 1. compute borrow from most significant bit + // Isolate bitfields inside A and B before subtraction (prevent carries from spilling over) + // do this by clamping most significant bit in A to 1, and msb in B to 0 + auto A_isolated = A | MSBs; // 1 op + auto B_isolated = B & ~MSBs; // 2 ops + auto borrows_into_sign_bit = ~(A_isolated - B_isolated); // 2 ops (total latency 4) + + // 2. determine what subtraction against most significant bit would give: + // A B borrow-in: (A-B-borrow-in) + // 0 0 0 (0-0-0) = 0 + // 0 0 1 (0-0-1) = 1 + borrow-out + // 0 1 0 (0-1-0) = 1 + borrow-out + // 0 1 1 (0-1-1) = 0 + borrow-out + // 1 0 0 (1-0-0) = 1 + // 1 0 1 (1-0-1) = 0 + // 1 1 0 (1-1-0) = 0 + // 1 1 1 (1-1-1) = 1 + borrow-out + // borrow-out = (~A & B) | (~A & borrow-in) | (A & B & borrow-in) + // The overflows are simply the borrow-out, now encoded into the sign bits of each field. + auto overflows = (~A & B) | (~A & borrows_into_sign_bit) | (A & B & borrows_into_sign_bit); + // ^ 6 ops, total latency 6 (4+2) + return overflows & MSBs; // 1 op, total latency 7 + // total of 12 ops and a latency of 7. On a beefy CPU 3-4 of those can run in parallel + // and still reach a combined latency of 10 or less. +} + +inline uint64_t find_all_fields_unsigned_LT(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return unsigned_LT_vector(MSBs, A, B); +} + +inline uint64_t find_all_fields_NE(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // 0 != A^B, same as asking 0 - (A^B) overflows. + return unsigned_LT_vector(MSBs, 0, A ^ B); +} + +inline uint64_t find_all_fields_EQ(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // get the fields which are EQ and negate the result + auto all_fields_NE = find_all_fields_NE(MSBs, A, B); + auto all_fields_NE_negated = ~all_fields_NE; + // must filter the negated vector so only MSB are left. + return MSBs & all_fields_NE_negated; +} + +inline uint64_t find_all_fields_unsigned_LE(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // Now A <= B is the same as !(A > B) so... + // reverse A and B to turn (A>B) --> (B B is the same as B < A + return find_all_fields_signed_LT(MSBs, B, A); +} + +inline uint64_t find_all_fields_signed_GE(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // A >= B is the same as B <= A + return find_all_fields_signed_LE(MSBs, B, A); +} + +constexpr uint32_t inverse_width[65] = { + 65536 * 64 / 1, // never used + 65536 * 64 / 1, 65536 * 64 / 2, 65536 * 64 / 3, 65536 * 64 / 4, 65536 * 64 / 5, 65536 * 64 / 6, + 65536 * 64 / 7, 65536 * 64 / 8, 65536 * 64 / 9, 65536 * 64 / 10, 65536 * 64 / 11, 65536 * 64 / 12, + 65536 * 64 / 13, 65536 * 64 / 14, 65536 * 64 / 15, 65536 * 64 / 16, 65536 * 64 / 17, 65536 * 64 / 18, + 65536 * 64 / 19, 65536 * 64 / 20, 65536 * 64 / 21, 65536 * 64 / 22, 65536 * 64 / 23, 65536 * 64 / 24, + 65536 * 64 / 25, 65536 * 64 / 26, 65536 * 64 / 27, 65536 * 64 / 28, 65536 * 64 / 29, 65536 * 64 / 30, + 65536 * 64 / 31, 65536 * 64 / 32, 65536 * 64 / 33, 65536 * 64 / 34, 65536 * 64 / 35, 65536 * 64 / 36, + 65536 * 64 / 37, 65536 * 64 / 38, 65536 * 64 / 39, 65536 * 64 / 40, 65536 * 64 / 41, 65536 * 64 / 42, + 65536 * 64 / 43, 65536 * 64 / 44, 65536 * 64 / 45, 65536 * 64 / 46, 65536 * 64 / 47, 65536 * 64 / 48, + 65536 * 64 / 49, 65536 * 64 / 50, 65536 * 64 / 51, 65536 * 64 / 52, 65536 * 64 / 53, 65536 * 64 / 54, + 65536 * 64 / 55, 65536 * 64 / 56, 65536 * 64 / 57, 65536 * 64 / 58, 65536 * 64 / 59, 65536 * 64 / 60, + 65536 * 64 / 61, 65536 * 64 / 62, 65536 * 64 / 63, 65536 * 64 / 64, +}; + +inline size_t countr_zero(uint64_t vector) +{ + unsigned long where; +#if defined(_WIN64) + if (_BitScanForward64(&where, vector)) + return static_cast(where); + return 0; +#elif defined(_WIN32) + uint32_t low = vector & 0xFFFFFFFF; + if (low) { + bool scan_ok = _BitScanForward(&where, low); + REALM_ASSERT_DEBUG(scan_ok); + return where; + } + else { + low = vector >> 32; + bool scan_ok = _BitScanForward(&where, low); + REALM_ASSERT_DEBUG(scan_ok); + return 32 + where; + } +#else + where = __builtin_ctzll(vector); + return static_cast(where); +#endif +} + +inline size_t first_field_marked(size_t width, uint64_t vector) +{ + const auto lz = countr_zero(vector); + const auto field = (lz * inverse_width[width]) >> 22; + REALM_ASSERT_DEBUG(width != 0); + REALM_ASSERT_DEBUG(field == (lz / width)); + return field; +} + +template +size_t parallel_subword_find(VectorCompare vector_compare, const uint64_t* data, size_t offset, size_t width, + uint64_t MSBs, uint64_t search_vector, size_t start, size_t end) +{ + const auto field_count = num_fields_for_width(width); + const auto bit_count_pr_iteration = num_bits_for_width(width); + const size_t fast_scan_limit = 4 * bit_count_pr_iteration; + // use signed to make it easier to ascertain correctness of loop condition below + auto total_bit_count_left = (end - start) * width; + REALM_ASSERT_DEBUG(end >= start); + UnalignedWordIter it(data, offset + start * width); + uint64_t found_vector = 0; + while (total_bit_count_left >= fast_scan_limit) { + // unrolling 2x + const auto word0 = it.get_with_unsafe_prefetch(bit_count_pr_iteration); + it.bump(bit_count_pr_iteration); + const auto word1 = it.get_with_unsafe_prefetch(bit_count_pr_iteration); + auto found_vector0 = vector_compare(MSBs, word0, search_vector); + auto found_vector1 = vector_compare(MSBs, word1, search_vector); + it.bump(bit_count_pr_iteration); + if (found_vector0) { + const auto sub_word_index = first_field_marked(width, found_vector0); + return start + sub_word_index; + } + if (found_vector1) { + const auto sub_word_index = first_field_marked(width, found_vector1); + return start + field_count + sub_word_index; + } + total_bit_count_left -= 2 * bit_count_pr_iteration; + start += 2 * field_count; + } + while (total_bit_count_left >= bit_count_pr_iteration) { + const auto word = it.get(bit_count_pr_iteration); + found_vector = vector_compare(MSBs, word, search_vector); + if (found_vector) { + const auto sub_word_index = first_field_marked(width, found_vector); + return start + sub_word_index; + } + total_bit_count_left -= bit_count_pr_iteration; + start += field_count; + it.bump(bit_count_pr_iteration); + } + if (total_bit_count_left) { // final subword, may be partial + const auto word = it.get(total_bit_count_left); // <-- limit lookahead to avoid touching memory beyond array + found_vector = vector_compare(MSBs, word, search_vector); + auto last_word_mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - total_bit_count_left); + found_vector &= last_word_mask; + if (found_vector) { + const auto sub_word_index = first_field_marked(width, found_vector); + return start + sub_word_index; + } + } + return end; +} + + +namespace impl { + +template +inline int64_t default_fetcher(const char* data, size_t ndx) +{ + return get_direct(data, ndx); +} + +template +struct CompressedDataFetcher { + + int64_t operator()(const char*, size_t ndx) const + { + return ptr->get(ndx); + } + const T* ptr; +}; + +// Lower and Upper bound are mainly used in the B+tree implementation, +// but also for indexing, we can exploit these functions when the array +// is encoded, just providing a way for fetching the data. +// In this case the width is going to be ignored. // Lower/upper bound in sorted sequence // ------------------------------------ @@ -222,8 +792,9 @@ inline std::pair get_two(const char* data, size_t width, size_ // // We currently use binary search. See for example // http://www.tbray.org/ongoing/When/200x/2003/03/22/Binary. -template -inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept +template +inline size_t lower_bound(const char* data, size_t start, size_t end, int64_t value, + F fetcher = default_fetcher) noexcept { // The binary search used here is carefully optimized. Key trick is to use a single // loop controlling variable (size) instead of high/low pair, and to keep updates @@ -233,7 +804,11 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept // might be slightly faster if we used branches instead. The loop unrolling yields // a final 5-20% speedup depending on circumstances. - size_t low = 0; + // size_t low = 0; + REALM_ASSERT_DEBUG(end >= start); + size_t size = end - start; + // size_t low = 0; + size_t low = start; while (size >= 8) { // The following code (at X, Y and Z) is 3 times manually unrolled instances of (A) below. @@ -244,7 +819,7 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept size_t other_half = size - half; size_t probe = low + half; size_t other_low = low + other_half; - int64_t v = get_direct(data, probe); + int64_t v = fetcher(data, probe); size = half; low = (v < value) ? other_low : low; @@ -253,7 +828,7 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept other_half = size - half; probe = low + half; other_low = low + other_half; - v = get_direct(data, probe); + v = fetcher(data, probe); size = half; low = (v < value) ? other_low : low; @@ -262,7 +837,7 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept other_half = size - half; probe = low + half; other_low = low + other_half; - v = get_direct(data, probe); + v = fetcher(data, probe); size = half; low = (v < value) ? other_low : low; } @@ -295,7 +870,7 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept size_t other_half = size - half; size_t probe = low + half; size_t other_low = low + other_half; - int64_t v = get_direct(data, probe); + int64_t v = fetcher(data, probe); size = half; // for max performance, the line below should compile into a conditional // move instruction. Not all compilers do this. To maximize chance @@ -308,16 +883,20 @@ inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept } // See lower_bound() -template -inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept +template +inline size_t upper_bound(const char* data, size_t start, size_t end, int64_t value, + F fetcher = default_fetcher) noexcept { - size_t low = 0; + REALM_ASSERT_DEBUG(end >= start); + size_t size = end - start; + // size_t low = 0; + size_t low = start; while (size >= 8) { size_t half = size / 2; size_t other_half = size - half; size_t probe = low + half; size_t other_low = low + other_half; - int64_t v = get_direct(data, probe); + int64_t v = fetcher(data, probe); size = half; low = (value >= v) ? other_low : low; @@ -325,7 +904,7 @@ inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept other_half = size - half; probe = low + half; other_low = low + other_half; - v = get_direct(data, probe); + v = fetcher(data, probe); size = half; low = (value >= v) ? other_low : low; @@ -333,7 +912,7 @@ inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept other_half = size - half; probe = low + half; other_low = low + other_half; - v = get_direct(data, probe); + v = fetcher(data, probe); size = half; low = (value >= v) ? other_low : low; } @@ -343,13 +922,41 @@ inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept size_t other_half = size - half; size_t probe = low + half; size_t other_low = low + other_half; - int64_t v = get_direct(data, probe); + int64_t v = fetcher(data, probe); size = half; low = (value >= v) ? other_low : low; }; return low; } +} // namespace impl + +template +inline size_t lower_bound(const char* data, size_t size, int64_t value) noexcept +{ + return impl::lower_bound(data, 0, size, value, impl::default_fetcher); +} + +template +inline size_t lower_bound(const char* data, size_t size, int64_t value, + const impl::CompressedDataFetcher& fetcher) noexcept +{ + return impl::lower_bound(data, 0, size, value, fetcher); +} + +template +inline size_t upper_bound(const char* data, size_t size, int64_t value) noexcept +{ + return impl::upper_bound(data, 0, size, value, impl::default_fetcher); +} + +template +inline size_t upper_bound(const char* data, size_t size, int64_t value, + const impl::CompressedDataFetcher& fetcher) noexcept +{ + return impl::upper_bound(data, 0, size, value, fetcher); +} + } // namespace realm #endif /* ARRAY_TPL_HPP_ */ diff --git a/src/realm/group_writer.cpp b/src/realm/group_writer.cpp index dd112444f80..2990e010d3a 100644 --- a/src/realm/group_writer.cpp +++ b/src/realm/group_writer.cpp @@ -682,8 +682,8 @@ ref_type GroupWriter::write_group() if (top.size() > Group::s_hist_ref_ndx) { if (ref_type history_ref = top.get_as_ref(Group::s_hist_ref_ndx)) { Allocator& alloc = top.get_alloc(); - ref_type new_history_ref = Array::write(history_ref, alloc, *writer, only_modified, false); // Throws - top.set(Group::s_hist_ref_ndx, from_ref(new_history_ref)); // Throws + ref_type new_history_ref = Array::write(history_ref, alloc, *writer, only_modified, false); // Throws + top.set(Group::s_hist_ref_ndx, from_ref(new_history_ref)); // Throws } } if (top.size() > Group::s_evacuation_point_ndx) { diff --git a/test/test_array.cpp b/test/test_array.cpp index 637a394670c..b3102253b2d 100644 --- a/test/test_array.cpp +++ b/test/test_array.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include "test.hpp" @@ -1559,4 +1560,525 @@ NONCONCURRENT_TEST(Array_count) c.destroy(); } +TEST(Array_Bits) +{ + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(0), 0); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(1), 1); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(2), 2); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(3), 2); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(4), 3); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(5), 3); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(7), 3); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(8), 4); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(0), 1); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(1), 2); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-1), 1); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-2), 2); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-3), 3); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-4), 3); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(3), 3); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(4), 4); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(7), 4); +} + +TEST(Array_cares_about) +{ + std::vector expected{ + 0xffffffffffffffff, 0xffffffffffffffff, 0xffffffffffffffff, 0x7fffffffffffffff, 0xffffffffffffffff, + 0xfffffffffffffff, 0xfffffffffffffff, 0x7fffffffffffffff, 0xffffffffffffffff, 0x7fffffffffffffff, + 0xfffffffffffffff, 0x7fffffffffffff, 0xfffffffffffffff, 0xfffffffffffff, 0xffffffffffffff, + 0xfffffffffffffff, 0xffffffffffffffff, 0x7ffffffffffff, 0x3fffffffffffff, 0x1ffffffffffffff, + 0xfffffffffffffff, 0x7fffffffffffffff, 0xfffffffffff, 0x3fffffffffff, 0xffffffffffff, + 0x3ffffffffffff, 0xfffffffffffff, 0x3fffffffffffff, 0xffffffffffffff, 0x3ffffffffffffff, + 0xfffffffffffffff, 0x3fffffffffffffff, 0xffffffffffffffff, 0x1ffffffff, 0x3ffffffff, + 0x7ffffffff, 0xfffffffff, 0x1fffffffff, 0x3fffffffff, 0x7fffffffff, + 0xffffffffff, 0x1ffffffffff, 0x3ffffffffff, 0x7ffffffffff, 0xfffffffffff, + 0x1fffffffffff, 0x3fffffffffff, 0x7fffffffffff, 0xffffffffffff, 0x1ffffffffffff, + 0x3ffffffffffff, 0x7ffffffffffff, 0xfffffffffffff, 0x1fffffffffffff, 0x3fffffffffffff, + 0x7fffffffffffff, 0xffffffffffffff, 0x1ffffffffffffff, 0x3ffffffffffffff, 0x7ffffffffffffff, + 0xfffffffffffffff, 0x1fffffffffffffff, 0x3fffffffffffffff, 0x7fffffffffffffff, 0xffffffffffffffff}; + std::vector res; + for (size_t i = 0; i <= 64; i++) { + res.push_back(cares_about(i)); + } + CHECK_EQUAL(res, expected); +} + +TEST(AlignDirectBitFields) +{ + uint64_t a[2]; + a[0] = a[1] = 0; + { + BfIterator it(a, 0, 7, 7, 8); + REALM_ASSERT(*it == 0); + auto it2(it); + ++it2; + it2.set_value(127 + 128); + REALM_ASSERT(*it == 0); + ++it; + REALM_ASSERT(*it == 127); + ++it; + REALM_ASSERT(*it == 0); + } + // reverse polarity + a[0] = a[1] = -1ULL; + { + BfIterator it(a, 0, 7, 7, 8); + REALM_ASSERT(*it == 127); + auto it2(it); + ++it2; + it2.set_value(42 + 128); + REALM_ASSERT(*it == 127); + ++it; + REALM_ASSERT(*it == 42); + ++it; + REALM_ASSERT(*it == 127); + } +} + +TEST(TestSignValuesStoredIterator) +{ + { + // positive values are easy. + uint64_t a[2]; + BfIterator it(a, 0, 8, 8, 0); + for (size_t i = 0; i < 16; ++i) { + it.set_value(i); + ++it; + } + it.move(0); + for (size_t i = 0; i < 16; ++i) { + auto v = *it; + CHECK_EQUAL(v, i); + ++it; + } + } + { + // negative values require a bit more work + uint64_t a[2]; + BfIterator it(a, 0, 8, 8, 0); + for (size_t i = 0; i < 16; ++i) { + it.set_value(-i); + ++it; + } + it.move(0); + for (int64_t i = 0; i < 16; ++i) { + const auto sv = sign_extend_value(8, *it); + CHECK_EQUAL(sv, -i); + ++it; + } + it.move(0); + const auto sign_mask = 1ULL << (7); + for (int64_t i = 0; i < 16; ++i) { + const auto sv = sign_extend_field_by_mask(sign_mask, *it); + CHECK_EQUAL(sv, -i); + ++it; + } + } +} + +TEST(VerifyIterationAcrossWords) +{ + uint64_t a[4]{0, 0, 0, 0}; // 4 64 bit words, let's store N elements of 5bits each + BfIterator it(a, 0, 5, 5, 0); + // 51 is the max amount of values we can fit in 4 words. Writting beyond this point is likely going + // to crash. Writing beyond the 4 words is not possible in practice because the Array has boundery checks + // and enough memory is reserved during compression. + srand((unsigned)time(0)); // no need to use complex stuff + std::vector values; + for (size_t i = 0; i < 51; i++) { + int64_t randomNumber = rand() % 16; // max value that can fit in 5 bits (4 bit for the value + 1 sign) + values.push_back(randomNumber); + it.set_value(randomNumber); + ++it; + } + + { + // normal bf iterator + it.move(0); // reset to first value. + // go through the memory, 5 bits by 5 bits. + // every 12 values, we will read the value across + // 2 words. + for (size_t i = 0; i < 51; ++i) { + const auto v = sign_extend_value(5, *it); + CHECK_EQUAL(v, values[i]); + ++it; + } + } + + { + // unaligned iterator + UnalignedWordIter u_it(a, 0); + for (size_t i = 0; i < 51; ++i) { + const auto v = sign_extend_value(5, u_it.get(5) & 0x1F); + CHECK_EQUAL(v, values[i]); + u_it.bump(5); + } + } +} + +TEST(LowerBoundCorrectness) +{ + constexpr auto size = 16; + int64_t a[size]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + std::vector expected_default_lb; + for (const auto v : a) { + const auto pos = lower_bound<64>((const char*)(&a), size, v); + expected_default_lb.push_back(pos); + } + + // now simulate the compression of a in less bits + uint64_t buff[2] = {0, 0}; + BfIterator it(buff, 0, 5, 5, 0); // 5 bits because 4 bits for the values + 1 bit for the sign + for (size_t i = 0; i < 16; ++i) { + it.set_value(i); + CHECK_EQUAL(*it, a[i]); + ++it; + } + struct MyClass { + uint64_t* _data; + int64_t get(size_t ndx) const + { + BfIterator it(_data, 0, 5, 5, ndx); + return sign_extend_value(5, *it); + } + }; + // a bit of set up here. + MyClass my_class; + my_class._data = buff; + using Fetcher = impl::CompressedDataFetcher; + Fetcher my_fetcher; + my_fetcher.ptr = &my_class; + + // verify that the fetcher returns the same values + for (size_t i = 0; i < size; ++i) { + CHECK_EQUAL(my_fetcher.ptr->get(i), a[i]); + } + + std::vector diffent_width_lb; + for (const auto v : a) { + const auto pos = lower_bound((const char*)buff, size, v, my_fetcher); + diffent_width_lb.push_back(pos); + } + + CHECK_EQUAL(expected_default_lb, diffent_width_lb); +} + +TEST(UpperBoundCorrectness) +{ + constexpr auto size = 16; + int64_t a[size]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + std::vector expected_default_ub; + for (const auto v : a) { + const auto pos = upper_bound<64>((const char*)(&a), size, v); + expected_default_ub.push_back(pos); + } + + // now simulate the compression of a in less bits + uint64_t buff[2] = {0, 0}; + BfIterator it(buff, 0, 5, 5, 0); // 5 bits because 4 bits for the values + 1 bit for the sign + for (size_t i = 0; i < size; ++i) { + it.set_value(i); + CHECK_EQUAL(*it, a[i]); + ++it; + } + struct MyClass { + uint64_t* _data; + int64_t get(size_t ndx) const + { + BfIterator it(_data, 0, 5, 5, ndx); + return sign_extend_value(5, *it); + } + }; + // a bit of set up here. + MyClass my_class; + my_class._data = buff; + using Fetcher = impl::CompressedDataFetcher; + Fetcher my_fetcher; + my_fetcher.ptr = &my_class; + + // verify that the fetcher returns the same values + for (size_t i = 0; i < size; ++i) { + CHECK_EQUAL(my_fetcher.ptr->get(i), a[i]); + } + + std::vector diffent_width_ub; + for (const auto v : a) { + const auto pos = upper_bound((const char*)buff, size, v, my_fetcher); + diffent_width_ub.push_back(pos); + } + + CHECK_EQUAL(expected_default_ub, diffent_width_ub); +} + +TEST(ParallelSearchEqualMatch) +{ + std::mt19937_64 gen64; + constexpr size_t buflen = 4; + uint64_t buff[buflen]; + std::vector values; + for (size_t width = 1; width <= 64; width++) { + const size_t size = (buflen * 64) / width; + const uint64_t bit_mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - width); // (1ULL << width) - 1; + + values.resize(size); + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + uint64_t u_val = gen64() & bit_mask; + int64_t val = sign_extend_value(width, u_val); + values[i] = val; + it.set_value(val); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, values[i]); + ++it; + } + + std::sort(values.begin(), values.end()); + auto last = std::unique(values.begin(), values.end()); + for (auto val = values.begin(); val != last; val++) { + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, *val); + + // perform the check with a normal iteration + size_t start = 0; + const auto end = size; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv == *val) + linear_scan_result.push_back(start); + ++start; + } + + // Now use the optimized version + static auto vector_compare_eq = [](auto msb, auto a, auto b) { + return find_all_fields_EQ(msb, a, b); + }; + + start = 0; + std::vector parallel_result; + while (start < end) { + start = + parallel_subword_find(vector_compare_eq, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + CHECK(!parallel_result.empty()); + CHECK(!linear_scan_result.empty()); + CHECK_EQUAL(linear_scan_result, parallel_result); + } + } +} + +TEST(ParallelSearchEqualNoMatch) +{ + uint64_t buff[2] = {0, 0}; + constexpr size_t width = 2; + constexpr size_t size = 64; + constexpr int64_t key = 2; + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + it.set_value(1); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, 1); + ++it; + } + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, key); + + static auto vector_compare_eq = [](auto msb, auto a, auto b) { + return find_all_fields_EQ(msb, a, b); + }; + + size_t start = 0; + const auto end = size; + std::vector parallel_result; + while (start < end) { + start = parallel_subword_find(vector_compare_eq, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + // perform the same check but with a normal iteration + start = 0; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv == key) + linear_scan_result.push_back(start); + ++start; + } + + CHECK(parallel_result.empty()); + CHECK(linear_scan_result.empty()); +} + +TEST(ParallelSearchNotEqual) +{ + uint64_t buff[2] = {0, 0}; + constexpr size_t width = 2; + constexpr size_t size = 64; + constexpr int64_t key = 2; + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + it.set_value(1); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, 1); + ++it; + } + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, key); + + static auto vector_compare_neq = [](auto msb, auto a, auto b) { + return find_all_fields_NE(msb, a, b); + }; + + size_t start = 0; + const auto end = size; + std::vector parallel_result; + while (start < end) { + start = parallel_subword_find(vector_compare_neq, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + // perform the same check but with a normal iteration + start = 0; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv != key) + linear_scan_result.push_back(start); + ++start; + } + + CHECK(!parallel_result.empty()); + CHECK(!linear_scan_result.empty()); + CHECK_EQUAL(parallel_result, linear_scan_result); +} + +TEST(ParallelSearchLessThan) +{ + uint64_t buff[2] = {0, 0}; + constexpr size_t width = 4; + constexpr size_t size = 32; + constexpr int64_t key = 3; + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + it.set_value(2); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, 2); + ++it; + } + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, key); + + static auto vector_compare_lt = [](auto msb, auto a, auto b) { + return find_all_fields_signed_LT(msb, a, b); + }; + + size_t start = 0; + const auto end = size; + std::vector parallel_result; + while (start < end) { + start = parallel_subword_find(vector_compare_lt, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + // perform the same check but with a normal iteration + start = 0; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv < key) + linear_scan_result.push_back(start); + ++start; + } + CHECK(!parallel_result.empty()); + CHECK(!linear_scan_result.empty()); + CHECK_EQUAL(parallel_result, linear_scan_result); +} + +TEST(ParallelSearchGreaterThan) +{ + uint64_t buff[2] = {0, 0}; + constexpr size_t width = 4; + constexpr size_t size = 32; + constexpr int64_t key = 2; + BfIterator it(buff, 0, width, width, 0); + for (size_t i = 0; i < size; ++i) { + it.set_value(3); + ++it; + } + it.move(0); + for (size_t i = 0; i < size; ++i) { + auto v = sign_extend_value(width, *it); + CHECK_EQUAL(v, 3); + ++it; + } + const auto mask = 1ULL << (width - 1); + const auto msb = populate(width, mask); + const auto search_vector = populate(width, key); + + static auto vector_compare_gt = [](auto msb, auto a, auto b) { + return find_all_fields_signed_GT(msb, a, b); + }; + + size_t start = 0; + const auto end = size; + std::vector parallel_result; + while (start < end) { + start = parallel_subword_find(vector_compare_gt, buff, size_t{0}, width, msb, search_vector, start, end); + if (start != end) + parallel_result.push_back(start); + start += 1; + } + + // perform the same check but with a normal iteration + start = 0; + std::vector linear_scan_result; + while (start < end) { + it.move(start); + const auto sv = sign_extend_value(width, *it); + if (sv > key) + linear_scan_result.push_back(start); + ++start; + } + CHECK(!parallel_result.empty()); + CHECK(!linear_scan_result.empty()); + CHECK_EQUAL(parallel_result, linear_scan_result); +} + + #endif // TEST_ARRAY From a02798ab25b7075125fb5e0c96c65d830b9b095c Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Tue, 21 May 2024 15:48:01 +0100 Subject: [PATCH 08/18] RCORE-2058 Adding clickbench importer and query tool for evaluating compression (#7664) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update next-major * specified part of new layout (new width encoding) * new header format for compressed arrays * code review * code review * start of classifying arrays for compression * classification down to column types * first attempt to cut through the BPlusTree madness * [wip] start on 'type driven' write process * all tests passing (but no compression enabled) * enabled compression for signed integer leafs only * removed some dubious constructions in cluster tree * delete tmp array while classifying arrays * enabled compression of links and backlinks (excl collections) * also compress bplustree of integers/links (experimental) * pref for compressing dicts (not working) * wip * wip * finally: compressing collections (incl dicts) * compressing timestamps now * enabled compression on ObjectID, TypedLink and UUID * also compressing Mixed properties (not list/dicts of Mixed) * Array compression with collections in Mixed (#7412) --------- Co-authored-by: Finn Schiermer Andersen * merge next-major + collection in mixed * enable dynamic choice of compression method * moved typed_write/typed_print for bptree into class * Merge pull request #7432 from realm/fsa/clean_typed_write moved typed_write/typed_print for bptree into class * cleanup unrelated code changes * fix compilation * cleanup * code review * code review * scaffolding for accessing unaligned memory inside compressed arrays * introducing parallel scan for scanning in one go multiple compressed arrays * Silence warning * tests + point fix for upper bound * fix ubsan * added clickbench data ingestor * Added 2 queries to clickbench * separated out a clickquery benchmark * separated out a clickquery benchmark * update to clickbench and query * fix warnings * attempt to fix windows builders * silence warnings * fix importer --------- Co-authored-by: Finn Schiermer Andersen Co-authored-by: Jørgen Edelbo Co-authored-by: Finn Schiermer Andersen --- src/realm/array_direct.hpp | 2 +- src/realm/exec/CMakeLists.txt | 22 ++ src/realm/exec/clickbench.cpp | 375 ++++++++++++++++++++++++++++++++++ src/realm/exec/clickquery.cpp | 86 ++++++++ src/realm/node_header.hpp | 2 +- test/test_array.cpp | 4 +- 6 files changed, 487 insertions(+), 4 deletions(-) create mode 100644 src/realm/exec/clickbench.cpp create mode 100644 src/realm/exec/clickquery.cpp diff --git a/src/realm/array_direct.hpp b/src/realm/array_direct.hpp index 97c5a2b0d33..5380876700f 100644 --- a/src/realm/array_direct.hpp +++ b/src/realm/array_direct.hpp @@ -693,7 +693,7 @@ inline size_t first_field_marked(size_t width, uint64_t vector) } template -size_t parallel_subword_find(VectorCompare vector_compare, const uint64_t* data, size_t offset, size_t width, +size_t parallel_subword_find(VectorCompare vector_compare, const uint64_t* data, size_t offset, uint8_t width, uint64_t MSBs, uint64_t search_vector, size_t start, size_t end) { const auto field_count = num_fields_for_width(width); diff --git a/src/realm/exec/CMakeLists.txt b/src/realm/exec/CMakeLists.txt index 16bb966d868..24592a4f61b 100644 --- a/src/realm/exec/CMakeLists.txt +++ b/src/realm/exec/CMakeLists.txt @@ -24,6 +24,28 @@ if (EMSCRIPTEN) set_target_properties(RealmTrawler PROPERTIES EXCLUDE_FROM_ALL TRUE) endif() +add_executable(ClickBench EXCLUDE_FROM_ALL clickbench.cpp ) +set_target_properties(ClickBench PROPERTIES + OUTPUT_NAME "clickbench" + DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} +) +if (EMSCRIPTEN) + set_target_properties(ClickBench PROPERTIES EXCLUDE_FROM_ALL TRUE) +endif() + +target_link_libraries(ClickBench Storage) + +add_executable(ClickQuery EXCLUDE_FROM_ALL clickquery.cpp ) +set_target_properties(ClickQuery PROPERTIES + OUTPUT_NAME "clickquery" + DEBUG_POSTFIX ${CMAKE_DEBUG_POSTFIX} +) +if (EMSCRIPTEN) + set_target_properties(ClickQuery PROPERTIES EXCLUDE_FROM_ALL TRUE) +endif() + +target_link_libraries(ClickQuery Storage) + add_executable(RealmEnumerate realm_enumerate.cpp) set_target_properties(RealmEnumerate PROPERTIES OUTPUT_NAME "realm-enumerate" diff --git a/src/realm/exec/clickbench.cpp b/src/realm/exec/clickbench.cpp new file mode 100644 index 00000000000..54c4b13e318 --- /dev/null +++ b/src/realm/exec/clickbench.cpp @@ -0,0 +1,375 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace realm; + +template +class Mailbox { +public: + void send(T* val) + { + std::unique_lock lck(m_mutex); + m_list.push_back(val); + m_cv.notify_one(); + } + T* receive() + { + std::unique_lock lck(m_mutex); + while (m_list.empty()) + m_cv.wait(lck); + T* ret = m_list.front(); + m_list.pop_front(); + return ret; + } + +private: + std::deque m_list; + std::mutex m_mutex; + std::condition_variable m_cv; +}; + +// remove this when enumerated strings are supported: +#define type_EnumString type_String + +static void create_table(TransactionRef tr) +{ + auto t = tr->add_table("Hits"); + t->add_column(type_Int, "WatchID"); + t->add_column(type_Int, "JavaEnable"); + t->add_column(type_EnumString, "Title", true); + t->add_column(type_Int, "GoodEvent"); + t->add_column(type_Timestamp, "EventTime"); + t->add_column(type_Timestamp, "EventDate"); + t->add_column(type_Int, "CounterID"); + t->add_column(type_Int, "ClientIP"); + t->add_column(type_Int, "RegionID"); + t->add_column(type_Int, "UserID"); + t->add_column(type_Int, "CounterClass"); + t->add_column(type_Int, "OS"); + t->add_column(type_Int, "UserAgent"); + t->add_column(type_EnumString, "URL", true); + t->add_column(type_EnumString, "Referer", true); + t->add_column(type_Int, "IsRefresh"); + t->add_column(type_Int, "RefererCategoryID"); + t->add_column(type_Int, "RefererRegionID"); + t->add_column(type_Int, "URLCategoryID"); + t->add_column(type_Int, "URLRegionID"); + t->add_column(type_Int, "ResolutionWidth"); + t->add_column(type_Int, "ResolutionHeight"); + t->add_column(type_Int, "ResolutionDepth"); + t->add_column(type_Int, "FlashMajor"); + t->add_column(type_Int, "FlashMinor"); + t->add_column(type_EnumString, "FlashMinor2", true); + t->add_column(type_Int, "NetMajor"); + t->add_column(type_Int, "NetMinor"); + t->add_column(type_Int, "UserAgentMajor"); + t->add_column(type_EnumString, "UserAgentMinor", true); + t->add_column(type_Int, "CookieEnable"); + t->add_column(type_Int, "JavascriptEnable"); + t->add_column(type_Int, "IsMobile"); + t->add_column(type_Int, "MobilePhone"); + t->add_column(type_EnumString, "MobilePhoneModel", true); + t->add_column(type_EnumString, "Params", true); + t->add_column(type_Int, "IPNetworkID"); + t->add_column(type_Int, "TraficSourceID"); + t->add_column(type_Int, "SearchEngineID"); + t->add_column(type_EnumString, "SearchPhrase", true); + t->add_column(type_Int, "AdvEngineID"); + t->add_column(type_Int, "IsArtifical"); + t->add_column(type_Int, "WindowClientWidth"); + t->add_column(type_Int, "WindowClientHeight"); + t->add_column(type_Int, "ClientTimeZone"); + t->add_column(type_Timestamp, "ClientEventTime"); + t->add_column(type_Int, "SilverlightVersion1"); + t->add_column(type_Int, "SilverlightVersion2"); + t->add_column(type_Int, "SilverlightVersion3"); + t->add_column(type_Int, "SilverlightVersion4"); + t->add_column(type_EnumString, "PageCharset", true); + t->add_column(type_Int, "CodeVersion"); + t->add_column(type_Int, "IsLink"); + t->add_column(type_Int, "IsDownload"); + t->add_column(type_Int, "IsNotBounce"); + t->add_column(type_Int, "FUniqID"); + t->add_column(type_EnumString, "OriginalURL", true); + t->add_column(type_Int, "HID"); + t->add_column(type_Int, "IsOldCounter"); + t->add_column(type_Int, "IsEvent"); + t->add_column(type_Int, "IsParameter"); + t->add_column(type_Int, "DontCountHits"); + t->add_column(type_Int, "WithHash"); + t->add_column(type_EnumString, "HitColor", true); + t->add_column(type_Timestamp, "LocalEventTime"); + t->add_column(type_Int, "Age"); + t->add_column(type_Int, "Sex"); + t->add_column(type_Int, "Income"); + t->add_column(type_Int, "Interests"); + t->add_column(type_Int, "Robotness"); + t->add_column(type_Int, "RemoteIP"); + t->add_column(type_Int, "WindowName"); + t->add_column(type_Int, "OpenerName"); + t->add_column(type_Int, "HistoryLength"); + t->add_column(type_EnumString, "BrowserLanguage", true); + t->add_column(type_EnumString, "BrowserCountry", true); + t->add_column(type_EnumString, "SocialNetwork", true); + t->add_column(type_EnumString, "SocialAction", true); + t->add_column(type_Int, "HTTPError"); + t->add_column(type_Int, "SendTiming"); + t->add_column(type_Int, "DNSTiming"); + t->add_column(type_Int, "ConnectTiming"); + t->add_column(type_Int, "ResponseStartTiming"); + t->add_column(type_Int, "ResponseEndTiming"); + t->add_column(type_Int, "FetchTiming"); + t->add_column(type_Int, "SocialSourceNetworkID"); + t->add_column(type_EnumString, "SocialSourcePage", true); + t->add_column(type_Int, "ParamPrice"); + t->add_column(type_EnumString, "ParamOrderID", true); + t->add_column(type_EnumString, "ParamCurrency", true); + t->add_column(type_Int, "ParamCurrencyID"); + t->add_column(type_EnumString, "OpenstatServiceName", true); + t->add_column(type_EnumString, "OpenstatCampaignID", true); + t->add_column(type_EnumString, "OpenstatAdID", true); + t->add_column(type_EnumString, "OpenstatSourceID", true); + t->add_column(type_EnumString, "UTMSource", true); + t->add_column(type_EnumString, "UTMMedium", true); + t->add_column(type_EnumString, "UTMCampaign", true); + t->add_column(type_EnumString, "UTMContent", true); + t->add_column(type_EnumString, "UTMTerm", true); + t->add_column(type_EnumString, "FromTag", true); + t->add_column(type_Int, "HasGCLID"); + t->add_column(type_Int, "RefererHash"); + t->add_column(type_Int, "URLHash"); + t->add_column(type_Int, "CLID"); + tr->commit(); +} + +static int strtoi(const char* p, char** endp) +{ + int ret = 0; + while (*p >= '0' && *p <= '9') { + ret *= 10; + ret += *p - '0'; + ++p; + } + *endp = const_cast(p); + return ret; +} + +inline int64_t epoch_days_fast(int y, int m, int d) +{ + const uint32_t year_base = 4800; /* Before min year, multiple of 400. */ + const int32_t m_adj = m - 3; /* March-based month. */ + const uint32_t carry = (m_adj > m) ? 1 : 0; + const uint32_t adjust = carry ? 12 : 0; + const uint32_t y_adj = y + year_base - carry; + const uint32_t month_days = ((m_adj + adjust) * 62719 + 769) / 2048; + const uint32_t leap_days = y_adj / 4 - y_adj / 100 + y_adj / 400; + return y_adj * 365 + leap_days + month_days + (d - 1) - 2472632; +} + +static Timestamp get_timestamp(const char* str) +{ + char* p; + int year = int(strtoi(str, &p)); + if (*p == '-') { + p++; + int mon = int(strtoi(p, &p)); + if (*p == '-') { + p++; + int day = int(strtoi(p, &p)); + time_t hms = 0; + if (*p == ' ' || *p == 'T') { + p++; + int h = int(strtoi(p, &p)); + int m = 0; + int s = 0; + if (*p == ':') { + p++; + m = int(strtoi(p, &p)); + if (*p == ':') { + p++; + s = int(strtoi(p, &p)); + } + } + hms = (h * 3600) + (m * 60) + s; + } + if (*p == '\0') { + return Timestamp(epoch_days_fast(year, mon, day) * 86400 + hms, 0); + } + } + } + return Timestamp(); +} + +struct BufferedValues { + std::vector buffer{256}; + std::vector values{256}; +}; + +Mailbox mbx; +Mailbox resp; + +static void parse_file(const char* filename) +{ + std::ifstream inp(filename); + + auto buf = resp.receive(); + auto str = buf->buffer.begin(); + auto it = buf->values.begin(); + auto end = buf->values.end(); + while (std::getline(inp, *str)) { + char* tok = str->data(); + for (FieldValue& val : *it) { + char* end = strchr(tok, '\t'); + if (end) { + *end = '\0'; + } + switch (val.col_key.get_type()) { + case col_type_Int: + val.value = Mixed(int64_t(strtoll(tok, nullptr, 10))); + break; + case col_type_String: + val.value = strlen(tok) ? Mixed(tok) : Mixed(); + break; + /* + case col_type_EnumString: + val.value = strlen(tok) ? Mixed(tok) : Mixed(); + break; + */ + case col_type_Timestamp: + val.value = Mixed(get_timestamp(tok)); + break; + default: + break; + } + tok = end + 1; + } + ++it; + ++str; + if (it == end) { + mbx.send(buf); + buf = resp.receive(); + str = buf->buffer.begin(); + it = buf->values.begin(); + end = buf->values.end(); + } + } + buf->values.erase(it, end); + if (buf->values.size()) { + mbx.send(buf); + } + mbx.send(nullptr); +} + +static void import(const char* filename) +{ + DBOptions options; + auto db = DB::create(make_in_realm_history(), "hits.realm"); + create_table(db->start_write()); + auto tr = db->start_write(); + auto t = tr->get_table("Hits"); + auto col_keys = t->get_column_keys(); + + std::cout << std::endl << "Reading data into realm" << std::endl; + auto time_start = std::chrono::high_resolution_clock::now(); + BufferedValues buf1; + BufferedValues buf2; + for (auto& val : buf1.values) { + for (auto col : col_keys) { + val.insert(col, Mixed()); + } + } + for (auto& val : buf2.values) { + for (auto col : col_keys) { + val.insert(col, Mixed()); + } + } + resp.send(&buf1); + resp.send(&buf2); + std::thread parse_file_thread(parse_file, filename); + + int buf_cnt = 0; + const int bufs_per_commit = 100; + while (auto buf = mbx.receive()) { + for (auto& val : buf->values) { + Obj o = t->create_object(ObjKey(), val); + // verify + for (auto& e : val) { + if (e.col_key.get_type() == col_type_Int) { + auto got_int = o.get(e.col_key); + auto the_int = e.value.get_int(); + REALM_ASSERT(got_int == the_int); + } + if (e.col_key.get_type() == col_type_String) { + auto got_string = o.get(e.col_key); + auto the_string = e.value.get_string(); + REALM_ASSERT(got_string == the_string); + } + } + } + resp.send(buf); + if (buf_cnt++ > bufs_per_commit) { + tr->commit_and_continue_as_read(); + tr->promote_to_write(); + std::cout << '.'; + std::cout.flush(); + buf_cnt = 0; + } + } + tr->commit_and_continue_as_read(); + + parse_file_thread.join(); + auto time_end = std::chrono::high_resolution_clock::now(); + std::cout << "Ingestion complete in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; +} + +static void dump_prop(const char* filename, const char* prop_name) +{ + auto db = DB::create(make_in_realm_history(), filename); + auto tr = db->start_read(); + auto t = tr->get_table("Hits"); + auto col = t->get_column_key(prop_name); + for (auto& o : *t) { + switch (col.get_type()) { + case col_type_Int: + std::cout << o.get(col) << std::endl; + break; + case col_type_String: + std::cout << o.get(col) << std::endl; + break; + /* + case col_type_EnumString: + REALM_ASSERT(false); + break; + */ + case col_type_Timestamp: + std::cout << o.get(col) << std::endl; + break; + default: + break; + } + } +} + +int main(int argc, const char* argv[]) +{ + if (argc == 1) { + import("mill.tsv"); + } + if (argc == 2) { + import(argv[1]); + } + if (argc == 3) { + dump_prop(argv[1], argv[2]); + } +} diff --git a/src/realm/exec/clickquery.cpp b/src/realm/exec/clickquery.cpp new file mode 100644 index 00000000000..e930eb27013 --- /dev/null +++ b/src/realm/exec/clickquery.cpp @@ -0,0 +1,86 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace realm; + +static void import(const char* filename) +{ + DBOptions options; + auto db = DB::create(make_in_realm_history(), filename); + auto tr = db->start_write(); + auto t = tr->get_table("Hits"); + auto time_start = std::chrono::high_resolution_clock::now(); + auto time_end = time_start; + { + std::cout << std::endl << "count of AdvEngineID <> 0" << std::endl; + time_start = std::chrono::high_resolution_clock::now(); + size_t q = 0; + for (int i = 0; i < 10; ++i) { + auto k = t->get_column_key("AdvEngineID"); + q = t->where().not_equal(k, 0).count(); + } + time_end = std::chrono::high_resolution_clock::now(); + std::cout << "result = " << q << " in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; + } + { + std::cout << std::endl << "Query result for AdvEngineID <> 0" << std::endl; + time_start = std::chrono::high_resolution_clock::now(); + TableView q; + for (int i = 0; i < 10; ++i) { + auto k = t->get_column_key("AdvEngineID"); + q = t->where().not_equal(k, 0).find_all(); + } + time_end = std::chrono::high_resolution_clock::now(); + std::cout << "result with size " << q.size() << " in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; + time_start = std::chrono::high_resolution_clock::now(); + size_t count = 0; + for (int i = 0; i < 10; ++i) { + auto limit = q.size(); + auto k = t->get_column_key("AdvEngineID"); + for (size_t i = 0; i < limit; ++i) { + count += q[i].get(k); + } + } + time_end = std::chrono::high_resolution_clock::now(); + std::cout << "Iterating over result to get count " << count << " in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; + } + { + std::cout << std::endl << "Max of EventDate" << std::endl; + time_start = std::chrono::high_resolution_clock::now(); + Mixed q; + for (int i = 0; i < 10; ++i) { + auto k = t->get_column_key("EventDate"); + q = *(t->max(k)); + } + // auto q = t->where().not_equal(k, 0).count(); + time_end = std::chrono::high_resolution_clock::now(); + std::cout << "result = " << q << " in " + << std::chrono::duration_cast(time_end - time_start).count() << " msecs" + << std::endl; + } +} + +int main(int argc, const char* argv[]) +{ + if (argc == 1) { + import("./hits.realm"); + } + if (argc == 2) { + import(argv[1]); + } +} diff --git a/src/realm/node_header.hpp b/src/realm/node_header.hpp index d8e6652106e..2ffe073b721 100644 --- a/src/realm/node_header.hpp +++ b/src/realm/node_header.hpp @@ -208,7 +208,7 @@ class NodeHeader { static size_t unsigned_to_num_bits(uint64_t value) { if constexpr (sizeof(size_t) == sizeof(uint64_t)) - return static_cast(1) + log2(value); + return 1 + log2(static_cast(value)); uint32_t high = value >> 32; if (high) return 33 + log2(high); diff --git a/test/test_array.cpp b/test/test_array.cpp index b3102253b2d..8a86ac15718 100644 --- a/test/test_array.cpp +++ b/test/test_array.cpp @@ -1598,7 +1598,7 @@ TEST(Array_cares_about) 0x7fffffffffffff, 0xffffffffffffff, 0x1ffffffffffffff, 0x3ffffffffffffff, 0x7ffffffffffffff, 0xfffffffffffffff, 0x1fffffffffffffff, 0x3fffffffffffffff, 0x7fffffffffffffff, 0xffffffffffffffff}; std::vector res; - for (size_t i = 0; i <= 64; i++) { + for (uint8_t i = 0; i <= 64; i++) { res.push_back(cares_about(i)); } CHECK_EQUAL(res, expected); @@ -1818,7 +1818,7 @@ TEST(ParallelSearchEqualMatch) constexpr size_t buflen = 4; uint64_t buff[buflen]; std::vector values; - for (size_t width = 1; width <= 64; width++) { + for (uint8_t width = 1; width <= 64; width++) { const size_t size = (buflen * 64) / width; const uint64_t bit_mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - width); // (1ULL << width) - 1; From 222832d9c16b2c9497f170135296b373a41d1c71 Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Tue, 21 May 2024 15:55:13 +0100 Subject: [PATCH 09/18] RCORE-2057 New builder for assessing that compression is working correctly (#7667) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update next-major * specified part of new layout (new width encoding) * new header format for compressed arrays * code review * code review * start of classifying arrays for compression * classification down to column types * first attempt to cut through the BPlusTree madness * [wip] start on 'type driven' write process * all tests passing (but no compression enabled) * enabled compression for signed integer leafs only * removed some dubious constructions in cluster tree * delete tmp array while classifying arrays * enabled compression of links and backlinks (excl collections) * also compress bplustree of integers/links (experimental) * pref for compressing dicts (not working) * wip * wip * finally: compressing collections (incl dicts) * compressing timestamps now * enabled compression on ObjectID, TypedLink and UUID * also compressing Mixed properties (not list/dicts of Mixed) * Array compression with collections in Mixed (#7412) --------- Co-authored-by: Finn Schiermer Andersen * merge next-major + collection in mixed * enable dynamic choice of compression method * moved typed_write/typed_print for bptree into class * Merge pull request #7432 from realm/fsa/clean_typed_write moved typed_write/typed_print for bptree into class * cleanup unrelated code changes * fix compilation * cleanup * code review * code review * scaffolding for accessing unaligned memory inside compressed arrays * introducing parallel scan for scanning in one go multiple compressed arrays * Silence warning * tests + point fix for upper bound * fix ubsan * added clickbench data ingestor * Added 2 queries to clickbench * separated out a clickquery benchmark * separated out a clickquery benchmark * update to clickbench and query * fix warnings * add new builder with compression always enabled for verifying that compression works correctly (#7524) * attempt to fix windows builders * silence warning windows * silence warnings --------- Co-authored-by: Finn Schiermer Andersen Co-authored-by: Jørgen Edelbo Co-authored-by: Finn Schiermer Andersen --- CMakeLists.txt | 7 ++++--- evergreen/config.yml | 21 +++++++++++++++++++++ src/realm/util/config.h.in | 1 + 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d5710f1077..6514f1e01ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -261,12 +261,13 @@ option(REALM_ENABLE_ALLOC_SET_ZERO "Zero all allocations." OFF) if(NOT EMSCRIPTEN) option(REALM_ENABLE_ENCRYPTION "Enable encryption." ON) endif() -option(REALM_ENABLE_MEMDEBUG "Add additional memory checks" OFF) -option(REALM_VALGRIND "Tell the test suite we are running with valgrind" OFF) -option(REALM_SYNC_MULTIPLEXING "Enables/disables sync session multiplexing by default" ON) +option(REALM_ENABLE_MEMDEBUG "Add additional memory checks." OFF) +option(REALM_VALGRIND "Tell the test suite we are running with valgrind." OFF) +option(REALM_SYNC_MULTIPLEXING "Enables/disables sync session multiplexing by default." ON) set(REALM_MAX_BPNODE_SIZE "1000" CACHE STRING "Max B+ tree node size.") option(REALM_ENABLE_GEOSPATIAL "Enable geospatial types and queries." ON) option(REALM_APP_SERVICES "Enable the default app services implementation." ON) +option(REALM_COMPRESS "Compress all the arrays by default in flex format." OFF) # Find dependencies set(THREADS_PREFER_PTHREAD_FLAG ON) diff --git a/evergreen/config.yml b/evergreen/config.yml index 7e5f4034782..df7e292791d 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -139,6 +139,10 @@ functions: set_cmake_var realm_vars REALM_BUILD_COMMANDLINE_TOOLS BOOL "${build_command_line_tools|On}" set_cmake_var realm_vars REALM_ENABLE_ENCRYPTION BOOL "${enable_realm_encryption|On}" + if [[ -n "${compress|}" ]]; then + set_cmake_var realm_vars REALM_COMPRESS PATH "${cmake_toolchain_file}" + fi + if [[ -n "${fetch_missing_dependencies|}" ]]; then set_cmake_var realm_vars REALM_FETCH_MISSING_DEPENDENCIES BOOL On @@ -1704,6 +1708,23 @@ buildvariants: - name: compile_test_coverage - name: finalize_coverage_data +- name: macos-array-compression + display_name: "MacOS 11 arm64 (Compress Arrays)" + run_on: macos-1100-arm64 + expansions: + cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-macos-universal.tar.gz" + cmake_bindir: "./cmake_binaries/CMake.app/Contents/bin" + cmake_toolchain_file: "./tools/cmake/xcode.toolchain.cmake" + cmake_generator: Xcode + max_jobs: $(sysctl -n hw.logicalcpu) + xcode_developer_dir: /Applications/Xcode13.1.app/Contents/Developer + extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=arm64 + compress: On + cmake_build_type: Debug + coveralls_flag_name: "macos-arm64" + tasks: + - name: compile_test + - name: windows-64-vs2019 display_name: "Windows x86_64 (VS 2019)" run_on: windows-vsCurrent-large diff --git a/src/realm/util/config.h.in b/src/realm/util/config.h.in index 508dc064a91..3bb6cd9df36 100644 --- a/src/realm/util/config.h.in +++ b/src/realm/util/config.h.in @@ -21,3 +21,4 @@ #cmakedefine01 REALM_VALGRIND #cmakedefine01 REALM_ASAN #cmakedefine01 REALM_TSAN +#cmakedefine01 REALM_COMPRESS From a6bb5e9db004a2fc31e7556bdf398b76f79eda5e Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Thu, 6 Jun 2024 15:28:51 +0100 Subject: [PATCH 10/18] RCORE-2094 Compressing Integer Arrays (#7668) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add integer compressor for arrays that have type WTypBits * Add packed handler * Add flex handler * Embed integer compression logic inside commit machinery * Tests * Optimization for integer array decompression and Array::find for compressed formats * code review --------- Co-authored-by: Finn Schiermer Andersen Co-authored-by: Jørgen Edelbo --- Package.swift | 4 + evergreen/config.yml | 4 + src/realm/CMakeLists.txt | 7 + src/realm/alloc_slab.cpp | 4 + src/realm/array.cpp | 606 ++----- src/realm/array.hpp | 329 ++-- src/realm/array_aggregate_optimizations.cpp | 369 ++++ src/realm/array_blobs_small.cpp | 3 +- src/realm/array_blobs_small.hpp | 3 +- src/realm/array_direct.hpp | 290 +--- src/realm/array_integer.cpp | 7 +- src/realm/array_integer.hpp | 14 +- src/realm/array_integer_tpl.hpp | 7 +- src/realm/array_mixed.cpp | 1 + src/realm/array_unsigned.cpp | 5 +- src/realm/array_unsigned.hpp | 10 +- src/realm/array_with_find.cpp | 36 +- src/realm/array_with_find.hpp | 36 +- src/realm/group.cpp | 12 +- src/realm/group.hpp | 1 + src/realm/group_writer.cpp | 14 +- src/realm/group_writer.hpp | 2 +- src/realm/impl/array_writer.hpp | 2 +- src/realm/impl/output_stream.cpp | 9 +- src/realm/impl/output_stream.hpp | 2 +- src/realm/integer_compressor.cpp | 318 ++++ src/realm/integer_compressor.hpp | 202 +++ src/realm/integer_flex_compressor.cpp | 79 + src/realm/integer_flex_compressor.hpp | 305 ++++ src/realm/integer_packed_compressor.cpp | 68 + src/realm/integer_packed_compressor.hpp | 229 +++ src/realm/node.cpp | 9 +- src/realm/node.hpp | 2 +- src/realm/node_header.hpp | 43 +- src/realm/obj.cpp | 5 +- src/realm/query_conditions.hpp | 149 ++ src/realm/query_engine.hpp | 1 + src/realm/query_state.hpp | 3 +- src/realm/table.hpp | 5 +- test/benchmark-common-tasks/main.cpp | 1 - test/object-store/results.cpp | 1 - test/test_array.cpp | 101 +- test/test_array_integer.cpp | 1681 +++++++++++++++++++ test/test_group.cpp | 194 +++ test/test_links.cpp | 2 + test/test_list.cpp | 35 + test/test_query.cpp | 34 + test/test_shared.cpp | 14 +- test/test_table.cpp | 123 +- test/test_unresolved_links.cpp | 31 +- 50 files changed, 4474 insertions(+), 938 deletions(-) create mode 100644 src/realm/array_aggregate_optimizations.cpp create mode 100644 src/realm/integer_compressor.cpp create mode 100644 src/realm/integer_compressor.hpp create mode 100644 src/realm/integer_flex_compressor.cpp create mode 100644 src/realm/integer_flex_compressor.hpp create mode 100644 src/realm/integer_packed_compressor.cpp create mode 100644 src/realm/integer_packed_compressor.hpp diff --git a/Package.swift b/Package.swift index 55a07398f7b..1a2e581eddb 100644 --- a/Package.swift +++ b/Package.swift @@ -52,6 +52,7 @@ let notSyncServerSources: [String] = [ "realm/array_blobs_small.cpp", "realm/array_decimal128.cpp", "realm/array_fixed_bytes.cpp", + "realm/array_aggregate_optimizations.cpp", "realm/array_integer.cpp", "realm/array_key.cpp", "realm/array_mixed.cpp", @@ -78,6 +79,9 @@ let notSyncServerSources: [String] = [ "realm/group.cpp", "realm/group_writer.cpp", "realm/history.cpp", + "realm/integer_compressor.cpp", + "realm/integer_flex_compressor.cpp", + "realm/integer_packed_compressor.cpp", "realm/impl", "realm/index_string.cpp", "realm/link_translator.cpp", diff --git a/evergreen/config.yml b/evergreen/config.yml index 8b0fed20697..debf15fd1fa 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -137,6 +137,10 @@ functions: set_cmake_var realm_vars REALM_LLVM_COVERAGE BOOL On fi + if [[ -n "${compress|}" ]]; then + set_cmake_var realm_vars REALM_COMPRESS PATH "${cmake_toolchain_file}" + fi + set_cmake_var realm_vars REALM_BUILD_COMMANDLINE_TOOLS BOOL "${build_command_line_tools|On}" set_cmake_var realm_vars REALM_ENABLE_ENCRYPTION BOOL "${enable_realm_encryption|On}" if [[ -n "${compress|}" ]]; then diff --git a/src/realm/CMakeLists.txt b/src/realm/CMakeLists.txt index b5aebd5d3bf..18583f3549a 100644 --- a/src/realm/CMakeLists.txt +++ b/src/realm/CMakeLists.txt @@ -13,6 +13,7 @@ set(REALM_SOURCES array_blobs_big.cpp array_decimal128.cpp array_fixed_bytes.cpp + array_aggregate_optimizations.cpp array_integer.cpp array_key.cpp array_mixed.cpp @@ -36,6 +37,9 @@ set(REALM_SOURCES db.cpp group_writer.cpp history.cpp + integer_compressor.cpp + integer_flex_compressor.cpp + integer_packed_compressor.cpp impl/copy_replication.cpp impl/output_stream.cpp impl/simulated_failure.cpp @@ -163,6 +167,9 @@ set(REALM_INSTALL_HEADERS handover_defs.hpp history.hpp index_string.hpp + integer_compressor.hpp + integer_flex_compressor.hpp + integer_packed_compressor.hpp keys.hpp list.hpp mixed.hpp diff --git a/src/realm/alloc_slab.cpp b/src/realm/alloc_slab.cpp index 24b122e50d6..5465603c882 100644 --- a/src/realm/alloc_slab.cpp +++ b/src/realm/alloc_slab.cpp @@ -388,6 +388,10 @@ SlabAlloc::FreeBlock* SlabAlloc::allocate_block(int size) if (remaining) push_freelist_entry(remaining); REALM_ASSERT_EX(size_from_block(block) >= size, size_from_block(block), size, get_file_path_for_assertions()); + const auto block_before = bb_before(block); + REALM_ASSERT_DEBUG(block_before && block_before->block_after_size >= size); + const auto after_block_size = size_from_block(block); + REALM_ASSERT_DEBUG(after_block_size >= size); return block; } diff --git a/src/realm/array.cpp b/src/realm/array.cpp index 2f96b15877d..be70388bb2b 100644 --- a/src/realm/array.cpp +++ b/src/realm/array.cpp @@ -42,7 +42,6 @@ #pragma warning(disable : 4127) // Condition is constant warning #endif - // Header format (8 bytes): // ------------------------ // @@ -190,38 +189,79 @@ using namespace realm::util; void QueryStateBase::dyncast() {} -size_t Array::bit_width(int64_t v) +uint8_t Array::bit_width(int64_t v) { // FIXME: Assuming there is a 64-bit CPU reverse bitscan // instruction and it is fast, then this function could be // implemented as a table lookup on the result of the scan - if ((uint64_t(v) >> 4) == 0) { static const int8_t bits[] = {0, 1, 2, 2, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}; return bits[int8_t(v)]; } - - // First flip all bits if bit 63 is set (will now always be zero) if (v < 0) v = ~v; - // Then check if bits 15-31 used (32b), 7-31 used (16b), else (8b) return uint64_t(v) >> 31 ? 64 : uint64_t(v) >> 15 ? 32 : uint64_t(v) >> 7 ? 16 : 8; } +template +struct Array::VTableForWidth { + struct PopulatedVTable : VTable { + PopulatedVTable() + { + getter = &Array::get; + setter = &Array::set; + chunk_getter = &Array::get_chunk; + finder[cond_Equal] = &Array::find_vtable; + finder[cond_NotEqual] = &Array::find_vtable; + finder[cond_Greater] = &Array::find_vtable; + finder[cond_Less] = &Array::find_vtable; + } + }; + static const PopulatedVTable vtable; +}; + +template +const typename Array::VTableForWidth::PopulatedVTable Array::VTableForWidth::vtable; + void Array::init_from_mem(MemRef mem) noexcept { - char* header = Node::init_from_mem(mem); - // Parse header + // Header is the type of header that has been allocated, in case we are decompressing, + // the header is of kind A, which is kind of deceiving the purpose of these checks. + // Since we will try to fetch some data from the just initialised header, and never reset + // important fields used for type A arrays, like width, lower, upper_bound which are used + // for expanding the array, but also query the data. + const auto header = mem.get_addr(); + const auto is_extended = m_integer_compressor.init(header); + m_is_inner_bptree_node = get_is_inner_bptree_node_from_header(header); m_has_refs = get_hasrefs_from_header(header); m_context_flag = get_context_flag_from_header(header); - update_width_cache_from_header(); + + if (is_extended) { + m_ref = mem.get_ref(); + m_data = get_data_from_header(header); + m_size = m_integer_compressor.size(); + m_width = m_integer_compressor.v_width(); + m_lbound = -m_integer_compressor.v_mask(); + m_ubound = m_integer_compressor.v_mask() - 1; + m_integer_compressor.set_vtable(*this); + m_getter = m_vtable->getter; + } + else { + // Old init phase. + Node::init_from_mem(mem); + update_width_cache_from_header(); + } +} + +MemRef Array::get_mem() const noexcept +{ + return MemRef(get_header_from_data(m_data), m_ref, m_alloc); } void Array::update_from_parent() noexcept { - REALM_ASSERT_DEBUG(is_attached()); ArrayParent* parent = get_parent(); REALM_ASSERT_DEBUG(parent); ref_type new_ref = get_ref_from_parent(); @@ -230,7 +270,7 @@ void Array::update_from_parent() noexcept void Array::set_type(Type type) { - REALM_ASSERT(is_attached()); + REALM_ASSERT_DEBUG(is_attached()); copy_on_write(); // Throws @@ -254,7 +294,6 @@ void Array::set_type(Type type) set_hasrefs_in_header(init_has_refs, header); } - void Array::destroy_children(size_t offset) noexcept { for (size_t i = offset; i != m_size; ++i) { @@ -275,15 +314,28 @@ void Array::destroy_children(size_t offset) noexcept } } +// size_t Array::get_byte_size() const noexcept +//{ +// const auto header = get_header(); +// auto num_bytes = get_byte_size_from_header(header); +// auto read_only = m_alloc.is_read_only(m_ref) == true; +// auto capacity = get_capacity_from_header(header); +// auto bytes_ok = num_bytes <= capacity; +// REALM_ASSERT(read_only || bytes_ok); +// REALM_ASSERT_7(m_alloc.is_read_only(m_ref), ==, true, ||, num_bytes, <=, get_capacity_from_header(header)); +// return num_bytes; +// } ref_type Array::do_write_shallow(_impl::ArrayWriterBase& out) const { - // Write flat array + // here we might want to compress the array and write down. const char* header = get_header_from_data(m_data); size_t byte_size = get_byte_size(); - uint32_t dummy_checksum = 0x41414141UL; // "AAAA" in ASCII - ref_type new_ref = out.write_array(header, byte_size, dummy_checksum); // Throws - REALM_ASSERT_3(new_ref % 8, ==, 0); // 8-byte alignment + const auto compressed = is_compressed(); + uint32_t dummy_checksum = compressed ? 0x42424242UL : 0x41414141UL; // + uint32_t dummy_checksum_bytes = compressed ? 2 : 4; // AAAA / BB (only 2 bytes for extended arrays) + ref_type new_ref = out.write_array(header, byte_size, dummy_checksum, dummy_checksum_bytes); // Throws + REALM_ASSERT_3(new_ref % 8, ==, 0); // 8-byte alignment return new_ref; } @@ -308,7 +360,6 @@ ref_type Array::do_write_deep(_impl::ArrayWriterBase& out, bool only_if_modified } new_array.add(value); // Throws } - return new_array.do_write_shallow(out); // Throws } @@ -333,8 +384,8 @@ void Array::move(size_t begin, size_t end, size_t dest_begin) if (bits_per_elem < 8) { // FIXME: Should be optimized for (size_t i = begin; i != end; ++i) { - int_fast64_t v = (this->*m_getter)(i); - (this->*(m_vtable->setter))(dest_begin++, v); + int_fast64_t v = m_getter(*this, i); + m_vtable->setter(*this, dest_begin++, v); } return; } @@ -360,8 +411,8 @@ void Array::move(Array& dst, size_t ndx) size_t sz = m_size; for (size_t i = ndx; i < sz; i++) { - auto v = (this->*getter)(i); - (dst.*setter)(dest_begin++, v); + auto v = getter(*this, i); + setter(dst, dest_begin++, v); } truncate(ndx); @@ -370,17 +421,15 @@ void Array::move(Array& dst, size_t ndx) void Array::set(size_t ndx, int64_t value) { REALM_ASSERT_3(ndx, <, m_size); - if ((this->*(m_vtable->getter))(ndx) == value) + if (m_vtable->getter(*this, ndx) == value) return; // Check if we need to copy before modifying copy_on_write(); // Throws - // Grow the array if needed to store this value ensure_minimum_width(value); // Throws - // Set the value - (this->*(m_vtable->setter))(ndx, value); + m_vtable->setter(*this, ndx, value); } void Array::set_as_ref(size_t ndx, ref_type ref) @@ -428,6 +477,7 @@ void Array::insert(size_t ndx, int_fast64_t value) { REALM_ASSERT_DEBUG(ndx <= m_size); + decompress_array(*this); const auto old_width = m_width; const auto old_size = m_size; const Getter old_getter = m_getter; // Save old getter before potential width expansion @@ -447,8 +497,8 @@ void Array::insert(size_t ndx, int_fast64_t value) size_t i = old_size; while (i > ndx) { --i; - int64_t v = (this->*old_getter)(i); - (this->*(m_vtable->setter))(i + 1, v); + int64_t v = old_getter(*this, i); + m_vtable->setter(*this, i + 1, v); } } else if (ndx != old_size) { @@ -462,19 +512,30 @@ void Array::insert(size_t ndx, int_fast64_t value) } // Insert the new value - (this->*(m_vtable->setter))(ndx, value); + m_vtable->setter(*this, ndx, value); // Expand values above insertion if (do_expand) { size_t i = ndx; while (i != 0) { --i; - int64_t v = (this->*old_getter)(i); - (this->*(m_vtable->setter))(i, v); + int64_t v = old_getter(*this, i); + m_vtable->setter(*this, i, v); } } } +void Array::copy_on_write() +{ + if (is_read_only() && !decompress_array(*this)) + Node::copy_on_write(); +} + +void Array::copy_on_write(size_t min_size) +{ + if (is_read_only() && !decompress_array(*this)) + Node::copy_on_write(min_size); +} void Array::truncate(size_t new_size) { @@ -499,7 +560,6 @@ void Array::truncate(size_t new_size) } } - void Array::truncate_and_destroy_children(size_t new_size) { REALM_ASSERT(is_attached()); @@ -528,10 +588,8 @@ void Array::truncate_and_destroy_children(size_t new_size) } } - void Array::do_ensure_minimum_width(int_fast64_t value) { - // Make room for the new value const size_t width = bit_width(value); @@ -544,353 +602,32 @@ void Array::do_ensure_minimum_width(int_fast64_t value) size_t i = m_size; while (i != 0) { --i; - int64_t v = (this->*old_getter)(i); - (this->*(m_vtable->setter))(i, v); + int64_t v = old_getter(*this, i); + m_vtable->setter(*this, i, v); } } -int64_t Array::sum(size_t start, size_t end) const +bool Array::compress_array(Array& arr) const { - REALM_TEMPEX(return sum, m_width, (start, end)); + if (m_integer_compressor.get_encoding() == NodeHeader::Encoding::WTypBits) { + return m_integer_compressor.compress(*this, arr); + } + return false; } -template -int64_t Array::sum(size_t start, size_t end) const +bool Array::decompress_array(Array& arr) const { - if (end == size_t(-1)) - end = m_size; - REALM_ASSERT_EX(end <= m_size && start <= end, start, end, m_size); - - if (w == 0 || start == end) - return 0; - - int64_t s = 0; - - // Sum manually until 128 bit aligned - for (; (start < end) && (((size_t(m_data) & 0xf) * 8 + start * w) % 128 != 0); start++) { - s += get(start); - } - - if (w == 1 || w == 2 || w == 4) { - // Sum of bitwidths less than a byte (which are always positive) - // uses a divide and conquer algorithm that is a variation of popolation count: - // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel - - // static values needed for fast sums - const uint64_t m2 = 0x3333333333333333ULL; - const uint64_t m4 = 0x0f0f0f0f0f0f0f0fULL; - const uint64_t h01 = 0x0101010101010101ULL; - - int64_t* data = reinterpret_cast(m_data + start * w / 8); - size_t chunks = (end - start) * w / 8 / sizeof(int64_t); - - for (size_t t = 0; t < chunks; t++) { - if (w == 1) { -#if 0 -#if defined(USE_SSE42) && defined(_MSC_VER) && defined(REALM_PTR_64) - s += __popcnt64(data[t]); -#elif !defined(_MSC_VER) && defined(USE_SSE42) && defined(REALM_PTR_64) - s += __builtin_popcountll(data[t]); -#else - uint64_t a = data[t]; - const uint64_t m1 = 0x5555555555555555ULL; - a -= (a >> 1) & m1; - a = (a & m2) + ((a >> 2) & m2); - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - s += a; -#endif -#endif - s += fast_popcount64(data[t]); - } - else if (w == 2) { - uint64_t a = data[t]; - a = (a & m2) + ((a >> 2) & m2); - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - - s += a; - } - else if (w == 4) { - uint64_t a = data[t]; - a = (a & m4) + ((a >> 4) & m4); - a = (a * h01) >> 56; - s += a; - } - } - start += sizeof(int64_t) * 8 / no0(w) * chunks; - } - -#ifdef REALM_COMPILER_SSE - if (sseavx<42>()) { - // 2000 items summed 500000 times, 8/16/32 bits, miliseconds: - // Naive, templated get<>: 391 371 374 - // SSE: 97 148 282 - - if ((w == 8 || w == 16 || w == 32) && end - start > sizeof(__m128i) * 8 / no0(w)) { - __m128i* data = reinterpret_cast<__m128i*>(m_data + start * w / 8); - __m128i sum_result = {0}; - __m128i sum2; - - size_t chunks = (end - start) * w / 8 / sizeof(__m128i); - - for (size_t t = 0; t < chunks; t++) { - if (w == 8) { - /* - // 469 ms AND disadvantage of handling max 64k elements before overflow - __m128i vl = _mm_cvtepi8_epi16(data[t]); - __m128i vh = data[t]; - vh.m128i_i64[0] = vh.m128i_i64[1]; - vh = _mm_cvtepi8_epi16(vh); - sum_result = _mm_add_epi16(sum_result, vl); - sum_result = _mm_add_epi16(sum_result, vh); - */ - - /* - // 424 ms - __m128i vl = _mm_unpacklo_epi8(data[t], _mm_set1_epi8(0)); - __m128i vh = _mm_unpackhi_epi8(data[t], _mm_set1_epi8(0)); - sum_result = _mm_add_epi32(sum_result, _mm_madd_epi16(vl, _mm_set1_epi16(1))); - sum_result = _mm_add_epi32(sum_result, _mm_madd_epi16(vh, _mm_set1_epi16(1))); - */ - - __m128i vl = _mm_cvtepi8_epi16(data[t]); // sign extend lower words 8->16 - __m128i vh = data[t]; - vh = _mm_srli_si128(vh, 8); // v >>= 64 - vh = _mm_cvtepi8_epi16(vh); // sign extend lower words 8->16 - __m128i sum1 = _mm_add_epi16(vl, vh); - __m128i sumH = _mm_cvtepi16_epi32(sum1); - __m128i sumL = _mm_srli_si128(sum1, 8); // v >>= 64 - sumL = _mm_cvtepi16_epi32(sumL); - sum_result = _mm_add_epi32(sum_result, sumL); - sum_result = _mm_add_epi32(sum_result, sumH); - } - else if (w == 16) { - // todo, can overflow for array size > 2^32 - __m128i vl = _mm_cvtepi16_epi32(data[t]); // sign extend lower words 16->32 - __m128i vh = data[t]; - vh = _mm_srli_si128(vh, 8); // v >>= 64 - vh = _mm_cvtepi16_epi32(vh); // sign extend lower words 16->32 - sum_result = _mm_add_epi32(sum_result, vl); - sum_result = _mm_add_epi32(sum_result, vh); - } - else if (w == 32) { - __m128i v = data[t]; - __m128i v0 = _mm_cvtepi32_epi64(v); // sign extend lower dwords 32->64 - v = _mm_srli_si128(v, 8); // v >>= 64 - __m128i v1 = _mm_cvtepi32_epi64(v); // sign extend lower dwords 32->64 - sum_result = _mm_add_epi64(sum_result, v0); - sum_result = _mm_add_epi64(sum_result, v1); - - /* - __m128i m = _mm_set1_epi32(0xc000); // test if overflow could happen (still need - underflow test). - __m128i mm = _mm_and_si128(data[t], m); - zz = _mm_or_si128(mm, zz); - sum_result = _mm_add_epi32(sum_result, data[t]); - */ - } - } - start += sizeof(__m128i) * 8 / no0(w) * chunks; - - // prevent taking address of 'state' to make the compiler keep it in SSE register in above loop - // (vc2010/gcc4.6) - sum2 = sum_result; - - // Avoid aliasing bug where sum2 might not yet be initialized when accessed by get_universal - char sum3[sizeof sum2]; - memcpy(&sum3, &sum2, sizeof sum2); - - // Sum elements of sum - for (size_t t = 0; t < sizeof(__m128i) * 8 / ((w == 8 || w == 16) ? 32 : 64); ++t) { - int64_t v = get_universal < (w == 8 || w == 16) ? 32 : 64 > (reinterpret_cast(&sum3), t); - s += v; - } - } - } -#endif - - // Sum remaining elements - for (; start < end; ++start) - s += get(start); - - return s; + return arr.is_compressed() ? m_integer_compressor.decompress(arr) : false; } -size_t Array::count(int64_t value) const noexcept +bool Array::try_compress(Array& arr) const { - const uint64_t* next = reinterpret_cast(m_data); - size_t value_count = 0; - const size_t end = m_size; - size_t i = 0; - - // static values needed for fast population count - const uint64_t m1 = 0x5555555555555555ULL; - const uint64_t m2 = 0x3333333333333333ULL; - const uint64_t m4 = 0x0f0f0f0f0f0f0f0fULL; - const uint64_t h01 = 0x0101010101010101ULL; - - if (m_width == 0) { - if (value == 0) - return m_size; - return 0; - } - if (m_width == 1) { - if (uint64_t(value) > 1) - return 0; - - const size_t chunkvals = 64; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - if (value == 0) - a = ~a; // reverse - - a -= (a >> 1) & m1; - a = (a & m2) + ((a >> 2) & m2); - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - - // Could use intrinsic instead: - // a = __builtin_popcountll(a); // gcc intrinsic - - value_count += to_size_t(a); - } - } - else if (m_width == 2) { - if (uint64_t(value) > 3) - return 0; - - const uint64_t v = ~0ULL / 0x3 * value; - - // Masks to avoid spillover between segments in cascades - const uint64_t c1 = ~0ULL / 0x3 * 0x1; - - const size_t chunkvals = 32; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - a ^= v; // zero matching bit segments - a |= (a >> 1) & c1; // cascade ones in non-zeroed segments - a &= m1; // isolate single bit in each segment - a ^= m1; // reverse isolated bits - // if (!a) continue; - - // Population count - a = (a & m2) + ((a >> 2) & m2); - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - - value_count += to_size_t(a); - } - } - else if (m_width == 4) { - if (uint64_t(value) > 15) - return 0; - - const uint64_t v = ~0ULL / 0xF * value; - const uint64_t m = ~0ULL / 0xF * 0x1; - - // Masks to avoid spillover between segments in cascades - const uint64_t c1 = ~0ULL / 0xF * 0x7; - const uint64_t c2 = ~0ULL / 0xF * 0x3; - - const size_t chunkvals = 16; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - a ^= v; // zero matching bit segments - a |= (a >> 1) & c1; // cascade ones in non-zeroed segments - a |= (a >> 2) & c2; - a &= m; // isolate single bit in each segment - a ^= m; // reverse isolated bits - - // Population count - a = (a + (a >> 4)) & m4; - a = (a * h01) >> 56; - - value_count += to_size_t(a); - } - } - else if (m_width == 8) { - if (value > 0x7FLL || value < -0x80LL) - return 0; // by casting? - - const uint64_t v = ~0ULL / 0xFF * value; - const uint64_t m = ~0ULL / 0xFF * 0x1; - - // Masks to avoid spillover between segments in cascades - const uint64_t c1 = ~0ULL / 0xFF * 0x7F; - const uint64_t c2 = ~0ULL / 0xFF * 0x3F; - const uint64_t c3 = ~0ULL / 0xFF * 0x0F; - - const size_t chunkvals = 8; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - a ^= v; // zero matching bit segments - a |= (a >> 1) & c1; // cascade ones in non-zeroed segments - a |= (a >> 2) & c2; - a |= (a >> 4) & c3; - a &= m; // isolate single bit in each segment - a ^= m; // reverse isolated bits - - // Population count - a = (a * h01) >> 56; - - value_count += to_size_t(a); - } - } - else if (m_width == 16) { - if (value > 0x7FFFLL || value < -0x8000LL) - return 0; // by casting? - - const uint64_t v = ~0ULL / 0xFFFF * value; - const uint64_t m = ~0ULL / 0xFFFF * 0x1; - - // Masks to avoid spillover between segments in cascades - const uint64_t c1 = ~0ULL / 0xFFFF * 0x7FFF; - const uint64_t c2 = ~0ULL / 0xFFFF * 0x3FFF; - const uint64_t c3 = ~0ULL / 0xFFFF * 0x0FFF; - const uint64_t c4 = ~0ULL / 0xFFFF * 0x00FF; - - const size_t chunkvals = 4; - for (; i + chunkvals <= end; i += chunkvals) { - uint64_t a = next[i / chunkvals]; - a ^= v; // zero matching bit segments - a |= (a >> 1) & c1; // cascade ones in non-zeroed segments - a |= (a >> 2) & c2; - a |= (a >> 4) & c3; - a |= (a >> 8) & c4; - a &= m; // isolate single bit in each segment - a ^= m; // reverse isolated bits - - // Population count - a = (a * h01) >> 56; - - value_count += to_size_t(a); - } - } - else if (m_width == 32) { - int32_t v = int32_t(value); - const int32_t* d = reinterpret_cast(m_data); - for (; i < end; ++i) { - if (d[i] == v) - ++value_count; - } - return value_count; - } - else if (m_width == 64) { - const int64_t* d = reinterpret_cast(m_data); - for (; i < end; ++i) { - if (d[i] == value) - ++value_count; - } - return value_count; - } - - // Check remaining elements - for (; i < end; ++i) - if (value == get(i)) - ++value_count; + return compress_array(arr); +} - return value_count; +bool Array::try_decompress() +{ + return decompress_array(*this); } size_t Array::calc_aligned_byte_size(size_t size, int width) @@ -990,9 +727,9 @@ MemRef Array::create(Type type, bool context_flag, WidthType width_type, size_t { REALM_ASSERT_DEBUG(value == 0 || width_type == wtype_Bits); REALM_ASSERT_DEBUG(size == 0 || width_type != wtype_Ignore); - int width = 0; + uint8_t width = 0; if (value != 0) - width = static_cast(bit_width(value)); + width = bit_width(value); auto mem = Node::create_node(size, alloc, context_flag, type, width_type, width); if (value != 0) { const auto header = mem.get_addr(); @@ -1004,52 +741,32 @@ MemRef Array::create(Type type, bool context_flag, WidthType width_type, size_t } // This is the one installed into the m_vtable->finder slots. -template -bool Array::find_vtable(int64_t value, size_t start, size_t end, size_t baseindex, QueryStateBase* state) const +template +bool Array::find_vtable(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) { - return ArrayWithFind(*this).find_optimized(value, start, end, baseindex, state); + REALM_TEMPEX2(return ArrayWithFind(arr).find_optimized, cond, arr.m_width, (value, start, end, baseindex, state)); } - -template -struct Array::VTableForWidth { - struct PopulatedVTable : Array::VTable { - PopulatedVTable() - { - getter = &Array::get; - setter = &Array::set; - chunk_getter = &Array::get_chunk; - finder[cond_Equal] = &Array::find_vtable; - finder[cond_NotEqual] = &Array::find_vtable; - finder[cond_Greater] = &Array::find_vtable; - finder[cond_Less] = &Array::find_vtable; - } - }; - static const PopulatedVTable vtable; -}; - -template -const typename Array::VTableForWidth::PopulatedVTable Array::VTableForWidth::vtable; - void Array::update_width_cache_from_header() noexcept { - auto width = get_width_from_header(get_header()); - m_lbound = lbound_for_width(width); - m_ubound = ubound_for_width(width); - - m_width = width; - - REALM_TEMPEX(m_vtable = &VTableForWidth, width, ::vtable); + m_width = get_width_from_header(get_header()); + m_lbound = lbound_for_width(m_width); + m_ubound = ubound_for_width(m_width); + REALM_ASSERT_DEBUG(m_lbound <= m_ubound); + REALM_ASSERT_DEBUG(m_width >= m_lbound); + REALM_ASSERT_DEBUG(m_width <= m_ubound); + REALM_TEMPEX(m_vtable = &VTableForWidth, m_width, ::vtable); m_getter = m_vtable->getter; } // This method reads 8 concecutive values into res[8], starting from index 'ndx'. It's allowed for the 8 values to // exceed array length; in this case, remainder of res[8] will be be set to 0. template -void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept +void Array::get_chunk(const Array& arr, size_t ndx, int64_t res[8]) noexcept { - REALM_ASSERT_3(ndx, <, m_size); - + auto sz = arr.size(); + REALM_ASSERT_3(ndx, <, sz); size_t i = 0; // if constexpr to avoid producing spurious warnings resulting from @@ -1061,7 +778,7 @@ void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept // Round m_size down to byte granularity as the trailing bits in the last // byte are uninitialized - size_t bytes_available = m_size / elements_per_byte; + size_t bytes_available = sz / elements_per_byte; // Round start and end to be byte-aligned. Start is rounded down and // end is rounded up as we may read up to 7 unused bits at each end. @@ -1073,7 +790,7 @@ void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept uint64_t c = 0; for (size_t i = end; i > start; --i) { c <<= 8; - c += *reinterpret_cast(m_data + i - 1); + c += *reinterpret_cast(arr.m_data + i - 1); } // Trim off leading bits which aren't part of the requested range c >>= (ndx - start * elements_per_byte) * w; @@ -1093,31 +810,31 @@ void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept } } - for (; i + ndx < m_size && i < 8; i++) - res[i] = get(ndx + i); + for (; i + ndx < sz && i < 8; i++) + res[i] = get(arr, ndx + i); for (; i < 8; i++) res[i] = 0; #ifdef REALM_DEBUG - for (int j = 0; j + ndx < m_size && j < 8; j++) { - int64_t expected = get(ndx + j); + for (int j = 0; j + ndx < sz && j < 8; j++) { + int64_t expected = Array::get_universal(arr.m_data, ndx + j); REALM_ASSERT(res[j] == expected); } #endif } template <> -void Array::get_chunk<0>(size_t ndx, int64_t res[8]) const noexcept +void Array::get_chunk<0>(const Array& arr, size_t ndx, int64_t res[8]) noexcept { - REALM_ASSERT_3(ndx, <, m_size); + REALM_ASSERT_3(ndx, <, arr.m_size); memset(res, 0, sizeof(int64_t) * 8); } template -void Array::set(size_t ndx, int64_t value) +void Array::set(Array& arr, size_t ndx, int64_t value) { - set_direct(m_data, ndx, value); + realm::set_direct(arr.m_data, ndx, value); } void Array::_mem_usage(size_t& mem) const noexcept @@ -1222,10 +939,15 @@ void Array::report_memory_usage_2(MemUsageHandler& handler) const void Array::verify() const { #ifdef REALM_DEBUG - REALM_ASSERT(is_attached()); - REALM_ASSERT(m_width == 0 || m_width == 1 || m_width == 2 || m_width == 4 || m_width == 8 || m_width == 16 || - m_width == 32 || m_width == 64); + REALM_ASSERT(is_attached()); + if (!wtype_is_extended(get_header())) { + REALM_ASSERT(m_width == 0 || m_width == 1 || m_width == 2 || m_width == 4 || m_width == 8 || m_width == 16 || + m_width == 32 || m_width == 64); + } + else { + REALM_ASSERT(m_width <= 64); + } if (!get_parent()) return; @@ -1238,35 +960,60 @@ void Array::verify() const size_t Array::lower_bound_int(int64_t value) const noexcept { + if (is_compressed()) + return lower_bound_int_compressed(value); REALM_TEMPEX(return lower_bound, m_width, (m_data, m_size, value)); } size_t Array::upper_bound_int(int64_t value) const noexcept { + if (is_compressed()) + return upper_bound_int_compressed(value); REALM_TEMPEX(return upper_bound, m_width, (m_data, m_size, value)); } - -size_t Array::find_first(int64_t value, size_t start, size_t end) const +size_t Array::lower_bound_int_compressed(int64_t value) const noexcept { - return find_first(value, start, end); + static impl::CompressedDataFetcher encoder; + encoder.ptr = &m_integer_compressor; + return lower_bound(m_data, m_size, value, encoder); } +size_t Array::upper_bound_int_compressed(int64_t value) const noexcept +{ + static impl::CompressedDataFetcher encoder; + encoder.ptr = &m_integer_compressor; + return upper_bound(m_data, m_size, value, encoder); +} int_fast64_t Array::get(const char* header, size_t ndx) noexcept { - const char* data = get_data_from_header(header); - uint_least8_t width = get_width_from_header(header); - return get_direct(data, width, ndx); + // this is very important. Most of the times we end up here + // because we are traversing the cluster, the keys/refs in the cluster + // are not compressed (because there is almost no gain), so the intent + // is avoiding to pollute traversing the cluster as little as possible. + // We need to check the header wtype and only initialise the + // integer compressor, if needed. Otherwise we should just call + // get_direct. On average there should be one more access to the header + // while traversing the cluster tree. + if (REALM_LIKELY(!NodeHeader::wtype_is_extended(header))) { + const char* data = get_data_from_header(header); + uint_least8_t width = get_width_from_header(header); + return get_direct(data, width, ndx); + } + // Ideally, we would not want to construct a compressor every time we end up here. + // However the compressor initalization should be fast enough. Creating an array, + // which owns a compressor internally, is the better approach if we intend to access + // the same data over and over again. The compressor basically caches the most important + // information about the layuot of the data itself. + IntegerCompressor s_compressor; + s_compressor.init(header); + return s_compressor.get(ndx); } - std::pair Array::get_two(const char* header, size_t ndx) noexcept { - const char* data = get_data_from_header(header); - uint_least8_t width = get_width_from_header(header); - std::pair p = ::get_two(data, width, ndx); - return std::make_pair(p.first, p.second); + return std::make_pair(get(header, ndx), get(header, ndx + 1)); } bool QueryStateCount::match(size_t, Mixed) noexcept @@ -1312,7 +1059,6 @@ bool QueryStateFindAll>::match(size_t index) noexcept ++m_match_count; int64_t key_value = (m_key_values ? m_key_values->get(index) : index) + m_key_offset; m_keys.push_back(ObjKey(key_value)); - return (m_limit > m_match_count); } diff --git a/src/realm/array.hpp b/src/realm/array.hpp index 1df0aa2b992..6b9569ebd82 100644 --- a/src/realm/array.hpp +++ b/src/realm/array.hpp @@ -21,8 +21,10 @@ #include #include +#include #include #include +#include namespace realm { @@ -90,12 +92,8 @@ class QueryStateFindFirst : public QueryStateBase { class Array : public Node, public ArrayParent { public: /// Create an array accessor in the unattached state. - explicit Array(Allocator& allocator) noexcept - : Node(allocator) - { - } - - ~Array() noexcept override {} + explicit Array(Allocator& allocator) noexcept; + virtual ~Array() noexcept = default; /// Create a new integer array of the specified type and size, and filled /// with the specified value, and attach this accessor to it. This does not @@ -126,6 +124,8 @@ class Array : public Node, public ArrayParent { init_from_ref(ref); } + MemRef get_mem() const noexcept; + /// Called in the context of Group::commit() to ensure that attached /// accessors stay valid across a commit. Please note that this works only /// for non-transactional commits. Accessors obtained during a transaction @@ -174,21 +174,23 @@ class Array : public Node, public ArrayParent { void set_as_ref(size_t ndx, ref_type ref); template - void set(size_t ndx, int64_t value); + static void set(Array&, size_t ndx, int64_t value); int64_t get(size_t ndx) const noexcept; + std::vector get_all(size_t b, size_t e) const; + template - int64_t get(size_t ndx) const noexcept; + static int64_t get(const Array& arr, size_t ndx) noexcept; void get_chunk(size_t ndx, int64_t res[8]) const noexcept; template - void get_chunk(size_t ndx, int64_t res[8]) const noexcept; + static void get_chunk(const Array&, size_t ndx, int64_t res[8]) noexcept; ref_type get_as_ref(size_t ndx) const noexcept; - RefOrTagged get_as_ref_or_tagged(size_t ndx) const noexcept; + void set(size_t ndx, RefOrTagged); void add(RefOrTagged); void ensure_minimum_width(RefOrTagged); @@ -198,12 +200,21 @@ class Array : public Node, public ArrayParent { void alloc(size_t init_size, size_t new_width) { - REALM_ASSERT_3(m_width, ==, get_width_from_header(get_header())); - REALM_ASSERT_3(m_size, ==, get_size_from_header(get_header())); + // Node::alloc is the one that triggers copy on write. If we call alloc for a B + // array we have a bug in our machinery, the array should have been decompressed + // way before calling alloc. + const auto header = get_header(); + REALM_ASSERT_3(m_width, ==, get_width_from_header(header)); + REALM_ASSERT_3(m_size, ==, get_size_from_header(header)); Node::alloc(init_size, new_width); update_width_cache_from_header(); } + bool is_empty() const noexcept + { + return size() == 0; + } + /// Remove the element at the specified index, and move elements at higher /// indexes to the next lower index. /// @@ -322,6 +333,8 @@ class Array : public Node, public ArrayParent { /// by doing a linear search for short sequences. size_t lower_bound_int(int64_t value) const noexcept; size_t upper_bound_int(int64_t value) const noexcept; + size_t lower_bound_int_compressed(int64_t value) const noexcept; + size_t upper_bound_int_compressed(int64_t value) const noexcept; //@} int64_t get_sum(size_t start = 0, size_t end = size_t(-1)) const @@ -351,6 +364,18 @@ class Array : public Node, public ArrayParent { /// (idempotency). void destroy_deep() noexcept; + /// check if the array is encoded (in B format) + inline bool is_compressed() const; + + inline const IntegerCompressor& integer_compressor() const; + + /// used only for testing, encode the array passed as argument + bool try_compress(Array&) const; + + /// used only for testing, decode the array, on which this method is invoked. If the array is not encoded, this is + /// a NOP + bool try_decompress(); + /// Shorthand for `destroy_deep(MemRef(ref, alloc), alloc)`. static void destroy_deep(ref_type ref, Allocator& alloc) noexcept; @@ -383,25 +408,35 @@ class Array : public Node, public ArrayParent { /// Same as non-static write() with `deep` set to true. This is for the /// cases where you do not already have an array accessor available. + /// Compression may be attempted if `compress_in_flight` is true. + /// This should be avoided if you rely on the size of the array beeing unchanged. static ref_type write(ref_type, Allocator&, _impl::ArrayWriterBase&, bool only_if_modified, bool compress_in_flight); - size_t find_first(int64_t value, size_t begin = 0, size_t end = size_t(-1)) const; + inline size_t find_first(int64_t value, size_t begin = 0, size_t end = size_t(-1)) const + { + return find_first(value, begin, end); + } // Wrappers for backwards compatibility and for simple use without // setting up state initialization etc template size_t find_first(int64_t value, size_t start = 0, size_t end = size_t(-1)) const { - REALM_ASSERT(start <= m_size && (end <= m_size || end == size_t(-1)) && start <= end); - // todo, would be nice to avoid this in order to speed up find_first loops QueryStateFindFirst state; Finder finder = m_vtable->finder[cond::condition]; - (this->*finder)(value, start, end, 0, &state); + finder(*this, value, start, end, 0, &state); + return state.m_state; + } - return static_cast(state.m_state); + template + bool find(int64_t value, size_t start, size_t end, size_t baseIndex, QueryStateBase* state) const + { + Finder finder = m_vtable->finder[cond::condition]; + return finder(*this, value, start, end, baseIndex, state); } + /// Get the specified element without the cost of constructing an /// array instance. If an array instance is already available, or /// you need to get multiple values, then this method will be @@ -463,11 +498,15 @@ class Array : public Node, public ArrayParent { /// Takes a 64-bit value and returns the minimum number of bits needed /// to fit the value. For alignment this is rounded up to nearest /// log2. Possible results {0, 1, 2, 4, 8, 16, 32, 64} - static size_t bit_width(int64_t value); + static uint8_t bit_width(int64_t value); void typed_print(std::string prefix) const; protected: + friend class NodeTree; + void copy_on_write(); + void copy_on_write(size_t min_size); + // This returns the minimum value ("lower bound") of the representable values // for the given bit width. Valid widths are 0, 1, 2, 4, 8, 16, 32, and 64. static constexpr int_fast64_t lbound_for_width(size_t width) noexcept; @@ -505,14 +544,17 @@ class Array : public Node, public ArrayParent { protected: // Getters and Setters for adaptive-packed arrays - typedef int64_t (Array::*Getter)(size_t) const; // Note: getters must not throw - typedef void (Array::*Setter)(size_t, int64_t); - typedef bool (Array::*Finder)(int64_t, size_t, size_t, size_t, QueryStateBase*) const; - typedef void (Array::*ChunkGetter)(size_t, int64_t res[8]) const; // Note: getters must not throw + typedef int64_t (*Getter)(const Array&, size_t); // Note: getters must not throw + typedef void (*Setter)(Array&, size_t, int64_t); + typedef bool (*Finder)(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + typedef void (*ChunkGetter)(const Array&, size_t, int64_t res[8]); // Note: getters must not throw + + typedef std::vector (*GetterAll)(const Array&, size_t, size_t); // Note: getters must not throw struct VTable { Getter getter; ChunkGetter chunk_getter; + GetterAll getter_all; Setter setter; Finder finder[cond_VTABLE_FINDER_COUNT]; // one for each active function pointer }; @@ -520,11 +562,12 @@ class Array : public Node, public ArrayParent { struct VTableForWidth; // This is the one installed into the m_vtable->finder slots. - template - bool find_vtable(int64_t value, size_t start, size_t end, size_t baseindex, QueryStateBase* state) const; + template + static bool find_vtable(const Array&, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state); template - int64_t get_universal(const char* const data, const size_t ndx) const; + static int64_t get_universal(const char* const data, const size_t ndx); protected: Getter m_getter = nullptr; // cached to avoid indirection @@ -538,6 +581,11 @@ class Array : public Node, public ArrayParent { bool m_has_refs; // Elements whose first bit is zero are refs to subarrays. bool m_context_flag; // Meaning depends on context. + IntegerCompressor m_integer_compressor; + // compress/decompress this array + bool compress_array(Array&) const; + bool decompress_array(Array& arr) const; + private: ref_type do_write_shallow(_impl::ArrayWriterBase&) const; ref_type do_write_deep(_impl::ArrayWriterBase&, bool only_if_modified, bool compress) const; @@ -548,10 +596,15 @@ class Array : public Node, public ArrayParent { void report_memory_usage_2(MemUsageHandler&) const; #endif + +private: friend class Allocator; friend class SlabAlloc; friend class GroupWriter; friend class ArrayWithFind; + friend class IntegerCompressor; + friend class PackedCompressor; + friend class FlexCompressor; }; class TempArray : public Array { @@ -573,6 +626,57 @@ class TempArray : public Array { // Implementation: +inline Array::Array(Allocator& allocator) noexcept + : Node(allocator) +{ +} + +inline bool Array::is_compressed() const +{ + const auto enc = m_integer_compressor.get_encoding(); + return enc == NodeHeader::Encoding::Flex || enc == NodeHeader::Encoding::Packed; +} + +inline const IntegerCompressor& Array::integer_compressor() const +{ + return m_integer_compressor; +} + +inline int64_t Array::get(size_t ndx) const noexcept +{ + REALM_ASSERT_DEBUG(is_attached()); + REALM_ASSERT_DEBUG_EX(ndx < m_size, ndx, m_size); + return m_getter(*this, ndx); + + // Two ideas that are not efficient but may be worth looking into again: + /* + // Assume correct width is found early in REALM_TEMPEX, which is the case for B tree offsets that + // are probably either 2^16 long. Turns out to be 25% faster if found immediately, but 50-300% slower + // if found later + REALM_TEMPEX(return get, (ndx)); + */ + /* + // Slightly slower in both of the if-cases. Also needs an matchcount m_size check too, to avoid + // reading beyond array. + if (m_width >= 8 && m_size > ndx + 7) + return get<64>(ndx >> m_shift) & m_widthmask; + else + return (this->*(m_vtable->getter))(ndx); + */ +} + +inline std::vector Array::get_all(size_t b, size_t e) const +{ + REALM_ASSERT_DEBUG(is_compressed()); + return m_vtable->getter_all(*this, b, e); +} + +template +inline int64_t Array::get(const Array& arr, size_t ndx) noexcept +{ + REALM_ASSERT_DEBUG(arr.is_attached()); + return get_universal(arr.m_data, ndx); +} constexpr inline int_fast64_t Array::lbound_for_width(size_t width) noexcept { @@ -673,7 +777,6 @@ inline void Array::create(Type type, bool context_flag, size_t length, int_fast6 init_from_mem(mem); } - inline Array::Type Array::get_type() const noexcept { if (m_is_inner_bptree_node) { @@ -689,41 +792,44 @@ inline Array::Type Array::get_type() const noexcept inline void Array::get_chunk(size_t ndx, int64_t res[8]) const noexcept { REALM_ASSERT_DEBUG(ndx < m_size); - (this->*(m_vtable->chunk_getter))(ndx, res); + m_vtable->chunk_getter(*this, ndx, res); } template -int64_t Array::get_universal(const char* data, size_t ndx) const +inline int64_t Array::get_universal(const char* data, size_t ndx) { - if (w == 0) { - return 0; - } - else if (w == 1) { - size_t offset = ndx >> 3; - return (data[offset] >> (ndx & 7)) & 0x01; + if (w == 64) { + size_t offset = ndx << 3; + return *reinterpret_cast(data + offset); } - else if (w == 2) { - size_t offset = ndx >> 2; - return (data[offset] >> ((ndx & 3) << 1)) & 0x03; + else if (w == 32) { + size_t offset = ndx << 2; + return *reinterpret_cast(data + offset); } - else if (w == 4) { - size_t offset = ndx >> 1; - return (data[offset] >> ((ndx & 1) << 2)) & 0x0F; + else if (w == 16) { + size_t offset = ndx << 1; + return *reinterpret_cast(data + offset); } else if (w == 8) { return *reinterpret_cast(data + ndx); } - else if (w == 16) { - size_t offset = ndx * 2; - return *reinterpret_cast(data + offset); + else if (w == 4) { + size_t offset = ndx >> 1; + auto d = data[offset]; + return (d >> ((ndx & 1) << 2)) & 0x0F; } - else if (w == 32) { - size_t offset = ndx * 4; - return *reinterpret_cast(data + offset); + else if (w == 2) { + size_t offset = ndx >> 2; + auto d = data[offset]; + return (d >> ((ndx & 3) << 1)) & 0x03; } - else if (w == 64) { - size_t offset = ndx * 8; - return *reinterpret_cast(data + offset); + else if (w == 1) { + size_t offset = ndx >> 3; + auto d = data[offset]; + return (d >> (ndx & 7)) & 0x01; + } + else if (w == 0) { + return 0; } else { REALM_ASSERT_DEBUG(false); @@ -731,35 +837,6 @@ int64_t Array::get_universal(const char* data, size_t ndx) const } } -template -int64_t Array::get(size_t ndx) const noexcept -{ - return get_universal(m_data, ndx); -} - -inline int64_t Array::get(size_t ndx) const noexcept -{ - REALM_ASSERT_DEBUG(is_attached()); - REALM_ASSERT_DEBUG_EX(ndx < m_size, ndx, m_size); - return (this->*m_getter)(ndx); - - // Two ideas that are not efficient but may be worth looking into again: - /* - // Assume correct width is found early in REALM_TEMPEX, which is the case for B tree offsets that - // are probably either 2^16 long. Turns out to be 25% faster if found immediately, but 50-300% slower - // if found later - REALM_TEMPEX(return get, (ndx)); - */ - /* - // Slightly slower in both of the if-cases. Also needs an matchcount m_size check too, to avoid - // reading beyond array. - if (m_width >= 8 && m_size > ndx + 7) - return get<64>(ndx >> m_shift) & m_widthmask; - else - return (this->*(m_vtable->getter))(ndx); - */ -} - inline int64_t Array::front() const noexcept { return get(0); @@ -848,34 +925,6 @@ inline void Array::destroy_deep() noexcept m_data = nullptr; } -inline ref_type Array::write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified, bool compress) const -{ - REALM_ASSERT(is_attached()); - - if (only_if_modified && m_alloc.is_read_only(m_ref)) - return m_ref; - - if (!deep || !m_has_refs) - return do_write_shallow(out); // Throws - - return do_write_deep(out, only_if_modified, compress); // Throws -} - -inline ref_type Array::write(ref_type ref, Allocator& alloc, _impl::ArrayWriterBase& out, bool only_if_modified, - bool compress) -{ - if (only_if_modified && alloc.is_read_only(ref)) - return ref; - - Array array(alloc); - array.init_from_ref(ref); - - if (!array.m_has_refs) - return array.do_write_shallow(out); // Throws - - return array.do_write_deep(out, only_if_modified, compress); // Throws -} - inline void Array::add(int_fast64_t value) { insert(m_size, value); @@ -986,7 +1035,6 @@ inline size_t Array::get_max_byte_size(size_t num_elems) noexcept return header_size + num_elems * max_bytes_per_elem; } - inline void Array::update_child_ref(size_t child_ndx, ref_type new_ref) { set(child_ndx, new_ref); @@ -1004,6 +1052,73 @@ inline void Array::ensure_minimum_width(int_fast64_t value) do_ensure_minimum_width(value); } +inline ref_type Array::write(_impl::ArrayWriterBase& out, bool deep, bool only_if_modified, + bool compress_in_flight) const +{ + REALM_ASSERT_DEBUG(is_attached()); + // The default allocator cannot be trusted wrt is_read_only(): + REALM_ASSERT_DEBUG(!only_if_modified || &m_alloc != &Allocator::get_default()); + if (only_if_modified && m_alloc.is_read_only(m_ref)) + return m_ref; + + if (!deep || !m_has_refs) { + // however - creating an array using ANYTHING BUT the default allocator during commit is also wrong.... + // it only works by accident, because the whole slab area is reinitialized after commit. + // We should have: Array encoded_array{Allocator::get_default()}; + Array compressed_array{Allocator::get_default()}; + if (compress_in_flight && compress_array(compressed_array)) { +#ifdef REALM_DEBUG + const auto encoding = compressed_array.m_integer_compressor.get_encoding(); + REALM_ASSERT_DEBUG(encoding == Encoding::Flex || encoding == Encoding::Packed); + REALM_ASSERT_DEBUG(size() == compressed_array.size()); + for (size_t i = 0; i < compressed_array.size(); ++i) { + REALM_ASSERT_DEBUG(get(i) == compressed_array.get(i)); + } +#endif + auto ref = compressed_array.do_write_shallow(out); + compressed_array.destroy(); + return ref; + } + return do_write_shallow(out); // Throws + } + + return do_write_deep(out, only_if_modified, compress_in_flight); // Throws +} + +inline ref_type Array::write(ref_type ref, Allocator& alloc, _impl::ArrayWriterBase& out, bool only_if_modified, + bool compress_in_flight) +{ + // The default allocator cannot be trusted wrt is_read_only(): + REALM_ASSERT_DEBUG(!only_if_modified || &alloc != &Allocator::get_default()); + if (only_if_modified && alloc.is_read_only(ref)) + return ref; + + Array array(alloc); + array.init_from_ref(ref); + REALM_ASSERT_DEBUG(array.is_attached()); + + if (!array.m_has_refs) { + Array compressed_array{Allocator::get_default()}; + if (compress_in_flight && array.compress_array(compressed_array)) { +#ifdef REALM_DEBUG + const auto encoding = compressed_array.m_integer_compressor.get_encoding(); + REALM_ASSERT_DEBUG(encoding == Encoding::Flex || encoding == Encoding::Packed); + REALM_ASSERT_DEBUG(array.size() == compressed_array.size()); + for (size_t i = 0; i < compressed_array.size(); ++i) { + REALM_ASSERT_DEBUG(array.get(i) == compressed_array.get(i)); + } +#endif + auto ref = compressed_array.do_write_shallow(out); + compressed_array.destroy(); + return ref; + } + else { + return array.do_write_shallow(out); // Throws + } + } + return array.do_write_deep(out, only_if_modified, compress_in_flight); // Throws +} + } // namespace realm diff --git a/src/realm/array_aggregate_optimizations.cpp b/src/realm/array_aggregate_optimizations.cpp new file mode 100644 index 00000000000..6242e6853dd --- /dev/null +++ b/src/realm/array_aggregate_optimizations.cpp @@ -0,0 +1,369 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#include +#include + +using namespace realm; + +int64_t Array::sum(size_t start, size_t end) const +{ + REALM_TEMPEX(return sum, m_width, (start, end)); +} + +template +int64_t Array::sum(size_t start, size_t end) const +{ + if (end == size_t(-1)) + end = m_size; + + REALM_ASSERT_EX(end <= m_size && start <= end, start, end, m_size); + + if (start == end) + return 0; + + int64_t s = 0; + + // Sum manually until 128 bit aligned + for (; (start < end) && (((size_t(m_data) & 0xf) * 8 + start * w) % 128 != 0); start++) { + s += get(*this, start); + } + + if (w == 1 || w == 2 || w == 4) { + // Sum of bitwidths less than a byte (which are always positive) + // uses a divide and conquer algorithm that is a variation of popolation count: + // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel + + // static values needed for fast sums + const uint64_t m2 = 0x3333333333333333ULL; + const uint64_t m4 = 0x0f0f0f0f0f0f0f0fULL; + const uint64_t h01 = 0x0101010101010101ULL; + + int64_t* data = reinterpret_cast(m_data + start * w / 8); + size_t chunks = (end - start) * w / 8 / sizeof(int64_t); + + for (size_t t = 0; t < chunks; t++) { + if (w == 1) { +#if 0 +#if defined(USE_SSE42) && defined(_MSC_VER) && defined(REALM_PTR_64) + s += __popcnt64(data[t]); +#elif !defined(_MSC_VER) && defined(USE_SSE42) && defined(REALM_PTR_64) + s += __builtin_popcountll(data[t]); +#else + uint64_t a = data[t]; + const uint64_t m1 = 0x5555555555555555ULL; + a -= (a >> 1) & m1; + a = (a & m2) + ((a >> 2) & m2); + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + s += a; +#endif +#endif + s += fast_popcount64(data[t]); + } + else if (w == 2) { + uint64_t a = data[t]; + a = (a & m2) + ((a >> 2) & m2); + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + + s += a; + } + else if (w == 4) { + uint64_t a = data[t]; + a = (a & m4) + ((a >> 4) & m4); + a = (a * h01) >> 56; + s += a; + } + } + start += sizeof(int64_t) * 8 / no0(w) * chunks; + } + +#ifdef REALM_COMPILER_SSE + if (sseavx<42>()) { + // 2000 items summed 500000 times, 8/16/32 bits, miliseconds: + // Naive, templated get<>: 391 371 374 + // SSE: 97 148 282 + + if ((w == 8 || w == 16 || w == 32) && end - start > sizeof(__m128i) * 8 / no0(w)) { + __m128i* data = reinterpret_cast<__m128i*>(m_data + start * w / 8); + __m128i sum_result = {0}; + __m128i sum2; + + size_t chunks = (end - start) * w / 8 / sizeof(__m128i); + + for (size_t t = 0; t < chunks; t++) { + if (w == 8) { + /* + // 469 ms AND disadvantage of handling max 64k elements before overflow + __m128i vl = _mm_cvtepi8_epi16(data[t]); + __m128i vh = data[t]; + vh.m128i_i64[0] = vh.m128i_i64[1]; + vh = _mm_cvtepi8_epi16(vh); + sum_result = _mm_add_epi16(sum_result, vl); + sum_result = _mm_add_epi16(sum_result, vh); + */ + + /* + // 424 ms + __m128i vl = _mm_unpacklo_epi8(data[t], _mm_set1_epi8(0)); + __m128i vh = _mm_unpackhi_epi8(data[t], _mm_set1_epi8(0)); + sum_result = _mm_add_epi32(sum_result, _mm_madd_epi16(vl, _mm_set1_epi16(1))); + sum_result = _mm_add_epi32(sum_result, _mm_madd_epi16(vh, _mm_set1_epi16(1))); + */ + + __m128i vl = _mm_cvtepi8_epi16(data[t]); // sign extend lower words 8->16 + __m128i vh = data[t]; + vh = _mm_srli_si128(vh, 8); // v >>= 64 + vh = _mm_cvtepi8_epi16(vh); // sign extend lower words 8->16 + __m128i sum1 = _mm_add_epi16(vl, vh); + __m128i sumH = _mm_cvtepi16_epi32(sum1); + __m128i sumL = _mm_srli_si128(sum1, 8); // v >>= 64 + sumL = _mm_cvtepi16_epi32(sumL); + sum_result = _mm_add_epi32(sum_result, sumL); + sum_result = _mm_add_epi32(sum_result, sumH); + } + else if (w == 16) { + // todo, can overflow for array size > 2^32 + __m128i vl = _mm_cvtepi16_epi32(data[t]); // sign extend lower words 16->32 + __m128i vh = data[t]; + vh = _mm_srli_si128(vh, 8); // v >>= 64 + vh = _mm_cvtepi16_epi32(vh); // sign extend lower words 16->32 + sum_result = _mm_add_epi32(sum_result, vl); + sum_result = _mm_add_epi32(sum_result, vh); + } + else if (w == 32) { + __m128i v = data[t]; + __m128i v0 = _mm_cvtepi32_epi64(v); // sign extend lower dwords 32->64 + v = _mm_srli_si128(v, 8); // v >>= 64 + __m128i v1 = _mm_cvtepi32_epi64(v); // sign extend lower dwords 32->64 + sum_result = _mm_add_epi64(sum_result, v0); + sum_result = _mm_add_epi64(sum_result, v1); + + /* + __m128i m = _mm_set1_epi32(0xc000); // test if overflow could happen (still need + underflow test). + __m128i mm = _mm_and_si128(data[t], m); + zz = _mm_or_si128(mm, zz); + sum_result = _mm_add_epi32(sum_result, data[t]); + */ + } + } + start += sizeof(__m128i) * 8 / no0(w) * chunks; + + // prevent taking address of 'state' to make the compiler keep it in SSE register in above loop + // (vc2010/gcc4.6) + sum2 = sum_result; + + // Avoid aliasing bug where sum2 might not yet be initialized when accessed by get_universal + char sum3[sizeof sum2]; + memcpy(&sum3, &sum2, sizeof sum2); + + // Sum elements of sum + for (size_t t = 0; t < sizeof(__m128i) * 8 / ((w == 8 || w == 16) ? 32 : 64); ++t) { + int64_t v = get_universal < (w == 8 || w == 16) ? 32 : 64 > (reinterpret_cast(&sum3), t); + s += v; + } + } + } +#endif + + // Sum remaining elements + for (; start < end; ++start) + s += get(*this, start); + + return s; +} + +size_t Array::count(int64_t value) const noexcept +{ + // This is not used anywhere in the code, I believe we can delete this + // since the query logic does not use this + const uint64_t* next = reinterpret_cast(m_data); + size_t value_count = 0; + const size_t end = m_size; + size_t i = 0; + + // static values needed for fast population count + const uint64_t m1 = 0x5555555555555555ULL; + const uint64_t m2 = 0x3333333333333333ULL; + const uint64_t m4 = 0x0f0f0f0f0f0f0f0fULL; + const uint64_t h01 = 0x0101010101010101ULL; + + if (m_width == 0) { + if (value == 0) + return m_size; + return 0; + } + if (m_width == 1) { + if (uint64_t(value) > 1) + return 0; + + const size_t chunkvals = 64; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + if (value == 0) + a = ~a; // reverse + + a -= (a >> 1) & m1; + a = (a & m2) + ((a >> 2) & m2); + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + + // Could use intrinsic instead: + // a = __builtin_popcountll(a); // gcc intrinsic + + value_count += to_size_t(a); + } + } + else if (m_width == 2) { + if (uint64_t(value) > 3) + return 0; + + const uint64_t v = ~0ULL / 0x3 * value; + + // Masks to avoid spillover between segments in cascades + const uint64_t c1 = ~0ULL / 0x3 * 0x1; + + const size_t chunkvals = 32; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + a ^= v; // zero matching bit segments + a |= (a >> 1) & c1; // cascade ones in non-zeroed segments + a &= m1; // isolate single bit in each segment + a ^= m1; // reverse isolated bits + // if (!a) continue; + + // Population count + a = (a & m2) + ((a >> 2) & m2); + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + + value_count += to_size_t(a); + } + } + else if (m_width == 4) { + if (uint64_t(value) > 15) + return 0; + + const uint64_t v = ~0ULL / 0xF * value; + const uint64_t m = ~0ULL / 0xF * 0x1; + + // Masks to avoid spillover between segments in cascades + const uint64_t c1 = ~0ULL / 0xF * 0x7; + const uint64_t c2 = ~0ULL / 0xF * 0x3; + + const size_t chunkvals = 16; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + a ^= v; // zero matching bit segments + a |= (a >> 1) & c1; // cascade ones in non-zeroed segments + a |= (a >> 2) & c2; + a &= m; // isolate single bit in each segment + a ^= m; // reverse isolated bits + + // Population count + a = (a + (a >> 4)) & m4; + a = (a * h01) >> 56; + + value_count += to_size_t(a); + } + } + else if (m_width == 8) { + if (value > 0x7FLL || value < -0x80LL) + return 0; // by casting? + + const uint64_t v = ~0ULL / 0xFF * value; + const uint64_t m = ~0ULL / 0xFF * 0x1; + + // Masks to avoid spillover between segments in cascades + const uint64_t c1 = ~0ULL / 0xFF * 0x7F; + const uint64_t c2 = ~0ULL / 0xFF * 0x3F; + const uint64_t c3 = ~0ULL / 0xFF * 0x0F; + + const size_t chunkvals = 8; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + a ^= v; // zero matching bit segments + a |= (a >> 1) & c1; // cascade ones in non-zeroed segments + a |= (a >> 2) & c2; + a |= (a >> 4) & c3; + a &= m; // isolate single bit in each segment + a ^= m; // reverse isolated bits + + // Population count + a = (a * h01) >> 56; + + value_count += to_size_t(a); + } + } + else if (m_width == 16) { + if (value > 0x7FFFLL || value < -0x8000LL) + return 0; // by casting? + + const uint64_t v = ~0ULL / 0xFFFF * value; + const uint64_t m = ~0ULL / 0xFFFF * 0x1; + + // Masks to avoid spillover between segments in cascades + const uint64_t c1 = ~0ULL / 0xFFFF * 0x7FFF; + const uint64_t c2 = ~0ULL / 0xFFFF * 0x3FFF; + const uint64_t c3 = ~0ULL / 0xFFFF * 0x0FFF; + const uint64_t c4 = ~0ULL / 0xFFFF * 0x00FF; + + const size_t chunkvals = 4; + for (; i + chunkvals <= end; i += chunkvals) { + uint64_t a = next[i / chunkvals]; + a ^= v; // zero matching bit segments + a |= (a >> 1) & c1; // cascade ones in non-zeroed segments + a |= (a >> 2) & c2; + a |= (a >> 4) & c3; + a |= (a >> 8) & c4; + a &= m; // isolate single bit in each segment + a ^= m; // reverse isolated bits + + // Population count + a = (a * h01) >> 56; + + value_count += to_size_t(a); + } + } + else if (m_width == 32) { + int32_t v = int32_t(value); + const int32_t* d = reinterpret_cast(m_data); + for (; i < end; ++i) { + if (d[i] == v) + ++value_count; + } + return value_count; + } + else if (m_width == 64) { + const int64_t* d = reinterpret_cast(m_data); + for (; i < end; ++i) { + if (d[i] == value) + ++value_count; + } + return value_count; + } + + // Check remaining elements + for (; i < end; ++i) + if (value == get(i)) + ++value_count; + + return value_count; +} diff --git a/src/realm/array_blobs_small.cpp b/src/realm/array_blobs_small.cpp index bca4d012a1f..4e93f40c5f4 100644 --- a/src/realm/array_blobs_small.cpp +++ b/src/realm/array_blobs_small.cpp @@ -91,7 +91,8 @@ void ArraySmallBlobs::erase(size_t ndx) REALM_ASSERT_3(ndx, <, m_offsets.size()); size_t start = ndx ? to_size_t(m_offsets.get(ndx - 1)) : 0; - size_t end = to_size_t(m_offsets.get(ndx)); + auto offset = m_offsets.get(ndx); + size_t end = to_size_t(offset); m_blob.erase(start, end); m_offsets.erase(ndx); diff --git a/src/realm/array_blobs_small.hpp b/src/realm/array_blobs_small.hpp index 8db3467a209..e1a08e43e4f 100644 --- a/src/realm/array_blobs_small.hpp +++ b/src/realm/array_blobs_small.hpp @@ -176,7 +176,8 @@ inline BinaryData ArraySmallBlobs::get(size_t ndx) const noexcept } else { size_t begin = ndx ? to_size_t(m_offsets.get(ndx - 1)) : 0; - size_t end = to_size_t(m_offsets.get(ndx)); + auto offset = m_offsets.get(ndx); + size_t end = to_size_t(offset); BinaryData bd = BinaryData(m_blob.get(begin), end - begin); // Old database file (non-nullable column should never return null) diff --git a/src/realm/array_direct.hpp b/src/realm/array_direct.hpp index 5380876700f..4b92141bf55 100644 --- a/src/realm/array_direct.hpp +++ b/src/realm/array_direct.hpp @@ -26,48 +26,48 @@ // clang-format off /* wid == 16/32 likely when accessing offsets in B tree */ #define REALM_TEMPEX(fun, wid, arg) \ - if (wid == 16) {fun<16> arg;} \ - else if (wid == 32) {fun<32> arg;} \ - else if (wid == 0) {fun<0> arg;} \ - else if (wid == 1) {fun<1> arg;} \ - else if (wid == 2) {fun<2> arg;} \ - else if (wid == 4) {fun<4> arg;} \ - else if (wid == 8) {fun<8> arg;} \ - else if (wid == 64) {fun<64> arg;} \ - else {REALM_ASSERT_DEBUG(false); fun<0> arg;} +if (wid == 16) {fun<16> arg;} \ +else if (wid == 32) {fun<32> arg;} \ +else if (wid == 0) {fun<0> arg;} \ +else if (wid == 1) {fun<1> arg;} \ +else if (wid == 2) {fun<2> arg;} \ +else if (wid == 4) {fun<4> arg;} \ +else if (wid == 8) {fun<8> arg;} \ +else if (wid == 64) {fun<64> arg;} \ +else {REALM_ASSERT_DEBUG(false); fun<0> arg;} #define REALM_TEMPEX2(fun, targ, wid, arg) \ - if (wid == 16) {fun arg;} \ - else if (wid == 32) {fun arg;} \ - else if (wid == 0) {fun arg;} \ - else if (wid == 1) {fun arg;} \ - else if (wid == 2) {fun arg;} \ - else if (wid == 4) {fun arg;} \ - else if (wid == 8) {fun arg;} \ - else if (wid == 64) {fun arg;} \ - else {REALM_ASSERT_DEBUG(false); fun arg;} +if (wid == 16) {fun arg;} \ +else if (wid == 32) {fun arg;} \ +else if (wid == 0) {fun arg;} \ +else if (wid == 1) {fun arg;} \ +else if (wid == 2) {fun arg;} \ +else if (wid == 4) {fun arg;} \ +else if (wid == 8) {fun arg;} \ +else if (wid == 64) {fun arg;} \ +else {REALM_ASSERT_DEBUG(false); fun arg;} #define REALM_TEMPEX3(fun, targ1, wid, targ3, arg) \ - if (wid == 16) {fun arg;} \ - else if (wid == 32) {fun arg;} \ - else if (wid == 0) {fun arg;} \ - else if (wid == 1) {fun arg;} \ - else if (wid == 2) {fun arg;} \ - else if (wid == 4) {fun arg;} \ - else if (wid == 8) {fun arg;} \ - else if (wid == 64) {fun arg;} \ - else {REALM_ASSERT_DEBUG(false); fun arg;} +if (wid == 16) {fun arg;} \ +else if (wid == 32) {fun arg;} \ +else if (wid == 0) {fun arg;} \ +else if (wid == 1) {fun arg;} \ +else if (wid == 2) {fun arg;} \ +else if (wid == 4) {fun arg;} \ +else if (wid == 8) {fun arg;} \ +else if (wid == 64) {fun arg;} \ +else {REALM_ASSERT_DEBUG(false); fun arg;} #define REALM_TEMPEX4(fun, targ1, targ3, targ4, wid, arg) \ - if (wid == 16) {fun arg;} \ - else if (wid == 32) {fun arg;} \ - else if (wid == 0) {fun arg;} \ - else if (wid == 1) {fun arg;} \ - else if (wid == 2) {fun arg;} \ - else if (wid == 4) {fun arg;} \ - else if (wid == 8) {fun arg;} \ - else if (wid == 64) {fun arg;} \ - else {REALM_ASSERT_DEBUG(false); fun arg;} +if (wid == 16) {fun arg;} \ +else if (wid == 32) {fun arg;} \ +else if (wid == 0) {fun arg;} \ +else if (wid == 1) {fun arg;} \ +else if (wid == 2) {fun arg;} \ +else if (wid == 4) {fun arg;} \ +else if (wid == 8) {fun arg;} \ +else if (wid == 64) {fun arg;} \ +else {REALM_ASSERT_DEBUG(false); fun arg;} // clang-format on namespace realm { @@ -194,21 +194,22 @@ class UnalignedWordIter { } // 'num_bits' number of bits which must be read // WARNING returned word may be garbage above the first 'num_bits' bits. - uint64_t get(size_t num_bits) + uint64_t consume(size_t num_bits) { auto first_word = m_word_ptr[0]; uint64_t result = first_word >> m_in_word_offset; // note: above shifts in zeroes - if (m_in_word_offset + num_bits <= 64) - return result; - // if we're here, in_word_offset > 0 - auto first_word_size = 64 - m_in_word_offset; - auto second_word = m_word_ptr[1]; - result |= second_word << first_word_size; - // note: above shifts in zeroes below the bits we want + if (m_in_word_offset + num_bits > 64) { + // if we're here, in_word_offset > 0 + auto first_word_size = 64 - m_in_word_offset; + auto second_word = m_word_ptr[1]; + result |= second_word << first_word_size; + // note: above shifts in zeroes below the bits we want + } + _bump(num_bits); return result; } - uint64_t get_with_unsafe_prefetch(size_t num_bits) + uint64_t consume_with_unsafe_prefetch(size_t num_bits) { auto first_word = m_word_ptr[0]; uint64_t result = first_word >> m_in_word_offset; @@ -216,21 +217,24 @@ class UnalignedWordIter { auto first_word_size = 64 - m_in_word_offset; auto second_word = m_word_ptr[1]; REALM_ASSERT_DEBUG(num_bits <= 64); - result |= (m_in_word_offset + num_bits > 64) ? (second_word << first_word_size) : 0; + if (num_bits > first_word_size) + result |= second_word << first_word_size; // note: above shifts in zeroes below the bits we want + _bump(num_bits); return result; } + +private: + const uint64_t* m_word_ptr; + unsigned m_in_word_offset; + // bump the iterator the specified number of bits - void bump(size_t num_bits) + void _bump(size_t num_bits) { auto total_offset = m_in_word_offset + num_bits; m_word_ptr += total_offset >> 6; m_in_word_offset = total_offset & 0x3F; } - -private: - const uint64_t* m_word_ptr; - unsigned m_in_word_offset; }; // Read a bit field of up to 64 bits. @@ -241,16 +245,19 @@ class UnalignedWordIter { // iterator useful for scanning arrays faster than by indexing each element // supports arrays of pairs by differentiating field size and step size. class BfIterator { + friend class FlexCompressor; + friend class PackedCompressor; + public: BfIterator() = default; BfIterator(const BfIterator&) = default; BfIterator(BfIterator&&) = default; BfIterator& operator=(const BfIterator&) = default; BfIterator& operator=(BfIterator&&) = default; - BfIterator(uint64_t* data_area, size_t initial_offset, size_t field_size, size_t step_size, size_t index) + BfIterator(uint64_t* data_area, size_t initial_offset, uint8_t field_size, uint8_t step_size, size_t index) : data_area(data_area) - , field_size(static_cast(field_size)) - , step_size(static_cast(step_size)) + , field_size(field_size) + , step_size(step_size) , offset(initial_offset) { if (field_size < 64) @@ -376,13 +383,13 @@ inline bool operator<(const BfIterator& a, const BfIterator& b) return a.field_position < b.field_position; } -inline uint64_t read_bitfield(uint64_t* data_area, size_t field_position, size_t width) +inline uint64_t read_bitfield(uint64_t* data_area, size_t field_position, uint8_t width) { BfIterator it(data_area, field_position, width, width, 0); return *it; } -inline void write_bitfield(uint64_t* data_area, size_t field_position, size_t width, uint64_t value) +inline void write_bitfield(uint64_t* data_area, size_t field_position, uint8_t width, uint64_t value) { BfIterator it(data_area, field_position, width, width, 0); it.set_value(value); @@ -414,26 +421,26 @@ inline std::pair get_two(const char* data, size_t width, size_ /* Subword parallel search - The following provides facilities for subword parallel search for bitfields of any size. - To simplify, the first bitfield must be aligned within the word: it must occupy the lowest - bits of the word. + The following provides facilities for subword parallel search for bitfields of any size. + To simplify, the first bitfield must be aligned within the word: it must occupy the lowest + bits of the word. - In general the metods here return a vector with the most significant bit in each field - marking that a condition was met when comparing the corresponding pair of fields in two - vectors. Checking if any field meets a condition is as simple as comparing the return - vector against 0. Finding the first to meet a condition is also supported. + In general the metods here return a vector with the most significant bit in each field + marking that a condition was met when comparing the corresponding pair of fields in two + vectors. Checking if any field meets a condition is as simple as comparing the return + vector against 0. Finding the first to meet a condition is also supported. - Vectors are "split" into fields according to a MSB vector, wich indicates the most - significant bit of each field. The MSB must be passed in as an argument to most - bit field comparison functions. It can be generated by the field_sign_bit template. + Vectors are "split" into fields according to a MSB vector, wich indicates the most + significant bit of each field. The MSB must be passed in as an argument to most + bit field comparison functions. It can be generated by the field_sign_bit template. - The simplest condition to test is any_field_NE(A,B), where A and B are words. - This condition should be true if any bitfield in A is not equal to the corresponding - field in B. + The simplest condition to test is any_field_NE(A,B), where A and B are words. + This condition should be true if any bitfield in A is not equal to the corresponding + field in B. - This is almost as simple as a direct word compare, but needs to take into account that - we may want to have part of the words undefined. -*/ + This is almost as simple as a direct word compare, but needs to take into account that + we may want to have part of the words undefined. + */ constexpr uint8_t num_fields_table[65] = {0, 64, 32, 21, 16, 12, 10, 9, // 0-7 8, 7, 6, 5, 5, 4, 4, 4, // 8-15 4, 3, 3, 3, 3, 3, 2, 2, // 16-23 @@ -521,127 +528,6 @@ constexpr uint64_t field_sign_bit(int width) return populate(width, 1ULL << (width - 1)); } -/* Unsigned LT. - - This can be determined by trial subtaction. However, some care must be exercised - since simply subtracting one vector from another will allow carries from one - bitfield to flow into the next one. To avoid this, we isolate bitfields by clamping - the MSBs to 1 in A and 0 in B before subtraction. After the subtraction the MSBs in - the result indicate borrows from the MSB. We then compute overflow (borrow OUT of MSB) - using boolean logic as described below. - - Unsigned LT is also used to find all zero fields or all non-zero fields, so it is - the backbone of all comparisons returning vectors. -*/ - -// compute the overflows in unsigned trial subtraction A-B. The overflows -// will be marked by 1 in the sign bit of each field in the result. Other -// bits in the result are zero. -// Overflow are detected for each field pair where A is less than B. -inline uint64_t unsigned_LT_vector(uint64_t MSBs, uint64_t A, uint64_t B) -{ - // 1. compute borrow from most significant bit - // Isolate bitfields inside A and B before subtraction (prevent carries from spilling over) - // do this by clamping most significant bit in A to 1, and msb in B to 0 - auto A_isolated = A | MSBs; // 1 op - auto B_isolated = B & ~MSBs; // 2 ops - auto borrows_into_sign_bit = ~(A_isolated - B_isolated); // 2 ops (total latency 4) - - // 2. determine what subtraction against most significant bit would give: - // A B borrow-in: (A-B-borrow-in) - // 0 0 0 (0-0-0) = 0 - // 0 0 1 (0-0-1) = 1 + borrow-out - // 0 1 0 (0-1-0) = 1 + borrow-out - // 0 1 1 (0-1-1) = 0 + borrow-out - // 1 0 0 (1-0-0) = 1 - // 1 0 1 (1-0-1) = 0 - // 1 1 0 (1-1-0) = 0 - // 1 1 1 (1-1-1) = 1 + borrow-out - // borrow-out = (~A & B) | (~A & borrow-in) | (A & B & borrow-in) - // The overflows are simply the borrow-out, now encoded into the sign bits of each field. - auto overflows = (~A & B) | (~A & borrows_into_sign_bit) | (A & B & borrows_into_sign_bit); - // ^ 6 ops, total latency 6 (4+2) - return overflows & MSBs; // 1 op, total latency 7 - // total of 12 ops and a latency of 7. On a beefy CPU 3-4 of those can run in parallel - // and still reach a combined latency of 10 or less. -} - -inline uint64_t find_all_fields_unsigned_LT(uint64_t MSBs, uint64_t A, uint64_t B) -{ - return unsigned_LT_vector(MSBs, A, B); -} - -inline uint64_t find_all_fields_NE(uint64_t MSBs, uint64_t A, uint64_t B) -{ - // 0 != A^B, same as asking 0 - (A^B) overflows. - return unsigned_LT_vector(MSBs, 0, A ^ B); -} - -inline uint64_t find_all_fields_EQ(uint64_t MSBs, uint64_t A, uint64_t B) -{ - // get the fields which are EQ and negate the result - auto all_fields_NE = find_all_fields_NE(MSBs, A, B); - auto all_fields_NE_negated = ~all_fields_NE; - // must filter the negated vector so only MSB are left. - return MSBs & all_fields_NE_negated; -} - -inline uint64_t find_all_fields_unsigned_LE(uint64_t MSBs, uint64_t A, uint64_t B) -{ - // Now A <= B is the same as !(A > B) so... - // reverse A and B to turn (A>B) --> (B B is the same as B < A - return find_all_fields_signed_LT(MSBs, B, A); -} - -inline uint64_t find_all_fields_signed_GE(uint64_t MSBs, uint64_t A, uint64_t B) -{ - // A >= B is the same as B <= A - return find_all_fields_signed_LE(MSBs, B, A); -} - constexpr uint32_t inverse_width[65] = { 65536 * 64 / 1, // never used 65536 * 64 / 1, 65536 * 64 / 2, 65536 * 64 / 3, 65536 * 64 / 4, 65536 * 64 / 5, 65536 * 64 / 6, @@ -706,12 +592,10 @@ size_t parallel_subword_find(VectorCompare vector_compare, const uint64_t* data, uint64_t found_vector = 0; while (total_bit_count_left >= fast_scan_limit) { // unrolling 2x - const auto word0 = it.get_with_unsafe_prefetch(bit_count_pr_iteration); - it.bump(bit_count_pr_iteration); - const auto word1 = it.get_with_unsafe_prefetch(bit_count_pr_iteration); + const auto word0 = it.consume_with_unsafe_prefetch(bit_count_pr_iteration); + const auto word1 = it.consume_with_unsafe_prefetch(bit_count_pr_iteration); auto found_vector0 = vector_compare(MSBs, word0, search_vector); auto found_vector1 = vector_compare(MSBs, word1, search_vector); - it.bump(bit_count_pr_iteration); if (found_vector0) { const auto sub_word_index = first_field_marked(width, found_vector0); return start + sub_word_index; @@ -723,8 +607,10 @@ size_t parallel_subword_find(VectorCompare vector_compare, const uint64_t* data, total_bit_count_left -= 2 * bit_count_pr_iteration; start += 2 * field_count; } + + // One word at a time while (total_bit_count_left >= bit_count_pr_iteration) { - const auto word = it.get(bit_count_pr_iteration); + const auto word = it.consume(bit_count_pr_iteration); found_vector = vector_compare(MSBs, word, search_vector); if (found_vector) { const auto sub_word_index = first_field_marked(width, found_vector); @@ -732,10 +618,12 @@ size_t parallel_subword_find(VectorCompare vector_compare, const uint64_t* data, } total_bit_count_left -= bit_count_pr_iteration; start += field_count; - it.bump(bit_count_pr_iteration); } - if (total_bit_count_left) { // final subword, may be partial - const auto word = it.get(total_bit_count_left); // <-- limit lookahead to avoid touching memory beyond array + + // final subword, may be partial + if (total_bit_count_left) { + // limit lookahead to avoid touching memory beyond array + const auto word = it.consume(total_bit_count_left); found_vector = vector_compare(MSBs, word, search_vector); auto last_word_mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - total_bit_count_left); found_vector &= last_word_mask; diff --git a/src/realm/array_integer.cpp b/src/realm/array_integer.cpp index 8cf854c671f..f86871c3225 100644 --- a/src/realm/array_integer.cpp +++ b/src/realm/array_integer.cpp @@ -24,6 +24,12 @@ using namespace realm; +ArrayInteger::ArrayInteger(Allocator& allocator) noexcept + : Array(allocator) +{ + m_is_inner_bptree_node = false; +} + Mixed ArrayInteger::get_any(size_t ndx) const { return Mixed(get(ndx)); @@ -112,7 +118,6 @@ void ArrayIntNull::replace_nulls_with(int64_t new_null) } } - void ArrayIntNull::avoid_null_collision(int64_t value) { if (m_width == 64) { diff --git a/src/realm/array_integer.hpp b/src/realm/array_integer.hpp index 3b50d3757d1..b8739414091 100644 --- a/src/realm/array_integer.hpp +++ b/src/realm/array_integer.hpp @@ -29,16 +29,10 @@ namespace realm { class ArrayInteger : public Array, public ArrayPayload { public: using value_type = int64_t; - - using Array::add; using Array::find_first; - using Array::get; - using Array::insert; - using Array::move; - using Array::set; explicit ArrayInteger(Allocator&) noexcept; - ~ArrayInteger() noexcept override {} + ~ArrayInteger() noexcept override = default; static value_type default_value(bool) { @@ -171,12 +165,6 @@ class ArrayIntNull : public Array, public ArrayPayload { // Implementation: -inline ArrayInteger::ArrayInteger(Allocator& allocator) noexcept - : Array(allocator) -{ - m_is_inner_bptree_node = false; -} - inline ArrayIntNull::ArrayIntNull(Allocator& allocator) noexcept : Array(allocator) { diff --git a/src/realm/array_integer_tpl.hpp b/src/realm/array_integer_tpl.hpp index 9d96584ab3c..0914b1bae65 100644 --- a/src/realm/array_integer_tpl.hpp +++ b/src/realm/array_integer_tpl.hpp @@ -27,9 +27,10 @@ namespace realm { template bool ArrayInteger::find(value_type value, size_t start, size_t end, QueryStateBase* state) const { - return ArrayWithFind(*this).find(value, start, end, 0, state); + return Array::find(value, start, end, 0, state); } + inline bool ArrayIntNull::find_impl(int cond, value_type value, size_t start, size_t end, QueryStateBase* state) const { switch (cond) { @@ -74,9 +75,7 @@ bool ArrayIntNull::find_impl(value_type opt_value, size_t start, size_t end, Que value = *opt_value; } } - - // Fall back to plain Array find. - return ArrayWithFind(*this).find(value, start2, end2, baseindex2, state); + return Array::find(value, start2, end2, baseindex2, state); } else { cond c; diff --git a/src/realm/array_mixed.cpp b/src/realm/array_mixed.cpp index b0542da93b0..7d00991ad5b 100644 --- a/src/realm/array_mixed.cpp +++ b/src/realm/array_mixed.cpp @@ -274,6 +274,7 @@ size_t ArrayMixed::find_first(Mixed value, size_t begin, size_t end) const noexc DataType type = value.get_type(); if (end == realm::npos) end = size(); + for (size_t i = begin; i < end; i++) { if (Mixed::data_types_are_comparable(this->get_type(i), type) && get(i) == value) { return i; diff --git a/src/realm/array_unsigned.cpp b/src/realm/array_unsigned.cpp index e1aac8dbf80..938fe5aece8 100644 --- a/src/realm/array_unsigned.cpp +++ b/src/realm/array_unsigned.cpp @@ -71,6 +71,7 @@ inline uint64_t ArrayUnsigned::_get(size_t ndx, uint8_t width) const return reinterpret_cast(m_data)[ndx]; } return get_direct(m_data, width, ndx); + REALM_UNREACHABLE(); } void ArrayUnsigned::create(size_t initial_size, uint64_t ubound_value) @@ -168,7 +169,8 @@ size_t ArrayUnsigned::upper_bound(uint64_t value) const noexcept void ArrayUnsigned::insert(size_t ndx, uint64_t value) { REALM_ASSERT_DEBUG(m_width >= 8); - bool do_expand = value > m_ubound; + + bool do_expand = value > (uint64_t)m_ubound; const uint8_t old_width = m_width; const uint8_t new_width = do_expand ? bit_width(value) : m_width; const auto old_size = m_size; @@ -215,6 +217,7 @@ void ArrayUnsigned::insert(size_t ndx, uint64_t value) void ArrayUnsigned::erase(size_t ndx) { REALM_ASSERT_DEBUG(m_width >= 8); + copy_on_write(); // Throws size_t w = m_width >> 3; diff --git a/src/realm/array_unsigned.hpp b/src/realm/array_unsigned.hpp index f1926ec7fc0..3e13b35e8dd 100644 --- a/src/realm/array_unsigned.hpp +++ b/src/realm/array_unsigned.hpp @@ -19,7 +19,7 @@ #ifndef REALM_ARRAY_UNSIGNED_HPP #define REALM_ARRAY_UNSIGNED_HPP -#include +#include namespace realm { @@ -81,13 +81,13 @@ class ArrayUnsigned : public Node { } private: - uint_least8_t m_width = 0; // Size of an element (meaning depend on type of array). - uint64_t m_ubound; // max number that can be stored with current m_width + uint_least8_t m_width = 0; + uint64_t m_ubound = 0; // max is 0xFFFFFFFFFFFFFFFFLL void init_from_mem(MemRef mem) noexcept { - Node::init_from_mem(mem); - set_width(get_width_from_header(get_header())); + auto header = Node::init_from_mem(mem); + set_width(get_width_from_header(header)); } void adjust(size_t ndx, int64_t diff) diff --git a/src/realm/array_with_find.cpp b/src/realm/array_with_find.cpp index e33513ef28e..2cf528a5c47 100644 --- a/src/realm/array_with_find.cpp +++ b/src/realm/array_with_find.cpp @@ -34,32 +34,6 @@ void ArrayWithFind::find_all(IntegerColumn* result, int64_t value, size_t col_of return; } - -bool ArrayWithFind::find(int cond, int64_t value, size_t start, size_t end, size_t baseindex, - QueryStateBase* state) const -{ - if (cond == cond_Equal) { - return find(value, start, end, baseindex, state); - } - if (cond == cond_NotEqual) { - return find(value, start, end, baseindex, state); - } - if (cond == cond_Greater) { - return find(value, start, end, baseindex, state); - } - if (cond == cond_Less) { - return find(value, start, end, baseindex, state); - } - if (cond == cond_None) { - return find(value, start, end, baseindex, state); - } - else if (cond == cond_LeftNotNull) { - return find(value, start, end, baseindex, state); - } - REALM_ASSERT_DEBUG(false); - return false; -} - size_t ArrayWithFind::first_set_bit(uint32_t v) const { // (v & -v) is UB when v is INT_MIN @@ -79,5 +53,15 @@ size_t ArrayWithFind::first_set_bit64(int64_t v) const return first_set_bit(v1) + 32; } +bool ArrayWithFind::find_all_will_match(size_t start2, size_t end, size_t baseindex, QueryStateBase* state) const +{ + REALM_ASSERT_DEBUG(state->match_count() < state->limit()); + size_t process = state->limit() - state->match_count(); + size_t end2 = end - start2 > process ? start2 + process : end; + for (; start2 < end2; start2++) + if (!state->match(start2 + baseindex)) + return false; + return true; +} } // namespace realm diff --git a/src/realm/array_with_find.hpp b/src/realm/array_with_find.hpp index 81d86d47e44..b35ed85e808 100644 --- a/src/realm/array_with_find.hpp +++ b/src/realm/array_with_find.hpp @@ -89,8 +89,6 @@ class ArrayWithFind { } // Main finding function - used for find_first, find_all, sum, max, min, etc. - bool find(int cond, int64_t value, size_t start, size_t end, size_t baseindex, QueryStateBase* state) const; - template bool find(int64_t value, size_t start, size_t end, size_t baseindex, QueryStateBase* state) const; @@ -161,7 +159,6 @@ class ArrayWithFind { private: const Array& m_array; - template bool find_all_will_match(size_t start, size_t end, size_t baseindex, QueryStateBase* state) const; }; //************************************************************************************* @@ -276,19 +273,6 @@ uint64_t ArrayWithFind::cascade(uint64_t a) const } } -template -REALM_NOINLINE bool ArrayWithFind::find_all_will_match(size_t start2, size_t end, size_t baseindex, - QueryStateBase* state) const -{ - REALM_ASSERT_DEBUG(state->match_count() < state->limit()); - size_t process = state->limit() - state->match_count(); - size_t end2 = end - start2 > process ? start2 + process : end; - for (; start2 < end2; start2++) - if (!state->match(start2 + baseindex)) - return false; - return true; -} - // This is the main finding function for Array. Other finding functions are just // wrappers around this one. Search for 'value' using condition cond (Equal, // NotEqual, Less, etc) and call QueryStateBase::match() for each match. Break and @@ -318,7 +302,7 @@ bool ArrayWithFind::find_optimized(int64_t value, size_t start, size_t end, size // optimization if all items are guaranteed to match (such as cond == NotEqual && value == 100 && m_ubound == 15) if (c.will_match(value, lbound, ubound)) { - return find_all_will_match(start2, end, baseindex, state); + return find_all_will_match(start2, end, baseindex, state); } // finder cannot handle this bitwidth @@ -567,14 +551,18 @@ inline bool ArrayWithFind::compare_equality(int64_t value, size_t start, size_t QueryStateBase* state) const { REALM_ASSERT_DEBUG(start <= m_array.m_size && (end <= m_array.m_size || end == size_t(-1)) && start <= end); + REALM_ASSERT_DEBUG(width == m_array.m_width); - size_t ee = round_up(start, 64 / no0(width)); + auto v = 64 / no0(width); + size_t ee = round_up(start, v); ee = ee > end ? end : ee; - for (; start < ee; ++start) - if (eq ? (m_array.get(start) == value) : (m_array.get(start) != value)) { + for (; start < ee; ++start) { + auto v = Array::get(m_array, start); + if (eq ? (v == value) : (v != value)) { if (!state->match(start + baseindex)) return false; } + } if (start >= end) return true; @@ -624,7 +612,7 @@ inline bool ArrayWithFind::compare_equality(int64_t value, size_t start, size_t } while (start < end) { - if (eq ? m_array.get(start) == value : m_array.get(start) != value) { + if (eq ? Array::get(m_array, start) == value : Array::get(m_array, start) != value) { if (!state->match(start + baseindex)) { return false; } @@ -903,8 +891,8 @@ bool ArrayWithFind::compare_relation(int64_t value, size_t start, size_t end, si size_t ee = round_up(start, 64 / no0(bitwidth)); ee = ee > end ? end : ee; for (; start < ee; start++) { - if (gt ? (m_array.get(start) > value) : (m_array.get(start) < value)) { - if (!state->match(start + baseindex, m_array.get(start))) + if (gt ? (Array::get(m_array, start) > value) : (Array::get(m_array, start) < value)) { + if (!state->match(start + baseindex, Array::get(m_array, start))) return false; } } @@ -969,7 +957,7 @@ bool ArrayWithFind::compare_relation(int64_t value, size_t start, size_t end, si // Test unaligned end and/or values of width > 16 manually while (start < end) { - if (gt ? m_array.get(start) > value : m_array.get(start) < value) { + if (gt ? Array::get(m_array, start) > value : Array::get(m_array, start) < value) { if (!state->match(start + baseindex)) return false; } diff --git a/src/realm/group.cpp b/src/realm/group.cpp index ab3bef4c68a..eeecbaed4f5 100644 --- a/src/realm/group.cpp +++ b/src/realm/group.cpp @@ -1012,10 +1012,6 @@ ref_type Group::DefaultTableWriter::write_names(_impl::OutputStream& out) } ref_type Group::DefaultTableWriter::write_tables(_impl::OutputStream& out) { - // bool deep = true; // Deep - // bool only_if_modified = false; // Always - // bool compress = false; // true; - // return m_group->m_tables.write(out, deep, only_if_modified, compress); // Throws return m_group->typed_write_tables(out); } @@ -1141,7 +1137,6 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table REALM_ASSERT(version_number == 0 || version_number == 1); } else { - // table_writer.typed_print(""); // Because we need to include the total logical file size in the // top-array, we have to start by writing everything except the // top-array, and then finally compute and write a correct version of @@ -1151,7 +1146,8 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table // DB to compact the database by writing only the live data // into a separate file. ref_type names_ref = table_writer.write_names(out_2); // Throws - ref_type tables_ref = table_writer.write_tables(out_2); // Throws + ref_type tables_ref = table_writer.write_tables(out_2); + SlabAlloc new_alloc; new_alloc.attach_empty(); // Throws Array top(new_alloc); @@ -1214,8 +1210,8 @@ void Group::write(std::ostream& out, int file_format_version, TableWriter& table top.set(2, RefOrTagged::make_tagged(final_file_size)); // Throws // Write the top array - bool deep = false; // Shallow - bool only_if_modified = false; // Always + bool deep = false; // Shallow + bool only_if_modified = false; // Always bool compress = false; top.write(out_2, deep, only_if_modified, compress); // Throws REALM_ASSERT_3(size_t(out_2.get_ref_of_next_array()), ==, final_file_size); diff --git a/src/realm/group.hpp b/src/realm/group.hpp index 434c0258336..08ddd9acd44 100644 --- a/src/realm/group.hpp +++ b/src/realm/group.hpp @@ -1133,6 +1133,7 @@ class Group::TableWriter { { m_group->typed_print(prefix); } + virtual ~TableWriter() noexcept {} void set_group(const Group* g) diff --git a/src/realm/group_writer.cpp b/src/realm/group_writer.cpp index 2990e010d3a..4ce470fec62 100644 --- a/src/realm/group_writer.cpp +++ b/src/realm/group_writer.cpp @@ -41,15 +41,16 @@ class InMemoryWriter : public _impl::ArrayWriterBase { , m_alloc(owner.m_alloc) { } - ref_type write_array(const char* data, size_t size, uint32_t checksum) override + ref_type write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) override { + REALM_ASSERT(checksum_bytes == 4 || checksum_bytes == 2); size_t pos = m_owner.get_free_space(size); // Write the block char* dest_addr = translate(pos); REALM_ASSERT_RELEASE(dest_addr && (reinterpret_cast(dest_addr) & 7) == 0); - memcpy(dest_addr, &checksum, 4); - memcpy(dest_addr + 4, data + 4, size - 4); + memcpy(dest_addr, &checksum, checksum_bytes); + memcpy(dest_addr + checksum_bytes, data + checksum_bytes, size - checksum_bytes); // return ref of the written array ref_type ref = to_ref(pos); return ref; @@ -1339,8 +1340,9 @@ bool inline is_aligned(char* addr) return (as_binary & 7) == 0; } -ref_type GroupWriter::write_array(const char* data, size_t size, uint32_t checksum) +ref_type GroupWriter::write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) { + REALM_ASSERT(checksum_bytes == 4 || checksum_bytes == 2); // Get position of free space to write in (expanding file if needed) size_t pos = get_free_space(size); @@ -1349,8 +1351,8 @@ ref_type GroupWriter::write_array(const char* data, size_t size, uint32_t checks char* dest_addr = window->translate(pos); REALM_ASSERT_RELEASE(is_aligned(dest_addr)); window->encryption_read_barrier(dest_addr, size); - memcpy(dest_addr, &checksum, 4); - memcpy(dest_addr + 4, data + 4, size - 4); + memcpy(dest_addr, &checksum, checksum_bytes); + memcpy(dest_addr + checksum_bytes, data + checksum_bytes, size - checksum_bytes); window->encryption_write_barrier(dest_addr, size); // return ref of the written array ref_type ref = to_ref(pos); diff --git a/src/realm/group_writer.hpp b/src/realm/group_writer.hpp index 438879114c6..b6caed048f6 100644 --- a/src/realm/group_writer.hpp +++ b/src/realm/group_writer.hpp @@ -135,7 +135,7 @@ class GroupWriter : public _impl::ArrayWriterBase { size_t get_file_size() const noexcept; - ref_type write_array(const char*, size_t, uint32_t) override; + ref_type write_array(const char*, size_t, uint32_t, uint32_t) override; #ifdef REALM_DEBUG void dump(); diff --git a/src/realm/impl/array_writer.hpp b/src/realm/impl/array_writer.hpp index 55fd42574bc..4096805e0fa 100644 --- a/src/realm/impl/array_writer.hpp +++ b/src/realm/impl/array_writer.hpp @@ -39,7 +39,7 @@ class ArrayWriterBase { /// /// Returns the ref (position in the target stream) of the written copy of /// the specified array data. - virtual ref_type write_array(const char* data, size_t size, uint32_t checksum) = 0; + virtual ref_type write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) = 0; }; } // namespace _impl diff --git a/src/realm/impl/output_stream.cpp b/src/realm/impl/output_stream.cpp index 04db91235b6..1b0d870aa2f 100644 --- a/src/realm/impl/output_stream.cpp +++ b/src/realm/impl/output_stream.cpp @@ -39,17 +39,18 @@ void OutputStream::write(const char* data, size_t size) } -ref_type OutputStream::write_array(const char* data, size_t size, uint32_t checksum) +ref_type OutputStream::write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) { REALM_ASSERT(size % 8 == 0); + REALM_ASSERT(checksum_bytes == 4 || checksum_bytes == 2); const char* data_1 = data; size_t size_1 = size; const char* cksum_bytes = reinterpret_cast(&checksum); - m_out.write(cksum_bytes, 4); // Throws - data_1 += 4; - size_1 -= 4; + m_out.write(cksum_bytes, checksum_bytes); // Throws + data_1 += checksum_bytes; + size_1 -= checksum_bytes; do_write(data_1, size_1); // Throws diff --git a/src/realm/impl/output_stream.hpp b/src/realm/impl/output_stream.hpp index eb459900485..ba287f92c30 100644 --- a/src/realm/impl/output_stream.hpp +++ b/src/realm/impl/output_stream.hpp @@ -41,7 +41,7 @@ class OutputStream : public ArrayWriterBase { void write(const char* data, size_t size); - ref_type write_array(const char* data, size_t size, uint32_t checksum) override; + ref_type write_array(const char* data, size_t size, uint32_t checksum, uint32_t checksum_bytes) override; private: ref_type m_next_ref; diff --git a/src/realm/integer_compressor.cpp b/src/realm/integer_compressor.cpp new file mode 100644 index 00000000000..5246928e775 --- /dev/null +++ b/src/realm/integer_compressor.cpp @@ -0,0 +1,318 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace realm; + +namespace { + +template +inline void init_compress_array(Array& arr, size_t byte_size, Arg&&... args) +{ + Allocator& allocator = arr.get_alloc(); + auto mem = allocator.alloc(byte_size); + auto h = mem.get_addr(); + T::init_header(h, std::forward(args)...); + NodeHeader::set_capacity_in_header(byte_size, h); + arr.init_from_mem(mem); +} + +} // namespace + +bool IntegerCompressor::always_compress(const Array& origin, Array& arr, NodeHeader::Encoding encoding) const +{ + using Encoding = NodeHeader::Encoding; + std::vector values; + std::vector indices; + compress_values(origin, values, indices); + if (!values.empty()) { + const uint8_t flags = NodeHeader::get_flags(origin.get_header()); + uint8_t v_width = std::max(Node::signed_to_num_bits(values.front()), Node::signed_to_num_bits(values.back())); + + if (encoding == Encoding::Packed) { + const auto packed_size = NodeHeader::calc_size(indices.size(), v_width, NodeHeader::Encoding::Packed); + init_compress_array(arr, packed_size, flags, v_width, origin.size()); + PackedCompressor::copy_data(origin, arr); + } + else if (encoding == Encoding::Flex) { + uint8_t ndx_width = NodeHeader::unsigned_to_num_bits(values.size()); + const auto flex_size = NodeHeader::calc_size(values.size(), indices.size(), v_width, ndx_width); + init_compress_array(arr, flex_size, flags, v_width, ndx_width, values.size(), + indices.size()); + FlexCompressor::copy_data(arr, values, indices); + } + else { + REALM_UNREACHABLE(); + } + return true; + } + return false; +} + +bool IntegerCompressor::compress(const Array& origin, Array& arr) const +{ + if (origin.m_width < 2 || origin.m_size == 0) + return false; + +#if REALM_COMPRESS + return always_compress(origin, arr, NodeHeader::Encoding::Flex); +#else + std::vector values; + std::vector indices; + compress_values(origin, values, indices); + REALM_ASSERT(!values.empty()); + const auto uncompressed_size = origin.get_byte_size(); + uint8_t ndx_width = NodeHeader::unsigned_to_num_bits(values.size()); + uint8_t v_width = std::max(Node::signed_to_num_bits(values.front()), Node::signed_to_num_bits(values.back())); + const auto packed_size = NodeHeader::calc_size(indices.size(), v_width, NodeHeader::Encoding::Packed); + const auto flex_size = NodeHeader::calc_size(values.size(), indices.size(), v_width, ndx_width); + // heuristic: only compress to packed if gain at least 11.1% + const auto adjusted_packed_size = packed_size + packed_size / 8; + // heuristic: only compress to flex if gain at least 20% + const auto adjusted_flex_size = flex_size + flex_size / 4; + if (adjusted_flex_size < adjusted_packed_size && adjusted_flex_size < uncompressed_size) { + const uint8_t flags = NodeHeader::get_flags(origin.get_header()); + init_compress_array(arr, flex_size, flags, v_width, ndx_width, values.size(), indices.size()); + FlexCompressor::copy_data(arr, values, indices); + return true; + } + else if (adjusted_packed_size < uncompressed_size) { + const uint8_t flags = NodeHeader::get_flags(origin.get_header()); + init_compress_array(arr, packed_size, flags, v_width, origin.size()); + PackedCompressor::copy_data(origin, arr); + return true; + } + return false; +#endif +} + +bool IntegerCompressor::decompress(Array& arr) const +{ + int64_t min_v = std::numeric_limits::max(); + int64_t max_v = std::numeric_limits::min(); + REALM_ASSERT_DEBUG(arr.is_attached()); + auto values_fetcher = [&]() { + const auto sz = arr.size(); + if (is_packed()) { + std::vector res; + res.reserve(sz); + for (size_t i = 0; i < sz; ++i) { + auto val = arr.get(i); + if (val > max_v) + max_v = val; + if (val < min_v) + min_v = val; + res.push_back(val); + } + return res; + } + min_v = FlexCompressor::min(*this); + max_v = FlexCompressor::max(*this); + return FlexCompressor::get_all(*this, 0, sz); + }; + const auto& values = values_fetcher(); + // do the reverse of compressing the array + REALM_ASSERT_DEBUG(!values.empty()); + using Encoding = NodeHeader::Encoding; + const auto flags = NodeHeader::get_flags(arr.get_header()); + const auto size = values.size(); + const auto width = std::max(Array::bit_width(min_v), Array::bit_width(max_v)); + REALM_ASSERT_DEBUG(width == 0 || width == 1 || width == 2 || width == 4 || width == 8 || width == 16 || + width == 32 || width == 64); + // 64 is some slab allocator magic number. + // The padding is needed in order to account for bit width expansion. + const auto byte_size = 64 + NodeHeader::calc_size(size, width, Encoding::WTypBits); + REALM_ASSERT_DEBUG(byte_size % 8 == 0); // nevertheless all the values my be aligned to 8 + + // Create new array with the correct width + const auto mem = arr.get_alloc().alloc(byte_size); + const auto header = mem.get_addr(); + init_header(header, Encoding::WTypBits, flags, width, size); + NodeHeader::set_capacity_in_header(byte_size, header); + + // Destroy old array before initializing + arr.destroy(); + arr.init_from_mem(mem); + + // this is copying the bits straight, without doing any COW, since the array is basically restored, we just need + // to copy the data straight back into it. This makes decompressing the array equivalent to copy on write for + // normal arrays, in fact for a compressed array, we skip COW and we just decompress, getting the same result. + auto setter = arr.m_vtable->setter; + for (size_t ndx = 0; ndx < size; ++ndx) + setter(arr, ndx, values[ndx]); + + // very important: since the ref of the current array has changed, the parent must be informed. + // Otherwise we will lose the link between parent array and child array. + arr.update_parent(); + REALM_ASSERT_DEBUG(width == arr.get_width()); + REALM_ASSERT_DEBUG(arr.size() == values.size()); + + return true; +} + +bool IntegerCompressor::init(const char* h) +{ + m_encoding = NodeHeader::get_encoding(h); + // avoid to check wtype here, it is another access to the header, that we can avoid. + // We just need to know if the encoding is packed or flex. + // This makes Array::init_from_mem faster. + if (REALM_LIKELY(!(is_packed() || is_flex()))) + return false; + + if (is_packed()) { + init_packed(h); + } + else { + init_flex(h); + } + return true; +} +int64_t IntegerCompressor::get_packed(const Array& arr, size_t ndx) +{ + return PackedCompressor::get(arr.m_integer_compressor, ndx); +} + +int64_t IntegerCompressor::get_flex(const Array& arr, size_t ndx) +{ + return FlexCompressor::get(arr.m_integer_compressor, ndx); +} + +std::vector IntegerCompressor::get_all_packed(const Array& arr, size_t begin, size_t end) +{ + return PackedCompressor::get_all(arr.m_integer_compressor, begin, end); +} + +std::vector IntegerCompressor::get_all_flex(const Array& arr, size_t begin, size_t end) +{ + return FlexCompressor::get_all(arr.m_integer_compressor, begin, end); +} + +void IntegerCompressor::get_chunk_packed(const Array& arr, size_t ndx, int64_t res[8]) +{ + PackedCompressor::get_chunk(arr.m_integer_compressor, ndx, res); +} + +void IntegerCompressor::get_chunk_flex(const Array& arr, size_t ndx, int64_t res[8]) +{ + FlexCompressor::get_chunk(arr.m_integer_compressor, ndx, res); +} + +void IntegerCompressor::set_packed(Array& arr, size_t ndx, int64_t val) +{ + PackedCompressor::set_direct(arr.m_integer_compressor, ndx, val); +} + +void IntegerCompressor::set_flex(Array& arr, size_t ndx, int64_t val) +{ + FlexCompressor::set_direct(arr.m_integer_compressor, ndx, val); +} + +template +bool IntegerCompressor::find_packed(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, + QueryStateBase* st) +{ + return PackedCompressor::find_all(arr, val, begin, end, base_index, st); +} + +template +bool IntegerCompressor::find_flex(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, + QueryStateBase* st) +{ + return FlexCompressor::find_all(arr, val, begin, end, base_index, st); +} + +void IntegerCompressor::set_vtable(Array& arr) +{ + static const Array::VTable vtable_packed = {get_packed, + get_chunk_packed, + get_all_packed, + set_packed, + { + find_packed, + find_packed, + find_packed, + find_packed, + }}; + static const Array::VTable vtable_flex = {get_flex, + get_chunk_flex, + get_all_flex, + set_flex, + { + find_flex, + find_flex, + find_flex, + find_flex, + }}; + if (is_packed()) { + arr.m_vtable = &vtable_packed; + } + else { + arr.m_vtable = &vtable_flex; + } +} + +int64_t IntegerCompressor::get(size_t ndx) const +{ + if (is_packed()) { + return PackedCompressor::get(*this, ndx); + } + else { + return FlexCompressor::get(*this, ndx); + } +} + +void IntegerCompressor::compress_values(const Array& arr, std::vector& values, + std::vector& indices) const +{ + // The main idea is to compress the values in flex format. If Packed is better it will be chosen by + // IntegerCompressor::compress. The algorithm is O(n lg n), it gives us nice properties, but we could use an + // efficient hash table and try to boost perf during insertion, although leaf arrays are relatively small in + // general (256 entries). The two compresion formats are packed and flex, and the data in the array is re-arranged + // in the following ways (if compressed): + // Packed: || node header || ..... values ..... || + // Flex: || node header || ..... values ..... || ..... indices ..... || + + const auto sz = arr.size(); + REALM_ASSERT_DEBUG(sz > 0); + values.reserve(sz); + indices.reserve(sz); + + for (size_t i = 0; i < sz; ++i) { + auto item = arr.get(i); + values.push_back(item); + } + + std::sort(values.begin(), values.end()); + auto last = std::unique(values.begin(), values.end()); + values.erase(last, values.end()); + + for (size_t i = 0; i < sz; ++i) { + auto pos = std::lower_bound(values.begin(), values.end(), arr.get(i)); + indices.push_back(unsigned(std::distance(values.begin(), pos))); + REALM_ASSERT_DEBUG(values[indices[i]] == arr.get(i)); + } +} diff --git a/src/realm/integer_compressor.hpp b/src/realm/integer_compressor.hpp new file mode 100644 index 00000000000..4e9023cfe18 --- /dev/null +++ b/src/realm/integer_compressor.hpp @@ -0,0 +1,202 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#ifndef REALM_INTEGER_COMPRESSOR_HPP +#define REALM_INTEGER_COMPRESSOR_HPP + +#include +#include +#include +#include +#include +#include + +namespace realm { + +class Array; +class QueryStateBase; +class IntegerCompressor { +public: + // commit => encode, COW/insert => decode + bool compress(const Array&, Array&) const; + bool decompress(Array&) const; + + bool init(const char*); + void set_vtable(Array&); + + // init from mem B + inline uint64_t* data() const; + inline size_t size() const; + inline NodeHeader::Encoding get_encoding() const; + inline uint8_t v_width() const; + inline uint8_t ndx_width() const; + inline size_t v_size() const; + inline size_t ndx_size() const; + + inline uint64_t v_mask() const; + inline uint64_t ndx_mask() const; + inline uint64_t msb() const; + inline uint64_t ndx_msb() const; + inline uint64_t bitmask_v() const; + inline uint64_t bitmask_ndx() const; + + int64_t get(size_t) const; + +private: + // getting and setting interface specifically for encoding formats + inline void init_packed(const char*); + inline void init_flex(const char*); + + static int64_t get_packed(const Array& arr, size_t ndx); + static int64_t get_flex(const Array& arr, size_t ndx); + + static std::vector get_all_packed(const Array& arr, size_t begin, size_t end); + static std::vector get_all_flex(const Array& arr, size_t begin, size_t end); + + static void get_chunk_packed(const Array& arr, size_t ndx, int64_t res[8]); + static void get_chunk_flex(const Array& arr, size_t ndx, int64_t res[8]); + static void set_packed(Array& arr, size_t ndx, int64_t val); + static void set_flex(Array& arr, size_t ndx, int64_t val); + // query interface + template + static bool find_packed(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, + QueryStateBase* st); + template + static bool find_flex(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, + QueryStateBase* st); + + // internal impl + void compress_values(const Array&, std::vector&, std::vector&) const; + inline bool is_packed() const; + inline bool is_flex() const; + + // for testing + bool always_compress(const Array&, Array&, Node::Encoding) const; + +private: + using Encoding = NodeHeader::Encoding; + Encoding m_encoding{NodeHeader::Encoding::WTypBits}; + uint64_t* m_data; + uint8_t m_v_width = 0, m_ndx_width = 0; + size_t m_v_size = 0, m_ndx_size = 0; +}; + +inline void IntegerCompressor::init_packed(const char* h) +{ + m_data = (uint64_t*)NodeHeader::get_data_from_header(h); + m_v_width = NodeHeader::get_element_size(h, Encoding::Packed); + m_v_size = NodeHeader::get_num_elements(h, Encoding::Packed); +} + +inline void IntegerCompressor::init_flex(const char* h) +{ + m_data = (uint64_t*)NodeHeader::get_data_from_header(h); + m_v_width = NodeHeader::get_elementA_size(h); + m_v_size = NodeHeader::get_arrayA_num_elements(h); + m_ndx_width = NodeHeader::get_elementB_size(h); + m_ndx_size = NodeHeader::get_arrayB_num_elements(h); +} + +inline uint64_t* IntegerCompressor::data() const +{ + return m_data; +} + +inline bool IntegerCompressor::is_packed() const +{ + return m_encoding == NodeHeader::Encoding::Packed; +} + +inline bool IntegerCompressor::is_flex() const +{ + return m_encoding == NodeHeader::Encoding::Flex; +} + +inline size_t IntegerCompressor::size() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return m_encoding == NodeHeader::Encoding::Packed ? v_size() : ndx_size(); +} + +inline size_t IntegerCompressor::v_size() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return m_v_size; +} + +inline size_t IntegerCompressor::ndx_size() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return m_ndx_size; +} + +inline uint8_t IntegerCompressor::v_width() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return m_v_width; +} + +inline uint8_t IntegerCompressor::ndx_width() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return m_ndx_width; +} + +inline NodeHeader::Encoding IntegerCompressor::get_encoding() const +{ + return m_encoding; +} + +inline uint64_t IntegerCompressor::v_mask() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return 1ULL << (m_v_width - 1); +} + +inline uint64_t IntegerCompressor::ndx_mask() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return 1ULL << (m_ndx_width - 1); +} + +inline uint64_t IntegerCompressor::msb() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return populate(m_v_width, v_mask()); +} + +inline uint64_t IntegerCompressor::ndx_msb() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return populate(m_ndx_width, ndx_mask()); +} + +inline uint64_t IntegerCompressor::bitmask_v() const +{ + REALM_ASSERT_DEBUG(is_packed() || is_flex()); + return 0xFFFFFFFFFFFFFFFFULL >> (64 - m_v_width); +} + +inline uint64_t IntegerCompressor::bitmask_ndx() const +{ + REALM_ASSERT_DEBUG(is_flex()); + return 0xFFFFFFFFFFFFFFFFULL >> (64 - m_ndx_width); +} + +} // namespace realm +#endif // REALM_INTEGER_COMPRESSOR_HPP diff --git a/src/realm/integer_flex_compressor.cpp b/src/realm/integer_flex_compressor.cpp new file mode 100644 index 00000000000..ef5e3b2fe6f --- /dev/null +++ b/src/realm/integer_flex_compressor.cpp @@ -0,0 +1,79 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#include +#include +#include + +#include +#include + +#ifdef REALM_DEBUG +#include +#include +#endif + +using namespace realm; + +void FlexCompressor::init_header(char* h, uint8_t flags, uint8_t v_width, uint8_t ndx_width, size_t v_size, + size_t ndx_size) +{ + using Encoding = NodeHeader::Encoding; + ::init_header(h, Encoding::Flex, flags, v_width, ndx_width, v_size, ndx_size); +} + +void FlexCompressor::copy_data(const Array& arr, const std::vector& values, + const std::vector& indices) +{ + using Encoding = NodeHeader::Encoding; + REALM_ASSERT_DEBUG(arr.is_attached()); + const auto& compressor = arr.integer_compressor(); + REALM_ASSERT_DEBUG(compressor.get_encoding() == Encoding::Flex); + const auto v_width = compressor.v_width(); + const auto ndx_width = compressor.ndx_width(); + const auto v_size = values.size(); + const auto data = (uint64_t*)arr.m_data; + const auto offset = static_cast(v_size * v_width); + BfIterator it_value{data, 0, v_width, v_width, 0}; + BfIterator it_index{data, offset, ndx_width, ndx_width, 0}; + for (size_t i = 0; i < v_size; ++i) { + it_value.set_value(values[i]); + REALM_ASSERT_DEBUG(sign_extend_value(v_width, it_value.get_value()) == values[i]); + ++it_value; + } + for (size_t i = 0; i < indices.size(); ++i) { + REALM_ASSERT_DEBUG(values[indices[i]] == + sign_extend_value(v_width, read_bitfield(data, indices[i] * v_width, v_width))); + it_index.set_value(indices[i]); + REALM_ASSERT_DEBUG(indices[i] == it_index.get_value()); + REALM_ASSERT_DEBUG(values[indices[i]] == + sign_extend_value(v_width, read_bitfield(data, indices[i] * v_width, v_width))); + ++it_index; + } +} + +bool FlexCompressor::find_all_match(size_t start, size_t end, size_t baseindex, QueryStateBase* state) +{ + REALM_ASSERT_DEBUG(state->match_count() < state->limit()); + const auto process = state->limit() - state->match_count(); + const auto end2 = end - start > process ? start + process : end; + for (; start < end2; start++) + if (!state->match(start + baseindex)) + return false; + return true; +} diff --git a/src/realm/integer_flex_compressor.hpp b/src/realm/integer_flex_compressor.hpp new file mode 100644 index 00000000000..a7338978af8 --- /dev/null +++ b/src/realm/integer_flex_compressor.hpp @@ -0,0 +1,305 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#ifndef FLEX_COMPRESSOR_HPP +#define FLEX_COMPRESSOR_HPP + +#include + +#include +#include +#include + +namespace realm { + +// +// Compress array in Flex format +// Decompress array in WTypeBits formats +// +class FlexCompressor { +public: + // encoding/decoding + static void init_header(char*, uint8_t, uint8_t, uint8_t, size_t, size_t); + static void copy_data(const Array&, const std::vector&, const std::vector&); + // getters/setters + static int64_t get(const IntegerCompressor&, size_t); + static std::vector get_all(const IntegerCompressor&, size_t, size_t); + static void get_chunk(const IntegerCompressor&, size_t, int64_t[8]); + static void set_direct(const IntegerCompressor&, size_t, int64_t); + + template + static bool find_all(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + + static int64_t min(const IntegerCompressor&); + static int64_t max(const IntegerCompressor&); + +private: + static bool find_all_match(size_t, size_t, size_t, QueryStateBase*); + + template + static bool find_linear(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + + template + static bool find_parallel(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + + template + static bool do_find_all(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + + template + static bool run_parallel_subscan(size_t, size_t, size_t); +}; + +inline int64_t FlexCompressor::get(const IntegerCompressor& c, size_t ndx) +{ + const auto offset = c.v_width() * c.v_size(); + const auto ndx_w = c.ndx_width(); + const auto v_w = c.v_width(); + const auto data = c.data(); + BfIterator ndx_iterator{data, offset, ndx_w, ndx_w, ndx}; + BfIterator data_iterator{data, 0, v_w, v_w, static_cast(*ndx_iterator)}; + return sign_extend_field_by_mask(c.v_mask(), *data_iterator); +} + +inline std::vector FlexCompressor::get_all(const IntegerCompressor& c, size_t b, size_t e) +{ + const auto offset = c.v_width() * c.v_size(); + const auto ndx_w = c.ndx_width(); + const auto v_w = c.v_width(); + const auto data = c.data(); + const auto sign_mask = c.v_mask(); + const auto range = (e - b); + const auto starting_bit = offset + b * ndx_w; + const auto bit_per_it = num_bits_for_width(ndx_w); + const auto ndx_mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - ndx_w); + const auto values_per_word = num_fields_for_width(ndx_w); + + // this is very important, x4 faster pre-allocating the array + std::vector res; + res.reserve(range); + + UnalignedWordIter unaligned_ndx_iterator(data, starting_bit); + BfIterator data_iterator{data, 0, v_w, v_w, 0}; + auto remaining_bits = ndx_w * range; + while (remaining_bits >= bit_per_it) { + auto word = unaligned_ndx_iterator.consume(bit_per_it); + for (int i = 0; i < values_per_word; ++i) { + const auto index = word & ndx_mask; + data_iterator.move(static_cast(index)); + const auto sv = sign_extend_field_by_mask(sign_mask, *data_iterator); + res.push_back(sv); + word >>= ndx_w; + } + remaining_bits -= bit_per_it; + } + if (remaining_bits) { + auto last_word = unaligned_ndx_iterator.consume(remaining_bits); + while (remaining_bits) { + const auto index = last_word & ndx_mask; + data_iterator.move(static_cast(index)); + const auto sv = sign_extend_field_by_mask(sign_mask, *data_iterator); + res.push_back(sv); + remaining_bits -= ndx_w; + last_word >>= ndx_w; + } + } + return res; +} + +inline int64_t FlexCompressor::min(const IntegerCompressor& c) +{ + const auto v_w = c.v_width(); + const auto data = c.data(); + const auto sign_mask = c.v_mask(); + BfIterator data_iterator{data, 0, v_w, v_w, 0}; + return sign_extend_field_by_mask(sign_mask, *data_iterator); +} + +inline int64_t FlexCompressor::max(const IntegerCompressor& c) +{ + const auto v_w = c.v_width(); + const auto data = c.data(); + const auto sign_mask = c.v_mask(); + BfIterator data_iterator{data, 0, v_w, v_w, c.v_size() - 1}; + return sign_extend_field_by_mask(sign_mask, *data_iterator); +} + +inline void FlexCompressor::get_chunk(const IntegerCompressor& c, size_t ndx, int64_t res[8]) +{ + auto sz = 8; + std::memset(res, 0, sizeof(int64_t) * sz); + auto supposed_end = ndx + sz; + size_t i = ndx; + size_t index = 0; + for (; i < supposed_end; ++i) { + res[index++] = get(c, i); + } + for (; index < 8; ++index) { + res[index++] = get(c, i++); + } +} + +inline void FlexCompressor::set_direct(const IntegerCompressor& c, size_t ndx, int64_t value) +{ + const auto offset = c.v_width() * c.v_size(); + const auto ndx_w = c.ndx_width(); + const auto v_w = c.v_width(); + const auto data = c.data(); + BfIterator ndx_iterator{data, offset, ndx_w, ndx_w, ndx}; + BfIterator data_iterator{data, 0, v_w, v_w, static_cast(*ndx_iterator)}; + data_iterator.set_value(value); +} + +template +inline bool FlexCompressor::find_all(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) +{ + REALM_ASSERT_DEBUG(start <= arr.m_size && (end <= arr.m_size || end == size_t(-1)) && start <= end); + Cond c; + + if (end == npos) + end = arr.m_size; + + if (!(arr.m_size > start && start < end)) + return true; + + const auto lbound = arr.m_lbound; + const auto ubound = arr.m_ubound; + + if (!c.can_match(value, lbound, ubound)) + return true; + + if (c.will_match(value, lbound, ubound)) { + return find_all_match(start, end, baseindex, state); + } + + REALM_ASSERT_DEBUG(arr.m_width != 0); + + if constexpr (std::is_same_v) { + return do_find_all(arr, value, start, end, baseindex, state); + } + else if constexpr (std::is_same_v) { + return do_find_all(arr, value, start, end, baseindex, state); + } + else if constexpr (std::is_same_v) { + return do_find_all(arr, value, start, end, baseindex, state); + } + else if constexpr (std::is_same_v) { + return do_find_all(arr, value, start, end, baseindex, state); + } + return true; +} + +template +inline bool FlexCompressor::do_find_all(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) +{ + const auto v_width = arr.m_width; + const auto v_range = arr.integer_compressor().v_size(); + const auto ndx_range = end - start; + if (!run_parallel_subscan(v_width, v_range, ndx_range)) + return find_linear(arr, value, start, end, baseindex, state); + return find_parallel(arr, value, start, end, baseindex, state); +} + +template +inline bool FlexCompressor::find_linear(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) +{ + const auto cmp = [](int64_t item, int64_t key) { + if constexpr (std::is_same_v) + return item == key; + if constexpr (std::is_same_v) + return item != key; + if constexpr (std::is_same_v) + return item < key; + if constexpr (std::is_same_v) + return item > key; + REALM_UNREACHABLE(); + }; + + const auto& c = arr.integer_compressor(); + const auto offset = c.v_width() * c.v_size(); + const auto ndx_w = c.ndx_width(); + const auto v_w = c.v_width(); + const auto data = c.data(); + const auto mask = c.v_mask(); + BfIterator ndx_iterator{data, offset, ndx_w, ndx_w, start}; + BfIterator data_iterator{data, 0, v_w, v_w, static_cast(*ndx_iterator)}; + while (start < end) { + const auto sv = sign_extend_field_by_mask(mask, *data_iterator); + if (cmp(sv, value) && !state->match(start + baseindex)) + return false; + ndx_iterator.move(++start); + data_iterator.move(static_cast(*ndx_iterator)); + } + return true; +} + +template +inline bool FlexCompressor::find_parallel(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) +{ + // + // algorithm idea: first try to find in the array of values (should be shorter in size but more bits) using + // VectorCond1. + // Then match the index found in the array of indices using VectorCond2 + // + + const auto& compressor = arr.integer_compressor(); + const auto v_width = compressor.v_width(); + const auto v_size = compressor.v_size(); + const auto ndx_width = compressor.ndx_width(); + const auto offset = v_size * v_width; + uint64_t* data = (uint64_t*)arr.m_data; + + auto MSBs = compressor.msb(); + auto search_vector = populate(v_width, value); + auto v_start = + parallel_subword_find(find_all_fields, data, 0, v_width, MSBs, search_vector, 0, v_size); + + if constexpr (!std::is_same_v) { + if (start == v_size) + return true; + } + + MSBs = compressor.ndx_msb(); + search_vector = populate(ndx_width, v_start); + while (start < end) { + start = parallel_subword_find(find_all_fields_unsigned, data, offset, ndx_width, MSBs, + search_vector, start, end); + + if (start < end && !state->match(start + baseindex)) + return false; + + ++start; + } + return true; +} + +template +inline bool FlexCompressor::run_parallel_subscan(size_t v_width, size_t v_range, size_t ndx_range) +{ + if constexpr (std::is_same_v || std::is_same_v) { + return v_width < 32 && v_range >= 20 && ndx_range >= 20; + } + // > and < need looks slower in parallel scan for large values + return v_width <= 16 && v_range >= 20 && ndx_range >= 20; +} + +} // namespace realm +#endif // FLEX_COMPRESSOR_HPP diff --git a/src/realm/integer_packed_compressor.cpp b/src/realm/integer_packed_compressor.cpp new file mode 100644 index 00000000000..2f7646b1b0c --- /dev/null +++ b/src/realm/integer_packed_compressor.cpp @@ -0,0 +1,68 @@ +/************************************************************************* + * + * Copyright 2023 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#include +#include +#include +#include +#include + +#include +#include + +#ifdef REALM_DEBUG +#include +#include +#endif + +using namespace realm; + +void PackedCompressor::init_header(char* h, uint8_t flags, uint8_t v_width, size_t v_size) +{ + using Encoding = NodeHeader::Encoding; + ::init_header((char*)h, Encoding::Packed, flags, static_cast(v_width), v_size); +} + +void PackedCompressor::copy_data(const Array& origin, Array& arr) +{ + // this can be boosted a little bit, with and size should be known at this stage. + using Encoding = NodeHeader::Encoding; + REALM_ASSERT_DEBUG(arr.is_attached()); + REALM_ASSERT_DEBUG(arr.integer_compressor().get_encoding() == Encoding::Packed); + // we don't need to access the header, init from mem must have been called + const auto v_width = arr.m_width; + const auto v_size = arr.m_size; + auto data = (uint64_t*)arr.m_data; + BfIterator it_value{data, 0, v_width, v_width, 0}; + for (size_t i = 0; i < v_size; ++i) { + it_value.set_value(origin.get(i)); + REALM_ASSERT_DEBUG(sign_extend_value(v_width, it_value.get_value()) == origin.get(i)); + ++it_value; + } +} + +bool PackedCompressor::find_all_match(size_t start, size_t end, size_t baseindex, QueryStateBase* state) +{ + REALM_ASSERT_DEBUG(state->match_count() < state->limit()); + const auto process = state->limit() - state->match_count(); + const auto end2 = end - start > process ? start + process : end; + for (; start < end2; start++) + if (!state->match(start + baseindex)) + return false; + return true; +} diff --git a/src/realm/integer_packed_compressor.hpp b/src/realm/integer_packed_compressor.hpp new file mode 100644 index 00000000000..91d94fc5eab --- /dev/null +++ b/src/realm/integer_packed_compressor.hpp @@ -0,0 +1,229 @@ +/************************************************************************* + * + * Copyright 2024 Realm Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + **************************************************************************/ + +#ifndef PACKED_COMPRESSOR_HPP +#define PACKED_COMPRESSOR_HPP + +#include +#include + +#include +#include + +namespace realm { + +// +// Compress array in Packed format +// Decompress array in WTypeBits formats +// +class PackedCompressor { +public: + // encoding/decoding + static void init_header(char*, uint8_t, uint8_t, size_t); + static void copy_data(const Array&, Array&); + // get or set + static int64_t get(const IntegerCompressor&, size_t); + static std::vector get_all(const IntegerCompressor& c, size_t b, size_t e); + static void get_chunk(const IntegerCompressor&, size_t, int64_t res[8]); + static void set_direct(const IntegerCompressor&, size_t, int64_t); + + template + static bool find_all(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + +private: + static bool find_all_match(size_t start, size_t end, size_t baseindex, QueryStateBase* state); + + template + static bool find_parallel(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + + template + static bool find_linear(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); + + template + static bool run_parallel_scan(size_t, size_t); +}; + +inline int64_t PackedCompressor::get(const IntegerCompressor& c, size_t ndx) +{ + BfIterator it{c.data(), 0, c.v_width(), c.v_width(), ndx}; + return sign_extend_field_by_mask(c.v_mask(), *it); +} + +inline std::vector PackedCompressor::get_all(const IntegerCompressor& c, size_t b, size_t e) +{ + const auto range = (e - b); + const auto v_w = c.v_width(); + const auto data = c.data(); + const auto sign_mask = c.v_mask(); + const auto starting_bit = b * v_w; + const auto total_bits = starting_bit + (v_w * range); + const auto mask = 0xFFFFFFFFFFFFFFFFULL >> (64 - v_w); + const auto bit_per_it = num_bits_for_width(v_w); + const auto values_per_word = num_fields_for_width(v_w); + + std::vector res; + res.reserve(range); + + UnalignedWordIter unaligned_data_iterator(data, starting_bit); + auto cnt_bits = starting_bit; + while (cnt_bits + bit_per_it < total_bits) { + auto word = unaligned_data_iterator.consume(bit_per_it); + for (int i = 0; i < values_per_word; ++i) { + res.push_back(sign_extend_field_by_mask(sign_mask, word & mask)); + word >>= v_w; + } + cnt_bits += bit_per_it; + } + if (cnt_bits < total_bits) { + auto last_word = unaligned_data_iterator.consume(static_cast(total_bits - cnt_bits)); + while (cnt_bits < total_bits) { + res.push_back(sign_extend_field_by_mask(sign_mask, last_word & mask)); + cnt_bits += v_w; + last_word >>= v_w; + } + } + return res; +} + +inline void PackedCompressor::set_direct(const IntegerCompressor& c, size_t ndx, int64_t value) +{ + BfIterator it{c.data(), 0, c.v_width(), c.v_width(), ndx}; + it.set_value(value); +} + +inline void PackedCompressor::get_chunk(const IntegerCompressor& c, size_t ndx, int64_t res[8]) +{ + auto sz = 8; + std::memset(res, 0, sizeof(int64_t) * sz); + auto supposed_end = ndx + sz; + size_t i = ndx; + size_t index = 0; + // this can be done better, in one go, retrieve both!!! + for (; i < supposed_end; ++i) { + res[index++] = get(c, i); + } + for (; index < 8; ++index) { + res[index++] = get(c, i++); + } +} + + +template +inline bool PackedCompressor::find_all(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) +{ + REALM_ASSERT_DEBUG(start <= arr.m_size && (end <= arr.m_size || end == size_t(-1)) && start <= end); + Cond c; + + if (end == npos) + end = arr.m_size; + + if (!(arr.m_size > start && start < end)) + return true; + + const auto lbound = arr.m_lbound; + const auto ubound = arr.m_ubound; + + if (!c.can_match(value, lbound, ubound)) + return true; + + if (c.will_match(value, lbound, ubound)) { + return find_all_match(start, end, baseindex, state); + } + + REALM_ASSERT_DEBUG(arr.m_width != 0); + + if (!run_parallel_scan(arr.m_width, end - start)) + return find_linear(arr, value, start, end, baseindex, state); + + return find_parallel(arr, value, start, end, baseindex, state); +} + +template +inline bool PackedCompressor::find_parallel(const Array& arr, int64_t value, size_t start, size_t end, + size_t baseindex, QueryStateBase* state) +{ + // + // Main idea around find parallel (applicable to flex arrays too). + // Try to find the starting point where the condition can be met, comparing as many values as a single 64bit can + // contain in parallel. Once we have found the starting point, keep matching values as much as we can between + // start and end. + // + // EG: let's store 6, it gets stored in 4 bits (0110). 6 is 4 bits because 110 (6) + sign bit 0. + // Inside 64bits we can fit max 16 times 6. If we go from index 0 to 15 throughout the same 64 bits, we need to + // apply a mask and a shift bits every time, then compare the extracted values. + // This is not the cheapest thing to do. Instead we can compare all values contained within 64 bits in one go, and + // see if there is a match with what we are looking for. Reducing the number of comparison by ~logk(N) where K is + // the width of each single value within a 64 bit word and N is the total number of values stored in the array. + + const auto data = (const uint64_t*)arr.m_data; + const auto width = arr.m_width; + const auto MSBs = arr.integer_compressor().msb(); + const auto search_vector = populate(arr.m_width, value); + while (start < end) { + start = parallel_subword_find(find_all_fields, data, 0, width, MSBs, search_vector, start, end); + if (start < end && !state->match(start + baseindex)) + return false; + ++start; + } + return true; +} + +template +inline bool PackedCompressor::find_linear(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, + QueryStateBase* state) +{ + auto compare = [](int64_t a, int64_t b) { + if constexpr (std::is_same_v) + return a == b; + if constexpr (std::is_same_v) + return a != b; + if constexpr (std::is_same_v) + return a > b; + if constexpr (std::is_same_v) + return a < b; + }; + const auto& c = arr.integer_compressor(); + BfIterator it{c.data(), 0, c.v_width(), c.v_width(), start}; + for (; start < end; ++start) { + it.move(start); + const auto sv = sign_extend_field_by_mask(c.v_mask(), *it); + if (compare(sv, value) && !state->match(start + baseindex)) + return false; + } + return true; +} + +template +inline bool PackedCompressor::run_parallel_scan(size_t width, size_t range) +{ + if constexpr (std::is_same_v) { + // we seem to be particularly slow doing parallel scan in packed for NotEqual. + // we are much better with a linear scan. TODO: investigate this. + return false; + } + if constexpr (std::is_same_v) { + return width < 32 && range >= 20; + } + // > and < need a different heuristic + return width <= 20 && range >= 20; +} + +} // namespace realm + +#endif // PACKED_COMPRESSOR_HPP diff --git a/src/realm/node.cpp b/src/realm/node.cpp index f23cff4316b..63ef4d3962c 100644 --- a/src/realm/node.cpp +++ b/src/realm/node.cpp @@ -26,7 +26,8 @@ using namespace realm; -MemRef Node::create_node(size_t size, Allocator& alloc, bool context_flag, Type type, WidthType width_type, int width) +MemRef Node::create_node(size_t size, Allocator& alloc, bool context_flag, Type type, WidthType width_type, + uint8_t width) { size_t byte_size_0 = calc_byte_size(width_type, size, width); size_t byte_size = std::max(byte_size_0, size_t(initial_capacity)); @@ -81,9 +82,9 @@ size_t Node::calc_item_count(size_t bytes, size_t width) const noexcept void Node::alloc(size_t init_size, size_t new_width) { - REALM_ASSERT(is_attached()); + REALM_ASSERT_DEBUG(is_attached()); char* header = get_header_from_data(m_data); - REALM_ASSERT(!wtype_is_extended(header)); + REALM_ASSERT_DEBUG(!wtype_is_extended(header)); size_t needed_bytes = calc_byte_len(init_size, new_width); // this method is not public and callers must (and currently do) ensure that // needed_bytes are never larger than max_array_payload. @@ -132,7 +133,7 @@ void Node::alloc(size_t init_size, size_t new_width) } // update width (important when we convert from normal uncompressed array into compressed format) if (new_width != orig_width) { - set_width_in_header(int(new_width), header); + set_width_in_header(new_width, header); } set_size_in_header(init_size, header); m_size = init_size; diff --git a/src/realm/node.hpp b/src/realm/node.hpp index 5cb637ab7d1..8a4b862a701 100644 --- a/src/realm/node.hpp +++ b/src/realm/node.hpp @@ -323,7 +323,7 @@ class Node : public NodeHeader { } static MemRef create_node(size_t size, Allocator& alloc, bool context_flag = false, Type type = type_Normal, - WidthType width_type = wtype_Ignore, int width = 1); + WidthType width_type = wtype_Ignore, uint8_t width = 1); void set_header_size(size_t value) noexcept { diff --git a/src/realm/node_header.hpp b/src/realm/node_header.hpp index 2ffe073b721..ca7d5638025 100644 --- a/src/realm/node_header.hpp +++ b/src/realm/node_header.hpp @@ -205,7 +205,7 @@ class NodeHeader { h[4] = h4; } - static size_t unsigned_to_num_bits(uint64_t value) + static uint8_t unsigned_to_num_bits(uint64_t value) { if constexpr (sizeof(size_t) == sizeof(uint64_t)) return 1 + log2(static_cast(value)); @@ -218,7 +218,7 @@ class NodeHeader { return 0; } - static inline size_t signed_to_num_bits(int64_t value) + static inline uint8_t signed_to_num_bits(int64_t value) { if (value >= 0) return 1 + unsigned_to_num_bits(value); @@ -292,7 +292,6 @@ class NodeHeader { (reinterpret_cast(header))[0] = static_cast(value >> 3); } } - static size_t get_byte_size_from_header(const char* header) noexcept; // ^ First 3 must overlap numerically with corresponding wtype_X enum. @@ -343,17 +342,18 @@ class NodeHeader { private: friend class Node; + friend class IntegerCompressor; // Setting element size for encodings with a single element size: - static void inline set_element_size(char* header, size_t bits_per_element, Encoding); + static void inline set_element_size(char* header, uint8_t bits_per_element, Encoding); // Getting element size for encodings with a single element size: - static inline size_t get_element_size(const char* header, Encoding); + static inline uint8_t get_element_size(const char* header, Encoding); // Used only by flex at this stage. // Setting element sizes for encodings with two element sizes (called A and B) - static inline void set_elementA_size(char* header, size_t bits_per_element); - static inline void set_elementB_size(char* header, size_t bits_per_element); + static inline void set_elementA_size(char* header, uint8_t bits_per_element); + static inline void set_elementB_size(char* header, uint8_t bits_per_element); // Getting element sizes for encodings with two element sizes (called A and B) - static inline size_t get_elementA_size(const char* header); - static inline size_t get_elementB_size(const char* header); + static inline uint8_t get_elementA_size(const char* header); + static inline uint8_t get_elementB_size(const char* header); // Setting num of elements for encodings with two element sizes (called A and B) static inline void set_arrayA_num_elements(char* header, size_t num_elements); static inline void set_arrayB_num_elements(char* header, size_t num_elements); @@ -366,9 +366,9 @@ class NodeHeader { static inline void set_num_elements(char* header, size_t num_elements, Encoding); static inline size_t calc_size(size_t num_elements); - static inline size_t calc_size(size_t num_elements, size_t element_size, Encoding); - static inline size_t calc_size(size_t arrayA_num_elements, size_t arrayB_num_elements, size_t elementA_size, - size_t elementB_size); + static inline size_t calc_size(size_t num_elements, uint8_t element_size, Encoding); + static inline size_t calc_size(size_t arrayA_num_elements, size_t arrayB_num_elements, uint8_t elementA_size, + uint8_t elementB_size); static size_t calc_byte_size(WidthType wtype, size_t size, uint_least8_t width) noexcept { @@ -441,7 +441,7 @@ class NodeHeader { } }; -inline void NodeHeader::set_element_size(char* header, size_t bits_per_element, Encoding encoding) +inline void NodeHeader::set_element_size(char* header, uint8_t bits_per_element, Encoding encoding) { switch (encoding) { case NodeHeader::Encoding::Packed: { @@ -469,7 +469,7 @@ inline void NodeHeader::set_element_size(char* header, size_t bits_per_element, } } -inline size_t NodeHeader::get_element_size(const char* header, Encoding encoding) +inline uint8_t NodeHeader::get_element_size(const char* header, Encoding encoding) { switch (encoding) { case NodeHeader::Encoding::Packed: { @@ -496,7 +496,7 @@ inline size_t NodeHeader::get_element_size(const char* header, Encoding encoding } } -inline void NodeHeader::set_elementA_size(char* header, size_t bits_per_element) +inline void NodeHeader::set_elementA_size(char* header, uint8_t bits_per_element) { // we're a bit low on bits for the Flex encoding, so we need to squeeze stuff REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); @@ -509,7 +509,7 @@ inline void NodeHeader::set_elementA_size(char* header, size_t bits_per_element) (reinterpret_cast(header))[1] = word; } -inline void NodeHeader::set_elementB_size(char* header, size_t bits_per_element) +inline void NodeHeader::set_elementB_size(char* header, uint8_t bits_per_element) { // we're a bit low on bits for the Flex encoding, so we need to squeeze stuff REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); @@ -522,7 +522,7 @@ inline void NodeHeader::set_elementB_size(char* header, size_t bits_per_element) (reinterpret_cast(header))[3] = word; } -inline size_t NodeHeader::get_elementA_size(const char* header) +inline uint8_t NodeHeader::get_elementA_size(const char* header) { const auto encoding = get_encoding(header); REALM_ASSERT_DEBUG(encoding == Encoding::Flex); @@ -536,7 +536,7 @@ inline size_t NodeHeader::get_elementA_size(const char* header) return bits_per_element; } -inline size_t NodeHeader::get_elementB_size(const char* header) +inline uint8_t NodeHeader::get_elementB_size(const char* header) { REALM_ASSERT_DEBUG(get_encoding(header) == Encoding::Flex); uint16_t word = (reinterpret_cast(header))[3]; @@ -643,7 +643,7 @@ inline size_t NodeHeader::calc_size(size_t num_elements) return calc_byte_size(wtype_Ignore, num_elements, 0); } -inline size_t NodeHeader::calc_size(size_t num_elements, size_t element_size, Encoding encoding) +inline size_t NodeHeader::calc_size(size_t num_elements, uint8_t element_size, Encoding encoding) { using Encoding = NodeHeader::Encoding; switch (encoding) { @@ -660,8 +660,8 @@ inline size_t NodeHeader::calc_size(size_t num_elements, size_t element_size, En } } -inline size_t NodeHeader::calc_size(size_t arrayA_num_elements, size_t arrayB_num_elements, size_t elementA_size, - size_t elementB_size) +inline size_t NodeHeader::calc_size(size_t arrayA_num_elements, size_t arrayB_num_elements, uint8_t elementA_size, + uint8_t elementB_size) { return NodeHeader::header_size + align_bits_to8(arrayA_num_elements * elementA_size + arrayB_num_elements * elementB_size); @@ -757,6 +757,7 @@ static inline void init_header(char* header, realm::NodeHeader::Encoding enc, ui REALM_ASSERT_DEBUG(num_elemsB < 1024); hw[1] = static_cast(((bits_pr_elemA - 1) << 10) | num_elemsA); hw[3] = static_cast(((bits_pr_elemB - 1) << 10) | num_elemsB); + REALM_ASSERT_DEBUG(realm::NodeHeader::get_encoding(header) == realm::NodeHeader::Encoding::Flex); } } // namespace diff --git a/src/realm/obj.cpp b/src/realm/obj.cpp index 8a1267029b9..eb8138dd8f5 100644 --- a/src/realm/obj.cpp +++ b/src/realm/obj.cpp @@ -549,12 +549,9 @@ int64_t Obj::_get(ColKey::Idx col_ndx) const if (current_version != m_storage_version) { update(); } - ref_type ref = to_ref(Array::get(m_mem.get_addr(), col_ndx.val + 1)); char* header = alloc.translate(ref); - int width = Array::get_width_from_header(header); - char* data = Array::get_data_from_header(header); - REALM_TEMPEX(return get_direct, width, (data, m_row_ndx)); + return Array::get(header, m_row_ndx); } template <> diff --git a/src/realm/query_conditions.hpp b/src/realm/query_conditions.hpp index cf3cf9e73d8..ea16fb4a736 100644 --- a/src/realm/query_conditions.hpp +++ b/src/realm/query_conditions.hpp @@ -1002,6 +1002,155 @@ struct GreaterEqual : public HackClass { static const int condition = -1; }; +/* Unsigned LT. + + This can be determined by trial subtaction. However, some care must be exercised + since simply subtracting one vector from another will allow carries from one + bitfield to flow into the next one. To avoid this, we isolate bitfields by clamping + the MSBs to 1 in A and 0 in B before subtraction. After the subtraction the MSBs in + the result indicate borrows from the MSB. We then compute overflow (borrow OUT of MSB) + using boolean logic as described below. + + Unsigned LT is also used to find all zero fields or all non-zero fields, so it is + the backbone of all comparisons returning vectors. + */ + +// compute the overflows in unsigned trial subtraction A-B. The overflows +// will be marked by 1 in the sign bit of each field in the result. Other +// bits in the result are zero. +// Overflow are detected for each field pair where A is less than B. +inline uint64_t unsigned_LT_vector(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // 1. compute borrow from most significant bit + // Isolate bitfields inside A and B before subtraction (prevent carries from spilling over) + // do this by clamping most significant bit in A to 1, and msb in B to 0 + auto A_isolated = A | MSBs; // 1 op + auto B_isolated = B & ~MSBs; // 2 ops + auto borrows_into_sign_bit = ~(A_isolated - B_isolated); // 2 ops (total latency 4) + + // 2. determine what subtraction against most significant bit would give: + // A B borrow-in: (A-B-borrow-in) + // 0 0 0 (0-0-0) = 0 + // 0 0 1 (0-0-1) = 1 + borrow-out + // 0 1 0 (0-1-0) = 1 + borrow-out + // 0 1 1 (0-1-1) = 0 + borrow-out + // 1 0 0 (1-0-0) = 1 + // 1 0 1 (1-0-1) = 0 + // 1 1 0 (1-1-0) = 0 + // 1 1 1 (1-1-1) = 1 + borrow-out + // borrow-out = (~A & B) | (~A & borrow-in) | (A & B & borrow-in) + // The overflows are simply the borrow-out, now encoded into the sign bits of each field. + auto overflows = (~A & B) | (~A & borrows_into_sign_bit) | (A & B & borrows_into_sign_bit); + // ^ 6 ops, total latency 6 (4+2) + return overflows & MSBs; // 1 op, total latency 7 + // total of 12 ops and a latency of 7. On a beefy CPU 3-4 of those can run in parallel + // and still reach a combined latency of 10 or less. +} + +template +uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B); + +template +uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B); + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // 0 != A^B, same as asking 0 - (A^B) overflows. + return unsigned_LT_vector(MSBs, 0, A ^ B); +} + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // get the fields which are EQ and negate the result + auto all_fields_NE = find_all_fields(MSBs, A, B); + auto all_fields_NE_negated = ~all_fields_NE; + // must filter the negated vector so only MSB are left. + return MSBs & all_fields_NE_negated; +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return find_all_fields(MSBs, A, B); +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return find_all_fields(MSBs, A, B); +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return unsigned_LT_vector(MSBs, A, B); +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // Now A <= B is the same as !(A > B) so... + // reverse A and B to turn (A>B) --> (B +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return find_all_fields_unsigned(MSBs, B, A); +} + +template <> +inline uint64_t find_all_fields_unsigned(uint64_t MSBs, uint64_t A, uint64_t B) +{ + return find_all_fields_unsigned(MSBs, B, A); +} + +/* + Handling signed values + + Trial subtraction only works as is for unsigned. We simply transform signed into unsigned + by pusing all values up by 1<<(field_width-1). This makes all negative values positive and positive + values remain positive, although larger. Any overflow during the push can be ignored. + After that transformation Trial subtraction should correctly detect the LT condition. + + */ + + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + auto sign_bits = MSBs; + return unsigned_LT_vector(MSBs, A ^ sign_bits, B ^ sign_bits); +} + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + auto sign_bits = MSBs; + return find_all_fields_unsigned(MSBs, A ^ sign_bits, B ^ sign_bits); +} + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // A > B is the same as B < A + return find_all_fields(MSBs, B, A); +} + +template <> +inline uint64_t find_all_fields(uint64_t MSBs, uint64_t A, uint64_t B) +{ + // A >= B is the same as B <= A + return find_all_fields(MSBs, B, A); +} + } // namespace realm #endif // REALM_QUERY_CONDITIONS_HPP diff --git a/src/realm/query_engine.hpp b/src/realm/query_engine.hpp index 8b7ecf2d1e8..26a07377536 100644 --- a/src/realm/query_engine.hpp +++ b/src/realm/query_engine.hpp @@ -449,6 +449,7 @@ static size_t find_first_haystack(LeafType& leaf, NeedleContainer& needles, size { // for a small number of conditions, it is faster to do a linear search than to compute the hash // the exact thresholds were found experimentally + if (needles.size() < linear_search_threshold) { for (size_t i = start; i < end; ++i) { auto element = leaf.get(i); diff --git a/src/realm/query_state.hpp b/src/realm/query_state.hpp index b2812276539..ac0480d7166 100644 --- a/src/realm/query_state.hpp +++ b/src/realm/query_state.hpp @@ -22,8 +22,6 @@ #include // size_t #include // unint8_t etc -#include - namespace realm { enum Action { act_ReturnFirst, act_Sum, act_Max, act_Min, act_Count, act_FindAll, act_Average }; @@ -34,6 +32,7 @@ enum { cond_Equal, cond_NotEqual, cond_Greater, cond_Less, cond_VTABLE_FINDER_CO class ArrayUnsigned; class Mixed; +class ArrayPayload; class QueryStateBase { public: diff --git a/src/realm/table.hpp b/src/realm/table.hpp index 3709669400d..0830d7c733f 100644 --- a/src/realm/table.hpp +++ b/src/realm/table.hpp @@ -544,6 +544,10 @@ class Table { return false; } + ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, bool deep, bool only_modified, + bool compress) const; + void typed_print(std::string prefix, ref_type ref) const; + private: template TableView find_all(ColKey col_key, T value); @@ -689,7 +693,6 @@ class Table { }; ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const; - void typed_print(std::string prefix, ref_type ref) const; private: enum LifeCycleCookie { diff --git a/test/benchmark-common-tasks/main.cpp b/test/benchmark-common-tasks/main.cpp index 5333e464dfc..b837834796b 100644 --- a/test/benchmark-common-tasks/main.cpp +++ b/test/benchmark-common-tasks/main.cpp @@ -1413,7 +1413,6 @@ struct BenchmarkQueryChainedOrIntsIndexed : BenchmarkQueryChainedOrInts { } }; - struct BenchmarkQueryIntEquality : BenchmarkQueryChainedOrInts { const char* name() const { diff --git a/test/object-store/results.cpp b/test/object-store/results.cpp index 5815d258e84..3fedacfeaec 100644 --- a/test/object-store/results.cpp +++ b/test/object-store/results.cpp @@ -103,7 +103,6 @@ struct TestContext : CppContext { } }; - TEST_CASE("notifications: async delivery", "[notifications]") { _impl::RealmCoordinator::assert_no_open_realms(); TestFile config; diff --git a/test/test_array.cpp b/test/test_array.cpp index 8a86ac15718..a77c698b7fa 100644 --- a/test/test_array.cpp +++ b/test/test_array.cpp @@ -96,6 +96,27 @@ void has_zero_byte(TestContext& test_context, int64_t value, size_t reps) } // anonymous namespace +TEST(Array_Bits) +{ + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(0), 0); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(1), 1); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(2), 2); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(3), 2); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(4), 3); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(5), 3); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(7), 3); + CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(8), 4); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(0), 1); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(1), 2); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-1), 1); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-2), 2); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-3), 3); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(-4), 3); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(3), 3); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(4), 4); + CHECK_EQUAL(NodeHeader::signed_to_num_bits(7), 4); +} + TEST(Array_General) { Array c(Allocator::get_default()); @@ -1560,25 +1581,56 @@ NONCONCURRENT_TEST(Array_count) c.destroy(); } -TEST(Array_Bits) +TEST(DirectBitFields) { - CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(0), 0); - CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(1), 1); - CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(2), 2); - CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(3), 2); - CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(4), 3); - CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(5), 3); - CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(7), 3); - CHECK_EQUAL(NodeHeader::unsigned_to_num_bits(8), 4); - CHECK_EQUAL(NodeHeader::signed_to_num_bits(0), 1); - CHECK_EQUAL(NodeHeader::signed_to_num_bits(1), 2); - CHECK_EQUAL(NodeHeader::signed_to_num_bits(-1), 1); - CHECK_EQUAL(NodeHeader::signed_to_num_bits(-2), 2); - CHECK_EQUAL(NodeHeader::signed_to_num_bits(-3), 3); - CHECK_EQUAL(NodeHeader::signed_to_num_bits(-4), 3); - CHECK_EQUAL(NodeHeader::signed_to_num_bits(3), 3); - CHECK_EQUAL(NodeHeader::signed_to_num_bits(4), 4); - CHECK_EQUAL(NodeHeader::signed_to_num_bits(7), 4); + uint64_t a[2]; + a[0] = a[1] = 0; + { + BfIterator it(a, 0, 7, 7, 8); + REALM_ASSERT(*it == 0); + auto it2(it); + ++it2; + it2.set_value(127 + 128); + REALM_ASSERT(*it == 0); + ++it; + REALM_ASSERT(*it == 127); + ++it; + REALM_ASSERT(*it == 0); + } + // reverse polarity + a[0] = a[1] = -1ULL; + { + BfIterator it(a, 0, 7, 7, 8); + REALM_ASSERT(*it == 127); + auto it2(it); + ++it2; + it2.set_value(42 + 128); + REALM_ASSERT(*it == 127); + ++it; + REALM_ASSERT(*it == 42); + ++it; + REALM_ASSERT(*it == 127); + } +} + +TEST(Extended_Array_encoding) +{ + using Encoding = NodeHeader::Encoding; + Array array(Allocator::get_default()); + auto mem = array.get_alloc().alloc(10); + init_header(mem.get_addr(), Encoding::Flex, 7, 1, 1, 1, 1); + array.init_from_mem(mem); + auto array_header = array.get_header(); + auto encoding = array.get_encoding(array_header); + CHECK(encoding == Encoding::Flex); + + Array another_array(Allocator::get_default()); + another_array.init_from_ref(array.get_ref()); + auto another_header = another_array.get_header(); + auto another_encoding = another_array.get_encoding(another_header); + CHECK(encoding == another_encoding); + + array.get_alloc().free_(mem); } TEST(Array_cares_about) @@ -1710,9 +1762,8 @@ TEST(VerifyIterationAcrossWords) // unaligned iterator UnalignedWordIter u_it(a, 0); for (size_t i = 0; i < 51; ++i) { - const auto v = sign_extend_value(5, u_it.get(5) & 0x1F); + const auto v = sign_extend_value(5, u_it.consume(5) & 0x1F); CHECK_EQUAL(v, values[i]); - u_it.bump(5); } } } @@ -1859,7 +1910,7 @@ TEST(ParallelSearchEqualMatch) // Now use the optimized version static auto vector_compare_eq = [](auto msb, auto a, auto b) { - return find_all_fields_EQ(msb, a, b); + return find_all_fields(msb, a, b); }; start = 0; @@ -1901,7 +1952,7 @@ TEST(ParallelSearchEqualNoMatch) const auto search_vector = populate(width, key); static auto vector_compare_eq = [](auto msb, auto a, auto b) { - return find_all_fields_EQ(msb, a, b); + return find_all_fields(msb, a, b); }; size_t start = 0; @@ -1951,7 +2002,7 @@ TEST(ParallelSearchNotEqual) const auto search_vector = populate(width, key); static auto vector_compare_neq = [](auto msb, auto a, auto b) { - return find_all_fields_NE(msb, a, b); + return find_all_fields(msb, a, b); }; size_t start = 0; @@ -2002,7 +2053,7 @@ TEST(ParallelSearchLessThan) const auto search_vector = populate(width, key); static auto vector_compare_lt = [](auto msb, auto a, auto b) { - return find_all_fields_signed_LT(msb, a, b); + return find_all_fields(msb, a, b); }; size_t start = 0; @@ -2052,7 +2103,7 @@ TEST(ParallelSearchGreaterThan) const auto search_vector = populate(width, key); static auto vector_compare_gt = [](auto msb, auto a, auto b) { - return find_all_fields_signed_GT(msb, a, b); + return find_all_fields(msb, a, b); }; size_t start = 0; diff --git a/test/test_array_integer.cpp b/test/test_array_integer.cpp index a26cecf52b2..9ccdf25653e 100644 --- a/test/test_array_integer.cpp +++ b/test/test_array_integer.cpp @@ -19,6 +19,7 @@ #include "testsettings.hpp" #include +#include #include #include @@ -31,6 +32,1575 @@ using namespace realm; using namespace realm::test_util; +// #define ARRAY_PERFORMANCE_TESTING +#if !defined(REALM_DEBUG) && defined(ARRAY_PERFORMANCE_TESTING) +NONCONCURRENT_TEST(perf_array_encode_get_vs_array_get_less_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " < 32 bit values " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) + REALM_ASSERT(a.get(i) == input_array[i]); + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Positive values - Array::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + t1 = high_resolution_clock::now(); + + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(compressed_a.get(i) == a.get(i)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Positive values - ArrayCompress::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) + REALM_ASSERT(a.get(i) == input_array[i]); + } + t2 = high_resolution_clock::now(); + + std::cout << std::endl; + + std::cout << " Negative values - Array::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Negative values - Array::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(compressed_a.get(i) == a.get(i)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Negative values - ArrayCompress::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + + +NONCONCURRENT_TEST(Test_basic_find_EQ_less_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth < 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = a.find_first(input_array[i]); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(a.get(ndx) == input_array[ndx]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto v = a.find_first(input_array[i]); + auto v1 = compressed_a.find_first(input_array[i]); + REALM_ASSERT(v == v1); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = compressed_a.find_first(input_array[i]); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(compressed_a.get(ndx) == input_array[ndx]); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto v = a.find_first(input_array[i]); + auto v1 = compressed_a.find_first(input_array[i]); + REALM_ASSERT(v == v1); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = a.find_first(input_array[i]); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(a.get(ndx) == input_array[ndx]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = compressed_a.find_first(input_array[i]); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(compressed_a.get(ndx) == a.get(ndx)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_NEQ_value_less_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth < 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(i, 0, a.size(), &state1); + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // NEQ for signed integers is not working. TODO: investigate this. + // verify that both find the same thing + + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-i, 0, a.size(), &state1); + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_LT_value_less_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth < 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1{}; + QueryStateFindFirst state2{}; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { // there is nothing less than 0 + a.find(i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() << " ms" + << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(i, 0, a.size(), &state1); + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { // there is nothing less than 0 + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-i, 0, a.size(), &state1); + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { // nothing less than the biggest negative number + a.find(-i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() << " ms" + << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { // nothing less than the biggest negative number + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_GT_value_less_32bit) +{ + // GT subword parallel search is not working... TODO : investigate + using namespace std; + using namespace std::chrono; + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth < 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { // nothing greatest than the last number + a.find(i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(i, 0, a.size(), &state1); + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { // nothing bigger than the last val + compressed_a.find(i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-i, 0, a.size(), &state1); + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { // nothing bigger than 0 + a.find(-i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { // nothing bigger than 0 + compressed_a.find(-i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(perf_array_encode_get_vs_array_get_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x0000000100000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " >= 32 bit values " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) + REALM_ASSERT(a.get(i) == input_array[i]); + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Positive values - Array::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + t1 = high_resolution_clock::now(); + + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(compressed_a.get(i) == a.get(i)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Positive values - ArrayCompress::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-i); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) + REALM_ASSERT(a.get(i) == input_array[i]); + } + t2 = high_resolution_clock::now(); + + std::cout << std::endl; + + std::cout << " Negative values - Array::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Negative values - Array::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(compressed_a.get(i) == a.get(i)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::get(): " << duration_cast(t2 - t1).count() << " ns" + << std::endl; + std::cout << " Negative values - ArrayCompress::get(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_EQ_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x000001000000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth >= 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = a.find_first(start_value + i); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(a.get(ndx) == input_array[ndx]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + REALM_ASSERT(a.find_first(start_value + i) == compressed_a.find_first(start_value + i)); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = compressed_a.find_first(start_value + i); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(compressed_a.get(ndx) == a.get(ndx)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-(start_value + i)); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + const auto k = -(start_value + i); + const auto v1 = a.find_first(k); + const auto v2 = compressed_a.find_first(k); + REALM_ASSERT(v1 == v2); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = a.find_first(-(start_value + i)); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(a.get(ndx) == input_array[ndx]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + auto ndx = compressed_a.find_first(-(start_value + i)); + REALM_ASSERT(ndx != realm::not_found); + REALM_ASSERT(compressed_a.get(ndx) == a.get(ndx)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_NEQ_value_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x0000000100000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth >= 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-(start_value + i)); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_LT_value_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x0000000100000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth >= 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() << " ms" + << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-(start_value + i)); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() << " ms" + << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +NONCONCURRENT_TEST(Test_basic_find_GT_value_greater_32bit) +{ + using namespace std; + using namespace std::chrono; + size_t start_value = 0x0000100000000; // 32 bit val + size_t n_values = 1000; + size_t n_runs = 100; + std::cout << " Value with bitwidth >= 32 " << std::endl; + std::cout << " N values = " << n_values << std::endl; + std::cout << " N runs = " << n_runs << std::endl; + + std::vector input_array; + ArrayInteger a(Allocator::get_default()); + ArrayInteger compressed_a(Allocator::get_default()); + a.create(); + + for (size_t i = 0; i < n_values; i++) + input_array.push_back(start_value + i); + std::random_device rd; + std::mt19937 g(rd()); + std::shuffle(input_array.begin(), input_array.end(), g); + for (const auto& v : input_array) + a.add(v); + + QueryStateFindFirst state1; + QueryStateFindFirst state2; + auto t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { + a.find(start_value + i, 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + auto t2 = high_resolution_clock::now(); + + std::cout << " Positive values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Positive values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + state1 = {}; + state2 = {}; + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + const auto k = start_value + i; + a.find(k, 0, a.size(), &state1); + compressed_a.find(k, 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values - 1; ++i) { + compressed_a.find(start_value + i, 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Positive values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Positive values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + std::cout << std::endl; + + a.destroy(); + compressed_a.destroy(); + a.create(); + input_array.clear(); + for (size_t i = 0; i < n_values; i++) + input_array.push_back(-(start_value + i)); + std::random_device rd1; + std::mt19937 g1(rd1()); + std::shuffle(input_array.begin(), input_array.end(), g1); + for (const auto& v : input_array) + a.add(v); + + a.try_compress(compressed_a); + CHECK(compressed_a.is_compressed()); + CHECK(compressed_a.size() == a.size()); + + // verify that both find the same thing + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state1.m_state == state2.m_state); + } + } + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 0; i < n_values; ++i) { + a.find(-(start_value + i), 0, a.size(), &state1); + REALM_ASSERT(state1.m_state != realm::not_found); + REALM_ASSERT(a.get(state1.m_state) == input_array[state1.m_state]); + } + } + t2 = high_resolution_clock::now(); + + std::cout << " Negative values - Array::find(): " << duration_cast(t2 - t1).count() + << " ms" << std::endl; + std::cout << " Negative values - Array::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + t1 = high_resolution_clock::now(); + for (size_t j = 0; j < n_runs; ++j) { + for (size_t i = 1; i < n_values; ++i) { + compressed_a.find(-(start_value + i), 0, compressed_a.size(), &state2); + REALM_ASSERT(state2.m_state != realm::not_found); + REALM_ASSERT(compressed_a.get(state2.m_state) == a.get(state2.m_state)); + } + } + t2 = high_resolution_clock::now(); + std::cout << " Negative values - ArrayCompress::find(): " + << duration_cast(t2 - t1).count() << " ms" << std::endl; + std::cout << " Negative values - ArrayCompress::find(): " + << (double)duration_cast(t2 - t1).count() / n_values / n_runs << " ns/value" << std::endl; + + a.destroy(); + compressed_a.destroy(); +} + +#endif + +// disable this test if forcing compression to Packed. +#if !REALM_COMPRESS +TEST(Test_ArrayInt_no_compress) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(10); + a.add(11); + a.add(12); + // the original array is never encoded. a1 is the array to write down to disk + // in this case compression is not needed + CHECK_NOT(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a.get(0) == 10); + CHECK(a.get(1) == 11); + CHECK(a.get(2) == 12); + a.destroy(); + a1.destroy(); +} + +TEST(Test_ArrayInt_compress_decompress_needed) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(10); + a.add(5); + a.add(5); + // uncompressed requires 3 x 4 bits, compressed takes 2 x 5 bits + 3 x 2 bits + // with 8 byte alignment this is both 16 bytes. + CHECK_NOT(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + a.add(10); + a.add(15); + // uncompressed is 5x4 bits, compressed is 3x5 bits + 5x2 bits + // with 8 byte alignment this is both 16 bytes + CHECK_NOT(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + a.add(10); + a.add(15); + a.add(10); + a.add(15); + // uncompressed is 9x4 bits, compressed is 3x5 bits + 9x2 bits + // with 8 byte alignment this is both 16 bytes + CHECK_NOT(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + a.add(-1); + // the addition of -1 forces the array from unsigned to signed form + // changing from 4 bits per element to 8 bits. + // (1,2,4 bit elements are unsigned, larger elements are signed) + // uncompressed is 10x8 bits, compressed is 3x5 bits + 10x2 bits + // with alignment, this is 24 bytes uncompressed and 16 bytes compressed + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a.get(0) == 10); + CHECK(a.get(1) == 5); + CHECK(a.get(2) == 5); + CHECK(a.get(3) == 10); + CHECK(a.get(4) == 15); + CHECK(a1.is_compressed()); + auto v = a1.get(0); + CHECK(v == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + a.destroy(); + a1.destroy(); +} +#endif + +TEST(Test_ArrayInt_get_all) +{ + std::vector vs = {3656152302, 2814021986, 4195757081, 3272933168, 3466127978, 2777289082, + 4247467684, 3825361855, 2496524560, 4052938301, 3765455798, 2527633011, + 3448934593, 3699340964, 4057735040, 3294068800}; + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + for (const auto i : vs) + a.add(i); + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + auto res = a1.get_all(0, a1.size()); + CHECK(res == vs); + a.destroy(); + a1.destroy(); +} + +TEST(Test_array_same_size_less_bits) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(1000000); + a.add(1000000); + a.add(1000000); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a.get_any(0) == 1000000); + CHECK(a.get_any(1) == 1000000); + CHECK(a.get_any(2) == 1000000); + CHECK(a1.is_compressed()); + CHECK(a1.get_any(0) == 1000000); + CHECK(a1.get_any(1) == 1000000); + CHECK(a1.get_any(2) == 1000000); + a.destroy(); + a1.destroy(); +} + +TEST(Test_ArrayInt_negative_nums) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(-1000000); + a.add(0); + a.add(1000000); + CHECK_NOT(a.is_compressed()); + CHECK(a.try_compress(a1)); + a1.destroy(); + CHECK(a.get(0) == -1000000); + CHECK(a.get(1) == 0); + CHECK(a.get(2) == 1000000); + a.add(-1000000); + a.add(-1000000); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a.get(0) == -1000000); + CHECK(a.get(1) == 0); + CHECK(a.get(2) == 1000000); + CHECK(a.get(3) == -1000000); + CHECK(a.get(4) == -1000000); + a.add(0); + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + + a.add(1000000); + a1.destroy(); // this decodes the array + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.try_decompress()); + a.add(-1000000); + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + a.add(0); + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + a.add(1000000); + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK_NOT(a.is_compressed()); + CHECK(a1.is_compressed()); + CHECK(a.size() == 10); + CHECK(a.size() == a1.size()); + CHECK(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + CHECK(a1.get(6) == a.get(6)); + CHECK(a1.get(7) == a.get(7)); + CHECK(a1.get(8) == a.get(8)); + a.destroy(); + a1.destroy(); +} + +TEST(Test_ArrayInt_compress_data) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + + a.create(); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(4); + a.add(5); + a.add(6); + a.add(7); + a.add(8); + a.add(4); + a.try_compress(a1); + bool ok = a1.is_compressed(); + CHECK(ok); + CHECK(a1.is_compressed()); + CHECK(a1.is_attached()); + CHECK(a.is_attached()); + for (size_t i = 0; i < a.size(); ++i) { + auto v0 = a1.get(i); + auto v1 = a.get(i); + CHECK(v0 == v1); + } + a.destroy(); + a1.destroy(); + + a.create(); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.add(-4427957085475570907); + a.try_compress(a1); + for (size_t i = 0; i < a.size(); ++i) + CHECK(a1.get(i) == a.get(i)); + a.destroy(); + a1.destroy(); + + a.create(); + + a.add(16388); + a.add(409); + a.add(16388); + a.add(16388); + a.add(409); + a.add(16388); + CHECK(a.size() == 6); + // Current: [16388:16, 409:16, 16388:16, 16388:16, 409:16, 16388:16], space needed: 6*16 bits = 96 bits + + // header + // compress the array is a good option. + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + // Compressed: [409:16, 16388:16][1:1,0:1,1:1,1:1,0:1,1:1], space needed: 2*16 bits + 6 * 1 bit = 38 bits + + // header + CHECK(a1.size() == a.size()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + // decompress + CHECK(a1.try_decompress()); + a.add(20); + // compress again, it should be a viable option + a1.destroy(); + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + CHECK(a1.size() == 7); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + CHECK(a1.get(6) == a.get(6)); + CHECK(a1.try_decompress()); + CHECK_NOT(a1.is_compressed()); + CHECK(a1.get(0) == a.get(0)); + CHECK(a1.get(1) == a.get(1)); + CHECK(a1.get(2) == a.get(2)); + CHECK(a1.get(3) == a.get(3)); + CHECK(a1.get(4) == a.get(4)); + CHECK(a1.get(5) == a.get(5)); + CHECK(a1.get(6) == a.get(6)); + a.destroy(); + a1.destroy(); +} + +TEST(Test_ArrayInt_compress_data_init_from_mem) +{ + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + a.add(16388); + a.add(409); + a.add(16388); + a.add(16388); + a.add(409); + a.add(16388); + const auto sz = a.size(); + CHECK(sz == 6); + // Current: [16388:16, 409:16, 16388:16, 16388:16, 409:16, 16388:16], + // space needed: 6*16 bits = 96 bits + header + // compress the array is a good option (it should already be compressed). + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + // Array should be in compressed form now + auto mem = a1.get_mem(); + ArrayInteger a2(Allocator::get_default()); + a2.init_from_mem(mem); // initialise a1 with a + // check a2 + CHECK(a2.is_compressed()); + const auto sz2 = a2.size(); + CHECK(sz2 == 6); + CHECK(a2.get(0) == 16388); + CHECK(a2.get(1) == 409); + CHECK(a2.get(2) == 16388); + CHECK(a2.get(3) == 16388); + CHECK(a2.get(4) == 409); + CHECK(a2.get(5) == 16388); + // decompress a2 and compresses again + CHECK(a2.is_compressed()); + CHECK(a2.try_decompress()); + CHECK_NOT(a2.is_compressed()); + a2.add(20); + CHECK(a2.try_compress(a1)); + CHECK(a1.is_compressed()); + CHECK(a1.size() == 7); + CHECK(a1.get(0) == 16388); + CHECK(a1.get(1) == 409); + CHECK(a1.get(2) == 16388); + CHECK(a1.get(3) == 16388); + CHECK(a1.get(4) == 409); + CHECK(a1.get(5) == 16388); + CHECK(a1.get(6) == 20); + CHECK(a1.try_decompress()); + a.destroy(); + a1.destroy(); + a2.destroy(); + CHECK_NOT(a.is_attached()); + CHECK_NOT(a1.is_attached()); + CHECK_NOT(a2.is_attached()); +} TEST(ArrayIntNull_SetNull) { @@ -244,3 +1814,114 @@ TEST(ArrayRef_Basic) a.destroy(); } + +TEST_TYPES(ArrayInt_comparison, Equal, NotEqual, Less, Greater) +{ + using Cond = TEST_TYPE; + ArrayInteger a(Allocator::get_default()); + ArrayInteger a1(Allocator::get_default()); + a.create(); + + // check first positive values < 32 bits + constexpr auto N = 300; + constexpr auto M = 3; + for (size_t i = 0; i < N; i++) + for (size_t j = 0; j < M; ++j) + a.add(i); + + auto sz = a.size(); + CHECK(sz == M * N); + + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + + // Array should be in compressed form now and values should match + for (size_t i = 0; i < sz; ++i) + CHECK(a.get(i) == a1.get(i)); + + for (int i = (int)(sz)-1; i >= 0; --i) { + QueryStateFindFirst m_first1, m_first2; + CHECK(a.find(i, 0, sz, &m_first1) == a1.find(i, 0, sz, &m_first2)); + CHECK(m_first1.m_state == m_first2.m_state); + } + + IntegerColumn accu1(Allocator::get_default()); + IntegerColumn accu2(Allocator::get_default()); + accu1.create(); + accu2.create(); + for (int i = (int)(sz)-1; i >= 0; --i) { + QueryStateFindAll m1{accu1}, m2{accu2}; + CHECK(a.find(i, 0, sz, &m1) == a1.find(i, 0, sz, &m2)); + CHECK(m1.match_count() == m2.match_count()); + } + + // check negative numbers now. + a1.destroy(); + a.clear(); + + for (size_t i = 0; i < N; i++) + for (size_t j = 0; j < M; ++j) + a.add(-i); + + sz = a.size(); + CHECK(sz == M * N); + + CHECK(a.try_compress(a1)); + CHECK(a1.is_compressed()); + + // Array should be in compressed form now and values should match + for (size_t i = 0; i < sz; ++i) + CHECK(a.get(i) == a1.get(i)); + + for (int64_t i = (int64_t)(sz)-1; i >= 0; --i) { + QueryStateFindFirst m_first1, m_first2; + CHECK(a.find(-i, 0, sz, &m_first1) == a1.find(-i, 0, sz, &m_first2)); + CHECK(m_first1.m_state == m_first2.m_state); + } + + accu1.clear(); + accu2.clear(); + for (int i = (int)(sz)-1; i >= 0; --i) { + QueryStateFindAll m1{accu1}, m2{accu2}; + CHECK(a.find(-i, 0, sz, &m1) == a1.find(-i, 0, sz, &m2)); + CHECK(m1.match_count() == m2.match_count()); + } + + accu1.destroy(); + accu2.destroy(); + a.destroy(); + a1.destroy(); + +#if REALM_COMPRESS + a.create(); + std::random_device dev; + std::mt19937 rng(dev()); + const auto min_range_t = (size_t)std::numeric_limits::min(); + const auto max_range_t = (size_t)std::numeric_limits::max(); + std::uniform_int_distribution dist(min_range_t, max_range_t); + sz = 100; + for (size_t i = 0; i < sz; ++i) { + auto v = (int)dist(rng); + a.add(v); + } + a.try_compress(a1); + + for (size_t i = 0; i < sz; ++i) + CHECK(a.get(i) == a1.get(i)); + + CHECK(a1.is_compressed()); + for (size_t i = 0; i < sz; ++i) { + QueryStateFindFirst m_first1, m_first2; + CHECK(a.find(a.get(i), 0, sz, &m_first1) == a1.find(a1.get(i), 0, sz, &m_first2)); + CHECK(m_first1.m_state == m_first2.m_state); + if (m_first1.m_state != realm::not_found) + CHECK(a.get(m_first1.m_state) == a1.get(m_first2.m_state)); + } + + a.destroy(); + a1.destroy(); +#endif + + CHECK_NOT(a.is_attached()); + CHECK_NOT(a1.is_attached()); +} diff --git a/test/test_group.cpp b/test/test_group.cpp index 54cd141485b..651a582463c 100644 --- a/test/test_group.cpp +++ b/test/test_group.cpp @@ -2315,4 +2315,198 @@ TEST(Group_UniqueColumnKeys) CHECK_NOT_EQUAL(col_foo, col_bar); } +TEST(Group_ArrayCompression_Correctness) +{ + GROUP_TEST_PATH(path); + + // Create group with one list which maps to array_integer + Group to_disk; + TableRef table = to_disk.add_table("test"); + auto col_key = table->add_column_list(type_Int, "lint"); + auto obj = table->create_object(); + auto array = obj.get_list(col_key); + array.add(16388); + array.add(409); + array.add(16388); + array.add(16388); + array.add(409); + array.add(16388); + CHECK_EQUAL(array.size(), 6); + CHECK_EQUAL(array.get_any(0).get_int(), 16388); + CHECK_EQUAL(array.get_any(1).get_int(), 409); + CHECK_EQUAL(array.get_any(2).get_int(), 16388); + CHECK_EQUAL(array.get_any(3).get_int(), 16388); + CHECK_EQUAL(array.get_any(4).get_int(), 409); + CHECK_EQUAL(array.get_any(5).get_int(), 16388); + + // Serialize to disk (compression should happen when the proper leaf array is serialized to disk) + to_disk.write(path, crypt_key()); + +#ifdef REALM_DEBUG + to_disk.verify(); +#endif + + // Load the tables + Group from_disk(path, crypt_key()); + TableRef read_table = from_disk.get_table("test"); + auto col_key1 = read_table->get_column_key("lint"); + auto obj1 = read_table->get_object(0); + auto l1 = obj1.get_list(col_key1); + CHECK(l1.size() == array.size()); + CHECK(*read_table == *table); + for (size_t i = 0; i < l1.size(); ++i) { + CHECK_EQUAL(l1.get_any(i), array.get_any(i)); + } + +#ifdef REALM_DEBUG + from_disk.verify(); +#endif +} + +TEST(Group_ArrayCompression_Correctness_Negative) +{ + GROUP_TEST_PATH(path); + + // Create group with one list which maps to array_integer + Group to_disk; + TableRef table = to_disk.add_table("test"); + auto col_key = table->add_column_list(type_Int, "lint"); + auto obj = table->create_object(); + auto array = obj.get_list(col_key); + + array.add(-1); + array.add(-1); + array.add(-1); + array.add(-1); + array.add(std::numeric_limits::max()); + array.add(std::numeric_limits::max()); + + CHECK_EQUAL(array.size(), 6); + CHECK_EQUAL(array.get_any(0).get_int(), -1); + CHECK_EQUAL(array.get_any(1).get_int(), -1); + CHECK_EQUAL(array.get_any(2).get_int(), -1); + CHECK_EQUAL(array.get_any(3).get_int(), -1); + CHECK_EQUAL(array.get_any(4).get_int(), std::numeric_limits::max()); + CHECK_EQUAL(array.get_any(5).get_int(), std::numeric_limits::max()); + + // Serialize to disk (compression should happen when the proper leaf array is serialized to disk) + to_disk.write(path, crypt_key()); + +#ifdef REALM_DEBUG + to_disk.verify(); +#endif + + // Load the tables + Group from_disk(path, crypt_key()); + TableRef read_table = from_disk.get_table("test"); + auto col_key1 = read_table->get_column_key("lint"); + auto obj1 = read_table->get_object(0); + auto l1 = obj1.get_list(col_key1); + CHECK(l1.size() == array.size()); + CHECK(*read_table == *table); + for (size_t i = 0; i < l1.size(); ++i) { + CHECK_EQUAL(l1.get_any(i), array.get_any(i)); + } + +#ifdef REALM_DEBUG + from_disk.verify(); +#endif +} + +TEST(Group_ArrayCompression_Correctness_Funny_Values) +{ + GROUP_TEST_PATH(path); + + // Create group with one list which maps to array_integer + Group to_disk; + TableRef table = to_disk.add_table("test"); + auto col_key = table->add_column_list(type_Int, "lint"); + auto obj = table->create_object(); + auto array = obj.get_list(col_key); + + std::vector vs = {3656152302, 2814021986, 4195757081, 3272933168, 3466127978, 2777289082, + 4247467684, 3825361855, 2496524560, 4052938301, 3765455798, 2527633011, + 3448934593, 3699340964, 4057735040, 3294068800}; + + size_t ndx = 0; + for (const auto v : vs) { + array.add(v); + CHECK_EQUAL(v, array.get(ndx++)); + } + CHECK_EQUAL(array.size(), vs.size()); + + // Serialize to disk (compression should happen when the proper leaf array is serialized to disk) + to_disk.write(path, crypt_key()); + +#ifdef REALM_DEBUG + to_disk.verify(); +#endif + + // Load the tables + Group from_disk(path, crypt_key()); + TableRef read_table = from_disk.get_table("test"); + auto col_key1 = read_table->get_column_key("lint"); + auto obj1 = read_table->get_object(0); + auto l1 = obj1.get_list(col_key1); + CHECK(l1.size() == array.size()); + CHECK(*read_table == *table); + for (size_t i = 0; i < l1.size(); ++i) { + CHECK_EQUAL(l1.get_any(i), array.get_any(i)); + } + +#ifdef REALM_DEBUG + from_disk.verify(); +#endif +} + + +TEST(Group_ArrayCompression_Correctness_Random_Input) +{ + GROUP_TEST_PATH(path); + + // Create group with one list which maps to array_integer + Group to_disk; + TableRef table = to_disk.add_table("test"); + auto col_key = table->add_column_list(type_Int, "lint"); + auto obj = table->create_object(); + auto array = obj.get_list(col_key); + + std::random_device dev; + std::mt19937 rng(dev()); + constexpr auto min = std::numeric_limits::min(); + constexpr auto max = std::numeric_limits::max(); + std::uniform_int_distribution dist6(static_cast(min), + static_cast(max)); + for (size_t i = 0; i < 1000; ++i) { + const auto v = dist6(rng); + array.add(v); + const auto stored_v = array.get_any(i).get_int(); + CHECK_EQUAL(stored_v, v); + } + + // Serialize to disk (compression should happen when the proper leaf array is serialized to disk) + to_disk.write(path, crypt_key()); + +#ifdef REALM_DEBUG + to_disk.verify(); +#endif + + // Load the tables + Group from_disk(path, crypt_key()); + TableRef read_table = from_disk.get_table("test"); + auto col_key1 = read_table->get_column_key("lint"); + auto obj1 = read_table->get_object(0); + auto l1 = obj1.get_list(col_key1); + CHECK(l1.size() == array.size()); + CHECK(*read_table == *table); + for (size_t i = 0; i < l1.size(); ++i) { + CHECK_EQUAL(l1.get_any(i), array.get_any(i)); + } + +#ifdef REALM_DEBUG + from_disk.verify(); +#endif +} + + #endif // TEST_GROUP diff --git a/test/test_links.cpp b/test/test_links.cpp index be08d2c7392..7561364089b 100644 --- a/test/test_links.cpp +++ b/test/test_links.cpp @@ -1167,11 +1167,13 @@ TEST(Links_FormerMemLeakCase) auto col = origin->add_column(*target, "link"); origin->create_object().set(col, k); origin->create_object().set(col, k); + wt.get_group().verify(); wt.commit(); } { WriteTransaction wt(sg_w); TableRef target = wt.get_table("target"); + wt.get_group().verify(); target->begin()->remove(); wt.get_group().verify(); wt.commit(); diff --git a/test/test_list.cpp b/test/test_list.cpp index b29935981b1..d8e3f1fc1de 100644 --- a/test/test_list.cpp +++ b/test/test_list.cpp @@ -633,6 +633,41 @@ TEST(List_AggOps) test_lists_numeric_agg(test_context, sg, type_Decimal, Decimal128(realm::null()), true); } +TEST(Test_Write_List_Nested_In_Mixed) +{ + SHARED_GROUP_TEST_PATH(path); + std::string message; + DBOptions options; + options.logger = test_context.logger; + DBRef db = DB::create(make_in_realm_history(), path, options); + auto tr = db->start_write(); + auto table = tr->add_table("table"); + auto col_any = table->add_column(type_Mixed, "something"); + + Obj obj = table->create_object(); + obj.set_any(col_any, Mixed{20}); + tr->verify(); + tr->commit_and_continue_writing(); // commit simple mixed + tr->verify(); + + obj.set_collection(col_any, CollectionType::List); + auto list = obj.get_list_ptr(col_any); + list->add(Mixed{10}); + list->add(Mixed{11}); + tr->verify(); + tr->commit_and_continue_writing(); // commit nested list in mixed + tr->verify(); + + // spicy it up a little bit... + list->insert_collection(2, CollectionType::List); + list->insert_collection(3, CollectionType::List); + list->get_list(2)->add(Mixed{20}); + list->get_list(3)->add(Mixed{21}); + tr->commit_and_continue_writing(); + tr->verify(); + tr->close(); +} + TEST(List_Nested_InMixed) { SHARED_GROUP_TEST_PATH(path); diff --git a/test/test_query.cpp b/test/test_query.cpp index 6df86fb2b1f..c2d6b196b32 100644 --- a/test/test_query.cpp +++ b/test/test_query.cpp @@ -5772,4 +5772,38 @@ TEST(Query_NestedLinkCount) CHECK_EQUAL(q.count(), 3); } +TEST_TYPES(Query_IntCompressed, Equal, NotEqual, Less, LessEqual, Greater, GreaterEqual) +{ + TEST_TYPE c; + SHARED_GROUP_TEST_PATH(path); + int ints[] = {-120, -111, -70, -61, -55, -45, -22, -15, -3, 2, 7, 18, 25, 33, 55, 56, 66, 78, 104, 125}; + std::vector values; + for (int j = 1; j < 21; j++) { + for (int i = 0; i < j; i++) { + values.push_back(ints[i]); + } + } + + auto db = DB::create(path); + auto wt = db->start_write(); + auto t = wt->add_table("table"); + auto col = t->add_column(type_Int, "id"); + for (auto val : values) { + t->create_object().set(col, val); + } + wt->commit_and_continue_as_read(); + + for (int val : {-1000, -125, 2, 3, 6, 126, 1000}) { + size_t num_matches = 0; + for (auto i : values) { + if (c(i, val)) + num_matches++; + } + + char query_str[20]; + snprintf(query_str, 20, "id %s %d", c.description().c_str(), val); + CHECK_EQUAL(t->query(query_str).count(), num_matches); + } +} + #endif // TEST_QUERY diff --git a/test/test_shared.cpp b/test/test_shared.cpp index 78ede3b4a0c..85c3de4f8ab 100644 --- a/test/test_shared.cpp +++ b/test/test_shared.cpp @@ -95,34 +95,32 @@ using unit_test::TestContext; // `experiments/testcase.cpp` and then run `sh build.sh // check-testcase` (or one of its friends) from the command line. -#if 0 + // Sorting benchmark -ONLY(Query_QuickSort2) +TEST(Query_QuickSort2) { Random random(random_int()); // Seed from slow global generator // Triggers QuickSort because range > len Table ttt; - auto ints = ttt.add_column(type_Int, "1"); + // auto ints = ttt.add_column(type_Int, "1"); auto strings = ttt.add_column(type_String, "2"); for (size_t t = 0; t < 10000; t++) { Obj o = ttt.create_object(); - // o.set(ints, random.draw_int_mod(1100)); + // o.set(ints, random.draw_int_mod(1100)); o.set(strings, "a"); } Query q = ttt.where(); - std::cerr << "GO"; - for (size_t t = 0; t < 1000; t++) { TableView tv = q.find_all(); tv.sort(strings); - // tv.ints(strings); + // tv.ints(strings); } } -#endif + #if REALM_WINDOWS namespace { diff --git a/test/test_table.cpp b/test/test_table.cpp index 80df42e1824..52e06fb2659 100644 --- a/test/test_table.cpp +++ b/test/test_table.cpp @@ -46,7 +46,7 @@ using namespace std::chrono; #include "test_types_helper.hpp" // #include -// #define PERFORMACE_TESTING +// #define PERFORMANCE_TESTING using namespace realm; using namespace realm::util; @@ -2954,9 +2954,122 @@ NONCONCURRENT_TEST(Table_QuickSort2) std::cout << " time: " << duration_cast(t2 - t1).count() / nb_reps << " ns/rep" << std::endl; } +NONCONCURRENT_TEST(Table_object_timestamp) +{ +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) + int nb_rows = 10'000'000; + int num_runs = 100; +#else + int nb_rows = 100'000; + int num_runs = 1; +#endif + SHARED_GROUP_TEST_PATH(path); + std::unique_ptr hist(make_in_realm_history()); + DBRef sg = DB::create(*hist, path, DBOptions(crypt_key())); + ColKey c0; + + CALLGRIND_START_INSTRUMENTATION; + + std::cout << nb_rows << " rows - timestamps" << std::endl; + + { + WriteTransaction wt(sg); + auto table = wt.add_table("test"); + + c0 = table->add_column(type_Timestamp, "ts"); + + + auto t1 = steady_clock::now(); + + for (int i = 0; i < nb_rows; i++) { + Timestamp t(i, i); + table->create_object(ObjKey(i)).set_all(t); + } + + auto t2 = steady_clock::now(); + std::cout << " insertion time: " << duration_cast(t2 - t1).count() / nb_rows << " ns/key" + << std::endl; + + CHECK_EQUAL(table->size(), nb_rows); + wt.commit(); + } + { + ReadTransaction rt(sg); + auto table = rt.get_table("test"); + + auto t1 = steady_clock::now(); + Timestamp t(nb_rows / 2, nb_rows / 2); + for (int j = 0; j < num_runs; ++j) { + auto result = table->where().equal(c0, t).find_all(); + } + + auto t2 = steady_clock::now(); + + std::cout << " find all : " << duration_cast(t2 - t1).count() / num_runs << " ms" + << std::endl; + } +} + +NONCONCURRENT_TEST(Table_object_search) +{ +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) + int nb_rows = 10'000'000; + int num_runs = 100; +#else + int nb_rows = 100'000; + int num_runs = 1; +#endif + SHARED_GROUP_TEST_PATH(path); + std::unique_ptr hist(make_in_realm_history()); + DBRef sg = DB::create(*hist, path, DBOptions(crypt_key())); + ColKey c0; + ColKey c1; + + CALLGRIND_START_INSTRUMENTATION; + + std::cout << nb_rows << " rows - sequential" << std::endl; + + { + WriteTransaction wt(sg); + auto table = wt.add_table("test"); + + c0 = table->add_column(type_Int, "int1"); + c1 = table->add_column(type_Int, "int2", true); + + + auto t1 = steady_clock::now(); + + for (int i = 0; i < nb_rows; i++) { + table->create_object(ObjKey(i)).set_all(i << 1, i << 2); + } + + auto t2 = steady_clock::now(); + std::cout << " insertion time: " << duration_cast(t2 - t1).count() / nb_rows << " ns/key" + << std::endl; + + CHECK_EQUAL(table->size(), nb_rows); + wt.commit(); + } + { + ReadTransaction rt(sg); + auto table = rt.get_table("test"); + + auto t1 = steady_clock::now(); + + for (int j = 0; j < num_runs; ++j) { + auto result = table->find_all_int(c0, nb_rows / 2); + } + + auto t2 = steady_clock::now(); + + std::cout << " find all : " << duration_cast(t2 - t1).count() / num_runs << " ms" + << std::endl; + } +} + NONCONCURRENT_TEST(Table_object_sequential) { -#ifdef PERFORMACE_TESTING +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) int nb_rows = 10'000'000; int num_runs = 1; #else @@ -3106,7 +3219,7 @@ NONCONCURRENT_TEST(Table_object_sequential) NONCONCURRENT_TEST(Table_object_seq_rnd) { -#ifdef PERFORMACE_TESTING +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) size_t rows = 1'000'000; int runs = 100; // runs for building scenario #else @@ -3149,7 +3262,7 @@ NONCONCURRENT_TEST(Table_object_seq_rnd) } // scenario established! int nb_rows = int(key_values.size()); -#ifdef PERFORMACE_TESTING +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) int num_runs = 10; // runs for timing access #else int num_runs = 1; // runs for timing access @@ -3221,7 +3334,7 @@ NONCONCURRENT_TEST(Table_object_seq_rnd) NONCONCURRENT_TEST(Table_object_random) { -#ifdef PERFORMACE_TESTING +#if !defined(REALM_DEBUG) && defined(PERFORMANCE_TESTING) int nb_rows = 1'000'000; int num_runs = 10; #else diff --git a/test/test_unresolved_links.cpp b/test/test_unresolved_links.cpp index adaf6981130..60f50ee3488 100644 --- a/test/test_unresolved_links.cpp +++ b/test/test_unresolved_links.cpp @@ -837,35 +837,6 @@ TEST(Links_ManyObjects) tr->commit(); } -TEST(Unresolved_PerformanceLinks) -{ - constexpr int nb_objects = 1000; - using namespace std::chrono; - - SHARED_GROUP_TEST_PATH(path); - auto hist = make_in_realm_history(); - DBRef db = DB::create(*hist, path); - - auto tr = db->start_write(); - auto table = tr->add_table_with_primary_key("table", type_Int, "id"); - auto origin = tr->add_table("origin"); - auto col = origin->add_column(*table, "link"); - auto key = table->get_objkey_from_primary_key(1); - for (int i = 0; i < nb_objects; i++) { - origin->create_object().set(col, key); - } - tr->commit_and_continue_as_read(); - tr->promote_to_write(); - auto t1 = steady_clock::now(); - table->create_object_with_primary_key(1); - auto t2 = steady_clock::now(); - tr->commit_and_continue_as_read(); - CHECK(t2 > t1); - // std::cout << "Time: " << duration_cast(t2 - t1).count() << " us" << std::endl; - tr->promote_to_write(); - tr->verify(); -} - TEST(Unresolved_PerformanceLinkList) { constexpr int nb_objects = 1000; @@ -889,6 +860,7 @@ TEST(Unresolved_PerformanceLinkList) ll.add(key3); } tr->commit_and_continue_as_read(); + // compresses tr->promote_to_write(); auto t1 = steady_clock::now(); table->create_object_with_primary_key(1); @@ -897,7 +869,6 @@ TEST(Unresolved_PerformanceLinkList) auto t2 = steady_clock::now(); tr->commit_and_continue_as_read(); CHECK(t2 > t1); - // std::cout << "Time: " << duration_cast(t2 - t1).count() << " us" << std::endl; tr->promote_to_write(); tr->verify(); } From 27d73a4c3eca350b029b1fba5568fe788928935b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Mon, 10 Jun 2024 15:05:25 +0200 Subject: [PATCH 11/18] Update release notes --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 393508adc3e..f03f417b3ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +# NEXT MAJOR RELEASE + +### Enhancements +* (PR [#????](https://github.com/realm/realm-core/pull/????)) +* Storage of integers changed so that they take up less space in the file. This can cause commits and some queries to take a bit longer (PR [#7668](https://github.com/realm/realm-core/pull/7668)) + +### Fixed +* ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) +* None. + +### Breaking changes +* None. + +### Compatibility +* Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. + +----------- + +### Internals +* None. + +---------------------------------------------- + # 14.10.0 Release notes ### Enhancements From ac249ef5505fd041d58784d205d3fb985848ad82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Mon, 10 Jun 2024 11:42:01 +0200 Subject: [PATCH 12/18] Improve FlexCompressor::find_all --- src/realm/integer_flex_compressor.cpp | 14 +- src/realm/integer_flex_compressor.hpp | 213 +++++++++++++------------- src/realm/query_conditions.hpp | 32 +++- 3 files changed, 141 insertions(+), 118 deletions(-) diff --git a/src/realm/integer_flex_compressor.cpp b/src/realm/integer_flex_compressor.cpp index ef5e3b2fe6f..a7d8c8134b0 100644 --- a/src/realm/integer_flex_compressor.cpp +++ b/src/realm/integer_flex_compressor.cpp @@ -70,10 +70,18 @@ void FlexCompressor::copy_data(const Array& arr, const std::vector& val bool FlexCompressor::find_all_match(size_t start, size_t end, size_t baseindex, QueryStateBase* state) { REALM_ASSERT_DEBUG(state->match_count() < state->limit()); - const auto process = state->limit() - state->match_count(); - const auto end2 = end - start > process ? start + process : end; - for (; start < end2; start++) + while (start < end) { if (!state->match(start + baseindex)) return false; + start++; + } return true; } + +size_t FlexCompressor::lower_bound(size_t size, int64_t value, uint64_t mask, BfIterator& data_iterator) noexcept +{ + return impl::lower_bound(nullptr, 0, size, value, [&](auto, size_t ndx) { + data_iterator.move(ndx); + return sign_extend_field_by_mask(mask, *data_iterator); + }); +} diff --git a/src/realm/integer_flex_compressor.hpp b/src/realm/integer_flex_compressor.hpp index a7338978af8..6b1c00008e9 100644 --- a/src/realm/integer_flex_compressor.hpp +++ b/src/realm/integer_flex_compressor.hpp @@ -50,18 +50,7 @@ class FlexCompressor { private: static bool find_all_match(size_t, size_t, size_t, QueryStateBase*); - - template - static bool find_linear(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); - - template - static bool find_parallel(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); - - template - static bool do_find_all(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); - - template - static bool run_parallel_subscan(size_t, size_t, size_t); + static size_t lower_bound(size_t, int64_t, uint64_t, BfIterator&) noexcept; }; inline int64_t FlexCompressor::get(const IntegerCompressor& c, size_t ndx) @@ -164,17 +153,32 @@ inline void FlexCompressor::set_direct(const IntegerCompressor& c, size_t ndx, i data_iterator.set_value(value); } +template +class IndexCond { +public: + using type = T; +}; + +template <> +class IndexCond { +public: + using type = GreaterEqual; +}; + template inline bool FlexCompressor::find_all(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, QueryStateBase* state) { + static constexpr size_t RANGE_LIMIT = 20; + static constexpr size_t WIDTH_LIMIT = 16; + REALM_ASSERT_DEBUG(start <= arr.m_size && (end <= arr.m_size || end == size_t(-1)) && start <= end); Cond c; if (end == npos) end = arr.m_size; - if (!(arr.m_size > start && start < end)) + if (start >= arr.m_size || start >= end) return true; const auto lbound = arr.m_lbound; @@ -189,116 +193,105 @@ inline bool FlexCompressor::find_all(const Array& arr, int64_t value, size_t sta REALM_ASSERT_DEBUG(arr.m_width != 0); - if constexpr (std::is_same_v) { - return do_find_all(arr, value, start, end, baseindex, state); + const auto& compressor = arr.integer_compressor(); + const auto v_width = arr.m_width; + const auto v_size = compressor.v_size(); + const auto mask = compressor.v_mask(); + uint64_t* data = (uint64_t*)arr.m_data; + size_t v_start = realm::not_found; + + /**************** Search the values ****************/ + + int64_t modified_value = value; + if constexpr (std::is_same_v) { + modified_value++; // We use GreaterEqual below, so this will effectively be Greater } - else if constexpr (std::is_same_v) { - return do_find_all(arr, value, start, end, baseindex, state); + + if (v_size >= RANGE_LIMIT) { + if (v_width <= WIDTH_LIMIT) { + auto search_vector = populate(v_width, modified_value); + v_start = parallel_subword_find(find_all_fields, data, 0, v_width, compressor.msb(), + search_vector, 0, v_size); + } + else { + BfIterator data_iterator{data, 0, v_width, v_width, 0}; + v_start = lower_bound(v_size, modified_value, mask, data_iterator); + } } - else if constexpr (std::is_same_v) { - return do_find_all(arr, value, start, end, baseindex, state); + else { + BfIterator data_iterator{data, 0, v_width, v_width, 0}; + size_t idx = 0; + while (idx < v_size) { + if (sign_extend_field_by_mask(mask, *data_iterator) >= modified_value) { + break; + } + data_iterator.move(++idx); + } + v_start = idx; } - else if constexpr (std::is_same_v) { - return do_find_all(arr, value, start, end, baseindex, state); + + if constexpr (realm::is_any_v) { + // Check for equality. + if (v_start < v_size) { + BfIterator it{data, 0, v_width, v_width, v_start}; + if (sign_extend_field_by_mask(mask, *it) > value) { + v_start = v_size; // Mark as not found + } + } } - return true; -} -template -inline bool FlexCompressor::do_find_all(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, - QueryStateBase* state) -{ - const auto v_width = arr.m_width; - const auto v_range = arr.integer_compressor().v_size(); - const auto ndx_range = end - start; - if (!run_parallel_subscan(v_width, v_range, ndx_range)) - return find_linear(arr, value, start, end, baseindex, state); - return find_parallel(arr, value, start, end, baseindex, state); -} + /***************** Some early outs *****************/ -template -inline bool FlexCompressor::find_linear(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, - QueryStateBase* state) -{ - const auto cmp = [](int64_t item, int64_t key) { - if constexpr (std::is_same_v) - return item == key; - if constexpr (std::is_same_v) - return item != key; - if constexpr (std::is_same_v) - return item < key; - if constexpr (std::is_same_v) - return item > key; - REALM_UNREACHABLE(); - }; - - const auto& c = arr.integer_compressor(); - const auto offset = c.v_width() * c.v_size(); - const auto ndx_w = c.ndx_width(); - const auto v_w = c.v_width(); - const auto data = c.data(); - const auto mask = c.v_mask(); - BfIterator ndx_iterator{data, offset, ndx_w, ndx_w, start}; - BfIterator data_iterator{data, 0, v_w, v_w, static_cast(*ndx_iterator)}; - while (start < end) { - const auto sv = sign_extend_field_by_mask(mask, *data_iterator); - if (cmp(sv, value) && !state->match(start + baseindex)) - return false; - ndx_iterator.move(++start); - data_iterator.move(static_cast(*ndx_iterator)); + if (v_start == v_size) { + if constexpr (realm::is_any_v) { + return true; // No Matches + } + if constexpr (realm::is_any_v) { + return find_all_match(start, end, baseindex, state); // All matches + } + } + else if (v_start == 0) { + if constexpr (std::is_same_v) { + // No index is less than 0 + return true; // No Matches + } + if constexpr (std::is_same_v) { + // All index is greater than or equal to 0 + return find_all_match(start, end, baseindex, state); + } } - return true; -} -template -inline bool FlexCompressor::find_parallel(const Array& arr, int64_t value, size_t start, size_t end, size_t baseindex, - QueryStateBase* state) -{ - // - // algorithm idea: first try to find in the array of values (should be shorter in size but more bits) using - // VectorCond1. - // Then match the index found in the array of indices using VectorCond2 - // + /*************** Search the indexes ****************/ - const auto& compressor = arr.integer_compressor(); - const auto v_width = compressor.v_width(); - const auto v_size = compressor.v_size(); + using U = typename IndexCond::type; + const auto ndx_range = end - start; const auto ndx_width = compressor.ndx_width(); - const auto offset = v_size * v_width; - uint64_t* data = (uint64_t*)arr.m_data; - - auto MSBs = compressor.msb(); - auto search_vector = populate(v_width, value); - auto v_start = - parallel_subword_find(find_all_fields, data, 0, v_width, MSBs, search_vector, 0, v_size); - - if constexpr (!std::is_same_v) { - if (start == v_size) - return true; + const auto v_offset = v_size * v_width; + if (ndx_range >= RANGE_LIMIT) { + auto search_vector = populate(ndx_width, v_start); + while (start < end) { + start = parallel_subword_find(find_all_fields_unsigned, data, v_offset, ndx_width, + compressor.ndx_msb(), search_vector, start, end); + if (start < end) { + if (!state->match(start + baseindex)) + return false; + } + ++start; + } } - - MSBs = compressor.ndx_msb(); - search_vector = populate(ndx_width, v_start); - while (start < end) { - start = parallel_subword_find(find_all_fields_unsigned, data, offset, ndx_width, MSBs, - search_vector, start, end); - - if (start < end && !state->match(start + baseindex)) - return false; - - ++start; + else { + U index_c; + BfIterator ndx_iterator{data, v_offset, ndx_width, ndx_width, start}; + while (start < end) { + if (index_c(int64_t(*ndx_iterator), int64_t(v_start))) { + if (!state->match(start + baseindex)) + return false; + } + ndx_iterator.move(++start); + } } - return true; -} -template -inline bool FlexCompressor::run_parallel_subscan(size_t v_width, size_t v_range, size_t ndx_range) -{ - if constexpr (std::is_same_v || std::is_same_v) { - return v_width < 32 && v_range >= 20 && ndx_range >= 20; - } - // > and < need looks slower in parallel scan for large values - return v_width <= 16 && v_range >= 20 && ndx_range >= 20; + return true; } } // namespace realm diff --git a/src/realm/query_conditions.hpp b/src/realm/query_conditions.hpp index ea16fb4a736..d2298d5d15a 100644 --- a/src/realm/query_conditions.hpp +++ b/src/realm/query_conditions.hpp @@ -302,6 +302,10 @@ struct Equal { { return (v == 0 && ubound == 0 && lbound == 0); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 == v2; + } static std::string description() { @@ -344,6 +348,10 @@ struct NotEqual { { return (v > ubound || v < lbound); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 != v2; + } template bool operator()(A, B, C, D) const = delete; @@ -816,6 +824,10 @@ struct Greater { static_cast(ubound); return lbound > v; } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 > v2; + } static std::string description() { @@ -890,7 +902,6 @@ struct NotNull { } }; - struct Less { static const int avx = 0x11; // _CMP_LT_OQ template @@ -907,6 +918,11 @@ struct Less { return Mixed::types_are_comparable(m1, m2) && (m1 < m2); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 < v2; + } + template bool operator()(A, B, C, D) const { @@ -914,14 +930,12 @@ struct Less { return false; } static const int condition = cond_Less; - bool can_match(int64_t v, int64_t lbound, int64_t ubound) + bool can_match(int64_t v, int64_t lbound, int64_t) { - static_cast(ubound); return lbound < v; } - bool will_match(int64_t v, int64_t lbound, int64_t ubound) + bool will_match(int64_t v, int64_t, int64_t ubound) { - static_cast(lbound); return ubound < v; } static std::string description() @@ -952,6 +966,10 @@ struct LessEqual : public HackClass { { return (m1.is_null() && m2.is_null()) || (Mixed::types_are_comparable(m1, m2) && (m1 <= m2)); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 <= v2; + } template bool operator()(A, B, C, D) const @@ -988,6 +1006,10 @@ struct GreaterEqual : public HackClass { { return (m1.is_null() && m2.is_null()) || (Mixed::types_are_comparable(m1, m2) && (m1 >= m2)); } + bool operator()(int64_t v1, int64_t v2) const + { + return v1 >= v2; + } template bool operator()(A, B, C, D) const From 5c521a758d3189c4db61ebdf86149482b859b2be Mon Sep 17 00:00:00 2001 From: Nicola Cabiddu Date: Wed, 19 Jun 2024 16:01:38 +0100 Subject: [PATCH 13/18] remove set_direct methods from integer compressors --- src/realm/integer_compressor.cpp | 14 ++------------ src/realm/integer_compressor.hpp | 2 -- src/realm/integer_flex_compressor.hpp | 12 ------------ src/realm/integer_packed_compressor.hpp | 7 ------- 4 files changed, 2 insertions(+), 33 deletions(-) diff --git a/src/realm/integer_compressor.cpp b/src/realm/integer_compressor.cpp index 5246928e775..ca8bca153e8 100644 --- a/src/realm/integer_compressor.cpp +++ b/src/realm/integer_compressor.cpp @@ -221,16 +221,6 @@ void IntegerCompressor::get_chunk_flex(const Array& arr, size_t ndx, int64_t res FlexCompressor::get_chunk(arr.m_integer_compressor, ndx, res); } -void IntegerCompressor::set_packed(Array& arr, size_t ndx, int64_t val) -{ - PackedCompressor::set_direct(arr.m_integer_compressor, ndx, val); -} - -void IntegerCompressor::set_flex(Array& arr, size_t ndx, int64_t val) -{ - FlexCompressor::set_direct(arr.m_integer_compressor, ndx, val); -} - template bool IntegerCompressor::find_packed(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, QueryStateBase* st) @@ -250,7 +240,7 @@ void IntegerCompressor::set_vtable(Array& arr) static const Array::VTable vtable_packed = {get_packed, get_chunk_packed, get_all_packed, - set_packed, + nullptr, { find_packed, find_packed, @@ -260,7 +250,7 @@ void IntegerCompressor::set_vtable(Array& arr) static const Array::VTable vtable_flex = {get_flex, get_chunk_flex, get_all_flex, - set_flex, + nullptr, { find_flex, find_flex, diff --git a/src/realm/integer_compressor.hpp b/src/realm/integer_compressor.hpp index 4e9023cfe18..c177d491125 100644 --- a/src/realm/integer_compressor.hpp +++ b/src/realm/integer_compressor.hpp @@ -70,8 +70,6 @@ class IntegerCompressor { static void get_chunk_packed(const Array& arr, size_t ndx, int64_t res[8]); static void get_chunk_flex(const Array& arr, size_t ndx, int64_t res[8]); - static void set_packed(Array& arr, size_t ndx, int64_t val); - static void set_flex(Array& arr, size_t ndx, int64_t val); // query interface template static bool find_packed(const Array& arr, int64_t val, size_t begin, size_t end, size_t base_index, diff --git a/src/realm/integer_flex_compressor.hpp b/src/realm/integer_flex_compressor.hpp index 6b1c00008e9..3296e920f6d 100644 --- a/src/realm/integer_flex_compressor.hpp +++ b/src/realm/integer_flex_compressor.hpp @@ -40,7 +40,6 @@ class FlexCompressor { static int64_t get(const IntegerCompressor&, size_t); static std::vector get_all(const IntegerCompressor&, size_t, size_t); static void get_chunk(const IntegerCompressor&, size_t, int64_t[8]); - static void set_direct(const IntegerCompressor&, size_t, int64_t); template static bool find_all(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); @@ -142,17 +141,6 @@ inline void FlexCompressor::get_chunk(const IntegerCompressor& c, size_t ndx, in } } -inline void FlexCompressor::set_direct(const IntegerCompressor& c, size_t ndx, int64_t value) -{ - const auto offset = c.v_width() * c.v_size(); - const auto ndx_w = c.ndx_width(); - const auto v_w = c.v_width(); - const auto data = c.data(); - BfIterator ndx_iterator{data, offset, ndx_w, ndx_w, ndx}; - BfIterator data_iterator{data, 0, v_w, v_w, static_cast(*ndx_iterator)}; - data_iterator.set_value(value); -} - template class IndexCond { public: diff --git a/src/realm/integer_packed_compressor.hpp b/src/realm/integer_packed_compressor.hpp index 91d94fc5eab..ce342a582e2 100644 --- a/src/realm/integer_packed_compressor.hpp +++ b/src/realm/integer_packed_compressor.hpp @@ -40,7 +40,6 @@ class PackedCompressor { static int64_t get(const IntegerCompressor&, size_t); static std::vector get_all(const IntegerCompressor& c, size_t b, size_t e); static void get_chunk(const IntegerCompressor&, size_t, int64_t res[8]); - static void set_direct(const IntegerCompressor&, size_t, int64_t); template static bool find_all(const Array&, int64_t, size_t, size_t, size_t, QueryStateBase*); @@ -100,12 +99,6 @@ inline std::vector PackedCompressor::get_all(const IntegerCompressor& c return res; } -inline void PackedCompressor::set_direct(const IntegerCompressor& c, size_t ndx, int64_t value) -{ - BfIterator it{c.data(), 0, c.v_width(), c.v_width(), ndx}; - it.set_value(value); -} - inline void PackedCompressor::get_chunk(const IntegerCompressor& c, size_t ndx, int64_t res[8]) { auto sz = 8; From 3e32e5f2a0c2d664f4b5df1e1df2d16c581f629c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Mon, 8 Jul 2024 13:17:04 +0200 Subject: [PATCH 14/18] Fix CHANGELOG.md --- CHANGELOG.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1ef2a9395..9753fd2f453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,34 @@ ---------------------------------------------- +# 14.10.3 Release notes + +### Enhancements +* "Next launch" metadata file actions are now performed in a multi-process safe manner ([#7576](https://github.com/realm/realm-core/pull/7576)). + +### Fixed +* Fixed a change of mode from Strong to All when removing links from an embedded object that links to a tombstone. This affects sync apps that use embedded objects which have a `Lst` that contains a link to another top level object which has been deleted by another sync client (creating a tombstone locally). In this particular case, the switch would cause any remaining link removals to recursively delete the destination object if there were no other links to it. ([#7828](https://github.com/realm/realm-core/issues/7828), since 14.0.0-beta.0) +* Fixed removing backlinks from the wrong objects if the link came from a nested list, nested dictionary, top-level dictionary, or list of mixed, and the source table had more than 256 objects. This could manifest as `array_backlink.cpp:112: Assertion failed: int64_t(value >> 1) == key.value` when removing an object. ([#7594](https://github.com/realm/realm-core/issues/7594), since v11 for dictionaries) +* Fixed the collapse/rejoin of clusters which contained nested collections with links. This could manifest as `array.cpp:319: Array::move() Assertion failed: begin <= end [2, 1]` when removing an object. ([#7839](https://github.com/realm/realm-core/issues/7839), since the introduction of nested collections in v14.0.0-beta.0) +* wait_for_upload_completion() was inconsistent in how it handled commits which did not produce any changesets to upload. Previously it would sometimes complete immediately if all commits waiting to be uploaded were empty, and at other times it would wait for a server roundtrip. It will now always complete immediately. ([PR #7796](https://github.com/realm/realm-core/pull/7796)). +* `realm_sync_session_handle_error_for_testing` parameter `is_fatal` was flipped changing the expected behavior. (#[7750](https://github.com/realm/realm-core/issues/7750)). + +### Breaking changes +* None. + +### Compatibility +* Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. + +----------- + +### Internals +* Fixed `Table::remove_object_recursive` which wouldn't recursively follow links through a single `Mixed` property. This feature is exposed publicly on `Table` but no SDK currently uses it, so this is considered internal. ([#7829](https://github.com/realm/realm-core/issues/7829), likely since the introduction of Mixed) +* Upload completion is now tracked in a multiprocess-compatible manner ([PR #7796](https://github.com/realm/realm-core/pull/7796)). +* The local realm will assume the the client file ident of the fresh realm during a client reset. ([PR #7850](https://github.com/realm/realm-core/pull/7850)) +* Building using C++20 on Windows. + +---------------------------------------------- + # 14.10.2 Release notes ### Enhancements @@ -517,7 +545,7 @@ * Fixed equality queries on a Mixed property with an index possibly returning the wrong result if values of different types happened to have the same StringIndex hash. ([6407](https://github.com/realm/realm-core/issues/6407) since v11.0.0-beta.5). * If you have more than 8388606 links pointing to one specific object, the program will crash. ([#6577](https://github.com/realm/realm-core/issues/6577), since v6.0.0) * Query for NULL value in Dictionary would give wrong results ([6748])(https://github.com/realm/realm-core/issues/6748), since v10.0.0) -* A Realm generated on a non-apple ARM 64 device and copied to another platform (and vice-versa) were non-portable due to a sorting order difference. This impacts strings or binaries that have their first difference at a non-ascii character. These items may not be found in a set, or in an indexed column if the strings had a long common prefix (> 200 characters). ([PR #6670](https://github.com/realm/realm-core/pull/6670), since 2.0.0-rc7 for indexes, and since since the introduction of sets in v10.2.0) +* A Realm generated on a non-apple ARM 64 device and copied to another platform (and vice-versa) were non-portable due to a sorting order difference. This impacts strings or binaries that have their first difference at a non-ascii character. These items may not be found in a set, or in an indexed column if the strings had a long common prefix (> 200 characters). ([PR #6670](https://github.com/realm/realm-core/pull/6670), since 2.0.0-rc7 for indexes, and since the introduction of sets in v10.2.0) ### Breaking changes * Support for upgrading from Realm files produced by RealmCore v5.23.9 or earlier is no longer supported. From 0e533a76b8825a607d0d93f1f5477ffd8070cf41 Mon Sep 17 00:00:00 2001 From: nicola cabiddu Date: Tue, 9 Jul 2024 14:24:48 +0100 Subject: [PATCH 15/18] Remove `typed_print` (#7868) * remove typed_print * fix compress all builder --- evergreen/config.yml | 11 +++--- src/realm/array.cpp | 34 ------------------ src/realm/array.hpp | 2 -- src/realm/bplustree.cpp | 31 ---------------- src/realm/bplustree.hpp | 2 -- src/realm/cluster.cpp | 72 -------------------------------------- src/realm/cluster.hpp | 8 ----- src/realm/cluster_tree.cpp | 37 -------------------- src/realm/cluster_tree.hpp | 11 ------ src/realm/group.cpp | 42 ---------------------- src/realm/group.hpp | 6 ---- src/realm/node.hpp | 5 --- src/realm/spec.hpp | 5 --- src/realm/table.cpp | 26 -------------- src/realm/table.hpp | 1 - 15 files changed, 5 insertions(+), 288 deletions(-) diff --git a/evergreen/config.yml b/evergreen/config.yml index c18cd110529..055977f7312 100644 --- a/evergreen/config.yml +++ b/evergreen/config.yml @@ -1863,19 +1863,18 @@ buildvariants: - name: finalize_coverage_data - name: macos-array-compression - display_name: "MacOS 11 arm64 (Compress Arrays)" - run_on: macos-1100-arm64 + display_name: "MacOS 14 arm64 (Compress Arrays)" + run_on: macos-14-arm64 expansions: - cmake_url: "https://s3.amazonaws.com/static.realm.io/evergreen-assets/cmake-3.26.3-macos-universal.tar.gz" - cmake_bindir: "./cmake_binaries/CMake.app/Contents/bin" + cmake_bindir: "/opt/homebrew/bin" cmake_toolchain_file: "./tools/cmake/xcode.toolchain.cmake" + cmake_build_tool_options: "-sdk macosx" cmake_generator: Xcode max_jobs: $(sysctl -n hw.logicalcpu) - xcode_developer_dir: /Applications/Xcode13.1.app/Contents/Developer + xcode_developer_dir: /Applications/Xcode15.2.app/Contents/Developer extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=arm64 compress: On cmake_build_type: Debug - coveralls_flag_name: "macos-arm64" tasks: - name: compile_test diff --git a/src/realm/array.cpp b/src/realm/array.cpp index be70388bb2b..a2f61e4491c 100644 --- a/src/realm/array.cpp +++ b/src/realm/array.cpp @@ -1080,40 +1080,6 @@ bool QueryStateFindAll::match(size_t index) noexcept return (m_limit > m_match_count); } -void Array::typed_print(std::string prefix) const -{ - std::cout << "Generic Array " << header_to_string(get_header()) << " @ " << m_ref; - if (!is_attached()) { - std::cout << " Unattached"; - return; - } - if (size() == 0) { - std::cout << " Empty" << std::endl; - return; - } - std::cout << " size = " << size() << " {"; - if (has_refs()) { - std::cout << std::endl; - for (unsigned n = 0; n < size(); ++n) { - auto pref = prefix + " " + to_string(n) + ":\t"; - RefOrTagged rot = get_as_ref_or_tagged(n); - if (rot.is_ref() && rot.get_as_ref()) { - Array a(m_alloc); - a.init_from_ref(rot.get_as_ref()); - std::cout << pref; - a.typed_print(pref); - } - else if (rot.is_tagged()) { - std::cout << pref << rot.get_as_int() << std::endl; - } - } - std::cout << prefix << "}" << std::endl; - } - else { - std::cout << " Leaf of unknown type }" << std::endl; - } -} - ref_type ArrayPayload::typed_write(ref_type ref, _impl::ArrayWriterBase& out, Allocator& alloc) { Array arr(alloc); diff --git a/src/realm/array.hpp b/src/realm/array.hpp index 47984bfe959..858915fe27f 100644 --- a/src/realm/array.hpp +++ b/src/realm/array.hpp @@ -506,8 +506,6 @@ class Array : public Node, public ArrayParent { /// log2. Possible results {0, 1, 2, 4, 8, 16, 32, 64} static uint8_t bit_width(int64_t value); - void typed_print(std::string prefix) const; - protected: friend class NodeTree; void copy_on_write(); diff --git a/src/realm/bplustree.cpp b/src/realm/bplustree.cpp index b07e68d0e2b..02ea994b303 100644 --- a/src/realm/bplustree.cpp +++ b/src/realm/bplustree.cpp @@ -867,37 +867,6 @@ ref_type BPlusTreeBase::typed_write(ref_type ref, _impl::ArrayWriterBase& out, A return written_node.write(out); } -void BPlusTreeBase::typed_print(std::string prefix, Allocator& alloc, ref_type root, ColumnType col_type) -{ - char* header = alloc.translate(root); - Array a(alloc); - a.init_from_ref(root); - if (NodeHeader::get_is_inner_bptree_node_from_header(header)) { - std::cout << "{" << std::endl; - REALM_ASSERT(a.has_refs()); - for (unsigned j = 0; j < a.size(); ++j) { - auto pref = prefix + " " + std::to_string(j) + ":\t"; - RefOrTagged rot = a.get_as_ref_or_tagged(j); - if (rot.is_ref() && rot.get_as_ref()) { - if (j == 0) { - std::cout << pref << "BPTree offsets as ArrayUnsigned as "; - Array a(alloc); - a.init_from_ref(rot.get_as_ref()); - a.typed_print(prefix); - } - else { - std::cout << pref << "Subtree beeing "; - BPlusTreeBase::typed_print(pref, alloc, rot.get_as_ref(), col_type); - } - } - } - } - else { - std::cout << "BPTree Leaf[" << col_type << "] as "; - a.typed_print(prefix); - } -} - size_t BPlusTreeBase::size_from_header(const char* header) { auto node_size = Array::get_size_from_header(header); diff --git a/src/realm/bplustree.hpp b/src/realm/bplustree.hpp index 1f78d32ac26..5f763892b7a 100644 --- a/src/realm/bplustree.hpp +++ b/src/realm/bplustree.hpp @@ -219,8 +219,6 @@ class BPlusTreeBase { } static ref_type typed_write(ref_type, _impl::ArrayWriterBase&, Allocator&, TypedWriteFunc); - static void typed_print(std::string prefix, Allocator& alloc, ref_type root, ColumnType col_type); - protected: template diff --git a/src/realm/cluster.cpp b/src/realm/cluster.cpp index 2406d1e2362..1734993ebce 100644 --- a/src/realm/cluster.cpp +++ b/src/realm/cluster.cpp @@ -1713,76 +1713,4 @@ ref_type Cluster::typed_write(ref_type ref, _impl::ArrayWriterBase& out) const } return written_cluster.write(out); } - -void Cluster::typed_print(std::string prefix) const -{ - REALM_ASSERT_DEBUG(!get_is_inner_bptree_node_from_header(get_header())); - std::cout << "Cluster of size " << size() << " " << header_to_string(get_header()) << std::endl; - const auto table = get_owning_table(); - for (unsigned j = 0; j < size(); ++j) { - RefOrTagged rot = get_as_ref_or_tagged(j); - auto pref = prefix + " " + std::to_string(j) + ":\t"; - if (rot.is_ref() && rot.get_as_ref()) { - if (j == 0) { - std::cout << pref << "Keys as ArrayUnsigned as "; - Array a(m_alloc); - a.init_from_ref(rot.get_as_ref()); - a.typed_print(pref); - } - else { - auto col_key = table->m_leaf_ndx2colkey[j - 1]; - auto col_type = col_key.get_type(); - auto col_attr = col_key.get_attrs(); - std::string attr_string; - if (col_attr.test(col_attr_Dictionary)) - attr_string = "Dict:"; - if (col_attr.test(col_attr_List)) - attr_string = "List:"; - if (col_attr.test(col_attr_Set)) - attr_string = "Set:"; - if (col_attr.test(col_attr_Nullable)) - attr_string += "Null:"; - std::cout << pref << "Column[" << attr_string << col_type << "] as "; - // special cases for the types we want to compress - if (col_attr.test(col_attr_List) || col_attr.test(col_attr_Set)) { - // That is a single bplustree - // propagation of nullable missing here? - // handling of mixed missing here? - BPlusTreeBase::typed_print(pref, m_alloc, rot.get_as_ref(), col_type); - } - else if (col_attr.test(col_attr_Dictionary)) { - Array dict_top(m_alloc); - dict_top.init_from_ref(rot.get_as_ref()); - if (dict_top.size() == 0) { - std::cout << "{ empty }" << std::endl; - continue; - } - std::cout << "{" << std::endl; - auto ref0 = dict_top.get_as_ref(0); - if (ref0) { - auto p = pref + " 0:\t"; - std::cout << p; - BPlusTreeBase::typed_print(p, m_alloc, ref0, col_type); - } - if (dict_top.size() == 1) { - continue; // is this really possible? or should all dicts have both trees? - } - auto ref1 = dict_top.get_as_ref(1); - if (ref1) { - auto p = pref + " 1:\t"; - std::cout << p; - BPlusTreeBase::typed_print(p, m_alloc, dict_top.get_as_ref(1), col_type); - } - } - else { - // handle all other cases as generic arrays - Array a(m_alloc); - a.init_from_ref(rot.get_as_ref()); - a.typed_print(pref); - } - } - } - } -} - } // namespace realm diff --git a/src/realm/cluster.hpp b/src/realm/cluster.hpp index 33be9fcbe8b..564f3aa107e 100644 --- a/src/realm/cluster.hpp +++ b/src/realm/cluster.hpp @@ -214,13 +214,6 @@ class ClusterNode : public Array { } virtual ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const = 0; - virtual void typed_print(std::string prefix) const - { - static_cast(get_owning_table()); - std::cout << "ClusterNode as "; - Array::typed_print(prefix); - } - protected: #if REALM_MAX_BPNODE_SIZE > 256 static constexpr int node_shift_factor = 8; @@ -329,7 +322,6 @@ class Cluster : public ClusterNode { void verify() const; void dump_objects(int64_t key_offset, std::string lead) const override; virtual ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out) const override; - virtual void typed_print(std::string prefix) const override; static void remove_backlinks(const Table* origin_table, ObjKey origin_key, ColKey col, const std::vector& keys, CascadeState& state); static void remove_backlinks(const Table* origin_table, ObjKey origin_key, ColKey col, diff --git a/src/realm/cluster_tree.cpp b/src/realm/cluster_tree.cpp index 5bc2231454c..9c6d7fb1fe7 100644 --- a/src/realm/cluster_tree.cpp +++ b/src/realm/cluster_tree.cpp @@ -165,43 +165,6 @@ class ClusterNodeInner : public ClusterNode { return written_node.write(out); } - virtual void typed_print(std::string prefix) const override - { - REALM_ASSERT(get_is_inner_bptree_node_from_header(get_header())); - REALM_ASSERT(has_refs()); - std::cout << "ClusterNodeInner " << header_to_string(get_header()) << std::endl; - for (unsigned j = 0; j < size(); ++j) { - RefOrTagged rot = get_as_ref_or_tagged(j); - auto pref = prefix + " " + std::to_string(j) + ":\t"; - if (rot.is_ref() && rot.get_as_ref()) { - if (j == 0) { - std::cout << pref << "Keys as ArrayUnsigned as "; - Array a(m_alloc); - a.init_from_ref(rot.get_as_ref()); - a.typed_print(pref); - } - else { - auto header = m_alloc.translate(rot.get_as_ref()); - MemRef m(header, rot.get_as_ref(), m_alloc); - if (get_is_inner_bptree_node_from_header(header)) { - ClusterNodeInner a(m_alloc, m_tree_top); - a.init(m); - std::cout << pref; - a.typed_print(pref); - } - else { - Cluster a(j, m_alloc, m_tree_top); - a.init(m); - std::cout << pref; - a.typed_print(pref); - } - } - } - // just ignore entries, which are not refs. - } - Array::typed_print(prefix); - } - private: static constexpr size_t s_key_ref_index = 0; static constexpr size_t s_sub_tree_depth_index = 1; diff --git a/src/realm/cluster_tree.hpp b/src/realm/cluster_tree.hpp index a03ed4aabde..1b0d05c759b 100644 --- a/src/realm/cluster_tree.hpp +++ b/src/realm/cluster_tree.hpp @@ -196,17 +196,6 @@ class ClusterTree { return m_root->typed_write(ref, out); } - void typed_print(std::string prefix) const - { - if (m_root) { - std::cout << prefix << "ClusterTree as "; - m_root->typed_print(prefix); - } - else { - std::cout << "Emtpy ClusterTree" << std::endl; - } - } - protected: friend class Obj; friend class Cluster; diff --git a/src/realm/group.cpp b/src/realm/group.cpp index 90b7d690b26..b6703b3af53 100644 --- a/src/realm/group.cpp +++ b/src/realm/group.cpp @@ -960,48 +960,6 @@ ref_type Group::typed_write_tables(_impl::ArrayWriterBase& out) const } return dest.write(out); } -void Group::table_typed_print(std::string prefix, ref_type ref) const -{ - REALM_ASSERT(m_top.get_as_ref(1) == ref); - Array a(m_alloc); - a.init_from_ref(ref); - REALM_ASSERT(a.has_refs()); - for (unsigned j = 0; j < a.size(); ++j) { - auto pref = prefix + " " + to_string(j) + ":\t"; - RefOrTagged rot = a.get_as_ref_or_tagged(j); - if (rot.is_tagged() || rot.get_as_ref() == 0) - continue; - auto table_accessor = do_get_table(j); - REALM_ASSERT(table_accessor); - table_accessor->typed_print(pref, rot.get_as_ref()); - } -} -void Group::typed_print(std::string prefix) const -{ - std::cout << "Group top array" << std::endl; - for (unsigned j = 0; j < m_top.size(); ++j) { - auto pref = prefix + " " + to_string(j) + ":\t"; - RefOrTagged rot = m_top.get_as_ref_or_tagged(j); - if (rot.is_ref() && rot.get_as_ref()) { - if (j == 1) { - // Tables - std::cout << pref << "All Tables" << std::endl; - table_typed_print(pref, rot.get_as_ref()); - } - else { - Array a(m_alloc); - a.init_from_ref(rot.get_as_ref()); - std::cout << pref; - a.typed_print(pref); - } - } - else { - std::cout << pref << rot.get_as_int() << std::endl; - } - } - std::cout << "}" << std::endl; -} - ref_type Group::DefaultTableWriter::write_names(_impl::OutputStream& out) { diff --git a/src/realm/group.hpp b/src/realm/group.hpp index 08ddd9acd44..352c5bd25fb 100644 --- a/src/realm/group.hpp +++ b/src/realm/group.hpp @@ -516,8 +516,6 @@ class Group : public ArrayParent { } #endif ref_type typed_write_tables(_impl::ArrayWriterBase& out) const; - void table_typed_print(std::string prefix, ref_type ref) const; - void typed_print(std::string prefix) const; protected: static constexpr size_t s_table_name_ndx = 0; @@ -1129,10 +1127,6 @@ class Group::TableWriter { virtual ref_type write_names(_impl::OutputStream&) = 0; virtual ref_type write_tables(_impl::OutputStream&) = 0; virtual HistoryInfo write_history(_impl::OutputStream&) = 0; - void typed_print(std::string prefix) - { - m_group->typed_print(prefix); - } virtual ~TableWriter() noexcept {} diff --git a/src/realm/node.hpp b/src/realm/node.hpp index 8a4b862a701..8d606b37708 100644 --- a/src/realm/node.hpp +++ b/src/realm/node.hpp @@ -263,11 +263,6 @@ class Node : public NodeHeader { } } - void typed_print(int) const - { - std::cout << "Generic Node ERROR\n"; - } - protected: /// The total size in bytes (including the header) of a new empty /// array. Must be a multiple of 8 (i.e., 64-bit aligned). diff --git a/src/realm/spec.hpp b/src/realm/spec.hpp index d1a17072f67..c9f3ff0c230 100644 --- a/src/realm/spec.hpp +++ b/src/realm/spec.hpp @@ -84,11 +84,6 @@ class Spec { void set_ndx_in_parent(size_t) noexcept; void verify() const; - void typed_print(std::string prefix) const - { - std::cout << prefix << "Spec as "; - m_top.typed_print(prefix); - } private: // Underlying array structure. diff --git a/src/realm/table.cpp b/src/realm/table.cpp index ad9435f45ca..a21820997d6 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -3381,29 +3381,3 @@ ref_type Table::typed_write(ref_type ref, _impl::ArrayWriterBase& out) const } return dest.write(out); } - -void Table::typed_print(std::string prefix, ref_type ref) const -{ - REALM_ASSERT(ref == m_top.get_mem().get_ref()); - std::cout << prefix << "Table with key = " << m_key << " " << NodeHeader::header_to_string(m_top.get_header()) - << " {" << std::endl; - for (unsigned j = 0; j < m_top.size(); ++j) { - auto pref = prefix + " " + to_string(j) + ":\t"; - auto rot = m_top.get_as_ref_or_tagged(j); - if (rot.is_ref() && rot.get_as_ref()) { - if (j == 0) { - m_spec.typed_print(pref); - } - else if (j == 2) { - m_clusters.typed_print(pref); - } - else { - Array a(m_alloc); - a.init_from_ref(rot.get_as_ref()); - std::cout << pref; - a.typed_print(pref); - } - } - } - std::cout << prefix << "}" << std::endl; -} diff --git a/src/realm/table.hpp b/src/realm/table.hpp index 0830d7c733f..ac0faa5140a 100644 --- a/src/realm/table.hpp +++ b/src/realm/table.hpp @@ -546,7 +546,6 @@ class Table { ref_type typed_write(ref_type ref, _impl::ArrayWriterBase& out, bool deep, bool only_modified, bool compress) const; - void typed_print(std::string prefix, ref_type ref) const; private: template From b69af9e5bad98cc96e6f70cf57b85b33099129f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Mon, 12 Aug 2024 14:11:43 +0200 Subject: [PATCH 16/18] Create test file in file-format 24 --- test/test_upgrade_database.cpp | 108 ++++++++++++++++++++++- test/test_upgrade_database_1000_24.realm | Bin 0 -> 719112 bytes test/test_upgrade_database_4_24.realm | Bin 0 -> 8336 bytes 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 test/test_upgrade_database_1000_24.realm create mode 100644 test/test_upgrade_database_4_24.realm diff --git a/test/test_upgrade_database.cpp b/test/test_upgrade_database.cpp index ae95d1a02da..dad101ce10d 100644 --- a/test/test_upgrade_database.cpp +++ b/test/test_upgrade_database.cpp @@ -36,6 +36,7 @@ #include #include "test.hpp" #include "test_table_helper.hpp" +#include "util/compare_groups.hpp" #include @@ -273,8 +274,8 @@ TEST(Upgrade_Database_11) TableRef foo = g.add_table_with_primary_key("foo", type_Int, "id", false); TableRef bar = g.add_table_with_primary_key("bar", type_String, "name", false); TableRef o = g.add_table("origin"); - auto col1 = o->add_column_link(type_Link, "link1", *foo); - auto col2 = o->add_column_link(type_Link, "link2", *bar); + auto col1 = o->add_column(*foo, "link1"); + auto col2 = o->add_column(*bar, "link2"); for (auto id : ids) { auto obj = foo->create_object_with_primary_key(id); @@ -884,4 +885,107 @@ TEST_IF(Upgrade_Database_23, REALM_MAX_BPNODE_SIZE == 4 || REALM_MAX_BPNODE_SIZE g.write(path); #endif // TEST_READ_UPGRADE_MODE } + +TEST_IF(Upgrade_Database_24, REALM_MAX_BPNODE_SIZE == 4 || REALM_MAX_BPNODE_SIZE == 1000) +{ + std::string path = test_util::get_test_resource_path() + "test_upgrade_database_" + + util::to_string(REALM_MAX_BPNODE_SIZE) + "_24.realm"; + +#if TEST_READ_UPGRADE_MODE + CHECK_OR_RETURN(File::exists(path)); + + SHARED_GROUP_TEST_PATH(temp_copy); + + + // Make a copy of the database so that we keep the original file intact and unmodified + File::copy(path, temp_copy); + auto hist = make_in_realm_history(); + DBOptions options; + options.logger = test_context.logger; + auto sg = DB::create(*hist, temp_copy, options); + auto wt = sg->start_write(); + // rt->to_json(std::cout); + wt->verify(); + + SHARED_GROUP_TEST_PATH(path2); + wt->write(path2); + auto db2 = DB::create(path2); + auto wt2 = db2->start_write(); + CHECK(test_util::compare_groups(*wt, *wt2)); +#else + // NOTE: This code must be executed from an old file-format-version 24 + // core in order to create a file-format-version 25 test file! + + const size_t cnt = 10 * REALM_MAX_BPNODE_SIZE; + + std::vector string_values{ + "white", "yellow", "red", "orange", "green", "blue", "grey", "violet", "purple", "black", + }; + + StringData long_string(R"(1. Jeg ved en lærkerede, +jeg siger ikke mer; +den findes på en hede, +et sted som ingen ser. + +2. I reden er der unger, +og ungerne har dun. +De pipper de har tunger, +og reden er så lun. + +3. Og de to gamle lærker, +de flyver tæt omkring. +Jeg tænker nok de mærker, +jeg gør dem ingenting. + +4. Jeg lurer bag en slåen. +Der står jeg ganske nær. +Jeg rækker mig på tåen +og holder på mit vejr. + +5. For ræven han vil bide +og drengen samle bær. +men ingen skal få vide, +hvor lærkereden er. +)"); + StringData dict_value( + R"({"Seven":7, "Six":6, "Points": [1.25, 4.5, 6.75], "Attributes": {"Height": 202, "Weight": 92}})"); + + Timestamp now(std::chrono::system_clock::now()); + auto now_seconds = now.get_seconds(); + + Group g; + + TableRef t = g.add_table_with_primary_key("table", type_ObjectId, "_id", false); + auto col_int = t->add_column(type_Int, "int"); + auto col_optint = t->add_column(type_Int, "optint", true); + /* auto col_bool = */ t->add_column(type_Bool, "bool"); + auto col_string = t->add_column(type_String, "string"); + /* auto col_binary = */ t->add_column(type_Binary, "binary"); + auto col_mixed = t->add_column(type_Mixed, "any"); + auto col_date = t->add_column(type_Timestamp, "date"); + /* auto col_float = */ t->add_column(type_Float, "float"); + /* auto col_double = */ t->add_column(type_Double, "double"); + /* auto col_decimal = */ t->add_column(type_Decimal, "decimal"); + /* auto col_uuid = */ t->add_column(type_UUID, "uuid"); + + TableRef target = g.add_table_with_primary_key("target", type_Int, "_id", false); + auto col_link = t->add_column(*target, "link"); + + auto target_key = target->create_object_with_primary_key(1).get_key(); + for (size_t i = 0; i < cnt; i++) { + + auto o = t->create_object_with_primary_key(ObjectId::gen()); + o.set(col_int, uint64_t(0xffff) + i); + o.set(col_optint, uint64_t(0xffff) + (i % 10)); + o.set(col_string, string_values[i % string_values.size()]); + o.set(col_date, Timestamp(now_seconds + i, 0)); + o.set(col_link, target_key); + } + auto obj = t->create_object_with_primary_key(ObjectId::gen()); + obj.set_json(col_mixed, dict_value); + obj.set(col_string, long_string); + g.write(path); +#endif // TEST_READ_UPGRADE_MODE +} + #endif // TEST_GROUP diff --git a/test/test_upgrade_database_1000_24.realm b/test/test_upgrade_database_1000_24.realm new file mode 100644 index 0000000000000000000000000000000000000000..58e86453c3a5c552e735e69d636913e455b64856 GIT binary patch literal 719112 zcmeF)1z6T=y6^FqkWlRI?!s2=ZpH2Z1wj!I!S2My#_sNJ#m2_&?rz0C-&dDs{mz=% z*V!|BpL6XQ9_RYFzT@q956^P_rJlu)AHTj?{`l4~Z>0(;=<^KSedB1;t+%~@Lt-q4JuYd5*zXk*bThqofAi$3w zu0mAFxFI{NDQOI$K;aSiHA=?iAazo2QFg zjd{}Zr58vqlwKsgSbB-{QeCi22Fs;aNUxM$CB0gDjr3YwuucZ+r8h`#l-?x0S$d1~ zR$Z`72HT}~Nbi*1CB0jEk934C*eip5()*1)#0rEf^zl)j}4Zp+|~^j+zD()Xnw zNI#T*qzfL);ED88>1WcG|Lehn$i%1uhE+$=E7nG2Jn{-L(QqrZR%Se}%E~g91 z%bqys?cGm^@RdPF=}yvq(w(LKr30h` zbwL*y1W9+54wep)?k3$`x`!_4DT7|py`}p|_m%D^-CufuE*L0-LDGYzhe!{V9wt3p zI#d^okikglQPQKO$4HNr9w!~93&zV}g7ie`Nz#+0r$|qg4%Y?KWH4QNhV)G7S<bXKL0rUx^{tl#NQgv8 zj3h{kWJrz_NQqQPjWkG$bV!d3$cRkHj4a5CY{-rr$cbFYjXcPUe8>-16hJ`~LSYm^ zQ4~XQlz9n%*8y+#{w+GA}q!dEX6V`#|o^(Dy+sDti?L4#|CV~CTzwQY{fQg#}4eo zF6_o0L|`xWVLuMwAP(U$B5?#qaSX?C0w-|_Q8l-FyT5tnY^n3<(OfzJJuy z&!Zjh5VbYGo;g4B`^Eh8VV(!(d20Uru-N@i|MOB`A4td_?AHHo!|MhAw{O;;$Hyn_ zdZZnYHZ<+8|K0D`EBJAT2!8DMw?>VA%^fT@Ki>}N^m&`NA3DPy0SH7F1feT}5rS^$ zjvnZVUg(WJ=!<^nj{z8nK^Tl77>Z#Sj!=xiNQ}a0jKNrpLm0+m0w!V-CSwYwA{^5& z9WyW!voITTFcb#W5Vm37o_!MBy~f;4IGJJTBlOF5xn+;3}@+I&R=5 zZs9iW;4bdrJ|5s99^o;b;3=NrIbPr;Ug0&~;4R+aJwD(gqVWlz@daP;4d3wtmM`19 zZD0#K*drDk;0Px;BR1l|1#uA%@sR)tkqC*A1WAz$$&msnkqW7i25FHF>5%~$kqMcR z1zC{|*^vV|kqfzz2YHbX`QeHJD2PHRj3OwCVknLha6?IyLTQviS(HP0R6s>kLSZ1V~q7fRS37VoAnxh3;q7_=B4cfv3o@j^m@Paowzz4qQ z2A$9_&kMpHnurz+~YP)=#w-s>DsiBX`?ebWlE4O zMZOIAs+F<++SZ@8T!znA_X{ZTqv7krYzXX!7}U!}iEf0zF8XV7_I1 zf{ZfAB%N70i*#1$Y|`1KbLfJcGRP&JTRM+)Ug><&`K4WTK>-;QlrAJ)Sh|RGQR!mR z#dSdm8MsN8lrAM*TDpvMS?O}Rpu7w!NLQ4uBwbm$igZ=!YPz7h3~ET%l&&RRTe^;P zU1@h+P)`Q+r5i{$lx`&5Sh|UHQ(e$Z2F;~gNVk-3CEZ%OjdWXG;2{G~>2}iXrM;xR zr8`Lb=mK9Ebd>HS?I+z?+Fv?AI#3sMkwK7jSLtBs5b18x-KBfzf}S$yCEZ)Pk91$@ ze$xG=2k3%p zBMw{;7x54u36KzpkQhmj6v>brDUcGWkQ!-_7U_^48ITc~kQrH!71@v-Igk^%kQ;fB z7x|DMt|)+lD1^c&f}$vf;wS+(ltd|%Mj4bvIh02QR753IMio>=HB?6p)I=@RMjh0J zJL;i68lWK>p)s1EDVm`cO{6TQ$Ieb5*E&>sUZ5Q8unLogJ>FdU&6fsq)6(HMiV7>6*7#{^8oBuvH> zOhq`RVLE1DCT3wa=3p-7VLldMAr@gVmS8ECVL4V{C01cI)?h8xVLdirBQ{|(wqPr^ zVLNtUCw5^s_8 zkLSS8dk(R97FeGNj`&x;H)>uE>wAK%?^WWN4m+6p;s3h#p#OjUezx9+wmuJj-*d2j zKJ)+d`(Wn(pSjP@MSt_}12F&o1AZI)&%eKFo>%61^{@22($@$6nfum{n!jou-Tc?b zC4awO!N)NjTw7RQNB#BfZ|+-j=h6?I;g0|Wq6>o16~PEWH*`l2^h7W8Mj!M=KlH}{ z48$M|#t;m}Fbqd1MqngHVKl~IEXE-W<1qmfF$t3~1yd1@X_$@~n2A}KjX9W$d62#(?yj^hMQ;uNBA8fS18=Wreua1obq8CP%>*Ki#-a1*z18+ULQ_i!H%@DPvi z7*FsN&+r^C@Di`^8gK9x@9-WU@Db7YgwObbulR=V_yNmT?pwnacCbe*IKUB3a7Jvz zfeYdy9^xYb5+V^2BMFis8ImIfQX&;nBMs6b9nvEMG9nW)BMY)38?qw@av~RUBM+CE$jVD237}gR&@x@~D7{sD#R>f~u&7>ZpO5sD;|7gSv1> zJ=8}7G(;mbMiVqeGc-pFv_vbkMjNz+2RzXZ?coJ)bbt?h(Gk|CfVD4MT^xp;Ma{Pn#ZX2DA(96xL~LZ|kpY{ds@6Z(VDiT(p=ky+C@Q^djlS(o3Y5 z>Wz?PGFUFXLVBh2D(ThIYoyoef^{-jFTFu}qx2@}&C*+>x9S2uS>gA6>ss5TcS!G) z-X*NjH{mBHdIMG?PJd=@!y0rCUk2 zmTn{6Ru_24z*D-NbbDzpX>aKc(muMtR|XxWJ4yRVcb4{-4v-Gi1zltiB;8dySUNS9=f2X40=iTmhL0nSGu2cf9V0bV4w^JNe`ADB0W@knDlVzP+c%W1|y|MNspEu zBRy7noOGBj7%zhf(i5d8Nl%uZB0W_)To+7}!F1^v(le!JNzay^BR$vpcEZp5*4(+| zqP4{awy=XeV!;89aDp>pBMw{;7x54u36KzpkQhmj6v>brDUcGWkQ!-_7U_^48ITc~ zkQrH!71@v-Igk^%kQ;fB7x|DMt|)+lD1^c&f}$vf;wS+(ltd|%Mj4bvIh02QR753I zMio>=HB?6p)I=@RMjh0JJL;i68lWK>p)s1EDVm`cO{6TQ$Ieb5*E&>sUZ5Q8unLogJ>FdU&6fsq)6 z(HMiV7>6*7#{^8oBuvH>Ohq`RVLE1DCT3wa=3p-7VLldMAr@gVmS8ECVL4V{C01cI z)?h8xVLdirBQ{|(wqPr^VLNtUCw5^s_8kLSS8`_|@uuDNe+?rF!|uUqa`>%F)2dGPyQl=btO&DXOw zU%#7oE1R!ZZoWRd`TF)=78~wMdzr7#Zk`7*jVGjed{XxxzRM-VrdH>{JvNH z`}GUv4oms*zxPewF6qP_Oh0snKLQYlE(k(b1S169&>cO{6TQ$Ieb5*E&>sUZ5Q8un zLogJ>FdU&6fsq)6(HMiV7>6*7#{^8oBuvH>Ohq`RVLE1DCT3wa=3p-7VLldMAr@gV zmS8ECVL4V{C01cI)?h8xVLdirBQ{|(wqPr^VLNtUCw5^s_82Q1&Xe+^sM!5*>T07p2%8L<%uE{KbG zh>rwFh(t(?BuI*6NRAXpiBw39G)RkdNRJH2h)l?gEXay%$c`MyiCoByJjjcD$PZT( zKtU8jVH8186hm>8fE!Ap6iTBE%Ay>~qXH_T5-Ot#s-haIqXufC7HXpo>cSoMP#+D@ z5RK3nP0$q0&>St$60Oi0ZO|4T@I*VbhZnrj0Y30WM_8W^`W)i9VH?GBG3NxV&j)k=I_CWSHZQ3;0doT81k4GT6VMZ|z8+xiU+amOHYZ?Cz?^_N z0doT81k4GT6EG)WPQaXiIRSG5<^-(oFZj>izvknP+`qP1ocdTSuAvr7@zEAbwJ?jt z9btStaxEV5G0A9H=JE5ESu6=qeA};&Q~u>%cI|m`A!EMu0_la)i=-DzFOgoVH%FGq zV7c@P>6Ox}q*qI?kzT6{*2!SK^akmT(wn3=OK*|hstfoehTr$HYj2m{A-z+2m-KGw zJ<<`nV6P1JN$;0FAbn8!kn~~cNL_G521lijNgtO!A$?N%lysCXI4y%S(r2a5NuQU# zAbnB#k}kL`gDcWkrLRd}m%br=Q~H)JxGjS_(s!lrN#B=#ApKDKkuG>FgD28YrJqSZ zmwqArQu>uHcrAlB(r=~TNxzr=ApKD~S{Hnh!Ds0&(qE;&Nq?9A@n_eu$bvdHGO(4l zleU+RCG8;XDD9*RoMjMOI*zo9bX@6p(($De=z@eYNF<$DI*D{r>15K$rBmpFlrl&q zomx7LbXw_j(&?o$=z@$g$RwRvI*W8x>1@*3rE}1w*5 zx(sSa*Oaa$U0b@2bX{q8T~JR3^`#p~Hf>Oa{%RTS&K*ZYAAXx{Y*O zUEm=DPw95j?WMh>y`?)y`{)8+8FZBHB<&~NS=wJZKsrztbdf=jbXVzM=@98|(%q$d z=z^Xy=q24-x{q{U>3-7vr3dJOfif5*Jy?2(^ib(x(!-@gb-@T3jFcWFJz9E<^jPU} z(qXz_ybLBtPn4b{Jz08+^i=6^T`)}s)1_xf&y=1eJzIK?^q=>#xtqxszqZ)G7Iv^l zEI7asPH;wS#DNRqA|B!+0TLn+5+ezcA{mk+1yUjvQX>u0A|28r12Q5LG9wGJA{(+J z2XZ18aw8A&A|LX@6$MZbg-{qpP!z>b93|j}k|>4JD1)*nhw`X^il~IjsDi4fhU%z+ zny7`^sDrw2M?KU>12jY1Wg$F#*4(;IuZ*+hUe9;k|;D^re zM*srR1wrVFV1%F>x}yhrq8ECj5Bj1X`eOhFVh{#n2!>)9h9eXsFcPCM8e=dP;}C}N zn1G3xgvpqKsR+k3OvenOCl9L&W$%*O&O#3C%l5-i0sEXNA0#44=D8mz@Stj7jy z#3pRU7Hq{fY{w4l#4hZ{9zTh(~ygCwPiyc#ao%iC1`yH+YM8c#jYGh-iGm zXMDj|e8YE?=NtcihM)Jc&DXy+UpLx(9dzq^>0;*Z1Gn0LdoMdM#C+Y-f2FTmYVOgS zd$W90z}%xZ_vp<%dUKE7?!WqV*R5|K{Juy1&)my?b)Jt?!gAr)$AIP5yQC9$GyTvR{s=%Ix*!N$5sVOYLwEE*PxL}>^g&=dVjRLS9uqJTlQ0=mFcsmLhUu7rnV5yyn1i{Phxu55g;<2eSc0Wk zhUHj+l~{$3?3if{OiAFzDqUN&rD2YbYV103N5XT(MvxF9a#AwCiy zArc`mk{~IPAvsbYB~l?Z(jYC;Aw4o6BQhZ~vLGw6Av%()S#}#ug`_Ct)DRTlb z_XMoZ2Xikw=KlRRFS$7Za{}fB%n8Ko6R^G>VD4qd?B4_Cr8g&FPQaXiIRSG5<^;?M zm=iE3U{1iCfH?tk0)IUL>-!7-(Ry*aW>2Fs;aNUxM$CB0gDjr3Yw zuucZ+r8h`#l-?x0S$d1~R$ag+G5o%lU1z)W4(XlJyQFtZ?~#ts1$$+%PkO)f0qKL% zholcnN9uwjGB_%IO!~O=3F(v4r=+8F!D$(skv=PZPWrs`1?h{@mvq5p8C;RRDt%4* zy7Ud{o6@&*!EG7bk-jT^Px`*}1L=p-k95If89b4GD*a6Qx%3O^m(s6v!D|`3k$x-v zPWrv{2kDQ}(YoN13_eSLk^UD1C`q|-{LlTI(4K^J6{ zK_=DB7=Qq>D-ylP<0c zO31)Xx}C)0=q{~W|(*@;aP(iw)bS3G^(p99ZN>|eb)n!max~6n3>DtnDr0Yt% z>wx}kI<>BiDcq?_u3W-@3l-9oyhbSvrB(ru*M>H-fLcuKdEZZGX6?JeCw z+D8}o%AlikCuu+F&eHzU0n&lGpoA})Nq=!lmlO8S|stZQQV5IaY>Cw_-q{m8+lMd4b<7F^GdZP3s>B-Vl zq^C-U>w;-Am@Yj-dZzR&>DkhAq~}`S9{Kt8vbmdSvF9xl*uoC>hy@2Y!U@iZjW}>Y zT*O0sBtSwWLSiIAQY1riq(DlfLTaQzTBJjIWI#q_LS|$^R%AnV^g&=dVjRLS9uqJTlQ0=mFcsmL zhUu7rnV5yyn1i{Phxu55g;<2eSc0WkhUHj+l~{$3?3if{OiKb`|W z?`7L@AJrbQVD9I}-21ua_W^(A@z21IAJ+Hc#hl*<)|b}$JotSN%ldiF=Ig_H_2gL0 zJy&zj)!cJ6_gw$W_gu~M)I3l9_~+jVGHz3kA$ynTpp{Ppz<>%HttfBg8* zp?s%rmvrK8rXM=P9{~tN7X+azf)Rpl=#C!fiC*Z9KIn^n=#K#yh(Q>PAsC8b7>-bk zz(|b3XpF&Fj6)d4V*(~(5+-8`rXn2EFdZ{66SFWIb1)b4FdqxB5R0%FORyBnupBF} z605KpYp@pUupS$*5u30XTd)<|upK+F6T7e*dk}%W*oXZ%fP*-M!-&KY9K|sl#|fOo zDMaBk&fqN0;XE$jA}--FuHY)J;W}>MCT`(2?%*!&;XWSVAs*o|p5Q5-;W=L5C0^k* z-rz0X;XOX!BckyMpYa7>@eSYc1C}4$%Z4rNV2@aEfFqpXjM#_+7sN$8#76=oL?R?c z5+p@3Bu5IQL@K048l*)!q(=s1L?&cL7Gy;>WJeCWVBsE-C{h(>6PCTNOg zXpRkH;yb`1O*Z(d|`0_FtF378WwC%_37{^j>!HkCvZbOU=iT&#jNc=g%X8&ttfS6nsoF2VB!|?;s0z z3*b}u*T*UU^7XRo&XWrn^Q9L^FO*&+y;ypQ^isV!vP=fcrB_I=lwKvhT6&H2T3xVC z2J59aNN<$hB)wUBi}Y4qz$Y>MzL#BhyYvp}ozlCccT4Y)j?e{rWw1|rzw`m=gVKkj z4@*btf+I3GDt%1)xbz9>lhUW8qjbS(8Jv+mD}7G-yz~X>i_({L!DSg-k-jQ@P5Qd@ z4e6WGw{*d68QhV+D}7J;zVrj>htiL9!DAUbk$x)uO!~R>3+b2AuXMp{8N88xEB#LT zz4Qm^kJ8b);FAnKOMj96D*a9RyY!Dg+ub4y+-+oFD{UujFC9zTLE2H;Nf$WFAhvWI zX&33Z(($C@ODE6;31yH-I7>%hq?1dh&;==FkV-nWbQ8#S(q_a!s&;>bVkV`tZbROxv()pzGOS|fV0x~ElT}ZmHbP?&I(#52U>w*$8 zaFZ@6T}ryNbQ$Tg(&cnPc^OoYt|(ney0UZ?>8jGzbU}3))R3+zT}!&QbRFrs((byT zo($?sH;`^9-AKBzbQ9^Ox}cd1noGBkZYkYLy0vs0>9)GSLk6DG?WEgFdr5mscaZkc z1->%qDBVfgPr9?TzjT0fpf2blgCOay(!tUp(%q!HOZU(PJ!Q~Ky0>&6>Auqar29(` z&;OONZ)$5i%GlJxY4C^cd-}(&MDVbisHTOpu-^JxO}9^c3l- z(&4&bnhd5(&yb!eJxhAF^c?9w?`3m0(-MofOkfK;*drDk;0Px;BR1l|1#uA%@sR)t zkqC*A1WAz$$&msnkqW7i25FHF>5%~$kqMcR1zC{|*^vV|kqfzz2YHbX`QeHJD2PHR zj3OwCVknLha6?IyLTQviS(HP0R6s>kLSZ1V~q7fRS z37VoAnxh3;q7_=B4cfv3o@j^m@Paowzz4qQh)(cBXZRxkf#`xDbVV>i&<)+u13l3T zz0n7K(GUGG00S`ygE0g{F$}{IiV+xzQ5cOe7>jWT!+1=cP#W|eE1zf}>T*eh##Wh^V4cx>n+{PW; z#Xa1|13bhdJjN3|#WOs|3%tZDyv7^6#XG#m2Yf^{KH)RI;48l2JFM@Q`2BV1V%j}a z^YWT|_%ZSR^?&sH{@&)mz-P>;9Y;R@?F{q?N(vIGD4@t;Ha&ieCyzg^OayP1CI z41WY55M2<2t_Vg5x}iIIpeK5vH~OG2`k_AtU?2uzFos|#hG95DF#;no3ZpRwV=)e4 z7>@~iFz)GybYOKLptiyV2 zz(#DsW^BP$Y{Pc!z)tMKZtOt>_F^CQ;{Xog5Dp^}M{pF!a2zLa5~mP_(>Q~(IEVAN zfQz_<%eaE8xQ6Svft$F6+qi?fxQF|AfQNX5$9RILc!uYAftPrN*LZ`sc!&4+fRBjA zCw#^ie8o3>#}8O6+r4>@fVVB|V2@aEfFqpXjM#_+7sN$8#76=oL?R?c5+p@3Bu5IQ zL@K048l*)!q(=s1L?&cL7Gy;>WJeCWVBsE-C{h(>6PCTNOgXpR9x9Goeb7XZ;;+7y-9kr^cLx@ zx`0n&_7CNMq<2g2k&e&>du6asdcX7m>4VaTqz_9+>VhLOI4XTi`ndE7 z>66l@q@#4fX&Ic6J}Z4r`n>c7>5I~rbirjAT#>#ieNFnh^bP5o(zkTMZ5iB=zAJrC z`o8o7>4(yfbird8Jdu7X{Y?6~^b6^i(yw&EYZ<(eek=V>`n~iA>5tOUy5N%xK1+X* z{wn=V`n&XxKf9hq7SywmfvvQiw7qmJX$NUXX(wIaEQ8q6aim?O<4VVqjxU`+7bKKH zBI(4^Nu-lXCzDPtokAC+ltC)#)Y56B(@Lk4PA{E77i5${Ch5%5S){W{XOqq@okJJo zltC`(+|qfZ^GfHF&M)n%3kt}fpmZVW!qP>gi%J)hF0Kno$iPjyq;x6i($Zz5%SxBi z1?6Q>LAs)JCF#o2Rivv*SJMU6Wl%%9rgSao+R}BT>q@)pf_gHjFWo@8p>!kZ#?nos zo9cpQGH5Q{Lb|1NE9utKZKT`k0uLE@O1G14FYP7mE!{!dM;G|YprdprX+P=C(*Du` z(t*05iwuIKyGjR3he&sm?k?R!7xa`tFX`UWeWd$J_ml1~JwO)>l))hB!O}ydhe{8V z9xffK3r5Ibr1U81(b8k2$4ZZr4$}qWWiUZ{qVy!`$+CE$jVD237} zgR&@x@~D7{sD#R>f~u&7>ZpO5sD;|7gSv1>J=8}7G(;mbMiVqeGc-pFv_vbkMjNz+ z2RzXZ?coJ)bbt?h(Gi{ChtBXv00PkkLFkHLgrFO`qX&AT7kZ-)`l28DV*mzX5C&ri zhGH0oBNQVr5~DC0V=xxu5Qg!XfQgud$(Vwv2*)%`#|+HGEX>9n%*8y+#{w+GA}q!d zEX6V`#|o^(Dy+sDti?L4#|CV~CTzwQY{fQg#}4eoF6_o0L|`xWVLuMwAP(U$B5?#q zaSX?C0w-|_Q8zxVm4uMhk)_p-N+C01D}oV%Zs?94=!stFjXvm$e&~+@7>Gd_j3F3`VHl23 zjKD~Y!f1@aSd2p$#$y5|ViG1}3Z^0)(=Z(~FcY&d8*?xh^DrL^un>!|7)!7e%di|P zuoA1V8f&l?>#!ahuo0WE8C$Rw+prxwuoJtm8+#Cez1WBSIDmsVgu{r$5gf%a9LEWq z#3@AKG|u2G&fz>R;36*JGOpk%uHiav;3jV2Htygq?%_Tj;2|F2F`nQlp5ZxO;3Zz+ zHQwMY-r+qy;3J~(37_!=U-1p!@dFl{?cO%9g&ph>3l4CE6Pytnao~cuh==${fP_ed z#7KgqNQUG{fs{yv)JTK0NQd;ufQ-n5%*cYQ$cF65ft<*N+{lBx$cOxJMFA8Yy&%Q4jUe01eRyjnM>6 z(G1Pe0xi)Bt#yLwk6^8y(;SUvz}^vGAAYhOH~ljdBysy=-}8nlvX6V^6^P zd@%R2W9;94^MacbFehM6z??t~KLP9O0p?zI4F64FUVL)`<^;?Mm=iE3U{1iCfH?tk z0_FtF378WwC-9dOu)e?GAHA3D!#(S;etcg0AU-#J7@vQRSA2d$0zM{LjgLt-Mcb)- z-u@guHy`0^{_ef(dh_H$#(e1o(hH>*NiUXOBE3{^jx3YGa_JS)E2URSua;gTy;c{j zlfioF4bmH>H%V`n-XgtK7i^OOpV098j&{8r(mSPhN$-~4BORd&_R3(N^nU3B(g&pv zNgtMu)CEUma8&x3^l|AE(kG=)Nk{2|(=s?CeOCIM^m*wE(if#K>4M8LxFUU3`kM50 z=^N5FrElqi+cLN#eOLOP^nK|E(hsE{>4L{Hcq08&`kC}|=@-&3rC;fS*D`n`{Z{&& z^n2+K(jTRxb-^bYe3t$q{Z;y#^mpkWe|CL~EU0fI16yf3X?y8d(hky&(oVX-Sq8DC z<4C(m$CZvJ9bY*~lR?@Ad+eo+71s*c+lx`>8UfN6ATe^d^ zk1p_)K}YFM(tgsNrTwJ?qyu$97a0Ueca;v74w3FA-CeqeF6b$PUedj#`$+ed?kC+} zdVnq%D1$-LgQbT^50xG!JzP3e7mSd>Na<11qov14kCh%L9i|J$%V2`^MCnP=lclFf zPn8bW1=D0OU3!M}OzBzDv!&-q|9LN)yO|b8-ZFtL>|l>raDXG6;EdRa0~f?aJj6!= zBt#-4MiL}NG9*U|q(myDMjE6=I;2MiWJD%pMiyj6He^Q*zQ3|C|24ztW>EXoyB= zj3#J`W@wHUXo*&6jW%ct4|t*-+QSQcg0Q6neBg_Y=mbA>hCc!jh%N|1R|F#j-OwF9 z&=bAT8-36h{m>r+Fc5<<7(*}=!!R777=e)(h0z#;u^5LijK>5_#3W3{6ih`preQi} zU?yf^Hs)Y1=3zb-U?CP^F_vH{mSH(oU?o;zHP&D))?qz1U?VnRGqzwWwqZMVU?+BA zH})U`d$AAuaR3K#2!|1gBRGm8zxN)g zdA^wE3m*gC^o-h14^Iv_wSl>SQ{qJe{XYOUYALs2~w1xGX{(9DX*>nH+@t;Ha z&ieCyzg^OayP1CI41WY55M2<2t_Vg5x}iIIpeK5vH~OG2`k_AtU?2uzFos|#hG95D zF#;no3ZpRwV=)e47>@~iF zz)GybYOKLptiyV2z(#DsW^BP$Y{Pc!z)tMKZtOt>_F^CQ;{Xog5Dp^}M{pF!a2zLa z5~mP_(>Q~(IEVANfQz_<%eaE8xQ6Svft$F6+qi?fxQF|AfQNX5$9RILc!uYAftPrN z*LZ`sc!&4+fRBjACw#^ie8o3>#}8O+xt9%F*ufsL-~dNB!5Og;2QG+-16hJ`~LSYm^ zQ4~XQlzN;*)jL;zj?{c378WwCtyw>W}krd^#F4(J7)hLFfYA10doT81k4GT6EG)W zPQaXiIRSG5<^;?Mm=pNx30U7>@Q>chwyfn&!cOiTL~wWD0QUnB%Eu#@;uas1e1+(9 zd`$8>pGOSW=f6Hq>1=WNKknhplM5O1r58vqlwKsgSbB-{QoT8{Oa{xPS4gjvUM0O+ zdX4m2U9e6D>!mkHZ;|`l-?!1TY8Ulgf7@CgMHHb zr4L9Sls+VVSUOS{9Ff6M>0{EzrB6tols+XLr3+5W;EeQH>2uQOr7uWdl)j`3F3aGG z^i}C=($}SLNZ*vcr3-G$;EwcN>3h=mr5{K?lzyZO9?Rf~^i%0)($A$|NWYYRr3+rm z;EnWK>37oar9Vi2l#bQ~pJeb^`it~e>2K2CrGNa{4J@*tfsG7orR}8crDI7uNIOb9 z=>lgN#FmaD?IImlI-YcV=>)nUp$rm9Czehkom4uRbaLqwx*(+tQc0(lP9vRGI-PWS zY5nt{K}H#5lFlrhMLMf=HtForIrJsSDT7?nxux?+=atSUonP8j7Zi{|LFq!$g{6x~ z7nLq1U0fHGkb#?YN$FD3rKQVAmz6H33(Cu&f^*~lR?@Ad+eo+71s*c+lx`>8 zUfN6ATe^d^k1p_)K}YFM(tgsNrTwJ?qyu$97a0Ueca;v74w3FA-CeqeF6b$PUedj# z`$+ed?kC+}dVnq%D1$-LgQbT^50xG!JzP3e7mSd>Na<11qov14kCh%L9i|J$%V2`^ zMCnP=lclFfPn8bW1=D0OU3!M}OzBzDv!&-q&$YfiV!ea-gS(j)C*CrFE$m>ASa5(N zoZyVuhyxeIMLfhu0whEtBt{Y>MKUBu3Zz6Tq(&N~MLMKM24qAgWJVTbMK)wd4&+2G z3ZXEHpeTx=I7+|`B~c2cQ3hpE4&_k+6;TP5Q3X{|4b@QtHBk$- zQ3rM5j(Vt%255*zXpAOkie_kz7HEl9XpJ^#3lDgr9ooYSe9stGI^ixPhCvh1IqU z?zjEF`g)}19;vxU%0G7I9;u&4yH5WN_ejn2$~>?5=gB@G$iu$_Uq_p-CvN^d82^Rm zi@rYa&)my?91$Kmt8#euOI(8l<%xR?|R3U2t*eIp(}zB zf^O)J9_Wc)=#4(;i+<>j0T_ru7>pqpieVUzP>jGxjKXM)!B~t#7{+4)CSnpMV+y7s z9MdozGcXggFdK6)7xOS53$PH2uoz3Q6w9z2E3gu)uo`Qy7VEGc8?X_Zuo+vh72B{K zJFpYGup4_2fxXy={WySwIE2H9#1S0DF&xJUoWvu0A|28r12Q5L zG9wGJA{(+J2XZ18aw8A&A|LX@6$MZbg-{qpP!z>b93|j}k|>4JD1)*nhw`X^il~Ij zsDi4fhU%z+ny7`^sDrw2M?KU>12jY1Wg$F#*4(;IuZ*+hU ze9;kq=ec3qmgh!bJlmho6LT+H9-Jo43B3)abCz4Qj@jnbQ>H%o7k-l_|>$$(F2 z_Wf@$NzAAl9`nvQD>6_BGbir*I+>yR3eNXzn^aJUK z(vNh(V;MY=ek%P;`nmKA>6g;4bir#Gypet@{Z9J5^atsW($Tu$lMFsff06zw{Z0D2 z^p8Kgp+y!nw2^_Ww4JoQbS!BHX-8=%UEnN(*wS&NU8Lhm$CHjPoj?~PltCit#L`Km zlS(I(PA;877o?OyD(TeHX{6Iir;|=Eok165ltCux%+gt;vr1=^&Muup7vz*dF6rFT zd8G47=abGa?Wzk3$e^HfA?d=>MWl;L7n3fo3rfhqO}eCXDe2PEWu(hWm(vC1Wl%x7 zqI4zc%F=zK)RuHBk9J{O{AOZf@U&k zF5N=9rF1Ll*3xaH+v)-j8F)&!lWs5VCG9QULE1+b_{yN8bSG&)>CV#r(gD(ex}b{; zf~31j2TO-Yca!ce-9s1jltC})-qL-f`%3qd?k_z+7YvlaAnC!3R#hwFlAGMFwsLwct4Ea}*BM*<{7A|yrCS*nyWJNY)M-JpfF62fY8T};g0|Wq6>o16~PEWH*`l2^h7W8Mj!M=KlH}{48$M|#t;m} zFbqd1MqngHVKl~IEXE-W<1qmfF$t3~1yd1@X_$@~n2A}KjX9W$d62#(?y zj^hMQ;uNBA8fS18=Wreua1obq8CP%>*Ki#-a1*z18+ULQ_i!H%@DPvi7*FsN&+r^C z@Di`^8gK9x@9-WU@Db7YgwObbulR=V_~SY7^IoR3U2t*eIp(}#_ z7khUdRE4_seSB>|K}E%G#m2@)?8e4G0Z}nfR8+)nY;5fA?pAEZ0z^D_BoEf zcOPfYv(NL_bGV)PIN$Md_BGsc)&Ukb3*FEiJMZw7yZy50}zUV7=*zXf}t3O z;RwSBjKnC6#u$vnID}(7CSW2aVKSy*DyCsNW?&{}VK(MqF6LoA7GNP3VKJ6qDVAY5 zR$wJoVKvrZE!JT@Hee$*VKcU1Dpfz zIEhm@jWallb2yI+xQI)*j4QZ`Yq*XZxQSc1jXSuDd$^AWc!)=Mj3_+8Q#`|SyueGm zLNs3E4c_7%-s1y4A_kxE8DH=f-|!tjV6o#~Hf&%EJH$a;*uw$w5FZKPh=fRl#Bf3q zBtCO9KAvLGw6AvC$Jh_+~l-#l*E#PGP`P}AJYmWND}<^*Ex30OZ~ zF!!=!?LWVH!_5hp6EG)WP9T<_fc4`5b1yrVzY~}@-<*Is0doT81k4GT6EG)WPQaXi zIRSG5<^;?M{N)6!UoZH(_p)8{*jT)a+gSWd^0T4Kau*vBJRf-lPCO^+3>Qxui>tSd z#SQ*}zrB~;V7^?)SRlPndXe;E=_S%jrI+c=k>xU2A-z(1mGo-qHPUOP*Xe@wGT0!! zQF@c~X6Y@`Tcsm(!8RH24Gll8T!znA_X z{ZTqb7krYzXX!7}U!}iEf0zF8tGz6;!0Qhg*ht$-+eycfjw@|1?VtGDs_(PCC7G2I-8_nWUX{L1r0bk40227k59^oq$^8Tk*+H3p$n?Xpt^Jo>6+5Dq-#spk@nODb!AXb zy1sM+X)o!9(v76Ob%Boze5D&p`$;#EZYteO+Futmmq82ZmeQ@H1EgC^w~-Fi1#M-} zPP)BxkaP#>j?$f^gLOe?8H7l8k?tzpO}e{u59yw|pqC7KOZSoPE8S1Jzw`j6y~Aq-RUdk)A94>s~f@GcED?%mg;Dg&pD`F6`lec!-Y#a702RLSi@}36dfi zk|PCDA{A024bmbV(jx;hA`_gE8Cj4O*^nJMkP|M*h1|%4yvT?AC;(R!L?IML5fnu+ z6o(s1z#Sz~3Z+p7Wl;|0Q2`ZE36)U=RpEhZsE!(_iCU?!849l?sE3pczu?B0g4(qW28?gzSu?1TZfo<519oUIo*o{4i#9r*f zejLC-9KvB7!BHH;ah$+OoWg0G!C9Qcd0fCnT*75s!Bt$tb=<&B+`?_#!ClUg8y^@fvUN7Vq#LAMg<|_=L~+g0J|7@34Md;^+MbbMMyNOE>qg z`Ts!fp2x<9VY`Ldz?y2)SyyjkNP^`F@ zY99YuS)a)=kAJcB@lQV<_)qR-ryk1llyHIdm;U#x_p$^3`R9KQx01L4Qi?IYtu?)+x0xPi!tFZ=au@398 z0UNOio3RC35rJ*kjvd&EUD%C1h{Rs(!+spVK^($i9Klf>!*QIzNu0uIoWWU~!+Bi5 zMO?yVT)|ab!*$%iP29q5+`(Pk!+ku!Lp;J`MBxdZ;u)Uf1zzG6qVXDU@D}gz9v|=# zG5Cbf_=2zahVS?ROC0WH!v?mnLmb3~Jsc1Z@sR+INQgv83@0Q(QY1riq(DlfLTaQz zTBJjIWI#q_f-^EB3$h{`vLgp_!UegI8+niy`H&w4;EIAMgu*C-q9}&qa6<{Wqa;e9 zG|HeX%Aq_epdu=vGOC~|JWvhQQ3Ewm3$;-Po~Vm@sE-ElLPIoyH+tXfuznn1?q$dFcLMX~n-ef6U{1iCfH?tk0_FtF378WwCtyy%oParjznp;e z>ji)JUiK00Sx0xZvBdP`XG8bnE;c-QJ~9}~cuq0`kz;tB?8!EkGl*II+k4qw^W{Ru z0_la)i=-DzFOgm6Ox}q*qI?kzOmkP8Y0~!3OD#(wn3=OK*|hDjlH< zw#k5RX!v6FdOX*kA(YoNZ4Bkk;m3}AvUiyRdN9h<{@JR-rrN2mj zmHsCEUHZqbZfKDO4gZjVjkK+_opc=OxYG904!R(o4B|^Ckam<#D4j?;v9yyeNFsxz z(#fQgOQ(=dDV<6>wJt~_gS67=q|-}hkj^NbN!nQ#WR^h|>8#S(q_a!skj^RXq6>1# zAh&cL>AceUr1MJ`kapDt1!Yi3y0CN+>7vrbq>D?t>4Fk6aF;GAT}ryNbQ$Tg(&cnP zc^OoYt|(ney0UZ?>8jEmx}cg2s!P|9t|?thy0&y3X-{2HR|fT@>q|F~_L6QW-ALM7 z7x>7)SGuvZpL7%Hrqa!%{dGZe8MKgYDcwpsK)SVb8|gq@&{hWRq}xjeNq3O$DBVdq zSQm7bL5Or0>8{eO zONZ%#5i%GlJxY4C^cd-}(&MDVb-{QUOpu-^JxO}9^c3l-($jRobQ#Q$o+&*`dbac& z>ABLs?qzc~(~^MCOke|B*dY$$!X6HYhxkYUMZpO5sD;|715eaNJ=8}7c%dO0!5cpCMPvA(37VoA{LvgO z&=Rc>fYxY(K(s|Wv_}v+pd&gV7@ZMnV#$p`8F&+~z5tA?(Q!o|NFdZ{66SFWIb1)b4FdqxB5R0%FORyBn zupBF}605KpYp@pUupS$*5u30XTd)-o*oN)cft}ce-PnUj?8QFp#{nF~AsogL9K|sl z#|fOoDV)X`oW(hu#|2!(C0xc8T*Wn9#|_-XE!@T(+{HcI#{)dXBRobFp5Q5-;W=L5 zC0-#Kuki+N@ec3t0Ur^APxy>4_=<1%4(rz?e%^ohZ+#tLbKg96-ZM7u&(<$OTEFVZ z|F`4c-8Xml`qz8e<{oZHaFDr&Ytt*h+{10d`+<4=cylk`JRVrg<3ULCjY z&!5xspWMq{w~yy35ee%r{qI@tWn1?DZ#SjxdbCNQ}a0jKNrpLpa7`0w!V-CSwYw zVj8An24-RwW@8TKVjkvW0TyBr7GnvPVi}fW1y*7eR$~p;Vjb3F12$q4He(C6A_Cj6 z9XqfSyRaL35Q)9mhy6H!gE)l4ID(@%hT}MalQ@ObID@k|hx53Ai@1c#xPq&=hU>V2 zo4AGBxP!a6hx>Sdhj@g?h{6**#WOs|3%tZDMB_Ey;4R+aJwD(gV(WJeCXfuznn1 z?q$dFcLMX~n-ef6U{1iCfH?tk0_FtF378WwCtyy%oParjznp;e>ji)JUbg=d8%x** z8%y{Wem3-W?qVZ~?{9G6ImvQ7C+P|A3)ahEgY-t}P12jCw@7c5j?e|$WWYBx{JfXl zaEJ6x>0Q#hrT0ij>Vmy8*eAVT`hfI7=|j?orH|->qcS)qeO&s4^hxPc(x;`*=z_B{ zI46Bx`hxUD=}Xd=rLX9Mt1`GIeO>y7^iAno(zm7W=z_a4xF>yI`hoOA=||F!rK5Dg z6B#^}ekT1~`i1mM=~vRxy5O}8-blZdekc82`h)aG=@?z`Nd}*#zesyT$)uA@r;tu5 zok}{jE=VJTw9@IM(@STN&M2Kp+F2K5mO&QjtkT(}vrFfY&MEDp3v$UIw{#xqywdrk z^Gg?ycGU$1Wl%`EuyhgWqSD2ri%Yxdf)X-tmo6z?O1iXk8R@dp<#a)L8B~z2C|yaq zvUC;cs?r|1pqdP-OV^ODDP2ptwsak7PhC(~2KA)tOE-}Al5Qy7NZMN$_{hLly0NsM zbQ9^O(#@p(bwP6(w2*Eo-AXz@y0vs0=|El3RtD{)+e-&YcaZKV-AOuF7j%|Eh;$d} zuF~D4yG!?w?x_oU$)LA%AL+i*{iOR#50DPk1p{R;NP4jJ5b2@P!=#5xhv|Y5G8id6 zN_w>P80oRnc$vuz@Y?5C?H#4+q3Ud?bJ)5+V^2!wE@{6v>brDUcGWkQ!-_7U_^48ITc~;Ec@3 zf~?4f?8t$fa6vBQMjqrvKIBIMxS}8mp)iV|D2kyt+)x7UD2Y-ijWQ^Uawv}qsEA6a zj4G%K4^%^S)Id$tLT%K6C+eae>Z1X?&=8H_4IlWTG5pX3P0&ImylbVWCGM-TKwFZ4zq^hH1P#{h(4AO>MDhF~a$VK~At0wXaB zqcH|!F%IDvj|rHFNtlc&n2Kqbjv1JVS(uGEn2ULsj|EtWMOcg_Sc+v>julvmRalKR zSc`R7j}6#}P1uYr*op{j!*=YzPVB;N>_H^opBG9u_i$s$JzVp69}r+3?=807+ja^H z=xX^xKOXo`?qz%M(_lQ|jlX_;VZE0Q2B8Bwq7#DA86oI` zuIPsD=z*T-h2H3czUYVk7=Ta=#2^gD5DdjI3`ZD7U?fIiG{#^o#vvTzF#!`X36n7e zQ!x$GF#|I(3$rl?b1@I|u>cFP2#c`vcx3ahaOYq1XNu>l*g37fG6TM>b6 z*p408iCx%@J&43!?8AN>z(E|sVI09x9K&&(z)76KX`I1XoWprsz(ribWn95kT*GzT zz)jr3ZQQ|K+{1l5z(YL3V?^Nzp5hsv;{{&g6{7JPZ}1lH@E#xV5i$6L&-j9`_=fNJ z0gFBNvS9;T*dY$$!X6HYhxkYUMZpO5sD;|715eaNJ=8}7c%dO0!5cpCMPvA(37VoA{LvgO&=Rc>fYxY( zK(s|W{N{1PCX&aEGj_Ire!h}`&r{4Va{}fBV($r9KVC5RvSaVRzj@2e378WwCtyw> zcAtRt;{bCnJ9fVhn77`XfH?tk0_FtF378WwCtyy%oPaq2a{}fB%nAJU1gu{#uzr8) zkAL&a`uF&Lg+zbwW8`DFXYCwqV{v)M&xVfSE;d4WK5`ze@SJ26q8)54F;0BXLlRqy zOXhz(PZ`hR=&Tnz=F5eQ1=0(p7fCOcULw6zdYLX*E`t@)E2URSua;gTy;gdiE?6&v z4bmH>H%V`n-XgtKIzkt0lL6n*@bg}FqaD&arFTj1mfj;BsSEbXV4w7U=>yUSr4LCT zmOi2jj>_Pe^l|AE(kG=)NuQQJqYKW;;GFb%=?l^qr7uZemcF73uFBw=^mXYQ(l@1V zN#B;fqYLiJ;GXn-=?Bser5{N@mX6W|Ph{{^`kC}|=@-&3rC&)$>w?!Zcq9E*`knN9 z=?~H$rDJr#CmDQ}{v!QV`kVB3=^ww^+ae3R|B!)=w5_zAbR6lp()Q90x*(nm;!7ux zc9c#iok%*dw39AKB7>yT$)uA@r;tu5ok}{jE=VJTw9@IM(@STN&M2KpTL1sRJF^V3 zNN1JKCY@b6hjdP97kvwI$so6M9_hT&`K0qp7m#+<1qEeLNV>3e5$U4R#iWZ%yXk@w zGH{nJDP2msv~(HiveM;rL3tTekgh0QNxHIh73r$d9=f2K45~}lkgh3ROS-mn9cfQp zP*(=^r0YvJkoJ;pDBVcfTNn7qz*oAlw4Zbn>88@nr2Tb4a~ZUdZYkYLIzYO$bQ|eF zUC>qr?WEgF2T6C3?kL?!I#?HUmO+Sg7wN9j-K4uq_mJ+X3wp_*w{#!rzS8}q`%4dy z4%Gz%WiUv3u=Eh=q0+;ohf9a)f)O$pDLqPhwDcJ1vC`wD!*#)U8BCC#C_PDfvh)<` zsnXMQ!E_nSke(?$OM15S9O=2zzwTvoH`9`k&rDzgTi78E;=&#dh==${07oQ5A|!?r zk{~IPAvsbYB~l?Z(jYC;Aw4o6BQn7mnUMuqkqz0A13BS>T*!?)$cuc)j{t^ z6hToGLvgsF1l&;)rBE7WP!{D-9u-g#l~5T~P!%4ihU%z+ny7`^r~^;bMLpC<19+h! z8o?Vr@I_4EzlCJ5P;TbgFv)JJG4g-I-nyuAsC$zf-dNaZs?94=!stF zjXvm$e&~+@2*p4Q!e9)+Pz=Lxgkc0mViZPW48~#{!Z98bFcFh58B;J7(=Z(~FcY&d z8*?xh^DrL^un>!|7)!7e%di|PuoA1V8f&l?>#!ahuo0WE8C$Rw5!i<9*nyqch27YL zNbJQv?8gBd#33BU5gf%a9LEWq#3`J{8JxvAoW})R#3fwD6o5K9S?^`Lmifn@|2dT3tbgz4=OyjAn;C=-=!i}T zMrVYe3%a5kx}yhrq8ECj5Bj1X`eOh>F%W|=7(*}=!!R6S7=e)(h0z#;u^5MNjK>5_ z#3W3{6imf5OvenOCl9L&W$%*O&O#3C%l5-i0sEXNA0#44=D8mz@Stj7jy#3pRU z7HmZXwqZMVU?+BAH})VBd$AAuaR3K#2#0Y5M{x|taRMiC3a4=fXK@baaRC=`372sN zS8)y3aRWDT3%79xcX1E*@c<9;2#*nkCwPiyc#ao%iC2imYrMf*yu*8Zz(>U36F%b$ zzTz9c;|DAb+{=ayY+;8uhzol-ARgi)0UVJKiI5mhNP?tDhU7?rlt_itNQ1OUhxEvR zjK~CMWJVTbMK)wd4&;Ojav?YJATRPEKMKGV1yKlvQ3OR%48`Gw5^zUJltO8gL0ObT zc~n3}R6=D`K~;F58mglPYN8fuqYgY#7xhpd4d8`_XasNgz!#0-hbCx>X7ERIv_MO= zLI7H$4Fb^??a(>RzdUZ(c=NasSi#)OmWND}<^*Ex30OZ~F!!=!?LWVH!_5hp6EG)W zP9T<_fc4`5b1yrVzY~}@-<*Is0doT81k4GT6EG)WPQaXiIRSG5<^;?M{N)6!UoZH( z_p-yeXC0Bx))HBepAB7mio-Y?N7Dz9Y zUL?I(dWrN>>1BFzWVsAhNUxM$CB0gDjr3aSb-G}^3^qt_l-?x0S$d1~R_O>`uuTSh zL&MK|+1@*(cS`S)-YvaHI#L(xmBBvg{n7`d4@w`BJ}iAi7aWzrG3n#dC!|kGpOQW; zeMT3YmBBgb^U@cjFG^pMzASx37hILWHR7V$-;%y9eMcAEmBBsf`_d1jA4)%x zek>iO3!cc}sq{1H=h82vUrN7{j@AXQW$;G&t@JzT_tGDvKT5~wf=@E|Ed53LtMoVN z@6tbhwU0#>`1~OQ8);iC6K^pZhu=|0kZrTaE!>w@tzm>@k-dXn^H=_%4vrKjnF=`xrhJyUv?^la%l(sQMM z-OJ`~CO-wr@&{~S3p>O?T-d__@em&g;E04sgv4+{5+p@3Bu5IQL@K048l*)!q(=s1 zL?$>RGqNBnvLQQiASYaq3%QX8d65tKQ2?$eh(aigA}EStC=NH2fICW}6iTBE%Ay>~ zqXH_T5-Ot#s=@=+P#rZ;6SYtqb>NA*sE7J!053E|BY49HzGw_TG(l4|gFl+11zMsN z0?-<55Qw&DhxQ0U2XsUy1fw%T&;?!54c*ZLJ<$uj(Fc9e5B)I!p%{ok7>pqpieVUz zFpR)RjKXM)!B~t#IL2cFCSnpMV+y8X8m40gW?~j*V-DtG9_C{K7Ge<=V+odG8J1%O zR$>)aV-40~9oAz5HewStV+*z-0^6`1JFpYGup4_2iM`l|{WySwIE2GEf}=Qw<2Zqn zIEB+VgR?k?^SFSExP;5Nf~&ZO>$riNxP{xegS)tg`*?tdc!bA@!V^5jGd#x&yu>R+ z<2BykE#BchKHwu_@Cl#s1z+(E-(mf_#LxQ=w*1^UJH!EhW?m1PGq8a<7xTK{vElu| z-TQrO>*Imm%jO=cdHqoC?rvPT~|!;|$K?9M0ncF5(g{;|i|g z8m{98ZsHbh;|}iP9`54-9^w%mBMMLO6wmM+FYpqt5RKP(gSU8x_xOO1h`}d(#ut3W zH+;trSmJRn8#b_o9pWG^?BRfTh>rwtL_#D&VmKiQk|G(BBLz|-6;dM&(jpzwBLgxb z6P%G5S&$XkkR3UY6E4Vw+{lBx$cOwW09O=5ArwXt6h$!minJ*VI7Dz9YUL?I( zdWrN>>1BFzWVsAhNUxM$CB0gDjr3aSb-G}^3^qt_l-?x0S$d1~R_O>`uuTShL&MK| z**-g@cS`S)-YvaHI#L(xmBBvg{n7`d4@w`BJ}iAi7aWzrG3n#dC!|kGpOQW;eMT3Y zmBBgb^U@cjFG^pMzASx37hILWHR7V$-;%y9eMcAEmBBsf`_d1jA4)%xek>iO z3!cc}sq{1H=h82vUrN7{j@AXQW$;G&t@JzT_tGDvKT5~wf=@E|Ed53LtMoVN@6tbh zwXa1M`2HaS8);iC6K^pZhu=|0kZrTaE!>w@tzm>@k-dXn^H=_%4vrKjnF=`xrhJyUv?^la%l(sQMM-OJ`~ zrX?|-nZO3NutOZgg*_Y)5Al%zj!1|^NDL<=K~f|`a-={?q(W+>L0Y6kdSpOGWP&p? zBMY)38?qw@a>50lN;D1^c&f}$vf;&4L=xT7RWp)|^%EXtugDxe}N zp)#tVDm+jP)lmaAQ46(E2cD>ldZ>>E@Ipg0f;W8Ni^lLn6EsCL_@g;mpe0%%0Iksm zfoO|%XpbOtKu2^!FghayUCcO{6TQ$Ieb5*E&>sU3ih&q}!5D&}7>3~p!w8JT zD2&D!jKw&FV>~8cA|_!nreG?jVLE1DCT3wa=3p-7VLldMAr@gVmS8ECVL4V{C01cI z)?h8xVLdirBQ{|(wqPqFunpU>13R$`yRiq6*o%GGj{`V}LpY2hIErI9juSYEQ#g$? zIE!;Qj|;enOSp_HxQc7IjvKg%TeyuoxQlzZj|X^&M|g}VJi${u!*jgAOT0ofUgHhk z;vL@O13n@KpYR!9@D<LL@)G4 zAM`~(^v3{%Vju=#Fos|#hG96uFajen3ZpRwV=)fl7>@~#}8QIb1xe1SOq1pWV(kf7KVC5RvSaN(zj?#W378WwCtyw>mY;z2;{bCnJC?r_m^a^? zfH?tk0_FtF378WwCtyy%oPaq2a{}fB%nAJE1gu{#uzsiOkAL&a`uF&Lg+zbYSS*p; zvyNJ3Yl&XP_cW~ME;c-PJ~9yVcusO1BKGn+*+*?HM-YAWxA(Gr=gWnR1=0(p7fCOc zULw6zdYRrFSuTSW(krD`Nw1b(BfVC7oi11}gALLfr8h}$mfj-0RXRc!Y?A@s(D3tK zw(kz1xICYO!~O=3F(v4r=(9ypV0+p zWpGaVyz~X>i_({*FH2w11y^NoP5Qd@4e6WGx1?`N-_ZqkWpGdWzVrj>htiLvA4^B+ zf+sR~D*a6Qx%3O^m(s7KqjkY+8N88xEB#LTz4Qm^kJ2%^;FAnKOMj96D*a9RyY!D= z-Pj@v8vh{!8);iC6K^pZhu=|0kZrTaE!>w@tzm>@k-dXn^H=_%4vrKjnF=`xrhJyUv?^la%l(sQMM-OJ`~ zrp1ZROke|B*dY$$!X6HYhxkYUMZpO5sD;|715eaNJ=8}7c%dO0!5cpCMPvA(37VoA{LvgO&=Rc>fYxY( zK(s|Wv_}v+pd&gV7@ZMnV#$p`8F&+~z5tA?(Q!o|NFdZ{66SFWIb1)b4FdqxB5R0%FORyBnupBF}605Kp zYp@pUupS$*5u30XTd)-o*oN)cft}ce-PnUj?8QFp#{nF~AsogL9K|sl#|fOoDV)X` zoW(hu#|2!(C0xc8T*Wn9#|_-XE!@T(+{HcI#{)dXBRobFp5Q5-;W=L5C0-#Kuki+N z@ec3t0Ur^APxy>4_=<1%4(rz&e%^nGZTC>k+iU$ImAQXy-Vgrqe&Fu=ulKTfy;$>l zsR5QhTEx=VOEvd2%{@)Mr)eG!LV|3`39FFWR+fBxrCezX3)pP!es=Wb>YI-nyuAsC$zf-dNaZs?94=!stF zjXvm$e&~+@2*p4Q!e9)+Pz=Lxgkc0mViZPW48~#{!Z98bFcFh58B;J7(=Z(~FcY&d z8*?xh^DrL^un>!|7)!7e%di|PuoA1V8f&l?>#!ahuo0WE8C$Rw5!i<9*nyqch27YL zNbJQv?8gBd#33BU5gf%a9LEWq#3`J{8JxvAoW})R#3fwD6Ld2B;Z~)Y+wsJ z#6eux!vXOS9|_=ygh+(Ma6%F!MKUBu3Zz6Tq(&N~MLMKM24qAgI3qK%AS<#VJ8~c= zT#yU7kq3E^5BX65t|*8?D2yT~iee}ZHqYTQT9Ll2tDxwl9qYA3R1JzI+ zHBb|^P#bmNiMptV`e*nl`*9-pcy=?DmwwBOGwwAD`{A}o# z+{MNn>-RD6oTMAiNqWHZ2d|TDZ)fpGSSq_ezwaTQ#qs~Uhc{m?WGs+gD7{E}vGfw@ zrP9mv=E!mxtdL$Qy-Iqu^cv~4((81=dKqkx-YC6Edb9Kv>8;Wcx?r0O_=bj`_p%%B zklrc1OM18T9_dJ3uvZ5Ar1wi7kUl7VNcyn!5nXUp2FIk2OP`QFDSb-%wDcKWa8?HA zq|ZxVkiIB=N&2$%6yq3Wm>9^AFq~A+_kp3tgqYFOC;Is4>>95k?q`yo5_|<+E zS>X4F3~Z!rrR}8SNXM17mv+zv@njHRI)SvKbVBJw(ut*=bU_jsB$ZAkom@JFbV}(| z(y4Vp8X2UOPA8pSI)ijZ=}gl4{|A1VWspTWt8_N$?9w@;b4t7DTaZfzxux?+=atSU zonN|uw5u*CD1$=Mg{6x~7nLq1U0m8t7nG2JyL3tEQqrZR%Se}%E~g91%bnAU%G*`mvlqvM$+E8z()qY(v7A4 zq?<@Lm2M{OuM3*XpoMfx=~mJK(ygW2NC)bIwlZiZ-CjCKx`T8_=}ywYx}dWRLZrJ$ zca`oY-CeqebWdH-O9s8A`$+ed?kC+}dVqAOE*L0-LDGYzhe!{V9wt3pI!qUgkikgl zQPQKO$4HNr9w!~H3&zV}g7ie`Nz#+0r$|qgo~8??%V37|OzBzDv!&-q&z1glFPpoW zmLz;;0vp)E4sj3{_HaNv#76=+A|VnXF`SSDNs$c6kpd}^3aOC>X^{@;kpUTz3C_rj zEXay%$c`My2^Zu-Zsb8;fQqPu z%BX^>@IW>6n3;n1$JxgSnW8`B;F3ScJt`f~8o79U zcX*Ev_=p&M!e@NJSA4^FSidgu^ZtYN>k~Gxg&oX&RqGdV%)Mpveh@p}58NC7>%DAq z?=B$Vzxm#s`S&&dzC60wc5mA$D8RfvwvG8Y1m>Q#dA$Gicwqhf;D2*3yIc`Fiw8Vm z{iXjs>%HvofByNOL;216_kMm}(w@7SLFj;v=!9T&MhLo~E4raOdY~tIp*Q-VFZ!WB z1|SpzF$jY(1Vb?l!x4rN7>Q9BjWHODaR|qFOu$4;!emUrR7}Hk%)m^{!fedJT+G9K zEWko6!eT7JQY^!AtiVdF!fLF+TCBr*Y`{ir!e(s2RzzSMwqpl&Vi$H}4-De za1e)Z7)Njv$8a1ca1y6*8fS18=Wreua1obq8CP%>*Ki#-a1*z18+ULQ_i!H%@DPvi z7*TkFr+9|vc!8IAg=oCS8@$CkyvGN8L<~ORGrr&}zTrE5z~acgY}mjSc8G(xu!jTU zAwCko5ebn9iQ$AKNQz`gjuc3VR7j09NQ-nxj||9&OmIeKWIb9BwE9ca%galtvkpMLCp51yn>OR7Mq4g$JskI%=RMYN0mj zz!P;*5B1RiUTBC$@P-e3(HMScf~IH&e>6u6v_vZepf%bc5N**8zj@rSiQsYLh}%ED z-ys2?qnlsm1k4G<-V?BXykPES$KHQ`^Ol1SifHIckg9KbI&?;Ham;6i=D+aubssW zJ$XKI8qV;X{2va`g%+3UCWvi;`Eg^UH#3#AuHFP2^+y;ORc-W*vjgB8*% zrB_L>mR=*hR(hQ-STBPO(i^2WNpF_kBE3~QLKke40pHN@^Io>!4(XlJyQFtZ?~#tw z1$$+%PkO)f0qKL%holcnAJGLzWpGUTxbz9>lhUW8PfMTC1!rY&PWrs`1?h{@m!vOC zU(p3uWpGXUy7Ud{o6@(WZ%g0N1$Sj|Px`*}1L=p-kE9<3+b2A zucV`O!D|`3k$x-vPWrv{2kDQ}F}mQB3_eSLk^Um2_%dkVXb+rPE2L zm(C!aQ96^fvo6RigDlcnrL#$Am(C%bQ`$ur4Neys32WYx{`Ed=_=Ayr9E^(H5pWw zt|47hx|Vcp={nM$x}dHM>Pgp^ZXoR?-B7xbw6`wsk%6ytV`)F>Celr%n@RiYg61-4 zA>C5Cm2`k~Yw0%9fx4ir4BAPzmkyHdAl*^AlXS2y=q!T}=`PY;rMpRYm+m3mQy27- zL2v0k(tV}-N%xl?ARVd;2FhTN^kC^B(nF<(Ne`C}(*+}BFj9Jy^l0fZ(qpB^Nr&r# z@iLeoJyCj+^knHN(o?0U>4NDpm?1q=dY1HT={eGKrGMSa=5D4XDW93Z2DY$69K?k^ z91sujkpPZJh(t&XCnP~qBtvqfKuV-SYNSD0q(gdSKt^POGcqF!vLYL@BL{NA1-XzL zd5{MZw7yZy50}zUV7=*zXf}t3O;RwSBjKnC6#u$vnID}(7CSW2a zVKSy*DyCsNW?&{}VK(MqF6LoA7GNP3VKJ6qDVAY5R$wJoVKvrZE!JT@Hee$*VKcU1 zDpfzIEhm@jWallb2yI+xQI)*j4QZ` zYq*XZxQSc1jXSuDd$^AWc!)=Mj3_+8Q#`|SyueGmLNs3E4c_7%-s1y4A_kxE8DH=f z-|!vQuS@*A|6tGeH#^|R59?Pi%zb2YAKAPg#HOE5;O_UY_p*b#nb(I6uvpCN!-h2P z*oM~$=Jk@zJ#}-B*}Q(ddA(@!c);VpfBWmlTR%Ve-`vZ-(wXNe5e4fn{qI@tWxIs@ zZ#SjxdbCNQ}a0jKNrpLpa7`0w!V-CSwYwVj8An24-RwW@8TKVjkvW0TyBr7GnvP zVi}fW1y*7eR$~p;Vjb3F12$q4He(C6A_Cj69XqfSyRaL35Q)9mhy6H!gE)l4ID(@% zhT}MalQ@ObID@k|hx53Ai@1c#xPq&=hU>V2o4AGBxP!a6hx>Sdhj@g?h{6**#WOs| z3%tZDMB_Ey;4R+aJwD(gV(WJeCXfuznn1?q$dFcLMX~n-ef6U{1iCfH?tk0_FtF z378WwCtyy%oParjznp;e>ji)JUUnGwtk*TSvqS{&v!UB^7aLAIAL)ufo|Ei}(B8aG z_8>b;I3gze_Fi_A`Ens+f%HP@Mbe96FdOX*kA(YoNZ4Bkk;m3}Av zUiyRdN9h<{@JR-rrN2mjmHsCEUHZqbZfcPQP5+RAjkK+_opc=OxYG904!R(o4B|^C zkam<#D4j?;v9yyeNFsxz(#fQgOQ(=dDV<6>wJt~_gS67=q|-}hkj^NbN!nQ#WR^h| z>8#S(q_a!skj^RXq6>1#Ah&cL>AceUr1MJ`kapDt1!Yi3y0CN+>7vrbq>D?t>4Fk6 zaF;GAT}ryNbQ$Tg(&cnPc^OoYt|(ney0UZ?>8jEmx}cg2s!P|9t|?thy0&y3X-{2H zR|fT@>q|F~_L6QW-ALM77x>7)SGuvZpL7%Hrqa!%{dGZe8MKgYDcwpsK)SVb8|gq@ z&{hWRq}xjeNq3O$DBVdqSQm7bL5Or0>8{eOONZ%#5i%GlJxY4C^cd-}(&MDVb-{QUOpu-^JxO}9^c3l- z($jRobQ#Q$o+&*`dbac&>ABLs?qzc~(~^wOOke|B*dY$$!X6HYhxkYUMZpO5sD;|715eaNJ=8}7c%dO0 z!5cpCMPvA(37VoA{LvgO&=Rc>fYxY(K(s|Wv_}v+pd&gV7@ZMnV{vUgH9aROo_I-SdfY{jB-Q9|Xjg5^> zh@he%7B+TwcPlnFHgM z@EKq572oh3KkyUQ=Otq9KbY6aHuuua>!tJGyjc486F9f|`@L*)57(y;Pvx4|6ZZG? z?#%sk^LoPO^@Ppq37f|QFY|cdAJE$pC&=f&`|-f~`N4m3FFSApU#CO}tUvVkv);>& z`19xg7|K7azb@wUlFr=C^hXzTMF0ZP4MFIR9_Wc)=#60XL0|Mke+BAf+Y#}vSEX`u!SA$5f2WCj|51FL`aMz zND4$ zq8N&!1WKY5N}~+Q!WrdI9u-g#mEeNPsDi4fhU%z+ns7xe)J7fDMLpC<1Gu3f8lf?o zpedT6Io#0#Ezt_C(FSeN4j%A?7rfCPKIi~n_@N^@!TPZ9m&Xm8XdX8{Z#MU`>E+TZq*qFZ>dlc=GFUCWMtZIEI_dS& z8>GW@!A2QulHM%6MS83BHtFrs;ksal4ETnIn0wi6cS-M--XpzNdY^QJF4!-F1JVbj z4@n=EJ|cZo`j{>_E`t-&C#6qGpO!u&eOCIME;uiP3(^;*FG*jPz9M~9I#L&0lfiZA z8`3wWZ%N;lz9Su_3+~F`p7ed`2htCvA4xx!j@AWFWbjn_ne=n%7t$}KUrE2#1#e{V zR{EXvd+86-AEiG@f7S(GWbjq`oAh_-AJRXifBo)u7Fp0Pjtp$1<4W5~+ezC?$CGx@ z1@UE&KsupxBI(4^Nu-lXJL-aDGDt3+LOP{%D(TeHX{4QWL0TE4lTI(4K{}&!Ch5%5 zS#&{G8Dx{rE}cUMRY+?85EN)E?q*pq;x6i z($ZygL0K6%OP7-_FI_>pqI4x`7hO`JvE?q;qrnIXrs3n8i(siWkO4pOF zFWo@eO&2tjK_ltL(oLkBN;i{kF72)hTF9WKbSvrB(ru*MO1G2t&;_0{@RIhHZZGX4 z-9g${+D{jBltCxy&eHzUU8K882S^9%f^IShlI||uL%OGQFX`UW!MdQ24Ejp4JD1)+aMmdy61yn>OxS%qspem}NI%=RMTu}?PQ3rKV5B1RiZfJ-` zXpAOkie_jIceFrDv_fmNL0hzg2Rz{gZ?uOGI=~lx=!j0}41aV%R|Fss-4KNC=z*T- zh2988AM`~(^v3`U#2|!VFos|#hG95HU?fIiG{#^o#$h}rU?L`AGNxcEreQi}U?yf^ zHs)Y1=3zb-U?CP^F_vH{mSH(oU?oDa3ahaOYq1XNu>oP&h)vjxE!c`}*p6`Qz)tMK zZtTHc>_Y_h;{Xog5Dw!Aj^Y@O;{;CP6i(v|&f*--;{qr6Sr_1 zcMyfUxQF|AfQNX5$B4!gJjF9S#|yl~E4;=Vyu~}b#|M1GCw#^ie8o3>#}E93^?8Yy z`w!-IvU%*|POf>KZ0nO()@Q5uZvxDH{n&9|-?{DI?`4~Nxc}Yv?#!RB`SUgRtj#@Z zbI;m8R@}4xcOUPqpC9}u_p%*c@O4T!!TLjgKkL10_dkFBkD>g-`s-poFX_zPOn-Di zR|Fss-4KNC=z*T-h2988AM`~(^v3`U#2|!VFos|#hG95HU?fIiG{#^o#$h}rU?L`A zGNxcEreQi}U?yf^Hs)Y1=3zb-U?CP^F_vH{mSH(oU?oDa3ahaOYq1XNu>oP&h)vjx zE!c`}*p6`Qz)tMKZtTHc>_Y_h;{Xog5Dw!Aj^Y@O;{;CP6i(v|&f*--;{qr6Sr_1cMyfUxQF|AfQNX5$B4!gJjF9S#|yl~E4;=Vyu~}b#|M1GCw#^i ze8o3>#}E9(FIbXtFB>+93tQO19`WFS_(*_+NQA^lf~0UnG9*U|q(myDMjALFEz%)9 zG9V)|Av3ZdE3zRwav&#iAvf|MFY+Nj3ZNhgp)iV|D2kytN}wc4p)|^%ESymeXB?fH?tk z0_Fr_`3YD*4lwt!WBEIQdGpN)m=iE3U{1iCfH?tk0_FtF378WwCtyy%oWNgB!1{c_ zzk4q`oO{+$H*GD^QT*M|5BR=@hqe|Mz8>j;S$s`06k#8Do$MdBmIy>Aw)^Az9^zXP z{jYm?i{(Pb66vMV%cPe}uaI6T9jZ4+R>@$s^cv~4((9zxOK*@4(*+x4ut|Ee^cLx@ z(%YoBONZ-%9Wvk>8e;Bcx7#JXTY8W5Ug>?(5xQW%3=T*ils+VVSo(Tl$W4lrFd{ zgL~5Vr5{K?lzt@rSUOr4Jdwdu>1Wch2K2CrGH5Ol>YU*JuI@oBaRGgq~l84O4~`>OUILT&;{{jkU%=2bRy}*(n+L~ zN;~R;WHLxDokBXLbSml8(rKifbU|7fq?1lBok2RIbSCM{()!;6kE}AtCY@b6hjdQq zT++Fv^XOZUR|ff{^Gg?yE+}0{y0CN+T~Jg8#iWZ%myj+gT}ryNbQxVxRtC<}<)q6? zSCFnKT}j$S7gUx(73r$d)ugLS*O0C$?WzlE$)L7$9qGE#^`z@dH;{JI1r24;NV>6f z6X~YX&7_-4yX%4$GH5B?O1ial8|k*v?W8?)fu{_-q`jrvOZ!N7koJ}K(*+%6&`G+p zw7+y0>8{cN(t*05n+$@ayG!?w?kU|%y0>(&F6bkJzS8}q`%4dy9wC&f7VY2xPk6x_?cswC@P!{bq7yp9A6?KD0SH7l1fe^6peK5vH-gazebEp7F#rQG z2q74ZAsC8b7>*GbiBTAhF&K++7>@~Q~(IEVANfQz_<%eaE8h{QEq#|_-XE!@T(MBy&(;XWSVAs*o| zqVWVz@eI%L0x$6juki+N@ec3t0Uz-RpYa7>@eSYc13zJXULxlHgSn4vebUCg*V zvGe`Fx!vFIW%GL3=Jld|EOE@eMRRY_m-hqn`tj!Rz~4L`*!1x+j|Z{g@xc1|!GCfu zdr>|+ODMu%{h_~~^ta4H>CD|se{?}t1RxOI5QOgNfu87v-Uvn? z^hH1P#{dk(AcSBrhF~a$VK_!$Bt~I0#$YVQVLT>aA|_!nreG?jVLE1DCT3wa=3p-7 zVLldMAr@gVmS8ECVL4V{B|@Sdhj@g?h{h8<#WOs|3%tZDyv7^6#XG#m2YkdQe8v}i#W#G%5B$V0SRA>R4I9LT zE$m>AcyK^`BtSwWLSiIAQaB5v{7kP(@X8Cj4O*^nJMkQ2F( z8+niy`H&w4P!NSs7)4MN#ZVk2P!gq38f8!x&M1fSsDO&71Q%3B6;wqvR7VZegez*H zHtL`*>Y+Xwzzq%22#wJMP0@PH@0;Ene1K?nH44;|47|M0kB zfAf}`6EG)WPQaW%>^=eO#{uSE zcIGRyCt!+1gEpJ zgk-X_Ov+|wnT1HckKr?l@HI&nxE8mwxRtZBxFe*_KiJB?2rN9&=7Mk+hdpX zZs|SJd!_eDN9cn6GB_Z8Q2LPcVd*2%N2QPHg5xqcA$?N%l=NxoGty_J&*_5mGPodp zQTmefW$7!@SEVC$!8I9Nm%br=Q~H+lZRtDGQM%x+4DLzamwq7qQ2LSdW9evJ@I(er zrJqSZmwqArQu>wjYhCb025+U`Nxzr=ApKGLlk{g@@I?k+rN2pkm;NFBQ~KBM_O!?X z&p0x$k&Y{ED{UujFC973HJq;pH>(FJ*BkWV_lbOGst(uJf8 zOBc}vMP*P-y0~-+>5|f=q)SVe(FJ8?;4EEEy1aA+>59^oq+N7DWf@eFt}0zky1H}? z>6+56x}cT}YD?FVt}9(ny1sM+X*XTaPzH^p8%sBlZYteOy1BHwE@&ZxmeQ@HTT8c* zZY$kR+Cvw3%D_w7Te`ipk8}rVUui#G&`}1Rq&rLdOLvj(Djgsls0+HuAV|8qbPwsC z(!Hd6O9$(MJ~HSl-A}r|^Z@CB(u1T!birU543QoxJxqGI^a$yZ(xY_2Xc>%=9xFXg zdc5=m>50;lbirg9Op%@{JxzML^bF~l(zA5IY#Gdvo+~|1dcO1m>4nn2?`3m0(~_Fc zOkjh!u!SA$5f2WCj|51FL`aMzND4$q8N&!1WKY5N}~+Q!WrdI9u-g#mEeNPsDi4fhU%z+ zns7xe)J7fDMLpC<1Gu3f8lf?opedT6Io#0#Ezt_C(FSeN4j%A?7rfCPKIi~n_@N^@ zp)>r^1zizr+Fc5!Y#VV}E8mz@Stj7j~ zVIwwSGqzwWwqZNMu>(7?3%jugd$A7@*pCA^h(kDxBRGm5nexiU0(n8-masJO+%gTCm8{uqFP7=#cE#t;m}Fbu~CjKnC6 z#u$vnIE=>xOvEHi#uQA&G)%_~%)~6r#vIJWJj}-eEW{!##u6;WGAzdmtVAeQVKvrZ zE!JT@HXsZeu?d^81zWKV+Yyc(*oj@(jXl_leTcw*9Kb;w!eJc2Q5?f@oWMz(!fBkr zS)9XpT);(K!ev~+RYc+%uHy!7;udb>4x(@u_i!H%@DPvi7}0owr+9|vc!8IAh1Yn4 zw|Iy5_<)c2gwObbulR=V_<^7J1xqsSWy1z>VGBFhBOV+O9|@2UiI5mckQ9zchU7?r zlt_itNCPLNMLMKM24qAgWJVTbMK)wd4&+2GbJSw0fD!~PnQ3X{|4b@QtHQ|a{sEs_|G(uxEK~pqCbGV}g zTA~$NqYc`k9X#L(FLb!Tcd-%3*CQw47+;f&L{y-iB|6y7@)}N~ z|M6b7=VG~#u|#^Q^fKw?(krA_N{8yrkySESExks1t@JwS_0k)p!*sz$8Elf?EWJf~ ztMoSM?b6}8V22F&hK87X*`B+kcT4Y)-YdONIzkuhm%#z)gVKkj4@)1BJ}P}o7aW(t z3F(v4r=(9ypOHQ*eNGpgm%#<;i_({*FH2vMzA7E53$DrFy7Ud{o6@(WZ%f~ij?x8p zWpGdWzVrj>htiLvA4^B;f+sR~D*a6Qx%3O^m(s7KU+aQ5GI%TfPWrv{2kDQ}pQJzQ zf-f@oD*a9RyYvs~pVGg6x0gj0c*T){jdWaTTWLFKd+B)64!R(|3=&8ulujg_SUQPx zQfWtBkW2>2rBg_!lujj`S~`uilP*XrgLKmAr87upl+GlbSvrd@$SQ+u(%GeRNavK! zC7oM3k1oh7gM8BYr3*+GlrAJ)Sh|QVC@O3-7vr3Xk4lpZ70#2trAJ7QlpduEM$2G~^jPU}(&MEkNKcfWqzfj?V2boq>1oo_rDsUbl%AyvX3JoX z^jztA((|PkNH3KBeJ`84nU*wsW&#_;g)Qu0k9crEd?Y|ZBtl{&K~gv(8ImIfQX&;n zBMqF87U_^48ITc~kQrH!71@v-Igk^%kQ;fB7x|DM1yB%$P#8r}6va>+B~TKjP#R@W z7S1S#@~D7{s00^OMio>=HB?6p)PyT)p*HHEF6yB^8o&(=(Fl#v1WnNl&Ebv~Xo*&6 zjW%eDcJP2Fyx@)Y@IeRo!VewM37z4OF6fE?1fm;)&>cO{6TQ$I!RUj&=!gCofPol< z5Ddl;48<@E#|VtXD2&D!jKw&N#{^8oBuvH>OvN-z#|+HGEX>9n%*8y+#{w+GA}q!d zEX6V`#|o@OC{|%L)?h8xVLdh=3>&cto3RC3u?^c1jvd&EUD%C1*o%FLz!*QIzNu0uIoWWU~!+Bi5MO?yVT)|aD;u@~w25#aOZsQK3a2NM*9}n;lkMJ1L zc!H;RhUa*Jmw1KOc!Rfihxhn^kNAYo_=2zahVS@+pRhhJ5p(~++($O|(s^Gn_tInO z-%ntDzS8=55OWX9`hCuUJ%a+x>%&^i>%#_lb~X1>Epg53$D79kAB%ZB2r!Qayc_)Y zKOX4E1IhmU_`v#g%>V0$^?%2`>$vBZ^L0u@!TLjgKkL2hEGz!?^MC#QtiLYi^ODZo z&GbhXbVUFH(G5Z9jvnZVUg(Wr^g&=dVjRX} z0w!V-CSwYwVj8An24-RwW@8TKVjkvW0TyBr7GnvPVi}fW1y&*ytFRhtuommE9vcvb zjo5_E*n+LthV2N)4(!A(?8YAK#Xdw}KMvp^4&gA4;3$saI8NXsPT@4p;4IGJJTBlO zF5xn+;3^_<4cBo4H*pKMaR*Vji+i|_2Y84_c#LQ~!BafLbG*Pyyuxd|!CSn;dwjr0 ze8OjZ!B>34cl^Ll{DLJp_p)JwxUhvC>=6$Rh>rwFh(t(?BuENJBtvqfKuV-SYNUY^ z(jpzwBLgxb6EY(UvLYL@BL{LK7jh#H@**GdqW}t`5DKFRilP{bqXbH#6iTBE%EB4t zP#zUf5tZPA%BX^>sD|pOftqkdE!0LG)I~kiM+3N_AsV4EnxH9~p*h^q0xi)BtpLVBfisNNh|C4<${YoymouajOcy+Jxm7i^TlCh5)6Tco#2ZTwM%-p^d9NG()*+%bisZZ9FRUJeMtJS^bzT!(#LecaT%PDJ}G@l`n2>J z>9f-3bisKUT#&vfeM$PV^cCr=(viC0nhdT>-;lm3eM|bb^d0FaU2s*W&!nGAzmR?@{Yv_^E_fq@x6<#V-%Edx{wV!P`m-+hB7?8e-=x1w z|B(JE{p)voTV#QE92wY1$Cb8~wv)D(jwkJ)3*yTlfpkLYMAC_+lSn6(cGLyQWRP4s zg>*{kRMM%X(?~n%g0wP7C!Jn8gLFpeOwyU9v*?1XGRP*KT{?$!PU&3Ixux^yg1j=w zC!JrqfOJ9WLehn$i|B%)GAJfpT)KpGN$FD3rKQX0g0eDjmM$kIy2B;8%QhjdTrUedj#gLOe48T6IzC*5Cqfb>A=LDC_*V6Y5^NDq}BCOuqwg!D-1 zQMzEX48};0l^!QOUV4J`MCnPoV6qIRNKciXCOutxhV)G7S-N1h4CYABm7XU(UwVP` zLh0Z4vbmdSapE%**dQ)!VF!D}g9G9t0TLn+5+ezc!V$@k94U|zsgN3J;DoeDhxEvR zjL3w{$bziMhV00JoXCaT$b-Ddhx{mjf+&Q-D1xFWhT4JD1)+aMmdy61yn>O zxS%qspem}NI%=RMTu}?PQ3rKV5B1RiZfJ-`XpAOkie_jIceFrDv_fmNL0hzg2Rz{g zZ?uOGI=~lx=!j0}41aV%R|Fss-4KNC=z*T-h2988AM`~(^v3`U#2|!VFos|#hG95H zU?fIiG{#^o#$h}rU?L`AGNxcEreQi}U?yf^Hs)Y1=3zb-U?CP^F_vH{mSH(oU?oDa z3ahaOYq1XNu>oP&h)vjxE!c`}*p6`Qz)tMKZtTHc>_Y_h;{Xog5Dw!Aj^Y@O;{;CP z6i(v|&f*--;{qr6Sr_1cMyfUxQF|AfQNX5$B4!gJjF9S#|yl~ zE4;=Vyu~}b#|M1GCw#^ie8o3>#}E93^?8Yy`wwxsZ($2NnEScbXKTzoZS#H*d)^P6 zz5f1s+1x`luOG_&vR)kn{C#>@;+T8t9ebF2sTNyvFSTc^xR+`k|9txJ)hzS)=kMv= znfv&?{bJYSpY`*Dn7^mx|L0z|-+jJLiC|cN=cO{6TQ$I!RUj&=!gCofPol<5Ddl;48<@E#|VtXD2&D!jKw&N#{^8o zBuvH>OvN-z#|+HGEX>9n%*8y+#{w+GA}q!dEX6V`#|o@OC{|%L)?h8xVLdh=3>&ct zo3RC3u?^c1jvd&EUD%C1*o%FLz!*QIzNu0uIoWWU~!+Bi5MO?yV zT)|aD;u@~w25#aOZsQK3a2NM*9}n;lkMJ1Lc!H;RhUa*Jmw1KOc!Rfihxhn^kNAYo z_=2zahVS@+pZEn!3hrgY2615vJJ=&091tG~kPwNG7)g*6j!1^&NP(0{h15s`C!|F> zq(=s1L?&cL7Gy;>WJeC41(i_+RZ$JqQ3EyMidv|RI;e|!sE-D4Lqjw|V>CfiG(&T^qXk-`6&F4+^|E96JArxg%?X$jFehM6z?^_N0doT81k4GT6EG)WPQaYNUrxaK ze1Y{lJb(QkKdirw?^j3?$A;??N9`;w7ws&rSNXf4Z*UhIi}-qE1U~aMNrz}&Py0Eq zll|V#k_E1|_J4feLwrl3|8);2O`JLk4_9L(IKw?_JWnrT0khmEI>Ep$qoQ;DGc& z=|j?orH@D-l|H5mj?3VL^hxPc(x;`*NS~EHrwh)@;DYo;=}Xd=rLRa|m5$T}*JN;A z`iAsP>08pbrSC{b>4Lj5xF>yI`hoOA=||F!rK5Gh6B#^}ekT1~`i1mM=~vRPb-^1M zyp?_@{a*Tm^hfDW(w}v~7a4q&{wDoh`iJyS>0iIQy+szZk0S#c>A2Fi(st7J(($An zbU}O>B#=%hok%*dbQ0;L(vG?ynGBLkr;tu5ok}{jbQ)7>(3XOPY)ok=>g zwEp*?eO4J{lg=)kLprB)F6rFTdGsyFD}#K}`K1d;7nCj}U0Aw^E+{I4V$#K>OGuZL zE+t)Bx{NL;D+6cga?<6cD@a$At|aZE3o6T?igZ=!YSPuEYe?6WcGU&7WKdhWj&xn= zdeZf!8%Vq9f`&3^B;8oLiF8xxX41{2-E~0=8MKscCEZ%OjdWY-cG4cYz*7cZ(%#bT zrG2D3Nc&3r>4J_j=p@})+F!bhbXVyB=|El3O$I^I-KBd-_mu7>-CH_X7xa-qU+I3* z{iO#;50oAx9ij^c%V3D~Q0ZaP!=*<^kCYyz3r5RejPzLPanj?ZCrD3}o}>#V%V3K1 zROxBb)1_xf&y=2}3uen;j`UpVdD8Qx7f3IZ{(Uc-yP1}>d}ab0#Dy*FV2^lkKzt-X zLL@?BBtcR*A{mk+1yUjvQX>tVkQV8X9vP4knUEP-kQLdG9XXH_xsV%qkQe!o9|cel zg-{qpP!z>b93@Z^rBE7WP!`T8hw`X^il_t^R7Mq4MKx4M4b+4yYN0mjpf2j6J{rIc z4bcdV(F9G=49($=7HEl9XpJ^#i+1pUC%oW|_V7Um_`(kz(FvX5k1ptn00g2Ng3uj3 z&=bAT8^P#va@jK>5_#3W3{6imf5Oven& z#4OCl9L&W$%*O&O#3C%l5-i0sEXNA0L?~8aHP&D))?qz1APgI^37fG6Td@t>5sn?$ ziCx%@J=lwVh`@dvz(E|sVI09x9K&&(z)76KX`I1XoWprsz(ribWn95kMB*B*;|6Zx z7H;DXqHq`Ya32rw5RdQ}(RhNVc!uYAftPrN*LZ`sc!&4+fRFfu&-j9`_=fNJfuFEG z-w<>E;ny$glNPb(9;$hJ%{_dFKc15_@AuZfkHOjd@AtA}+dVY%_VRp<#oRmg@;CR6 z&EG>1`|jbI$3OG<*OmL{F^_-N&kz2Sd)bz(_7(>?!umshKkL10w?BXWkD>g-`s-po zFX_zPOn-DiR|Fss-4KNC=z*T-h2988AM`~(^v3`U#2|!VFos|#hG95HU?fIiG{#^o z#$h}rU?L`AGNxcEreQi}U?yf^Hs)Y1=3zb-U?CP^F_vH{mSH(oU?oDa3ahaOYq1XN zu>oP&h)vjxE!c`}*p6`Qz)tMKZtTHc>_Y_h;{Xog5Dw!Aj^Y@O;{;CP6i(v|&f*-- z;{qr6Sr_1cMyfUxQF|AfQNX5$B4!gJjF9S#|yl~E4;=Vyu~}b z#|M1GCw#^ie8o3>#}E9(FIZA?FB>+93tQO19`WFS_(*_+NQA^lf~0UnG9*U|q(myD zMjALFEz%)9G9V)|Av3ZdE3zRwav&#iAvf|MFY+Nj3ZNhgp)iV|D2kytN}wc4p)|^% zESyme4dm>-!xN^0|xoVNSrDK*IsDmmPcm{LNc#PQaXiIRSG5vHJw99|xFw*|Gb5z`XV51k4GT6EG)WPQaXiIRSG5 z<^;?Mm=iE3U{2t#Ct!WP;NQKM9mYNDNJo21R7!hZ6U(00y|TBI;p>s^n8ep47a=sS zy(O%uy(Jt`d?$l-&tL9kw_hw5GL}d$m0l*jTzZA{O6gF&IkHLytEJaSua#aWyD|(Mr1whila9~@`(BG`Tq>oA;(*?(6a6C4hrq_0Xx>Vj)BxGsG| z`lj?P>D$tGq@#4fT^ZbyzAybi`l0kA>BrL1y5NZno=QKHelGn&`la+M>DRj8jSSvO zzmtA1{XzPp^e5@hy5NfpzDj?S{x1DP`ls}--_6VF$iKvqwvmo2Z7XdjZ7&^9+Cdk@ zmq7yQgwlzm6H6zNPAcuF3zEqoxpWHYl+vlBQ%k3jcG3lDWspufy>tfYjMAB;GfQXD z1zBZ~O**@D4(XiIxukPT=g|duWspxgzjOiVg3^Vg3riQ#1x000OuD#q3F(s3rKC$s zm(c}fW#BAbPP)8w1?h^?m84yCL1h_Kk*+FTO}e^t4e6TFuDYO>3~Ec)k*+IUPrANz z18Fy1&`<`Aq#H{&k!~v8OuD(WyDn%UgO<{*q+3h3k!~y9PTE5kc*?*_+FQE4w2yQL zXbmouoTU`%8C`?kXK19jFVs$skC&yL1ofp3=RfdrJrFf<7|nE8S1Jzw`j< zfzpGdLv+Dl84QsgDm_ekxbz6=k=6$Rh>rwFh(t(? zBuENJBtvqfKuV-SYNUY^(jpzwBLgxb6EY(UvLYL@BL{LK7jh#H@**GdqW}t`5DKFR zilP{bqXbH#6iTBE%EB4tP#zUf5tZPA%BX^>sD|pOftqkdE!0LG)I~kiM+3N_AsV4E znxH9~p*h^q0xi)Bt>6n3;n1$Jx zgSnW8`B;F3ScJt`f~8o7pfzIEhm@jWallb2yI+xQI)*j4QZ`NL<5p+`vuT!fo6^ z6z<|4?&AR-;t?Jr8c*;P&+r^C@Di`^8gK9x@9-WU@DZQz8DH=f-|!tj@DtYO3u5j+ znAgd+K5Jv{rJMJI*!h0o-2U(PvU$B&^LnX1mN;Iq^z~BBJxy~@Q}1b-$Ah3ifAe@? z)5pi(KcKfIPLPlJ|Hpsx|39psAN(ixvS+pB>y!wE^@sj`)_d8}fByU*L-~jG*TsBZ z(wV!N{^)|P2tXjZAqd^k13l3Ty%CH)=!<^nj{z8nK?uQM48c$g!*GniNQ}a0jKNrp z!+1=;hY0M)0UX339L5nG#W5Vm37o_!oW>cP#W|eE z1zf}>T*eh#MI^4_I&R=5Zs9iWAPRSJ5BKo^5Ag_(5sfE!if4F^7kG(Rc#SuBi+6aB z5BP{r_>3?3if{OiANYx1u%zN%Hf#_Vwy=Xe;=uv&kpKyi2#JvdN#TfONRAXpiBw39 zG;l&%q(gdSKt^OjW@JHDWJ7l3Ku+XBZsb8;8KuMHBX_P@( zIHMfOqXH_T5?oLjRZtbxP#rZ;6RxO*+NguNsE7J!05>#5BQ!=6G(|HshdWxJC0e01 z+Mq4k!2_P~f;Za32OZ!GKXgPV{KMmhjbl4|OBQc)FIyfmO_~#kwI^Wxc){Guj_E`t-& zC#6qGpO!u&eOCIME;uiP3(^;*FG*jPz9M~9I#L&0lfiZA8`3wWZ%N;lz9Su_3+~F` zp7ed`2htCvA4xx!j@AWFWbjn_ne=n%7t$}KUrE2#1#e{VR{EXvd+86-AEiG@f7S(G zWbjq`oAh_-AJRXifBkO0twR1Kj2lKLr7K8R zl&&Q0q6;d^po(-=>1xu|rE5snly=nxwPa9Rx{h>R>3Y)jr5i}Q>4JtbXe8ZOx`}jC z>1NW+rQLNw3mLSOZYAAXx{Y*O>2}f{y1-KgUeeyu?WKLBJ4pLV`{{y?GUz1TS=wK^ zi*#4%0O>$o&`kzG(%q$dNcWWPCEZ&(SQqq>L0{>9(*30eNDq`ABpsp)2FqZG^ib(x z(!-@kNRN~rr3*&OV2t!w>2cEIr6)*Fl%AvuCd*)o^i=6-($l49NY9j>r3+@uV2<=$ z>3P!gr58vql>U7$o4c8o^n7Ll8^nbz>|l?0a6o({Ktd!!VkALQI3gL6BLz|-6;dM& zoRAjjkRBP35t)z~S&$XkkR3UY6SfQqOD7gRYy&_p*|YG4Gqx6(G1PujuvQ%R%nej zXp466fG51*jrQO+%gTCm8{uqFP7=#cE z#t;m}Fbu~CjKnC6#u$vnIE=>xOvEHi#uQA&G)%_~%)~6r#vIJWJj}-eEW{!##u6;W zGAzdmtVAeQVKvrZE!JT@HXsZeu?d^81zWKV+Yyc(*oj@(jXl_leTcw*9Kb;w!eJc2 zQ5?f@oWMz(!fBkrS)9XpT);(K!ev~+RYc+%uHy!7;udb>4x(@u_i!H%@DPvi7}0ow zr+9|vc!8IAh1Yn4w|Iy5_<)c2gwObbulR=V_<^6WJ}(h-|H1nFgbm`t`ue%%ey;UN z9CJ_GydT7l_XF!&X?;A1xrb%_K4)|9&d2A!`QDxR^EH3I{NFxquZ{u!=Jm17>tmb8 z1M_&Gj|ckkK(aqSKCpg7{D1wh{_nVV9k<9lzD@}jSbym6$J{>T&!7M6?`QpWF`t)o z=5D4xx}Ylp5QuIFLU;5)PxL}>1fvi7q96KW00v?ZLNFLZFciZu93wCiqc9p{Fc#x5 z9uqJTlQ0=mFcs4<9WyW!voITTFcUg8yA;|<>89p2*u zKH?KT;|spx8@}TQe&QD_skxU88^nbz>|l?0a6o({Ktd!!VkALQI3gL6BLz|-6;dM& zoRAjjkRBP35t)z~S&$XkkR3UY6SfQqOD7gRYy&_p*|YG4Gqx6(G1PujuvQ%R%nej zXp466fG51*jrQN-vXMF1w@Po5-Yy-k3wFqWZ)k|Qm)&8P^ls@r(tD-%Nk`~{{W3TpeNg(4^kL~E z(nqC_>4M`jI3ayf`jqr(=`+%2rO)Yt^D?*~eNp<7^kwNQ(pRM;b-^_mT$jEfeN+0D z^lj-o(owqLt_tfYjMAB;GfQXD1zBZ~ zO**@D4(XiIxukPT=g|duWspxgzjOiVg3^Vg3riQ#1x000OuD#q3F(s3rKC$sm(c}f zW#BAbPP)8w1?h^?m84yCL1h_Kk*+FTO}e^t4e6TFuDYO>3~Ec)k*+IUPrANz18Fy1 z&`<`Aq#H{&k!~v8OuD(WyDn%UgO<{*q+3h3k!~y9PTE5kc*?*_+FQE4w2yQLXbmouoTU`%8C`?kXK19jFVs$skC&yL1ofp3=RfdrJrFf<7|nE8S1Jzw`j=6$Rh>rwFh(t(?BuENJ zBtvqfKuV-SYNUY^(jpzwBLgxb6EY(UvLYL@BL{LK7jh#H@**GdqW}t`5DKFRilP{b zqXbH#6iTBE%EB4tP#zUf5tZPA%BX^>sD|pOftqkdE!0LG)I~kiM+3N_AsV4EnxH9~ zp*h^q0xi)Bt>6n3;n1$JxgSnW8 z`B;F3ScJt`f~8o7pfzIEhm@jWallb2yI+xQI)*j4QZ`NL<5p+`vuT!fo6^6z<|4 z?&AR-;t?Jr8c*;P&+r^C@Di`^8gK9x@9-WU@DZQz8DH=f-|!tj@DtYO8)EK1#N+u1 zhd-X9us(BP?jxJ~$mabZHvRns&K>^#df9*ohCe^1N*&%Nwp*Z4XmB4PbHrv84`d)ZFc?e)u~|I^lA z7xQ^ZXYOYDqYJts0Dk+_EIxPhCvh1g(H$7IZ_}c zQXw_czzJ!Q4(X8r8IcK@kp)?i4cU6bB~c2c zQ3hqkb<{vjxS|$nqYmn#9_ph3+|Uq>&=^h76wS~a?r4FQ zXoc2jgSKb~4|u{0-e?aWbbv4X&=H*w(AyH{FOM5GE2tc^ybLZ#UzENieOda7^i}CdU2shX*QIYr-;};3eOvmD zbd)Z*D}#H|_oW|5Ka_qX{a89$7d(-{Q|V{Y&!t~Tzm$F@{aP2ik-=N(chc{rKS+O+ z{v`cb7krVySLtuk-=%*@|CIjqyZtP(z%PyrY^38#+e+I>+e^ojcF+a!WspERp>!hY z#L`KmlS(`4f@CsCE}cR;rF1Ik)Y56BopeE38Kjd=FP%X;qjVBiDcq?<}NlWs2Ut_xboprv#x>DJP1q}xijllIUBo-*)~_Lgoh?IYbm+E?077j%?C zC+W`8{?c8fyGjR02kL@uG6<6HF5N@Ar*tpr-qOLkppOjtO81lQFFindp!6W=5M3}> z21BHWN)MABE7uXEg8@szv6ePv&?r!W>Z0zpB z#>VdMZv9@{(eXX!oHg^C&#d1+hsU*EXD+>6-oqnjAL?QMzL(A2OiNncGl30kVTYKo zhXWkpgjk4;IEagQh>rwFh(t(?BuI*6NRAXpiBw39G)RkdNDpUZKt^OjW@JHDWP=N` zBL{LK7jh#H^1>DQkRJt55QR_}Mc{^_a7Qr|M+uZfDR`hX%AhRDp*$*}B0NzEl~Dy% zQ4Q5m12s_#wNVFkQ4jUe01eRyjnM>6(G1Pe0xi)BUTBRr@J3s-gAaVs9)9pg00I$& zV01tTI-(OgqYJvC8@i(hdZHJ4qYwI`ANpee24WBfV+e*~7=~j6Mq(63V+_V(9L8e; zCSnpMV+y7s6w@#rGcXggFdK6)7xOS53$PH2uoz1ahNW1B9UcX*Ev_=r#V zj4$|#Z?Jw`;@ABL^E}z+Ub=a{bpD$cO}{^Zd%Hj1%Qp9Ly}R?lT=RUw{;gX3aX-DQ zk8dY)Pu*e{WbUc+YVh~JmuepW%;TRw-;cs8nm+zn-#_?Y?q#=3?qKmjFswiH&$Hgk z4*&V+k#Zeu*Dr+ zFc5<<7(*}=!!R5pFcPCM8e=dP<1ii*FcFh58B;J7p_qp0n1Pv?h1r;cxtNFfSb&9C zgvD5bFf7F~EXNA0#44;tIM!e-)?qz1U?VnRGqzwWwqZMVU?+BAH}+sJ_F+E`;2;hm z0*7$~M{x|taRMh1iBmX@GdPQLIFAd6!bM!dWn95kT*GzTz)jr3ZQQ|K+{1l5z(YL3 zV?4oAJi~Lmz)QTsYrMf*yu*8Zz(;(-XMDj|e8YEGQgAODHn4>qV!|E{aD)?LAvWS5 zF5)3R5+ETGAu*C5DUu;MQXnN#AvMwp`gVZ1mmR%d2h2-vPQaXi zIRSG5<^;?Mm=iE3U{1iCfH?tk0_Ft%cmmds7yP&PvJ-r^x41btSUh4oSZc<1urx#{ zpO4&(7kp0AlE%T}=Fv@x zq<2d1lHRQg_Q+tb^gikR(g&muN*|Jr&;^HOa76m3^fBq<(kG-(N=NE~Q!+R$eMb7M z^f~GC(ifzobiqX#T#~*leMS1J^fl@0(l>O$O&Q#hzAb%6`mXdn>HE?TbiqRzJd%Da z{Y3hy^fT$_(l2zuOBuY9el7h*`mOXk>G#qfbiqd%e3Jew{YCn#^f&46zuVU$3w&e9 zz((3u+D7Gfg~;vyd6BLNa35fUQ_k|G(BBLz|-6;dM&(jpzw!xArwXtxS=TAQ4GaV0wqxj9w?16D2s9^j|!*= zPgFu>R6$i#Lv_?ZP1Hhd)InX;Lwz(rLo`BTG(l4|Lvyr1OSFO)TB8lT(H8CC17Ea< zAN&!3Km;Ke9T0+!=!DMbg0AR>?&yJ@=!M?sgTCm8{uqFP7=*zXf}t3O;TVCD7=_Uo zgRvNg@tA;#n1sogf~g3_G)%_~%)~6r#vIJWJj}-eEW{!##u9{KDVAY5R$wJoVKu_B z25Yen>#+eFu?d^81zWKV+pz;Xu?xGg2Yay(`*8pVaR?DOj3YRTV>pfzIEhG{!fBkr zS)9XpTtE~q;u0?73a;WBuHy!7;udb>4({R}?&AR-;t?L>37+B^p5p~x;uT)w4c_7% z-s1y4;uAjO3%=qTtRI*7b^qb}ck4$jtRJP|u>j_twz;QmUJs(rJ#BZNKi|tX_i)=- z9~8|!T>sAI9&RT~4DQ->5l*eA_&3gfDm*OhqWBVLE1DCT3wa=3p-7VLldM zAr@gVmLLpEu?)+x0xPi!s}YVhSc`R7j}6#}P1uYr*otk~jvd&EUD%C1*o%GGj{`V} zLx{j(9Klf>!*QIzNkrllPU8&D;vCN70-|scmv9+Za23~Z9XD_jw{RPGa2NM*9}n;l zkMI~z@D$JR953(^ukadg@D}gz9v|=#pYR!9@D<rwFh(t(?BuI*6NRAXpiBw39G)RkdNDpUZKt^OjW@JHDWP=N`BL{LK7jh#H z^1>DQkRJt55QR_}Mc{^_a7Qr|M+uZfDR`hX%AhRDp*$*}B0NzEl~Dy%Q4Q5m12s_# zwNVFkQ4jUe01eRyjnM>6(G1Pe0xi)BUTBRr@J3s-gAaVs9)Iz;VH3>bM!!bpUbZ}B znlvX6ZBM}Z_JX;W9c};m%?oZ$z?^_N0doS;`~<9T2bg==(fpOby!hq>%n6tiFehM6 zz?^_N0doT81k4GT6EG)WPT&tGVEuT(e|s-GjCin1YvVG^tg^c;q3#1oHFOps?y+k@pZ;mXL!7}OP(krA_O0SY$ zEgh~4*2rM3^g8MF(i@~VN^g?htP8ftfG=qHbuZg@oAh?+9nw3ccS-No1$$($S9+iH ze(3|!2c-{5N9cmXGB_fARQj0oap@D%C#55G!6_M>mOdkWR{EUudFcz%QM%xw3@%Au zmcAl=Rr;Frb?FNT-!fC!Jo} zSr=rGK}P9J(wU{RNN1JKChejNvdbWcbWZ79(z&JcNavMy)dl%vkYBohbV2Dt(uJjq zNW1BRqB3xoE+$=Ex`cE|=~B`jx}dZS%1D=$E+<`Hx`K2?X-{2HNd}drt4LRst|nbw zx`uR3T~JE~wWaGw*Ojg(U0=F^bVFUxNCu6in@Bg6ZYJGax`lL0UC>GfUec|l+emv$ zx0P-u?V}5PWzb&QPugEPKsr!5NIF;-bdW)abVun<(w(KdNOzU)rVF~upoer%>0Z*k zrTa+tmF}kt`paN|^g!uB(u1XkNDq}BrVED4V1)EY=~2?7rN>B*l^&-H#>-%W^hD`N z(vzj9NKcgx)dkaJFkO0v^i1hl(zB)KNdLZ<&D~5(dfqdE4QyeDn6QTf9N~mmh>bXi zi+G5S1W1TPNQ@*%ieyNR6iA6wNR2c|i*!g2XJkM|WI|?SK~`jg3$h~zav~RUBMo_0a$g(Fl#v1WnNl&Cvoa(F$H@jW+N`TeO1@e9<0$@J9dw5rklLKnOab6FQ>{ zx}qDpqX&AT7kZ-)`l28DV*mzX5C&rihGH0oV+2NG6h>nV#$p`CV*(~(5+-8`rXm#6 zFdZ{66SFWIb1)b4FdqxB5R0%FOAv;oScc_Tft6T=)d4_=<0^eq7?$ z{fC&`#ipa}^97rG={)xH_itVgtRK9Ke&3(K-S^M;vU$E(^L)|K`1w-Jy?k>oKal51 zH}`OZqQyO2^LQT;9G#yZ`;U+J*7pzomwVacCh&Pm%!T!b{(07W**AXv`j4Uf!}|Mv zyMDhF~a$VK_!$ zBt~I0#$YVQVLT>aA|_!nreG>UF%8o(12ZuTvoQyAF%R>x01L4Qi?IY@Sc+v>julvm zRalL1tif8W!+LDMMr^`nY{6D+!*=YzPVB;N?7?2_!+spVK^#H^4&w-p;uwzO1WqCn zr*Il)a2Drq9v2XWi@1c#xPq&=hU>V2o4AGBxP!a6hx>Sdhj@g?c!H;RhUa*Jmw1KO zc!Rfihxhn^kNAYo_=2zahVQVX;$AjvU<*6MggqSK2q(lsY{Wra#6x@}Ktd!!VkALQ zBtvqfKuV-SYNSD0q(gc*BLgxb6EY(UvLYK?kR3UY6SprBMcDQ4Zx%0TtniN~nw~sETT+jvArFWaMwgT-fngC%$f-y3=acd-${*D*YRGoO=mh1(3CCwqZ|#S_8n z{_4nmZq!&vskq*7CNMq<8CrJu=uUy-#|-^a1IE(ubrYbirX6 z9FaaMeN6he^a<&c(viC0lnhQwpOHQ*eNOtk^abfCU2stbm!vOCUy;5leNFnh^bK8b zQwF!BZ%f~izAJrC`o8o7UGPu_kE9<`n~iA zUGPx`pQJxaf06zw{Z0D&@Ak9E0>2nCu#vWvwv&!2Z7=O0?WhZ!WDrX_wsainxYF^Y z<4Y&d1qo%4NIJ1}66vJU$)uA@r_co{WspibwR9Tkw9@IM(@Q(+f($aqD4j_D?J zkS-}*O4>sgl$Jpm>9W%0q{~ZJkgh20sS7H}pt5uo>8jGzq^nEUkgll88@nq?=2(kZ!39TFJmmy0vs0X>aMa((R;ubb+r7+DrRM z`%4E%2TBJ?2kU|kG6<3GDBVfAvve2fuF~CfL3bJSknSnnOS-poAL+i*{d7Tp84Qpf zC_PAeu=Eh=q0+;2!EhOjkRB;LN_w>P80oRn<8;Az8BCC#C_PDfvh)<`snVgkV44i3 zOV5y=DLqSiw)7n7-}kb)n`v?8Jrmf#7IugUdpN)mPKbrrh=aI@hxkZcFP2#c`vtiyV2z(#DsW^BP$Y{Pc!z)tMKZtTHc?8AN> zz(E{B1PASQd7f;} z!rV(YuLsfc`xCgg|MR`<5N~rY)!a+9*mm(X_fq*tmdzjUEt-EnyuHo8ALic=FaK7p z&A%U(7=O3N1MB+-7TzAT{%=A)r)m9P>-!1+>4)|AaqrsJC6dom!VT6R)?fei=UMM% zcl-J4fBf^Tzwg)kC4St^^hW>!5rklLKnOab6FQ>{x}qDpqX&AT7kZ-)`l28DV*mzX z5C&rihGH0oV+2NG6h>nV#$p`CV*(~(5+-8`rXm#6FdZ{66SFWIb1)b4FdqxB5R0%F zOAv;oScc_Tft6T=)d4_=<1%4ohn8Wy1!xutQAP!vT(PLM+5a9K=OD z#76=oL?R?c5+p@3Bu5IQL@K048l*)!q=z#yAR{s%GqNBnvcUz}kpnrA3%QX8dEttD z$d3Xjh(aigB5*@dxT6?~qXbH#6g*HGWl$F7P#zUf5uT`o%BX^>sD|pOftsj=+NguN zsE7J!fQD#<#%O}3Xolu!ftF|mFSJG*c%v=a!3Vx*kH2`_u$jx_M)+BCFZ+*=o95@{ z1ft^!Sl?cV^Z()PlIVCg=4Cb~U{1iCfH?tk0)O`tu)ZDO^7G>rPJCs;-~Ht^FP}L9 za{}fB%n6tiFehM6z?^_N0doT81k4GT6EG+6pHIO0@dE4DGw}7#|MrLV_wn@#@nYCm zEK%IEw(NGWIPT|rLq~8I8!h>KWIt@?bCMB=jN*B+Z#Yo z1HPc)*S&1NZPMGNcS!G)-X*?(`=t*^ACx{M9ia;j%ixIgQR!pS$E8n5 zpOlW&1*c?iTKbIiS?P1q=cO-5N9lr#GPoptS^A3fRq1Qe*QIahf}1k9C4F1^j`UsW zd(!u%ALxRIGI%8YSo(?dQ|V{Y&!u1Jf|oLQCH-3Zjr3dTchc{rKj?ywGWaC@S^A6g zSLtuk-+#BiMHcwSkb#Y~t+btVOlf;*2WdxL;3R`s(y^uENXM0qCmmlpfi6fWgGADa zrISb}l};v|Tsnm=NGXF<(y67>NT-!fC!Jo}Sr=rGK}P9J(wU{RNN1JKChejNvdbWc zbWZ79(z&JcNavMy)dl%vkYBohbV2Dt(uJjqNW1BRqB3xoE+$=Ex`cE|=~B`jx}dZS z%1D=$E+<`Hx`K2?X-{2HNd}drt4LRst|nbwx`uR3T~JE~wWaGw*Ojg(U0=F^bVFUx zNCu6in@Bg6ZYJGax`lL0UC>GfUec|l+emv$x0P-u?V}5PWzb&QPugEPKsr!5NIF;- zbdW)abVun<(w(KdNOzU)rVF~upoer%>0Z*krTa+tmF}kt`paN|^g!uB(u1XkNDq}B zrVED4V1)EY=~2?7rN>B*l^&-H#>-%W^hD`N(vzj9NKcgx)dkaJFkO0v^i1hl(zB)K zNYAytpJBa&_}!VWvChDICa{4m><|<7aDXG65DT#p2XPS(@sR)tkqC*A1WAz$$&msn zkqW7i25FHF>EVnF$cRkHj4a5CY;Zw#q7VwB2;5KY+XwpdlKeF`A$$nxQ#b zpe0(t3$4)x-e`+<@PRMd!w>!lKp=t;j1CAvM|47GbU{~iLwEE*PxL}>^g&;hy6H!gE)i; z9L5nG#W5Vm37kYEPT@4p;4IGJJT4##7jX%faRpa#4cBo4H*pKMaR+yC5BKo^5Ag_( z@dQut4A1cbFYyYm@dj`44)5^+AMpvF@daP;4L=_Te%#A;;PJrm$HyqFAJ4FUw8Gre zHutp6>p@KZJ%|pk2kw4mAL;pPMz3c?>{_*QS zhVl>V@8fN*AMcm=aW~T+0SH78g3$pX=!j0}j4tSkZs?94=!stFjXvm$e&~+@7>Gd_ zj3F3`VHl1P7>Q9BjWHODaTt#Yn21T3j47CkP)x&g%)m^{!fedJT+G9KEWko6!eT5z z7?xrgmSY80Vii^+9BZ%^>#!ahuo0WE8C$Rw+prxwuoJtm8+))9`>-Dea1e(Ofx|e0 zqd11+IDwOh#3`J{8JxvAoW})3;UX^KGOpk%uHiav;3jV2Htygq?%_Tj;2|F2F`nQl zp5ZxO;3Zz+HQwMY-r+qy;3GcaGrr&}zTrD8X}FgS8`#1QF<}n}IKm0B5F2q27x54u z36KzpkQhmj6v>brDUcGWkQ!-_7U_^4&d7j_$b`(uf~?2}7i32c|q7o{j3aX+Ss-p&Kq84hS4(g&F z>Z1V~q7fRS37VoAnxh3;q7}T*8g1Z>wrB?*_@X`j;&H>qjmHhoM2ei4*lw2nH$5=5+2jh zvKb%v`i2yIPO>JSlWYl}WR8~LbdHv8n49MxpQm)<8SV69$Go=xylCt{UwVP`Lg_`) zi=~%Hhv|Z)GFT?PTzZA{O6gV7tEIzr!5SH?m0l;kUV4M{M(It`n{~k!8Sn)SzwTxG zZ7l8!AMM>?)_Jn8t-33Ne186=WUES*F;sdO^w zgbQbBX(%Gb4^d-nHgB;R1rE^K= zmd+!cSK3t<u=>pOPr3*=?2mbbwMK;G?s26 z-Bh}nbaUwz(k*pCD;ao6x0Y@r?JeC_x}CI-F7TB>ducyuf9U|}K?q*st@}3E7U<*6MggqSK2q(lsY{Wra#6x@}Ktd!!VkALQBtvqfKuV-SYNSD0 zq(gc*BLgxb6EY(UvLYK?kR3UY6Sp zrBMcDQ4Zx%0TtniN~nw~sETT+jvA3CO72VJsJMZw7yZy5127PSFc?EH z6vHqaBQO%9FdAbp7UM7;6EG2zFd0)Y6``1h>6n3;n1$JxgSnW8`B;F3ScJt`f-o$_ zGAzdmti&p;MmW}BE!JT@Hee$*VKcU1E4E=fc3>xVVK??*FZN+S4&WdTAp(bS1V?cU z$8iED5s6bcjWallb2yI+h{8o&!ev~+Rb0b$+`vuT!fo8aUEITcJitRd!eczaQ#`|S zyueGm!fU+2TfD=2e85M1!e@NJSA4_I$AKUBA8dI%utRjYhiYD4a}VE!*LU-J@Q>F6 zcmF@%%Qp8ky}R?VTysy;JRh#Pmv8QE+wyp59uIguFwc)2J-$ze_5FkYQU0D%ZXFghRv9nlG$(FI-64c*ZLJ<$uj(Fc9e z5B)I!12G7LF$6;~48t)3BQXl2F$QBX4&yNa6EO*sF$Gf*ifNdR8JLM#n2kA@i+Pxj z1z3nhSd1kI!%{56a;(5gtio!9V-40~9oAz5HewStV+*!o8@6Kyc48NHV-NOXANJz_ z4&o3Za2Q8$6vuEJCvXyxIEB+VgR?k?^SFR0T*M_@#uZ$}HC)FH+{7*1#vR16w zJj5eB#uGfnGd#x&yu>TK#v8oFJG{pSe8eYw#ut3WH++XBE%&lv16$Z3ChXw=M>ruC zVj~XXA|B!+0TLn+5+ezcA{mk+1yUjvQX>u0A|2Ah85xifnUEP-kQLeBg6znFoXCaT z$b-CaMLy(50Te_b6h;xap(xx@48>6bB~c0kb<{vj z)Ix34L0!~CeKbHrG(uxEK~pqCbF@H9w1O8}qYb>#7VY2zU$n!jC9Z;;+7y-9krF4!UizM$dPz3hN((%YqXNbi*1CB0i0 z?2*A<>3!1sr4L9Sls+UKp$iVn;E425>0{EzrB6tol#bK|r(|$i`i%5h>2uQOr7uWF z>4J+gxFmg9`ik^b>1)#0rElnhn=-g1eOvmD^j+zD()Xnw=z@nbcqIK;`ib;Y>1Wc< zrC;cRmoj)I{aX5s^jqn7((k1|=z@2K2Ce|Mlo76itSfsM4Sw4HQJ zX?tl0X-8e)B!gJev8Cfk$CZvJ9bY*gmg*iQqmr}ptKCiNSBo^CtY5;f^3& zNLQ7vCS6^+hICC`P)i22rRzx7m98gUU%G*GLtW5F292egNH>*kCf!`Rg>*|@&`Jhg z(ygW2NPA1Sm2M~PqYHdx&|cb4+Fv?AI#4=DI#?HUkU@xaN9j(|ou#`-ca`p@3%bjo zhjdTrUedj#`$+ed?xzd-%V2=?K;Jg!D-1QPQKO$4HNr9;XY& z%V2`^MCnP=lclFfPn8bU1=D0OU3!M}OzBzDv!&-q|Gt;a-Aqd+-ZOy>Y+;9(u!jR2 z;e=R-jW~#lc!-Y#NQgv8j3h{kWJrz_NQqQPjWkG$bVv_pWI#q_LS|$^R%C+=dVjRX}0w!V- zCSwYwA{5gw9WyW!voITTFc3?3 zif<^*k6+JA_x-!|V;0trQkdt-jt?&yJ@=!M?sgTCm8{uqFP7=*zXf}t3O z;TVCD7=_UogRvNg@tA;#n1sogf~g3_G)%_~%)~6r#vIJWJj}-eEW{!##u9{KDVAY5 zR$wJoVKu_B25Yen>#+eFu?d^81zWKV+pz;Xu?xGg2Yay(`*8pVaR?DOj3YRTV>pfz zIEhG{!fBkrS)9XpTtE~q;u0?73a;WBuHy!7;udb>4({R}?&AR-;t?L>37+B^p5p~x z;uT)w4c_7%-s1y4;uAjO3%=qTzQdA^d)csoE$k2z_HckBoDd7K5eIP*5Al%z36Thi zkpxMR49SrKDUk}Pkp^jz4(Z{H49JK~$c!w=ifnK}cH}@#rgp)#tVDypG6YM>@+p*HHEF6yB^8lWK> zp)s1EDVm`@bUf!n0FOYe~0DZNX2w=UQtgT2!Gr1wi7kUl7V zNIF6n9G1Zm>7&xeq>oFVkUl9LsS8fY;I#A^>9f-3q|ZxVkdD#?7iDls`m*#D>8sM$ zq_0cg&;>VTa7+5O^d0HD()XnAOFz&B4`uL3`myvA>8H}qq@PQ_&;>7L@Jjl%^c(57 z((k0-OMlP>A7$`K`m^*G>95k?q`&{}Ad4&riXj6VX6p^?(hky&y1+>Wv7}>5 z$B~XJ9Zx#GbOK$FPzH&l6H6zNPAZ*DI=OTTU64`+siad&r;$!8olZKvw6iYAAcKt3 znWQsIXOYe-olV+B7i5<~4(XiIxukPT=aJ4U?Wzm%$soUU0qKI$g`^8h7m;?;1x01x zE?rEzxO55WlG3H5J#;~78I+MOD_u^yymSTWiqf9CpppzKOIMMuDqT&wx^xZcn!2Et z3~Ec)k*+IUPrANz1L=mkppgt3OE-~jD&0)FxpWKZmb##o47{XUOSh5smToKEPTEHo z_{yNYw4b!UbbxfAbdYqgF6bbG5b2K6ouoTUcaiQY-AxyCmq8Edp3=RfdrS9`?kn9- z7xb6G0O^6!gQN#b50M@!Jxmu2m%#|>k5h1|%4yl_Q6 zCfiG(&T=KuffO7h0nYywMiz-~(T@hadbAfItKx7#$FTj_8EW=z^~3 zhVJNrp6G?%=!3rKhyECVff$6r7=ob~hT#~2kr;*17=y7Ghw+$ziI{}Rn1ZPY#WYOE z49vtV%*Gtd#XQW%0xZNLEXERqVJVhjIaXjLR$(>5u?B0g4(qW28?gzSu?1VP4coB; zJFyG9u?Ksx5BqTd2XP1yIE*7WieosA6F7-ToWg0G!C9Qcd0apgF5(g{;|i|g8m{98 zZsHbh;|}iP9`54-9^w%m;|ZSP8J^<>Ug8yA;|<>89p2*uKH?KT;|spx8-6|x{CK`> z^E}zskJ*@e>E`tydcGdG2fl4&gS`HLxTOy^TXP7_l_RV4{N>;K^yCX zpZPun=H79%xOZ&+`}Xhu`=)OXB;xOvEHi#uQ9ND5haLW?&{}VK(MqF6LoA7GNP3 zVKJ5<3`?;L%drA0u?njZjx|_|by$xL*oaNoj4jxTZP<<-*oj@(jXl_leb|o!IEX`t zz+oK0Q5?f@oWMy$;uKEf49?;l&f@~2a1obq8CP%>*Ki#-a1*z18+ULQ_i!H%@DPvi z7*FsN&+r^C@Di`^8gK9x@9-WU@DZQz8DH=f-|!ul^xVsa4QyeDn6QTf9N~mmh>bXi zi+G5S1W1TPNQ@*%ieyNR6iA6wNR2c|i*!g2XJkM|WI|?SK~`jg3$h~zav~RUBMo_0a$g(Fl#v1WnNl&Cvoa(F$H@jW+N`TeO1@e9<0%@wj1QImGv(KWgq}|M79t z{M?*CbUXp;+Y9Dic69ukH!rg}0doT81k4GT6Zpr+7tFnE^O|5zz?^_N0doT81k4GT z6EG)WPQaXiIRSG5<^;?M=m}UqZ}8vV%XVAlXlc38(c-g}?+v|^yV%&x*D>5c3O*<4 z0@sr~Pxb{ziwAt}|K+{xpm}m3W4`nP>4nmZq!&vskq*Hy5O)3jz}MsJ|=x!`h@gJ=}290N(QH;&q$w@J|}%%`hs+nF1RRzOVXF6 zuSj2&z9xNL`i3sJDT7Izes7|`@ zK?WIQl+GlbSvre!R_ScgF1jGQ401^4l+GocTRM+)UTIfdkWU8rr3*+GlrAJ)Sh|R` zn=U9S19$0S(#54qNSBl@CGDXLO3R>(bXnR>3Y)jr5i{$)CG-X&{(>ObW`bO(#@q?NVn7ltz_UO-CDYhw6}Cy z>2}gSy1-Wk?WO&s{iOq>1EqtcgLOd%8H7l8lcFP2#c`vtiyV2z(#DsW^BP$Y{Pc! zz)tMKZtTHc?8AN>z(E{B1Pjqn!FyEd)wym zz&svUY`gfH#{<7;@OWT-|KQg>>i^5VY-c+siwj&~{h@!J^>&e z{J5Lxj{pQB2*K!p5OhQ*bVe6+MK^Ru5A;MY^hO`_ML+b%01U(+48{-)#V`!V2#mxi zjK&y@#W;+|1Wd#vOvV&UMJT3WI%Z%dW??qwU@qoiJ{Djh7GW`#APh^f49l?sE3pcz z5so!ji*;C!4cLfH*o-aMif!1A9oUIo*o{5di+$LS12~97h`?bS!BHH;ah$+OMB)@q z;|$K?9M0ncqHqzHa2Z!{71wYbH*gcTa2t1U7x!=<5AYC=@EA|<6wmM+FYpqt@EULM z7Vq#LAMg>M@EKq572oh37H95d!v?mnLrmDi0giA&EW}0}#6>*BM*<{7A|yr#1WKY5JWv{CP!{D-9u-g#o~VS%sDi4fhU%z+ny7`^sDrwwhx%xMhG>MwXo99_ zhURF2mS_bpv_>0vqb=IO2fk>Jzj)lR3FUDk%+cw`<4J7ZKQ}+j378Xzz9(RPd%@hx zj=q2Y<|Q{LU{1iCfH{HaeFE0E1I)eb=>0liUV3u^<^;?Mm=iE3U{1iCfH?tk0_FtF z378WwC-BD;uztM2`jwsE|HBXK@8jzg;>EDBSR%P+{oys=XZ{1<8~O`(vC)vvM|MLv zpOf5;h&WD`$RtjdC|I)moRYz5=`+%2rO!#9m%bn! zr3)^~;F9!Z=_}G#rLRd}m%gD3Zpz@6^lj-o(s!lrN#B=#pbH+#;F0uW=_k@prJqSZ zmwurOUdrH=^lRxi(r=~TNxzr=pbI|A;FI)c=`Ye>rN2pk|J@xdvY15K$rBmpFlrl&qomx7L zbXw_j(&?r3zXu&M$RMM1Ch5%5S){W{XOni(mms?ga!BWt&Ly2&I*)W-X;)p4PX_s= z3rH7~E+k!8x`?!!E+{Gkcj;o%#idI~my|9g?V$@w%b<*OS?O}p<)te~SCsbD1(jq_ zS-OgJRq1Nd)un4l*VF~IWKdhWj&xn=deZf!8%Q_Q1&w6TSh|UHQ|V^X&81sNx6}o# zWZ)&;TDpz2w{%2A89 zy9|0r_mu7>-CMekbYJOyx}d)d21pN-9wa?jdWiH;>0!EHxC}-}kCYxIJz9E<^jPU} zx?sEvCP+_|o+Le4dW!T^=}=uTO$O7YXGqVKo+Uk7dXDt(d)eI0v}ENy6WG8Oc8Cdk zIKUB3h=tgQgSd!?_(*_+NQA^lf}}`>MDhF~a$VK_!$Bt~I0#$YVQVLT>a zA|_!nreG>UF%8o(12ZuTvoQyAF%R>x01L4Qi?IY@Sc+v>julvmRalL1tif8W!+LDM zMr^`nY{6D+!*=YzPVB;N?7?2_!+spVK^#H^4&w-p;uwzO1WqCnr*Il)a2Drq9v2XW zi@1c#xPq&=hU>V2o4AGBxP!a6hx>Sdhj@g?c!H;RhUa*Jmw1KOc!Rfihxhn^kNAYo z_=2za2J6Qge%*g?;NugHKR!lb{piKt{~oG&dCh%ubKm^;zvu43f4-L;-1*->pQgFD z?d@&uX_|YQUjD6GM~8cw=JCL{lQ;Kq&GW~%3T)?XiNW(to5w$%7y9pdPu=?d!T)kE zdvj?%Pl*Uvf9RiQy_fA+<{!WQV<`Wy{=Q%Dm-ulv(;oo{L=b|}0U_v!PUws-=!$OW zjvnZVUg(WJ=!<^nj{z8nK^Tl77>Z#Sju9A%Q5cOe7>jWjj|rHFNtlc&n2JzL!*tBR zOw7V;%)wmD!+b2jLM*~!EI}BSVi}fW1y*7eRwEp1uommE9viR`o3I&Muoc^|9XqfS zyRaL3uowHV9|v#{hY*3oID(@%hT}MalZeDAoW>cP#W|eE1w`Q@F5xn+;3}@+I&R=5 zZs9iW;4bdrJ|5s99^o;b;3=NrIbPr;Ug0&~;4R+aJwD(gKH)RI;48l2J1iNtwv7QB z*uoAmVGjp5!U?eu8*va9@em&gkPwNG7)g*6$&ef=kP@ko8flOg>5v}I$bgK z5-Ot#s-haIqXufC7HXpo>Y^U%qX8PC5gMZjnxYw+qXk-`6}-?IZQzZzXa^toqCKn+ z3x9aruyN&a!=r+^mn{#OCd~;%+Y_)pKA3yi(e|(3yx`^p%n6tiFeebrPr&+ifVr0) z&0h)3i*HWAoPaq2a{}fB%n6tiFehM6z?^_N0doT81paUW){htbxA(GrxMw{spOYoD z5Z@cRD0i{(g0F8#z~>}A`J7}!w5;r8@u}@(2}Y>buY1{+lX|gZo?OV7FTFr|q4Xl@ z#nMZp!*sz?87z}tF15Ifi3J16ZUX`Bb*Qm zu@MJx5f9cMwtUUG_3y*pVoAg``M%bl+u{HI{9xbvTK_!$=!Dk)<#`3{e}28vk3IZ6 z=FjKb`RC7>cenn3`T6@DetuogKY#x7>)(F;-2Ug+{r&UjKcBbd*Uue(ex2B_pWFTX zdf8t;cL)mk@pV+ce*XXS|GyYd^W|}_?A^|}i+3Am??7k&eTzfej9$rM?*{i5*846}0SJ_)m)eNp4=_>n&&bZyLt@y-&My|H+{drK9b2b9P=!s!K*R zR^|SE%A+N`v1)Y3mfXtA9$6x%MvC&EV~uWnPS%a?SeT#pcYX<(JF+G1!;40=WJXIK z7UqwU`?s~^R#LZ)mllogXs@v?Ih#GQG;+xA~Z_`I&%R?H3(6 zn~S)Ln;6C2WM=;7IgJ0RPk&#xYH!O=cKp4x)Em!rwSGTO{CrN!@mjtQ{2$@|6aIhw zd`x{G`R{p$1_ZR}J7&!2{F_V1?RLXI>6DJPPY?h9 zem>3iqwcr;Wm@)^zSi*p@vY*$yQ5`2sHI8!d-=7`KcB~4^v`GXMJ?M~;{V#dKmWfR z#aulyJczF|No>s zZdls#&XWJmkAVN>cz28M;eX?A`)|8l;F+Bf6f%+`Mp4XYN*F^aV;RTolyL|DIX|C& zx7#i0_}VefzU`lU2>Cy_j#}dQ-z_cke_Hav_3&TU@BeO}|J&PJ&gYDO(;Xbps>A>7 zxJGZyc27S-luA%c?LznS5XB4=mxN5{d z*VTIg0r4%nw^Z7=D+2;XT^JDX_4t5*n;N`==eK}>htD+^b#$wMw`2IJR+ClmN5icx=iZVJE$cUry}8~0 zSP!}X%3ke9UEQnwb=UT4f6?{5+MjV_ulC2@+^hZXxxLzdo!_hd`-Q#Qzg*m_eH|^& zb^jAj>+dTr@74YpDtVSFo}-%Qso@0{@*<0PiCSK!j#sGXRTlFaOL(0I-e4(jvW&M_ z&fBcu9U6I;mAuC)-e)x*u!av=%SWu^W7hKt8~Buse8wg|XER^0g)iC4S8U^Jw(|`; z_?De~$1c97i67X_k2LcWd-$2X{K7tdWk0`hfZsXD9~|OO4)YgB2pHI_eJhTlHEjr_ zEkPVjFzpDTJ)s;!7{?OMadhB#A~=CaP9%zxh~{KsIE7d`(uq@v<1{*RI$bz}uAE6c zXOTcRx^p%?IEO^eC5iJ$=6rh6ixe)PHy6@}i|EV6^y3oxb14J3j8raXAXkvam85eO zgSeUut|60a$>KT&b3H@2fuY>UFm587o5|r8a=Dc}hLg{26flB9MpDEmiWyA_V<=@T z4 zGPs6Jt|g1>7|ity;Rc3sBg43fY;GopTgc^B@)%A&w^6_d3K>ZeqbO!HC5)kzv5ez( z%D98^+{py)Vj_1liF=sLy-eXgrgA^!Jis&_WI7KqgNK>PBh2DaX7dWo>DtVSFo}-%Qso@0{@*<0PiCSK!j#sGXRTlFaOL(0I-e4(jvW&M_&fBcu z9U6I;mAuC)-e)x*u!av=%SWu^W7hKt8~Buse8wg|XER^0g)iC4S8U^Jw(|`;_?De~ z$1c97i67X_k2LcWd-$2X{K7tdWk0`hfZsXD9~|OO4)YgB2uM@^If~Y_A&|BNaWuiS zBZT&ZatvV{OE|~Tf#Zqb1R^<+C{7}plZoLJV(CaHP9=`h=*;PK;S9QRCh?p_0^R7& z+4SHX5;>P7&Lf%g=}9kAxPabVNFOeuFBj8~OX$y~4B#?SxtxJqK^j+*&Q%QJYBIQn zOs*x1>ln=S4B-ZbawEgIiEM5rhg-if&5tzm6MOiXz5K#Heq}$uae&`B$R8ZyPY&}JM+k7w){3KOO&bDfOAtpBOglno zPbkL_#<7HR9342G2u>i96N%y^qB)ruP9c_#bmCOvIE~JnP8ZIgD`yhVStQVn?wm~z z&LNR=N#Z<`IiH^NB83a+&4u*gBKmSM{kVkwT*?40BbCb;$Q7hhsoT_6z*dx_tSEperrJMrk4x#7P;YX>ATRc7oZm*prxlzUs-^D)POKr zI=Naplp=^EifCepr4udvC!Og+S6Vtm66j735=kPNo}|#5mJZIo^rJrmNM#^tq%(*N zbXx>uF_<9?Wf5^4SjZx3siU67ETMs=EMqw&aK$t-3whq=sSJ`1Rzk}9gHVIhmCrH*;`bGRP#0!3<$2!^kFw zT=K}LfI^BWri4<)QO0;CFp)`2W(rd&XByL)!Axc`n>oy79`jj11(j4$O$`fKL@jmH zvzR3`u#{yiX9bO{WEHDf!&=s{o(*hd6Pww>R<^O79qeQmP3)$bJ?v#4`#Hct4sn{a5@k{BvC{YLoB+l1LEjR7rGKp0^R9BB1t6ElN5T>hraZqKLbc* zAZesChzv5xVlYD($}qCYA(uSzDWH%diYcL#ag;Hh33yl_U=ov=!c@wc#&l*dlUdAW z4s)5ud=^kaB~?^Y!$KBOOC9wrW(f@}Wf{v^K_e?!#cI~DmUXOW0~^`IX11`EZER-; zJK04OyJ=<*d)dc+4seh|9OekE3=p;^kRXByA(Sw}=|BXACIX^}CWcr#5l3gb(3N-+ z=uQt3Ng|n^q|lo_^ravD89*unNh6&>WMD`nAdA5aVJO4MCWl<|$ftlpiYTUpQpQon zcqTBBNla!6Qz>T})0x3cW-*&N%w-<)SwIDqR8dV03t2=hb=0$%B{Z;*l~0tzXjm=a1EM;YUpz(gi7nJG-AoM}vF1~Zw(Z00bRdCX@46;x71H8m_` z5w+A&&tjI)z*3g6oE0>(l2xo`4QpA)dN#0;O>AZhTiM2TcCeFOG_jjz_OO?I?B@Un zImBU((8|DHYXS)(m=Hn!;xI>O6{P$LB#2-_2qlbgIuJo5QA86%ES-p>GhOIP zJPCBC2Z)Q;Fj=I&(T*ID@X7NjztfKsUN`Ha$3pM9w9N z^GN1=deVy&E}%CT(ua%a%fkoigrVJa;mIyO_w` zOyV9Ub1zf4kEz^GIS(+62bs=8%-~^W@(8ndl-WGS93E#bPcV-sna@)!;Atv&hDx5L zisz{2d1`opg}lfjUZR$lspA#ud6mVy#u8qqfj3ynn=Iokmh(0%c!x&bWhL*iiuYN~ z2dv>k*76bS_?Y#4!UjHNBcHK}&)LiuY~f3`@)g_on(chU4!&h4-?5ADY2pWV^CQjt z#2$WTFTb#lU)j%Z9N>2j@&||blf(SQ5dyAK|2c}*v>}kT1aUOMv?GM}gmMgF97{OI z(ShTM-~=K$ktj|gnv;p)6k_Q}Cr%}f)9B3Ubm0uTawhSdMFQRE&e`5y>naLx};!$Sv7;|`>xjex9_4LkUjoqWeGzNd*F*v*eL^Amgc znZ5kNK7M6Czj1)yImjOz;!h6q7e@#fr2caht!YCbZ3*INf@wzx?Fr==!Z?<2j-vy| z6Tt~Yaw1WjL^LN8!zsklkxraS9H-Hl)9Jz)bmdIqIg13k(VesD!8s&yE=in6GUwBi zUZijVy}6J+Ttr_krXQEkpGz6QWu$UB1G$1Ut|XnS7{t|Na1EJUOBUBLnCltB4GiT* zhH(?w+)NI)kjt&)F`RsEqks_`4>OZTn8l;a<}v2*ICFV|c|6H{o?-z{Q^7M-@+?(6M>Wq= z!wW3rMHcZAwY*FnuTamcEao+q@H!2=!BXC28E>(iw^_kEH1aMhd5=}R&uTtk4Ii?W zk66dYtmhLp@F^Slj7@ybX1-txU$T|2*v8jv=NoqLEj#&+U3^ayKd_r0Y33*P@H2b) zg?;?WetzQszjKg3IK-bE<}Z#A;5qA797Su|5J+2sIGSME5kh-HIfgKfC7k2v!0|+I z0+F0Z6ekhQ$;5C9v2>&prxM3$bmnxra0XpDlX%V|fo^o?Y_3qs_u#rt{W(!-{#&&kFlU+2i zn`ZW~mwoK#00%k5VUEzsPyDtfkRXByA(Sw}=|BXLL=jC4v2-Gi&UB$G@g&fl9wd@P zGCfJ5H+|?!Kl(F(R0fhpI)lg{lPm@^grN*0n;deNXB&KfE0Su2UC{=`q7^Oq%x2+(iubsnPf4TAq-_0 z+2oK*9;Vp@m{t=|ND;-9P|7&U7|#SIGKtAdVJhWJV>&aK$t-3whq=sSJ`1Rzk}9gH zVIhmCrH*S;;C^ zvxc>-V?7(#$R;+kg{^F3J3H9PE}Ga)Gke&}KK65fgB;>8M`&dNYHI=sBA5_D2_u{i zL=Z_7(Zmo-C*tT#7rGKp0^R9BB1t6ElN5T>hn5Mj{pimCQW;1Z=?o%+OtKiv5QZ|0 zY;wpYk9-O!q=;flC}kXFjAsH9nZ#tKFqLwqF`XIAWEQiT!(8Sup9NGhN;%V*&J1QUi`mR! zF7uer0xGDaifU?D$RcW~qn^bqp@F3=V>v5mWF@Ou%^KFSj`eI{Bb(UF7PhjD?d)JD zyJ%uJ&Fo<>``FI`4swXY9HG@Q%AY`j2quJ3!U(4W5kwM2G%>`|i8wmbg|5VtKzDkO zND|5PB!%Aep)dXD&j3;xNE+!3B7;n_7|alcGK_3;$R&?_3MizAVoE4w9A%7W0u!0U zWTr5ca;7nz8O&rBvzfzO<}sfIR8UD3)zq+%MbuJ9J&RdF14~)Pa#qmDN>;I&HLPVF z>)F6YHnEv4Y-JnU*}+bB(Zp_=*~4D;v7ZAR*aJ3UAwiDY_`LT~!emwxnT0I3WljdTW)K_*!YW(Y$WMm9O*l1DxT z6jDSnC6qFbGR8B3iA-WLQ?9%d$wFpEc-&11~rapv*_^LUc^ zJjDW@rh;dvSp3;Xz${rtuOe&-;6aEL!S%wHTKAVa?$N70%#1k#oujwYCP zgwUQ)jvBB|z6=FZe$oYk=Qau$K_Me4Vid)Uri3w+GL~`NP8oMF zo;#VqT}yh9`J zvXb{$#rv%01J>{%Yx#(Ee9U@2VFRDCkNE9a#&B?@Y3bAyg6Q>f#X>{gvx^MHvdqO#e zFpedhq!XtS$7yutbh>Z`T{)9@&LV+sbmwe(a1M!_ zOA_ai%=z@B7b#poZ!V+{7txoC>BlAX=TZi68L3>(K&~K-D@o@n25~hRTtg<;lErll z=6Z&314FryVcbMEH%{3@4x4C}0GIjHHNB6f>F<#!$*w#&J7k+`)M6WCC|F zk-M41Jxu0arf?rqxu0?#U>XlHorjpg!_4FnX7MPqd5k$c&Rm{g9#1l#r&z$#RPYRy zJWCbNQO)zz@B#~Ykwv^jEiY5YE7bEUi+PPDyiNmeu#`7h##=1sZC3COjl9cB-eVQ- zvziZB!-uTpBi8XT>-mHYe9A^XV-ugVnJ?JFmu%%Lw(&LF`Gy^Q%TB&y7vIyw5A5bg zn)!)6{LEf{VIRM;pWisZ?;PY04)G_4`HLe2csE)rj-oYf2&64R98ECo2%$Zp977n# z63%gS;CLc9fk;jyij#=uWMVjlSUS>)Q;Fj=I&(T*ID@X7NjztfKsUN`Ha$3pM9w9N z^GN1=deVy&E}%CT(ua%a%fRfJdKR;U29~mn<*cBQm8@bl zYgo%V*0X_)Y+^H8*vdAxvxA-NqKVx!vxmLxV?PHt$RQ4MgjV6opFn~LCWKJJ2&V%P zL=r_bF~rh|I6BjXuEdi-cY2UW63O%=h2Hd`<%QP$=+6LB8AuxG3?hR}vKY(|hBAz7 za>yl*d9OMv( zIYO)Bls|z45ljf7gb_{$B8Vi4Xkv(^6LEB=3tfpPf$sDmktCAoNeaE`LtpyQp8=#Y zkTlX6LRfJdKR;U29~mn<*cBQm8@blYgo%V*0X_)Y+^H8*vdAxvxA-N zqKVx!vxmLxV?PHt$RQ4MgqFASwkD7uf(ap%Fv96T1d&7$O$@PgB96{*l~0tzXjm=a1EM;YUpz(gi7 znJG-AoM}vF1~Zw(Z00bRdCX@46;x71H8m_`5w+A&&tjI)z*3g6oE0>(l2xo`4QpA) zdN#0;O>AZhTiM2TcCeFOG_jjz_OO?I?B@UnImBU((CT>QPar`A6GA9qgwuftB8eiJ z7-H!}9G&SxSK>*aJ3UAwiDY_`LT~!emwxnT0I3WljdTW)K_*!YW(Y$WMm9O*l1DxT z6jDSnC6qFbGR8B3iA-WLQhN;%V* z&J1QUi`mR!F7uer0xGDaifU?D$RcW~qn^bqp@F3=V>v5mWF@Ou%^KFSj`eI{Bb(UF z7PhjD?d)JDyJ%uJ&Fo<>``FI`4swXY9HCXD@+XiWf(ap%Fv96T1d&7$O$@PgB96{< zp)2tu(48J6l0-5+Nuf7==u1EPGk{bEl14g%$RLv}1~Y`A3?rKya>*l~0tzYOe~zmU zcJF>YL%4yV+{iF)BAc7Z;TCeal{|)%&utVifkoigrVJa;mI zyO_w`OyV9Ub1zf4kEz^GIS(+62bs=8%-~^W@(8ndl-WGS93E#bPcV-sna@)!;Atv& zhDx5Lisz{2d1`opg}lfjUZR$lspA#ud6mVy#u8qqfj3ynn=Iokmh(0%c!x&bWhL*i ziuYN~2dv>k*76bS_?Y#4!UjHNBcHK}&)LiuY~f3`@)g_on(chU4!&h4-?5ADY2pWV z^CQjt#2$WTFTb#lU)j%Z9N>2j@&||blf(SQ5dyOG+i?`FX+t1w3F2siX-5d{3FR2V zIF@jZqXWki!3jiiB2k<~G$#|oDa6u|PMk^{r_q_y>B1RwvVB%P}m#MNYQ4Vhd^7S}PD z>lwlg4CO|KaTD3xOb)k@%dO-woP2JhfDsfjk|IV?%xFp&Ln&h!$L*AH2jjVu3Eag* z?q(AAFqwOq!hKBTe#&`(X*|eu9%2R$Gm}S{#iPvTG3M|%b9sV!Jjr~XVgXN6!826y zELA*5HP2JS3oPVC7V#3byi6UhP|vF@<~5e^It{$RQr=`4Z?T-WS;0Fr@-8cRk5#>Gkf`kef-LPe&YbYbC5qc#Gf4IFOCp!o%+vFw5AP#v?Yk638oz(v?r8f2;*47 zIgSn-PXs3r$%#a9649JY45tuFM>=sTahyhHPNxfJ(3LZZ=PVNFMt9Dp2j`H;xg>EO z$(&D5dXd5f^yWhPa1njEn0{PBe=cPJmyycl4CD&ZxRP|PVh~r8!8K%ZEm>U0V6JBf zH!zeN8OBXyb2B;ILN2$G$8hqwjRHnc$ViG9MKPl(VGN~=WgNFt#vP33P9|^{6S|&YVsc&Y&x2637 zH!_Ty$mV8pxP@G9C6D3ca~lPWppcOiF^XbFQ^FWZ8Ou0sr;Ixo&z(%*E+%p}lemY; z+{+a1V=DJk&I3&2L8kK%GkBPpJi;sUf2EUS%<_v4q!Y;0>1YCd+t><-E-b-l36qS;>2>;(b>00c-e> zwS2@nK4v|iuz^q6$Y*Tgb2jq@TlkW#e8o1tW;@@ogKyc%ckJSOn)rd;{75rDv4@}8 z%P;KXSN8K82l$76gaE&1){3KOO&bDfOAtpBOglnoPbkL_#<7HR9342G z2u>i96N%y^qB)ruP9c_#bmCOvIE~JnP8ZIgD`yhVStQVn?wm~z&LNR=N#Z<`IiH^N zB83a+&4u)#<@enFGp_!+iL1r#-AgEC9A%7W0u!0UWTr5ca;7nz8O&rBvzfzO<}sfI zR8UD3)zq+%MbuJ9J&RdF14~)Pa#qmDN>;I&HLPVF>)F6YHnEv4Y-JnU*}+bB(Zp_= z*~4D;v7ZAR7~ymvf=Hr>CWcr#5l3gb(3N-+=uQt3Ng|n^ zq|lo_^ravD89*unNh6&>WROV~gBik5hLKGUx#W>g0fiJ%ObMloqm1!PU?P*4%oL_l z&NQYogPF`?HglNEJm#~23M#3hni>|eh+68XXE94?U@6O3&I%e?$tqT}hPA9?Jsa4_ zCN{H$t!!gEJJ`uCn%GS{a5@k{BvC{YLoA($ zqcdITN<0a4rw557kxWlg=uIE`(vSWOAeDinkwe^2n!vLW(G+ zgi^*)#&{+$kx5Ku3R5X(8q=A|_^B?53GL>}4POIlw^ZoTiOK4yz%UI3|8d=FIR>(8$u64MO*4Dg%RcsV zfP)<3Fh^(=t^5fjh+skpC5&)75J4nSL=!_Sort3|UFb?Y33R6ii6oIsPg3YjANtad z{tO_MfuxbnATr1#i@^+GD8tAmhg|Z=r+`9=D5iu`#!<$2CNPmnOlAsGDQ6nfnZZnE zF`GHeWghccKn0alQB4gCSwt;$)U%i+G_aIqEN2CctYj6dS;Jb^v7QZVWD}d&!dAAi zogM6C7ftM@nLX@fANx7LK@M@4BeXhM`4dPG!GsV>7~ymvf=Hr>CWcr#5l3gb(3N-+ z=uQt3Ng|n^q|lo_^ravD89*unNh6&>WROV~gBik5hLKGUx#W>g0fiJ%ObMloqm1!P zU?P*4%oL_l&NQYogPF`?HglNEJm#~23M#3hni>|eh+68XXE94?U@6O3&I%e?$tqT} zhPA9?Jsa4_CN{H$t!!gEJJ`uCn%GS{a5@k{ zBvC{YLoA($qcdITN<0a4rw557kxWlg=uIE`(vSWOAeDinkwe z^2n!vLW(G+gi^*)#&{+$kx5Ku3R5X(8q=Awvz`rXWD}d&!dAAiogM6C7rWWRUiLBIIQ<#OAO7~w<^Nfgn<5KA2Kj3a?Wl1L_nRMHsF1ST?x$xI=g3^K_gn;de>(8$u4%YhrR4$K!E-XWDtWH z!cc}WoDqy<6r&l#SON$nh+skpC5&()h$M<=Vu&S&r-s3tmJuC@dB%Pku|);T3%)yuTaXXl<^wnyiNsgP|2HA@fOv* zO%3l*%e&O^9`(FW10T@HhcxjK&3sG?pU}#uwDB43d`<^n(8-r{@fF>CO%LDD%eVCL z9sPXIdVXL7KeCaZ*u>9l<`=f`D_i-EZT!x5{$K}xvXj5q#oz4aANKGsd-;!j?3eCt z$Nn6^fehpz25~ThIfNk`%1{nt7>6^QBN)MvjN~XraWtbjhA|w=SdJrr;|b&hf;f?2 zP9lVp3FQ>RIF)ctBZAY3#~9O5{ac+O)S=aaw%Byu51TtqS#lfosW zaw%zC#&|Ag0#`7RE1ASqOy+8)a1H5PO9t1G$@OG$1KHe24mXj@&E#jcSYF=awFR_-FS;s4s@+xJ#Mmeuj!5dWaCRMyeHE&bH zJJj+nb-YJC@6*5sH1Z)$d_*%J)50gT@+ob6MmwL=!54J$C0%?)H(%4kH}vu?eSAkh z-?N?{*ual$=qGn@H^E&R$>eq$TIvz0~DQR5B zcrIrGS1^$)nZ#90=4z&J4e4A<2G^0v^<;4a+1yADH<8QDW}8&vWpRlG$tZ&SlN)bcKM zyhlCn)4&Hb@*z!pL^B`L!Y8!yDQ$d4JD=0R7j*I^U3^71U(>@k^ztozd`Cauvz{N= zz>jR?CpPgjoB4$;{K{5-V;jG-oj=&YpX}r>cJVj6`G-CH%U=FtANyTr{@I@cIFNxH z#2^l4Fo!UNLmA3p4C8Qya|9zel93$6D2`?{$1sLt8Ow15a6EyWKoBPq%t?fBGXHy5 zKim(U6YTDZB#}%CsiZNU2~1=XlbJ#~8Dx@0HaX;y$5f^;I&HLPVFrIb-l1(j4$O%1iwQBMPnG|@~8t+dfj2c2}$ zO%J{F(a(A|u#rt{W(!-{#&&kFlU?j)4}00ifaBe57|0+7GlZcGV>lxi$tXrMhOqCWTbe7|#SIGKtAdA)O2|$s(H^a>-*V)0oZ- zW-^P}%waC`m`^?nC}1IrSj-ZZvW(@dppYVpDPbk6Sj`&NvW`;9D5ru-s;H)hTI#5$ zfkv8WriE78Xs3fty6C2dUi#=~Jsa4_CN{H$t!!gEJJ`uCcC&}Q>|;Qn{tRRggBik5 zhB2HGjARs}8N*lt2qcJLLI@>{a3Y8#ifCepC60K;kw79zB$GlaX^dwA6Pd(hrjSks znPibo4!Ptpm1#_81~Zw(Z00bRdCVuD1r)H5MJ#3sOIgNpR!~S0#gwp;Rjg(WYgtDr zWt3AvB~?^YLoIdG(?BClG}A&WZM4%tCtY;YLoa>wvz`rXWD}d&!dAAiogM6C7rWWR zUiLBI1pOJvAO7~w<^Nfgn<5KA2Kj3a?Wl1L_n zRMHsF1ST?x$xI=g3^K_gn;de>(8$u4%YhrR4$K#=|nWDtWH!cc}WoDqy<6r&l#SON$nh+skpC5&()h$M<= zVu&S9BTY2ZLMv^w(?KU) zbkjpGee|=Q4Qyl+o7uuvwy~WZ>|__a*~4D;G2le~8OR_8GlZcGV>lxi$tXrMhOqCWTbe7|#SIGKtAdA)O2|$s(H^a>-*V)0oZ- zW-^P}%waC`m`^?nC}1IrSj-ZZvW(@dppYVpDPbk6Sj`&NvW`;9D5ru-s;H)hTI#5$ zfkv8WriE78Xs3fty6C2dUi#=~Jsa4_CN{H$t!!gEJJ`uCcC&}Q>|;Q%{tRRggBik5 zhB2HGjARs}8N*lt2qcJLLI@>{a3Y8#ifCepC60K;kw79zB$GlaX^dwA6Pd(hrjSks znPibo4!Ptpm1#_81~Zw(Z00bRdCVuD1r)H5MJ#3sOIgNpR!~S0#gwp;Rjg(WYgtDr zWt3AvB~?^YLoIdG(?BClG}A&WZM4%tCtY;YLoa>wvz`rXWD}d&!dAAiogM6C7rWWR zUiLBIB>fr4AO7~w<^Nfgn<5KA2Kj3a?Wl1L_n zRMHsF1ST?x$xI=g3^K_gn;de>(8$u4%YhrR4$K#2YfWDtWH!cd0s|G)pguOGDE0Y8V@-A^HmQwirZA~>B$ z&LE02iRLU~IGb3`A&zs2=RC%7J_%evA{UaxMI>`EDO^G-my*V1jOTJDa0L^&l1W^} zWUgik*O1P&WN;msTu&A^kj;(ca1*)QOdhu|m0Ov{ZA|BOW^e~HxszGk#cb|o4)-vZ zdzr_5%;$dcd4L5xNC6MAkcU~sBP`}omhc!$d7NcD!E&Bt1y51P(-iRx#XL(1&#{u{ zS;Y&i=0(=<5^H&xb-Y3;uTsWql=C_jyg?;zQpHO*>Eqp>NpVG!>wDUO~d_gB)(#2PF^EEwuLoeUb$9MGeJ?r^_4gAPPeqs|pvzcGm z!mn)QH@5LR+xdeX{K-!KVi$k2n}68DzwG5d_OV~4yB+&;00%OVgBZlY4CWAqa417L zjA0zkaE@REM>3M57{$?y<`~9sEMqy20FEb+6A0o&f;ov0P9~I72;)@3IgJQTCz3OW z;!L7Bix|!(mUD>XT;e&8ahy*A7m&z>BykbRTucg=kjkZ`aT(*eoC#dPM6P5KS23BZ znZh-sb1fNMM<&;k#SLV0BRSkeE;p0MEllNBrg0n7xt$r@!A$OC7I!h5yP3m1%;jF@ zaUb)!pL`x*0S{8ZLoDQB7V!v+d6XqQ#!?<<8BegBCt1N$6!J7hJVP4ewCPyVUU>^}J65AJE8$H1QG5 zd`t_U(8{N@@fq!WP6uDm$(MBT72SMI58u$sxAgHH{d~`QeqaMXvXP(I#LsNz7q;*# zTltM`{LXg%U;Eu#}UBs1abmFoJcSy5yHuYatdLbN;szx!RbVD22q?z zG-naR*~D@Vahyv$=P{1+N#Ft!xsW6-BAJUx;Sy50lr%15JeM-7pgFBeXoy_7cW^*@lxQDsi%RKI5 zKKGN)11#V{3V4WxJj@~wVoY$$~4Jvt)D&C@+x2fSBYI&DB-lLxPY2X7I`H&_)qM46r z;S*ZPbyZD>k{KFppWiS7+kNvXDKl^h42QrX@7{tL0<`9N(C__1nVI0nI zj$i~wGLoYh#nFuB7{+icV>ylhjwg^42;xM7If)QXCX`bM<5a>qjR;OBk~4_nOrkl9 z7|te^bBN&u{4^qHGEaYJp@d%4~lqEdIQXXd+Pq3UPS;12j@-#&}Lov@%!gH+Tc~8zMmP~f5=AsI#1cn5<47QpB$7!Xl{CgPfr(6F zGE+z=gG{o>CWl<|n94M!GlQATVm5P_%RJ_j&jJcq$RZZAgrzKFIV&inh+;}u$tqT} zhPAAtlrqYxppq)8siBrS>S>^nCYouXl{VVxpp!1T>7kcC`dQBgHnNG$Y+)xG)t69TZ)=^3sh{Y^nDa%;S3JNKrm=ad9iq))PE$b+yjB+Zdq>5^4sHKj28fc`6W?E>a zjdnWdq>FBP=%tT-*0X_)Y+^H8*vdAxvxA-NVmEu(%RUC2qCW!}#9)Rnlwk~K1S1*6 zXvQ#>00Idjm=HnT3aU_sP63L{HN*d#tz(gi7nJJ``K_*#blS3|f zOl2C=nZZnEF`GHeWghd%X8{E)WD$#5!cvy8oD~#OL@_0-WEHDf!&=r+N*U!;P)QZl z)KE(u^)%2(6V0^HN*nET&`B5F^w3Km{j6sL8`;EWwy>3LY-a~M*~M=5u$O%d2-BZ| z3}P@t7|Jk)GlG$fVl-nIO8|ic5ljf7gb_{zkwg(q46(!!&o~lDB#C5FNF|N&Okg6D zn9LN?$sm&~vdJNrJf<>@>C9jzvzW~s<}#1@YDyXE2YHFyZj(Qqsq={x)Xr+yII_RW}ZhGjYkABv(fsJfpGh5ioHny{a zo$O*ad)Ui92Arxt0~y3%hA@<23}*x*8O3PEFqQxU2_l#fLJ1?B2qK9hniyhxG)t69TZ)=^3sh{Y^nDa%;S3JNKrm=ad9iq))PE$b+yjB+Zdq>5^4sHKj28fc`6W?E>a zjdnWdq>FBP=%tT-*0X_)Y+^H8*vdAxvxA-NVmEu(%RUC2rauE2#9)Rnlwk~K1S1*6 zXvQ#>00Idjm=HnT3aU_sP63L{HN*d#tz(gi7nJJ``K_*#blS3|f zOl2C=nZZnEF`GHeWghd%X8{E)WD$#5!cvy8oD~#OL@_0-WEHDf!&=r+N*U!;P)QZl z)KE(u^)%2(6V0^HN*nET&`B5F^w3Km{rul|_2G8+BN)MvjN~XraWtbjhA|w=SdJrr z;|b&hf;f?2P9lVp3FQ>RIF)ctBZAY3#~9O5{ac+O)S=aaw%Byu51 zTtqS#lfosWaw%zC#&|Ag0#`7RE1ASqOy+8)a1H5PO9t1G$@OG$1KHe24mXj@&E#jcSYF=awFR_-FS;s4s@+xJ#Mmeuj!5dWa zCRMyeHE&bHJJj+nb-YJC@6*5sH1Z)$d_*%J)50gT@+ob6MmwL=!54J$C0%?)H(%4k zH}vu?eSAkh-?N?{*ual$=qGn@H^E&R$>eq$TIvz#3 z%*CW|38`F48kaGi%bCCxOyo)?aTSxfnkigEI@glHb!2iqS=>N2HyS;I@Ls0UtmApw6 zZ&A(L)bI|qyh|PLQP2A{@Bxi{NE08?%*V9w39Woe8=uk6=XCG|oqS0bU(wCi^zaS6 zd`lnS(a-m+=La_MBOCdNP5jJeeqjs0vX$T1#_w$B4|eb;JNb)U{LOCuVGsYZm;czu zem9zb_U8Z&WFQAIh=Uo-Aq?SAhH@CgIGo`e!3d6IBu6odqZ!RHjNw?uavT91Par1{ z#EArR5+R&SD5ns{sf2SH5u8pWXAs4iL~|A~oJ}m}5XZU1a~|V3p9C%-kqb%UB9gh7 z6fPl^OG)E0#&bCnxPpmX$t12~GFLN&Ye?r>GPsURt|yBd$mT|JxQSeDCXZW~%B@V} zHl}ksGq{79+{rBNVm5a(hkKaIz0Bi2=5s&!Jir1Tq=1K5$ipn+5f<|(OL&Z>JkBzn zU^!2+f~P3tX^MD;VxFah=UB<}tl|Y$^CD|_iM71UI$oiaS1IE)%6Xj%-k_2F&Bbxb`7CxbsPif;b+WDLgzMzvY>EbK8`I;WSp_gyz z<2(BKp7s2|27Y8CKe36Q*~~9&;a9fu8{7Du?fk(G{$wYAv5UXi%|GnnU-t4J``9nX z{Ifp?a3BLYh(R38U=CpjhccAI7{=iY=Lkk{BqKSBQ5?-^j$sVPGM3{A;CKQ#fgnyK zn3D+MWI{QGFis_$(}>`7A~}O7&Lo<%h~aEvIfppTC7$yb$N4010f}5l5*Lxo#iVcv zsa#4Lmoc8pnZOlH z+nK=~%;ZjHaTl|>n>pOWT<&Ea_c5RQ$>#wU@E`>|#6lis5s$E#M_IySEah>Q@dV3x zk`+8fAx~4pGZgbIB|OJUo@W&=u$mWH!%M8?W!CWurMyZRuTjqHRPY9syh#;rQO(=b z@D8=SOC9e~&-*m+0gZe}6Ccsc$F%SXt$a!wpV7|ebnpe8d`TBy(aqQN@D06uOCR6S z&-bk72R85{8~KS%{LE&4VGF;qmEYLL?`-D}cJL=V`HNlr&Hvuj-w!=xzx8(a4Qyl+ zo7uuvwy~WZ>|__a*~4D;F(AU-hJg%XFhdy1ForXNk&I$AV;D;Sfdmmu2%&@#P6Uxe z5lsxS#1YRp5=bP8WKu{ajqyxiB9oZR6w=8clPt2yA(uR+GL7lXU?#Je%^c=3kNM=Y zfC3h>h{Y^nDa%;S3JNKrm=ad9iq))PE$b+yjB+Zdq>5^4sHKj28fc`6W?E>ajdnWd zq>FBP=%tT-*0X_)Y+^H8*vdAxvxA-NVmEu(%RUC2u0I1A#9)Rnlwk~K1S1*6XvQ#> z00Idjm=HnT3aU_sP63L{HN*d#tz(gi7nJJ``K_*#blS3|fOl2C= znZZnEF`GHeWghd%X8{E)WD$#5!cvy8oD~#OL@_0-WEHDf!&=r+N*U!;P)QZl)KE(u z^)%2(6V0^HN*nET&`B5F^w3Km{j6sL8`;EWwy>3LY-a~M*~M=5u$O%dh}55f3}P@t z7|Jk)GlG$fVl-nIO8|ic5ljf7gb_{zkwg(q46(!!&o~lDB#C5FNF|N&Okg6Dn9LN? z$sm&~vdJNrJf<>@>C9jzvzW~s<}#1@YDyXE2YHFyZj(Qqsq={x)Xr+yII_RW}ZhGjYkABv(fsJfpGh5ioHny{ao$O*a zd)Ui92ArWk0~y3%hA@<23}*x*8O3PEFqQxU2_l#fLJ1?B2qK9hniyhxG)t69TZ)=^3sh{Y^nDa%;S3JNKrm=ad9iq))PE$b+yjB+Zdq>5^4sHKj28fc`6W?E>ajdnWd zq>FBP=%tT-*0X_)Y+^H8*vdAxvxA-NVmEu(%RUC2sXqf5#9)Rnlwk~K1S1*6XvQ#> z00Idjm=HnT3aU_sP63L{HN*d#tz(gi7nJJ``K_*#blS3|fOl2C= znZZnEF`GHeWghd%X8{E)WD$#5!cvy8oD~#OL@_0-WEHDf!&=r+N*U!;P)QZl)KE(u z^)%2(6V0^HN*nET&`B5F^w3Km{j6sL8`;EWwy>3LY-a~M*~M=5u$O%dh}NHh3}P@t z7|Jk)GlG$fVl-nIO8|ic5ljf7gb_{zkwg(q46(!!&o~lDB#C5FNF|N&Okg6Dn9LN? z$sm&~vdJNrJf<>@>C9jzvzW~s<}#1@YDyXE2YHFyZj(Qqsq={x)Xr+yII_RW}ZhGjYkABv(fsJfpGh5ioHny{ao$O*a zd)Ui92Ario0~y3%hA@<23}*x*8O3PEFqQxU2_l#fLJ1?B2qK9hniyhxG)t69TZ)=^3s<^11w_1$*&f7rvn?Bzf9vENPZcI?jq9LPWpVh{&2m_r!C zp$z3PhH*H6@_B#-JV*f#v5<#Z#3L-`QI_x+OL?4SJi&6FWCc%A$kP<@48=T43D2>T z=UK%ItmZ}5@DgiznRUEEDX&t-Yn1ak6}&+uZ&JluRP#19yhAPTQpbDL^F9rHKqDX0 z#78vqF)e&TE1%NFXSDM<9ehD2U(&@_bn`Vmd_yna(#Lo7^F8bNferk~Mt))wKeL%% z*ut-DmO8G%jO2motGYn8=k(;wmO{ zHB-2Tbgm_X>&WDKvbce4ZX}1B$mM48xP__Q$~10cI=3@}JDACx%;GL)b2oFihq>I# zJnmyY_mj^9EZ{*3c!-5O%px9PF^{r@$5_haEaM55^CT;Hib9^Ih-WC~SxR`0l|0WX zUSKsZvWAyf%ge0e6-s%PGG3#c*QwwQDtVJC-lCefso@=Jd6zogqn`I^-~$@@kS0E& znU87V6I%I{Ha?@B&*|U`I{A_=zM`A2>ERoC`IbJuqo40t&ktg^koA{Z{{K6J~ zWh=k2jo;bMAMD^ycJddy_?zAQ!yf)+FaNQR{cbk@?9Txl$UqKa5C=1uLm0xL4COF} zaX7;{f)O0aNRDC@M>CpZ7{jrQB$&LE02 ziRLU~IGb3`A&zs2=RC%7J_%evA{UaxMI>`EDO^G-my*V1jOTJDa0L^&l1W^}WUgik z*O1P&WN;msTu&A^kj;(ca1*)QOdhu|m0Ov{ZA|BOW^e~HxszGk#cb|o4)-vZdzr_5 z%;$dcd4L5xNC6MAkcU~sBP`}omhc!$d7NcD!E&Bt1y51P(-iRx#XL(1&#{u{S;Y&i z=0(=<5^H&xb-Y3;uTsWql=C_jyg?;zQpHO*> zEqp>NpVG!>wDUO~d_gB)(#2PF^EEwuLoeUb$9MGeJ?r^_4gAPPeqs|pvzcGm!mn)Q zH@5LR+xdeX{K-!KVi$k2n}68DzwG5d_OV}{`DcF);6Mg)5Q8|F!5qR64rM5ZF^t0* z&Jm2@NJerLqd1z;9K#rnWh}=L!0`lf0zsTeFeeeh$%JwWVVp`hrxC&FL~;gEoJlli z5yRQUat?8vOFZW>j`K<20us59BrYPEi%H=UQn{2gE@M2GGl46Z$dyduDkgI^Q@Dn7 zt|f!($mDvmxPfeLB!`>Gg$x!lV2&%dF!SN_mwsUZb4Xso)JNd6O#MqMEm<;T>vumpa~~p7&|s0~+~|CO)E>k7?l( zTKSYVKBJw_>EH`G`I0WaqMNVj;TwASmOj3tpYQqKyZX^7hwN8jcdw+1YHFyZj(Qqs zq={x)Xr+yII_RW}ZhGjYkABv(fsJfpGh5ioHny{ao$O*ad)Ui92E@4AFpxnEW(Y$W z#&AY3l2MFi3}Xo(kRXByA(Sw}i6D|FqKP4vIN}*c0*NG%ObV%_F`fxbWD=8^LOK~_ zl0`N-F`s-EP{2YKv6v++Wf{v^K_NvHQ^HDCv6?lkWgVrI zQBDPwR8dV0wbW5h1C2D%Obe~F(M|`QbkR)@z4Xz~dN#0;O>AZhTiM2TcCeFO>}C&p z*~fsh^=BZ17|alcGK}GjU?ig$%^1cKKp;T`6GA9qgcCs|QA86%EOEp$jsy}(BAFCY zNn<<{n8+k1Glg_A$Rvwwa>yl*sZ3)!GnmONW;2Jm%ws;I&HLPVFrIb-l1(j4$O%1iwQBMPnG|@~8t+dfj2c2}$ zO%J{F(a(A|u#rt{W(!-{#&&kFlU?j)4}00ifOGU`AcGjp5QZ|0;f!D;qZrK?#u7jv zK?D;*C}D&XK_pQ`6GJR<#50Zr5=kPN6jDiJJQJA6BqlS3bTY^!i)?bpC6B30V>&aK z$t-3whq=sSKKU%5fQ2k#F-us=GM2M~LW(G+gq5sfHEUSQI!Y;{oC+$bqM90NsiU3- z8fl`L7FubeoenzbqMIIi>7$?ZY+xgs*vuBTvW@NRU?;oS%^vo$j{$M|Gmt?HW(Y$W z#&AY3l2MFi3}Xo(kRXByA(Sw}i6D|FqKP4vIN}*c0*NG%ObV%_F`fxbWD=8^LOK~_ zl0`N-F`s-EP{2YKv6v++Wf{v^K_NvHQ^HDCv6?lkWgVrI zQBDPwR8dV0wbW5h1C2D%Obe~F(M|`QbkR)@z4Xz~dN#0;O>AZhTiM2TcCeFO>}C&p z*~fr$^=BZ17|alcGK}GjU?ig$%^1cKKp;T`6GA9qgcCs|QA86%EOEp$jsy}(BAFCY zNn<<{n8+k1Glg_A$Rvwwa>yl*sZ3)!GnmONW;2Jm%ws;I&HLPVFrIb-l1(j4$O%1iwQBMPnG|@~8t+dfj2c2}$ zO%J{F(a(A|u#rt{W(!-{#&&kFlU?j)4}00ifb;ZcAcGjp5QZ|0;f!D;qZrK?#u7jv zK?D;*C}D&XK_pQ`6GJR<#50Zr5=kPN6jDk1f3)3YbQD3`;Qe+N+}+*Xf_recU;%<# zfB+$QaED;QA-KD{ySux)L-x<}-uJuvbw6yK^P8@ko|@`8*YxyU)jelW6SYwn_0bTG z(G<-QgjQ&SV6;aFI-v`?p$B>)6nznf0SHF~{?GI356Img;t?L>37+B^p5p~x;uT)w z4c_7%-s1y4;uAjO3%=qTzT*de;un775B?$+LPzH9AVG!#6&iFHFk!)l0~a2A#6eud zLwqDaLL@?BBtcRnLvo}*N~A(+q(NGwLwaODMr1-}WIt^6hToGLvfTqNt8kWN}~+Qq8!Sj0xF^sDx(Ujq8h3r5H(N}wNM*%P#5)39}UnD zjnEiP&=k$k94!!pmS~06XoI#0Mmw}e2ZW#_I-xVVpewqeJ9?ledZ9N$(Fc9e4`Jw! z0T_sI3_=73V+e*~7=~j6Mj{fUFdAbp7UM7;6EG2zFd0)Y71J;sGZ2NDn1$JxgSm*t zJj}-eEW{!##u6;WGQ?mxR$wJoVKvrZE!JT@Hee$*VKcU1E4E=fc3>xVVK??*FZN+S z4&WdT;V_QiD30McPT(X?;WW>EXoyB=j3#J`W@wHU2trG=LTj`^TLhyW+M@$P&=H-`8C}p7-OwF9 z&=bAT8=>ffzUYTA^v3`UL^uW^0)sIGLop1)F#;nIiBTAhF&K++7>@~r z6Sr_1cW@W?a32rw5RdQ}Pw*7a@EkAj60h(YZ}1lH@E#xV5uflGU+@**@Et$!6Tk2q zfAANv5IVE}LxKziDm3UYV8Vh82QEDLh=aI@hxkZ1$ltvkpMLCp5 z1yn>OR7Mq4MKx4MAZnl{YN0mjpf2j6J{q7Q8lf?opedT6Ia(kHEzt_C(FSc1jCN>` z4hTUBA!f*V+U&Mk5OGktR846Tr&|$!Y1se`rc<>PiaS;#kkpKyi2#Jvd zNs$c6kpd}^3aOC>X^{@;kpUTz37L@vS&CfiG(&T= zKoDA@6kHSf{y5f&gg=!=!Wj-fu87v-Uvk>^hG~}p+5#-Ai^;S5g3di z7>Z#Sju9A%NQ}a0jKNrp!+1=gXLI(l~{$;hy6H!gE)l4ID(@% zhU56}dG+ocEMW+_`!I|^Bt~N_#$zHTV=AU23bQZ=(U^~gSd68J!3wOx8mz+xY{C|7 z!w&4i9_+&b9KsPC!wHwz zAH+foVBQ8QbeOQ=!be=hM?xe1fUGc zp#mzQ3aTLxHBlRNQ6CM_7){X}L1=|G2u6E^pcA^F8+xD@LeUpt7=Um@UkN|7xrKu4&V@u;22Ke6wcrr zF5nWb;2Lh=7Vh949^et4;2B=v72e<-KHw9+;2VD67ycj?Vrk+J6*^4VaN#2^;v*pv zBPo(2B~l|T(jy}>BP+5aCvqb%@}nRMqbQ1_Bmz(dE4-fDNPw)&c z@CtA64j=FdU+@h-@C$zs3$YCGhYB4gY`E|d7x9r0iIEh^krJtq7U_`@nUNLQkrTO* z7x_^Tg;5m6Q4#?tgL0^VN~nTr2t-ZPMqSiLLo`NHG)E9xp$&r39wF$2F6f3H=!H=9 zMHmJk91$3TVHkl(jK)}u$3#rVR7^({W?>GZF&_)D7)ud@6Q40h{Q;WwzAH+f|NBp5ehY1@le8fe3Bt&8)MRKG> zYNSPaWJG3UMRw#wZsbLN6hvVZMRAlw0Lq{oDxeaopc(>E6SYwn_0bTG(G<-QgjQ&S zV6;aFI-v`?p$B>)6nznf0SHF~hF};*AQGc77UMAylQ9+35rtWpgJ{ghLM+Bo#9#$h zVGY({12$m`wqXZ$VGs7<01n{@j^PAO;SA2<0xsbSuHgo5;STQM0UqHAp5X;v;SJv5 z13uvkzTpRc;SXXVmM8vDp~Hj?7e3-5J`y4^k|H@$A~n(?Ju)IQvLZWjA~*6PKMJBS zilR75A^>Gj4i!)dRZtCqsEOLBi~4AY#%PM>2tq5gK``1Q1f9?Y-OvNQ5Q@GC!vKUM z0z)tiBM^zv7>n_kh{>3W>4?HC%t18fV<8q}DPph!tFQ*^umPK}1>3L#yRZlQZ~%vJ z1jld!r*H=6Z~>Qa1=nx`w{Qpd@BokS1kdmSukZ%%@ByFj1>f)kzwigK5GxRWsL)}; zh6^8Y5g!SW7)g;FDUlj!kscY58Cj7XIguNAksk$77)4PWB@uu!D2EEDges_pK-5HS z)J1(XL}N5Xa|EFk+8`M15rR(Wf^O)6UI;~Bgkb=}5rH8Xh7pLwXpF^pOvGeN#dJhr z7Um!t^RW<%u@o^_fmK+8b=ZJS*n(}?fnC^xeK>$aID%t1fm1kxbGU#@xPoiAfm^tP zdw76Hc!Fnmfme8gcldx$_=0cvfnWH8ScnygKUC;2VZ()wxQLI0NQ|UNj+97^v`CMP z$c(JWj-1GiyvUD&D2$>gj*R6-S0Lm+D6U-_4>|NDRS6XfnEaSEq#24`^& z=WziSaS4}k1y^wm*Kq?kaSOL`2X}D~_wfJ^@d%Ic1W)k{&+!5;@d~f;25<2W@9_a2 z@d=;t1z+(E-|+)K@e9B42Y(R@p)2!tkRU^W3Jp38n6O~OfeQ~l;vg>KAwCiyArc`m zk{~IPAvsbYB~l?Z(jYC;Aw4o6BQhZ~vLGw6Av=!3rKhcNWV01QMp1|b52 zF$6;~48t)3BN2&F7>zL)i*Xo_37CjUn2afyifNdR8HmD6%))HU!CXXR9_C{K7Ge<= zV+odG8Dg*;E3gu)uo`Qy7VEGc8?X_Zuo+vh72B{KJFpYGup4`@7yGau2XGLFa2Q8$ z6vuEJCvXy{a2jWD7Uyst7jO}ma2Z!{71wYbH*gcTa2t1U7x!=<5AYC=@EA|<6wmM+ zFYpqt@EULM7Vq#LAMg>M@EKq572oh3KkyU3@Ed>d7qQ?XV@HGp846Tr&|$!Y1se`r zc<>PiaS;#kkpKyi2#JvdNs$c6kpd}^3aOC>X^{@;kpUTz37L@vS&CfiG(&T=KoDA@6kHSf{y5f&gg=!=!Wj-fu87v-Uvk> z^hG~}p+5#-Ai^;S5g3di7>Z#Sju9A%NQ}a0jKNrp!+1=gXLI(l~{$;hy6H!gE)l4ID(@%hT}MalQ@ObID@k|hx53Ai@1c#xPq&=hU>V2o4AGBxP!a6 zhx>Sdhj@g?c!H;RhUa*Jmw1KOc!Rfihxhn^kNAYo_=2zahVS@+pZJB}_=CTQh0vY- z9};9JP@zGG0TUK%IB?;?M;ydOJj6!=Bt#-4MiL}NG9*U|q(myDMjE6=I;2MiWJD%p zMiyj6He^Q*o_0a$g(Fl#v1WnNl&CvouXo*&6jW%eDV6;PfbU+9?q7yo! z3%a5kx}yhrq8EB26n)Sa{Sb!!7=VEY#~?&tFos|#hG95HU?d_j3ZpRwV=)fnF#!`X z36n7eQ!x$GF#}PUiCLJ9Ihc!R%)@*vz(Op-Vl2T@EJF;IV+B@X6;@*n)?yvjV*@r~ z6EL0Y6kdSpOGWI|?SK~`i#cH}@#IBTLIeh52!>)9hGPUq zA`+u88e=dP<1ii*FcFh58B;J7(=Z(~5QUkTh1r;cxroL*%*O&O#3C%l5-i0s#9%pA zU?o;zHP&D))?q#VdtUu`q$SiQcdv{3Xo$vWislGHE3`o{+9L#=&;{Ml1HBN6z6iqr zgd+k&FbpFQiP0E~@tBCon2PC$!Ys@|H0EO=7Go)5umY>F2J5f^o3I7jumiiW2m5dU zhj0YPZ~~`r2Ip`Amv9Bwa09n+2lwy*kMIP~@B**!2Ji3zpYR3W@B_c_2eA+eDE zI!xGb;Ug~MBOwwaDUu^4QX?(WBO@{+E3zXeaw9MDqaX^SD2k&b0#F9!Pyv-t1=SFU zny8JssE>wdjHYOgAhbdo1fxAd&5&nckrmmI6S9Rq(ypUL}p|~cH~5EHUq5~DE|<1rDFF%{Dhg;|(` zXw1h#EXGpAUum}5a0Ech{$8Z9ta0cga0he$E*Kh;3a0mDB0FUql&+r1T@CNVj0iW;%-|z#! z@CUIFs}p~y&|$)c3mCr`1fdn$AQtt2t{9nVF1Drfgu=%5s1WS zjKz3N#AHmxbVOkm<{%pLu@H-~6fszVRak>{*nmygf^FD=UD$(tIDkVqf@3&=Q#gZj zxPVKzf@`>eTeyRJcz{QEf@gSvS9pVW_<&FNf^YbNU-*Mqh=If(Ds-5z;lf8;#79CT zMp7h4N~A_wq(??%Mpk4;PUJ>jHV8(0grF0;pc{Ii7edh&VHkjLL|_PpVFV&E8e=gY6EPW6F&$Bug*k}Ed@RIb zEJX}fU=`M29X4PSwqP4}U>Eja9}eIUj^G$h;1tf_94_DzuHYJO;1=%S9vHDul{Lk(GgQX~VrT-22Z#-VF7w114u%NRk{HxC7_5WM`bDesU<(82CKd}*oRR427 ztNy3IOkD2&Yf%0@pG~)*khcGSn~L=BevKvd(*MT(*JfPpIL~_>&+*^#p8CJ_j4l2f z|6lpnR=LjN-xf)NEGU91Xo4;nf+<*nEjWTJc!Dp)5#kE*g!n=NA)$~+NGv1~k_yR$ z3_?aBlaN`+B4ib^3E71lLQWx`Go>PL7|XP zSSTVC6^aSPg%UzZp_C9HlorYeWrcD=d7*+(QK%$T7ODtUg=#`|AyB9x)D&t7wS_uD zU7?;(UuYmS6dDPQg(gB%p_$NJXdwg%ErnJ>YoU$MRtOf_3GIarLWs~&=p=L&x(Ho` zZbEmVhtN~#CG-|Tg+4-Gp`Q>Y^cMyQ1BGy5kPsmZ7KR8zg<-;QVT3SJh!jQ%qlGcT zSYeznUYHRVVW>qm?1<7Glf~gY+;TtSBMto3G;;o!a`w@uvl0kEESds zF~V|Tg|JdsC9D?K2y2CP!g^tYuu<3~Y!Z zI3yevjtEDEW5RLagm6+gC7c${2xoa8bAW6^Q@ADE7VZdl zg?qw%;eqf_cqBX)o(NBcXTo#gh44~%CA=2i2ycaV!h7L^@KN|Ad=|b4UxjbNcj1Td zQ}`wP7XAo-h1h@Z2vHJcQ4v*96LrxLP0>_p*yNTV!9%4_im)Khj75j*N#eQO#*k2qV4iv-1L1KhBSR5h_6^Dt##S!92F;W~Q zjuyv=W5sdecyWR_QJf@B7N>|)#cASnafTQr&J<^fv&A{$TrpalC(aiahzrF<;$m@$ zxKvyw#)!+s72-;9mAG15Bd!(KiR;A;;zn_kxLMpHZWXtQ+r=H?PH~sGTihe=759nz z#RK9&@sM~}JR%+ykBP^{6XHqnlz3V^Bc2t{iRZ-&;zjY2cv-w6UKOv2*Toy+P4Sj^ zTf8IQ74M1n#RuX;@sapgd?G#-pNY@K7vf9tmH1kGBfb^iiSNY^;z#k5_*wiSeigro z-^Cx|Pw|)dTl^#b6=T`?MM;umNs&}ZlXS_DOv#dL$&p;ilYA+T6jzET#g`ID38h3* zVkwD~R7xf#mr_V6rBqUCDUFm?N++e4GDsPvOj2eki3 zsgu-M>LPWOx=G!o9#T)Km(*JdmHJ41rG8SF)L$AP4V1#AK~jV?SQ;V?m4->fr4iCd zDN-6Gjh4npW2JG@cxi$(QJN%8mZnHkrD@W1X@(Rf&6H+Iv!yxGTq#5z0-IwBpFj!DO*6VgfPlyq7;Bb}AbN#~^t(naZ#bXmG0U6rm$*QFcM zP3e|&Te>6NmF`LRr3cbO>5=qUdLliQo=MN87t%}VmGoMABfXX0N$;f((nsl&^jZ2M zeU-jR-=!bYPwAKRTlypYm16nm7iCG7WkptHP1a>YHf2k;Wk+^pPxj?Fa$Gr{9A8c# zCzKP(iRC16QaPENTuvdUlvBy6=7nBRhh2~+){2Ox0c(;ZRKFOo!nmT zAcx2uk`HB2gekT9-xc*#zA-|Mg$*<)%@>}_x{9gVb zf0RGTpXD#|SNWU#UH&2elz+*;qA04ODY{}PreZ0!;wY}-DZUa% ziL1m@;wuT1gi0bMv64hdsw7jAD=CzeN-8C_l153Zq*Kx>8I+7lCMC0yMaimUQ?e^L zl$=T~CAX4C$*bg3@+$?Df=VH!uu?=RsuWX-D8}h>1}fpoASFT> ztPD|xD#MiF$_Qno5~++*Mk`~KvC24QyfQ(Vs7z8OD^rxI$~0xVGDC?{W-7Ck*~%Pc zt`e=xQ|2oRl!eM7WwEkES*k2kVwB~|3T36TN?EO}QPwK!l=aF6Wuvl5*{p0)wkq3{ z?aB^ir?N}gt?W_uD*KfE$^qq|a!5I>98r!c$CTsB3FV}6N;$2ZQO+vol=I33<)U&) zxvX4Kt}54*>&gw~rgBTUt=v)WD)*H8$^+%0@<@5CJW-x1&y?rN3+1KqN_nlkQQj)= zl=sR9<)iXR`K)|VzAE38@5&G5r}9hrt^85`DzS>7imIf_s-mi@rs}GpnyRJRs-wE9 zr}}CfHLe;@jjtw96RL^S#A*^XshUhpuBK2^s;Sh}Y8o}InodoxW>7P#nbgc`7B#Dy zP0g<6P;;ue)ZA(wHLsda&94?v3#x_G!fFw#FtC`f3BUq1s4otTs`bs?F5qY6~?;ZK<|W zTdQrPtWHsI^kXovF@J zXRCA6xoWgJPo1wWP#3C;)Wzx&b*Z{cjZv4YE7XLK;8dPF^{9#fC2C)AVbDfP5^Mm?*ZQ_rgx z)QjpR^|E?Jy{cYQud6rIo9Zp~wt7dstKL)Zs}IzN>Lc~B`b2%IK2x8oFVvUnEA_Sd zMt!TkQ{Sr})Q{>X^|Sg#{i=RbzpFpgpXx95xB5r@tH!E=CTfxp5|+Dw76P4ExwjOOQT3+G@dCJFUIeK?~73YMr#sS{JRW)=lfK_0W21y|mt1sMbg8tM${uwEo%v zZJ-ve4bmdC!P*dQs5VR+u8q(}YLVI~ZL~H<8>@}e#%mL_iP|J>vNlDVs!h|TYcsSc zZKgI$o2|{!=4#Q}JZ-+VKwGFS(iUq=w58fIEk;|ety@aermt8-`XGTuNJEbx~NOKtSh>zYr3u* zx~W^btvkA_d%Ca3(c|jz^!R!LJ)xdRPpl`=lj_Ozgn|KdImkC zo=MNFXVJ6j+4Sss4n3!yOV6$6(evv0^!$1Oy`Wx5FRT~Qi|WPn;(7_aq+Uu7&`axO z^s;(6y}VvQuc%kjE9+JCs(LlOx*n+4&}-_o^xAqIy{=wQudg@I8|sbp#(ERIsoqR) zuD8&G^p<)ny|vy(Z>tCE?ez9~2R%gZsCUvk>s|D&dN;kh-b3%H_tJaop?V*^uij4& z)BEcK^nrS~K1h$y2kS%hq53d=xIRK3sYmLg^wIhleXKrCAFof)C+d^*$@&z1syL7?ck6rfz4|_VzkWbJs2|b~>qqpX`Z4{uenLN~pVCk3 zXY{lBIsL!K_4E1#{i1$JzpP);uj<$I>-r7-rhZGmt>4k_>i6{f`UCx;{z!kUKhdA+ z&-CZ|3;m`3N`I}t(ckLt^!NG){iFU#|Ezz}zv|!g@A?n@r~XU-t^d*g>an_Dh=ydy zhGM9OX6S}tn1*H8hGV#fXZS`OBd!t8h;Jk?5*mq}V$sgcY`Zlo|$8mWxbMj9in zkX&B$)#Fmf8XjNC>ZBd?Lq$Zr%d3L1rs!bTCJs8P%)Zj>-e z8l{W?qqI@RC~K56${Q7oibf@)vQfpTYE(0-8-Yd*qoz^IsBP3S>KgTo`bGnzq0z`_ zY&0>N8qJL6MhhdzXlb-ES{rSQwnnhg&S-CRFhY!uMkk}Q(Z%R$bThgeJ&c}4FQc~+ zYV{@x}yWqA|&s zY)mnx8q= z##n2tGu9g$jE%-7W3#cv*lKJuwi`Q)oyIO>x3S0AYwR=j8wZSo#v$Xdal|-k95ap^ zCybNEDdV(p#yD%7GtL_qjElx4FISzfVC zhGrwPvDw6IYBn>Qn=Q;Bv!&U}Y;CqN+nT{3_Aq;z zz0BTbsM*KtYxXn4%>L#8bD$Y+4l*Om!R8Qis5#6WZjLZVnvv!xbF?|e9BYm<$D0$( ziRL78vN^?^YECn!n={NPbEY}VoNdlA=bF*xJafLez+7l9G8dam%%$csGsawQt}s`c ztIXBr8gs3=&RlP9FgKc;%+2N&bE~<{+-~kLcbdD*-R2&1ues0MZyqoYnupB8<`MI# zdCWX+o-j|Er_9sl8S|`p&OC2kFfW>y%**B#^Qw8xyl&nwZ<@Ev+vXkfu6fVAZ$2;| znvcxK<`eU&`OJK7zA#^!ugurx8}qIC&U|lvFh81~%+KZ*^Q-yI{BHg*f11C{-{v3l zuNi9!mS{qR zrIpG`ZKbi&TIsCxRt77hmC4F%WwEka*{tkV4lAdX%gSx#vGQ8^to&91tDsfLDr^<8 zidx02;#LW(q*clauu5BHtg==)tGrdgs%TZRDqB^os#Z0tx)o^EuxeVhtlCx`tFBeg zs&6&08d{C4##R%nsnyJCZndz2td>?QtF_g}YHI~s?X31z2P?$tXmzqWTV1TKRyV7= z)x+v(^|E?fp;jNOuhq{Av-(>DtbtazHOPvv23td{q1G^KxHZBWX+>J2tkKpOYpgZS z8gEUoCR&rM$<`EWsx{4;Zq2ZwteMs+G*{wc3XR_z1BW!zjeSmXdSW+ zTSu&;)-mh2b;3Gnow80_XRNc$-Krx@q0AZd-S(yVgDH zzV*O*Xg#tXTTiT~)-&t5^}>2-y|P|gZ>+c0JL|pm!TM-@vOZg1tgqHL>$~;C`f2^L zep`R6zgDaz*rF}jvaQ&vt=YP5*rsjSw(Z!i?b*H^$Bt{qv*X(d?1XkAJF%U_PHHE! zliMlmly)jRwVlRJYp1i*+ZpVPb|yQsoyE>-XS1{0IqaNvE<3lK$Iff#v-8^p?1FY7 zyRcovE@~IEi`ymal6EOOz%FfA<>UN-A!>(!9vTNIQ z?7DV6yT0APZfG~M8{19nrgk&Cx!uAJvRm4%?ACT0yR98;x3k;Z9qbUhqut5wYdL9&8V>huXvJ;r0l7q#bFGvPauv z?6LMZd%Qito@h_9C)-o(srEE`x;?{=vS-?}?Ai7ld#)XA&$H*-3+#pVB73pD#9nGI zvt#V#_6mEYy~+JRR279Bu$=+;lvA5dW?EfCux7$1Ho%Sw!x4p;SYwxr7 z+Xw7}_96SQeZ)R$AG43!C+w5*^&JC5Tzp5r@loVZRrC%%)wN$4bU z5<5wpq)sv?xs$?4>7;T}J87J>PC6&OlflX8WO6b)S)8m+HYdB2!^!F7a&kL)oV-px zC%;p`Dd-e(3OhxdqE0cVxKqL@>6CH;oYGDir>s-XDeqKpDms;%%1#xhs#DFW?gTnD zoSIH8r?ykasq55p>N^dbhE5}=vD3t9>NInjJ1v|br=`=%Y3;Oe+B(5bJEy(V!3l9X zI-Q)(P8X-E)6MDb^l*AQy`0`osME*k>-2NNoc_)LXP^`A400lz!Ojq8s58tN?u>9o zI+4yOXS6fM8S9L5#yb<7iOwWvvNOe*>P&N{J2RXpXQngDneEJR<~q^NJZHYMz**=l zauz#FoTbh(C&pRstZ-I3tDM!&8fUGu&ROqla5g%doXyS_XREW#+3xIcb~?M9-Oe6o zud~nD?;LOrI)|LY&JpLRbIdvJoN!J$r<~Ky8Rx8X&N=T~a4tHRoXgG?=c;qfx$fL> zZaTM|+s+;5u5-`1?>ulGI***k&J*XU^UQhfyl`GRubkJ;8|SU_&Ux>Aa6USpoX^e| z=d1J0`R@F1emcLL-_9TBuM_JCuINgx>?*G6YOd}YuIXB??K-aOdam!rapSu2-1u$+ zH=&!zP3$Idle)>={*AE4x+Ps%|y6 zx*O=$aBI4?+}ds(x2{{yt?xE)8@i3$#%>e0soTtL?zV7)+?H-Dx3$~GZR-ZR?cDZm z2RFp+=yq~DyItI_Za25P+r#bY_Hui>p>7|yuiMWJbNjmk+<|VmJIIZ22fIVuq3$qu zxI4lf=|;Mv+|lkBcdR?k9q&$XC%Ti|$?g<)syoe{?#^(d+?nnyceXpno$E%s^W6FF z0(YUi$X)C%ahJNw+!%McyTV=Ru5wqqYuvT&I(NOh!QJR?ayPqM+^z05ce}g8-RbUf zce{Jsz3x7Dzk9$v=pJ$pyGPuk?lJecd%`{Go^nsSXWX;yIrqGK!M*5Saxc4A+^g<2 z_quz-z3JX^Z@YKgyY4;rzWcy^=st2EyHDJw?lbqf`@((czH(o?Z``-;JNLc&!Tsod zazDFY+^_C8_q+SU{ptR4f4hI&zizB6c%mnHvZr{er+K<(c&2B0w&!@R=Xt&t$BXO5 z^Wu96yo6pNFR_=zOX?-_l6xt!tJ3dl|fpUM4TIm&MEKW%IIoIlP=+ zE-$y2$II*G^YVKIynVihCuzl3pnRzB%!>j4l@@ji^yt-aJufEs7Yv?ud8hcH=rd~6zx!1xA@>+VWyw+YDudNsC zwe#A09lQ{)qu0sn>~-|^Z z&GY7a3%rHiB5$#`#9Qhu^J2W^-U@G}x5``Xt?|}+>%8^e25+Oc$=mF0@wR%~yzSl& zZ>P7*+wJY~_Imrg{oVoZpm)eS>>crrddIxu-U;udcgj2Mo$=0k=e+aY1@EGF$-C@b z@veH;yzAZ#@1}RlyY1ca?t1sU``!cZq4&sp>^~0D z{I-6u-_CFEckn~}j(#VUZHZ8q%Ae`a@@M;V z{JDO#KhK}<|M$4Qz+dPu@)!F{{H6XfKgM6~ukcs;tNhje8h@?7&R_3u@HhIK{LTIr zf2+UE-|p}5clx{h-ToebufNaV?;r3F`iK0({t^GEf6PDbpYTumr~K3Y8UL(*&Oh&8 z@Gtt8{LB6o|Ehn@zwY1gZ~C|V+x{K@u7A(J??3P#`j7m_{uBSH|IB~xzwlrBul(2l ztBG?87o=C(_8QNu!Ahq)=`_-vbkbR4+qP}nwr$(CZQHhOoBzCf@Avw@I!T_&Q&l;u zvpecl)a$4>QE#K(MZJ&u5cM(YQ`G0EFHv8kzD0eH`VsXr>Q~h7s6SB=AO?sDVu9Eo z4u}ilf%qT+hy)2iB9Is)0Z{+|4M3m+10aBb2?(%&4IJPC3OwKg1~>?S01|{ifoPBv zBm>Do3Xl?{0;xe7kQSr^=|KjN5o7|HK^Bk|WCPhj4v-V%0=YpRkQd|w`9T3t5EKH1 zK@m_C6a&RU2~ZN00;NG2P!^N}!bOYT%56~0z0=+>W&=>Rr z{lNe*5DWr?!4NPM3%j)F5o`jR!4|L;Yy;cD4zLsK z0=vN;uovtD`@sQl5F7%B!4Ys290SL}32+je0;j8l@5TV7;VrsFp*jgMdt`<*=uO-kTwS-zCEwPqFi_(CmX;9NOLqi(Y zOpR!kW^0b-YE<(yUt=2A0!?UA3pJ%hYe}_aT5>IgmQqWlrPk7DX|;4(dM$&NQOl%d z*0N|>wQO2;Er*s<%cbSk@@RRrd|H03fL2f|q!reRXhpSRT5+v}R#GdamDb8=Wwmlz zd98w0QLCg?)~aY#wQ5>*t%g=ptEJV}>S%SfdRl$0f!0uKq&3!>Xic?dT63+1)>3Pw zwbt5bZMAkt%ufA>!tP9`e=Q%ep-KRfHqJYqz%@FXhXGO z+Hh@zHc}g9!W3_SGcx{3uN_ELMLz1H4nZ?$*Yd+mevQTwEQ*1l+8 zwQt&Y?T7YL`=$NX{%8>}28;<~!Pqbkj0@wz_%H#Cgb86Hm>4F3Q4l~4La0LnB8Z_0 z3ACUM9q2*|J?KLQISinH5{6L0XqXfxgUMkEm=dOfsbLzJ7N&#gVFs8HW`dbv7MK-g zgV|vYm=orLxnUld7v_WcVF6eW7J`Lg5m*!!gT-M9SQ3_krC}LZ7M6qMVFg$bR)Upb z6<8HkgVkXTSQFNQwP77t7uJLIVFTC@HiC^|6WA0sgUw+J*b=satzjG37Pf=!VF%a| zc7mN@7uXecgWX{d*c0}Gy5I7VLgTvtnI1-M6qv04h7LJ4C z;RHAlPJ)x+6gU-5gVW&*I1|o-v*8>#7tVw8;R3i2E`p2U61WsDgUjIxxDu{{tKk~B z7OsQq;Rd)7Zi1WP7Pu8|gWKT_xD)PzyWt+V7w&`m;Q@FM9)gGA5qK0HgU8_scoLq1 zr{NiR7M_FW;RSdRUV@k56?he1gV*5=coW`&x8WUl7v6*S;RE;(K7xaq0LdK^8j9#4<2 zC(tAHgnA-9v7SVa(t)n&P}g-sM>^I`o#>Ws>yGZ~RQGgWXFAsdUFcE|b)`q^N%drU zay^BfQctC)*3;-|^>liAJ%gT6&!lJ8v*=m%YK*30N+^>TW7y@Fm*ucTMjtLRnrYI=3OhF(*zrPtQ$=ymma zdVRft-cWC(H`bfzP4#AabG?P$Qg5ZV*4yZ9^>%uDy@TFS@1%FuyXal@ZhCjUhu%}~ zrT5nR=zaBmdVhU@K2RT|57vk1|L*mn`Y?UCK0+Ur3>d`Z9gFzCvHAuhLiRYxK4HI(@yq zLEorv(l_f{^sV|feY?Ix->L7?ck6rfz4|_VzkWbJs2|b~>qqpX`Z4{uenLN~pVCk3 zXY{lBIsLqTLBFV9(l6^*^sD+c{kncbzp3BSZ|isTyZSx-zWzXes6Wyl>reEj`ZN8x z{z8ALztUgpZ}hkNJN>=>LI0?K(m(58^so9i{k#4{|Ed4df9rqr2qT6O(}-onHsTm@ zjd(_UBY_cVBs3BkiH#&ilmQIQfQD`u1~RZ=8pN;++i(ompoVAo1~a%37{ZW7XecAv zNNOZAk{c}Mkk}Q(Z%R$bThge zJ&c}4FQd27$LMSHGx{3?jDf}=W3VyA7-|eNh8rV{k;W)vv@ymQYm76-8xxF)#w261 zF~yi_Of#k%GmM$WEMvAY$Czu(Gv*r$jD^M`W3jQsSZXXYmK!UKmBuP#wXw!nYpgTY z8yk#`#wKI4vBlVGY%{hSJB*#iE@QW`$JlG^Gxi$?jDyA@=_VGN6no6UvOTpsXkx%8qiNoG2H{jq;$pC?Cp?3ZR0h5Gss{ zprWW4DvnB^lBg6ajmn_1s2nPfDxiv}5~_@W+G#o~Retrm1q@Ojn<&GXdPOQHlU4Y6WWZnpsi>d+KzUhooE-@ zjrO3uXdl{-4xoeR5IT&Gprhy*I*v}DljsyWjn1I6=o~taE})C(61t48psVN_x{hw3 zo9Gt0jqaek=pMR{9-xQl5qgZCpr_~=dX8S8m*^FGjozTQ=pA~GKA?~26Z(w4ps(l~ z`i_2}pXe9*jsBnr90SM1v2bi02gk+naD1EqN8*Gy5l)Pg;3y2Rh9TCmff2^o!~|Q| z#twEd#UA!C!yE@#V2MMla5PSeli}nz1x|@m;nX+{PK(pw^f&{~h%@2LI1A2-v*GMG z2hNFe;oLY6&WrQm{I~!vhzsGuxCkzai{aw91TKk7;nKJaE{n_I^0)%7h%4dBxC*X{ ztKsUn2Cj)~;o7(ku8Zs8`nUmZh#TR?xCw5Go8jiT1#XF3;nuhfZj0OD_P7J?h&$oV zxC`!zyW#G*2kwb`;oi6p?u+~3{&)Z$hzH@pcnBVfhvDIP1RjY;;n8>u9*f7}@puBB zh$rF6cnY41r{U>%2A+v$;n{c&o{Q(<`FH_dh!^3-cnMyLm*M4j1zw3);njEzUW?b^ z^>_o`h&SQQcnjW&x8d!02i}Qy;oW!--i!C){rCVrh!5ez_y|6VkKyC^1U`vR;nVmG zK8w%c^Y{Y3h%e#G_zJ#?ui@+X2EK`J;oJBQzKieS`}hHVh#%p{_z8ZBpW)~D1%8QN z;n(;Lev9AX_xJ<;h(F=a_zV7uzv1ur2mXnF;otZVjxb}GG0j+JY%`7-*NkVzHxrnV zWSn^nxJW;L_AS;MSp)-r3Gbq zz-(wXG8>ys%%)~Dv$@&AY-zSKTbpgnfA@M@vz^)A>|k~@JDHu$E@oG=o7vs$VfHk8 znZ3R% zfGi}7$YQdDEG5gxa&SYtfovq3$Y!#IY$e;scCv%)B)iCNvWM&? z`^bKBfE*-;$YFAX93{udadLv3B&Wz}a)z8G=g4_-fm|e)$YpYcTqW1Yb#jB;B)7nLC9$F`U}+Y#bjz@ig)P$}mSx$N zW4RW!Jj=J3#jU^+mb5}kSI8BttM7etC`i@YGJjsT3M~FHdb4! zoz>pzV0E-QS)HveR#&T=)!pi0^|X3fy{$f0U#p+h-x^>Iv<6v&ts&M>YnV0M8exsJ zMp>h+G1gdXoHgE>U`@0pS(B|P)>LbnHQkzF&9r7&v#mMSTx*^+-&$ZTv=&*5ttHk{ zYnip&T4AlUR#~g9HP%{doweTDU~RNES(~jb)>dnqwcXlb?X-4TyRALeUTdGV-#TC& zv<_K^ts~Y^>zH-iI$@o(PFbg|GuBz_oORy1U|qB>S(mLV)>Z48b=|sQ-L!65x2-$Y zUF)88-+Ev@v>sWHttZw~>zVc3dSSh^URkfLH`ZJ0o%P=OV12YcS)Z*h)>rGB_1*em z{j`2rzpX!3gdM|DqwbR+@?F@ECJCmK+&SGb^ zv)S409Cl7Smz~?rW9PN=+4=1Pc0s$4UDz&S7qyGo#qAPyNxPI?+Ad?4waeM%?Fx29 zyOLemu3}fUtJ&4<8g@;)mR;MfW7oCo+4b!Pc0;?7-PmqoH?^DD&FvO;OS_fb+HPaF zwcFY4?GAQFyOZ77?qYYfyV>3C9(GT=m)+a$WB0ZD+5PPS_CR}(J=h*%54DHc!|f6F zNPCn$+8$$%wa3}x?Fsfody+lbo?=h6r`gl(8TL$jmOa~^W6!nc+4JoM_CkA+z1Uu2 zFSVE1%k35RN_&;P+FoO?wb$9}?G5%udy~D{-ePaHx7pk69rjLpm%ZEGWAC;1+57DS z_CfoQeb_!?AGMF!$L$mLN&A$2+CF2Swa?k-?F;rr`;vXxzG7dsui4k_8}?26mVMj4 zW8by!+4t=S_Cx!T{n&nDKeeCP&+QlXOZ%1m+J0ldwcpwA?GN@x`;-0I{$hW%zuDjI zANEiCm;KxRV@EhKoS055C$ z<=BqnxDItZ$9I^+oxl-}bV5ft(N0n)nUmZ};iPm@IjNmAPFg3OlitbTWOOn)nVl?7 zRwtX2-O1tPbaFYlojgunC!dqwDc}@z3OR+HB2H1Km{Z&-;gobrIi;O4PFbg%Q{JiI zRCFpim7OZizk9u^Q_ZRF)NpD#wVc{c9jC5S&#CV;a2h&|oW@QQr>WD-Y3{UeS~{(q z)=nFzt<%nF?{siFI-Q)(P8X-E)6MDb^l*AQy`0`oAE&R=&*|?Ba0WVqoWafzXQ(sG z8Sad5MmnRM(asoWtTWCT?@VweI+L8q&J<^=GtHUq%y4Epvz*z^9A~aG&zbKma27g? zoW;%(XQ{KyS?;WGRywPk)y^7ct+UQq?`&{3I-8u$&K75@v(4G=>~MBEyPVz59%rw! z&)M%Ba1J_$oWsr$=csecIqsZrPCBQY)6N;^taHve?_6*$I+vWw&K2jXbIrN#+;DC> zx18I~9p|od&$;hBa2`63oX5@+=c)6|dG5S$UOKOw*UlT~t@F-#?|g7RI-i`+&KKva z^UeA0{BV9czntIBA1A_%;l^}hxv||iZd^B>8{bXfM!E^zL~dd?i5uktS977OyM~Kg z?3yldE!TD(*LA7uxxUL>?gp-Kr5n1+jdqi|$=u{_3OA*j%1!O2anri#-1Ke+H=~=$ z&Fp4zv%1;b>~0P>r<=>o?dEawy7}DvZUMKTTgWZ!7IBNZ#oXd<3AdzM$}R1dam%{p z-12S(x1w9gt?X8DtGd

TV6Urd!Lc?bdPYy7k=pZUeWW+sJL~HgTJ}&D`d03%8}) z%5Ckoaof7>-1cqFamTvj-0|)NccMGVo$O9=r@GVJ>Fx}7raQ}>?ap!My7S!m?gDqA zyU1PaE^(K-%iQJe3U{Tu%3bZQao4)*-1Y7TccZ(>-Ry30x4PTh?d}eDr@PDD?e1~+ zy8GPy?g96pd&oWP9&wMl$K2!Y3HPLX%02C#anHKv-1F`Q_o92rz3g6bue#UV>+TKr zrhCi1?cQy5`lzP;sjB*-KK_v~TqR})dO-7T`6f`AGMN`u>G%Za>)6)zzBh5rJ(=0SA z%|^4+95g4*MRU_UG%w9Z^V0&fAT2}-(;~DeEk=ve60{^OMN88%v@9)0%hL+9BCSL# z(<-zotwyWU8nh;@MQhVKv@Wej>(d6bA#Fq((x(=N0t?MA!P9<(RzMSIgev@h*P`_ln*ARR;p(;;*y9Y%-K5p*OSMMu*ybSxc5 z$I}UPBArAh(-A1?59dsw%MR(IZbT8dU_tOLPAU#A6($2({dI`NmUScna7v%v@^Ps1DhKD@tnI7>h&-NV8^{D50zQ;W71)lJv7kbKz_L6$Z zyyRXAFQu2tOYNoc(t7E<^j-!pqnF9c>}Bz?dfB|}UJfs(m&?oT@rruIyy9L7ucTMXEA5r>%6jFz@?Hh6qF2eQ>{aoqdeyw@UJb9NSIevI)$!_j z^}PCC1Fxai$ZPC1@tS(gyyjjDucg<@Ywfl1+IsE0_Ff0Cqu0sn>~-`n2e zdegk=-VAT1H_Myt&GF`X^St@q0&k(W$Xo0!@s@hayye~sZ>6`&TkWm!)_Uu__1*?= zqqoW1>}~P3dfUA1-VSf4x69k@?eX?{`@H?$0q>x9$UE#E@s4`OyyM;p@1%FiJMEqE z&U)v(^WFvTqIb!=>|OD$de^+`-VN`jcgwr&-SO^v_q_Yw1Mi{t$b0NP@t%6myyxBv z@1^(3d+ojP-g@u6_udEZqxZ@C?0xaRdf&Y7-Vg7m_sje3{qZ9F7=BDYmLJ=XPerdmqU)C?@m-j3975z$nWxtAF)vxAP_iOky{aSu) zzm8wmujkkI8~6?VMt)2j4+GY%waC0%ws-djI)3VCRxZ7i)Kk# zGM1dBU@2KDmYStuX<0g!o@HPeStgd5Wno!aHkO^`U^!VXmYd~ad09S|pA}#QSs_-K z6=6kLF;<+FU?o{8R+^PzWm!2^o>gEKStVAPRbf?GHCCO~U^Q7SR-4sfby+=DpEY0& zStHh%HDOIzGuE87U@ciI)|$0pZCN|ko^@ayStr(+bzxmuH`bl?U_Dtc)|>TVeOW)& zpABFG*&sHU4PissFgBcxU?bTmHkyrLW7#-1o=spA*(5fZO<_~nG&Y^hU^Ce)Hk-|1 zbJ;vLpDkbu*&?=>En!R9GPazpU@O@wwwkSBYuP%so^4_mKbH+IjxZsk9T=8h0lqciKc?zDAr{bx38lIM?H^lsDtec?;f>x8kjN8{U?;IAKsVu*>`3OFekK&{G7(SMd(a z8Ge?Zf zB7=lMq9AdQB!~(?pan3{10z5I4$OcAR$vEC;084C0zY5@4}w4hG6(||L=)GsqR>4)O$fgM2~$pg>SCC=?V9 ziUdW2VnOksL{Ks)6_gIj1Z9JALHVFUP%)?!R1T^HRfB3l^`J&jGpH5R4(bGTgL*;z zph3_uXcROKngmUQWrTZ3)E_FzY_GuRdE4)z3lgMGpN;6QLNI20TXjs!=8W5My@L~t@V6`T&v1ZRVD z!TI1qa51u}Cb$TM5K@FfiD>cfUQa5LiR2=M zNGVc@)FO>YE7FPdB7?{%GKtJ0i^wXniR>bW$SHD(+#-+2EAol_qJStU3W>s^h$t$G ziQ=M!C@D&b(xQwgE6R!TqJpR>Dv8RXil{28iRz+;s3~fR+Mt7*aiR0pgI4Mqv)8dRcE6$1Y;)1v+E{V(HinuDSiRT59*M`|iFhiWiRa>lcqv|q*W!(ME8dCs;)D1oK8erbi})(OiSOcv z_$hvg-{OymkTGOT8B4~Nab#Q>PsW!CWTZ?e6UoFfiHwp!Y7$Ca8WKq?O-ZCBZRto? zQt3%wGRb8ig_JUsN=D11GMP*+Q^=Gul}s(u$h0z@OfNIYj53qVEVIb0GMmgUbI6=B zm&`5m$hvYM=jNEW60A zvYYHKd&r)$m+US3$iA|l>@Nq%fpU->EQiRUa+n-0N63+KlpHO`$gy&q94{xxiE@&h zET_n+a+;hjXULgymYgl;$hmT!oG%y1g>sQxESJcoa+zE%SICuem0T^?$hC5vTrW4s zjdGLREVsz5a+};PcgUS`m)tG)$h~r(+%FHvgYu9(ERV>e@|Zj>Pso$9B#`XLK>7=$8}VHm0~I!qcS3zLT_!jxgE zFm;$FOdF;P(}x+tjA5oQbC@N}8fFW#hdIKWVXiQDm?z8|<_q(O1;T=1p|EgRBrF;h z3yX&(!jfUBuyj}^EE|>!%ZC-hieaU&a#$s-8deLdhc&{QVXd%sSSPF-)(h*04Z?i2hdsicVXv@v*eC28 z_6z%m1Hysfpm1hdaWZ;jVCZxF_5j?hE&a2f~Blq402cBs>}(3y+5g(C--jQvZ|aauPUgDs*aF^yzN(+RA|sF7-v8m-2tv1*(euO_I8YLc3)rl_fEnwqX=sF`Y(nyu!jxoV!8uNJ6X16Dj;N#Rm^!XbsFUiHI<3yAv+A5WuP&&I>XN#wuBfZ(n!2uTsGI7Rx~=Z0yXv00 zuO6s}>XCY^o~WnlnR>2XsF&)Mdad55x9XjGuRf@c>XZ7czNoM2oBFPPsGsVW`mO$` zi0By6F{5Ke$BvE@9XC2&bo}T9(UH*!qZ36Zj!qIC75&d%Km3<3cX{^jD`jf{BvBr@XYoBz)x{L8hAG55=Vd3OKw`Kx37 zW#mP$zkRO$pM2cEzQ5`JAnyM0HIDd;w)^^D?%x0EVuHW=pZG`r^~d~&3HY!6`?Y`c zKhY;qM55UL=8OF0`9=Jl^?&2(|K|SlkAH&wukU|`|JDDVY5$kc`@i#F|9^`9>+Ac^ v{a^n-_kZHw{|^1j?yLH@_J2AS`TKVEuYUIq{l|Cre|?XtM_J0ljA{QLv}{e& literal 0 HcmV?d00001 diff --git a/test/test_upgrade_database_4_24.realm b/test/test_upgrade_database_4_24.realm new file mode 100644 index 0000000000000000000000000000000000000000..7d8280b0f55683b5397eea7699b7a31d731ef357 GIT binary patch literal 8336 zcmdT}O>7&-6`tiT$>py|in914Q|mZ!RoGGOCP9=qKXhU{vSY{cUrgs$RJ0;#Q6xc8 zihMGUI{HuqJo1u2M;(>+B0w-;^rWK>J?PK?xisLRC{(A7`@Pwn^>AYeJ+uc6@pk^+ zd-LA=-p(v{pFXt;v&*fQ&%c*K`y5J`F;A;FQ#EQ|GGnA|ySt z^(9(ZnKi$vEEKD3X^AUVtCiY3@d-N^i=j(oU1{70`xqNw3dTIJMMXYCb9b!}2BXbr8Sb+w-3j~oBIg!4J6{hri*A7D_Sco6=6V6KA#pr8wXF(>u` zA&1(*zp>P8(7rfREq;ZK(Bs|@XdS_SApYJR@2>Yb_`^t1-1xJfzn@b;9*_zmK6yRl zkH_%kxds|PWcYi_-nO^vUGw(5x^L1~@HKoLU)R@j{Bh%dUc&i;)V?USKd^&I@qi*N z1kA&r2q@~ppH2lj6U08?jxX(-?)~1qAJ96GgK9lfjf9Dg1$hKQ-`7+t(*)4_z|&WlM2 z|4UN)vef?2jxxzpUeMD@3={_?K=wF4cg0yD_6HKBc~s86;64Z3`-IGYP+t5P=mqv{ zzl44|7g9l5)hh11(2)xPP!JT_Cod*~1+IbYI9z#=53~cFz(!y*pa+fMbg&Uz4Xy__ zf;!xIKK@rEoUcmlkEHfBJC+m=!haB$CqYA?BQE^Kc_8)&u`X%{fA=}y-X~}s!G9qB zpKWZC{%8h>vn8!g0(BOXfM+;m7={;olN7|82WFxW}(W6FmI@*Y?M%SYo zQJw#VAb;HZ|1AmU+fw^useQ)|C&?53SztZ_Itt3U@E7~O*at|IIyZ3wdejxK7 zi2p!uU=N)?;*o~pgqtT*cjLx&|^kyI@XA- z#@1sSF`dH5s6Gvuc`FA5U=ccu0xQu|XLVN0I!VidVxfR2G)aOH(K z55)c;>_P41h5H6-W%tR8sYHWoAUh6M zUKHY;_A# z95Va|JA>$i2S~A+8#%x&^&67UGqMk{r`4m z+_-F(jAe7yFl$D2>$~Q>iF?>Q6}*Sx#Y)L+8kPBZ(^xQ@XM?lgnXA-h%|)ZJ^({%1 zIj`9=7F$rUSYI&kx)tM#W^+6k{QbCb$slz#1A1mbOPFn*3f4UoRN>Vo-sQ*7O`xVxPz~Z8`nzEX|(D_skl%zh4rUkrZHE&zYI;St?yb!ePNzn z0*?pDQ;gK$qEV~QlkkG&vE^=QYlEz`eZY%k$OK=tB2-;!LTaX1BCi*#Ti=@87 z`nG9Mv`WR=A|h3TN^7m=)^`Y9(^#mKc)Y-A*2wpAy-LBL7%o&=i2gl_)@$R&#d_0# z)G|f3STmL@Rb!?yYm&@t)3hSXeVyTpEP&6B_k6Kx%t7BW*iQw^%TV+TO$z_G^MA-h zKCJSraqcfWAFRHDdE^Q&lrK1s)rZ5K;qI_C@5TH#(mWpDv~w-IXrEbXnTveFJ5|SP zc&kkVftx0LvSm;OZ&p^G^%x7&I?$_q!~UQse#J+9JivP<%}S|^_xX41n#QqDd6YlH zh|es0V4p8f9}(;#_63Z4#!^$Mrw<&v)syU`auRP?Ne};X&1L>>zcx+eEfv{3&6D1h z)T*Od)>}@kIePKJPW#7CmZXqN*;1RS?UXFHJ-qA4LBFzR^O2mFT>4YLo=wD@ik?oT zg&u(85de`KrWLBm-+X#1P5jiy{wCOXAMxz#<9)Y6eAqYmvjxey3(aFpp&I(5Dk}^% zhE|5!L+#=9;jwffT~4p0SJMyE>(085=)FE(Sq!q%gQtf!(&G8V8uzlBY%GwE?>hWo zR_I+ntxfAt|L&E0SEauCLG;8=cRENEn;?n@txNhR(Y^$_3?d$3 z&s6ppJC4)*HjNJu&0wNw^ zPk!Vad!JokA4vMi-_OAFIp_-z=@a~okxT4Dc9mW0=U3$QV7I3`avdC2e!^aDY@{%< zIRoV`O6^o4il_MrM0N@KZmQ*>(CCa8CDlkE?WD0(V4QV0Xm&PDkJB{yP#k zIN609CwVFl`1Su#r!SQo%T44+IM1&%jx`nA94-$b?XSdwLi1QpyS3ck z@BFd9|Ce8$^&erK09(ne<~q5x+%YGAY5$MK6V7uu^^-$K_|8r1P(PKym6dZ1jFEkaCxwZJiuxwv`*75X9QDEP=}Phxf66Ig2l9;iR36CRwQ&zd zr#KqKX_dCE2eId6-R=vVM7?Q`Z@c(yF*3GIZNV?bg}pmRf8>&OUrn)TLpjG-!>~9X NpZMYR8;Ms>{|Ci15EuXe literal 0 HcmV?d00001 From f3a5d90e064b47bf747944aa8883fecd04a59737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Mon, 12 Aug 2024 10:56:36 +0200 Subject: [PATCH 17/18] Bump file format version --- CHANGELOG.md | 2 +- src/realm/backup_restore.cpp | 7 ++++--- src/realm/group.cpp | 1 + src/realm/group.hpp | 5 ++++- src/realm/transaction.cpp | 7 ++++++- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af0d058f5ea..dab8322b469 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ * None. ### Compatibility -* Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. +* Fileformat: Generates files with format v25. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier. ----------- diff --git a/src/realm/backup_restore.cpp b/src/realm/backup_restore.cpp index 8070642af0d..164ece35907 100644 --- a/src/realm/backup_restore.cpp +++ b/src/realm/backup_restore.cpp @@ -34,13 +34,14 @@ using VersionList = BackupHandler::VersionList; using VersionTimeList = BackupHandler::VersionTimeList; // Note: accepted versions should have new versions added at front -const VersionList BackupHandler::accepted_versions_ = {24, 23, 22, 21, 20, 11, 10}; +const VersionList BackupHandler::accepted_versions_ = {25, 24, 23, 22, 21, 20, 11, 10}; // the pair is // we keep backup files in 3 months. static constexpr int three_months = 3 * 31 * 24 * 60 * 60; -const VersionTimeList BackupHandler::delete_versions_{{23, three_months}, {22, three_months}, {21, three_months}, - {20, three_months}, {11, three_months}, {10, three_months}}; +const VersionTimeList BackupHandler::delete_versions_{{24, three_months}, {23, three_months}, {22, three_months}, + {21, three_months}, {20, three_months}, {11, three_months}, + {10, three_months}}; // helper functions diff --git a/src/realm/group.cpp b/src/realm/group.cpp index b6703b3af53..05879f7a874 100644 --- a/src/realm/group.cpp +++ b/src/realm/group.cpp @@ -419,6 +419,7 @@ int Group::read_only_version_check(SlabAlloc& alloc, ref_type top_ref, const std case 0: file_format_ok = (top_ref == 0); break; + case 24: case g_current_file_format_version: file_format_ok = true; break; diff --git a/src/realm/group.hpp b/src/realm/group.hpp index 352c5bd25fb..945a28dd585 100644 --- a/src/realm/group.hpp +++ b/src/realm/group.hpp @@ -764,6 +764,9 @@ class Group : public ArrayParent { /// Backlinks in BPlusTree /// Sort order of Strings changed (affects sets and the string index) /// + /// 25 Enhanced layout of NodeHeader to support compression. + /// Integer arrays are stored in a compressed format. + /// /// IMPORTANT: When introducing a new file format version, be sure to review /// the file validity checks in Group::open() and DB::do_open, the file /// format selection logic in @@ -771,7 +774,7 @@ class Group : public ArrayParent { /// upgrade logic in Group::upgrade_file_format(), AND the lists of accepted /// file formats and the version deletion list residing in "backup_restore.cpp" - static constexpr int g_current_file_format_version = 24; + static constexpr int g_current_file_format_version = 25; int get_file_format_version() const noexcept; void set_file_format_version(int) noexcept; diff --git a/src/realm/transaction.cpp b/src/realm/transaction.cpp index 9e93875923f..8221d65aa3f 100644 --- a/src/realm/transaction.cpp +++ b/src/realm/transaction.cpp @@ -533,7 +533,7 @@ void Transaction::upgrade_file_format(int target_file_format_version) // Be sure to revisit the following upgrade logic when a new file format // version is introduced. The following assert attempt to help you not // forget it. - REALM_ASSERT_EX(target_file_format_version == 24, target_file_format_version); + REALM_ASSERT_EX(target_file_format_version == 25, target_file_format_version); // DB::do_open() must ensure that only supported version are allowed. // It does that by asking backup if the current file format version is @@ -584,6 +584,11 @@ void Transaction::upgrade_file_format(int target_file_format_version) t->migrate_col_keys(); } } + if (current_file_format_version < 25) { + for (auto k : table_keys) { + get_table(k); + } + } // NOTE: Additional future upgrade steps go here. } From d5da3b91e0876d0fbb37b53ab6f24d11cc11af82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rgen=20Edelbo?= Date: Fri, 25 Oct 2024 11:21:11 +0200 Subject: [PATCH 18/18] RCORE-2198: Support for additional properties (#7519) (#7886) This will allow some initial experiments in the SDKs. This is not a fileformat breaking change as we just use already existing features. --- CHANGELOG.md | 1 + bindgen/spec.yml | 21 +- src/realm.h | 67 +++++++ src/realm/db.cpp | 3 +- src/realm/db.hpp | 1 + src/realm/db_options.hpp | 3 + src/realm/group.cpp | 7 +- src/realm/group.hpp | 4 +- src/realm/impl/array_writer.hpp | 4 +- src/realm/obj.cpp | 126 +++++++++++- src/realm/obj.hpp | 47 ++++- src/realm/object-store/c_api/config.cpp | 5 + src/realm/object-store/c_api/object.cpp | 189 +++++++++++++++--- src/realm/object-store/collection.cpp | 5 + src/realm/object-store/collection.hpp | 1 + .../impl/object_accessor_impl.hpp | 24 ++- .../object-store/impl/realm_coordinator.cpp | 1 + src/realm/object-store/object.hpp | 5 + src/realm/object-store/object_accessor.hpp | 113 ++++++++--- src/realm/object-store/shared_realm.hpp | 1 + src/realm/sync/instruction_applier.cpp | 7 +- src/realm/sync/instruction_replication.cpp | 79 +++++--- src/realm/sync/instruction_replication.hpp | 4 +- .../server/server_file_access_cache.hpp | 1 + src/realm/table.cpp | 24 +++ src/realm/table.hpp | 22 +- src/realm/transaction.cpp | 1 + test/object-store/c_api/c_api.cpp | 100 +++++++++ test/object-store/object.cpp | 103 +++++++++- test/test_sync.cpp | 64 ++++++ 30 files changed, 909 insertions(+), 124 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dab8322b469..19627784a74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Enhancements * (PR [#????](https://github.com/realm/realm-core/pull/????)) * Storage of integers changed so that they take up less space in the file. This can cause commits and some queries to take a bit longer (PR [#7668](https://github.com/realm/realm-core/pull/7668)) +* We now allow synchronizing, getting and setting properties that are not defined in the object schema (PR [#7886](https://github.com/realm/realm-core/pull/7886)) ### Fixed * ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?) diff --git a/bindgen/spec.yml b/bindgen/spec.yml index 91fac6394fa..c4db39ca7a2 100644 --- a/bindgen/spec.yml +++ b/bindgen/spec.yml @@ -440,6 +440,9 @@ records: schema_mode: type: SchemaMode default: SchemaMode::Automatic + flexible_schema: + type: bool + default: false disable_format_upgrade: type: bool default: false @@ -785,6 +788,7 @@ classes: remove_object: '(key: ObjKey)' get_link_target: '(column: ColKey) -> TableRef' clear: () + get_column_key: '(column: StringData) -> ColKey' get_primary_key_column: '() -> ColKey' Obj: @@ -807,17 +811,28 @@ classes: - '(column: ColKey, value: Mixed)' - sig: '(column: ColKey, value: Mixed, is_default: bool)' suffix: with_default - set_collection: '(column: ColKey, type: CollectionType) -> Obj' + - sig: '(column: StringData, value: Mixed)' + suffix: by_name + set_collection: + - '(column: ColKey, type: CollectionType) -> Obj' + - sig: '(prop_name: StringData, type: CollectionType)' + suffix: by_name add_int: '(column: ColKey, value: int64_t) -> Obj' get_linked_object: '(column: ColKey) const -> Nullable' to_string: () const -> std::string get_backlink_count: '() const -> count_t' + erase_additional_prop: '(prop_name: StringData) const -> Obj' + get_additional_properties: '() -> std::vector' get_backlink_view: '(src_table: TableRef, src_col_key: ColKey) -> TableView' create_and_set_linked_object: '(column: ColKey) -> Obj' - + has_schema_property: '(column: StringData) -> bool' + get_collection_ptr: '(prop_name: StringData) -> CollectionPointer' Transaction: sharedPtrWrapped: TransactionRef + CollectionPointer: + cppName: CollectionBasePtr + ObjectStore: staticMethods: get_schema_version: '(group: Group) -> SchemaVersion' @@ -1035,6 +1050,7 @@ classes: Collection: cppName: object_store::Collection + sharedPtrWrapped: SharedCollection abstract: true properties: get_type: PropertyType @@ -1063,6 +1079,7 @@ classes: base: Collection constructors: make: '(r: SharedRealm, parent: const Obj&, col: ColKey)' + make_other: '(r: SharedRealm, parent: const Obj&, prop_name: StringData)' methods: get: - sig: '(ndx: count_t) -> Obj' diff --git a/src/realm.h b/src/realm.h index bfc8b304694..f885f05e7e1 100644 --- a/src/realm.h +++ b/src/realm.h @@ -925,6 +925,11 @@ RLM_API bool realm_config_get_cached(realm_config_t*) RLM_API_NOEXCEPT; */ RLM_API void realm_config_set_automatic_backlink_handling(realm_config_t*, bool) RLM_API_NOEXCEPT; +/** + * Allow realm objects in the realm to have additional properties that are not defined in the schema. + */ +RLM_API void realm_config_set_flexible_schema(realm_config_t*, bool) RLM_API_NOEXCEPT; + /** * Create a custom scheduler object from callback functions. * @@ -1645,6 +1650,13 @@ RLM_API realm_object_t* realm_object_from_thread_safe_reference(const realm_t*, */ RLM_API bool realm_get_value(const realm_object_t*, realm_property_key_t, realm_value_t* out_value); +/** + * Get the value for a property. + * + * @return True if no exception occurred. + */ +RLM_API bool realm_get_value_by_name(const realm_object_t*, const char* property_name, realm_value_t* out_value); + /** * Get the values for several properties. * @@ -1680,6 +1692,41 @@ RLM_API bool realm_get_values(const realm_object_t*, size_t num_values, const re */ RLM_API bool realm_set_value(realm_object_t*, realm_property_key_t, realm_value_t new_value, bool is_default); +/** + * Set the value for a property. Property need not be defined in schema if flexible + * schema is enabled in configuration + * + * @param property_name The name of the property. + * @param new_value The new value for the property. + * @return True if no exception occurred. + */ +RLM_API bool realm_set_value_by_name(realm_object_t*, const char* property_name, realm_value_t new_value); + +/** + * Examines if the object has a property with the given name. + * @param out_has_property will be true if the property exists. + * @return True if no exception occurred. + */ +RLM_API bool realm_has_property(realm_object_t*, const char* property_name, bool* out_has_property); + +/** + * Get a list of properties set on the object that are not defined in the schema. + * + * @param out_prop_names A pointer to an array of const char* of size @a max. If the pointer is NULL, + * no names will be copied, but @a out_n will be set to the required size. + * @param max size of @a out_prop_names + * @param out_n number of names actually returned. + */ +RLM_API void realm_get_additional_properties(realm_object_t*, const char** out_prop_names, size_t max, size_t* out_n); + +/** + * Erases a property from an object. You can't erase a property that is defined in the current schema. + * + * @param property_name The name of the property. + * @return True if the property was removed. + */ +RLM_API bool realm_erase_additional_property(realm_object_t*, const char* property_name); + /** * Assign a JSON formatted string to a Mixed property. Underlying structures will be created as needed * @@ -1701,6 +1748,8 @@ RLM_API realm_object_t* realm_set_embedded(realm_object_t*, realm_property_key_t */ RLM_API realm_list_t* realm_set_list(realm_object_t*, realm_property_key_t); RLM_API realm_dictionary_t* realm_set_dictionary(realm_object_t*, realm_property_key_t); +RLM_API realm_list_t* realm_set_list_by_name(realm_object_t*, const char* property_name); +RLM_API realm_dictionary_t* realm_set_dictionary_by_name(realm_object_t*, const char* property_name); /** Return the object linked by the given property * @@ -1753,6 +1802,15 @@ RLM_API bool realm_set_values(realm_object_t*, size_t num_values, const realm_pr */ RLM_API realm_list_t* realm_get_list(realm_object_t*, realm_property_key_t); +/** + * Get a list instance for the property of an object by name. + * + * Note: It is up to the caller to call `realm_release()` on the returned list. + * + * @return A non-null pointer if no exception occurred. + */ +RLM_API realm_list_t* realm_get_list_by_name(realm_object_t*, const char*); + /** * Create a `realm_list_t` from a pointer to a `realm::List`, copy-constructing * the internal representation. @@ -2258,6 +2316,15 @@ RLM_API realm_set_t* realm_set_from_thread_safe_reference(const realm_t*, realm_ */ RLM_API realm_dictionary_t* realm_get_dictionary(realm_object_t*, realm_property_key_t); +/** + * Get a dictionary instance for the property of an object by name. + * + * Note: It is up to the caller to call `realm_release()` on the returned dictionary. + * + * @return A non-null pointer if no exception occurred. + */ +RLM_API realm_dictionary_t* realm_get_dictionary_by_name(realm_object_t*, const char*); + /** * Create a `realm_dictionary_t` from a pointer to a `realm::object_store::Dictionary`, * copy-constructing the internal representation. diff --git a/src/realm/db.cpp b/src/realm/db.cpp index f6f258616d2..6198cac10ae 100644 --- a/src/realm/db.cpp +++ b/src/realm/db.cpp @@ -2806,7 +2806,8 @@ void DB::async_request_write_mutex(TransactionRef& tr, util::UniqueFunction { SharedInfo* m_info = nullptr; bool m_wait_for_change_enabled = true; // Initially wait_for_change is enabled bool m_write_transaction_open GUARDED_BY(m_mutex) = false; + bool m_allow_flexible_schema; std::string m_db_path; int m_file_format_version = 0; util::InterprocessMutex m_writemutex; diff --git a/src/realm/db_options.hpp b/src/realm/db_options.hpp index a11eded2352..68e2c8098f3 100644 --- a/src/realm/db_options.hpp +++ b/src/realm/db_options.hpp @@ -106,6 +106,9 @@ struct DBOptions { /// will clear and reinitialize the file. bool clear_on_invalid_file = false; + /// Allow setting properties not supported by a specific column on an object + bool allow_flexible_schema = false; + /// sys_tmp_dir will be used if the temp_dir is empty when creating DBOptions. /// It must be writable and allowed to create pipe/fifo file on it. /// set_sys_tmp_dir is not a thread-safe call and it is only supposed to be called once diff --git a/src/realm/group.cpp b/src/realm/group.cpp index 05879f7a874..039a76ea535 100644 --- a/src/realm/group.cpp +++ b/src/realm/group.cpp @@ -69,12 +69,13 @@ Group::Group() } -Group::Group(const std::string& file_path, const char* encryption_key) +Group::Group(const std::string& file_path, const char* encryption_key, bool allow_additional_properties) : m_local_alloc(new SlabAlloc) // Throws , m_alloc(*m_local_alloc) , m_top(m_alloc) , m_tables(m_alloc) , m_table_names(m_alloc) + , m_allow_additional_properties(allow_additional_properties) { init_array_parents(); @@ -761,6 +762,10 @@ Table* Group::do_add_table(StringData name, Table::Type table_type, bool do_repl Table* table = create_table_accessor(j); table->do_set_table_type(table_type); + if (m_allow_additional_properties && name.begins_with(g_class_name_prefix)) { + table->do_add_additional_prop_column(); + } + return table; } diff --git a/src/realm/group.hpp b/src/realm/group.hpp index 945a28dd585..24cd6df1b1c 100644 --- a/src/realm/group.hpp +++ b/src/realm/group.hpp @@ -117,7 +117,8 @@ class Group : public ArrayParent { /// types that are derived from FileAccessError, the /// derived exception type is thrown. Note that InvalidDatabase is /// among these derived exception types. - explicit Group(const std::string& file, const char* encryption_key = nullptr); + explicit Group(const std::string& file, const char* encryption_key = nullptr, + bool allow_additional_properties = false); /// Attach this Group instance to the specified memory buffer. /// @@ -599,6 +600,7 @@ class Group : public ArrayParent { mutable int m_num_tables = 0; bool m_attached = false; bool m_is_writable = true; + bool m_allow_additional_properties = false; static std::optional fake_target_file_format; util::UniqueFunction m_notify_handler; diff --git a/src/realm/impl/array_writer.hpp b/src/realm/impl/array_writer.hpp index 4096805e0fa..1b2694a8e60 100644 --- a/src/realm/impl/array_writer.hpp +++ b/src/realm/impl/array_writer.hpp @@ -30,9 +30,7 @@ class ArrayWriterBase { bool only_modified = true; bool compress = true; const Table* table; - virtual ~ArrayWriterBase() - { - } + virtual ~ArrayWriterBase() {} /// Write the specified array data and its checksum into free /// space. diff --git a/src/realm/obj.cpp b/src/realm/obj.cpp index 8d968971fc7..a569957f990 100644 --- a/src/realm/obj.cpp +++ b/src/realm/obj.cpp @@ -631,6 +631,37 @@ BinaryData Obj::_get(ColKey::Idx col_ndx) const return ArrayBinary::get(alloc.translate(ref), m_row_ndx, alloc); } +bool Obj::has_property(StringData prop_name) const +{ + if (m_table->get_column_key(prop_name)) + return true; + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + return dict.contains(prop_name); + } + return false; +} + +bool Obj::has_schema_property(StringData prop_name) const +{ + if (m_table->get_column_key(prop_name)) + return true; + return false; +} + +std::vector Obj::get_additional_properties() const +{ + std::vector ret; + + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + dict.for_all_keys([&ret](StringData key) { + ret.push_back(key); + }); + } + return ret; +} + Mixed Obj::get_any(ColKey col_key) const { m_table->check_column(col_key); @@ -676,6 +707,19 @@ Mixed Obj::get_any(ColKey col_key) const return {}; } +Mixed Obj::get_additional_prop(StringData prop_name) const +{ + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + if (auto val = dict.try_get(prop_name)) { + return *val; + } + } + throw InvalidArgument(ErrorCodes::InvalidProperty, + util::format("Property '%1.%2' does not exist", m_table->get_class_name(), prop_name)); + return {}; +} + Mixed Obj::get_primary_key() const { auto col = m_table->get_primary_key_column(); @@ -1107,7 +1151,8 @@ StablePath Obj::get_stable_path() const noexcept void Obj::add_index(Path& path, const CollectionParent::Index& index) const { if (path.empty()) { - path.emplace_back(get_table()->get_column_key(index)); + auto ck = m_table->get_column_key(index); + path.emplace_back(ck); } else { StringData col_name = get_table()->get_column_name(index); @@ -1229,6 +1274,32 @@ Obj& Obj::set(ColKey col_key, Mixed value, bool is_default) return *this; } +Obj& Obj::erase_additional_prop(StringData prop_name) +{ + bool erased = false; + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + erased = dict.try_erase(prop_name); + } + if (!erased) { + throw InvalidArgument(ErrorCodes::InvalidProperty, util::format("Could not erase property: %1", prop_name)); + } + return *this; +} + +Obj& Obj::set_additional_prop(StringData prop_name, const Mixed& value) +{ + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + dict.insert(prop_name, value); + } + else { + throw InvalidArgument(ErrorCodes::InvalidProperty, + util::format("Property '%1.%2' does not exist", m_table->get_class_name(), prop_name)); + } + return *this; +} + Obj& Obj::set_any(ColKey col_key, Mixed value, bool is_default) { if (value.is_null()) { @@ -1983,7 +2054,6 @@ Dictionary Obj::get_dictionary(ColKey col_key) const Obj& Obj::set_collection(ColKey col_key, CollectionType type) { - REALM_ASSERT(col_key.get_type() == col_type_Mixed); if ((col_key.is_dictionary() && type == CollectionType::Dictionary) || (col_key.is_list() && type == CollectionType::List)) { return *this; @@ -1991,11 +2061,34 @@ Obj& Obj::set_collection(ColKey col_key, CollectionType type) if (type == CollectionType::Set) { throw IllegalOperation("Set nested in Mixed is not supported"); } + if (col_key.get_type() != col_type_Mixed) { + throw IllegalOperation("Collection can only be nested in Mixed"); + } set(col_key, Mixed(0, type)); return *this; } +Obj& Obj::set_collection(StringData prop_name, CollectionType type) +{ + if (auto ck = get_column_key(prop_name)) { + return set_collection(ck, type); + } + return set_additional_collection(prop_name, type); +} + +Obj& Obj::set_additional_collection(StringData prop_name, CollectionType type) +{ + if (auto ck = m_table->m_additional_prop_col) { + Dictionary dict(*this, ck); + dict.insert_collection(prop_name, type); + } + else { + throw InvalidArgument(ErrorCodes::InvalidProperty, util::format("Property not found: %1", prop_name)); + } + return *this; +} + DictionaryPtr Obj::get_dictionary_ptr(ColKey col_key) const { return std::make_shared(get_dictionary(col_key)); @@ -2011,14 +2104,33 @@ Dictionary Obj::get_dictionary(StringData col_name) const return get_dictionary(get_column_key(col_name)); } -CollectionPtr Obj::get_collection_ptr(const Path& path) const +CollectionBasePtr Obj::get_collection_ptr(const Path& path) const { REALM_ASSERT(path.size() > 0); // First element in path must be column name auto col_key = path[0].is_col_key() ? path[0].get_col_key() : m_table->get_column_key(path[0].get_key()); - REALM_ASSERT(col_key); + + CollectionBasePtr collection; size_t level = 1; - CollectionBasePtr collection = get_collection_ptr(col_key); + if (col_key) { + collection = get_collection_ptr(col_key); + } + else { + if (auto ck = m_table->m_additional_prop_col) { + auto prop_name = path[0].get_key(); + Dictionary dict(*this, ck); + auto ref = dict.get(prop_name); + if (ref.is_type(type_List)) { + collection = dict.get_list(prop_name); + } + else if (ref.is_type(type_Dictionary)) { + collection = dict.get_dictionary(prop_name); + } + else { + throw InvalidArgument("Wrong path"); + } + } + } while (level < path.size()) { auto& path_elem = path[level]; @@ -2044,7 +2156,7 @@ CollectionPtr Obj::get_collection_ptr(const Path& path) const return collection; } -CollectionPtr Obj::get_collection_by_stable_path(const StablePath& path) const +CollectionBasePtr Obj::get_collection_by_stable_path(const StablePath& path) const { // First element in path is phony column key ColKey col_key = m_table->get_column_key(path[0]); @@ -2108,7 +2220,7 @@ CollectionBasePtr Obj::get_collection_ptr(ColKey col_key) const CollectionBasePtr Obj::get_collection_ptr(StringData col_name) const { - return get_collection_ptr(get_column_key(col_name)); + return get_collection_ptr(Path{{col_name}}); } LinkCollectionPtr Obj::get_linkcollection_ptr(ColKey col_key) const diff --git a/src/realm/obj.hpp b/src/realm/obj.hpp index 67c82a0cada..96525dd8200 100644 --- a/src/realm/obj.hpp +++ b/src/realm/obj.hpp @@ -117,17 +117,30 @@ class Obj { template U get(ColKey col_key) const; + bool has_property(StringData prop_name) const; + bool has_schema_property(StringData prop_name) const; + + std::vector get_additional_properties() const; + Mixed get_any(ColKey col_key) const; Mixed get_any(StringData col_name) const { - return get_any(get_column_key(col_name)); + if (auto ck = get_column_key(col_name)) { + return get_any(ck); + } + return get_additional_prop(col_name); } + Mixed get_additional_prop(StringData col_name) const; + Mixed get_primary_key() const; template U get(StringData col_name) const { - return get(get_column_key(col_name)); + if (auto ck = get_column_key(col_name)) { + return get(ck); + } + return get_additional_prop(col_name).get(); } bool is_unresolved(ColKey col_key) const; @@ -187,17 +200,26 @@ class Obj { // default state. If the object does not exist, create a // new object and link it. (To Be Implemented) Obj clear_linked_object(ColKey col_key); + + Obj& erase_additional_prop(StringData prop_name); Obj& set_any(ColKey col_key, Mixed value, bool is_default = false); Obj& set_any(StringData col_name, Mixed value, bool is_default = false) { - return set_any(get_column_key(col_name), value, is_default); + if (auto ck = get_column_key(col_name)) { + return set_any(ck, value, is_default); + } + return set_additional_prop(col_name, value); } template Obj& set(StringData col_name, U value, bool is_default = false) { - return set(get_column_key(col_name), value, is_default); + if (auto ck = get_column_key(col_name)) { + return set(ck, value, is_default); + } + return set_additional_prop(col_name, Mixed(value)); } + Obj& set_additional_prop(StringData prop_name, const Mixed& value); Obj& set_null(ColKey col_key, bool is_default = false); Obj& set_null(StringData col_name, bool is_default = false) @@ -206,6 +228,7 @@ class Obj { } Obj& set_json(ColKey col_key, StringData json); + Obj& add_int(ColKey col_key, int64_t value); Obj& add_int(StringData col_name, int64_t value) { @@ -248,6 +271,11 @@ class Obj { { return std::dynamic_pointer_cast>(get_collection_ptr(path)); } + template + std::shared_ptr> get_list_ptr(StringData prop_name) const + { + return get_list_ptr(Path{prop_name}); + } template Lst get_list(StringData col_name) const @@ -285,17 +313,24 @@ class Obj { LnkSet get_linkset(StringData col_name) const; LnkSetPtr get_linkset_ptr(ColKey col_key) const; SetBasePtr get_setbase_ptr(ColKey col_key) const; + Dictionary get_dictionary(ColKey col_key) const; Dictionary get_dictionary(StringData col_name) const; Obj& set_collection(ColKey col_key, CollectionType type); + Obj& set_collection(StringData, CollectionType type); + Obj& set_additional_collection(StringData, CollectionType type); DictionaryPtr get_dictionary_ptr(ColKey col_key) const; DictionaryPtr get_dictionary_ptr(const Path& path) const; + DictionaryPtr get_dictionary_ptr(StringData prop_name) const + { + return get_dictionary_ptr(Path{prop_name}); + } CollectionBasePtr get_collection_ptr(ColKey col_key) const; CollectionBasePtr get_collection_ptr(StringData col_name) const; - CollectionPtr get_collection_ptr(const Path& path) const; - CollectionPtr get_collection_by_stable_path(const StablePath& path) const; + CollectionBasePtr get_collection_ptr(const Path& path) const; + CollectionBasePtr get_collection_by_stable_path(const StablePath& path) const; LinkCollectionPtr get_linkcollection_ptr(ColKey col_key) const; void assign_pk_and_backlinks(Obj& other); diff --git a/src/realm/object-store/c_api/config.cpp b/src/realm/object-store/c_api/config.cpp index 7b1b00498c2..2d76bd9368f 100644 --- a/src/realm/object-store/c_api/config.cpp +++ b/src/realm/object-store/c_api/config.cpp @@ -243,3 +243,8 @@ RLM_API void realm_config_set_automatic_backlink_handling(realm_config_t* realm_ { realm_config->automatically_handle_backlinks_in_migrations = enable_automatic_handling; } + +RLM_API void realm_config_set_flexible_schema(realm_config_t* realm_config, bool flexible_schema) noexcept +{ + realm_config->flexible_schema = flexible_schema; +} diff --git a/src/realm/object-store/c_api/object.cpp b/src/realm/object-store/c_api/object.cpp index 44bc55ab548..4971a818c66 100644 --- a/src/realm/object-store/c_api/object.cpp +++ b/src/realm/object-store/c_api/object.cpp @@ -253,31 +253,31 @@ RLM_API realm_object_t* realm_object_from_thread_safe_reference(const realm_t* r }); } -RLM_API bool realm_get_value(const realm_object_t* obj, realm_property_key_t col, realm_value_t* out_value) +RLM_API bool realm_get_value(const realm_object_t* object, realm_property_key_t col, realm_value_t* out_value) { - return realm_get_values(obj, 1, &col, out_value); + return realm_get_values(object, 1, &col, out_value); } -RLM_API bool realm_get_values(const realm_object_t* obj, size_t num_values, const realm_property_key_t* properties, +RLM_API bool realm_get_values(const realm_object_t* object, size_t num_values, const realm_property_key_t* properties, realm_value_t* out_values) { return wrap_err([&]() { - obj->verify_attached(); + object->verify_attached(); - auto o = obj->get_obj(); + auto obj = object->get_obj(); for (size_t i = 0; i < num_values; ++i) { auto col_key = ColKey(properties[i]); if (col_key.is_collection()) { - auto table = o.get_table(); - auto& schema = schema_for_table(obj->get_realm(), table->get_key()); + auto table = obj.get_table(); + auto& schema = schema_for_table(object->get_realm(), table->get_key()); throw PropertyTypeMismatch{schema.name, table->get_column_name(col_key)}; } - auto val = o.get_any(col_key); + auto val = obj.get_any(col_key); if (out_values) { - auto converted = objkey_to_typed_link(val, col_key, *o.get_table()); + auto converted = objkey_to_typed_link(val, col_key, *obj.get_table()); out_values[i] = to_capi(converted); } } @@ -286,18 +286,34 @@ RLM_API bool realm_get_values(const realm_object_t* obj, size_t num_values, cons }); } -RLM_API bool realm_set_value(realm_object_t* obj, realm_property_key_t col, realm_value_t new_value, bool is_default) +RLM_API bool realm_get_value_by_name(const realm_object_t* object, const char* property_name, + realm_value_t* out_value) +{ + return wrap_err([&]() { + object->verify_attached(); + + auto obj = object->get_obj(); + auto val = obj.get_any(property_name); + if (out_value) { + *out_value = to_capi(val); + } + return true; + }); +} + +RLM_API bool realm_set_value(realm_object_t* object, realm_property_key_t col, realm_value_t new_value, + bool is_default) { - return realm_set_values(obj, 1, &col, &new_value, is_default); + return realm_set_values(object, 1, &col, &new_value, is_default); } -RLM_API bool realm_set_values(realm_object_t* obj, size_t num_values, const realm_property_key_t* properties, +RLM_API bool realm_set_values(realm_object_t* object, size_t num_values, const realm_property_key_t* properties, const realm_value_t* values, bool is_default) { return wrap_err([&]() { - obj->verify_attached(); - auto o = obj->get_obj(); - auto table = o.get_table(); + object->verify_attached(); + auto obj = object->get_obj(); + auto table = obj.get_table(); // Perform validation up front to avoid partial updates. This is // unlikely to incur performance overhead because the object itself is @@ -308,12 +324,12 @@ RLM_API bool realm_set_values(realm_object_t* obj, size_t num_values, const real table->check_column(col_key); if (col_key.is_collection()) { - auto& schema = schema_for_table(obj->get_realm(), table->get_key()); + auto& schema = schema_for_table(object->get_realm(), table->get_key()); throw PropertyTypeMismatch{schema.name, table->get_column_name(col_key)}; } auto val = from_capi(values[i]); - check_value_assignable(obj->get_realm(), *table, col_key, val); + check_value_assignable(object->get_realm(), *table, col_key, val); } // Actually write the properties. @@ -321,36 +337,94 @@ RLM_API bool realm_set_values(realm_object_t* obj, size_t num_values, const real for (size_t i = 0; i < num_values; ++i) { auto col_key = ColKey(properties[i]); auto val = from_capi(values[i]); - o.set_any(col_key, val, is_default); + obj.set_any(col_key, val, is_default); } return true; }); } -RLM_API bool realm_set_json(realm_object_t* obj, realm_property_key_t col, const char* json_string) +RLM_API bool realm_set_value_by_name(realm_object_t* object, const char* property_name, realm_value_t new_value) { return wrap_err([&]() { - obj->verify_attached(); - auto o = obj->get_obj(); + object->verify_attached(); + auto obj = object->get_obj(); + obj.set_any(property_name, from_capi(new_value)); + return true; + }); +} + +RLM_API bool realm_has_property(realm_object_t* object, const char* property_name, bool* out_has_property) +{ + return wrap_err([&]() { + object->verify_attached(); + if (out_has_property) { + auto obj = object->get_obj(); + *out_has_property = obj.has_property(property_name); + } + return true; + }); +} + +RLM_API void realm_get_additional_properties(realm_object_t* object, const char** out_prop_names, size_t max, + size_t* out_n) +{ + size_t copied = 0; + wrap_err([&]() { + object->verify_attached(); + auto obj = object->get_obj(); + auto vec = obj.get_additional_properties(); + copied = vec.size(); + if (out_prop_names) { + if (max < copied) { + copied = max; + } + auto it = vec.begin(); + auto to_copy = copied; + while (to_copy--) { + *out_prop_names++ = (*it++).data(); + } + } + return true; + }); + if (out_n) { + *out_n = copied; + } +} + +RLM_API bool realm_erase_additional_property(realm_object_t* object, const char* property_name) +{ + return wrap_err([&]() { + object->verify_attached(); + auto obj = object->get_obj(); + obj.erase_additional_prop(property_name); + return true; + }); +} + +RLM_API bool realm_set_json(realm_object_t* object, realm_property_key_t col, const char* json_string) +{ + return wrap_err([&]() { + object->verify_attached(); + auto obj = object->get_obj(); ColKey col_key(col); if (col_key.get_type() != col_type_Mixed) { - auto table = o.get_table(); - auto& schema = schema_for_table(obj->get_realm(), table->get_key()); + auto table = obj.get_table(); + auto& schema = schema_for_table(object->get_realm(), table->get_key()); throw PropertyTypeMismatch{schema.name, table->get_column_name(col_key)}; } - o.set_json(ColKey(col), json_string); + obj.set_json(ColKey(col), json_string); return true; }); } -RLM_API realm_object_t* realm_set_embedded(realm_object_t* obj, realm_property_key_t col) +RLM_API realm_object_t* realm_set_embedded(realm_object_t* object, realm_property_key_t col) { return wrap_err([&]() { - obj->verify_attached(); - auto& o = obj->get_obj(); - return new realm_object_t({obj->get_realm(), o.create_and_set_linked_object(ColKey(col))}); + object->verify_attached(); + auto& obj = object->get_obj(); + return new realm_object_t({object->get_realm(), obj.create_and_set_linked_object(ColKey(col))}); }); } @@ -380,12 +454,35 @@ RLM_API realm_dictionary_t* realm_set_dictionary(realm_object_t* object, realm_p }); } -RLM_API realm_object_t* realm_get_linked_object(realm_object_t* obj, realm_property_key_t col) +RLM_API realm_list_t* realm_set_list_by_name(realm_object_t* object, const char* property_name) { return wrap_err([&]() { - obj->verify_attached(); - const auto& o = obj->get_obj().get_linked_object(ColKey(col)); - return o ? new realm_object_t({obj->get_realm(), o}) : nullptr; + object->verify_attached(); + + auto& obj = object->get_obj(); + obj.set_collection(property_name, CollectionType::List); + return new realm_list_t{List{object->get_realm(), obj.get_list_ptr(property_name)}}; + }); +} + +RLM_API realm_dictionary_t* realm_set_dictionary_by_name(realm_object_t* object, const char* property_name) +{ + return wrap_err([&]() { + object->verify_attached(); + + auto& obj = object->get_obj(); + obj.set_collection(property_name, CollectionType::Dictionary); + return new realm_dictionary_t{ + object_store::Dictionary{object->get_realm(), obj.get_dictionary_ptr(property_name)}}; + }); +} + +RLM_API realm_object_t* realm_get_linked_object(realm_object_t* object, realm_property_key_t col) +{ + return wrap_err([&]() { + object->verify_attached(); + const auto& obj = object->get_obj().get_linked_object(ColKey(col)); + return obj ? new realm_object_t({object->get_realm(), obj}) : nullptr; }); } @@ -408,6 +505,20 @@ RLM_API realm_list_t* realm_get_list(realm_object_t* object, realm_property_key_ }); } +RLM_API realm_list_t* realm_get_list_by_name(realm_object_t* object, const char* prop_name) +{ + return wrap_err([&]() -> realm_list_t* { + object->verify_attached(); + + const auto& obj = object->get_obj(); + auto collection = obj.get_collection_ptr(StringData(prop_name)); + if (collection->get_collection_type() == CollectionType::List) { + return new realm_list_t{List{object->get_realm(), std::move(collection)}}; + } + return nullptr; + }); +} + RLM_API realm_set_t* realm_get_set(realm_object_t* object, realm_property_key_t key) { return wrap_err([&]() { @@ -445,6 +556,20 @@ RLM_API realm_dictionary_t* realm_get_dictionary(realm_object_t* object, realm_p }); } +RLM_API realm_dictionary_t* realm_get_dictionary_by_name(realm_object_t* object, const char* prop_name) +{ + return wrap_err([&]() -> realm_dictionary_t* { + object->verify_attached(); + + const auto& obj = object->get_obj(); + auto collection = obj.get_collection_ptr(StringData(prop_name)); + if (collection->get_collection_type() == CollectionType::Dictionary) { + return new realm_dictionary_t{object_store::Dictionary{object->get_realm(), std::move(collection)}}; + } + return nullptr; + }); +} + RLM_API char* realm_object_to_string(realm_object_t* object) { return wrap_err([&]() { diff --git a/src/realm/object-store/collection.cpp b/src/realm/object-store/collection.cpp index 6efe683844a..ab42eca58be 100644 --- a/src/realm/object-store/collection.cpp +++ b/src/realm/object-store/collection.cpp @@ -42,6 +42,11 @@ Collection::Collection(const Object& parent_obj, const Property* prop) { } +Collection::Collection(std::shared_ptr r, const Obj& parent_obj, const StringData prop_name) + : Collection(std::shared_ptr(r), parent_obj.get_collection_ptr(prop_name), PropertyType::Mixed) +{ +} + Collection::Collection(std::shared_ptr r, const Obj& parent_obj, ColKey col) : Collection(std::move(r), parent_obj.get_collection_ptr(col), ObjectSchema::from_core_type(col) & ~PropertyType::Collection) diff --git a/src/realm/object-store/collection.hpp b/src/realm/object-store/collection.hpp index e809ee2bab1..649155a6e6e 100644 --- a/src/realm/object-store/collection.hpp +++ b/src/realm/object-store/collection.hpp @@ -45,6 +45,7 @@ class Collection { Collection(std::shared_ptr r, const Obj& parent_obj, ColKey col); Collection(std::shared_ptr r, const CollectionBase& coll); Collection(std::shared_ptr r, CollectionBasePtr coll); + Collection(std::shared_ptr r, const Obj& parent_obj, const StringData prop_name); const std::shared_ptr& get_realm() const { diff --git a/src/realm/object-store/impl/object_accessor_impl.hpp b/src/realm/object-store/impl/object_accessor_impl.hpp index c7371589335..0093a8e5117 100644 --- a/src/realm/object-store/impl/object_accessor_impl.hpp +++ b/src/realm/object-store/impl/object_accessor_impl.hpp @@ -70,15 +70,15 @@ class CppContext { // value present. The property is identified both by the name of the // property and its index within the ObjectScehma's persisted_properties // array. - util::Optional value_for_property(std::any& dict, const Property& prop, + util::Optional value_for_property(std::any& dict, const std::string& name, size_t /* property_index */) const { #if REALM_ENABLE_GEOSPATIAL if (auto geo = std::any_cast(&dict)) { - if (prop.name == Geospatial::c_geo_point_type_col_name) { + if (name == Geospatial::c_geo_point_type_col_name) { return geo->get_type_string(); } - else if (prop.name == Geospatial::c_geo_point_coords_col_name) { + else if (name == Geospatial::c_geo_point_coords_col_name) { std::vector coords; auto&& point = geo->get(); // throws coords.push_back(point.longitude); @@ -88,11 +88,11 @@ class CppContext { } return coords; } - REALM_ASSERT_EX(false, prop.name); // unexpected property type + REALM_ASSERT_EX(false, name); // unexpected property type } #endif auto const& v = util::any_cast(dict); - auto it = v.find(prop.name); + auto it = v.find(name); return it == v.end() ? util::none : util::make_optional(it->second); } @@ -118,6 +118,20 @@ class CppContext { template void enumerate_dictionary(std::any& value, Func&& fn) { +#if REALM_ENABLE_GEOSPATIAL + if (auto geo = std::any_cast(&value)) { + fn(Geospatial::c_geo_point_type_col_name, std::any(geo->get_type_string())); + std::vector coords; + auto&& point = geo->get(); // throws + coords.push_back(point.longitude); + coords.push_back(point.latitude); + if (point.has_altitude()) { + coords.push_back(*point.get_altitude()); + } + fn(Geospatial::c_geo_point_coords_col_name, std::any(coords)); + return; + } +#endif for (auto&& v : util::any_cast(value)) fn(v.first, v.second); } diff --git a/src/realm/object-store/impl/realm_coordinator.cpp b/src/realm/object-store/impl/realm_coordinator.cpp index 3d9f2a95034..06a47e6041c 100644 --- a/src/realm/object-store/impl/realm_coordinator.cpp +++ b/src/realm/object-store/impl/realm_coordinator.cpp @@ -478,6 +478,7 @@ bool RealmCoordinator::open_db() options.durability = m_config.in_memory ? DBOptions::Durability::MemOnly : DBOptions::Durability::Full; options.is_immutable = m_config.immutable(); options.logger = util::Logger::get_default_logger(); + options.allow_flexible_schema = m_config.flexible_schema; if (!m_config.fifo_files_fallback_path.empty()) { options.temp_dir = util::normalize_dir(m_config.fifo_files_fallback_path); diff --git a/src/realm/object-store/object.hpp b/src/realm/object-store/object.hpp index 53fa43a2653..8d5a7527f7d 100644 --- a/src/realm/object-store/object.hpp +++ b/src/realm/object-store/object.hpp @@ -206,7 +206,12 @@ class Object { void set_property_value_impl(ContextType& ctx, const Property& property, ValueType value, CreatePolicy policy, bool is_default); template + void set_additional_property_value_impl(ContextType& ctx, StringData prop_name, ValueType value, + CreatePolicy policy); + template ValueType get_property_value_impl(ContextType& ctx, const Property& property) const; + template + ValueType get_additional_property_value_impl(ContextType& ctx, StringData prop_name) const; template static ObjKey get_for_primary_key_in_migration(ContextType& ctx, Table const& table, const Property& primary_prop, diff --git a/src/realm/object-store/object_accessor.hpp b/src/realm/object-store/object_accessor.hpp index 824ee437204..6022db7a0eb 100644 --- a/src/realm/object-store/object_accessor.hpp +++ b/src/realm/object-store/object_accessor.hpp @@ -42,9 +42,13 @@ namespace realm { template void Object::set_property_value(ContextType& ctx, StringData prop_name, ValueType value, CreatePolicy policy) { - auto& property = property_for_name(prop_name); - validate_property_for_setter(property); - set_property_value_impl(ctx, property, value, policy, false); + if (auto prop = m_object_schema->property_for_name(prop_name)) { + validate_property_for_setter(*prop); + set_property_value_impl(ctx, *prop, value, policy, false); + } + else { + set_additional_property_value_impl(ctx, prop_name, value, policy); + } } template @@ -62,7 +66,12 @@ ValueType Object::get_property_value(ContextType& ctx, const Property& property) template ValueType Object::get_property_value(ContextType& ctx, StringData prop_name) const { - return get_property_value_impl(ctx, property_for_name(prop_name)); + if (auto prop = m_object_schema->property_for_name(prop_name)) { + return get_property_value_impl(ctx, *prop); + } + else { + return get_additional_property_value_impl(ctx, prop_name); + } } namespace { @@ -205,6 +214,28 @@ void Object::set_property_value_impl(ContextType& ctx, const Property& property, ctx.did_change(); } +template +void Object::set_additional_property_value_impl(ContextType& ctx, StringData prop_name, ValueType value, + CreatePolicy policy) +{ + Mixed new_val = ctx.template unbox(value, policy); + if (new_val.is_type(type_Dictionary)) { + m_obj.set_additional_collection(prop_name, CollectionType::Dictionary); + object_store::Dictionary dict(m_realm, m_obj.get_collection_ptr(prop_name)); + dict.assign(ctx, value, policy); + ctx.did_change(); + return; + } + if (new_val.is_type(type_List)) { + m_obj.set_additional_collection(prop_name, CollectionType::List); + List list(m_realm, m_obj.get_collection_ptr(prop_name)); + list.assign(ctx, value, policy); + ctx.did_change(); + return; + } + m_obj.set_additional_prop(prop_name, new_val); +} + template ValueType Object::get_property_value_impl(ContextType& ctx, const Property& property) const { @@ -269,6 +300,20 @@ ValueType Object::get_property_value_impl(ContextType& ctx, const Property& prop } } +template +ValueType Object::get_additional_property_value_impl(ContextType& ctx, StringData prop_name) const +{ + verify_attached(); + auto value = m_obj.get_additional_prop(prop_name); + if (value.is_type(type_Dictionary)) { + return ctx.box(object_store::Dictionary(m_realm, m_obj.get_collection_ptr(prop_name))); + } + if (value.is_type(type_List)) { + return ctx.box(List(m_realm, m_obj.get_collection_ptr(prop_name))); + } + return ctx.box(value); +} + template Object Object::create(ContextType& ctx, std::shared_ptr const& realm, StringData object_type, ValueType value, CreatePolicy policy, ObjKey current_obj, Obj* out_row) @@ -319,7 +364,7 @@ Object Object::create(ContextType& ctx, std::shared_ptr const& realm, Obj // or throw an exception if updating is disabled. if (auto primary_prop = object_schema.primary_key_property()) { auto primary_value = - ctx.value_for_property(value, *primary_prop, primary_prop - &object_schema.persisted_properties[0]); + ctx.value_for_property(value, primary_prop->name, primary_prop - &object_schema.persisted_properties[0]); if (!primary_value) primary_value = ctx.default_value_for_property(object_schema, *primary_prop); if (!primary_value && !is_nullable(primary_prop->type)) @@ -372,30 +417,44 @@ Object Object::create(ContextType& ctx, std::shared_ptr const& realm, Obj // that. if (out_row && object_schema.table_type != ObjectSchema::ObjectType::TopLevelAsymmetric) *out_row = obj; - for (size_t i = 0; i < object_schema.persisted_properties.size(); ++i) { - auto& prop = object_schema.persisted_properties[i]; - // If table has primary key, it must have been set during object creation - if (prop.is_primary && skip_primary) - continue; - - auto v = ctx.value_for_property(value, prop, i); - if (!created && !v) - continue; - - bool is_default = false; - if (!v) { - v = ctx.default_value_for_property(object_schema, prop); - is_default = true; + + std::unordered_set props_supplied; + ctx.enumerate_dictionary(value, [&](StringData name, auto&& value) { + if (auto prop = object_schema.property_for_name(name)) { + if (!prop->is_primary || !skip_primary) + object.set_property_value_impl(ctx, *prop, value, policy, false); + props_supplied.insert(name); + } + else { + object.set_additional_property_value_impl(ctx, name, value, policy); } - // We consider null or a missing value to be equivalent to an empty - // array/set for historical reasons; the original implementation did this - // accidentally and it's not worth changing. - if ((!v || ctx.is_null(*v)) && !is_nullable(prop.type) && !is_collection(prop.type)) { - if (prop.is_primary || !ctx.allow_missing(value)) - throw MissingPropertyValueException(object_schema.name, prop.name); + }); + + if (created) { + // assign default values + for (size_t i = 0; i < object_schema.persisted_properties.size(); ++i) { + auto& prop = object_schema.persisted_properties[i]; + // If table has primary key, it must have been set during object creation + if (prop.is_primary && skip_primary) + continue; + + bool already_set = props_supplied.count(prop.name); + if (already_set) + continue; + + bool is_default = true; + auto v = ctx.default_value_for_property(object_schema, prop); + + // We consider null or a missing value to be equivalent to an empty + // array/set for historical reasons; the original implementation did this + // accidentally and it's not worth changing. + if ((!v || ctx.is_null(*v)) && !is_nullable(prop.type) && !is_collection(prop.type)) { + if (prop.is_primary || !ctx.allow_missing(value)) + throw MissingPropertyValueException(object_schema.name, prop.name); + } + if (v) + object.set_property_value_impl(ctx, prop, *v, policy, is_default); } - if (v) - object.set_property_value_impl(ctx, prop, *v, policy, is_default); } if (object_schema.table_type == ObjectSchema::ObjectType::TopLevelAsymmetric) { return Object{}; diff --git a/src/realm/object-store/shared_realm.hpp b/src/realm/object-store/shared_realm.hpp index 5e98c96149f..23b79b1ea9a 100644 --- a/src/realm/object-store/shared_realm.hpp +++ b/src/realm/object-store/shared_realm.hpp @@ -104,6 +104,7 @@ struct RealmConfig { std::string fifo_files_fallback_path; bool in_memory = false; + bool flexible_schema = false; SchemaMode schema_mode = SchemaMode::Automatic; SchemaSubsetMode schema_subset_mode = SchemaSubsetMode::Strict; diff --git a/src/realm/sync/instruction_applier.cpp b/src/realm/sync/instruction_applier.cpp index 60dfa4ae0c9..701503abdf9 100644 --- a/src/realm/sync/instruction_applier.cpp +++ b/src/realm/sync/instruction_applier.cpp @@ -1495,8 +1495,13 @@ InstructionApplier::PathResolver::Status InstructionApplier::PathResolver::resol InstructionApplier::PathResolver::Status InstructionApplier::PathResolver::resolve_field(Obj& obj, InternString field) { auto field_name = get_string(field); - ColKey col = obj.get_table()->get_column_key(field_name); + auto table = obj.get_table().unchecked_ptr(); + ColKey col = table->get_column_key(field_name); if (!col) { + if (auto ck = table->get_additional_prop_col()) { + auto dict = obj.get_dictionary(ck); + return resolve_dictionary_element(dict, field); + } on_error(util::format("%1: No such field: '%2' in class '%3'", m_instr_name, field_name, obj.get_table()->get_name())); return Status::DidNotResolve; diff --git a/src/realm/sync/instruction_replication.cpp b/src/realm/sync/instruction_replication.cpp index c845bd9c0bf..190a4c92dd3 100644 --- a/src/realm/sync/instruction_replication.cpp +++ b/src/realm/sync/instruction_replication.cpp @@ -611,51 +611,62 @@ void SyncReplication::set_clear(const CollectionBase& set) } } -void SyncReplication::dictionary_update(const CollectionBase& dict, const Mixed& key, const Mixed& value) +void SyncReplication::dictionary_update(const CollectionBase& dict, const Mixed& key, const Mixed* value) { // If link is unresolved, it should not be communicated. - if (value.is_unresolved_link()) { + if (value && value->is_unresolved_link()) { return; } - if (select_collection(dict)) { - Instruction::Update instr; - REALM_ASSERT(key.get_type() == type_String); - populate_path_instr(instr, dict); + Instruction::Update instr; + REALM_ASSERT(key.get_type() == type_String); + + const Table* source_table = dict.get_table().unchecked_ptr(); + auto col = dict.get_col_key(); + ObjKey obj_key = dict.get_owner_key(); + if (source_table->is_additional_props_col(col)) { + // Here we have to fake it and pretend we are setting/erasing a property on the object + if (!select_table(*source_table)) { + return; + } + + populate_path_instr(instr, *source_table, obj_key, {key.get_string()}); + } + else { + if (!select_collection(dict)) { + return; + } + + populate_path_instr(instr, *source_table, obj_key, dict.get_short_path()); StringData key_value = key.get_string(); instr.path.push_back(m_encoder.intern_string(key_value)); - instr.value = as_payload(dict, value); - instr.is_default = false; - emit(instr); } + if (value) { + instr.value = as_payload(*source_table, col, *value); + } + else { + instr.value = Instruction::Payload::Erased{}; + } + instr.is_default = false; + emit(instr); } void SyncReplication::dictionary_insert(const CollectionBase& dict, size_t ndx, Mixed key, Mixed value) { Replication::dictionary_insert(dict, ndx, key, value); - dictionary_update(dict, key, value); + dictionary_update(dict, key, &value); } void SyncReplication::dictionary_set(const CollectionBase& dict, size_t ndx, Mixed key, Mixed value) { Replication::dictionary_set(dict, ndx, key, value); - dictionary_update(dict, key, value); + dictionary_update(dict, key, &value); } void SyncReplication::dictionary_erase(const CollectionBase& dict, size_t ndx, Mixed key) { Replication::dictionary_erase(dict, ndx, key); - - if (select_collection(dict)) { - Instruction::Update instr; - REALM_ASSERT(key.get_type() == type_String); - populate_path_instr(instr, dict); - StringData key_value = key.get_string(); - instr.path.push_back(m_encoder.intern_string(key_value)); - instr.value = Instruction::Payload::Erased{}; - instr.is_default = false; - emit(instr); - } + dictionary_update(dict, key, nullptr); } void SyncReplication::dictionary_clear(const CollectionBase& dict) @@ -750,7 +761,25 @@ void SyncReplication::populate_path_instr(Instruction::PathInstruction& instr, c { REALM_ASSERT(key); // The first path entry will be the column key - REALM_ASSERT(path[0].is_col_key()); + std::string field_name; + if (path[0].is_col_key()) { + auto ck = path[0].get_col_key(); + if (table.is_additional_props_col(ck)) { + // We are modifying a collection nested in an additional property + REALM_ASSERT(path.size() > 1); + field_name = path[1].get_key(); + // Erase the "__additional" part of the path + path.erase(path.begin()); + } + else { + field_name = table.get_column_name(ck); + } + } + else { + // In the case of an additional property directly on an object, + // the first element is a string. + field_name = path[0].get_key(); + } if (table.is_embedded()) { // For embedded objects, Obj::traverse_path() yields the top object @@ -760,7 +789,7 @@ void SyncReplication::populate_path_instr(Instruction::PathInstruction& instr, c // Populate top object in the normal way. auto top_table = table.get_parent_group()->get_table(full_path.top_table); - full_path.path_from_top.emplace_back(table.get_column_name(path[0].get_col_key())); + full_path.path_from_top.emplace_back(field_name); for (auto it = path.begin() + 1; it != path.end(); ++it) { full_path.path_from_top.emplace_back(std::move(*it)); @@ -782,8 +811,6 @@ void SyncReplication::populate_path_instr(Instruction::PathInstruction& instr, c m_last_primary_key = instr.object; } - StringData field_name = table.get_column_name(path[0].get_col_key()); - if (m_last_field_name == field_name) { instr.field = m_last_interned_field_name; } diff --git a/src/realm/sync/instruction_replication.hpp b/src/realm/sync/instruction_replication.hpp index ff7fbd62de4..46f683dcec6 100644 --- a/src/realm/sync/instruction_replication.hpp +++ b/src/realm/sync/instruction_replication.hpp @@ -132,13 +132,13 @@ class SyncReplication : public Replication { void populate_path_instr(Instruction::PathInstruction&, const CollectionBase&); void populate_path_instr(Instruction::PathInstruction&, const CollectionBase&, uint32_t ndx); - void dictionary_update(const CollectionBase&, const Mixed& key, const Mixed& val); + void dictionary_update(const CollectionBase&, const Mixed& key, const Mixed* val); // Cache information for the purpose of avoiding excessive string comparisons / interning // lookups. const Table* m_last_table = nullptr; ObjKey m_last_object; - StringData m_last_field_name; + std::string m_last_field_name; InternString m_last_class_name; util::Optional m_last_primary_key; InternString m_last_interned_field_name; diff --git a/src/realm/sync/noinst/server/server_file_access_cache.hpp b/src/realm/sync/noinst/server/server_file_access_cache.hpp index 24af8a4adc4..51186f5b2ba 100644 --- a/src/realm/sync/noinst/server/server_file_access_cache.hpp +++ b/src/realm/sync/noinst/server/server_file_access_cache.hpp @@ -229,6 +229,7 @@ inline DBOptions ServerFileAccessCache::Slot::make_shared_group_options() const options.encryption_key = m_cache.m_encryption_key->data(); if (m_disable_sync_to_disk) options.durability = DBOptions::Durability::Unsafe; + options.allow_flexible_schema = true; return options; } diff --git a/src/realm/table.cpp b/src/realm/table.cpp index a21820997d6..fb910594ec5 100644 --- a/src/realm/table.cpp +++ b/src/realm/table.cpp @@ -262,6 +262,7 @@ using namespace realm; using namespace realm::util; Replication* Table::g_dummy_replication = nullptr; +static const StringData additional_properties_colum_name{"__additional"}; bool TableVersions::operator==(const TableVersions& other) const { @@ -634,12 +635,18 @@ void Table::init(ref_type top_ref, ArrayParent* parent, size_t ndx_in_parent, bo auto rot_pk_key = m_top.get_as_ref_or_tagged(top_position_for_pk_col); m_primary_key_col = rot_pk_key.is_tagged() ? ColKey(rot_pk_key.get_as_int()) : ColKey(); + m_additional_prop_col = ColKey(); if (m_top.size() <= top_position_for_flags) { m_table_type = Type::TopLevel; } else { uint64_t flags = m_top.get_as_ref_or_tagged(top_position_for_flags).get_as_int(); m_table_type = Type(flags & table_type_mask); + if (flags & additional_prop_mask) { + // If we have an additional properties column, it will always be first + REALM_ASSERT(m_spec.m_names.size() > 0 && m_spec.m_names.get(0) == additional_properties_colum_name); + m_additional_prop_col = ColKey(m_spec.m_keys.get(0)); + } } m_has_any_embedded_objects.reset(); @@ -2952,6 +2959,23 @@ void Table::do_set_primary_key_column(ColKey col_key) m_primary_key_col = col_key; } +void Table::do_add_additional_prop_column() +{ + ColumnAttrMask attr; + attr.set(col_attr_Dictionary); + attr.set(col_attr_Nullable); + ColKey col_key = generate_col_key(col_type_Mixed, attr); + + uint64_t flags = m_top.get_as_ref_or_tagged(top_position_for_flags).get_as_int(); + flags |= additional_prop_mask; + m_top.set(top_position_for_flags, RefOrTagged::make_tagged(flags)); + + m_additional_prop_col = + do_insert_root_column(col_key, col_type_Mixed, additional_properties_colum_name, type_String); + // Be sure that it will always be first + REALM_ASSERT(m_additional_prop_col.get_index().val == 0); +} + bool Table::contains_unique_values(ColKey col) const { if (search_index_type(col) == IndexType::General) { diff --git a/src/realm/table.hpp b/src/realm/table.hpp index ac0faa5140a..278f756041c 100644 --- a/src/realm/table.hpp +++ b/src/realm/table.hpp @@ -93,6 +93,7 @@ class Table { /// . enum class Type : uint8_t { TopLevel = 0, Embedded = 0x1, TopLevelAsymmetric = 0x2 }; constexpr static uint8_t table_type_mask = 0x3; + constexpr static uint8_t additional_prop_mask = 0x4; /// Construct a new freestanding top-level table with static /// lifetime. For debugging only. @@ -146,6 +147,15 @@ class Table { DataType get_dictionary_key_type(ColKey column_key) const noexcept; ColKey get_column_key(StringData name) const noexcept; ColKey get_column_key(StableIndex) const noexcept; + bool is_additional_props_col(ColKey ck) const + { + return ck == m_additional_prop_col; + } + ColKey get_additional_prop_col() const + { + return m_additional_prop_col; + } + ColKeys get_column_keys() const; typedef util::Optional> BacklinkOrigin; BacklinkOrigin find_backlink_origin(StringData origin_table_name, StringData origin_col_name) const noexcept; @@ -738,6 +748,7 @@ class Table { Array m_opposite_column; // 8th slot in m_top std::vector> m_index_accessors; ColKey m_primary_key_col; + ColKey m_additional_prop_col; Replication* const* m_repl; static Replication* g_dummy_replication; bool m_is_frozen = false; @@ -793,6 +804,7 @@ class Table { ColKey find_backlink_column(ColKey origin_col_key, TableKey origin_table) const; ColKey find_or_add_backlink_column(ColKey origin_col_key, TableKey origin_table); void do_set_primary_key_column(ColKey col_key); + void do_add_additional_prop_column(); void validate_column_is_unique(ColKey col_key) const; ObjKey get_next_valid_key(); @@ -954,6 +966,7 @@ class ColKeys { public: ColKeys(ConstTableRef&& t) : m_table(std::move(t)) + , m_offset(m_table->get_additional_prop_col() ? 1 : 0) { } @@ -964,7 +977,7 @@ class ColKeys { size_t size() const { - return m_table->get_column_count(); + return m_table->get_column_count() - m_offset; } bool empty() const { @@ -972,19 +985,20 @@ class ColKeys { } ColKey operator[](size_t p) const { - return ColKeyIterator(m_table, p).operator*(); + return ColKeyIterator(m_table, p + m_offset).operator*(); } ColKeyIterator begin() const { - return ColKeyIterator(m_table, 0); + return ColKeyIterator(m_table, m_offset); } ColKeyIterator end() const { - return ColKeyIterator(m_table, size()); + return ColKeyIterator(m_table, m_table->get_column_count()); } private: ConstTableRef m_table; + unsigned m_offset = 0; }; // Class used to collect a chain of links when building up a Query following links. diff --git a/src/realm/transaction.cpp b/src/realm/transaction.cpp index 8221d65aa3f..7deb3451184 100644 --- a/src/realm/transaction.cpp +++ b/src/realm/transaction.cpp @@ -170,6 +170,7 @@ Transaction::Transaction(DBRef _db, SlabAlloc* alloc, DB::ReadLockInfo& rli, DB: { bool writable = stage == DB::transact_Writing; m_transact_stage = DB::transact_Ready; + m_allow_additional_properties = db->m_allow_flexible_schema; set_transact_stage(stage); attach_shared(m_read_lock.m_top_ref, m_read_lock.m_file_size, writable, VersionID{rli.m_version, rli.m_reader_idx}); diff --git a/test/object-store/c_api/c_api.cpp b/test/object-store/c_api/c_api.cpp index 3b80efa75c6..4e124f21c0e 100644 --- a/test/object-store/c_api/c_api.cpp +++ b/test/object-store/c_api/c_api.cpp @@ -2630,6 +2630,12 @@ TEST_CASE("C API - properties", "[c_api]") { CHECK(strings.get() != list2.get()); } + SECTION("get_by_name") { + auto list2 = cptr_checked(realm_get_list_by_name(obj2.get(), "strings")); + CHECK(realm_equals(strings.get(), list2.get())); + CHECK(strings.get() != list2.get()); + } + SECTION("insert, then get") { write([&]() { CHECK(checked(realm_list_insert(strings.get(), 0, a))); @@ -3739,6 +3745,12 @@ TEST_CASE("C API - properties", "[c_api]") { CHECK(strings.get() != dict2.get()); } + SECTION("get by name") { + auto dict2 = cptr_checked(realm_get_dictionary_by_name(obj1.get(), "nullable_string_dict")); + CHECK(realm_equals(strings.get(), dict2.get())); + CHECK(strings.get() != dict2.get()); + } + SECTION("insert, then get, then erase") { write([&]() { bool inserted = false; @@ -5841,6 +5853,94 @@ TEST_CASE("C API: convert", "[c_api]") { realm_release(realm); } +TEST_CASE("C API: flexible schema", "[c_api]") { + TestFile test_file; + ObjectSchema object_schema = {"Foo", {{"_id", PropertyType::Int, Property::IsPrimary{true}}}}; + + auto config = make_config(test_file.path.c_str(), false); + config->schema = Schema{object_schema}; + config->schema_version = 0; + config->flexible_schema = 1; + auto realm = realm_open(config.get()); + realm_class_info_t class_foo; + bool found = false; + CHECK(checked(realm_find_class(realm, "Foo", &found, &class_foo))); + REQUIRE(found); + + SECTION("Simple set/get/delete") { + checked(realm_begin_write(realm)); + + realm_value_t pk = rlm_int_val(42); + auto obj1 = cptr_checked(realm_object_create_with_primary_key(realm, class_foo.key, pk)); + checked(realm_set_value_by_name(obj1.get(), "age", rlm_int_val(23))); + const char* prop_names[10]; + size_t actual; + realm_get_additional_properties(obj1.get(), prop_names, 10, &actual); + REQUIRE(actual == 1); + CHECK(prop_names[0] == std::string_view("age")); + realm_has_property(obj1.get(), "age", &found); + REQUIRE(found); + realm_has_property(obj1.get(), "_id", &found); + REQUIRE(found); + realm_has_property(obj1.get(), "weight", &found); + REQUIRE(!found); + + realm_value_t value; + CHECK(checked(realm_get_value_by_name(obj1.get(), "age", &value))); + CHECK(value.type == RLM_TYPE_INT); + CHECK(value.integer == 23); + + checked(realm_erase_additional_property(obj1.get(), "age")); + realm_get_additional_properties(obj1.get(), nullptr, 0, &actual); + REQUIRE(actual == 0); + realm_commit(realm); + } + + SECTION("Set/get nested list") { + checked(realm_begin_write(realm)); + + realm_value_t pk = rlm_int_val(42); + auto obj1 = cptr_checked(realm_object_create_with_primary_key(realm, class_foo.key, pk)); + auto list = cptr_checked(realm_set_list_by_name(obj1.get(), "scores")); + REQUIRE(list); + realm_has_property(obj1.get(), "scores", &found); + REQUIRE(found); + + realm_value_t value; + CHECK(checked(realm_get_value_by_name(obj1.get(), "scores", &value))); + CHECK(value.type == RLM_TYPE_LIST); + + auto list1 = cptr_checked(realm_get_list_by_name(obj1.get(), "scores")); + REQUIRE(list1); + + realm_commit(realm); + } + + SECTION("Set/get nested dictionary") { + checked(realm_begin_write(realm)); + + realm_value_t pk = rlm_int_val(42); + auto obj1 = cptr_checked(realm_object_create_with_primary_key(realm, class_foo.key, pk)); + auto dict = cptr_checked(realm_set_dictionary_by_name(obj1.get(), "properties")); + REQUIRE(dict); + realm_has_property(obj1.get(), "properties", &found); + REQUIRE(found); + + realm_value_t value; + CHECK(checked(realm_get_value_by_name(obj1.get(), "properties", &value))); + CHECK(value.type == RLM_TYPE_DICTIONARY); + + auto dict1 = cptr_checked(realm_get_dictionary_by_name(obj1.get(), "properties")); + REQUIRE(dict1); + + realm_commit(realm); + } + + realm_close(realm); + REQUIRE(realm_is_closed(realm)); + realm_release(realm); +} + struct Userdata { std::atomic called{false}; bool has_error; diff --git a/test/object-store/object.cpp b/test/object-store/object.cpp index 37ebc08ec72..6b105f0ecd0 100644 --- a/test/object-store/object.cpp +++ b/test/object-store/object.cpp @@ -155,6 +155,100 @@ class CreatePolicyRecordingContext { mutable CreatePolicy last_create_policy; }; +TEST_CASE("object with flexible schema") { + using namespace std::string_literals; + _impl::RealmCoordinator::assert_no_open_realms(); + + InMemoryTestFile config; + config.automatic_change_notifications = false; + config.schema_mode = SchemaMode::AdditiveExplicit; + config.flexible_schema = true; + config.schema = Schema{{ + "table", + { + {"_id", PropertyType::Int, Property::IsPrimary{true}}, + }, + }}; + + config.schema_version = 0; + auto r = Realm::get_shared_realm(config); + + TestContext d(r); + auto create = [&](std::any&& value, CreatePolicy policy = CreatePolicy::ForceCreate) { + r->begin_transaction(); + auto obj = Object::create(d, r, *r->schema().find("table"), value, policy); + r->commit_transaction(); + return obj; + }; + + SECTION("create object") { + auto object = create(AnyDict{ + {"_id", INT64_C(1)}, + {"bool", true}, + {"int", INT64_C(5)}, + {"float", 2.2f}, + {"double", 3.3}, + {"string", "hello"s}, + {"date", Timestamp(10, 20)}, + {"object id", ObjectId("000000000000000000000001")}, + {"decimal", Decimal128("1.23e45")}, + {"uuid", UUID("3b241101-abba-baba-caca-4136c566a962")}, + {"mixed", "mixed"s}, + + {"bool array", AnyVec{true, false}}, + {"int array", AnyVec{INT64_C(5), INT64_C(6)}}, + {"float array", AnyVec{1.1f, 2.2f}}, + {"double array", AnyVec{3.3, 4.4}}, + {"string array", AnyVec{"a"s, "b"s, "c"s}}, + {"date array", AnyVec{Timestamp(10, 20), Timestamp(30, 40)}}, + {"object id array", AnyVec{ObjectId("AAAAAAAAAAAAAAAAAAAAAAAA"), ObjectId("BBBBBBBBBBBBBBBBBBBBBBBB")}}, + {"decimal array", AnyVec{Decimal128("1.23e45"), Decimal128("6.78e9")}}, + {"uuid array", AnyVec{UUID(), UUID("3b241101-e2bb-4255-8caf-4136c566a962")}}, + {"mixed array", + AnyVec{25, "b"s, 1.45, util::none, Timestamp(30, 40), Decimal128("1.23e45"), + ObjectId("AAAAAAAAAAAAAAAAAAAAAAAA"), UUID("3b241101-e2bb-4255-8caf-4136c566a962")}}, + {"dictionary", AnyDict{{"key", "value"s}}}, + }); + + Obj obj = object.get_obj(); + REQUIRE(obj.get("_id") == 1); + REQUIRE(obj.get("bool") == true); + REQUIRE(obj.get("int") == 5); + REQUIRE(obj.get("float") == 2.2f); + REQUIRE(obj.get("double") == 3.3); + REQUIRE(obj.get("string") == "hello"); + REQUIRE(obj.get("date") == Timestamp(10, 20)); + REQUIRE(obj.get("object id") == ObjectId("000000000000000000000001")); + REQUIRE(obj.get("decimal") == Decimal128("1.23e45")); + REQUIRE(obj.get("uuid") == UUID("3b241101-abba-baba-caca-4136c566a962")); + REQUIRE(obj.get("mixed") == Mixed("mixed")); + + auto check_array = [&](StringData prop, auto... values) { + auto vec = get_vector({values...}); + using U = typename decltype(vec)::value_type; + auto list = obj.get_list_ptr(prop); + size_t i = 0; + for (auto value : vec) { + CAPTURE(i); + REQUIRE(i < list->size()); + REQUIRE(value == list->get(i).get()); + ++i; + } + }; + check_array("bool array", true, false); + check_array("int array", INT64_C(5), INT64_C(6)); + check_array("float array", 1.1f, 2.2f); + check_array("double array", 3.3, 4.4); + check_array("string array", StringData("a"), StringData("b"), StringData("c")); + check_array("date array", Timestamp(10, 20), Timestamp(30, 40)); + check_array("object id array", ObjectId("AAAAAAAAAAAAAAAAAAAAAAAA"), ObjectId("BBBBBBBBBBBBBBBBBBBBBBBB")); + check_array("decimal array", Decimal128("1.23e45"), Decimal128("6.78e9")); + check_array("uuid array", UUID(), UUID("3b241101-e2bb-4255-8caf-4136c566a962")); + + REQUIRE(obj.get_dictionary_ptr("dictionary")->get("key") == Mixed("value")); + } +} + TEST_CASE("object") { using namespace std::string_literals; _impl::RealmCoordinator::assert_no_open_realms(); @@ -281,6 +375,7 @@ TEST_CASE("object") { {"person", { {"_id", PropertyType::String, Property::IsPrimary{true}}, + {"name", PropertyType::String}, {"age", PropertyType::Int}, {"scores", PropertyType::Array | PropertyType::Int}, {"assistant", PropertyType::Object | PropertyType::Nullable, "person"}, @@ -1180,7 +1275,6 @@ TEST_CASE("object") { {"data", "olleh"s}, {"date", Timestamp(10, 20)}, {"object", AnyDict{{"_id", INT64_C(10)}, {"value", INT64_C(10)}}}, - {"array", AnyVector{AnyDict{{"value", INT64_C(20)}}}}, {"object id", ObjectId("000000000000000000000001")}, {"decimal", Decimal128("1.23e45")}, {"uuid", UUID("3b241101-0000-0000-0000-4136c566a962")}, @@ -1532,7 +1626,6 @@ TEST_CASE("object") { {"string array", AnyVec{"a"s, "b"s, "c"s}}, {"data array", AnyVec{"d"s, "e"s, "f"s}}, {"date array", AnyVec{}}, - {"object array", AnyVec{AnyDict{{"_id", INT64_C(20)}, {"value", INT64_C(20)}}}}, {"object id array", AnyVec{ObjectId("000000000000000000000001")}}, {"decimal array", AnyVec{Decimal128("1.23e45")}}, {"uuid array", AnyVec{UUID("3b241101-1111-bbbb-cccc-4136c566a962")}}, @@ -1672,7 +1765,6 @@ TEST_CASE("object") { {"data", "olleh"s}, {"date", Timestamp(10, 20)}, {"object", AnyDict{{"_id", INT64_C(10)}, {"value", INT64_C(10)}}}, - {"array", AnyVector{AnyDict{{"value", INT64_C(20)}}}}, {"object id", ObjectId("000000000000000000000001")}, {"decimal", Decimal128("1.23e45")}, {"uuid", UUID("3b241101-aaaa-bbbb-cccc-4136c566a962")}, @@ -1688,7 +1780,6 @@ TEST_CASE("object") { {"data", "olleh"s}, {"date", Timestamp(10, 20)}, {"object", AnyDict{{"_id", INT64_C(10)}, {"value", INT64_C(10)}}}, - {"array", AnyVector{AnyDict{{"value", INT64_C(20)}}}}, {"object id", ObjectId("000000000000000000000001")}, {"decimal", Decimal128("1.23e45")}, {"uuid", UUID("3b241101-aaaa-bbbb-cccc-4136c566a962")}, @@ -1720,9 +1811,9 @@ TEST_CASE("object") { obj = create(AnyDict{{"_id", d.null_value()}}, "nullable string pk"); REQUIRE(obj.get_obj().is_null(col_pk_str)); - obj = create(AnyDict{{}}, "nullable int pk"); + obj = create(AnyDict{}, "nullable int pk"); REQUIRE(obj.get_obj().get>(col_pk_int) == 10); - obj = create(AnyDict{{}}, "nullable string pk"); + obj = create(AnyDict{}, "nullable string pk"); REQUIRE(obj.get_obj().get(col_pk_str) == "value"); } diff --git a/test/test_sync.cpp b/test/test_sync.cpp index 933e3516f50..57dd1d11c67 100644 --- a/test/test_sync.cpp +++ b/test/test_sync.cpp @@ -6223,6 +6223,70 @@ TEST(Sync_DeleteCollectionInCollection) } } +TEST(Sync_AdditionalProperties) +{ + DBOptions options; + options.allow_flexible_schema = true; + SHARED_GROUP_TEST_PATH(db_1_path); + SHARED_GROUP_TEST_PATH(db_2_path); + auto db_1 = DB::create(make_client_replication(), db_1_path, options); + auto db_2 = DB::create(make_client_replication(), db_2_path, options); + + TEST_DIR(dir); + fixtures::ClientServerFixture fixture{dir, test_context}; + fixture.start(); + + Session session_1 = fixture.make_session(db_1, "/test"); + Session session_2 = fixture.make_session(db_2, "/test"); + + write_transaction(db_1, [&](WriteTransaction& tr) { + auto& g = tr.get_group(); + auto table = g.add_table_with_primary_key("class_Table", type_Int, "id"); + auto col_any = table->add_column(type_Mixed, "any"); + auto foo = table->create_object_with_primary_key(123); + foo.set_any(col_any, "FooBar"); + foo.set("age", 10); + foo.set_collection("scores", CollectionType::List); + auto list = foo.get_list_ptr("scores"); + list->add(4.6); + }); + + session_1.wait_for_upload_complete_or_client_stopped(); + session_2.wait_for_download_complete_or_client_stopped(); + + write_transaction(db_2, [&](WriteTransaction& tr) { + auto table = tr.get_table("class_Table"); + CHECK_EQUAL(table->size(), 1); + + auto obj = table->get_object_with_primary_key(123); + auto col_keys = table->get_column_keys(); + CHECK_EQUAL(col_keys.size(), 2); + CHECK_EQUAL(table->get_column_name(col_keys[0]), "id"); + CHECK_EQUAL(table->get_column_name(col_keys[1]), "any"); + auto props = obj.get_additional_properties(); + CHECK_EQUAL(props.size(), 2); + CHECK_EQUAL(obj.get("age"), 10); + CHECK_EQUAL(obj.get_any("any"), Mixed("FooBar")); + auto list = obj.get_list_ptr("scores"); + CHECK_EQUAL(list->get(0), Mixed(4.6)); + CHECK_THROW_ANY(obj.get_any("some")); + CHECK_THROW_ANY(obj.erase_additional_prop("any")); + obj.erase_additional_prop("age"); + }); + + session_2.wait_for_upload_complete_or_client_stopped(); + session_1.wait_for_download_complete_or_client_stopped(); + + write_transaction(db_1, [&](WriteTransaction& tr) { + auto table = tr.get_table("class_Table"); + CHECK_EQUAL(table->size(), 1); + + auto obj = table->get_object_with_primary_key(123); + auto props = obj.get_additional_properties(); + CHECK_EQUAL(props.size(), 1); + }); +} + TEST(Sync_NestedCollectionClear) { TEST_CLIENT_DB(db_1);