Merge Next major

CHANGELOG.md

@@ -1,3 +1,27 @@
+# NEXT MAJOR RELEASE
+
+### Enhancements
+* <New feature description> (PR [#????](https://github.com/realm/realm-core/pull/????))
+* Storage of integers changed so that they take up less space in the file. This can cause commits and some queries to take a bit longer (PR [#7668](https://github.com/realm/realm-core/pull/7668))
+* We now allow synchronizing, getting and setting properties that are not defined in the object schema (PR [#7886](https://github.com/realm/realm-core/pull/7886))
+
+### Fixed
+* <How do the end-user experience this issue? what was the impact?> ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?)
+* None.
+
+### Breaking changes
+* None.
+
+### Compatibility
+* Fileformat: Generates files with format v25. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier.
+
+-----------
+
+### Internals
+* None.
+
+----------------------------------------------
+
 # 14.12.0 Release notes
 
 ### Enhancements
@@ -33,7 +57,6 @@
 
 ### Fixed
 * Sync client may report duplicate compensating write errors ([#7708](https://github.com/realm/realm-core/issues/7708), since v14.8.0).
-* String serialization of timestamps with a sufficiently large timestamp value could overflow an int causing undefined behavior, causing potentially bad values for the month/day/year values in stringified dates. ([PR #7934](https://github.com/realm/realm-core/pull/7934)).
 
 ### Breaking changes
 * None.
@@ -174,7 +197,7 @@
 
 ### Enhancements
 * It is no longer an error to set a base url for an App with a trailing slash - for example, `https://services.cloud.mongodb.com/` instead of `https://services.cloud.mongodb.com` - before this change that would result in a 404 error from the server ([PR #7791](https://github.com/realm/realm-core/pull/7791)).
-* Performance has been improved for range queries on integers and timestamps. Requires that you use the "BETWEEN" operation in RQL or the Query::between() method when you build the query. (PR [#7785](https://github.com/realm/realm-core/pull/7785))
+* Performance has been improved for range queries on integers and timestamps. Requires that you use the "BETWEEN" operation in MQL or the Query::between() method when you build the query. (PR [#7785](https://github.com/realm/realm-core/pull/7785))
 * Expose `Obj::add_int()` in the bindgen spec. ([PR #7797](https://github.com/realm/realm-core/pull/7797)).
 
 ### Fixed

CMakeLists.txt

@@ -266,12 +266,13 @@ option(REALM_ENABLE_ALLOC_SET_ZERO "Zero all allocations." OFF)
 if(NOT EMSCRIPTEN)
     option(REALM_ENABLE_ENCRYPTION "Enable encryption." ON)
 endif()
-option(REALM_ENABLE_MEMDEBUG "Add additional memory checks" OFF)
-option(REALM_VALGRIND "Tell the test suite we are running with valgrind" OFF)
-option(REALM_SYNC_MULTIPLEXING "Enables/disables sync session multiplexing by default" ON)
+option(REALM_ENABLE_MEMDEBUG "Add additional memory checks." OFF)
+option(REALM_VALGRIND "Tell the test suite we are running with valgrind." OFF)
+option(REALM_SYNC_MULTIPLEXING "Enables/disables sync session multiplexing by default." ON)
 set(REALM_MAX_BPNODE_SIZE "1000" CACHE STRING "Max B+ tree node size.")
 option(REALM_ENABLE_GEOSPATIAL "Enable geospatial types and queries." ON)
 option(REALM_APP_SERVICES "Enable the default app services implementation." ON)
+option(REALM_COMPRESS "Compress all the arrays by default in flex format." OFF)
 
 # Find dependencies
 set(THREADS_PREFER_PTHREAD_FLAG ON)

Package.swift

@@ -52,6 +52,7 @@ let notSyncServerSources: [String] = [
     "realm/array_blobs_small.cpp",
     "realm/array_decimal128.cpp",
     "realm/array_fixed_bytes.cpp",
+    "realm/array_aggregate_optimizations.cpp",
     "realm/array_integer.cpp",
     "realm/array_key.cpp",
     "realm/array_mixed.cpp",
@@ -78,6 +79,9 @@ let notSyncServerSources: [String] = [
     "realm/group.cpp",
     "realm/group_writer.cpp",
     "realm/history.cpp",
+    "realm/integer_compressor.cpp",
+    "realm/integer_flex_compressor.cpp",
+    "realm/integer_packed_compressor.cpp",
     "realm/impl",
     "realm/index_string.cpp",
     "realm/link_translator.cpp",

bindgen/spec.yml

@@ -440,6 +440,9 @@ records:
       schema_mode:
         type: SchemaMode
         default: SchemaMode::Automatic
+      flexible_schema:
+        type: bool
+        default: false
       disable_format_upgrade:
         type: bool
         default: false
@@ -785,6 +788,7 @@ classes:
       remove_object: '(key: ObjKey)'
       get_link_target: '(column: ColKey) -> TableRef'
       clear: ()
+      get_column_key: '(column: StringData) -> ColKey'
       get_primary_key_column: '() -> ColKey'
 
   Obj:
@@ -807,17 +811,28 @@ classes:
         - '(column: ColKey, value: Mixed)'
         - sig: '(column: ColKey, value: Mixed, is_default: bool)'
           suffix: with_default
-      set_collection: '(column: ColKey, type: CollectionType) -> Obj'
+        - sig: '(column: StringData, value: Mixed)'
+          suffix: by_name
+      set_collection:
+        - '(column: ColKey, type: CollectionType) -> Obj'
+        - sig: '(prop_name: StringData, type: CollectionType)'
+          suffix: by_name
       add_int: '(column: ColKey, value: int64_t) -> Obj'
       get_linked_object: '(column: ColKey) const -> Nullable<Obj>'
       to_string: () const -> std::string
       get_backlink_count: '() const -> count_t'
+      erase_additional_prop: '(prop_name: StringData) const -> Obj'
+      get_additional_properties: '() -> std::vector<StringData>'
       get_backlink_view: '(src_table: TableRef, src_col_key: ColKey) -> TableView'
       create_and_set_linked_object: '(column: ColKey) -> Obj'
-
+      has_schema_property: '(column: StringData) -> bool'
+      get_collection_ptr: '(prop_name: StringData) -> CollectionPointer'
   Transaction:
     sharedPtrWrapped: TransactionRef
 
+  CollectionPointer:
+    cppName: CollectionBasePtr
+
   ObjectStore:
     staticMethods:
       get_schema_version: '(group: Group) -> SchemaVersion'
@@ -1035,6 +1050,7 @@ classes:
 
   Collection:
     cppName: object_store::Collection
+    sharedPtrWrapped: SharedCollection
     abstract: true
     properties:
       get_type: PropertyType
@@ -1063,6 +1079,7 @@ classes:
     base: Collection
     constructors:
       make: '(r: SharedRealm, parent: const Obj&, col: ColKey)'
+      make_other: '(r: SharedRealm, parent: const Obj&, prop_name: StringData)'
     methods:
       get:
         - sig: '(ndx: count_t) -> Obj'

evergreen/config.yml

@@ -137,8 +137,16 @@ functions:
               set_cmake_var realm_vars REALM_LLVM_COVERAGE BOOL On
           fi
 
+          if [[ -n "${compress|}" ]]; then
+              set_cmake_var realm_vars REALM_COMPRESS PATH "${cmake_toolchain_file}"
+          fi
+
           set_cmake_var realm_vars REALM_BUILD_COMMANDLINE_TOOLS BOOL "${build_command_line_tools|On}"
           set_cmake_var realm_vars REALM_ENABLE_ENCRYPTION BOOL "${enable_realm_encryption|On}"
+          if [[ -n "${compress|}" ]]; then
+              set_cmake_var realm_vars REALM_COMPRESS PATH "${cmake_toolchain_file}"
+          fi
+
 
           if [[ -n "${fetch_missing_dependencies|}" ]]; then
               set_cmake_var realm_vars REALM_FETCH_MISSING_DEPENDENCIES BOOL On
@@ -1980,6 +1988,22 @@ buildvariants:
   - name: compile_test_coverage
   - name: finalize_coverage_data
 
+- name: macos-array-compression
+  display_name: "MacOS 14 arm64 (Compress Arrays)"
+  run_on: macos-14-arm64
+  expansions:
+    cmake_bindir: "/opt/homebrew/bin"
+    cmake_toolchain_file: "./tools/cmake/xcode.toolchain.cmake"
+    cmake_build_tool_options: "-sdk macosx"
+    cmake_generator: Xcode
+    max_jobs: $(sysctl -n hw.logicalcpu)
+    xcode_developer_dir: /Applications/Xcode15.2.app/Contents/Developer
+    extra_flags: -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_OSX_ARCHITECTURES=arm64
+    compress: On
+    cmake_build_type: Debug
+  tasks:
+  - name: compile_test
+
 - name: ubuntu-trace-logging
   display_name: "Ubuntu (Trace Logging Enabled)"
   run_on: ubuntu2204-arm64-large

src/realm.h

@@ -925,6 +925,11 @@ RLM_API bool realm_config_get_cached(realm_config_t*) RLM_API_NOEXCEPT;
  */
 RLM_API void realm_config_set_automatic_backlink_handling(realm_config_t*, bool) RLM_API_NOEXCEPT;
 
+/**
+ * Allow realm objects in the realm to have additional properties that are not defined in the schema.
+ */
+RLM_API void realm_config_set_flexible_schema(realm_config_t*, bool) RLM_API_NOEXCEPT;
+
 /**
  * Create a custom scheduler object from callback functions.
  *
@@ -1645,6 +1650,13 @@ RLM_API realm_object_t* realm_object_from_thread_safe_reference(const realm_t*,
  */
 RLM_API bool realm_get_value(const realm_object_t*, realm_property_key_t, realm_value_t* out_value);
 
+/**
+ * Get the value for a property.
+ *
+ * @return True if no exception occurred.
+ */
+RLM_API bool realm_get_value_by_name(const realm_object_t*, const char* property_name, realm_value_t* out_value);
+
 /**
  * Get the values for several properties.
  *
@@ -1680,6 +1692,41 @@ RLM_API bool realm_get_values(const realm_object_t*, size_t num_values, const re
  */
 RLM_API bool realm_set_value(realm_object_t*, realm_property_key_t, realm_value_t new_value, bool is_default);
 
+/**
+ * Set the value for a property. Property need not be defined in schema if flexible
+ * schema is enabled in configuration
+ *
+ * @param property_name The name of the property.
+ * @param new_value The new value for the property.
+ * @return True if no exception occurred.
+ */
+RLM_API bool realm_set_value_by_name(realm_object_t*, const char* property_name, realm_value_t new_value);
+
+/**
+ * Examines if the object has a property with the given name.
+ * @param out_has_property will be true if the property exists.
+ * @return True if no exception occurred.
+ */
+RLM_API bool realm_has_property(realm_object_t*, const char* property_name, bool* out_has_property);
+
+/**
+ * Get a list of properties set on the object that are not defined in the schema.
+ *
+ * @param out_prop_names A pointer to an array of const char* of size @a max. If the pointer is NULL,
+ *                       no names will be copied, but @a out_n will be set to the required size.
+ * @param max size of @a out_prop_names
+ * @param out_n number of names actually returned.
+ */
+RLM_API void realm_get_additional_properties(realm_object_t*, const char** out_prop_names, size_t max, size_t* out_n);
+
+/**
+ * Erases a property from an object. You can't erase a property that is defined in the current schema.
+ *
+ * @param property_name The name of the property.
+ * @return True if the property was removed.
+ */
+RLM_API bool realm_erase_additional_property(realm_object_t*, const char* property_name);
+
 /**
  * Assign a JSON formatted string to a Mixed property. Underlying structures will be created as needed
  *
@@ -1701,6 +1748,8 @@ RLM_API realm_object_t* realm_set_embedded(realm_object_t*, realm_property_key_t
  */
 RLM_API realm_list_t* realm_set_list(realm_object_t*, realm_property_key_t);
 RLM_API realm_dictionary_t* realm_set_dictionary(realm_object_t*, realm_property_key_t);
+RLM_API realm_list_t* realm_set_list_by_name(realm_object_t*, const char* property_name);
+RLM_API realm_dictionary_t* realm_set_dictionary_by_name(realm_object_t*, const char* property_name);
 
 /** Return the object linked by the given property
  *
@@ -1753,6 +1802,15 @@ RLM_API bool realm_set_values(realm_object_t*, size_t num_values, const realm_pr
  */
 RLM_API realm_list_t* realm_get_list(realm_object_t*, realm_property_key_t);
 
+/**
+ * Get a list instance for the property of an object by name.
+ *
+ * Note: It is up to the caller to call `realm_release()` on the returned list.
+ *
+ * @return A non-null pointer if no exception occurred.
+ */
+RLM_API realm_list_t* realm_get_list_by_name(realm_object_t*, const char*);
+
 /**
  * Create a `realm_list_t` from a pointer to a `realm::List`, copy-constructing
  * the internal representation.
@@ -2258,6 +2316,15 @@ RLM_API realm_set_t* realm_set_from_thread_safe_reference(const realm_t*, realm_
  */
 RLM_API realm_dictionary_t* realm_get_dictionary(realm_object_t*, realm_property_key_t);
 
+/**
+ * Get a dictionary instance for the property of an object by name.
+ *
+ * Note: It is up to the caller to call `realm_release()` on the returned dictionary.
+ *
+ * @return A non-null pointer if no exception occurred.
+ */
+RLM_API realm_dictionary_t* realm_get_dictionary_by_name(realm_object_t*, const char*);
+
 /**
  * Create a `realm_dictionary_t` from a pointer to a `realm::object_store::Dictionary`,
  * copy-constructing the internal representation.

src/realm/CMakeLists.txt

@@ -13,6 +13,7 @@ set(REALM_SOURCES
     array_blobs_big.cpp
     array_decimal128.cpp
     array_fixed_bytes.cpp
+    array_aggregate_optimizations.cpp
     array_integer.cpp
     array_key.cpp
     array_mixed.cpp
@@ -36,6 +37,9 @@ set(REALM_SOURCES
     db.cpp
     group_writer.cpp
     history.cpp
+    integer_compressor.cpp
+    integer_flex_compressor.cpp
+    integer_packed_compressor.cpp
     impl/copy_replication.cpp
     impl/output_stream.cpp
     impl/simulated_failure.cpp
@@ -163,6 +167,9 @@ set(REALM_INSTALL_HEADERS
     handover_defs.hpp
     history.hpp
     index_string.hpp
+    integer_compressor.hpp
+    integer_flex_compressor.hpp
+    integer_packed_compressor.hpp
     keys.hpp
     list.hpp
     mixed.hpp

src/realm/alloc.cpp

@@ -113,14 +113,17 @@ Allocator& Allocator::get_default() noexcept
 // * adding a cross-over mapping. (if the array crosses a mapping boundary)
 // * using an already established cross-over mapping. (ditto)
 // this can proceed concurrently with other calls to translate()
-char* Allocator::translate_less_critical(RefTranslation* ref_translation_ptr, ref_type ref) const noexcept
+char* Allocator::translate_less_critical(RefTranslation* ref_translation_ptr, ref_type ref,
+                                         bool known_in_slab) const noexcept
 {
     size_t idx = get_section_index(ref);
     RefTranslation& txl = ref_translation_ptr[idx];
     size_t offset = ref - get_section_base(idx);
     char* addr = txl.mapping_addr + offset;
     util::encryption_read_barrier(addr, NodeHeader::header_size, txl.encrypted_mapping);
-    auto size = NodeHeader::get_byte_size_from_header(addr);
+    // if we know the translation is inside the slab area, we don't need to check
+    // for anything beyond the header, and we don't need to check if decryption is needed
+    auto size = known_in_slab ? 8 : NodeHeader::get_byte_size_from_header(addr);
     bool crosses_mapping = offset + size > (1 << section_shift);
     // Move the limit on use of the existing primary mapping.
     // Take into account that another thread may attempt to change / have changed it concurrently,

src/realm/alloc.hpp

@@ -113,6 +113,9 @@ class Allocator {
     /// Calls do_translate().
     char* translate(ref_type ref) const noexcept;
 
+    /// Simpler version if we know the ref points inside the slab area
+    char* translate_in_slab(ref_type ref) const noexcept;
+
     /// Returns true if, and only if the object at the specified 'ref'
     /// is in the immutable part of the memory managed by this
     /// allocator. The method by which some objects become part of the
@@ -247,8 +250,8 @@ class Allocator {
     /// then entirely the responsibility of the caller that the memory
     /// is not modified by way of the returned memory pointer.
     virtual char* do_translate(ref_type ref) const noexcept = 0;
-    char* translate_critical(RefTranslation*, ref_type ref) const noexcept;
-    char* translate_less_critical(RefTranslation*, ref_type ref) const noexcept;
+    char* translate_critical(RefTranslation*, ref_type ref, bool known_in_slab = false) const noexcept;
+    char* translate_less_critical(RefTranslation*, ref_type ref, bool known_in_slab = false) const noexcept;
     virtual void get_or_add_xover_mapping(RefTranslation&, size_t, size_t, size_t) = 0;
     Allocator() noexcept = default;
     size_t get_section_index(size_t pos) const noexcept;
@@ -544,7 +547,8 @@ inline bool Allocator::is_read_only(ref_type ref) const noexcept
 }
 
 // performance critical part of the translation process. Less critical code is in translate_less_critical.
-inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr, ref_type ref) const noexcept
+inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr, ref_type ref,
+                                           bool known_in_slab) const noexcept
 {
     size_t idx = get_section_index(ref);
     RefTranslation& txl = ref_translation_ptr[idx];
@@ -560,7 +564,7 @@ inline char* Allocator::translate_critical(RefTranslation* ref_translation_ptr,
             return addr;
         }
         // the lowest possible xover offset may grow concurrently, but that will be handled inside the call
-        return translate_less_critical(ref_translation_ptr, ref);
+        return translate_less_critical(ref_translation_ptr, ref, known_in_slab);
     }
     realm::util::terminate("Invalid ref translation entry", __FILE__, __LINE__, txl.cookie, 0x1234567890, ref, idx);
 }
@@ -573,6 +577,17 @@ inline char* Allocator::translate(ref_type ref) const noexcept
     return do_translate(ref);
 }
 
+inline char* Allocator::translate_in_slab(ref_type ref) const noexcept
+{
+    auto ref_translation_ptr = m_ref_translation_ptr.load(std::memory_order_acquire);
+    if (REALM_LIKELY(ref_translation_ptr)) {
+        return translate_critical(ref_translation_ptr, ref, true);
+    }
+    else {
+        return do_translate(ref);
+    }
+}
+
 
 } // namespace realm