From 9b62144ea477c7cdad1ce9ad06daf8de9ddf5a6e Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 18 Sep 2023 16:04:34 -0600 Subject: [PATCH 01/13] let's see if this'll start up an atlas cluster... --- .evergreen/config.yml | 57 ++++++++- .evergreen/config/commands.yml.erb | 55 +++++++++ .evergreen/config/run-tests-atlas-full.sh | 23 ++++ lib/mongoid/composable.rb | 2 + lib/mongoid/search_indexable.rb | 77 ++++++++++++ spec/mongoid/search_indexable_spec.rb | 139 ++++++++++++++++++++++ 6 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 .evergreen/config/run-tests-atlas-full.sh create mode 100644 lib/mongoid/search_indexable.rb create mode 100644 spec/mongoid/search_indexable_spec.rb diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 3aff112ac7..c2a1bc0004 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -307,10 +307,65 @@ post: #- func: "upload test results" - func: "upload test results to s3" +task_groups: + - name: testatlas_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch source + - func: create expansions + - command: shell.exec + params: + shell: "bash" + working_dir: "src" + script: | + ${PREPARE_SHELL} + + DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \ + DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \ + DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \ + DRIVERS_ATLAS_LAMBDA_USER="${DRIVERS_ATLAS_LAMBDA_USER}" \ + DRIVERS_ATLAS_LAMBDA_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" \ + LAMBDA_STACK_NAME="dbx-ruby-lambda" \ + MONGODB_VERSION="7.0" \ + task_id="${task_id}" \ + execution="${execution}" \ + $DRIVERS_TOOLS/.evergreen/atlas/setup-atlas-cluster.sh + - command: expansions.update + params: + file: src/atlas-expansion.yml + teardown_group: + - command: shell.exec + params: + shell: "bash" + working_dir: "src" + script: | + ${PREPARE_SHELL} + + DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \ + DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \ + DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \ + LAMBDA_STACK_NAME="dbx-ruby-lambda" \ + task_id="${task_id}" \ + execution="${execution}" \ + $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh + tasks: + - test-full-atlas-task + tasks: - name: "test" commands: - func: "run tests" + - name: "test-full-atlas-task" + commands: + - command: shell.exec + type: test + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh axes: - id: "mongodb-version" display_name: MongoDB Version @@ -682,7 +737,7 @@ buildvariants: rails: ['7.0'] os: ubuntu-22.04 fle: helper - display_name: "${rails}, ${driver}, ${mongodb-version}" + display_name: "${rails}, ${driver}, ${mongodb-version} (FLE ${fle})" tasks: - name: "test" diff --git a/.evergreen/config/commands.yml.erb b/.evergreen/config/commands.yml.erb index 6d16b19908..d73a193bcf 100644 --- a/.evergreen/config/commands.yml.erb +++ b/.evergreen/config/commands.yml.erb @@ -281,7 +281,62 @@ post: #- func: "upload test results" - func: "upload test results to s3" +task_groups: + - name: testatlas_task_group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch source + - func: create expansions + - command: shell.exec + params: + shell: "bash" + working_dir: "src" + script: | + ${PREPARE_SHELL} + + DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \ + DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \ + DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \ + DRIVERS_ATLAS_LAMBDA_USER="${DRIVERS_ATLAS_LAMBDA_USER}" \ + DRIVERS_ATLAS_LAMBDA_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" \ + LAMBDA_STACK_NAME="dbx-ruby-lambda" \ + MONGODB_VERSION="7.0" \ + task_id="${task_id}" \ + execution="${execution}" \ + $DRIVERS_TOOLS/.evergreen/atlas/setup-atlas-cluster.sh + - command: expansions.update + params: + file: src/atlas-expansion.yml + teardown_group: + - command: shell.exec + params: + shell: "bash" + working_dir: "src" + script: | + ${PREPARE_SHELL} + + DRIVERS_ATLAS_PUBLIC_API_KEY="${DRIVERS_ATLAS_PUBLIC_API_KEY}" \ + DRIVERS_ATLAS_PRIVATE_API_KEY="${DRIVERS_ATLAS_PRIVATE_API_KEY}" \ + DRIVERS_ATLAS_GROUP_ID="${DRIVERS_ATLAS_GROUP_ID}" \ + LAMBDA_STACK_NAME="dbx-ruby-lambda" \ + task_id="${task_id}" \ + execution="${execution}" \ + $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh + tasks: + - test-full-atlas-task + tasks: - name: "test" commands: - func: "run tests" + - name: "test-full-atlas-task" + commands: + - command: shell.exec + type: test + params: + working_dir: "src" + shell: "bash" + script: | + ${PREPARE_SHELL} + MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh diff --git a/.evergreen/config/run-tests-atlas-full.sh b/.evergreen/config/run-tests-atlas-full.sh new file mode 100644 index 0000000000..c36fc64c44 --- /dev/null +++ b/.evergreen/config/run-tests-atlas-full.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -ex + +. `dirname "$0"`/../spec/shared/shlib/distro.sh +. `dirname "$0"`/../spec/shared/shlib/set_env.sh +. `dirname "$0"`/functions.sh + +set_env_vars +set_env_python +set_env_ruby + +bundle_install + +ATLAS_URI=$MONGODB_URI \ + EXAMPLE_TIMEOUT=600 \ + bundle exec rspec -fd spec/mongoid/search_indexable_spec.rb + +test_status=$? + +kill_jruby + +exit ${test_status} diff --git a/lib/mongoid/composable.rb b/lib/mongoid/composable.rb index 4afb69c24e..d935e88ca5 100644 --- a/lib/mongoid/composable.rb +++ b/lib/mongoid/composable.rb @@ -12,6 +12,7 @@ require "mongoid/matchable" require "mongoid/persistable" require "mongoid/reloadable" +require 'mongoid/search_indexable' require "mongoid/selectable" require "mongoid/scopable" require "mongoid/serializable" @@ -50,6 +51,7 @@ module Composable include Association include Reloadable include Scopable + include SearchIndexable include Selectable include Serializable include Shardable diff --git a/lib/mongoid/search_indexable.rb b/lib/mongoid/search_indexable.rb new file mode 100644 index 0000000000..c8884804e5 --- /dev/null +++ b/lib/mongoid/search_indexable.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Mongoid + # Encapsulates behavior around managing search indexes. This feature + # is only supported when connected to an Atlas cluster. + module SearchIndexable + extend ActiveSupport::Concern + + included do + cattr_accessor :search_index_specs + self.search_index_specs = [] + end + + # Implementations for the feature's class-level methods. + module ClassMethods + # Request the creation of all registered search indices. Note + # that the search indexes are created asynchronously, and may take + # several minutes to be fully available. + # + # @return [ Array ] The names of the search indexes. + def create_search_indexes + return if search_index_specs.empty? + + collection.search_indexes.create_many(search_index_specs) + end + + def search_indexes(options = {}) + collection.search_indexes(options) + end + + def remove_search_index(name: nil, id: nil) + logger.info("MONGOID: Removing search index '#{name || id}' " \ + "on collection '#{self.collection.name}'.") + collection.search_indexes.drop_one(name: name, id: id) + end + + # Request the removal of all registered search indices. Note + # that the search indexes are removed asynchronously, and may take + # several minutes to be fully deleted. + # + # @note It would be nice if this could remove ONLY the search indices + # that have been declared on the model, but because the model may not + # name the index, we can't guarantee that we'll know the name or id of + # the corresponding indices. It is not unreasonable to assume, though, + # that the intention is for the model to declare, one-to-one, all + # desired search indices, so removing all search indices ought to suffice. + # If a specific index or set of indices needs to be removed instead, + # consider using search_indexes.each with remove_search_index. + def remove_search_indexes + search_indexes.each do |spec| + remove_search_index id: spec['id'] + end + end + + # Adds an index definition for the provided single or compound keys. + # + # @example Create a basic index. + # class Person + # include Mongoid::Document + # field :name, type: String + # search_index({ ... }) + # search_index :name_of_index, { ... } + # end + # + # @param [ Symbol | String ] name_or_defn Either the name of the index to + # define, or the index definition. + # @param [ Hash ] defn The search index definition. + def search_index(name_or_defn, defn = nil) + name = name_or_defn + name, defn = nil, name if name.is_a?(Hash) + + spec = { definition: defn }.tap { |s| s[:name] = name if name } + search_index_specs.push(spec) + end + end + end +end diff --git a/spec/mongoid/search_indexable_spec.rb b/spec/mongoid/search_indexable_spec.rb new file mode 100644 index 0000000000..8fdf97e343 --- /dev/null +++ b/spec/mongoid/search_indexable_spec.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +require "spec_helper" + +class SearchIndexHelper + attr_reader :model + + def initialize(model) + @model = model + model.collection.create + end + + def collection + model.collection + end + + # Wait for all of the indexes with the given names to be ready; then return + # the list of index definitions corresponding to those names. + def wait_for(*names, &condition) + names.flatten! + + timeboxed_wait do + result = collection.search_indexes + return filter_results(result, names) if names.all? { |name| ready?(result, name, &condition) } + end + end + + # Wait until all of the indexes with the given names are absent from the + # search index list. + def wait_for_absense_of(*names) + names.flatten.each do |name| + timeboxed_wait do + break if collection.search_indexes(name: name).empty? + end + end + end + + private + + def timeboxed_wait(step: 5, max: 300) + start = Mongo::Utils.monotonic_time + + loop do + yield + + sleep step + raise Timeout::Error, 'wait took too long' if Mongo::Utils.monotonic_time - start > max + end + end + + # Returns true if the list of search indexes includes one with the given name, + # which is ready to be queried. + def ready?(list, name, &condition) + condition ||= ->(index) { index['queryable'] } + list.any? { |index| index['name'] == name && condition[index] } + end + + def filter_results(result, names) + result.select { |index| names.include?(index['name']) } + end +end + +class SearchablePerson + include Mongoid::Document + + search_index mappings: { dynamic: false } + search_index :with_dynamic_mappings, mappings: { dynamic: true } +end + +describe Mongoid::SearchIndexable do + before do + skip "#{described_class} requires at Atlas environment (set ATLAS_URI)" if ENV['ATLAS_URI'].nil? + end + + let(:helper) { SearchIndexHelper.new(Person) } + + describe '.search_index_specs' do + context 'when no search indexes have been defined' do + it 'has no search index specs' do + expect(Person.search_index_specs).to be_empty + end + end + + context 'when search indexes have been defined' do + it 'has search index specs' do + expect(SearchablePerson.search_index_specs).to be == [ + { definition: { dynamic: false } }, + { name: :with_dynamic_mappings, definition: { dynamic: true } } + ] + end + end + end + + context 'when needing to first create search indexes' do + let(:requested_definitions) { SearchablePerson.search_index_specs.map { |spec| spec[:definition] } } + let(:index_names) { SearchablePerson.create_search_indexes } + let(:actual_definitions) { helper.wait_for(*index_names) } + + describe '.create_search_indexes' do + it 'creates the indexes' do + expect(actual_definitions).to be == requested_definitions + end + end + + describe '.search_indexes' do + before { actual_definitions } # wait for the indices to be created + let(:queried_definitions) { SearchablePerson.search_indexes.map { |i| i['latestDefinition'] } } + + it 'queries the available search indexes' do + expect(queried_definitions).to be == requested_definitions + end + end + + describe '.remove_search_index' do + let(:target_index) { actual_definitions.first } + + before do + SearchablePerson.remove_search_index id: target_index['id'] + helper.wait_for_absense_of target_index['name'] + end + + it 'removes the requested index' do + expect(SearchablePerson.search_indexes(id: target_index['id'])).to be_empty + end + end + + describe '.remove_search_indexes' do + before do + actual_definitions # wait for the indexes to be created + Person.remove_search_indexes + helper.wait_for_absense_of(actual_definitions.map { |i| i['name'] }) + end + + it 'removes the indexes' do + expect(SearchablePerson.search_indexes).to be_empty + end + end + end +end From 9a80b01500ac4d06e84cdc9b65fe4ff1aa34ce19 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 18 Sep 2023 16:10:30 -0600 Subject: [PATCH 02/13] need to add a variant so the spec will actually run --- .evergreen/config.yml | 8 ++++++++ .evergreen/config/variants.yml.erb | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c2a1bc0004..a6320ff4f4 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -864,3 +864,11 @@ buildvariants: display_name: "FLE: ${rails}, ${driver}, ${mongodb-version}" tasks: - name: "test" + +- matrix_name: atlas-full + matrix_spec: + ruby: ruby-3.2 + os: rhel8 + display_name: "Atlas (Full)" + tasks: + - name: testatlas_task_group diff --git a/.evergreen/config/variants.yml.erb b/.evergreen/config/variants.yml.erb index b0d633844e..cbec69fd10 100644 --- a/.evergreen/config/variants.yml.erb +++ b/.evergreen/config/variants.yml.erb @@ -245,3 +245,11 @@ buildvariants: display_name: "FLE: ${rails}, ${driver}, ${mongodb-version}" tasks: - name: "test" + +- matrix_name: atlas-full + matrix_spec: + ruby: ruby-3.2 + os: ubuntu-22.04 + display_name: "Atlas (Full)" + tasks: + - name: testatlas_task_group From f4c94ad384e8121d6db7319ca9e8e27df60bb35a Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 18 Sep 2023 16:12:05 -0600 Subject: [PATCH 03/13] appease rubocop --- lib/mongoid/search_indexable.rb | 7 +++++-- spec/mongoid/search_indexable_spec.rb | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/mongoid/search_indexable.rb b/lib/mongoid/search_indexable.rb index c8884804e5..5ff876a45a 100644 --- a/lib/mongoid/search_indexable.rb +++ b/lib/mongoid/search_indexable.rb @@ -29,8 +29,11 @@ def search_indexes(options = {}) end def remove_search_index(name: nil, id: nil) - logger.info("MONGOID: Removing search index '#{name || id}' " \ - "on collection '#{self.collection.name}'.") + logger.info( + "MONGOID: Removing search index '#{name || id}' " \ + "on collection '#{collection.name}'." + ) + collection.search_indexes.drop_one(name: name, id: id) end diff --git a/spec/mongoid/search_indexable_spec.rb b/spec/mongoid/search_indexable_spec.rb index 8fdf97e343..a76f39bd3e 100644 --- a/spec/mongoid/search_indexable_spec.rb +++ b/spec/mongoid/search_indexable_spec.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "spec_helper" +require 'spec_helper' class SearchIndexHelper attr_reader :model @@ -104,6 +104,7 @@ class SearchablePerson describe '.search_indexes' do before { actual_definitions } # wait for the indices to be created + let(:queried_definitions) { SearchablePerson.search_indexes.map { |i| i['latestDefinition'] } } it 'queries the available search indexes' do From 4434ddb4ae75f7a72e537d6e6c8c9c8f7a46dd39 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 18 Sep 2023 16:37:12 -0600 Subject: [PATCH 04/13] run on a supported os --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index a6320ff4f4..b8c1d9cc05 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -868,7 +868,7 @@ buildvariants: - matrix_name: atlas-full matrix_spec: ruby: ruby-3.2 - os: rhel8 + os: ubuntu-22.04 display_name: "Atlas (Full)" tasks: - name: testatlas_task_group From f7c1253af6e0db906b30d0b3783f4a3fe5c8e43b Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Mon, 18 Sep 2023 16:51:40 -0600 Subject: [PATCH 05/13] need .mod/drivers-evergreen-tools --- .evergreen/config.yml | 2 +- .evergreen/config/commands.yml.erb | 2 +- .gitmodules | 3 +++ .mod/drivers-evergreen-tools | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) create mode 160000 .mod/drivers-evergreen-tools diff --git a/.evergreen/config.yml b/.evergreen/config.yml index b8c1d9cc05..f0d95fcd88 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -51,7 +51,7 @@ functions: CURRENT_VERSION=latest fi - export DRIVERS_TOOLS="$(pwd)/../drivers-tools" + export DRIVERS_TOOLS="$(pwd)/.mod/drivers-evergreen-tools" export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" diff --git a/.evergreen/config/commands.yml.erb b/.evergreen/config/commands.yml.erb index d73a193bcf..19d3406201 100644 --- a/.evergreen/config/commands.yml.erb +++ b/.evergreen/config/commands.yml.erb @@ -25,7 +25,7 @@ functions: CURRENT_VERSION=latest fi - export DRIVERS_TOOLS="$(pwd)/../drivers-tools" + export DRIVERS_TOOLS="$(pwd)/.mod/drivers-evergreen-tools" export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" diff --git a/.gitmodules b/.gitmodules index 805feb77d5..05f15e6b98 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "spec/shared"] path = spec/shared url = https://github.com/mongodb-labs/mongo-ruby-spec-shared +[submodule ".mod/drivers-evergreen-tools"] + path = .mod/drivers-evergreen-tools + url = https://github.com/mongodb-labs/drivers-evergreen-tools diff --git a/.mod/drivers-evergreen-tools b/.mod/drivers-evergreen-tools new file mode 160000 index 0000000000..1f018c7a24 --- /dev/null +++ b/.mod/drivers-evergreen-tools @@ -0,0 +1 @@ +Subproject commit 1f018c7a248c4fcda6cb7a77043fd673755e0986 From afd5c8544c5874e48c0d25030845445f0da20313 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Tue, 19 Sep 2023 08:33:28 -0600 Subject: [PATCH 06/13] this got put in the wrong directory --- .evergreen/{config => }/run-tests-atlas-full.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .evergreen/{config => }/run-tests-atlas-full.sh (100%) diff --git a/.evergreen/config/run-tests-atlas-full.sh b/.evergreen/run-tests-atlas-full.sh similarity index 100% rename from .evergreen/config/run-tests-atlas-full.sh rename to .evergreen/run-tests-atlas-full.sh From 78b831439b483588a262dca4912492602d7e7deb Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Tue, 19 Sep 2023 14:53:18 -0600 Subject: [PATCH 07/13] get specs to pass --- .evergreen/config.yml | 108 ++++++++++++++------------ .evergreen/config/axes.yml.erb | 5 +- .evergreen/config/commands.yml.erb | 89 +++++++++++---------- .evergreen/config/variants.yml.erb | 4 +- .evergreen/run-tests-atlas-full.sh | 3 +- lib/mongoid/search_indexable.rb | 4 +- spec/lite_spec_helper.rb | 33 +++++--- spec/mongoid/search_indexable_spec.rb | 51 ++++++------ spec/spec_helper.rb | 39 ++++++---- spec/support/spec_config.rb | 8 +- 10 files changed, 198 insertions(+), 146 deletions(-) mode change 100644 => 100755 .evergreen/run-tests-atlas-full.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f0d95fcd88..70e4ae92e6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -44,50 +44,56 @@ functions: params: working_dir: "src" script: | - # Get the current unique version of this checkout - if [ "${is_patch}" = "true" ]; then - CURRENT_VERSION=$(git describe)-patch-${version_id} - else - CURRENT_VERSION=latest - fi - - export DRIVERS_TOOLS="$(pwd)/.mod/drivers-evergreen-tools" - - export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" - export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" - export UPLOAD_BUCKET="${project}" - export PROJECT_DIRECTORY="$(pwd)" - - cat < expansion.yml - CURRENT_VERSION: "$CURRENT_VERSION" - DRIVERS_TOOLS: "$DRIVERS_TOOLS" - MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" - MONGODB_BINARIES: "$MONGODB_BINARIES" - UPLOAD_BUCKET: "$UPLOAD_BUCKET" - PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" - PREPARE_SHELL: | - set -o errexit - set -o xtrace - export DRIVERS_TOOLS="$DRIVERS_TOOLS" - export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" - export MONGODB_BINARIES="$MONGODB_BINARIES" - export UPLOAD_BUCKET="$UPLOAD_BUCKET" - export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" - - export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" - export PATH="$MONGODB_BINARIES:$PATH" - export PROJECT="${project}" - - export MONGODB_VERSION=${VERSION} - export TOPOLOGY=${TOPOLOGY} - export SINGLE_MONGOS=${SINGLE_MONGOS} - export AUTH=${AUTH} - export SSL=${SSL} - export APP_TESTS=${APP_TESTS} - export DOCKER_DISTRO=${DOCKER_DISTRO} - EOT - # See what we've done - cat expansion.yml + # Get the current unique version of this checkout + if [ "${is_patch}" = "true" ]; then + CURRENT_VERSION=$(git describe)-patch-${version_id} + else + CURRENT_VERSION=latest + fi + + export DRIVERS_TOOLS="$(pwd)/.mod/drivers-evergreen-tools" + + export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" + export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" + export UPLOAD_BUCKET="${project}" + export PROJECT_DIRECTORY="$(pwd)" + + cat < expansion.yml + CURRENT_VERSION: "$CURRENT_VERSION" + DRIVERS_TOOLS: "$DRIVERS_TOOLS" + MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" + MONGODB_BINARIES: "$MONGODB_BINARIES" + UPLOAD_BUCKET: "$UPLOAD_BUCKET" + PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" + PREPARE_SHELL: | + set -o errexit + set -o xtrace + export DRIVERS_TOOLS="$DRIVERS_TOOLS" + export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" + export MONGODB_BINARIES="$MONGODB_BINARIES" + export UPLOAD_BUCKET="$UPLOAD_BUCKET" + export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" + + export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" + export PATH="$MONGODB_BINARIES:$PATH" + export PROJECT="${project}" + + export MONGODB_VERSION="${VERSION}" + export TOPOLOGY="${TOPOLOGY}" + export SINGLE_MONGOS="${SINGLE_MONGOS}" + export AUTH="${AUTH}" + export SSL="${SSL}" + export APP_TESTS="${APP_TESTS}" + export DOCKER_DISTRO="${DOCKER_DISTRO}" + export RVM_RUBY="${RVM_RUBY}" + export RAILS="${RAILS}" + export DRIVER="${DRIVER}" + export I18N="${I18N}" + export TEST_I18N_FALLBACKS="${TEST_I18N_FALLBACKS}" + export FLE="${FLE}" + EOT + # See what we've done + cat expansion.yml # Load the expansion file to make an evergreen variable with the current unique version - command: expansions.update @@ -266,7 +272,7 @@ functions: ${PREPARE_SHELL} env \ MONGODB_URI="${MONGODB_URI}" \ - TOPOLOGY=${TOPOLOGY} \ + TOPOLOGY="${TOPOLOGY}" \ RVM_RUBY="${RVM_RUBY}" \ RAILS="${RAILS}" \ DRIVER="${DRIVER}" \ @@ -365,7 +371,8 @@ tasks: shell: "bash" script: | ${PREPARE_SHELL} - MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh + MONGODB_URI="${MONGODB_URI}" \ + .evergreen/run-tests-atlas-full.sh axes: - id: "mongodb-version" display_name: MongoDB Version @@ -487,13 +494,16 @@ axes: - id: "os" display_name: OS values: + - id: actual-ubuntu-22.04 + display_name: "Ubuntu 22.04" + run_on: ubuntu2204-small - id: ubuntu-18.04 display_name: "Ubuntu 18.04" run_on: ubuntu2004-small variables: DOCKER_DISTRO: ubuntu1804 - id: ubuntu-22.04 - display_name: "Ubuntu 20.04" + display_name: "Ubuntu 22.04" run_on: ubuntu2004-small variables: DOCKER_DISTRO: ubuntu2204 @@ -868,7 +878,9 @@ buildvariants: - matrix_name: atlas-full matrix_spec: ruby: ruby-3.2 - os: ubuntu-22.04 + os: actual-ubuntu-22.04 + auth: auth + ssl: ssl display_name: "Atlas (Full)" tasks: - name: testatlas_task_group diff --git a/.evergreen/config/axes.yml.erb b/.evergreen/config/axes.yml.erb index c5e8b8f878..2c1e5a44c3 100644 --- a/.evergreen/config/axes.yml.erb +++ b/.evergreen/config/axes.yml.erb @@ -119,13 +119,16 @@ axes: - id: "os" display_name: OS values: + - id: actual-ubuntu-22.04 + display_name: "Ubuntu 22.04" + run_on: ubuntu2204-small - id: ubuntu-18.04 display_name: "Ubuntu 18.04" run_on: ubuntu2004-small variables: DOCKER_DISTRO: ubuntu1804 - id: ubuntu-22.04 - display_name: "Ubuntu 20.04" + display_name: "Ubuntu 22.04" run_on: ubuntu2004-small variables: DOCKER_DISTRO: ubuntu2204 diff --git a/.evergreen/config/commands.yml.erb b/.evergreen/config/commands.yml.erb index 19d3406201..396ab4531b 100644 --- a/.evergreen/config/commands.yml.erb +++ b/.evergreen/config/commands.yml.erb @@ -18,50 +18,56 @@ functions: params: working_dir: "src" script: | - # Get the current unique version of this checkout - if [ "${is_patch}" = "true" ]; then - CURRENT_VERSION=$(git describe)-patch-${version_id} - else - CURRENT_VERSION=latest - fi + # Get the current unique version of this checkout + if [ "${is_patch}" = "true" ]; then + CURRENT_VERSION=$(git describe)-patch-${version_id} + else + CURRENT_VERSION=latest + fi - export DRIVERS_TOOLS="$(pwd)/.mod/drivers-evergreen-tools" + export DRIVERS_TOOLS="$(pwd)/.mod/drivers-evergreen-tools" - export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" - export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" - export UPLOAD_BUCKET="${project}" - export PROJECT_DIRECTORY="$(pwd)" + export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" + export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" + export UPLOAD_BUCKET="${project}" + export PROJECT_DIRECTORY="$(pwd)" - cat < expansion.yml - CURRENT_VERSION: "$CURRENT_VERSION" - DRIVERS_TOOLS: "$DRIVERS_TOOLS" - MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" - MONGODB_BINARIES: "$MONGODB_BINARIES" - UPLOAD_BUCKET: "$UPLOAD_BUCKET" - PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" - PREPARE_SHELL: | - set -o errexit - set -o xtrace - export DRIVERS_TOOLS="$DRIVERS_TOOLS" - export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" - export MONGODB_BINARIES="$MONGODB_BINARIES" - export UPLOAD_BUCKET="$UPLOAD_BUCKET" - export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" + cat < expansion.yml + CURRENT_VERSION: "$CURRENT_VERSION" + DRIVERS_TOOLS: "$DRIVERS_TOOLS" + MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" + MONGODB_BINARIES: "$MONGODB_BINARIES" + UPLOAD_BUCKET: "$UPLOAD_BUCKET" + PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" + PREPARE_SHELL: | + set -o errexit + set -o xtrace + export DRIVERS_TOOLS="$DRIVERS_TOOLS" + export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" + export MONGODB_BINARIES="$MONGODB_BINARIES" + export UPLOAD_BUCKET="$UPLOAD_BUCKET" + export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" - export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" - export PATH="$MONGODB_BINARIES:$PATH" - export PROJECT="${project}" + export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" + export PATH="$MONGODB_BINARIES:$PATH" + export PROJECT="${project}" - export MONGODB_VERSION=${VERSION} - export TOPOLOGY=${TOPOLOGY} - export SINGLE_MONGOS=${SINGLE_MONGOS} - export AUTH=${AUTH} - export SSL=${SSL} - export APP_TESTS=${APP_TESTS} - export DOCKER_DISTRO=${DOCKER_DISTRO} - EOT - # See what we've done - cat expansion.yml + export MONGODB_VERSION="${VERSION}" + export TOPOLOGY="${TOPOLOGY}" + export SINGLE_MONGOS="${SINGLE_MONGOS}" + export AUTH="${AUTH}" + export SSL="${SSL}" + export APP_TESTS="${APP_TESTS}" + export DOCKER_DISTRO="${DOCKER_DISTRO}" + export RVM_RUBY="${RVM_RUBY}" + export RAILS="${RAILS}" + export DRIVER="${DRIVER}" + export I18N="${I18N}" + export TEST_I18N_FALLBACKS="${TEST_I18N_FALLBACKS}" + export FLE="${FLE}" + EOT + # See what we've done + cat expansion.yml # Load the expansion file to make an evergreen variable with the current unique version - command: expansions.update @@ -240,7 +246,7 @@ functions: ${PREPARE_SHELL} env \ MONGODB_URI="${MONGODB_URI}" \ - TOPOLOGY=${TOPOLOGY} \ + TOPOLOGY="${TOPOLOGY}" \ RVM_RUBY="${RVM_RUBY}" \ RAILS="${RAILS}" \ DRIVER="${DRIVER}" \ @@ -339,4 +345,5 @@ tasks: shell: "bash" script: | ${PREPARE_SHELL} - MONGODB_URI="${MONGODB_URI}" .evergreen/run-tests-atlas-full.sh + MONGODB_URI="${MONGODB_URI}" \ + .evergreen/run-tests-atlas-full.sh diff --git a/.evergreen/config/variants.yml.erb b/.evergreen/config/variants.yml.erb index cbec69fd10..9348e6b05b 100644 --- a/.evergreen/config/variants.yml.erb +++ b/.evergreen/config/variants.yml.erb @@ -249,7 +249,9 @@ buildvariants: - matrix_name: atlas-full matrix_spec: ruby: ruby-3.2 - os: ubuntu-22.04 + os: actual-ubuntu-22.04 + auth: auth + ssl: ssl display_name: "Atlas (Full)" tasks: - name: testatlas_task_group diff --git a/.evergreen/run-tests-atlas-full.sh b/.evergreen/run-tests-atlas-full.sh old mode 100644 new mode 100755 index c36fc64c44..f3114b8168 --- a/.evergreen/run-tests-atlas-full.sh +++ b/.evergreen/run-tests-atlas-full.sh @@ -10,7 +10,8 @@ set_env_vars set_env_python set_env_ruby -bundle_install +export BUNDLE_GEMFILE=gemfiles/driver_master.gemfile +bundle install ATLAS_URI=$MONGODB_URI \ EXAMPLE_TIMEOUT=600 \ diff --git a/lib/mongoid/search_indexable.rb b/lib/mongoid/search_indexable.rb index 5ff876a45a..8f4abe0dfa 100644 --- a/lib/mongoid/search_indexable.rb +++ b/lib/mongoid/search_indexable.rb @@ -65,14 +65,14 @@ def remove_search_indexes # search_index :name_of_index, { ... } # end # - # @param [ Symbol | String ] name_or_defn Either the name of the index to + # @param [ Symbol | String | Hash ] name_or_defn Either the name of the index to # define, or the index definition. # @param [ Hash ] defn The search index definition. def search_index(name_or_defn, defn = nil) name = name_or_defn name, defn = nil, name if name.is_a?(Hash) - spec = { definition: defn }.tap { |s| s[:name] = name if name } + spec = { definition: defn }.tap { |s| s[:name] = name.to_s if name } search_index_specs.push(spec) end end diff --git a/spec/lite_spec_helper.rb b/spec/lite_spec_helper.rb index 19041be61f..cb4f6b617a 100644 --- a/spec/lite_spec_helper.rb +++ b/spec/lite_spec_helper.rb @@ -51,6 +51,28 @@ TimeoutInterrupt = Timeout end +STANDARD_TIMEOUTS = { + app: 500, # App tests under JRuby take a REALLY long time (over 5 minutes per test). + default: 30, +}.freeze + +def timeout_type + if ENV['EXAMPLE_TIMEOUT'].to_i > 0 + :custom + elsif SpecConfig.instance.app_tests? + :app + else + :default + end +end + +def example_timeout_seconds + STANDARD_TIMEOUTS.fetch( + timeout_type, + (ENV['EXAMPLE_TIMEOUT'] || STANDARD_TIMEOUTS[:default]).to_i + ) +end + RSpec.configure do |config| config.expect_with(:rspec) do |c| c.syntax = [:should, :expect] @@ -61,17 +83,8 @@ end if SpecConfig.instance.ci? && !%w(1 true yes).include?(ENV['INTERACTIVE']&.downcase) - timeout = if SpecConfig.instance.app_tests? - # App tests under JRuby take a REALLY long time (over 5 minutes per test). - 500 - else - # Allow a max of 30 seconds per test. - # Tests should take under 10 seconds ideally but it seems - # we have some that run for more than 10 seconds in CI. - 30 - end config.around(:each) do |example| - TimeoutInterrupt.timeout(timeout) do + TimeoutInterrupt.timeout(example_timeout_seconds) do example.run end end diff --git a/spec/mongoid/search_indexable_spec.rb b/spec/mongoid/search_indexable_spec.rb index a76f39bd3e..a1514c8c03 100644 --- a/spec/mongoid/search_indexable_spec.rb +++ b/spec/mongoid/search_indexable_spec.rb @@ -7,6 +7,7 @@ class SearchIndexHelper def initialize(model) @model = model + model.collection.drop model.collection.create end @@ -60,19 +61,22 @@ def filter_results(result, names) end end -class SearchablePerson - include Mongoid::Document - - search_index mappings: { dynamic: false } - search_index :with_dynamic_mappings, mappings: { dynamic: true } -end - describe Mongoid::SearchIndexable do before do skip "#{described_class} requires at Atlas environment (set ATLAS_URI)" if ENV['ATLAS_URI'].nil? end - let(:helper) { SearchIndexHelper.new(Person) } + let(:model) do + Class.new do + include Mongoid::Document + store_in collection: BSON::ObjectId.new.to_s + + search_index mappings: { dynamic: false } + search_index :with_dynamic_mappings, mappings: { dynamic: true } + end + end + + let(:helper) { SearchIndexHelper.new(model) } describe '.search_index_specs' do context 'when no search indexes have been defined' do @@ -83,18 +87,19 @@ class SearchablePerson context 'when search indexes have been defined' do it 'has search index specs' do - expect(SearchablePerson.search_index_specs).to be == [ - { definition: { dynamic: false } }, - { name: :with_dynamic_mappings, definition: { dynamic: true } } + expect(model.search_index_specs).to be == [ + { definition: { mappings: { dynamic: false } } }, + { name: 'with_dynamic_mappings', definition: { mappings: { dynamic: true } } } ] end end end context 'when needing to first create search indexes' do - let(:requested_definitions) { SearchablePerson.search_index_specs.map { |spec| spec[:definition] } } - let(:index_names) { SearchablePerson.create_search_indexes } - let(:actual_definitions) { helper.wait_for(*index_names) } + let(:requested_definitions) { model.search_index_specs.map { |spec| spec[:definition].with_indifferent_access } } + let(:index_names) { model.create_search_indexes } + let(:actual_indexes) { helper.wait_for(*index_names) } + let(:actual_definitions) { actual_indexes.map { |i| i['latestDefinition'] } } describe '.create_search_indexes' do it 'creates the indexes' do @@ -103,9 +108,9 @@ class SearchablePerson end describe '.search_indexes' do - before { actual_definitions } # wait for the indices to be created + before { actual_indexes } # wait for the indices to be created - let(:queried_definitions) { SearchablePerson.search_indexes.map { |i| i['latestDefinition'] } } + let(:queried_definitions) { model.search_indexes.map { |i| i['latestDefinition'] } } it 'queries the available search indexes' do expect(queried_definitions).to be == requested_definitions @@ -113,27 +118,27 @@ class SearchablePerson end describe '.remove_search_index' do - let(:target_index) { actual_definitions.first } + let(:target_index) { actual_indexes.first } before do - SearchablePerson.remove_search_index id: target_index['id'] + model.remove_search_index id: target_index['id'] helper.wait_for_absense_of target_index['name'] end it 'removes the requested index' do - expect(SearchablePerson.search_indexes(id: target_index['id'])).to be_empty + expect(model.search_indexes(id: target_index['id'])).to be_empty end end describe '.remove_search_indexes' do before do - actual_definitions # wait for the indexes to be created - Person.remove_search_indexes - helper.wait_for_absense_of(actual_definitions.map { |i| i['name'] }) + actual_indexes # wait for the indexes to be created + model.remove_search_indexes + helper.wait_for_absense_of(actual_indexes.map { |i| i['name'] }) end it 'removes the indexes' do - expect(SearchablePerson.search_indexes).to be_empty + expect(model.search_indexes).to be_empty end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ba0f61ac4f..9da2629fb2 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -51,10 +51,13 @@ def database_id_alt require 'support/constraints' require 'support/crypt' +use_ssl = %w[ ssl 1 true ].include?(ENV['SSL']) +ssl_options = { ssl: use_ssl }.freeze + # Give MongoDB servers time to start up in CI environments if SpecConfig.instance.ci? starting = true - client = Mongo::Client.new(SpecConfig.instance.addresses) + client = Mongo::Client.new(SpecConfig.instance.addresses, ssl_options) while starting begin client.command(ping: 1) @@ -71,15 +74,15 @@ def database_id_alt default: { database: database_id, hosts: SpecConfig.instance.addresses, - options: { + options: ssl_options.merge( server_selection_timeout: 3.42, wait_queue_timeout: 1, max_pool_size: 5, heartbeat_frequency: 180, - user: MONGOID_ROOT_USER.name, - password: MONGOID_ROOT_USER.password, - auth_source: Mongo::Database::ADMIN, - } + user: SpecConfig.instance.uri.client_options[:user] || MONGOID_ROOT_USER.name, + password: SpecConfig.instance.uri.client_options[:password] || MONGOID_ROOT_USER.password, + auth_source: Mongo::Database::ADMIN + ) } }, options: { @@ -121,17 +124,19 @@ class Query require "i18n/backend/fallbacks" end -# The user must be created before any of the tests are loaded, until -# https://jira.mongodb.org/browse/MONGOID-4827 is implemented. -client = Mongo::Client.new(SpecConfig.instance.addresses, server_selection_timeout: 3.03) -begin - # Create the root user administrator as the first user to be added to the - # database. This user will need to be authenticated in order to add any - # more users to any other databases. - client.database.users.create(MONGOID_ROOT_USER) -rescue Mongo::Error::OperationFailure => e -ensure - client.close +unless SpecConfig.instance.atlas? + # The user must be created before any of the tests are loaded, until + # https://jira.mongodb.org/browse/MONGOID-4827 is implemented. + client = Mongo::Client.new(SpecConfig.instance.addresses, server_selection_timeout: 3.03) + begin + # Create the root user administrator as the first user to be added to the + # database. This user will need to be authenticated in order to add any + # more users to any other databases. + client.database.users.create(MONGOID_ROOT_USER) + rescue Mongo::Error::OperationFailure => e + ensure + client.close + end end RSpec.configure do |config| diff --git a/spec/support/spec_config.rb b/spec/support/spec_config.rb index 12ebcfc66f..c4000f0d0c 100644 --- a/spec/support/spec_config.rb +++ b/spec/support/spec_config.rb @@ -17,8 +17,8 @@ def initialize STDERR.puts "Please consider providing the correct uri via MONGODB_URI environment variable." @uri_str = DEFAULT_MONGODB_URI end - - @uri = Mongo::URI.new(@uri_str) + + @uri = Mongo::URI.get(@uri_str) end attr_reader :uri_str @@ -56,6 +56,10 @@ def ci? !!ENV['CI'] end + def atlas? + !!ENV['ATLAS_URI'] + end + def rails_version v = ENV['RAILS'] if v == '' From b2cf976b96e80912bd6ead9dfee074a6602acc72 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Tue, 19 Sep 2023 14:57:09 -0600 Subject: [PATCH 08/13] rubocop --- spec/mongoid/search_indexable_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/mongoid/search_indexable_spec.rb b/spec/mongoid/search_indexable_spec.rb index a1514c8c03..22115623bc 100644 --- a/spec/mongoid/search_indexable_spec.rb +++ b/spec/mongoid/search_indexable_spec.rb @@ -61,6 +61,7 @@ def filter_results(result, names) end end +# rubocop:disable RSpec/MultipleMemoizedHelpers describe Mongoid::SearchIndexable do before do skip "#{described_class} requires at Atlas environment (set ATLAS_URI)" if ENV['ATLAS_URI'].nil? @@ -143,3 +144,4 @@ def filter_results(result, names) end end end +# rubocop:enable RSpec/MultipleMemoizedHelpers From a25d212a6958795770dbc4a4b2f2736ecc095fd9 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Fri, 22 Sep 2023 08:48:00 -0600 Subject: [PATCH 09/13] finish specs for atlas index management --- lib/mongoid/railties/database.rake | 5 ++ lib/mongoid/search_indexable.rb | 100 +++++++++++++++++++++-- lib/mongoid/tasks/database.rake | 11 +++ lib/mongoid/tasks/database.rb | 56 +++++++++++++ lib/mongoid/utils.rb | 11 +++ spec/lite_spec_helper.rb | 12 +++ spec/mongoid/tasks/database_rake_spec.rb | 70 ++++++++++++++++ spec/mongoid/tasks/database_spec.rb | 60 +++++++++++++- 8 files changed, 319 insertions(+), 6 deletions(-) diff --git a/lib/mongoid/railties/database.rake b/lib/mongoid/railties/database.rake index d839602c29..5847b265ae 100644 --- a/lib/mongoid/railties/database.rake +++ b/lib/mongoid/railties/database.rake @@ -71,6 +71,11 @@ namespace :db do task :create_indexes => "mongoid:create_indexes" end + unless Rake::Task.task_defined?("db:create_search_indexes") + desc "Create search indexes specified in Mongoid models" + task :create_search_indexes => "mongoid:create_search_indexes" + end + unless Rake::Task.task_defined?("db:remove_indexes") desc "Remove indexes specified in Mongoid models" task :remove_indexes => "mongoid:remove_indexes" diff --git a/lib/mongoid/search_indexable.rb b/lib/mongoid/search_indexable.rb index 8f4abe0dfa..80a0e44fae 100644 --- a/lib/mongoid/search_indexable.rb +++ b/lib/mongoid/search_indexable.rb @@ -6,6 +6,50 @@ module Mongoid module SearchIndexable extend ActiveSupport::Concern + # Represents the status of the indexes returned by a search_indexes + # call. + # + # @api private + class Status + # @return [ Array ] the raw index documents + attr_reader :indexes + + # Create a new Status object. + # + # @param [ Array ] indexes the raw index documents + def initialize(indexes) + @indexes = indexes + end + + # Returns the subset of indexes that have status == 'READY' + # + # @return [ Array ] index documents for "ready" indices + def ready + indexes.select { |i| i['status'] == 'READY' } + end + + # Returns the subset of indexes that have status == 'PENDING' + # + # @return [ Array ] index documents for "pending" indices + def pending + indexes.select { |i| i['status'] == 'PENDING' } + end + + # Returns the subset of indexes that are marked 'queryable' + # + # @return [ Array ] index documents for 'queryable' indices + def queryable + indexes.select { |i| i['queryable'] } + end + + # Returns true if all the given indexes are 'ready' and 'queryable'. + # + # @return [ true | false ] ready status of all indexes + def ready? + indexes.all? { |i| i['status'] == 'READY' && i['queryable'] } + end + end + included do cattr_accessor :search_index_specs self.search_index_specs = [] @@ -24,10 +68,45 @@ def create_search_indexes collection.search_indexes.create_many(search_index_specs) end + # Waits for the named search indexes to be created. + # + # @param [ Integer ] interval the number of seconds to wait before + # polling again (only used when a progress callback is given). + # @param [ Proc ] progress an optional callback for reporting the + # status of the new indexes. + # + # @yield [ SearchIndexable::Status ] the status object + def wait_for_search_indexes(names, interval: 5, &progress) + loop do + status = Status.new(get_indexes(names)) + progress.call(status) + break if status.ready? + sleep interval + end + end + + # A convenience method for querying the search indexes available on the + # current model's collection. + # + # @param [ Hash ] options the options to pass through to the search + # index query. + # + # @option options [ String ] :id The id of the specific index to query (optional) + # @option options [ String ] :name The name of the specific index to query (optional) + # @option options [ Hash ] :aggregate The options hash to pass to the + # aggregate command (optional) + # + # @return [ self ] the model class def search_indexes(options = {}) collection.search_indexes(options) + self end + # Removes the search index specified by the given name or id. Either + # name OR id must be given, but not both. + # + # @param [ String | nil ] name the name of the index to remove + # @param [ String | nil ] id the id of the index to remove def remove_search_index(name: nil, id: nil) logger.info( "MONGOID: Removing search index '#{name || id}' " \ @@ -37,17 +116,17 @@ def remove_search_index(name: nil, id: nil) collection.search_indexes.drop_one(name: name, id: id) end - # Request the removal of all registered search indices. Note + # Request the removal of all registered search indexes. Note # that the search indexes are removed asynchronously, and may take # several minutes to be fully deleted. # - # @note It would be nice if this could remove ONLY the search indices + # @note It would be nice if this could remove ONLY the search indexes # that have been declared on the model, but because the model may not # name the index, we can't guarantee that we'll know the name or id of - # the corresponding indices. It is not unreasonable to assume, though, + # the corresponding indexes. It is not unreasonable to assume, though, # that the intention is for the model to declare, one-to-one, all - # desired search indices, so removing all search indices ought to suffice. - # If a specific index or set of indices needs to be removed instead, + # desired search indexes, so removing all search indexes ought to suffice. + # If a specific index or set of indexes needs to be removed instead, # consider using search_indexes.each with remove_search_index. def remove_search_indexes search_indexes.each do |spec| @@ -75,6 +154,17 @@ def search_index(name_or_defn, defn = nil) spec = { definition: defn }.tap { |s| s[:name] = name.to_s if name } search_index_specs.push(spec) end + + private + + # Retrieves the index records for the indexes with the given names. + # + # @param [ Array ] names the index names to query + # + # @return [ Array ] the raw index documents + def get_indexes(names) + collection.search_indexes.select { |i| names.include?(i['name']) } + end end end end diff --git a/lib/mongoid/tasks/database.rake b/lib/mongoid/tasks/database.rake index 36a01de776..76786cfab1 100644 --- a/lib/mongoid/tasks/database.rake +++ b/lib/mongoid/tasks/database.rake @@ -17,6 +17,12 @@ namespace :db do ::Mongoid::Tasks::Database.create_indexes end + desc "Create search indexes specified in Mongoid models" + task :create_search_indexes => [:environment, :load_models] do + wait = Mongoid::Utils.truthy_string?(ENV['WAIT_FOR_SEARCH_INDEXES'] || '1') + ::Mongoid::Tasks::Database.create_search_indexes(wait: wait) + end + desc "Remove indexes that exist in the database but are not specified in Mongoid models" task :remove_undefined_indexes => [:environment, :load_models] do ::Mongoid::Tasks::Database.remove_undefined_indexes @@ -27,6 +33,11 @@ namespace :db do ::Mongoid::Tasks::Database.remove_indexes end + desc "Remove search indexes specified in Mongoid models" + task :remove_search_indexes => [:environment, :load_models] do + ::Mongoid::Tasks::Database.remove_search_indexes + end + desc "Shard collections with shard keys specified in Mongoid models" task :shard_collections => [:environment, :load_models] do ::Mongoid::Tasks::Database.shard_collections diff --git a/lib/mongoid/tasks/database.rb b/lib/mongoid/tasks/database.rb index a779b7cfb5..aa3edbcc79 100644 --- a/lib/mongoid/tasks/database.rb +++ b/lib/mongoid/tasks/database.rb @@ -56,6 +56,26 @@ def create_indexes(models = ::Mongoid.models) end.compact end + # Submit requests for the search indexes to be created. This will happen + # asynchronously. If "wait" is true, the method will block while it + # waits for the indexes to be created. + # + # @param [ Array ] models the models to build search + # indexes for. + # @param [ true | false ] wait whether to wait for the indexes to be + # built. + def create_search_indexes(models = ::Mongoid.models, wait: true) + searchable = models.select { |m| m.search_index_specs.any? } + + # queue up the search index creation requests + index_names_by_model = searchable.each_with_object({}) do |model, obj| + logger.info("MONGOID: Creating search indexes on #{model}...") + obj[model] = model.create_search_indexes + end + + wait_for_search_indexes(index_names_by_model) if wait + end + # Return the list of indexes by model that exist in the database but aren't # specified on the models. # @@ -128,6 +148,17 @@ def remove_indexes(models = ::Mongoid.models) end.compact end + # Remove all search indexes from the given models. + # + # @params [ Array ] models the models to remove + # search indexes from. + def remove_search_indexes(models = ::Mongoid.models) + models.each do |model| + next if model.embedded? + model.remove_search_indexes + end + end + # Shard collections for models that declare shard keys. # # Returns the model classes that have had their collections sharded, @@ -216,6 +247,31 @@ def shard_collections(models = ::Mongoid.models) def logger Mongoid.logger end + + # Waits for the search indexes to be built on the given models. + # + # @param [ Hash> ] models a mapping of + # index names for each model + def wait_for_search_indexes(models) + logger.info('MONGOID: Waiting for search indexes to be created') + logger.info('MONGOID: Press ctrl-c to skip the wait and let the indexes be created in the background') + + models.each do |model, names| + model.wait_for_search_indexes(names) do |status| + if status.ready? + puts + logger.info("MONGOID: Search indexes on #{model} are READY") + else + print '.' + $stdout.flush + end + end + end + rescue Interrupt + # ignore ctrl-C here; we assume it is meant only to skip + # the wait, and that subsequent tasks ought to continue. + logger.info('MONGOID: Skipping the wait for search indexes; they will be created in the background') + end end end end diff --git a/lib/mongoid/utils.rb b/lib/mongoid/utils.rb index b3ac761410..ec6c885516 100644 --- a/lib/mongoid/utils.rb +++ b/lib/mongoid/utils.rb @@ -37,5 +37,16 @@ def placeholder?(value) def monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC) end + + # Returns true if the string is any of the following values: "1", + # "yes", "true", "on". Anything else is assumed to be false. Case is + # ignored, as are leading or trailing spaces. + # + # @param [ String ] string the string value to consider + # + # @return [ true | false ] + def truthy_string?(string) + %w( 1 yes true on ).include?(string.strip.downcase) + end end end diff --git a/spec/lite_spec_helper.rb b/spec/lite_spec_helper.rb index cb4f6b617a..d3868211e0 100644 --- a/spec/lite_spec_helper.rb +++ b/spec/lite_spec_helper.rb @@ -90,6 +90,18 @@ def example_timeout_seconds end end + def local_env(env = nil, &block) + around do |example| + env ||= block.call + saved_env = ENV.to_h + ENV.update(env) + + example.run + ensure + ENV.replace(saved_env) if saved_env + end + end + config.extend(Mrss::LiteConstraints) end diff --git a/spec/mongoid/tasks/database_rake_spec.rb b/spec/mongoid/tasks/database_rake_spec.rb index 9340b9d484..c61adba2c5 100644 --- a/spec/mongoid/tasks/database_rake_spec.rb +++ b/spec/mongoid/tasks/database_rake_spec.rb @@ -31,6 +31,34 @@ end end + shared_examples_for 'create_search_indexes' do + [ nil, *%w( 1 true yes on ) ].each do |truthy| + context "when WAIT_FOR_SEARCH_INDEXES is #{truthy.inspect}" do + local_env 'WAIT_FOR_SEARCH_INDEXES' => truthy + + it 'receives create_search_indexes with wait: true' do + expect(Mongoid::Tasks::Database) + .to receive(:create_search_indexes) + .with(wait: true) + task.invoke + end + end + end + + %w( 0 false no off bogus ).each do |falsey| + context "when WAIT_FOR_SEARCH_INDEXES is #{falsey.inspect}" do + local_env 'WAIT_FOR_SEARCH_INDEXES' => falsey + + it 'receives create_search_indexes with wait: false' do + expect(Mongoid::Tasks::Database) + .to receive(:create_search_indexes) + .with(wait: false) + task.invoke + end + end + end + end + shared_examples_for "create_collections" do it "receives create_collections" do @@ -203,6 +231,26 @@ end end +describe 'db:mongoid:create_search_indexes' do + include_context 'rake task' + + it_behaves_like 'create_search_indexes' + + it 'calls load_models' do + expect(task.prerequisites).to include('load_models') + end + + it 'calls environment' do + expect(task.prerequisites).to include('environment') + end + + context 'when using rails task' do + include_context 'rails rake task' + + it_behaves_like 'create_search_indexes' + end +end + describe "db:mongoid:create_collections" do include_context "rake task" @@ -287,6 +335,28 @@ end end +describe 'db:mongoid:remove_search_indexes' do + include_context 'rake task' + + it 'receives remove_search_indexes' do + expect(Mongoid::Tasks::Database).to receive(:remove_search_indexes) + task.invoke + end + + it 'calls environment' do + expect(task.prerequisites).to include('environment') + end + + context 'when using rails task' do + include_context 'rails rake task' + + it 'receives remove_search_indexes' do + expect(Mongoid::Tasks::Database).to receive(:remove_search_indexes) + task.invoke + end + end +end + describe "db:mongoid:drop" do include_context "rake task" diff --git a/spec/mongoid/tasks/database_spec.rb b/spec/mongoid/tasks/database_spec.rb index d5ef01960b..91a0ecc9a7 100644 --- a/spec/mongoid/tasks/database_spec.rb +++ b/spec/mongoid/tasks/database_spec.rb @@ -3,7 +3,7 @@ require "spec_helper" -describe "Mongoid::Tasks::Database" do +describe Mongoid::Tasks::Database do before(:all) do module DatabaseSpec @@ -213,6 +213,64 @@ class Note end end + describe '.create_search_indexes' do + let(:searchable_model) do + Class.new do + include Mongoid::Document + store_in collection: BSON::ObjectId.new.to_s + + search_index mappings: { dynamic: true } + end + end + + let(:index_names) { %w[ name1 name2 ] } + let(:searchable_model_spy) do + class_spy(searchable_model, + create_search_indexes: index_names, + search_index_specs: [ { mappings: { dynamic: true } } ]) + end + + context 'when wait is true' do + before do + allow(described_class).to receive(:wait_for_search_indexes) + described_class.create_search_indexes([ searchable_model_spy ], wait: true) + end + + it 'invokes both create_search_indexes and wait_for_search_indexes' do + expect(searchable_model_spy).to have_received(:create_search_indexes) + expect(described_class).to have_received(:wait_for_search_indexes).with(searchable_model_spy => index_names) + end + end + + context 'when wait is false' do + before do + allow(described_class).to receive(:wait_for_search_indexes) + described_class.create_search_indexes([ searchable_model_spy ], wait: false) + end + + it 'invokes only create_search_indexes' do + expect(searchable_model_spy).to have_received(:create_search_indexes) + expect(described_class).not_to have_received(:wait_for_search_indexes) + end + end + end + + describe '.remove_search_indexes' do + before do + models.each do |model| + allow(model).to receive(:remove_search_indexes) unless model.embedded? + end + + described_class.remove_search_indexes(models) + end + + it 'calls remove_search_indexes on all non-embedded models' do + models.each do |model| + expect(model).to have_received(:remove_search_indexes) unless model.embedded? + end + end + end + describe ".undefined_indexes" do before(:each) do From 000c73758904181f412a9127ed2e39fbf7e93b85 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Fri, 22 Sep 2023 10:28:51 -0600 Subject: [PATCH 10/13] docs for the search index stuff --- docs/reference/indexes.txt | 61 ++++++++++++++++++++++++++++++ docs/release-notes/mongoid-9.0.txt | 47 +++++++++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/docs/reference/indexes.txt b/docs/reference/indexes.txt index 6602dd0ff6..7a5d11a475 100644 --- a/docs/reference/indexes.txt +++ b/docs/reference/indexes.txt @@ -131,6 +131,28 @@ The ``background`` option has `no effect as of MongoDB 4.2 `_. +Specifying Search Indexes on MongoDB Atlas +========================================== + +If your application is connected to MongoDB Atlas, you can declare and manage +search indexes on your models. (This feature is only available on MongoDB +Atlas.) + +To declare a search index, use the ``search_index`` macro in your model: + +.. code-block:: ruby + + class Message + include Mongoid::Document + + search_index { ... } + search_index :named_index, { ... } + end + +Search indexes may be given an explicit name; this is necessary if you have +more than one search index on a model. + + Index Management Rake Tasks =========================== @@ -162,6 +184,45 @@ in Rails console: # Remove indexes for Model Model.remove_indexes +Managing Search Indexes on MongoDB Atlas +---------------------------------------- + +If you have defined search indexes on your model, there are rake tasks available +for creating and removing those search indexes: + +.. code-block:: bash + + $ rake db:mongoid:create_search_indexes + $ rake db:mongoid:remove_search_indexes + +By default, creating search indexes will wait for the indexes to be created, +which can take quite some time. If you want to simply let the database create +the indexes in the background, you can set the ``WAIT_FOR_SEARCH_INDEXES`` +environment variable to 0, like this: + +.. code-block:: bash + + $ rake WAIT_FOR_SEARCH_INDEXES=0 db:mongoid:create_search_indexes + +Note that the task for removing search indexes will remove all search indexes +from all models, and should be used with caution. + +You can also add and remove search indexes for a single model by invoking the +following in a Rails console: + +.. code-block:: ruby + + # Create all defined search indexes on the model; this will return + # immediately and the indexes will be created in the background. + Model.create_search_indexes + + # Remove all search indexes from the model + Model.remove_search_indexes + + # Enumerate all search indexes on the model + Model.search_indexes.each { |index| ... } + + Telling Mongoid Where to Look For Models ---------------------------------------- diff --git a/docs/release-notes/mongoid-9.0.txt b/docs/release-notes/mongoid-9.0.txt index b47977010a..1c6642e60d 100644 --- a/docs/release-notes/mongoid-9.0.txt +++ b/docs/release-notes/mongoid-9.0.txt @@ -338,6 +338,53 @@ Mongoid to allow literal BSON::Decimal128 fields: BSON 5 and later. BSON 4 and earlier ignore the setting entirely. +Search Index Management with MongoDB Atlas +------------------------------------------ + +When connected to MongoDB Atlas, Mongoid now supports creating and removing +search indexes. You may do so programmatically, via the Mongoid::SearchIndexable +API: + +.. code-block:: ruby + + class SearchablePerson + include Mongoid::Document + + search_index { ... } # define the search index here + end + + # create the declared search indexes; this returns immediately, but the + # search indexes may take several minutes before they are available. + SearchablePerson.create_search_indexes + + # query the available search indexes + SearchablePerson.search_indexes.each do |index| + # ... + end + + # remove all search indexes from the model's collection + SearchablePerson.remove_search_indexes + +If you are not connected to MongoDB Atlas, the search index definitions are +ignored. Trying to create, enumerate, or remove search indexes will result in +an error. + +There are also rake tasks available, for convenience: + +.. code-block:: bash + + # create search indexes for all models; waits for indexes to be created + # and shows progress on the terminal. + $ rake mongoid:db:create_search_indexes + + # as above, but returns immediately and lets the indexes be created in the + # background + $ rake WAIT_FOR_SEARCH_INDEXES=0 mongoid:db:create_search_indexes + + # removes search indexes from all models + $ rake mongoid:db:remove_search_indexes + + Bug Fixes and Improvements -------------------------- From 262306f00bebaf45862239095891ce0ed84a04d0 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Fri, 22 Sep 2023 10:31:52 -0600 Subject: [PATCH 11/13] rubocop --- lib/mongoid/search_indexable.rb | 7 +++---- lib/mongoid/utils.rb | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/mongoid/search_indexable.rb b/lib/mongoid/search_indexable.rb index 80a0e44fae..420d6f9147 100644 --- a/lib/mongoid/search_indexable.rb +++ b/lib/mongoid/search_indexable.rb @@ -72,15 +72,14 @@ def create_search_indexes # # @param [ Integer ] interval the number of seconds to wait before # polling again (only used when a progress callback is given). - # @param [ Proc ] progress an optional callback for reporting the - # status of the new indexes. # # @yield [ SearchIndexable::Status ] the status object - def wait_for_search_indexes(names, interval: 5, &progress) + def wait_for_search_indexes(names, interval: 5) loop do status = Status.new(get_indexes(names)) - progress.call(status) + yield status if block_given? break if status.ready? + sleep interval end end diff --git a/lib/mongoid/utils.rb b/lib/mongoid/utils.rb index ec6c885516..20f8dc2395 100644 --- a/lib/mongoid/utils.rb +++ b/lib/mongoid/utils.rb @@ -46,7 +46,7 @@ def monotonic_time # # @return [ true | false ] def truthy_string?(string) - %w( 1 yes true on ).include?(string.strip.downcase) + %w[ 1 yes true on ].include?(string.strip.downcase) end end end From e89213b141e3067e180d24cb0ab809f3ae3ce586 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Fri, 22 Sep 2023 15:16:37 -0600 Subject: [PATCH 12/13] return the correct value --- lib/mongoid/search_indexable.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/mongoid/search_indexable.rb b/lib/mongoid/search_indexable.rb index 420d6f9147..e2fbd1824f 100644 --- a/lib/mongoid/search_indexable.rb +++ b/lib/mongoid/search_indexable.rb @@ -94,11 +94,8 @@ def wait_for_search_indexes(names, interval: 5) # @option options [ String ] :name The name of the specific index to query (optional) # @option options [ Hash ] :aggregate The options hash to pass to the # aggregate command (optional) - # - # @return [ self ] the model class def search_indexes(options = {}) collection.search_indexes(options) - self end # Removes the search index specified by the given name or id. Either From dfd784f7ddd251a5532a053d79f8427c79bf9c32 Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Wed, 27 Sep 2023 14:56:26 -0600 Subject: [PATCH 13/13] add missing documentation --- lib/mongoid/search_indexable.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/mongoid/search_indexable.rb b/lib/mongoid/search_indexable.rb index e2fbd1824f..1342539223 100644 --- a/lib/mongoid/search_indexable.rb +++ b/lib/mongoid/search_indexable.rb @@ -70,6 +70,7 @@ def create_search_indexes # Waits for the named search indexes to be created. # + # @param [ Array ] names the list of index names to wait for # @param [ Integer ] interval the number of seconds to wait before # polling again (only used when a progress callback is given). #