From 2eb8cf2f6ef2824c7c136fa298ae3b86eeb49c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Wed, 13 Nov 2024 17:33:28 +0100 Subject: [PATCH 1/8] PHPORM-28 Add Scout engined to index into MongoDB Search Move scout engine declaration to the main MongoDBServiceProvider --- .github/workflows/build-ci.yml | 3 + composer.json | 1 + docker-compose.yml | 6 +- phpstan-baseline.neon | 5 + src/MongoDBServiceProvider.php | 21 + src/Scout/ScoutEngine.php | 542 +++++++++++++++++ tests/AtlasSearchTest.php | 7 +- tests/ModelTest.php | 3 +- tests/Models/SchemaVersion.php | 4 +- tests/Scout/Models/ScoutUser.php | 43 ++ .../Models/SearchableInSameNamespace.php | 30 + tests/Scout/Models/SearchableModel.php | 50 ++ tests/Scout/ScoutEngineTest.php | 548 ++++++++++++++++++ tests/Scout/ScoutIntegrationTest.php | 242 ++++++++ 14 files changed, 1497 insertions(+), 8 deletions(-) create mode 100644 src/Scout/ScoutEngine.php create mode 100644 tests/Scout/Models/ScoutUser.php create mode 100644 tests/Scout/Models/SearchableInSameNamespace.php create mode 100644 tests/Scout/Models/SearchableModel.php create mode 100644 tests/Scout/ScoutEngineTest.php create mode 100644 tests/Scout/ScoutIntegrationTest.php diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml index 4fea1b84d..16bd213ec 100644 --- a/.github/workflows/build-ci.yml +++ b/.github/workflows/build-ci.yml @@ -65,6 +65,9 @@ jobs: until docker exec --tty mongodb mongosh 127.0.0.1:27017 --eval "db.runCommand({ ping: 1 })"; do sleep 1 done + until docker exec --tty mongodb mongosh 127.0.0.1:27017 --eval "db.createCollection('connection_test') && db.getCollection('connection_test').createSearchIndex({mappings:{dynamic: true}})"; do + sleep 1 + done - name: "Show MongoDB server status" run: | diff --git a/composer.json b/composer.json index 68ec8bc4f..dce593ed5 100644 --- a/composer.json +++ b/composer.json @@ -35,6 +35,7 @@ }, "require-dev": { "mongodb/builder": "^0.2", + "laravel/scout": "^11", "league/flysystem-gridfs": "^3.28", "league/flysystem-read-only": "^3.0", "phpunit/phpunit": "^10.3", diff --git a/docker-compose.yml b/docker-compose.yml index f757ec3cd..fb35e0c63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.5' - services: app: tty: true @@ -16,11 +14,11 @@ services: mongodb: container_name: mongodb - image: mongo:latest + image: mongodb/mongodb-atlas-local:latest ports: - "27017:27017" healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh mongodb:27017 --quiet + test: mongosh mongodb:27017 --quiet --eval 'db.runCommand("ping").ok' interval: 10s timeout: 10s retries: 5 diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 7b34210ad..737e31f17 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -24,3 +24,8 @@ parameters: message: "#^Method Illuminate\\\\Database\\\\Schema\\\\Blueprint\\:\\:create\\(\\) invoked with 1 parameter, 0 required\\.$#" count: 1 path: src/Schema/Builder.php + + - + message: "#^Call to an undefined method Illuminate\\\\Support\\\\HigherOrderCollectionProxy\\<\\(int\\|string\\), Illuminate\\\\Database\\\\Eloquent\\\\Model\\>\\:\\:pushSoftDeleteMetadata\\(\\)\\.$#" + count: 1 + path: src/Scout/ScoutEngine.php diff --git a/src/MongoDBServiceProvider.php b/src/MongoDBServiceProvider.php index 9db2122dc..b0c085b8e 100644 --- a/src/MongoDBServiceProvider.php +++ b/src/MongoDBServiceProvider.php @@ -7,12 +7,14 @@ use Closure; use Illuminate\Cache\CacheManager; use Illuminate\Cache\Repository; +use Illuminate\Container\Container; use Illuminate\Filesystem\FilesystemAdapter; use Illuminate\Filesystem\FilesystemManager; use Illuminate\Foundation\Application; use Illuminate\Session\SessionManager; use Illuminate\Support\ServiceProvider; use InvalidArgumentException; +use Laravel\Scout\EngineManager; use League\Flysystem\Filesystem; use League\Flysystem\GridFS\GridFSAdapter; use League\Flysystem\ReadOnly\ReadOnlyFilesystemAdapter; @@ -20,6 +22,7 @@ use MongoDB\Laravel\Cache\MongoStore; use MongoDB\Laravel\Eloquent\Model; use MongoDB\Laravel\Queue\MongoConnector; +use MongoDB\Laravel\Scout\ScoutEngine; use RuntimeException; use Symfony\Component\HttpFoundation\Session\Storage\Handler\MongoDbSessionHandler; @@ -102,6 +105,7 @@ public function register() }); $this->registerFlysystemAdapter(); + $this->registerScoutEngine(); } private function registerFlysystemAdapter(): void @@ -155,4 +159,21 @@ private function registerFlysystemAdapter(): void }); }); } + + private function registerScoutEngine(): void + { + $this->app->resolving(EngineManager::class, function (EngineManager $engineManager) { + $engineManager->extend('mongodb', function (Container $app) { + $connectionName = $app->get('config')->get('scout.mongodb.connection', 'mongodb'); + $connection = $app->get('db')->connection($connectionName); + $softDelete = (bool) $app->get('config')->get('scout.soft_delete', false); + + assert($connection instanceof Connection, new InvalidArgumentException(sprintf('The connection "%s" is not a MongoDB connection.', $connectionName))); + + return new ScoutEngine($connection->getMongoDB(), $softDelete); + }); + + return $engineManager; + }); + } } diff --git a/src/Scout/ScoutEngine.php b/src/Scout/ScoutEngine.php new file mode 100644 index 000000000..d3dcdcba2 --- /dev/null +++ b/src/Scout/ScoutEngine.php @@ -0,0 +1,542 @@ + [ + 'dynamic' => true, + ], + ]; + + public function __construct( + private Database $database, + private bool $softDelete, + ) { + } + + /** + * Update the given model in the index. + * + * @see Engine::update() + * + * @param EloquentCollection $models + * + * @throws MongoDBRuntimeException + */ + #[Override] + public function update($models) + { + if ($models->isEmpty()) { + return; + } + + if ($this->softDelete && $this->usesSoftDelete($models)) { + $models->each->pushSoftDeleteMetadata(); + } + + $bulk = []; + foreach ($models as $model) { + $searchableData = $model->toSearchableArray(); + $searchableData = self::serialize($searchableData); + + if ($searchableData) { + unset($searchableData['_id']); + + $searchableData = array_merge($searchableData, $model->scoutMetadata()); + if (isset($searchableData['__soft_deleted'])) { + $searchableData['__soft_deleted'] = (bool) $searchableData['__soft_deleted']; + } + + $bulk[] = [ + 'updateOne' => [ + ['_id' => $model->getScoutKey()], + [ + '$setOnInsert' => ['_id' => $model->getScoutKey()], + '$set' => $searchableData, + ], + ['upsert' => true], + ], + ]; + } + } + + $this->getIndexableCollection($models)->bulkWrite($bulk); + } + + /** + * Remove the given model from the index. + * + * @see Engine::delete() + * + * @param EloquentCollection $models + */ + #[Override] + public function delete($models): void + { + assert($models instanceof Collection, new TypeError(sprintf('Argument #1 ($models) must be of type %s, %s given', Collection::class, get_debug_type($models)))); + + if ($models->isEmpty()) { + return; + } + + $collection = $this->getIndexableCollection($models); + $ids = $models->map(fn (Model $model) => $model->getScoutKey())->all(); + $collection->deleteMany(['_id' => ['$in' => $ids]]); + } + + /** + * Perform the given search on the engine. + * + * @see Engine::search() + * + * @return mixed + */ + #[Override] + public function search(Builder $builder) + { + return $this->performSearch($builder); + } + + /** + * Perform the given search on the engine. + * + * @see Engine::paginate() + * + * @param int $perPage + * @param int $page + * + * @return mixed + */ + #[Override] + public function paginate(Builder $builder, $perPage, $page) + { + assert(is_int($perPage), new TypeError(sprintf('Argument #2 ($perPage) must be of type int, %s given', get_debug_type($perPage)))); + assert(is_int($page), new TypeError(sprintf('Argument #3 ($page) must be of type int, %s given', get_debug_type($page)))); + + $builder = clone $builder; + $builder->take($perPage); + + return $this->performSearch($builder, $perPage * ($page - 1)); + } + + /** + * Perform the given search on the engine. + */ + protected function performSearch(Builder $builder, ?int $offset = null): array + { + $collection = $this->getSearchableCollection($builder->model); + + if ($builder->callback) { + $result = call_user_func( + $builder->callback, + $collection, + $builder->query, + $offset, + ); + assert($result instanceof CursorInterface, new LogicException(sprintf('The search builder closure must return a MongoDB cursor, %s returned', get_debug_type($result)))); + + return $result->toArray(); + } + + $compound = [ + 'should' => [ + [ + 'text' => [ + 'query' => $builder->query, + 'path' => ['wildcard' => '*'], + 'fuzzy' => ['maxEdits' => 2], + 'score' => ['boost' => ['value' => 5]], + ], + ], + [ + 'wildcard' => [ + 'query' => $builder->query . '*', + 'path' => ['wildcard' => '*'], + 'allowAnalyzedField' => true, + ], + ], + ], + 'minimumShouldMatch' => 1, + ]; + + // "filter" specifies conditions on exact values to match + // "mustNot" specifies conditions on exact values that must not match + // They don't contribute to the relevance score + // https://www.mongodb.com/docs/atlas/atlas-search/compound/#options + foreach ($builder->wheres as $field => $value) { + if ($field === '__soft_deleted') { + $value = (bool) $value; + } + + $compound['filter'][] = ['equals' => ['path' => $field, 'value' => $value]]; + } + + foreach ($builder->whereIns as $field => $value) { + $compound['filter'][] = ['in' => ['path' => $field, 'value' => $value]]; + } + + foreach ($builder->whereNotIns as $field => $value) { + $compound['mustNot'][] = ['in' => ['path' => $field, 'value' => $value]]; + } + + // Sort by field value only if specified + $sort = []; + foreach ($builder->orders as $order) { + $sort[$order['column']] = $order['direction'] === 'asc' ? 1 : -1; + } + + $pipeline = [ + [ + '$search' => [ + 'index' => self::INDEX_NAME, + 'compound' => $compound, + 'count' => ['type' => 'lowerBound'], + ...($sort ? ['sort' => $sort] : []), + ], + ], + [ + '$addFields' => [ + // Metadata field with the total count of documents + '__count' => '$$SEARCH_META.count.lowerBound', + ], + ], + ]; + + if ($offset) { + $pipeline[] = ['$skip' => $offset]; + } + + if ($builder->limit) { + $pipeline[] = ['$limit' => $builder->limit]; + } + + $options = [ + 'typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array'], + ]; + + return $collection->aggregate($pipeline, $options)->toArray(); + } + + /** + * Pluck and return the primary keys of the given results. + * + * @see Engine::mapIds() + * + * @param list $results + */ + #[Override] + public function mapIds($results): Collection + { + assert(is_array($results), new TypeError(sprintf('Argument #1 ($results) must be of type array, %s given', get_debug_type($results)))); + + return new Collection(array_column($results, '_id')); + } + + /** + * Map the given results to instances of the given model. + * + * @see Engine::map() + * + * @param array $results + * @param Model $model + */ + #[Override] + public function map(Builder $builder, $results, $model): Collection + { + assert(is_array($results), new TypeError(sprintf('Argument #2 ($results) must be of type array, %s given', get_debug_type($results)))); + assert($model instanceof Model, new TypeError(sprintf('Argument #3 ($model) must be of type %s, %s given', Model::class, get_debug_type($model)))); + + if (! $results) { + return $model->newCollection(); + } + + $objectIds = array_column($results, '_id'); + $objectIdPositions = array_flip($objectIds); + + return $model->getScoutModelsByIds( + $builder, + $objectIds, + )->filter(function ($model) use ($objectIdPositions) { + return array_key_exists($model->getScoutKey(), $objectIdPositions); + })->map(function ($model) use ($results, $objectIdPositions) { + $result = $results[$objectIdPositions[$model->getScoutKey()]] ?? []; + + foreach ($result as $key => $value) { + if ($key[0] === '_') { + $model->withScoutMetadata($key, $value); + } + } + + return $model; + })->sortBy(function ($model) use ($objectIdPositions) { + return $objectIdPositions[$model->getScoutKey()]; + })->values(); + } + + /** + * Get the total count from a raw result returned by the engine. + * This is an estimate if the count is larger than 1000. + * + * @see Engine::getTotalCount() + * + * @param mixed $results + */ + #[Override] + public function getTotalCount($results): int + { + if (! $results) { + return 0; + } + + return $results[0]['__count']; + } + + /** + * Flush all records from the engine. + * + * @see Engine::flush() + * + * @param Model $model + */ + #[Override] + public function flush($model): void + { + assert($model instanceof Model, new TypeError(sprintf('Argument #1 ($model) must be of type %s, %s given', Model::class, get_debug_type($model)))); + + $collection = $this->getIndexableCollection($model); + + $collection->deleteMany([]); + } + + /** + * Map the given results to instances of the given model via a lazy collection. + * + * @see Engine::lazyMap() + * + * @param Builder $builder + * @param array|Cursor $results + * @param Model $model + * + * @return LazyCollection + */ + #[Override] + public function lazyMap(Builder $builder, $results, $model): LazyCollection + { + assert($results instanceof Cursor || is_array($results), new TypeError(sprintf('Argument #2 ($results) must be of type %s|array, %s given', Cursor::class, get_debug_type($results)))); + assert($model instanceof Model, new TypeError(sprintf('Argument #3 ($model) must be of type %s, %s given', Model::class, get_debug_type($model)))); + + if (! $results) { + return LazyCollection::make($model->newCollection()); + } + + $objectIds = array_column($results, '_id'); + $objectIdPositions = array_flip($objectIds); + + return $model->queryScoutModelsByIds( + $builder, + $objectIds, + )->cursor()->filter(function ($model) use ($objectIds) { + return in_array($model->getScoutKey(), $objectIds); + })->map(function ($model) use ($results, $objectIdPositions) { + $result = $results[$objectIdPositions[$model->getScoutKey()]] ?? []; + + foreach ($result as $key => $value) { + if (substr($key, 0, 1) === '_') { + $model->withScoutMetadata($key, $value); + } + } + + return $model; + })->sortBy(function ($model) use ($objectIdPositions) { + return $objectIdPositions[$model->getScoutKey()]; + })->values(); + } + + /** + * Create the MongoDB Atlas Search index. + * + * Accepted options: + * - wait: bool, default true. Wait for the index to be created. + * + * @see Engine::createIndex() + * + * @param string $name Collection name + * @param array{wait?:bool} $options + */ + #[Override] + public function createIndex($name, array $options = []): void + { + assert(is_string($name), new TypeError(sprintf('Argument #1 ($name) must be of type string, %s given', get_debug_type($name)))); + + // Ensure the collection exists before creating the search index + $this->database->createCollection($name); + + $collection = $this->database->selectCollection($name); + $collection->createSearchIndex( + self::DEFAULT_DEFINITION, + ['name' => self::INDEX_NAME], + ); + + if ($options['wait'] ?? true) { + $this->wait(function () use ($collection) { + $indexes = $collection->listSearchIndexes([ + 'name' => self::INDEX_NAME, + 'typeMap' => ['root' => 'bson'], + ]); + + return $indexes->current() && $indexes->current()->status === 'READY'; + }); + } + } + + /** + * Delete a "search index", i.e. a MongoDB collection. + * + * @see Engine::deleteIndex() + */ + #[Override] + public function deleteIndex($name): void + { + assert(is_string($name), new TypeError(sprintf('Argument #1 ($name) must be of type string, %s given', get_debug_type($name)))); + + $this->database->selectCollection($name)->drop(); + } + + /** Get the MongoDB collection used to search for the provided model */ + private function getSearchableCollection(Model|EloquentCollection $model): MongoDBCollection + { + if ($model instanceof EloquentCollection) { + $model = $model->first(); + } + + assert(method_exists($model, 'searchableAs'), sprintf('Model "%s" must use "%s" trait', $model::class, Searchable::class)); + + return $this->database->selectCollection($model->searchableAs()); + } + + /** Get the MongoDB collection used to index the provided model */ + private function getIndexableCollection(Model|EloquentCollection $model): MongoDBCollection + { + if ($model instanceof EloquentCollection) { + $model = $model->first(); + } + + assert($model instanceof Model); + assert(method_exists($model, 'indexableAs'), sprintf('Model "%s" must use "%s" trait', $model::class, Searchable::class)); + + if ( + $model->getConnection() instanceof Connection + && $model->getConnection()->getDatabaseName() === $this->database->getDatabaseName() + && $model->getTable() === $model->indexableAs() + ) { + throw new LogicException(sprintf('The MongoDB Scout collection "%s.%s" must use a different collection from the collection name of the model "%s". Set the "scout.prefix" configuration or use a distinct MongoDB database', $this->database->getDatabaseName(), $model->indexableAs(), $model::class)); + } + + return $this->database->selectCollection($model->indexableAs()); + } + + private static function serialize(mixed $value): mixed + { + if ($value instanceof DateTimeInterface) { + return new UTCDateTime($value); + } + + if ($value instanceof Serializable || ! is_iterable($value)) { + return $value; + } + + // Convert Laravel Collections and other Iterators to arrays + if ($value instanceof Traversable) { + $value = iterator_to_array($value); + } + + // Recursively serialize arrays + return array_map(self::serialize(...), $value); + } + + private function usesSoftDelete(Model|EloquentCollection $model): bool + { + if ($model instanceof EloquentCollection) { + $model = $model->first(); + } + + return in_array(SoftDeletes::class, class_uses_recursive($model)); + } + + /** + * Wait for the callback to return true, use it for asynchronous + * Atlas Search index management operations. + */ + private function wait(Closure $callback): void + { + // Fallback to time() if hrtime() is not supported + $timeout = (hrtime()[0] ?? time()) + self::WAIT_TIMEOUT_SEC; + while ((hrtime()[0] ?? time()) < $timeout) { + if ($callback()) { + return; + } + + sleep(1); + } + + throw new MongoDBRuntimeException('Atlas search index operation time out'); + } +} diff --git a/tests/AtlasSearchTest.php b/tests/AtlasSearchTest.php index c9cd2d5e3..c6f5a9a65 100644 --- a/tests/AtlasSearchTest.php +++ b/tests/AtlasSearchTest.php @@ -14,10 +14,12 @@ use function array_map; use function assert; + use function mt_getrandmax; use function rand; use function range; use function srand; +use function sort; use function usleep; use function usort; @@ -202,11 +204,14 @@ public function testDatabaseBuilderAutocomplete() self::assertInstanceOf(LaravelCollection::class, $results); self::assertCount(3, $results); + // Sort results, because order is not guaranteed + $results = $results->all(); + sort($results); self::assertSame([ 'Database System Concepts', 'Modern Operating Systems', 'Operating System Concepts', - ], $results->sort()->values()->all()); + ], $results); } public function testDatabaseBuilderVectorSearch() diff --git a/tests/ModelTest.php b/tests/ModelTest.php index c532eea55..ef71a5fe0 100644 --- a/tests/ModelTest.php +++ b/tests/ModelTest.php @@ -406,8 +406,9 @@ public function testSoftDelete(): void $this->assertEquals(2, Soft::count()); } + /** @param class-string $model */ #[DataProvider('provideId')] - public function testPrimaryKey(string $model, $id, $expected, bool $expectedFound): void + public function testPrimaryKey(string $model, mixed $id, mixed $expected, bool $expectedFound): void { $model::truncate(); $expectedType = get_debug_type($expected); diff --git a/tests/Models/SchemaVersion.php b/tests/Models/SchemaVersion.php index 8acd73545..b142d8bda 100644 --- a/tests/Models/SchemaVersion.php +++ b/tests/Models/SchemaVersion.php @@ -5,9 +5,9 @@ namespace MongoDB\Laravel\Tests\Models; use MongoDB\Laravel\Eloquent\HasSchemaVersion; -use MongoDB\Laravel\Eloquent\Model as Eloquent; +use MongoDB\Laravel\Eloquent\Model; -class SchemaVersion extends Eloquent +class SchemaVersion extends Model { use HasSchemaVersion; diff --git a/tests/Scout/Models/ScoutUser.php b/tests/Scout/Models/ScoutUser.php new file mode 100644 index 000000000..50fa39a94 --- /dev/null +++ b/tests/Scout/Models/ScoutUser.php @@ -0,0 +1,43 @@ +dropIfExists('scout_users'); + $schema->create('scout_users', function (Blueprint $table) { + $table->increments('id'); + $table->string('name'); + $table->string('email')->nullable(); + $table->date('email_verified_at')->nullable(); + $table->timestamps(); + $table->softDeletes(); + }); + } +} diff --git a/tests/Scout/Models/SearchableInSameNamespace.php b/tests/Scout/Models/SearchableInSameNamespace.php new file mode 100644 index 000000000..91b909067 --- /dev/null +++ b/tests/Scout/Models/SearchableInSameNamespace.php @@ -0,0 +1,30 @@ +getTable(); + } +} diff --git a/tests/Scout/Models/SearchableModel.php b/tests/Scout/Models/SearchableModel.php new file mode 100644 index 000000000..c0c438979 --- /dev/null +++ b/tests/Scout/Models/SearchableModel.php @@ -0,0 +1,50 @@ +getAttribute($this->getScoutKeyName()) ?: 'key_' . $this->getKey(); + } + + /** + * This method must be overridden when the `getScoutKey` method is also overridden, + * to support model serialization for async indexation jobs. + * + * @see Searchable::getScoutKeyName() + */ + public function getScoutKeyName(): string + { + return 'scout_key'; + } +} diff --git a/tests/Scout/ScoutEngineTest.php b/tests/Scout/ScoutEngineTest.php new file mode 100644 index 000000000..e15e9ff9d --- /dev/null +++ b/tests/Scout/ScoutEngineTest.php @@ -0,0 +1,548 @@ + ['root' => 'array', 'document' => 'array', 'array' => 'array']]; + + /** @param callable(): Builder $builder */ + #[DataProvider('provideSearchPipelines')] + public function testSearch(Closure $builder, array $expectedPipeline): void + { + $data = [['_id' => 'key_1', '__count' => 15], ['_id' => 'key_2', '__count' => 15]]; + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_searchable') + ->andReturn($collection); + $cursor = m::mock(CursorInterface::class); + $cursor->shouldReceive('toArray')->once()->with()->andReturn($data); + + $collection->shouldReceive('aggregate') + ->once() + ->withArgs(function ($pipeline, $options) use ($expectedPipeline) { + self::assertEquals($expectedPipeline, $pipeline); + self::assertEquals(self::EXPECTED_SEARCH_OPTIONS, $options); + + return true; + }) + ->andReturn($cursor); + + $engine = new ScoutEngine($database, softDelete: false); + $result = $engine->search($builder()); + $this->assertEquals($data, $result); + } + + public function provideSearchPipelines(): iterable + { + $defaultPipeline = [ + [ + '$search' => [ + 'index' => 'scout', + 'compound' => [ + 'should' => [ + [ + 'text' => [ + 'path' => ['wildcard' => '*'], + 'query' => 'lar', + 'fuzzy' => ['maxEdits' => 2], + 'score' => ['boost' => ['value' => 5]], + ], + ], + [ + 'wildcard' => [ + 'query' => 'lar*', + 'path' => ['wildcard' => '*'], + 'allowAnalyzedField' => true, + ], + ], + ], + 'minimumShouldMatch' => 1, + ], + 'count' => [ + 'type' => 'lowerBound', + ], + ], + ], + [ + '$addFields' => [ + '__count' => '$$SEARCH_META.count.lowerBound', + ], + ], + ]; + + yield 'simple string' => [ + function () { + return new Builder(new SearchableModel(), 'lar'); + }, + $defaultPipeline, + ]; + + yield 'where conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->where('foo', 'bar'); + $builder->where('key', 'value'); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => 'foo', 'value' => 'bar']], + ['equals' => ['path' => 'key', 'value' => 'value']], + ], + ], + ], + ], + ]), + ]; + + yield 'where in conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->where('foo', 'bar'); + $builder->where('bar', 'baz'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => 'foo', 'value' => 'bar']], + ['equals' => ['path' => 'bar', 'value' => 'baz']], + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + ], + ], + ], + ]), + ]; + + yield 'where not in conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->where('foo', 'bar'); + $builder->where('bar', 'baz'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + $builder->whereNotIn('eaea', [3]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => 'foo', 'value' => 'bar']], + ['equals' => ['path' => 'bar', 'value' => 'baz']], + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + 'mustNot' => [ + ['in' => ['path' => 'eaea', 'value' => [3]]], + ], + ], + ], + ], + ]), + ]; + + yield 'where in conditions without other conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + ], + ], + ], + ]), + ]; + + yield 'where not in conditions without other conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + $builder->whereNotIn('eaea', [3]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + 'mustNot' => [ + ['in' => ['path' => 'eaea', 'value' => [3]]], + ], + ], + ], + ], + ]), + ]; + + yield 'empty where in conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + $builder->whereNotIn('eaea', [3]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + 'mustNot' => [ + ['in' => ['path' => 'eaea', 'value' => [3]]], + ], + ], + ], + ], + ]), + ]; + + yield 'exclude soft-deleted' => [ + function () { + return new Builder(new SearchableModel(), 'lar', softDelete: true); + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => '__soft_deleted', 'value' => 0]], + ], + ], + ], + ], + ]), + ]; + + yield 'only trashed' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar', softDelete: true); + $builder->onlyTrashed(); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => '__soft_deleted', 'value' => 1]], + ], + ], + ], + ], + ]), + ]; + + yield 'with callback' => [ + fn () => new Builder(new SearchableModel(), 'query', callback: function (...$args) { + $this->assertCount(3, $args); + $this->assertInstanceOf(Collection::class, $args[0]); + $this->assertSame('query', $args[1]); + $this->assertNull($args[2]); + + return $args[0]->aggregate(['pipeline'], self::EXPECTED_SEARCH_OPTIONS); + }), + ['pipeline'], + ]; + + yield 'ordered' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->orderBy('name', 'desc'); + $builder->orderBy('age', 'asc'); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'sort' => [ + 'name' => -1, + 'age' => 1, + ], + ], + ], + ]), + ]; + } + + public function testPaginate() + { + $perPage = 5; + $page = 3; + + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $cursor = m::mock(CursorInterface::class); + $database->shouldReceive('selectCollection') + ->with('collection_searchable') + ->andReturn($collection); + $collection->shouldReceive('aggregate') + ->once() + ->withArgs(function (...$args) { + self::assertSame([ + [ + '$search' => [ + 'index' => 'scout', + 'compound' => [ + 'should' => [ + [ + 'text' => [ + 'query' => 'mustang', + 'path' => ['wildcard' => '*'], + 'fuzzy' => ['maxEdits' => 2], + 'score' => ['boost' => ['value' => 5]], + ], + ], + [ + 'wildcard' => [ + 'query' => 'mustang*', + 'path' => ['wildcard' => '*'], + 'allowAnalyzedField' => true, + ], + ], + ], + 'minimumShouldMatch' => 1, + ], + 'count' => [ + 'type' => 'lowerBound', + ], + 'sort' => [ + 'name' => -1, + ], + ], + ], + [ + '$addFields' => [ + '__count' => '$$SEARCH_META.count.lowerBound', + ], + ], + [ + '$skip' => 10, + ], + [ + '$limit' => 5, + ], + ], $args[0]); + + $this->assertSame(self::EXPECTED_SEARCH_OPTIONS, $args[1]); + + return true; + }) + ->andReturn($cursor); + $cursor->shouldReceive('toArray') + ->once() + ->with() + ->andReturn([['_id' => 'key_1', '__count' => 17], ['_id' => 'key_2', '__count' => 17]]); + + $engine = new ScoutEngine($database, softDelete: false); + $builder = new Builder(new SearchableModel(), 'mustang'); + $builder->orderBy('name', 'desc'); + $engine->paginate($builder, $perPage, $page); + } + + #[DataProvider('provideResultsForMapIds')] + public function testMapIds(array $results): void + { + $engine = new ScoutEngine(m::mock(Database::class), softDelete: false); + + $ids = $engine->mapIds($results); + + $this->assertInstanceOf(IlluminateCollection::class, $ids); + $this->assertEquals(['key_1', 'key_2'], $ids->all()); + } + + public static function provideResultsForMapIds(): iterable + { + yield 'array' => [ + [ + ['_id' => 'key_1', 'foo' => 'bar'], + ['_id' => 'key_2', 'foo' => 'bar'], + ], + ]; + + yield 'object' => [ + [ + (object) ['_id' => 'key_1', 'foo' => 'bar'], + (object) ['_id' => 'key_2', 'foo' => 'bar'], + ], + ]; + + yield Document::class => [ + [ + Document::fromPHP(['_id' => 'key_1', 'foo' => 'bar']), + Document::fromPHP(['_id' => 'key_2', 'foo' => 'bar']), + ], + ]; + + yield BSONDocument::class => [ + [ + new BSONDocument(['_id' => 'key_1', 'foo' => 'bar']), + new BSONDocument(['_id' => 'key_2', 'foo' => 'bar']), + ], + ]; + } + + public function testUpdate(): void + { + $date = new DateTimeImmutable('2000-01-02 03:04:05'); + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_indexable') + ->andReturn($collection); + $collection->shouldReceive('bulkWrite') + ->once() + ->with([ + [ + 'updateOne' => [ + ['_id' => 'key_1'], + ['$setOnInsert' => ['_id' => 'key_1'], '$set' => ['id' => 1, 'date' => new UTCDateTime($date)]], + ['upsert' => true], + ], + ], + [ + 'updateOne' => [ + ['_id' => 'key_2'], + ['$setOnInsert' => ['_id' => 'key_2'], '$set' => ['id' => 2]], + ['upsert' => true], + ], + ], + ]); + + $engine = new ScoutEngine($database, softDelete: false); + $engine->update(EloquentCollection::make([ + new SearchableModel([ + 'id' => 1, + 'date' => $date, + ]), + new SearchableModel([ + 'id' => 2, + ]), + ])); + } + + public function testUpdateWithSoftDelete(): void + { + $date = new DateTimeImmutable('2000-01-02 03:04:05'); + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_indexable') + ->andReturn($collection); + $collection->shouldReceive('bulkWrite') + ->once() + ->with([ + [ + 'updateOne' => [ + ['_id' => 'key_1'], + ['$setOnInsert' => ['_id' => 'key_1'], '$set' => ['id' => 1]], + ['upsert' => true], + ], + ], + ]); + + $model = new SearchableModel(['id' => 1]); + $model->delete(); + + $engine = new ScoutEngine($database, softDelete: false); + $engine->update(EloquentCollection::make([$model])); + } + + public function testDelete(): void + { + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_indexable') + ->andReturn($collection); + $collection->shouldReceive('deleteMany') + ->once() + ->with(['_id' => ['$in' => ['key_1', 'key_2']]]); + + $engine = new ScoutEngine($database, softDelete: false); + $engine->delete(EloquentCollection::make([ + new SearchableModel(['id' => 1]), + new SearchableModel(['id' => 2]), + ])); + } + + public function testDeleteWithRemoveableScoutCollection(): void + { + $job = new RemoveFromSearch(EloquentCollection::make([ + new SearchableModel(['id' => 5, 'scout_key' => 'key_5']), + ])); + + $job = unserialize(serialize($job)); + + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_indexable') + ->andReturn($collection); + $collection->shouldReceive('deleteMany') + ->once() + ->with(['_id' => ['$in' => ['key_5']]]); + + $engine = new ScoutEngine($database, softDelete: false); + $engine->delete($job->models); + } +} diff --git a/tests/Scout/ScoutIntegrationTest.php b/tests/Scout/ScoutIntegrationTest.php new file mode 100644 index 000000000..404165862 --- /dev/null +++ b/tests/Scout/ScoutIntegrationTest.php @@ -0,0 +1,242 @@ +set('scout.driver', 'mongodb'); + $app['config']->set('scout.prefix', 'prefix_'); + } + + public function setUp(): void + { + parent::setUp(); + + // Init the SQL database with some objects that will be indexed + // Test data copied from Laravel Scout tests + // https://github.com/laravel/scout/blob/10.x/tests/Integration/SearchableTests.php + ScoutUser::executeSchema(); + + $collect = LazyCollection::make(function () { + yield ['name' => 'Laravel Framework']; + + foreach (range(2, 10) as $key) { + yield ['name' => 'Example ' . $key]; + } + + yield ['name' => 'Larry Casper', 'email_verified_at' => null]; + yield ['name' => 'Reta Larkin']; + + foreach (range(13, 19) as $key) { + yield ['name' => 'Example ' . $key]; + } + + yield ['name' => 'Prof. Larry Prosacco DVM', 'email_verified_at' => null]; + + foreach (range(21, 38) as $key) { + yield ['name' => 'Example ' . $key, 'email_verified_at' => null]; + } + + yield ['name' => 'Linkwood Larkin', 'email_verified_at' => null]; + yield ['name' => 'Otis Larson MD']; + yield ['name' => 'Gudrun Larkin']; + yield ['name' => 'Dax Larkin']; + yield ['name' => 'Dana Larson Sr.']; + yield ['name' => 'Amos Larson Sr.']; + }); + + $id = 0; + $date = new DateTimeImmutable('2021-01-01 00:00:00'); + foreach ($collect as $data) { + $data = array_merge(['id' => ++$id, 'email_verified_at' => $date], $data); + ScoutUser::create($data)->save(); + } + + self::assertSame(44, ScoutUser::count()); + } + + /** This test create the search index for tests performing search */ + public function testItCanCreateTheCollection() + { + $this->skipIfSearchIndexManagementIsNotSupported(); + + $collection = DB::connection('mongodb')->getCollection('prefix_scout_users'); + $collection->drop(); + + // Recreate the indexes using the artisan commands + // Ensure they return a success exit code (0) + self::assertSame(0, artisan($this, 'scout:delete-index', ['name' => ScoutUser::class])); + self::assertSame(0, artisan($this, 'scout:index', ['name' => ScoutUser::class])); + self::assertSame(0, artisan($this, 'scout:import', ['model' => ScoutUser::class])); + + self::assertSame(44, $collection->countDocuments()); + + $searchIndexes = $collection->listSearchIndexes(['name' => 'scout']); + self::assertCount(1, $searchIndexes); + + // Wait for all documents to be indexed asynchronously + $i = 100; + while (true) { + $indexedDocuments = $collection->aggregate([ + ['$search' => ['index' => 'scout', 'exists' => ['path' => 'name']]], + ])->toArray(); + + if (count($indexedDocuments) > 0) { + break; + } + + if ($i-- === 0) { + self::fail('Documents not indexed'); + } + + usleep(100_000); + } + + self::assertCount(44, $indexedDocuments); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearch() + { + // All the search queries use "sort" option to ensure the results are deterministic + $results = ScoutUser::search('lar')->take(10)->orderBy('id')->get(); + + self::assertSame([ + 1 => 'Laravel Framework', + 11 => 'Larry Casper', + 12 => 'Reta Larkin', + 20 => 'Prof. Larry Prosacco DVM', + 39 => 'Linkwood Larkin', + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $results->pluck('name', 'id')->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearchWithQueryCallback() + { + $results = ScoutUser::search('lar')->take(10)->orderBy('id')->query(function ($query) { + return $query->whereNotNull('email_verified_at'); + })->get(); + + self::assertSame([ + 1 => 'Laravel Framework', + 12 => 'Reta Larkin', + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $results->pluck('name', 'id')->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearchToFetchKeys() + { + $results = ScoutUser::search('lar')->orderBy('id')->take(10)->keys(); + + self::assertSame([1, 11, 12, 20, 39, 40, 41, 42, 43, 44], $results->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearchWithQueryCallbackToFetchKeys() + { + $results = ScoutUser::search('lar')->take(10)->orderBy('id', 'desc')->query(function ($query) { + return $query->whereNotNull('email_verified_at'); + })->keys(); + + self::assertSame([44, 43, 42, 41, 40, 39, 20, 12, 11, 1], $results->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUsePaginatedSearch() + { + $page1 = ScoutUser::search('lar')->take(10)->orderBy('id')->paginate(5, 'page', 1); + $page2 = ScoutUser::search('lar')->take(10)->orderBy('id')->paginate(5, 'page', 2); + + self::assertSame([ + 1 => 'Laravel Framework', + 11 => 'Larry Casper', + 12 => 'Reta Larkin', + 20 => 'Prof. Larry Prosacco DVM', + 39 => 'Linkwood Larkin', + ], $page1->pluck('name', 'id')->all()); + + self::assertSame([ + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $page2->pluck('name', 'id')->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUsePaginatedSearchWithQueryCallback() + { + $queryCallback = function ($query) { + return $query->whereNotNull('email_verified_at'); + }; + + $page1 = ScoutUser::search('lar')->take(10)->orderBy('id')->query($queryCallback)->paginate(5, 'page', 1); + $page2 = ScoutUser::search('lar')->take(10)->orderBy('id')->query($queryCallback)->paginate(5, 'page', 2); + + self::assertSame([ + 1 => 'Laravel Framework', + 12 => 'Reta Larkin', + ], $page1->pluck('name', 'id')->all()); + + self::assertSame([ + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $page2->pluck('name', 'id')->all()); + } + + public function testItCannotIndexInTheSameNamespace() + { + self::expectException(LogicException::class); + self::expectExceptionMessage(sprintf( + 'The MongoDB Scout collection "%s.searchable_in_same_namespaces" must use a different collection from the collection name of the model "%s". Set the "scout.prefix" configuration or use a distinct MongoDB database', + env('MONGODB_DATABASE', 'unittest'), + SearchableInSameNamespace::class, + ),); + + SearchableInSameNamespace::create(['name' => 'test']); + } +} From 22f40a07a39b52f709dc30ff40d142e7839f2d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 14 Jan 2025 15:01:07 +0100 Subject: [PATCH 2/8] Update default configuration --- docker-compose.yml | 3 ++- phpunit.xml.dist | 2 +- src/Scout/ScoutEngine.php | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fb35e0c63..38fee69dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,8 @@ services: ports: - "27017:27017" healthcheck: - test: mongosh mongodb:27017 --quiet --eval 'db.runCommand("ping").ok' + test: mongosh --quiet --eval 'db.runCommand("ping").ok' interval: 10s timeout: 10s retries: 5 +s diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 5431164d8..7044f9069 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -17,7 +17,7 @@ - + diff --git a/src/Scout/ScoutEngine.php b/src/Scout/ScoutEngine.php index d3dcdcba2..1cefc6bf8 100644 --- a/src/Scout/ScoutEngine.php +++ b/src/Scout/ScoutEngine.php @@ -84,6 +84,8 @@ public function __construct( #[Override] public function update($models) { + assert($models instanceof EloquentCollection, new TypeError(sprintf('Argument #1 ($models) must be of type %s, %s given', EloquentCollection::class, get_debug_type($models)))); + if ($models->isEmpty()) { return; } From bd218acb002ae703b79ec4ec66c40fc49cc9468e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 14 Jan 2025 18:22:22 +0100 Subject: [PATCH 3/8] Set typemap on cursor, only root._id is mapped --- src/Scout/ScoutEngine.php | 70 ++++++++++++++++++++------------- tests/AtlasSearchTest.php | 3 +- tests/Scout/ScoutEngineTest.php | 15 ++++--- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/src/Scout/ScoutEngine.php b/src/Scout/ScoutEngine.php index 1cefc6bf8..4ecaf4f56 100644 --- a/src/Scout/ScoutEngine.php +++ b/src/Scout/ScoutEngine.php @@ -66,6 +66,8 @@ final class ScoutEngine extends Engine ], ]; + private const TYPEMAP = ['root' => 'object', 'document' => 'bson', 'array' => 'bson']; + public function __construct( private Database $database, private bool $softDelete, @@ -96,28 +98,42 @@ public function update($models) $bulk = []; foreach ($models as $model) { + assert($model instanceof Model && method_exists($model, 'toSearchableArray'), new LogicException(sprintf('Model "%s" must use "%s" trait', $model::class, Searchable::class))); + $searchableData = $model->toSearchableArray(); $searchableData = self::serialize($searchableData); - if ($searchableData) { - unset($searchableData['_id']); - - $searchableData = array_merge($searchableData, $model->scoutMetadata()); - if (isset($searchableData['__soft_deleted'])) { - $searchableData['__soft_deleted'] = (bool) $searchableData['__soft_deleted']; - } - + // Skip/remove the model if it doesn't provide any searchable data + if (! $searchableData) { $bulk[] = [ - 'updateOne' => [ + 'deleteOne' => [ ['_id' => $model->getScoutKey()], - [ - '$setOnInsert' => ['_id' => $model->getScoutKey()], - '$set' => $searchableData, - ], - ['upsert' => true], ], ]; + + continue; } + + unset($searchableData['_id']); + + $searchableData = array_merge($searchableData, $model->scoutMetadata()); + + /** Convert the __soft_deleted set by {@see Searchable::pushSoftDeleteMetadata()} + * into a boolean for efficient storage and indexing. */ + if (isset($searchableData['__soft_deleted'])) { + $searchableData['__soft_deleted'] = (bool) $searchableData['__soft_deleted']; + } + + $bulk[] = [ + 'updateOne' => [ + ['_id' => $model->getScoutKey()], + [ + '$setOnInsert' => ['_id' => $model->getScoutKey()], + '$set' => $searchableData, + ], + ['upsert' => true], + ], + ]; } $this->getIndexableCollection($models)->bulkWrite($bulk); @@ -133,7 +149,7 @@ public function update($models) #[Override] public function delete($models): void { - assert($models instanceof Collection, new TypeError(sprintf('Argument #1 ($models) must be of type %s, %s given', Collection::class, get_debug_type($models)))); + assert($models instanceof EloquentCollection, new TypeError(sprintf('Argument #1 ($models) must be of type %s, %s given', Collection::class, get_debug_type($models)))); if ($models->isEmpty()) { return; @@ -149,7 +165,7 @@ public function delete($models): void * * @see Engine::search() * - * @return mixed + * @return array */ #[Override] public function search(Builder $builder) @@ -158,14 +174,14 @@ public function search(Builder $builder) } /** - * Perform the given search on the engine. + * Perform the given search on the engine with pagination. * * @see Engine::paginate() * * @param int $perPage * @param int $page * - * @return mixed + * @return array */ #[Override] public function paginate(Builder $builder, $perPage, $page) @@ -182,20 +198,21 @@ public function paginate(Builder $builder, $perPage, $page) /** * Perform the given search on the engine. */ - protected function performSearch(Builder $builder, ?int $offset = null): array + private function performSearch(Builder $builder, ?int $offset = null): array { $collection = $this->getSearchableCollection($builder->model); if ($builder->callback) { - $result = call_user_func( + $cursor = call_user_func( $builder->callback, $collection, $builder->query, $offset, ); - assert($result instanceof CursorInterface, new LogicException(sprintf('The search builder closure must return a MongoDB cursor, %s returned', get_debug_type($result)))); + assert($cursor instanceof CursorInterface, new LogicException(sprintf('The search builder closure must return a MongoDB cursor, %s returned', get_debug_type($cursor)))); + $cursor->setTypeMap(self::TYPEMAP); - return $result->toArray(); + return $cursor->toArray(); } $compound = [ @@ -270,11 +287,10 @@ protected function performSearch(Builder $builder, ?int $offset = null): array $pipeline[] = ['$limit' => $builder->limit]; } - $options = [ - 'typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array'], - ]; + $cursor = $collection->aggregate($pipeline); + $cursor->setTypeMap(self::TYPEMAP); - return $collection->aggregate($pipeline, $options)->toArray(); + return $cursor->toArray(); } /** @@ -539,6 +555,6 @@ private function wait(Closure $callback): void sleep(1); } - throw new MongoDBRuntimeException('Atlas search index operation time out'); + throw new MongoDBRuntimeException(sprintf('Atlas search index operation time out after %s seconds', self::WAIT_TIMEOUT_SEC)); } } diff --git a/tests/AtlasSearchTest.php b/tests/AtlasSearchTest.php index c6f5a9a65..763e4ca2c 100644 --- a/tests/AtlasSearchTest.php +++ b/tests/AtlasSearchTest.php @@ -14,12 +14,11 @@ use function array_map; use function assert; - use function mt_getrandmax; use function rand; use function range; -use function srand; use function sort; +use function srand; use function usleep; use function usort; diff --git a/tests/Scout/ScoutEngineTest.php b/tests/Scout/ScoutEngineTest.php index e15e9ff9d..f5ac64e03 100644 --- a/tests/Scout/ScoutEngineTest.php +++ b/tests/Scout/ScoutEngineTest.php @@ -27,7 +27,7 @@ /** Unit tests that do not require an Atlas Search cluster */ class ScoutEngineTest extends TestCase { - private const EXPECTED_SEARCH_OPTIONS = ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']]; + private const EXPECTED_TYPEMAP = ['root' => 'object', 'document' => 'bson', 'array' => 'bson']; /** @param callable(): Builder $builder */ #[DataProvider('provideSearchPipelines')] @@ -40,13 +40,16 @@ public function testSearch(Closure $builder, array $expectedPipeline): void ->with('collection_searchable') ->andReturn($collection); $cursor = m::mock(CursorInterface::class); + $cursor->shouldReceive('setTypeMap')->once()->with(self::EXPECTED_TYPEMAP); $cursor->shouldReceive('toArray')->once()->with()->andReturn($data); + $collection->shouldReceive('getCollectionName') + ->zeroOrMoreTimes() + ->andReturn('collection_searchable'); $collection->shouldReceive('aggregate') ->once() - ->withArgs(function ($pipeline, $options) use ($expectedPipeline) { + ->withArgs(function ($pipeline) use ($expectedPipeline) { self::assertEquals($expectedPipeline, $pipeline); - self::assertEquals(self::EXPECTED_SEARCH_OPTIONS, $options); return true; }) @@ -295,10 +298,11 @@ function () { fn () => new Builder(new SearchableModel(), 'query', callback: function (...$args) { $this->assertCount(3, $args); $this->assertInstanceOf(Collection::class, $args[0]); + $this->assertSame('collection_searchable', $args[0]->getCollectionName()); $this->assertSame('query', $args[1]); $this->assertNull($args[2]); - return $args[0]->aggregate(['pipeline'], self::EXPECTED_SEARCH_OPTIONS); + return $args[0]->aggregate(['pipeline']); }), ['pipeline'], ]; @@ -383,11 +387,10 @@ public function testPaginate() ], ], $args[0]); - $this->assertSame(self::EXPECTED_SEARCH_OPTIONS, $args[1]); - return true; }) ->andReturn($cursor); + $cursor->shouldReceive('setTypeMap')->once()->with(self::EXPECTED_TYPEMAP); $cursor->shouldReceive('toArray') ->once() ->with() From 1ae5af1678821d307984399de634c62c4a0696fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 14 Jan 2025 19:40:52 +0100 Subject: [PATCH 4/8] Factorize map/lazyMap into performMap --- src/Scout/ScoutEngine.php | 119 +++++++++++-------------- tests/AtlasSearchTest.php | 6 +- tests/Scout/Models/SearchableModel.php | 2 +- tests/Scout/ScoutEngineTest.php | 4 +- tests/Scout/ScoutIntegrationTest.php | 20 +++++ 5 files changed, 74 insertions(+), 77 deletions(-) diff --git a/src/Scout/ScoutEngine.php b/src/Scout/ScoutEngine.php index 4ecaf4f56..5a2ea8aa7 100644 --- a/src/Scout/ScoutEngine.php +++ b/src/Scout/ScoutEngine.php @@ -27,7 +27,6 @@ use function array_column; use function array_flip; -use function array_key_exists; use function array_map; use function array_merge; use function assert; @@ -44,7 +43,6 @@ use function method_exists; use function sleep; use function sprintf; -use function substr; use function time; /** @@ -313,40 +311,67 @@ public function mapIds($results): Collection * * @see Engine::map() * - * @param array $results - * @param Model $model + * @param Builder $builder + * @param array|Cursor $results + * @param Model $model + * + * @return Collection */ #[Override] public function map(Builder $builder, $results, $model): Collection { - assert(is_array($results), new TypeError(sprintf('Argument #2 ($results) must be of type array, %s given', get_debug_type($results)))); - assert($model instanceof Model, new TypeError(sprintf('Argument #3 ($model) must be of type %s, %s given', Model::class, get_debug_type($model)))); + return $this->performMap($builder, $results, $model, false); + } + + /** + * Map the given results to instances of the given model via a lazy collection. + * + * @see Engine::lazyMap() + * + * @param Builder $builder + * @param array|Cursor $results + * @param Model $model + * + * @return LazyCollection + */ + #[Override] + public function lazyMap(Builder $builder, $results, $model): LazyCollection + { + return $this->performMap($builder, $results, $model, true); + } + /** @return ($lazy is true ? LazyCollection : Collection) */ + private function performMap(Builder $builder, array $results, Model $model, bool $lazy): Collection|LazyCollection + { if (! $results) { - return $model->newCollection(); + $collection = $model->newCollection(); + + return $lazy ? LazyCollection::make($collection) : $collection; } $objectIds = array_column($results, '_id'); $objectIdPositions = array_flip($objectIds); - return $model->getScoutModelsByIds( - $builder, - $objectIds, - )->filter(function ($model) use ($objectIdPositions) { - return array_key_exists($model->getScoutKey(), $objectIdPositions); - })->map(function ($model) use ($results, $objectIdPositions) { - $result = $results[$objectIdPositions[$model->getScoutKey()]] ?? []; - - foreach ($result as $key => $value) { - if ($key[0] === '_') { - $model->withScoutMetadata($key, $value); + return $model->queryScoutModelsByIds($builder, $objectIds) + ->{$lazy ? 'cursor' : 'get'}() + ->filter(function ($model) use ($objectIds) { + return in_array($model->getScoutKey(), $objectIds); + }) + ->map(function ($model) use ($results, $objectIdPositions) { + $result = $results[$objectIdPositions[$model->getScoutKey()]] ?? []; + + foreach ($result as $key => $value) { + if ($key[0] === '_' && $key !== '_id') { + $model->withScoutMetadata($key, $value); + } } - } - return $model; - })->sortBy(function ($model) use ($objectIdPositions) { - return $objectIdPositions[$model->getScoutKey()]; - })->values(); + return $model; + }) + ->sortBy(function ($model) use ($objectIdPositions) { + return $objectIdPositions[$model->getScoutKey()]; + }) + ->values(); } /** @@ -364,7 +389,7 @@ public function getTotalCount($results): int return 0; } - return $results[0]['__count']; + return $results[0]->__count; } /** @@ -384,50 +409,6 @@ public function flush($model): void $collection->deleteMany([]); } - /** - * Map the given results to instances of the given model via a lazy collection. - * - * @see Engine::lazyMap() - * - * @param Builder $builder - * @param array|Cursor $results - * @param Model $model - * - * @return LazyCollection - */ - #[Override] - public function lazyMap(Builder $builder, $results, $model): LazyCollection - { - assert($results instanceof Cursor || is_array($results), new TypeError(sprintf('Argument #2 ($results) must be of type %s|array, %s given', Cursor::class, get_debug_type($results)))); - assert($model instanceof Model, new TypeError(sprintf('Argument #3 ($model) must be of type %s, %s given', Model::class, get_debug_type($model)))); - - if (! $results) { - return LazyCollection::make($model->newCollection()); - } - - $objectIds = array_column($results, '_id'); - $objectIdPositions = array_flip($objectIds); - - return $model->queryScoutModelsByIds( - $builder, - $objectIds, - )->cursor()->filter(function ($model) use ($objectIds) { - return in_array($model->getScoutKey(), $objectIds); - })->map(function ($model) use ($results, $objectIdPositions) { - $result = $results[$objectIdPositions[$model->getScoutKey()]] ?? []; - - foreach ($result as $key => $value) { - if (substr($key, 0, 1) === '_') { - $model->withScoutMetadata($key, $value); - } - } - - return $model; - })->sortBy(function ($model) use ($objectIdPositions) { - return $objectIdPositions[$model->getScoutKey()]; - })->values(); - } - /** * Create the MongoDB Atlas Search index. * @@ -475,7 +456,7 @@ public function deleteIndex($name): void { assert(is_string($name), new TypeError(sprintf('Argument #1 ($name) must be of type string, %s given', get_debug_type($name)))); - $this->database->selectCollection($name)->drop(); + $this->database->dropCollection($name); } /** Get the MongoDB collection used to search for the provided model */ diff --git a/tests/AtlasSearchTest.php b/tests/AtlasSearchTest.php index 763e4ca2c..c9cd2d5e3 100644 --- a/tests/AtlasSearchTest.php +++ b/tests/AtlasSearchTest.php @@ -17,7 +17,6 @@ use function mt_getrandmax; use function rand; use function range; -use function sort; use function srand; use function usleep; use function usort; @@ -203,14 +202,11 @@ public function testDatabaseBuilderAutocomplete() self::assertInstanceOf(LaravelCollection::class, $results); self::assertCount(3, $results); - // Sort results, because order is not guaranteed - $results = $results->all(); - sort($results); self::assertSame([ 'Database System Concepts', 'Modern Operating Systems', 'Operating System Concepts', - ], $results); + ], $results->sort()->values()->all()); } public function testDatabaseBuilderVectorSearch() diff --git a/tests/Scout/Models/SearchableModel.php b/tests/Scout/Models/SearchableModel.php index c0c438979..e53200f1a 100644 --- a/tests/Scout/Models/SearchableModel.php +++ b/tests/Scout/Models/SearchableModel.php @@ -39,7 +39,7 @@ public function getScoutKey(): string /** * This method must be overridden when the `getScoutKey` method is also overridden, - * to support model serialization for async indexation jobs. + * to support model serialization for async indexing jobs. * * @see Searchable::getScoutKeyName() */ diff --git a/tests/Scout/ScoutEngineTest.php b/tests/Scout/ScoutEngineTest.php index f5ac64e03..5ae22b17a 100644 --- a/tests/Scout/ScoutEngineTest.php +++ b/tests/Scout/ScoutEngineTest.php @@ -403,11 +403,11 @@ public function testPaginate() } #[DataProvider('provideResultsForMapIds')] - public function testMapIds(array $results): void + public function testLazyMapIds(array $results): void { $engine = new ScoutEngine(m::mock(Database::class), softDelete: false); - $ids = $engine->mapIds($results); + $ids = $engine->lazyMap($results); $this->assertInstanceOf(IlluminateCollection::class, $ids); $this->assertEquals(['key_1', 'key_2'], $ids->all()); diff --git a/tests/Scout/ScoutIntegrationTest.php b/tests/Scout/ScoutIntegrationTest.php index 404165862..9a7cb83e9 100644 --- a/tests/Scout/ScoutIntegrationTest.php +++ b/tests/Scout/ScoutIntegrationTest.php @@ -145,6 +145,26 @@ public function testItCanUseBasicSearch() ], $results->pluck('name', 'id')->all()); } + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearchCursor() + { + // All the search queries use "sort" option to ensure the results are deterministic + $results = ScoutUser::search('lar')->take(10)->orderBy('id')->cursor(); + + self::assertSame([ + 1 => 'Laravel Framework', + 11 => 'Larry Casper', + 12 => 'Reta Larkin', + 20 => 'Prof. Larry Prosacco DVM', + 39 => 'Linkwood Larkin', + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $results->pluck('name', 'id')->all()); + } + #[Depends('testItCanCreateTheCollection')] public function testItCanUseBasicSearchWithQueryCallback() { From b22192b188ad29b44f187bbcf6c0f1a04a89a91e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 14 Jan 2025 20:34:08 +0100 Subject: [PATCH 5/8] Update tests --- src/Scout/ScoutEngine.php | 13 ++++++------- tests/Scout/ScoutEngineTest.php | 26 +++++++++++++++----------- tests/Scout/ScoutIntegrationTest.php | 6 +++--- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/Scout/ScoutEngine.php b/src/Scout/ScoutEngine.php index 5a2ea8aa7..db5ef8373 100644 --- a/src/Scout/ScoutEngine.php +++ b/src/Scout/ScoutEngine.php @@ -17,7 +17,6 @@ use MongoDB\BSON\UTCDateTime; use MongoDB\Collection as MongoDBCollection; use MongoDB\Database; -use MongoDB\Driver\Cursor; use MongoDB\Driver\CursorInterface; use MongoDB\Exception\RuntimeException as MongoDBRuntimeException; use MongoDB\Laravel\Connection; @@ -311,9 +310,9 @@ public function mapIds($results): Collection * * @see Engine::map() * - * @param Builder $builder - * @param array|Cursor $results - * @param Model $model + * @param Builder $builder + * @param array $results + * @param Model $model * * @return Collection */ @@ -328,9 +327,9 @@ public function map(Builder $builder, $results, $model): Collection * * @see Engine::lazyMap() * - * @param Builder $builder - * @param array|Cursor $results - * @param Model $model + * @param Builder $builder + * @param array $results + * @param Model $model * * @return LazyCollection */ diff --git a/tests/Scout/ScoutEngineTest.php b/tests/Scout/ScoutEngineTest.php index 5ae22b17a..438cb8b91 100644 --- a/tests/Scout/ScoutEngineTest.php +++ b/tests/Scout/ScoutEngineTest.php @@ -266,7 +266,7 @@ function () { '$search' => [ 'compound' => [ 'filter' => [ - ['equals' => ['path' => '__soft_deleted', 'value' => 0]], + ['equals' => ['path' => '__soft_deleted', 'value' => false]], ], ], ], @@ -286,7 +286,7 @@ function () { '$search' => [ 'compound' => [ 'filter' => [ - ['equals' => ['path' => '__soft_deleted', 'value' => 1]], + ['equals' => ['path' => '__soft_deleted', 'value' => true]], ], ], ], @@ -493,20 +493,24 @@ public function testUpdateWithSoftDelete(): void ->andReturn($collection); $collection->shouldReceive('bulkWrite') ->once() - ->with([ - [ - 'updateOne' => [ - ['_id' => 'key_1'], - ['$setOnInsert' => ['_id' => 'key_1'], '$set' => ['id' => 1]], - ['upsert' => true], + ->withArgs(function ($pipeline) { + $this->assertSame([ + [ + 'updateOne' => [ + ['_id' => 'key_1'], + ['$setOnInsert' => ['_id' => 'key_1'], '$set' => ['id' => 1, '__soft_deleted' => false]], + ['upsert' => true], + ], ], - ], - ]); + ], $pipeline); + + return true; + }); $model = new SearchableModel(['id' => 1]); $model->delete(); - $engine = new ScoutEngine($database, softDelete: false); + $engine = new ScoutEngine($database, softDelete: true); $engine->update(EloquentCollection::make([$model])); } diff --git a/tests/Scout/ScoutIntegrationTest.php b/tests/Scout/ScoutIntegrationTest.php index 9a7cb83e9..7b9d704f6 100644 --- a/tests/Scout/ScoutIntegrationTest.php +++ b/tests/Scout/ScoutIntegrationTest.php @@ -42,6 +42,8 @@ public function setUp(): void { parent::setUp(); + $this->skipIfSearchIndexManagementIsNotSupported(); + // Init the SQL database with some objects that will be indexed // Test data copied from Laravel Scout tests // https://github.com/laravel/scout/blob/10.x/tests/Integration/SearchableTests.php @@ -88,8 +90,6 @@ public function setUp(): void /** This test create the search index for tests performing search */ public function testItCanCreateTheCollection() { - $this->skipIfSearchIndexManagementIsNotSupported(); - $collection = DB::connection('mongodb')->getCollection('prefix_scout_users'); $collection->drop(); @@ -111,7 +111,7 @@ public function testItCanCreateTheCollection() ['$search' => ['index' => 'scout', 'exists' => ['path' => 'name']]], ])->toArray(); - if (count($indexedDocuments) > 0) { + if (count($indexedDocuments) >= 44) { break; } From 1ffcc19d9632f5c1ae845354c8f61dc6cd97d691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 14 Jan 2025 21:13:23 +0100 Subject: [PATCH 6/8] Update tests for map/lazyMap --- tests/Scout/ScoutEngineTest.php | 105 ++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 39 deletions(-) diff --git a/tests/Scout/ScoutEngineTest.php b/tests/Scout/ScoutEngineTest.php index 438cb8b91..0aaa1374f 100644 --- a/tests/Scout/ScoutEngineTest.php +++ b/tests/Scout/ScoutEngineTest.php @@ -5,22 +5,24 @@ use Closure; use DateTimeImmutable; use Illuminate\Database\Eloquent\Collection as EloquentCollection; -use Illuminate\Support\Collection as IlluminateCollection; +use Illuminate\Support\Collection as LaravelCollection; +use Illuminate\Support\LazyCollection; use Laravel\Scout\Builder; use Laravel\Scout\Jobs\RemoveFromSearch; use Mockery as m; -use MongoDB\BSON\Document; use MongoDB\BSON\UTCDateTime; use MongoDB\Collection; use MongoDB\Database; use MongoDB\Driver\CursorInterface; +use MongoDB\Laravel\Eloquent\Model; use MongoDB\Laravel\Scout\ScoutEngine; +use MongoDB\Laravel\Tests\Scout\Models\ScoutUser; use MongoDB\Laravel\Tests\Scout\Models\SearchableModel; use MongoDB\Laravel\Tests\TestCase; -use MongoDB\Model\BSONDocument; use PHPUnit\Framework\Attributes\DataProvider; use function array_replace_recursive; +use function count; use function serialize; use function unserialize; @@ -402,46 +404,71 @@ public function testPaginate() $engine->paginate($builder, $perPage, $page); } - #[DataProvider('provideResultsForMapIds')] - public function testLazyMapIds(array $results): void + public function testMapMethodRespectsOrder() { - $engine = new ScoutEngine(m::mock(Database::class), softDelete: false); - - $ids = $engine->lazyMap($results); - - $this->assertInstanceOf(IlluminateCollection::class, $ids); - $this->assertEquals(['key_1', 'key_2'], $ids->all()); + $database = m::mock(Database::class); + $engine = new ScoutEngine($database, false); + + $model = m::mock(Model::class); + $model->shouldReceive(['getScoutKeyName' => 'id']); + $model->shouldReceive('queryScoutModelsByIds->get') + ->andReturn(LaravelCollection::make([ + new ScoutUser(['id' => 1]), + new ScoutUser(['id' => 2]), + new ScoutUser(['id' => 3]), + new ScoutUser(['id' => 4]), + ])); + + $builder = m::mock(Builder::class); + + $results = $engine->map($builder, [ + ['_id' => 1, '__count' => 4], + ['_id' => 2, '__count' => 4], + ['_id' => 4, '__count' => 4], + ['_id' => 3, '__count' => 4], + ], $model); + + $this->assertEquals(4, count($results)); + $this->assertEquals([ + 0 => ['id' => 1], + 1 => ['id' => 2], + 2 => ['id' => 4], + 3 => ['id' => 3], + ], $results->toArray()); } - public static function provideResultsForMapIds(): iterable + public function testLazyMapMethodRespectsOrder() { - yield 'array' => [ - [ - ['_id' => 'key_1', 'foo' => 'bar'], - ['_id' => 'key_2', 'foo' => 'bar'], - ], - ]; - - yield 'object' => [ - [ - (object) ['_id' => 'key_1', 'foo' => 'bar'], - (object) ['_id' => 'key_2', 'foo' => 'bar'], - ], - ]; - - yield Document::class => [ - [ - Document::fromPHP(['_id' => 'key_1', 'foo' => 'bar']), - Document::fromPHP(['_id' => 'key_2', 'foo' => 'bar']), - ], - ]; - - yield BSONDocument::class => [ - [ - new BSONDocument(['_id' => 'key_1', 'foo' => 'bar']), - new BSONDocument(['_id' => 'key_2', 'foo' => 'bar']), - ], - ]; + $lazy = false; + $database = m::mock(Database::class); + $engine = new ScoutEngine($database, false); + + $model = m::mock(Model::class); + $model->shouldReceive(['getScoutKeyName' => 'id']); + $model->shouldReceive('queryScoutModelsByIds->cursor') + ->andReturn(LazyCollection::make([ + new ScoutUser(['id' => 1]), + new ScoutUser(['id' => 2]), + new ScoutUser(['id' => 3]), + new ScoutUser(['id' => 4]), + ])); + + $builder = m::mock(Builder::class); + + $results = $engine->lazyMap($builder, [ + ['_id' => 1, '__count' => 4], + ['_id' => 2, '__count' => 4], + ['_id' => 4, '__count' => 4], + ['_id' => 3, '__count' => 4], + ], $model); + + $this->assertEquals(4, count($results)); + $this->assertEquals([ + 0 => ['id' => 1], + 1 => ['id' => 2], + 2 => ['id' => 4], + 3 => ['id' => 3], + ], $results->toArray()); } public function testUpdate(): void From aabec4d795d26a8977b7adfc26f58f95c8b915e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 14 Jan 2025 21:28:56 +0100 Subject: [PATCH 7/8] Improve comments --- docker-compose.yml | 1 - src/Scout/ScoutEngine.php | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 38fee69dc..fc0f0e49a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,4 +22,3 @@ services: interval: 10s timeout: 10s retries: 5 -s diff --git a/src/Scout/ScoutEngine.php b/src/Scout/ScoutEngine.php index db5ef8373..72f897c8f 100644 --- a/src/Scout/ScoutEngine.php +++ b/src/Scout/ScoutEngine.php @@ -21,6 +21,7 @@ use MongoDB\Exception\RuntimeException as MongoDBRuntimeException; use MongoDB\Laravel\Connection; use Override; +use stdClass; use Traversable; use TypeError; @@ -212,6 +213,13 @@ private function performSearch(Builder $builder, ?int $offset = null): array return $cursor->toArray(); } + // Using compound to combine search operators + // https://www.mongodb.com/docs/atlas/atlas-search/compound/#options + // "should" specifies conditions that contribute to the relevance score + // at least one of them must match, + // - "text" search for the text including fuzzy matching + // - "wildcard" allows special characters like * and ?, similar to LIKE in SQL + // These are the only search operators to accept wildcard path. $compound = [ 'should' => [ [ @@ -236,7 +244,6 @@ private function performSearch(Builder $builder, ?int $offset = null): array // "filter" specifies conditions on exact values to match // "mustNot" specifies conditions on exact values that must not match // They don't contribute to the relevance score - // https://www.mongodb.com/docs/atlas/atlas-search/compound/#options foreach ($builder->wheres as $field => $value) { if ($field === '__soft_deleted') { $value = (bool) $value; @@ -378,8 +385,9 @@ private function performMap(Builder $builder, array $results, Model $model, bool * This is an estimate if the count is larger than 1000. * * @see Engine::getTotalCount() + * @see https://www.mongodb.com/docs/atlas/atlas-search/counting/ * - * @param mixed $results + * @param stdClass[] $results */ #[Override] public function getTotalCount($results): int @@ -388,6 +396,8 @@ public function getTotalCount($results): int return 0; } + // __count field is added by the aggregation pipeline in performSearch() + // using the count.lowerBound in the $search stage return $results[0]->__count; } From cc3126652219c06a97e8706ebb2a12ddabbdfd45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Tamarelle?= Date: Tue, 14 Jan 2025 21:47:03 +0100 Subject: [PATCH 8/8] Explicit $setOnInsert not necessary --- src/Scout/ScoutEngine.php | 3 ++- tests/Scout/ScoutEngineTest.php | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Scout/ScoutEngine.php b/src/Scout/ScoutEngine.php index 72f897c8f..e3c9c68c3 100644 --- a/src/Scout/ScoutEngine.php +++ b/src/Scout/ScoutEngine.php @@ -126,7 +126,8 @@ public function update($models) 'updateOne' => [ ['_id' => $model->getScoutKey()], [ - '$setOnInsert' => ['_id' => $model->getScoutKey()], + // The _id field is added automatically when the document is inserted + // Update all other fields '$set' => $searchableData, ], ['upsert' => true], diff --git a/tests/Scout/ScoutEngineTest.php b/tests/Scout/ScoutEngineTest.php index 0aaa1374f..a079ae530 100644 --- a/tests/Scout/ScoutEngineTest.php +++ b/tests/Scout/ScoutEngineTest.php @@ -485,14 +485,14 @@ public function testUpdate(): void [ 'updateOne' => [ ['_id' => 'key_1'], - ['$setOnInsert' => ['_id' => 'key_1'], '$set' => ['id' => 1, 'date' => new UTCDateTime($date)]], + ['$set' => ['id' => 1, 'date' => new UTCDateTime($date)]], ['upsert' => true], ], ], [ 'updateOne' => [ ['_id' => 'key_2'], - ['$setOnInsert' => ['_id' => 'key_2'], '$set' => ['id' => 2]], + ['$set' => ['id' => 2]], ['upsert' => true], ], ], @@ -525,7 +525,7 @@ public function testUpdateWithSoftDelete(): void [ 'updateOne' => [ ['_id' => 'key_1'], - ['$setOnInsert' => ['_id' => 'key_1'], '$set' => ['id' => 1, '__soft_deleted' => false]], + ['$set' => ['id' => 1, '__soft_deleted' => false]], ['upsert' => true], ], ],