diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml index 4fea1b84d..16bd213ec 100644 --- a/.github/workflows/build-ci.yml +++ b/.github/workflows/build-ci.yml @@ -65,6 +65,9 @@ jobs: until docker exec --tty mongodb mongosh 127.0.0.1:27017 --eval "db.runCommand({ ping: 1 })"; do sleep 1 done + until docker exec --tty mongodb mongosh 127.0.0.1:27017 --eval "db.createCollection('connection_test') && db.getCollection('connection_test').createSearchIndex({mappings:{dynamic: true}})"; do + sleep 1 + done - name: "Show MongoDB server status" run: | diff --git a/composer.json b/composer.json index 68ec8bc4f..dce593ed5 100644 --- a/composer.json +++ b/composer.json @@ -35,6 +35,7 @@ }, "require-dev": { "mongodb/builder": "^0.2", + "laravel/scout": "^11", "league/flysystem-gridfs": "^3.28", "league/flysystem-read-only": "^3.0", "phpunit/phpunit": "^10.3", diff --git a/docker-compose.yml b/docker-compose.yml index f757ec3cd..fc0f0e49a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.5' - services: app: tty: true @@ -16,11 +14,11 @@ services: mongodb: container_name: mongodb - image: mongo:latest + image: mongodb/mongodb-atlas-local:latest ports: - "27017:27017" healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh mongodb:27017 --quiet + test: mongosh --quiet --eval 'db.runCommand("ping").ok' interval: 10s timeout: 10s retries: 5 diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 7b34210ad..737e31f17 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -24,3 +24,8 @@ parameters: message: "#^Method Illuminate\\\\Database\\\\Schema\\\\Blueprint\\:\\:create\\(\\) invoked with 1 parameter, 0 required\\.$#" count: 1 path: src/Schema/Builder.php + + - + message: "#^Call to an undefined method Illuminate\\\\Support\\\\HigherOrderCollectionProxy\\<\\(int\\|string\\), Illuminate\\\\Database\\\\Eloquent\\\\Model\\>\\:\\:pushSoftDeleteMetadata\\(\\)\\.$#" + count: 1 + path: src/Scout/ScoutEngine.php diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 5431164d8..7044f9069 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -17,7 +17,7 @@ - + diff --git a/src/MongoDBServiceProvider.php b/src/MongoDBServiceProvider.php index 9db2122dc..b0c085b8e 100644 --- a/src/MongoDBServiceProvider.php +++ b/src/MongoDBServiceProvider.php @@ -7,12 +7,14 @@ use Closure; use Illuminate\Cache\CacheManager; use Illuminate\Cache\Repository; +use Illuminate\Container\Container; use Illuminate\Filesystem\FilesystemAdapter; use Illuminate\Filesystem\FilesystemManager; use Illuminate\Foundation\Application; use Illuminate\Session\SessionManager; use Illuminate\Support\ServiceProvider; use InvalidArgumentException; +use Laravel\Scout\EngineManager; use League\Flysystem\Filesystem; use League\Flysystem\GridFS\GridFSAdapter; use League\Flysystem\ReadOnly\ReadOnlyFilesystemAdapter; @@ -20,6 +22,7 @@ use MongoDB\Laravel\Cache\MongoStore; use MongoDB\Laravel\Eloquent\Model; use MongoDB\Laravel\Queue\MongoConnector; +use MongoDB\Laravel\Scout\ScoutEngine; use RuntimeException; use Symfony\Component\HttpFoundation\Session\Storage\Handler\MongoDbSessionHandler; @@ -102,6 +105,7 @@ public function register() }); $this->registerFlysystemAdapter(); + $this->registerScoutEngine(); } private function registerFlysystemAdapter(): void @@ -155,4 +159,21 @@ private function registerFlysystemAdapter(): void }); }); } + + private function registerScoutEngine(): void + { + $this->app->resolving(EngineManager::class, function (EngineManager $engineManager) { + $engineManager->extend('mongodb', function (Container $app) { + $connectionName = $app->get('config')->get('scout.mongodb.connection', 'mongodb'); + $connection = $app->get('db')->connection($connectionName); + $softDelete = (bool) $app->get('config')->get('scout.soft_delete', false); + + assert($connection instanceof Connection, new InvalidArgumentException(sprintf('The connection "%s" is not a MongoDB connection.', $connectionName))); + + return new ScoutEngine($connection->getMongoDB(), $softDelete); + }); + + return $engineManager; + }); + } } diff --git a/src/Scout/ScoutEngine.php b/src/Scout/ScoutEngine.php new file mode 100644 index 000000000..e3c9c68c3 --- /dev/null +++ b/src/Scout/ScoutEngine.php @@ -0,0 +1,551 @@ + [ + 'dynamic' => true, + ], + ]; + + private const TYPEMAP = ['root' => 'object', 'document' => 'bson', 'array' => 'bson']; + + public function __construct( + private Database $database, + private bool $softDelete, + ) { + } + + /** + * Update the given model in the index. + * + * @see Engine::update() + * + * @param EloquentCollection $models + * + * @throws MongoDBRuntimeException + */ + #[Override] + public function update($models) + { + assert($models instanceof EloquentCollection, new TypeError(sprintf('Argument #1 ($models) must be of type %s, %s given', EloquentCollection::class, get_debug_type($models)))); + + if ($models->isEmpty()) { + return; + } + + if ($this->softDelete && $this->usesSoftDelete($models)) { + $models->each->pushSoftDeleteMetadata(); + } + + $bulk = []; + foreach ($models as $model) { + assert($model instanceof Model && method_exists($model, 'toSearchableArray'), new LogicException(sprintf('Model "%s" must use "%s" trait', $model::class, Searchable::class))); + + $searchableData = $model->toSearchableArray(); + $searchableData = self::serialize($searchableData); + + // Skip/remove the model if it doesn't provide any searchable data + if (! $searchableData) { + $bulk[] = [ + 'deleteOne' => [ + ['_id' => $model->getScoutKey()], + ], + ]; + + continue; + } + + unset($searchableData['_id']); + + $searchableData = array_merge($searchableData, $model->scoutMetadata()); + + /** Convert the __soft_deleted set by {@see Searchable::pushSoftDeleteMetadata()} + * into a boolean for efficient storage and indexing. */ + if (isset($searchableData['__soft_deleted'])) { + $searchableData['__soft_deleted'] = (bool) $searchableData['__soft_deleted']; + } + + $bulk[] = [ + 'updateOne' => [ + ['_id' => $model->getScoutKey()], + [ + // The _id field is added automatically when the document is inserted + // Update all other fields + '$set' => $searchableData, + ], + ['upsert' => true], + ], + ]; + } + + $this->getIndexableCollection($models)->bulkWrite($bulk); + } + + /** + * Remove the given model from the index. + * + * @see Engine::delete() + * + * @param EloquentCollection $models + */ + #[Override] + public function delete($models): void + { + assert($models instanceof EloquentCollection, new TypeError(sprintf('Argument #1 ($models) must be of type %s, %s given', Collection::class, get_debug_type($models)))); + + if ($models->isEmpty()) { + return; + } + + $collection = $this->getIndexableCollection($models); + $ids = $models->map(fn (Model $model) => $model->getScoutKey())->all(); + $collection->deleteMany(['_id' => ['$in' => $ids]]); + } + + /** + * Perform the given search on the engine. + * + * @see Engine::search() + * + * @return array + */ + #[Override] + public function search(Builder $builder) + { + return $this->performSearch($builder); + } + + /** + * Perform the given search on the engine with pagination. + * + * @see Engine::paginate() + * + * @param int $perPage + * @param int $page + * + * @return array + */ + #[Override] + public function paginate(Builder $builder, $perPage, $page) + { + assert(is_int($perPage), new TypeError(sprintf('Argument #2 ($perPage) must be of type int, %s given', get_debug_type($perPage)))); + assert(is_int($page), new TypeError(sprintf('Argument #3 ($page) must be of type int, %s given', get_debug_type($page)))); + + $builder = clone $builder; + $builder->take($perPage); + + return $this->performSearch($builder, $perPage * ($page - 1)); + } + + /** + * Perform the given search on the engine. + */ + private function performSearch(Builder $builder, ?int $offset = null): array + { + $collection = $this->getSearchableCollection($builder->model); + + if ($builder->callback) { + $cursor = call_user_func( + $builder->callback, + $collection, + $builder->query, + $offset, + ); + assert($cursor instanceof CursorInterface, new LogicException(sprintf('The search builder closure must return a MongoDB cursor, %s returned', get_debug_type($cursor)))); + $cursor->setTypeMap(self::TYPEMAP); + + return $cursor->toArray(); + } + + // Using compound to combine search operators + // https://www.mongodb.com/docs/atlas/atlas-search/compound/#options + // "should" specifies conditions that contribute to the relevance score + // at least one of them must match, + // - "text" search for the text including fuzzy matching + // - "wildcard" allows special characters like * and ?, similar to LIKE in SQL + // These are the only search operators to accept wildcard path. + $compound = [ + 'should' => [ + [ + 'text' => [ + 'query' => $builder->query, + 'path' => ['wildcard' => '*'], + 'fuzzy' => ['maxEdits' => 2], + 'score' => ['boost' => ['value' => 5]], + ], + ], + [ + 'wildcard' => [ + 'query' => $builder->query . '*', + 'path' => ['wildcard' => '*'], + 'allowAnalyzedField' => true, + ], + ], + ], + 'minimumShouldMatch' => 1, + ]; + + // "filter" specifies conditions on exact values to match + // "mustNot" specifies conditions on exact values that must not match + // They don't contribute to the relevance score + foreach ($builder->wheres as $field => $value) { + if ($field === '__soft_deleted') { + $value = (bool) $value; + } + + $compound['filter'][] = ['equals' => ['path' => $field, 'value' => $value]]; + } + + foreach ($builder->whereIns as $field => $value) { + $compound['filter'][] = ['in' => ['path' => $field, 'value' => $value]]; + } + + foreach ($builder->whereNotIns as $field => $value) { + $compound['mustNot'][] = ['in' => ['path' => $field, 'value' => $value]]; + } + + // Sort by field value only if specified + $sort = []; + foreach ($builder->orders as $order) { + $sort[$order['column']] = $order['direction'] === 'asc' ? 1 : -1; + } + + $pipeline = [ + [ + '$search' => [ + 'index' => self::INDEX_NAME, + 'compound' => $compound, + 'count' => ['type' => 'lowerBound'], + ...($sort ? ['sort' => $sort] : []), + ], + ], + [ + '$addFields' => [ + // Metadata field with the total count of documents + '__count' => '$$SEARCH_META.count.lowerBound', + ], + ], + ]; + + if ($offset) { + $pipeline[] = ['$skip' => $offset]; + } + + if ($builder->limit) { + $pipeline[] = ['$limit' => $builder->limit]; + } + + $cursor = $collection->aggregate($pipeline); + $cursor->setTypeMap(self::TYPEMAP); + + return $cursor->toArray(); + } + + /** + * Pluck and return the primary keys of the given results. + * + * @see Engine::mapIds() + * + * @param list $results + */ + #[Override] + public function mapIds($results): Collection + { + assert(is_array($results), new TypeError(sprintf('Argument #1 ($results) must be of type array, %s given', get_debug_type($results)))); + + return new Collection(array_column($results, '_id')); + } + + /** + * Map the given results to instances of the given model. + * + * @see Engine::map() + * + * @param Builder $builder + * @param array $results + * @param Model $model + * + * @return Collection + */ + #[Override] + public function map(Builder $builder, $results, $model): Collection + { + return $this->performMap($builder, $results, $model, false); + } + + /** + * Map the given results to instances of the given model via a lazy collection. + * + * @see Engine::lazyMap() + * + * @param Builder $builder + * @param array $results + * @param Model $model + * + * @return LazyCollection + */ + #[Override] + public function lazyMap(Builder $builder, $results, $model): LazyCollection + { + return $this->performMap($builder, $results, $model, true); + } + + /** @return ($lazy is true ? LazyCollection : Collection) */ + private function performMap(Builder $builder, array $results, Model $model, bool $lazy): Collection|LazyCollection + { + if (! $results) { + $collection = $model->newCollection(); + + return $lazy ? LazyCollection::make($collection) : $collection; + } + + $objectIds = array_column($results, '_id'); + $objectIdPositions = array_flip($objectIds); + + return $model->queryScoutModelsByIds($builder, $objectIds) + ->{$lazy ? 'cursor' : 'get'}() + ->filter(function ($model) use ($objectIds) { + return in_array($model->getScoutKey(), $objectIds); + }) + ->map(function ($model) use ($results, $objectIdPositions) { + $result = $results[$objectIdPositions[$model->getScoutKey()]] ?? []; + + foreach ($result as $key => $value) { + if ($key[0] === '_' && $key !== '_id') { + $model->withScoutMetadata($key, $value); + } + } + + return $model; + }) + ->sortBy(function ($model) use ($objectIdPositions) { + return $objectIdPositions[$model->getScoutKey()]; + }) + ->values(); + } + + /** + * Get the total count from a raw result returned by the engine. + * This is an estimate if the count is larger than 1000. + * + * @see Engine::getTotalCount() + * @see https://www.mongodb.com/docs/atlas/atlas-search/counting/ + * + * @param stdClass[] $results + */ + #[Override] + public function getTotalCount($results): int + { + if (! $results) { + return 0; + } + + // __count field is added by the aggregation pipeline in performSearch() + // using the count.lowerBound in the $search stage + return $results[0]->__count; + } + + /** + * Flush all records from the engine. + * + * @see Engine::flush() + * + * @param Model $model + */ + #[Override] + public function flush($model): void + { + assert($model instanceof Model, new TypeError(sprintf('Argument #1 ($model) must be of type %s, %s given', Model::class, get_debug_type($model)))); + + $collection = $this->getIndexableCollection($model); + + $collection->deleteMany([]); + } + + /** + * Create the MongoDB Atlas Search index. + * + * Accepted options: + * - wait: bool, default true. Wait for the index to be created. + * + * @see Engine::createIndex() + * + * @param string $name Collection name + * @param array{wait?:bool} $options + */ + #[Override] + public function createIndex($name, array $options = []): void + { + assert(is_string($name), new TypeError(sprintf('Argument #1 ($name) must be of type string, %s given', get_debug_type($name)))); + + // Ensure the collection exists before creating the search index + $this->database->createCollection($name); + + $collection = $this->database->selectCollection($name); + $collection->createSearchIndex( + self::DEFAULT_DEFINITION, + ['name' => self::INDEX_NAME], + ); + + if ($options['wait'] ?? true) { + $this->wait(function () use ($collection) { + $indexes = $collection->listSearchIndexes([ + 'name' => self::INDEX_NAME, + 'typeMap' => ['root' => 'bson'], + ]); + + return $indexes->current() && $indexes->current()->status === 'READY'; + }); + } + } + + /** + * Delete a "search index", i.e. a MongoDB collection. + * + * @see Engine::deleteIndex() + */ + #[Override] + public function deleteIndex($name): void + { + assert(is_string($name), new TypeError(sprintf('Argument #1 ($name) must be of type string, %s given', get_debug_type($name)))); + + $this->database->dropCollection($name); + } + + /** Get the MongoDB collection used to search for the provided model */ + private function getSearchableCollection(Model|EloquentCollection $model): MongoDBCollection + { + if ($model instanceof EloquentCollection) { + $model = $model->first(); + } + + assert(method_exists($model, 'searchableAs'), sprintf('Model "%s" must use "%s" trait', $model::class, Searchable::class)); + + return $this->database->selectCollection($model->searchableAs()); + } + + /** Get the MongoDB collection used to index the provided model */ + private function getIndexableCollection(Model|EloquentCollection $model): MongoDBCollection + { + if ($model instanceof EloquentCollection) { + $model = $model->first(); + } + + assert($model instanceof Model); + assert(method_exists($model, 'indexableAs'), sprintf('Model "%s" must use "%s" trait', $model::class, Searchable::class)); + + if ( + $model->getConnection() instanceof Connection + && $model->getConnection()->getDatabaseName() === $this->database->getDatabaseName() + && $model->getTable() === $model->indexableAs() + ) { + throw new LogicException(sprintf('The MongoDB Scout collection "%s.%s" must use a different collection from the collection name of the model "%s". Set the "scout.prefix" configuration or use a distinct MongoDB database', $this->database->getDatabaseName(), $model->indexableAs(), $model::class)); + } + + return $this->database->selectCollection($model->indexableAs()); + } + + private static function serialize(mixed $value): mixed + { + if ($value instanceof DateTimeInterface) { + return new UTCDateTime($value); + } + + if ($value instanceof Serializable || ! is_iterable($value)) { + return $value; + } + + // Convert Laravel Collections and other Iterators to arrays + if ($value instanceof Traversable) { + $value = iterator_to_array($value); + } + + // Recursively serialize arrays + return array_map(self::serialize(...), $value); + } + + private function usesSoftDelete(Model|EloquentCollection $model): bool + { + if ($model instanceof EloquentCollection) { + $model = $model->first(); + } + + return in_array(SoftDeletes::class, class_uses_recursive($model)); + } + + /** + * Wait for the callback to return true, use it for asynchronous + * Atlas Search index management operations. + */ + private function wait(Closure $callback): void + { + // Fallback to time() if hrtime() is not supported + $timeout = (hrtime()[0] ?? time()) + self::WAIT_TIMEOUT_SEC; + while ((hrtime()[0] ?? time()) < $timeout) { + if ($callback()) { + return; + } + + sleep(1); + } + + throw new MongoDBRuntimeException(sprintf('Atlas search index operation time out after %s seconds', self::WAIT_TIMEOUT_SEC)); + } +} diff --git a/tests/ModelTest.php b/tests/ModelTest.php index c532eea55..ef71a5fe0 100644 --- a/tests/ModelTest.php +++ b/tests/ModelTest.php @@ -406,8 +406,9 @@ public function testSoftDelete(): void $this->assertEquals(2, Soft::count()); } + /** @param class-string $model */ #[DataProvider('provideId')] - public function testPrimaryKey(string $model, $id, $expected, bool $expectedFound): void + public function testPrimaryKey(string $model, mixed $id, mixed $expected, bool $expectedFound): void { $model::truncate(); $expectedType = get_debug_type($expected); diff --git a/tests/Models/SchemaVersion.php b/tests/Models/SchemaVersion.php index 8acd73545..b142d8bda 100644 --- a/tests/Models/SchemaVersion.php +++ b/tests/Models/SchemaVersion.php @@ -5,9 +5,9 @@ namespace MongoDB\Laravel\Tests\Models; use MongoDB\Laravel\Eloquent\HasSchemaVersion; -use MongoDB\Laravel\Eloquent\Model as Eloquent; +use MongoDB\Laravel\Eloquent\Model; -class SchemaVersion extends Eloquent +class SchemaVersion extends Model { use HasSchemaVersion; diff --git a/tests/Scout/Models/ScoutUser.php b/tests/Scout/Models/ScoutUser.php new file mode 100644 index 000000000..50fa39a94 --- /dev/null +++ b/tests/Scout/Models/ScoutUser.php @@ -0,0 +1,43 @@ +dropIfExists('scout_users'); + $schema->create('scout_users', function (Blueprint $table) { + $table->increments('id'); + $table->string('name'); + $table->string('email')->nullable(); + $table->date('email_verified_at')->nullable(); + $table->timestamps(); + $table->softDeletes(); + }); + } +} diff --git a/tests/Scout/Models/SearchableInSameNamespace.php b/tests/Scout/Models/SearchableInSameNamespace.php new file mode 100644 index 000000000..91b909067 --- /dev/null +++ b/tests/Scout/Models/SearchableInSameNamespace.php @@ -0,0 +1,30 @@ +getTable(); + } +} diff --git a/tests/Scout/Models/SearchableModel.php b/tests/Scout/Models/SearchableModel.php new file mode 100644 index 000000000..e53200f1a --- /dev/null +++ b/tests/Scout/Models/SearchableModel.php @@ -0,0 +1,50 @@ +getAttribute($this->getScoutKeyName()) ?: 'key_' . $this->getKey(); + } + + /** + * This method must be overridden when the `getScoutKey` method is also overridden, + * to support model serialization for async indexing jobs. + * + * @see Searchable::getScoutKeyName() + */ + public function getScoutKeyName(): string + { + return 'scout_key'; + } +} diff --git a/tests/Scout/ScoutEngineTest.php b/tests/Scout/ScoutEngineTest.php new file mode 100644 index 000000000..a079ae530 --- /dev/null +++ b/tests/Scout/ScoutEngineTest.php @@ -0,0 +1,582 @@ + 'object', 'document' => 'bson', 'array' => 'bson']; + + /** @param callable(): Builder $builder */ + #[DataProvider('provideSearchPipelines')] + public function testSearch(Closure $builder, array $expectedPipeline): void + { + $data = [['_id' => 'key_1', '__count' => 15], ['_id' => 'key_2', '__count' => 15]]; + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_searchable') + ->andReturn($collection); + $cursor = m::mock(CursorInterface::class); + $cursor->shouldReceive('setTypeMap')->once()->with(self::EXPECTED_TYPEMAP); + $cursor->shouldReceive('toArray')->once()->with()->andReturn($data); + + $collection->shouldReceive('getCollectionName') + ->zeroOrMoreTimes() + ->andReturn('collection_searchable'); + $collection->shouldReceive('aggregate') + ->once() + ->withArgs(function ($pipeline) use ($expectedPipeline) { + self::assertEquals($expectedPipeline, $pipeline); + + return true; + }) + ->andReturn($cursor); + + $engine = new ScoutEngine($database, softDelete: false); + $result = $engine->search($builder()); + $this->assertEquals($data, $result); + } + + public function provideSearchPipelines(): iterable + { + $defaultPipeline = [ + [ + '$search' => [ + 'index' => 'scout', + 'compound' => [ + 'should' => [ + [ + 'text' => [ + 'path' => ['wildcard' => '*'], + 'query' => 'lar', + 'fuzzy' => ['maxEdits' => 2], + 'score' => ['boost' => ['value' => 5]], + ], + ], + [ + 'wildcard' => [ + 'query' => 'lar*', + 'path' => ['wildcard' => '*'], + 'allowAnalyzedField' => true, + ], + ], + ], + 'minimumShouldMatch' => 1, + ], + 'count' => [ + 'type' => 'lowerBound', + ], + ], + ], + [ + '$addFields' => [ + '__count' => '$$SEARCH_META.count.lowerBound', + ], + ], + ]; + + yield 'simple string' => [ + function () { + return new Builder(new SearchableModel(), 'lar'); + }, + $defaultPipeline, + ]; + + yield 'where conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->where('foo', 'bar'); + $builder->where('key', 'value'); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => 'foo', 'value' => 'bar']], + ['equals' => ['path' => 'key', 'value' => 'value']], + ], + ], + ], + ], + ]), + ]; + + yield 'where in conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->where('foo', 'bar'); + $builder->where('bar', 'baz'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => 'foo', 'value' => 'bar']], + ['equals' => ['path' => 'bar', 'value' => 'baz']], + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + ], + ], + ], + ]), + ]; + + yield 'where not in conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->where('foo', 'bar'); + $builder->where('bar', 'baz'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + $builder->whereNotIn('eaea', [3]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => 'foo', 'value' => 'bar']], + ['equals' => ['path' => 'bar', 'value' => 'baz']], + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + 'mustNot' => [ + ['in' => ['path' => 'eaea', 'value' => [3]]], + ], + ], + ], + ], + ]), + ]; + + yield 'where in conditions without other conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + ], + ], + ], + ]), + ]; + + yield 'where not in conditions without other conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + $builder->whereNotIn('eaea', [3]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + 'mustNot' => [ + ['in' => ['path' => 'eaea', 'value' => [3]]], + ], + ], + ], + ], + ]), + ]; + + yield 'empty where in conditions' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->whereIn('qux', [1, 2]); + $builder->whereIn('quux', [1, 2]); + $builder->whereNotIn('eaea', [3]); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['in' => ['path' => 'qux', 'value' => [1, 2]]], + ['in' => ['path' => 'quux', 'value' => [1, 2]]], + ], + 'mustNot' => [ + ['in' => ['path' => 'eaea', 'value' => [3]]], + ], + ], + ], + ], + ]), + ]; + + yield 'exclude soft-deleted' => [ + function () { + return new Builder(new SearchableModel(), 'lar', softDelete: true); + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => '__soft_deleted', 'value' => false]], + ], + ], + ], + ], + ]), + ]; + + yield 'only trashed' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar', softDelete: true); + $builder->onlyTrashed(); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'compound' => [ + 'filter' => [ + ['equals' => ['path' => '__soft_deleted', 'value' => true]], + ], + ], + ], + ], + ]), + ]; + + yield 'with callback' => [ + fn () => new Builder(new SearchableModel(), 'query', callback: function (...$args) { + $this->assertCount(3, $args); + $this->assertInstanceOf(Collection::class, $args[0]); + $this->assertSame('collection_searchable', $args[0]->getCollectionName()); + $this->assertSame('query', $args[1]); + $this->assertNull($args[2]); + + return $args[0]->aggregate(['pipeline']); + }), + ['pipeline'], + ]; + + yield 'ordered' => [ + function () { + $builder = new Builder(new SearchableModel(), 'lar'); + $builder->orderBy('name', 'desc'); + $builder->orderBy('age', 'asc'); + + return $builder; + }, + array_replace_recursive($defaultPipeline, [ + [ + '$search' => [ + 'sort' => [ + 'name' => -1, + 'age' => 1, + ], + ], + ], + ]), + ]; + } + + public function testPaginate() + { + $perPage = 5; + $page = 3; + + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $cursor = m::mock(CursorInterface::class); + $database->shouldReceive('selectCollection') + ->with('collection_searchable') + ->andReturn($collection); + $collection->shouldReceive('aggregate') + ->once() + ->withArgs(function (...$args) { + self::assertSame([ + [ + '$search' => [ + 'index' => 'scout', + 'compound' => [ + 'should' => [ + [ + 'text' => [ + 'query' => 'mustang', + 'path' => ['wildcard' => '*'], + 'fuzzy' => ['maxEdits' => 2], + 'score' => ['boost' => ['value' => 5]], + ], + ], + [ + 'wildcard' => [ + 'query' => 'mustang*', + 'path' => ['wildcard' => '*'], + 'allowAnalyzedField' => true, + ], + ], + ], + 'minimumShouldMatch' => 1, + ], + 'count' => [ + 'type' => 'lowerBound', + ], + 'sort' => [ + 'name' => -1, + ], + ], + ], + [ + '$addFields' => [ + '__count' => '$$SEARCH_META.count.lowerBound', + ], + ], + [ + '$skip' => 10, + ], + [ + '$limit' => 5, + ], + ], $args[0]); + + return true; + }) + ->andReturn($cursor); + $cursor->shouldReceive('setTypeMap')->once()->with(self::EXPECTED_TYPEMAP); + $cursor->shouldReceive('toArray') + ->once() + ->with() + ->andReturn([['_id' => 'key_1', '__count' => 17], ['_id' => 'key_2', '__count' => 17]]); + + $engine = new ScoutEngine($database, softDelete: false); + $builder = new Builder(new SearchableModel(), 'mustang'); + $builder->orderBy('name', 'desc'); + $engine->paginate($builder, $perPage, $page); + } + + public function testMapMethodRespectsOrder() + { + $database = m::mock(Database::class); + $engine = new ScoutEngine($database, false); + + $model = m::mock(Model::class); + $model->shouldReceive(['getScoutKeyName' => 'id']); + $model->shouldReceive('queryScoutModelsByIds->get') + ->andReturn(LaravelCollection::make([ + new ScoutUser(['id' => 1]), + new ScoutUser(['id' => 2]), + new ScoutUser(['id' => 3]), + new ScoutUser(['id' => 4]), + ])); + + $builder = m::mock(Builder::class); + + $results = $engine->map($builder, [ + ['_id' => 1, '__count' => 4], + ['_id' => 2, '__count' => 4], + ['_id' => 4, '__count' => 4], + ['_id' => 3, '__count' => 4], + ], $model); + + $this->assertEquals(4, count($results)); + $this->assertEquals([ + 0 => ['id' => 1], + 1 => ['id' => 2], + 2 => ['id' => 4], + 3 => ['id' => 3], + ], $results->toArray()); + } + + public function testLazyMapMethodRespectsOrder() + { + $lazy = false; + $database = m::mock(Database::class); + $engine = new ScoutEngine($database, false); + + $model = m::mock(Model::class); + $model->shouldReceive(['getScoutKeyName' => 'id']); + $model->shouldReceive('queryScoutModelsByIds->cursor') + ->andReturn(LazyCollection::make([ + new ScoutUser(['id' => 1]), + new ScoutUser(['id' => 2]), + new ScoutUser(['id' => 3]), + new ScoutUser(['id' => 4]), + ])); + + $builder = m::mock(Builder::class); + + $results = $engine->lazyMap($builder, [ + ['_id' => 1, '__count' => 4], + ['_id' => 2, '__count' => 4], + ['_id' => 4, '__count' => 4], + ['_id' => 3, '__count' => 4], + ], $model); + + $this->assertEquals(4, count($results)); + $this->assertEquals([ + 0 => ['id' => 1], + 1 => ['id' => 2], + 2 => ['id' => 4], + 3 => ['id' => 3], + ], $results->toArray()); + } + + public function testUpdate(): void + { + $date = new DateTimeImmutable('2000-01-02 03:04:05'); + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_indexable') + ->andReturn($collection); + $collection->shouldReceive('bulkWrite') + ->once() + ->with([ + [ + 'updateOne' => [ + ['_id' => 'key_1'], + ['$set' => ['id' => 1, 'date' => new UTCDateTime($date)]], + ['upsert' => true], + ], + ], + [ + 'updateOne' => [ + ['_id' => 'key_2'], + ['$set' => ['id' => 2]], + ['upsert' => true], + ], + ], + ]); + + $engine = new ScoutEngine($database, softDelete: false); + $engine->update(EloquentCollection::make([ + new SearchableModel([ + 'id' => 1, + 'date' => $date, + ]), + new SearchableModel([ + 'id' => 2, + ]), + ])); + } + + public function testUpdateWithSoftDelete(): void + { + $date = new DateTimeImmutable('2000-01-02 03:04:05'); + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_indexable') + ->andReturn($collection); + $collection->shouldReceive('bulkWrite') + ->once() + ->withArgs(function ($pipeline) { + $this->assertSame([ + [ + 'updateOne' => [ + ['_id' => 'key_1'], + ['$set' => ['id' => 1, '__soft_deleted' => false]], + ['upsert' => true], + ], + ], + ], $pipeline); + + return true; + }); + + $model = new SearchableModel(['id' => 1]); + $model->delete(); + + $engine = new ScoutEngine($database, softDelete: true); + $engine->update(EloquentCollection::make([$model])); + } + + public function testDelete(): void + { + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_indexable') + ->andReturn($collection); + $collection->shouldReceive('deleteMany') + ->once() + ->with(['_id' => ['$in' => ['key_1', 'key_2']]]); + + $engine = new ScoutEngine($database, softDelete: false); + $engine->delete(EloquentCollection::make([ + new SearchableModel(['id' => 1]), + new SearchableModel(['id' => 2]), + ])); + } + + public function testDeleteWithRemoveableScoutCollection(): void + { + $job = new RemoveFromSearch(EloquentCollection::make([ + new SearchableModel(['id' => 5, 'scout_key' => 'key_5']), + ])); + + $job = unserialize(serialize($job)); + + $database = m::mock(Database::class); + $collection = m::mock(Collection::class); + $database->shouldReceive('selectCollection') + ->with('collection_indexable') + ->andReturn($collection); + $collection->shouldReceive('deleteMany') + ->once() + ->with(['_id' => ['$in' => ['key_5']]]); + + $engine = new ScoutEngine($database, softDelete: false); + $engine->delete($job->models); + } +} diff --git a/tests/Scout/ScoutIntegrationTest.php b/tests/Scout/ScoutIntegrationTest.php new file mode 100644 index 000000000..7b9d704f6 --- /dev/null +++ b/tests/Scout/ScoutIntegrationTest.php @@ -0,0 +1,262 @@ +set('scout.driver', 'mongodb'); + $app['config']->set('scout.prefix', 'prefix_'); + } + + public function setUp(): void + { + parent::setUp(); + + $this->skipIfSearchIndexManagementIsNotSupported(); + + // Init the SQL database with some objects that will be indexed + // Test data copied from Laravel Scout tests + // https://github.com/laravel/scout/blob/10.x/tests/Integration/SearchableTests.php + ScoutUser::executeSchema(); + + $collect = LazyCollection::make(function () { + yield ['name' => 'Laravel Framework']; + + foreach (range(2, 10) as $key) { + yield ['name' => 'Example ' . $key]; + } + + yield ['name' => 'Larry Casper', 'email_verified_at' => null]; + yield ['name' => 'Reta Larkin']; + + foreach (range(13, 19) as $key) { + yield ['name' => 'Example ' . $key]; + } + + yield ['name' => 'Prof. Larry Prosacco DVM', 'email_verified_at' => null]; + + foreach (range(21, 38) as $key) { + yield ['name' => 'Example ' . $key, 'email_verified_at' => null]; + } + + yield ['name' => 'Linkwood Larkin', 'email_verified_at' => null]; + yield ['name' => 'Otis Larson MD']; + yield ['name' => 'Gudrun Larkin']; + yield ['name' => 'Dax Larkin']; + yield ['name' => 'Dana Larson Sr.']; + yield ['name' => 'Amos Larson Sr.']; + }); + + $id = 0; + $date = new DateTimeImmutable('2021-01-01 00:00:00'); + foreach ($collect as $data) { + $data = array_merge(['id' => ++$id, 'email_verified_at' => $date], $data); + ScoutUser::create($data)->save(); + } + + self::assertSame(44, ScoutUser::count()); + } + + /** This test create the search index for tests performing search */ + public function testItCanCreateTheCollection() + { + $collection = DB::connection('mongodb')->getCollection('prefix_scout_users'); + $collection->drop(); + + // Recreate the indexes using the artisan commands + // Ensure they return a success exit code (0) + self::assertSame(0, artisan($this, 'scout:delete-index', ['name' => ScoutUser::class])); + self::assertSame(0, artisan($this, 'scout:index', ['name' => ScoutUser::class])); + self::assertSame(0, artisan($this, 'scout:import', ['model' => ScoutUser::class])); + + self::assertSame(44, $collection->countDocuments()); + + $searchIndexes = $collection->listSearchIndexes(['name' => 'scout']); + self::assertCount(1, $searchIndexes); + + // Wait for all documents to be indexed asynchronously + $i = 100; + while (true) { + $indexedDocuments = $collection->aggregate([ + ['$search' => ['index' => 'scout', 'exists' => ['path' => 'name']]], + ])->toArray(); + + if (count($indexedDocuments) >= 44) { + break; + } + + if ($i-- === 0) { + self::fail('Documents not indexed'); + } + + usleep(100_000); + } + + self::assertCount(44, $indexedDocuments); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearch() + { + // All the search queries use "sort" option to ensure the results are deterministic + $results = ScoutUser::search('lar')->take(10)->orderBy('id')->get(); + + self::assertSame([ + 1 => 'Laravel Framework', + 11 => 'Larry Casper', + 12 => 'Reta Larkin', + 20 => 'Prof. Larry Prosacco DVM', + 39 => 'Linkwood Larkin', + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $results->pluck('name', 'id')->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearchCursor() + { + // All the search queries use "sort" option to ensure the results are deterministic + $results = ScoutUser::search('lar')->take(10)->orderBy('id')->cursor(); + + self::assertSame([ + 1 => 'Laravel Framework', + 11 => 'Larry Casper', + 12 => 'Reta Larkin', + 20 => 'Prof. Larry Prosacco DVM', + 39 => 'Linkwood Larkin', + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $results->pluck('name', 'id')->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearchWithQueryCallback() + { + $results = ScoutUser::search('lar')->take(10)->orderBy('id')->query(function ($query) { + return $query->whereNotNull('email_verified_at'); + })->get(); + + self::assertSame([ + 1 => 'Laravel Framework', + 12 => 'Reta Larkin', + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $results->pluck('name', 'id')->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearchToFetchKeys() + { + $results = ScoutUser::search('lar')->orderBy('id')->take(10)->keys(); + + self::assertSame([1, 11, 12, 20, 39, 40, 41, 42, 43, 44], $results->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUseBasicSearchWithQueryCallbackToFetchKeys() + { + $results = ScoutUser::search('lar')->take(10)->orderBy('id', 'desc')->query(function ($query) { + return $query->whereNotNull('email_verified_at'); + })->keys(); + + self::assertSame([44, 43, 42, 41, 40, 39, 20, 12, 11, 1], $results->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUsePaginatedSearch() + { + $page1 = ScoutUser::search('lar')->take(10)->orderBy('id')->paginate(5, 'page', 1); + $page2 = ScoutUser::search('lar')->take(10)->orderBy('id')->paginate(5, 'page', 2); + + self::assertSame([ + 1 => 'Laravel Framework', + 11 => 'Larry Casper', + 12 => 'Reta Larkin', + 20 => 'Prof. Larry Prosacco DVM', + 39 => 'Linkwood Larkin', + ], $page1->pluck('name', 'id')->all()); + + self::assertSame([ + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $page2->pluck('name', 'id')->all()); + } + + #[Depends('testItCanCreateTheCollection')] + public function testItCanUsePaginatedSearchWithQueryCallback() + { + $queryCallback = function ($query) { + return $query->whereNotNull('email_verified_at'); + }; + + $page1 = ScoutUser::search('lar')->take(10)->orderBy('id')->query($queryCallback)->paginate(5, 'page', 1); + $page2 = ScoutUser::search('lar')->take(10)->orderBy('id')->query($queryCallback)->paginate(5, 'page', 2); + + self::assertSame([ + 1 => 'Laravel Framework', + 12 => 'Reta Larkin', + ], $page1->pluck('name', 'id')->all()); + + self::assertSame([ + 40 => 'Otis Larson MD', + 41 => 'Gudrun Larkin', + 42 => 'Dax Larkin', + 43 => 'Dana Larson Sr.', + 44 => 'Amos Larson Sr.', + ], $page2->pluck('name', 'id')->all()); + } + + public function testItCannotIndexInTheSameNamespace() + { + self::expectException(LogicException::class); + self::expectExceptionMessage(sprintf( + 'The MongoDB Scout collection "%s.searchable_in_same_namespaces" must use a different collection from the collection name of the model "%s". Set the "scout.prefix" configuration or use a distinct MongoDB database', + env('MONGODB_DATABASE', 'unittest'), + SearchableInSameNamespace::class, + ),); + + SearchableInSameNamespace::create(['name' => 'test']); + } +}