From cab429e22bbb60468b84e677dd79019bdde54f85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Fri, 16 Aug 2024 14:12:40 +0200 Subject: [PATCH 1/9] feat(dsl): Sanitize embedded objects --- dsl/src/main/kotlin/DocumentWriter.kt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dsl/src/main/kotlin/DocumentWriter.kt b/dsl/src/main/kotlin/DocumentWriter.kt index a5aaaa8..adf1f69 100644 --- a/dsl/src/main/kotlin/DocumentWriter.kt +++ b/dsl/src/main/kotlin/DocumentWriter.kt @@ -54,5 +54,11 @@ internal inline fun BsonWriter.writeArray(name: String, block: () -> Unit) { internal fun BsonWriter.writeObject(value: T, codec: CodecRegistry) { @Suppress("UNCHECKED_CAST") // Kotlin doesn't smart-cast here, but should, this is safe (codec.get(value!!::class.java) as Encoder) - .encode(this, value, EncoderContext.builder().build()) + .encode( + this, + value, + EncoderContext.builder() + .isEncodingCollectibleDocument(true) + .build() + ) } From 177adfc612d3c3145c892fddf91688cf4ceedd9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Fri, 16 Aug 2024 14:50:21 +0200 Subject: [PATCH 2/9] build(docker): Add a MongoDB database to the project --- .idea/dataSources.xml | 17 +++++++++++++++++ .idea/runConfigurations/Docker_Compose.xml | 12 ++++++++++++ docker/docker-compose.yml | 9 +++++++++ 3 files changed, 38 insertions(+) create mode 100644 .idea/dataSources.xml create mode 100644 .idea/runConfigurations/Docker_Compose.xml create mode 100644 docker/docker-compose.yml diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml new file mode 100644 index 0000000..04a0647 --- /dev/null +++ b/.idea/dataSources.xml @@ -0,0 +1,17 @@ + + + + + mongo + true + com.dbschema.MongoJdbcDriver + mongodb://localhost:27017 + + + + + + $ProjectFileDir$ + + + \ No newline at end of file diff --git a/.idea/runConfigurations/Docker_Compose.xml b/.idea/runConfigurations/Docker_Compose.xml new file mode 100644 index 0000000..244dadd --- /dev/null +++ b/.idea/runConfigurations/Docker_Compose.xml @@ -0,0 +1,12 @@ + + + + + + + + + \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..964dae7 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,9 @@ + +services: + mongodb: + image: mongo:7 + environment: + - MONGODB_INITDB_ROOT_USERNAME=user + - MONGODB_INITDB_ROOT_PASSWORD=password + ports: + - "27017:27017" From 569fbebd6de08633223bc72bf30f8bc071365fe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Fri, 16 Aug 2024 14:51:14 +0200 Subject: [PATCH 3/9] fix(sync): Fix malformed update --- .../src/main/kotlin/NativeMongoCollection.kt | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/driver-sync/src/main/kotlin/NativeMongoCollection.kt b/driver-sync/src/main/kotlin/NativeMongoCollection.kt index 7ccb7ea..ed04dfb 100644 --- a/driver-sync/src/main/kotlin/NativeMongoCollection.kt +++ b/driver-sync/src/main/kotlin/NativeMongoCollection.kt @@ -32,6 +32,19 @@ class NativeMongoCollection( return bson } + @OptIn(LowLevelApi::class) + private fun CompoundExpression.toNestedBsonDocument(): BsonDocument { + val bson = BsonDocument() + + BsonDocumentWriter(bson).use { writer -> + writer.writeStartDocument() + this.writeTo(writer) + writer.writeEndDocument() + } + + return bson + } + // region Find override fun find(): FindIterable = @@ -94,7 +107,7 @@ class NativeMongoCollection( val updateBson = UpdateExpression(unsafe.codecRegistry) .apply(update) - .toBsonDocument() + .toNestedBsonDocument() return when (val session = getCurrentSession()) { null -> unsafe.updateOne(filterBson, updateBson, options) @@ -113,7 +126,7 @@ class NativeMongoCollection( val updateBson = UpdateExpression(unsafe.codecRegistry) .apply(update) - .toBsonDocument() + .toNestedBsonDocument() return when (val session = getCurrentSession()) { null -> unsafe.updateMany(filterBson, updateBson, options) @@ -132,7 +145,7 @@ class NativeMongoCollection( val updateBson = UpdateExpression(unsafe.codecRegistry) .apply(update) - .toBsonDocument() + .toNestedBsonDocument() return when (val session = getCurrentSession()) { null -> unsafe.findOneAndUpdate(filterBson, updateBson, options) From 64f0a155377e2c87b91e470ee5212e9f51ae0d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Fri, 16 Aug 2024 14:56:35 +0200 Subject: [PATCH 4/9] refactor(sync): Refactor expressions to interfaces --- .../src/main/kotlin/NativeMongoCollection.kt | 6 +-- dsl/src/main/kotlin/expr/FilterExpression.kt | 13 ++--- .../main/kotlin/expr/PredicateExpression.kt | 10 ++-- dsl/src/main/kotlin/expr/UpdateExpression.kt | 21 ++++---- ...ssion.kt => AbstractCompoundExpression.kt} | 45 ++++++++++++---- dsl/src/main/kotlin/expr/common/Expression.kt | 52 ++++++++++++++++--- 6 files changed, 107 insertions(+), 40 deletions(-) rename dsl/src/main/kotlin/expr/common/{CompoundExpression.kt => AbstractCompoundExpression.kt} (66%) diff --git a/driver-sync/src/main/kotlin/NativeMongoCollection.kt b/driver-sync/src/main/kotlin/NativeMongoCollection.kt index ed04dfb..d9fe569 100644 --- a/driver-sync/src/main/kotlin/NativeMongoCollection.kt +++ b/driver-sync/src/main/kotlin/NativeMongoCollection.kt @@ -9,7 +9,7 @@ import com.mongodb.kotlin.client.FindIterable import fr.qsh.ktmongo.dsl.LowLevelApi import fr.qsh.ktmongo.dsl.expr.FilterExpression import fr.qsh.ktmongo.dsl.expr.UpdateExpression -import fr.qsh.ktmongo.dsl.expr.common.CompoundExpression +import fr.qsh.ktmongo.dsl.expr.common.AbstractCompoundExpression import org.bson.BsonDocument import org.bson.BsonDocumentWriter import com.mongodb.kotlin.client.MongoCollection as OfficialMongoCollection @@ -22,7 +22,7 @@ class NativeMongoCollection( fun asOfficialMongoCollection() = unsafe @OptIn(LowLevelApi::class) - private fun CompoundExpression.toBsonDocument(): BsonDocument { + private fun AbstractCompoundExpression.toBsonDocument(): BsonDocument { val bson = BsonDocument() BsonDocumentWriter(bson).use { writer -> @@ -33,7 +33,7 @@ class NativeMongoCollection( } @OptIn(LowLevelApi::class) - private fun CompoundExpression.toNestedBsonDocument(): BsonDocument { + private fun AbstractCompoundExpression.toNestedBsonDocument(): BsonDocument { val bson = BsonDocument() BsonDocumentWriter(bson).use { writer -> diff --git a/dsl/src/main/kotlin/expr/FilterExpression.kt b/dsl/src/main/kotlin/expr/FilterExpression.kt index d6ed364..9ed70d9 100644 --- a/dsl/src/main/kotlin/expr/FilterExpression.kt +++ b/dsl/src/main/kotlin/expr/FilterExpression.kt @@ -2,7 +2,8 @@ package fr.qsh.ktmongo.dsl.expr import fr.qsh.ktmongo.dsl.KtMongoDsl import fr.qsh.ktmongo.dsl.LowLevelApi -import fr.qsh.ktmongo.dsl.expr.common.CompoundExpression +import fr.qsh.ktmongo.dsl.expr.common.AbstractExpression +import fr.qsh.ktmongo.dsl.expr.common.AbstractCompoundExpression import fr.qsh.ktmongo.dsl.expr.common.Expression import fr.qsh.ktmongo.dsl.path.PropertySyntaxScope import fr.qsh.ktmongo.dsl.writeArray @@ -22,12 +23,12 @@ import kotlin.reflect.KProperty1 @KtMongoDsl class FilterExpression( codec: CodecRegistry, -) : CompoundExpression(codec), PropertySyntaxScope { +) : AbstractCompoundExpression(codec), PropertySyntaxScope { // region Low-level operations @LowLevelApi - override fun simplify(children: List): Expression? = + override fun simplify(children: List): AbstractExpression? = when (children.size) { 0 -> null 1 -> this @@ -35,7 +36,7 @@ class FilterExpression( } @LowLevelApi - private sealed class FilterExpressionNode(codec: CodecRegistry) : Expression(codec) + private sealed class FilterExpressionNode(codec: CodecRegistry) : AbstractExpression(codec) // endregion // region $and, $or @@ -78,7 +79,7 @@ class FilterExpression( codec: CodecRegistry, ) : FilterExpressionNode(codec) { - override fun simplify(): Expression? { + override fun simplify(): AbstractExpression? { if (declaredChildren.isEmpty()) return null @@ -152,7 +153,7 @@ class FilterExpression( codec: CodecRegistry, ) : FilterExpressionNode(codec) { - override fun simplify(): Expression? { + override fun simplify(): AbstractExpression? { if (declaredChildren.isEmpty()) return null diff --git a/dsl/src/main/kotlin/expr/PredicateExpression.kt b/dsl/src/main/kotlin/expr/PredicateExpression.kt index 423d41b..36d26dc 100644 --- a/dsl/src/main/kotlin/expr/PredicateExpression.kt +++ b/dsl/src/main/kotlin/expr/PredicateExpression.kt @@ -1,8 +1,8 @@ package fr.qsh.ktmongo.dsl.expr import fr.qsh.ktmongo.dsl.* -import fr.qsh.ktmongo.dsl.expr.common.CompoundExpression -import fr.qsh.ktmongo.dsl.expr.common.Expression +import fr.qsh.ktmongo.dsl.expr.common.AbstractCompoundExpression +import fr.qsh.ktmongo.dsl.expr.common.AbstractExpression import fr.qsh.ktmongo.dsl.path.PropertySyntaxScope import org.bson.BsonType import org.bson.BsonWriter @@ -15,12 +15,12 @@ import org.bson.codecs.configuration.CodecRegistry @KtMongoDsl class PredicateExpression( codec: CodecRegistry, -) : CompoundExpression(codec), PropertySyntaxScope { +) : AbstractCompoundExpression(codec), PropertySyntaxScope { // region Low-level operations @LowLevelApi - private sealed class PredicateExpressionNode(codec: CodecRegistry) : Expression(codec) + private sealed class PredicateExpressionNode(codec: CodecRegistry) : AbstractExpression(codec) // endregion // region $eq @@ -332,7 +332,7 @@ class PredicateExpression( codec: CodecRegistry, ) : PredicateExpressionNode(codec) { - override fun simplify(): Expression? { + override fun simplify(): AbstractExpression? { if (expression.children.isEmpty()) return null diff --git a/dsl/src/main/kotlin/expr/UpdateExpression.kt b/dsl/src/main/kotlin/expr/UpdateExpression.kt index 675e0b2..1388948 100644 --- a/dsl/src/main/kotlin/expr/UpdateExpression.kt +++ b/dsl/src/main/kotlin/expr/UpdateExpression.kt @@ -2,7 +2,8 @@ package fr.qsh.ktmongo.dsl.expr import fr.qsh.ktmongo.dsl.KtMongoDsl import fr.qsh.ktmongo.dsl.LowLevelApi -import fr.qsh.ktmongo.dsl.expr.common.CompoundExpression +import fr.qsh.ktmongo.dsl.expr.common.AbstractExpression +import fr.qsh.ktmongo.dsl.expr.common.AbstractCompoundExpression import fr.qsh.ktmongo.dsl.expr.common.Expression import fr.qsh.ktmongo.dsl.expr.common.acceptAll import fr.qsh.ktmongo.dsl.path.Path @@ -23,21 +24,21 @@ import kotlin.reflect.KProperty1 @KtMongoDsl class UpdateExpression( codec: CodecRegistry, -) : CompoundExpression(codec), PropertySyntaxScope { +) : AbstractCompoundExpression(codec), PropertySyntaxScope { // region Low-level operations - private class OperatorCombinator( + private class OperatorCombinator( val type: KClass, val combinator: (List, CodecRegistry) -> T ) { @Suppress("UNCHECKED_CAST") // This is a private class, it should not be used incorrectly - operator fun invoke(sources: List, codec: CodecRegistry) = + operator fun invoke(sources: List, codec: CodecRegistry) = combinator(sources as List, codec) } @LowLevelApi - override fun simplify(children: List): Expression? { + override fun simplify(children: List): AbstractExpression? { if (children.isEmpty()) return null @@ -60,7 +61,7 @@ class UpdateExpression( } @LowLevelApi - private sealed class UpdateExpressionNode(codec: CodecRegistry) : Expression(codec) + private sealed class UpdateExpressionNode(codec: CodecRegistry) : AbstractExpression(codec) // endregion // region $set @@ -157,7 +158,7 @@ class UpdateExpression( val mappings: List>, codec: CodecRegistry, ) : UpdateExpressionNode(codec) { - override fun simplify(): Expression? = + override fun simplify(): AbstractExpression? = this.takeUnless { mappings.isEmpty() } override fun write(writer: BsonWriter) { @@ -212,7 +213,7 @@ class UpdateExpression( val mappings: List>, codec: CodecRegistry, ) : UpdateExpressionNode(codec) { - override fun simplify(): Expression? = + override fun simplify(): AbstractExpression? = this.takeUnless { mappings.isEmpty() } override fun write(writer: BsonWriter) { @@ -263,7 +264,7 @@ class UpdateExpression( val fields: List, codec: CodecRegistry, ) : UpdateExpressionNode(codec) { - override fun simplify(): Expression? = + override fun simplify(): AbstractExpression? = this.takeUnless { fields.isEmpty() } override fun write(writer: BsonWriter) { @@ -311,7 +312,7 @@ class UpdateExpression( val fields: List>, codec: CodecRegistry, ) : UpdateExpressionNode(codec) { - override fun simplify(): Expression? = + override fun simplify(): AbstractExpression? = this.takeUnless { fields.isEmpty() } override fun write(writer: BsonWriter) { diff --git a/dsl/src/main/kotlin/expr/common/CompoundExpression.kt b/dsl/src/main/kotlin/expr/common/AbstractCompoundExpression.kt similarity index 66% rename from dsl/src/main/kotlin/expr/common/CompoundExpression.kt rename to dsl/src/main/kotlin/expr/common/AbstractCompoundExpression.kt index 98fa9c5..9d2bf16 100644 --- a/dsl/src/main/kotlin/expr/common/CompoundExpression.kt +++ b/dsl/src/main/kotlin/expr/common/AbstractCompoundExpression.kt @@ -8,7 +8,34 @@ import java.util.* /** * A compound node in the BSON AST. - * This class is an implementation detail of all operator DSLs. + * This is the supertype for all DSL scopes. + * + * A compound node is a node that may have children. + * It may have 0…n children. + * + * A new child expression may be added by calling the [accept] function. + */ +interface CompoundExpression : Expression { + + /** + * Adds a new [expression] as a child of this one. + * + * Since [Expression] subtypes may generate arbitrary BSON, it is possible to + * use this method to inject arbitrary BSON into any KtMongo DSL. + * However, this is not recommended, because raw BSON is easy to write incorrectly + * (leading to performance issues, syntax errors, or security vulnerabilities). + * + * Instead, we recommend calling the other methods provided by DSLs, which are type-safe + * helpers to call this function. + */ + @LowLevelApi + @KtMongoDsl + fun accept(expression: Expression) + +} + +/** + * Helper to implement [CompoundExpression]. * * This class adds the method [accept] which allows binding a child expression * into the current one. @@ -17,9 +44,9 @@ import java.util.* * * @see Expression */ -abstract class CompoundExpression( +abstract class AbstractCompoundExpression( codec: CodecRegistry, -) : Expression(codec) { +) : AbstractExpression(codec), CompoundExpression { // region Sub-expression binding @@ -38,7 +65,7 @@ abstract class CompoundExpression( * It is added to this expression as-is. * * This function is only publicly available to allow users to add missing operators themselves - * by implementing [Expression] for their operator. + * by implementing [AbstractExpression] for their operator. * * **An incorrectly written expression may allow arbitrary code execution on the database, * data corruption, or data leaks. Only call this function on expressions you are sure @@ -46,7 +73,7 @@ abstract class CompoundExpression( */ @LowLevelApi @KtMongoDsl - fun accept(expression: Expression) { + override fun accept(expression: Expression) { require(!frozen) { "This expression has already been frozen, it cannot accept the child expression $expression" } val simplifiedExpression = expression.simplify() @@ -68,18 +95,18 @@ abstract class CompoundExpression( * **These children have already been simplified.** */ @LowLevelApi - protected open fun simplify(children: List): Expression? = + protected open fun simplify(children: List): AbstractExpression? = this @LowLevelApi - final override fun simplify(): Expression? = + final override fun simplify(): AbstractExpression? = simplify(children) // endregion // region Writing /** - * See [Expression.write]. + * See [AbstractExpression.write]. * * @param children The list of expressions that have been [bound][accept] into this * expression. @@ -105,7 +132,7 @@ abstract class CompoundExpression( /** * Binds any arbitrary [expressions] as sub-expressions of the receiver. * - * To learn more about the security implications, see [CompoundExpression.accept]. + * To learn more about the security implications, see [AbstractCompoundExpression.accept]. */ @LowLevelApi @KtMongoDsl diff --git a/dsl/src/main/kotlin/expr/common/Expression.kt b/dsl/src/main/kotlin/expr/common/Expression.kt index 4ea5021..bfd4ae9 100644 --- a/dsl/src/main/kotlin/expr/common/Expression.kt +++ b/dsl/src/main/kotlin/expr/common/Expression.kt @@ -15,16 +15,54 @@ import org.bson.codecs.configuration.CodecRegistry * * ### Security * - * Implementing this interface allows to inject arbitrary BSON into a request. - * Be very careful not to allow request injections. + * Implementing this interface allows injecting arbitrary BSON into a request. + * Be very careful not to allow injections. + * + * ### Implementation notes + * + * Prefer implementing [AbstractExpression] than implementing this interface directly. * * ### Debugging notes * * Use [toString] to generate the JSON of this expression. */ -abstract class Expression( +interface Expression { + + /** + * Makes this expression immutable. + * + * After this method has been called, the expression can never be modified again. + * This ensures that requests cannot change after they have been used by other requests. + */ + @LowLevelApi + fun freeze() + + /** + * Returns a simplified (but equivalent) expression to the current expression. + * + * If `null` is returned, it means the current expression was simplified into a no-op + * (i.e. it does nothing). + */ + @LowLevelApi + fun simplify(): Expression? + + /** + * Writes this expression into a [writer]. + * + * Depending on the type of expression, the expected current context may be different. + */ + @LowLevelApi + fun writeTo(writer: BsonWriter) + + companion object +} + +/** + * Utility implementation for [Expression], which handles the [codec], [toString] representation and [freezing][freeze]. + */ +abstract class AbstractExpression( protected val codec: CodecRegistry, -) { +) : Expression { /** * See [freeze]. @@ -36,7 +74,7 @@ abstract class Expression( * Forbid further mutations to this expression. */ @LowLevelApi - fun freeze() { + final override fun freeze() { frozen = true } @@ -64,7 +102,7 @@ abstract class Expression( * Returning `null` means that the entire expression has been simplified to a no-op, and can be removed. */ @LowLevelApi - open fun simplify(): Expression? = this + override fun simplify(): AbstractExpression? = this /** * Writes this expression into a [writer]. @@ -72,7 +110,7 @@ abstract class Expression( * This function is guaranteed to be pure. */ @LowLevelApi - fun writeTo(writer: BsonWriter) { + final override fun writeTo(writer: BsonWriter) { this.simplify()?.write(writer) } From 48ca8c2410a5f1d69b094e8b757e3cdd01834043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Fri, 16 Aug 2024 17:19:06 +0200 Subject: [PATCH 5/9] fix(dsl): Simplify subexpressions before calling FilterExpression.invoke --- dsl/src/main/kotlin/expr/FilterExpression.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dsl/src/main/kotlin/expr/FilterExpression.kt b/dsl/src/main/kotlin/expr/FilterExpression.kt index 9ed70d9..5b83592 100644 --- a/dsl/src/main/kotlin/expr/FilterExpression.kt +++ b/dsl/src/main/kotlin/expr/FilterExpression.kt @@ -214,10 +214,14 @@ class FilterExpression( @LowLevelApi private class PredicateInFilterExpression( val target: String, - val expression: PredicateExpression<*>, + val expression: Expression, codec: CodecRegistry, ) : FilterExpressionNode(codec) { + override fun simplify(): AbstractExpression? = + expression.simplify() + ?.let { PredicateInFilterExpression(target, it, codec) } + override fun write(writer: BsonWriter) { writer.writeDocument { writer.writeDocument(target) { From cb88c04863cb9981b61b7b1f0b2beaca25d2bee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Fri, 16 Aug 2024 18:28:25 +0200 Subject: [PATCH 6/9] feat(dsl): $elemMatch --- dsl/README.md | 1 + dsl/src/main/kotlin/expr/FilterExpression.kt | 307 +++++++++++++++++- .../test/kotlin/expr/FilterExpressionTest.kt | 179 +++++++++- 3 files changed, 467 insertions(+), 20 deletions(-) diff --git a/dsl/README.md b/dsl/README.md index cba87df..89d7267 100644 --- a/dsl/README.md +++ b/dsl/README.md @@ -5,6 +5,7 @@ ### Filter - [`$and`][fr.qsh.ktmongo.dsl.expr.FilterExpression.and] +- [`$elemMatch`][fr.qsh.ktmongo.dsl.expr.FilterExpression.anyObject] - [`$eq`][fr.qsh.ktmongo.dsl.expr.FilterExpression.eq] - [`$exists`][fr.qsh.ktmongo.dsl.expr.FilterExpression.exists] - [`$gt`][fr.qsh.ktmongo.dsl.expr.FilterExpression.gt] diff --git a/dsl/src/main/kotlin/expr/FilterExpression.kt b/dsl/src/main/kotlin/expr/FilterExpression.kt index 5b83592..6374cab 100644 --- a/dsl/src/main/kotlin/expr/FilterExpression.kt +++ b/dsl/src/main/kotlin/expr/FilterExpression.kt @@ -2,8 +2,8 @@ package fr.qsh.ktmongo.dsl.expr import fr.qsh.ktmongo.dsl.KtMongoDsl import fr.qsh.ktmongo.dsl.LowLevelApi -import fr.qsh.ktmongo.dsl.expr.common.AbstractExpression import fr.qsh.ktmongo.dsl.expr.common.AbstractCompoundExpression +import fr.qsh.ktmongo.dsl.expr.common.AbstractExpression import fr.qsh.ktmongo.dsl.expr.common.Expression import fr.qsh.ktmongo.dsl.path.PropertySyntaxScope import fr.qsh.ktmongo.dsl.writeArray @@ -18,7 +18,71 @@ import kotlin.reflect.KProperty1 /** * DSL for MongoDB operators that are used as predicates in conditions. * - * For example, these operators are available when querying with `find`, or as the filter in `updateOne`. + * ### Example + * + * This expression type is available in multiple operators, most commonly `find`: + * ```kotlin + * class User( + * val name: String, + * val age: Int, + * ) + * + * collection.find { + * User::age gte 18 + * } + * ``` + * + * ### Beware of arrays! + * + * MongoDB operators do not discriminate between scalars and arrays. + * When an array is encountered, all operators attempt to match on the array itself. + * If the match fails, the operators attempt to match array elements. + * + * It is not possible to mimic this behavior in KtMongo while still keeping type-safety, + * so operators may behave strangely when arrays are encountered. + * + * Note that if the collection corresponds to the declared Kotlin type, + * these situations can never happen, as the Kotlin type system doesn't allow them to. + * + * When developers attempt to perform an operator on the entire array, + * they should use operators as normal: + * ```kotlin + * class User( + * val name: String, + * val favoriteNumbers: List + * ) + * + * collection.find { + * User::favoriteNumbers eq listOf(1, 2) + * } + * ``` + * Developers should use the request above when they want to match a document similar to: + * ```json + * { + * favoriteNumbers: [1, 2] + * } + * ``` + * The following document will NOT match: + * ```json + * { + * favoriteNumbers: [3] + * } + * ``` + * + * However, due to MongoDB's behavior when encountering arrays, it should be noted + * that the following document WILL match: + * ```json + * { + * favoriteNumbers: [ + * [3], + * [1, 2], + * [7, 2] + * ] + * } + * ``` + * + * To execute an operator on one of the elements of an array, see [any]. + * */ @KtMongoDsl class FilterExpression( @@ -103,10 +167,10 @@ class FilterExpression( } override fun write(writer: BsonWriter) { - writer.writeDocument { - writer.writeName("\$and") - writer.writeArray { - for (child in declaredChildren) { + writer.writeName("\$and") + writer.writeArray { + for (child in declaredChildren) { + writer.writeDocument { child.writeTo(writer) } } @@ -164,10 +228,10 @@ class FilterExpression( } override fun write(writer: BsonWriter) { - writer.writeDocument { - writer.writeName("\$or") - writer.writeArray { - for (child in declaredChildren) { + writer.writeName("\$or") + writer.writeArray { + for (child in declaredChildren) { + writer.writeDocument { child.writeTo(writer) } } @@ -223,10 +287,8 @@ class FilterExpression( ?.let { PredicateInFilterExpression(target, it, codec) } override fun write(writer: BsonWriter) { - writer.writeDocument { - writer.writeDocument(target) { - expression.writeTo(writer) - } + writer.writeDocument(target) { + expression.writeTo(writer) } } } @@ -677,7 +739,6 @@ class FilterExpression( this { gteNotNull(value) } } - /** * Selects documents for which this field has a value strictly lesser than [value]. * @@ -851,5 +912,221 @@ class FilterExpression( isOneOf(values.asList()) } + // endregion + // region $elemMatch + + /** + * Specify operators on array elements. + * + * ### Example + * + * Find any user who has 12 as one of their favorite numbers. + * + * ```kotlin + * class User( + * val name: String, + * val favoriteNumbers: List + * ) + * + * collection.find { + * User::favoriteNumbers.any eq 12 + * } + * ``` + * + * ### Repeated usages will match different items + * + * Note that if `any` is used multiple times, it may test different items. + * For example, the following request will match the following document: + * ```kotlin + * collection.find { + * User::favoriteNumbers.any gt 2 + * User::favoriteNumbers.any lte 7 + * } + * ``` + * ```json + * { + * "name": "Nicolas", + * "favoriteNumbers": [ 1, 9 ] + * } + * ``` + * Because 1 is less than 7, and 9 is greater than 2, the document is returned. + * + * If you want to apply multiple filters to the same item, use the [any] function. + * + * ### Arrays don't exist in finds! + * + * MongoDB operators do not discriminate between scalars and arrays. + * When an array is encountered, all operators attempt to match on the array itself. + * If the match fails, the operators attempt to match array elements. + * + * It is not possible to mimic this behavior in KtMongo while still keeping type-safety, + * so KtMongo has different operators to filter a collection itself or its elements. + * + * As a consequence, the request: + * ```kotlin + * collection.find { + * User::favoriteNumbers.any eq 5 + * } + * ``` + * will, as expected, match the following document: + * ```json + * { + * favoriteNumbers: [1, 4, 5, 10] + * } + * ``` + * + * It is important to note that it WILL also match this document: + * ```json + * { + * favoriteNumbers: 5 + * } + * ``` + * + * Since this document doesn't conform to the Kotlin declared type `List`, + * it is unlikely that such an element exists, but developers should keep it in mind. + * + * ### External resources + * + * - [Official document](https://www.mongodb.com/docs/manual/tutorial/query-arrays/) + */ + @KtMongoDsl + val KProperty1>.any: KProperty1 + @Suppress("UNCHECKED_CAST") // The type parameters are fake anyway + get() = this as KProperty1 + + /** + * Specify multiple operators on a single array element. + * + * ### Example + * + * Find students with a grade between 8 and 10, that may be eligible to perform + * an exam a second time. + * + * ```kotlin + * class Student( + * val name: String, + * val grades: List + * ) + * + * collection.find { + * Student::grades.any { + * gte(8) + * lte(10) + * } + * } + * ``` + * + * The following document will match because the grade 9 is in the interval. + * ```json + * { + * "name": "John", + * "grades": [9, 3] + * } + * ``` + * + * The following document will NOT match, because none of the grades are in the interval. + * ```json + * { + * "name": "Lea", + * "grades": [18, 19] + * } + * ``` + * + * If you want to perform multiple checks on different elements of an array, + * see the [any] property. + * + * This function only allows specifying operators on array elements directly. + * To specify operators on sub-fields of array elements, see [anyObject]. + * + * ### External resources + * + * - [Official documentation](https://www.mongodb.com/docs/manual/reference/operator/query/elemMatch/) + */ + @OptIn(LowLevelApi::class) + @KtMongoDsl + fun KProperty1>.any(block: PredicateExpression.() -> Unit) { + accept(ElementMatchExpressionNode(this.path().toString(), PredicateExpression(codec).apply(block), codec)) + } + + /** + * Specify multiple operators on fields of a single array element. + * + * ### Example + * + * Find customers who have a pet that is born this month, as they may be eligible for a discount. + * + * ```kotlin + * class Customer( + * val name: String, + * val pets: List, + * ) + * + * class Pet( + * val name: String, + * val birthMonth: Int + * ) + * + * val currentMonth = 3 + * + * collection.find { + * Customer::pets.anyObject { + * Pet::birthMonth gte currentMonth + * Pet::birthMonth lte (currentMonth + 1) + * } + * } + * ``` + * + * The following document will match: + * ```json + * { + * "name": "Fred", + * "pets": [ + * { + * "name": "Arthur", + * "birthMonth": 5 + * }, + * { + * "name": "Gwen", + * "birthMonth": 3 + * } + * ] + * } + * ``` + * because the pet "Gwen" has a matching birth month. + * + * If you want to perform operators on the elements directly (not on their fields), use + * [any] instead. + * + * ### External resources + * + * - [Official documentation](https://www.mongodb.com/docs/manual/reference/operator/query/elemMatch/) + */ + @OptIn(LowLevelApi::class) + @KtMongoDsl + fun KProperty1>.anyObject(block: FilterExpression.() -> Unit) { + accept(ElementMatchExpressionNode(this.path().toString(), FilterExpression(codec).apply(block), codec)) + } + + @LowLevelApi + private class ElementMatchExpressionNode( + val target: String, + val expression: Expression, + codec: CodecRegistry, + ) : FilterExpressionNode(codec) { + + override fun simplify(): AbstractExpression = + ElementMatchExpressionNode(target, expression.simplify() + ?: OrFilterExpressionNode(emptyList(), codec), codec) + + override fun write(writer: BsonWriter) { + writer.writeDocument(target) { + writer.writeName("\$elemMatch") + writer.writeStartDocument() + expression.writeTo(writer) + writer.writeEndDocument() + } + } + } + // endregion } diff --git a/dsl/src/test/kotlin/expr/FilterExpressionTest.kt b/dsl/src/test/kotlin/expr/FilterExpressionTest.kt index 32e526d..d044c34 100644 --- a/dsl/src/test/kotlin/expr/FilterExpressionTest.kt +++ b/dsl/src/test/kotlin/expr/FilterExpressionTest.kt @@ -23,6 +23,10 @@ class FilterExpressionTest : FunSpec({ val type = "\$type" val not = "\$not" val isOneOf = "\$in" + val gt = "\$gt" + val gte = "\$gte" + val lt = "\$lt" + val lte = "\$lte" context("Operator $eq") { test("Integer") { @@ -376,11 +380,6 @@ class FilterExpressionTest : FunSpec({ } context("Comparison operators") { - val gt = "\$gt" - val gte = "\$gte" - val lt = "\$lt" - val lte = "\$lte" - test("int $gt") { filter { User::age gt 12 @@ -429,4 +428,174 @@ class FilterExpressionTest : FunSpec({ """.trimIndent() } } + + context("Array operators") { + val elemMatch = "\$elemMatch" + + class Pet( + val name: String, + val age: Int, + ) + + class User( + val scores: List, + val pets: List, + ) + + test("Test on an array element") { + filter { + User::scores.any eq 12 + } shouldBeBson """ + { + "scores": { + "$eq": 12 + } + } + """.trimIndent() + } + + test("Test on different array elements") { + filter { + User::scores.any gt 12 + User::scores.any lte 15 + } shouldBeBson """ + { + "$and": [ + { + "scores": { + "$gt": 12 + } + }, + { + "scores": { + "$lte": 15 + } + } + ] + } + """.trimIndent() + } + + test("Test on a single array element") { + filter { + User::scores.any { + gt(12) + lte(15) + } + } shouldBeBson """ + { + "scores": { + "$elemMatch": { + "$gt": 12, + "$lte": 15 + } + } + } + """.trimIndent() + } + + test("Test on subfields of different array elements") { + filter { + User::pets.any / Pet::age gt 15 + User::pets.any / Pet::age lte 18 + } shouldBeBson """ + { + "$and": [ + { + "pets.age": { + "$gt": 15 + } + }, + { + "pets.age": { + "$lte": 18 + } + } + ] + } + """.trimIndent() + } + + test("Test on subfields of a single array element") { + filter { + User::pets.anyObject { + Pet::age gt 15 + Pet::age lte 18 + } + } shouldBeBson """ + { + "pets": { + "$elemMatch": { + "$and": [ + { + "age": { + "$gt": 15 + } + }, + { + "age": { + "$lte": 18 + } + } + ] + } + } + } + """.trimIndent() + } + + test("Test on a single subfield of a single array element") { + filter { + User::pets.anyObject { + Pet::age { + gt(15) + lte(18) + } + } + } shouldBeBson """ + { + "pets": { + "$elemMatch": { + "age": { + "$gt": 15, + "$lte": 18 + } + } + } + } + """.trimIndent() + } + + test("Everything combined") { + filter { + User::pets.any / Pet::age gt 3 + User::pets.anyObject { + Pet::age gte 1 + Pet::name eq "Chocolat" + } + } shouldBeBson """ + { + "$and": [ + { + "pets.age": {"$gt": 3} + }, + { + "pets": { + "$elemMatch": { + "$and": [ + { + "age": {"$gte": 1} + }, + { + "name": {"$eq": "Chocolat"} + } + ] + } + } + } + ] + } + """.trimIndent() + } + } }) From 651c11a676cc52a7ade86adda8885b9d301539ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Fri, 16 Aug 2024 18:34:22 +0200 Subject: [PATCH 7/9] feat(dsl): Shorthand '.any /' by just '/' --- dsl/src/main/kotlin/expr/FilterExpression.kt | 31 +++++++++++++++++++ .../test/kotlin/expr/FilterExpressionTest.kt | 4 +-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/dsl/src/main/kotlin/expr/FilterExpression.kt b/dsl/src/main/kotlin/expr/FilterExpression.kt index 6374cab..f2f155a 100644 --- a/dsl/src/main/kotlin/expr/FilterExpression.kt +++ b/dsl/src/main/kotlin/expr/FilterExpression.kt @@ -994,6 +994,37 @@ class FilterExpression( @Suppress("UNCHECKED_CAST") // The type parameters are fake anyway get() = this as KProperty1 + /** + * Combines Kotlin properties into a path usable to point to any item in an array. + * + * ### Example + * + * ```kotlin + * class User( + * val grades: List + * ) + * + * class Grade( + * val name: Int + * ) + * + * collection.find { + * User::grades / Grade::name eq 19 + * } + * ``` + * + * This function is a shorthand for `any`: + * ```kotlin + * collection.find { + * User::grades.any / Gradle::name eq 19 + * } + * ``` + */ + @KtMongoDsl + @JvmName("anyChild") + operator fun KProperty1>.div(other: KProperty1): KProperty1 = + this.any.div(other) + /** * Specify multiple operators on a single array element. * diff --git a/dsl/src/test/kotlin/expr/FilterExpressionTest.kt b/dsl/src/test/kotlin/expr/FilterExpressionTest.kt index d044c34..65241b7 100644 --- a/dsl/src/test/kotlin/expr/FilterExpressionTest.kt +++ b/dsl/src/test/kotlin/expr/FilterExpressionTest.kt @@ -497,7 +497,7 @@ class FilterExpressionTest : FunSpec({ test("Test on subfields of different array elements") { filter { User::pets.any / Pet::age gt 15 - User::pets.any / Pet::age lte 18 + User::pets / Pet::age lte 18 // without 'any', the / does the same thing } shouldBeBson """ { "$and": [ @@ -568,7 +568,7 @@ class FilterExpressionTest : FunSpec({ test("Everything combined") { filter { - User::pets.any / Pet::age gt 3 + User::pets / Pet::age gt 3 User::pets.anyObject { Pet::age gte 1 Pet::name eq "Chocolat" From 34f8b67c77306ac4bcfb05bf8a6f1851a926779b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Fri, 16 Aug 2024 18:56:32 +0200 Subject: [PATCH 8/9] feat(dsl): $in should accept Sets --- dsl/src/main/kotlin/expr/PredicateExpression.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dsl/src/main/kotlin/expr/PredicateExpression.kt b/dsl/src/main/kotlin/expr/PredicateExpression.kt index 36d26dc..73fb9de 100644 --- a/dsl/src/main/kotlin/expr/PredicateExpression.kt +++ b/dsl/src/main/kotlin/expr/PredicateExpression.kt @@ -782,13 +782,13 @@ class PredicateExpression( */ @OptIn(LowLevelApi::class) @KtMongoDsl - fun isOneOf(values: List) { + fun isOneOf(values: Collection) { accept(OneOfPredicateExpressionNode(values, codec)) } @LowLevelApi private class OneOfPredicateExpressionNode( - val values: List, + val values: Collection, codec: CodecRegistry, ) : PredicateExpressionNode(codec) { From e8f12163135ca1b79a3644c260d25234371e3b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20=E2=80=9CCLOVIS=E2=80=9D=20Canet?= Date: Fri, 16 Aug 2024 18:57:03 +0200 Subject: [PATCH 9/9] feat(dsl): $containsAll --- dsl/README.md | 1 + dsl/src/main/kotlin/expr/FilterExpression.kt | 51 +++++++++++++++++-- .../test/kotlin/expr/ExpressionTestUtils.kt | 1 + .../test/kotlin/expr/FilterExpressionTest.kt | 17 +++++++ 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/dsl/README.md b/dsl/README.md index 89d7267..e44e8e4 100644 --- a/dsl/README.md +++ b/dsl/README.md @@ -4,6 +4,7 @@ ### Filter +- [`$all`][fr.qsh.ktmongo.dsl.expr.FilterExpression.containsAll] - [`$and`][fr.qsh.ktmongo.dsl.expr.FilterExpression.and] - [`$elemMatch`][fr.qsh.ktmongo.dsl.expr.FilterExpression.anyObject] - [`$eq`][fr.qsh.ktmongo.dsl.expr.FilterExpression.eq] diff --git a/dsl/src/main/kotlin/expr/FilterExpression.kt b/dsl/src/main/kotlin/expr/FilterExpression.kt index f2f155a..28d6221 100644 --- a/dsl/src/main/kotlin/expr/FilterExpression.kt +++ b/dsl/src/main/kotlin/expr/FilterExpression.kt @@ -1,13 +1,10 @@ package fr.qsh.ktmongo.dsl.expr -import fr.qsh.ktmongo.dsl.KtMongoDsl -import fr.qsh.ktmongo.dsl.LowLevelApi +import fr.qsh.ktmongo.dsl.* import fr.qsh.ktmongo.dsl.expr.common.AbstractCompoundExpression import fr.qsh.ktmongo.dsl.expr.common.AbstractExpression import fr.qsh.ktmongo.dsl.expr.common.Expression import fr.qsh.ktmongo.dsl.path.PropertySyntaxScope -import fr.qsh.ktmongo.dsl.writeArray -import fr.qsh.ktmongo.dsl.writeDocument import org.bson.BsonType import org.bson.BsonWriter import org.bson.codecs.configuration.CodecRegistry @@ -1159,5 +1156,51 @@ class FilterExpression( } } + // endregion + // region $all + + /** + * Selects documents where the value of a field is an array that contains all the specified [values]. + * + * ### Example + * + * ```kotlin + * class User( + * val grades: List + * ) + * + * collection.find { + * User::grades containsAll listOf(2, 3, 7) + * } + * ``` + * + * ### External resources + * + * - [Official documentation](https://www.mongodb.com/docs/manual/reference/operator/query/all/) + */ + @OptIn(LowLevelApi::class) + @KtMongoDsl + infix fun KProperty1.containsAll(values: Collection) { + accept(ArrayAllExpressionNode(this.path().toString(), values, codec)) + } + + @LowLevelApi + private class ArrayAllExpressionNode( + val path: String, + val values: Collection, + codec: CodecRegistry, + ) : FilterExpressionNode(codec) { + + @LowLevelApi + override fun write(writer: BsonWriter) { + writer.writeDocument(path) { + writer.writeArray("\$all") { + for (value in values) + writer.writeObject(value, codec) + } + } + } + } + // endregion } diff --git a/dsl/src/test/kotlin/expr/ExpressionTestUtils.kt b/dsl/src/test/kotlin/expr/ExpressionTestUtils.kt index c5a4584..ebe90a9 100644 --- a/dsl/src/test/kotlin/expr/ExpressionTestUtils.kt +++ b/dsl/src/test/kotlin/expr/ExpressionTestUtils.kt @@ -70,4 +70,5 @@ infix fun String.shouldBeBson(expected: String) { .replace("\n", "") .replace("\t", "") .replace(",", ", ") + .replace(" ", " ") } diff --git a/dsl/src/test/kotlin/expr/FilterExpressionTest.kt b/dsl/src/test/kotlin/expr/FilterExpressionTest.kt index 65241b7..e6050f4 100644 --- a/dsl/src/test/kotlin/expr/FilterExpressionTest.kt +++ b/dsl/src/test/kotlin/expr/FilterExpressionTest.kt @@ -27,6 +27,7 @@ class FilterExpressionTest : FunSpec({ val gte = "\$gte" val lt = "\$lt" val lte = "\$lte" + val all = "\$all" context("Operator $eq") { test("Integer") { @@ -598,4 +599,20 @@ class FilterExpressionTest : FunSpec({ """.trimIndent() } } + + test("Operator $all") { + class User( + val grades: List, + ) + + filter { + User::grades containsAll listOf(1, 2, 3) + } shouldBeBson """ + { + "grades": { + "$all": [1, 2, 3] + } + } + """.trimIndent() + } })