Make build succeed with OS 2.11

Signed-off-by: Heemin Kim <[email protected]>

build.gradle

@@ -13,7 +13,7 @@ buildscript {
     ext {
         // build.version_qualifier parameter applies to knn plugin artifacts only. OpenSearch version must be set
         // explicitly as 'opensearch.version' property, for instance opensearch.version=2.0.0-rc1-SNAPSHOT
-        opensearch_version = System.getProperty("opensearch.version", "2.16.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.11.1")
         version_qualifier = System.getProperty("build.version_qualifier", "")
         opensearch_group = "org.opensearch"
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
@@ -293,9 +293,9 @@ dependencies {
     api group: 'com.google.guava', name: 'guava', version:'32.1.3-jre'
     api group: 'commons-lang', name: 'commons-lang', version: '2.6'
     testFixturesImplementation "org.opensearch.test:framework:${opensearch_version}"
-    testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.9'
+    testImplementation group: 'net.bytebuddy', name: 'byte-buddy', version: '1.14.7'
     testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.2'
-    testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.9'
+    testImplementation group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.14.7'
     testFixturesImplementation "org.opensearch:common-utils:${version}"
     implementation 'com.github.oshi:oshi-core:6.4.13'
     api "net.java.dev.jna:jna:5.13.0"

src/main/java/org/opensearch/knn/index/IndexUtil.java

@@ -33,6 +33,7 @@
 import java.util.Locale;
 import java.util.Map;
 
+import static org.opensearch.Version.CURRENT;
 import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES;
 import static org.opensearch.knn.common.KNNConstants.HNSW_ALGO_EF_SEARCH;
 import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE;
@@ -45,11 +46,11 @@ public class IndexUtil {
 
     private static final Version MINIMAL_SUPPORTED_VERSION_FOR_LUCENE_HNSW_FILTER = Version.V_2_4_0;
     private static final Version MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED = Version.V_2_11_0;
-    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT = Version.V_2_12_0;
-    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT = Version.V_2_13_0;
-    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH = Version.V_2_14_0;
-    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS = Version.V_2_16_0;
-    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE = Version.V_2_16_0;
+    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_NODE_ASSIGNMENT = null;
+    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_METHOD_COMPONENT_CONTEXT = null;
+    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_RADIAL_SEARCH = null;
+    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_METHOD_PARAMETERS = null;
+    private static final Version MINIMAL_SUPPORTED_VERSION_FOR_MODEL_VECTOR_DATA_TYPE = null;
     // public so neural search can access it
     public static final Map<String, Version> minimalRequiredVersionMap = initializeMinimalRequiredVersionMap();
 
@@ -281,18 +282,23 @@ public static Map<String, Object> getParametersAtLoading(
         return Collections.unmodifiableMap(loadParameters);
     }
 
+    // Placeholder for feature flag
+    private static boolean enable216() {
+        return true;
+    }
+
     public static boolean isClusterOnOrAfterMinRequiredVersion(String key) {
         Version minimalRequiredVersion = minimalRequiredVersionMap.get(key);
         if (minimalRequiredVersion == null) {
-            return false;
+            return enable216();
         }
         return KNNClusterUtil.instance().getClusterMinVersion().onOrAfter(minimalRequiredVersion);
     }
 
     public static boolean isVersionOnOrAfterMinRequiredVersion(Version version, String key) {
         Version minimalRequiredVersion = minimalRequiredVersionMap.get(key);
         if (minimalRequiredVersion == null) {
-            return false;
+            ;
         }
         return version.onOrAfter(minimalRequiredVersion);
     }

src/main/java/org/opensearch/knn/index/KNNVectorSimilarityFunction.java

@@ -8,14 +8,31 @@
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.opensearch.knn.plugin.script.KNNScoringUtil;
 
+import static org.apache.lucene.util.VectorUtil.dotProduct;
+
 /**
  * Wrapper class of VectorSimilarityFunction to support more function than what Lucene provides
  */
 public enum KNNVectorSimilarityFunction {
     EUCLIDEAN(VectorSimilarityFunction.EUCLIDEAN),
     DOT_PRODUCT(VectorSimilarityFunction.DOT_PRODUCT),
     COSINE(VectorSimilarityFunction.COSINE),
-    MAXIMUM_INNER_PRODUCT(VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT),
+    MAXIMUM_INNER_PRODUCT(null) {
+        @Override
+        public float compare(float[] v1, float[] v2) {
+            return scaleMaxInnerProductScore(dotProduct(v1, v2));
+        }
+
+        @Override
+        public float compare(byte[] v1, byte[] v2) {
+            return scaleMaxInnerProductScore(dotProduct(v1, v2));
+        }
+
+        @Override
+        public VectorSimilarityFunction getVectorSimilarityFunction() {
+            throw new IllegalStateException("VectorSimilarityFunction is not available for Hamming space");
+        }
+    },
     HAMMING(null) {
         @Override
         public float compare(float[] v1, float[] v2) {
@@ -50,4 +67,15 @@ public float compare(float[] var1, float[] var2) {
     public float compare(byte[] var1, byte[] var2) {
         return vectorSimilarityFunction.compare(var1, var2);
     }
+
+    /**
+     * @param vectorDotProductSimilarity the raw similarity between two vectors
+     * @return A scaled score preventing negative scores for maximum-inner-product
+     */
+    public static float scaleMaxInnerProductScore(float vectorDotProductSimilarity) {
+        if (vectorDotProductSimilarity < 0) {
+            return 1 / (1 + -1 * vectorDotProductSimilarity);
+        }
+        return vectorDotProductSimilarity + 1;
+    }
 }

src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java

@@ -111,11 +111,6 @@ public KnnVectorsFormat getKnnVectorsFormatForField(final String field) {
         return vectorsFormatSupplier.apply(knnVectorsFormatParams);
     }
 
-    @Override
-    public int getMaxDimensions(String fieldName) {
-        return getKnnVectorsFormatForField(fieldName).getMaxDimensions(fieldName);
-    }
-
     private boolean isKnnVectorFieldType(final String field) {
         return mapperService.isPresent() && mapperService.get().fieldType(field) instanceof KNNVectorFieldMapper.KNNVectorFieldType;
     }

src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java

@@ -5,10 +5,9 @@
 
 package org.opensearch.knn.index.codec.KNN950Codec;
 
-import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat;
-import org.opensearch.knn.index.util.KNNEngine;
 
 import java.util.Optional;
 
@@ -29,15 +28,4 @@ public KNN950PerFieldKnnVectorsFormat(final Optional<MapperService> mapperServic
             )
         );
     }
-
-    @Override
-    /**
-     * This method returns the maximum dimension allowed from KNNEngine for Lucene codec
-     *
-     * @param fieldName Name of the field, ignored
-     * @return Maximum constant dimension set by KNNEngine
-     */
-    public int getMaxDimensions(String fieldName) {
-        return KNNEngine.getMaxDimensionByEngine(KNNEngine.LUCENE);
-    }
 }

src/main/java/org/opensearch/knn/index/codec/KNNCodecVersion.java

@@ -11,8 +11,7 @@
 import org.apache.lucene.backward_codecs.lucene92.Lucene92Codec;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.backward_codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.backward_codecs.lucene95.Lucene95Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
 import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.knn.index.codec.KNN80Codec.KNN80CompoundFormat;
@@ -24,8 +23,6 @@
 import org.opensearch.knn.index.codec.KNN940Codec.KNN940PerFieldKnnVectorsFormat;
 import org.opensearch.knn.index.codec.KNN950Codec.KNN950Codec;
 import org.opensearch.knn.index.codec.KNN950Codec.KNN950PerFieldKnnVectorsFormat;
-import org.opensearch.knn.index.codec.KNN990Codec.KNN990Codec;
-import org.opensearch.knn.index.codec.KNN990Codec.KNN990PerFieldKnnVectorsFormat;
 
 import java.util.Optional;
 import java.util.function.BiFunction;
@@ -95,24 +92,9 @@ public enum KNNCodecVersion {
             .knnVectorsFormat(new KNN950PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService)))
             .build(),
         KNN950Codec::new
-    ),
-
-    V_9_9_0(
-        "KNN990Codec",
-        new Lucene99Codec(),
-        new KNN990PerFieldKnnVectorsFormat(Optional.empty()),
-        (delegate) -> new KNNFormatFacade(
-            new KNN80DocValuesFormat(delegate.docValuesFormat()),
-            new KNN80CompoundFormat(delegate.compoundFormat())
-        ),
-        (userCodec, mapperService) -> KNN990Codec.builder()
-            .delegate(userCodec)
-            .knnVectorsFormat(new KNN990PerFieldKnnVectorsFormat(Optional.ofNullable(mapperService)))
-            .build(),
-        KNN990Codec::new
     );
 
-    private static final KNNCodecVersion CURRENT = V_9_9_0;
+    private static final KNNCodecVersion CURRENT = V_9_5_0;
 
     private final String codecName;
     private final Codec defaultCodecDelegate;

src/main/java/org/opensearch/knn/index/query/KNNQueryFactory.java

@@ -11,8 +11,6 @@
 import org.apache.lucene.search.KnnFloatVectorQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.join.BitSetProducer;
-import org.apache.lucene.search.join.DiversifyingChildrenByteKnnVectorQuery;
-import org.apache.lucene.search.join.DiversifyingChildrenFloatKnnVectorQuery;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.knn.index.VectorDataType;
 import org.opensearch.knn.index.util.KNNEngine;
@@ -132,9 +130,9 @@ public static Query create(CreateQueryRequest createQueryRequest) {
         log.debug(String.format("Creating Lucene k-NN query for index: %s \"\", field: %s \"\", k: %d", indexName, fieldName, k));
         switch (vectorDataType) {
             case BYTE:
-                return getKnnByteVectorQuery(fieldName, byteVector, luceneK, filterQuery, parentFilter);
+                return new KnnByteVectorQuery(fieldName, byteVector, luceneK, filterQuery);
             case FLOAT:
-                return getKnnFloatVectorQuery(fieldName, vector, luceneK, filterQuery, parentFilter);
+                return new KnnFloatVectorQuery(fieldName, vector, luceneK, filterQuery);
             default:
                 throw new IllegalArgumentException(
                     String.format(
@@ -155,40 +153,4 @@ private static Query validateFilterQuerySupport(final Query filterQuery, final K
         }
         return null;
     }
-
-    /**
-     * If parentFilter is not null, it is a nested query. Therefore, we return {@link DiversifyingChildrenByteKnnVectorQuery}
-     * which will dedupe search result per parent so that we can get k parent results at the end.
-     */
-    private static Query getKnnByteVectorQuery(
-        final String fieldName,
-        final byte[] byteVector,
-        final int k,
-        final Query filterQuery,
-        final BitSetProducer parentFilter
-    ) {
-        if (parentFilter == null) {
-            return new KnnByteVectorQuery(fieldName, byteVector, k, filterQuery);
-        } else {
-            return new DiversifyingChildrenByteKnnVectorQuery(fieldName, byteVector, filterQuery, k, parentFilter);
-        }
-    }
-
-    /**
-     * If parentFilter is not null, it is a nested query. Therefore, we return {@link DiversifyingChildrenFloatKnnVectorQuery}
-     * which will dedupe search result per parent so that we can get k parent results at the end.
-     */
-    private static Query getKnnFloatVectorQuery(
-        final String fieldName,
-        final float[] floatVector,
-        final int k,
-        final Query filterQuery,
-        final BitSetProducer parentFilter
-    ) {
-        if (parentFilter == null) {
-            return new KnnFloatVectorQuery(fieldName, floatVector, k, filterQuery);
-        } else {
-            return new DiversifyingChildrenFloatKnnVectorQuery(fieldName, floatVector, filterQuery, k, parentFilter);
-        }
-    }
 }