Elasticssearch 8 upgrade (#7599)

* Update to Elasticsearch 8. Use of Elasticsearch Java API Client instead of Java High Level REST Client * Update to Elasticsearch 8 / WFS indexing draft. (#88) * Update to Elasticsearch 8 / remove TODOs * Update Elasticsearch client to version 8.11.3 * Elasticsearch / Update maven plugin. * Associated record / Store all relations in index / Remove experimental feature, not used. Related to #4912 * Elasticsearch / Update maven plugin configuration. Avoid error like ERROR: Elasticsearch exited unexpectedly, with exit code 143 * Elasticsearch / Update MetadataUtils.getAssociated to retrieve scripted overview field * Elasticsearch / Update MetadataUtils.getAssociated remove TODO comment * Elasticsearch / Fix and refactor index readonly health check * Elasticsearch / Log query error details * Elasticsearch / Sonarlint improvements * Elasticsearch / WrapperQuery use base64 encoded JSON string query. * Elasticsearch / Remove unused commented code from EsSearchManager * Elasticsearch / More strict Xlink query based on UUID and fix check on hits. A request may return no hits but can be used to check number of hits. In such case we should avoid using hits.hits.size and use hits.total.value to get number of match. * Elasticsearch / Health check / Fix number of hits info. * Elasticsearch / Cleaning / No need to retrieve hits to only get matches. * Elasticsearch / Deprecated field [include] used, expected [includes] instead. * Elasticsearch / Remove 'Clear XLink cache' from Administration > Tools, clear the Xlink cache automatically before indexing and remove non-implemented code to retrieve metadata with XLink (not required anymore) * Kibana / Update install instruction Related to elastic/kibana#82521. * Elasticsearch / Remove unused imports * Kibana / Update default dashboards. * Elasticsearch / Documentation / Update Elasticsearch version * Elasticsearch / Fix logger module name typo --------- Co-authored-by: François Prunayre <[email protected]>
geonetwork · Feb 9, 2024 · fbbefc2 · fbbefc2
1 parent da9623f
commit fbbefc2
Show file tree

Hide file tree

Showing 71 changed files with 716 additions and 950 deletions.
diff --git a/core/pom.xml b/core/pom.xml
@@ -579,9 +579,10 @@
       <artifactId>spring-test</artifactId>
       <scope>compile</scope>
     </dependency>
+
     <dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-high-level-client</artifactId>
+      <groupId>co.elastic.clients</groupId>
+      <artifactId>elasticsearch-java</artifactId>
     </dependency>
 
     <dependency>
@@ -695,6 +696,7 @@
               <clusterName>test</clusterName>
               <transportPort>9300</transportPort>
               <httpPort>9200</httpPort>
+              <environmentVariables><ES_JAVA_OPTS>-Xmx2g</ES_JAVA_OPTS></environmentVariables>
             </configuration>
             <executions>
               <execution>

diff --git a/core/src/main/java/org/fao/geonet/constants/Geonet.java b/core/src/main/java/org/fao/geonet/constants/Geonet.java
@@ -630,7 +630,7 @@ public static final class Namespaces {
 
     public static class IndexFieldNames {
         public static final String HASXLINKS = "_hasxlinks";
-        public static final String XLINK = "_xlink";
+        public static final String XLINK = "xlink";
         public static final String ROOT = "_root";
         public static final String SCHEMA = "documentStandard";
         public static final String DATABASE_CREATE_DATE = "createDate";

diff --git a/core/src/main/java/org/fao/geonet/kernel/DataManager.java b/core/src/main/java/org/fao/geonet/kernel/DataManager.java
@@ -3,7 +3,7 @@
 //=== DataManager
 //===
 //=============================================================================
-//===	Copyright (C) 2001-2007 Food and Agriculture Organization of the
+//===	Copyright (C) 2001-2023 Food and Agriculture Organization of the
 //===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
 //===	and United Nations Environment Programme (UNEP)
 //===
@@ -143,11 +143,6 @@ public void init(ServiceContext context, Boolean force) throws Exception {
         }
     }
 
-    @Deprecated
-    public synchronized void rebuildIndexXLinkedMetadata(final ServiceContext context) throws Exception {
-        metadataIndexer.rebuildIndexXLinkedMetadata(context);
-    }
-
     @Deprecated
     public synchronized void rebuildIndexForSelection(final ServiceContext context, String bucket, boolean clearXlink) throws Exception {
         metadataIndexer.rebuildIndexForSelection(context, bucket, clearXlink);

diff --git a/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java b/core/src/main/java/org/fao/geonet/kernel/SelectionManager.java
@@ -23,13 +23,14 @@
 
 package org.fao.geonet.kernel;
 
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Hit;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import jeeves.server.UserSession;
 import jeeves.server.context.ServiceContext;
 
 import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.search.SearchHit;
 import org.fao.geonet.ApplicationContextHolder;
 import org.fao.geonet.constants.Edit;
 import org.fao.geonet.constants.Geonet;
@@ -248,8 +249,9 @@ public void selectAll(String type, ServiceContext context, UserSession session)
                     EsSearchManager searchManager = context.getBean(EsSearchManager.class);
                     searchResponse = searchManager.query(request.get("query"), FIELDLIST_UUID, 0, maxhits);
                     List<String> uuidList = new ArrayList();
-                    for (SearchHit h : Arrays.asList(searchResponse.getHits().getHits())) {
-                        uuidList.add((String) h.getSourceAsMap().get(Geonet.IndexFieldNames.UUID));
+                    ObjectMapper objectMapper = new ObjectMapper();
+                    for (Hit h : (List<Hit>) searchResponse.hits().hits()) {
+                        uuidList.add((String) objectMapper.convertValue(h.source(), Map.class).get(Geonet.IndexFieldNames.UUID));
                     }
 
                     if (selection != null) {

diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/IMetadataIndexer.java
@@ -1,5 +1,5 @@
 //=============================================================================
-//===	Copyright (C) 2001-2011 Food and Agriculture Organization of the
+//===	Copyright (C) 2001-2023 Food and Agriculture Organization of the
 //===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
 //===	and United Nations Environment Programme (UNEP)
 //===
@@ -24,11 +24,9 @@
 package org.fao.geonet.kernel.datamanager;
 
 import java.io.IOException;
-import java.util.Calendar;
 import java.util.List;
 
 import org.fao.geonet.domain.AbstractMetadata;
-import org.fao.geonet.kernel.search.ISearchManager;
 import org.fao.geonet.kernel.search.IndexingMode;
 import org.jdom.Element;
 import org.springframework.data.jpa.domain.Specification;
@@ -65,11 +63,6 @@ public interface IMetadataIndexer {
      */
     int batchDeleteMetadataAndUpdateIndex(Specification<? extends AbstractMetadata> specification) throws Exception;
 
-    /**
-     * Search for all records having XLinks (ie. indexed with _hasxlinks flag), clear the cache and reindex all records found.
-     */
-    void rebuildIndexXLinkedMetadata(ServiceContext context) throws Exception;
-
     /**
      * Reindex all records in current selection.
      */

diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataIndexer.java
@@ -200,31 +200,6 @@ public int batchDeleteMetadataAndUpdateIndex(Specification<? extends AbstractMet
         return metadataToDelete.size();
     }
 
-    @Override
-    /**
-     * Search for all records having XLinks (ie. indexed with _hasxlinks flag),
-     * clear the cache and reindex all records found.
-     */
-    public synchronized void rebuildIndexXLinkedMetadata(final ServiceContext context) throws Exception {
-
-        // get all metadata with XLinks
-        Set<Integer> toIndex = searchManager.getDocsWithXLinks();
-
-        if (Log.isDebugEnabled(Geonet.DATA_MANAGER))
-            Log.debug(Geonet.DATA_MANAGER, "Will index " + toIndex.size() + " records with XLinks");
-        if (toIndex.size() > 0) {
-            // clean XLink Cache so that cache and index remain in sync
-            Processor.clearCache();
-
-            ArrayList<String> stringIds = new ArrayList<String>();
-            for (Integer id : toIndex) {
-                stringIds.add(id.toString());
-            }
-            // execute indexing operation
-            batchIndexInThreadPool(context, stringIds);
-        }
-    }
-
     /**
      * Reindex all records in current selection.
      */
@@ -350,9 +325,9 @@ public void indexMetadata(final String metadataId,
                 List<Attribute> xlinks = Processor.getXLinks(md);
                 if (xlinks.size() > 0) {
                     fields.put(Geonet.IndexFieldNames.HASXLINKS, true);
-                    StringBuilder sb = new StringBuilder();
                     for (Attribute xlink : xlinks) {
                         fields.put(Geonet.IndexFieldNames.XLINK, xlink.getValue());
+                        fields.put(Geonet.IndexFieldNames.XLINK, xlink.getValue().replaceAll("local://srv/api/registries/entries/(.*)\\?.*", "$1"));
                     }
                     Processor.detachXLink(md, getServiceContext());
                 } else {

diff --git a/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java b/core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java
@@ -1342,8 +1342,8 @@ public boolean isValid(Integer id) {
     }
 
     boolean hasReferencingMetadata(ServiceContext context, AbstractMetadata metadata) throws Exception {
-        StringBuilder query = new StringBuilder(String.format("xlink:*%s*", metadata.getUuid()));
-        return this.searchManager.query(query.toString(), null, 0, 0).getHits().getTotalHits().value > 0;
+        StringBuilder query = new StringBuilder(String.format("xlink:\"%s\"", metadata.getUuid()));
+        return this.searchManager.query(query.toString(), null, 0, 0).hits().total().value() > 0;
     }
 
 }
diff --git a/core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java b/core/src/main/java/org/fao/geonet/kernel/mef/MEF2Exporter.java
@@ -23,10 +23,11 @@
 
 package org.fao.geonet.kernel.mef;
 
+import co.elastic.clients.elasticsearch.core.SearchResponse;
+import co.elastic.clients.elasticsearch.core.search.Hit;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import jeeves.server.context.ServiceContext;
 import org.apache.commons.io.FileUtils;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.search.SearchHit;
 import org.fao.geonet.Constants;
 import org.fao.geonet.GeonetContext;
 import org.fao.geonet.ZipUtil;
@@ -137,8 +138,9 @@ public static Path doExport(ServiceContext context, Set<String> uuids,
                 String mdSchema = null, mdTitle = null, mdAbstract = null, isHarvested = null;
                 MetadataType mdType = null;
 
-                SearchHit[] hits = result.getHits().getHits();
-                final Map<String, Object> source = hits[0].getSourceAsMap();
+                List<Hit> hits = result.hits().hits();
+                ObjectMapper objectMapper = new ObjectMapper();
+                final Map<String, Object> source = objectMapper.convertValue(hits.get(0).source(), Map.class);
                 mdSchema = (String) source.get(Geonet.IndexFieldNames.SCHEMA);
                 mdTitle = (String) source.get(Geonet.IndexFieldNames.RESOURCETITLE);
                 mdAbstract = (String) source.get(Geonet.IndexFieldNames.RESOURCEABSTRACT);