diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000000..20f2716957f --- /dev/null +++ b/docker/README.md @@ -0,0 +1,3 @@ +# Docker Test Environments + +These docker images are intended for development and debugging. For production we recommend the official GeoNetwork docker images at https://github.com/geonetwork/docker-geonetwork.git repository. diff --git a/docker/docker-geonetwork.txt b/docker/docker-geonetwork.txt deleted file mode 100644 index 1e1ae93bf58..00000000000 --- a/docker/docker-geonetwork.txt +++ /dev/null @@ -1 +0,0 @@ -Please see https://github.com/geonetwork/docker-geonetwork.git. diff --git a/docker/gn-cas-ldap/README.md b/docker/gn-cas-ldap/README.md index 326641c3a64..ae16f5181ed 100644 --- a/docker/gn-cas-ldap/README.md +++ b/docker/gn-cas-ldap/README.md @@ -1,4 +1,4 @@ -# Introduction +# GeoNetwork CAS Test Environment This composition is meant to make runtime testing the CAS integration of GeoNetwork easier. @@ -6,6 +6,8 @@ GeoNetwork easier. This composition also integrates a LDAP, so that testing the config-spring-cas-ldap configuration is also possible. +These docker images are intended for development and debugging. For production we recommend the official GeoNetwork docker images at https://github.com/geonetwork/docker-geonetwork.git repository. + # Prerequisites It requires the GeoNetwork webapp to be built first: @@ -18,7 +20,6 @@ Then it can be launched: ``` $ docker-compose up - ``` # Accessing the CAS login page from GeoNetwork diff --git a/docker/gn-postgres/README.md b/docker/gn-postgres/README.md new file mode 100644 index 00000000000..fb5aa0e58d2 --- /dev/null +++ b/docker/gn-postgres/README.md @@ -0,0 +1,8 @@ +# GeoNetwork PostgreSQL Test Environment + +This composition is meant to make runtime testing the PostgreSQL integration of +GeoNetwork easier. + +This folder provides a ``docker-compose.yml`` file for local testing. + +These docker images are intended for development and debugging. For production we recommend the official GeoNetwork docker images at https://github.com/geonetwork/docker-geonetwork.git repository. diff --git a/docker/docker-compose.yml b/docker/gn-postgres/docker-compose.yml similarity index 97% rename from docker/docker-compose.yml rename to docker/gn-postgres/docker-compose.yml index c6d942427d9..3c40e613034 100644 --- a/docker/docker-compose.yml +++ b/docker/gn-postgres/docker-compose.yml @@ -4,7 +4,7 @@ volumes: services: geonetwork: - image: geonetwork:3.99.0 + image: geonetwork:latest restart: always ports: - 8080:8080 diff --git a/docker/gn-postgres/kibana/kibana.txt b/docker/gn-postgres/kibana/kibana.txt new file mode 100644 index 00000000000..942837bbb76 --- /dev/null +++ b/docker/gn-postgres/kibana/kibana.txt @@ -0,0 +1,5 @@ +server.basePath: "/geonetwork/dashboards" +server.rewriteBasePath: false +kibana.index: ".dashboards" +elasticsearch.hosts: ["http://elasticsearch:9200"] + diff --git a/es/README.md b/es/README.md index c27cee0b4b0..0ef407c7f22 100644 --- a/es/README.md +++ b/es/README.md @@ -1,33 +1,48 @@ # Install, configure and start Elasticsearch +## Installation options + This section describes several methods for configuring Elasticsearch for development. These configurations should not be used for a production deployment. -## Manual installation +### Docker installation (Recommended) -1. Download Elasticsearch 8.x (tested with 8.11.3 for Geonetwork 4.4.x) from https://www.elastic.co/downloads/elasticsearch -and copy to the ES module, e.g., es/elasticsearch-8.11.3 +1. Use docker pull to download the image (you can check version in the :file:`pom.xml` file): -2. Disable the security + ``` + docker pull docker.elastic.co/elasticsearch/elasticsearch:8.11.3 + ``` -Elasticsearch 8 has security enabled by default. To disable this configuration for development, update the file `config/elasticsearch.yml` adding at the end: +2. Use docker run, leaving 9200 available: -``` -xpack.security.enabled: false -xpack.security.enrollment.enabled: false -``` + ``` + docker run -p 9200:9200 -p 9300:9300 \ + -e "discovery.type=single-node" \ + -e "xpack.security.enabled=false" \ + -e "xpack.security.enrollment.enabled=false" \ + docker.elastic.co/elasticsearch/elasticsearch:8.11.3 + ``` + +3. Check that elasticsearch is running by visiting http://localhost:9200 in a browser +### Docker compose installation -3. Start ES using: +1. Use docker compose with the provided [docker-compose.yml](docker-compose.yml): - ```shell script - ./bin/elasticsearch + ``` + cd es + docker-compose up ``` -4. Check that elasticsearch is running by visiting http://localhost:9200 in a browser +3. Check that it is running using your browser: + + * Elasticsearch: http://localhost:9200 + * Kibana: http://localhost:5601 + +### Maven installation -## Maven installation +Maven installation ensure you always are using the ``es.version`` version specified in ``pom.xml``. 1. Maven can take care of the installation steps: @@ -44,42 +59,32 @@ xpack.security.enrollment.enabled: false ``` 3. Check that elasticsearch is running by visiting http://localhost:9200 in a browser -## Docker installation +## Manual installation -1. Use docker pull to download the image (you can check version in the :file:`pom.xml` file): +1. Download Elasticsearch 8.11.3 from https://www.elastic.co/downloads/elasticsearch +and copy to the ES module, e.g., ``es/elasticsearch-8.11.3` - ``` - docker pull docker.elastic.co/elasticsearch/elasticsearch:8.11.3 - ``` +2. Disable the security -2. Use docker run, leaving 9200 available: + Elasticsearch 8 has security enabled by default. To disable this configuration for development, update the file `config/elasticsearch.yml` adding at the end: ``` - docker run -p 9200:9200 -p 9300:9300 \ - -e "discovery.type=single-node" \ - -e "xpack.security.enabled=false" \ - -e "xpack.security.enrollment.enabled=false" \ - docker.elastic.co/elasticsearch/elasticsearch:8.11.3 + xpack.security.enabled: false + xpack.security.enrollment.enabled: false ``` -3. Check that elasticsearch is running by visiting http://localhost:9200 in a browser - -## Docker compose installation - -1. Use docker compose with the provided [docker-compose.yml](docker-compose.yml): +3. Start ES using: - ``` - cd es - docker-compose up + ```shell script + ./bin/elasticsearch ``` -3. Check that it is running using your browser: - - * Elasticsearch: http://localhost:9200 - * Kibana: http://localhost:5601 +4. Check that elasticsearch is running by visiting http://localhost:9200 in a browser # Configuration +## Index management + Optionally you can manually create index but they will be created by the catalogue when the Elastic instance is available and if index does not exist. @@ -122,7 +127,7 @@ Don't hesitate to propose a Pull Request with the new language. 1. Configure ES to start on server startup. It is recommended to protect `gn-records` index from the Internet access. - * Note that for debian-based servers the current deb download (7.3.2) can be installed rather than installing manually and can be configured to run as a service using the instructions here: https://www.elastic.co/guide/en/elasticsearch/reference/current/starting-elasticsearch.html + * Note that for debian-based servers the current deb download (8.11.3) can be installed rather than installing manually and can be configured to run as a service using the instructions here: https://www.elastic.co/guide/en/elasticsearch/reference/current/starting-elasticsearch.html # Troubleshoot @@ -163,3 +168,22 @@ field expansion for [*] matches too many fields, limit: 1024 An option is to restrict `queryBase` to limit the number of field to query on. `any:(${any}) resourceTitleObject.default:(${any})^2` is a good default. Using `${any}` will probably trigger the error if the number of records is high. The other option is to increase `indices.query.bool.max_clause_count`. + + +## Disk space threshold + +The server application will refuse to write new content unless there is enough free space available (by default 1/4 of your hard drive). + +To turn off this check: + +``` + curl -XPUT http://localhost:9200/_cluster/settings -H 'Content-Type: application/json' -d '{ "transient" : { "cluster.routing.allocation.disk.threshold_enabled" : false } }' +``` + +## Blocked by index read-only / allow delete + +To recover: + +``` +curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_all/_settings -d '{"index.blocks.read_only_allow_delete": null}' +``` diff --git a/es/es-dashboards/README.md b/es/es-dashboards/README.md index b95aa299cb4..b39111ae58e 100644 --- a/es/es-dashboards/README.md +++ b/es/es-dashboards/README.md @@ -1,24 +1,20 @@ # Install, configure and start Kibana -## Manual installation - -Download Kibana from https://www.elastic.co/downloads/kibana. For Geonetwork 3.8.x download at least version 7.2.1 +## Installation options -Set Kibana base path and index name in config/kibana.yml: +### Docker compose installation (Recommended) -``` -server.basePath: "/geonetwork/dashboards" -server.rewriteBasePath: false -``` +1. Use docker compose with the provided [docker-compose.yml](es/docker-compose.yml): -Adapt if needed ```elasticsearch.url``` and ```server.host```. - -Start Kibana manually: + ``` + cd es + docker-compose up + ``` -``` -cd kibana/bin -./kibana -``` +3. Check that it is running using your browser: + + * Elasticsearch: http://localhost:9200 + * Kabana: http://localhost:5601 ## Maven installation @@ -41,20 +37,26 @@ cd kibana/bin mvn exec:exec -Dkb-start ``` -## Docker compose installation +## Manual installation + +1. Download Kibana 8.11.3 from https://www.elastic.co/downloads/kibana + +2. Set Kibana base path and index name in config/kibana.yml: + + ``` + server.basePath: "/geonetwork/dashboards" + server.rewriteBasePath: false + ``` + +3. Adapt if needed ```elasticsearch.url``` and ```server.host```. -1. Use docer compose with the provided [docker-compose.yml](docker-compose.yml): +4. Start Kibana manually: ``` - cd es - docker-compose up + cd kibana/bin + ./kibana ``` -3. Check that it is running using your browser: - - * Elasticsearch: http://localhost:9200 - * Kabana: http://localhost:5601 - ## Import Configuration 1. Kibana should be running from: @@ -69,16 +71,17 @@ cd kibana/bin http://localhost:8080/geonetwork/dashboards ``` + ## Troubleshoot If it does not start properly, check Kibana log files (eg. it may fail if Elasticsearch version is not compatible with Kibana version). -Visit Kibana in a browser using one of the above links and go to 'Saved Objects'. Import export.ndjson from https://github.com/geonetwork/core-geonetwork/blob/4.0.x/es/es-dashboards/data/export.ndjson +Visit Kibana in a browser using one of the above links and go to 'Saved Objects'. Import export.ndjson from https://github.com/geonetwork/core-geonetwork/blob/main/es/es-dashboards/data/export.ndjson ### Production Use -Kibana can be installed from the debian files, and 7.3.2 is confirmed as working with Geonetwork 3.8.x. +Kibana can be installed from the debian files, and Kibana 8.11.3 is confirmed as working with Geonetwork 4.4.x. Set Kibana to start when the server starts up, using the instructions at https://www.elastic.co/guide/en/kibana/current/start-stop.html diff --git a/es/es-dashboards/pom.xml b/es/es-dashboards/pom.xml index fd86d42981d..551d62b0f0b 100644 --- a/es/es-dashboards/pom.xml +++ b/es/es-dashboards/pom.xml @@ -24,12 +24,74 @@ 4.0.0 gn-es-dashboards GeoNetwork dashboard app based on Kibana - gn-es org.geonetwork-opensource 4.4.4-SNAPSHOT + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + check-readme + + enforce + + + + + Update README.md examples for Elasticsearch ${es.version} + + import java.util.regex.Pattern; + + esVersion = "${es.version}"; + print("Scanning README for " + esVersion); + + docker = Pattern.compile("Kibana (\\d.\\d\\d.\\d)"); + download = Pattern.compile("Download Kibana (\\d.\\d\\d.\\d)"); + + patterns = new Pattern[]{ docker, download}; + + readme = new BufferedReader(new FileReader("README.md")); + + number = 0; + while ((line = readme.readLine()) != null) { + number++; + for (pattern : patterns ){ + matcher = pattern.matcher(line); + if (matcher.find()) { + if (!esVersion.equals(matcher.group(1))) { + print("README.md:"+number+" FAILURE: " + line); + return false; + } + } + } + } + readme.close(); + true; + + + + + + + check-docker + + enforce + + + true + + + + + + + kb-download diff --git a/es/pom.xml b/es/pom.xml index d9338a9fc8f..883c046d3a0 100644 --- a/es/pom.xml +++ b/es/pom.xml @@ -12,6 +12,105 @@ GeoNetwork index using Elasticsearch pom + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + check-readme + + enforce + + + + + Update README.md examples for Elasticsearch ${es.version} + + import java.util.regex.Pattern; + + esVersion = "${es.version}"; + print("Scanning README for " + esVersion); + + docker = Pattern.compile("elasticsearch:(\\d.\\d\\d.\\d)"); + download = Pattern.compile("Download Elasticsearch (\\d.\\d\\d.\\d)"); + folder = Pattern.compile("es/elasticsearch-(\\d.\\d\\d.\\d)"); + + patterns = new Pattern[]{ docker, download, folder}; + + readme = new BufferedReader(new FileReader("README.md")); + + number = 0; + while ((line = readme.readLine()) != null) { + number++; + for (pattern : patterns ){ + matcher = pattern.matcher(line); + if (matcher.find()) { + if (!esVersion.equals(matcher.group(1))) { + print("README.md:"+number+" FAILURE: " + line); + return false; + } + } + } + } + readme.close(); + true; + + + + + + + check-docker + + enforce + + + + + Update docker-compose.yml for Elasticsearch ${es.version} + + import java.util.regex.Pattern; + + boolean scanDockerCompose(String filename){ + esVersion = "${es.version}"; + print("Scanning "+filename+" for " + esVersion); + + docker = Pattern.compile("elasticsearch:(\\d.\\d\\d.\\d)"); + kibana = Pattern.compile("kibana:(\\d.\\d\\d.\\d)"); + patterns = new Pattern[]{ docker, kibana}; + + reader = new BufferedReader(new FileReader("${project.basedir}"+"/"+filename)); + + number = 0; + while ((line = reader.readLine()) != null) { + number++; + for (pattern : patterns ){ + matcher = pattern.matcher(line); + if (matcher.find()) { + if (!esVersion.equals(matcher.group(1))) { + print(filename+":"+number+" FAILURE: " + line); + return false; + } + } + } + } + reader.close(); + return true; + } + + return scanDockerCompose("docker-compose.yml"); + + + + + + + + + + es-download