From 8a73dc05e64ebbbafdf48713e111ae1e50fbb849 Mon Sep 17 00:00:00 2001 From: "Drori, Eynan" Date: Mon, 7 Oct 2019 14:04:55 +0300 Subject: [PATCH 01/13] add support for es 7.x --- .../ElasticRestClient.java | 23 +++++++- .../embeddedelasticsearch/IndexSettings.java | 58 ++++++++++++++++--- .../InstallFromVersion.java | 3 +- .../TypeWithMapping.java | 2 +- 4 files changed, 73 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java index 1f2b88a..cbcd1e4 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java @@ -164,10 +164,13 @@ private String indexMetadataJson(String indexName, String indexType, String id, joiner.add("\"_id\": \"" + id + "\""); } - if(routing != null) { - joiner.add("\"_routing\": \"" + routing + "\""); + if (routing != null) { + if (newESVersion()) { + joiner.add("\"routing\": \"" + routing + "\""); + } else { + joiner.add("\"_routing\": \"" + routing + "\""); + } } - return "{ \"index\": {" + joiner.toString() + "} }"; } @@ -180,6 +183,20 @@ void refresh() { } } + private boolean newESVersion() { + HttpGet request = new HttpGet(url("/")); + return httpClient.execute(request, response -> { + JsonNode jsonNode; + try { + jsonNode = OBJECT_MAPPER.readTree(readBodySafely(response)); + } catch (IOException e) { + return false; + } + String esV = jsonNode.get("version").get("number").asText(); + return Integer.parseInt(esV.substring(0,1)) >= 7; //if version is 7 and above + }); + } + private void performBulkRequest(String requestUrl, String bulkRequestBody) { HttpPost request = new HttpPost(requestUrl); request.setHeader(new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json")); diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/IndexSettings.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/IndexSettings.java index bf2e999..edd8415 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/IndexSettings.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/IndexSettings.java @@ -17,31 +17,65 @@ public class IndexSettings { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private final Optional mappings; private final List types; private final Optional settings; private final Optional aliases; - + private boolean includeTypeName=false; public static Builder builder() { return new Builder(); } - public IndexSettings(List types, Optional settings) { + private IndexSettings(List types, Optional settings, Optional aliases) { + this.mappings = rawToJson(Optional.of("{}")); this.types = types; this.settings = rawToJson(settings); this.aliases = Optional.empty(); } - private IndexSettings(List types, Optional settings, Optional aliases) { - this.types = types; + public IndexSettings(Optional mapping, Optional settings) { + includeTypeName = true; + this.mappings = rawToJson(mapping); + this.settings = rawToJson(settings); + this.aliases = Optional.empty(); + this.types = new ArrayList<>(); + } + + private IndexSettings(Optional mapping, Optional settings, Optional aliases) { + this.mappings = rawToJson(mapping); + includeTypeName = true; this.settings = rawToJson(settings); this.aliases = rawToJson(aliases); + this.types = new ArrayList<>(); } public static class Builder { - private final List types = new ArrayList<>(); + private Optional mapping = Optional.empty(); private Optional settings = Optional.empty(); private Optional aliases = Optional.empty(); + private final List types = new ArrayList<>(); + + /** + * Type with mappings to create with index + * + * @param mapping mappings for created type + */ + public Builder withMapping(Object mapping) throws IOException { + String mappingString; + if (mapping == null) { + return this; + } + else if (mapping instanceof InputStream) { + InputStream mappingStream = (InputStream) mapping; + mappingString = IOUtils.toString(mappingStream, UTF_8); + } + else { + mappingString = (String) mapping; + } + this.mapping = Optional.of(mappingString); + return this; + } /** * Specify type inside created index @@ -53,6 +87,7 @@ public Builder withType(String type, InputStream mapping) throws IOException { return withType(type, IOUtils.toString(mapping, UTF_8)); } + /** * Type with mappings to create with index * @@ -106,7 +141,7 @@ public Builder withAliases(String aliases) { * @return IndexSettings with specified parameters */ public IndexSettings build() { - return new IndexSettings(types, settings, aliases); + return new IndexSettings(mapping, settings, aliases); } } @@ -114,8 +149,15 @@ public ObjectNode toJson() { ObjectNode objectNode = new ObjectMapper().createObjectNode(); objectNode.set("settings", settings.orElse(OBJECT_MAPPER.createObjectNode())); objectNode.set("aliases", aliases.orElse(OBJECT_MAPPER.createObjectNode())); - ObjectNode mappingsObject = prepareMappingsObject(); - objectNode.set("mappings", mappingsObject); + + if (includeTypeName){ + objectNode.set("mappings",mappings.orElse(OBJECT_MAPPER.createObjectNode())); + } + else { + ObjectNode mappingsObject = prepareMappingsObject(); + objectNode.set("mappings", mappingsObject); + } + return objectNode; } diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java index 6b7524d..3d75ea6 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java @@ -39,7 +39,8 @@ private enum ElsDownloadUrl { ELS_1x("1.", "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-{VERSION}.zip"), ELS_2x("2.", "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/zip/elasticsearch/{VERSION}/elasticsearch-{VERSION}.zip"), ELS_5x("5.", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{VERSION}.zip"), - ELS_6x("6.", ELS_5x.downloadUrl); + ELS_6x("6.", ELS_5x.downloadUrl), + ELS_7x("7.", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{VERSION}-windows-x86_64.zip"); String versionPrefix; String downloadUrl; diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/TypeWithMapping.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/TypeWithMapping.java index 159561f..72f06ef 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/TypeWithMapping.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/TypeWithMapping.java @@ -30,4 +30,4 @@ public String getType() { public JsonNode getMapping() { return mapping; } -} +} \ No newline at end of file From 4df08504d0464a3f83112ce02fadf72a219055dd Mon Sep 17 00:00:00 2001 From: "Drori, Eynan" Date: Mon, 7 Oct 2019 14:05:34 +0300 Subject: [PATCH 02/13] add es 7.3 unit tests --- es73-test/build.gradle | 10 ++ .../EmbeddedElasticSpec.groovy | 125 ++++++++++++++++++ .../PluginsInstallationSpec.groovy | 31 +++++ .../test/resources/audio-book-mapping.json | 21 +++ es73-test/src/test/resources/car-mapping.json | 15 +++ .../src/test/resources/elastic-settings.json | 15 +++ .../test/resources/paper-book-mapping.json | 15 +++ settings.gradle | 1 + .../EmbeddedElasticCoreApiBaseSpec.groovy | 7 +- .../SampleIndices.groovy | 8 ++ .../src/main/resources/cars-template-7x.json | 18 +++ 11 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 es73-test/build.gradle create mode 100644 es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/EmbeddedElasticSpec.groovy create mode 100644 es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy create mode 100644 es73-test/src/test/resources/audio-book-mapping.json create mode 100644 es73-test/src/test/resources/car-mapping.json create mode 100644 es73-test/src/test/resources/elastic-settings.json create mode 100644 es73-test/src/test/resources/paper-book-mapping.json create mode 100644 test-base/src/main/resources/cars-template-7x.json diff --git a/es73-test/build.gradle b/es73-test/build.gradle new file mode 100644 index 0000000..02f817f --- /dev/null +++ b/es73-test/build.gradle @@ -0,0 +1,10 @@ +dependencies { + testCompile project(':test-base') + + testCompile group: 'org.elasticsearch', name: 'elasticsearch', version: '7.3.2' + testCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.3.2' + testCompile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-client', version: '7.3.2' + testCompile group: 'org.locationtech.spatial4j', name: 'spatial4j', version: '0.6' + testCompile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.6.2' + testCompile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.6.2' +} \ No newline at end of file diff --git a/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/EmbeddedElasticSpec.groovy b/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/EmbeddedElasticSpec.groovy new file mode 100644 index 0000000..8bd0a7e --- /dev/null +++ b/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/EmbeddedElasticSpec.groovy @@ -0,0 +1,125 @@ +package pl.allegro.tech.embeddedelasticsearch + + +import org.apache.http.HttpHost +import org.elasticsearch.action.get.GetRequest +import org.elasticsearch.action.search.SearchRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.search.builder.SearchSourceBuilder + +import static java.util.concurrent.TimeUnit.MINUTES +import static pl.allegro.tech.embeddedelasticsearch.PopularProperties.HTTP_PORT +import static pl.allegro.tech.embeddedelasticsearch.SampleIndices.* + +class EmbeddedElasticSpec extends EmbeddedElasticCoreApiBaseSpec { + + static final ELASTIC_VERSION = "7.3.2" + static final HTTP_PORT_VALUE = 9999 + static final DOC_TYPE = "_doc" + + static EmbeddedElastic embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(HTTP_PORT, HTTP_PORT_VALUE) + .withEsJavaOpts("-Xms128m -Xmx512m") + .withTemplate(CARS_TEMPLATE_NAME, CARS_TEMPLATE_7x) + .withIndex(CARS_INDEX_NAME,CARS_INDEX_7x) + .withIndex(BOOKS_INDEX_NAME, BOOKS_INDEX) + .withStartTimeout(2, MINUTES) + .build() + .start() + + static RestHighLevelClient client = createClient() + + def setup() { + embeddedElastic.recreateIndices() + } + + def cleanupSpec() { + client.close() + embeddedElastic.stop() + } + + static RestHighLevelClient createClient() { + return new RestHighLevelClient(RestClient.builder(new HttpHost("localhost", HTTP_PORT_VALUE))) + } + + @Override + void index(IndexRequest indexRequest) { + IndexRequest newIndexRequest = new IndexRequest.IndexRequestBuilder(indexRequest.getIndexName(),"_doc",indexRequest.getJson()).build() + index(Arrays.asList(newIndexRequest)) + } + + @Override + void index(List indexRequests) { + ArrayList newIndexRequests = new ArrayList<>() + for (IndexRequest newIndexRequest : indexRequests) { + newIndexRequests.add( new IndexRequest.IndexRequestBuilder(newIndexRequest.getIndexName(),DOC_TYPE,newIndexRequest.getJson()).withId(newIndexRequest.getId()).withRouting(newIndexRequest.getRouting()).build()) + } + embeddedElastic.index(newIndexRequests) + } + + @Override + void index(SampleIndices.PaperBook book) { + index(new IndexRequest.IndexRequestBuilder(BOOKS_INDEX_NAME, DOC_TYPE, toJson(book)).build()) + } + + @Override + void index(SampleIndices.Car car) { + index(new IndexRequest.IndexRequestBuilder(CARS_INDEX_NAME, DOC_TYPE, toJson(car)).build()) + } + + @Override + void index(String indexName, String indexType, Map idJsonMap) { + embeddedElastic.index(indexName, DOC_TYPE,idJsonMap) + } + + @Override + List fetchAllDocuments() { + fetchAllDocuments(CARS_INDEX_NAME) + fetchAllDocuments(BOOKS_INDEX_NAME) + } + + @Override + List fetchAllDocuments(String indexName) { + final searchRequest = new SearchRequest(indexName) + .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())); + + client.search(searchRequest, RequestOptions.DEFAULT) + .hits.hits.toList() + .collect { it.sourceAsString } + } + + @Override + List fetchAllDocuments(String indexName, String typeName) { + fetchAllDocuments(indexName) + } + + @Override + List fetchAllDocuments(String indexName, String typeName, String routing) { + final searchRequest = new SearchRequest(indexName) + .routing(routing) + .source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())) + + client.search(searchRequest, RequestOptions.DEFAULT) + .hits.hits.toList() + .collect { it.sourceAsString } + } + + @Override + List searchByTerm(String indexName, String typeName, String fieldName, String value) { + final searchRequest = new SearchRequest() + .source(new SearchSourceBuilder().query(QueryBuilders.termQuery(fieldName, value))) + + client.search(searchRequest, RequestOptions.DEFAULT) + .hits.hits.toList() + .collect { it.sourceAsString } + } + + @Override + String getById(String indexName, String typeName, String id) { + final getRequest = new GetRequest(indexName, DOC_TYPE, id) + client.get(getRequest,RequestOptions.DEFAULT).sourceAsString + } +} diff --git a/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy b/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy new file mode 100644 index 0000000..7e73afa --- /dev/null +++ b/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy @@ -0,0 +1,31 @@ +package pl.allegro.tech.embeddedelasticsearch + +import static java.util.concurrent.TimeUnit.MINUTES + +class PluginsInstallationSpec extends PluginsInstallationBaseSpec { + + static final HTTP_PORT_VALUE = 9200 + + EmbeddedElastic.Builder baseEmbeddedElastic() { + return EmbeddedElastic.builder() + .withElasticVersion("6.3.0") + .withEsJavaOpts("-Xms128m -Xmx512m") + .withSetting(PopularProperties.HTTP_PORT, HTTP_PORT_VALUE) + .withStartTimeout(2, MINUTES) + } + + @Override + String pluginByUrlUrl() { + return "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-stempel/analysis-stempel-6.3.0.zip" + } + + @Override + String pluginByUrlName() { + return "analysis-stempel" + } + + @Override + String pluginByName() { + return "discovery-file" + } +} \ No newline at end of file diff --git a/es73-test/src/test/resources/audio-book-mapping.json b/es73-test/src/test/resources/audio-book-mapping.json new file mode 100644 index 0000000..2327dee --- /dev/null +++ b/es73-test/src/test/resources/audio-book-mapping.json @@ -0,0 +1,21 @@ +{ + "audio_book": { + "properties": { + "author": { + "type": "text", + "index": "false" + }, + "title": { + "type": "text", + "index": "false" + }, + "readBy": { + "type": "text", + "index": "false" + }, + "description": { + "type": "text" + } + } + } +} \ No newline at end of file diff --git a/es73-test/src/test/resources/car-mapping.json b/es73-test/src/test/resources/car-mapping.json new file mode 100644 index 0000000..ae25c2f --- /dev/null +++ b/es73-test/src/test/resources/car-mapping.json @@ -0,0 +1,15 @@ +{ + "properties": { + "manufacturer": { + "type": "text", + "index": "false" + }, + "model": { + "type": "text", + "index": "true" + }, + "description": { + "type": "text" + } + } +} \ No newline at end of file diff --git a/es73-test/src/test/resources/elastic-settings.json b/es73-test/src/test/resources/elastic-settings.json new file mode 100644 index 0000000..8ebf291 --- /dev/null +++ b/es73-test/src/test/resources/elastic-settings.json @@ -0,0 +1,15 @@ +{ + "number_of_shards" : 5, + "analysis": { + "analyzer": { + "custom-analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": [ + "asciifolding", + "lowercase" + ] + } + } + } +} \ No newline at end of file diff --git a/es73-test/src/test/resources/paper-book-mapping.json b/es73-test/src/test/resources/paper-book-mapping.json new file mode 100644 index 0000000..a2b790d --- /dev/null +++ b/es73-test/src/test/resources/paper-book-mapping.json @@ -0,0 +1,15 @@ +{ + "properties": { + "author": { + "type": "text", + "index": "false" + }, + "title": { + "type": "text", + "index": "false" + }, + "description": { + "type": "text" + } + } +} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 1a3217d..191545b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,3 +8,4 @@ include 'es50-test' include 'es55-test' include 'es60-test' include 'es63-test' +include 'es73-test' diff --git a/test-base/src/main/groovy/pl/allegro/tech/embeddedelasticsearch/EmbeddedElasticCoreApiBaseSpec.groovy b/test-base/src/main/groovy/pl/allegro/tech/embeddedelasticsearch/EmbeddedElasticCoreApiBaseSpec.groovy index 834c126..736ed58 100644 --- a/test-base/src/main/groovy/pl/allegro/tech/embeddedelasticsearch/EmbeddedElasticCoreApiBaseSpec.groovy +++ b/test-base/src/main/groovy/pl/allegro/tech/embeddedelasticsearch/EmbeddedElasticCoreApiBaseSpec.groovy @@ -1,5 +1,6 @@ package pl.allegro.tech.embeddedelasticsearch +import groovy.json.JsonOutput import org.skyscreamer.jsonassert.JSONAssert import spock.lang.Specification @@ -46,7 +47,7 @@ abstract class EmbeddedElasticCoreApiBaseSpec extends Specification { final document = toJson(FIAT_126p) when: - embeddedElastic.index(CARS_INDEX_NAME, CAR_INDEX_TYPE, ["$id": document]) + index(CARS_INDEX_NAME, CAR_INDEX_TYPE, ["$id": document]) then: final result = getById(CARS_INDEX_NAME, CAR_INDEX_TYPE, id) @@ -164,6 +165,10 @@ abstract class EmbeddedElasticCoreApiBaseSpec extends Specification { index(new IndexRequest.IndexRequestBuilder(CARS_INDEX_NAME, CAR_INDEX_TYPE, toJson(car)).build()) } + void index(String indexName, String indexType, Map idJsonMap) { + embeddedElastic.index(indexName, indexType, idJsonMap) + } + void index(SampleIndices.PaperBook book) { index(new IndexRequest.IndexRequestBuilder(BOOKS_INDEX_NAME, PAPER_BOOK_INDEX_TYPE, toJson(book)).build()) } diff --git a/test-base/src/main/groovy/pl/allegro/tech/embeddedelasticsearch/SampleIndices.groovy b/test-base/src/main/groovy/pl/allegro/tech/embeddedelasticsearch/SampleIndices.groovy index 1599a0c..f928796 100644 --- a/test-base/src/main/groovy/pl/allegro/tech/embeddedelasticsearch/SampleIndices.groovy +++ b/test-base/src/main/groovy/pl/allegro/tech/embeddedelasticsearch/SampleIndices.groovy @@ -9,10 +9,12 @@ class SampleIndices { static final BOOK_ALIAS_1 = "book_alias_1" static final BOOK_ALIAS_2 = "book_alias_2" static final CARS_INDEX_NAME = "cars" + static final CARS_MAPPING_NAME = "car-mapping.json" static final CAR_INDEX_TYPE = "car" static final CARS_TEMPLATE_NAME = "cars_template" static final CARS_TEMPLATE = resourceToString("cars-template.json") static final CARS_TEMPLATE_6x = getSystemResourceAsStream("cars-template-6x.json") + static final CARS_TEMPLATE_7x = getSystemResourceAsStream("cars-template-7x.json") static final BOOKS_INDEX_NAME = "books" static final PAPER_BOOK_INDEX_TYPE = "paper_book" static final AUDIO_BOOK_INDEX_TYPE = "audio_book" @@ -29,6 +31,12 @@ class SampleIndices { .withAliases(getSystemResourceAsStream("elastic-aliases.json")) .build() + static final CARS_INDEX_7x = IndexSettings.builder() + .withMapping(getSystemResourceAsStream("car-mapping.json")) + .withSettings(getSystemResourceAsStream("elastic-settings.json")) + .withAliases(getSystemResourceAsStream("elastic-aliases.json")) + .build() + static String toJson(Car car) { """ { diff --git a/test-base/src/main/resources/cars-template-7x.json b/test-base/src/main/resources/cars-template-7x.json new file mode 100644 index 0000000..3443bb5 --- /dev/null +++ b/test-base/src/main/resources/cars-template-7x.json @@ -0,0 +1,18 @@ +{ + "index_patterns": [ + "car*" + ], + "mappings": { + "properties": { + "manufacturer": { + "type": "keyword" + }, + "model": { + "type": "keyword" + }, + "description": { + "type": "text" + } + } + } +} \ No newline at end of file From b304743d4f3100d161cc48a1a7641ba010fff6df Mon Sep 17 00:00:00 2001 From: eynand <48277393+eynand@users.noreply.github.com> Date: Thu, 10 Oct 2019 10:54:21 +0300 Subject: [PATCH 03/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e0ce191..b9c6d51 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build Status](https://travis-ci.org/allegro/embedded-elasticsearch.svg?branch=master)](https://travis-ci.org/allegro/embedded-elasticsearch) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/pl.allegro.tech/embedded-elasticsearch/badge.svg)](http://central.maven.org/maven2/pl/allegro/tech/embedded-elasticsearch) -Small utility for creating integration tests that use Elasticsearch. Instead of using `Node` it downloads Elasticsearch in specified version and starts it in a separate process. It also allows you to install required plugins which is not possible when using `NodeBuilder`. Utility was tested with 1.x, 2.x, 5.x and 6.x versions of Elasticsearch. +Small utility for creating integration tests that use Elasticsearch. Instead of using `Node` it downloads Elasticsearch in specified version and starts it in a separate process. It also allows you to install required plugins which is not possible when using `NodeBuilder`. Utility was tested with 1.x, 2.x, 5.x, 6.x and 7.x versions of Elasticsearch. ## Introduction From 5bd0d56ac7b660acaf0ddbfca1648b2ab007c45a Mon Sep 17 00:00:00 2001 From: eynand <48277393+eynand@users.noreply.github.com> Date: Thu, 10 Oct 2019 10:57:46 +0300 Subject: [PATCH 04/13] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b9c6d51..ab6371f 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ Available `IndexSettings.Builder` options | Method | Description | | ------------- | ------------- | | `withType(String type, String mapping)` | specify type and it's mappings | +| `withMapping(String mapping)` | starting from Elasticseatch 7, there is not more types, so when using an ES version 7.0 and above this method should be used insted of withType | | `withSettings(String settings)` | specify index settings | From 0f12d096b231f2a60a0316c803b905f99f41b3a7 Mon Sep 17 00:00:00 2001 From: eynand <48277393+eynand@users.noreply.github.com> Date: Wed, 23 Oct 2019 15:12:19 +0300 Subject: [PATCH 05/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab6371f..ffac889 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Available `IndexSettings.Builder` options | Method | Description | | ------------- | ------------- | | `withType(String type, String mapping)` | specify type and it's mappings | -| `withMapping(String mapping)` | starting from Elasticseatch 7, there is not more types, so when using an ES version 7.0 and above this method should be used insted of withType | +| `withMapping(String mapping)` | starting from Elasticseatch 7, there is no more types, so when using an ES version 7.0 and above this method should be used insted of withType method | | `withSettings(String settings)` | specify index settings | From a82f1066b17ed2d33b0c9bdb0ec6cb76b55b1ac2 Mon Sep 17 00:00:00 2001 From: Benjamin Kihm Date: Thu, 20 Feb 2020 16:45:24 +0100 Subject: [PATCH 06/13] Change dist to trusty --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 18268ff..2cb3134 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ --- language: java -script: +dist: trusty +script: - ./gradlew clean check --info jdk: - oraclejdk8 From 5794b0d70e5f17a4ea4d59a45bf5404e733d6594 Mon Sep 17 00:00:00 2001 From: Pablo Saavedra Date: Fri, 10 Apr 2020 15:33:55 -0300 Subject: [PATCH 07/13] Added support for downloading platform specific versions of elastic search. This includes choosing the right URL for elastic 7 and the ability to install from tar.gz (besides zip) --- .../ElasticSearchInstaller.java | 19 +++++++++++-- .../InstallFromVersion.java | 27 +++++++++++++++++-- .../InstallationSourceSpec.groovy | 14 ++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java index a861a77..00a5149 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java @@ -7,9 +7,13 @@ import org.slf4j.LoggerFactory; import pl.allegro.tech.embeddedelasticsearch.InstallationDescription.Plugin; +import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.lang.ProcessBuilder.Redirect; import java.nio.file.Path; @@ -17,6 +21,7 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.zip.GZIPInputStream; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.commons.io.FileUtils.forceMkdir; @@ -72,8 +77,18 @@ private void installElastic(Path downloadedTo) throws IOException { } private void unzip(Path downloadedTo, File destination) throws IOException { - Archiver archiver = ArchiverFactory.createArchiver("zip"); - archiver.extract(downloadedTo.toFile(), destination); + Archiver archiver = ArchiverFactory.createArchiver(downloadedTo.endsWith("zip") ? "zip" : "tar"); + try (InputStream is = toStream(downloadedTo)) { + archiver.extract(is, destination); + } + } + + private InputStream toStream(Path downloadedTo) throws IOException { + InputStream result = new FileInputStream(downloadedTo.toFile()); + if (downloadedTo.endsWith(".gz")) { + result = new GZIPInputStream(result); + } + return new BufferedInputStream(result); } private void configureElastic() throws IOException { diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java index 3d75ea6..208c546 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/InstallFromVersion.java @@ -1,6 +1,7 @@ package pl.allegro.tech.embeddedelasticsearch; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.SystemUtils; import java.net.MalformedURLException; import java.net.URL; @@ -29,7 +30,7 @@ public URL resolveDownloadUrl() { private URL urlFromVersion(String version) { ElsDownloadUrl elsDownloadUrl = ElsDownloadUrl.getByVersion(version); try { - return new URL(StringUtils.replace(elsDownloadUrl.downloadUrl, "{VERSION}", version)); + return new URL(StringUtils.replace(elsDownloadUrl.getDownloadUrl(), "{VERSION}", version)); } catch (MalformedURLException e) { throw new RuntimeException(e); } @@ -40,7 +41,25 @@ private enum ElsDownloadUrl { ELS_2x("2.", "https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/zip/elasticsearch/{VERSION}/elasticsearch-{VERSION}.zip"), ELS_5x("5.", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{VERSION}.zip"), ELS_6x("6.", ELS_5x.downloadUrl), - ELS_7x("7.", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{VERSION}-windows-x86_64.zip"); + ELS_7x("7.", "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{VERSION}-{PLATFORM}-x86_64.{EXTENSION}") { + @Override + String getDownloadUrl() { + if (SystemUtils.IS_OS_LINUX) { + String result = StringUtils.replace(downloadUrl, PLATFORM, "linux"); + return StringUtils.replace(result, EXTENSION, "tar.gz"); + } else if (SystemUtils.IS_OS_MAC) { + String result = StringUtils.replace(downloadUrl, PLATFORM, "darwin"); + return StringUtils.replace(result, EXTENSION, "tar.gz"); + } else if (SystemUtils.IS_OS_WINDOWS) { + String result = StringUtils.replace(downloadUrl, PLATFORM, "windows"); + return StringUtils.replace(result, EXTENSION, "zip"); + } + throw new IllegalArgumentException(("Unsupported OS version " + SystemUtils.OS_NAME)); + } + }; + + private static final String PLATFORM = "{PLATFORM}"; + private static final String EXTENSION = "{EXTENSION}"; String versionPrefix; String downloadUrl; @@ -54,6 +73,10 @@ boolean versionMatch(String elasticVersion) { return elasticVersion.startsWith(versionPrefix); } + String getDownloadUrl() { + return downloadUrl; + } + static ElsDownloadUrl getByVersion(String elasticVersion) { return Arrays.stream(ElsDownloadUrl.values()) .filter(u -> u.versionMatch(elasticVersion)) diff --git a/core/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/InstallationSourceSpec.groovy b/core/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/InstallationSourceSpec.groovy index f9cc10c..f775ee9 100644 --- a/core/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/InstallationSourceSpec.groovy +++ b/core/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/InstallationSourceSpec.groovy @@ -4,6 +4,10 @@ import spock.lang.Specification import spock.lang.Unroll class InstallationSourceSpec extends Specification { + static { + //Need to do this here since we read the value statically + System.setProperty("os.name", "Linux") + } def "should construct valid url for version"() { given: @@ -14,6 +18,16 @@ class InstallationSourceSpec extends Specification { resolvedUrl != null } + def "should construct valid url for platform specific version"() { + given: + final installationSource = new InstallFromVersion("7.6.2") + when: + final resolvedUrl = installationSource.resolveDownloadUrl() + then: + resolvedUrl != null + resolvedUrl.toExternalForm() == "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.6.2-linux-x86_64.tar.gz" + } + def "should extract properly version from normal url"() { given: final expectedVersion = "2.3.4" From f434fc2e590f886b3a09f0c97f97057c1728bd1c Mon Sep 17 00:00:00 2001 From: Pablo Saavedra Date: Fri, 10 Apr 2020 16:16:40 -0300 Subject: [PATCH 08/13] Setting executable permission if nedded and fixed a bug when downloading tar.gz files --- .../tech/embeddedelasticsearch/ElasticSearchInstaller.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java index 00a5149..8197461 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java @@ -69,6 +69,10 @@ private void installElastic(Path downloadedTo) throws IOException { logger.info("Installing Elasticsearch" + " into " + destination + "..."); try { unzip(downloadedTo, destination); + File executable = getFile(getInstallationDirectory(), "bin", "elasticsearch"); + if (!executable.canExecute()) { + executable.setExecutable(true); + } logger.info("Done"); } catch (IOException e) { logger.info("Failure : " + e); @@ -85,7 +89,7 @@ private void unzip(Path downloadedTo, File destination) throws IOException { private InputStream toStream(Path downloadedTo) throws IOException { InputStream result = new FileInputStream(downloadedTo.toFile()); - if (downloadedTo.endsWith(".gz")) { + if (downloadedTo.toFile().getName().endsWith(".gz")) { result = new GZIPInputStream(result); } return new BufferedInputStream(result); From 883a33c03ad1d55e8e7702b44b1e7c98f4668488 Mon Sep 17 00:00:00 2001 From: Pablo Saavedra Date: Fri, 10 Apr 2020 17:34:43 -0300 Subject: [PATCH 09/13] Another attempt to fix tests --- .../ElasticSearchInstaller.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java index 8197461..40d9fbe 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java @@ -69,10 +69,8 @@ private void installElastic(Path downloadedTo) throws IOException { logger.info("Installing Elasticsearch" + " into " + destination + "..."); try { unzip(downloadedTo, destination); - File executable = getFile(getInstallationDirectory(), "bin", "elasticsearch"); - if (!executable.canExecute()) { - executable.setExecutable(true); - } + makeExecutable("elasticsearch"); + makeExecutable("plugin"); logger.info("Done"); } catch (IOException e) { logger.info("Failure : " + e); @@ -80,6 +78,13 @@ private void installElastic(Path downloadedTo) throws IOException { } } + private void makeExecutable(String name) { + File executable = getFile(getInstallationDirectory(), "bin", name); + if (!executable.canExecute()) { + executable.setExecutable(true); + } + } + private void unzip(Path downloadedTo, File destination) throws IOException { Archiver archiver = ArchiverFactory.createArchiver(downloadedTo.endsWith("zip") ? "zip" : "tar"); try (InputStream is = toStream(downloadedTo)) { From 5fbf7dd26b5630dee0aea78a79800af300c970e9 Mon Sep 17 00:00:00 2001 From: Pablo Saavedra Date: Fri, 10 Apr 2020 17:44:57 -0300 Subject: [PATCH 10/13] Another file that needs to be executable --- .../tech/embeddedelasticsearch/ElasticSearchInstaller.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java index 40d9fbe..1630096 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java @@ -71,6 +71,7 @@ private void installElastic(Path downloadedTo) throws IOException { unzip(downloadedTo, destination); makeExecutable("elasticsearch"); makeExecutable("plugin"); + makeExecutable("elasticsearch-plugin"); logger.info("Done"); } catch (IOException e) { logger.info("Failure : " + e); From b42153afcbfd0b97d29b562f5c3e20adddf54cdf Mon Sep 17 00:00:00 2001 From: Pablo Saavedra Date: Fri, 10 Apr 2020 17:59:18 -0300 Subject: [PATCH 11/13] This is probably too hacky by now, but for some reason zip files are not maintaining exec permissions --- .../embeddedelasticsearch/ElasticSearchInstaller.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java index 1630096..fe6c047 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticSearchInstaller.java @@ -69,9 +69,10 @@ private void installElastic(Path downloadedTo) throws IOException { logger.info("Installing Elasticsearch" + " into " + destination + "..."); try { unzip(downloadedTo, destination); - makeExecutable("elasticsearch"); - makeExecutable("plugin"); - makeExecutable("elasticsearch-plugin"); + makeExecutable("bin", "elasticsearch"); + makeExecutable("bin", "plugin"); + makeExecutable("bin", "elasticsearch-plugin"); + makeExecutable("modules", "x-pack", "x-pack-ml", "platform", "linux-x86_64", "bin", "controller"); logger.info("Done"); } catch (IOException e) { logger.info("Failure : " + e); @@ -79,8 +80,8 @@ private void installElastic(Path downloadedTo) throws IOException { } } - private void makeExecutable(String name) { - File executable = getFile(getInstallationDirectory(), "bin", name); + private void makeExecutable(String... names) { + File executable = getFile(getInstallationDirectory(), names); if (!executable.canExecute()) { executable.setExecutable(true); } From 43b15ffbefcdd86512f262ac6d46bbf522847097 Mon Sep 17 00:00:00 2001 From: Pablo Saavedra Date: Thu, 16 Apr 2020 11:48:13 -0300 Subject: [PATCH 12/13] Fixed code review comments --- .../allegro/tech/embeddedelasticsearch/ElasticRestClient.java | 2 +- .../tech/embeddedelasticsearch/PluginsInstallationSpec.groovy | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java index cbcd1e4..ed898d9 100644 --- a/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java +++ b/core/src/main/java/pl/allegro/tech/embeddedelasticsearch/ElasticRestClient.java @@ -193,7 +193,7 @@ private boolean newESVersion() { return false; } String esV = jsonNode.get("version").get("number").asText(); - return Integer.parseInt(esV.substring(0,1)) >= 7; //if version is 7 and above + return Integer.parseInt(esV.substring(0, esV.indexOf('.'))) >= 7; //if version is 7 and above }); } diff --git a/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy b/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy index 7e73afa..7904a52 100644 --- a/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy +++ b/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy @@ -8,7 +8,7 @@ class PluginsInstallationSpec extends PluginsInstallationBaseSpec { EmbeddedElastic.Builder baseEmbeddedElastic() { return EmbeddedElastic.builder() - .withElasticVersion("6.3.0") + .withElasticVersion("7.3.2") .withEsJavaOpts("-Xms128m -Xmx512m") .withSetting(PopularProperties.HTTP_PORT, HTTP_PORT_VALUE) .withStartTimeout(2, MINUTES) @@ -16,7 +16,7 @@ class PluginsInstallationSpec extends PluginsInstallationBaseSpec { @Override String pluginByUrlUrl() { - return "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-stempel/analysis-stempel-6.3.0.zip" + return "https://artifacts.elastic.co/downloads/elasticsearch-plugins/analysis-stempel/analysis-stempel-7.3.2.zip" } @Override From bf62032c35b48ef89a5d70e9b4cc766c73077cf1 Mon Sep 17 00:00:00 2001 From: Pablo Saavedra Date: Mon, 20 Apr 2020 18:56:41 -0300 Subject: [PATCH 13/13] Replaced deprecated plugin --- .../tech/embeddedelasticsearch/PluginsInstallationSpec.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy b/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy index 7904a52..cffb2a7 100644 --- a/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy +++ b/es73-test/src/test/groovy/pl/allegro/tech/embeddedelasticsearch/PluginsInstallationSpec.groovy @@ -26,6 +26,6 @@ class PluginsInstallationSpec extends PluginsInstallationBaseSpec { @Override String pluginByName() { - return "discovery-file" + return "mapper-size" } } \ No newline at end of file