diff --git a/.github/workflows/create-documentation-issue.yml b/.github/workflows/create-documentation-issue.yml index df63847f8afca..b45e053cc25c2 100644 --- a/.github/workflows/create-documentation-issue.yml +++ b/.github/workflows/create-documentation-issue.yml @@ -29,7 +29,7 @@ jobs: - name: Create Issue From File id: create-issue - uses: peter-evans/create-issue-from-file@v4 + uses: peter-evans/create-issue-from-file@v5 with: title: Add documentation related to new feature content-filepath: ./ci/documentation/issue.md diff --git a/CHANGELOG.md b/CHANGELOG.md index aa31819ffae97..1a3ea103ed118 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800)) - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625)) - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887)) +- [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028)) ### Dependencies - Bump `log4j-core` from 2.18.0 to 2.19.0 @@ -85,6 +86,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Update supported version for max_shard_size parameter in Shrink API ([#11439](https://github.com/opensearch-project/OpenSearch/pull/11439)) - Fix typo in API annotation check message ([11836](https://github.com/opensearch-project/OpenSearch/pull/11836)) - Update supported version for must_exist parameter in update aliases API ([#11872](https://github.com/opensearch-project/OpenSearch/pull/11872)) +- [Bug] Check phase name before SearchRequestOperationsListener onPhaseStart ([#12035](https://github.com/opensearch-project/OpenSearch/pull/12035)) ### Security @@ -152,12 +154,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `stefanzweifel/git-auto-commit-action` from 4 to 5 ([#11171](https://github.com/opensearch-project/OpenSearch/pull/11171)) - Bump `actions/github-script` from 6 to 7 ([#11271](https://github.com/opensearch-project/OpenSearch/pull/11271)) - Bump `jackson` and `jackson_databind` from 2.15.2 to 2.16.0 ([#11273](https://github.com/opensearch-project/OpenSearch/pull/11273)) -- Bump `netty` from 4.1.100.Final to 4.1.104.Final ([#11294](https://github.com/opensearch-project/OpenSearch/pull/11294), [#11775](https://github.com/opensearch-project/OpenSearch/pull/11775)) +- Bump `netty` from 4.1.100.Final to 4.1.106.Final ([#11294](https://github.com/opensearch-project/OpenSearch/pull/11294), [#11775](https://github.com/opensearch-project/OpenSearch/pull/11775)), [#12034](https://github.com/opensearch-project/OpenSearch/pull/12034)) - Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.16.12 to 0.17.6 ([#10163](https://github.com/opensearch-project/OpenSearch/pull/10163), [#11692](https://github.com/opensearch-project/OpenSearch/pull/11692)) - Bump `com.squareup.okhttp3:okhttp` from 4.11.0 to 4.12.0 ([#10861](https://github.com/opensearch-project/OpenSearch/pull/10861)) - Bump `org.apache.commons:commons-text` from 1.10.0 to 1.11.0 ([#11344](https://github.com/opensearch-project/OpenSearch/pull/11344)) -- Bump `reactor-netty-core` from 1.1.12 to 1.1.13 ([#11350](https://github.com/opensearch-project/OpenSearch/pull/11350)) -- Bump `com.gradle.enterprise` from 3.14.1 to 3.16.1 ([#11339](https://github.com/opensearch-project/OpenSearch/pull/11339), [#11629](https://github.com/opensearch-project/OpenSearch/pull/11629)) +- Bump `reactor-netty-core` from 1.1.12 to 1.1.15 ([#11350](https://github.com/opensearch-project/OpenSearch/pull/11350)), ([#12042](https://github.com/opensearch-project/OpenSearch/pull/12042)) +- Bump `com.gradle.enterprise` from 3.14.1 to 3.16.2 ([#11339](https://github.com/opensearch-project/OpenSearch/pull/11339), [#11629](https://github.com/opensearch-project/OpenSearch/pull/11629), [#12056](https://github.com/opensearch-project/OpenSearch/pull/12056)) - Bump `actions/setup-java` from 3 to 4 ([#11447](https://github.com/opensearch-project/OpenSearch/pull/11447)) - Bump `commons-net:commons-net` from 3.9.0 to 3.10.0 ([#11450](https://github.com/opensearch-project/OpenSearch/pull/11450)) - Bump `org.apache.maven:maven-model` from 3.9.4 to 3.9.6 ([#11445](https://github.com/opensearch-project/OpenSearch/pull/11445)) @@ -174,12 +176,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `com.maxmind.db:maxmind-db` from 3.0.0 to 3.1.0 ([#11693](https://github.com/opensearch-project/OpenSearch/pull/11693)) - Bump `net.java.dev.jna:jna` from 5.13.0 to 5.14.0 ([#11798](https://github.com/opensearch-project/OpenSearch/pull/11798)) - Bump `lycheeverse/lychee-action` from 1.8.0 to 1.9.1 ([#11795](https://github.com/opensearch-project/OpenSearch/pull/11795), [#11887](https://github.com/opensearch-project/OpenSearch/pull/11887)) -- Bump `Lucene` from 9.8.0 to 9.9.1 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421)) +- Bump `Lucene` from 9.8.0 to 9.9.2 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421)), ([#12063](https://github.com/opensearch-project/OpenSearch/pull/12063)) - Bump `com.networknt:json-schema-validator` from 1.0.86 to 1.2.0 ([#11886](https://github.com/opensearch-project/OpenSearch/pull/11886), [#11963](https://github.com/opensearch-project/OpenSearch/pull/11963)) - Bump `com.google.api:gax-httpjson` from 0.103.1 to 2.39.0 ([#11794](https://github.com/opensearch-project/OpenSearch/pull/11794)) - Bump `com.google.oauth-client:google-oauth-client` from 1.34.1 to 1.35.0 ([#11960](https://github.com/opensearch-project/OpenSearch/pull/11960)) -- Bump `com.diffplug.spotless` from 6.23.2 to 6.24.0 ([#11962](https://github.com/opensearch-project/OpenSearch/pull/11962)) +- Bump `com.diffplug.spotless` from 6.23.2 to 6.25.0 ([#11962](https://github.com/opensearch-project/OpenSearch/pull/11962), [#12055](https://github.com/opensearch-project/OpenSearch/pull/12055)) - Bump `com.google.cloud:google-cloud-core` from 2.5.10 to 2.30.0 ([#11961](https://github.com/opensearch-project/OpenSearch/pull/11961)) +- Bump `reactor-core` from 3.5.11 to 3.5.14 ([#12042](https://github.com/opensearch-project/OpenSearch/pull/12042)) +- Bump `com.google.http-client:google-http-client-jackson2` from 1.43.3 to 1.44.1 ([#12059](https://github.com/opensearch-project/OpenSearch/pull/12059)) +- Bump `peter-evans/create-issue-from-file` from 4 to 5 ([#12057](https://github.com/opensearch-project/OpenSearch/pull/12057)) ### Changed - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840)) diff --git a/build.gradle b/build.gradle index 4d29ca8664af3..e082967255b10 100644 --- a/build.gradle +++ b/build.gradle @@ -54,7 +54,7 @@ plugins { id 'lifecycle-base' id 'opensearch.docker-support' id 'opensearch.global-build-info' - id "com.diffplug.spotless" version "6.24.0" apply false + id "com.diffplug.spotless" version "6.25.0" apply false id "org.gradle.test-retry" version "1.5.4" apply false id "test-report-aggregation" id 'jacoco-report-aggregation' diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 8f8e445b04437..95ae1ddb578a1 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.9.1 +lucene = 9.9.2 bundled_jdk_vendor = adoptium bundled_jdk = 21.0.2+13 @@ -26,12 +26,12 @@ jakarta_annotation = 1.3.5 # when updating the JNA version, also update the version in buildSrc/build.gradle jna = 5.13.0 -netty = 4.1.104.Final +netty = 4.1.106.Final joda = 2.12.2 # project reactor -reactor_netty = 1.1.13 -reactor = 3.5.11 +reactor_netty = 1.1.15 +reactor = 3.5.14 # client dependencies httpclient5 = 5.2.1 @@ -55,7 +55,7 @@ bouncycastle=1.76 randomizedrunner = 2.7.1 junit = 4.13.2 hamcrest = 2.1 -mockito = 5.5.0 +mockito = 5.10.0 objenesis = 3.2 bytebuddy = 1.14.7 diff --git a/libs/core/licenses/lucene-core-9.9.1.jar.sha1 b/libs/core/licenses/lucene-core-9.9.1.jar.sha1 deleted file mode 100644 index ae596196d9e6a..0000000000000 --- a/libs/core/licenses/lucene-core-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -55249fa9a0ed321adcf8283c6f3b649a6812b0a9 \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.9.2.jar.sha1 b/libs/core/licenses/lucene-core-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..2d03d69369b9f --- /dev/null +++ b/libs/core/licenses/lucene-core-9.9.2.jar.sha1 @@ -0,0 +1 @@ +7699f80220fc80b08413902560904623b88beb9f \ No newline at end of file diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java index 6a92993f5dd42..307da89c18d48 100644 --- a/libs/core/src/main/java/org/opensearch/Version.java +++ b/libs/core/src/main/java/org/opensearch/Version.java @@ -98,8 +98,8 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_11_0 = new Version(2110099, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0); public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0); - public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_1); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_9_1); + public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_9_2); public static final Version CURRENT = V_3_0_0; public static Version fromId(int id) { diff --git a/modules/lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1 deleted file mode 100644 index 402cc36ba3d68..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1782a69d0e83af9cc3c65db0dcd2e7e7c1e5f90e \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.9.2.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..e073455415e24 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.9.2.jar.sha1 @@ -0,0 +1 @@ +045df3828b6460c032a6551040e31ea432b0aad6 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.104.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.104.Final.jar.sha1 deleted file mode 100644 index 30f215e47f8ad..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -756797174b94a3aee11ce83522473f3c18287a43 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..1f170375e9347 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +c5495ba59a627641b3a7c23f6bcb801874c7f7b0 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.104.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.104.Final.jar.sha1 deleted file mode 100644 index 9ed9b896d4b4e..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f51fcfd3baac88b2c0b8dc715932ad5622d17429 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..a75ea81b7ee03 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +025171b63aa1e7a5fd8a7e4e660d6d3110241ea7 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.104.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.104.Final.jar.sha1 deleted file mode 100644 index 478e7cfba1470..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2db1556de1b8dc07695604bf51a0a133263ad63f \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..7f5d1adbff740 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +21a07cdf0fc46b313fe2248f1275cdbdac0ba87b \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 deleted file mode 100644 index f0242709f34f7..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5bb757929f7c4d1bf12740a378a99643caaad1ac \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..c4a0c1fae8e24 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +41515e8c51eeaaddceabdb4f86fbc5dbfc25b70e \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.104.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.104.Final.jar.sha1 deleted file mode 100644 index 1b533eea3b3b3..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec5fc4a7c5475eb20805e14f7274aa28872b5ba1 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..bda8b9376e992 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +132defb4689f41b51b483b7202b22b6e89fe35fd \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.104.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.104.Final.jar.sha1 deleted file mode 100644 index 70777be4dc636..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -18c0e659950cdef5f12c36eccfa14cbd2ad2049d \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..749cc807bcce2 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +874c970c4ff958b1140dde52bc17e6a9e7cde662 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.104.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.104.Final.jar.sha1 deleted file mode 100644 index d7c15af9312fe..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dfa4fe5c3a6eabb7af09902eb63266829876d8a2 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..bd160c07ad0ff --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +e185ae573db04939215f94d6ba869758dcecbde9 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.104.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.104.Final.jar.sha1 deleted file mode 100644 index 5cacaf11a29ce..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da7b263b6fedc5add70e78ee8927c8bd2b9bb589 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..2dab4bff2cc0e --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +8fcca44ae16b98e15965093e7696832019fd6f27 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 deleted file mode 100644 index 64797bf11aedc..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d4da9f7237ac3ac292891e0b2d5364acbce128cf \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..b4b977fdad7de --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +2da179bd95903f0fa73218b8f0d02690c0cfbc94 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1 deleted file mode 100644 index dde9b7c100dc7..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -147cb42a90a29501d9ca6094ea0db1d213f3076a \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.2.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..b318a2d89db7d --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.2.jar.sha1 @@ -0,0 +1 @@ +f7e549fdac07140f4cd379a0f517c38434165e95 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1 deleted file mode 100644 index b70a22e9db096..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b034dd3a975763e083c7e11b5d0f7d516ab72590 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.2.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..87510efa881bc --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.2.jar.sha1 @@ -0,0 +1 @@ +eb19738fd3ca85071fef96978a056a7c94d8e793 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1 deleted file mode 100644 index 323f165c62790..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c405f2f7d0fc127d88dfbadd753469b2028fdf52 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.2.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..fb3746eb27840 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.2.jar.sha1 @@ -0,0 +1 @@ +7e83a83741155ececf9193a4f967e570e170236d \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.1.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.1.jar.sha1 deleted file mode 100644 index dd659ddf4de95..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -970e5775876c2d7e1b9af7421a4b17d96f63faf4 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.2.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..e88299f106bb2 --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.2.jar.sha1 @@ -0,0 +1 @@ +dabdea96c7a6d00363b0093a580e7d783efa69a4 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.1.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.1.jar.sha1 deleted file mode 100644 index ed0e81d8f1f75..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2421e5238e9b8484929291744d709dd743c01da1 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.2.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..05dd3328b032d --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.2.jar.sha1 @@ -0,0 +1 @@ +fb551d9ae6a1bf12bc90a4d26dd8fa3eefac8cb9 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.1.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.1.jar.sha1 deleted file mode 100644 index fd8e000088180..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a23e7de4cd9ae7af285c89dc1c55e0ac3f157fd3 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.2.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..b47a1e4052407 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.2.jar.sha1 @@ -0,0 +1 @@ +0d495b478861b2d29e0a58d273ca0e6e755292e9 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.1.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.1.jar.sha1 deleted file mode 100644 index d0e7a3b0c751c..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8d9bce1ea51db279878c51091dd9aefc7b335da4 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.2.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..b7d54cb230445 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.2.jar.sha1 @@ -0,0 +1 @@ +9958d813d4cfdd890b4611c679ed36775480fa0d \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.104.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.104.Final.jar.sha1 deleted file mode 100644 index 0232fc58f9357..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e237ce67ab230ed1ba749d6651b278333c21b3f \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..005cc2388bd89 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +9d4b3315bb625ec2d73fa569fb6bce4589243d5e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 deleted file mode 100644 index f0242709f34f7..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5bb757929f7c4d1bf12740a378a99643caaad1ac \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..c4a0c1fae8e24 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +41515e8c51eeaaddceabdb4f86fbc5dbfc25b70e \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.104.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.104.Final.jar.sha1 deleted file mode 100644 index 3b6cd3524d978..0000000000000 --- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5b126ceba61275f38297cacd5ea0cd6d3addee04 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..4fc3f5c43edb0 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +129f9bd6aa0cf28bfd7d45a8b6a598eed9c67702 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.104.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.104.Final.jar.sha1 deleted file mode 100644 index 9d01e814971f2..0000000000000 --- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -50a2d899a8f8a68daed1a9b6d7750184310cc45f \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..3dcfa7f26db06 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +bffc88b7c56d7b553453e4244247a1b0ba1fdc8a \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.104.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.104.Final.jar.sha1 deleted file mode 100644 index 987b524aedc98..0000000000000 --- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1210e5856fecb9182d58c0d33fa6e946b344b40 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..0617c6999c586 --- /dev/null +++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +3970474ed55aa1e3e8de5a5602c342c6b8155371 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 deleted file mode 100644 index 64797bf11aedc..0000000000000 --- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d4da9f7237ac3ac292891e0b2d5364acbce128cf \ No newline at end of file diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..b4b977fdad7de --- /dev/null +++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +2da179bd95903f0fa73218b8f0d02690c0cfbc94 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.13.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.13.jar.sha1 deleted file mode 100644 index 5eaf96739ed72..0000000000000 --- a/plugins/repository-azure/licenses/reactor-netty-core-1.1.13.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -faea23e582978a34f6a932b81e86206ec2314990 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-core-1.1.15.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-core-1.1.15.jar.sha1 new file mode 100644 index 0000000000000..c30a99a2338b4 --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-core-1.1.15.jar.sha1 @@ -0,0 +1 @@ +3221d405ad55a573cf29875a8244a4217cf07185 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.13.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.13.jar.sha1 deleted file mode 100644 index 091125169c696..0000000000000 --- a/plugins/repository-azure/licenses/reactor-netty-http-1.1.13.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c5af7bc746050d080891a5446cca2c96a0c51d03 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/reactor-netty-http-1.1.15.jar.sha1 b/plugins/repository-azure/licenses/reactor-netty-http-1.1.15.jar.sha1 new file mode 100644 index 0000000000000..ab3171cd02b73 --- /dev/null +++ b/plugins/repository-azure/licenses/reactor-netty-http-1.1.15.jar.sha1 @@ -0,0 +1 @@ +c79756fa2dfc28ac81fc9d23a14b17c656c3e560 \ No newline at end of file diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle index e0651da8d39c8..2ce67b1f25a80 100644 --- a/plugins/repository-gcs/build.gradle +++ b/plugins/repository-gcs/build.gradle @@ -78,7 +78,7 @@ dependencies { api 'com.google.http-client:google-http-client:1.43.3' api 'com.google.http-client:google-http-client-appengine:1.43.3' api 'com.google.http-client:google-http-client-gson:1.43.3' - api 'com.google.http-client:google-http-client-jackson2:1.43.3' + api 'com.google.http-client:google-http-client-jackson2:1.44.1' api 'com.google.oauth-client:google-oauth-client:1.34.1' diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.3.jar.sha1 deleted file mode 100644 index 8380b9fb770b5..0000000000000 --- a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.43.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -689da86469d19a01c726c8c24477b95c8a834bbe \ No newline at end of file diff --git a/plugins/repository-gcs/licenses/google-http-client-jackson2-1.44.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.44.1.jar.sha1 new file mode 100644 index 0000000000000..4472ffbbebe1c --- /dev/null +++ b/plugins/repository-gcs/licenses/google-http-client-jackson2-1.44.1.jar.sha1 @@ -0,0 +1 @@ +3f1947de0fd9eb250af16abe6103c11e68d11635 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.104.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.104.Final.jar.sha1 deleted file mode 100644 index 9110503f67304..0000000000000 --- a/plugins/repository-hdfs/licenses/netty-all-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d75246285e5fac6f6dad47e387ed4f46f36e521d \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.106.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..f9affd7887093 --- /dev/null +++ b/plugins/repository-hdfs/licenses/netty-all-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +6ec5bd2be65d5529e58e9e482b747c1135b3736b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.104.Final.jar.sha1 deleted file mode 100644 index 30f215e47f8ad..0000000000000 --- a/plugins/repository-s3/licenses/netty-buffer-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -756797174b94a3aee11ce83522473f3c18287a43 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..1f170375e9347 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-buffer-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +c5495ba59a627641b3a7c23f6bcb801874c7f7b0 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.104.Final.jar.sha1 deleted file mode 100644 index 9ed9b896d4b4e..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f51fcfd3baac88b2c0b8dc715932ad5622d17429 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..a75ea81b7ee03 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +025171b63aa1e7a5fd8a7e4e660d6d3110241ea7 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.104.Final.jar.sha1 deleted file mode 100644 index 478e7cfba1470..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2db1556de1b8dc07695604bf51a0a133263ad63f \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..7f5d1adbff740 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +21a07cdf0fc46b313fe2248f1275cdbdac0ba87b \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 deleted file mode 100644 index f0242709f34f7..0000000000000 --- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5bb757929f7c4d1bf12740a378a99643caaad1ac \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..c4a0c1fae8e24 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +41515e8c51eeaaddceabdb4f86fbc5dbfc25b70e \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.104.Final.jar.sha1 deleted file mode 100644 index 1b533eea3b3b3..0000000000000 --- a/plugins/repository-s3/licenses/netty-common-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec5fc4a7c5475eb20805e14f7274aa28872b5ba1 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-common-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..bda8b9376e992 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-common-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +132defb4689f41b51b483b7202b22b6e89fe35fd \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.104.Final.jar.sha1 deleted file mode 100644 index 70777be4dc636..0000000000000 --- a/plugins/repository-s3/licenses/netty-handler-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -18c0e659950cdef5f12c36eccfa14cbd2ad2049d \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..749cc807bcce2 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-handler-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +874c970c4ff958b1140dde52bc17e6a9e7cde662 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.104.Final.jar.sha1 deleted file mode 100644 index d7c15af9312fe..0000000000000 --- a/plugins/repository-s3/licenses/netty-resolver-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dfa4fe5c3a6eabb7af09902eb63266829876d8a2 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..bd160c07ad0ff --- /dev/null +++ b/plugins/repository-s3/licenses/netty-resolver-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +e185ae573db04939215f94d6ba869758dcecbde9 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.104.Final.jar.sha1 deleted file mode 100644 index 5cacaf11a29ce..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da7b263b6fedc5add70e78ee8927c8bd2b9bb589 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..2dab4bff2cc0e --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +8fcca44ae16b98e15965093e7696832019fd6f27 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.104.Final.jar.sha1 deleted file mode 100644 index 522d85a3bf12e..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -700fdbabab44709b0eccffe8f91c4226a5787356 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..58b668b7e80a9 --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +c058d5c712e00e8560e519970b3d27747778b8f2 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 deleted file mode 100644 index 64797bf11aedc..0000000000000 --- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d4da9f7237ac3ac292891e0b2d5364acbce128cf \ No newline at end of file diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..b4b977fdad7de --- /dev/null +++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +2da179bd95903f0fa73218b8f0d02690c0cfbc94 \ No newline at end of file diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java index 4fda0ee95a3ec..e44f408e6dd12 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java @@ -201,6 +201,12 @@ final class S3ClientSettings { key -> Setting.intSetting(key, 500, Property.NodeScope) ); + static final Setting.AffixSetting MAX_SYNC_CONNECTIONS_SETTING = Setting.affixKeySetting( + PREFIX, + "max_sync_connections", + key -> Setting.intSetting(key, 500, Property.NodeScope) + ); + /** Connection acquisition timeout for new connections to S3. */ static final Setting.AffixSetting CONNECTION_ACQUISITION_TIMEOUT = Setting.affixKeySetting( PREFIX, @@ -284,10 +290,13 @@ final class S3ClientSettings { /** The connection TTL for the s3 client */ final int connectionTTLMillis; - /** The max number of connections for the s3 client */ + /** The max number of connections for the s3 async client */ final int maxConnections; - /** The connnection acquisition timeout for the s3 async client */ + /** The max number of connections for the s3 sync client */ + final int maxSyncConnections; + + /** The connnection acquisition timeout for the s3 sync and async client */ final int connectionAcquisitionTimeoutMillis; /** The number of retries to use for the s3 client. */ @@ -318,6 +327,7 @@ private S3ClientSettings( int connectionTimeoutMillis, int connectionTTLMillis, int maxConnections, + int maxSyncConnections, int connectionAcquisitionTimeoutMillis, int maxRetries, boolean throttleRetries, @@ -336,6 +346,7 @@ private S3ClientSettings( this.connectionTimeoutMillis = connectionTimeoutMillis; this.connectionTTLMillis = connectionTTLMillis; this.maxConnections = maxConnections; + this.maxSyncConnections = maxSyncConnections; this.connectionAcquisitionTimeoutMillis = connectionAcquisitionTimeoutMillis; this.maxRetries = maxRetries; this.throttleRetries = throttleRetries; @@ -386,6 +397,9 @@ S3ClientSettings refine(Settings repositorySettings) { ).millis() ); final int newMaxConnections = Math.toIntExact(getRepoSettingOrDefault(MAX_CONNECTIONS_SETTING, normalizedSettings, maxConnections)); + final int newMaxSyncConnections = Math.toIntExact( + getRepoSettingOrDefault(MAX_SYNC_CONNECTIONS_SETTING, normalizedSettings, maxConnections) + ); final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); final boolean newPathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); @@ -433,6 +447,7 @@ S3ClientSettings refine(Settings repositorySettings) { newConnectionTimeoutMillis, newConnectionTTLMillis, newMaxConnections, + newMaxSyncConnections, newConnectionAcquisitionTimeoutMillis, newMaxRetries, newThrottleRetries, @@ -563,6 +578,7 @@ static S3ClientSettings getClientSettings(final Settings settings, final String Math.toIntExact(getConfigValue(settings, clientName, CONNECTION_TIMEOUT_SETTING).millis()), Math.toIntExact(getConfigValue(settings, clientName, CONNECTION_TTL_SETTING).millis()), Math.toIntExact(getConfigValue(settings, clientName, MAX_CONNECTIONS_SETTING)), + Math.toIntExact(getConfigValue(settings, clientName, MAX_SYNC_CONNECTIONS_SETTING)), Math.toIntExact(getConfigValue(settings, clientName, CONNECTION_ACQUISITION_TIMEOUT).millis()), getConfigValue(settings, clientName, MAX_RETRIES_SETTING), getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java index 24387fb98a425..fe81da31432f4 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java @@ -279,6 +279,8 @@ protected PasswordAuthentication getPasswordAuthentication() { } clientBuilder.socketTimeout(Duration.ofMillis(clientSettings.readTimeoutMillis)); + clientBuilder.maxConnections(clientSettings.maxSyncConnections); + clientBuilder.connectionAcquisitionTimeout(Duration.ofMillis(clientSettings.connectionAcquisitionTimeoutMillis)); return clientBuilder; } diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java index f27c8387b6e45..b47749553aeba 100644 --- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java +++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java @@ -74,6 +74,8 @@ public void testThereIsADefaultClientByDefault() { assertThat(defaultSettings.connectionTimeoutMillis, is(10 * 1000)); assertThat(defaultSettings.connectionTTLMillis, is(5 * 1000)); assertThat(defaultSettings.maxConnections, is(500)); + assertThat(defaultSettings.maxSyncConnections, is(500)); + assertThat(defaultSettings.connectionAcquisitionTimeoutMillis, is(15 * 60 * 1000)); assertThat(defaultSettings.maxRetries, is(3)); assertThat(defaultSettings.throttleRetries, is(true)); } diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.104.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.104.Final.jar.sha1 deleted file mode 100644 index 30f215e47f8ad..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -756797174b94a3aee11ce83522473f3c18287a43 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..1f170375e9347 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +c5495ba59a627641b3a7c23f6bcb801874c7f7b0 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.104.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.104.Final.jar.sha1 deleted file mode 100644 index 9ed9b896d4b4e..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f51fcfd3baac88b2c0b8dc715932ad5622d17429 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..a75ea81b7ee03 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +025171b63aa1e7a5fd8a7e4e660d6d3110241ea7 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.104.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.104.Final.jar.sha1 deleted file mode 100644 index 478e7cfba1470..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2db1556de1b8dc07695604bf51a0a133263ad63f \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..7f5d1adbff740 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +21a07cdf0fc46b313fe2248f1275cdbdac0ba87b \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.104.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.104.Final.jar.sha1 deleted file mode 100644 index 1b533eea3b3b3..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec5fc4a7c5475eb20805e14f7274aa28872b5ba1 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..bda8b9376e992 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +132defb4689f41b51b483b7202b22b6e89fe35fd \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.104.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.104.Final.jar.sha1 deleted file mode 100644 index 70777be4dc636..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -18c0e659950cdef5f12c36eccfa14cbd2ad2049d \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..749cc807bcce2 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +874c970c4ff958b1140dde52bc17e6a9e7cde662 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.104.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.104.Final.jar.sha1 deleted file mode 100644 index d7c15af9312fe..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dfa4fe5c3a6eabb7af09902eb63266829876d8a2 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..bd160c07ad0ff --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +e185ae573db04939215f94d6ba869758dcecbde9 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.104.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.104.Final.jar.sha1 deleted file mode 100644 index 5cacaf11a29ce..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da7b263b6fedc5add70e78ee8927c8bd2b9bb589 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.106.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..2dab4bff2cc0e --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +8fcca44ae16b98e15965093e7696832019fd6f27 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.104.Final.jar.sha1 deleted file mode 100644 index 30f215e47f8ad..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -756797174b94a3aee11ce83522473f3c18287a43 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..1f170375e9347 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +c5495ba59a627641b3a7c23f6bcb801874c7f7b0 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.104.Final.jar.sha1 deleted file mode 100644 index 9ed9b896d4b4e..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f51fcfd3baac88b2c0b8dc715932ad5622d17429 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..a75ea81b7ee03 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +025171b63aa1e7a5fd8a7e4e660d6d3110241ea7 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.104.Final.jar.sha1 deleted file mode 100644 index 0232fc58f9357..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8e237ce67ab230ed1ba749d6651b278333c21b3f \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..005cc2388bd89 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +9d4b3315bb625ec2d73fa569fb6bce4589243d5e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.104.Final.jar.sha1 deleted file mode 100644 index 478e7cfba1470..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2db1556de1b8dc07695604bf51a0a133263ad63f \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..7f5d1adbff740 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +21a07cdf0fc46b313fe2248f1275cdbdac0ba87b \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 deleted file mode 100644 index f0242709f34f7..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5bb757929f7c4d1bf12740a378a99643caaad1ac \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..c4a0c1fae8e24 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +41515e8c51eeaaddceabdb4f86fbc5dbfc25b70e \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.104.Final.jar.sha1 deleted file mode 100644 index 1b533eea3b3b3..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec5fc4a7c5475eb20805e14f7274aa28872b5ba1 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..bda8b9376e992 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +132defb4689f41b51b483b7202b22b6e89fe35fd \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.104.Final.jar.sha1 deleted file mode 100644 index 70777be4dc636..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -18c0e659950cdef5f12c36eccfa14cbd2ad2049d \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..749cc807bcce2 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +874c970c4ff958b1140dde52bc17e6a9e7cde662 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.104.Final.jar.sha1 deleted file mode 100644 index d7c15af9312fe..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dfa4fe5c3a6eabb7af09902eb63266829876d8a2 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..bd160c07ad0ff --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +e185ae573db04939215f94d6ba869758dcecbde9 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.104.Final.jar.sha1 deleted file mode 100644 index 987b524aedc98..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f1210e5856fecb9182d58c0d33fa6e946b344b40 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..0617c6999c586 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +3970474ed55aa1e3e8de5a5602c342c6b8155371 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.104.Final.jar.sha1 deleted file mode 100644 index 5cacaf11a29ce..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da7b263b6fedc5add70e78ee8927c8bd2b9bb589 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..2dab4bff2cc0e --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +8fcca44ae16b98e15965093e7696832019fd6f27 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 deleted file mode 100644 index 64797bf11aedc..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.104.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d4da9f7237ac3ac292891e0b2d5364acbce128cf \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 new file mode 100644 index 0000000000000..b4b977fdad7de --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.106.Final.jar.sha1 @@ -0,0 +1 @@ +2da179bd95903f0fa73218b8f0d02690c0cfbc94 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.13.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.13.jar.sha1 deleted file mode 100644 index 5eaf96739ed72..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.13.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -faea23e582978a34f6a932b81e86206ec2314990 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.15.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.15.jar.sha1 new file mode 100644 index 0000000000000..c30a99a2338b4 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-core-1.1.15.jar.sha1 @@ -0,0 +1 @@ +3221d405ad55a573cf29875a8244a4217cf07185 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.13.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.13.jar.sha1 deleted file mode 100644 index 091125169c696..0000000000000 --- a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.13.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c5af7bc746050d080891a5446cca2c96a0c51d03 \ No newline at end of file diff --git a/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.15.jar.sha1 b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.15.jar.sha1 new file mode 100644 index 0000000000000..ab3171cd02b73 --- /dev/null +++ b/plugins/transport-reactor-netty4/licenses/reactor-netty-http-1.1.15.jar.sha1 @@ -0,0 +1 @@ +c79756fa2dfc28ac81fc9d23a14b17c656c3e560 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.9.1.jar.sha1 b/server/licenses/lucene-analysis-common-9.9.1.jar.sha1 deleted file mode 100644 index c9e6120da7497..0000000000000 --- a/server/licenses/lucene-analysis-common-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -24c8401b530308f9568eb7b408c2029c63f564c6 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.9.2.jar.sha1 b/server/licenses/lucene-analysis-common-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..6eefe1b002fde --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.9.2.jar.sha1 @@ -0,0 +1 @@ +2a4712ee9d87e0f9942a81195dbd5223dbeaa541 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.9.1.jar.sha1 b/server/licenses/lucene-backward-codecs-9.9.1.jar.sha1 deleted file mode 100644 index 69ecf6aa68200..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11c46007366bb037be7d271ab0a5849b1d544662 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.9.2.jar.sha1 b/server/licenses/lucene-backward-codecs-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..377acc22a1d6b --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.9.2.jar.sha1 @@ -0,0 +1 @@ +b53707366cf2891bb301f85a44c5108fc2d5d1b5 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.9.1.jar.sha1 b/server/licenses/lucene-core-9.9.1.jar.sha1 deleted file mode 100644 index ae596196d9e6a..0000000000000 --- a/server/licenses/lucene-core-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -55249fa9a0ed321adcf8283c6f3b649a6812b0a9 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.9.2.jar.sha1 b/server/licenses/lucene-core-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..2d03d69369b9f --- /dev/null +++ b/server/licenses/lucene-core-9.9.2.jar.sha1 @@ -0,0 +1 @@ +7699f80220fc80b08413902560904623b88beb9f \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.9.1.jar.sha1 b/server/licenses/lucene-grouping-9.9.1.jar.sha1 deleted file mode 100644 index e7df056400661..0000000000000 --- a/server/licenses/lucene-grouping-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2f2785e17c5c823cc8f41a7ddb4647aaca8ee773 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.9.2.jar.sha1 b/server/licenses/lucene-grouping-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..1e321d751bff4 --- /dev/null +++ b/server/licenses/lucene-grouping-9.9.2.jar.sha1 @@ -0,0 +1 @@ +72b6c47fd80933d259859d38325e3e020c8e017b \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.9.1.jar.sha1 b/server/licenses/lucene-highlighter-9.9.1.jar.sha1 deleted file mode 100644 index 828c7294aa586..0000000000000 --- a/server/licenses/lucene-highlighter-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -30928513461bf79a5cb057e84da7d34a1e53227d \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.9.2.jar.sha1 b/server/licenses/lucene-highlighter-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..d5c2e2a5504f2 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.9.2.jar.sha1 @@ -0,0 +1 @@ +0968d74d6794af65798819304f10ceb947080332 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.9.1.jar.sha1 b/server/licenses/lucene-join-9.9.1.jar.sha1 deleted file mode 100644 index 34b44ca8c6ad5..0000000000000 --- a/server/licenses/lucene-join-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b9c8cc99632280148f92b4c0a64111c482d5d0ac \ No newline at end of file diff --git a/server/licenses/lucene-join-9.9.2.jar.sha1 b/server/licenses/lucene-join-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..60da109fb5c83 --- /dev/null +++ b/server/licenses/lucene-join-9.9.2.jar.sha1 @@ -0,0 +1 @@ +902e4b65686e29c1489a7cdf43ed2ce3b7fe8ace \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.9.1.jar.sha1 b/server/licenses/lucene-memory-9.9.1.jar.sha1 deleted file mode 100644 index b75fba4c331e9..0000000000000 --- a/server/licenses/lucene-memory-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -49f820b1b321860fa42a4f7583e8ed8f77b9c1c2 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.9.2.jar.sha1 b/server/licenses/lucene-memory-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..bebb36d7ed684 --- /dev/null +++ b/server/licenses/lucene-memory-9.9.2.jar.sha1 @@ -0,0 +1 @@ +08175beb4135c23e6918c822381cd51cd2e118a8 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.9.1.jar.sha1 b/server/licenses/lucene-misc-9.9.1.jar.sha1 deleted file mode 100644 index f1e1e056004e9..0000000000000 --- a/server/licenses/lucene-misc-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -db7c30217602dfcda394a4d0f0a9e68140d385a6 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.9.2.jar.sha1 b/server/licenses/lucene-misc-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..15f572ec715bf --- /dev/null +++ b/server/licenses/lucene-misc-9.9.2.jar.sha1 @@ -0,0 +1 @@ +39b015aa603db42b55f0833193bdfc75d38842ba \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.9.1.jar.sha1 b/server/licenses/lucene-queries-9.9.1.jar.sha1 deleted file mode 100644 index 888b9b4a05ec8..0000000000000 --- a/server/licenses/lucene-queries-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d157547bd24edc8e9d9d59c273107dc3ac5fde5e \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.9.2.jar.sha1 b/server/licenses/lucene-queries-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..9a94e40b69e85 --- /dev/null +++ b/server/licenses/lucene-queries-9.9.2.jar.sha1 @@ -0,0 +1 @@ +671252594e15280818f56d4b369bc48c0ab00dac \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.9.1.jar.sha1 b/server/licenses/lucene-queryparser-9.9.1.jar.sha1 deleted file mode 100644 index 1ce8a069a0f4e..0000000000000 --- a/server/licenses/lucene-queryparser-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -12d844fe224f6f97c510ac20d68903ed7f626f6c \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.9.2.jar.sha1 b/server/licenses/lucene-queryparser-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..f23292999ae6a --- /dev/null +++ b/server/licenses/lucene-queryparser-9.9.2.jar.sha1 @@ -0,0 +1 @@ +7cf8dea63ab511ea9a322916198c3abd0402c8b2 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.9.1.jar.sha1 b/server/licenses/lucene-sandbox-9.9.1.jar.sha1 deleted file mode 100644 index 14fd86dadc404..0000000000000 --- a/server/licenses/lucene-sandbox-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -272e588fd3d8c0a401b28a1ac715f27044bf62ec \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.9.2.jar.sha1 b/server/licenses/lucene-sandbox-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..a30f00d72a689 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.9.2.jar.sha1 @@ -0,0 +1 @@ +4e2bdc6a5b65dd15b78477cf2c49cf7b707ba757 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.9.1.jar.sha1 b/server/licenses/lucene-spatial-extras-9.9.1.jar.sha1 deleted file mode 100644 index 0efd5a7595bfe..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e066432e7ab02b2a4914f989bcd8c44adbf340ad \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.9.2.jar.sha1 b/server/licenses/lucene-spatial-extras-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..dafeee401659a --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.9.2.jar.sha1 @@ -0,0 +1 @@ +2693d4d4d3b9a0bf5b632ccbb01132eade1c4e14 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.9.1.jar.sha1 b/server/licenses/lucene-spatial3d-9.9.1.jar.sha1 deleted file mode 100644 index 7f06466e4c721..0000000000000 --- a/server/licenses/lucene-spatial3d-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fa54c9b962778e28ebc0efb9f75297781350361a \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.9.2.jar.sha1 b/server/licenses/lucene-spatial3d-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..47a1c05187098 --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.9.2.jar.sha1 @@ -0,0 +1 @@ +61866a44eafce499ebbc20bcb75c8b8a60a5449b \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.9.1.jar.sha1 b/server/licenses/lucene-suggest-9.9.1.jar.sha1 deleted file mode 100644 index 06732480d1b6c..0000000000000 --- a/server/licenses/lucene-suggest-9.9.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9554de5b22ae7483b344b94a9a956960b7a5d49c \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.9.2.jar.sha1 b/server/licenses/lucene-suggest-9.9.2.jar.sha1 new file mode 100644 index 0000000000000..115983c932983 --- /dev/null +++ b/server/licenses/lucene-suggest-9.9.2.jar.sha1 @@ -0,0 +1 @@ +352ad5296f48a482e2b65bb7d5d6ba2977bb035e \ No newline at end of file diff --git a/server/licenses/reactor-core-3.5.11.jar.sha1 b/server/licenses/reactor-core-3.5.11.jar.sha1 deleted file mode 100644 index e5ffdbc8a7840..0000000000000 --- a/server/licenses/reactor-core-3.5.11.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -db2299757f562261eb775d13658e86ff06f91e8a \ No newline at end of file diff --git a/server/licenses/reactor-core-3.5.14.jar.sha1 b/server/licenses/reactor-core-3.5.14.jar.sha1 new file mode 100644 index 0000000000000..3b58e7a68bade --- /dev/null +++ b/server/licenses/reactor-core-3.5.14.jar.sha1 @@ -0,0 +1 @@ +6e0c97c2e78273a00fd4ed38016b19ff3c6de59e \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java index 5b41c2a13b596..6db44d580f518 100644 --- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java @@ -58,6 +58,7 @@ import org.opensearch.search.internal.SearchContext; import org.opensearch.search.internal.ShardSearchRequest; import org.opensearch.search.pipeline.PipelinedRequest; +import org.opensearch.telemetry.tracing.SpanScope; import org.opensearch.transport.Transport; import java.util.ArrayDeque; @@ -220,6 +221,7 @@ public final void start() { null ) ); + searchRequestContext.getSearchRequestOperationsListener().onRequestEnd(searchRequestContext); return; } executePhase(this); @@ -432,26 +434,30 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha } private void onPhaseEnd(SearchRequestContext searchRequestContext) { - if (getCurrentPhase() != null) { + if (getCurrentPhase() != null && SearchPhaseName.isValidName(getName())) { long tookInNanos = System.nanoTime() - getCurrentPhase().getStartTimeInNanos(); searchRequestContext.updatePhaseTookMap(getCurrentPhase().getName(), TimeUnit.NANOSECONDS.toMillis(tookInNanos)); + this.searchRequestContext.getSearchRequestOperationsListener().onPhaseEnd(this, searchRequestContext); } - this.searchRequestContext.getSearchRequestOperationsListener().onPhaseEnd(this, searchRequestContext); } - private void onPhaseStart(SearchPhase phase) { + void onPhaseStart(SearchPhase phase, SearchRequestContext searchRequestContext) { setCurrentPhase(phase); - this.searchRequestContext.getSearchRequestOperationsListener().onPhaseStart(this); + if (SearchPhaseName.isValidName(phase.getName())) { + this.searchRequestContext.getSearchRequestOperationsListener().onPhaseStart(this, searchRequestContext); + } } private void onRequestEnd(SearchRequestContext searchRequestContext) { - this.searchRequestContext.getSearchRequestOperationsListener().onRequestEnd(this, searchRequestContext); + this.searchRequestContext.getSearchRequestOperationsListener().onRequestEnd(searchRequestContext); } private void executePhase(SearchPhase phase) { try { - onPhaseStart(phase); - phase.recordAndRun(); + onPhaseStart(phase, searchRequestContext); + try (SpanScope spanScope = searchRequestContext.getTracer().withSpanInScope(searchRequestContext.getPhaseSpan())) { + phase.recordAndRun(); + } } catch (Exception e) { if (logger.isDebugEnabled()) { logger.debug(new ParameterizedMessage("Failed to execute [{}] while moving to [{}] phase", request, phase.getName()), e); @@ -703,6 +709,7 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At searchContextId = null; } } + searchRequestContext.setSearchTask(getTask()); searchRequestContext.setTotalHits(internalSearchResponse.hits().getTotalHits()); searchRequestContext.setShardStats(results.getNumShards(), successfulOps.get(), skippedOps.get(), failures.length); onPhaseEnd(searchRequestContext); @@ -714,7 +721,9 @@ public void sendSearchResponse(InternalSearchResponse internalSearchResponse, At @Override public final void onPhaseFailure(SearchPhase phase, String msg, Throwable cause) { - this.searchRequestContext.getSearchRequestOperationsListener().onPhaseFailure(this); + if (SearchPhaseName.isValidName(phase.getName())) { + this.searchRequestContext.getSearchRequestOperationsListener().onPhaseFailure(this, searchRequestContext); + } raisePhaseFailure(new SearchPhaseExecutionException(phase.getName(), msg, cause, buildShardFailures())); } diff --git a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java index 8cf92934c8a52..c6f3d4c70632d 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java +++ b/server/src/main/java/org/opensearch/action/search/SearchPhaseName.java @@ -10,6 +10,9 @@ import org.opensearch.common.annotation.PublicApi; +import java.util.HashSet; +import java.util.Set; + /** * Enum for different Search Phases in OpenSearch * @@ -25,6 +28,12 @@ public enum SearchPhaseName { CAN_MATCH("can_match"); private final String name; + private static final Set PHASE_NAMES = new HashSet<>(); + static { + for (SearchPhaseName phaseName : SearchPhaseName.values()) { + PHASE_NAMES.add(phaseName.name); + } + } SearchPhaseName(final String name) { this.name = name; @@ -33,4 +42,8 @@ public enum SearchPhaseName { public String getName() { return name; } + + public static boolean isValidName(String phaseName) { + return PHASE_NAMES.contains(phaseName); + } } diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java index eceac7204b196..6fe54a1435897 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestContext.java @@ -8,11 +8,17 @@ package org.opensearch.action.search; +import org.apache.logging.log4j.LogManager; import org.apache.lucene.search.TotalHits; import org.opensearch.common.annotation.InternalApi; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.telemetry.tracing.noop.NoopSpan; +import org.opensearch.telemetry.tracing.noop.NoopTracer; import java.util.EnumMap; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -23,20 +29,59 @@ */ @InternalApi class SearchRequestContext { + private final SearchRequest searchRequest; + private SearchTask searchTask; private final SearchRequestOperationsListener searchRequestOperationsListener; private long absoluteStartNanos; private final Map phaseTookMap; private TotalHits totalHits; private final EnumMap shardStats; + private Tracer tracer; + private Span requestSpan; + private Span phaseSpan; - private final SearchRequest searchRequest; + /** + * This constructor is for testing only + */ + SearchRequestContext() { + this(new SearchRequest()); + } + + /** + * This constructor is for testing only + */ + SearchRequestContext(SearchRequest searchRequest) { + this(new SearchRequestOperationsListener.CompositeListener(List.of(), LogManager.getLogger()), searchRequest); + } + + /** + * This constructor is for testing only + */ + SearchRequestContext(SearchRequestOperationsListener searchRequestOperationsListener, SearchRequest searchRequest) { + this(searchRequestOperationsListener, searchRequest, NoopTracer.INSTANCE); + } - SearchRequestContext(final SearchRequestOperationsListener searchRequestOperationsListener, final SearchRequest searchRequest) { + SearchRequestContext(SearchRequestOperationsListener searchRequestOperationsListener, SearchRequest searchRequest, Tracer tracer) { this.searchRequestOperationsListener = searchRequestOperationsListener; + this.searchRequest = searchRequest; + this.tracer = tracer; this.absoluteStartNanos = System.nanoTime(); this.phaseTookMap = new HashMap<>(); this.shardStats = new EnumMap<>(ShardStatsFieldNames.class); - this.searchRequest = searchRequest; + this.requestSpan = NoopSpan.INSTANCE; + this.phaseSpan = NoopSpan.INSTANCE; + } + + public SearchRequest getSearchRequest() { + return searchRequest; + } + + public void setSearchTask(SearchTask searchTask) { + this.searchTask = searchTask; + } + + public SearchTask getSearchTask() { + return searchTask; } SearchRequestOperationsListener getSearchRequestOperationsListener() { @@ -78,7 +123,7 @@ void setTotalHits(TotalHits totalHits) { this.totalHits = totalHits; } - TotalHits totalHits() { + public TotalHits totalHits() { return totalHits; } @@ -89,7 +134,7 @@ void setShardStats(int total, int successful, int skipped, int failed) { this.shardStats.put(ShardStatsFieldNames.SEARCH_REQUEST_SLOWLOG_SHARD_FAILED, failed); } - String formattedShardStats() { + public String formattedShardStats() { if (shardStats.isEmpty()) { return ""; } else { @@ -107,6 +152,26 @@ String formattedShardStats() { ); } } + + public Tracer getTracer() { + return tracer; + } + + public void setRequestSpan(Span requestSpan) { + this.requestSpan = requestSpan; + } + + public Span getRequestSpan() { + return requestSpan; + } + + public void setPhaseSpan(Span phaseSpan) { + this.phaseSpan = phaseSpan; + } + + public Span getPhaseSpan() { + return phaseSpan; + } } enum ShardStatsFieldNames { diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestCoordinatorTrace.java b/server/src/main/java/org/opensearch/action/search/SearchRequestCoordinatorTrace.java new file mode 100644 index 0000000000000..7aa69d365012b --- /dev/null +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestCoordinatorTrace.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.action.search; + +import org.opensearch.telemetry.tracing.AttributeNames; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanBuilder; +import org.opensearch.telemetry.tracing.SpanContext; +import org.opensearch.telemetry.tracing.Tracer; + +import static org.opensearch.core.common.Strings.capitalize; + +/** + * Listener for search request tracing on the coordinator node + * + * @opensearch.internal + */ +public final class SearchRequestCoordinatorTrace extends SearchRequestOperationsListener { + private final Tracer tracer; + + public SearchRequestCoordinatorTrace(Tracer tracer) { + this.tracer = tracer; + } + + @Override + void onPhaseStart(SearchPhaseContext context, SearchRequestContext searchRequestContext) { + searchRequestContext.setPhaseSpan( + tracer.startSpan( + SpanBuilder.from( + "coord" + capitalize(context.getCurrentPhase().getName()), + new SpanContext(searchRequestContext.getRequestSpan()) + ) + ) + ); + } + + @Override + void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) { + searchRequestContext.getPhaseSpan().endSpan(); + } + + @Override + void onPhaseFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext) { + searchRequestContext.getPhaseSpan().endSpan(); + } + + @Override + void onRequestEnd(SearchRequestContext searchRequestContext) { + Span requestSpan = searchRequestContext.getRequestSpan(); + + // add response-related attributes on request end + requestSpan.addAttribute( + AttributeNames.TOTAL_HITS, + searchRequestContext.totalHits() == null ? "0" : searchRequestContext.totalHits().toString() + ); + requestSpan.addAttribute( + AttributeNames.SHARDS, + searchRequestContext.formattedShardStats().isEmpty() ? "null" : searchRequestContext.formattedShardStats() + ); + requestSpan.addAttribute( + AttributeNames.SOURCE, + searchRequestContext.getSearchRequest().source() == null + ? "null" + : searchRequestContext.getSearchRequest().source().toString() + ); + } +} diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestOperationsListener.java b/server/src/main/java/org/opensearch/action/search/SearchRequestOperationsListener.java index 2a09cc084f79f..7eaa78588ac5b 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestOperationsListener.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestOperationsListener.java @@ -31,15 +31,15 @@ protected SearchRequestOperationsListener(final boolean enabled) { this.enabled = enabled; } - abstract void onPhaseStart(SearchPhaseContext context); + abstract void onPhaseStart(SearchPhaseContext context, SearchRequestContext searchRequestContext); abstract void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext); - abstract void onPhaseFailure(SearchPhaseContext context); + abstract void onPhaseFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext); void onRequestStart(SearchRequestContext searchRequestContext) {} - void onRequestEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) {} + void onRequestEnd(SearchRequestContext searchRequestContext) {} boolean isEnabled(SearchRequest searchRequest) { return isEnabled(); @@ -69,10 +69,10 @@ static final class CompositeListener extends SearchRequestOperationsListener { } @Override - void onPhaseStart(SearchPhaseContext context) { + void onPhaseStart(SearchPhaseContext context, SearchRequestContext searchRequestContext) { for (SearchRequestOperationsListener listener : listeners) { try { - listener.onPhaseStart(context); + listener.onPhaseStart(context, searchRequestContext); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("onPhaseStart listener [{}] failed", listener), e); } @@ -91,10 +91,10 @@ void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRequestCo } @Override - void onPhaseFailure(SearchPhaseContext context) { + void onPhaseFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext) { for (SearchRequestOperationsListener listener : listeners) { try { - listener.onPhaseFailure(context); + listener.onPhaseFailure(context, searchRequestContext); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("onPhaseFailure listener [{}] failed", listener), e); } @@ -113,10 +113,10 @@ void onRequestStart(SearchRequestContext searchRequestContext) { } @Override - public void onRequestEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) { + public void onRequestEnd(SearchRequestContext searchRequestContext) { for (SearchRequestOperationsListener listener : listeners) { try { - listener.onRequestEnd(context, searchRequestContext); + listener.onRequestEnd(searchRequestContext); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("onRequestEnd listener [{}] failed", listener), e); } diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestSlowLog.java b/server/src/main/java/org/opensearch/action/search/SearchRequestSlowLog.java index 7f25f9026f215..621bff9595ced 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestSlowLog.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestSlowLog.java @@ -134,29 +134,29 @@ public SearchRequestSlowLog(ClusterService clusterService) { } @Override - void onPhaseStart(SearchPhaseContext context) {} + void onPhaseStart(SearchPhaseContext context, SearchRequestContext searchRequestContext) {} @Override void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) {} @Override - void onPhaseFailure(SearchPhaseContext context) {} + void onPhaseFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext) {} @Override void onRequestStart(SearchRequestContext searchRequestContext) {} @Override - void onRequestEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) { + void onRequestEnd(SearchRequestContext searchRequestContext) { long tookInNanos = System.nanoTime() - searchRequestContext.getAbsoluteStartNanos(); if (warnThreshold >= 0 && tookInNanos > warnThreshold && level.isLevelEnabledFor(SlowLogLevel.WARN)) { - logger.warn(new SearchRequestSlowLogMessage(context, tookInNanos, searchRequestContext)); + logger.warn(new SearchRequestSlowLogMessage(tookInNanos, searchRequestContext)); } else if (infoThreshold >= 0 && tookInNanos > infoThreshold && level.isLevelEnabledFor(SlowLogLevel.INFO)) { - logger.info(new SearchRequestSlowLogMessage(context, tookInNanos, searchRequestContext)); + logger.info(new SearchRequestSlowLogMessage(tookInNanos, searchRequestContext)); } else if (debugThreshold >= 0 && tookInNanos > debugThreshold && level.isLevelEnabledFor(SlowLogLevel.DEBUG)) { - logger.debug(new SearchRequestSlowLogMessage(context, tookInNanos, searchRequestContext)); + logger.debug(new SearchRequestSlowLogMessage(tookInNanos, searchRequestContext)); } else if (traceThreshold >= 0 && tookInNanos > traceThreshold && level.isLevelEnabledFor(SlowLogLevel.TRACE)) { - logger.trace(new SearchRequestSlowLogMessage(context, tookInNanos, searchRequestContext)); + logger.trace(new SearchRequestSlowLogMessage(tookInNanos, searchRequestContext)); } } @@ -167,15 +167,11 @@ void onRequestEnd(SearchPhaseContext context, SearchRequestContext searchRequest */ static final class SearchRequestSlowLogMessage extends OpenSearchLogMessage { - SearchRequestSlowLogMessage(SearchPhaseContext context, long tookInNanos, SearchRequestContext searchRequestContext) { - super(prepareMap(context, tookInNanos, searchRequestContext), message(context, tookInNanos, searchRequestContext)); + SearchRequestSlowLogMessage(long tookInNanos, SearchRequestContext searchRequestContext) { + super(prepareMap(tookInNanos, searchRequestContext), message(tookInNanos, searchRequestContext)); } - private static Map prepareMap( - SearchPhaseContext context, - long tookInNanos, - SearchRequestContext searchRequestContext - ) { + private static Map prepareMap(long tookInNanos, SearchRequestContext searchRequestContext) { final Map messageFields = new HashMap<>(); messageFields.put("took", TimeValue.timeValueNanos(tookInNanos)); messageFields.put("took_millis", TimeUnit.NANOSECONDS.toMillis(tookInNanos)); @@ -185,22 +181,24 @@ private static Map prepareMap( } else { messageFields.put("total_hits", "-1"); } - messageFields.put("search_type", context.getRequest().searchType()); + messageFields.put("search_type", searchRequestContext.getSearchRequest().searchType()); messageFields.put("shards", searchRequestContext.formattedShardStats()); - - if (context.getRequest().source() != null) { - String source = escapeJson(context.getRequest().source().toString(FORMAT_PARAMS)); + if (searchRequestContext.getSearchRequest().source() != null) { + String source = escapeJson(searchRequestContext.getSearchRequest().source().toString(FORMAT_PARAMS)); messageFields.put("source", source); } else { messageFields.put("source", "{}"); } - - messageFields.put("id", context.getTask().getHeader(Task.X_OPAQUE_ID)); + if (searchRequestContext.getSearchTask() != null) { + messageFields.put("id", searchRequestContext.getSearchTask().getHeader(Task.X_OPAQUE_ID)); + } else { + messageFields.put("id", ""); + } return messageFields; } // Message will be used in plaintext logs - private static String message(SearchPhaseContext context, long tookInNanos, SearchRequestContext searchRequestContext) { + private static String message(long tookInNanos, SearchRequestContext searchRequestContext) { final StringBuilder sb = new StringBuilder(); sb.append("took[").append(TimeValue.timeValueNanos(tookInNanos)).append("], "); sb.append("took_millis[").append(TimeUnit.NANOSECONDS.toMillis(tookInNanos)).append("], "); @@ -210,15 +208,15 @@ private static String message(SearchPhaseContext context, long tookInNanos, Sear } else { sb.append("total_hits[-1]"); } - sb.append("search_type[").append(context.getRequest().searchType()).append("], "); + sb.append("search_type[").append(searchRequestContext.getSearchRequest().searchType()).append("], "); sb.append("shards[").append(searchRequestContext.formattedShardStats()).append("], "); - if (context.getRequest().source() != null) { - sb.append("source[").append(context.getRequest().source().toString(FORMAT_PARAMS)).append("], "); + if (searchRequestContext.getSearchRequest().source() != null) { + sb.append("source[").append(searchRequestContext.getSearchRequest().source().toString(FORMAT_PARAMS)).append("], "); } else { sb.append("source[], "); } - if (context.getTask().getHeader(Task.X_OPAQUE_ID) != null) { - sb.append("id[").append(context.getTask().getHeader(Task.X_OPAQUE_ID)).append("]"); + if (searchRequestContext.getSearchTask() != null) { + sb.append("id[").append(searchRequestContext.getSearchTask().getHeader(Task.X_OPAQUE_ID)).append("]"); } else { sb.append("id[]"); } diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java b/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java index 88d599a0dcdaa..739407b773fac 100644 --- a/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java +++ b/server/src/main/java/org/opensearch/action/search/SearchRequestStats.java @@ -58,7 +58,7 @@ public long getPhaseMetric(SearchPhaseName searchPhaseName) { } @Override - void onPhaseStart(SearchPhaseContext context) { + void onPhaseStart(SearchPhaseContext context, SearchRequestContext searchRequestContext) { phaseStatsMap.get(context.getCurrentPhase().getSearchPhaseName()).current.inc(); } @@ -71,7 +71,7 @@ void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRequestCo } @Override - void onPhaseFailure(SearchPhaseContext context) { + void onPhaseFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext) { phaseStatsMap.get(context.getCurrentPhase().getSearchPhaseName()).current.dec(); } diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java index 842c10b700d24..2512988f56f01 100644 --- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java @@ -88,6 +88,11 @@ import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.Task; import org.opensearch.telemetry.metrics.MetricsRegistry; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanBuilder; +import org.opensearch.telemetry.tracing.SpanScope; +import org.opensearch.telemetry.tracing.Tracer; +import org.opensearch.telemetry.tracing.listener.TraceableActionListener; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.RemoteClusterAware; import org.opensearch.transport.RemoteClusterService; @@ -173,6 +178,7 @@ public class TransportSearchAction extends HandledTransportAction) SearchRequest::new); this.client = client; @@ -215,6 +222,7 @@ public TransportSearchAction( this.searchRequestOperationsCompositeListenerFactory = searchRequestOperationsCompositeListenerFactory; clusterService.getClusterSettings() .addSettingsUpdateConsumer(SEARCH_QUERY_METRICS_ENABLED_SETTING, this::setSearchQueryMetricsEnabled); + this.tracer = tracer; } private void setSearchQueryMetricsEnabled(boolean searchQueryMetricsEnabled) { @@ -433,11 +441,16 @@ private void executeRequest( } SearchRequestOperationsListener.CompositeListener requestOperationsListeners = searchRequestOperationsCompositeListenerFactory .buildCompositeListener(originalSearchRequest, logger); - SearchRequestContext searchRequestContext = new SearchRequestContext(requestOperationsListeners, originalSearchRequest); + SearchRequestContext searchRequestContext = new SearchRequestContext(requestOperationsListeners, originalSearchRequest, tracer); searchRequestContext.getSearchRequestOperationsListener().onRequestStart(searchRequestContext); PipelinedRequest searchRequest; ActionListener listener; + + Span requestSpan = tracer.startSpan(SpanBuilder.from("coordReq")); + originalListener = TraceableActionListener.create(originalListener, requestSpan, tracer); + searchRequestContext.setRequestSpan(requestSpan); + try { searchRequest = searchPipelineService.resolvePipeline(originalSearchRequest); listener = searchRequest.transformResponseListener(originalListener); @@ -473,7 +486,9 @@ private void executeRequest( ); } }, listener::onFailure); - searchRequest.transformRequest(requestTransformListener); + try (SpanScope spanScope = tracer.withSpanInScope(requestSpan)) { + searchRequest.transformRequest(requestTransformListener); + } } private ActionListener buildRewriteListener( @@ -1238,7 +1253,7 @@ private AbstractSearchAsyncAction searchAsyncAction clusters, searchRequestContext ); - return new SearchPhase(action.getName()) { + return new SearchPhase("none") { @Override public void run() { action.start(); diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java index 9c1e902606cab..dab99fd25b192 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -164,7 +164,7 @@ public RemoteSegmentMetadata init() throws IOException { */ public RemoteSegmentMetadata initializeToSpecificCommit(long primaryTerm, long commitGeneration, String acquirerId) throws IOException { String metadataFilePrefix = MetadataFilenameUtils.getMetadataFilePrefixForCommit(primaryTerm, commitGeneration); - String metadataFile = ((RemoteStoreMetadataLockManager) mdLockManager).fetchLock(metadataFilePrefix, acquirerId); + String metadataFile = ((RemoteStoreMetadataLockManager) mdLockManager).fetchLockedMetadataFile(metadataFilePrefix, acquirerId); RemoteSegmentMetadata remoteSegmentMetadata = readMetadataFile(metadataFile); if (remoteSegmentMetadata != null) { this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(remoteSegmentMetadata.getMetadata()); @@ -749,20 +749,16 @@ public void deleteStaleSegments(int lastNMetadataFilesToKeep) throws IOException lastNMetadataFilesToKeep, sortedMetadataFileList.size() ); - List metadataFilesToBeDeleted = metadataFilesEligibleToDelete.stream().filter(metadataFile -> { - try { - return !isLockAcquired(metadataFile); - } catch (IOException e) { - logger.error( - "skipping metadata file (" - + metadataFile - + ") deletion for this run," - + " as checking lock for metadata is failing with error: " - + e - ); - return false; - } - }).collect(Collectors.toList()); + Set allLockFiles; + try { + allLockFiles = ((RemoteStoreMetadataLockManager) mdLockManager).fetchLockedMetadataFiles(MetadataFilenameUtils.METADATA_PREFIX); + } catch (Exception e) { + logger.error("Exception while fetching segment metadata lock files, skipping deleteStaleSegments", e); + return; + } + List metadataFilesToBeDeleted = metadataFilesEligibleToDelete.stream() + .filter(metadataFile -> allLockFiles.contains(metadataFile) == false) + .collect(Collectors.toList()); sortedMetadataFileList.removeAll(metadataFilesToBeDeleted); logger.debug( diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java index 5ebd00f59ef49..9c29e03c225e4 100644 --- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java +++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManager.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; /** @@ -75,7 +76,7 @@ public void release(LockInfo lockInfo) throws IOException { } } - public String fetchLock(String filenamePrefix, String acquirerId) throws IOException { + public String fetchLockedMetadataFile(String filenamePrefix, String acquirerId) throws IOException { Collection lockFiles = lockDirectory.listFilesByPrefix(filenamePrefix); List lockFilesForAcquirer = lockFiles.stream() .filter(lockFile -> acquirerId.equals(FileLockInfo.LockFileUtils.getAcquirerIdFromLock(lockFile))) @@ -88,6 +89,11 @@ public String fetchLock(String filenamePrefix, String acquirerId) throws IOExcep return lockFilesForAcquirer.get(0); } + public Set fetchLockedMetadataFiles(String filenamePrefix) throws IOException { + Collection lockFiles = lockDirectory.listFilesByPrefix(filenamePrefix); + return lockFiles.stream().map(FileLockInfo.LockFileUtils::getFileToLockNameFromLock).collect(Collectors.toSet()); + } + /** * Checks whether a given file have any lock on it or not. * @param lockInfo File Lock Info instance for which we need to check if lock is acquired. diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 8510122c39fcb..62d5813a1d1a1 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -48,6 +48,7 @@ import org.opensearch.action.search.SearchPhaseController; import org.opensearch.action.search.SearchRequestOperationsCompositeListenerFactory; import org.opensearch.action.search.SearchRequestOperationsListener; +import org.opensearch.action.search.SearchRequestCoordinatorTrace; import org.opensearch.action.search.SearchRequestSlowLog; import org.opensearch.action.search.SearchRequestStats; import org.opensearch.action.search.SearchTransportService; @@ -787,6 +788,7 @@ protected Node( final SearchRequestStats searchRequestStats = new SearchRequestStats(clusterService.getClusterSettings()); final SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService); + final SearchRequestCoordinatorTrace searchRequestCoordinatorTrace = new SearchRequestCoordinatorTrace(tracer); remoteStoreStatsTrackerFactory = new RemoteStoreStatsTrackerFactory(clusterService, settings); final IndicesService indicesService = new IndicesService( @@ -1168,7 +1170,8 @@ protected Node( searchModule.getFetchPhase(), responseCollectorService, circuitBreakerService, - searchModule.getIndexSearcherExecutor(threadPool) + searchModule.getIndexSearcherExecutor(threadPool), + tracer ); final List> tasksExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) @@ -1284,6 +1287,7 @@ protected Node( b.bind(Tracer.class).toInstance(tracer); b.bind(SearchRequestStats.class).toInstance(searchRequestStats); b.bind(SearchRequestSlowLog.class).toInstance(searchRequestSlowLog); + b.bind(SearchRequestCoordinatorTrace.class).toInstance(searchRequestCoordinatorTrace); b.bind(MetricsRegistry.class).toInstance(metricsRegistry); b.bind(RemoteClusterStateService.class).toProvider(() -> remoteClusterStateService); b.bind(PersistedStateRegistry.class).toInstance(persistedStateRegistry); @@ -1802,7 +1806,8 @@ protected SearchService newSearchService( FetchPhase fetchPhase, ResponseCollectorService responseCollectorService, CircuitBreakerService circuitBreakerService, - Executor indexSearcherExecutor + Executor indexSearcherExecutor, + Tracer tracer ) { return new SearchService( clusterService, @@ -1814,7 +1819,8 @@ protected SearchService newSearchService( fetchPhase, responseCollectorService, circuitBreakerService, - indexSearcherExecutor + indexSearcherExecutor, + tracer ); } diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java index 2c85fcbb25f35..712c432f933fe 100644 --- a/server/src/main/java/org/opensearch/search/SearchService.java +++ b/server/src/main/java/org/opensearch/search/SearchService.java @@ -84,6 +84,7 @@ import org.opensearch.index.query.QueryRewriteContext; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.Rewriteable; +import org.opensearch.index.search.stats.ShardSearchStats; import org.opensearch.index.shard.IndexEventListener; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.SearchOperationListener; @@ -135,6 +136,10 @@ import org.opensearch.search.sort.SortOrder; import org.opensearch.search.suggest.Suggest; import org.opensearch.search.suggest.completion.CompletionSuggestion; +import org.opensearch.telemetry.tracing.Span; +import org.opensearch.telemetry.tracing.SpanBuilder; +import org.opensearch.telemetry.tracing.SpanScope; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.Scheduler.Cancellable; import org.opensearch.threadpool.ThreadPool; import org.opensearch.threadpool.ThreadPool.Names; @@ -318,6 +323,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv private final AtomicInteger openPitContexts = new AtomicInteger(); private final String sessionId = UUIDs.randomBase64UUID(); private final Executor indexSearcherExecutor; + private final Tracer tracer; public SearchService( ClusterService clusterService, @@ -329,7 +335,8 @@ public SearchService( FetchPhase fetchPhase, ResponseCollectorService responseCollectorService, CircuitBreakerService circuitBreakerService, - Executor indexSearcherExecutor + Executor indexSearcherExecutor, + Tracer tracer ) { Settings settings = clusterService.getSettings(); this.threadPool = threadPool; @@ -346,6 +353,7 @@ public SearchService( circuitBreakerService.getBreaker(CircuitBreaker.REQUEST) ); this.indexSearcherExecutor = indexSearcherExecutor; + this.tracer = tracer; TimeValue keepAliveInterval = KEEPALIVE_INTERVAL_SETTING.get(settings); setKeepAlives(DEFAULT_KEEPALIVE_SETTING.get(settings), MAX_KEEPALIVE_SETTING.get(settings)); setPitKeepAlives(DEFAULT_KEEPALIVE_SETTING.get(settings), MAX_PIT_KEEPALIVE_SETTING.get(settings)); @@ -606,7 +614,10 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh SearchContext context = createContext(readerContext, request, task, true) ) { final long afterQueryTime; - try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context)) { + try ( + SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, tracer); + SpanScope spanScope = tracer.withSpanInScope(executor.getQuerySpan()) + ) { loadOrExecuteQueryPhase(request, context); if (context.queryResult().hasSearchContext() == false && readerContext.singleSession()) { freeReaderContext(readerContext.id()); @@ -637,7 +648,10 @@ private SearchPhaseResult executeQueryPhase(ShardSearchRequest request, SearchSh } private QueryFetchSearchResult executeFetchPhase(ReaderContext reader, SearchContext context, long afterQueryTime) { - try (SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, true, afterQueryTime)) { + try ( + SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(context, tracer, true, afterQueryTime); + SpanScope spanScope = tracer.withSpanInScope(executor.getFetchSpan()) + ) { shortcutDocIdsToLoad(context); fetchPhase.execute(context); if (reader.singleSession()) { @@ -666,7 +680,8 @@ public void executeQueryPhase( final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null); try ( SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false); - SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext) + SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext, tracer); + SpanScope spanScope = tracer.withSpanInScope(executor.getQuerySpan()) ) { searchContext.searcher().setAggregatedDfs(readerContext.getAggregatedDfs(null)); processScroll(request, readerContext, searchContext); @@ -690,7 +705,8 @@ public void executeQueryPhase(QuerySearchRequest request, SearchShardTask task, readerContext.setAggregatedDfs(request.dfs()); try ( SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, true); - SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext) + SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext, tracer); + SpanScope spanScope = tracer.withSpanInScope(executor.getQuerySpan()) ) { searchContext.searcher().setAggregatedDfs(request.dfs()); queryPhase.execute(searchContext); @@ -745,7 +761,8 @@ public void executeFetchPhase( final ShardSearchRequest shardSearchRequest = readerContext.getShardSearchRequest(null); try ( SearchContext searchContext = createContext(readerContext, shardSearchRequest, task, false); - SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext) + SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext, tracer); + SpanScope spanScope = tracer.withSpanInScope(executor.getQuerySpan()) ) { searchContext.assignRescoreDocIds(readerContext.getRescoreDocIds(null)); searchContext.searcher().setAggregatedDfs(readerContext.getAggregatedDfs(null)); @@ -776,7 +793,13 @@ public void executeFetchPhase(ShardFetchRequest request, SearchShardTask task, A searchContext.searcher().setAggregatedDfs(readerContext.getAggregatedDfs(request.getAggregatedDfs())); searchContext.docIdsToLoad(request.docIds(), 0, request.docIdsSize()); try ( - SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor(searchContext, true, System.nanoTime()) + SearchOperationListenerExecutor executor = new SearchOperationListenerExecutor( + searchContext, + tracer, + true, + System.nanoTime() + ); + SpanScope spanScope = tracer.withSpanInScope(executor.getFetchSpan()) ) { fetchPhase.execute(searchContext); if (readerContext.singleSession()) { @@ -1168,10 +1191,10 @@ private void checkKeepAliveLimit(long keepAlive) { if (keepAlive > maxKeepAlive) { throw new IllegalArgumentException( "Keep alive for request (" - + TimeValue.timeValueMillis(keepAlive) + + timeValueMillis(keepAlive) + ") is too large. " + "It must be less than (" - + TimeValue.timeValueMillis(maxKeepAlive) + + timeValueMillis(maxKeepAlive) + "). " + "This limit can be set by changing the [" + MAX_KEEPALIVE_SETTING.getKey() @@ -1187,10 +1210,10 @@ private void checkPitKeepAliveLimit(long keepAlive) { if (keepAlive > maxPitKeepAlive) { throw new IllegalArgumentException( "Keep alive for request (" - + TimeValue.timeValueMillis(keepAlive) + + timeValueMillis(keepAlive) + ") is too large. " + "It must be less than (" - + TimeValue.timeValueMillis(maxPitKeepAlive) + + timeValueMillis(maxPitKeepAlive) + "). " + "This limit can be set by changing the [" + MAX_PIT_KEEPALIVE_SETTING.getKey() @@ -1352,7 +1375,7 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc + "] index level setting." ); } - for (org.opensearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) { + for (SearchSourceBuilder.ScriptField field : source.scriptFields()) { FieldScript.Factory factory = scriptService.compile(field.script(), FieldScript.CONTEXT); SearchLookup lookup = context.getQueryShardContext().lookup(); FieldScript.LeafFactory searchScript = factory.newFactory(field.script().getParams(), lookup); @@ -1657,29 +1680,20 @@ public IndicesService getIndicesService() { } /** - * Returns a builder for {@link InternalAggregation.ReduceContext}. This + * Returns a builder for {@link ReduceContext}. This * builder retains a reference to the provided {@link SearchSourceBuilder}. */ public InternalAggregation.ReduceContextBuilder aggReduceContextBuilder(SearchSourceBuilder searchSourceBuilder) { return new InternalAggregation.ReduceContextBuilder() { @Override - public InternalAggregation.ReduceContext forPartialReduction() { - return InternalAggregation.ReduceContext.forPartialReduction( - bigArrays, - scriptService, - () -> requestToPipelineTree(searchSourceBuilder) - ); + public ReduceContext forPartialReduction() { + return ReduceContext.forPartialReduction(bigArrays, scriptService, () -> requestToPipelineTree(searchSourceBuilder)); } @Override public ReduceContext forFinalReduction() { PipelineTree pipelineTree = requestToPipelineTree(searchSourceBuilder); - return InternalAggregation.ReduceContext.forFinalReduction( - bigArrays, - scriptService, - multiBucketConsumerService.create(), - pipelineTree - ); + return ReduceContext.forFinalReduction(bigArrays, scriptService, multiBucketConsumerService.create(), pipelineTree); } }; } @@ -1728,7 +1742,7 @@ public MinAndMax estimatedMinAndMax() { /** * This helper class ensures we only execute either the success or the failure path for {@link SearchOperationListener}. - * This is crucial for some implementations like {@link org.opensearch.index.search.stats.ShardSearchStats}. + * This is crucial for some implementations like {@link ShardSearchStats}. */ private static final class SearchOperationListenerExecutor implements AutoCloseable { private final SearchOperationListener listener; @@ -1737,20 +1751,24 @@ private static final class SearchOperationListenerExecutor implements AutoClosea private final boolean fetch; private long afterQueryTime = -1; private boolean closed = false; + private Span querySpan = null; + private Span fetchSpan = null; - SearchOperationListenerExecutor(SearchContext context) { - this(context, false, System.nanoTime()); + SearchOperationListenerExecutor(SearchContext context, Tracer tracer) { + this(context, tracer, false, System.nanoTime()); } - SearchOperationListenerExecutor(SearchContext context, boolean fetch, long startTime) { + SearchOperationListenerExecutor(SearchContext context, Tracer tracer, boolean fetch, long startTime) { this.listener = context.indexShard().getSearchOperationListener(); this.context = context; time = startTime; this.fetch = fetch; if (fetch) { listener.onPreFetchPhase(context); + fetchSpan = tracer.startSpan(SpanBuilder.from("shardFetch", context)); } else { listener.onPreQueryPhase(context); + querySpan = tracer.startSpan(SpanBuilder.from("shardQuery", context)); } } @@ -1766,17 +1784,29 @@ public void close() { if (afterQueryTime != -1) { if (fetch) { listener.onFetchPhase(context, afterQueryTime - time); + fetchSpan.endSpan(); } else { listener.onQueryPhase(context, afterQueryTime - time); + querySpan.endSpan(); } } else { if (fetch) { listener.onFailedFetchPhase(context); + fetchSpan.endSpan(); } else { listener.onFailedQueryPhase(context); + querySpan.endSpan(); } } } } + + Span getQuerySpan() { + return querySpan; + } + + Span getFetchSpan() { + return fetchSpan; + } } } diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java b/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java index b6b2cf360d1c5..024b74ab6d2d0 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/AttributeNames.java @@ -94,4 +94,19 @@ private AttributeNames() { * Refresh Policy */ public static final String REFRESH_POLICY = "refresh_policy"; + + /** + * Search Request Source + */ + public static final String SOURCE = "source"; + + /** + * Search Response Shard Stats + */ + public static final String SHARDS = "shards"; + + /** + * Search Response Total Hits + */ + public static final String TOTAL_HITS = "total_hits"; } diff --git a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java index 1dce422943b7a..5cca1c4f21c46 100644 --- a/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java +++ b/server/src/main/java/org/opensearch/telemetry/tracing/SpanBuilder.java @@ -14,6 +14,7 @@ import org.opensearch.core.common.Strings; import org.opensearch.http.HttpRequest; import org.opensearch.rest.RestRequest; +import org.opensearch.search.internal.SearchContext; import org.opensearch.telemetry.tracing.attributes.Attributes; import org.opensearch.transport.TcpChannel; import org.opensearch.transport.Transport; @@ -42,6 +43,15 @@ private SpanBuilder() { } + /** + * Creates {@link SpanCreationContext} from String + * @param spanName String. + * @return context. + */ + public static SpanCreationContext from(String spanName) { + return SpanCreationContext.server().name(spanName); + } + /** * Creates {@link SpanCreationContext} from the {@link HttpRequest} * @param request Http request. @@ -170,4 +180,31 @@ private static Attributes buildSpanAttributes(String nodeId, ReplicatedWriteRequ return attributes; } + /** + * Creates {@link SpanCreationContext} with parent set to specified SpanContext. + * @param spanName name of span. + * @param parentSpan target parent span. + * @return context + */ + public static SpanCreationContext from(String spanName, SpanContext parentSpan) { + return SpanCreationContext.server().name(spanName).parent(parentSpan); + } + + /** + * Creates {@link SpanCreationContext} from SearchRequest. + * @param spanName name of span. + * @param searchContext SearchRequest object. + * @return context + */ + public static SpanCreationContext from(String spanName, SearchContext searchContext) { + return SpanCreationContext.server().name(spanName).attributes(buildSpanAttributes(searchContext)); + } + + private static Attributes buildSpanAttributes(SearchContext searchContext) { + Attributes attributes = Attributes.create() + .addAttribute(AttributeNames.SHARD_ID, searchContext.request().shardId().getId()) + .addAttribute(AttributeNames.INDEX, searchContext.request().shardId().getIndexName()); + return attributes; + } + } diff --git a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java index 76129341fc9a2..799306ce1a735 100644 --- a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java +++ b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java @@ -338,7 +338,14 @@ public void testOnPhaseFailureAndVerifyListeners() { SearchQueryThenFetchAsyncAction action = createSearchQueryThenFetchAsyncAction(requestOperationListeners); action.start(); assertEquals(1, testListener.getPhaseCurrent(action.getSearchPhaseName())); - action.onPhaseFailure(new SearchPhase("test") { + action.onPhaseFailure(new SearchPhase("none") { + @Override + public void run() { + + } + }, "message", null); + assertEquals(1, testListener.getPhaseCurrent(action.getSearchPhaseName())); + action.onPhaseFailure(new SearchPhase(action.getName()) { @Override public void run() { @@ -352,14 +359,14 @@ public void run() { ); searchDfsQueryThenFetchAsyncAction.start(); assertEquals(1, testListener.getPhaseCurrent(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); - searchDfsQueryThenFetchAsyncAction.onPhaseFailure(new SearchPhase("test") { + searchDfsQueryThenFetchAsyncAction.onPhaseFailure(new SearchPhase(searchDfsQueryThenFetchAsyncAction.getName()) { @Override public void run() { } }, "message", null); - assertEquals(0, testListener.getPhaseCurrent(action.getSearchPhaseName())); - assertEquals(0, testListener.getPhaseTotal(action.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseCurrent(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); + assertEquals(0, testListener.getPhaseTotal(searchDfsQueryThenFetchAsyncAction.getSearchPhaseName())); FetchSearchPhase fetchPhase = createFetchSearchPhase(); ShardId shardId = new ShardId(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLength(10), randomInt()); @@ -368,7 +375,7 @@ public void run() { action.skipShard(searchShardIterator); action.executeNextPhase(action, fetchPhase); assertEquals(1, testListener.getPhaseCurrent(fetchPhase.getSearchPhaseName())); - action.onPhaseFailure(new SearchPhase("test") { + action.onPhaseFailure(new SearchPhase(fetchPhase.getName()) { @Override public void run() { @@ -403,6 +410,30 @@ public void run() { assertEquals(requestIds, releasedContexts); } + public void testOnPhaseStart() { + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + SearchRequestStats testListener = new SearchRequestStats(clusterSettings); + + final List requestOperationListeners = new ArrayList<>(List.of(testListener)); + SearchQueryThenFetchAsyncAction action = createSearchQueryThenFetchAsyncAction(requestOperationListeners); + + action.onPhaseStart(new SearchPhase("test") { + @Override + public void run() {} + }, new SearchRequestContext()); + action.onPhaseStart(new SearchPhase("none") { + @Override + public void run() {} + }, new SearchRequestContext()); + assertEquals(0, testListener.getPhaseCurrent(action.getSearchPhaseName())); + + action.onPhaseStart(new SearchPhase(action.getName()) { + @Override + public void run() {} + }, new SearchRequestContext()); + assertEquals(1, testListener.getPhaseCurrent(action.getSearchPhaseName())); + } + public void testShardNotAvailableWithDisallowPartialFailures() { SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false); AtomicReference exception = new AtomicReference<>(); diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsCompositeListenerFactoryTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsCompositeListenerFactoryTests.java index 78c5ba4412c68..d36fa74fdaabb 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsCompositeListenerFactoryTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsCompositeListenerFactoryTests.java @@ -119,13 +119,13 @@ public void testStandardListenerAndPerRequestListenerDisabled() { public SearchRequestOperationsListener createTestSearchRequestOperationsListener() { return new SearchRequestOperationsListener() { @Override - void onPhaseStart(SearchPhaseContext context) {} + void onPhaseStart(SearchPhaseContext context, SearchRequestContext searchRequestContext) {} @Override void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRequestContext) {} @Override - void onPhaseFailure(SearchPhaseContext context) {} + void onPhaseFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext) {} }; } } diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerSupport.java b/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerSupport.java index 0f737e00478cb..6e4cfb70d204b 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerSupport.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerSupport.java @@ -17,7 +17,7 @@ */ public interface SearchRequestOperationsListenerSupport { default void onPhaseStart(SearchRequestOperationsListener listener, SearchPhaseContext context) { - listener.onPhaseStart(context); + listener.onPhaseStart(context, new SearchRequestContext()); } default void onPhaseEnd(SearchRequestOperationsListener listener, SearchPhaseContext context) { diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerTests.java index a53c35a8401b3..b825fa43039bf 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestOperationsListenerTests.java @@ -29,7 +29,7 @@ public void testListenersAreExecuted() { SearchRequestOperationsListener testListener = new SearchRequestOperationsListener() { @Override - public void onPhaseStart(SearchPhaseContext context) { + public void onPhaseStart(SearchPhaseContext context, SearchRequestContext searchRequestContext) { searchPhaseMap.get(context.getCurrentPhase().getSearchPhaseName()).current.inc(); } @@ -40,7 +40,7 @@ public void onPhaseEnd(SearchPhaseContext context, SearchRequestContext searchRe } @Override - public void onPhaseFailure(SearchPhaseContext context) { + public void onPhaseFailure(SearchPhaseContext context, SearchRequestContext searchRequestContext) { searchPhaseMap.get(context.getCurrentPhase().getSearchPhaseName()).current.dec(); } }; @@ -62,7 +62,7 @@ public void onPhaseFailure(SearchPhaseContext context) { for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { when(ctx.getCurrentPhase()).thenReturn(searchPhase); when(searchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); - compositeListener.onPhaseStart(ctx); + compositeListener.onPhaseStart(ctx, new SearchRequestContext()); assertEquals(totalListeners, searchPhaseMap.get(searchPhaseName).current.count()); } } diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestSlowLogTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestSlowLogTests.java index f009988ffae17..0d6551b200a9f 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestSlowLogTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestSlowLogTests.java @@ -114,7 +114,6 @@ public void testMultipleSlowLoggersUseSingleLog4jLogger() { public void testOnRequestEnd() throws InterruptedException { final Logger logger = mock(Logger.class); final SearchRequestContext searchRequestContext = mock(SearchRequestContext.class); - final SearchPhaseContext searchPhaseContext = mock(SearchPhaseContext.class); final SearchRequest searchRequest = mock(SearchRequest.class); final SearchTask searchTask = mock(SearchTask.class); @@ -132,11 +131,11 @@ public void testOnRequestEnd() throws InterruptedException { new SearchRequestOperationsListener.CompositeListener(searchListenersList, logger) ); when(searchRequestContext.getAbsoluteStartNanos()).thenReturn(System.nanoTime() - 1L); - when(searchPhaseContext.getRequest()).thenReturn(searchRequest); - when(searchPhaseContext.getTask()).thenReturn(searchTask); + when(searchRequestContext.getSearchRequest()).thenReturn(searchRequest); + when(searchRequestContext.getSearchTask()).thenReturn(searchTask); when(searchRequest.searchType()).thenReturn(SearchType.QUERY_THEN_FETCH); - searchRequestContext.getSearchRequestOperationsListener().onRequestEnd(searchPhaseContext, searchRequestContext); + searchRequestContext.getSearchRequestOperationsListener().onRequestEnd(searchRequestContext); verify(logger, never()).warn(any(SearchRequestSlowLog.SearchRequestSlowLogMessage.class)); verify(logger, times(1)).info(any(SearchRequestSlowLog.SearchRequestSlowLogMessage.class)); @@ -146,7 +145,6 @@ public void testOnRequestEnd() throws InterruptedException { public void testConcurrentOnRequestEnd() throws InterruptedException { final Logger logger = mock(Logger.class); - final SearchPhaseContext searchPhaseContext = mock(SearchPhaseContext.class); final SearchRequest searchRequest = mock(SearchRequest.class); final SearchTask searchTask = mock(SearchTask.class); @@ -161,8 +159,6 @@ public void testConcurrentOnRequestEnd() throws InterruptedException { SearchRequestSlowLog searchRequestSlowLog = new SearchRequestSlowLog(clusterService, logger); final List searchListenersList = new ArrayList<>(List.of(searchRequestSlowLog)); - when(searchPhaseContext.getRequest()).thenReturn(searchRequest); - when(searchPhaseContext.getTask()).thenReturn(searchTask); when(searchRequest.searchType()).thenReturn(SearchType.QUERY_THEN_FETCH); int numRequests = 50; @@ -179,6 +175,7 @@ public void testConcurrentOnRequestEnd() throws InterruptedException { new SearchRequestOperationsListener.CompositeListener(searchListenersList, logger), searchRequest ); + searchRequestContext.setSearchTask(searchTask); searchRequestContext.setAbsoluteStartNanos((i < numRequestsLogged) ? 0 : System.nanoTime()); searchRequestContexts.add(searchRequestContext); } @@ -188,7 +185,7 @@ public void testConcurrentOnRequestEnd() throws InterruptedException { threads[i] = new Thread(() -> { phaser.arriveAndAwaitAdvance(); SearchRequestContext thisContext = searchRequestContexts.get(finalI); - thisContext.getSearchRequestOperationsListener().onRequestEnd(searchPhaseContext, thisContext); + thisContext.getSearchRequestOperationsListener().onRequestEnd(thisContext); countDownLatch.countDown(); }); threads[i].start(); @@ -211,7 +208,6 @@ public void testSearchRequestSlowLogHasJsonFields_EmptySearchRequestContext() th searchRequest ); SearchRequestSlowLog.SearchRequestSlowLogMessage p = new SearchRequestSlowLog.SearchRequestSlowLogMessage( - searchPhaseContext, 10, searchRequestContext ); @@ -223,7 +219,7 @@ public void testSearchRequestSlowLogHasJsonFields_EmptySearchRequestContext() th assertThat(p.getValueFor("search_type"), equalTo("QUERY_THEN_FETCH")); assertThat(p.getValueFor("shards"), equalTo("")); assertThat(p.getValueFor("source"), equalTo("{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}")); - assertThat(p.getValueFor("id"), equalTo(null)); + assertThat(p.getValueFor("id"), equalTo("")); } public void testSearchRequestSlowLogHasJsonFields_NotEmptySearchRequestContext() throws IOException { @@ -239,11 +235,7 @@ public void testSearchRequestSlowLogHasJsonFields_NotEmptySearchRequestContext() searchRequestContext.updatePhaseTookMap(SearchPhaseName.EXPAND.getName(), 5L); searchRequestContext.setTotalHits(new TotalHits(3L, TotalHits.Relation.EQUAL_TO)); searchRequestContext.setShardStats(10, 8, 1, 1); - SearchRequestSlowLog.SearchRequestSlowLogMessage p = new SearchRequestSlowLog.SearchRequestSlowLogMessage( - searchPhaseContext, - 10, - searchRequestContext - ); + SearchRequestSlowLog.SearchRequestSlowLogMessage p = new SearchRequestSlowLog.SearchRequestSlowLogMessage(10, searchRequestContext); assertThat(p.getValueFor("took"), equalTo("10nanos")); assertThat(p.getValueFor("took_millis"), equalTo("0")); @@ -252,7 +244,7 @@ public void testSearchRequestSlowLogHasJsonFields_NotEmptySearchRequestContext() assertThat(p.getValueFor("search_type"), equalTo("QUERY_THEN_FETCH")); assertThat(p.getValueFor("shards"), equalTo("{total:10, successful:8, skipped:1, failed:1}")); assertThat(p.getValueFor("source"), equalTo("{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}")); - assertThat(p.getValueFor("id"), equalTo(null)); + assertThat(p.getValueFor("id"), equalTo("")); } public void testSearchRequestSlowLogHasJsonFields_PartialContext() throws IOException { @@ -269,7 +261,6 @@ public void testSearchRequestSlowLogHasJsonFields_PartialContext() throws IOExce searchRequestContext.setTotalHits(new TotalHits(3L, TotalHits.Relation.EQUAL_TO)); searchRequestContext.setShardStats(5, 3, 1, 1); SearchRequestSlowLog.SearchRequestSlowLogMessage p = new SearchRequestSlowLog.SearchRequestSlowLogMessage( - searchPhaseContext, 10000000000L, searchRequestContext ); @@ -281,7 +272,7 @@ public void testSearchRequestSlowLogHasJsonFields_PartialContext() throws IOExce assertThat(p.getValueFor("search_type"), equalTo("QUERY_THEN_FETCH")); assertThat(p.getValueFor("shards"), equalTo("{total:5, successful:3, skipped:1, failed:1}")); assertThat(p.getValueFor("source"), equalTo("{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}")); - assertThat(p.getValueFor("id"), equalTo(null)); + assertThat(p.getValueFor("id"), equalTo("")); } public void testSearchRequestSlowLogSearchContextPrinterToLog() throws IOException { @@ -298,7 +289,6 @@ public void testSearchRequestSlowLogSearchContextPrinterToLog() throws IOExcepti searchRequestContext.setTotalHits(new TotalHits(3L, TotalHits.Relation.EQUAL_TO)); searchRequestContext.setShardStats(10, 8, 1, 1); SearchRequestSlowLog.SearchRequestSlowLogMessage p = new SearchRequestSlowLog.SearchRequestSlowLogMessage( - searchPhaseContext, 100000, searchRequestContext ); diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestStatsTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestStatsTests.java index 377ccebbfd418..87598ad1e8271 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestStatsTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestStatsTests.java @@ -34,9 +34,9 @@ public void testSearchRequestPhaseFailure() { for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); - testRequestStats.onPhaseStart(ctx); + testRequestStats.onPhaseStart(ctx, new SearchRequestContext()); assertEquals(1, testRequestStats.getPhaseCurrent(searchPhaseName)); - testRequestStats.onPhaseFailure(ctx); + testRequestStats.onPhaseFailure(ctx, new SearchRequestContext()); assertEquals(0, testRequestStats.getPhaseCurrent(searchPhaseName)); } } @@ -52,7 +52,7 @@ public void testSearchRequestStats() { for (SearchPhaseName searchPhaseName : SearchPhaseName.values()) { when(mockSearchPhase.getSearchPhaseName()).thenReturn(searchPhaseName); long tookTimeInMillis = randomIntBetween(1, 10); - testRequestStats.onPhaseStart(ctx); + testRequestStats.onPhaseStart(ctx, new SearchRequestContext()); long startTime = System.nanoTime() - TimeUnit.MILLISECONDS.toNanos(tookTimeInMillis); when(mockSearchPhase.getStartTimeInNanos()).thenReturn(startTime); assertEquals(1, testRequestStats.getPhaseCurrent(searchPhaseName)); @@ -84,7 +84,7 @@ public void testSearchRequestStatsOnPhaseStartConcurrently() throws InterruptedE for (int i = 0; i < numTasks; i++) { threads[i] = new Thread(() -> { phaser.arriveAndAwaitAdvance(); - testRequestStats.onPhaseStart(ctx); + testRequestStats.onPhaseStart(ctx, new SearchRequestContext()); countDownLatch.countDown(); }); threads[i].start(); @@ -155,8 +155,8 @@ public void testSearchRequestStatsOnPhaseFailureConcurrently() throws Interrupte for (int i = 0; i < numTasks; i++) { threads[i] = new Thread(() -> { phaser.arriveAndAwaitAdvance(); - testRequestStats.onPhaseStart(ctx); - testRequestStats.onPhaseFailure(ctx); + testRequestStats.onPhaseStart(ctx, new SearchRequestContext()); + testRequestStats.onPhaseFailure(ctx, new SearchRequestContext()); countDownLatch.countDown(); }); threads[i].start(); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java index 2c6c4afed69fd..7944ee681f5fc 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -988,6 +988,53 @@ public void testDeleteStaleCommitsActualDelete() throws Exception { verify(remoteMetadataDirectory).deleteFile(metadataFilename3); } + public void testDeleteStaleCommitsActualDeleteWithLocks() throws Exception { + Map> metadataFilenameContentMapping = populateMetadata(); + remoteSegmentStoreDirectory.init(); + + // Locking one of the metadata files to ensure that it is not getting deleted. + when(mdLockManager.fetchLockedMetadataFiles(any())).thenReturn(Set.of(metadataFilename2)); + + // popluateMetadata() adds stub to return 3 metadata files + // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(1); + + for (String metadata : metadataFilenameContentMapping.get(metadataFilename3).values()) { + String uploadedFilename = metadata.split(RemoteSegmentStoreDirectory.UploadedSegmentMetadata.SEPARATOR)[1]; + verify(remoteDataDirectory).deleteFile(uploadedFilename); + } + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); + verify(remoteMetadataDirectory).deleteFile(metadataFilename3); + verify(remoteMetadataDirectory, times(0)).deleteFile(metadataFilename2); + } + + public void testDeleteStaleCommitsNoDeletesDueToLocks() throws Exception { + remoteSegmentStoreDirectory.init(); + + // Locking all the old metadata files to ensure that none of the segment files are getting deleted. + when(mdLockManager.fetchLockedMetadataFiles(any())).thenReturn(Set.of(metadataFilename2, metadataFilename3)); + + // popluateMetadata() adds stub to return 3 metadata files + // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(1); + + assertBusy(() -> assertThat(remoteSegmentStoreDirectory.canDeleteStaleCommits.get(), is(true))); + verify(remoteMetadataDirectory, times(0)).deleteFile(any()); + } + + public void testDeleteStaleCommitsExceptionWhileFetchingLocks() throws Exception { + remoteSegmentStoreDirectory.init(); + + // Locking one of the metadata files to ensure that it is not getting deleted. + when(mdLockManager.fetchLockedMetadataFiles(any())).thenThrow(new RuntimeException("Rate limit exceeded")); + + // popluateMetadata() adds stub to return 3 metadata files + // We are passing lastNMetadataFilesToKeep=2 here so that oldest 1 metadata file will be deleted + remoteSegmentStoreDirectory.deleteStaleSegmentsAsync(1); + + verify(remoteMetadataDirectory, times(0)).deleteFile(any()); + } + public void testDeleteStaleCommitsDeleteDedup() throws Exception { Map> metadataFilenameContentMapping = new HashMap<>(populateMetadata()); metadataFilenameContentMapping.put(metadataFilename4, metadataFilenameContentMapping.get(metadataFilename3)); diff --git a/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManagerTests.java b/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManagerTests.java index b4eac2c4548d5..299100b65a43e 100644 --- a/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManagerTests.java +++ b/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreMetadataLockManagerTests.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import java.util.Set; import junit.framework.TestCase; @@ -96,4 +97,26 @@ public void testIsAcquiredExceptionCase() { // metadata file is not passed durin FileLockInfo testLockInfo = FileLockInfo.getLockInfoBuilder().withAcquirerId(testAcquirerId).build(); assertThrows(IllegalArgumentException.class, () -> remoteStoreMetadataLockManager.isAcquired(testLockInfo)); } + + public void testFetchLocksEmpty() throws IOException { + when(lockDirectory.listFilesByPrefix("metadata")).thenReturn(Set.of()); + assertEquals(0, remoteStoreMetadataLockManager.fetchLockedMetadataFiles("metadata").size()); + } + + public void testFetchLocksNonEmpty() throws IOException { + String metadata1 = "metadata_1_2_3"; + String metadata2 = "metadata_4_5_6"; + when(lockDirectory.listFilesByPrefix("metadata")).thenReturn( + Set.of( + FileLockInfo.LockFileUtils.generateLockName(metadata1, "snapshot1"), + FileLockInfo.LockFileUtils.generateLockName(metadata2, "snapshot2") + ) + ); + assertEquals(Set.of(metadata1, metadata2), remoteStoreMetadataLockManager.fetchLockedMetadataFiles("metadata")); + } + + public void testFetchLocksException() throws IOException { + when(lockDirectory.listFilesByPrefix("metadata")).thenThrow(new IOException("Something went wrong")); + assertThrows(IOException.class, () -> remoteStoreMetadataLockManager.fetchLockedMetadataFiles("metadata")); + } } diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index 9bb1f51c51cf6..9f357b129988e 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -2279,7 +2279,8 @@ public void onFailure(final Exception e) { new FetchPhase(Collections.emptyList()), responseCollectorService, new NoneCircuitBreakerService(), - null + null, + NoopTracer.INSTANCE ); SearchPhaseController searchPhaseController = new SearchPhaseController( writableRegistry(), @@ -2313,7 +2314,8 @@ public void onFailure(final Exception e) { client ), NoopMetricsRegistry.INSTANCE, - searchRequestOperationsCompositeListenerFactory + searchRequestOperationsCompositeListenerFactory, + NoopTracer.INSTANCE ) ); actions.put( diff --git a/settings.gradle b/settings.gradle index 24ab4a7a22237..8fbf32504215b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,7 @@ */ plugins { - id "com.gradle.enterprise" version "3.16.1" + id "com.gradle.enterprise" version "3.16.2" } ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE') diff --git a/test/framework/src/main/java/org/opensearch/node/MockNode.java b/test/framework/src/main/java/org/opensearch/node/MockNode.java index e6c7e21d5b3ea..c3f36f44ed409 100644 --- a/test/framework/src/main/java/org/opensearch/node/MockNode.java +++ b/test/framework/src/main/java/org/opensearch/node/MockNode.java @@ -155,7 +155,8 @@ protected SearchService newSearchService( FetchPhase fetchPhase, ResponseCollectorService responseCollectorService, CircuitBreakerService circuitBreakerService, - Executor indexSearcherExecutor + Executor indexSearcherExecutor, + Tracer tracer ) { if (getPluginsService().filterPlugins(MockSearchService.TestPlugin.class).isEmpty()) { return super.newSearchService( @@ -168,7 +169,8 @@ protected SearchService newSearchService( fetchPhase, responseCollectorService, circuitBreakerService, - indexSearcherExecutor + indexSearcherExecutor, + tracer ); } return new MockSearchService( @@ -180,7 +182,8 @@ protected SearchService newSearchService( queryPhase, fetchPhase, circuitBreakerService, - indexSearcherExecutor + indexSearcherExecutor, + tracer ); } diff --git a/test/framework/src/main/java/org/opensearch/search/MockSearchService.java b/test/framework/src/main/java/org/opensearch/search/MockSearchService.java index a0bbcb7be05f9..85a547239df10 100644 --- a/test/framework/src/main/java/org/opensearch/search/MockSearchService.java +++ b/test/framework/src/main/java/org/opensearch/search/MockSearchService.java @@ -42,6 +42,7 @@ import org.opensearch.search.fetch.FetchPhase; import org.opensearch.search.internal.ReaderContext; import org.opensearch.search.query.QueryPhase; +import org.opensearch.telemetry.tracing.Tracer; import org.opensearch.threadpool.ThreadPool; import java.util.HashMap; @@ -96,7 +97,8 @@ public MockSearchService( QueryPhase queryPhase, FetchPhase fetchPhase, CircuitBreakerService circuitBreakerService, - Executor indexSearcherExecutor + Executor indexSearcherExecutor, + Tracer tracer ) { super( clusterService, @@ -108,7 +110,8 @@ public MockSearchService( fetchPhase, null, circuitBreakerService, - indexSearcherExecutor + indexSearcherExecutor, + tracer ); }