Merge branch 'main' into health-monitor-fixes

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.19", "8.12.2", "8.13.0", "8.14.0"]
+            BWC_VERSION: ["7.17.19", "8.12.3", "8.13.0", "8.14.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1873,6 +1873,22 @@ steps:
         env:
           BWC_VERSION: 8.12.2
 
+      - label: "{{matrix.image}} / 8.12.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.3
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.3
+
       - label: "{{matrix.image}} / 8.13.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -1152,6 +1152,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.2
+      - label: 8.12.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.3#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.3
       - label: 8.13.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.0#bwcTest
         timeout_in_minutes: 300

.ci/bwcVersions

@@ -114,5 +114,6 @@ BWC_VERSION:
   - "8.12.0"
   - "8.12.1"
   - "8.12.2"
+  - "8.12.3"
   - "8.13.0"
   - "8.14.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.19"
-  - "8.12.2"
+  - "8.12.3"
   - "8.13.0"
   - "8.14.0"

README.asciidoc

@@ -1,20 +1,24 @@
 = Elasticsearch
 
-Elasticsearch is a distributed, RESTful search engine optimized for speed and relevance on production-scale workloads. You can use Elasticsearch to perform real-time search over massive datasets for applications including:
+Elasticsearch is a distributed search and analytics engine optimized for speed and relevance on production-scale workloads. Elasticsearch is the foundation of Elastic's open Stack platform. Search in near real-time over massive datasets, perform vector searches, integrate with generative AI applications, and much more.
 
-* Vector search
+Use cases enabled by Elasticsearch include:
+
+* https://www.elastic.co/search-labs/blog/articles/retrieval-augmented-generation-rag[Retrieval Augmented Generation (RAG)] 
+* https://www.elastic.co/search-labs/blog/categories/vector-search[Vector search]
 * Full-text search
 * Logs
 * Metrics
 * Application performance monitoring (APM)
 * Security logs
 
-
 \... and more!
 
 To learn more about Elasticsearch's features and capabilities, see our
 https://www.elastic.co/products/elasticsearch[product page].
 
+To access information on https://www.elastic.co/search-labs/blog/categories/ml-research[machine learning innovations] and the latest https://www.elastic.co/search-labs/blog/categories/lucene[Lucene contributions from Elastic], more information can be found in https://www.elastic.co/search-labs[Search Labs]. 
+
 [[get-started]]
 == Get started
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -32,7 +32,9 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 
 import javax.inject.Inject;
 
@@ -142,6 +144,8 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
 
                     System.out.println("Generating buildscan link for artifact...");
 
+                    // Output should be in the format: "<UUID><space><ISO-8601-timestamp>\n"
+                    // and multiple artifacts could be returned
                     Process process = new ProcessBuilder(
                         "buildkite-agent",
                         "artifact",
@@ -150,7 +154,7 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                         "--step",
                         System.getenv("BUILDKITE_JOB_ID"),
                         "--format",
-                        "%i"
+                        "%i %c"
                     ).start();
                     process.waitFor();
                     String processOutput;
@@ -159,7 +163,17 @@ public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNo
                     } catch (IOException e) {
                         processOutput = "";
                     }
-                    String artifactUuid = processOutput.trim();
+
+                    // Sort them by timestamp, and grab the most recent one
+                    Optional<String> artifact = Arrays.stream(processOutput.trim().split("\n")).map(String::trim).min((a, b) -> {
+                        String[] partsA = a.split(" ");
+                        String[] partsB = b.split(" ");
+                        // ISO-8601 timestamps can be sorted lexicographically
+                        return partsB[1].compareTo(partsA[1]);
+                    });
+
+                    // Grab just the UUID from the artifact
+                    String artifactUuid = artifact.orElse("").split(" ")[0];
 
                     System.out.println("Artifact UUID: " + artifactUuid);
                     if (artifactUuid.isEmpty() == false) {

build-tools-internal/version.properties

@@ -13,7 +13,7 @@ supercsv          = 2.4.0
 log4j             = 2.19.0
 slf4j             = 2.0.6
 ecsLogging        = 1.2.0
-jna               = 5.10.0
+jna               = 5.12.1
 netty             = 4.1.107.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1

distribution/archives/build.gradle

@@ -11,7 +11,7 @@ import java.nio.file.Path
 
 apply plugin: 'elasticsearch.internal-distribution-archive-setup'
 
-CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, String architecture, boolean isTestDistro) {
+CopySpec archiveFiles(String distributionType, String os, String architecture, boolean isTestDistro) {
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
@@ -29,9 +29,9 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
       into('bin') {
         with binFiles(distributionType, isTestDistro)
       }
-      into("darwin".equals(platform) ? 'jdk.app' : 'jdk') {
+      into("darwin".equals(os) ? 'jdk.app' : 'jdk') {
         if (isTestDistro == false) {
-          with jdkFiles(project, platform, architecture)
+          with jdkFiles(project, os, architecture)
         }
       }
       into('') {
@@ -56,7 +56,11 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
 
       with noticeFile(isTestDistro)
       into('modules') {
-        with modulesFiles
+        if (isTestDistro) {
+          with integTestModulesFiles
+        } else {
+          with modulesFiles(os, architecture)
+        }
       }
     }
   }
@@ -65,42 +69,42 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
 distribution_archives {
   integTestZip {
     content {
-      archiveFiles(integTestModulesFiles, 'zip', null, 'x64', true)
+      archiveFiles('zip', null, null, true)
     }
   }
 
   windowsZip {
     archiveClassifier = 'windows-x86_64'
     content {
-      archiveFiles(modulesFiles('windows-x86_64'), 'zip', 'windows', 'x64', false)
+      archiveFiles('zip', 'windows', 'x64', false)
     }
   }
 
   darwinTar {
     archiveClassifier = 'darwin-x86_64'
     content {
-      archiveFiles(modulesFiles('darwin-x86_64'), 'tar', 'darwin', 'x64', false)
+      archiveFiles('tar', 'darwin', 'x64', false)
     }
   }
 
   darwinAarch64Tar {
     archiveClassifier = 'darwin-aarch64'
     content {
-      archiveFiles(modulesFiles('darwin-aarch64'), 'tar', 'darwin', 'aarch64', false)
+      archiveFiles('tar', 'darwin', 'aarch64', false)
     }
   }
 
   linuxAarch64Tar {
     archiveClassifier = 'linux-aarch64'
     content {
-      archiveFiles(modulesFiles('linux-aarch64'), 'tar', 'linux', 'aarch64', false)
+      archiveFiles('tar', 'linux', 'aarch64', false)
     }
   }
 
   linuxTar {
     archiveClassifier = 'linux-x86_64'
     content {
-      archiveFiles(modulesFiles('linux-x86_64'), 'tar', 'linux', 'x64', false)
+      archiveFiles('tar', 'linux', 'x64', false)
     }
   }
 }

distribution/build.gradle

@@ -332,10 +332,10 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
         }
       }
 
-    modulesFiles = { platform ->
+    modulesFiles = { os, architecture ->
       copySpec {
         eachFile {
-          if (it.relativePath.segments[-2] == 'bin' || ((platform == 'darwin-x86_64' || platform == 'darwin-aarch64') && it.relativePath.segments[-2] == 'MacOS')) {
+          if (it.relativePath.segments[-2] == 'bin' || (os == 'darwin' && it.relativePath.segments[-2] == 'MacOS')) {
             // bin files, wherever they are within modules (eg platform specific) should be executable
             // and MacOS is an alternative to bin on macOS
             it.mode = 0755
@@ -344,7 +344,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           }
         }
         List excludePlatforms = ['linux-x86_64', 'linux-aarch64', 'windows-x86_64', 'darwin-x86_64', 'darwin-aarch64']
-        if (platform != null) {
+        if (os != null) {
+          String platform = os + '-' + architecture
+          if (architecture == 'x64') {
+            // ML platform dir uses the x86_64 nomenclature
+            platform = os + '-x86_64'
+          }
           excludePlatforms.remove(excludePlatforms.indexOf(platform))
         } else {
           excludePlatforms = []
@@ -430,15 +435,15 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       }
     }
 
-    jdkFiles = { Project project, String platform, String architecture ->
+    jdkFiles = { Project project, String os, String architecture ->
       return copySpec {
-        from project.jdks."bundled_${platform}_${architecture}"
+        from project.jdks."bundled_${os}_${architecture}"
         exclude "demo/**"
         /*
          * The Contents/MacOS directory interferes with notarization, and is unused by our distribution, so we exclude
          * it from the build.
          */
-        if ("darwin".equals(platform)) {
+        if ("darwin".equals(os)) {
           exclude "Contents/MacOS"
         }
         eachFile { FileCopyDetails details ->

distribution/packages/build.gradle

@@ -143,7 +143,7 @@ def commonPackageConfig(String type, String architecture) {
         with libFiles
       }
       into('modules') {
-        with modulesFiles('linux-' + ((architecture == 'x64') ? 'x86_64' : architecture))
+        with modulesFiles('linux', architecture)
       }
       into('jdk') {
         with jdkFiles(project, 'linux', architecture)

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java

@@ -101,13 +101,13 @@
 import java.util.zip.ZipOutputStream;
 
 import static org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase.forEachFileRecursively;
-import static org.elasticsearch.test.hamcrest.RegexMatcher.matches;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.hasToString;
+import static org.hamcrest.Matchers.matchesRegex;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
@@ -1286,7 +1286,7 @@ public void testInvalidShaFileMismatchFilename() throws Exception {
             )
         );
         assertEquals(ExitCodes.IO_ERROR, e.exitCode);
-        assertThat(e, hasToString(matches("checksum file at \\[.*\\] is not for this plugin")));
+        assertThat(e, hasToString(matchesRegex(".*checksum file at \\[.*\\] is not for this plugin.*")));
     }
 
     public void testInvalidShaFileContainingExtraLine() throws Exception {

docs/changelog/103542.yaml

@@ -0,0 +1,7 @@
+pr: 103542
+summary: Flatten object mappings when subobjects is false
+area: Mapping
+type: feature
+issues:
+ - 99860
+ - 103497

docs/changelog/104711.yaml

@@ -0,0 +1,5 @@
+pr: 104711
+summary: "Fixing NPE when requesting [_none_] for `stored_fields`"
+area: Search
+type: bug
+issues: []

docs/changelog/104830.yaml

@@ -0,0 +1,5 @@
+pr: 104830
+summary: All new `shard_seed` parameter for `random_sampler` agg
+area: Aggregations
+type: enhancement
+issues: []

docs/changelog/105067.yaml

@@ -0,0 +1,5 @@
+pr: 105067
+summary: "ESQL: Use faster field caps"
+area: ES|QL
+type: enhancement
+issues: []