Merge branch 'master' into reporting/usage-stricter-ts

.ci/es-snapshots/Jenkinsfile_verify_es

@@ -21,6 +21,7 @@ def SNAPSHOT_MANIFEST = "https://storage.googleapis.com/kibana-ci-es-snapshots-d
 
 kibanaPipeline(timeoutMinutes: 120) {
   catchErrors {
+    retryable.enable(2)
     withEnv(["ES_SNAPSHOT_MANIFEST=${SNAPSHOT_MANIFEST}"]) {
       parallel([
         'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),

docs/discover/search.asciidoc

@@ -1,25 +1,53 @@
 [[search]]
 == Searching your data
-You can search the indices that match the current <<index-patterns, index pattern>> by entering
-your search criteria in the Query bar. By default you can use Kibana's <<kuery-query, standard query language>>
-which features autocomplete and a simple, easy to use syntax. Kibana's legacy query
-language (based on Lucene https://lucene.apache.org/core/2_9_4/queryparsersyntax.html[query syntax])
-is still available for the time being under the options menu in the Query Bar. When this
-legacy query language is selected, the full JSON-based {ref}/query-dsl.html[Elasticsearch Query DSL]
-can also be used.
-
-When you submit a search request, the histogram, Documents table, and Fields
-list are updated to reflect the search results. The total number of hits
-(matching documents) is shown in the toolbar. The Documents table shows the
-first five hundred hits. By default, the hits are listed in reverse
-chronological order, with the newest documents shown first. You can reverse
-the sort order by clicking the Time column header. You can also sort the table
-by the values in any indexed field. For more information, see <<sorting,
-Sorting the Documents Table>>.
-
-To search your data, enter your search criteria in the Query bar and
-press *Enter* or click *Search* image:images/search-button.jpg[] to submit
-the request to Elasticsearch.
+Many Kibana apps embed a query bar for real-time search, including
+*Discover*, *Visualize*, and *Dashboard*.
+
+[float]
+=== Search your data
+
+To search the indices that match the current <<index-patterns, index pattern>>,
+enter your search criteria in the query bar. By default, you'll use
+{kib}'s <<kuery-query, standard query language>> (KQL), which
+features autocomplete and a simple, easy-to-use syntax. If you prefer to use
+{kib}'s legacy query
+language, based on the
+Lucene https://lucene.apache.org/core/2_9_4/queryparsersyntax.html[query syntax],
+you can switch to it from the KQL popup in the query bar. When you enable the
+legacy query language, you can use the full
+JSON-based {ref}/query-dsl.html[Elasticsearch Query DSL].
+
+
+[float]
+[[autorefresh]]
+=== Refresh search results
+As more documents are added to the indices you're searching, the search results
+shown in *Discover*, and used to display visualizations, get stale. Using the
+time filter, you can
+configure a refresh interval to periodically resubmit your searches to
+retrieve the latest results.
+
+[role="screenshot"]
+image::images/autorefresh-interval.png[]
+
+You can also manually refresh the search results by
+clicking the *Refresh* button.
+
+[float]
+=== Searching large amounts of data
+
+Sometimes you want to search through large amounts of data no matter how long
+the search takes.  While this might not happen often, there are times
+that long-running queries are required.  Consider a threat hunting scenario
+where you need to search through years of data.
+
+If you run a query, and the run time gets close to the
+timeout, you're presented the option to ignore the timeout.  This enables you to
+run queries with large amounts of data to completion.
+
+By default, a query times out after 30 seconds.
+The timeout is in place to avoid unintentional load on the cluster.
+
 
 include::kuery.asciidoc[]
 
@@ -160,31 +188,3 @@ To completely delete a query:
 image::discover/images/saved-query-management-component-delete-query-button.png["Example of the saved query management popover when a query is hovered over and we are about to delete a query",width="80%"]
 
 You can import, export, and delete saved queries from <<managing-saved-objects, Saved Objects in Management>>.
-
-[[select-pattern]]
-=== Change the indices you're searching
-When you submit a search request, the indices that match the currently-selected
-index pattern are searched.
-To change the indices you are searching, click the index pattern and select a
-different <<settings-create-pattern, index pattern>>.
-
-[[autorefresh]]
-=== Refresh the search results
-As more documents are added to the indices you're searching, the search results
-shown in Discover and used to display visualizations get stale. You can
-configure a refresh interval to periodically resubmit your searches to
-retrieve the latest results.
-
-. Click image:images/time-filter-calendar.png[].
-
-. In the *Refresh every* field, enter the refresh rate, then select the interval
-  from the dropdown.
-
-. Click *Start*.
-+
-image::images/autorefresh-intervals.png[]
-
-To disable auto refresh, click *Stop*.
-
-If auto refresh is not enabled, click *Refresh* to manually refresh the search
-results.

docs/user/discover.asciidoc

@@ -21,6 +21,7 @@ image::images/Discover-Start.png[Discover]
 
 
 [float]
+[[select-pattern]]
 === Set up your index pattern
 
 The first thing to do in *Discover* is to select an <<index-patterns, index pattern>>, which

docs/user/reporting/development/pdf-integration.asciidoc

@@ -63,3 +63,5 @@ If there are multiple visualizations, the `data-shared-items-count` attribute sh
 many Visualizations to look for. Reporting will look at every element with the `data-shared-item` attribute and use the corresponding
 `data-render-complete` attribute and `renderComplete` events to listen for rendering to complete. When rendering is complete for a visualization
 the `data-render-complete` attribute should be set to "true" and it should dispatch a custom DOM `renderComplete` event.
+
+If the reporting job uses multiple URLs, before looking for any of the `data-shared-item` or `data-shared-items-count` attributes, it waits for a `data-shared-page` attribute that specifies which page is being loaded.

src/core/server/elasticsearch/elasticsearch_service.mock.ts

@@ -18,6 +18,7 @@
  */
 
 import { BehaviorSubject } from 'rxjs';
+import { Client } from 'elasticsearch';
 import { IClusterClient, ICustomClusterClient } from './cluster_client';
 import { IScopedClusterClient } from './scoped_cluster_client';
 import { ElasticsearchConfig } from './elasticsearch_config';
@@ -130,6 +131,55 @@ const createMock = () => {
   return mocked;
 };
 
+const createElasticsearchClientMock = () => {
+  const mocked: jest.Mocked<Client> = {
+    cat: {} as any,
+    cluster: {} as any,
+    indices: {} as any,
+    ingest: {} as any,
+    nodes: {} as any,
+    snapshot: {} as any,
+    tasks: {} as any,
+    bulk: jest.fn(),
+    clearScroll: jest.fn(),
+    count: jest.fn(),
+    create: jest.fn(),
+    delete: jest.fn(),
+    deleteByQuery: jest.fn(),
+    deleteScript: jest.fn(),
+    deleteTemplate: jest.fn(),
+    exists: jest.fn(),
+    explain: jest.fn(),
+    fieldStats: jest.fn(),
+    get: jest.fn(),
+    getScript: jest.fn(),
+    getSource: jest.fn(),
+    getTemplate: jest.fn(),
+    index: jest.fn(),
+    info: jest.fn(),
+    mget: jest.fn(),
+    msearch: jest.fn(),
+    msearchTemplate: jest.fn(),
+    mtermvectors: jest.fn(),
+    ping: jest.fn(),
+    putScript: jest.fn(),
+    putTemplate: jest.fn(),
+    reindex: jest.fn(),
+    reindexRethrottle: jest.fn(),
+    renderSearchTemplate: jest.fn(),
+    scroll: jest.fn(),
+    search: jest.fn(),
+    searchShards: jest.fn(),
+    searchTemplate: jest.fn(),
+    suggest: jest.fn(),
+    termvectors: jest.fn(),
+    update: jest.fn(),
+    updateByQuery: jest.fn(),
+    close: jest.fn(),
+  };
+  return mocked;
+};
+
 export const elasticsearchServiceMock = {
   create: createMock,
   createInternalSetup: createInternalSetupContractMock,
@@ -138,4 +188,5 @@ export const elasticsearchServiceMock = {
   createClusterClient: createClusterClientMock,
   createCustomClusterClient: createCustomClusterClientMock,
   createScopedClusterClient: createScopedClusterClientMock,
+  createElasticsearchClient: createElasticsearchClientMock,
 };

src/core/server/http/integration_tests/core_service.test.mocks.ts

@@ -24,3 +24,14 @@ jest.doMock('../../elasticsearch/scoped_cluster_client', () => ({
     return elasticsearchServiceMock.createScopedClusterClient();
   }),
 }));
+
+jest.doMock('elasticsearch', () => {
+  const realES = jest.requireActual('elasticsearch');
+  return {
+    ...realES,
+    // eslint-disable-next-line object-shorthand
+    Client: function() {
+      return elasticsearchServiceMock.createElasticsearchClient();
+    },
+  };
+});

src/core/server/http/integration_tests/core_services.test.ts

@@ -43,7 +43,7 @@ describe('http service', () => {
   describe('auth', () => {
     let root: ReturnType<typeof kbnTestServer.createRoot>;
     beforeEach(async () => {
-      root = kbnTestServer.createRoot({ migrations: { skip: true } });
+      root = kbnTestServer.createRoot();
     }, 30000);
 
     afterEach(async () => {
@@ -192,7 +192,7 @@ describe('http service', () => {
 
       let root: ReturnType<typeof kbnTestServer.createRoot>;
       beforeEach(async () => {
-        root = kbnTestServer.createRoot({ migrations: { skip: true } });
+        root = kbnTestServer.createRoot();
       }, 30000);
 
       afterEach(async () => {
@@ -326,7 +326,7 @@ describe('http service', () => {
     describe('#basePath()', () => {
       let root: ReturnType<typeof kbnTestServer.createRoot>;
       beforeEach(async () => {
-        root = kbnTestServer.createRoot({ migrations: { skip: true } });
+        root = kbnTestServer.createRoot();
       }, 30000);
 
       afterEach(async () => await root.shutdown());
@@ -355,7 +355,7 @@ describe('http service', () => {
   describe('elasticsearch', () => {
     let root: ReturnType<typeof kbnTestServer.createRoot>;
     beforeEach(async () => {
-      root = kbnTestServer.createRoot({ migrations: { skip: true } });
+      root = kbnTestServer.createRoot();
     }, 30000);
 
     afterEach(async () => {

vars/prChanges.groovy

@@ -7,6 +7,7 @@ def getSkippablePaths() {
     /^docs\//,
     /^rfcs\//,
     /^.ci\/.+\.yml$/,
+    /^.ci\/es-snapshots\//,
     /^\.github\//,
     /\.md$/,
   ]