Migrate Index Management and enrichers to the new ES JS client (#105863)

* Destructure index API request bodies consistently.
* Remove unnecessary calls to encodeURIComponent on the server.
* Migrate routes to handleEsError. Delete wrapEsError helpers. Remove unused isEsError and parseEsError dependencies. Remove isEsError from es_ui_shared.
* Update tests and migrate API integration tests.
* Clarify test details in CCR README. Update Index Management README with steps for testing Cloud-managed index templates and steps for testing indices and data streams that contain special characters.

docs/developer/plugin-list.asciidoc

@@ -457,7 +457,7 @@ Index Management by running this series of requests in Console:
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/index_management/README.md[indexManagement]
-|Create a data stream using Console and you'll be able to view it in the UI:
+|Create an index with special characters and verify it renders correctly:
 
 
 |{kib-repo}blob/{branch}/x-pack/plugins/infra/README.md[infra]

src/plugins/es_ui_shared/__packages_do_not_import__/errors/index.ts

@@ -6,6 +6,5 @@
  * Side Public License, v 1.
  */
 
-export { isEsError } from './is_es_error';
 export { handleEsError } from './handle_es_error';
 export { parseEsError } from './es_error_parser';

src/plugins/es_ui_shared/server/errors/index.ts

@@ -6,4 +6,4 @@
  * Side Public License, v 1.
  */
 
-export { isEsError, handleEsError, parseEsError } from '../../__packages_do_not_import__/errors';
+export { handleEsError, parseEsError } from '../../__packages_do_not_import__/errors';

src/plugins/es_ui_shared/server/index.ts

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-export { isEsError, handleEsError, parseEsError } from './errors';
+export { handleEsError, parseEsError } from './errors';
 
 /** dummy plugin*/
 export function plugin() {

x-pack/plugins/cross_cluster_replication/README.md

@@ -4,14 +4,15 @@
 
 You can run a local cluster and simulate a remote cluster within a single Kibana directory.
 
-1. Ensure Kibana isn't running so it doesn't load up any data into your cluster. Run `yarn es snapshot --license=trial` to install a fresh snapshot. Wait for ES to finish setting up.
+1. Ensure Kibana isn't running so it doesn't load up any data into your cluster. Run `yarn es snapshot --license=trial` to install a fresh snapshot. Wait for ES to finish setting up and activate the license.
 2. Create a "remote" copy of your ES snapshot by running: `cp -R .es/8.0.0 .es/8.0.0-2`.
-4. Start your "remote" cluster by running `.es/8.0.0-2/bin/elasticsearch -E cluster.name=europe -E transport.port=9400`.
-4. Run `yarn start` to start Kibana.
-5. Index a document into your remote cluster by running `curl -X PUT http://elastic:changeme@localhost:9201/my-leader-index --data '{"settings":{"number_of_shards":1,"soft_deletes.enabled":true}}' --header "Content-Type: application/json"`. Note that these settings are required for testing auto-follow pattern conflicts errors (see below).
+4. Start your "local" cluster by running `.es/8.0.0/bin/elasticsearch`.
+5. Run `yarn start` to start Kibana so that it connects to the "local" cluster.
+6. Start your "remote" cluster by running `.es/8.0.0-2/bin/elasticsearch -E cluster.name=europe -E transport.port=9400`.
+7. Index a document into your "remote" cluster by running `curl -X PUT http://elastic:changeme@localhost:9201/my-leader-index --data '{"settings":{"number_of_shards":1,"soft_deletes.enabled":true}}' --header "Content-Type: application/json"`. Note that these settings are required for testing auto-follow pattern conflicts errors (see below).
 
 Now you can create follower indices and auto-follow patterns to replicate the `my-leader-index`
-index on the remote cluster that's available at `127.0.0.1:9400`.
+index on the "remote" cluster that's available at `127.0.0.1:9400`.
 
 ### Auto-follow pattern conflict errors
 

x-pack/plugins/cross_cluster_replication/server/plugin.ts

@@ -7,14 +7,8 @@
 
 import { Observable } from 'rxjs';
 import { first } from 'rxjs/operators';
-import {
-  CoreSetup,
-  CoreStart,
-  Plugin,
-  Logger,
-  PluginInitializerContext,
-  LegacyAPICaller,
-} from 'src/core/server';
+import { CoreSetup, CoreStart, Plugin, Logger, PluginInitializerContext } from 'src/core/server';
+import { IScopedClusterClient } from 'kibana/server';
 
 import { Index } from '../../index_management/server';
 import { PLUGIN } from '../common/constants';
@@ -23,20 +17,18 @@ import { registerApiRoutes } from './routes';
 import { CrossClusterReplicationConfig } from './config';
 import { License, handleEsError } from './shared_imports';
 
-// TODO replace deprecated ES client after Index Management is updated
-const ccrDataEnricher = async (indicesList: Index[], callWithRequest: LegacyAPICaller) => {
+const ccrDataEnricher = async (indicesList: Index[], client: IScopedClusterClient) => {
   if (!indicesList?.length) {
     return indicesList;
   }
-  const params = {
-    path: '/_all/_ccr/info',
-    method: 'GET',
-  };
+
   try {
-    const { follower_indices: followerIndices } = await callWithRequest(
-      'transport.request',
-      params
-    );
+    const {
+      body: { follower_indices: followerIndices },
+    } = await client.asCurrentUser.ccr.followInfo({
+      index: '_all',
+    });
+
     return indicesList.map((index) => {
       const isFollowerIndex = !!followerIndices.find(
         (followerIndex: { follower_index: string }) => {

x-pack/plugins/index_lifecycle_management/server/plugin.ts

@@ -6,44 +6,36 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import {
-  CoreSetup,
-  Plugin,
-  Logger,
-  PluginInitializerContext,
-  LegacyAPICaller,
-} from 'src/core/server';
-import { handleEsError } from './shared_imports';
+import { CoreSetup, Plugin, Logger, PluginInitializerContext } from 'src/core/server';
+import { IScopedClusterClient } from 'kibana/server';
 
 import { Index as IndexWithoutIlm } from '../../index_management/common/types';
 import { PLUGIN } from '../common/constants';
-import { Index, IndexLifecyclePolicy } from '../common/types';
+import { Index } from '../common/types';
 import { Dependencies } from './types';
 import { registerApiRoutes } from './routes';
 import { License } from './services';
 import { IndexLifecycleManagementConfig } from './config';
+import { handleEsError } from './shared_imports';
 
 const indexLifecycleDataEnricher = async (
   indicesList: IndexWithoutIlm[],
-  // TODO replace deprecated ES client after Index Management is updated
-  callAsCurrentUser: LegacyAPICaller
+  client: IScopedClusterClient
 ): Promise<Index[]> => {
   if (!indicesList || !indicesList.length) {
     return [];
   }
 
-  const params = {
-    path: '/*/_ilm/explain',
-    method: 'GET',
-  };
-
-  const { indices: ilmIndicesData } = await callAsCurrentUser<{
-    indices: { [indexName: string]: IndexLifecyclePolicy };
-  }>('transport.request', params);
+  const {
+    body: { indices: ilmIndicesData },
+  } = await client.asCurrentUser.ilm.explainLifecycle({
+    index: '*',
+  });
 
   return indicesList.map((index: IndexWithoutIlm) => {
     return {
       ...index,
+      // @ts-expect-error @elastic/elasticsearch Element implicitly has an 'any' type
       ilm: { ...(ilmIndicesData[index.name] || {}) },
     };
   });

x-pack/plugins/index_management/README.md

@@ -1,5 +1,16 @@
 # Index Management UI
 
+## Indices tab
+
+### Quick steps for testing
+
+Create an index with special characters and verify it renders correctly:
+
+```
+# Renders as %{[@metadata][beat]}-%{[@metadata][version]}-2020.08.23
+PUT %25%7B%5B%40metadata%5D%5Bbeat%5D%7D-%25%7B%5B%40metadata%5D%5Bversion%5D%7D-2020.08.23
+```
+
 ## Data streams tab
 
 ### Quick steps for testing
@@ -19,4 +30,56 @@ POST ds/_doc
 {
   "@timestamp": "2020-01-27"
 }
-```
+```
+
+Create a data stream with special characters and verify it renders correctly:
+
+```
+# Configure template for creating a data stream
+PUT _index_template/special_ds
+{
+  "index_patterns": ["%{[@metadata][beat]}-%{[@metadata][version]}-2020.08.23"],
+  "data_stream": {}
+}
+
+# Add a document to the data stream, which will render as %{[@metadata][beat]}-%{[@metadata][version]}-2020.08.23
+POST %25%7B%5B%40metadata%5D%5Bbeat%5D%7D-%25%7B%5B%40metadata%5D%5Bversion%5D%7D-2020.08.23/_doc
+{
+  "@timestamp": "2020-01-27"
+}
+```
+
+## Index templates tab
+
+### Quick steps for testing
+
+By default, **legacy index templates** are not shown in the UI. Make them appear by creating one in Console:
+
+```
+PUT _template/template_1
+{
+  "index_patterns": ["foo*"]
+}
+```
+
+To test **Cloud-managed templates**:
+
+1. Add `cluster.metadata.managed_index_templates` setting via Dev Tools:
+```
+PUT /_cluster/settings
+{
+  "persistent": {
+    "cluster.metadata.managed_index_templates": ".cloud-"
+  }
+}
+```
+
+2. Create a template with the format: `.cloud-<template_name>` via Dev Tools.
+```
+PUT _template/.cloud-example
+{
+  "index_patterns": [ "foobar*"]
+}
+```
+
+The UI will now prevent you from editing or deleting this template.

x-pack/plugins/index_management/public/application/components/index_templates/simulate_template/simulate_template.tsx

@@ -6,7 +6,6 @@
  */
 
 import React, { useState, useCallback, useEffect } from 'react';
-import uuid from 'uuid';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { EuiCodeBlock, EuiCallOut } from '@elastic/eui';
 
@@ -37,11 +36,6 @@ export const SimulateTemplate = React.memo(({ template, filters }: Props) => {
     }
 
     const indexTemplate = serializeTemplate(stripEmptyFields(template) as TemplateDeserialized);
-
-    // Until ES fixes a bug on their side we will send a random index pattern to the simulate API.
-    // Issue: https://github.com/elastic/elasticsearch/issues/59152
-    indexTemplate.index_patterns = [uuid.v4()];
-
     const { data, error } = await simulateIndexTemplate(indexTemplate);
     let filteredTemplate = data;