[Data Explorer][Discover 2.0] Fix issues when change index pattern

* allow side nav show only selected index pattern fields when switch
* allow reset column state when switch index pattern

Issue Resolve:
opensearch-project#4840
opensearch-project#4846

Signed-off-by: ananzh <ananzh@amazon.com>

src/plugins/discover/public/application/components/sidebar/lib/get_index_pattern_field_list.ts

@@ -36,18 +36,5 @@ export function getIndexPatternFieldList(
   fieldCounts?: Record<string, number>
 ) {
   if (!indexPattern || !fieldCounts) return [];
-
-  const fieldNamesInDocs = Object.keys(fieldCounts);
-  const fieldNamesInIndexPattern = indexPattern.fields.getAll().map((fld) => fld.name);
-  const unknownTypes: IndexPatternField[] = [];
-
-  difference(fieldNamesInDocs, fieldNamesInIndexPattern).forEach((unknownFieldName) => {
-    unknownTypes.push({
-      displayName: String(unknownFieldName),
-      name: String(unknownFieldName),
-      type: 'unknown',
-    } as IndexPatternField);
-  });
-
-  return [...indexPattern.fields.getAll(), ...unknownTypes];
+  return [...indexPattern.fields.getAll()];
 }

src/plugins/discover/public/application/view_components/canvas/index.tsx

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import React, { useEffect, useState } from 'react';
+import React, { useEffect, useState, useMemo, useRef } from 'react';
 import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiCallOut, EuiLink } from '@elastic/eui';
 import { TopNav } from './top_nav';
 import { ViewProps } from '../../../../../data_explorer/public';
@@ -14,10 +14,18 @@ import { ResultStatus, SearchData } from '../utils/use_search';
 import { DiscoverNoResults } from '../../components/no_results/no_results';
 import { DiscoverUninitialized } from '../../components/uninitialized/uninitialized';
 import { LoadingSpinner } from '../../components/loading_spinner/loading_spinner';
+import { setColumns, useDispatch, useSelector } from '../../utils/state_management';
+import { filterColumns } from '../utils/filter_columns';
 
 // eslint-disable-next-line import/no-default-export
 export default function DiscoverCanvas({ setHeaderActionMenu, history }: ViewProps) {
   const { data$, refetch$, indexPattern } = useDiscoverContext();
+  const { columns } = useSelector((state) => state.discover);
+  const filteredColumns = filterColumns(columns, indexPattern);
+  const dispatch = useDispatch();
+
+  const prevColumns = useRef(filteredColumns);
+  const prevIndexPattern = useRef(indexPattern);
 
   const [fetchState, setFetchState] = useState<SearchData>({
     status: data$.getValue().status,
@@ -69,6 +77,15 @@ export default function DiscoverCanvas({ setHeaderActionMenu, history }: ViewPro
     };
   }, [data$, fetchState]);
 
+  useEffect(() => {
+    if (indexPattern !== prevIndexPattern.current) {
+      dispatch(setColumns({ columns: filteredColumns }));
+      prevColumns.current = filteredColumns;
+      prevIndexPattern.current = indexPattern;
+    }
+    prevColumns.current = filteredColumns;
+  }, [dispatch, filteredColumns, indexPattern]);
+
   const timeField = indexPattern?.timeFieldName ? indexPattern.timeFieldName : undefined;
 
   return (

src/plugins/discover/public/application/view_components/utils/filter_columns.test.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { filterColumns } from './filter_columns';
+import { IndexPattern } from '../../../opensearch_dashboards_services';
+
+describe('filterColumns', () => {
+  const indexPatternMock = {
+    fields: {
+      getAll: () => [{ name: 'a' }, { name: 'c' }, { name: 'd' }],
+    },
+  } as IndexPattern;
+
+  it('should return columns that exist in the index pattern fields', () => {
+    const columns = ['a', 'b'];
+    const result = filterColumns(columns, indexPatternMock);
+    expect(result).toEqual(['a']);
+  });
+
+  it('should return _source if no columns exist in the index pattern fields', () => {
+    const columns = ['b', 'e'];
+    const result = filterColumns(columns, indexPatternMock);
+    expect(result).toEqual(['_source']);
+  });
+
+  it('should return empty array if no columns and indexPattern is null', () => {
+    const columns = ['b', 'e'];
+    const result = filterColumns(columns, null);
+    expect(result).toEqual(['_source']);
+  });
+});

src/plugins/discover/public/application/view_components/utils/filter_columns.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { IndexPattern } from '../../../opensearch_dashboards_services';
+
+/**
+ * Helper function to filter columns based on the fields of the index pattern.
+ * This function is used when we switch between index patterns. We want to keep the columns that are
+ * still available in the new index pattern and remove the ones that are not.
+ * If the resulting array is empty, it provides a fallback to a single '_source' column.
+ * @param columns Array of column names
+ * @param indexPattern Index pattern object
+ */
+export function filterColumns(columns: string[], indexPattern: IndexPattern) {
+  const fieldsName = indexPattern?.fields.getAll().map((fld) => fld.name) || [];
+  const filteredColumns = columns.filter((column) => fieldsName.includes(column));
+  return filteredColumns.length > 0 ? filteredColumns : ['_source'];
+}