Skip to content

Commit

Permalink
[dataset-nav][bug] get connections by cluster and update client names…
Browse files Browse the repository at this point in the history 
…pace (#7609) (#7610)

* changes from abby

* refactor api

* polling db not populating

* fixing datasource cache

* fixing session id stuff and running async queries

* Changeset file for PR #7609 created/updated

---------

(cherry picked from commit d09895a)

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>
Signed-off-by: abbyhu2000 <abigailhu2000@gmail.com>
Signed-off-by: Sean Li <lnse@amazon.com>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: abbyhu2000 <abigailhu2000@gmail.com>
Co-authored-by: Sean Li <lnse@amazon.com>
Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>
(cherry picked from commit 7605de7)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@github-actions @abbyhu2000
@sejli @opensearch-changeset-bot
4 people committed Sep 20, 2024
1 parent 10bb541 commit 73e67f7
Show file tree
Hide file tree
Showing 26 changed files with 350 additions and 361 deletions.
2 changes: 2 additions & 0 deletions changelogs/fragments/7609.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
fix:
- Dataset nav to load external connections and update namespace ([#7609](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/7609))
59 changes: 21 additions & 38 deletions src/plugins/data/public/ui/dataset_navigator/dataset_navigator.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,7 @@ import {
getSearchService,
getUiService,
} from '../../services';
import {
fetchDataSources,
fetchIndexPatterns,
fetchIndices,
isCatalogCacheFetching,
fetchIfExternalDataSourcesEnabled,
} from './lib';
import { fetchDataSources, fetchIndexPatterns, fetchIndices, isCatalogCacheFetching } from './lib';
import { useDataSetManager } from '../search_bar/lib/use_dataset_manager';
import { DataSetContract } from '../../query';

Expand Down Expand Up @@ -76,7 +70,6 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
const isInitialized = useRef(false);
const [isOpen, setIsOpen] = useState(false);
const [isLoading, setIsLoading] = useState(false);
const [isExternalDataSourcesEnabled, setIsExternalDataSourcesEnabled] = useState(false);
const [dataSources, setDataSources] = useState<SimpleDataSource[]>([]);
const [externalDataSources, setExternalDataSources] = useState<SimpleDataSource[]>([]);
const [indexPatterns, setIndexPatterns] = useState<any[]>([]);
Expand All @@ -85,6 +78,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
const [selectedDataSetState, setSelectedDataSetState] = useState<
SelectedDataSetState | undefined
>(undefined);
const isExternalDataSourcesEnabled = externalDataSources.length > 0;

const {
loadStatus: dataSourcesLoadStatus,
Expand All @@ -104,7 +98,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({

const onRefresh = useCallback(() => {
if (!isCatalogCacheFetching(dataSourcesLoadStatus) && dataSources.length > 0) {
startLoadingDataSources(dataSources.map((dataSource) => dataSource.id));
startLoadingDataSources(dataSources);
}
}, [dataSourcesLoadStatus, dataSources, startLoadingDataSources]);

Expand Down Expand Up @@ -158,19 +152,27 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({

setIsLoading(true);
try {
const [
fetchedIndexPatterns,
fetchedDataSources,
fetchedIsExternalDataSourcesEnabled,
] = await Promise.all([
const [fetchedIndexPatterns, fetchedDataSources] = await Promise.all([
fetchIndexPatterns(savedObjectsClient!, ''),
fetchDataSources(savedObjectsClient!),
fetchIfExternalDataSourcesEnabled(http!),
]);

const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache();
if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) {
setExternalDataSources(
externalDataSourcesCache.dataSources.map((ds) => ({
id: ds.dataSourceRef,
name: ds.name,
type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL,
}))
);
} else if (fetchedDataSources.length > 0) {
setExternalDataSources(await startLoadingDataSources(fetchedDataSources));
}

setIndexPatterns(fetchedIndexPatterns);
setDataSources(fetchedDataSources);
setIsExternalDataSourcesEnabled(fetchedIsExternalDataSourcesEnabled);

if (dataSet) {
setSelectedDataSetState({
id: dataSet.id,
Expand Down Expand Up @@ -206,7 +208,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache();
if (status === DirectQueryLoadingStatus.SUCCESS) {
setExternalDataSources(
externalDataSourcesCache.externalDataSources.map((ds) => ({
externalDataSourcesCache.dataSources.map((ds) => ({
id: ds.dataSourceRef,
name: ds.name,
type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL,
Expand Down Expand Up @@ -462,25 +464,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
{
name: S3DataSourcesLabel,
panel: 4,
onClick: () => {
const externalDataSourcesCache = CatalogCacheManager.getExternalDataSourcesCache();
if (
(externalDataSourcesCache.status === CachedDataSourceStatus.Empty ||
externalDataSourcesCache.status === CachedDataSourceStatus.Failed) &&
!isCatalogCacheFetching(dataSourcesLoadStatus) &&
dataSources.length > 0
) {
startLoadingDataSources(dataSources.map((dataSource) => dataSource.id));
} else if (externalDataSourcesCache.status === CachedDataSourceStatus.Updated) {
setExternalDataSources(
externalDataSourcesCache.externalDataSources.map((ds) => ({
id: ds.dataSourceRef,
name: ds.name,
type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL,
}))
);
}
},
onClick: () => {},
},
]
: []),
Expand Down Expand Up @@ -527,7 +511,7 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
),
items: externalDataSources.map((dataSource) => ({
name: dataSource.name,
onClick: () => handleSelectExternalDataSource(dataSource),
onClick: async () => await handleSelectExternalDataSource(dataSource),
panel: 5,
})),
content: isCatalogCacheFetching(dataSourcesLoadStatus) && createLoadingSpinner(),
Expand Down Expand Up @@ -652,7 +636,6 @@ export const DataSetNavigator: React.FC<DataSetNavigatorProps> = ({
databasesLoadStatus,
cachedTables,
tablesLoadStatus,
startLoadingDataSources,
handleSelectedDataSet,
handleSelectedDataSource,
handleSelectedObject,
Expand Down
1 change: 0 additions & 1 deletion src/plugins/data/public/ui/dataset_navigator/index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,3 @@

export { DataSetNavigator, DataSetNavigatorProps } from './dataset_navigator';
export { createDataSetNavigator } from './create_dataset_navigator';
export { setAsyncSessionId, getAsyncSessionId, setAsyncSessionIdByObj } from './lib';
45 changes: 31 additions & 14 deletions src/plugins/data/public/ui/dataset_navigator/lib/catalog_cache/cache_loader.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import { useEffect, useRef, useState } from 'react';
import { HttpStart, NotificationsStart } from 'opensearch-dashboards/public';
import { ASYNC_POLLING_INTERVAL, SPARK_HIVE_TABLE_REGEX, SPARK_PARTITION_INFO } from '../constants';
import { ASYNC_QUERY, SPARK } from '../constants';
import {
AsyncPollingResult,
CachedColumn,
Expand All @@ -16,12 +16,12 @@ import {
DirectQueryLoadingStatus,
DirectQueryRequest,
} from '../types';
import { getAsyncSessionId, setAsyncSessionIdByObj } from '../utils/query_session_utils';
import { SIMPLE_DATA_SOURCE_TYPES, SimpleDataSource } from '../../../../../common';
import { addBackticksIfNeeded, combineSchemaAndDatarows, formatError } from '../utils/shared';
import { usePolling } from '../utils/use_polling';
import { SQLService } from '../requests/sql';
import { CatalogCacheManager } from './cache_manager';
import { fetchExternalDataSources } from '../utils';
import { getUiService } from '../../../../services';

export const updateDatabasesToCache = (
dataSourceName: string,
Expand Down Expand Up @@ -96,10 +96,9 @@ export const updateTablesToCache = (
);
return;
}

const combinedData = combineSchemaAndDatarows(pollingResult.schema, pollingResult.datarows);
const newTables = combinedData
.filter((row: any) => !SPARK_HIVE_TABLE_REGEX.test(row.information))
.filter((row: any) => !SPARK.HIVE_TABLE_REGEX.test(row.information))
.map((row: any) => ({
name: row.tableName,
}));
Expand Down Expand Up @@ -184,7 +183,7 @@ export const updateTableColumnsToCache = (

const tableColumns: CachedColumn[] = [];
for (const row of combinedData) {
if (row.col_name === SPARK_PARTITION_INFO) {
if (row.col_name === SPARK.PARTITION_INFO) {
break;
}
tableColumns.push({
Expand Down Expand Up @@ -282,7 +281,6 @@ export const useLoadToCache = (
http: HttpStart,
notifications: NotificationsStart
) => {
const sqlService = new SQLService(http);
const [currentDataSourceName, setCurrentDataSourceName] = useState('');
const [currentDatabaseName, setCurrentDatabaseName] = useState<string | undefined>('');
const [currentTableName, setCurrentTableName] = useState<string | undefined>('');
Expand All @@ -298,8 +296,13 @@ export const useLoadToCache = (
startPolling,
stopPolling: stopLoading,
} = usePolling<any, any>((params) => {
return sqlService.fetchWithJobId(params, dataSourceMDSClientId.current);
}, ASYNC_POLLING_INTERVAL);
return http.fetch(`../../api/enhancements/datasource/jobs`, {
query: {
id: dataSourceMDSClientId.current,
queryId: params.queryId,
},
});
}, ASYNC_QUERY.POLLING_INTERVAL);

const onLoadingFailed = () => {
setLoadStatus(DirectQueryLoadingStatus.FAILED);
Expand All @@ -319,6 +322,7 @@ export const useLoadToCache = (
databaseName,
tableName,
}: StartLoadingParams) => {
const uiService = getUiService();
setLoadStatus(DirectQueryLoadingStatus.SCHEDULED);
setCurrentDataSourceName(dataSourceName);
setCurrentDatabaseName(databaseName);
Expand All @@ -331,14 +335,19 @@ export const useLoadToCache = (
datasource: dataSourceName,
};

const sessionId = getAsyncSessionId(dataSourceName);
const sessionId = uiService.Settings.getUserQuerySessionId(dataSourceName);
if (sessionId) {
requestPayload = { ...requestPayload, sessionId };
}
await sqlService
.fetch(requestPayload, dataSourceMDSId)
await http
.post(`../../api/enhancements/datasource/jobs`, {
body: JSON.stringify(requestPayload),
query: {
id: dataSourceMDSClientId.current,
},
})
.then((result) => {
setAsyncSessionIdByObj(dataSourceName, result);
uiService.Settings.setUserQuerySessionIdByObj(dataSourceName, result);
if (result.queryId) {
startPolling({
queryId: result.queryId,
Expand Down Expand Up @@ -443,7 +452,9 @@ export const useLoadExternalDataSourcesToCache = (
DirectQueryLoadingStatus.INITIAL
);

const loadExternalDataSources = async (connectedClusters: string[]) => {
const loadExternalDataSources = async (
connectedClusters: SimpleDataSource[]
): Promise<SimpleDataSource[]> => {
setLoadStatus(DirectQueryLoadingStatus.SCHEDULED);
CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Empty);

Expand All @@ -452,13 +463,19 @@ export const useLoadExternalDataSourcesToCache = (
CatalogCacheManager.updateExternalDataSources(externalDataSources);
setLoadStatus(DirectQueryLoadingStatus.SUCCESS);
CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Updated);
return externalDataSources.map((dataSource) => ({
id: dataSource.dataSourceRef,
name: dataSource.name,
type: SIMPLE_DATA_SOURCE_TYPES.EXTERNAL,
}));
} catch (error) {
setLoadStatus(DirectQueryLoadingStatus.FAILED);
CatalogCacheManager.setExternalDataSourcesLoadingStatus(CachedDataSourceStatus.Failed);
notifications.toasts.addError(error, {
title: 'Failed to load external datasources',
});
}
return [];
};

return { loadStatus, loadExternalDataSources };
Expand Down
Loading

0 comments on commit 73e67f7

Please sign in to comment.