[WIP][discover] PPL and SQL use default internal search route

docs/_sidebar.md

@@ -167,6 +167,7 @@
     - [Opensearch dashboards.release notes 2.14.0](../release-notes/opensearch-dashboards.release-notes-2.14.0.md)
     - [Opensearch dashboards.release notes 2.15.0](../release-notes/opensearch-dashboards.release-notes-2.15.0.md)
     - [Opensearch dashboards.release notes 2.16.0](../release-notes/opensearch-dashboards.release-notes-2.16.0.md)
+    - [Opensearch dashboards.release notes 2.17.0](../release-notes/opensearch-dashboards.release-notes-2.17.0.md)
     - [Opensearch dashboards.release notes 2.2.0](../release-notes/opensearch-dashboards.release-notes-2.2.0.md)
     - [Opensearch dashboards.release notes 2.2.1](../release-notes/opensearch-dashboards.release-notes-2.2.1.md)
     - [Opensearch dashboards.release notes 2.3.0](../release-notes/opensearch-dashboards.release-notes-2.3.0.md)

src/plugins/data/common/search/search_source/fetch/get_search_params.ts

@@ -53,19 +53,16 @@
   searchRequest: SearchRequest,
   dependencies: {
     getConfig: GetConfigFn;
-    getDataFrame: GetDataFrameFn;
   }
 ): ISearchRequestParams {
-  const { getConfig, getDataFrame } = dependencies;
+  const { getConfig } = dependencies;
   const searchParams = getSearchParams(getConfig);
-  const dataFrame = getDataFrame();
   const indexTitle = searchRequest.index.title || searchRequest.index;
 
   return {
     index: indexTitle,
     body: {
       ...searchRequest.body,
-      ...(dataFrame && dataFrame?.name === indexTitle ? { df: dataFrame } : {}),
     },
     ...searchParams,
   };

src/plugins/data/common/search/search_source/search_source.ts

@@ -424,30 +424,25 @@
   private async fetchExternalSearch(searchRequest: SearchRequest, options: ISearchOptions) {
     const { search, getConfig, onResponse } = this.dependencies;
 
-    const currentDataframe = this.getDataFrame();
-    if (!currentDataframe || currentDataframe.name !== searchRequest.index?.id) {
-      await this.createDataFrame(searchRequest);
-    }
-
     const params = getExternalSearchParamsFromRequest(searchRequest, {
       getConfig,
-      getDataFrame: this.getDataFrame.bind(this),
     });
 
     return search({ params }, options).then(async (response: any) => {
-      if (response.hasOwnProperty('type')) {
-        if ((response as IDataFrameResponse).type === DATA_FRAME_TYPES.DEFAULT) {
-          const dataFrameResponse = response as IDataFrameResponse;
+      const rawResponse = response.rawResponse;
+      if (rawResponse.hasOwnProperty('type')) {
+        if ((rawResponse as IDataFrameResponse).type === DATA_FRAME_TYPES.DEFAULT) {
+          const dataFrameResponse = rawResponse as IDataFrameResponse;
           await this.setDataFrame(dataFrameResponse.body as IDataFrame);
-          return onResponse(searchRequest, convertResult(response as IDataFrameResponse));
+          return onResponse(searchRequest, convertResult(rawResponse as IDataFrameResponse));
         }
-        if ((response as IDataFrameResponse).type === DATA_FRAME_TYPES.POLLING) {
-          const dataFrameResponse = response as IDataFrameResponse;
+        if ((rawResponse as IDataFrameResponse).type === DATA_FRAME_TYPES.POLLING) {
+          const dataFrameResponse = rawResponse as IDataFrameResponse;
           await this.setDataFrame(dataFrameResponse.body as IDataFrame);
-          return onResponse(searchRequest, convertResult(response as IDataFrameResponse));
+          return onResponse(searchRequest, convertResult(rawResponse as IDataFrameResponse));
         }
-        if ((response as IDataFrameResponse).type === DATA_FRAME_TYPES.ERROR) {
-          const dataFrameError = response as IDataFrameError;
+        if ((rawResponse as IDataFrameResponse).type === DATA_FRAME_TYPES.ERROR) {
+          const dataFrameError = rawResponse as IDataFrameError;
           throw new RequestFailure(null, dataFrameError);
         }
       }

src/plugins/data/server/search/routes/shim_hits_total.ts

@@ -38,7 +38,10 @@
  * @internal
  */
 export function shimHitsTotal(response: SearchResponse<any>) {
-  const total = (response.hits?.total as any)?.value ?? response.hits?.total;
+  if (!response.hits) {
+    return response;
+  }
+  const total = (response.hits.total as any)?.value ?? response.hits.total;
   const hits = { ...response.hits, total };
   return { ...response, hits };
 }

src/plugins/query_enhancements/common/constants.ts

@@ -20,10 +20,6 @@ export const SEARCH_STRATEGY = {
 };
 
 export const API = {
-  SEARCH: `${BASE_API}/search`,
-  PPL_SEARCH: `${BASE_API}/search/${SEARCH_STRATEGY.PPL}`,
-  SQL_SEARCH: `${BASE_API}/search/${SEARCH_STRATEGY.SQL}`,
-  SQL_ASYNC_SEARCH: `${BASE_API}/search/${SEARCH_STRATEGY.SQL_ASYNC}`,
   QUERY_ASSIST: {
     LANGUAGES: `${BASE_API}/assist/languages`,
     GENERATE: `${BASE_API}/assist/generate`,

src/plugins/query_enhancements/common/types.ts

@@ -23,7 +23,7 @@ export interface QueryStatusConfig {
 
 export interface EnhancedFetchContext {
   http: CoreSetup['http'];
-  path: string;
+  strategy?: string;
   signal?: AbortSignal;
 }
 

src/plugins/query_enhancements/common/utils.ts

@@ -3,15 +3,11 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { Query } from 'src/plugins/data/common';
-import { from, throwError, timer } from 'rxjs';
+import { IOpenSearchDashboardsSearchRequest, Query } from 'src/plugins/data/common';
+import { from, timer } from 'rxjs';
 import { filter, mergeMap, take, takeWhile } from 'rxjs/operators';
-import {
-  EnhancedFetchContext,
-  QueryAggConfig,
-  QueryStatusConfig,
-  QueryStatusOptions,
-} from './types';
+import { trimEnd } from 'lodash';
+import { EnhancedFetchContext, QueryStatusConfig, QueryStatusOptions } from './types';
 
 export const formatDate = (dateString: string) => {
   const date = new Date(dateString);
@@ -37,6 +33,12 @@
   }));
 };
 
+export const getQuery = (searchRequest: IOpenSearchDashboardsSearchRequest): Query => {
+  return (
+    searchRequest.params?.body?.query?.queries[1] ?? searchRequest.params?.body?.query?.queries[0]
+  );
+};
+
 export const removeKeyword = (queryString: string | undefined) => {
   return queryString?.replace(new RegExp('.keyword'), '') ?? '';
 };
@@ -47,9 +49,14 @@
   throw error;
 };
 
-export const fetch = (context: EnhancedFetchContext, query: Query, aggConfig?: QueryAggConfig) => {
-  const { http, path, signal } = context;
-  const body = JSON.stringify({ query: { ...query, format: 'jdbc' }, aggConfig });
+export const fetch = (
+  context: EnhancedFetchContext,
+  request: IOpenSearchDashboardsSearchRequest
+) => {
+  const { http, strategy, signal } = context;
+  const { id, ...searchRequest } = request;
+  const path = trimEnd(`/internal/search/${strategy}/${id || ''}`, '/');
+  const body = JSON.stringify(searchRequest);
   return from(
     http.fetch({
       method: 'POST',

src/plugins/query_enhancements/public/datasets/s3_type.ts

@@ -4,6 +4,7 @@
  */
 
 import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public';
+import { trimEnd } from 'lodash';
 import {
   DATA_STRUCTURE_META_TYPES,
   DEFAULT_DATA,
@@ -14,7 +15,7 @@
   DatasetField,
 } from '../../../data/common';
 import { DatasetTypeConfig, IDataPluginServices } from '../../../data/public';
-import { DATASET, handleQueryStatus } from '../../common';
+import { API, DATASET, handleQueryStatus } from '../../common';
 import S3_ICON from '../assets/s3_mark.svg';
 
 export const s3TypeConfig: DatasetTypeConfig = {
@@ -115,7 +116,7 @@
   try {
     const response = await handleQueryStatus({
       fetchStatus: () =>
-        http.fetch('../../api/enhancements/datasource/jobs', {
+        http.fetch(trimEnd(API.DATA_SOURCE.ASYNC_JOBS), {
           query: {
             id: dataSource?.id,
             queryId: meta.queryId,
@@ -172,7 +173,7 @@
   dataSource: DataStructure
 ): Promise<DataStructure[]> => {
   const query = (dataSource.meta as DataStructureCustomMeta).query;
-  const response = await http.fetch(`../../api/enhancements/datasource/external`, {
+  const response = await http.fetch(trimEnd(API.DATA_SOURCE.EXTERNAL), {
     query,
   });
 
@@ -193,7 +194,7 @@
   const dataSource = path.find((ds) => ds.type === 'DATA_SOURCE');
   const connection = path[path.length - 1];
   const meta = connection.meta as DataStructureCustomMeta;
-  const response = await http.post(`../../api/enhancements/datasource/jobs`, {
+  const response = await http.post(trimEnd(API.DATA_SOURCE.ASYNC_JOBS), {
     body: JSON.stringify({
       lang: 'sql',
       query: `SHOW DATABASES in ${connection.title}`,
@@ -215,7 +216,7 @@
   const connection = path.find((ds) => ds.type === 'CONNECTION');
   const sessionId = (connection?.meta as DataStructureCustomMeta).sessionId;
   const database = path[path.length - 1];
-  const response = await http.post(`../../api/enhancements/datasource/jobs`, {
+  const response = await http.post(trimEnd(API.DATA_SOURCE.ASYNC_JOBS), {
     body: JSON.stringify({
       lang: 'sql',
       query: `SHOW TABLES in ${database.title}`,

src/plugins/query_enhancements/public/search/ppl_search_interceptor.ts

@@ -3,8 +3,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { trimEnd } from 'lodash';
-import { Observable } from 'rxjs';
+import { Observable, throwError } from 'rxjs';
+import { catchError } from 'rxjs/operators';
 import { formatTimePickerDate, Query } from '../../../data/common';
 import {
   DataPublicPluginStart,
@@ -17,10 +17,10 @@ import {
 import {
   formatDate,
   SEARCH_STRATEGY,
-  API,
+  QueryAggConfig,
+  getQuery,
   EnhancedFetchContext,
   fetch,
-  QueryAggConfig,
 } from '../../common';
 import { QueryEnhancementsPluginStartDependencies } from '../types';
 
@@ -42,24 +42,46 @@ export class PPLSearchInterceptor extends SearchInterceptor {
     signal?: AbortSignal,
     strategy?: string
   ): Observable<IOpenSearchDashboardsSearchResponse> {
-    const { id, ...searchRequest } = request;
     const context: EnhancedFetchContext = {
       http: this.deps.http,
-      path: trimEnd(API.PPL_SEARCH),
+      strategy,
       signal,
     };
 
-    const query = this.buildQuery();
-
-    return fetch(context, query, this.getAggConfig(searchRequest, query));
+    return fetch(context, request).pipe(
+      catchError((error) => {
+        return throwError(error);
+      })
+    );
   }
 
-  public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) {
-    return this.runSearch(request, options.abortSignal, SEARCH_STRATEGY.PPL);
+  public search(
+    request: IOpenSearchDashboardsSearchRequest,
+    options: ISearchOptions
+  ): Observable<IOpenSearchDashboardsSearchResponse> {
+    const query = this.buildQuery(request);
+    const aggConfig = this.getAggConfig(request, query);
+
+    // Create a new request object with the updated query and aggConfig
+    const enhancedRequest: IOpenSearchDashboardsSearchRequest = {
+      ...request,
+      params: {
+        ...request.params,
+        body: {
+          ...request.params.body,
+          query: {
+            queries: [getQuery(request), query],
+          },
+          aggConfig,
+        },
+      },
+    };
+
+    return this.runSearch(enhancedRequest, options.abortSignal!, SEARCH_STRATEGY.PPL);
   }
 
-  private buildQuery() {
-    const query: Query = this.queryService.queryString.getQuery();
+  private buildQuery(request: IOpenSearchDashboardsSearchRequest) {
+    const query: Query = getQuery(request);
     const dataset = query.dataset;
     if (!dataset || !dataset.timeFieldName) return query;
     const timeFilter = this.getTimeFilter(dataset.timeFieldName);

src/plugins/query_enhancements/public/search/sql_search_interceptor.ts

@@ -3,20 +3,19 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { trimEnd } from 'lodash';
+import { CoreStart } from 'opensearch-dashboards/public';
 import { Observable, throwError } from 'rxjs';
 import { catchError, tap } from 'rxjs/operators';
-import { CoreStart } from 'opensearch-dashboards/public';
 import {
   DataPublicPluginStart,
   IOpenSearchDashboardsSearchRequest,
-  IOpenSearchDashboardsSearchResponse,
   ISearchOptions,
   SearchInterceptor,
   SearchInterceptorDeps,
 } from '../../../data/public';
-import { API, DATASET, EnhancedFetchContext, SEARCH_STRATEGY, fetch } from '../../common';
+import { DATASET, EnhancedFetchContext, SEARCH_STRATEGY, fetch } from '../../common';
 import { QueryEnhancementsPluginStartDependencies } from '../types';
+import { IOpenSearchDashboardsSearchResponse } from '../../../data/common';
 
 export class SQLSearchInterceptor extends SearchInterceptor {
   protected queryService!: DataPublicPluginStart['query'];
@@ -39,12 +38,12 @@ export class SQLSearchInterceptor extends SearchInterceptor {
     const isAsync = strategy === SEARCH_STRATEGY.SQL_ASYNC;
     const context: EnhancedFetchContext = {
       http: this.deps.http,
-      path: trimEnd(isAsync ? API.SQL_ASYNC_SEARCH : API.SQL_SEARCH),
+      strategy,
       signal,
     };
 
     if (isAsync) this.notifications.toasts.add('Fetching data...');
-    return fetch(context, this.queryService.queryString.getQuery()).pipe(
+    return fetch(context, request).pipe(
       tap(() => isAsync && this.notifications.toasts.addSuccess('Fetch complete...')),
       catchError((error) => {
         return throwError(error);
@@ -55,6 +54,6 @@ export class SQLSearchInterceptor extends SearchInterceptor {
   public search(request: IOpenSearchDashboardsSearchRequest, options: ISearchOptions) {
     const dataset = this.queryService.queryString.getQuery().dataset;
     const strategy = dataset?.type === DATASET.S3 ? SEARCH_STRATEGY.SQL_ASYNC : SEARCH_STRATEGY.SQL;
-    return this.runSearch(request, options.abortSignal, strategy);
+    return this.runSearch(request, options.abortSignal!, strategy);
   }
 }

src/plugins/query_enhancements/server/plugin.ts

@@ -82,11 +82,7 @@ export class QueryEnhancementsPlugin
       dataSourceEnabled: !!dataSource,
     }));
 
-    defineRoutes(this.logger, router, client, {
-      ppl: pplSearchStrategy,
-      sql: sqlSearchStrategy,
-      sqlasync: sqlAsyncSearchStrategy,
-    });
+    defineRoutes(router, client);
 
     this.logger.info('queryEnhancements: Setup complete');
     return {};