[Maps] support term joins for Elasticsearch document source with vector tile scaling

x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts

@@ -28,6 +28,7 @@ export type DataFilters = {
   timeslice?: Timeslice;
   zoom: number;
   isReadOnly: boolean;
+  joinKeyFilter?: Filter;
 };
 
 export type VectorSourceRequestMeta = DataFilters & {
@@ -77,7 +78,10 @@ export type VectorTileLayerMeta = {
 };
 
 // Partial because objects are justified downstream in constructors
-export type DataRequestMeta = Partial<
+export type DataRequestMeta = {
+  // request stop time in milliseconds since epoch
+  requestStopTime?: number;
+} & Partial<
   VectorSourceRequestMeta &
     VectorJoinSourceRequestMeta &
     VectorStyleRequestMeta &

x-pack/plugins/maps/public/actions/layer_actions.ts

@@ -13,6 +13,7 @@ import {
   createLayerInstance,
   getEditState,
   getLayerById,
+  getLayerDescriptor,
   getLayerList,
   getLayerListRaw,
   getMapColors,
@@ -55,11 +56,12 @@ import {
   LayerDescriptor,
   StyleDescriptor,
   TileMetaFeature,
+  VectorLayerDescriptor,
 } from '../../common/descriptor_types';
 import { ILayer } from '../classes/layers/layer';
 import { IVectorLayer } from '../classes/layers/vector_layer';
 import { OnSourceChangeArgs } from '../classes/sources/source';
-import { DRAW_MODE, LAYER_STYLE_TYPE, LAYER_TYPE } from '../../common/constants';
+import { DRAW_MODE, LAYER_STYLE_TYPE, LAYER_TYPE, SCALING_TYPES } from '../../common/constants';
 import { IVectorStyle } from '../classes/styles/vector/vector_style';
 import { notifyLicensedFeatureUsage } from '../licensed_features';
 import { IESAggField } from '../classes/fields/agg';
@@ -362,13 +364,17 @@ function updateSourcePropWithoutSync(
   value: unknown,
   newLayerType?: LAYER_TYPE
 ) {
-  return async (dispatch: ThunkDispatch<MapStoreState, void, AnyAction>) => {
+  return async (
+    dispatch: ThunkDispatch<MapStoreState, void, AnyAction>,
+    getState: () => MapStoreState
+  ) => {
     if (propName === 'metrics') {
       if (newLayerType) {
         throw new Error('May not change layer-type when modifying metrics source-property');
       }
       return await dispatch(updateMetricsProp(layerId, value));
     }
+
     dispatch({
       type: UPDATE_SOURCE_PROP,
       layerId,
@@ -378,6 +384,37 @@ function updateSourcePropWithoutSync(
     if (newLayerType) {
       dispatch(updateLayerType(layerId, newLayerType));
     }
+
+    if (propName === 'scalingType') {
+      // get joins from layer descriptor instead of layer.getJoins()
+      // 1) IVectorLayer implemenations my return empty array when descriptor has joins
+      // 2) getJoins returns instances and descriptors are needed.
+      const layerDescriptor = getLayerDescriptor(getState(), layerId) as VectorLayerDescriptor;
+      const joins = layerDescriptor.joins ? layerDescriptor.joins : [];
+      if (value === SCALING_TYPES.CLUSTERS && joins.length) {
+        // Blended scaling type does not support joins
+        // It is not possible to display join metrics when showing clusters
+        dispatch({
+          type: SET_JOINS,
+          layerId,
+          joins: [],
+        });
+        await dispatch(updateStyleProperties(layerId));
+      } else if (value === SCALING_TYPES.MVT) {
+        if (joins.length > 1) {
+          // Maplibre feature-state join uses promoteId and there is a limit to one promoteId
+          // Therefore, Vector tile scaling supports only one join
+          dispatch({
+            type: SET_JOINS,
+            layerId,
+            joins: [joins[0]],
+          });
+        }
+        // update style props regardless of updating joins
+        // Allow style to clean-up data driven style properties with join fields that do not support feature-state.
+        await dispatch(updateStyleProperties(layerId));
+      }
+    }
   };
 }
 
@@ -562,7 +599,7 @@ function removeLayerFromLayerList(layerId: string) {
   };
 }
 
-function updateStyleProperties(layerId: string, previousFields: IField[]) {
+function updateStyleProperties(layerId: string, previousFields?: IField[]) {
   return async (
     dispatch: ThunkDispatch<MapStoreState, void, AnyAction>,
     getState: () => MapStoreState
@@ -584,7 +621,7 @@ function updateStyleProperties(layerId: string, previousFields: IField[]) {
     const nextFields = await (targetLayer as IVectorLayer).getFields(); // take into account all fields, since labels can be driven by any field (source or join)
     const { hasChanges, nextStyleDescriptor } = await (
       style as IVectorStyle
-    ).getDescriptorWithUpdatedStyleProps(nextFields, previousFields, getMapColors(getState()));
+    ).getDescriptorWithUpdatedStyleProps(nextFields, getMapColors(getState()), previousFields);
     if (hasChanges && nextStyleDescriptor) {
       dispatch(updateLayerStyle(layerId, nextStyleDescriptor));
     }
@@ -626,9 +663,9 @@ export function updateLayerStyleForSelectedLayer(styleDescriptor: StyleDescripto
 export function setJoinsForLayer(layer: ILayer, joins: JoinDescriptor[]) {
   return async (dispatch: ThunkDispatch<MapStoreState, void, AnyAction>) => {
     const previousFields = await (layer as IVectorLayer).getFields();
-    await dispatch({
+    dispatch({
       type: SET_JOINS,
-      layer,
+      layerId: layer.getId(),
       joins,
     });
     await dispatch(updateStyleProperties(layer.getId(), previousFields));

x-pack/plugins/maps/public/classes/layers/vector_layer/bounds_data.ts

@@ -34,6 +34,7 @@ export async function syncBoundsData({
     timeFilters: dataFilters.timeFilters,
     timeslice: dataFilters.timeslice,
     filters: dataFilters.filters,
+    joinKeyFilter: dataFilters.joinKeyFilter,
     applyGlobalQuery: source.getApplyGlobalQuery(),
     applyGlobalTime: source.getApplyGlobalTime(),
   };

x-pack/plugins/maps/public/classes/layers/vector_layer/geojson_vector_layer/geojson_vector_layer.tsx

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import _ from 'lodash';
 import React from 'react';
 import { i18n } from '@kbn/i18n';
 import { EuiIcon } from '@elastic/eui';
@@ -13,35 +14,30 @@ import type { Map as MbMap, GeoJSONSource as MbGeoJSONSource } from '@kbn/mapbox
 import {
   EMPTY_FEATURE_COLLECTION,
   FEATURE_VISIBLE_PROPERTY_NAME,
-  FIELD_ORIGIN,
   LAYER_TYPE,
 } from '../../../../../common/constants';
 import {
   StyleMetaDescriptor,
   Timeslice,
-  VectorJoinSourceRequestMeta,
   VectorLayerDescriptor,
 } from '../../../../../common/descriptor_types';
-import { PropertiesMap } from '../../../../../common/elasticsearch_util';
 import { TimesliceMaskConfig } from '../../../util/mb_filter_expressions';
 import { DataRequestContext } from '../../../../actions';
 import { IVectorStyle, VectorStyle } from '../../../styles/vector/vector_style';
 import { ISource } from '../../../sources/source';
 import { IVectorSource } from '../../../sources/vector_source';
 import { AbstractLayer, LayerIcon } from '../../layer';
-import { InnerJoin } from '../../../joins/inner_join';
 import {
   AbstractVectorLayer,
   noResultsIcon,
   NO_RESULTS_ICON_AND_TOOLTIPCONTENT,
 } from '../vector_layer';
 import { DataRequestAbortError } from '../../../util/data_request';
-import { canSkipSourceUpdate } from '../../../util/can_skip_fetch';
 import { getFeatureCollectionBounds } from '../../../util/get_feature_collection_bounds';
 import { GEOJSON_FEATURE_ID_PROPERTY_NAME } from './assign_feature_ids';
 import { syncGeojsonSourceData } from './geojson_source_data';
-import { JoinState, performInnerJoins } from './perform_inner_joins';
-import { buildVectorRequestMeta } from '../../build_vector_request_meta';
+import { performInnerJoins } from './perform_inner_joins';
+import { pluckStyleMetaFromFeatures } from './pluck_style_meta_from_features';
 
 export class GeoJsonVectorLayer extends AbstractVectorLayer {
   static createDescriptor(
@@ -127,7 +123,12 @@ export class GeoJsonVectorLayer extends AbstractVectorLayer {
     if (!style || !sourceDataRequest) {
       return null;
     }
-    return await style.pluckStyleMetaFromSourceDataRequest(sourceDataRequest);
+
+    return await pluckStyleMetaFromFeatures(
+      _.get(sourceDataRequest.getData(), 'features', []),
+      await this.getSource().getSupportedShapeTypes(),
+      this.getCurrentStyle().getDynamicPropertiesArray()
+    );
   }
 
   _requiresPrevSourceCleanup(mbMap: MbMap) {
@@ -159,6 +160,13 @@ export class GeoJsonVectorLayer extends AbstractVectorLayer {
     this._setMbLinePolygonProperties(mbMap, undefined, timesliceMaskConfig);
   }
 
+  _getJoinFilterExpression(): unknown | undefined {
+    return this.hasJoins()
+      ? // Remove unjoined source features by filtering out features without GeoJSON feature.property[FEATURE_VISIBLE_PROPERTY_NAME] is true
+        ['==', ['get', FEATURE_VISIBLE_PROPERTY_NAME], true]
+      : undefined;
+  }
+
   _syncFeatureCollectionWithMb(mbMap: MbMap) {
     const mbGeoJSONSource = mbMap.getSource(this.getId()) as MbGeoJSONSource;
     const featureCollection = this._getSourceFeatureCollection();
@@ -260,118 +268,6 @@ export class GeoJsonVectorLayer extends AbstractVectorLayer {
     }
   }
 
-  async _syncJoin({
-    join,
-    startLoading,
-    stopLoading,
-    onLoadError,
-    registerCancelCallback,
-    dataFilters,
-    isForceRefresh,
-  }: { join: InnerJoin } & DataRequestContext): Promise<JoinState> {
-    const joinSource = join.getRightJoinSource();
-    const sourceDataId = join.getSourceDataRequestId();
-    const requestToken = Symbol(`layer-join-refresh:${this.getId()} - ${sourceDataId}`);
-
-    const joinRequestMeta: VectorJoinSourceRequestMeta = buildVectorRequestMeta(
-      joinSource,
-      joinSource.getFieldNames(),
-      dataFilters,
-      joinSource.getWhereQuery(),
-      isForceRefresh
-    ) as VectorJoinSourceRequestMeta;
-
-    const prevDataRequest = this.getDataRequest(sourceDataId);
-    const canSkipFetch = await canSkipSourceUpdate({
-      source: joinSource,
-      prevDataRequest,
-      nextRequestMeta: joinRequestMeta,
-      extentAware: false, // join-sources are term-aggs that are spatially unaware (e.g. ESTermSource/TableSource).
-      getUpdateDueToTimeslice: () => {
-        return true;
-      },
-    });
-
-    if (canSkipFetch) {
-      return {
-        dataHasChanged: false,
-        join,
-        propertiesMap: prevDataRequest?.getData() as PropertiesMap,
-      };
-    }
-
-    try {
-      startLoading(sourceDataId, requestToken, joinRequestMeta);
-      const leftSourceName = await this._source.getDisplayName();
-      const propertiesMap = await joinSource.getPropertiesMap(
-        joinRequestMeta,
-        leftSourceName,
-        join.getLeftField().getName(),
-        registerCancelCallback.bind(null, requestToken)
-      );
-      stopLoading(sourceDataId, requestToken, propertiesMap);
-      return {
-        dataHasChanged: true,
-        join,
-        propertiesMap,
-      };
-    } catch (error) {
-      if (!(error instanceof DataRequestAbortError)) {
-        onLoadError(sourceDataId, requestToken, `Join error: ${error.message}`);
-      }
-      throw error;
-    }
-  }
-
-  async _syncJoins(syncContext: DataRequestContext, style: IVectorStyle) {
-    const joinSyncs = this.getValidJoins().map(async (join) => {
-      await this._syncJoinStyleMeta(syncContext, join, style);
-      await this._syncJoinFormatters(syncContext, join, style);
-      return this._syncJoin({ join, ...syncContext });
-    });
-
-    return await Promise.all(joinSyncs);
-  }
-
-  async _syncJoinStyleMeta(syncContext: DataRequestContext, join: InnerJoin, style: IVectorStyle) {
-    const joinSource = join.getRightJoinSource();
-    return this._syncStyleMeta({
-      source: joinSource,
-      style,
-      sourceQuery: joinSource.getWhereQuery(),
-      dataRequestId: join.getSourceMetaDataRequestId(),
-      dynamicStyleProps: this.getCurrentStyle()
-        .getDynamicPropertiesArray()
-        .filter((dynamicStyleProp) => {
-          const matchingField = joinSource.getFieldByName(dynamicStyleProp.getFieldName());
-          return (
-            dynamicStyleProp.getFieldOrigin() === FIELD_ORIGIN.JOIN &&
-            !!matchingField &&
-            dynamicStyleProp.isFieldMetaEnabled()
-          );
-        }),
-      ...syncContext,
-    });
-  }
-
-  async _syncJoinFormatters(syncContext: DataRequestContext, join: InnerJoin, style: IVectorStyle) {
-    const joinSource = join.getRightJoinSource();
-    return this._syncFormatters({
-      source: joinSource,
-      dataRequestId: join.getSourceFormattersDataRequestId(),
-      fields: style
-        .getDynamicPropertiesArray()
-        .filter((dynamicStyleProp) => {
-          const matchingField = joinSource.getFieldByName(dynamicStyleProp.getFieldName());
-          return dynamicStyleProp.getFieldOrigin() === FIELD_ORIGIN.JOIN && !!matchingField;
-        })
-        .map((dynamicStyleProp) => {
-          return dynamicStyleProp.getField()!;
-        }),
-      ...syncContext,
-    });
-  }
-
   _getSourceFeatureCollection() {
     const sourceDataRequest = this.getSourceDataRequest();
     return sourceDataRequest ? (sourceDataRequest.getData() as FeatureCollection) : null;

x-pack/plugins/maps/public/classes/layers/vector_layer/geojson_vector_layer/perform_inner_joins.ts

@@ -9,20 +9,13 @@ import { FeatureCollection } from 'geojson';
 import { i18n } from '@kbn/i18n';
 import { FEATURE_VISIBLE_PROPERTY_NAME } from '../../../../../common/constants';
 import { DataRequestContext } from '../../../../actions';
-import { InnerJoin } from '../../../joins/inner_join';
-import { PropertiesMap } from '../../../../../common/elasticsearch_util';
+import { JoinState } from '../types';
 
 interface SourceResult {
   refreshed: boolean;
   featureCollection: FeatureCollection;
 }
 
-export interface JoinState {
-  dataHasChanged: boolean;
-  join: InnerJoin;
-  propertiesMap?: PropertiesMap;
-}
-
 export async function performInnerJoins(
   sourceResult: SourceResult,
   joinStates: JoinState[],