[BACKPORT] Backport 2.x to main

.github/workflows/dashboards-observability-test-and-build-workflow.yml

@@ -20,6 +20,11 @@ jobs:
       - name: Checkout Plugin
         uses: actions/checkout@v1
 
+      # Enable longer filenames for windows
+      - name: Enable longer filenames
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: git config --system core.longpaths true
+
       - name: Checkout OpenSearch Dashboards
         uses: actions/checkout@v2
         with:

dashboards-observability/.cypress/integration/VisualizationCharts/10_scatter_chart.spec.js

@@ -9,7 +9,7 @@ import {
     TEST_QUERIES,
     querySearch,
     landOnEventVisualizations
-  } from '../../utils/event_constants';
+  } from '../../utils/event_analytics/constants';
 
   const numberOfWindow = 4;
   const legendSize = 20;

dashboards-observability/.cypress/integration/VisualizationCharts/11_horizontalBar_chart.spec.js

@@ -9,7 +9,7 @@ import {
     TEST_QUERIES,
     querySearch,
     landOnEventVisualizations,
-  } from '../../utils/event_constants';
+  } from '../../utils/event_analytics/constants';
 
   const numberOfWindow = 4;
   const labelSize = 20;

dashboards-observability/.cypress/integration/VisualizationCharts/12_boxPlot_chart.spec.js

@@ -9,7 +9,7 @@ import {
   TEST_QUERIES,
   querySearch,
   landOnEventVisualizations,
-} from '../../utils/event_constants';
+} from '../../utils/event_analytics/constants';
 
 const numberOfWindow = 4;
 const labelSize = 20;

dashboards-observability/.cypress/integration/VisualizationCharts/13_logsView.spec.js

@@ -11,7 +11,7 @@ import {
   landOnEventVisualizations,
   saveVisualizationAndVerify,
   deleteVisualization,
-} from '../../utils/event_constants';
+} from '../../utils/event_analytics/constants';
 
 const renderLogsView = () => {
   landOnEventVisualizations();

dashboards-observability/.cypress/integration/VisualizationCharts/13_stats_chart.spec.js

@@ -9,7 +9,7 @@ import {
   TEST_QUERIES,
   querySearch,
   landOnEventVisualizations
-} from '../../utils/event_constants';
+} from '../../utils/event_analytics/constants';
 
 const numberOfWindow = 4;
 const metricsPrecisionUpdated = 2;

dashboards-observability/.cypress/integration/VisualizationCharts/7_pie_chart.spec.js

@@ -12,7 +12,7 @@ import {
     renderDataConfig,
     saveVisualizationAndVerify,
     deleteVisualization,
-  } from '../../utils/event_constants';
+  } from '../../utils/event_analytics/constants';
 
   const renderPieChart = () => {
     landOnEventVisualizations();

dashboards-observability/.cypress/integration/VisualizationCharts/9_coordinateMap_chart.spec.js

@@ -8,7 +8,7 @@ import {
     TEST_QUERIES,
     querySearch,
     landOnEventVisualizations
-} from '../../utils/event_constants';
+} from '../../utils/event_analytics/constants';
 
 const numberOfWindow = 4;
 const labelSize = 20;

dashboards-observability/.cypress/utils/constants.js

@@ -4,6 +4,7 @@
  */
 
 export const delay = 1500;
+export const COMMAND_TIMEOUT_LONG = 10000;
 
 // trace analytics
 export const TRACE_ID = '8832ed6abbb2a83516461960c89af49d';

dashboards-observability/.cypress/utils/event_constants.js → dashboards-observability/.cypress/utils/event_analytics/constants.js

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { supressResizeObserverIssue } from './constants';
+import { COMMAND_TIMEOUT_LONG, supressResizeObserverIssue, TIMEOUT_DELAY } from '../constants';
 
 export const delay = 1000;
 export const YEAR_TO_DATE_DOM_ID = '[data-test-subj="superDatePickerCommonlyUsed_Year_to date"]';
@@ -72,10 +72,13 @@ export const aggregationValues = [
 ];
 
 export const querySearch = (query, rangeSelected) => {
-  cy.get('[data-test-subj="searchAutocompleteTextArea"]').type(query);
+  cy.get('[data-test-subj="searchAutocompleteTextArea"]', { timeout: COMMAND_TIMEOUT_LONG })
+    .clear()
+    .focus()
+    .type(query, { delay: 50 });
   cy.get('[data-test-subj="superDatePickerToggleQuickMenuButton"]').click();
-  cy.wait(delay);
   cy.get(rangeSelected).click();
+  cy.wait(delay * 2);
   cy.get('[data-test-subj="superDatePickerApplyTimeButton"]').contains('Refresh').click();
 };
 
@@ -88,6 +91,7 @@ export const landOnEventExplorer = () => {
   cy.visit(
     `${Cypress.env('opensearchDashboards')}/app/observability-dashboards#/event_analytics/explorer`
   );
+  supressResizeObserverIssue();
   cy.wait(delay);
 };
 

dashboards-observability/.cypress/utils/event_analytics/helpers.js

@@ -0,0 +1,12 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { COMMAND_TIMEOUT_LONG } from '../constants';
+
+export const clearQuerySearchBoxText = (testSubjectName) => {
+  cy.get(`[data-test-subj="${testSubjectName}"]`, {
+    timeout: COMMAND_TIMEOUT_LONG,
+  }).clear({ force: true });
+};

dashboards-observability/common/constants/explorer.ts

@@ -15,6 +15,9 @@ export const RAW_QUERY = 'rawQuery';
 export const FINAL_QUERY = 'finalQuery';
 export const SELECTED_DATE_RANGE = 'selectedDateRange';
 export const INDEX = 'index';
+export const SELECTED_PATTERN_FIELD = 'selectedPatternField';
+export const PATTERN_REGEX = 'patternRegex';
+export const FILTERED_PATTERN = 'filteredPattern';
 export const SELECTED_TIMESTAMP = 'selectedTimestamp';
 export const SELECTED_FIELDS = 'selectedFields';
 export const UNSELECTED_FIELDS = 'unselectedFields';
@@ -29,7 +32,7 @@ export const TAB_CHART_ID_TXT_PFX = 'main-content-vis-';
 export const TAB_EVENT_ID = 'main-content-events';
 export const TAB_CHART_ID = 'main-content-vis';
 export const HAS_SAVED_TIMESTAMP = 'hasSavedTimestamp';
-export const FILTER_OPTIONS = ['Visualization', 'Query'];
+export const FILTER_OPTIONS = ['Visualization', 'Query', 'Metric'];
 export const SAVED_QUERY = 'savedQuery';
 export const SAVED_VISUALIZATION = 'savedVisualization';
 export const SAVED_OBJECT_ID = 'savedObjectId';
@@ -78,11 +81,16 @@ export const REDUX_EXPL_SLICE_FIELDS = 'fields';
 export const REDUX_EXPL_SLICE_QUERY_TABS = 'queryTabs';
 export const REDUX_EXPL_SLICE_VISUALIZATION = 'explorerVisualization';
 export const REDUX_EXPL_SLICE_COUNT_DISTRIBUTION = 'countDistributionVisualization';
+export const REDUX_EXPL_SLICE_PATTERNS = 'patterns';
 export const PLOTLY_GAUGE_COLUMN_NUMBER = 4;
 export const APP_ANALYTICS_TAB_ID_REGEX = /application-analytics-tab.+/;
 export const DEFAULT_AVAILABILITY_QUERY = 'stats count() by span( timestamp, 1h )';
+export const PPL_DEFAULT_PATTERN_REGEX_FILETER = '[a-zA-Z\\d]';
+// Greedily matches the longest substring for example (patterns referer | patterns pattern='[0-9]' message | where ...) used to modify the query for patterns table
+export const PATTERNS_REGEX = /\|\s*patterns.+?\|.*\s*where\s+patterns_field\s*\=\s*'[^a-zA-Z0-9]+'/;
+// Used to extract the initial pattern applied
+export const PATTERNS_EXTRACTOR_REGEX = /patterns\s+(?<pattern>\S+)/;
 export const ADD_BUTTON_TEXT = '+ Add color theme';
-export const ADD_SERIES_POSITION_TEXT = '+ Add label position';
 export const NUMBER_INPUT_MIN_LIMIT = 1;
 
 export const VIZ_CONTAIN_XY_AXIS = [
@@ -281,7 +289,13 @@ export const DATA_CONFIG_HINTS_INFO = {
   [BREAKDOWNS]:
     "Defines how each series is broken down. Breakdowns are 'by' clauses that subdivide the existing series.",
 };
-export const SERIES_POSITION_OPTIONS = [
-  { id: htmlIdGenerator('ct')(), label: 'Left', side: 'left' },
-  { id: htmlIdGenerator('ct')(), label: 'Right', side: 'right' },
-];
+
+// Metrics constants
+export const UNITS_OF_MEASURE = [
+  'seconds (s)',
+  'hours (h)',
+  'celsius (C)',
+  'farenheit (F)',
+  'meters (m)',
+  'kilometers (k)',
+]

dashboards-observability/common/constants/metrics.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// requests constants
+export const VISUALIZATION = 'viz';
+export const SAVED_VISUALIZATION = 'savedVisualization';
+export const PPL_DATASOURCES_REQUEST =
+  'show datasources | where CONNECTOR_TYPE="PROMETHEUS" | fields DATASOURCE_NAME';
+
+// redux
+export const REDUX_SLICE_METRICS = 'metrics';
+
+export const resolutionOptions = [
+  { value: 's', text: 'seconds' },
+  { value: 'm', text: 'minutes' },
+  { value: 'h', text: 'hours' },
+  { value: 'd', text: 'days' },
+  // { value: 'M', text: 'Months' }, // commenting it here as prometheus doesn't have support
+  // { value: 'q', text: 'quarters' },
+  { value: 'y', text: 'years' },
+];
+
+export const DEFAULT_METRIC_HEIGHT = 2;
+export const DEFAULT_METRIC_WIDTH = 12;

dashboards-observability/common/constants/shared.ts

@@ -31,6 +31,8 @@ export const observabilityPluginOrder = 6000;
 export const SQL_DOCUMENTATION_URL = 'https://opensearch.org/docs/latest/search-plugins/sql/index/';
 export const PPL_DOCUMENTATION_URL =
   'https://opensearch.org/docs/latest/observability-plugin/ppl/commands/';
+export const PPL_PATTERNS_DOCUMENTATION_URL =
+  'https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/cmd/patterns.rst#description';
 export const UI_DATE_FORMAT = 'MM/DD/YYYY hh:mm A';
 export const PPL_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss.SSSSSS';
 export const SPAN_REGEX = /span/;

dashboards-observability/common/query_manager/antlr/grammar/OpenSearchPPLLexer.g4

@@ -11,6 +11,8 @@ channels { WHITESPACE, ERRORCHANNEL }
 
 // COMMAND KEYWORDS
 SEARCH:                             'SEARCH';
+DESCRIBE:                           'DESCRIBE';
+SHOW:                               'SHOW';
 FROM:                               'FROM';
 WHERE:                              'WHERE';
 FIELDS:                             'FIELDS';
@@ -23,8 +25,16 @@ HEAD:                               'HEAD';
 TOP:                                'TOP';
 RARE:                               'RARE';
 PARSE:                              'PARSE';
+METHOD:                             'METHOD';
+REGEX:                              'REGEX';
+PUNCT:                              'PUNCT';
+GROK:                               'GROK';
+PATTERN:                            'PATTERN';
+PATTERNS:                           'PATTERNS';
+NEW_FIELD:                          'NEW_FIELD';
 KMEANS:                             'KMEANS';
 AD:                                 'AD';
+ML:                                 'ML';
 
 // COMMAND ASSIST KEYWORDS
 AS:                                 'AS';
@@ -33,6 +43,7 @@ SOURCE:                             'SOURCE';
 INDEX:                              'INDEX';
 D:                                  'D';
 DESC:                               'DESC';
+CATALOGS:                           'CATALOGS';
 
 // CLAUSE KEYWORDS
 SORTBY:                             'SORTBY';
@@ -59,6 +70,7 @@ SAMPLE_SIZE:                        'SAMPLE_SIZE';
 OUTPUT_AFTER:                       'OUTPUT_AFTER';
 TIME_DECAY:                         'TIME_DECAY';
 ANOMALY_RATE:                       'ANOMALY_RATE';
+CATEGORY_FIELD:                     'CATEGORY_FIELD';
 TIME_FIELD:                         'TIME_FIELD';
 TIME_ZONE:                          'TIME_ZONE';
 TRAINING_DATA_SIZE:                 'TRAINING_DATA_SIZE';
@@ -101,6 +113,7 @@ DAY_SECOND:                         'DAY_SECOND';
 DAY_MINUTE:                         'DAY_MINUTE';
 DAY_HOUR:                           'DAY_HOUR';
 YEAR_MONTH:                         'YEAR_MONTH';
+CONVERT_TZ:                         'CONVERT_TZ';
 
 // DATASET TYPES
 DATAMODEL:                          'DATAMODEL';
@@ -168,6 +181,7 @@ VAR_POP:                            'VAR_POP';
 STDDEV_SAMP:                        'STDDEV_SAMP';
 STDDEV_POP:                         'STDDEV_POP';
 PERCENTILE:                         'PERCENTILE';
+TAKE:                               'TAKE';
 FIRST:                              'FIRST';
 LAST:                               'LAST';
 LIST:                               'LIST';
@@ -222,21 +236,39 @@ TAN:                                'TAN';
 
 // DATE AND TIME FUNCTIONS
 ADDDATE:                            'ADDDATE';
+CURDATE:                            'CURDATE';
+CURRENT_DATE:                       'CURRENT_DATE';
+CURRENT_TIME:                       'CURRENT_TIME';
+CURRENT_TIMESTAMP:                  'CURRENT_TIMESTAMP';
+CURTIME:                            'CURTIME';
 DATE:                               'DATE';
 DATE_ADD:                           'DATE_ADD';
+DATE_FORMAT:                        'DATE_FORMAT';
 DATE_SUB:                           'DATE_SUB';
+DAYNAME:                            'DAYNAME';
 DAYOFMONTH:                         'DAYOFMONTH';
 DAYOFWEEK:                          'DAYOFWEEK';
 DAYOFYEAR:                          'DAYOFYEAR';
-DAYNAME:                            'DAYNAME';
 FROM_DAYS:                          'FROM_DAYS';
+LOCALTIME:                          'LOCALTIME';
+LOCALTIMESTAMP:                     'LOCALTIMESTAMP';
+FROM_UNIXTIME:                      'FROM_UNIXTIME';
+MAKEDATE:                           'MAKEDATE';
+MAKETIME:                           'MAKETIME';
 MONTHNAME:                          'MONTHNAME';
+NOW:                                'NOW';
+PERIOD_ADD:                         'PERIOD_ADD';
+PERIOD_DIFF:                        'PERIOD_DIFF';
 SUBDATE:                            'SUBDATE';
+SYSDATE:                            'SYSDATE';
 TIME:                               'TIME';
 TIME_TO_SEC:                        'TIME_TO_SEC';
 TIMESTAMP:                          'TIMESTAMP';
-DATE_FORMAT:                        'DATE_FORMAT';
 TO_DAYS:                            'TO_DAYS';
+UTC_DATE:                           'UTC_DATE';
+UTC_TIME:                           'UTC_TIME';
+UTC_TIMESTAMP:                      'UTC_TIMESTAMP';
+UNIX_TIMESTAMP:                     'UNIX_TIMESTAMP';
 
 // TEXT FUNCTIONS
 SUBSTR:                             'SUBSTR';
@@ -267,11 +299,16 @@ ISNOTNULL:                          'ISNOTNULL';
 IFNULL:                             'IFNULL';
 NULLIF:                             'NULLIF';
 IF:                                 'IF';
+TYPEOF:                             'TYPEOF';
 
 // RELEVANCE FUNCTIONS AND PARAMETERS
 MATCH:                              'MATCH';
 MATCH_PHRASE:                       'MATCH_PHRASE';
+MATCH_PHRASE_PREFIX:                'MATCH_PHRASE_PREFIX';
+MATCH_BOOL_PREFIX:                  'MATCH_BOOL_PREFIX';
 SIMPLE_QUERY_STRING:                'SIMPLE_QUERY_STRING';
+MULTI_MATCH:                        'MULTI_MATCH';
+QUERY_STRING:                       'QUERY_STRING';
 
 ALLOW_LEADING_WILDCARD:             'ALLOW_LEADING_WILDCARD';
 ANALYZE_WILDCARD:                   'ANALYZE_WILDCARD';
@@ -282,6 +319,7 @@ CUTOFF_FREQUENCY:                   'CUTOFF_FREQUENCY';
 DEFAULT_FIELD:                      'DEFAULT_FIELD';
 DEFAULT_OPERATOR:                   'DEFAULT_OPERATOR';
 ENABLE_POSITION_INCREMENTS:         'ENABLE_POSITION_INCREMENTS';
+ESCAPE:                             'ESCAPE';
 FLAGS:                              'FLAGS';
 FUZZY_MAX_EXPANSIONS:               'FUZZY_MAX_EXPANSIONS';
 FUZZY_PREFIX_LENGTH:                'FUZZY_PREFIX_LENGTH';