[MQL] support enhancing language selector (#6613)

Enable with `data.enhancements.enabled: true` Allows for enhancing the data plugin UI service and search service. #### Remaining work * Address issue with time range being invalid if previous state successfully queried and set it with a time range format that is invalid for the new query language * For example, DQL with quick time range (4 weeks to now), get results. Switch to PPL, even though PPL has a default time range enhancement. The props date range saved in the app state takes priority and sets the time range to quick range causing an error. I can still modify the time range and get a successful query but it will first fail until the user updates it to a non quick time range. * Add tests * Disable for plugins that do not support the functionality * By default index patterns are created with a unique ID. However, it can be enabled to create an index pattern with a custom ID that matches the name of the index pattern (which in turn maps to indices). * For seamless integration, the temp data frame would need to check if the index pattern that maps to the data frame name. And get it's id. * This means that dashboards with visualizations that were created with an index pattern unique ID still require the existing index pattern to exist in memory. ### Issues Resolved closes #6639 closes #6311 partially resolves: #5504 * add error data frame Signed-off-by: Paul Sebastian <paulstn@amazon.com> move language to left, some styling and disable per app name Signed-off-by: Kawika Avilla <kavilla414@gmail.com> --------- Signed-off-by: Kawika Avilla <kavilla414@gmail.com> Signed-off-by: Paul Sebastian <paulstn@amazon.com> Co-authored-by: Paul Sebastian <paulstn@amazon.com> Co-authored-by: opensearch-changeset-bot[bot] <154024398+opensearch-changeset-bot[bot]@users.noreply.github.com>
opensearch-project · May 9, 2024 · 7d9b3cd · 7d9b3cd
1 parent 884dbb3
commit 7d9b3cd
Show file tree

Hide file tree

Showing 92 changed files with 2,484 additions and 255 deletions.
diff --git a/changelogs/fragments/6613.yml b/changelogs/fragments/6613.yml
@@ -0,0 +1,2 @@
+feat:
+- Support language selector from the data plugin ([#6613](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/6613))
diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml
@@ -331,3 +331,6 @@
 # Set the value to true to enable Ui Metric Collectors in Usage Collector
 # This publishes the Application Usage and UI Metrics into the saved object, which can be accessed by /api/stats?extended=true&legacy=true&exclude_usage=false
 # usageCollection.uiMetric.enabled: false
+
+# Set the value to true to enable enhancements for the data plugin
+# data.enhancements.enabled: false
diff --git a/package.json b/package.json
@@ -154,6 +154,7 @@
     "@hapi/vision": "^6.1.0",
     "@hapi/wreck": "^17.1.0",
     "@opensearch-project/opensearch": "^2.6.0",
+    "@opensearch/datemath": "5.0.3",
     "@osd/ace": "1.0.0",
     "@osd/analytics": "1.0.0",
     "@osd/apm-config-loader": "1.0.0",

diff --git a/packages/opensearch-datemath/index.d.ts b/packages/opensearch-datemath/index.d.ts
@@ -43,6 +43,8 @@ declare const datemath: {
   unitsAsc: Unit[];
   unitsDesc: Unit[];
 
+  isDateTime(input: any): boolean;
+
   /**
    * Parses a string into a moment object. The string can be something like "now - 15m".
    * @param options.forceNow If this optional parameter is supplied, "now" will be treated as this

diff --git a/packages/opensearch-datemath/index.js b/packages/opensearch-datemath/index.js
@@ -49,6 +49,7 @@ const isDate = (d) => Object.prototype.toString.call(d) === '[object Date]';
 
 const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
 
+const isDateTime = (d, momentInstance = moment) => momentInstance.isMoment(d);
 /*
  * This is a simplified version of opensearch's date parser.
  * If you pass in a momentjs instance as the third parameter the calculation
@@ -57,7 +58,7 @@ const isValidDate = (d) => isDate(d) && !isNaN(d.valueOf());
  */
 function parse(text, { roundUp = false, momentInstance = moment, forceNow } = {}) {
   if (!text) return undefined;
-  if (momentInstance.isMoment(text)) return text;
+  if (isDateTime(text, momentInstance)) return text;
   if (isDate(text)) return momentInstance(text);
   if (forceNow !== undefined && !isValidDate(forceNow)) {
     throw new Error('forceNow must be a valid Date');
@@ -164,6 +165,7 @@ function parseDateMath(mathString, time, roundUp) {
 
 module.exports = {
   parse: parse,
+  isDateTime: isDateTime,
   unitsMap: Object.freeze(unitsMap),
   units: Object.freeze(units),
   unitsAsc: Object.freeze(unitsAsc),

diff --git a/packages/opensearch-datemath/index.test.ts b/packages/opensearch-datemath/index.test.ts
@@ -122,19 +122,19 @@ describe('dateMath', function () {
     });
 
     it('should return a moment if passed a date', function () {
-      expect(dateMath.parse(date).format(format)).to.eql(mmnt.format(format));
+      expect(dateMath.parse(date)!.format(format)).to.eql(mmnt.format(format));
     });
 
     it('should return a moment if passed an ISO8601 string', function () {
-      expect(dateMath.parse(string).format(format)).to.eql(mmnt.format(format));
+      expect(dateMath.parse(string)!.format(format)).to.eql(mmnt.format(format));
     });
 
     it('should return the current time when parsing now', function () {
-      expect(dateMath.parse('now').format(format)).to.eql(now.format(format));
+      expect(dateMath.parse('now')!.format(format)).to.eql(now.format(format));
     });
 
     it('should use the forceNow parameter when parsing now', function () {
-      expect(dateMath.parse('now', { forceNow: anchoredDate }).valueOf()).to.eql(unix);
+      expect(dateMath.parse('now', { forceNow: anchoredDate })!.valueOf()).to.eql(unix);
     });
   });
 
@@ -158,17 +158,17 @@ describe('dateMath', function () {
         const thenEx = `${anchor}||-${len}${span}`;
 
         it('should return ' + len + span + ' ago', function () {
-          const parsed = dateMath.parse(nowEx).format(format);
+          const parsed = dateMath.parse(nowEx)!.format(format);
           expect(parsed).to.eql(now.subtract(len, span).format(format));
         });
 
         it('should return ' + len + span + ' before ' + anchor, function () {
-          const parsed = dateMath.parse(thenEx).format(format);
+          const parsed = dateMath.parse(thenEx)!.format(format);
           expect(parsed).to.eql(anchored.subtract(len, span).format(format));
         });
 
         it('should return ' + len + span + ' before forceNow', function () {
-          const parsed = dateMath.parse(nowEx, { forceNow: anchoredDate }).valueOf();
+          const parsed = dateMath.parse(nowEx, { forceNow: anchoredDate })!.valueOf();
           expect(parsed).to.eql(anchored.subtract(len, span).valueOf());
         });
       });
@@ -195,17 +195,17 @@ describe('dateMath', function () {
         const thenEx = `${anchor}||+${len}${span}`;
 
         it('should return ' + len + span + ' from now', function () {
-          expect(dateMath.parse(nowEx).format(format)).to.eql(now.add(len, span).format(format));
+          expect(dateMath.parse(nowEx)!.format(format)).to.eql(now.add(len, span).format(format));
         });
 
         it('should return ' + len + span + ' after ' + anchor, function () {
-          expect(dateMath.parse(thenEx).format(format)).to.eql(
+          expect(dateMath.parse(thenEx)!.format(format)).to.eql(
             anchored.add(len, span).format(format)
           );
         });
 
         it('should return ' + len + span + ' after forceNow', function () {
-          expect(dateMath.parse(nowEx, { forceNow: anchoredDate }).valueOf()).to.eql(
+          expect(dateMath.parse(nowEx, { forceNow: anchoredDate })!.valueOf()).to.eql(
             anchored.add(len, span).valueOf()
           );
         });
@@ -229,26 +229,26 @@ describe('dateMath', function () {
 
     spans.forEach((span) => {
       it(`should round now to the beginning of the ${span}`, function () {
-        expect(dateMath.parse('now/' + span).format(format)).to.eql(
+        expect(dateMath.parse('now/' + span)!.format(format)).to.eql(
           now.startOf(span).format(format)
         );
       });
 
       it(`should round now to the beginning of forceNow's ${span}`, function () {
-        expect(dateMath.parse('now/' + span, { forceNow: anchoredDate }).valueOf()).to.eql(
+        expect(dateMath.parse('now/' + span, { forceNow: anchoredDate })!.valueOf()).to.eql(
           anchored.startOf(span).valueOf()
         );
       });
 
       it(`should round now to the end of the ${span}`, function () {
-        expect(dateMath.parse('now/' + span, { roundUp: true }).format(format)).to.eql(
+        expect(dateMath.parse('now/' + span, { roundUp: true })!.format(format)).to.eql(
           now.endOf(span).format(format)
         );
       });
 
       it(`should round now to the end of forceNow's ${span}`, function () {
         expect(
-          dateMath.parse('now/' + span, { roundUp: true, forceNow: anchoredDate }).valueOf()
+          dateMath.parse('now/' + span, { roundUp: true, forceNow: anchoredDate })!.valueOf()
         ).to.eql(anchored.endOf(span).valueOf());
       });
     });
@@ -269,28 +269,28 @@ describe('dateMath', function () {
     });
 
     it('should round to the nearest second with 0 value', function () {
-      const val = dateMath.parse('now-0s/s').format(format);
+      const val = dateMath.parse('now-0s/s')!.format(format);
       expect(val).to.eql(now.startOf('s').format(format));
     });
 
     it('should subtract 17s, rounded to the nearest second', function () {
-      const val = dateMath.parse('now-17s/s').format(format);
+      const val = dateMath.parse('now-17s/s')!.format(format);
       expect(val).to.eql(now.startOf('s').subtract(17, 's').format(format));
     });
 
     it('should add 555ms, rounded to the nearest millisecond', function () {
-      const val = dateMath.parse('now+555ms/ms').format(format);
+      const val = dateMath.parse('now+555ms/ms')!.format(format);
       expect(val).to.eql(now.add(555, 'ms').startOf('ms').format(format));
     });
 
     it('should subtract 555ms, rounded to the nearest second', function () {
-      const val = dateMath.parse('now-555ms/s').format(format);
+      const val = dateMath.parse('now-555ms/s')!.format(format);
       expect(val).to.eql(now.subtract(555, 'ms').startOf('s').format(format));
     });
 
     it('should round weeks to Sunday by default', function () {
       const val = dateMath.parse('now-1w/w');
-      expect(val.isoWeekday()).to.eql(7);
+      expect(val!.isoWeekday()).to.eql(7);
     });
 
     it('should round weeks based on the passed moment locale start of week setting', function () {
@@ -300,7 +300,7 @@ describe('dateMath', function () {
         week: { dow: 2 },
       });
       const val = dateMath.parse('now-1w/w', { momentInstance: m });
-      expect(val.isoWeekday()).to.eql(2);
+      expect(val!.isoWeekday()).to.eql(2);
     });
 
     it('should round up weeks based on the passed moment locale start of week setting', function () {
@@ -315,11 +315,11 @@ describe('dateMath', function () {
       });
       // The end of the range (rounding up) should be the last day of the week (so one day before)
       // our start of the week, that's why 3 - 1
-      expect(val.isoWeekday()).to.eql(3 - 1);
+      expect(val!.isoWeekday()).to.eql(3 - 1);
     });
 
     it('should round relative to forceNow', function () {
-      const val = dateMath.parse('now-0s/s', { forceNow: anchoredDate }).valueOf();
+      const val = dateMath.parse('now-0s/s', { forceNow: anchoredDate })!.valueOf();
       expect(val).to.eql(anchored.startOf('s').valueOf());
     });
 

diff --git a/packages/osd-opensearch/src/cli_commands/snapshot.js b/packages/osd-opensearch/src/cli_commands/snapshot.js
@@ -50,6 +50,7 @@ exports.help = (defaults = {}) => {
       --download-only   Download the snapshot but don't actually start it
       --ssl             Sets up SSL on OpenSearch
       --security        Installs and sets up the OpenSearch Security plugin on the cluster
+      --sql             Installs and sets up the required OpenSearch SQL/PPL plugins on the cluster
       --P               OpenSearch plugin artifact URL to install it on the cluster. We can use the flag multiple times
                         to install multiple plugins on the cluster snapshot. The argument value can be url to zip file, maven coordinates of the plugin 
                         or for local zip files, use file:<followed by the absolute or relative path to the plugin zip file>.
@@ -77,6 +78,8 @@ exports.run = async (defaults = {}) => {
 
     boolean: ['security'],
 
+    boolean: ['sql'],
+
     default: defaults,
   });
 
@@ -98,6 +101,10 @@ exports.run = async (defaults = {}) => {
       await cluster.setupSecurity(installPath, options.version ?? defaults.version);
     }
 
+    if (options.sql) {
+      await cluster.setupSql(installPath, options.version ?? defaults.version);
+    }
+
     options.bundledJDK = true;
 
     await cluster.run(installPath, options);

diff --git a/packages/osd-opensearch/src/cluster.js b/packages/osd-opensearch/src/cluster.js
@@ -70,10 +70,11 @@ const first = (stream, map) =>
   });
 
 exports.Cluster = class Cluster {
-  constructor({ log = defaultLog, ssl = false, security = false } = {}) {
+  constructor({ log = defaultLog, ssl = false, security = false, sql = false } = {}) {
     this._log = log;
     this._ssl = ssl;
     this._security = security;
+    this._sql = sql;
     this._caCertPromise = ssl ? readFile(CA_CERT_PATH) : undefined;
   }
 
@@ -224,6 +225,28 @@ exports.Cluster = class Cluster {
     }
   }
 
+  /**
+   * Setups cluster with SQL/PPL plugins
+   *
+   * @param {string} installPath
+   * @property {String} version - version of OpenSearch
+   */
+  async setupSql(installPath, version) {
+    await this.installSqlPlugin(installPath, version, 'opensearch-sql');
+    await this.installSqlPlugin(installPath, version, 'opensearch-observability');
+  }
+
+  async installSqlPlugin(installPath, version, id) {
+    this._log.info(`Setting up: ${id}`);
+    try {
+      const pluginUrl = generateEnginePluginUrl(version, id);
+      await this.installOpenSearchPlugins(installPath, pluginUrl);
+      this._log.info(`Completed setup: ${id}`);
+    } catch (ex) {
+      this._log.warning(`Failed to setup: ${id}`);
+    }
+  }
+
   /**
    * Starts OpenSearch and returns resolved promise once started
    *

diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js
@@ -249,6 +249,7 @@ export default function (program) {
       .option('--dev', 'Run the server with development mode defaults')
       .option('--ssl', 'Run the dev server using HTTPS')
       .option('--security', 'Run the dev server using security defaults')
+      .option('--sql', 'Run the dev server using SQL/PPL defaults')
       .option('--dist', 'Use production assets from osd/optimizer')
       .option(
         '--no-base-path',