[Circuit-Breaker] Add memory circuit breaker configuration

Add responseMaxHeapPercentage, memoryCircuitBreakerEnabled flags in opensearch_dashboards.yml

Signed-off-by: Zuocheng Ding <zding817@gmail.com>

config/opensearch_dashboards.yml

@@ -76,6 +76,16 @@
 # must be a positive integer.
 #opensearch.requestTimeout: 30000
 
+
+# Enables the memory circuit breaker that prevents heap out of memory errors for large query responses.
+# If `enabled` == true, we will provide additional check to prevent potential out of memory error in @openserach-project/opensearch.
+# The default threshold is based on the `max-old-space-size` of NodeJS. It is configurable by tuning `opensearch.memoryCircuitBreaker.maxPercentage`.
+#opensearch.memoryCircuitBreaker.enabled: false
+
+# The pecentage of maximum heap allowed for processing response. The default value of the pecentage is `1.0`. The valid input range should be [0, 1] inclusively.
+# For reference, the `threshold of memoryCircuitBreaker` = `max-old-space-size of NodeJS` * `opensearch.memoryCircuitBreaker.maxPercentage`
+#opensearch.memoryCircuitBreaker.maxPercentage: 1.0
+
 # List of OpenSearch Dashboards client-side headers to send to OpenSearch. To send *no* client-side
 # headers, set this value to [] (an empty list).
 #opensearch.requestHeadersWhitelist: [ authorization ]

package.json

@@ -124,7 +124,7 @@
     "@hapi/podium": "^4.1.3",
     "@hapi/vision": "^6.1.0",
     "@hapi/wreck": "^17.1.0",
-    "@opensearch-project/opensearch": "^1.0.2",
+    "@opensearch-project/opensearch": "^1.1.0",
     "@osd/ace": "1.0.0",
     "@osd/analytics": "1.0.0",
     "@osd/apm-config-loader": "1.0.0",

packages/osd-opensearch/package.json

@@ -12,7 +12,7 @@
     "osd:watch": "node scripts/build --watch"
   },
   "dependencies": {
-    "@opensearch-project/opensearch": "^1.0.2",
+    "@opensearch-project/opensearch": "^1.1.0",
     "@osd/dev-utils": "1.0.0",
     "abort-controller": "^3.0.0",
     "chalk": "^4.1.0",

src/core/server/opensearch/client/client_config.ts

@@ -53,6 +53,9 @@ export type OpenSearchClientConfig = Pick<
   | 'username'
   | 'password'
 > & {
+  memoryCircuitBreaker?:
+    | OpenSearchConfig['memoryCircuitBreaker']
+    | ClientOptions['memoryCircuitBreaker'];
   pingTimeout?: OpenSearchConfig['pingTimeout'] | ClientOptions['pingTimeout'];
   requestTimeout?: OpenSearchConfig['requestTimeout'] | ClientOptions['requestTimeout'];
   ssl?: Partial<OpenSearchConfig['ssl']>;
@@ -79,6 +82,9 @@ export function parseClientOptions(config: OpenSearchClientConfig, scoped: boole
   if (config.pingTimeout != null) {
     clientOptions.pingTimeout = getDurationAsMs(config.pingTimeout);
   }
+  if (config.memoryCircuitBreaker != null) {
+    clientOptions.memoryCircuitBreaker = config.memoryCircuitBreaker;
+  }
   if (config.requestTimeout != null) {
     clientOptions.requestTimeout = getDurationAsMs(config.requestTimeout);
   }

src/core/server/opensearch/opensearch_config.test.ts

@@ -78,6 +78,10 @@ test('set correct defaults', () => {
       ],
       "ignoreVersionMismatch": false,
       "logQueries": false,
+      "memoryCircuitBreaker": Object {
+        "enabled": false,
+        "maxPercentage": 1,
+      },
       "optimizedHealthcheckId": undefined,
       "password": undefined,
       "pingTimeout": "PT30S",

src/core/server/opensearch/opensearch_config.ts

@@ -80,6 +80,10 @@ export const configSchema = schema.object({
   customHeaders: schema.recordOf(schema.string(), schema.string(), { defaultValue: {} }),
   shardTimeout: schema.duration({ defaultValue: '30s' }),
   requestTimeout: schema.duration({ defaultValue: '30s' }),
+  memoryCircuitBreaker: schema.object({
+    enabled: schema.boolean({ defaultValue: false }),
+    maxPercentage: schema.number({ defaultValue: 1.0 }),
+  }),
   pingTimeout: schema.duration({ defaultValue: schema.siblingRef('requestTimeout') }),
   logQueries: schema.boolean({ defaultValue: false }),
   optimizedHealthcheckId: schema.maybe(schema.string()),
@@ -244,6 +248,13 @@ export class OpenSearchConfig {
    */
   public readonly shardTimeout: Duration;
 
+  /**
+   * Set of options to configure memory circuit breaker for query response.
+   * The `maxPercentage` field is to determine the threshold for maximum heap size for memory circuit breaker. By default the value is `1.0`.
+   * The `enabled` field specifies whether the client should protect large response that can't fit into memory.
+   */
+
+  public readonly memoryCircuitBreaker: OpenSearchConfigType['memoryCircuitBreaker'];
   /**
    * Specifies whether the client should attempt to detect the rest of the cluster
    * when it is first instantiated.
@@ -302,6 +313,7 @@ export class OpenSearchConfig {
       : [rawConfig.requestHeadersWhitelist];
     this.pingTimeout = rawConfig.pingTimeout;
     this.requestTimeout = rawConfig.requestTimeout;
+    this.memoryCircuitBreaker = rawConfig.memoryCircuitBreaker;
     this.shardTimeout = rawConfig.shardTimeout;
     this.sniffOnStart = rawConfig.sniffOnStart;
     this.sniffOnConnectionFault = rawConfig.sniffOnConnectionFault;
@@ -313,7 +325,6 @@ export class OpenSearchConfig {
 
     const { alwaysPresentCertificate, verificationMode } = rawConfig.ssl;
     const { key, keyPassphrase, certificate, certificateAuthorities } = readKeyAndCerts(rawConfig);
-
     this.ssl = {
       alwaysPresentCertificate,
       key,

src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker

@@ -32,6 +32,8 @@ opensearch_dashboards_vars=(
     opensearch.pingTimeout
     opensearch.requestHeadersWhitelist
     opensearch.requestTimeout
+    opensearch.responseMaxHeapPercentage
+    opensearch.memoryCircuitBreakerEnabled
     opensearch.shardTimeout
     opensearch.sniffInterval
     opensearch.sniffOnConnectionFault

yarn.lock

@@ -2216,10 +2216,10 @@
     universal-user-agent "^2.0.0"
     url-template "^2.0.8"
 
-"@opensearch-project/opensearch@^1.0.2":
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/@opensearch-project/opensearch/-/opensearch-1.0.2.tgz#b5946bcd2b6b655b4ae8fbb4562411cf283a7503"
-  integrity sha512-gQ2CjbS7/pJ4A3IBWd+AD0KbCaAnE1Ur9baRxqM1NMH/7A8GQxwtVxx8raUOf4HZExkZZOUYBMzJgWM1OyELyw==
+"@opensearch-project/opensearch@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@opensearch-project/opensearch/-/opensearch-1.1.0.tgz#8b3c8b4cbcea01755ba092d2997bf0b4ca7f22f7"
+  integrity sha512-1TDw92JL8rD1b2QGluqBsIBLIiD5SGciIpz4qkrGAe9tcdfQ1ptub5e677rhWl35UULSjr6hP8M6HmISZ/M5HQ==
   dependencies:
     debug "^4.3.1"
     hpagent "^0.1.1"
@@ -16231,7 +16231,6 @@ reselect@^4.0.0:
   integrity sha512-qUgANli03jjAyGlnbYVAV5vvnOmJnODyABz51RdBN7M4WaVu8mecZWgyQNkG8Yqe3KRGRt0l4K4B3XVEULC4CA==
 
 resize-observer-polyfill@^1.5.1:
-  name "resize-observer-polyfill"
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/@4lolo/resize-observer-polyfill/-/resize-observer-polyfill-1.5.2.tgz#58868fc7224506236b5550d0c68357f0a874b84b"
   integrity sha512-HY4JYLITsWBOdeqCF/x3q7Aa2PVl/BmfkPv4H/Qzplc4Lrn9cKmWz6jHyAREH9tFuD0xELjJVgX3JaEmdcXu3g==