[Circuit-Breaker] Add memory circuit breaker configuration (#1347)

Add opensearch.memoryCircuitBreaker.enabled, opensearch.memoryCircuitBreaker.maxPercentage flags in opensearch_dashboards.yml Signed-off-by: Zuocheng Ding <zding817@gmail.com>
opensearch-project · Apr 12, 2022 · 7bc0f85 · 7bc0f85
1 parent db7cb6d
commit 7bc0f85
Show file tree

Hide file tree

Showing 8 changed files with 39 additions and 7 deletions.
diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml
@@ -76,6 +76,15 @@
 # must be a positive integer.
 #opensearch.requestTimeout: 30000
 
+# Enables the memory circuit breaker that prevents heap out of memory errors for large query responses.
+# If enabled, we will provide additional check to prevent potential out of memory error in @openserach-project/opensearch.
+# The default threshold is based on the `max-old-space-size` of NodeJS. It is configurable by tuning `opensearch.memoryCircuitBreaker.maxPercentage`.
+#opensearch.memoryCircuitBreaker.enabled: false
+
+# The pecentage of maximum heap allowed for processing response. The default value of the pecentage is `1.0`. The valid input range should be [0, 1] inclusively.
+# For reference, the `threshold of memoryCircuitBreaker` = `max-old-space-size of NodeJS` * `opensearch.memoryCircuitBreaker.maxPercentage`
+#opensearch.memoryCircuitBreaker.maxPercentage: 1.0
+
 # List of OpenSearch Dashboards client-side headers to send to OpenSearch. To send *no* client-side
 # headers, set this value to [] (an empty list).
 #opensearch.requestHeadersWhitelist: [ authorization ]

diff --git a/package.json b/package.json
@@ -123,7 +123,7 @@
     "@hapi/podium": "^4.1.3",
     "@hapi/vision": "^6.1.0",
     "@hapi/wreck": "^17.1.0",
-    "@opensearch-project/opensearch": "^1.0.2",
+    "@opensearch-project/opensearch": "^1.1.0",
     "@osd/ace": "1.0.0",
     "@osd/analytics": "1.0.0",
     "@osd/apm-config-loader": "1.0.0",

diff --git a/packages/osd-opensearch/package.json b/packages/osd-opensearch/package.json
@@ -12,7 +12,7 @@
     "osd:watch": "node scripts/build --watch"
   },
   "dependencies": {
-    "@opensearch-project/opensearch": "^1.0.2",
+    "@opensearch-project/opensearch": "^1.1.0",
     "@osd/dev-utils": "1.0.0",
     "abort-controller": "^3.0.0",
     "chalk": "^4.1.0",

diff --git a/src/core/server/opensearch/client/client_config.ts b/src/core/server/opensearch/client/client_config.ts
@@ -53,6 +53,9 @@ export type OpenSearchClientConfig = Pick<
   | 'username'
   | 'password'
 > & {
+  memoryCircuitBreaker?:
+    | OpenSearchConfig['memoryCircuitBreaker']
+    | ClientOptions['memoryCircuitBreaker'];
   pingTimeout?: OpenSearchConfig['pingTimeout'] | ClientOptions['pingTimeout'];
   requestTimeout?: OpenSearchConfig['requestTimeout'] | ClientOptions['requestTimeout'];
   ssl?: Partial<OpenSearchConfig['ssl']>;
@@ -75,13 +78,15 @@ export function parseClientOptions(config: OpenSearchClientConfig, scoped: boole
       ...config.customHeaders,
     },
   };
-
-  if (config.pingTimeout != null) {
-    clientOptions.pingTimeout = getDurationAsMs(config.pingTimeout);
+  if (config.memoryCircuitBreaker != null) {
+    clientOptions.memoryCircuitBreaker = config.memoryCircuitBreaker;
   }
   if (config.requestTimeout != null) {
     clientOptions.requestTimeout = getDurationAsMs(config.requestTimeout);
   }
+  if (config.pingTimeout != null) {
+    clientOptions.pingTimeout = getDurationAsMs(config.pingTimeout);
+  }
   if (config.sniffInterval != null) {
     clientOptions.sniffInterval =
       typeof config.sniffInterval === 'boolean'

diff --git a/src/core/server/opensearch/opensearch_config.test.ts b/src/core/server/opensearch/opensearch_config.test.ts
@@ -78,6 +78,10 @@ test('set correct defaults', () => {
       ],
       "ignoreVersionMismatch": false,
       "logQueries": false,
+      "memoryCircuitBreaker": Object {
+        "enabled": false,
+        "maxPercentage": 1,
+      },
       "optimizedHealthcheckId": undefined,
       "password": undefined,
       "pingTimeout": "PT30S",

diff --git a/src/core/server/opensearch/opensearch_config.ts b/src/core/server/opensearch/opensearch_config.ts
@@ -77,6 +77,10 @@ export const configSchema = schema.object({
   requestHeadersWhitelist: schema.oneOf([schema.string(), schema.arrayOf(schema.string())], {
     defaultValue: ['authorization'],
   }),
+  memoryCircuitBreaker: schema.object({
+    enabled: schema.boolean({ defaultValue: false }),
+    maxPercentage: schema.number({ defaultValue: 1.0 }),
+  }),
   customHeaders: schema.recordOf(schema.string(), schema.string(), { defaultValue: {} }),
   shardTimeout: schema.duration({ defaultValue: '30s' }),
   requestTimeout: schema.duration({ defaultValue: '30s' }),
@@ -244,6 +248,13 @@ export class OpenSearchConfig {
    */
   public readonly shardTimeout: Duration;
 
+  /**
+   * Set of options to configure memory circuit breaker for query response.
+   * The `maxPercentage` field is to determine the threshold for maximum heap size for memory circuit breaker. By default the value is `1.0`.
+   * The `enabled` field specifies whether the client should protect large response that can't fit into memory.
+   */
+
+  public readonly memoryCircuitBreaker: OpenSearchConfigType['memoryCircuitBreaker'];
   /**
    * Specifies whether the client should attempt to detect the rest of the cluster
    * when it is first instantiated.
@@ -300,6 +311,7 @@ export class OpenSearchConfig {
     this.requestHeadersWhitelist = Array.isArray(rawConfig.requestHeadersWhitelist)
       ? rawConfig.requestHeadersWhitelist
       : [rawConfig.requestHeadersWhitelist];
+    this.memoryCircuitBreaker = rawConfig.memoryCircuitBreaker;
     this.pingTimeout = rawConfig.pingTimeout;
     this.requestTimeout = rawConfig.requestTimeout;
     this.shardTimeout = rawConfig.shardTimeout;

diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/opensearch-dashboards-docker
@@ -30,8 +30,10 @@ opensearch_dashboards_vars=(
     opensearch.hosts
     opensearch.logQueries
     opensearch.password
-    opensearch.pingTimeout
     opensearch.requestHeadersWhitelist
+    opensearch.memoryCircuitBreaker.enabled
+    opensearch.memoryCircuitBreaker.maxPercentage
+    opensearch.pingTimeout
     opensearch.requestTimeout
     opensearch.shardTimeout
     opensearch.sniffInterval

diff --git a/yarn.lock b/yarn.lock
@@ -2365,7 +2365,7 @@
   dependencies:
     "@octokit/openapi-types" "^11.2.0"
 
-"@opensearch-project/opensearch@^1.0.2":
+"@opensearch-project/opensearch@^1.1.0":
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/@opensearch-project/opensearch/-/opensearch-1.1.0.tgz#8b3c8b4cbcea01755ba092d2997bf0b4ca7f22f7"
   integrity sha512-1TDw92JL8rD1b2QGluqBsIBLIiD5SGciIpz4qkrGAe9tcdfQ1ptub5e677rhWl35UULSjr6hP8M6HmISZ/M5HQ==