[Remote Store] Add support to provide separate segment metadata repository

CHANGELOG.md

@@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686))
 - Add support for more than one protocol for transport ([#12967](https://github.com/opensearch-project/OpenSearch/pull/12967))
 - Add changes for overriding remote store and replication settings during snapshot restore. ([#11868](https://github.com/opensearch-project/OpenSearch/pull/11868))
+- [Remote Store] Add support to provide separate segment metadata repository ([#12993](https://github.com/opensearch-project/OpenSearch/pull/12993))
 
 ### Dependencies
 - Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))

server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java

@@ -31,8 +31,8 @@
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
-import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
-import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY;
 
 public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase {
 
@@ -89,15 +89,15 @@ public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailu
         );
 
         return Settings.builder()
-            .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
+            .put("node.attr." + REMOTE_STORE_SEGMENT_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
             .put(segmentRepoTypeAttributeKey, "mock")
             .put(segmentRepoSettingsAttributeKeyPrefix + "location", repoLocation)
             .put(segmentRepoSettingsAttributeKeyPrefix + "random_control_io_exception_rate", ioFailureRate)
             .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_verification_file", true)
             .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_list_blobs", true)
             .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_blobs", skipExceptionBlobList)
             .put(segmentRepoSettingsAttributeKeyPrefix + "max_failure_number", maxFailure)
-            .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME)
+            .put("node.attr." + REMOTE_STORE_TRANSLOG_DATA_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME)
             .put(translogRepoTypeAttributeKey, "mock")
             .put(translogRepoSettingsAttributeKeyPrefix + "location", repoLocation)
             .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)

server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java

@@ -20,9 +20,9 @@
 
 import java.util.Locale;
 
-import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
-import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
 import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -149,7 +149,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI
         Settings settings = Settings.builder()
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo")
+            .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo")
             .build();
         IllegalArgumentException exc = expectThrows(
             IllegalArgumentException.class,
@@ -161,7 +161,7 @@ public void testRemoteStoreSegmentRepoWithoutRemoteEnabledAndSegmentReplicationI
                 String.format(
                     Locale.ROOT,
                     "Settings %s can only be set/enabled when %s is set to true",
-                    SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
+                    SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY,
                     SETTING_REMOTE_STORE_ENABLED
                 )
             )
@@ -174,7 +174,7 @@ public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException()
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
             .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(SETTING_REMOTE_STORE_ENABLED, true)
-            .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo")
+            .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo")
             .build();
 
         IllegalArgumentException exc = expectThrows(
@@ -188,7 +188,7 @@ public void testRemoteStoreEnabledByUserWithRemoteRepoIllegalArgumentException()
                     Locale.ROOT,
                     "Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;",
                     SETTING_REMOTE_STORE_ENABLED,
-                    SETTING_REMOTE_SEGMENT_STORE_REPOSITORY
+                    SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY
                 )
             )
         );
@@ -198,7 +198,7 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th
         Settings settings = Settings.builder()
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
-            .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo")
+            .put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo")
             .build();
         IllegalArgumentException exc = expectThrows(
             IllegalArgumentException.class,
@@ -210,7 +210,7 @@ public void testRemoteStoreOverrideOnlyTranslogRepoIllegalArgumentException() th
                 String.format(
                     Locale.ROOT,
                     "Settings %s can only be set/enabled when %s is set to true",
-                    SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY,
+                    SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY,
                     SETTING_REMOTE_STORE_ENABLED
                 )
             )
@@ -223,8 +223,8 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception {
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
             .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(SETTING_REMOTE_STORE_ENABLED, true)
-            .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, "my-custom-repo")
-            .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, "my-custom-repo")
+            .put(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, "my-custom-repo")
+            .put(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY, "my-custom-repo")
             .build();
         IllegalArgumentException exc = expectThrows(
             IllegalArgumentException.class,
@@ -237,8 +237,8 @@ public void testRemoteStoreOverrideTranslogRepoCorrectly() throws Exception {
                     Locale.ROOT,
                     "Validation Failed: 1: private index setting [%s] can not be set explicitly;2: private index setting [%s] can not be set explicitly;3: private index setting [%s] can not be set explicitly;",
                     SETTING_REMOTE_STORE_ENABLED,
-                    SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
-                    SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY
+                    SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY,
+                    SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY
                 )
             )
         );
@@ -254,8 +254,8 @@ protected void verifyRemoteStoreIndexSettings(
     ) {
         assertEquals(replicationType, indexSettings.get(SETTING_REPLICATION_TYPE));
         assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
-        assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
-        assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
+        assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY));
+        assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY));
         assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings));
     }
 

server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

@@ -710,7 +710,7 @@ public void testInvalidRestoreRequestScenarios() throws Exception {
 
         // try index restore with remote store repository modified
         Settings remoteStoreIndexSettings = Settings.builder()
-            .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo)
+            .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY, newRemoteStoreRepo)
             .build();
 
         exception = expectThrows(
@@ -735,8 +735,8 @@ public void testInvalidRestoreRequestScenarios() throws Exception {
                 .prepareRestoreSnapshot(snapshotRepo, snapshotName1)
                 .setWaitForCompletion(false)
                 .setIgnoreIndexSettings(
-                    IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
-                    IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY
+                    IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY,
+                    IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY
                 )
                 .setIndices(index)
                 .setRenamePattern(index)

server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java

@@ -63,6 +63,7 @@
 
 public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase {
     protected static final String REPOSITORY_NAME = "test-remote-store-repo";
+    protected static final String SEGMENT_METADATA_REPOSITORY_NAME = "test-remote-segment-metadata-repo";
     protected static final String REPOSITORY_2_NAME = "test-remote-store-repo-2";
     protected static final int SHARD_COUNT = 1;
     protected static int REPLICA_COUNT = 1;
@@ -72,6 +73,7 @@ public class RemoteStoreBaseIntegTestCase extends OpenSearchIntegTestCase {
     protected static final String MAX_SEQ_NO_REFRESHED_OR_FLUSHED = "max-seq-no-refreshed-or-flushed";
 
     protected Path segmentRepoPath;
+    protected Path segmentMetadataRepoPath;
     protected Path translogRepoPath;
     protected boolean clusterSettingsSuppliedByTest = false;
     private final List<String> documentKeys = List.of(
@@ -133,15 +135,29 @@ protected Map<String, Long> indexData(int numberOfIterations, boolean invokeFlus
     protected Settings nodeSettings(int nodeOrdinal) {
         if (segmentRepoPath == null || translogRepoPath == null) {
             segmentRepoPath = randomRepoPath().toAbsolutePath();
+            segmentMetadataRepoPath = randomRepoPath().toAbsolutePath();
             translogRepoPath = randomRepoPath().toAbsolutePath();
         }
         if (clusterSettingsSuppliedByTest) {
             return Settings.builder().put(super.nodeSettings(nodeOrdinal)).build();
         } else {
-            return Settings.builder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath))
-                .build();
+            Settings.Builder settingsBuilder = Settings.builder().put(super.nodeSettings(nodeOrdinal));
+            if (randomBoolean()) {
+                settingsBuilder.put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath));
+            } else {
+                // Use a separate segment metadata repository
+                settingsBuilder.put(
+                    remoteStoreClusterSettings(
+                        REPOSITORY_NAME,
+                        segmentRepoPath,
+                        SEGMENT_METADATA_REPOSITORY_NAME,
+                        segmentMetadataRepoPath,
+                        REPOSITORY_2_NAME,
+                        translogRepoPath
+                    )
+                );
+            }
+            return settingsBuilder.build();
         }
     }
 

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -321,9 +321,11 @@ public Iterator<Setting<?>> settings() {
 
     public static final String SETTING_REMOTE_STORE_ENABLED = "index.remote_store.enabled";
 
-    public static final String SETTING_REMOTE_SEGMENT_STORE_REPOSITORY = "index.remote_store.segment.repository";
+    public static final String SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY = "index.remote_store.segment.repository";
 
-    public static final String SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY = "index.remote_store.translog.repository";
+    public static final String SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY = "index.remote_store.segment.metadata_repository";
+
+    public static final String SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY = "index.remote_store.translog.repository";
 
     /**
      * Used to specify if the index data should be persisted in the remote store.
@@ -362,86 +364,73 @@ public Iterator<Setting<?>> settings() {
         Property.Dynamic
     );
 
-    /**
-     * Used to specify remote store repository to use for this index.
-     */
-    public static final Setting<String> INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING = Setting.simpleString(
-        SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
-        new Setting.Validator<>() {
+    static class RepositoryNameValidator implements Setting.Validator<String> {
 
-            @Override
-            public void validate(final String value) {}
+        String remoteStoreRepositorySettingKey;
 
-            @Override
-            public void validate(final String value, final Map<Setting<?>, Object> settings) {
-                if (value == null || value.isEmpty()) {
-                    throw new IllegalArgumentException(
-                        "Setting "
-                            + INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey()
-                            + " should be provided with non-empty repository ID"
-                    );
-                } else {
-                    validateRemoteStoreSettingEnabled(settings, INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING);
-                }
-            }
+        RepositoryNameValidator(String remoteStoreRepositorySettingKey) {
+            this.remoteStoreRepositorySettingKey = remoteStoreRepositorySettingKey;
+        }
 
-            @Override
-            public Iterator<Setting<?>> settings() {
-                final List<Setting<?>> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING);
-                return settings.iterator();
+        @Override
+        public void validate(final String value) {}
+
+        @Override
+        public void validate(final String value, final Map<Setting<?>, Object> settings) {
+            if (value == null || value.isEmpty()) {
+                throw new IllegalArgumentException(
+                    "Setting " + remoteStoreRepositorySettingKey + " should be provided with non-empty repository ID"
+                );
+            } else {
+                validateRemoteStoreSettingEnabled(settings, remoteStoreRepositorySettingKey);
             }
-        },
+        }
+
+        @Override
+        public Iterator<Setting<?>> settings() {
+            final List<Setting<?>> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING);
+            return settings.iterator();
+        }
+    }
+
+    /**
+     * Used to specify remote store data repository to use for this index.
+     */
+    public static final Setting<String> INDEX_REMOTE_SEGMENT_STORE_DATA_REPOSITORY_SETTING = Setting.simpleString(
+        SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY,
+        new RepositoryNameValidator(SETTING_REMOTE_SEGMENT_STORE_DATA_REPOSITORY),
+        Property.IndexScope,
+        Property.PrivateIndex,
+        Property.Dynamic
+    );
+
+    /**
+     * Used to specify remote store metadata repository to use for this index.
+     */
+    public static final Setting<String> INDEX_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY_SETTING = Setting.simpleString(
+        SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY,
+        new RepositoryNameValidator(SETTING_REMOTE_SEGMENT_STORE_METADATA_REPOSITORY),
         Property.IndexScope,
         Property.PrivateIndex,
         Property.Dynamic
     );
 
-    private static void validateRemoteStoreSettingEnabled(final Map<Setting<?>, Object> settings, Setting<?> setting) {
+    private static void validateRemoteStoreSettingEnabled(final Map<Setting<?>, Object> settings, String settingKey) {
         final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
         if (isRemoteSegmentStoreEnabled == false) {
             throw new IllegalArgumentException(
                 "Settings "
-                    + setting.getKey()
+                    + settingKey
                     + " can only be set/enabled when "
                     + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()
                     + " is set to true"
             );
         }
     }
 
-    public static final Setting<String> INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING = Setting.simpleString(
-        SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY,
-        new Setting.Validator<>() {
-
-            @Override
-            public void validate(final String value) {}
-
-            @Override
-            public void validate(final String value, final Map<Setting<?>, Object> settings) {
-                if (value == null || value.isEmpty()) {
-                    throw new IllegalArgumentException(
-                        "Setting " + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey() + " should be provided with non-empty repository ID"
-                    );
-                } else {
-                    final Boolean isRemoteTranslogStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
-                    if (isRemoteTranslogStoreEnabled == null || isRemoteTranslogStoreEnabled == false) {
-                        throw new IllegalArgumentException(
-                            "Settings "
-                                + INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey()
-                                + " can only be set/enabled when "
-                                + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()
-                                + " is set to true"
-                        );
-                    }
-                }
-            }
-
-            @Override
-            public Iterator<Setting<?>> settings() {
-                final List<Setting<?>> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING);
-                return settings.iterator();
-            }
-        },
+    public static final Setting<String> INDEX_REMOTE_TRANSLOG_DATA_REPOSITORY_SETTING = Setting.simpleString(
+        SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY,
+        new RepositoryNameValidator(SETTING_REMOTE_TRANSLOG_STORE_DATA_REPOSITORY),
         Property.IndexScope,
         Property.PrivateIndex,
         Property.Dynamic