diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bc32e5268c2f..a0a33583c5b0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Create NamedRoute to map extension routes to a shortened name ([#6870](https://github.com/opensearch-project/OpenSearch/pull/6870)) - Added @dbwiddis as on OpenSearch maintainer ([#7665](https://github.com/opensearch-project/OpenSearch/pull/7665)) - [Extensions] Add ExtensionAwarePlugin extension point to add custom settings for extensions ([#7526](https://github.com/opensearch-project/OpenSearch/pull/7526)) +- Add new cluster setting to set default index replication type ([#7420](https://github.com/opensearch-project/OpenSearch/pull/7420)) ### Dependencies - Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.3 (#7564) diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index c2d84c6c71582..107fe345c942b 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -102,10 +102,9 @@ ${path.logs} # # ---------------------------------- Experimental Features ----------------------------------- # -# Gates the visibility of the index setting that allows changing of replication type. -# Once the feature is ready for production release, this feature flag can be removed. +# Gates the visibility of the experimental segment replication features until they are production ready. # -#opensearch.experimental.feature.replication_type.enabled: false +#opensearch.experimental.feature.segment_replication_experimental.enabled: false # # # Gates the visibility of the index setting that allows persisting data to remote store along with local disk. diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java index 9ae577912283a..a3c9af0bef5fb 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationClusterSettingIT.java @@ -8,11 +8,14 @@ package org.opensearch.indices.replication; -import org.opensearch.action.admin.indices.replication.SegmentReplicationStatsResponse; +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.Index; import org.opensearch.index.IndexModule; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.plugins.Plugin; @@ -24,8 +27,8 @@ import java.util.Collections; import java.util.Arrays; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) public class SegmentReplicationClusterSettingIT extends OpenSearchIntegTestCase { @@ -77,71 +80,119 @@ protected Collection> nodePlugins() { return Arrays.asList(SegmentReplicationClusterSettingIT.TestPlugin.class, MockTransportService.TestPlugin.class); } - public void testReplicationWithSegmentReplicationClusterSetting() throws Exception { - - boolean isSystemIndex = randomBoolean(); - String indexName = isSystemIndex ? SYSTEM_INDEX_NAME : INDEX_NAME; + public void testSystemIndexWithSegmentReplicationClusterSetting() throws Exception { // Starting two nodes with primary and replica shards respectively. final String primaryNode = internalCluster().startNode(); - createIndex(indexName); - ensureYellowAndNoInitializingShards(indexName); + createIndex(SYSTEM_INDEX_NAME); + ensureYellowAndNoInitializingShards(SYSTEM_INDEX_NAME); final String replicaNode = internalCluster().startNode(); - ensureGreen(indexName); - - final int initialDocCount = scaledRandomIntBetween(20, 30); - for (int i = 0; i < initialDocCount; i++) { - client().prepareIndex(indexName).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); - } - - refresh(indexName); - assertBusy(() -> { - assertHitCount(client(replicaNode).prepareSearch(indexName).setSize(0).setPreference("_only_local").get(), initialDocCount); - }); - - SegmentReplicationStatsResponse segmentReplicationStatsResponse = client().admin() + ensureGreen(SYSTEM_INDEX_NAME); + final GetSettingsResponse response = client().admin() .indices() - .prepareSegmentReplicationStats(indexName) - .execute() + .getSettings(new GetSettingsRequest().indices(SYSTEM_INDEX_NAME).includeDefaults(true)) .actionGet(); - if (isSystemIndex) { - // Verify that Segment Replication did not happen on the replica shard. - assertNull(segmentReplicationStatsResponse.getReplicationStats().get(indexName)); - } else { - // Verify that Segment Replication happened on the replica shard. - assertFalse(segmentReplicationStatsResponse.getReplicationStats().get(indexName).get(0).getReplicaStats().isEmpty()); - } + assertEquals(response.getSetting(SYSTEM_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); + + // Verify index setting isSegRepEnabled is false. + Index index = resolveIndex(SYSTEM_INDEX_NAME); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, primaryNode); + assertEquals(indicesService.indexService(index).getIndexSettings().isSegRepEnabled(), false); } - public void testIndexReplicationSettingOverridesClusterSetting() throws Exception { + public void testIndexReplicationSettingOverridesSegRepClusterSetting() throws Exception { + Settings settings = Settings.builder().put(CLUSTER_SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build(); + final String ANOTHER_INDEX = "test-index"; + // Starting two nodes with primary and replica shards respectively. - final String primaryNode = internalCluster().startNode(); + final String primaryNode = internalCluster().startNode(settings); prepareCreate( INDEX_NAME, Settings.builder() // we want to override cluster replication setting by passing a index replication setting .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) ).get(); - ensureYellowAndNoInitializingShards(INDEX_NAME); - final String replicaNode = internalCluster().startNode(); - ensureGreen(INDEX_NAME); + createIndex(ANOTHER_INDEX); + ensureYellowAndNoInitializingShards(INDEX_NAME, ANOTHER_INDEX); + final String replicaNode = internalCluster().startNode(settings); + + // Randomly close and open index. + if (randomBoolean()) { + logger.info("--> Closing the index "); + client().admin().indices().prepareClose(INDEX_NAME).get(); - final int initialDocCount = scaledRandomIntBetween(20, 30); - for (int i = 0; i < initialDocCount; i++) { - client().prepareIndex(INDEX_NAME).setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + logger.info("--> Opening the index"); + client().admin().indices().prepareOpen(INDEX_NAME).get(); } + ensureGreen(INDEX_NAME, ANOTHER_INDEX); - refresh(INDEX_NAME); - assertBusy(() -> { - assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), initialDocCount); - }); + final GetSettingsResponse response = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(INDEX_NAME, ANOTHER_INDEX).includeDefaults(true)) + .actionGet(); + assertEquals(response.getSetting(INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); + assertEquals(response.getSetting(ANOTHER_INDEX, SETTING_REPLICATION_TYPE), ReplicationType.SEGMENT.toString()); + + // Verify index setting isSegRepEnabled. + Index index = resolveIndex(INDEX_NAME); + Index anotherIndex = resolveIndex(ANOTHER_INDEX); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, primaryNode); + assertEquals(indicesService.indexService(index).getIndexSettings().isSegRepEnabled(), false); + assertEquals(indicesService.indexService(anotherIndex).getIndexSettings().isSegRepEnabled(), true); + } - SegmentReplicationStatsResponse segmentReplicationStatsResponse = client().admin() + public void testIndexReplicationSettingOverridesDocRepClusterSetting() throws Exception { + Settings settings = Settings.builder().put(CLUSTER_SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT).build(); + final String ANOTHER_INDEX = "test-index"; + final String primaryNode = internalCluster().startNode(settings); + prepareCreate( + INDEX_NAME, + Settings.builder() + // we want to override cluster replication setting by passing a index replication setting + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + ).get(); + createIndex(ANOTHER_INDEX); + ensureYellowAndNoInitializingShards(INDEX_NAME, ANOTHER_INDEX); + final String replicaNode = internalCluster().startNode(settings); + ensureGreen(INDEX_NAME, ANOTHER_INDEX); + + final GetSettingsResponse response = client().admin() .indices() - .prepareSegmentReplicationStats(INDEX_NAME) - .execute() + .getSettings(new GetSettingsRequest().indices(INDEX_NAME, ANOTHER_INDEX).includeDefaults(true)) .actionGet(); - // Verify that Segment Replication did not happen on the replica shard. - assertNull(segmentReplicationStatsResponse.getReplicationStats().get(INDEX_NAME)); + assertEquals(response.getSetting(INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.SEGMENT.toString()); + assertEquals(response.getSetting(ANOTHER_INDEX, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); + + // Verify index setting isSegRepEnabled. + Index index = resolveIndex(INDEX_NAME); + Index anotherIndex = resolveIndex(ANOTHER_INDEX); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, primaryNode); + assertEquals(indicesService.indexService(index).getIndexSettings().isSegRepEnabled(), true); + assertEquals(indicesService.indexService(anotherIndex).getIndexSettings().isSegRepEnabled(), false); } + + public void testHiddenIndicesWithReplicationStrategyClusterSetting() throws Exception { + final String primaryNode = internalCluster().startNode(); + final String replicaNode = internalCluster().startNode(); + prepareCreate( + INDEX_NAME, + Settings.builder() + // we want to set index as hidden + .put("index.hidden", true) + ).get(); + ensureGreen(INDEX_NAME); + + // Verify that document replication strategy is used for hidden indices. + final GetSettingsResponse response = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(INDEX_NAME).includeDefaults(true)) + .actionGet(); + assertEquals(response.getSetting(INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); + + // Verify index setting isSegRepEnabled. + Index index = resolveIndex(INDEX_NAME); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, primaryNode); + assertEquals(indicesService.indexService(index).getIndexSettings().isSegRepEnabled(), false); + } + } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index f8b56857168b8..82d93f8bf2b72 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -31,6 +31,7 @@ import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_STORE_ENABLED_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; +import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) @@ -120,7 +121,15 @@ public void testRemoteStoreDisabledByUser() throws Exception { .getIndex(new GetIndexRequest().indices("test-idx-1").includeDefaults(true)) .get(); Settings indexSettings = getIndexResponse.settings().get("test-idx-1"); - verifyRemoteStoreIndexSettings(indexSettings, "false", null, null, null, null, null); + verifyRemoteStoreIndexSettings( + indexSettings, + "false", + null, + null, + null, + client().settings().get(CLUSTER_SETTING_REPLICATION_TYPE), + null + ); } public void testRemoteStoreEnabledByUserWithoutRemoteRepoAndSegmentReplicationIllegalArgumentException() throws Exception { diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java index e22165ae3d7fa..0ad2da0715a24 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java @@ -17,7 +17,10 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.FeatureFlags; +import org.opensearch.index.Index; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.rest.RestStatus; import org.opensearch.test.InternalTestCluster; @@ -28,6 +31,8 @@ import java.util.List; import java.util.concurrent.TimeUnit; +import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; +import static org.opensearch.indices.IndicesService.CLUSTER_SETTING_REPLICATION_TYPE; import static org.opensearch.indices.replication.SegmentReplicationBaseIT.waitForSearchableDocs; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; @@ -261,4 +266,45 @@ public void testRestoreOnReplicaNode() throws Exception { SearchResponse resp = client().prepareSearch(RESTORED_INDEX_NAME).setQuery(QueryBuilders.matchAllQuery()).get(); assertHitCount(resp, DOC_COUNT); } + + public void testSnapshotRestoreOnIndexWithSegRepClusterSetting() throws Exception { + Settings settings = Settings.builder() + .put(super.featureFlagSettings()) + .put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true") + .put(CLUSTER_SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + + // Starting two nodes with primary and replica shards respectively. + final String primaryNode = internalCluster().startNode(settings); + prepareCreate( + INDEX_NAME, + Settings.builder() + // we want to override cluster replication setting by passing a index replication setting + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT) + ).get(); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replicaNode = internalCluster().startNode(settings); + ensureGreen(INDEX_NAME); + + createSnapshot(); + // Delete index + assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get()); + assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME)); + + RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(null); + + // Assertions + assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED); + ensureGreen(RESTORED_INDEX_NAME); + GetSettingsResponse settingsResponse = client().admin() + .indices() + .getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME).includeDefaults(true)) + .get(); + assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.DOCUMENT.toString()); + + // Verify index setting isSegRepEnabled. + Index index = resolveIndex(RESTORED_INDEX_NAME); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class, primaryNode); + assertEquals(indicesService.indexService(index).getIndexSettings().isSegRepEnabled(), false); + } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 89abdbdb5419f..6e22de7c72d07 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -583,7 +583,8 @@ private ClusterState applyCreateIndexRequestWithV1Templates( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders + indexSettingProviders, + systemIndices.validateSystemIndex(request.index()) ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -647,7 +648,8 @@ private ClusterState applyCreateIndexRequestWithV2Template( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders + indexSettingProviders, + systemIndices.validateSystemIndex(request.index()) ); int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, null); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -727,7 +729,8 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( settings, indexScopedSettings, shardLimitValidator, - indexSettingProviders + indexSettingProviders, + sourceMetadata.isSystem() ); final int routingNumShards = getIndexNumberOfRoutingShards(aggregatedIndexSettings, sourceMetadata); IndexMetadata tmpImd = buildAndValidateTemporaryIndexMetadata(currentState, aggregatedIndexSettings, request, routingNumShards); @@ -810,7 +813,8 @@ static Settings aggregateIndexSettings( Settings settings, IndexScopedSettings indexScopedSettings, ShardLimitValidator shardLimitValidator, - Set indexSettingProviders + Set indexSettingProviders, + boolean isSystemIndex ) { // Create builders for the template and request settings. We transform these into builders // because we may want settings to be "removed" from these prior to being set on the new @@ -894,6 +898,7 @@ static Settings aggregateIndexSettings( indexSettingsBuilder.put(IndexMetadata.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName()); indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); + updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings, isSystemIndex); updateRemoteStoreSettings(indexSettingsBuilder, request.settings(), settings); if (sourceMetadata != null) { @@ -928,6 +933,27 @@ static Settings aggregateIndexSettings( return indexSettings; } + /** + * Updates index settings to set replication strategy by default based on cluster level settings + * @param settingsBuilder index settings builder to be updated with relevant settings + * @param requestSettings settings passed in during index create request + * @param clusterSettings cluster level settings + */ + private static void updateReplicationStrategy( + Settings.Builder settingsBuilder, + Settings requestSettings, + Settings clusterSettings, + boolean isSystemIndex + ) { + if (isSystemIndex || IndexMetadata.INDEX_HIDDEN_SETTING.get(requestSettings)) { + settingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT); + return; + } + if (CLUSTER_REPLICATION_TYPE_SETTING.exists(clusterSettings) && INDEX_REPLICATION_TYPE_SETTING.exists(requestSettings) == false) { + settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.get(clusterSettings)); + } + } + /** * Updates index settings to enable remote store by default based on cluster level settings * @param settingsBuilder index settings builder to be updated with relevant settings diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 4b63a4ca5e42a..de7dc102939ce 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -48,7 +48,6 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.common.Strings; import org.opensearch.index.translog.Translog; -import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.ingest.IngestService; import org.opensearch.node.Node; @@ -765,13 +764,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti nodeName = Node.NODE_NAME_SETTING.get(settings); this.indexMetadata = indexMetadata; numberOfShards = settings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, null); - if (FeatureFlags.isEnabled(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL) - && indexMetadata.isSystem() == false - && settings.get(IndexMetadata.SETTING_REPLICATION_TYPE) == null) { - replicationType = IndicesService.CLUSTER_REPLICATION_TYPE_SETTING.get(settings); - } else { - replicationType = IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings); - } + replicationType = IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings); isRemoteStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, false); isRemoteTranslogStoreEnabled = settings.getAsBoolean(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_ENABLED, false); remoteStoreTranslogRepository = settings.get(IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 00d496fde0434..cfc83328d1ee8 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -817,7 +817,8 @@ public void testAggregateSettingsAppliesSettingsFromTemplatesAndRequest() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(aggregatedIndexSettings.get("template_setting"), equalTo("value1")); @@ -879,7 +880,8 @@ public void testRequestDataHavePriorityOverTemplateData() throws Exception { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(resolvedAliases.get(0).getSearchRouting(), equalTo("fromRequest")); @@ -901,7 +903,8 @@ public void testDefaultSettings() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("1")); @@ -916,7 +919,8 @@ public void testSettingsFromClusterState() { Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 15).build(), IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(aggregatedIndexSettings.get(SETTING_NUMBER_OF_SHARDS), equalTo("15")); @@ -953,7 +957,8 @@ public void testTemplateOrder() throws Exception { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); List resolvedAliases = resolveAndValidateAliases( request.index(), @@ -992,7 +997,8 @@ public void testAggregateIndexSettingsIgnoresTemplatesOnCreateFromSourceIndex() Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertThat(aggregatedIndexSettings.get("templateSetting"), is(nullValue())); @@ -1214,7 +1220,8 @@ public void testRemoteStoreNoUserOverrideConflictingReplicationTypeIndexSettings settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ) ); assertThat( @@ -1245,7 +1252,8 @@ public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettin settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1277,7 +1285,8 @@ public void testRemoteStoreNoUserOverrideIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1311,9 +1320,10 @@ public void testRemoteStoreDisabledByUserIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); - verifyRemoteStoreIndexSettings(indexSettings, "false", null, null, null, null, null); + verifyRemoteStoreIndexSettings(indexSettings, "false", null, null, null, ReplicationType.SEGMENT.toString(), null); } public void testRemoteStoreTranslogDisabledByUserIndexSettings() { @@ -1337,7 +1347,8 @@ public void testRemoteStoreTranslogDisabledByUserIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings(indexSettings, "true", "my-segment-repo-1", "false", null, ReplicationType.SEGMENT.toString(), null); } @@ -1366,7 +1377,8 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1400,7 +1412,8 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings( indexSettings, @@ -1434,7 +1447,8 @@ public void testRemoteStoreOverrideReplicationTypeIndexSettings() { settings, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); verifyRemoteStoreIndexSettings(indexSettings, null, null, null, null, ReplicationType.DOCUMENT.toString(), null); } @@ -1508,7 +1522,8 @@ public void testSoftDeletesDisabledIsRejected() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); }); assertThat( @@ -1537,7 +1552,8 @@ public void testValidateTranslogRetentionSettings() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertWarnings( "Translog retention settings [index.translog.retention.age] " @@ -1584,7 +1600,8 @@ public void testDeprecatedSimpleFSStoreSettings() { Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), - Collections.emptySet() + Collections.emptySet(), + false ); assertWarnings( "[simplefs] is deprecated and will be removed in 2.0. Use [niofs], which offers equal " @@ -1592,6 +1609,92 @@ public void testDeprecatedSimpleFSStoreSettings() { ); } + public void testClusterReplicationSetting() { + Settings settings = Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build(); + FeatureFlagSetter.set(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL); + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + Settings indexSettings = aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + false + ); + assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); + } + + public void testIndexSettingOverridesClusterReplicationSetting() { + Settings settings = Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build(); + FeatureFlagSetter.set(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL); + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + final Settings.Builder requestSettings = Settings.builder(); + // Set index setting replication type as DOCUMENT + requestSettings.put(SETTING_REPLICATION_TYPE, ReplicationType.DOCUMENT); + request.settings(requestSettings.build()); + Settings indexSettings = aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + false + ); + // Verify if index setting overrides cluster replication setting + assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); + } + + public void testHiddenIndexUsesDocumentReplication() { + Settings settings = Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build(); + FeatureFlagSetter.set(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL); + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + final Settings.Builder requestSettings = Settings.builder(); + // Set index setting replication type as DOCUMENT + requestSettings.put("index.hidden", true); + request.settings(requestSettings.build()); + Settings indexSettings = aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + false + ); + // Verify replication type is Document Replication + assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); + } + + public void testSystemIndexUsesDocumentReplication() { + Settings settings = Settings.builder().put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT).build(); + FeatureFlagSetter.set(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL); + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + final Settings.Builder requestSettings = Settings.builder(); + request.settings(requestSettings.build()); + // set isSystemIndex parameter as true + Settings indexSettings = aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + settings, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet(), + true + ); + // Verify replication type is Document Replication + assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE)); + } + private IndexTemplateMetadata addMatchingTemplate(Consumer configurator) { IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*"); configurator.accept(builder); @@ -1653,4 +1756,5 @@ private void verifyRemoteStoreIndexSettings( assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY)); assertEquals(translogBufferInterval, indexSettings.get(SETTING_REMOTE_TRANSLOG_BUFFER_INTERVAL)); } + }