diff --git a/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/FullClusterRestartIT.java index 3c6232569f3f1..81b6408b213f6 100644 --- a/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/opensearch/upgrades/FullClusterRestartIT.java @@ -696,7 +696,6 @@ public void testEmptyShard() throws IOException { // before timing out .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster - settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); if (randomBoolean()) { settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), "-1"); } @@ -828,6 +827,11 @@ public void testSnapshotRestore() throws IOException { if (isRunningAgainstOldCluster()) { // Create the index count = between(200, 300); + Settings.Builder settings = Settings.builder(); + if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } + createIndex(index, settings.build()); indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject()); } else { count = countOfIndexedRandomDocuments(); @@ -1337,7 +1341,9 @@ public void testOperationBasedRecovery() throws Exception { final Settings.Builder settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1); - settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + if (minimumNodeVersion().before(Version.V_2_0_0)) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } createIndex(index, settings.build()); ensureGreen(index); int committedDocs = randomIntBetween(100, 200); @@ -1392,7 +1398,9 @@ public void testRecoveryWithTranslogRetentionDisabled() throws Exception { final Settings.Builder settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1); - settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + if (minimumNodeVersion().before(Version.V_2_0_0)) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } if (randomBoolean()) { settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), "-1"); } @@ -1424,6 +1432,72 @@ public void testRecoveryWithTranslogRetentionDisabled() throws Exception { assertTotalHits(numDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search")))); } + public void testResize() throws Exception { + int numDocs; + if (isRunningAgainstOldCluster()) { + final Settings.Builder settings = Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 3) + .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1); + if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false); + } + final String mappings = randomBoolean() ? "\"_source\": { \"enabled\": false}" : null; + createIndex(index, settings.build(), mappings); + numDocs = randomIntBetween(10, 1000); + for (int i = 0; i < numDocs; i++) { + indexDocument(Integer.toString(i)); + if (rarely()) { + flush(index, randomBoolean()); + } + } + saveInfoDocument("num_doc_" + index, Integer.toString(numDocs)); + ensureGreen(index); + } else { + ensureGreen(index); + numDocs = Integer.parseInt(loadInfoDocument("num_doc_" + index)); + int moreDocs = randomIntBetween(0, 100); + for (int i = 0; i < moreDocs; i++) { + indexDocument(Integer.toString(numDocs + i)); + if (rarely()) { + flush(index, randomBoolean()); + } + } + Request updateSettingsRequest = new Request("PUT", "/" + index + "/_settings"); + updateSettingsRequest.setJsonEntity("{\"settings\": {\"index.blocks.write\": true}}"); + client().performRequest(updateSettingsRequest); + { + final String target = index + "_shrunken"; + Request shrinkRequest = new Request("PUT", "/" + index + "/_shrink/" + target); + Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1); + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); + } + shrinkRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}"); + client().performRequest(shrinkRequest); + ensureGreenLongWait(target); + assertNumHits(target, numDocs + moreDocs, 1); + } + { + final String target = index + "_split"; + Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 6); + if (randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true); + } + Request splitRequest = new Request("PUT", "/" + index + "/_split/" + target); + splitRequest.setJsonEntity("{\"settings\":" + Strings.toString(settings.build()) + "}"); + client().performRequest(splitRequest); + ensureGreenLongWait(target); + assertNumHits(target, numDocs + moreDocs, 6); + } + { + final String target = index + "_cloned"; + client().performRequest(new Request("PUT", "/" + index + "/_clone/" + target)); + ensureGreenLongWait(target); + assertNumHits(target, numDocs + moreDocs, 3); + } + } + } + @SuppressWarnings("unchecked") public void testSystemIndexMetadataIsUpgraded() throws Exception { final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " + @@ -1529,12 +1603,13 @@ public void testSystemIndexMetadataIsUpgraded() throws Exception { } public void testEnableSoftDeletesOnRestore() throws Exception { + assumeTrue("soft deletes must be enabled on 2.0+", getOldClusterVersion().before(Version.V_2_0_0)); final String snapshot = "snapshot-" + index; if (isRunningAgainstOldCluster()) { final Settings.Builder settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1); - settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false); createIndex(index, settings.build()); ensureGreen(index); int numDocs = randomIntBetween(0, 100); diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java index 5507a5a221473..0e0dedf00b929 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/RecoveryIT.java @@ -339,7 +339,7 @@ public void testRecovery() throws Exception { // before timing out .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster - if (randomBoolean()) { + if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) { settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); } createIndex(index, settings.build()); @@ -370,8 +370,10 @@ public void testRetentionLeasesEstablishedWhenPromotingPrimary() throws Exceptio .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5)) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(1, 2)) // triggers nontrivial promotion .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") - .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster + if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } createIndex(index, settings.build()); int numDocs = randomInt(10); indexDocs(index, 0, numDocs); @@ -391,8 +393,10 @@ public void testRetentionLeasesEstablishedWhenRelocatingPrimary() throws Excepti .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), between(1, 5)) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), between(0, 1)) .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms") - .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0") // fail faster - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster + if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } createIndex(index, settings.build()); int numDocs = randomInt(10); indexDocs(index, 0, numDocs); @@ -713,10 +717,13 @@ private void assertNoopRecoveries(String indexName, Predicate targetNode public void testOperationBasedRecovery() throws Exception { final String index = "test_operation_based_recovery"; if (CLUSTER_TYPE == ClusterType.OLD) { - createIndex(index, Settings.builder() + final Settings.Builder settings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build()); + .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 2); + if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) { + settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); + } + createIndex(index, settings.build()); ensureGreen(index); indexDocs(index, 0, randomIntBetween(100, 200)); flush(index, randomBoolean()); @@ -791,7 +798,7 @@ public void testSoftDeletesDisabledWarning() throws Exception { if (CLUSTER_TYPE == ClusterType.OLD) { boolean softDeletesEnabled = true; Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1); - if (randomBoolean()) { + if (minimumNodeVersion().before(Version.V_2_0_0) && randomBoolean()) { softDeletesEnabled = randomBoolean(); settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), softDeletesEnabled); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml index c8ede7cd90284..30419f7738bbf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -155,19 +155,3 @@ - match: { error.type: "illegal_argument_exception" } - match: { error.reason: "The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." } ---- -"Create index without soft deletes": - - skip: - version: " - 7.5.99" - reason: "indices without soft deletes are deprecated in 7.6" - features: "allowed_warnings" - - - do: - allowed_warnings: - - Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions. - Please do not specify value for setting [index.soft_deletes.enabled] of index [test_index]. - indices.create: - index: test_index - body: - settings: - soft_deletes.enabled: false diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml index a308f798df692..711bdc8d4ca3d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/20_translog.yml @@ -1,83 +1,5 @@ --- -"Translog retention without soft_deletes": - - skip: - version: " - 7.5.99" - reason: "indices without soft deletes are deprecated in 7.6" - features: "allowed_warnings" - - - do: - indices.create: - index: test - body: - settings: - soft_deletes.enabled: false - allowed_warnings: - - Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions. - Please do not specify value for setting [index.soft_deletes.enabled] of index [test]. - - do: - cluster.health: - wait_for_no_initializing_shards: true - wait_for_events: languid - - do: - indices.stats: - metric: [ translog ] - - set: { indices.test.primaries.translog.size_in_bytes: creation_size } - - - do: - index: - index: test - id: 1 - body: { "foo": "bar" } - - - do: - indices.stats: - metric: [ translog ] - - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.operations: 1 } -# we can't check this yet as creation size will contain two empty translog generations. A single -# non empty generation with one op may be smaller or larger than that. -# - gt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.uncommitted_operations: 1 } - - - do: - indices.flush: - index: test - - - do: - indices.stats: - metric: [ translog ] - - gt: { indices.test.primaries.translog.size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.operations: 1 } - ## creation translog size has some overhead due to an initial empty generation that will be trimmed later - - lt: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.uncommitted_operations: 0 } - - - do: - indices.put_settings: - index: test - body: - index.translog.retention.size: -1 - index.translog.retention.age: -1 - - - do: - indices.flush: - index: test - force: true # force flush as we don't have pending ops - - - do: - indices.stats: - metric: [ translog ] - ## creation translog size has some overhead due to an initial empty generation that will be trimmed later - - lte: { indices.test.primaries.translog.size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.operations: 0 } - - lte: { indices.test.primaries.translog.uncommitted_size_in_bytes: $creation_size } - - match: { indices.test.primaries.translog.uncommitted_operations: 0 } - ---- -"Translog retention with soft_deletes": - - skip: - version: " - 7.3.99" - reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4" +"Translog retention": - do: indices.create: index: test @@ -179,70 +101,7 @@ - gte: { indices.test.primaries.translog.earliest_last_modified_age: 0 } --- -"Translog stats on closed indices without soft-deletes": - - skip: - version: " - 7.5.99" - reason: "indices without soft deletes are deprecated in 7.6" - features: "allowed_warnings" - - - do: - indices.create: - index: test - body: - settings: - soft_deletes.enabled: false - routing.rebalance.enable: "none" # prevents shard relocations while we are closing an index - allowed_warnings: - - Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions. - Please do not specify value for setting [index.soft_deletes.enabled] of index [test]. - - - do: - cluster.health: - wait_for_no_initializing_shards: true - wait_for_events: languid - - do: - index: - index: test - id: 1 - body: { "foo": "bar" } - - - do: - index: - index: test - id: 2 - body: { "foo": "bar" } - - - do: - index: - index: test - id: 3 - body: { "foo": "bar" } - - - do: - indices.stats: - metric: [ translog ] - - match: { indices.test.primaries.translog.operations: 3 } - - match: { indices.test.primaries.translog.uncommitted_operations: 3 } - - - do: - indices.close: - index: test - wait_for_active_shards: 1 - - is_true: acknowledged - - - do: - indices.stats: - metric: [ translog ] - expand_wildcards: all - forbid_closed_indices: false - - match: { indices.test.primaries.translog.operations: 3 } - - match: { indices.test.primaries.translog.uncommitted_operations: 0 } - ---- -"Translog stats on closed indices with soft-deletes": - - skip: - version: " - 7.3.99" - reason: "start ignoring translog retention policy with soft-deletes enabled in 7.4" +"Translog stats on closed indices": - do: indices.create: index: test diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java index 23432b4bd15b1..178fe49b1f0e7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/gateway/ReplicaShardAllocatorIT.java @@ -98,7 +98,6 @@ public void testPreferCopyCanPerformNoopRecovery() throws Exception { .prepareCreate(indexName) .setSettings( Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) .put(IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.getKey(), 1.0f) @@ -280,7 +279,6 @@ public void testFullClusterRestartPerformNoopRecovery() throws Exception { .prepareCreate(indexName) .setSettings( Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb") .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) @@ -342,7 +340,6 @@ public void testPreferCopyWithHighestMatchingOperations() throws Exception { .prepareCreate(indexName) .setSettings( Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), randomIntBetween(10, 100) + "kb") .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) @@ -473,7 +470,6 @@ public void testPeerRecoveryForClosedIndices() throws Exception { Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms") .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms") .build() diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index 4087dd9f9290b..59e719c3e16a8 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -910,7 +910,6 @@ public void testLimitNumberOfRetainedTranslogFiles() throws Exception { Settings.Builder settings = Settings.builder() .put(SETTING_NUMBER_OF_SHARDS, 1) .put(SETTING_NUMBER_OF_REPLICAS, 0) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) .put(IndexSettings.INDEX_TRANSLOG_RETENTION_TOTAL_FILES_SETTING.getKey(), translogRetentionTotalFiles); if (randomBoolean()) { settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), new ByteSizeValue(between(1, 1024 * 1024))); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java index ae99949c62b3e..e22bae9cf3ad1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/state/OpenCloseIndexIT.java @@ -49,7 +49,6 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexNotFoundException; -import org.opensearch.index.IndexSettings; import org.opensearch.index.query.QueryBuilders; import org.opensearch.rest.RestStatus; import org.opensearch.test.OpenSearchIntegTestCase; @@ -395,9 +394,6 @@ public void testOpenCloseIndexWithBlocks() { public void testTranslogStats() throws Exception { final String indexName = "test"; createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()); - boolean softDeletesEnabled = IndexSettings.INDEX_SOFT_DELETES_SETTING.get( - client().admin().indices().prepareGetSettings(indexName).get().getIndexToSettings().get(indexName) - ); final int nbDocs = randomIntBetween(0, 50); int uncommittedOps = 0; @@ -419,7 +415,7 @@ public void testTranslogStats() throws Exception { assertThat(stats.getIndex(indexName), notNullValue()); assertThat( stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), - equalTo(softDeletesEnabled ? uncommittedTranslogOps : nbDocs) + equalTo(uncommittedTranslogOps) ); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(uncommittedTranslogOps)); }); @@ -435,10 +431,7 @@ public void testTranslogStats() throws Exception { .setTranslog(true) .get(); assertThat(stats.getIndex(indexName), notNullValue()); - assertThat( - stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), - equalTo(softDeletesEnabled ? 0 : nbDocs) - ); + assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().estimatedNumberOfOperations(), equalTo(0)); assertThat(stats.getIndex(indexName).getPrimaries().getTranslog().getUncommittedOperations(), equalTo(0)); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java index c37066962cc11..6949409ae5f63 100644 --- a/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/recovery/RelocationIT.java @@ -749,7 +749,6 @@ public void testRelocationEstablishedPeerRecoveryRetentionLeases() throws Except .prepareCreate(indexName) .setSettings( Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomIntBetween(0, halfNodes - 1)) .put("index.routing.allocation.include.color", "blue") ) diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java index 441acc5ce0566..b8b2d4c1b665a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -102,7 +102,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; -import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.opensearch.index.shard.IndexShardTests.getEngineFromShard; import static org.opensearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -1416,12 +1415,8 @@ public void testSnapshotMoreThanOnce() throws InterruptedException { List shards = snapshotStatus.getShards(); for (SnapshotIndexShardStatus status : shards) { // we flush before the snapshot such that we have to process the segments_N files plus the .del file - if (INDEX_SOFT_DELETES_SETTING.get(settings)) { - // soft-delete generates DV files. - assertThat(status.getStats().getProcessedFileCount(), greaterThan(2)); - } else { - assertThat(status.getStats().getProcessedFileCount(), equalTo(2)); - } + // soft-delete generates DV files. + assertThat(status.getStats().getProcessedFileCount(), greaterThan(2)); } } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 5f74c783577d8..7ef1248f7d3e1 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -929,13 +929,11 @@ static Settings aggregateIndexSettings( * that will be used to create this index. */ shardLimitValidator.validateShardLimit(indexSettings, currentState); - if (indexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) { - DEPRECATION_LOGGER.deprecate( - "soft_deletes_disabled", - "Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions. " - + "Please do not specify value for setting [index.soft_deletes.enabled] of index [" - + request.index() - + "]." + if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) == false + && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(indexSettings).onOrAfter(Version.V_2_0_0)) { + throw new IllegalArgumentException( + "Creating indices with soft-deletes disabled is no longer supported. " + + "Please do not specify a value for setting [index.soft_deletes.enabled]." ); } validateTranslogRetentionSettings(indexSettings); diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 8da77ab1d0012..572caecc3671f 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -353,11 +353,11 @@ public final class IndexSettings { /** * Specifies if the index should use soft-delete instead of hard-delete for update/delete operations. - * Soft-deletes is enabled by default for 7.0+ indices. + * Soft-deletes is enabled by default for Legacy 7.x and 1.x indices and mandatory for 2.0+ indices. */ public static final Setting INDEX_SOFT_DELETES_SETTING = Setting.boolSetting( "index.soft_deletes.enabled", - settings -> Boolean.toString(IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(LegacyESVersion.V_7_0_0)), + true, Property.IndexScope, Property.Final ); @@ -667,6 +667,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mergeSchedulerConfig = new MergeSchedulerConfig(this); gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis(); softDeleteEnabled = scopedSettings.get(INDEX_SOFT_DELETES_SETTING); + assert softDeleteEnabled || version.before(Version.V_2_0_0) : "soft deletes must be enabled in version " + version; softDeleteRetentionOperations = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING); retentionLeaseMillis = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING).millis(); warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 25bd128fbdc53..45fc92ab66062 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -1096,7 +1096,6 @@ public void testBuildIndexMetadata() { Settings indexSettings = Settings.builder() .put("index.version.created", Version.CURRENT) - .put(INDEX_SOFT_DELETES_SETTING.getKey(), false) .put(SETTING_NUMBER_OF_REPLICAS, 0) .put(SETTING_NUMBER_OF_SHARDS, 1) .build(); @@ -1112,7 +1111,6 @@ public void testBuildIndexMetadata() { false ); - assertThat(indexMetadata.getSettings().getAsBoolean(INDEX_SOFT_DELETES_SETTING.getKey(), true), is(false)); assertThat(indexMetadata.getAliases().size(), is(1)); assertThat(indexMetadata.getAliases().keys().iterator().next().value, is("alias1")); assertThat("The source index primary term must be used", indexMetadata.primaryTerm(0), is(3L)); @@ -1154,36 +1152,27 @@ public void testGetIndexNumberOfRoutingShardsYieldsSourceNumberOfShards() { assertThat(targetRoutingNumberOfShards, is(6)); } - public void testSoftDeletesDisabledDeprecation() { - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); - aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - Settings.EMPTY, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() - ); - assertWarnings( - "Creating indices with soft-deletes disabled is deprecated and will be removed in future OpenSearch versions. " - + "Please do not specify value for setting [index.soft_deletes.enabled] of index [test]." - ); - request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); - if (randomBoolean()) { - request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), true).build()); - } - aggregateIndexSettings( - ClusterState.EMPTY_STATE, - request, - Settings.EMPTY, - null, - Settings.EMPTY, - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, - randomShardLimitService(), - Collections.emptySet() + public void testSoftDeletesDisabledIsRejected() { + final IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> { + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + request.settings(Settings.builder().put(INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); + aggregateIndexSettings( + ClusterState.EMPTY_STATE, + request, + Settings.EMPTY, + null, + Settings.EMPTY, + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + randomShardLimitService(), + Collections.emptySet() + ); + }); + assertThat( + error.getMessage(), + equalTo( + "Creating indices with soft-deletes disabled is no longer supported. " + + "Please do not specify a value for setting [index.soft_deletes.enabled]." + ) ); } diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 48c3094ee4b56..d67534bbfbddf 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -40,7 +40,6 @@ import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.ByteSizeUnit; import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.translog.Translog; @@ -758,38 +757,4 @@ public void testIgnoreTranslogRetentionSettingsIfSoftDeletesEnabled() { assertThat(indexSettings.getTranslogRetentionAge().millis(), equalTo(-1L)); assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L)); } - - public void testUpdateTranslogRetentionSettingsWithSoftDeletesDisabled() { - Settings.Builder settings = Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT); - - TimeValue ageSetting = TimeValue.timeValueHours(12); - if (randomBoolean()) { - ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000)); - settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting); - } - ByteSizeValue sizeSetting = new ByteSizeValue(512, ByteSizeUnit.MB); - if (randomBoolean()) { - sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024)); - settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting); - } - IndexMetadata metadata = newIndexMeta("index", settings.build()); - IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); - assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); - assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); - - Settings.Builder newSettings = Settings.builder().put(settings.build()); - if (randomBoolean()) { - ageSetting = randomBoolean() ? TimeValue.MINUS_ONE : TimeValue.timeValueMillis(randomIntBetween(0, 10000)); - newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), ageSetting); - } - if (randomBoolean()) { - sizeSetting = randomBoolean() ? new ByteSizeValue(-1) : new ByteSizeValue(randomIntBetween(0, 1024)); - newSettings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.getKey(), sizeSetting); - } - indexSettings.updateIndexMetadata(newIndexMeta("index", newSettings.build())); - assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); - assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); - } } diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index be955be882160..0a7dda8c60341 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -313,202 +313,6 @@ public void testVersionMapAfterAutoIDDocument() throws IOException { } } - public void testSegmentsWithoutSoftDeletes() throws Exception { - Settings settings = Settings.builder() - .put(defaultSettings.getSettings()) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .build(); - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( - IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build() - ); - try ( - Store store = createStore(); - InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null)) - ) { - List segments = engine.segments(false); - assertThat(segments.isEmpty(), equalTo(true)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(0L)); - assertThat(engine.segmentsStats(false, false).getMemoryInBytes(), equalTo(0L)); - - // create two docs and refresh - ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null); - Engine.Index first = indexForDoc(doc); - Engine.IndexResult firstResult = engine.index(first); - ParsedDocument doc2 = testParsedDocument("2", null, testDocumentWithTextField(), B_2, null); - Engine.Index second = indexForDoc(doc2); - Engine.IndexResult secondResult = engine.index(second); - assertThat(secondResult.getTranslogLocation(), greaterThan(firstResult.getTranslogLocation())); - engine.refresh("test"); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(1)); - SegmentsStats stats = engine.segmentsStats(false, false); - assertThat(stats.getCount(), equalTo(1L)); - assertThat(stats.getTermsMemoryInBytes(), greaterThan(0L)); - assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L)); - assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0L)); - assertThat(stats.getNormsMemoryInBytes(), greaterThan(0L)); - assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0L)); - assertThat(segments.get(0).isCommitted(), equalTo(false)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(2)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - assertThat(segments.get(0).ramTree, nullValue()); - engine.flush(); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(1)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(1L)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(2)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - ParsedDocument doc3 = testParsedDocument("3", null, testDocumentWithTextField(), B_3, null); - engine.index(indexForDoc(doc3)); - engine.refresh("test"); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(2)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L)); - assertThat(engine.segmentsStats(false, false).getTermsMemoryInBytes(), greaterThan(stats.getTermsMemoryInBytes())); - assertThat( - engine.segmentsStats(false, false).getStoredFieldsMemoryInBytes(), - greaterThan(stats.getStoredFieldsMemoryInBytes()) - ); - assertThat(engine.segmentsStats(false, false).getTermVectorsMemoryInBytes(), equalTo(0L)); - assertThat(engine.segmentsStats(false, false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes())); - assertThat(engine.segmentsStats(false, false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes())); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(2)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - engine.delete(new Engine.Delete("test", "1", newUid(doc), primaryTerm.get())); - engine.refresh("test"); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(2)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L)); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - engine.onSettingsChanged( - indexSettings.getTranslogRetentionAge(), - indexSettings.getTranslogRetentionSize(), - indexSettings.getSoftDeleteRetentionOperations() - ); - ParsedDocument doc4 = testParsedDocument("4", null, testDocumentWithTextField(), B_3, null); - engine.index(indexForDoc(doc4)); - engine.refresh("test"); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(3)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(3L)); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - assertThat(segments.get(2).isCommitted(), equalTo(false)); - assertThat(segments.get(2).isSearch(), equalTo(true)); - assertThat(segments.get(2).getNumDocs(), equalTo(1)); - assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(2).isCompound(), equalTo(true)); - - // internal refresh - lets make sure we see those segments in the stats - ParsedDocument doc5 = testParsedDocument("5", null, testDocumentWithTextField(), B_3, null); - engine.index(indexForDoc(doc5)); - engine.refresh("test", Engine.SearcherScope.INTERNAL, true); - - segments = engine.segments(false); - assertThat(segments.size(), equalTo(4)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L)); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - assertThat(segments.get(2).isCommitted(), equalTo(false)); - assertThat(segments.get(2).isSearch(), equalTo(true)); - assertThat(segments.get(2).getNumDocs(), equalTo(1)); - assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(2).isCompound(), equalTo(true)); - - assertThat(segments.get(3).isCommitted(), equalTo(false)); - assertThat(segments.get(3).isSearch(), equalTo(false)); - assertThat(segments.get(3).getNumDocs(), equalTo(1)); - assertThat(segments.get(3).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(3).isCompound(), equalTo(true)); - - // now refresh the external searcher and make sure it has the new segment - engine.refresh("test"); - segments = engine.segments(false); - assertThat(segments.size(), equalTo(4)); - assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L)); - assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); - assertThat(segments.get(0).isCommitted(), equalTo(true)); - assertThat(segments.get(0).isSearch(), equalTo(true)); - assertThat(segments.get(0).getNumDocs(), equalTo(1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); - assertThat(segments.get(0).isCompound(), equalTo(true)); - - assertThat(segments.get(1).isCommitted(), equalTo(false)); - assertThat(segments.get(1).isSearch(), equalTo(true)); - assertThat(segments.get(1).getNumDocs(), equalTo(1)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).isCompound(), equalTo(true)); - - assertThat(segments.get(2).isCommitted(), equalTo(false)); - assertThat(segments.get(2).isSearch(), equalTo(true)); - assertThat(segments.get(2).getNumDocs(), equalTo(1)); - assertThat(segments.get(2).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(2).isCompound(), equalTo(true)); - - assertThat(segments.get(3).isCommitted(), equalTo(false)); - assertThat(segments.get(3).isSearch(), equalTo(true)); - assertThat(segments.get(3).getNumDocs(), equalTo(1)); - assertThat(segments.get(3).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(3).isCompound(), equalTo(true)); - } - } - public void testVerboseSegments() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { List segments = engine.segments(true); @@ -1688,59 +1492,6 @@ public void testVersioningNewIndex() throws IOException { assertThat(indexResult.getVersion(), equalTo(1L)); } - public void testForceMergeWithoutSoftDeletes() throws IOException { - Settings settings = Settings.builder() - .put(defaultSettings.getSettings()) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - .build(); - IndexMetadata indexMetadata = IndexMetadata.builder(defaultSettings.getIndexMetadata()).settings(settings).build(); - try ( - Store store = createStore(); - Engine engine = createEngine( - config(IndexSettingsModule.newIndexSettings(indexMetadata), store, createTempDir(), new LogByteSizeMergePolicy(), null) - ) - ) { // use log MP here we test some behavior in ESMP - int numDocs = randomIntBetween(10, 100); - for (int i = 0; i < numDocs; i++) { - ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), B_1, null); - Engine.Index index = indexForDoc(doc); - engine.index(index); - engine.refresh("test"); - } - try (Engine.Searcher test = engine.acquireSearcher("test")) { - assertEquals(numDocs, test.getIndexReader().numDocs()); - } - engine.forceMerge(true, 1, false, false, false, UUIDs.randomBase64UUID()); - engine.refresh("test"); - assertEquals(engine.segments(true).size(), 1); - - ParsedDocument doc = testParsedDocument(Integer.toString(0), null, testDocument(), B_1, null); - Engine.Index index = indexForDoc(doc); - engine.delete(new Engine.Delete(index.type(), index.id(), index.uid(), primaryTerm.get())); - // expunge deletes - engine.forceMerge(true, 10, true, false, false, UUIDs.randomBase64UUID()); - engine.refresh("test"); - - assertEquals(engine.segments(true).size(), 1); - try (Engine.Searcher test = engine.acquireSearcher("test")) { - assertEquals(numDocs - 1, test.getIndexReader().numDocs()); - assertEquals(engine.config().getMergePolicy().toString(), numDocs - 1, test.getIndexReader().maxDoc()); - } - - doc = testParsedDocument(Integer.toString(1), null, testDocument(), B_1, null); - index = indexForDoc(doc); - engine.delete(new Engine.Delete(index.type(), index.id(), index.uid(), primaryTerm.get())); - // expunge deletes - engine.forceMerge(true, 10, false, false, false, UUIDs.randomBase64UUID()); - engine.refresh("test"); - assertEquals(engine.segments(true).size(), 1); - try (Engine.Searcher test = engine.acquireSearcher("test")) { - assertEquals(numDocs - 2, test.getIndexReader().numDocs()); - assertEquals(numDocs - 1, test.getIndexReader().maxDoc()); - } - } - } - /* * we are testing an edge case here where we have a fully deleted segment that is retained but has all it's IDs pruned away. */ @@ -6972,63 +6723,6 @@ public void testRebuildLocalCheckpointTrackerAndVersionMap() throws Exception { } } - public void testOpenSoftDeletesIndexWithSoftDeletesDisabled() throws Exception { - try (Store store = createStore()) { - Path translogPath = createTempDir(); - final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); - final IndexSettings softDeletesEnabled = IndexSettingsModule.newIndexSettings( - IndexMetadata.builder(defaultSettings.getIndexMetadata()) - .settings( - Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) - ) - .build() - ); - final List docs; - try ( - InternalEngine engine = createEngine( - config(softDeletesEnabled, store, translogPath, newMergePolicy(), null, null, globalCheckpoint::get) - ) - ) { - List ops = generateHistoryOnReplica(between(1, 100), randomBoolean(), randomBoolean(), randomBoolean()); - applyOperations(engine, ops); - engine.syncTranslog(); // to advance persisted checkpoint - globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint())); - engine.flush(); - docs = getDocIds(engine, true); - } - final IndexSettings softDeletesDisabled = IndexSettingsModule.newIndexSettings( - IndexMetadata.builder(defaultSettings.getIndexMetadata()) - .settings( - Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - ) - .build() - ); - EngineConfig config = config(softDeletesDisabled, store, translogPath, newMergePolicy(), null, null, globalCheckpoint::get); - try (InternalEngine engine = createEngine(config)) { - assertThat(getDocIds(engine, true), equalTo(docs)); - } - } - } - - public void testRequireSoftDeletesWhenAccessingChangesSnapshot() throws Exception { - try (Store store = createStore()) { - final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( - IndexMetadata.builder(defaultSettings.getIndexMetadata()) - .settings( - Settings.builder().put(defaultSettings.getSettings()).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) - ) - .build() - ); - try (InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), newMergePolicy(), null))) { - AssertionError error = expectThrows( - AssertionError.class, - () -> engine.newChangesSnapshot("test", createMapperService("test"), 0, randomNonNegativeLong(), randomBoolean()) - ); - assertThat(error.getMessage(), containsString("does not have soft-deletes enabled")); - } - } - } - void assertLuceneOperations(InternalEngine engine, long expectedAppends, long expectedUpdates, long expectedDeletes) { String message = "Lucene operations mismatched;" + " appends [actual:" @@ -7377,10 +7071,6 @@ public void testDeleteFailureSoftDeletesEnabled() throws IOException { runTestDeleteFailure(true, (engine, op) -> {}); } - public void testDeleteFailureSoftDeletesDisabled() throws IOException { - runTestDeleteFailure(false, (engine, op) -> {}); - } - private void runTestDeleteFailure( final boolean softDeletesEnabled, final CheckedBiConsumer consumer diff --git a/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java b/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java index 8023c19a9648c..c59d4849feffb 100644 --- a/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java +++ b/server/src/test/java/org/opensearch/index/replication/RecoveryDuringReplicationTests.java @@ -51,7 +51,6 @@ import org.opensearch.common.lucene.uid.Versions; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.internal.io.IOUtils; import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.engine.DocIdSeqNoAndSource; @@ -134,74 +133,6 @@ public void testIndexingDuringFileRecovery() throws Exception { } } - public void testRecoveryOfDisconnectedReplica() throws Exception { - Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build(); - try (ReplicationGroup shards = createGroup(1, settings)) { - shards.startAll(); - int docs = shards.indexDocs(randomInt(50)); - shards.flush(); - final IndexShard originalReplica = shards.getReplicas().get(0); - for (int i = 0; i < randomInt(2); i++) { - final int indexedDocs = shards.indexDocs(randomInt(5)); - docs += indexedDocs; - - final boolean flush = randomBoolean(); - if (flush) { - originalReplica.flush(new FlushRequest()); - } - } - - // simulate a background global checkpoint sync at which point we expect the global checkpoint to advance on the replicas - shards.syncGlobalCheckpoint(); - long globalCheckpointOnReplica = originalReplica.getLastSyncedGlobalCheckpoint(); - Optional safeCommitOnReplica = originalReplica.store() - .findSafeIndexCommit(globalCheckpointOnReplica); - assertTrue(safeCommitOnReplica.isPresent()); - shards.removeReplica(originalReplica); - - final int missingOnReplica = shards.indexDocs(randomInt(5)); - docs += missingOnReplica; - - final boolean translogTrimmed; - if (randomBoolean()) { - shards.flush(); - translogTrimmed = randomBoolean(); - if (translogTrimmed) { - final Translog translog = getTranslog(shards.getPrimary()); - translog.getDeletionPolicy().setRetentionAgeInMillis(0); - translog.trimUnreferencedReaders(); - } - } else { - translogTrimmed = false; - } - originalReplica.close("disconnected", false); - IOUtils.close(originalReplica.store()); - final IndexShard recoveredReplica = shards.addReplicaWithExistingPath( - originalReplica.shardPath(), - originalReplica.routingEntry().currentNodeId() - ); - shards.recoverReplica(recoveredReplica); - if (translogTrimmed && missingOnReplica > 0) { - // replica has something to catch up with, but since we trimmed the primary translog, we should fall back to full recovery - assertThat(recoveredReplica.recoveryState().getIndex().fileDetails(), not(empty())); - } else { - assertThat(recoveredReplica.recoveryState().getIndex().fileDetails(), empty()); - assertThat( - recoveredReplica.recoveryState().getTranslog().recoveredOperations(), - equalTo(Math.toIntExact(docs - 1 - safeCommitOnReplica.get().localCheckpoint)) - ); - assertThat( - recoveredReplica.recoveryState().getTranslog().totalLocal(), - equalTo(Math.toIntExact(globalCheckpointOnReplica - safeCommitOnReplica.get().localCheckpoint)) - ); - } - - docs += shards.indexDocs(randomInt(5)); - - shards.assertAllEqual(docs); - } - } - /* * Simulate a scenario with two replicas where one of the replicas receives an extra document, the other replica is promoted on primary * failure, the receiving replica misses the primary/replica re-sync and then recovers from the primary. We expect that a diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardRetentionLeaseTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardRetentionLeaseTests.java index 77093327dcc19..3de5175bac7c5 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardRetentionLeaseTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardRetentionLeaseTests.java @@ -337,40 +337,6 @@ public void testRetentionLeaseStats() throws IOException { } } - public void testRetentionLeasesActionsFailWithSoftDeletesDisabled() throws Exception { - IndexShard shard = newStartedShard(true, Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build()); - assertThat( - expectThrows( - AssertionError.class, - () -> shard.addRetentionLease( - randomAlphaOfLength(10), - randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE), - "test", - ActionListener.wrap(() -> {}) - ) - ).getMessage(), - equalTo("retention leases requires soft deletes but [index] does not have soft deletes enabled") - ); - assertThat( - expectThrows( - AssertionError.class, - () -> shard.renewRetentionLease( - randomAlphaOfLength(10), - randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE), - "test" - ) - ).getMessage(), - equalTo("retention leases requires soft deletes but [index] does not have soft deletes enabled") - ); - assertThat( - expectThrows(AssertionError.class, () -> shard.removeRetentionLease(randomAlphaOfLength(10), ActionListener.wrap(() -> {}))) - .getMessage(), - equalTo("retention leases requires soft deletes but [index] does not have soft deletes enabled") - ); - shard.syncRetentionLeases(); - closeShards(shard); - } - private void assertRetentionLeases( final IndexShard indexShard, final int size, diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index e92cd14aa00e6..ee2cde5c57267 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -2344,12 +2344,7 @@ public void testPrimaryHandOffUpdatesLocalCheckpoint() throws IOException { indexDoc(primarySource, "_doc", Integer.toString(i)); } IndexShardTestCase.updateRoutingEntry(primarySource, primarySource.routingEntry().relocate(randomAlphaOfLength(10), -1)); - final IndexShard primaryTarget = newShard( - primarySource.routingEntry().getTargetRelocatingShard(), - Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), primarySource.indexSettings().isSoftDeleteEnabled()) - .build() - ); + final IndexShard primaryTarget = newShard(primarySource.routingEntry().getTargetRelocatingShard()); updateMappings(primaryTarget, primarySource.indexSettings().getIndexMetadata()); recoverReplica(primaryTarget, primarySource, true); @@ -2367,12 +2362,11 @@ public void testPrimaryHandOffUpdatesLocalCheckpoint() throws IOException { /* This test just verifies that we fill up local checkpoint up to max seen seqID on primary recovery */ public void testRecoverFromStoreWithNoOps() throws IOException { - final Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).build(); - final IndexShard shard = newStartedShard(true, settings); + final IndexShard shard = newStartedShard(true); indexDoc(shard, "_doc", "0"); indexDoc(shard, "_doc", "1"); // start a replica shard and index the second doc - final IndexShard otherShard = newStartedShard(false, settings); + final IndexShard otherShard = newStartedShard(false); updateMappings(otherShard, shard.indexSettings().getIndexMetadata()); SourceToParse sourceToParse = new SourceToParse( shard.shardId().getIndexName(), @@ -2590,10 +2584,7 @@ public void testRecoveryFailsAfterMovingToRelocatedState() throws InterruptedExc public void testRestoreShard() throws IOException { final IndexShard source = newStartedShard(true); - IndexShard target = newStartedShard( - true, - Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), source.indexSettings().isSoftDeleteEnabled()).build() - ); + IndexShard target = newStartedShard(true); indexDoc(source, "_doc", "0"); EngineTestCase.generateNewSeqNo(source.getEngine()); // create a gap in the history diff --git a/server/src/test/java/org/opensearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/recovery/RecoverySourceHandlerTests.java index 960929ee7af9a..720356cb49588 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RecoverySourceHandlerTests.java @@ -661,7 +661,6 @@ public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOE Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 5)) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5)) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random())) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random())) ); diff --git a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java index 6286921688e67..d105da6565897 100644 --- a/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java +++ b/server/src/test/java/org/opensearch/indices/recovery/RecoveryTests.java @@ -41,6 +41,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.store.AlreadyClosedException; import org.opensearch.ExceptionsHelper; +import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.flush.FlushRequest; import org.opensearch.action.bulk.BulkShardRequest; @@ -68,6 +69,7 @@ import org.opensearch.index.store.Store; import org.opensearch.index.translog.SnapshotMatchers; import org.opensearch.index.translog.Translog; +import org.opensearch.test.VersionUtils; import java.io.IOException; import java.util.HashMap; @@ -156,7 +158,10 @@ public void testRecoveryWithOutOfOrderDeleteWithTranslog() throws Exception { * - index #5 * - If flush and the translog retention disabled, delete #1 will be removed while index #0 is still retained and replayed. */ - Settings settings = Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build(); + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_2_0_0)) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) + .build(); try (ReplicationGroup shards = createGroup(1, settings)) { shards.startAll(); // create out of order delete and index op on replica diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java index 9334d88717505..ac02bf92a9072 100644 --- a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java @@ -200,11 +200,7 @@ protected Settings indexSettings() { IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(), between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY)) ) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put( - IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000) - ) + .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)) .build(); } diff --git a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java index 97443f6cba811..ee92e3b134394 100644 --- a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java @@ -152,11 +152,7 @@ protected IndexMetadata buildIndexMetadata(int replicas, Settings indexSettings, .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put( - IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000) - ) + .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)) .put(indexSettings) .build(); IndexMetadata.Builder metadata = IndexMetadata.builder(index.getName()).settings(settings).primaryTerm(0, randomIntBetween(1, 100)); diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 2124e309f283f..54b3ffbfd3a1c 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -271,11 +271,7 @@ protected IndexShard newShard( .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put( - IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) : between(0, 1000) - ) + .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)) .put(settings) .build(); IndexMetadata.Builder metadata = IndexMetadata.builder(shardRouting.getIndexName()) diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 7cae8e30d0d74..cdfd1b15a09f3 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -748,7 +748,6 @@ public Settings indexSettings() { } // always default delayed allocation to 0 to make sure we have tests are not delayed builder.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), 0); - builder.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()); if (randomBoolean()) { builder.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)); } diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java index 0b5474b21cba6..960400019f7ea 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchSingleNodeTestCase.java @@ -113,16 +113,7 @@ protected void startNode(long seed) throws Exception { .preparePutTemplate("random-soft-deletes-template") .setPatterns(Collections.singletonList("*")) .setOrder(0) - .setSettings( - Settings.builder() - .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()) - .put( - IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), - randomBoolean() - ? IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.get(Settings.EMPTY) - : between(0, 1000) - ) - ) + .setSettings(Settings.builder().put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000))) .get(); }