Merge branch 'main' into noop-rrt

CHANGELOG.md

@@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054))
 - [Workload Management] Add Get QueryGroup API Logic ([14709](https://github.com/opensearch-project/OpenSearch/pull/14709))
 - [Workload Management] Add Settings for Workload Management feature ([#15028](https://github.com/opensearch-project/OpenSearch/pull/15028))
+- [Workload Management] Add Update QueryGroup API Logic ([#14775](https://github.com/opensearch-project/OpenSearch/pull/14775))
 - [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897))
 - Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774))
 - Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153))
@@ -38,15 +39,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428)))
 - Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494))
 - Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290))
-- Add fieldType to AbstractQueryBuilder and FieldSortBuilder ([#15328](https://github.com/opensearch-project/OpenSearch/pull/15328)))
-- [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410))
+- [Reader Writer Separation] Add experimental search replica shard type to achieve reader writer separation ([#15237](https://github.com/opensearch-project/OpenSearch/pull/15237))
 - [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788))
 - [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527))
+- Add support for pluggable deciders for concurrent search ([#15363](https://github.com/opensearch-project/OpenSearch/pull/15363))
 - Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426))
 - [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291)))
 - Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409))
 - Add prefix support to hashed prefix & infix path types on remote store ([#15557](https://github.com/opensearch-project/OpenSearch/pull/15557))
+- Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568))
 - [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218))
+- Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010))
+- ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508))
+- Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131))
+- Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471))
+- MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637))
 
 ### Dependencies
 - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))
@@ -100,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393))
 - Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394))
 - Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069))
+- Fix terms query on wildcard field returns nothing ([#15607](https://github.com/opensearch-project/OpenSearch/pull/15607))
 
 ### Security
 

client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java

@@ -238,7 +238,7 @@ public void testSnapshotsStatus() {
         boolean ignoreUnavailable = randomBoolean();
         String endpoint = "/_snapshot/" + repository + "/" + snapshotNames.toString() + "/_status";
 
-        SnapshotsStatusRequest snapshotsStatusRequest = new SnapshotsStatusRequest(repository, snapshots, indices);
+        SnapshotsStatusRequest snapshotsStatusRequest = (new SnapshotsStatusRequest(repository, snapshots)).indices(indices);
         RequestConvertersTests.setRandomClusterManagerTimeout(snapshotsStatusRequest, expectedParams);
         snapshotsStatusRequest.ignoreUnavailable(ignoreUnavailable);
         expectedParams.put("ignore_unavailable", Boolean.toString(ignoreUnavailable));

libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java

@@ -969,9 +969,13 @@ public <T extends Writeable> void writeOptionalArray(@Nullable T[] array) throws
     }
 
     public void writeOptionalWriteable(@Nullable Writeable writeable) throws IOException {
+        writeOptionalWriteable((out, writable) -> writable.writeTo(out), writeable);
+    }
+
+    public <T extends Writeable> void writeOptionalWriteable(final Writer<T> writer, @Nullable T writeable) throws IOException {
         if (writeable != null) {
             writeBoolean(true);
-            writeable.writeTo(this);
+            writer.write(this, writeable);
         } else {
             writeBoolean(false);
         }

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

@@ -49,6 +49,7 @@
 import org.opensearch.common.xcontent.support.XContentMapValues;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.core.xcontent.XContentParser.Token;
+import org.opensearch.index.compositeindex.datacube.DimensionType;
 import org.opensearch.index.fielddata.FieldData;
 import org.opensearch.index.fielddata.IndexFieldData;
 import org.opensearch.index.fielddata.IndexNumericFieldData;
@@ -71,10 +72,12 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Supplier;
 
 /** A {@link FieldMapper} for scaled floats. Values are internally multiplied
- *  by a scaling factor and rounded to the closest long. */
+ * by a scaling factor and rounded to the closest long.
+ */
 public class ScaledFloatFieldMapper extends ParametrizedFieldMapper {
 
     public static final String CONTENT_TYPE = "scaled_float";
@@ -162,11 +165,21 @@ public ScaledFloatFieldMapper build(BuilderContext context) {
             );
             return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo.build(), this);
         }
+
+        @Override
+        public Optional<DimensionType> getSupportedDataCubeDimensionType() {
+            return Optional.of(DimensionType.NUMERIC);
+        }
+
+        @Override
+        public boolean isDataCubeMetricSupported() {
+            return true;
+        }
     }
 
     public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings()));
 
-    public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder {
+    public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder, FieldValueConverter {
 
         private final double scalingFactor;
         private final Double nullValue;
@@ -340,6 +353,12 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
         private double scale(Object input) {
             return new BigDecimal(Double.toString(parse(input))).multiply(BigDecimal.valueOf(scalingFactor)).doubleValue();
         }
+
+        @Override
+        public double toDoubleValue(long value) {
+            double inverseScalingFactor = 1d / scalingFactor;
+            return value * inverseScalingFactor;
+        }
     }
 
     private final Explicit<Boolean> ignoreMalformed;

modules/mapper-extras/src/main/java/org/opensearch/index/query/RankFeatureQueryBuilder.java

@@ -400,11 +400,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.endObject();
     }
 
-    @Override
-    public final String fieldName() {
-        return getDefaultFieldName();
-    }
-
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
         final MappedFieldType ft = context.fieldMapper(field);

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -34,18 +34,24 @@
 
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
 import org.opensearch.plugins.Plugin;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 
 import static java.util.Collections.singletonList;
+import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX;
 import static org.hamcrest.Matchers.containsString;
 
 public class ScaledFloatFieldMapperTests extends MapperTestCase {
@@ -91,24 +97,112 @@ public void testExistsQueryDocValuesDisabled() throws IOException {
         assertParseMinimalWarnings();
     }
 
-    public void testDefaults() throws Exception {
-        XContentBuilder mapping = fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0));
+    @BeforeClass
+    public static void createMapper() {
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(STAR_TREE_INDEX, "true").build());
+    }
+
+    @AfterClass
+    public static void clearMapper() {
+        FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
+    }
+
+    public void testScaledFloatWithStarTree() throws Exception {
+
+        double scalingFactorField1 = randomDouble() * 100;
+        double scalingFactorField2 = randomDouble() * 100;
+        double scalingFactorField3 = randomDouble() * 100;
+
+        XContentBuilder mapping = getStarTreeMappingWithScaledFloat(scalingFactorField1, scalingFactorField2, scalingFactorField3);
         DocumentMapper mapper = createDocumentMapper(mapping);
-        assertEquals(mapping.toString(), mapper.mappingSource().toString());
+        assertTrue(mapping.toString().contains("startree"));
 
-        ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
-        IndexableField[] fields = doc.rootDoc().getFields("field");
+        long randomLongField1 = randomLong();
+        long randomLongField2 = randomLong();
+        long randomLongField3 = randomLong();
+        ParsedDocument doc = mapper.parse(
+            source(b -> b.field("field1", randomLongField1).field("field2", randomLongField2).field("field3", randomLongField3))
+        );
+        validateScaledFloatFields(doc, "field1", randomLongField1, scalingFactorField1);
+        validateScaledFloatFields(doc, "field2", randomLongField2, scalingFactorField2);
+        validateScaledFloatFields(doc, "field3", randomLongField3, scalingFactorField3);
+    }
+
+    @Override
+    protected Settings getIndexSettings() {
+        return Settings.builder()
+            .put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
+            .put(super.getIndexSettings())
+            .build();
+    }
+
+    private static void validateScaledFloatFields(ParsedDocument doc, String field, long value, double scalingFactor) {
+        IndexableField[] fields = doc.rootDoc().getFields(field);
         assertEquals(2, fields.length);
         IndexableField pointField = fields[0];
         assertEquals(1, pointField.fieldType().pointDimensionCount());
         assertFalse(pointField.fieldType().stored());
-        assertEquals(1230, pointField.numericValue().longValue());
+        assertEquals((long) (value * scalingFactor), pointField.numericValue().longValue());
         IndexableField dvField = fields[1];
         assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
-        assertEquals(1230, dvField.numericValue().longValue());
+        assertEquals((long) (value * scalingFactor), dvField.numericValue().longValue());
         assertFalse(dvField.fieldType().stored());
     }
 
+    private XContentBuilder getStarTreeMappingWithScaledFloat(
+        double scalingFactorField1,
+        double scalingFactorField2,
+        double scalingFactorField3
+    ) throws IOException {
+        return topMapping(b -> {
+            b.startObject("composite");
+            b.startObject("startree");
+            b.field("type", "star_tree");
+            b.startObject("config");
+            b.field("max_leaf_docs", 100);
+            b.startArray("ordered_dimensions");
+            b.startObject();
+            b.field("name", "field1");
+            b.endObject();
+            b.startObject();
+            b.field("name", "field2");
+            b.endObject();
+            b.endArray();
+            b.startArray("metrics");
+            b.startObject();
+            b.field("name", "field3");
+            b.startArray("stats");
+            b.value("sum");
+            b.value("value_count");
+            b.endArray();
+            b.endObject();
+            b.endArray();
+            b.endObject();
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            b.startObject("field1");
+            b.field("type", "scaled_float").field("scaling_factor", scalingFactorField1);
+            b.endObject();
+            b.startObject("field2");
+            b.field("type", "scaled_float").field("scaling_factor", scalingFactorField2);
+            b.endObject();
+            b.startObject("field3");
+            b.field("type", "scaled_float").field("scaling_factor", scalingFactorField3);
+            b.endObject();
+            b.endObject();
+        });
+    }
+
+    public void testDefaults() throws Exception {
+        XContentBuilder mapping = fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0));
+        DocumentMapper mapper = createDocumentMapper(mapping);
+        assertEquals(mapping.toString(), mapper.mappingSource().toString());
+
+        ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
+        validateScaledFloatFields(doc, "field", 123, 10.0);
+    }
+
     public void testMissingScalingFactor() {
         Exception e = expectThrows(
             MapperParsingException.class,

modules/parent-join/src/main/java/org/opensearch/join/query/HasChildQueryBuilder.java

@@ -264,11 +264,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.endObject();
     }
 
-    @Override
-    public final String fieldName() {
-        return getDefaultFieldName();
-    }
-
     public static HasChildQueryBuilder fromXContent(XContentParser parser) throws IOException {
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String childType = null;

modules/parent-join/src/main/java/org/opensearch/join/query/HasParentQueryBuilder.java

@@ -233,11 +233,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.endObject();
     }
 
-    @Override
-    public final String fieldName() {
-        return getDefaultFieldName();
-    }
-
     public static HasParentQueryBuilder fromXContent(XContentParser parser) throws IOException {
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String parentType = null;

modules/parent-join/src/main/java/org/opensearch/join/query/ParentIdQueryBuilder.java

@@ -130,11 +130,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.endObject();
     }
 
-    @Override
-    public final String fieldName() {
-        return getDefaultFieldName();
-    }
-
     public static ParentIdQueryBuilder fromXContent(XContentParser parser) throws IOException {
         float boost = AbstractQueryBuilder.DEFAULT_BOOST;
         String type = null;

modules/percolator/src/main/java/org/opensearch/percolator/PercolateQueryBuilder.java

@@ -356,11 +356,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         builder.endObject();
     }
 
-    @Override
-    public final String fieldName() {
-        return getDefaultFieldName();
-    }
-
     private static final ConstructingObjectParser<PercolateQueryBuilder, Void> PARSER = new ConstructingObjectParser<>(NAME, args -> {
         String field = (String) args[0];
         BytesReference document = (BytesReference) args[1];

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/WorkloadManagementPlugin.java

@@ -24,9 +24,12 @@
 import org.opensearch.plugin.wlm.action.TransportCreateQueryGroupAction;
 import org.opensearch.plugin.wlm.action.TransportDeleteQueryGroupAction;
 import org.opensearch.plugin.wlm.action.TransportGetQueryGroupAction;
+import org.opensearch.plugin.wlm.action.TransportUpdateQueryGroupAction;
+import org.opensearch.plugin.wlm.action.UpdateQueryGroupAction;
 import org.opensearch.plugin.wlm.rest.RestCreateQueryGroupAction;
 import org.opensearch.plugin.wlm.rest.RestDeleteQueryGroupAction;
 import org.opensearch.plugin.wlm.rest.RestGetQueryGroupAction;
+import org.opensearch.plugin.wlm.rest.RestUpdateQueryGroupAction;
 import org.opensearch.plugin.wlm.service.QueryGroupPersistenceService;
 import org.opensearch.plugins.ActionPlugin;
 import org.opensearch.plugins.Plugin;
@@ -52,7 +55,8 @@ public WorkloadManagementPlugin() {}
         return List.of(
             new ActionPlugin.ActionHandler<>(CreateQueryGroupAction.INSTANCE, TransportCreateQueryGroupAction.class),
             new ActionPlugin.ActionHandler<>(GetQueryGroupAction.INSTANCE, TransportGetQueryGroupAction.class),
-            new ActionPlugin.ActionHandler<>(DeleteQueryGroupAction.INSTANCE, TransportDeleteQueryGroupAction.class)
+            new ActionPlugin.ActionHandler<>(DeleteQueryGroupAction.INSTANCE, TransportDeleteQueryGroupAction.class),
+            new ActionPlugin.ActionHandler<>(UpdateQueryGroupAction.INSTANCE, TransportUpdateQueryGroupAction.class)
         );
     }
 
@@ -66,7 +70,12 @@ public List<RestHandler> getRestHandlers(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<DiscoveryNodes> nodesInCluster
     ) {
-        return List.of(new RestCreateQueryGroupAction(), new RestGetQueryGroupAction(), new RestDeleteQueryGroupAction());
+        return List.of(
+            new RestCreateQueryGroupAction(),
+            new RestGetQueryGroupAction(),
+            new RestDeleteQueryGroupAction(),
+            new RestUpdateQueryGroupAction()
+        );
     }
 
     @Override

plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/action/CreateQueryGroupRequest.java

@@ -40,15 +40,15 @@ public class CreateQueryGroupRequest extends ActionRequest {
      * Constructor for CreateQueryGroupRequest
      * @param queryGroup - A {@link QueryGroup} object
      */
-    public CreateQueryGroupRequest(QueryGroup queryGroup) {
+    CreateQueryGroupRequest(QueryGroup queryGroup) {
         this.queryGroup = queryGroup;
     }
 
     /**
      * Constructor for CreateQueryGroupRequest
      * @param in - A {@link StreamInput} object
      */
-    public CreateQueryGroupRequest(StreamInput in) throws IOException {
+    CreateQueryGroupRequest(StreamInput in) throws IOException {
         super(in);
         queryGroup = new QueryGroup(in);
     }