From b81345088182dc9eadda6ec824133aa2f64b6069 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Fri, 28 Jun 2024 15:45:26 +0530 Subject: [PATCH] adding some more tests and fix Signed-off-by: Bharathwaj G --- distribution/src/config/jvm.options | 1 - .../index/mapper/StarTreeMapperIT.java | 76 ++++++++++++++ .../index/mapper/StarTreeMapperTests.java | 99 +++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index d6bc810355396..f0ac98faffda9 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -85,4 +85,3 @@ ${error.file} # HDFS ForkJoinPool.common() support by SecurityManager -Djava.util.concurrent.ForkJoinPool.common.threadFactory=org.opensearch.secure_sm.SecuredForkJoinWorkerThreadFactory --Dopensearch.experimental.feature.composite_index.star_tree.enabled=true diff --git a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java index 39deac33a0f06..8e5193b650868 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java @@ -19,6 +19,7 @@ import org.opensearch.index.compositeindex.datacube.DateDimension; import org.opensearch.index.compositeindex.datacube.MetricStat; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings; import org.opensearch.indices.IndicesService; import org.opensearch.test.OpenSearchIntegTestCase; import org.junit.After; @@ -89,6 +90,55 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool } } + private static XContentBuilder createMaxDimTestMapping() { + try { + return jsonBuilder().startObject() + .startObject("composite") + .startObject("startree-1") + .field("type", "star_tree") + .startObject("config") + .startArray("ordered_dimensions") + .startObject() + .field("name", "timestamp") + .startArray("calendar_intervals") + .value("day") + .value("month") + .endArray() + .endObject() + .startObject() + .field("name", "dim2") + .endObject() + .startObject() + .field("name", "dim3") + .endObject() + .endArray() + .startArray("metrics") + .startObject() + .field("name", "dim2") + .endObject() + .endArray() + .endObject() + .endObject() + .endObject() + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("dim2") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .startObject("dim3") + .field("type", "integer") + .field("doc_values", true) + .endObject() + .endObject() + .endObject(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + private static XContentBuilder createTestMappingWithoutStarTree(boolean invalidDim, boolean invalidMetric, boolean keywordDim) { try { return jsonBuilder().startObject() @@ -329,6 +379,32 @@ public void testInvalidDimCompositeIndex() { ); } + public void testMaxDimsCompositeIndex() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) + .setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DIMENSIONS_SETTING.getKey(), 2)) + .get() + ); + assertEquals( + "Failed to parse mapping [_doc]: ordered_dimensions cannot have more than 2 dimensions for star tree field [startree-1]", + ex.getMessage() + ); + } + + public void testMaxCalendarIntervalsCompositeIndex() { + MapperParsingException ex = expectThrows( + MapperParsingException.class, + () -> prepareCreate(TEST_INDEX).setMapping(createMaxDimTestMapping()) + .setSettings(Settings.builder().put(StarTreeIndexSettings.STAR_TREE_MAX_DATE_INTERVALS_SETTING.getKey(), 1)) + .get() + ); + assertEquals( + "Failed to parse mapping [_doc]: At most [1] calendar intervals are allowed in dimension [timestamp]", + ex.getMessage() + ); + } + public void testUnsupportedDim() { MapperParsingException ex = expectThrows( MapperParsingException.class, diff --git a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java index af72a39f67000..a50a7d3127b77 100644 --- a/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/StarTreeMapperTests.java @@ -14,12 +14,17 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.compositeindex.datacube.DateDimension; +import org.opensearch.index.compositeindex.datacube.Dimension; +import org.opensearch.index.compositeindex.datacube.Metric; import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.NumericDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.junit.After; import org.junit.Before; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -207,6 +212,100 @@ public void testInvalidSingleDim() { ); } + public void testMetric() { + List m1 = new ArrayList<>(); + m1.add(MetricStat.MAX); + Metric metric1 = new Metric("name", m1); + Metric metric2 = new Metric("name", m1); + assertEquals(metric1, metric2); + List m2 = new ArrayList<>(); + m2.add(MetricStat.MAX); + m2.add(MetricStat.COUNT); + metric2 = new Metric("name", m2); + assertNotEquals(metric1, metric2); + + assertEquals(MetricStat.COUNT, MetricStat.fromTypeName("count")); + assertEquals(MetricStat.MAX, MetricStat.fromTypeName("max")); + assertEquals(MetricStat.MIN, MetricStat.fromTypeName("min")); + assertEquals(MetricStat.SUM, MetricStat.fromTypeName("sum")); + assertEquals(MetricStat.AVG, MetricStat.fromTypeName("avg")); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> MetricStat.fromTypeName("invalid")); + assertEquals("Invalid metric stat: invalid", ex.getMessage()); + } + + public void testDimensions() { + List d1CalendarIntervals = new ArrayList<>(); + d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + DateDimension d1 = new DateDimension("name", d1CalendarIntervals); + DateDimension d2 = new DateDimension("name", d1CalendarIntervals); + assertEquals(d1, d2); + d2 = new DateDimension("name1", d1CalendarIntervals); + assertNotEquals(d1, d2); + List d2CalendarIntervals = new ArrayList<>(); + d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + d2CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + d2 = new DateDimension("name", d2CalendarIntervals); + assertNotEquals(d1, d2); + NumericDimension n1 = new NumericDimension("name"); + NumericDimension n2 = new NumericDimension("name"); + assertEquals(n1, n2); + n2 = new NumericDimension("name1"); + assertNotEquals(n1, n2); + } + + public void testStarTreeField() { + List m1 = new ArrayList<>(); + m1.add(MetricStat.MAX); + Metric metric1 = new Metric("name", m1); + List d1CalendarIntervals = new ArrayList<>(); + d1CalendarIntervals.add(Rounding.DateTimeUnit.HOUR_OF_DAY); + DateDimension d1 = new DateDimension("name", d1CalendarIntervals); + NumericDimension n1 = new NumericDimension("numeric"); + NumericDimension n2 = new NumericDimension("name1"); + + List metrics = List.of(metric1); + List dims = List.of(d1, n2); + StarTreeFieldConfiguration config = new StarTreeFieldConfiguration( + 100, + Set.of("name"), + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP + ); + + StarTreeField field1 = new StarTreeField("starTree", dims, metrics, config); + StarTreeField field2 = new StarTreeField("starTree", dims, metrics, config); + assertEquals(field1, field2); + + dims = List.of(d1, n2, n1); + field2 = new StarTreeField("starTree", dims, metrics, config); + assertNotEquals(field1, field2); + + dims = List.of(d1, n2); + metrics = List.of(metric1, metric1); + field2 = new StarTreeField("starTree", dims, metrics, config); + assertNotEquals(field1, field2); + + dims = List.of(d1, n2); + metrics = List.of(metric1); + StarTreeFieldConfiguration config1 = new StarTreeFieldConfiguration( + 1000, + Set.of("name"), + StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP + ); + field2 = new StarTreeField("starTree", dims, metrics, config1); + assertNotEquals(field1, field2); + + config1 = new StarTreeFieldConfiguration(100, Set.of("name", "field2"), StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP); + field2 = new StarTreeField("starTree", dims, metrics, config1); + assertNotEquals(field1, field2); + + config1 = new StarTreeFieldConfiguration(100, Set.of("name"), StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP); + field2 = new StarTreeField("starTree", dims, metrics, config1); + assertNotEquals(field1, field2); + + field2 = new StarTreeField("starTree", dims, metrics, config); + assertEquals(field1, field2); + } + private XContentBuilder getExpandedMapping(String dim, String metric) throws IOException { return topMapping(b -> { b.startObject("composite");