-
Notifications
You must be signed in to change notification settings - Fork 34
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This service will build final XContent for stats from upload API across Nodes. Signed-off-by: Vijayan Balasubramanian <balasvij@amazon.com>
- Loading branch information
Showing
5 changed files
with
234 additions
and
17 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
89 changes: 89 additions & 0 deletions
89
src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.geospatial.stats.upload; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.AbstractMap; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import org.opensearch.common.xcontent.ToXContentFragment; | ||
| import org.opensearch.common.xcontent.XContentBuilder; | ||
|
|
||
| // Service to calculate summary of upload stats and generate XContent for StatsResponse | ||
| public class UploadStatsService implements ToXContentFragment { | ||
|
|
||
| public static final String UPLOADS = "uploads"; | ||
| public static final String TOTAL = "total"; | ||
| public static final String METRICS = "metrics"; | ||
| public static final String NODE_ID = "node_id"; | ||
| private final Map<String, UploadStats> uploadStats; | ||
| private final TotalUploadStats totalUploadStats; | ||
|
|
||
| public UploadStatsService(Map<String, UploadStats> uploadStats) { | ||
| this.uploadStats = Objects.requireNonNull(uploadStats, "upload stats map cannot be null"); | ||
| this.totalUploadStats = new TotalUploadStats(new ArrayList<>(uploadStats.values())); | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| /* | ||
| { | ||
| "uploads": { | ||
| "total": { | ||
| request_count : # of request, | ||
| "upload" : sum of documents to upload across API, | ||
| "success" : sum of successfully uploaded documents across API, | ||
| "failed" : sum of failed to upload documents across API, | ||
| "duration" : sum of duration in milliseconds to ingest document across API | ||
| }, | ||
| "metrics" : [ | ||
| { | ||
| "id" : <metric-id>, | ||
| "node_id" : node-id | ||
| "upload" : # of documents to upload, | ||
| "success" : # of successfully uploaded documents, | ||
| "failed" : # of failed to upload documents, | ||
| "duration" : duration in milliseconds to ingest document | ||
| }, ...... | ||
| ] | ||
| } | ||
| } | ||
| */ | ||
| builder.startObject(UPLOADS); | ||
| totalUploadStats.toXContent(builder, params); | ||
| builder.startArray(METRICS); | ||
| if (totalUploadStats.isUploadStatsEmpty()) { | ||
| builder.endArray(); | ||
| return builder.endObject(); | ||
| } | ||
| final List<AbstractMap.SimpleEntry<String, UploadMetric>> metricsByNodeID = groupMetricsByNodeID(uploadStats); | ||
| for (AbstractMap.SimpleEntry<String, UploadMetric> entry : metricsByNodeID) { | ||
| addMetrics(builder, params, entry); | ||
| } | ||
| builder.endArray(); | ||
| return builder.endObject(); | ||
| } | ||
|
|
||
| private void addMetrics(XContentBuilder builder, Params params, AbstractMap.SimpleEntry<String, UploadMetric> entry) | ||
| throws IOException { | ||
| builder.startObject(); | ||
| builder.field(NODE_ID, entry.getKey()); | ||
| entry.getValue().toXContent(builder, params); | ||
| builder.endObject(); | ||
| } | ||
|
|
||
| private List<AbstractMap.SimpleEntry<String, UploadMetric>> groupMetricsByNodeID(Map<String, UploadStats> uploadStats) { | ||
| return uploadStats.entrySet() | ||
| .stream() | ||
| .flatMap(stat -> stat.getValue().getMetrics().stream().map(metric -> new AbstractMap.SimpleEntry<>(stat.getKey(), metric))) | ||
| .collect(Collectors.toList()); | ||
|
|
||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
32 changes: 32 additions & 0 deletions
32
src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsBuilder.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.geospatial.stats.upload; | ||
|
|
||
| import static org.opensearch.test.OpenSearchTestCase.randomIntBetween; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.stream.IntStream; | ||
|
|
||
| import org.opensearch.geospatial.GeospatialTestHelper; | ||
|
|
||
| public class UploadStatsBuilder { | ||
| private static final int MAX_METRIC_COUNT = 10; | ||
| private static final int MIN_METRIC_COUNT = 2; | ||
|
|
||
| public static UploadStats randomUploadStats() { | ||
| int randomMetricCount = randomIntBetween(MIN_METRIC_COUNT, MAX_METRIC_COUNT); | ||
| UploadStats stats = new UploadStats(); | ||
| IntStream.range(0, randomMetricCount).forEach(unUsed -> stats.addMetric(GeospatialTestHelper.generateRandomUploadMetric())); | ||
| return stats; | ||
| } | ||
|
|
||
| public static List<UploadStats> randomUploadStats(int max) { | ||
| List<UploadStats> stats = new ArrayList<>(); | ||
| IntStream.range(0, max).forEach(unUsed -> stats.add(randomUploadStats())); | ||
| return stats; | ||
| } | ||
| } |
112 changes: 112 additions & 0 deletions
112
src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsServiceTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.geospatial.stats.upload; | ||
|
|
||
| import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; | ||
| import static org.opensearch.geospatial.GeospatialTestHelper.buildFieldNameValuePair; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| import org.opensearch.common.Strings; | ||
| import org.opensearch.common.xcontent.ToXContent; | ||
| import org.opensearch.common.xcontent.XContentBuilder; | ||
| import org.opensearch.geospatial.GeospatialTestHelper; | ||
| import org.opensearch.test.OpenSearchTestCase; | ||
|
|
||
| public class UploadStatsServiceTests extends OpenSearchTestCase { | ||
|
|
||
| private String removeStartAndEndObject(String content) { | ||
| assertNotNull(content); | ||
| assertTrue("content length should be at least 2", content.length() > 1); | ||
| return content.substring(1, content.length() - 1); | ||
| } | ||
|
|
||
| public void testInstanceCreation() { | ||
| Map<String, UploadStats> randomMap = new HashMap<>(); | ||
| randomMap.put(GeospatialTestHelper.randomLowerCaseString(), UploadStatsBuilder.randomUploadStats()); | ||
| randomMap.put(GeospatialTestHelper.randomLowerCaseString(), UploadStatsBuilder.randomUploadStats()); | ||
|
|
||
| UploadStatsService service = new UploadStatsService(randomMap); | ||
| assertNotNull(service); | ||
| } | ||
|
|
||
| public void testInstanceCreationFails() { | ||
| assertThrows(NullPointerException.class, () -> new UploadStatsService(null)); | ||
| } | ||
|
|
||
| public void testXContentWithNodeID() { | ||
| Map<String, UploadStats> randomMap = new HashMap<>(); | ||
| randomMap.put(GeospatialTestHelper.randomLowerCaseString(), UploadStatsBuilder.randomUploadStats()); | ||
| randomMap.put(GeospatialTestHelper.randomLowerCaseString(), UploadStatsBuilder.randomUploadStats()); | ||
| UploadStatsService service = new UploadStatsService(randomMap); | ||
| String xContent = Strings.toString(service); | ||
| assertNotNull(xContent); | ||
| for (String nodeID : randomMap.keySet()) { | ||
| assertTrue(nodeID + " is missing", xContent.contains(buildFieldNameValuePair(UploadStatsService.NODE_ID, nodeID))); | ||
| } | ||
| } | ||
|
|
||
| public void testXContentWithEmptyStats() throws IOException { | ||
| UploadStatsService service = new UploadStatsService(new HashMap<>()); | ||
| final XContentBuilder contentBuilder = jsonBuilder().startObject(); | ||
| service.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); | ||
| contentBuilder.endObject(); | ||
| String emptyContent = "{\"uploads\":{\"total\":{},\"metrics\":[]}}"; | ||
| assertEquals(emptyContent, Strings.toString(contentBuilder)); | ||
| } | ||
|
|
||
| public void testXContentWithTotalUploadStats() throws IOException { | ||
| Map<String, UploadStats> randomMap = new HashMap<>(); | ||
| final List<UploadStats> uploadStats = List.of(UploadStatsBuilder.randomUploadStats(), UploadStatsBuilder.randomUploadStats()); | ||
|
|
||
| for (UploadStats stats : uploadStats) { | ||
| randomMap.put(GeospatialTestHelper.randomLowerCaseString(), stats); | ||
| } | ||
| UploadStatsService service = new UploadStatsService(randomMap); | ||
| final XContentBuilder serviceContentBuilder = jsonBuilder().startObject(); | ||
| service.toXContent(serviceContentBuilder, ToXContent.EMPTY_PARAMS); | ||
| serviceContentBuilder.endObject(); | ||
| String content = Strings.toString(serviceContentBuilder); | ||
| assertNotNull(content); | ||
|
|
||
| final XContentBuilder summary = jsonBuilder().startObject(); | ||
| TotalUploadStats expectedSummary = new TotalUploadStats(uploadStats); | ||
| expectedSummary.toXContent(summary, ToXContent.EMPTY_PARAMS); | ||
| summary.endObject(); | ||
| final String totalUploadStatsSummary = Strings.toString(summary); | ||
| assertNotNull(totalUploadStatsSummary); | ||
| assertTrue(content.contains(removeStartAndEndObject(totalUploadStatsSummary))); | ||
| } | ||
|
|
||
| public void testXContentWithMetrics() throws IOException { | ||
| Map<String, UploadStats> randomMap = new HashMap<>(); | ||
| List<UploadMetric> randomMetrics = new ArrayList<>(); | ||
| final List<UploadStats> uploadStats = List.of(UploadStatsBuilder.randomUploadStats(), UploadStatsBuilder.randomUploadStats()); | ||
| for (UploadStats stats : uploadStats) { | ||
| randomMap.put(GeospatialTestHelper.randomLowerCaseString(), stats); | ||
| randomMetrics.addAll(stats.getMetrics()); | ||
| } | ||
| UploadStatsService service = new UploadStatsService(randomMap); | ||
| final XContentBuilder serviceContentBuilder = jsonBuilder().startObject(); | ||
| service.toXContent(serviceContentBuilder, ToXContent.EMPTY_PARAMS); | ||
| serviceContentBuilder.endObject(); | ||
| String content = Strings.toString(serviceContentBuilder); | ||
| assertNotNull(content); | ||
|
|
||
| for (UploadMetric metric : randomMetrics) { | ||
| XContentBuilder metricsAsContent = jsonBuilder().startObject(); | ||
| metric.toXContent(metricsAsContent, ToXContent.EMPTY_PARAMS); | ||
| metricsAsContent.endObject(); | ||
| final String metricsAsString = Strings.toString(metricsAsContent); | ||
| assertNotNull(metricsAsString); | ||
| assertTrue(content.contains(removeStartAndEndObject(metricsAsString))); | ||
| } | ||
| } | ||
| } |