Skip to content

Commit

Permalink
Merge pull request #285 from bsinno/bugfix/shard-region-detail-stats
Browse files Browse the repository at this point in the history 
fixed retrieving shard region stats and aggregation
  • Loading branch information
@yufei-cai
yufei-cai committed Nov 14, 2018
2 parents d14631b + 3e5d6a3 commit 51228ba
Show file tree
Hide file tree
Showing 14 changed files with 252 additions and 99 deletions.
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.0</version>
<version>3.0.0-M1</version>
<configuration>
<systemProperties>
<kamon.auto-start>true</kamon.auto-start>
Expand All @@ -315,7 +315,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<version>2.20.1</version>
<version>3.0.0-M1</version>
<executions>
<execution>
<id>integration-test</id>
Expand Down
24 changes: 17 additions & 7 deletions ...src/main/java/org/eclipse/ditto/services/concierge/starter/actors/ConciergeRootActor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,14 @@
import org.eclipse.ditto.services.models.concierge.actors.ConciergeForwarderActor;
import org.eclipse.ditto.services.utils.akka.LogUtil;
import org.eclipse.ditto.services.utils.cluster.ClusterStatusSupplier;
import org.eclipse.ditto.services.utils.cluster.RetrieveStatisticsDetailsResponseSupplier;
import org.eclipse.ditto.services.utils.config.ConfigUtil;
import org.eclipse.ditto.services.utils.config.MongoConfig;
import org.eclipse.ditto.services.utils.health.DefaultHealthCheckingActorFactory;
import org.eclipse.ditto.services.utils.health.HealthCheckingActorOptions;
import org.eclipse.ditto.services.utils.health.routes.StatusRoute;
import org.eclipse.ditto.services.utils.persistence.mongo.MongoClientActor;
import org.eclipse.ditto.signals.commands.devops.RetrieveStatisticsDetails;

import com.typesafe.config.Config;

Expand All @@ -53,7 +55,6 @@
import akka.actor.SupervisorStrategy;
import akka.cluster.Cluster;
import akka.cluster.pubsub.DistributedPubSubMediator;
import akka.cluster.sharding.ShardRegion;
import akka.cluster.singleton.ClusterSingletonManager;
import akka.cluster.singleton.ClusterSingletonManagerSettings;
import akka.event.DiagnosticLoggingAdapter;
Expand All @@ -64,6 +65,7 @@
import akka.japi.pf.DeciderBuilder;
import akka.japi.pf.ReceiveBuilder;
import akka.pattern.AskTimeoutException;
import akka.pattern.PatternsCS;
import akka.stream.ActorMaterializer;

/**
Expand All @@ -79,9 +81,6 @@ public final class ConciergeRootActor extends AbstractActor {
private static final String RESTARTING_CHILD_MSG = "Restarting child...";

private final DiagnosticLoggingAdapter log = LogUtil.obtain(this);

private final ActorRef conciergeShardRegion;

private final SupervisorStrategy supervisorStrategy = new OneForOneStrategy(true, DeciderBuilder
.match(NullPointerException.class, e -> {
log.error(e, "NullPointer in child actor: {}", e.getMessage());
Expand Down Expand Up @@ -129,6 +128,8 @@ public final class ConciergeRootActor extends AbstractActor {
return SupervisorStrategy.escalate();
}).build());

private final RetrieveStatisticsDetailsResponseSupplier retrieveStatisticsDetailsResponseSupplier;

private <C extends AbstractConciergeConfigReader> ConciergeRootActor(final C configReader,
final ActorRef pubSubMediator,
final AbstractEnforcerActorFactory<C> authorizationProxyPropsFactory,
Expand All @@ -141,7 +142,11 @@ private <C extends AbstractConciergeConfigReader> ConciergeRootActor(final C con

final ActorContext context = getContext();

conciergeShardRegion = authorizationProxyPropsFactory.startEnforcerActor(context, configReader, pubSubMediator);
final ActorRef conciergeShardRegion =
authorizationProxyPropsFactory.startEnforcerActor(context, configReader, pubSubMediator);

retrieveStatisticsDetailsResponseSupplier = RetrieveStatisticsDetailsResponseSupplier.of(conciergeShardRegion,
ConciergeMessagingConstants.SHARD_REGION, log);

final ActorRef conciergeForwarder = startChildActor(context, ConciergeForwarderActor.ACTOR_NAME,
ConciergeForwarderActor.props(pubSubMediator, conciergeShardRegion));
Expand Down Expand Up @@ -233,15 +238,20 @@ public SupervisorStrategy supervisorStrategy() {
@Override
public Receive createReceive() {
return ReceiveBuilder.create()
.matchEquals(ShardRegion.getShardRegionStateInstance(), getShardRegionState ->
conciergeShardRegion.forward(getShardRegionState, getContext()))
.match(RetrieveStatisticsDetails.class, this::handleRetrieveStatisticsDetails)
.match(Status.Failure.class, f -> log.error(f.cause(), "Got failure <{}>!", f))
.matchAny(m -> {
log.warning("Unknown message <{}>.", m);
unhandled(m);
}).build();
}

private void handleRetrieveStatisticsDetails(final RetrieveStatisticsDetails command) {
log.info("Sending the namespace stats of the concierge shard as requested..");
PatternsCS.pipe(retrieveStatisticsDetailsResponseSupplier
.apply(command.getDittoHeaders()), getContext().dispatcher()).to(getSender());
}

private void bindHttpStatusRoute(final ActorRef healthCheckingActor, final HttpConfigReader httpConfig,
final ActorMaterializer materializer) {
String hostname = httpConfig.getHostname();
Expand Down
90 changes: 38 additions & 52 deletions .../proxy/src/main/java/org/eclipse/ditto/services/gateway/proxy/actors/StatisticsActor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,7 @@
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
Expand All @@ -28,6 +26,7 @@
import org.eclipse.ditto.json.JsonFieldDefinition;
import org.eclipse.ditto.json.JsonObject;
import org.eclipse.ditto.json.JsonObjectBuilder;
import org.eclipse.ditto.json.JsonValue;
import org.eclipse.ditto.model.base.json.FieldType;
import org.eclipse.ditto.model.base.json.JsonSchemaVersion;
import org.eclipse.ditto.model.base.json.Jsonifiable;
Expand All @@ -40,6 +39,7 @@
import org.eclipse.ditto.services.utils.metrics.instruments.gauge.Gauge;
import org.eclipse.ditto.signals.commands.devops.RetrieveStatistics;
import org.eclipse.ditto.signals.commands.devops.RetrieveStatisticsDetails;
import org.eclipse.ditto.signals.commands.devops.RetrieveStatisticsDetailsResponse;
import org.eclipse.ditto.signals.commands.devops.RetrieveStatisticsResponse;

import akka.actor.AbstractActor;
Expand Down Expand Up @@ -144,10 +144,6 @@ private static Map<String, ShardStatisticsWrapper> initShardStatisticsMap() {
return shardStatisticsMap;
}

private static String ensureNonemptyString(final String possiblyEmptyString) {
return possiblyEmptyString.isEmpty() ? EMPTY_STRING_TAG : possiblyEmptyString;
}

@Override
public Receive createReceive() {
// ignore - the message is too late
Expand All @@ -168,16 +164,16 @@ public Receive createReceive() {
final ActorRef self = getSelf();
final ActorRef sender = getSender();
becomeStatisticsAwaiting(statistics ->
sender.tell(RetrieveStatisticsResponse.of(statistics.toJson(),
retrieveStatistics.getDittoHeaders()), self)
sender.tell(RetrieveStatisticsResponse.of(statistics.toJson(),
retrieveStatistics.getDittoHeaders()), self)
);
})
.match(RetrieveStatisticsDetails.class, retrieveStatistics -> {

tellRootActorToGetShardRegionState(THINGS_ROOT);
tellRootActorToGetShardRegionState(POLICIES_ROOT);
tellRootActorToGetShardRegionState(CONCIERGE_ROOT);
tellRootActorToGetShardRegionState(SEARCH_UPDATER_ROOT);
tellRootActorToRetrieveStatistics(THINGS_ROOT, retrieveStatistics);
tellRootActorToRetrieveStatistics(POLICIES_ROOT, retrieveStatistics);
tellRootActorToRetrieveStatistics(CONCIERGE_ROOT, retrieveStatistics);
tellRootActorToRetrieveStatistics(SEARCH_UPDATER_ROOT, retrieveStatistics);

final ActorRef self = getSelf();
final ActorRef sender = getSender();
Expand All @@ -199,9 +195,10 @@ private void tellShardRegionToSendClusterShardingStats() {
tellShardRegionToSendClusterShardingStats(SR_SEARCH_UPDATER);
}

private void tellRootActorToGetShardRegionState(final String rootActorPath) {
pubSubMediator.tell(new DistributedPubSubMediator.SendToAll(
rootActorPath, ShardRegion.getShardRegionStateInstance(), false), getSelf());
private void tellRootActorToRetrieveStatistics(final String rootActorPath,
final RetrieveStatisticsDetails retrieveStatistics) {
pubSubMediator.tell(new DistributedPubSubMediator.SendToAll(rootActorPath, retrieveStatistics, false),
getSelf());
}

private void tellShardRegionToSendClusterShardingStats(final String shardRegion) {
Expand Down Expand Up @@ -232,7 +229,8 @@ private void becomeStatisticsAwaiting(final Consumer<Statistics> statisticsConsu
currentStatistics.toJson(), retrieveStatistics.getDittoHeaders()), getSelf())
)
.match(ShardRegion.ClusterShardingStats.class, clusterShardingStats -> {
final ShardStatisticsWrapper shardStatistics = getShardStatistics(shardStatisticsMap);
final ShardStatisticsWrapper shardStatistics = getShardStatistics(shardStatisticsMap,
getSender().path().name());
final Map<Address, ShardRegion.ShardRegionStats> regions = clusterShardingStats.getRegions();
shardStatistics.count = regions.isEmpty() ? 0 : regions.values().stream()
.mapToInt(shardRegionStats -> shardRegionStats.getStats().isEmpty() ? 0 :
Expand Down Expand Up @@ -270,40 +268,25 @@ private void becomeStatisticsDetailsAwaiting(final Consumer<StatisticsDetails> s
retrieveStatistics -> getSender().tell(RetrieveStatisticsResponse.of(
currentStatisticsDetails.toJson(), retrieveStatistics.getDittoHeaders()), getSelf())
)
.match(ShardRegion.CurrentShardRegionState.class, currentShardRegionState -> {
final ShardStatisticsWrapper shardStatistics = getShardStatistics(shardStatisticsMap);
final Map<String, Long> shards = currentShardRegionState.getShards()
.match(RetrieveStatisticsDetailsResponse.class, retrieveStatisticsDetailsResponse -> {
final String shardRegion = retrieveStatisticsDetailsResponse.getStatisticsDetails()
.stream()
.map(ShardRegion.ShardState::getEntityIds)
.flatMap(strSet -> strSet.stream()
.map(str -> {
// groupKey may be either namespace or resource-type+namespace (in case of concierge)
final String[] groupKeys = str.split(":", 3);
// assume String.split(String, int) may not return an empty array
switch (groupKeys.length) {
case 0:
// should not happen with Java 8 strings, but just in case
return EMPTY_STRING_TAG;
case 1:
case 2:
// normal: namespace
return ensureNonemptyString(groupKeys[0]);
default:
// concierge: resource-type + namespace
return groupKeys[0] + ":" + groupKeys[1];
}
})
)
.collect(Collectors.groupingBy(Function.identity(),
Collectors.mapping(Function.identity(), Collectors.counting())));

shards.forEach((key, value) -> {
if (shardStatistics.hotnessMap.containsKey(key)) {
shardStatistics.hotnessMap.put(key, shardStatistics.hotnessMap.get(key) + value);
} else {
shardStatistics.hotnessMap.put(key, value);
}
});
.findFirst()
.map(JsonField::getKeyName)
.orElse(null);

if (shardRegion != null) {
final ShardStatisticsWrapper shardStatistics =
getShardStatistics(shardStatisticsMap, shardRegion);

retrieveStatisticsDetailsResponse.getStatisticsDetails().getValue(shardRegion)
.map(JsonValue::asObject)
.map(JsonObject::stream)
.ifPresent(namespaceEntries -> namespaceEntries.forEach(field ->
shardStatistics.hotnessMap
.merge(field.getKeyName(), field.getValue().asLong(), Long::sum)
));
}
})
.match(AskTimeoutException.class, askTimeout -> {
currentStatisticsDetails = new StatisticsDetails(
Expand All @@ -320,8 +303,10 @@ private void becomeStatisticsDetailsAwaiting(final Consumer<StatisticsDetails> s
);
}

private ShardStatisticsWrapper getShardStatistics(final Map<String, ShardStatisticsWrapper> shardStatisticsMap) {
ShardStatisticsWrapper shardStatistics = shardStatisticsMap.get(getSender().path().name());
private ShardStatisticsWrapper getShardStatistics(final Map<String, ShardStatisticsWrapper> shardStatisticsMap,
final String shardRegion) {

ShardStatisticsWrapper shardStatistics = shardStatisticsMap.get(shardRegion);
if (shardStatistics == null) {
if (getSender().path().toStringWithoutAddress().contains(SR_SEARCH_UPDATER)) {
shardStatistics = shardStatisticsMap.get(SR_SEARCH_UPDATER);
Expand Down Expand Up @@ -548,7 +533,8 @@ public String toString() {

}

private static class InternalRetrieveStatistics{
private static class InternalRetrieveStatistics {

private InternalRetrieveStatistics() {
// no-op
}
Expand Down
10 changes: 7 additions & 3 deletions ...n/java/org/eclipse/ditto/services/gateway/streaming/actors/EventAndResponsePublisher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
import akka.event.DiagnosticLoggingAdapter;
import akka.japi.Creator;
import akka.japi.pf.ReceiveBuilder;
import akka.stream.actor.AbstractActorPublisher;
import akka.stream.actor.AbstractActorPublisherWithStash;
import akka.stream.actor.ActorPublisherMessage;
import scala.concurrent.duration.FiniteDuration;

Expand All @@ -38,7 +38,7 @@
* necessary.
*/
public final class EventAndResponsePublisher
extends AbstractActorPublisher<Jsonifiable.WithPredicate<JsonObject, JsonField>> {
extends AbstractActorPublisherWithStash<Jsonifiable.WithPredicate<JsonObject, JsonField>> {

private static final int MESSAGE_CONSUMPTION_CHECK_SECONDS = 2;
private final DiagnosticLoggingAdapter logger = LogUtil.obtain(this);
Expand Down Expand Up @@ -79,11 +79,15 @@ public Receive createReceive() {
logger.debug("Established new connection: {}", connectionCorrelationId);
getContext().become(connected(connectionCorrelationId));
})
.matchAny(any -> logger.warning("Got unknown message during init phase '{}'", any)).build();
.matchAny(any -> {
logger.info("Got unknown message during init phase '{}' - stashing..", any);
stash();
}).build();
}

private Receive connected(final String connectionCorrelationId) {
this.connectionCorrelationId = connectionCorrelationId;
unstashAll();

return ReceiveBuilder.create()
.match(Signal.class, signal -> buffer.size() >= backpressureBufferSize, signal -> {
Expand Down
2 changes: 1 addition & 1 deletion ...ch/src/main/java/org/eclipse/ditto/services/models/thingsearch/ThingsSearchConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public final class ThingsSearchConstants {
/**
* Path of the updater root actor.
*/
public static final String UPDATER_ROOT_ACTOR_PATH = USER_PATH + "/searchUpdaterRoot";
public static final String UPDATER_ROOT_ACTOR_PATH = ROOT_ACTOR_PATH + "/searchUpdaterRoot";

/**
* Path of the search actor.
Expand Down
Loading

0 comments on commit 51228ba

Please sign in to comment.