Merge 4685 with 4680

build.gradle

@@ -99,6 +99,7 @@ configure([project(':cli'),
            project(':seednode'),
            project(':statsnode'),
            project(':pricenode'),
+           project(':inventory'),
            project(':apitest')]) {
 
     apply plugin: 'application'
@@ -594,6 +595,21 @@ configure(project(':daemon')) {
     }
 }
 
+configure(project(':inventory')) {
+    apply plugin: 'com.github.johnrengelman.shadow'
+
+    mainClassName = 'bisq.inventory.InventoryMonitorMain'
+
+    dependencies {
+        compile project(':core')
+        compile "com.google.guava:guava:$guavaVersion"
+        compile "com.sparkjava:spark-core:$sparkVersion"
+
+        compileOnly "org.projectlombok:lombok:$lombokVersion"
+        annotationProcessor "org.projectlombok:lombok:$lombokVersion"
+    }
+}
+
 configure(project(':apitest')) {
     mainClassName = 'bisq.apitest.ApiTestMain'
 

common/src/main/java/bisq/common/util/Profiler.java

@@ -32,9 +32,13 @@ public static void printSystemLoad() {
     }
 
     public static long getUsedMemoryInMB() {
+        return getUsedMemoryInBytes() / 1024 / 1024;
+    }
+
+    public static long getUsedMemoryInBytes() {
         Runtime runtime = Runtime.getRuntime();
-        long free = runtime.freeMemory() / 1024 / 1024;
-        long total = runtime.totalMemory() / 1024 / 1024;
+        long free = runtime.freeMemory();
+        long total = runtime.totalMemory();
         return total - free;
     }
 

common/src/main/java/bisq/common/util/Utilities.java

@@ -40,6 +40,8 @@
 import javafx.scene.input.KeyCombination;
 import javafx.scene.input.KeyEvent;
 
+import java.text.DecimalFormat;
+
 import java.net.URI;
 import java.net.URISyntaxException;
 
@@ -523,4 +525,11 @@ public static <T> Predicate<T> distinctByKey(Function<? super T, Object> keyExtr
         return t -> map.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
     }
 
+    public static String readableFileSize(long size) {
+        if (size <= 0) return "0";
+        String[] units = new String[]{"B", "kB", "MB", "GB", "TB"};
+        int digitGroups = (int) (Math.log10(size) / Math.log10(1024));
+        return new DecimalFormat("#,##0.###").format(size / Math.pow(1024, digitGroups)) + " " + units[digitGroups];
+    }
+
 }

core/src/main/java/bisq/core/account/witness/AccountAgeWitnessStore.java

@@ -35,7 +35,7 @@
 @Slf4j
 public class AccountAgeWitnessStore extends PersistableNetworkPayloadStore<AccountAgeWitness> {
 
-    AccountAgeWitnessStore() {
+    public AccountAgeWitnessStore() {
     }
 
 

core/src/main/java/bisq/core/network/p2p/inventory/DeviationSeverity.java

@@ -0,0 +1,24 @@
+/*
+ * This file is part of Bisq.
+ *
+ * Bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.core.network.p2p.inventory;
+
+public enum DeviationSeverity {
+    OK,
+    WARN,
+    ALERT
+}

core/src/main/java/bisq/core/network/p2p/inventory/GetInventoryRequestHandler.java

@@ -0,0 +1,186 @@
+/*
+ * This file is part of Bisq.
+ *
+ * Bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.core.network.p2p.inventory;
+
+import bisq.core.dao.monitoring.BlindVoteStateMonitoringService;
+import bisq.core.dao.monitoring.DaoStateMonitoringService;
+import bisq.core.dao.monitoring.ProposalStateMonitoringService;
+import bisq.core.dao.monitoring.model.BlindVoteStateBlock;
+import bisq.core.dao.monitoring.model.DaoStateBlock;
+import bisq.core.dao.monitoring.model.ProposalStateBlock;
+import bisq.core.dao.state.DaoStateService;
+import bisq.core.network.p2p.inventory.messages.GetInventoryRequest;
+import bisq.core.network.p2p.inventory.messages.GetInventoryResponse;
+
+import bisq.network.p2p.network.Connection;
+import bisq.network.p2p.network.MessageListener;
+import bisq.network.p2p.network.NetworkNode;
+import bisq.network.p2p.network.Statistic;
+import bisq.network.p2p.peers.PeerManager;
+import bisq.network.p2p.storage.P2PDataStorage;
+import bisq.network.p2p.storage.payload.ProtectedStorageEntry;
+
+import bisq.common.app.Version;
+import bisq.common.config.Config;
+import bisq.common.proto.network.NetworkEnvelope;
+import bisq.common.util.Profiler;
+import bisq.common.util.Utilities;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+import com.google.common.base.Enums;
+import com.google.common.base.Optional;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import java.lang.management.ManagementFactory;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class GetInventoryRequestHandler implements MessageListener {
+    private final NetworkNode networkNode;
+    private final PeerManager peerManager;
+    private final P2PDataStorage p2PDataStorage;
+    private final DaoStateService daoStateService;
+    private final DaoStateMonitoringService daoStateMonitoringService;
+    private final ProposalStateMonitoringService proposalStateMonitoringService;
+    private final BlindVoteStateMonitoringService blindVoteStateMonitoringService;
+    private final int maxConnections;
+    private final Set<String> permittedRequestersPubKey = new HashSet<>();
+
+    @Inject
+    public GetInventoryRequestHandler(NetworkNode networkNode,
+                                      PeerManager peerManager,
+                                      P2PDataStorage p2PDataStorage,
+                                      DaoStateService daoStateService,
+                                      DaoStateMonitoringService daoStateMonitoringService,
+                                      ProposalStateMonitoringService proposalStateMonitoringService,
+                                      BlindVoteStateMonitoringService blindVoteStateMonitoringService,
+                                      @Named(Config.MAX_CONNECTIONS) int maxConnections) {
+        this.networkNode = networkNode;
+        this.peerManager = peerManager;
+        this.p2PDataStorage = p2PDataStorage;
+        this.daoStateService = daoStateService;
+        this.daoStateMonitoringService = daoStateMonitoringService;
+        this.proposalStateMonitoringService = proposalStateMonitoringService;
+        this.blindVoteStateMonitoringService = blindVoteStateMonitoringService;
+        this.maxConnections = maxConnections;
+
+        this.networkNode.addMessageListener(this);
+    }
+
+    @Override
+    public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) {
+        if (networkEnvelope instanceof GetInventoryRequest) {
+            if (permittedRequestersPubKey.isEmpty()) {
+                return;
+            }
+
+            // Data
+            GetInventoryRequest getInventoryRequest = (GetInventoryRequest) networkEnvelope;
+            Map<InventoryItem, Integer> dataObjects = new HashMap<>();
+            p2PDataStorage.getMapForDataResponse(getInventoryRequest.getVersion()).values().stream()
+                    .map(e -> e.getClass().getSimpleName())
+                    .forEach(className -> {
+                        Optional<InventoryItem> optionalEnum = Enums.getIfPresent(InventoryItem.class, className);
+                        if (optionalEnum.isPresent()) {
+                            InventoryItem key = optionalEnum.get();
+                            dataObjects.putIfAbsent(key, 0);
+                            int prev = dataObjects.get(key);
+                            dataObjects.put(key, prev + 1);
+                        }
+                    });
+            p2PDataStorage.getMap().values().stream()
+                    .map(ProtectedStorageEntry::getProtectedStoragePayload)
+                    .filter(Objects::nonNull)
+                    .map(e -> e.getClass().getSimpleName())
+                    .forEach(className -> {
+                        Optional<InventoryItem> optionalEnum = Enums.getIfPresent(InventoryItem.class, className);
+                        if (optionalEnum.isPresent()) {
+                            InventoryItem key = optionalEnum.get();
+                            dataObjects.putIfAbsent(key, 0);
+                            int prev = dataObjects.get(key);
+                            dataObjects.put(key, prev + 1);
+                        }
+                    });
+            Map<InventoryItem, String> inventory = new HashMap<>();
+            dataObjects.forEach((key, value) -> inventory.put(key, String.valueOf(value)));
+
+            // DAO
+            int numBsqBlocks = daoStateService.getBlocks().size();
+            inventory.put(InventoryItem.numBsqBlocks, String.valueOf(numBsqBlocks));
+
+            int daoStateChainHeight = daoStateService.getChainHeight();
+            inventory.put(InventoryItem.daoStateChainHeight, String.valueOf(daoStateChainHeight));
+
+            LinkedList<DaoStateBlock> daoStateBlockChain = daoStateMonitoringService.getDaoStateBlockChain();
+            if (!daoStateBlockChain.isEmpty()) {
+                String daoStateHash = Utilities.bytesAsHexString(daoStateBlockChain.getLast().getMyStateHash().getHash());
+                inventory.put(InventoryItem.daoStateHash, daoStateHash);
+            }
+
+            LinkedList<ProposalStateBlock> proposalStateBlockChain = proposalStateMonitoringService.getProposalStateBlockChain();
+            if (!proposalStateBlockChain.isEmpty()) {
+                String proposalHash = Utilities.bytesAsHexString(proposalStateBlockChain.getLast().getMyStateHash().getHash());
+                inventory.put(InventoryItem.proposalHash, proposalHash);
+            }
+
+            LinkedList<BlindVoteStateBlock> blindVoteStateBlockChain = blindVoteStateMonitoringService.getBlindVoteStateBlockChain();
+            if (!blindVoteStateBlockChain.isEmpty()) {
+                String blindVoteHash = Utilities.bytesAsHexString(blindVoteStateBlockChain.getLast().getMyStateHash().getHash());
+                inventory.put(InventoryItem.blindVoteHash, blindVoteHash);
+            }
+
+            // network
+            inventory.put(InventoryItem.maxConnections, String.valueOf(maxConnections));
+            inventory.put(InventoryItem.numConnections, String.valueOf(networkNode.getAllConnections().size()));
+            inventory.put(InventoryItem.peakNumConnections, String.valueOf(peerManager.getPeakNumConnections()));
+            inventory.put(InventoryItem.numAllConnectionsLostEvents, String.valueOf(peerManager.getNumAllConnectionsLostEvents()));
+            inventory.put(InventoryItem.sentBytes, String.valueOf(Statistic.totalSentBytesProperty().get()));
+            inventory.put(InventoryItem.sentBytesPerSec, String.valueOf(Statistic.totalSentBytesPerSecProperty().get()));
+            inventory.put(InventoryItem.receivedBytes, String.valueOf(Statistic.totalReceivedBytesProperty().get()));
+            inventory.put(InventoryItem.receivedBytesPerSec, String.valueOf(Statistic.totalReceivedBytesPerSecProperty().get()));
+            inventory.put(InventoryItem.receivedMessagesPerSec, String.valueOf(Statistic.numTotalReceivedMessagesPerSecProperty().get()));
+            inventory.put(InventoryItem.sentMessagesPerSec, String.valueOf(Statistic.numTotalSentMessagesPerSecProperty().get()));
+
+            // node
+            inventory.put(InventoryItem.version, Version.VERSION);
+            inventory.put(InventoryItem.usedMemory, String.valueOf(Profiler.getUsedMemoryInBytes()));
+            inventory.put(InventoryItem.jvmStartTime, String.valueOf(ManagementFactory.getRuntimeMXBean().getStartTime()));
+
+            log.info("Send inventory {} to {}", inventory, connection.getPeersNodeAddressOptional());
+            GetInventoryResponse getInventoryResponse = new GetInventoryResponse(inventory);
+            networkNode.sendMessage(connection, getInventoryResponse);
+        }
+    }
+
+    public void shutDown() {
+        networkNode.removeMessageListener(this);
+    }
+
+    public void addPermittedRequestersPubKey(String pubKey) {
+        permittedRequestersPubKey.add(pubKey);
+    }
+}

p2p/src/main/java/bisq/network/p2p/inventory/GetInventoryRequestManager.java → core/src/main/java/bisq/core/network/p2p/inventory/GetInventoryRequestManager.java

@@ -15,7 +15,7 @@
  * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
  */
 
-package bisq.network.p2p.inventory;
+package bisq.core.network.p2p.inventory;
 
 import bisq.network.p2p.NodeAddress;
 import bisq.network.p2p.network.NetworkNode;
@@ -41,11 +41,12 @@ public GetInventoryRequestManager(NetworkNode networkNode) {
     }
 
     public void request(NodeAddress nodeAddress,
-                        Consumer<Map<String, Integer>> resultHandler,
+                        Consumer<Map<InventoryItem, String>> resultHandler,
                         ErrorMessageHandler errorMessageHandler) {
         if (requesterMap.containsKey(nodeAddress)) {
-            log.warn("There is still an open request pending for {}", nodeAddress.getFullAddress());
-            return;
+            log.warn("There was still a pending request for {}. We shut it down and make a new request",
+                    nodeAddress.getFullAddress());
+            requesterMap.get(nodeAddress).shutDown();
         }
 
         GetInventoryRequester getInventoryRequester = new GetInventoryRequester(networkNode,