Add inventory monitor module

build.gradle

@@ -99,6 +99,7 @@ configure([project(':cli'),
            project(':seednode'),
            project(':statsnode'),
            project(':pricenode'),
+           project(':inventory'),
            project(':apitest')]) {
 
     apply plugin: 'application'
@@ -594,6 +595,21 @@ configure(project(':daemon')) {
     }
 }
 
+configure(project(':inventory')) {
+    apply plugin: 'com.github.johnrengelman.shadow'
+
+    mainClassName = 'bisq.inventory.InventoryMonitorMain'
+
+    dependencies {
+        compile project(':core')
+        compile "com.google.guava:guava:$guavaVersion"
+        compile "com.sparkjava:spark-core:$sparkVersion"
+
+        compileOnly "org.projectlombok:lombok:$lombokVersion"
+        annotationProcessor "org.projectlombok:lombok:$lombokVersion"
+    }
+}
+
 configure(project(':apitest')) {
     mainClassName = 'bisq.apitest.ApiTestMain'
 

common/src/main/java/bisq/common/setup/CommonSetup.java

@@ -102,13 +102,13 @@ protected static void setSystemProperties() {
 
     protected static void setupSigIntHandlers(GracefulShutDownHandler gracefulShutDownHandler) {
         Signal.handle(new Signal("INT"), signal -> {
-            gracefulShutDownHandler.gracefulShutDown(() -> {
-            });
+            UserThread.execute(() -> gracefulShutDownHandler.gracefulShutDown(() -> {
+            }));
         });
 
         Signal.handle(new Signal("TERM"), signal -> {
-            gracefulShutDownHandler.gracefulShutDown(() -> {
-            });
+            UserThread.execute(() -> gracefulShutDownHandler.gracefulShutDown(() -> {
+            }));
         });
     }
 

common/src/main/java/bisq/common/util/Profiler.java

@@ -32,9 +32,13 @@ public static void printSystemLoad() {
     }
 
     public static long getUsedMemoryInMB() {
+        return getUsedMemoryInBytes() / 1024 / 1024;
+    }
+
+    public static long getUsedMemoryInBytes() {
         Runtime runtime = Runtime.getRuntime();
-        long free = runtime.freeMemory() / 1024 / 1024;
-        long total = runtime.totalMemory() / 1024 / 1024;
+        long free = runtime.freeMemory();
+        long total = runtime.totalMemory();
         return total - free;
     }
 

common/src/main/java/bisq/common/util/Utilities.java

@@ -40,6 +40,8 @@
 import javafx.scene.input.KeyCombination;
 import javafx.scene.input.KeyEvent;
 
+import java.text.DecimalFormat;
+
 import java.net.URI;
 import java.net.URISyntaxException;
 
@@ -523,4 +525,11 @@ public static <T> Predicate<T> distinctByKey(Function<? super T, Object> keyExtr
         return t -> map.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
     }
 
+    public static String readableFileSize(long size) {
+        if (size <= 0) return "0";
+        String[] units = new String[]{"B", "kB", "MB", "GB", "TB"};
+        int digitGroups = (int) (Math.log10(size) / Math.log10(1024));
+        return new DecimalFormat("#,##0.###").format(size / Math.pow(1024, digitGroups)) + " " + units[digitGroups];
+    }
+
 }

core/src/main/java/bisq/core/app/BisqSetup.java

@@ -260,7 +260,6 @@ public void start() {
     }
 
     private void step2() {
-        torSetup.cleanupTorFiles();
         readMapsFromResources(this::step3);
         checkForCorrectOSArchitecture();
         checkOSXVersion();

core/src/main/java/bisq/core/app/TorSetup.java

@@ -46,15 +46,7 @@ public TorSetup(@Named(Config.TOR_DIR) File torDir) {
         this.torDir = checkDir(torDir);
     }
 
-    public void cleanupTorFiles() {
-        cleanupTorFiles(null, null);
-    }
-
-    // We get sometimes Tor startup problems which is related to some tor files in the tor directory. It happens
-    // more often if the application got killed (not graceful shutdown).
-    // Creating all tor files newly takes about 3-4 sec. longer and it does not benefit from cache files.
-    // TODO: We should fix those startup problems in the netlayer library, once fixed there we can remove that call at the
-    // Bisq startup again.
+    // Should only be called if needed. Slows down Tor startup from about 5 sec. to 30 sec. if it gets deleted.
     public void cleanupTorFiles(@Nullable Runnable resultHandler, @Nullable ErrorMessageHandler errorMessageHandler) {
         File hiddenservice = new File(Paths.get(torDir.getAbsolutePath(), "hiddenservice").toString());
         try {

core/src/main/java/bisq/core/app/misc/AppSetupWithP2P.java

@@ -73,7 +73,6 @@ public AppSetupWithP2P(P2PService p2PService,
 
     @Override
     public void initPersistedDataHosts() {
-        torSetup.cleanupTorFiles();
         persistedDataHosts.add(p2PService);
 
         // we apply at startup the reading of persisted data but don't want to get it triggered in the constructor

core/src/main/java/bisq/core/filter/FilterManager.java

@@ -216,13 +216,13 @@ public void setFilterWarningHandler(Consumer<String> filterWarningHandler) {
         addListener(filter -> {
             if (filter != null && filterWarningHandler != null) {
                 if (filter.getSeedNodes() != null && !filter.getSeedNodes().isEmpty()) {
-                    log.info(Res.get("popup.warning.nodeBanned", Res.get("popup.warning.seed")));
+                    log.info("One of the seed nodes got banned. {}", filter.getSeedNodes());
                     // Let's keep that more silent. Might be used in case a node is unstable and we don't want to confuse users.
                     // filterWarningHandler.accept(Res.get("popup.warning.nodeBanned", Res.get("popup.warning.seed")));
                 }
 
                 if (filter.getPriceRelayNodes() != null && !filter.getPriceRelayNodes().isEmpty()) {
-                    log.info(Res.get("popup.warning.nodeBanned", Res.get("popup.warning.priceRelay")));
+                    log.info("One of the price relay nodes got banned. {}", filter.getPriceRelayNodes());
                     // Let's keep that more silent. Might be used in case a node is unstable and we don't want to confuse users.
                     // filterWarningHandler.accept(Res.get("popup.warning.nodeBanned", Res.get("popup.warning.priceRelay")));
                 }

core/src/main/java/bisq/core/network/p2p/inventory/GetInventoryRequestHandler.java

@@ -0,0 +1,176 @@
+/*
+ * This file is part of Bisq.
+ *
+ * Bisq is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or (at
+ * your option) any later version.
+ *
+ * Bisq is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package bisq.core.network.p2p.inventory;
+
+import bisq.core.dao.monitoring.BlindVoteStateMonitoringService;
+import bisq.core.dao.monitoring.DaoStateMonitoringService;
+import bisq.core.dao.monitoring.ProposalStateMonitoringService;
+import bisq.core.dao.monitoring.model.BlindVoteStateBlock;
+import bisq.core.dao.monitoring.model.DaoStateBlock;
+import bisq.core.dao.monitoring.model.ProposalStateBlock;
+import bisq.core.dao.state.DaoStateService;
+import bisq.core.network.p2p.inventory.messages.GetInventoryRequest;
+import bisq.core.network.p2p.inventory.messages.GetInventoryResponse;
+import bisq.core.network.p2p.inventory.model.InventoryItem;
+
+import bisq.network.p2p.network.Connection;
+import bisq.network.p2p.network.MessageListener;
+import bisq.network.p2p.network.NetworkNode;
+import bisq.network.p2p.network.Statistic;
+import bisq.network.p2p.peers.PeerManager;
+import bisq.network.p2p.storage.P2PDataStorage;
+import bisq.network.p2p.storage.payload.ProtectedStorageEntry;
+
+import bisq.common.app.Version;
+import bisq.common.config.Config;
+import bisq.common.proto.network.NetworkEnvelope;
+import bisq.common.util.Profiler;
+import bisq.common.util.Utilities;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+import com.google.common.base.Enums;
+import com.google.common.base.Optional;
+
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Objects;
+
+import java.lang.management.ManagementFactory;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class GetInventoryRequestHandler implements MessageListener {
+    private final NetworkNode networkNode;
+    private final PeerManager peerManager;
+    private final P2PDataStorage p2PDataStorage;
+    private final DaoStateService daoStateService;
+    private final DaoStateMonitoringService daoStateMonitoringService;
+    private final ProposalStateMonitoringService proposalStateMonitoringService;
+    private final BlindVoteStateMonitoringService blindVoteStateMonitoringService;
+    private final int maxConnections;
+
+    @Inject
+    public GetInventoryRequestHandler(NetworkNode networkNode,
+                                      PeerManager peerManager,
+                                      P2PDataStorage p2PDataStorage,
+                                      DaoStateService daoStateService,
+                                      DaoStateMonitoringService daoStateMonitoringService,
+                                      ProposalStateMonitoringService proposalStateMonitoringService,
+                                      BlindVoteStateMonitoringService blindVoteStateMonitoringService,
+                                      @Named(Config.MAX_CONNECTIONS) int maxConnections) {
+        this.networkNode = networkNode;
+        this.peerManager = peerManager;
+        this.p2PDataStorage = p2PDataStorage;
+        this.daoStateService = daoStateService;
+        this.daoStateMonitoringService = daoStateMonitoringService;
+        this.proposalStateMonitoringService = proposalStateMonitoringService;
+        this.blindVoteStateMonitoringService = blindVoteStateMonitoringService;
+        this.maxConnections = maxConnections;
+
+        this.networkNode.addMessageListener(this);
+    }
+
+    @Override
+    public void onMessage(NetworkEnvelope networkEnvelope, Connection connection) {
+        if (networkEnvelope instanceof GetInventoryRequest) {
+            // Data
+            GetInventoryRequest getInventoryRequest = (GetInventoryRequest) networkEnvelope;
+            Map<InventoryItem, Integer> dataObjects = new HashMap<>();
+            p2PDataStorage.getMapForDataResponse(getInventoryRequest.getVersion()).values().stream()
+                    .map(e -> e.getClass().getSimpleName())
+                    .forEach(className -> {
+                        Optional<InventoryItem> optionalEnum = Enums.getIfPresent(InventoryItem.class, className);
+                        if (optionalEnum.isPresent()) {
+                            InventoryItem key = optionalEnum.get();
+                            dataObjects.putIfAbsent(key, 0);
+                            int prev = dataObjects.get(key);
+                            dataObjects.put(key, prev + 1);
+                        }
+                    });
+            p2PDataStorage.getMap().values().stream()
+                    .map(ProtectedStorageEntry::getProtectedStoragePayload)
+                    .filter(Objects::nonNull)
+                    .map(e -> e.getClass().getSimpleName())
+                    .forEach(className -> {
+                        Optional<InventoryItem> optionalEnum = Enums.getIfPresent(InventoryItem.class, className);
+                        if (optionalEnum.isPresent()) {
+                            InventoryItem key = optionalEnum.get();
+                            dataObjects.putIfAbsent(key, 0);
+                            int prev = dataObjects.get(key);
+                            dataObjects.put(key, prev + 1);
+                        }
+                    });
+            Map<InventoryItem, String> inventory = new HashMap<>();
+            dataObjects.forEach((key, value) -> inventory.put(key, String.valueOf(value)));
+
+            // DAO
+            int numBsqBlocks = daoStateService.getBlocks().size();
+            inventory.put(InventoryItem.numBsqBlocks, String.valueOf(numBsqBlocks));
+
+            int daoStateChainHeight = daoStateService.getChainHeight();
+            inventory.put(InventoryItem.daoStateChainHeight, String.valueOf(daoStateChainHeight));
+
+            LinkedList<DaoStateBlock> daoStateBlockChain = daoStateMonitoringService.getDaoStateBlockChain();
+            if (!daoStateBlockChain.isEmpty()) {
+                String daoStateHash = Utilities.bytesAsHexString(daoStateBlockChain.getLast().getMyStateHash().getHash());
+                inventory.put(InventoryItem.daoStateHash, daoStateHash);
+            }
+
+            LinkedList<ProposalStateBlock> proposalStateBlockChain = proposalStateMonitoringService.getProposalStateBlockChain();
+            if (!proposalStateBlockChain.isEmpty()) {
+                String proposalHash = Utilities.bytesAsHexString(proposalStateBlockChain.getLast().getMyStateHash().getHash());
+                inventory.put(InventoryItem.proposalHash, proposalHash);
+            }
+
+            LinkedList<BlindVoteStateBlock> blindVoteStateBlockChain = blindVoteStateMonitoringService.getBlindVoteStateBlockChain();
+            if (!blindVoteStateBlockChain.isEmpty()) {
+                String blindVoteHash = Utilities.bytesAsHexString(blindVoteStateBlockChain.getLast().getMyStateHash().getHash());
+                inventory.put(InventoryItem.blindVoteHash, blindVoteHash);
+            }
+
+            // network
+            inventory.put(InventoryItem.maxConnections, String.valueOf(maxConnections));
+            inventory.put(InventoryItem.numConnections, String.valueOf(networkNode.getAllConnections().size()));
+            inventory.put(InventoryItem.peakNumConnections, String.valueOf(peerManager.getPeakNumConnections()));
+            inventory.put(InventoryItem.numAllConnectionsLostEvents, String.valueOf(peerManager.getNumAllConnectionsLostEvents()));
+            inventory.put(InventoryItem.sentBytes, String.valueOf(Statistic.totalSentBytesProperty().get()));
+            inventory.put(InventoryItem.sentBytesPerSec, String.valueOf(Statistic.totalSentBytesPerSecProperty().get()));
+            inventory.put(InventoryItem.receivedBytes, String.valueOf(Statistic.totalReceivedBytesProperty().get()));
+            inventory.put(InventoryItem.receivedBytesPerSec, String.valueOf(Statistic.totalReceivedBytesPerSecProperty().get()));
+            inventory.put(InventoryItem.receivedMessagesPerSec, String.valueOf(Statistic.numTotalReceivedMessagesPerSecProperty().get()));
+            inventory.put(InventoryItem.sentMessagesPerSec, String.valueOf(Statistic.numTotalSentMessagesPerSecProperty().get()));
+
+            // node
+            inventory.put(InventoryItem.version, Version.VERSION);
+            inventory.put(InventoryItem.usedMemory, String.valueOf(Profiler.getUsedMemoryInBytes()));
+            inventory.put(InventoryItem.jvmStartTime, String.valueOf(ManagementFactory.getRuntimeMXBean().getStartTime()));
+
+            log.info("Send inventory {} to {}", inventory, connection.getPeersNodeAddressOptional());
+            GetInventoryResponse getInventoryResponse = new GetInventoryResponse(inventory);
+            networkNode.sendMessage(connection, getInventoryResponse);
+        }
+    }
+
+    public void shutDown() {
+        networkNode.removeMessageListener(this);
+    }
+}

p2p/src/main/java/bisq/network/p2p/inventory/GetInventoryRequestManager.java → core/src/main/java/bisq/core/network/p2p/inventory/GetInventoryRequestManager.java

@@ -15,7 +15,9 @@
  * along with Bisq. If not, see <http://www.gnu.org/licenses/>.
  */
 
-package bisq.network.p2p.inventory;
+package bisq.core.network.p2p.inventory;
+
+import bisq.core.network.p2p.inventory.model.InventoryItem;
 
 import bisq.network.p2p.NodeAddress;
 import bisq.network.p2p.network.NetworkNode;
@@ -41,11 +43,12 @@ public GetInventoryRequestManager(NetworkNode networkNode) {
     }
 
     public void request(NodeAddress nodeAddress,
-                        Consumer<Map<String, Integer>> resultHandler,
+                        Consumer<Map<InventoryItem, String>> resultHandler,
                         ErrorMessageHandler errorMessageHandler) {
         if (requesterMap.containsKey(nodeAddress)) {
-            log.warn("There is still an open request pending for {}", nodeAddress.getFullAddress());
-            return;
+            log.warn("There was still a pending request for {}. We shut it down and make a new request",
+                    nodeAddress.getFullAddress());
+            requesterMap.get(nodeAddress).shutDown();
         }
 
         GetInventoryRequester getInventoryRequester = new GetInventoryRequester(networkNode,