Use boost::asio::thread_pool instead of std::async

This fixes a crash on some systems due to too many files being opened at once.
It also reduces the memory footprint significantly.

CMakeLists.txt

@@ -14,7 +14,8 @@ add_subdirectory("libs/doboz")
 
 # boost
 set(Boost_USE_STATIC_LIBS ON) 
-FIND_PACKAGE(Boost 1.65 REQUIRED COMPONENTS filesystem regex)
+FIND_PACKAGE(Boost 1.66 REQUIRED COMPONENTS filesystem system date_time regex)
+
 INCLUDE_DIRECTORIES( ${Boost_INCLUDE_DIR} )
 
 # Qt5

DSCSTools/MDB1.cpp

@@ -9,6 +9,7 @@
 #include <deque>
 #include "../libs/doboz/Compressor.h"
 #include "../libs/doboz/Decompressor.h"
+#include <boost/asio.hpp>
 
 #define MDB1_MAGIC_VALUE 0x3142444D
 const uint8_t CRYPT_KEY_1[997] = { 0xD3, 0x53, 0xD2, 0x85, 0xDC, 0x87, 0x77, 0xA7, 0x16, 0xFA, 0x8D, 0x45, 0x9D, 0x14, 0x60, 0x3B, 0x9B, 0x7B, 0xDA, 0xED, 0x25, 0xFD, 0xF5, 0x8D, 0x44, 0xD0, 0xEB, 0x8B, 0xAB, 0x4B, 0x6A, 0x3E, 0x01, 0x28, 0x63, 0xA3, 0xE3, 0x23, 0x63, 0xA3, 0xE2, 0x55, 0x6D, 0xA5, 0x7C, 0xA8, 0xE4, 0xF0, 0x8B, 0xAA, 0x7D, 0x74, 0x40, 0x9C, 0x47, 0x36, 0x9A, 0xAE, 0xB1, 0x19, 0x60, 0x3B, 0x9A, 0xAD, 0xE4, 0xEF, 0xBE, 0x82, 0x76, 0xDA, 0xED, 0x25, 0xFD, 0xF5, 0x8D, 0x45, 0x9C, 0x47, 0x37, 0x67, 0xD6, 0xB9, 0x81, 0xA8, 0xE3, 0x22, 0x96, 0x79, 0x40, 0x9C, 0x48, 0x04, 0x90, 0xAB, 0x4B, 0x6B, 0x0A, 0x5E, 0xA1, 0x48, 0x03, 0xC3, 0x83, 0x42, 0x35, 0xCD, 0x85, 0xDD, 0x55, 0x6C, 0xD7, 0x87, 0x76, 0xD9, 0x20, 0xFC, 0x28, 0x63, 0xA2, 0x15, 0x2D, 0x64, 0x6F, 0x3E, 0x02, 0xF6, 0x5A, 0x6D, 0xA5, 0x7D, 0x74, 0x3F, 0xCE, 0x51, 0x39, 0x00, 0x5C, 0x08, 0xC3, 0x82, 0x75, 0x0C, 0xF8, 0xF3, 0xF2, 0x26, 0xCA, 0x1E, 0x62, 0xD5, 0xED, 0x24, 0x30, 0xCC, 0xB8, 0xB3, 0xB3, 0xB2, 0xE6, 0x89, 0x11, 0xF9, 0xC0, 0x1B, 0xFA, 0x8E, 0x12, 0xC6, 0xE9, 0xF1, 0x58, 0xD4, 0x20, 0xFB, 0x5B, 0x3B, 0x9A, 0xAD, 0xE4, 0xF0, 0x8B, 0xAA, 0x7D, 0x74, 0x40, 0x9C, 0x47, 0x36, 0x9A, 0xAD, 0xE4, 0xF0, 0x8C, 0x77, 0xA7, 0x16, 0xFA, 0x8D, 0x45, 0x9C, 0x47, 0x36, 0x99, 0xE0, 0xBB, 0x1B, 0xFB, 0x5B, 0x3B, 0x9B, 0x7A, 0x0E, 0x91, 0x78, 0x73, 0x73, 0x72, 0xA5, 0x7D, 0x75, 0x0C, 0xF7, 0x26, 0xC9, 0x51, 0x38, 0x34, 0x00, 0x5C, 0x08, 0xC4, 0x50, 0x6C, 0xD7, 0x86, 0xAA, 0x7D, 0x75, 0x0C, 0xF7, 0x26, 0xC9, 0x50, 0x6B, 0x0B, 0x2A, 0xFE, 0xC2, 0xB6, 0x19, 0x60, 0x3C, 0x68, 0xA4, 0xB0, 0x4C, 0x38, 0x33, 0x32, 0x65, 0x3D, 0x34, 0x00, 0x5C, 0x07, 0xF6, 0x59, 0xA0, 0x7C, 0xA7, 0x16, 0xF9, 0xC1, 0xE8, 0x24, 0x2F, 0xFF, 0x8E, 0x12, 0xC6, 0xE9, 0xF0, 0x8C, 0x78, 0x74, 0x40, 0x9B, 0x7A, 0x0E, 0x91, 0x79, 0x41, 0x69, 0x71, 0xD9, 0x20, 0xFB, 0x5B, 0x3B, 0x9A, 0xAE, 0xB2, 0xE6, 0x8A, 0xDE, 0x22, 0x95, 0xAC, 0x18, 0x93, 0x12, 0xC5, 0x1D, 0x95, 0xAC, 0x18, 0x93, 0x13, 0x93, 0x12, 0xC6, 0xEA, 0xBD, 0xB5, 0x4C, 0x38, 0x34, 0x00, 0x5B, 0x3B, 0x9A, 0xAD, 0xE5, 0xBD, 0xB5, 0x4C, 0x38, 0x34, 0xFF, 0x8E, 0x11, 0xF8, 0xF4, 0xC0, 0x1B, 0xFB, 0x5B, 0x3B, 0x9A, 0xAE, 0xB2, 0xE5, 0xBD, 0xB5, 0x4D, 0x05, 0x5D, 0xD5, 0xED, 0x24, 0x30, 0xCC, 0xB8, 0xB4, 0x7F, 0x0F, 0x5E, 0xA2, 0x15, 0x2D, 0x64, 0x6F, 0x3E, 0x02, 0xF6, 0x59, 0xA1, 0x48, 0x03, 0xC2, 0xB6, 0x1A, 0x2E, 0x31, 0x98, 0x13, 0x93, 0x12, 0xC5, 0x1D, 0x95, 0xAD, 0xE4, 0xF0, 0x8C, 0x77, 0xA7, 0x16, 0xF9, 0xC1, 0xE9, 0xF1, 0x58, 0xD4, 0x20, 0xFB, 0x5B, 0x3A, 0xCD, 0x84, 0x10, 0x2C, 0x98, 0x14, 0x5F, 0x6E, 0x72, 0xA5, 0x7C, 0xA8, 0xE4, 0xEF, 0xBE, 0x81, 0xA9, 0xB0, 0x4B, 0x6B, 0x0A, 0x5D, 0xD4, 0x20, 0xFC, 0x27, 0x97, 0x47, 0x37, 0x66, 0x09, 0x90, 0xAB, 0x4A, 0x9E, 0xE2, 0x55, 0x6C, 0xD8, 0x54, 0x9F, 0xAE, 0xB2, 0xE6, 0x89, 0x11, 0xF9, 0xC0, 0x1C, 0xC7, 0xB6, 0x1A, 0x2E, 0x32, 0x66, 0x09, 0x91, 0x79, 0x41, 0x68, 0xA4, 0xB0, 0x4B, 0x6A, 0x3E, 0x02, 0xF6, 0x59, 0xA1, 0x48, 0x04, 0x90, 0xAB, 0x4B, 0x6A, 0x3E, 0x01, 0x28, 0x63, 0xA3, 0xE2, 0x56, 0x39, 0x01, 0x28, 0x63, 0xA2, 0x16, 0xF9, 0xC0, 0x1B, 0xFA, 0x8E, 0x11, 0xF9, 0xC1, 0xE9, 0xF1, 0x59, 0xA1, 0x48, 0x03, 0xC3, 0x82, 0x76, 0xD9, 0x20, 0xFC, 0x27, 0x96, 0x79, 0x40, 0x9B, 0x7B, 0xDA, 0xEE, 0xF1, 0x59, 0xA0, 0x7C, 0xA7, 0x17, 0xC7, 0xB7, 0xE6, 0x89, 0x11, 0xF9, 0xC1, 0xE9, 0xF1, 0x59, 0xA0, 0x7C, 0xA7, 0x16, 0xFA, 0x8D, 0x44, 0xCF, 0x1E, 0x62, 0xD5, 0xED, 0x25, 0xFD, 0xF4, 0xBF, 0x4E, 0xD1, 0xB8, 0xB3, 0xB2, 0xE5, 0xBC, 0xE7, 0x57, 0x06, 0x2A, 0xFE, 0xC2, 0xB5, 0x4D, 0x04, 0x8F, 0xDE, 0x22, 0x96, 0x79, 0x40, 0x9B, 0x7B, 0xDA, 0xED, 0x25, 0xFC, 0x28, 0x64, 0x70, 0x0C, 0xF7, 0x27, 0x97, 0x46, 0x6A, 0x3D, 0x35, 0xCC, 0xB7, 0xE7, 0x56, 0x3A, 0xCD, 0x84, 0x0F, 0x5E, 0xA1, 0x48, 0x04, 0x90, 0xAC, 0x18, 0x94, 0xDF, 0xEE, 0xF1, 0x59, 0xA1, 0x49, 0xD1, 0xB9, 0x80, 0xDC, 0x88, 0x43, 0x03, 0xC3, 0x82, 0x76, 0xD9, 0x20, 0xFB, 0x5B, 0x3A, 0xCE, 0x52, 0x06, 0x29, 0x31, 0x98, 0x14, 0x60, 0x3C, 0x67, 0xD7, 0x86, 0xAA, 0x7E, 0x42, 0x35, 0xCD, 0x85, 0xDD, 0x55, 0x6D, 0xA5, 0x7D, 0x75, 0x0D, 0xC5, 0x1D, 0x94, 0xE0, 0xBB, 0x1A, 0x2D, 0x64, 0x6F, 0x3E, 0x01, 0x29, 0x30, 0xCB, 0xEA, 0xBE, 0x81, 0xA9, 0xB0, 0x4C, 0x38, 0x34, 0xFF, 0x8F, 0xDE, 0x22, 0x95, 0xAD, 0xE5, 0xBD, 0xB5, 0x4C, 0x37, 0x66, 0x09, 0x91, 0x79, 0x40, 0x9C, 0x47, 0x37, 0x67, 0xD7, 0x86, 0xAA, 0x7D, 0x74, 0x40, 0x9C, 0x47, 0x37, 0x66, 0x09, 0x90, 0xAB, 0x4B, 0x6B, 0x0A, 0x5D, 0xD5, 0xEC, 0x58, 0xD3, 0x53, 0xD3, 0x53, 0xD3, 0x52, 0x06, 0x29, 0x30, 0xCC, 0xB8, 0xB4, 0x7F, 0x0F, 0x5F, 0x6F, 0x3E, 0x02, 0xF5, 0x8D, 0x45, 0x9D, 0x14, 0x5F, 0x6F, 0x3E, 0x01, 0x29, 0x31, 0x98, 0x13, 0x93, 0x13, 0x92, 0x45, 0x9D, 0x14, 0x5F, 0x6E, 0x71, 0xD8, 0x54, 0xA0, 0x7B, 0xDB, 0xBA, 0x4D, 0x05, 0x5C, 0x08, 0xC3, 0x82, 0x75, 0x0D, 0xC4, 0x4F, 0x9F, 0xAE, 0xB1, 0x19, 0x60, 0x3C, 0x68, 0xA4, 0xAF, 0x7F, 0x0E, 0x92, 0x45, 0x9D, 0x14, 0x60, 0x3C, 0x67, 0xD7, 0x86, 0xA9, 0xB0, 0x4C, 0x37, 0x67, 0xD6, 0xBA, 0x4D, 0x04, 0x90, 0xAB, 0x4A, 0x9D, 0x14, 0x5F, 0x6E, 0x72, 0xA6, 0x49, 0xD1, 0xB9, 0x80, 0xDB, 0xBB, 0x1B, 0xFA, 0x8D, 0x44, 0xCF, 0x1E, 0x62, 0xD6, 0xB9, 0x80, 0xDC, 0x87, 0x77, 0xA6, 0x49, 0xD1, 0xB9, 0x80, 0xDB, 0xBB, 0x1B, 0xFA, 0x8D, 0x44, 0xD0, 0xEB, 0x8A, 0xDE, 0x21, 0xC8, 0x84, 0x0F, 0x5E, 0xA1, 0x49, 0xD1, 0xB8, 0xB4, 0x80, 0xDC, 0x88, 0x43, 0x03, 0xC3, 0x83, 0x42, 0x35, 0xCD, 0x84, 0x0F, 0x5E, 0xA1, 0x48, 0x04, 0x8F, 0xDF, 0xEE, 0xF1, 0x59, 0xA0, 0x7C, 0xA7, 0x17, 0xC7, 0xB6, 0x19, 0x61, 0x08, 0xC4, 0x4F, 0x9F, 0xAE, 0xB1, 0x18, 0x93, 0x12, 0xC6, 0xEA, 0xBD, 0xB4, 0x80, 0xDC, 0x88, 0x44, 0xD0, 0xEB, 0x8B, 0xAB, 0x4B, 0x6B, 0x0B, 0x2A, 0xFE, 0xC2, 0xB6, 0x1A, 0x2D, 0x65, 0x3D, 0x35, 0xCC, 0xB8, 0xB4, 0x80, 0xDC, 0x88, 0x43, 0x03, 0xC2, 0xB5, 0x4D, 0x04, 0x8F, 0xDF, 0xEF, 0xBE, 0x81, 0xA8, 0xE3, 0x23, 0x63, 0xA2, 0x16, 0xF9, 0xC0, 0x1B, 0xFA, 0x8E, 0x11, 0xF9, 0xC1, 0xE9, 0xF0, 0x8B, 0xAA, 0x7E, 0x42, 0x35, 0xCD, 0x84, 0x10, 0x2C, 0x97, 0x46, 0x69, 0x70, 0x0C, 0xF7, 0x27, 0x97, 0x47, 0x37, 0x66, 0x0A, 0x5E, 0xA1, 0x49, 0xD0, 0xEC, 0x58, 0xD4, 0x20, 0xFC, 0x28, 0x64, 0x6F, 0x3E, 0x01, 0x28, 0x63, 0xA2, 0x15, 0x2C, 0x98, 0x14, 0x60, 0x3B, 0x9B };
@@ -339,13 +340,20 @@ void packMDB1(const boost::filesystem::path source, const boost::filesystem::pat
 		if (boost::filesystem::is_regular_file(i))
 			files.push_back(i);
 
-	std::map<std::string, std::future<CompressionResult>> futureMap;
-
-	std::cout << "Start compressing files..." << std::endl;
 	// start compressing files
-	// this may have to be replaced with a thread pool in the future, since std::async may not use one in all cases
-	for (auto file : files)
-		futureMap[file.string()] = std::async(getFileData, file, compress);
+	std::map<std::string, std::promise<CompressionResult>> futureMap;
+
+	uint32_t coreCount = std::thread::hardware_concurrency();
+	boost::asio::thread_pool pool(coreCount * 2); // twice the core count to account for blocking threads
+	std::cout << "Start compressing files with " << coreCount * 2 << " threads..." << std::endl;
+
+	for (auto file : files) {
+		futureMap[file.string()] = std::promise<CompressionResult>();
+
+		boost::asio::post(pool, [&promise=futureMap[file.string()], file, compress] {
+			promise.set_value(getFileData(file, compress));
+		});
+	}
 
 	std::vector<FileEntry> header1(files.size() + 1);
 	std::vector<FileNameEntry> header2(files.size() + 1);
@@ -389,7 +397,7 @@ void packMDB1(const boost::filesystem::path source, const boost::filesystem::pat
 		ptrdiff_t nodeId = std::distance(nodes.begin(), found);
 
 		// get data and write it
-		CompressionResult data = futureMap[file.string()].get();
+		CompressionResult data = futureMap[file.string()].get_future().get();
 
 		output.seekp(dataStart + offset);
 		output.write(data.data, data.size);

README.md

@@ -16,11 +16,10 @@ DSCSTools are modding tools for the Steam release of "Digimon Story Cyber Sleuth
 * more MBE structure files
 
 # Usage
-Currently only a command line interface is available. It is recommended to use a Windows system, due to the MDB1 format using Windows file separator.
+Currently only a command line interface is available. It is recommended to use a Windows system, due to the MDB1 format using Windows file separators.
 The tool *should* work on *nix systems, but is not as well tested.
 
-In order to use the repack function for large archives (-> DSDB) it is recommended that you have at least 16 GiB of RAM installed.
-An alternative is to use the DSDBA archive for any modified files, since it's contents have a higher priority than the main DSDB.
+To create mods it is strong recommended to use Pherakki's [SimpleDSCSModManager](https://github.com/Pherakki/SimpleDSCSModManager).
 
 **Do not use Microsoft Excel to modify extracted CSV files, it does *not* create RFC 4180 compliant CSV.** Use LibreOffice/OpenOffice as an alternative.
 
@@ -78,7 +77,7 @@ $ make install
 With the binary being located in /DSCSTools/
 
 ## Requirements
-* Boost 1.65 or higher
+* Boost 1.66 or higher
 * CMake 3.10 or higher
 * A C++ compiler
 
@@ -90,6 +89,12 @@ The tool uses the [doboz compression library](https://voxelium.wordpress.com/201
 * E-Mail: sydmontague@web.de
 * Reddit: [/u/Sydmontague](https://reddit.com/u/sydmontague)
 
+# Other DSCS Modding Projects/Tools
+* [SimpleDSCSModManager](https://github.com/Pherakki/SimpleDSCSModManager) by Pherakki
+* [Blender-Tools-for-DSCS](https://github.com/Pherakki/Blender-Tools-for-DSCS/) by Pherakki
+* [NutCracker](https://github.com/SydMontague/NutCracker)
+  * a decompiler for the game's Squirrel script files
+
 # Disclaimer
 This project is work-in-progress and I'm still learning how to properly use C++ and CMake, so excuse me if I make some nooby mistakes. :)