From ef2bd8ff335433e695eb561d7ecd44f233e58bf0 Mon Sep 17 00:00:00 2001 From: Todd Kordenbrock Date: Mon, 4 Oct 2021 15:44:11 -0500 Subject: [PATCH] REPO: Transfer internal updates for 1.2108.1 to github --- CMakeLists.txt | 214 ++--- INSTALL.md | 765 +++++++++-------- LICENSE.md | 2 +- NEWS.md | 34 +- README.md | 44 +- cmake/FaodelCheckTransports.cmake | 74 ++ cmake/FaodelConfig.cmake.in | 1 + cmake/FaodelRpathHelpers.cmake | 92 ++ cmake/FaodelSummary.cmake | 110 +++ cmake/FaodelTPLs.cmake | 167 ++-- cmake/faodelConfig.h.in | 7 + docs/ConfigurationFileCookbook.md | 220 +++++ docs/Doxyfile | 9 + docs/WhatsAFaodel.md | 23 + docs/faodhail.jpg | Bin 0 -> 114583 bytes examples/CMakeLists.txt | 49 +- examples/common/CMakeLists.txt | 9 +- examples/common/bootstrap/CMakeLists.txt | 9 +- .../common/bootstrap/bootstrap_example.cpp | 6 +- examples/common/data-types/CMakeLists.txt | 8 +- examples/common/data-types/main.cpp | 6 +- examples/common/data-types/nodeid_example.cpp | 6 +- .../common/data-types/resourceurl_example.cpp | 6 +- examples/common/info-interface/CMakeLists.txt | 9 +- .../common/info-interface/info_interface.cpp | 10 +- .../common/logging-interface/CMakeLists.txt | 9 +- examples/common/logging-interface/ClassA.cpp | 6 +- examples/common/logging-interface/ClassA.hh | 6 +- examples/common/logging-interface/ClassB.cpp | 6 +- examples/common/logging-interface/ClassB.hh | 6 +- .../logging-interface/example1_selectiveA.cpp | 6 +- .../logging-interface/example2_disableB.cpp | 6 +- .../logging-interface/logging_interface.cpp | 8 +- examples/common/serialization/CMakeLists.txt | 23 + examples/common/serialization/MyContainer.hh | 62 ++ .../serialization/serialization_example.cpp | 49 ++ examples/common/singleton/CMakeLists.txt | 30 +- examples/common/singleton/ServiceA.cpp | 6 +- examples/common/singleton/ServiceA.hh | 6 +- examples/common/singleton/ServiceB.cpp | 6 +- examples/common/singleton/ServiceB.hh | 6 +- examples/common/singleton/ServiceC.cpp | 6 +- examples/common/singleton/ServiceC.hh | 6 +- .../common/singleton/singleton_example.cpp | 6 +- examples/dirman/CMakeLists.txt | 41 +- .../dirman/dirman1-preload-configuration.cpp | 8 +- examples/dirman/dirman2-runtime-define.cpp | 6 +- examples/dirman/example1.cpp | 6 +- examples/dirman/example2.cpp | 6 +- examples/dirman/example3.cpp | 6 +- examples/dirman/example4.cpp | 6 +- examples/dirman/example5.cpp | 6 +- examples/faodel-cli/basic-startstop.sh | 2 +- .../playback-scripts/s01_basic_local.play | 81 ++ .../s02_trace_pool_example.play | 51 ++ .../s03_advanced_file_handling.play | 75 ++ .../playback-scripts/s04_bucket_renaming.play | 50 ++ .../s05_self_contained_pool.play | 36 + examples/kelpie/CMakeLists.txt | 20 +- examples/kelpie/README.md | 10 +- examples/kelpie/compute/CMakeLists.txt | 24 + examples/kelpie/compute/ex_compute1_pick.cpp | 100 +++ examples/kelpie/compute/ex_compute2_udf.cpp | 112 +++ examples/kelpie/nonet/dim-sum/CMakeLists.txt | 11 +- examples/kelpie/nonet/dim-sum/Consumer.cpp | 6 +- examples/kelpie/nonet/dim-sum/Consumer.hh | 6 +- examples/kelpie/nonet/dim-sum/DimSum.cpp | 6 +- examples/kelpie/nonet/dim-sum/DimSum.hh | 6 +- examples/kelpie/nonet/dim-sum/Producer.cpp | 6 +- examples/kelpie/nonet/dim-sum/Producer.hh | 6 +- .../kelpie/nonet/dim-sum/WorkerThread.cpp | 6 +- examples/kelpie/nonet/dim-sum/WorkerThread.hh | 6 +- examples/kelpie/nonet/dim-sum/dim-sum.cpp | 17 +- .../nonet/local-pool-basics/CMakeLists.txt | 14 +- .../local-pool-basics/ex1_get_pool_handle.cpp | 6 +- .../local-pool-basics/ex2_publish_need.cpp | 6 +- .../local-pool-basics/local-pool-basics.cpp | 7 +- .../kelpie/nonet/start-finish/CMakeLists.txt | 10 +- .../nonet/start-finish/start-finish.cpp | 7 +- .../nonet/storing-in-iom/CMakeLists.txt | 15 +- .../nonet/storing-in-iom/storing-in-iom.cpp | 18 +- .../kelpie/nonet/using-whookie/CMakeLists.txt | 10 +- .../nonet/using-whookie/using-whookie.cpp | 7 +- examples/kelpie/prod-con/CMakeLists.txt | 28 +- examples/kelpie/prod-con/Globals.cpp | 6 +- examples/kelpie/prod-con/Globals.hh | 6 +- examples/kelpie/prod-con/prod-con.cpp | 12 +- examples/lunasa/CMakeLists.txt | 10 +- examples/lunasa/bootstrap/CMakeLists.txt | 9 +- .../lunasa/bootstrap/bootstrap_example.cpp | 6 +- examples/lunasa/ldo_packing/CMakeLists.txt | 19 + examples/lunasa/ldo_packing/MyData.hh | 55 ++ examples/lunasa/ldo_packing/MyHelpers.hh | 95 ++ .../ldo_packing/ldo_packing_example.cpp | 279 ++++++ .../lunasa/simple_ldo_send/CMakeLists.txt | 15 +- .../simple_ldo_send_example.cpp | 14 +- .../lunasa/simple_user_ldo_put/CMakeLists.txt | 15 +- .../simple_user_ldo_put_example.cpp | 13 +- examples/nnti/CMakeLists.txt | 9 +- examples/nnti/cpp/PingPong.cpp | 6 +- examples/nnti/cpp/example_utils.cpp | 6 +- examples/nnti/cpp/example_utils.hpp | 6 +- examples/opbox/CMakeLists.txt | 70 +- examples/opbox/advanced/CMakeLists.txt | 2 + .../opbox/advanced/job2job/CMakeLists.txt | 6 +- .../opbox/advanced/job2job/OpInterjobPing.cpp | 12 +- .../opbox/advanced/job2job/OpInterjobPing.hh | 6 +- .../opbox/advanced/job2job/job2job-client.cpp | 8 +- .../opbox/advanced/job2job/job2job-server.cpp | 8 +- .../advanced/lingering-data/CMakeLists.txt | 6 +- .../lingering-data/lingering-data.cpp | 8 +- examples/opbox/basic/CMakeLists.txt | 3 + .../basic/message-packing/CMakeLists.txt | 32 +- .../opbox/basic/message-packing/example1.cpp | 7 +- .../opbox/basic/message-packing/example2.cpp | 6 +- .../opbox/basic/message-packing/example3.cpp | 9 +- .../opbox/basic/message-packing/example4.cpp | 27 +- .../opbox/basic/message-packing/example5.cpp | 6 +- .../opbox/basic/message-packing/example6.cpp | 6 +- .../opbox/basic/message-packing/example7.cpp | 16 +- .../message_packing_example.cpp | 12 +- .../opbox/basic/my-simple-ping/CMakeLists.txt | 4 +- .../basic/my-simple-ping/OpMySimplePing.cpp | 12 +- .../basic/my-simple-ping/OpMySimplePing.hh | 6 +- .../basic/my-simple-ping/my_simple_ping.cpp | 14 +- examples/opbox/basic/rdma-ping/CMakeLists.txt | 4 +- examples/opbox/basic/rdma-ping/OpRdmaPing.cpp | 12 +- examples/opbox/basic/rdma-ping/OpRdmaPing.hh | 6 +- examples/opbox/basic/rdma-ping/rdma_ping.cpp | 14 +- .../opbox/benchmarks/atomics/CMakeLists.txt | 4 +- .../atomics/OpBenchmarkCompareSwap.cpp | 12 +- .../atomics/OpBenchmarkCompareSwap.hh | 6 +- .../atomics/OpBenchmarkFetchAdd.cpp | 12 +- .../benchmarks/atomics/OpBenchmarkFetchAdd.hh | 6 +- examples/opbox/benchmarks/atomics/atomics.cpp | 15 +- .../opbox/benchmarks/connect/CMakeLists.txt | 4 +- examples/opbox/benchmarks/connect/connect.cpp | 6 +- .../benchmarks/msgplusrdma/CMakeLists.txt | 4 +- .../msgplusrdma/OpBenchmarkMsgRdma.cpp | 12 +- .../msgplusrdma/OpBenchmarkMsgRdma.hh | 6 +- .../benchmarks/msgplusrdma/msgplusrdma.cpp | 14 +- examples/opbox/benchmarks/rdma/CMakeLists.txt | 4 +- .../opbox/benchmarks/rdma/OpBenchmarkGet.cpp | 76 +- .../opbox/benchmarks/rdma/OpBenchmarkGet.hh | 13 +- .../opbox/benchmarks/rdma/OpBenchmarkPut.cpp | 71 +- .../opbox/benchmarks/rdma/OpBenchmarkPut.hh | 11 +- examples/opbox/benchmarks/rdma/rdma.cpp | 14 +- .../benchmarks/short_message/CMakeLists.txt | 4 +- .../short_message/OpBenchmarkPing.cpp | 10 +- .../short_message/OpBenchmarkPing.hh | 6 +- .../short_message/short_message.cpp | 14 +- examples/opbox/collectives/CMakeLists.txt | 3 + .../collectives/rapidfire/CMakeLists.txt | 4 +- .../collectives/rapidfire/OpRapidFire.cpp | 14 +- .../collectives/rapidfire/OpRapidFire.hh | 6 +- .../opbox/collectives/rapidfire/rapidfire.cpp | 14 +- .../opbox/collectives/ringer/CMakeLists.txt | 4 +- .../opbox/collectives/ringer/OpRinger.cpp | 26 +- examples/opbox/collectives/ringer/OpRinger.hh | 6 +- examples/opbox/collectives/ringer/RingInfo.hh | 6 +- examples/opbox/collectives/ringer/ringer.cpp | 14 +- .../collectives/scatter-gather/CMakeLists.txt | 4 +- .../collectives/scatter-gather/OpGather.cpp | 12 +- .../collectives/scatter-gather/OpGather.hh | 6 +- .../collectives/scatter-gather/OpScatter.cpp | 12 +- .../collectives/scatter-gather/OpScatter.hh | 6 +- .../scatter-gather/scatter_gather.cpp | 14 +- .../opbox-example-support/CMakeLists.txt | 11 + .../opbox/opbox-example-support/Globals.cpp | 6 +- .../opbox/opbox-example-support/Globals.hh | 6 +- .../opbox-example-support/SimpleDataStore.cpp | 6 +- .../opbox-example-support/SimpleDataStore.hh | 6 +- examples/services/CMakeLists.txt | 10 +- examples/services/backburner/CMakeLists.txt | 12 +- .../backburner/backburner_example.cpp | 6 +- .../services/mpi-sync-start/CMakeLists.txt | 22 + .../mpi_sync_start_advanced.cpp | 6 +- .../mpi-sync-start/mpi_sync_start_basic.cpp | 6 +- examples/whookie/CMakeLists.txt | 21 +- examples/whookie/bootstrap_example.cpp | 31 +- examples/whookie/killit_example.cpp | 18 +- src/dirman/CMakeLists.txt | 2 +- src/dirman/DirMan.cpp | 6 +- src/dirman/DirMan.hh | 6 +- src/dirman/README_DirMan.md | 67 +- src/dirman/common/DirectoryCache.cpp | 14 +- src/dirman/common/DirectoryCache.hh | 6 +- src/dirman/common/DirectoryOwnerCache.cpp | 12 +- src/dirman/common/DirectoryOwnerCache.hh | 6 +- src/dirman/core/DirManCoreBase.cpp | 30 +- src/dirman/core/DirManCoreBase.hh | 10 +- src/dirman/core/DirManCoreCentralized.cpp | 16 +- src/dirman/core/DirManCoreCentralized.hh | 6 +- src/dirman/core/DirManCoreStatic.cpp | 14 +- src/dirman/core/DirManCoreStatic.hh | 6 +- src/dirman/core/DirManCoreUnconfigured.cpp | 6 +- src/dirman/core/DirManCoreUnconfigured.hh | 8 +- src/dirman/core/Singleton.cpp | 8 +- src/dirman/core/Singleton.hh | 6 +- src/dirman/ops/OpDirManCentralized.cpp | 22 +- src/dirman/ops/OpDirManCentralized.hh | 6 +- src/dirman/ops/OpDirManCentralized_Origin.cpp | 10 +- src/dirman/ops/OpDirManCentralized_Target.cpp | 8 +- src/dirman/ops/msg_dirman.cpp | 19 +- src/dirman/ops/msg_dirman.hh | 10 +- src/faodel-common/Bootstrap.cpp | 27 +- src/faodel-common/Bootstrap.hh | 8 +- src/faodel-common/BootstrapImplementation.cpp | 224 ++++- src/faodel-common/BootstrapImplementation.hh | 19 +- src/faodel-common/BootstrapInterface.hh | 6 +- src/faodel-common/Bucket.cpp | 7 +- src/faodel-common/Bucket.hh | 23 +- src/faodel-common/CMakeLists.txt | 11 +- src/faodel-common/Common.hh | 6 +- src/faodel-common/Configuration.cpp | 328 +++---- src/faodel-common/Configuration.hh | 18 +- src/faodel-common/Debug.cpp | 31 +- src/faodel-common/Debug.hh | 50 +- src/faodel-common/DirectoryInfo.cpp | 10 +- src/faodel-common/DirectoryInfo.hh | 6 +- src/faodel-common/FaodelTypes.hh | 6 +- src/faodel-common/InfoInterface.hh | 6 +- src/faodel-common/LoggingInterface.cpp | 6 +- src/faodel-common/LoggingInterface.hh | 6 +- src/faodel-common/LoggingInterfaceMacros.hh | 6 +- src/faodel-common/MutexWrapper.cpp | 12 +- src/faodel-common/MutexWrapper.hh | 6 +- src/faodel-common/NodeID.cpp | 6 +- src/faodel-common/NodeID.hh | 24 +- src/faodel-common/QuickHTML.cpp | 6 +- src/faodel-common/QuickHTML.hh | 6 +- src/faodel-common/README_Common.md | 16 +- src/faodel-common/ReplyStream.cpp | 19 +- src/faodel-common/ReplyStream.hh | 10 +- src/faodel-common/ResourceURL.cpp | 41 +- src/faodel-common/ResourceURL.hh | 25 +- .../SerializationHelpersBoost.hh | 42 +- .../SerializationHelpersCereal.hh | 42 + src/faodel-common/StringHelpers.cpp | 269 ++++-- src/faodel-common/StringHelpers.hh | 40 +- src/faodel-services/BackBurner.cpp | 78 +- src/faodel-services/BackBurner.hh | 11 +- src/faodel-services/MPISyncStart.cpp | 48 +- src/faodel-services/MPISyncStart.hh | 6 +- src/faodel-services/README_Services.md | 35 +- src/kelpie/CMakeLists.txt | 23 +- src/kelpie/Kelpie.cpp | 57 +- src/kelpie/Kelpie.hh | 22 +- src/kelpie/Key.cpp | 164 +++- src/kelpie/Key.hh | 23 +- src/kelpie/README_Kelpie.md | 72 +- src/kelpie/common/ComputeRegistry.cpp | 151 ++++ src/kelpie/common/ComputeRegistry.hh | 61 ++ src/kelpie/common/KelpieInternal.cpp | 6 +- src/kelpie/common/KelpieInternal.hh | 6 +- src/kelpie/common/ObjectCapacities.cpp | 11 +- src/kelpie/common/ObjectCapacities.hh | 12 +- src/kelpie/common/OpArgsObjectAvailable.cpp | 6 +- src/kelpie/common/OpArgsObjectAvailable.hh | 19 +- src/kelpie/common/Types.cpp | 80 +- src/kelpie/common/Types.hh | 80 +- src/kelpie/core/KelpieCoreBase.cpp | 13 +- src/kelpie/core/KelpieCoreBase.hh | 18 +- src/kelpie/core/KelpieCoreNoNet.cpp | 43 +- src/kelpie/core/KelpieCoreNoNet.hh | 15 +- src/kelpie/core/KelpieCoreStandard.cpp | 72 +- src/kelpie/core/KelpieCoreStandard.hh | 16 +- src/kelpie/core/KelpieCoreUnconfigured.cpp | 22 +- src/kelpie/core/KelpieCoreUnconfigured.hh | 14 +- src/kelpie/core/Singleton.cpp | 58 +- src/kelpie/core/Singleton.hh | 16 +- src/kelpie/ioms/IomBase.cpp | 28 +- src/kelpie/ioms/IomBase.hh | 26 +- src/kelpie/ioms/IomCassandra.cpp | 44 +- src/kelpie/ioms/IomCassandra.hh | 38 +- src/kelpie/ioms/IomHDF5.cpp | 68 +- src/kelpie/ioms/IomHDF5.hh | 28 +- src/kelpie/ioms/IomLevelDB.cpp | 73 +- src/kelpie/ioms/IomLevelDB.hh | 26 +- src/kelpie/ioms/IomPosixIndividualObjects.cpp | 147 +++- src/kelpie/ioms/IomPosixIndividualObjects.hh | 23 +- src/kelpie/ioms/IomRegistry.cpp | 134 ++- src/kelpie/ioms/IomRegistry.hh | 18 +- src/kelpie/localkv/LocalKV.cpp | 529 ++++++------ src/kelpie/localkv/LocalKV.hh | 71 +- src/kelpie/localkv/LocalKVCell.cpp | 79 +- src/kelpie/localkv/LocalKVCell.hh | 21 +- src/kelpie/localkv/LocalKVRow.cpp | 220 +++-- src/kelpie/localkv/LocalKVRow.hh | 41 +- src/kelpie/localkv/LocalKVTypes.hh | 36 + src/kelpie/ops/direct/OpKelpieCompute.cpp | 235 +++++ src/kelpie/ops/direct/OpKelpieCompute.hh | 120 +++ src/kelpie/ops/direct/OpKelpieDrop.cpp | 176 ++++ src/kelpie/ops/direct/OpKelpieDrop.hh | 99 +++ src/kelpie/ops/direct/OpKelpieGetBounded.cpp | 86 +- src/kelpie/ops/direct/OpKelpieGetBounded.hh | 22 +- .../ops/direct/OpKelpieGetUnbounded.cpp | 106 +-- src/kelpie/ops/direct/OpKelpieGetUnbounded.hh | 22 +- src/kelpie/ops/direct/OpKelpieList.cpp | 81 +- src/kelpie/ops/direct/OpKelpieList.hh | 20 +- src/kelpie/ops/direct/OpKelpieMeta.cpp | 106 ++- src/kelpie/ops/direct/OpKelpieMeta.hh | 43 +- src/kelpie/ops/direct/OpKelpiePublish.cpp | 77 +- src/kelpie/ops/direct/OpKelpiePublish.hh | 25 +- src/kelpie/ops/direct/msg_direct.cpp | 162 +++- src/kelpie/ops/direct/msg_direct.hh | 127 ++- src/kelpie/pools/DHTPool/DHTPool.cpp | 404 ++++++--- src/kelpie/pools/DHTPool/DHTPool.hh | 35 +- src/kelpie/pools/LocalPool/LocalPool.cpp | 107 +-- src/kelpie/pools/LocalPool/LocalPool.hh | 33 +- src/kelpie/pools/NullPool/NullPool.cpp | 234 +++++ src/kelpie/pools/NullPool/NullPool.hh | 73 ++ src/kelpie/pools/Pool.cpp | 237 ++++- src/kelpie/pools/Pool.hh | 50 +- src/kelpie/pools/PoolBase.cpp | 21 +- src/kelpie/pools/PoolBase.hh | 27 +- src/kelpie/pools/PoolRegistry.cpp | 117 ++- src/kelpie/pools/PoolRegistry.hh | 21 +- src/kelpie/pools/RFTPool/RFTPool.cpp | 21 +- src/kelpie/pools/RFTPool/RFTPool.hh | 16 +- src/kelpie/pools/ResultCollector.cpp | 61 ++ src/kelpie/pools/ResultCollector.hh | 64 ++ src/kelpie/pools/TracePool/TracePool.cpp | 181 ++++ src/kelpie/pools/TracePool/TracePool.hh | 91 ++ .../UnconfiguredPool/UnconfiguredPool.cpp | 49 +- .../UnconfiguredPool/UnconfiguredPool.hh | 43 +- src/kelpie/services/PoolServerDriver.cpp | 6 +- src/kelpie/services/PoolServerDriver.hh | 6 +- src/lunasa/CMakeLists.txt | 9 +- src/lunasa/DataObject.cpp | 116 ++- src/lunasa/DataObject.hh | 15 +- src/lunasa/Lunasa.cpp | 24 +- src/lunasa/Lunasa.hh | 10 +- src/lunasa/allocators/AllocatorBase.cpp | 6 +- src/lunasa/allocators/AllocatorBase.hh | 6 +- src/lunasa/allocators/AllocatorMalloc.cpp | 20 +- src/lunasa/allocators/AllocatorMalloc.hh | 6 +- src/lunasa/allocators/AllocatorTcmalloc.cpp | 34 +- src/lunasa/allocators/AllocatorTcmalloc.hh | 6 +- .../allocators/AllocatorUnconfigured.cpp | 6 +- .../allocators/AllocatorUnconfigured.hh | 6 +- src/lunasa/allocators/Allocators.cpp | 7 +- src/lunasa/allocators/Allocators.hh | 6 +- src/lunasa/common/Allocation.hh | 43 +- src/lunasa/common/DataObjectPacker.cpp | 422 +++++++++ src/lunasa/common/DataObjectPacker.hh | 116 +++ src/lunasa/common/DataObjectTypeRegistry.cpp | 6 +- src/lunasa/common/DataObjectTypeRegistry.hh | 6 +- src/lunasa/common/GenericRandomDataBundle.hh | 13 +- .../common/GenericSequentialDataBundle.hh | 8 +- src/lunasa/common/Helpers.cpp | 31 + src/lunasa/common/Helpers.hh | 20 + src/lunasa/common/Types.hh | 6 +- src/lunasa/core/LunasaCoreBase.cpp | 9 +- src/lunasa/core/LunasaCoreBase.hh | 6 +- src/lunasa/core/LunasaCoreSplit.cpp | 9 +- src/lunasa/core/LunasaCoreSplit.hh | 6 +- src/lunasa/core/LunasaCoreUnconfigured.cpp | 6 +- src/lunasa/core/LunasaCoreUnconfigured.hh | 6 +- src/lunasa/core/Singleton.cpp | 6 +- src/lunasa/core/Singleton.hh | 6 +- src/nnti/CMakeLists.txt | 35 +- src/nnti/nnti.cpp | 15 +- src/nnti/nnti.h | 29 +- src/nnti/nntiConfig.h.in | 2 + src/nnti/nntiProbeXDR.cmake | 7 + src/nnti/nnti_buffer.cpp | 6 +- src/nnti/nnti_buffer.hpp | 38 +- src/nnti/nnti_callback.cpp | 32 +- src/nnti/nnti_callback.hpp | 10 +- src/nnti/nnti_connection.cpp | 6 +- src/nnti/nnti_connection.hpp | 20 +- src/nnti/nnti_datatype.hpp | 8 +- src/nnti/nnti_eq.hpp | 33 +- src/nnti/nnti_freelist.hpp | 6 +- src/nnti/nnti_logger.cpp | 6 +- src/nnti/nnti_logger.h | 6 +- src/nnti/nnti_logger.hpp | 6 +- src/nnti/nnti_op.cpp | 6 +- src/nnti/nnti_op.hpp | 6 +- src/nnti/nnti_pch.hpp | 6 +- src/nnti/nnti_peer.hpp | 6 +- src/nnti/nnti_pid.cpp | 6 +- src/nnti/nnti_pid.hpp | 6 +- src/nnti/nnti_serialize.cpp | 8 +- src/nnti/nnti_serialize.hpp | 6 +- src/nnti/nnti_state_machine.hpp | 6 +- src/nnti/nnti_threads.cpp | 6 +- src/nnti/nnti_threads.h | 6 +- src/nnti/nnti_threads_types.h | 6 +- src/nnti/nnti_transport.cpp | 6 +- src/nnti/nnti_transport.hpp | 14 +- src/nnti/nnti_types.c | 32 + src/nnti/nnti_types.h | 36 +- src/nnti/nnti_url.cpp | 17 +- src/nnti/nnti_url.hpp | 7 +- src/nnti/nnti_util.cpp | 76 ++ src/nnti/nnti_util.hpp | 82 +- src/nnti/nnti_vector.hpp | 6 +- src/nnti/nnti_wid.cpp | 8 +- src/nnti/nnti_wid.hpp | 10 +- src/nnti/nnti_wr.cpp | 36 +- src/nnti/nnti_wr.hpp | 10 +- src/nnti/serializers/cereal/nnti_cereal.hpp | 6 +- src/nnti/serializers/cereal/nnti_packable.hpp | 14 +- src/nnti/serializers/xdr/nnti_xdr.h | 6 +- src/nnti/serializers/xdr/nnti_xdr.hpp | 6 +- src/nnti/transport_factory.cpp | 38 +- src/nnti/transport_factory.hpp | 6 +- src/nnti/transports/base/base_transport.cpp | 13 +- src/nnti/transports/base/base_transport.hpp | 14 +- .../transports/ibverbs/ibverbs_atomic_op.hpp | 13 +- .../transports/ibverbs/ibverbs_buffer.cpp | 15 +- .../transports/ibverbs/ibverbs_buffer.hpp | 6 +- .../transports/ibverbs/ibverbs_cmd_buffer.cpp | 14 +- .../transports/ibverbs/ibverbs_cmd_buffer.hpp | 10 +- .../transports/ibverbs/ibverbs_cmd_msg.cpp | 6 +- .../transports/ibverbs/ibverbs_cmd_msg.hpp | 6 +- .../transports/ibverbs/ibverbs_cmd_op.hpp | 10 +- .../transports/ibverbs/ibverbs_connection.hpp | 84 +- src/nnti/transports/ibverbs/ibverbs_peer.hpp | 6 +- .../transports/ibverbs/ibverbs_rdma_op.hpp | 13 +- .../transports/ibverbs/ibverbs_transport.cpp | 321 +++++-- .../transports/ibverbs/ibverbs_transport.hpp | 46 +- src/nnti/transports/ibverbs/ibverbs_wr.hpp | 6 +- src/nnti/transports/mpi/mpi_buffer.cpp | 7 +- src/nnti/transports/mpi/mpi_buffer.hpp | 6 +- src/nnti/transports/mpi/mpi_cmd_buffer.cpp | 29 +- src/nnti/transports/mpi/mpi_cmd_buffer.hpp | 7 +- src/nnti/transports/mpi/mpi_cmd_msg.cpp | 9 +- src/nnti/transports/mpi/mpi_cmd_msg.hpp | 7 +- src/nnti/transports/mpi/mpi_cmd_op.hpp | 12 +- src/nnti/transports/mpi/mpi_connection.hpp | 73 +- src/nnti/transports/mpi/mpi_peer.hpp | 6 +- src/nnti/transports/mpi/mpi_transport.cpp | 137 ++- src/nnti/transports/mpi/mpi_transport.hpp | 38 +- src/nnti/transports/null/null_transport.hpp | 184 ++-- src/nnti/transports/ugni/ugni_atomic_op.hpp | 10 +- src/nnti/transports/ugni/ugni_buffer.cpp | 8 +- src/nnti/transports/ugni/ugni_buffer.hpp | 6 +- src/nnti/transports/ugni/ugni_cmd_buffer.cpp | 6 +- src/nnti/transports/ugni/ugni_cmd_buffer.hpp | 6 +- src/nnti/transports/ugni/ugni_cmd_msg.cpp | 84 +- src/nnti/transports/ugni/ugni_cmd_msg.hpp | 6 +- src/nnti/transports/ugni/ugni_cmd_op.cpp | 8 +- src/nnti/transports/ugni/ugni_cmd_op.hpp | 8 +- src/nnti/transports/ugni/ugni_cmd_tgt.hpp | 84 +- src/nnti/transports/ugni/ugni_connection.cpp | 37 +- src/nnti/transports/ugni/ugni_connection.hpp | 8 +- src/nnti/transports/ugni/ugni_mailbox.cpp | 10 +- src/nnti/transports/ugni/ugni_mailbox.hpp | 6 +- src/nnti/transports/ugni/ugni_peer.hpp | 6 +- src/nnti/transports/ugni/ugni_rdma_op.hpp | 26 +- src/nnti/transports/ugni/ugni_transport.cpp | 341 ++++++-- src/nnti/transports/ugni/ugni_transport.hpp | 107 +-- src/nnti/transports/ugni/ugni_wr.hpp | 6 +- src/opbox/CMakeLists.txt | 8 +- src/opbox/OpBox.cpp | 27 +- src/opbox/OpBox.hh | 10 +- src/opbox/common/Message.cpp | 10 +- src/opbox/common/Message.hh | 17 +- src/opbox/common/MessageHelpers.cpp | 22 +- src/opbox/common/MessageHelpers.hh | 56 +- src/opbox/common/OpArgs.cpp | 14 +- src/opbox/common/OpArgs.hh | 6 +- src/opbox/common/OpRegistry.cpp | 29 +- src/opbox/common/OpRegistry.hh | 8 +- src/opbox/common/Types.cpp | 6 +- src/opbox/common/Types.hh | 6 +- src/opbox/core/OpBoxCoreBase.cpp | 6 +- src/opbox/core/OpBoxCoreBase.hh | 15 +- .../core/OpBoxCoreDeprecatedStandard.cpp | 119 ++- src/opbox/core/OpBoxCoreDeprecatedStandard.hh | 30 +- src/opbox/core/OpBoxCoreThreaded.cpp | 222 ++--- src/opbox/core/OpBoxCoreThreaded.hh | 25 +- src/opbox/core/OpBoxCoreUnconfigured.cpp | 9 +- src/opbox/core/OpBoxCoreUnconfigured.hh | 7 +- src/opbox/core/OpTimer.cpp | 87 ++ src/opbox/core/OpTimer.hh | 79 ++ src/opbox/core/Singleton.cpp | 10 +- src/opbox/core/Singleton.hh | 6 +- .../net/libfabric_wrapper/fab_transport.cpp | 11 +- .../net/libfabric_wrapper/fab_transport.hh | 6 +- .../libfabric_wrapper/libfabric_wrapper.cpp | 13 +- src/opbox/net/libfabric_wrapper/shared.hh | 6 +- src/opbox/net/nbr.hh | 6 +- src/opbox/net/net.cpp | 8 +- src/opbox/net/net.hh | 6 +- src/opbox/net/net_internal.hh | 6 +- src/opbox/net/nnti/NetNnti.cpp | 117 ++- src/opbox/net/peer.hh | 6 +- src/opbox/ops/Op.cpp | 6 +- src/opbox/ops/Op.hh | 6 +- src/opbox/ops/OpCount.cpp | 8 +- src/opbox/ops/OpCount.hh | 6 +- src/opbox/ops/OpHelpers.hh | 6 +- src/opbox/ops/OpPing.cpp | 12 +- src/opbox/ops/OpPing.hh | 6 +- src/sbl/README_SBL.md | 2 +- src/sbl/sbl_boost_headers.hh | 6 +- src/sbl/sbl_logger.cpp | 6 +- src/sbl/sbl_logger.hh | 6 +- src/sbl/sbl_source.cpp | 8 +- src/sbl/sbl_source.hh | 6 +- src/sbl/sbl_stream.cpp | 6 +- src/sbl/sbl_stream.hh | 6 +- src/sbl/sbl_types.hh | 6 +- src/whookie/Server.hh | 6 +- src/whookie/Whookie.hh | 6 +- src/whookie/client/Client.cpp | 6 +- src/whookie/client/Client.hh | 6 +- src/whookie/server/ServerBoost.cpp | 6 +- src/whookie/server/boost/request_handler.cpp | 71 ++ src/whookie/server/boost/request_handler.hpp | 2 + src/whookie/server/boost/server.cpp | 22 +- src/whookie/server/boost/server.hpp | 7 +- tests/common/unit/tb_common_bootstrap.cpp | 79 +- tests/common/unit/tb_common_configuration.cpp | 41 +- .../unit/tb_common_logginginterface.cpp | 6 +- tests/common/unit/tb_common_mutex.cpp | 9 +- tests/common/unit/tb_common_resourceurl.cpp | 20 +- tests/common/unit/tb_common_serialization.cpp | 45 +- tests/common/unit/tb_common_stringhelpers.cpp | 32 +- tests/common/unit/tb_common_structs.cpp | 10 +- tests/dirman/CMakeLists.txt | 2 - .../component/mpi_dirman_centralized.cpp | 6 +- .../component/mpi_dirman_op_messages.cpp | 11 +- tests/dirman/component/mpi_dirman_restart.cpp | 6 +- .../dirman/unit/tb_dirman_corecentralized.cpp | 6 +- .../dirman/unit/tb_dirman_directorycache.cpp | 6 +- .../unit/tb_dirman_directoryownercache.cpp | 6 +- tests/kelpie/CMakeLists.txt | 38 +- .../kelpie/component/mpi_kelpie_behaviors.cpp | 148 ++++ tests/kelpie/component/mpi_kelpie_compute.cpp | 170 ++++ tests/kelpie/component/mpi_kelpie_dht.cpp | 292 +++++-- tests/kelpie/component/mpi_kelpie_iom_dht.cpp | 65 +- .../component/mpi_kelpie_many_items.cpp | 245 ++++++ tests/kelpie/component/mpi_kelpie_rft.cpp | 114 +-- .../component/mpi_kelpie_simple_peer.cpp | 72 +- .../nonet/tb_kelpie_nonet_compute.cpp | 165 ++++ .../nonet/tb_kelpie_nonet_iom_from_url.cpp | 214 +++++ .../nonet/tb_kelpie_nonet_iompio.cpp | 207 +++-- .../component/support/ExperimentLauncher.cpp | 6 +- .../component/support/ExperimentLauncher.hh | 6 +- tests/kelpie/component/support/Globals.cpp | 15 +- tests/kelpie/component/support/Globals.hh | 8 +- .../tb_kelpie_multiple_hidden_inits.cpp | 169 ++++ .../unit/ioms/tb_kelpie_iom_all_basic.cpp | 28 +- .../unit/ioms/tb_kelpie_iom_pio_basic.cpp | 16 +- .../unit/ioms/tb_kelpie_iom_service_basic.cpp | 42 +- .../messages/tb_kelpie_message_direct.cpp | 148 +++- tests/kelpie/unit/tb_kelpie_key.cpp | 59 +- tests/kelpie/unit/tb_kelpie_localkv.cpp | 294 ++++++- tests/kelpie/unit/tb_kelpie_types.cpp | 77 +- tests/lunasa/CMakeLists.txt | 2 +- .../component/nnti/mpi_lunasa_nnti_put.cpp | 10 +- .../nnti/mpi_lunasa_nnti_register_memory.cpp | 10 +- .../component/nnti/mpi_lunasa_nnti_send.cpp | 7 +- .../nnti/mpi_lunasa_nnti_send_user.cpp | 7 +- .../component/tb_lunasa_backburner_ldo.cpp | 8 +- .../component/tb_lunasa_basic_allocations.cpp | 96 ++- tests/lunasa/component/tb_lunasa_copy_ldo.cpp | 6 +- .../tb_lunasa_data_type_registry.cpp | 7 +- .../component/tb_lunasa_dataobjectpacker.cpp | 316 +++++++ .../tb_lunasa_generic_data_bundle.cpp | 15 +- tests/lunasa/component/tb_lunasa_ldo.cpp | 15 +- .../component/tb_lunasa_performance.cpp | 9 +- .../lunasa/component/tb_lunasa_statistics.cpp | 12 +- .../tb_lunasa_threaded_performance.cpp | 15 +- .../unit/tb_lunasa_configuration1_valid.cpp | 7 +- .../tb_lunasa_configuration2_reinit_tc.cpp | 12 +- .../tb_lunasa_configuration3_double_tc.cpp | 20 +- tests/nnti/benchmarks/AtomicsWithCallback.cpp | 13 +- tests/nnti/benchmarks/CMakeLists.txt | 9 +- tests/nnti/benchmarks/MemoryRegistration.cpp | 6 +- tests/nnti/benchmarks/RdmaWithCallback.cpp | 13 +- tests/nnti/benchmarks/SendWithCallback.cpp | 13 +- tests/nnti/benchmarks/bench_utils.cpp | 24 +- tests/nnti/benchmarks/bench_utils.hpp | 8 +- tests/nnti/c-api/CMakeLists.txt | 6 +- tests/nnti/c-api/IBConnectTest.c | 6 +- tests/nnti/c-api/IBQueueSendTest1.c | 6 +- tests/nnti/c-api/IBQueueSendTest2.c | 6 +- tests/nnti/c-api/IBRdmaOpTest.c | 6 +- tests/nnti/c-api/IBShortSendTest.c | 6 +- tests/nnti/c-api/IBUnexpectedSendTest.c | 6 +- tests/nnti/c-api/NntiAllocFreeTest.c | 6 +- tests/nnti/c-api/NntiInitTest1.c | 6 +- tests/nnti/c-api/NntiInitTest2.c | 6 +- tests/nnti/c-api/NntiInitTest3.c | 6 +- tests/nnti/c-api/NntiInitTest4.c | 6 +- tests/nnti/c-api/NntiLoggerTest1.c | 6 +- tests/nnti/c-api/NntiLoggerTest2.c | 6 +- tests/nnti/c-api/NntiLoggerTest3.c | 6 +- tests/nnti/c-api/test_utils.cpp | 24 +- tests/nnti/c-api/test_utils.h | 6 +- tests/nnti/cpp-api/AllocFreeTest.cpp | 6 +- tests/nnti/cpp-api/AtomicOpTest.cpp | 15 +- tests/nnti/cpp-api/CMakeLists.txt | 51 +- .../nnti/cpp-api/CallbackStateMachineTest.cpp | 26 +- tests/nnti/cpp-api/ConnectTest.cpp | 6 +- tests/nnti/cpp-api/LongSendTest.cpp | 12 +- tests/nnti/cpp-api/MultiPingPongTest.cpp | 13 +- tests/nnti/cpp-api/NntiEqClassTest.cpp | 50 +- .../cpp-api/NntiEventCallbackClassTest.cpp | 6 +- tests/nnti/cpp-api/NntiFreelistClassTest.cpp | 6 +- tests/nnti/cpp-api/NntiLoggerClassTest.cpp | 6 +- tests/nnti/cpp-api/NntiOpClassTest.cpp | 6 +- tests/nnti/cpp-api/NntiOpVectorTest.cpp | 10 +- tests/nnti/cpp-api/NntiPidClassTest.cpp | 6 +- tests/nnti/cpp-api/NntiUrlClassTest.cpp | 6 +- tests/nnti/cpp-api/NntiWidClassTest.cpp | 6 +- tests/nnti/cpp-api/PingPongCallbackTest.cpp | 11 +- tests/nnti/cpp-api/QueueSendTest1.cpp | 12 +- tests/nnti/cpp-api/QueueSendTest2.cpp | 12 +- tests/nnti/cpp-api/RdmaAlignmentTest.cpp | 545 ++++++++++++ tests/nnti/cpp-api/RdmaLengthTest.cpp | 249 +++--- tests/nnti/cpp-api/RdmaOpTest.cpp | 35 +- tests/nnti/cpp-api/SelfConnectTest.cpp | 97 +++ tests/nnti/cpp-api/ShortSendTest.cpp | 12 +- tests/nnti/cpp-api/UnexpectedCallbackTest.cpp | 13 +- tests/nnti/cpp-api/UnexpectedLongSendTest.cpp | 9 +- tests/nnti/cpp-api/UnexpectedSendTest.cpp | 14 +- tests/nnti/cpp-api/UrlPidTest.cpp | 6 +- tests/nnti/cpp-api/ZeroCopySendTest.cpp | 6 +- tests/nnti/cpp-api/test_env.hpp | 114 +++ tests/nnti/cpp-api/test_utils.cpp | 27 +- tests/nnti/cpp-api/test_utils.hpp | 8 +- tests/opbox/CMakeLists.txt | 45 +- tests/opbox/component/mpi_opbox_atomics.cpp | 8 +- tests/opbox/component/mpi_opbox_connect.cpp | 6 +- tests/opbox/component/mpi_opbox_get.cpp | 6 +- tests/opbox/component/mpi_opbox_hello.cpp | 6 +- tests/opbox/component/mpi_opbox_long_send.cpp | 6 +- tests/opbox/component/mpi_opbox_ping.cpp | 6 +- tests/opbox/component/mpi_opbox_put.cpp | 8 +- .../component/mpi_opbox_remote_buffer.cpp | 20 +- tests/opbox/component/mpi_opbox_self_send.cpp | 6 +- tests/opbox/component/mpi_opbox_send.cpp | 6 +- tests/opbox/component/support/Globals.cpp | 6 +- tests/opbox/component/support/Globals.hh | 6 +- tests/opbox/ops/tb_FabConnTest.cpp | 6 +- tests/opbox/ops/tb_OpboxOpPingFabTest.cpp | 6 +- .../opbox/unit/mpi_opbox_message_helpers.cpp | 16 +- tests/opbox/unit/mpi_opbox_net_init.cpp | 14 +- tests/opbox/unit/mpi_opbox_opargs.cpp | 8 +- tests/opbox/unit/mpi_opbox_triggerop.cpp | 12 +- .../unit/support/default_config_string.hh | 6 +- tests/sbl/unit/tb_sbl_compile_speed.cpp | 6 +- tests/sbl/unit/tb_sbl_logger_class.cpp | 6 +- tests/sbl/unit/tb_sbl_source_class.cpp | 6 +- .../component/mpi_service_mpisyncstart.cpp | 20 +- tests/services/unit/tb_service_backburner.cpp | 38 +- tests/whookie/CMakeLists.txt | 1 - .../mpi_whookie_multiprocess_threaded.cpp | 6 +- .../whookie/component/mpi_whookie_restart.cpp | 7 +- .../component/tb_whookie_clientserver.cpp | 20 +- .../standalone/simple_whookie_example.cpp | 6 +- tools/faodel-cli/ActionInterface.hh | 47 + tools/faodel-cli/CMakeLists.txt | 9 +- tools/faodel-cli/KelpieBlastParams.cpp | 221 +++++ tools/faodel-cli/KelpieBlastParams.hh | 71 ++ tools/faodel-cli/KelpieClientAction.cpp | 268 ++++++ tools/faodel-cli/KelpieClientAction.hh | 50 ++ tools/faodel-cli/PlayAction.cpp | 123 +++ tools/faodel-cli/PlayAction.hh | 36 + tools/faodel-cli/README_Faodel_Cli.md | 362 ++++++++ tools/faodel-cli/ResourceAction.cpp | 49 ++ tools/faodel-cli/ResourceAction.hh | 28 + tools/faodel-cli/all_in_one.cpp | 133 +++ tools/faodel-cli/build_info.cpp | 4 + tools/faodel-cli/build_info_ib.cpp | 4 + tools/faodel-cli/config_info.cpp | 146 +++- tools/faodel-cli/dirman_server.cpp | 4 + tools/faodel-cli/faodel_cli.cpp | 34 +- tools/faodel-cli/faodel_cli.hh | 25 +- tools/faodel-cli/kelpie_blast.cpp | 269 ++++++ tools/faodel-cli/kelpie_client.cpp | 666 +++++++------- tools/faodel-cli/kelpie_client.hh | 15 + tools/faodel-cli/kelpie_server.cpp | 15 +- tools/faodel-cli/play.cpp | 301 +++++++ tools/faodel-cli/resource_client.cpp | 155 ++-- tools/faodel-cli/resource_client.hh | 12 + tools/faodel-cli/whookie_client.cpp | 34 +- tools/faodel-stress/CMakeLists.txt | 37 + tools/faodel-stress/Job.cpp | 97 +++ tools/faodel-stress/Job.hh | 87 ++ tools/faodel-stress/JobKeys.cpp | 55 ++ tools/faodel-stress/JobKeys.hh | 49 ++ tools/faodel-stress/JobLocalPool.cpp | 101 +++ tools/faodel-stress/JobLocalPool.hh | 46 + tools/faodel-stress/JobMemoryAlloc.cpp | 84 ++ tools/faodel-stress/JobMemoryAlloc.hh | 47 + tools/faodel-stress/JobSerdes.cpp | 61 ++ tools/faodel-stress/JobSerdes.hh | 80 ++ tools/faodel-stress/JobWebClient.cpp | 118 +++ tools/faodel-stress/JobWebClient.hh | 39 + tools/faodel-stress/README_Faodel_Stress.md | 54 ++ tools/faodel-stress/Worker.cpp | 40 + tools/faodel-stress/Worker.hh | 58 ++ tools/faodel-stress/faodel-stress.cpp | 141 +++ .../serdes/SerdesParticleBundleObject.cpp | 100 +++ .../serdes/SerdesParticleBundleObject.hh | 52 ++ .../serdes/SerdesStringObject.cpp | 63 ++ .../serdes/SerdesStringObject.hh | 39 + tools/faodel-stress/serdes/WorkerSerdes.hh | 128 +++ tools/kelpie-server/kelpie_server_main.cpp | 4 + tpl/cereal/CMakeLists.txt | 11 +- .../cereal/external/rapidxml/license.txt | 104 +-- .../cereal/external/rapidxml/manual.html | 810 +++++++++--------- tpl/cereal/vs2013/sandbox/sandbox.vcxproj | 510 +++++------ .../vs2013/sandbox/sandbox.vcxproj.filters | 42 +- .../vs2013/sandbox_json/sandbox_json.vcxproj | 494 +++++------ .../sandbox_json/sandbox_json.vcxproj.filters | 42 +- .../vs2013/sandbox_rtti/sandbox_rtti.vcxproj | 502 +++++------ .../sandbox_rtti/sandbox_rtti.vcxproj.filters | 42 +- .../vs2013/sandbox_vs/sandbox_vs.vcxproj | 530 ++++++------ .../sandbox_vs/sandbox_vs.vcxproj.filters | 42 +- .../sandbox_vs_dll/sandbox_vs_dll.vcxproj | 520 +++++------ .../sandbox_vs_dll.vcxproj.filters | 64 +- tpl/cereal/vs2013/unittests/unittests.vcxproj | 652 +++++++------- .../unittests/unittests.vcxproj.filters | 252 +++--- tpl/cereal/vs2013/vs2013.sln | 294 +++---- tpl/gperftools/README_windows.txt | 240 +++--- .../src/gperftools/malloc_extension.h | 2 +- tpl/gperftools/src/page_heap.h | 2 +- tpl/gperftools/src/tcmalloc.cc | 4 +- tpl/gperftools/src/windows/shortproc.asm | 274 +++--- 728 files changed, 23523 insertions(+), 9353 deletions(-) create mode 100644 cmake/FaodelCheckTransports.cmake create mode 100644 cmake/FaodelRpathHelpers.cmake create mode 100644 cmake/FaodelSummary.cmake create mode 100644 docs/ConfigurationFileCookbook.md create mode 100644 docs/WhatsAFaodel.md create mode 100644 docs/faodhail.jpg create mode 100644 examples/common/serialization/CMakeLists.txt create mode 100644 examples/common/serialization/MyContainer.hh create mode 100644 examples/common/serialization/serialization_example.cpp create mode 100644 examples/faodel-cli/playback-scripts/s01_basic_local.play create mode 100644 examples/faodel-cli/playback-scripts/s02_trace_pool_example.play create mode 100644 examples/faodel-cli/playback-scripts/s03_advanced_file_handling.play create mode 100644 examples/faodel-cli/playback-scripts/s04_bucket_renaming.play create mode 100644 examples/faodel-cli/playback-scripts/s05_self_contained_pool.play create mode 100644 examples/kelpie/compute/CMakeLists.txt create mode 100644 examples/kelpie/compute/ex_compute1_pick.cpp create mode 100644 examples/kelpie/compute/ex_compute2_udf.cpp create mode 100644 examples/lunasa/ldo_packing/CMakeLists.txt create mode 100644 examples/lunasa/ldo_packing/MyData.hh create mode 100644 examples/lunasa/ldo_packing/MyHelpers.hh create mode 100644 examples/lunasa/ldo_packing/ldo_packing_example.cpp create mode 100644 examples/opbox/advanced/CMakeLists.txt create mode 100644 examples/opbox/basic/CMakeLists.txt create mode 100644 examples/opbox/collectives/CMakeLists.txt create mode 100644 examples/opbox/opbox-example-support/CMakeLists.txt create mode 100644 examples/services/mpi-sync-start/CMakeLists.txt create mode 100644 src/faodel-common/SerializationHelpersCereal.hh mode change 100644 => 100755 src/kelpie/CMakeLists.txt create mode 100644 src/kelpie/common/ComputeRegistry.cpp create mode 100644 src/kelpie/common/ComputeRegistry.hh create mode 100644 src/kelpie/localkv/LocalKVTypes.hh create mode 100644 src/kelpie/ops/direct/OpKelpieCompute.cpp create mode 100644 src/kelpie/ops/direct/OpKelpieCompute.hh create mode 100644 src/kelpie/ops/direct/OpKelpieDrop.cpp create mode 100644 src/kelpie/ops/direct/OpKelpieDrop.hh create mode 100644 src/kelpie/pools/NullPool/NullPool.cpp create mode 100644 src/kelpie/pools/NullPool/NullPool.hh create mode 100644 src/kelpie/pools/ResultCollector.cpp create mode 100644 src/kelpie/pools/ResultCollector.hh create mode 100644 src/kelpie/pools/TracePool/TracePool.cpp create mode 100644 src/kelpie/pools/TracePool/TracePool.hh create mode 100644 src/lunasa/common/DataObjectPacker.cpp create mode 100644 src/lunasa/common/DataObjectPacker.hh create mode 100644 src/lunasa/common/Helpers.cpp create mode 100644 src/lunasa/common/Helpers.hh create mode 100644 src/nnti/nnti_types.c create mode 100644 src/nnti/nnti_util.cpp create mode 100644 src/opbox/core/OpTimer.cpp create mode 100644 src/opbox/core/OpTimer.hh create mode 100644 tests/kelpie/component/mpi_kelpie_behaviors.cpp create mode 100644 tests/kelpie/component/mpi_kelpie_compute.cpp create mode 100644 tests/kelpie/component/mpi_kelpie_many_items.cpp create mode 100644 tests/kelpie/component/nonet/tb_kelpie_nonet_compute.cpp create mode 100644 tests/kelpie/component/nonet/tb_kelpie_nonet_iom_from_url.cpp create mode 100644 tests/kelpie/component/tb_kelpie_multiple_hidden_inits.cpp create mode 100644 tests/lunasa/component/tb_lunasa_dataobjectpacker.cpp create mode 100644 tests/nnti/cpp-api/RdmaAlignmentTest.cpp create mode 100644 tests/nnti/cpp-api/SelfConnectTest.cpp create mode 100644 tests/nnti/cpp-api/test_env.hpp create mode 100644 tools/faodel-cli/ActionInterface.hh create mode 100644 tools/faodel-cli/KelpieBlastParams.cpp create mode 100644 tools/faodel-cli/KelpieBlastParams.hh create mode 100644 tools/faodel-cli/KelpieClientAction.cpp create mode 100644 tools/faodel-cli/KelpieClientAction.hh create mode 100644 tools/faodel-cli/PlayAction.cpp create mode 100644 tools/faodel-cli/PlayAction.hh create mode 100644 tools/faodel-cli/README_Faodel_Cli.md create mode 100644 tools/faodel-cli/ResourceAction.cpp create mode 100644 tools/faodel-cli/ResourceAction.hh create mode 100644 tools/faodel-cli/all_in_one.cpp create mode 100644 tools/faodel-cli/kelpie_blast.cpp create mode 100644 tools/faodel-cli/kelpie_client.hh create mode 100644 tools/faodel-cli/play.cpp create mode 100644 tools/faodel-cli/resource_client.hh create mode 100644 tools/faodel-stress/CMakeLists.txt create mode 100644 tools/faodel-stress/Job.cpp create mode 100644 tools/faodel-stress/Job.hh create mode 100644 tools/faodel-stress/JobKeys.cpp create mode 100644 tools/faodel-stress/JobKeys.hh create mode 100644 tools/faodel-stress/JobLocalPool.cpp create mode 100644 tools/faodel-stress/JobLocalPool.hh create mode 100644 tools/faodel-stress/JobMemoryAlloc.cpp create mode 100644 tools/faodel-stress/JobMemoryAlloc.hh create mode 100644 tools/faodel-stress/JobSerdes.cpp create mode 100644 tools/faodel-stress/JobSerdes.hh create mode 100644 tools/faodel-stress/JobWebClient.cpp create mode 100644 tools/faodel-stress/JobWebClient.hh create mode 100644 tools/faodel-stress/README_Faodel_Stress.md create mode 100644 tools/faodel-stress/Worker.cpp create mode 100644 tools/faodel-stress/Worker.hh create mode 100644 tools/faodel-stress/faodel-stress.cpp create mode 100644 tools/faodel-stress/serdes/SerdesParticleBundleObject.cpp create mode 100644 tools/faodel-stress/serdes/SerdesParticleBundleObject.hh create mode 100644 tools/faodel-stress/serdes/SerdesStringObject.cpp create mode 100644 tools/faodel-stress/serdes/SerdesStringObject.hh create mode 100644 tools/faodel-stress/serdes/WorkerSerdes.hh diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b92abf..653b1ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.8.0) project( Faodel LANGUAGES CXX C - VERSION 1.1906.2 + VERSION 1.2108.1 ) @@ -40,7 +40,7 @@ endif() include( FindPkgConfig ) include( FindPackageHandleStandardArgs ) include( CMakePackageConfigHelpers ) - +include( CheckFunctionExists ) # Macros @@ -114,21 +114,24 @@ endfunction() # Options option( BUILD_SHARED_LIBS "Build Faodel as shared libs" OFF ) -option( BUILD_TESTS "Faodel testing gtest and ctest" ON ) +option( BUILD_TESTS "Faodel testing gtest and ctest" OFF ) +option( BUILD_EXAMPLES "Build examples" OFF) option( BUILD_DOCS "Generate documentation using Doxygen" ON ) -option( Faodel_ENABLE_CEREAL "Enable use of Cereal for serialization in NNTI (disables XDR)" OFF ) option( Faodel_ENABLE_MPI_SUPPORT "Enable use of MPI communication" ON ) option( Faodel_ENABLE_TCMALLOC "Use tcmalloc from gperftools in Lunasa, potentially other places" ON ) option( Faodel_ENABLE_IOM_HDF5 "Build the HDF5-based IOM in Kelpie" OFF ) option( Faodel_ENABLE_IOM_LEVELDB "Build the LevelDB-based IOM in Kelpie" OFF ) option( Faodel_ENABLE_IOM_CASSANDRA "Build the Cassandra-based IOM in Kelpie" OFF ) +option( Faodel_ENABLE_DEBUG_TIMERS "Enable timers for debug purposed" OFF ) + +mark_as_advanced(Faodel_ENABLE_DEBUG_TIMERS) set( Faodel_NETWORK_LIBRARY "nnti" CACHE STRING "RDMA Network library to use for low-level communication" ) set_property(CACHE Faodel_NETWORK_LIBRARY PROPERTY STRINGS nnti libfabric) - - +set( Faodel_NNTI_SERIALIZATION_METHOD "XDR" CACHE STRING "Serialization library for use in NNTI " ) +set_property(CACHE Faodel_NNTI_SERIALIZATION_METHOD PROPERTY STRINGS XDR CEREAL ) # Configure the world @@ -197,18 +200,18 @@ set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS Debug Pedantic Release RelW # Set the build type in case a project doesn't. This can be overridden by providing e.g. -DCMAKE_BUILD_TYPE=Release at # cmake time. if( NOT CMAKE_BUILD_TYPE ) - set( CMAKE_BUILD_TYPE Debug ) + set( CMAKE_BUILD_TYPE Debug CACHE STRING "Specify a build configuration" FORCE ) endif() -if( ${CMAKE_SYSTEM_NAME} MATCHES "CrayLinuxEnvironment" ) - if( BUILD_SHARED_LIBS ) - message( FATAL_ERROR - "Faodel does not support shared libraies on Cray compute nodes. Please set BUILD_SHARED_LIBS=OFF." - ) - endif() - set( Boost_USE_STATIC_LIBS ON ) -endif() +#if( ${CMAKE_SYSTEM_NAME} MATCHES "CrayLinuxEnvironment" ) +# if( BUILD_SHARED_LIBS ) +# message( FATAL_ERROR +# "Faodel does not support shared libraies on Cray compute nodes. Please set BUILD_SHARED_LIBS=OFF." +# ) +# endif() +# set( Boost_USE_STATIC_LIBS ON ) +#endif() macro( disable_isystem_flag ) # Disable use of the -isystem compiler flag by telling CMake to use -I instead @@ -241,6 +244,29 @@ if( ${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" ) endif() +#### +# +# Handle asserts in debugging +# +#### +set( Faodel_ASSERT_METHOD "cassert" CACHE STRING "Mechanism for dealing with asserts in Faodel") +set_property(CACHE Faodel_ASSERT_METHOD PROPERTY STRINGS cassert debugExit debugHalt debugWarn none) +mark_as_advanced(Faodel_ASSERT_METHOD) +if( Faodel_ASSERT_METHOD MATCHES "cassert") + set( Faodel_ASSERT_METHOD_CASSERT TRUE) +elseif( Faodel_ASSERT_METHOD MATCHES "debugExit") + set( Faodel_ASSERT_METHOD_DEBUG_EXIT TRUE) +elseif( Faodel_ASSERT_METHOD MATCHES "debugHalt") + set( Faodel_ASSERT_METHOD_DEBUG_HALT TRUE) +elseif( Faodel_ASSERT_METHOD MATCHES "debugWarn") + set( Faodel_ASSERT_METHOD_DEBUG_WARN TRUE) +elseif( Faodel_ASSERT_METHOD MATCHES "none") + set( Faodel_ASSERT_METHOD_NONE TRUE) +endif() + + + + #### # # Handle threads @@ -275,6 +301,19 @@ else() ) endif() +## Handle crc32() +## Some tests use crc32() to validate network transfers. +## Assume that crc32() is in zlib and make sure we can find it. +find_package(ZLIB) +if (ZLIB_FOUND) + set (CMAKE_REQUIRED_LIBRARIES ${ZLIB_LIBRARIES}) + check_function_exists(crc32 Faodel_HAVE_CRC32) + set (CMAKE_REQUIRED_LIBRARIES "") +endif(ZLIB_FOUND) +if (NOT Faodel_HAVE_CRC32) + message(STATUS "crc32() not found - some tests will not be built") +endif (NOT Faodel_HAVE_CRC32) + ## Handle logging set( Faodel_LOGGING_METHOD "stdout" CACHE STRING "Select where logging interface output is routed" ) set_property(CACHE Faodel_LOGGING_METHOD PROPERTY STRINGS stdout sbl disabled) @@ -305,9 +344,21 @@ set( CMAKE_NO_SYSTEM_FROM_IMPORTED TRUE ) include( ${CMAKE_CURRENT_LIST_DIR}/cmake/FaodelTPLs.cmake ) +include( ${CMAKE_CURRENT_LIST_DIR}/cmake/FaodelRpathHelpers.cmake ) +set_faodel_install_rpath() +set_cmake_install_rpath() +# The CMake Cray system module does not define the RPATH flags (probably +# because Cray has historically only supported static linking). Now that +# dynamic linking is supported, we define them ourselves. +if( ${CMAKE_SYSTEM_NAME} MATCHES "CrayLinuxEnvironment" ) + if( BUILD_SHARED_LIBS ) + set_cmake_rpath_flags() + endif() +endif() + ############################## # -# Stanza 4 : Locate optional TPLs and services +# Stanza 3 : Locate optional TPLs and services # ############################## @@ -336,7 +387,7 @@ set( CMAKE_NO_SYSTEM_FROM_IMPORTED ${savevar} ) ################################ ## -## Stanza 5 : Define Faodel build and install targets +## Stanza 4 : Define Faodel build and install targets ## ################################ if( Faodel_ENABLE_TCMALLOC ) @@ -402,6 +453,7 @@ endif() # Build tools add_subdirectory( tools/faodel-cli ) +add_subdirectory( tools/faodel-stress ) add_subdirectory( tools/kelpie-server ) @@ -424,6 +476,18 @@ if( BUILD_TESTS ) endif() +if( BUILD_EXAMPLES ) + # Optional: add the examples. Examples are scoped to only use libs from a certain + # point down. Currently missing the nnti examples. + foreach(ex_lib common dirman kelpie lunasa opbox services) + set(EXAMPLE_LIBS ${ex_lib}) + add_subdirectory( examples/${ex_lib} ) + endforeach(ex_lib) + + set(EXAMPLE_LIBS whookie common) + add_subdirectory( examples/whookie ) +endif() + export( EXPORT FaodelTargets FILE FaodelTargets.cmake ) install( EXPORT FaodelTargets @@ -489,6 +553,8 @@ if( BUILD_DOCS ) WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMENT "Generating API documentation with Doxygen" VERBATIM ) + add_custom_target(docs) + add_dependencies( docs doc) add_custom_target( install-doc COMMAND mkdir -p ${CMAKE_INSTALL_PREFIX}/share/doc/${PROJECT_NAME} && cp -R docs/ ${CMAKE_INSTALL_PREFIX}/share/doc/${PROJECT_NAME} @@ -518,110 +584,12 @@ endif( BUILD_DOCS ) ## Stanza 5 : Any status reporting or cleanup items ## ################################ -function( found_status ) - if( ${${ARGV1}} ) - set( found_str "Found" ) - if( ${ARGC} EQUAL 4 AND NOT ${ARGV3} EQUAL "" ) - set( version_str "(${ARGV2}${ARGV3})" ) - elseif( ${ARGC} EQUAL 3 AND NOT ${ARGV2} EQUAL "" ) - set( version_str "(${ARGV2})" ) - endif() - else() - set( found_str "Not Found" ) - endif() - set( fill_len 25 ) - set( padded_len 25 ) - string( RANDOM LENGTH ${fill_len} ALPHABET " " fill_str ) - string( SUBSTRING "${ARGV0}:${fill_str}" 0 ${padded_len} padded_str ) - message( STATUS " ${padded_str}${found_str} ${version_str}" ) -endfunction() - -function( message_status ) - set( fill_len 25 ) - set( padded_len 25 ) - string( RANDOM LENGTH ${fill_len} ALPHABET " " fill_str ) - string( SUBSTRING "${ARGV0}:${fill_str}" 0 ${padded_len} padded_str ) - message( STATUS " ${padded_str}${ARGV1}" ) -endfunction() +include( ${CMAKE_CURRENT_LIST_DIR}/cmake/FaodelSummary.cmake ) +print_faodel_config_summary() -message( STATUS "" ) -message( STATUS "======================================================================" ) -message( STATUS "" ) -message( STATUS "Configured to build Faodel with ${CMAKE_MAKE_PROGRAM}" ) -if( CMAKE_INSTALL_PREFIX ) - message( STATUS "Installation prefix is ${CMAKE_INSTALL_PREFIX}" ) -endif() -message( STATUS "" ) -message( STATUS "External Programs:" ) -message_status( "compiler" "${CMAKE_CXX_COMPILER_ID} (${CMAKE_CXX_COMPILER_VERSION})" ) -found_status( "doxygen" DOXYGEN_FOUND "v" "${DOXYGEN_VERSION}" ) -message( STATUS "" ) -message( STATUS "TPLs:" ) -found_status( "Boost" Boost_FOUND "v" "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}" ) -found_status( "MPI" MPI_FOUND "implements MPI-" "${MPI_C_VERSION}" ) -found_status( "googletest" GTest_FOUND ) -if (Faodel_NETWORK_LIBRARY STREQUAL "libfabric") - found_status( "libfabric" LIBFABRIC_FOUND "v" "${Libfabric_pc_VERSION}" ) -endif() -if (Faodel_NETWORK_LIBRARY STREQUAL "nnti") - found_status( "libugni" UGNI_FOUND "v" "${UGNI_PC_VERSION}" ) - found_status( "Cray DRC" DRC_FOUND ) - found_status( "ibverbs" IBVerbs_FOUND ) -endif() -message( STATUS "" ) -message( STATUS "Faodel Common Config:" ) -message( STATUS " Threading Model: ${Faodel_THREADING_MODEL}" ) -message( STATUS " Logging Method: ${Faodel_LOGGING_METHOD}" ) -message( STATUS "" ) -message( STATUS "Lunasa Config:" ) -if ( ${Faodel_ENABLE_TCMALLOC} ) - message( STATUS " Building with tcmalloc from gperftools" ) -endif() -message( STATUS "" ) -message( STATUS "NNTI Config:" ) -if( ${NNTI_BUILD_IBVERBS} ) - if( ${NNTI_HAVE_VERBS_EXP_H} ) - message( STATUS " Building the IBVerbs Transport with the libverbs expanded API (mlx4 or mlx5)" ) - else() - message( STATUS " Building the IBVerbs Transport with the libverbs standard API (mlx4 ONLY)" ) - endif() -else() - if( ${NNTI_DISABLE_IBVERBS_TRANSPORT} ) - message( STATUS " IBVerbs Transport explicitly disabled" ) - else() - message( STATUS " Not building the IBVerbs Transport" ) - endif() -endif() -if( ${NNTI_BUILD_UGNI} ) - message( STATUS " Building the UGNI Transport" ) -else() - if( ${NNTI_DISABLE_UGNI_TRANSPORT} ) - message( STATUS " UGNI Transport explicitly disabled" ) - else() - message( STATUS " Not building the UGNI Transport" ) - endif() -endif() -if( ${NNTI_BUILD_MPI} ) - message( STATUS " Building the MPI Transport" ) -else() - if( ${NNTI_DISABLE_MPI_TRANSPORT} ) - message( STATUS " MPI Transport explicitly disabled" ) - else() - message( STATUS " Not building the MPI Transport" ) - endif() -endif() -if( ${NNTI_USE_XDR} ) - message( STATUS " Using XDR for serialization" ) -else() - if( ${NNTI_USE_CEREAL} ) - message( STATUS " Using Cereal for serialization" ) - else() - message( STATUS " ERROR - Couldn't find a serialization library" ) - endif() -endif() -message( STATUS "" ) -message( STATUS "Opbox Config:" ) -message( STATUS " Network Library: ${Faodel_NETWORK_LIBRARY}" ) -message( STATUS "" ) -message( STATUS "======================================================================" ) -message( STATUS "" ) +## +## Next we check for disabled NNTI transports that seem like +## they could be errors or unintended misconfigurations. +## +include( ${CMAKE_CURRENT_LIST_DIR}/cmake/FaodelCheckTransports.cmake ) +check_preferred_transports() diff --git a/INSTALL.md b/INSTALL.md index 7ea0f79..c9c74f9 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,17 +1,50 @@ FAODEL Installation =================== -The FAODEL software is written in C++11 and use CMake for -building. Users must install a small number of third-party libraries -(TPLs) in order to build FAODEL. +The FAODEL software is written in C++11 and uses CMake for building. Users must +install a small number of third-party libraries (TPLs) to build FAODEL. + +Quickstart +---------- +FAODEL can now be built with the [Spack package manager](https://spack.io). The easiest way to build FAODEL and all its dependencies is through: +``` +spack install faodel +``` +Create a config file: +``` +export FAODEL_CONFIG=~/my-faodel.conf +vi $FAODEL_CONFIG +``` +Put basic settings in your $FAODEL_CONFIG file: +``` +whookie.interfaces ib0,eth0 # list of ip interface where web info will live +net.transport.name mpi # The HPC nic type, eg infiniband, ugni +whookie.info true # Print the web address to stdout at boot +bootstrap.debug true # Print more detailed debug messages about boot + # lunasa,opbox,kelpie,dirman all have debug/info +``` +Then use the `faodel` tool to run services, put/get data, etc: +``` +faodel help # Show all the options +faodel binfo # Verify your build looks right and devices found +faodel cinfo # Verify your config looks right + +# Create a pool in one window, then put/get/list data in another +mpirun -np 4 faodel all-in-one "dht:/my/dht ALL" +faodel kput -p /my/dht -k bob -f myfile.txt +faodel kget -p /my/dht -k bob +faodel klist -p /my/dht -k "b*" +``` +When you're ready to connect to FAODEL from your own code, look +in `faodel/examples/kelpie`. + Prerequisites ------------- FAODEL has a small number of prerequisites for building. A target platform should have: -- C++11 Compiler: The current implementation is typically built with - GCC and requires version 4.8.5 or higher in order to use C++11 - features. +- C++11 Compiler: The current implementation is typically built with GCC and + requires version 4.8.5 or higher in order to use C++11 features. - CMake 3.8 - Doxygen @@ -29,37 +62,30 @@ section later in this document. Configuring and Building FAODEL ------------------------------- FAODEL uses a standard CMake workflow for configuring, building, and installing -the FAODEL software. On a system where all compilers and TPLs have already -been setup (i.e., CC, BOOST_ROOT, and GTEST_ROOT environment variables are -correct), you can simply change into a build directory, run cmake to configure -the build, and then run make install to build/install the software. -For example: +the FAODEL software. On a system where all compilers and TPLs have already been +setup (i.e., CC, BOOST_ROOT, and GTEST_ROOT environment variables are correct), +you can simply change into a build directory, run cmake to configure the build, +and then run make install to build/install the software. For example: cd faodel - ROOT_DIR=$(pwd) - INSTALL_DIR=${ROOT_DIR}/install mkdir build cd build cmake \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ + -DCMAKE_INSTALL_PREFIX=$(pwd)/../install \ .. make -j 4 doc install -Often TPLs are not installed where they should be and it is necessary -to specify where things are located during configuration. You can also -specify a number of FAODEL-specific options to CMake during configuration. -A complete list of CMake options is provided in **Appendix A** of this -Install guide. The following provides a more thorough -configure/build/install example: - +Often TPLs are not installed where they should be and it is necessary to specify +where things are located during configuration. You can also specify a number of +FAODEL-specific options to CMake during configuration. A complete list of CMake +options is provided in **Appendix A** of this Install guide. The following +provides a more thorough configure/build/install example: cd faodel - ROOT_DIR=$(pwd) - INSTALL_DIR=${ROOT_DIR}/install mkdir build cd build cmake \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DCMAKE_INSTALL_PREFIX=$(pwd)/../install \ -DCMAKE_BUILD_TYPE=Release \ \ -DGTEST_ROOT=${GTEST_ROOT} \ @@ -69,157 +95,184 @@ configure/build/install example: -DFaodel_ENABLE_MPI_SUPPORT=ON \ \ -DBUILD_TESTS=ON \ + -DBUILD_EXAMPLES=ON \ -DBUILD_DOCS=ON \ .. make -j 4 doc install -Doing a plain 'make' will build the libraries (and tests) in the -build directory. The 'make install' operation will install the -libraries and headers into the CMAKE_INSTALL_PREFIX location -specified during configuration. The install step also generates -an use Building FAODEL --------------- -After setting up the environment, the rest of the configure and build is -the same as any other platform with two exceptions: +After setting up the environment, the rest of the configure and build is the +same as any other platform with two exceptions: -1. Use the Cray Linux Environment (CLE): You need to tell CMake that we - are building for the CLE using the `CMAKE_SYSTEM_NAME` variable. -2. Disable IBVerbs Transport: Mutrino has IBVerbs header files installed - that will cause NNTI to mistakenly think IB nics are available on the - system. You need to set the `NNTI_DISABLE_IBVERBS_TRANSPORT` variable - option. +1. Use the Cray Linux Environment (CLE): You need to tell CMake that we are + building for the CLE using the `CMAKE_SYSTEM_NAME` variable. +2. Disable IBVerbs Transport: Mutrino has IBVerbs header files installed that + will cause NNTI to mistakenly think IB nics are available on the system. You + need to set the `NNTI_DISABLE_IBVERBS_TRANSPORT` variable option. A generic FAODEL build on Mutrino starts with the following: @@ -535,43 +581,58 @@ export Faodel_DIR=$(pwd)/install mkdir build cd build cmake \ - -DCMAKE_SYSTEM_NAME=CrayLinuxEnvironment \ - -DCMAKE_INSTALL_PREFIX=${Faodel_DIR} \ - - -DFaodel_NETWORK_LIBRARY={libfabric|nnti} \ - -DREQUIRE_STATIC_LINKING:BOOL=TRUE \ + -DCMAKE_SYSTEM_NAME=CrayLinuxEnvironment \ + -DCMAKE_INSTALL_PREFIX=${Faodel_DIR} \ + -DFaodel_NETWORK_LIBRARY={libfabric|nnti} \ + -DREQUIRE_STATIC_LINKING:BOOL=TRUE \ -DNNTI_DISABLE_IBVERBS_TRANSPORT:BOOL=TRUE \ - -DCMAKE_C_COMPILER=cc \ - -DCMAKE_CXX_COMPILER=CC \ - -DGTEST_ROOT=${GTEST_ROOT} \ - -DBOOST_ROOT=${BOOST_ROOT} \ + -DCMAKE_C_COMPILER=cc \ + -DCMAKE_CXX_COMPILER=CC \ + -DGTEST_ROOT=${GTEST_ROOT} \ + -DBOOST_ROOT=${BOOST_ROOT} \ .. make -j 20 install ``` FAODEL Configuration Settings for Mutrino ----------------------------------------- - -When FAODEL starts, its default behavior is to read a configuration file -specified by the environment variable `FAODEL_CONFIG`. The Cray platforms -use an IP over Gemini Fabric nic for socket communication (usually -defined as ipogif0). Thus, you should add the following info to the -configuration file specified by `FAODEL_CONFIG`: +There are three Cray-specific system settings that you'll want to add to +your `FAODEL_CONFIG` file when running on mutrino: (1) the name of the +IP-over-Gemini device that whookie can use to setup connections +(eg `ipogif0`), (2) the network transport that should be used (eg, `ugni`), +and (3) the DRC credential id you've been assigned by a system admin if you plan +on doing job-to-job communication. ``` whookie.interfaces ipogif0 net.transport.name ugni +nnti.transport.credential_id 42 ``` +Inter-job Communication on Mutrino +---------------------------------- +Mutrino has an Aries/Gemini interconnect that is programmed using the UGNI +library. The UGNI library protects applications from outside data corruption ( +benign or malicious) by preventing RDMA operations between applications. In +order for two processes to communicate, they must have the same credentials. At +launch, the typical application is assigned a system-wide unique credential that +allows communication between processes making up the application and prevents +communication with other processes outside the application even if they are +launched by the same user within the same allocation. + +In order to communicate between applications, an administrator must create a +Dynamic RDMA Credential (DRC) and assign it to a user. Then a group of +applications using the same credential can transfer data to/from each other. + Mutrino Caveats --------------- Building and running has a few caveats: -1. Missing mpiexec breaks tests: CMake's mpi testing expects to launch - with mpiexec, while mutrino expects users to use srun. If you look - at the CTest file to get the path of mpiexec, you can use the following - sed script to replace it: +1. Missing mpiexec breaks tests: CMake's mpi testing expects to launch with + mpiexec, while mutrino expects users to use srun. If you look at the CTest + file to get the path of mpiexec, you can use the following sed script to + replace it: ``` cd faodel/build @@ -587,57 +648,51 @@ find . -name CTestTestfile.cmake | xargs sed -i 's@/opt/cray/elogin/eproxy/2.0.2 -DCMAKE_C_COMPILER=cc ``` -3. Gethost Warnings: The build will give many warning messages about - gethostbyname in a statically linked application requiring a runtime with - the shared libraries from glibc. This is normal and does not affect - FAODEL. - +3. Gethost Warnings: The build will give many warning messages about + gethostbyname in a statically linked application requiring a runtime with the + shared libraries from glibc. This is normal and does not affect FAODEL. Installing on Astra (ARM-based Mellanox InfiniBand Cluster) =========================================================== -Astra is an ARM-based cluster with a Mellanox InfiniBand interconnect. -In general, Faodel operates the same on Astra as on any other -InfiniBand platform. One extra feature of Astra is that it -has full support for Mellanox's On-Demand Paging feature that -allows a process' entire virtual address space to be registered -without pinning (locking) pages in memory. The availability of -ODP is detected during configuration, but it is disable by -default because it is still experimental. To enable it, add the -following to your configuration file. +Astra is an ARM-based cluster with a Mellanox InfiniBand interconnect. In +general, Faodel operates the same on Astra as on any other InfiniBand platform. +One extra feature of Astra is that it has full support for Mellanox's On-Demand +Paging feature that allows a process' entire virtual address space to be +registered without pinning (locking) pages in memory. The availability of ODP is +detected during configuration, but it is disable by default because it is still +experimental. To enable it, add the following to your configuration file. ``` net.transport.use_odp true ``` -This feature is still experimental with no guarantees of -performance or correctness. +This feature is still experimental with no guarantees of performance or +correctness. Installing on Kahuna (Generic InfiniBand Cluster) ================================================= Kahuna is a generic InfiniBand cluster that -uses [Spack](https://spack.readthedocs.io) to generate its software -modules. The following instrutions provide an example of how to build -the environment. +uses [Spack](https://spack.readthedocs.io) to generate its software modules. The +following instrutions provide an example of how to build the environment. Loading Modules --------------- -Kahuna has all the required TPLs installed in its modules. The -current recommended environment setting is as follows: +Kahuna has all the required TPLs installed in its modules. The current +recommended environment setting is as follows: ``` -module load mpi-gcc6 -module load cmake/3.11.0 -module load pkg-config -module load gcc/6.3.0 -module load boost/1.63.0 -module load googletest/1.8.0 +module load doxygen module load ninja -module use /opt/spack/c7.3/new-mpi4-gcc6.3.0/modules -module load openmpi/3.1.0 +module load cmake/3.15.4 +module load gcc9-support +module load gcc/9.2.0 +module load boost/1.70.0 +module load googletest +module load openmpi/3.1.3 export GTEST_ROOT=${GOOGLETEST_ROOT} ``` @@ -646,17 +701,15 @@ FAODEL Configuration Settings for Kahuna ---------------------------------------- When FAODEL starts, its default behavior is to read a configuration file specified by the environment variable `FAODEL_CONFIG`. Kahuna is a generic -InfiniBand Cluster with both 10GigE (on eth0) and IPoIB (on ib0) network -ports for sockets. Thus, you should add the following info to the -configuration file specified by `FAODEL_CONFIG`: +InfiniBand Cluster with both 10GigE (on eth0) and IPoIB (on ib0) network ports +for sockets. Thus, you should add the following info to the configuration file +specified by `FAODEL_CONFIG`: ``` whookie.interfaces eth0,ib0 net.transport.name ibverbs ``` - - Appendix A: FAODEL CMake Options ================================ The following CMake options can be toggled when building FAODEL: @@ -664,35 +717,39 @@ The following CMake options can be toggled when building FAODEL: Basic Options ------------- -| CMake Variable | Type | Purpose | -| ------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------- | -| BUILD_DOCS | Boolean | Use Doxygen to build and install docs in FAODEL/docs | -| BUILD_SHARED_LIBS | Boolean | Build FAODEL as shared libraries (instead of static) | -| BUILD_TESTS | Boolean | Build FAODEL unit and component tests | -| CMAKE_INSTALL_PREFIX | Path | Standard CMake variable for where to install FAODEL. Will create include,lib,bin subdirectories | -| Faodel_ENABLE_IOM_HDF5 | Boolean | Build a Kelpie IO module for writing objects to an HDF5 file. Requires HDF5_DIR | -| Faodel_ENABLE_IOM_LEVELDB | Boolean | Build a Kelpie IO module for writing objects to a LevelDB store. Required leveldb_DIR | -| Faodel_ENABLE_MPI_SUPPORT | Boolean | Include MPI capabilities (MPISyncStart service, mpi-based testing, and NNTI MPI Transport) | -| Faodel_ENABLE_TCMALLOC | Boolean | Compile the included tpl/gperftools tcmalloc and use in Lunasa memory allocator | -| Faodel_LOGGING_METHOD | stdout, sbl, disabled | Determines where the logging interface writes. Select disabled to optimize away | - -| Faodel_NETWORK_LIBRARY | nnti,libfabric | Select which RDMA network library to use | - +| CMake Variable | Type | Purpose | +| -------------------------------- | --------------------- | ------------------------------------------------------------------------------------------------- | +| BUILD_DOCS | Boolean | Use Doxygen to build and install docs in FAODEL/docs | +| BUILD_SHARED_LIBS | Boolean | Build FAODEL as shared libraries (instead of static) | +| BUILD_TESTS | Boolean | Build FAODEL unit and component tests | +| CMAKE_INSTALL_PREFIX | Path | Standard CMake variable for where to install FAODEL. Will create include,lib,bin subdirectories | +| Faodel_ENABLE_IOM_HDF5 | Boolean | Build a Kelpie IO module for writing objects to an HDF5 file. Requires HDF5_DIR | +| Faodel_ENABLE_IOM_LEVELDB | Boolean | Build a Kelpie IO module for writing objects to a LevelDB store. Required leveldb_DIR | +| Faodel_ENABLE_MPI_SUPPORT | Boolean | Include MPI capabilities (MPISyncStart service, mpi-based testing, and NNTI MPI Transport) | +| Faodel_ENABLE_TCMALLOC | Boolean | Compile the included tpl/gperftools tcmalloc and use in Lunasa memory allocator | +| Faodel_LOGGING_METHOD | stdout, sbl, disabled | Determines where the logging interface writes. Select disabled to optimize away | +| Faodel_NETWORK_LIBRARY | nnti,libfabric | Select which RDMA network library to use | +| Faodel_NNTI_SERIALIZATION_METHOD | CEREAL, XDR | Controls whether NNTI serializes data with XDR or Cereal | Advanced Options ---------------- -| CMake Variable | Type | Purpose | -| ------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------- | -| HDF5_DIR | Path | Location of HDF5 libs. Only used when FAODEL_ENABLE_IOM_HDF5 used | -| leveldb_DIR | Path | Location of leveldb libs. Only used when FAODEL_ENABLE_IOM_LEVELDB used | -| Faodel_ENABLE_CEREAL | Boolean | If XDR is not fully implemented (eg. MacOS Mojave), Cereal is used. CMake should autodetect, but this forces Cereal. | -| Faodel_NO_ISYSTEM_FLAG | Boolean | Some compilers use "-isystem" to identify system libs instead of "-I". CMake should autodetect, but this can override | -| Faodel_OPBOX_NET_NNTI | Boolean | Set to true if Faodel_NETWORK_LIBRARY is nnti | -| Faodel_PERFTOOLS_* | - | These variables are used by the tpl/gperftools library. See their documentation for more info | -| Faodel_THREADING_LIBRARY | pthreads, openmp | Select fundamental threading lib we're using. Almost always is pthreads. This may be modified in the future (eg QTHREADS) | + +| CMake Variable | Type | Purpose | +| -------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| HDF5_DIR | Path | Location of HDF5 libs. Only used when FAODEL_ENABLE_IOM_HDF5 used | +| leveldb_DIR | Path | Location of leveldb libs. Only used when FAODEL_ENABLE_IOM_LEVELDB used | +| Faodel_ASSERT_METHOD | cassert,debug*,none | Select assert behavior. Cassert is std c. debug(Exit/Halt/Warn) prints more info. none does no assertion checks | +| Faodel_ENABLE_DEBUG_TIMERS | Boolean | When enabled, some components such as opbox can capture timing traces that are dumped on exit. use opbox.enable_timers | +| Faodel_NO_ISYSTEM_FLAG | Boolean | Some compilers use "-isystem" to identify system libs instead of "-I". CMake should autodetect, but this can override | +| Faodel_OPBOX_NET_NNTI | Boolean | Set to true if Faodel_NETWORK_LIBRARY is nnti | +| Faodel_PERFTOOLS_* | - | These variables are used by the tpl/gperftools library. See their documentation for more info | +| Faodel_THREADING_LIBRARY | pthreads, openmp | Select fundamental threading lib we're using. Almost always is pthreads. This may be modified in the future (eg QTHREADS) | Environment Variables --------------------- The following environment variables are used during configuration -- BOOST_ROOT: Path to the boost installation (inside should have an include and lib directory for Boost) -- GTEST_ROOT: Path to the googletest installation is BUILD_TESTS is enabled + +- `BOOST_ROOT`: Path to the boost installation (inside should have an include and + lib directory for Boost). On some OS's where boost is a system package, + you can use `/usr/lib`. +- `GTEST_ROOT`: Path to the googletest installation is BUILD_TESTS is enabled diff --git a/LICENSE.md b/LICENSE.md index 9011ab8..6f13d5b 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright 2018 National Technology & Engineering Solutions of Sandia, +Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. Government retains certain rights in this software. diff --git a/NEWS.md b/NEWS.md index ff0f029..0ff206c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,10 +1,36 @@ Release Information =================== -This file provides information about different releases of the -faodel tools. Releases are named alphabetically and have a -4-digit ID associated with them that designates the year and month of -the release. +This file provides information about different releases of the faodel tools. +Releases are named alphabetically and have a 4-digit ID associated with them +that designates the year and month of the release. + +Fluid (1.2108.1) +---------------- +- Summary: Trace record/playback, Stress tools, UDFs, and testing +- Release Improvements + - Kelpie: Added Drop and RowInfo operations for remote use + - Kelpie: Added ResultCollector to simplify async requests + - Kelpie: New trace pool for client-side pool activities + - Lunasa: Added DataObjectPacker for easier packing + - Lunasa: Added StringObject for easier packing of text + - NNTI: Added support for RoCE + - faodel-cli: config-info dumps out configuration options + - faodel-cli: New playback option that can use traces + - faodel-cli: New kblast option generates parallel kelpie traffic + - faodel-stress: New tool for benchmarking CPU for non-net activities + - OpBox: Ability to capture timing traces for ops + - Backburner: New notification methods to decrease CPU usage +- Significant User-Visible Changes: + - Kelpie: kv_row/col_info_t replaced by object_info_t (smaller, simpler) + - config: "ioms" are now "kelpie.ioms" + config: backburner.notification_method for pipe, polling, sleep_polling + - Examples cam now be built inside the build via BUILD_EXAMPLES + - OpBoxStandard is now OpBoxDeprecatedStandard +- Experimental Features + - kelpie 'Compute' allows server to perform computations on objects via UDFs +- Known Issues: todo + Excelsior! (1.1906.1) --------------------- diff --git a/README.md b/README.md index 9757556..78f3dd8 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,28 @@ FAODEL Overview FAODEL (Flexible, Asynchronous, Object Data-Exchange Libraries) is a collection of software libraries that are used to implement different data management services on high-performance computing (HPC) -platforms. This project is part of the Advanced Technology Development -and Mitigation (ATDM) effort for NNSA's ASC program at Sandia -National Laboratories. +platforms. This project was funded through NNSA's ASC program at Sandia +National Laboratories. + +- **What Problem Does This Solve?** HPC workflows often need a way to + move large datasets between two or more MPI applications. Rather than route + intermediate data through the filesystem, FAODEL lets you pass the data + *directly* between the two MPI applications or *indirectly* through a + separate distributed memory application. The filesystem can also be used + if applications in the workflow do not run concurrently. + +- **Who Is the Intended Audience?** This software is intended for HPC + developers that write parallel MPI applications in C++ and run + workflows on cluster computers with hundreds to thousands of compute + nodes. FAODEL requires an HPC network fabric such as InfiniBand, RoCE, + OmniPath, or Gemini. + +- **Pronounciation**: We say *"Fay-oh-Dell"*. + +**Note**: FAODEL development takes place in a private repository due to Sandia's + software release process. The Github repository is only updated when + there are minor bug fixes or new release snapshots. This message will + be updated if/when FAODEL is no longer being developed. Components ---------- @@ -47,6 +66,17 @@ FAODEL is composed of multiple libraries: provides a way to map log information in FAODEL components to Boost's logging library. +There are two main command-line tools users may find useful: + +- [faodel-cli](tools/faodel-cli/README_Faodel_Cli.md): This tool is an + all-in-one tool for hosting and configuring different faodel services + such as dirman and kelpie. Users can obtain build and runtime info, as + well as query services and put/get/delete objects. Commands can also be + replayed through the play command. +- [faodel-stress](tools/faodel-stress/README_Faodel_Stress.md): This + standalone tool can be used to benchmark a system and estimate how + well it performs different data management operations. + Additional Information ====================== This release includes files to help guide users. The files are: @@ -59,6 +89,10 @@ This release includes files to help guide users. The files are: - [NEWS](NEWS.md): The news file provides a history of major changes provided with each release of this software. Developers should review this document when switching to a new release. +- [Configuration File Cookbook](docs/ConfigurationFileCookbook.md): This page + provides a summary of runtime configuration settings to plug into your + $FAODEL_CONFIG file +- [What's a Faodel?](docs/WhatsAFaodel.md) How we picked the acronym. Contributors ============ @@ -85,8 +119,10 @@ and copyright info: - gperftools (in tpl/gperftools) - Boost ASIO examples (in src/whookie/server) + + Copyright ========= -Copyright 2018 National Technology & Engineering Solutions of Sandia, LLC +Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. Government retains certain rights in this software. diff --git a/cmake/FaodelCheckTransports.cmake b/cmake/FaodelCheckTransports.cmake new file mode 100644 index 0000000..9257654 --- /dev/null +++ b/cmake/FaodelCheckTransports.cmake @@ -0,0 +1,74 @@ + +function( check_preferred_transports ) + # + # First we check for disabled transports that probably should be enabled + # + if( NOT "${NNTI_BUILD_UGNI}" ) + if( "${CMAKE_SYSTEM_NAME}" MATCHES "CrayLinuxEnvironment" ) + message( STATUS "======================================================================" ) + message( STATUS " WARNING " ) + message( STATUS "======================================================================" ) + message( STATUS " This is a Cray System, but the UGNI transport is disabled. " ) + found_status( "libugni" UGNI_FOUND "v" "${UGNI_PC_VERSION}" ) + found_status( "Cray DRC" DRC_FOUND ) + message( STATUS "======================================================================" ) + message( STATUS "" ) + endif() + endif() + + if( NOT "${NNTI_BUILD_IBVERBS}" ) + find_program(IBV_DEVINFO ibv_devinfo ) + if( IBV_DEVINFO ) + execute_process( COMMAND ${IBV_DEVINFO} RESULT_VARIABLE DEVINFO_RESULT OUTPUT_VARIABLE DEVINFO_STDOUT ERROR_VARIABLE DEVINFO_STDERR ) + if( ${DEVINFO_RESULT} EQUAL 0 ) + message( STATUS "======================================================================" ) + message( STATUS " WARNING " ) + message( STATUS "======================================================================" ) + message( STATUS "This system has an IB device, but the ibverbs transport is disabled. " ) + found_status( "ibverbs" IBVerbs_FOUND ) + if( ${NNTI_DISABLE_IBVERBS_TRANSPORT} ) + message( STATUS " IBVerbs Transport explicitly disabled" ) + else() + message( STATUS " Not building the IBVerbs Transport" ) + endif() + message( STATUS "======================================================================" ) + message( STATUS "" ) + endif() + endif() + endif() + + # + # Now we check for enabled transports that probably shouldn't be + # + if( "${NNTI_BUILD_IBVERBS}" ) + find_program(IBV_DEVINFO ibv_devinfo ) + if( IBV_DEVINFO ) + execute_process( COMMAND ${IBV_DEVINFO} RESULT_VARIABLE DEVINFO_RESULT OUTPUT_VARIABLE DEVINFO_STDOUT ERROR_VARIABLE DEVINFO_STDERR ) + if( NOT ${DEVINFO_RESULT} EQUAL 0 ) + message( STATUS "======================================================================" ) + message( STATUS " WARNING " ) + message( STATUS "======================================================================" ) + message( STATUS "The ibverbs transport is enabled, but ibv_devinfo didn't report any " ) + message( STATUS "device information. This can happen if the ibverbs headers, libraries" ) + message( STATUS "and tools are installed, but the system doesn't have any IB hardware. " ) + message( STATUS "To disable the ibverbs transport, add this to your cmake command: " ) + message( STATUS " -DNNTI_DISABLE_IBVERBS_TRANSPORT:BOOL=TRUE " ) + message( STATUS "======================================================================" ) + message( STATUS "" ) + endif() + else() + message( STATUS "======================================================================" ) + message( STATUS " WARNING " ) + message( STATUS "======================================================================" ) + message( STATUS "The ibverbs transport is enabled, but ibv_devinfo couldn't be found. " ) + message( STATUS "This can happen if the ibverbs headers and libraries are installed, " ) + message( STATUS "but the runtime tools are not installed. The tools are not required " ) + message( STATUS "to build and use the ibverbs transport, but this could indicate a " ) + message( STATUS "partial installation or missing IB hardware. " ) + message( STATUS "To disable the ibverbs transport, add this to your cmake command: " ) + message( STATUS " -DNNTI_DISABLE_IBVERBS_TRANSPORT:BOOL=TRUE " ) + message( STATUS "======================================================================" ) + message( STATUS "" ) + endif() + endif() +endfunction() diff --git a/cmake/FaodelConfig.cmake.in b/cmake/FaodelConfig.cmake.in index 4bc98e9..4e33b89 100644 --- a/cmake/FaodelConfig.cmake.in +++ b/cmake/FaodelConfig.cmake.in @@ -59,6 +59,7 @@ message( STATUS "[Faodel] Locating required third-party libraries" ) # These are options in the Faodel build set( Faodel_ENABLE_IOM_LEVELDB @Faodel_ENABLE_IOM_LEVELDB@ ) set( Faodel_ENABLE_IOM_HDF5 @Faodel_ENABLE_IOM_HDF5@ ) +set( Faodel_ENABLE_IOM_CASSANDRA @Faodel_ENABLE_IOM_CASSANDRA@ ) include( ${CMAKE_CURRENT_LIST_DIR}/FaodelTPLs.cmake ) diff --git a/cmake/FaodelRpathHelpers.cmake b/cmake/FaodelRpathHelpers.cmake new file mode 100644 index 0000000..c441f01 --- /dev/null +++ b/cmake/FaodelRpathHelpers.cmake @@ -0,0 +1,92 @@ + +function( add_library_paths_to_var lib_list add_to_var ) + set(target_var "${${add_to_var}}") + foreach(lib IN LISTS lib_list) + get_filename_component(lib_path ${lib} DIRECTORY ) + set(target_var "${target_var}:${lib_path}") + endforeach() + set(${add_to_var} "${target_var}" PARENT_SCOPE) +endfunction() + +function( set_faodel_build_rpath ) + set(FAODEL_BUILD_RPATH "" CACHE STRING "list of paths to use for RPATH in the build tree") + ## add the Faodel library build paths + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/tpl/gperftools") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/src/sbl") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/src/faodel-common") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/src/whookie") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/src/faodel-services") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/src/nnti") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/src/lunasa") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/src/opbox") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/src/dirman") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/src/kelpie") + ## add the Faodel test library build paths + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/tests/nnti/benchmarks") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/tests/nnti/c-api") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/tests/nnti/cpp-api") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/tests/opbox") + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${CMAKE_CURRENT_BINARY_DIR}/tests/kelpie") + ## add the Faodel TPL library build paths + if (Boost_FOUND) + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}:${Boost_LIBRARY_DIRS}") + endif() + if (GTest_FOUND) + add_library_paths_to_var(${GTEST_BOTH_LIBRARIES} FAODEL_BUILD_RPATH) + endif() + + ## put into the parent scope + set(FAODEL_BUILD_RPATH "${FAODEL_BUILD_RPATH}" PARENT_SCOPE) + +# message( STATUS "FAODEL_BUILD_RPATH=${FAODEL_BUILD_RPATH}" ) +endfunction() + +function( set_faodel_install_rpath ) + set(FAODEL_INSTALL_RPATH "" CACHE STRING "list of paths to use for RPATH in the install tree") + set(FAODEL_INSTALL_RPATH "${FAODEL_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/lib") + if (Boost_FOUND) + set(FAODEL_INSTALL_RPATH "${FAODEL_INSTALL_RPATH}:${Boost_LIBRARY_DIRS}") + endif() + if (GTest_FOUND) + add_library_paths_to_var("${GTEST_BOTH_LIBRARIES}" FAODEL_INSTALL_RPATH) + endif() + + ## put into the parent scope + set(FAODEL_INSTALL_RPATH "${FAODEL_INSTALL_RPATH}" PARENT_SCOPE) + +# message( STATUS "FAODEL_INSTALL_RPATH=${FAODEL_INSTALL_RPATH}" ) +endfunction() + +function( set_cmake_rpath_flags ) + set(CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG "-Wl,-rpath," PARENT_SCOPE) + set(CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG_SEP ":" PARENT_SCOPE) + set(CMAKE_EXECUTABLE_RUNTIME_C_FLAG "-Wl,-rpath," PARENT_SCOPE) + set(CMAKE_EXECUTABLE_RUNTIME_C_FLAG_SEP ":" PARENT_SCOPE) + + set(CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG "-Wl,-rpath," PARENT_SCOPE) + set(CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP ":" PARENT_SCOPE) + set(CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG "-Wl,-rpath," PARENT_SCOPE) + set(CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG_SEP ":" PARENT_SCOPE) + + set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG "-Wl,-soname," PARENT_SCOPE) + set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "-Wl,-soname," PARENT_SCOPE) + + set(CMAKE_EXECUTABLE_RPATH_LINK_CXX_FLAG "-Wl,-rpath-link," PARENT_SCOPE) + set(CMAKE_EXECUTABLE_RPATH_LINK_C_FLAG "-Wl,-rpath-link," PARENT_SCOPE) + set(CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG "-Wl,-rpath-link," PARENT_SCOPE) + set(CMAKE_SHARED_LIBRARY_RPATH_LINK_C_FLAG "-Wl,-rpath-link," PARENT_SCOPE) + + set(CMAKE_SKIP_INSTALL_RPATH NO PARENT_SCOPE) + set(CMAKE_SKIP_RPATH NO PARENT_SCOPE) +endfunction() + +function( set_cmake_linker_rpath_flags ) + set (CMAKE_EXE_LINKER_FLAGS "-Wl,-rpath,${FAODEL_BUILD_RPATH}" PARENT_SCOPE) + set (CMAKE_SHARED_LINKER_FLAGS "-Wl,-rpath,${FAODEL_BUILD_RPATH}" PARENT_SCOPE) +endfunction() + +function( set_cmake_install_rpath ) + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${FAODEL_INSTALL_RPATH}") + set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" PARENT_SCOPE) +# message( STATUS "CMAKE_INSTALL_RPATH=${CMAKE_INSTALL_RPATH}" ) +endfunction() diff --git a/cmake/FaodelSummary.cmake b/cmake/FaodelSummary.cmake new file mode 100644 index 0000000..4268c77 --- /dev/null +++ b/cmake/FaodelSummary.cmake @@ -0,0 +1,110 @@ +function( found_status ) + if( ${${ARGV1}} ) + set( found_str "Found" ) + if( ${ARGC} EQUAL 4 AND NOT ${ARGV3} EQUAL "" ) + set( version_str "(${ARGV2}${ARGV3})" ) + elseif( ${ARGC} EQUAL 3 AND NOT ${ARGV2} EQUAL "" ) + set( version_str "(${ARGV2})" ) + endif() + else() + set( found_str "Not Found" ) + endif() + set( fill_len 25 ) + set( padded_len 25 ) + string( RANDOM LENGTH ${fill_len} ALPHABET " " fill_str ) + string( SUBSTRING "${ARGV0}:${fill_str}" 0 ${padded_len} padded_str ) + message( STATUS " ${padded_str}${found_str} ${version_str}" ) +endfunction() + +function( message_status ) + set( fill_len 25 ) + set( padded_len 25 ) + string( RANDOM LENGTH ${fill_len} ALPHABET " " fill_str ) + string( SUBSTRING "${ARGV0}:${fill_str}" 0 ${padded_len} padded_str ) + message( STATUS " ${padded_str}${ARGV1}" ) +endfunction() + +function( print_faodel_config_summary ) + message( STATUS "" ) + message( STATUS "======================================================================" ) + message( STATUS "" ) + message( STATUS "Configured to build Faodel with ${CMAKE_MAKE_PROGRAM}" ) + if( CMAKE_INSTALL_PREFIX ) + message( STATUS "Installation prefix is ${CMAKE_INSTALL_PREFIX}" ) + endif() + message( STATUS "" ) + message( STATUS "External Programs:" ) + message_status( "compiler" "${CMAKE_CXX_COMPILER_ID} (${CMAKE_CXX_COMPILER_VERSION})" ) + found_status( "doxygen" DOXYGEN_FOUND "v" "${DOXYGEN_VERSION}" ) + message( STATUS "" ) + message( STATUS "TPLs:" ) + found_status( "Boost" Boost_FOUND "v" "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}" ) + found_status( "MPI" MPI_FOUND "implements MPI-" "${MPI_C_VERSION}" ) + found_status( "googletest" GTest_FOUND ) + if (Faodel_NETWORK_LIBRARY STREQUAL "libfabric") + found_status( "libfabric" LIBFABRIC_FOUND "v" "${Libfabric_pc_VERSION}" ) + endif() + if (Faodel_NETWORK_LIBRARY STREQUAL "nnti") + found_status( "libugni" UGNI_FOUND "v" "${UGNI_PC_VERSION}" ) + found_status( "Cray DRC" DRC_FOUND ) + found_status( "ibverbs" IBVerbs_FOUND ) + endif() + message( STATUS "" ) + message( STATUS "Faodel Common Config:" ) + message( STATUS " Threading Model: ${Faodel_THREADING_MODEL}" ) + message( STATUS " Logging Method: ${Faodel_LOGGING_METHOD}" ) + message( STATUS "" ) + message( STATUS "Lunasa Config:" ) + if ( ${Faodel_ENABLE_TCMALLOC} ) + message( STATUS " Building with tcmalloc from gperftools" ) + endif() + message( STATUS "" ) + message( STATUS "NNTI Config:" ) + if( ${NNTI_BUILD_IBVERBS} ) + if( ${NNTI_HAVE_VERBS_EXP_H} ) + message( STATUS " Building the IBVerbs Transport with the libverbs expanded API (mlx4 or mlx5)" ) + else() + message( STATUS " Building the IBVerbs Transport with the libverbs standard API (mlx4 ONLY)" ) + endif() + else() + if( ${NNTI_DISABLE_IBVERBS_TRANSPORT} ) + message( STATUS " IBVerbs Transport explicitly disabled" ) + else() + message( STATUS " Not building the IBVerbs Transport" ) + endif() + endif() + if( ${NNTI_BUILD_UGNI} ) + message( STATUS " Building the UGNI Transport" ) + else() + if( ${NNTI_DISABLE_UGNI_TRANSPORT} ) + message( STATUS " UGNI Transport explicitly disabled" ) + else() + message( STATUS " Not building the UGNI Transport" ) + endif() + endif() + if( ${NNTI_BUILD_MPI} ) + message( STATUS " Building the MPI Transport" ) + else() + if( ${NNTI_DISABLE_MPI_TRANSPORT} ) + message( STATUS " MPI Transport explicitly disabled" ) + else() + message( STATUS " Not building the MPI Transport" ) + endif() + endif() + if( ${NNTI_USE_XDR} ) + message( STATUS " Using XDR for serialization" ) + else() + if( ${NNTI_USE_CEREAL} ) + message( STATUS " Using Cereal for serialization" ) + else() + message( STATUS " ERROR - Couldn't find a serialization library" ) + endif() + endif() + message( STATUS "" ) + message( STATUS "Opbox Config:" ) + message( STATUS " Network Library: ${Faodel_NETWORK_LIBRARY}" ) + message( STATUS "" ) + message( STATUS "======================================================================" ) + message( STATUS "" ) + message( STATUS "" ) +endfunction() diff --git a/cmake/FaodelTPLs.cmake b/cmake/FaodelTPLs.cmake index e3eb10b..1cca1d9 100644 --- a/cmake/FaodelTPLs.cmake +++ b/cmake/FaodelTPLs.cmake @@ -1,3 +1,4 @@ +include( CheckCXXSymbolExists ) ########################## ## Get the Boost targets @@ -60,21 +61,40 @@ if(BUILD_TESTS) # We only care about GTEST_ROOT if tests are being built endif() find_package(GTest REQUIRED) + if ( GTest_FOUND ) + set( save_CMAKE_REQUIRED_INCLUDES "${CMAKE_REQUIRED_INCLUDES}" ) + set( CMAKE_REQUIRED_INCLUDES "${GTEST_INCLUDE_DIRS}" ) + + check_cxx_symbol_exists(TYPED_TEST_SUITE_P "gtest/gtest.h" HAVE_TYPED_TEST_SUITE_P) + check_cxx_symbol_exists(REGISTER_TYPED_TEST_SUITE_P "gtest/gtest.h" HAVE_REGISTER_TYPED_TEST_SUITE_P) + check_cxx_symbol_exists(INSTANTIATE_TYPED_TEST_SUITE_P "gtest/gtest.h" HAVE_INSTANTIATE_TYPED_TEST_SUITE_P) + + set( CMAKE_REQUIRED_INCLUDES "${save_CMAKE_REQUIRED_INCLUDES}" ) + + if( HAVE_TYPED_TEST_SUITE_P AND HAVE_REGISTER_TYPED_TEST_SUITE_P AND HAVE_INSTANTIATE_TYPED_TEST_SUITE_P ) + set( Faodel_HAVE_GTEST_TYPED_TEST_SUITE_API 1 ) + endif() + endif( GTest_FOUND ) endif() ######################## ## HDF5 IOM ######################## +set( Faodel_REQUIRED_HDF5_VERSION "1.8" ) if( Faodel_ENABLE_IOM_HDF5 ) find_package( HDF5 COMPONENTS C MODULE ) if( HDF5_FOUND AND NOT TARGET Faodel::HDF5 ) - add_library( Faodel::HDF5 INTERFACE IMPORTED ) - target_compile_definitions( Faodel::HDF5 INTERFACE ${HDF5_DEFINITIONS} ) - target_link_libraries( Faodel::HDF5 INTERFACE ${HDF5_LIBRARIES} ) - target_include_directories( Faodel::HDF5 INTERFACE ${HDF5_INCLUDE_DIRS} ) - set( FAODEL_HAVE_HDF5 TRUE ) - message( STATUS "Will build HDF5 IOM, Faodel_ENABLE_IOM_HDF5 set and HDF5 found" ) + if( HDF5_VERSION VERSION_GREATER_EQUAL ${Faodel_REQUIRED_HDF5_VERSION} ) + add_library( Faodel::HDF5 INTERFACE IMPORTED ) + target_compile_definitions( Faodel::HDF5 INTERFACE ${HDF5_DEFINITIONS} ) + target_link_libraries( Faodel::HDF5 INTERFACE ${HDF5_LIBRARIES} ) + target_include_directories( Faodel::HDF5 INTERFACE ${HDF5_INCLUDE_DIRS} ) + set( FAODEL_HAVE_HDF5 TRUE ) + message( STATUS "Will build HDF5 IOM, Faodel_ENABLE_IOM_HDF5 set and HDF5 found" ) + else() + message( STATUS "Cannot build HDF5 IOM as requested. Faodel requires ${Faodel_REQUIRED_HDF5_VERSION}, but found ${HDF5_VERSION}." ) + endif() else() message( STATUS "Cannot build HDF5 IOM as requested, HDF5 not found. Set HDF5_ROOT" ) endif() @@ -171,6 +191,46 @@ endif() # Stanza 3 : Determine Faodel's network library # ############################## + +function( create_pc_imported_target _prefix _target_name ) + + # This pretty closely mimics what happens in FindPkgConfig.cmake from the CMake distro. + # The only significant difference (should be) is that we get the STATIC versions of + # information from the .pc file, since we always want to link network libs statically. + # pkg_search_module( ... IMPORTED_TARGET ) sets up a target that only works with shared libraries. + # It's possible on some platforms that the STATIC variables won't be created + # (because pkg-config --static returns nothing, which might happen on platforms where no dynamic + # linking is ever done), so check for empty vars and fall back to the non-static variables if necessary. + + add_library( ${_target_name} INTERFACE IMPORTED ) + if( NOT BUILD_SHARED_LIBS AND ${_prefix}_STATIC_LDFLAGS ) + # pkg-config found a STATIC configuration + set_property( TARGET ${_target_name} + PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${${_prefix}_STATIC_INCLUDE_DIRS} + ) + set_property( TARGET ${_target_name} + PROPERTY INTERFACE_LINK_LIBRARIES ${${_prefix}_STATIC_LDFLAGS} + ) + set_property( TARGET ${_target_name} + PROPERTY INTERFACE_COMPILE_OPTIONS ${${_prefix}_STATIC_CFLAGS_OTHER} + ) + else() + if ( ${_prefix}_LDFLAGS ) + # Unlikely that we have any STATIC info, use the regular variabes + set_property( TARGET ${_target_name} + PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${${_prefix}_INCLUDE_DIRS} + ) + set_property( TARGET ${_target_name} + PROPERTY INTERFACE_LINK_LIBRARIES ${${_prefix}_LDFLAGS} + ) + set_property( TARGET ${_target_name} + PROPERTY INTERFACE_COMPILE_OPTIONS ${${_prefix}_CFLAGS_OTHER} + ) + endif() + endif() + +endfunction() + # When we're outside of the Faodel project (ie, this file has been installed and # the user is importing Faodel to use in their project), we can pull in some of # our settings from the installed lib when the user hasn't defined them. @@ -190,40 +250,9 @@ if (Faodel_NETWORK_LIBRARY STREQUAL "libfabric") if( Libfabric_pc_FOUND ) set( LIBFABRIC_FOUND TRUE ) - - # This pretty closely mimics what happens in FindPkgConfig.cmake from the CMake distro. - # The only significant difference (should be) is that we get the STATIC versions of - # information from the .pc file, since we always want to link network libs statically. - # pkg_search_module( ... IMPORTED_TARGET ) sets up a target that only works with shared libraries. - # It's possible on some platforms that the STATIC variables won't be created - # (because pkg-config --static returns nothing, which might happen on platforms where no dynamic - # linking is ever done), so check for empty vars and fall back to the non-static variables if necessary. - add_library( Libfabric INTERFACE IMPORTED ) - if( Libfabric_pc_STATIC_LDFLAGS ) - # Assume we have STATIC information - set_property( TARGET Libfabric - PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${Libfabric_pc_STATIC_INCLUDE_DIRS} - ) - set_property( TARGET Libfabric - PROPERTY INTERFACE_LINK_LIBRARIES ${Libfabric_pc_STATIC_LDFLAGS} - ) - set_property( TARGET Libfabric - PROPERTY INTERFACE_COMPILE_OPTIONS ${Libfabric_pc_STATIC_CFLAGS_OTHER} - ) - else() - # Unlikely that we have any STATIC info, use the regular variabes - set_property( TARGET Libfabric - PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${Libfabric_pc_INCLUDE_DIRS} - ) - set_property( TARGET Libfabric - PROPERTY INTERFACE_LINK_LIBRARIES ${Libfabric_pc_LDFLAGS} - ) - set_property( TARGET Libfabric - PROPERTY INTERFACE_COMPILE_OPTIONS ${Libfabric_pc_CFLAGS_OTHER} - ) - endif() - + create_pc_imported_target( Libfabric_pc Libfabric ) + LIST( APPEND FaodelNetlib_TARGETS Libfabric ) set( PKGCONFIG_REQUIRES "${PKGCONFIG_REQUIRES} libfabric" ) message( STATUS "Found Libfabric, target appended to FaodelNetlib_TARGETS" ) @@ -238,28 +267,36 @@ elseif( Faodel_NETWORK_LIBRARY STREQUAL "nnti" ) # For NNTI we need to locate UGNI, IBVerbs, and CrayDRC + # + # Cray UGNI + # pkg_check_modules(UGNI_PC cray-ugni) - SET(UGNI_FOUND ${UGNI_PC_FOUND}) - if(UGNI_PC_FOUND) - foreach(ugnilib ${UGNI_PC_LIBRARIES}) - find_library(${ugnilib}_LIBRARY NAMES ${ugnilib} HINTS ${UGNI_PC_LIBRARY_DIRS}) - if (${ugnilib}_LIBRARY) - LIST(APPEND UGNI_LIBRARIES ${${ugnilib}_LIBRARY}) - add_library( ${ugnilib} IMPORTED UNKNOWN ) - set_property( TARGET ${ugnilib} PROPERTY IMPORTED_LOCATION ${${ugnilib}_LIBRARY} ) - set_property( TARGET ${ugnilib} PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${UGNI_PC_INCLUDE_DIRS} ) - LIST( APPEND UGNI_TARGETS ${ugnilib} ) - endif (${ugnilib}_LIBRARY) - endforeach(ugnilib) - SET(UGNI_INCLUDE_DIRS ${UGNI_PC_INCLUDE_DIRS}) - LIST( APPEND FaodelNetlib_TARGETS ${UGNI_TARGETS} ) + + create_pc_imported_target( UGNI_PC UGNI ) + + LIST( APPEND UGNI_TARGETS UGNI ) + LIST( APPEND FaodelNetlib_TARGETS UGNI ) SET( NNTI_HAVE_UGNI 1 ) SET( FAODEL_HAVE_UGNI 1 ) set( PKGCONFIG_REQUIRES "${PKGCONFIG_REQUIRES} cray-ugni" ) endif(UGNI_PC_FOUND) + # + # Cray DRC + # + pkg_check_modules(DRC_PC cray-drc) + SET(DRC_FOUND ${DRC_PC_FOUND}) + if(DRC_PC_FOUND) + + create_pc_imported_target( DRC_PC DRC ) + + LIST( APPEND DRC_TARGETS DRC ) + LIST( APPEND FaodelNetlib_TARGETS DRC ) + set( PKGCONFIG_REQUIRES "${PKGCONFIG_REQUIRES} cray-drc" ) + endif(DRC_PC_FOUND) + # IBVERBS # - Try to find IBVerbs @@ -296,30 +333,6 @@ elseif( Faodel_NETWORK_LIBRARY STREQUAL "nnti" ) mark_as_advanced(IBVerbs_INCLUDE_DIR IBVerbs_LIBRARY ) - # - # CrayDRC - # - - pkg_check_modules(DRC_PC cray-drc) - SET(DRC_FOUND ${DRC_PC_FOUND}) - if(DRC_PC_FOUND) - foreach(drclib ${DRC_PC_LIBRARIES}) - find_library(${drclib}_LIBRARY NAMES ${drclib} HINTS ${DRC_PC_LIBRARY_DIRS}) - if (${drclib}_LIBRARY) - if( NOT TARGET ${drclib} ) - add_library( ${drclib} IMPORTED UNKNOWN ) - set_property( TARGET ${drclib} PROPERTY IMPORTED_LOCATION ${${drclib}_LIBRARY} ) - set_property( TARGET ${drclib} PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${DRC_PC_INCLUDE_DIRS}" ) - LIST( APPEND DRC_TARGETS ${drclib} ) - LIST( APPEND DRC_LIBRARIES ${${drclib}_LIBRARY}) - endif ( NOT TARGET ${drclib} ) - endif (${drclib}_LIBRARY) - endforeach(drclib) - LIST(APPEND DRC_INCLUDE_DIRS ${DRC_PC_INCLUDE_DIRS}) - LIST( APPEND FaodelNetlib_TARGETS ${DRC_TARGETS} ) - set( PKGCONFIG_REQUIRES "${PKGCONFIG_REQUIRES} cray-drc" ) - endif(DRC_PC_FOUND) - else() message(FATAL_ERROR "Faodel_NETWORK_LIBRARY must be defined as one of (nnti | libfabric). The build cannot continue." ) endif() diff --git a/cmake/faodelConfig.h.in b/cmake/faodelConfig.h.in index 3da39be..ac5af73 100644 --- a/cmake/faodelConfig.h.in +++ b/cmake/faodelConfig.h.in @@ -47,12 +47,19 @@ /* Faodel Common Info */ +#cmakedefine01 Faodel_ASSERT_METHOD_NONE +#cmakedefine01 Faodel_ASSERT_METHOD_CASSERT +#cmakedefine01 Faodel_ASSERT_METHOD_DEBUG_WARN +#cmakedefine01 Faodel_ASSERT_METHOD_DEBUG_HALT +#cmakedefine01 Faodel_ASSERT_METHOD_DEBUG_EXIT + #cmakedefine Faodel_THREADING_MODEL @Faodel_THREADING_MODEL@ #cmakedefine Faodel_THREADING_MODEL_PTHREADS 1 #cmakedefine Faodel_THREADING_MODEL_OPENMP 1 #cmakedefine01 Faodel_LOGGINGINTERFACE_DISABLED #cmakedefine01 Faodel_LOGGINGINTERFACE_USE_SBL +#cmakedefine01 Faodel_ENABLE_DEBUG_TIMERS /* NNTI Config Info */ #cmakedefine01 NNTI_BUILD_IBVERBS diff --git a/docs/ConfigurationFileCookbook.md b/docs/ConfigurationFileCookbook.md new file mode 100644 index 0000000..e561d1d --- /dev/null +++ b/docs/ConfigurationFileCookbook.md @@ -0,0 +1,220 @@ +# Configuration File Cookbook + +FAODEL applications load a configuration file at start time to determine +how different components should behave. FAODEL reads the environment +variable `FAODEL_CONFIG` to determine which file to load. This file provides +a summary of different options that can be supplied in the config file. + +## General Configuration Options +### Using Node Role to select from multiple configurations +The `node_role` setting allows you to specify multiple behaviors for different +classes of systems in the same config file. Just prefix all your settings with +the role name, and then append specify the role for each machine: + +``` +server.backburner.num_threads 32 # crank up the threads on a dedicated server +client.backburner.num_threads 1 # minimize threads on a client + +node_role server # usually code sets this via config.Append("node_role","server"); +``` +### Setting the Security Bucket +It isn't used in many of our examples, but all kelpie calls use a security bucket +to help isolate different users or workloads in the store. Set the bucket: + +``` +secutity_bucket my_special_stuff +``` + + +### Read Additional Configuration Files +While not recommended, your main configuration file can load additional config +files. +``` +config.additional_files config_file1;config_file2;config_file3 +``` + +## Logging +### Changing the log levels +Components that have logging capabilities can dump out different levels of detail: +``` +kelpie.debug true # shortcut for seeing all debug messages + +kelpie.log.debug true # enable debug, info, and warning messages +kelpie.log.info true # enable info and warning messages +kelpie.log.warn true # enable warning messages +``` + +### Top-level components with logging +These top-level components support logging: + +``` +backburner.debug true +bootstrap.debug true +dirman.debug true +kelpie.debug true +lunasa.debug true +mpisyncstart.debug true +opbox.debug true +whookie.debug true +``` + +## Lower-level components with logging +Several lower-level components also have logging capabilities. + +``` +backburner.worker # individual threads doing backburner work +dirman.cache.mine # stores list of known resources on this nodes +dirman.cache.others # stores list of known resources on other nodes +dirman.cache.owners # stores nodes of where to find items +kelpie.compute # stores list of known compute functions +kelpie.iom # stores list of known io modules +kelpie.lkv # manages local key/blob storage +kelpie.op.compute # state machine for compute Ops +kelpie.op.drop # state machine for drop Ops +kelpie.op.getbounded # state machine for want/need known size Ops +kelpie.op.getunbounded # state machine for want/need unknown size Ops +kelpie.op.list # state machien for list Ops +kelpie.op.meta # state machine for meta query Ops +kelpie.op.publish # state machine for publish Ops +kelpie.pool # stores list of known pools +lunasa.allocator # controls memory allocations +``` + +## Bootstrap Tricks +### Dumping the parsed configuration at Start +You can tell a node to dump out its configuration when bootstrap initializes: + +``` +bootstrap.show_config_at_init +``` + +### Delaying Finish +It can be beneficial to use whookie as a means of querying the state of +a running (or halted) simulation. You can either halt or delay shutdown +via bootstrap options. Also, you can display a large "ok" message on a +successful exit to make it more obvious how the sim ended. + +``` +bootstrap.status_on_shutdown true # Dumps a visible "ok" message +bootstrap.halt_on_shutdown true # Halt without exiting +bootstrap.sleep_seconds_before_shutdown 60 # Halt for 60s and then exit +``` + +## Managing DirMan +### Designating the Root Node and using it +Services that use DirMan generally need to know which node is the root node. If +you want to designate your node as the root and use a file to convey the id, +do the following on the server: + +``` +dirman.host_root true +dirman.write_root myroot_file.txt +``` +and the following on the other nodes: +``` +dirman.root_node.file myroot_file.txt +``` + +### Use MPISyncStart to setup the first rank as DirMan Root Node +If you're running inside an mpi job, MPISyncStart can automate picking +rank 0 as the dirman root: + +``` + +mpisyncstart.enable true +dirman.root_node_mpi 0 + +``` + +Make sure you remember to add mpisyncstart to bootstraps: + +``` + +faodel::mpisyncstart::bootstrap(); +faodel::bootstrap::Start(faodel::Configuration(default_config_string), kelpie::bootstrap); + +``` + +## Defining Resources +### Defining DHT Resources Using MPISyncStart + +In test programs you often need a simple way to define pool resources. One +way to do this is to write some startup code that creates pools and then +has nodes join the pool. An easier way to define basic resources is to +use the *mpisyncstart* service to populate dirman. MPISyncStart does some +communication operations at startup to convert MPI ranks to FAODEL node ids. + +This example enables mpisyncstart and then creates two pool resources. + +Note: Resources are ONLY written to the dirman node at boot. It is assumed + that the dirman service will be used to retrieve the info + +Note: This currently does not work with the static dirman implementation + +``` + +mpisyncstart.enable true +dirman.type centralized +dirman.root_node_mpi 0 + +dirman.resources_mpi[] local:/EMPIRE/particles ALL +dirman.resources_mpi[] dht:/EMPIRE/fluid ALL + +``` + +### Pointing I/O Modules (IOMs) at storage +An I/O module is essentially a driver for exchanging data with a storage +device. Each IOM needs to have a type and a path specified. + +``` +default.iom.type PosixIndividualObjects +default.iom.path ./faodel_data + +default.ioms empire_particles;empire_fields +dirman.resources[] local:/EMPIRE/particles&iom=empire_particles +dirman.resources[] local:/EMPIRE/fields&iom=empire_fields + +dirman.type static +``` + +# Summary of Options (Excluding Logging) + +| Property | Type | Default | Description | +| --------------------------------------- | ---------------------------- | ----------- | -------------------------------------------------------------------------------- | +| backburner.threads | integer | 1 | Number of worker threads to use | +| backburner.notification_method | pipe, polling, sleep_polling | pipe | Controls how workers are notified of new work | +| backburner.sleep_polling_time | time (us) | 100us | Set polling delay in sleep_polling mode | +| bootstrap.show_config_at_init | boolean | false | Display config used at init time | +| bootstrap.halt_on_shutdown | boolean | false | Do infinite-wait instead of exit at shutdown | +| bootstrap.status_on_shutdown | boolean | false | Dump an ok message on successful exit | +| bootstrap.exit_on_errors | boolean | false | Exit on errors instead of throwing exception | +| bootstrap.sleep_seconds_before_shutdown | int | 0 | Delay Finish shutdown for specified seconds | +| mpisyncstart.enable | boolean | false | Enable the service | +| mpisyncstop.enable | boolean | false | Perform an mpi barrier before Finish | +| config.additional_files | string list | "" | Load additional info for listed files | +| config.purge | boolean | false | Removes all tags and reloads current config | +| dirman.type | none,static,centralized | centralized | Static assumes all info is in config, centralized uses a single server | +| dirman.host_root | bool | false | When true, this node is the dirman root node | +| dirman.write_root | filename | "" | Instruct root node to write id to file | +| dirman.root_node | nodeid | "" | The nodeid of the root node (hex value) | +| dirman.root_node.file | filename | "" | Read the supplied file to find the root node | +| dirman.root_node_mpi | Rank | -1 | Specify rank for dirman root node | +| dirman.resource[] | url | "" | Define a static resource in the configuration | +| dirman.resources_mpi[] | url node(s) | "" | Specify resource using mpi ranks | +| kelpie.type | standard, nonet | standard | Select between the standard networked core, or a debug option without networking | +| lunasa.eager_memory_manager | tcmalloc, malloc | tcmalloc | Select memory allocator used on eager allocations | +| lunasa.lazy_memory_manager | tcmalloc, malloc | malloc | Select memory allocator used on lazy allocations | +| lunasa.tcmalloc.min_system_alloc | size | - | Override tcmalloc's minimum allocation size | +| opbox.type | standard | standard | Select the opbox implementation type | +| opbox.enable_timers | bool | false | Enable performance timers on Ops and dump timelines on shutdown | +| net.transport.name | verbs,gni,mpi,sockets | none | Network library to use, default is system dependent | +| net.log.debug | bool | false | When true, output debug messages | +| net.log.info | bool | false | When true, output info messages | +| net.log.warn | bool | false | When true, output warning messages | +| net.log.error | bool | false | When true, output error messages | +| net.log.fatal | bool | false | When true, output fatal messages | +| net.log.filename | string | none | When set, direct log messages to file else stdout | +| whookie.interfaces | string list | eth0,lo | Order of interfaces to get IP from | +| whookie.app_name | string | Whookie | Name of the application to display up front | +| whookie.address | string | 0.0.0.0 | The IP address the app would like to bind to | +| whookie.port | integer | 1990 | The desired port to use | diff --git a/docs/Doxyfile b/docs/Doxyfile index 6b004f5..79b7e2d 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -789,6 +789,8 @@ INPUT += src/lunasa INPUT += src/opbox INPUT += src/faodel-services INPUT += src/kelpie +INPUT += tools/faodel-cli +INPUT += tools/faodel-stress INPUT += README.md INPUT += INSTALL.md INPUT += LICENSE.md @@ -802,6 +804,10 @@ INPUT += src/dirman/README_DirMan.md INPUT += src/faodel-services/README_Services.md INPUT += src/faodel-common/README_Common.md INPUT += src/sbl/README_SBL.md +INPUT += tools/faodel-cli/README_Faodel_Cli.md +INPUT += tools/faodel-stress/README_Faodel_Stress.md +INPUT += docs/ConfigurationFileCookbook.md +INPUT += docs/WhatsAFaodel.md # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -827,6 +833,9 @@ INPUT_ENCODING = UTF-8 # *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. FILE_PATTERNS = *.cpp \ + *.c \ + *.h \ + *.hpp \ *.hh # The RECURSIVE tag can be used to specify whether or not subdirectories should diff --git a/docs/WhatsAFaodel.md b/docs/WhatsAFaodel.md new file mode 100644 index 0000000..da0a940 --- /dev/null +++ b/docs/WhatsAFaodel.md @@ -0,0 +1,23 @@ +What's a FAODEL? +================ + +[Faodhail](https://en.wikisource.org/wiki/An_Etymological_Dictionary_of_the_Gaelic_Language/F) +is a Gaelic term for a shallow spot in the water that allows +you to pass from one island to another. If you look at maps of the coastlines +of Ireland and Scottland, you'll find a few places where there are roads in +the water that connect two land masses. These roads are usually right on +the beach and are only available when the tide is out (follow the arrow +signs). [Faodhail A' Chinn Ear](https://getoutside.ordnancesurvey.co.uk/local/faodhail-a-chinn-ear-beach-na-h-eileanan-an-iar) +is one example. Omey Strand near Claddaghduff in Ireland (below) is another. +Pro tip: check your rental car agreement and the tide schedule before +driving out on one of these roads. + +We liked the idea of our communication software serving as a type of land +bridge that connects two or more independent application islands. The FAODEL +acronym was picked as a simplification of the faodhail terms. + +We pronounce it **"fay-oh-dell"**, although that's almost certainly incorrect. + + + +![faodhail](./faodhail.jpg) diff --git a/docs/faodhail.jpg b/docs/faodhail.jpg new file mode 100644 index 0000000000000000000000000000000000000000..84d2dd776610c9e77c2bbb3cf12746054474d636 GIT binary patch literal 114583 zcmeFa2V4|C_b)uVELB8$QOZh@-nP)Yfb=FEER?19jv^>FM5)rdNUur}P_PS1m9B^t z0a0ue5kz=13#uN&?1On0l z9$J5CkfRmm?gBx&x*`x21VOYA9Rdk$11Jw1+W@8jVGTIw0j2|C12`BFWDo*`@Ub01 zP7H4Yykir00xY!2g9KO_$ZiGU05~uJGlTF2IGB(ddSD*+(+MbO34);A4nDp<8)vkZ@m{p7tR`F?qC6?wqibk_#z;s@Nl43qT%|D5 z3R3b47%W;^N&zdYAccqcp{(eQvR8ltpgpj{rvR^R8FIc*0 zqg^NgJ(wbLq7C%`#sNAv(BT0v9^fqi#{djFxQvP5qzXbBaA2gQSWhBQ zk_9k)MKS=UQbU5i13_f~Qv(e5ng)Ow0R9Rxy9zK9z?^_yM+reJ0KW&cUVvEvW&!k8 z&_&n*-U09cz#IU>wfq1u8sO~!OMouO4{#5_Isgj*%mgqixI%&eO8_haun53FHDnDi z7GNA`4hI3=1?u=1&{G&8&?5m66jU2@M}!&~qyb|z+{3Ae_JM7*4e;oJuu}|z(jj0G zQ2G7hpmhSw2I%|o;7S7?xb=qsOaWr>C=m&88o=-sItOqn!0;7K1Gob46YT>qJ&=U+ zp+Z4NUjRA-z&8P%31kH8d}G|rQs74oy>W&iBrSu+$)Tk&QfLgC)D+>6 zbii#zmegRt_zx#-o-t?~5CmiC4=;_CK}!RoGzJa-dddLa|CvtmGmHOk5V6=lPZ-eG zvC?P^9!m_s5CF%p8!=!(5R;ZC#-wFri7{XZmYG-V`G4cpi6)q7TaD&Pd zi-!R%UIthVru`pncg$u@HVlp<+K*UM4ABB``Eh7@q7C7$M>GmChQSgU;OhpO0c=hsd^ zN+s&sxVr$ujFc=|mh>C)ALK+DSHK-5NjF9sCySHA%HU*)*-AqgFkH!kLV)1}9$I0D z!^q?0Wn?j6FoE*~6g=qhf1rbb2d2w{d$2#y!FUIL$-t0?ft&a*iU0+Sba0+h7&s3i z0D~X^AQxB{_zi=N@rw)?S&0dFFmnGPL3T4i7K~-TCV+bWrBi|D4R|z`$HV!{;qW+l ztQ;8fF?eYiyc`Bx3D9NbaPr{CSq6{CVIW`*VA&t{b8zIh|80Ta7WnrTfa(w^NE#A{ z{MZV%!3Qhif0ZXa75=Z%&$=<)0J6y=S4VvYv2ANNzB#U-mj4)ArT8^%l_C$!5Rf?G zm^$i@vCoJNDlA2E*O1qc_GGY!(KX~gL=DmJL%{wTJgM=+!4$JKv$cyvzdUcH0XV?a9`Ue%jzja%I8M^%wfbqtwe4$kQM||V8=q7bAe|H;5mnX=is@EzdACgib&kh_}6rpLb^2Q z&1A$63S28v>Sp3!f^|?@YG4NxgeJ}f{mKB#{$l=Ll4zk1!2eIX3SeDHa3#5kb1h_$ z0C8T9j5y!JPRtCp16<<2dN}#dR4{7=xfA97E2pQ>Q*cE$+#g%j-)UL@VPSZ-isVd) zGti{i&yaLANFgbOln*H+(MjQ-`APKCn}C%5=a9rl3K77*wooDj&v1bWI_NZ%1X+RJ z5ea6){@lipP#>gAtQ`fIiIW5}@GR+H9b9HozJKWae`Wng3Q10F({4wJv-y8ze+XJM z>|Si<=trC*1hb8sS8V@gI^$;8K&&O0$KEXae^vjV^ZRp&{+#~j_`l=-b2`b6f2Nb- z|I|KyZTY|I_$NAI(>^4<{}j>?`CyHtQH6A+{wVQlo_|dFHTGAopXK|y$@o{w2?+f| z38@qoV04Ck)Ml%t-sFI1%V7tH2!E@)o9vf-QkSyc?sr|}G#M0osffP6}ASMH8K|&A?cwrDrflh+6 zIyk@tJ@g7(&oyxPfH(wcLT`vE@bB3FjEmn?L-+@E`9DVf4F=~;K9@i7!*6I{_fO$Z zn{M#^ch54!ns3JevCiSzn1vB)Vq`j)EXHU;ORo zVp+~om@vn|FEUY8`kR|xF*Mjx{uLY3KW~dE9ZQ@cQJ+MkZ8|YJmb^{J= zaA-jq;O?Ltcz9M|!&gF;f1<$jGN=^zGLTZXiF2WB@YIt-JZFJ57a5?xUjQEN_dihq zcSFMtq6MZjO&q-=0I18e^8^FP|a(I1EiY`0~vi$F>ARrD^<$a)}nnDr_YKz1wh zqq<`RQ0Ex3k*mHNy>3J9J3Py>(ZG=WWMv2$q%(3DG5JeF;sMw70Jz)Ehi-t`7WheV zJMrot0yex3p2lFxUwJse+=>l!n8>ps^;e!h>p_64^j~wNepo3MBtb9uUz7dK{o4Zn zCs=?6%piibi}2&^h6iKxUn%i_;rT1&uTp=cBmuqg;Mh2;gAr66jK>P#7$*7=a2^4J zgE1Hf?-Tp=ul5NAy?ydm!VmoO_X4b>V>%7)KO8qneqe;;BeQ{IkvND0$pOV6d4EX# z&jaWGd6mC4{#y%dSlwdNZLq_%4La!;5{_?pbz%zr=8w+5RsSy;{`#BplU>|Z07SHu zbHNfrG`M%T3Yj2CrTp)PLg1|t0?ed6V z-a$Nb63?{6GZ*o^jRH=EXFgC64GIC>ff%S@3N!IcO9`*M{7)SGUi!EHZGqnw_-%pT z7Wi#}-xm14$^wwKdw_SSLjVD7NeBpZ_w_+bOGrt;FI&JnvKl8Y9pd*)KfYvA*OS8H zFj80yq@^vsS6WUSEco~KM!QI2CGg^S2^qBRUbMwtL-4%`nmEfE24(_yp+xpa+u(;2 zW`q=UoV}0qiu{+`lYH>ih97j0_-=vpJOEE3z;`qx_Vp1WsCCUnRZvBU0e?4G1tx0q6Et#|ucLGQ+9s>n6;k zx9ji(CHyuD0<%D9L=~_A%mH�iO;>Hr+-gV2&ap1%9gumx)wH_`Rh4+WI>D-cE_g zv2pt2_h(1r?4V3oJ+(4q223R-2QC&^KQASX1z(xK@9p7y;L?$d_G|dBuPM+XC53Tv z!O3q(!_(*B9(J?p;C&YH?sh{pViSkK9N=p^n2A;n)O_>HJ9vE!#V?p;6T^H6Y5 zjDR1xiH~i*eub?D)*Jl#y&zl|P>J;?V0w>=f`XEQoQjf?lA4-|hL(9N?UpUH9E?oc zn7Ov|aC2?vq#_&7O*WQE0~q;WVLkD$D=97ah3G;CN2f|{C|b_*^0)~)QA zot!(t!ukKATfYfz#i1~8GmqefkPHYi2E_VZcr^@y9NhMA-1&oQ!WAV$kyB7oQPXSz z42|@#rO1#dGIDYhuq@&T{5c+qft+!tv^oWosRJdiKQrc7(gi9$jmx)K%=$m^V;uvc zscBf**g3Wf>=G0b7Qx|VWaZ=)c57;B@6plK+iPxNX=QC=>*P#uadmU|2n-4i2@MNB zbo|80Q>SCj#3r9lNli=7$jmCdSX5k6T2@|BSKrWhrK!2)>g_vs@3nW_f6zHFI5a#m z`snf4%U2VxC#R<0%)I?PH@~p>W$Ekk3S2L^o+L-C)}QOe0P2NAp~z5_aJ>-7P?&)M zMZQy-f>GU+(!rmJ7juk?StIGfMVW(WWS^1|d;$W~a`4JwQk0*5eUs>OU$ zs4!N8##T>={a4-}LTc1n{)V4b@7(07_M4TyslGZNdr|eCzBJ~nrAK9B#`Q-8sMzQ5Rk)FBP8>Kf@ z8!7O$Fd$*aZkPP0@$`XTue!h?S3-4@=Oke) z>}-4BTMo8#*2&RncfK9|?M77XktsO?L&dCp0X-~JyYJb)k2@>S`tWHtMbyCF+Cg{r zRnJ@=mQ@Y*ay!G<6Ygg}d{^->t>o}xJRU#YG<9w?=GJG`0~W5ycHvZxevUU$uMxL6 zxR(3yOc4g3nFm>8zu+k2qM2__T+ueP!tCm30U~GaeI~p6?E%~8=6CZqGq}fTnA)!k zJwbb0(=KqE`7bO-<+5$zew?LtGnvw-@x{`9na@noY-Sl>#!vffdl}o8eb=^YR{Thq zePDC<-TaaB$?8sTvP7qrK2ubWtxP(veN8$Z8D9Q!bg|t&%HzQM2cqAWCO-E_1hJ}| zDXz_(8f{#j-gCrgUV34dEBk7o+4Z(p9NLfZ?-~QbOamrVQ&pT5dDI6(%lo=j_O~w7 zW|dwWmY@(A`cT9*)ZGn)lK3-S!VM=h=aBrjy!-OP1*fYk%OuCq?LVH2|HP=Gm$(XvAmpY-9;L@!bqHhDoQkWAbP2PXO z&6PLBy;c7py)vS3GLxHkEOLKfyj7OzlnAG86VL2Wafl?{vPKQr7f$_`U|NLnGxV-zJyoxd-1aw5y5@Hd|Z7y3$_L z?(@&tyMNq{tA(zcl183u`9dKdL}Aa9ni3rte|TuW^#hBf;TDt-$LdMZV|-O63HhRV z{?Agm7}B&HLvkPWmA*98jM-)1`S?xujR|AqE*`O()`^qvmQ8METYYAE`ufX96oaNi zl(b(cp=EI;tM|Ogh%Z&V-6&0m;MQ~7?#x(fXEZUU=@x~whoJ66wt3tdtl?l3<&iG& zPY|09+-5|1pcwD0;;!7yajL1PFIX|EWj5pRnv9{_ z(S7C=<|*xLlTVXI%PX3*9GJ1;7_kAZWOWpGo7)+`V|vktT$7f~8Y?PN z8Ouxp1_JySS9?^JbPjtizfYd-o;%>~C;ddQw#0o-KRx?`tkq>EGWITKWEi5uxc5Rv z5;qxHLfia0q~|xd?~HglI$qy@uRuya=N_6!r>s0rmhvrETjTq8n)9cc1s;3klIb(Z zf46144eC?3TLwURo=UXQQ=<&Q<_QNi#yuue z`HBqYIX!Wu=X0iG{Zq{)9?U&EN8Qn6o0O<0!XQt_i+iihZ ztt01H)#RbYTZ>bT$c}h4ln@{s$90XXeP^O{ZpZ4E3+m5J=L_q?eGl@wb$v8#zB-)~ zTvfIz;8xT^el{wCEFsFt?j0=+GNyy! zyUBhZMZ5BS>bolISeQcZmyu_0t`=^|-3|oidfmeIZP8-TB7{T%@p94}AM@ zFm-Kr-7hE&SkSA-n4jp;Os`P#4*vwMd>%_yQw9mJV z=TvvqUb&Spt@8}gOmkyK>vGQ4i}7IKyLRd9gL%Tl_X=q5IwW4S`UV!Dx{k;_-KScr zuJep6gR`h2?8R)ylxXYRIyBSJP(VA;7qQ~MhyVCfrKL5N@12V+JWCG>jZZHLvNtC@ z7cxA=mS()#6!?`!hCA=@nt8x|<-y(?FAo^8DNeYM6=-0;4pJW7#po-3?oe_-l*mGY z_&PLwEmBnH{!6>>d`(O4V@^xP__|L{`>fUd+g;}W)h-oqgK(O75-eEbca~* z<8B_HJtfR^^g=t_zKRY`vRt?VSD@qj{BE;miM?C8MvjdKYtMat?!FE^)}rRVa>&DI zd`0l9W#&~A3cBzPb@dh#V>HcR=9ZKTO_o|>V>zb>rD-FhQ<*FRjK7>ewsyOmY@Wrh zH#>fuc15V~ov~fKR83^D+myy6L2#e?-sz#~TgnkBssrggL!m-;ul5Aor(El@(m!RI z_H^8bc`;84dZxC*pmisuZ%f$&!RdS_#70%NxA>Ud6LDJb-zLDu30`z-j3A{1Lxxj;z?#d$zkM$ovzX znq7EC3Y|}PlMAFU^lrKQ9lJ*=ppNSL*vD`8U;s3FRO{1P_ZS3{2!~ThiRlCh#Q=GlVeG7Q$q=y9d*CFfBJIAvH_1pPwxODAH1?98Yvru{> zSAE{6$SY69LT>fsjwvzfu#Va;z*+E7f~uM9Z9!zpKJtu_ZyJ}7x@xL-V^6GUnwJUQ z+V#q5ZNX#PT*c&vrGtiay?5)$RxDDTpL7W+_v$`a3))91%7ziX)BW}1-0_5g97{8G z3|6wP_N$rrZA9XC1y|LPq#C}Nxns{MIabR9cg|gY>C(5-PR2!3`*p8#tTx|Ljj?a4 z^lJGH_j68ereDoZy~SJ=4n{I>O(b!dN-@mize+0+DGrNO!Tk9lUdeJ-2I5jv<@rK396$5&btRvK~6$d#X4 zN^fFi$cjRp>3y!uNhMD1r!lyF_x?u~_i`>h*wW`%B@&-1 zm{~l=fWCE->dp%%?nOTgQs;bBjR3{jld(IGqQfJ;3*YBYNxr1pQ8-mdshsf2eMVw% zE{(f+S3RDZQ#+*DT>d7le-G8{=}DC9!&fEWG$vA9RNt$3$V9|5Ft0;(Hzu@?mBnFm zK6R*+q=aYmc5GR8$r4PKPzN{0mRFmV<|0eip;|S1d|OkVtIW)FQd7^Etka;f`&+$Z z4KkDiMf28+G?B5wZARLvB34@vu4@u`UrW**Ar#YBvuO4gn0ya=&Vx}R5VibE<7xfV z?coKxdL7pv zJt!i7Ul&!cdj`8ISC#*Mh_kKe#g`XOJh=kb%lMPBc7#57%k4+`TKBuqmTY#%urW80 zXQ?{&Pg@kd3%_(f|L&nH*mCQ+@3zrC^g8qgN8S}9*O_)p`4&%ax>HmrK0Yf)rZz#C zjnl!IKT0HYm0y~6(&)&s5;nYn>$j53Nv2h$Vf+ZAdWwC%<7u9+i;CaWPg}Def0Nnw z+8v>|yQxNXyQiAYOYO+UZBK5=+wI|aV|Z;HIw|tpzCPkoL_v_-zVMYKqZe{dcs>M` zG+stWw=x_zO#bqTh1#1+uu_qpawf~7ty#h%d6at~|5M>hXa11un!22xC$wk^a$an! z)?vB1mNTUNP9IC(Z?Pio-%sI}u zyWh8JF)tbWb$W8jGseUe?mJMDDVk}b> zS`=iV*Cv~E=o?*i=cF%+oAvJx>zi^zYV|#%We>S0#2>7hShh3V8Ko3(FFw07XU)LF z@Z_-_y>5sO<*0OA$DJ2S1H3C_O{xjDrW_w0uL>JDvAgn{kd1z4jkRC4c|5#HJ=ek(3;p#VjDBzz@mv21)DO1L_BLP%~)LMb!*o^J=19i{#|Tj*ga+w6tx#i7qyH0tTTUPqa3GWtPuOL5G*(5*;HOQ$44z@1U1&@ zX1o&Jl!mHdqtJcRyls&su|f#lb|VV^q0gotT{?QLxGZ$gYxyq$v#^-<9-fm}f66JL z$cxOTI|fps;Z!s!W6>1{k#c5 z4(Kp%FP}h#FlAn%as_~4vLr8>$P(nK%xeuc5!>wxX6|tkI0-3nuvuCNh8O4vaCT8J z)70M70Ak9#n?(%`4V4JRO85r2N`lQ9pI30; zuXzOr5?~UstX|G8lAxHxD@Njn_h$RS4;#$Y-P@1I4ReyXoSlB~_yq@e5!0QWBne&w zco(=ppi%k&<>X!<$@!{WM{%Dix{3eLVx?#>Du&_Tvo79)cfcaWFE zisNvOQsRyTX^gm|j0+xYU*P26gpnud(e()oa`15?zCvkacEI}ME@8m4&pU?IME*J1jV1F?e1yY(xch6J2FqjD*x?)Wpjn$i+9nTbb9u-6uF~lWOkn zP1qac0Gmi!28+Q-%gJEmWaKe2(x5m*HJk6HU{%Bo^LWwDP72yUkppNI?%ocr1W6A+ zf-AA|3cCXc4ne*F=DxmO%Dg|e+1p4na|m+xaaHD(f}8ke`&BUTbpm8xAEF?^d9(Xy zIJimzDEZG^e@@x!<{RqsZ=^{)Bw6)*4#-yB`9UPsdq{W>bo#n)F7#wIDcsZQ7l#`1z-bLDh;7E`m$p`wn1i`Nz)m=e@ z2bT&ocjAtSXdz<#ww z^KvHy{H!NY_HP+}5;t4VKMMU!C(7*h^$Q1G#VyD<)aUQB`U?k9&cNM?;1lTkw}ty9 zO%kr@FT*e>7Lf$Tb{PKnsZc@xcE2s~+XBBW@Y@2vE%4g{|Mywo=fft!2i&BEf=9{q zSIA4cyLTTjF*DZIHP8YJ&cR}u0xu`;Kor=)!`mk)z-*5O8mtXLQ%-_iT-d;ud^?~m z4o-o7>Lw-z8|yg!?)+tBW987GIIQdMIQ|7~t22101S?z6fJe>AF97VRgE$DVOlXiF zj57dc2KSoI0K>c2hy{QG0{jT3J8s}v7`qZNSls5b}eA6MRdk3mJk{`Dn-y3I;pF zc!9lo#6jo-D1qQB#66J4Kgh#7BXa`DU?AxQB%FYRCgclwgWV2bc?iA+fCsprwhaVN zYD5x(Rs(ED@oas49(;{Jc^-nkC$6u5yRg3gy$~#>8V4H^`}~ymEdU!rF9LevPa1zd zSU`6Ig09^CNpl42&YHj)F!slO4gn6teBgCgNEfhI!CWcWeAxzUa{Lf%VQsfzH;@LW zLl?oEm^rYO0odU=6YPt$%N6Lh|A%_R`y2eQ+rQ-5v>$N+7a0->{{e>rf8)Lnl7+1YWp8o zMZw7%sq34oi|EO~qCUjhiDx=O(MO5WrZi}H=Wui(RDh&G7oZ_H&Q5iZnE2}nTo06h zFrzdr+?yfr!4=F2{tCefmLWM5_^2etk)0HULli zu&`5;BY`y04hS_AEJc(?AsmF&ko3qHct?NGj>*9aI)I_ACO2W8O_#V4f5Tj6h5D@t=|CXtf^^xD|8wVj2gTtXgb8mfaa z2RC^@Wp09cL-cp0ASxj$c-OEWZa4kiu#QV)5IeeHqmU3VH25or%#5&Chxr;nVRVFm zZDJh3frJWpwbW>#ZQaJ*R;994YcV5gK$W^0q}{^JA3RzVcKV9-QDB8I`OA|1zDj}t zKv$vTHC|}@(G#pOqbDX4Fe>hc>U3E@TBU2|EWYKKnW$}anVZ+Wef>zz<<9Gzhk~E_ zCh%*U9=fr|TALx&Adg!zsr2H3{x%HQ=v=YyuGMF+5%~ z2imyP87uqFUkD3i7;hb~vtEiSxVB$csc}+eFPrF*g*n#By*&xlE#I~p>nahx%13(2 zcz*GAeO>%@_WAklhOv<(AG6uyP<{Qb%%Rp{ouV)4<4<0%81t6t7advZS^6%S(9al= zP~Y=u9ol`XUsAduc*?6aW-+iQDSL9l+EroF)|DO?D?W(rzr%;iJ``Fl zKA!cu6D$9I)K~*QFx4RcuEdD5hwUVncJ6X(#Jqf9NDmIHP%~iGeQD;+RQICzIiuJ~ z){lB$xKmosOg^46>mD02zh>odzmLMP>6*f1yY%3|Nd3*}GxblWpO23kr(e%|*Y!?L zx_vs|cSkMvK7v)(H>K-7%L`rOt(koh*ImYJBA&9RzZvz_IeIZ$Ha*{vLAQz4wQ2-v z^f_Cqbv5?smzbHZ_VcxYNgn$ew5{#WpJibQPFFe96B3j$tzG685Ret3!6BSGnR47( z!8etf#lg0V(lXdg>za3BJ?}G5p|<@)mhVoZdQ09}cb}~1XXCNz=wGZ}J1}}fFiMi5 zc`CdOLuFxda^LvV@arYM=gZ&J+P7wA1U;@gZ?iBRR`_LV`E8$MlW2S7`2<6&zM=MB zhbiCF`}Doje>A0@ z4w>rB!DS{7pQ_fUJ(bksEK+mi*>$(7MXiCyPfyIG@sFiR`s}z;G=FuzOVW5pXOBcm z%c%I|1J{H%uH}A`70oWo-`Xv%+fB`+YugGo_U%868>sWXy->c(c42PfR2$ZWFTr{= zyEeNik*2j~SU}URqj|CXQLWkZy=%9x%`03s=ry>lWrXi)deS!?<^SnM!8#=SMOAub zzbe^z)vhCl`#-r&r@B^}5p3^OjETRQOy6~=^;=_RzzoBc$Mw~&Sm|{ACzAd86PDUv z(<_#gR&9r$88|(7#_+f^NIxy-P^LxAi;$<2XEU3&A(!n3V1D9#})C+O1gHN>l+4jrC2oIV*@ZB}zVP!U}?ny+j4sm+Y5o-WG}yM4rGWS88Mj9GH0 zujIZwli7e*fiwO4a)r-Cl*ki&*p!Y~aF_VhM~r1;?YesXNUUprZ%fs}ht@MET09H| zdUCOcE{EQ!&3O_k;NMf8>f7f3pd^Vcv*DV{f&=vy3yfUzA^T=Q9^0lv_$rq}0#mxZ zhw9CMvS2<6e6QK4ytE^J?P=d{kF$ z?o{bdscq%T(y@OhfAxT2@z~e{)0h>J@wo5y@ygh#r$JjH5|h-2lFKlK1|=Cvlesjid(&g*Pwpr^ctNj17$S zNT2(hD{cC8rf*wjc+1Rb^`J%t?BV)@?Rrf%3h6Y@p4s=><+^-m@i`>Grxo01k@I$> zdlBColo;N`J2%ssIclJ9)-7l*@TSTB*`v3;txqT3e$;pt*xnnVcszVirf6!x>40zk z(6>AuvbaUnonML>8tQGuiVWHCNE!{B7S?#X+@=~nT}CVI=FEAmq{{H~G}wN-wAaBS zS4Z5#1G~qsC)h8yHa%nyHcOxJJy1M-!(_Ql#m;6Cdu06i1!b@L!8z{MR|@F^-S@&h zOH{5YeBqdT_j*x9(t71_NS{&TOP55WGlTi|t2^QwX0uk#)F|fDOb5@sAM3Nr_gT(& zRc_-S?eCI2z;kJNEnduX?%JI`!pgt$+#x9= z&}Y*VaU4KC>M>qBt5e0?s-?N)DZjd1h^B+d{T|vOY`M z>tlJY&E!m8&veho5hJ@-y;`JF`I>R$uUs{!>ZJ@U z>U&$4gisARbMqd)Ts@9*cWho))0|A}I+l8^xF>MrG(LCvhUCoTm;I3$d-9yBrXq_I zoGS^USli3H>I#_@25{DOgf6p}+pCfYyW=hhGzu$Zzn*HeHWYi38?5ExP&ssLM_=Dc z_@W1a>vno={MB#|Ld7!O>e`J{YX`2VRK`EeoVKegr{i}Pw|-H~V7S&a>@mM8RYL^8>i}Tu;1f1C-kSv{CgxMoD zNUfeIv_1I+{q90)-LPO0`_U2`JqZE(h-dQ)8KZqc(+iwe<(wFFDvD78x*`|N&C`l5 znKJmL7Srh#TcqH1Ey%Xh+$>j>`Vzhl8Gc?7v5X_1SlKVUkJ6NiLbqIiBMnNf#+j!G zxbvr&6r+w!=4H+)JDG$8 zP;yBkR4flfQt0(;mQU|jEeZb?ds0MpA^fh%`@0v-lNdG2#d<I~`Xr&y>Z&@T0D+p)CNeZ^U=8ttF!p*Q2=bk?j> z?C#crsxE#FS^f)jg+f|`9Oy`k@|7Al`m;*8VudI#IY#dcb3wI{0XB!n+Ga80iF{e~ zteCf$Mwc`kYk4YNU20LygFX>_VSO}jgG%U*m3_X7jQ8xzaH>NmoNpeLqUwM_iiGlO-bHAhk#DtglOMe12Y&)nHXUASO#>q5_~eMhu< zZ_D=>RZ5*7y3*?yqi0ubs;wQX?b()H%24=*YLv{+D7Nq%>uf-`r%vPQWPVFlgOevH z;5o5d3WVMDRJ~WXmYkJISB!T`bq_lKum;r=sz|t)D_8ZB|BJvys)4Jz3TNcwalylq z?k8K#0`zBU?Fr7c;<>Cy>7MqA+Y;9(#ckr!*}3#zPo`xHmNie?J=!BbXvTQ^9Zm1q zT?QUg&1cUo=ABP@QC~IoETuy>zC~a;Zh9f2u$ix-QBq;-GFDdT9z)^lEd61Z-Ev;{ zXXDOQx9itXA3h&^Ge*DHKIa{)+j+Y^>U%hHTCCIi>%-V~XPUNWo18Q-D(P=%FmKcE z*7P+ADAmUDh#)8cmrVzFjodK~ww-K-2oarq2^OH84n_Wb>> zhE==Mg#)e`3ixD6 zomQ%J#mx)vKFwkCR{!C4ON(s1bEcI;DQX&lwE+{^Z`p*JL~ONL7%rvDb2Duz)4t*u*fqp=w0YTk`hBt4RROBWM>VKL(Yy(p@>F$$gIR1EqGIY|78)lS#rXD` z4Fr~#(skEFJe_#hJAOToJEtkQ-OZ+sPi}fJ6a6NAhtfo4puttvyaHp{tgzw-b=97C3sjWLNNV&E*yC;JUX42g7re zagDT=LJ&$(VJT}wEa#a^%04|m!87F_KJL)Jhi`3LzN+6Iq<^BOC$P%L{FYGl8sOx6&Urk1YGq=$J) zav~(6Pp*^7Us8AFM2NfvISpiXla0Be%+pLnj|NkB(xyPqY(Q_`eoPmqSFyLV*36RH ztl_;6V>;@hDYB07zM()?4|?l;Ax|{!Bw%F`c4tWKt+%a=xy+G<)zvd4=-C8xQY12X z=$&-HWl9ud_Qy2FW{cQpw7$zXa_=msf~a*1tpXXr_&R&vC0J)NDJ#xTd0S6 z)3SJI&U1PqLayYHhN*|Z%;W2a$1f_qJFC#4$e>r%=+`ZFqLC)mx+id-d|=(nnB=3! z^2M)5o(i6oNSrTSef=?#2XQ2!;D9xSyv~gN8|_N z6&e=&FITu&YVQ<#q-#jcOeg)mb5d82Ip}fC<>HLg>>hl#Zofo%u2GPUr5u}ZIsLt@ zSVeK;mPxz3tBJiGmY-!N~T2>-jR3{ifUm|9k_;Rf&SI3bwGcn%i(X9^2Pfiu0YyzSg@w7I*&(4@hZ9kJ`Xy&&k zQ?Pn}&KQe(iuo>?cg)c-)$Ltr#bpB}t!ERm&RJ=G+12K8pZAS^W9Gfbg|9MfrY0wZ zf;-Cb+jH5J1MNDKr&3Q?BvuQ1m4_&;WsKRHM-T8)MBmoRYP2!23+QK)#K-3t@_HTD z#d(;C6!%sz>NaS(vX~d*-o&Q&)g%{N$`?LhWYogilrrd>YI554ITtQ_J+XfiHyLDe zC;fcYcmC(5t=HJZJoq1k?M0>bVCxI*_MHzHQgBR-$-0M?DdT@+c1HUwnfU{Wpycx1 zCIj0qVDQ447j1iM?m9W!R5k4EqT0^bVt>z^;L<;wyA#~4;Ds8qQg5EDcEHx#=hkKJ z>Y?G)E`wfolHNy#P>ZZr32Iy_HEndq|U zyLV%Q!wt1E#rLRDo0-KFPqhpOr@I&El=Djn-%qt`k7Z&pZRMX*+P+*FIW0Sy%Eu^x zF4q>-WfAJP-9n9TxTKYtrGNQEL|SPJ_wD3{#`dXVcMntXVpP>upP+N|0X0YPseBkq z8x4ysh7->)s|8NeDc7iyTWXpA_ISjL+j7v0sBHff2* zi0nhw>2o%Fc<$)hqja@jvo7;-QQ@#~T6thykyei=e?`Vt%S4;#8f?X>@}!8h*2gBy zRc>h~pmM$eARMCzhK0*9yP08-}Hg`w|A3;Jqj3#dlUi!N6Nij1=Cw*Ck#@` zwX$B$^md|Hb@eKamR#1RLYxeq)H_qi;B288e@kP_33EJx{g@klmRMra-jsS>O^Df%ku^E09G%jL!uL+a+7G;M zp&!aVE1{=*wOmT!93J2PYKloGJ0v}?vA3Zm#`0{=?f#ncHV>*Vm$T6rox#0eMivJq2$4tx3V84jBf%6in2?3Y zPz;RZOw2ne=vjEB)hSugd>DQ<0gYW!ST&qXBABbC0`E=`(DstZaeK{dR+FRePqfc~SYuq0_eQGgn?N&KkTcQ87;PZTWB*+~`a~J+Ub4x>7`NK{Aam>v8(Tjr7un%5#>EF z_blCP9f~|SUpd*f>tUqvz;4y96)ji3Y2{rp>(J2t_@hx~hr&g_>$OXQ)#ha!;^)tN z*LJBdi>>&l%sIfGtxIYi|Ye6+1zj8sDU$00lsT^@PDjBYeA3k-k{9EnUZ|l$&A3Y|)$u>R~iM5Nb zI5NNQe(TFN&E?&Dz_PdJene^A6S>rT5AK&g5!^MFFDc-DcRA08Le_Zgf~rzf^1^J9 z-Md2r_b!p$D2{I!o#V-|Nt~X{wQgVLw%dEazPD%{3JA8_MVar`v5bGp!*!!PV_a;l z=vjZ{;h~FBW>Gsp?QBMKRI?`+<_FiDn?D}toA%?S3~wv=FcJL+R(g?+@G%Br=M zw`sfQTW4b_?XS!bz2zDAH0zN)V}L%NB|)TWe5Qs?Ao??lOkM}Rv4tCZJ@kmk zW$Cy~mzF2vJPTJwEu2msCAa?g+;X{(rflt^lMR@&%?)0PxLEN7qcrA!4l(I-nS?Iz*G=67js;b>jQGa%DL-g2VHu-gics zpwjYp7mXUDtWN4?%v=GxnO?29(uH8+e#hd8*Z0yVvy5#!ahUN$&33y?%PLU?TA%v- zGu}6NJKHcTdD`aBhs_^7l`W#NN||gKS`l&+Yvg^Vb>KWchgx<{KgQ;b^nI@wPod7# z#9obw5z9WtW@%4y={J&Ns7IcKiume|lqJakNyUc;w*|C(J^yu<`O!L*E4I_toOR@L z!Oddigkw?&OHkIQp;MYQOzLMWwkVocR9arfwuMKw4xP!V;2z6+7KFo@-k&g^F*aKA z5onj$VWlWa8B$Pkdn7P=H7nJp)6x4ujf~-nrHGMB;iJ&|w!(HMiLb9zhF-Z=9QJf_ zf1J~$+lPwdEL{0csU3_VA9kmcl~{CWpAFJi=M0lDeB<5Zl_t?5>F~gfZc4nptgq(= z%Sm~oxZ~RUCW@Vul^wJE_UPSiZz{c^b-U&U85`H!p&A)!RdhQmPtt(k68=zcXRRpu zN!>YvX117mJXOn$T_w@?Pv(3au+vrXMus=^$z)EO?kE#w?Ef$=|ILa{;<80rF}jSY z(aGq3O-TE-apsUOS9ZpX2|HoMds@$OS9~iypXd3;x`j;Ncjv2qk9SwJ#0KkF_9qUu zPg02QZCCu+^sb5sOl9+JmAwrTATK)x~U{VbxhL-mtR@STeS%xyGJ8kDk7J z`y5}T*w-1+^tFofwNq9{hJs7mZ(pmJbUu6N{ z4*mXM*X*ca$?%nm?zRUW(l(wfT%V*QtNjyh>di4l5+V+WwotsQ(A$(@H{S{})wrYgj zMf!#0q#tSDTU>|N6J+Y~sH5?ba$4eZ2${@0w~veC8>d}cJ8c)2MV`N#y-)8)y2nm8 zkB=#`$Y-?4H8hbOx3RNe_G;YU%4V*Y6(%FsaZ5C~Ipk@Q3#E6nmk35%A8p4iqj9uJ z{F3O=#j#rqm(SWgQ_OF1WW6lJYguhieqZQr?EPp&OLgy#t=1>aJaxv$Au6n zx~if-D%O^indubidh9gU1NJn%Bul)#%k|TQ{Ku~*?`f{CvWB66buNp^>;ih5YxUO zWGd4b_x94^xa(45g4Znsnj~4>y?3&mrFZw_ow$@IhxIgdReXAWE*q=u;kozpl}f+i zW$`WYUT&ve4c~roPh>FfC1%z?D^hXj?VCr|-zDbK69>;xcZRfGH#}$AC16!|E3wdf zpPi$bd`CqhQk2I1a9raj^NM$8`+fNg-?2zGq+V~19f|Hp_kTwB=G+yQDf0jy-iLb6 zP4o1SiXQrVYux4ek!7u#6wMgX_w$vpcIpD6qPG*RwsB>5v};ug2t^PMS6r|8zBM6K z#*X*Z{^3SepKgoV9Xt%r`9#uuO1`$bdJP{55*QBhIkse=@5Y>7>UWL4=6ZrCDtnL;G49vpk?l?>bp`GZwDB zd+5DAqxHkZJed>wUBtaQ8{6t{-nTjJ6PMAvOdm8|7&5#2%;&dP8)YK6U#G6}VUO&A z7N*U|E4dzCQ+%|1+IrTDb*L-hAx7-me!j=Sl@3nNcV7E{XnG5PIC|${nBr0>Uff-Z zyE}&*E(gUAcXw!UcPQ>~xE)^H-Q9{?fdYj>v3~db{_i)*WwY7a&F*B9ooq&OLpJb{ z4@AC(;#+}7d0vIl)bpyQ(7#-RHf2Q2Ns3D47nGUp_D1=KhYv`}LMrStC=T~XLagI| zHW%J(F$;eEP+`^(ZZb>ma9kd-dhVXsm{7L=d4Bg?i{ABN)AB}Te7t>{!!qR=ul&VTylQI~jN zuuZoaY|?sxmAvTmoxQR;#%h1*ELllJFSN}m3!rcYyiD{R{rR5#yxZ!+EYW(C2(sZ= zV72dXwh;+`luyukA(SsNgDK+NP4QXGFB{pb4NC+r5WnL+zh5ttiOs&%&?+Yab|IXN zk=K`um0VB32B`@O_c%Ugz5_jQ0DawiC+$DZbcI8>)PbcnkMN9MHy) zub73m`1~tFi;Vj90SC9@!o(Y(&3!?aeDR|^Zb*Uc2!<}V9(>Jyv0<`)!hpxm)PX~M z&|gS6$QeiFMlzHn|Ci1w1ty=VMfoQ$LRwFhGf%a)1@^YGRa4`19WM?Q#5R?Bue*)x zz__S9j71MzwKWW($MviAn;pDot3lF(ZRnqH)<1sDdAJ>akhz56)1SHFgaxgidEA`B ztKNpS+YWvyI;B5SM14!VJbUfbq=;QJ$+VBRxLVXhb{K4mk#&V;1=obPSGh7nb!#0 zLSxApiuPgScOD8KymUcsHjU)pvxoFlocMVSY89i+vR|U$y;hCf8uwi=c>10n5Cm@` z!gB*%a;a2-cwu&f2^)Q#;rt@zUKw7^nCNEp?roRnSlwUiDJ3l`A}JT=35!t@gx;Hm zsr$wmvn_w87?0J0cLg{j`M!rEZeaBapxW<3s`AM-sA~Id$c5C@rKh<>7)k1AL}v2v zq4*tVC6^Zp5g`GOR%9Js7+>@T_R4#^$WI%<1>k@hNyovesNK|!eLlUkPqZj6&mdr* z?Rp3nbbX9pn0GOA+55v}ugCIfrJ{Z!(^m3=LMLP}Iq3i*n8H=^PB(z7(a%A)COZ)AakzizAvomHw}jo!B#Va9gj<_HxmeQtiXu(y&}N?-dbSe`@jm& z8G|ctQed0Lo5Y(0+2+&MCs{)b+zZBJk+7Mo!tZInFfsb$&0cD#1+*te?uHAN`JEk3vpnLc98`3-^|!B( zg1&@wZ2Mk+PShOupz`*{Ix8MaSM>JAY-%Lm zm`9D4f-KkeXdl%t1AX?psoHmsQ{fEcp1rT4FU%*VA>5o0Z!8Q~MsIBF$DETm^V~q> za*8RKlKhv&dWt-Do1&X4o94yO?-(6b1eYDOV<}saY_fa*!CheJgDSSHg{x6maV)I% z8g5;(`Q=3j9o#W2~QIq3g=UM&4x!9sfQl$b}5Q(|WMA%E%_j=Byo@7|v0 z{K-1s;88$6)EL~>kP`R}oSk4Nx`w%ue@Ol^(x%ZnI7MG3qSSVX%~u14^qb zjJj|q88VYA(T-P+#QxQkBWmNqE%vK%Dn;ks*_eZh6c%G^X$`OL?~A)l6a}=8AJU$k z_Rgy7TlE^}&$v3pv@X+XR75R40IMWaYGzw)u<{G@3u3H8btBjTNCDPw;?D&7AK}|w z_(@UETrNTDjgWW`19rSdd*JrshmO|;gJCCUq_*g7---3N=wVrZ*9n>hNrm`$+q@p^ z!&T)TIfCq_#%&ptTnGA$ZFcXPrtewb!#{je#Bh|%-&EGZ9_aKM+o;L5Wl?)MSdbH8mQ$01I8{f7m3NXmF_Av=zBNWD*CGH2q=A zB_uQ0Y9&`=?HiTH&sR+#%k}0oKL;faRRcYNn>@%gK`Q=+dD+p6U$=TV_zk%n5XGKM z_-Td03AMjku#d1FR`T@p8`X_nd(=E%g>0J3)rRKvVAJr zpeYn9!gR2Bh$=lIJUfSd>*~7X>Q`Ldpm*w*%Ny5`F&?t}D$bmXh+a-+(Ned>7B;B3 zp;;~X1m1%KC5a^aw}c^+*oSjX8QgS;=&7q*5;Xmhxa8^4&fUo1HxD<4C=;IZ6~F$( z<Ak5^*qN3&(rx1yS;nF(t|MQOh2ZI?x4MDke@}Fk~@B_~nalC`m^AU2je8FDS&4 zwc12;J!9-Foetah&j&n16laaf)jag9k4lgb6BbWnm;F1IXBh=Yew!6fejMJqu16UF zi?VCBt(jX|3RVrA1Lgx0{PM<3;xH#j@qwlTJKX~Uf;AKn4pqy-@8BfN=jCvvEqe1e zG7hfBOKMKGADc^RgiGOmDFi`g!8L!=zsjK$9-$mZgB*ZJAIS&!{mry$SE+Y24#k!l2ea0Z~dyz@ecr~h7s+pHt zF<%7LZ)Jn$_Stva<_N9{Z9E#urt&?6T&jM@B#e%F5>Gw)gia%?hfwiwp9Er@v@yU#gLOx1dwbMg}wcRoS?fV+(7 zMD#H#KV}A~IXuJB)HAZ#ya{dAWuurh>}qBgd+WQQfj802LE{SqGk+MIP3S_FUudn| z)cTh(ek+t15|=bbh~J>f`%?HpY&KYJb@5&ZmCdNK94GF#XY_MUlUp$4h}(ptEwW&x zL)^0(cax1+KQWzUs71OgX45y&*5~k&t#XH|wec_UR^D*Vk43Gh5LaXp9f*0mOXJ2T zLgD^c$-EZq#JaGheMX|!xu=5q`d??2m6ao|;nu|UW$X9{vnqmq2fSA0y?w7@v+j=k zzf(tgwYq=y4ESPxq&`WR19#o(1HYQS9q}U|m02nAtNLjBcwnH0%*Z6?wg?3ZMo+lq zLg;yxOd!82TOz|2IQOXm1KbW?%}RDGt((dsUnKPh0^i(LE@MD7s=V`0wnd23H@Vpk zGhm>5?lGgFzUsOA18i70}(vZRtd4Ml1J3D0(W;xW$LJan`i4l@QY8VAle8?e_o@AJ*KjVw3qB&7!5wGj!wd69|MA_#x(4=t|6JOb% zooh3%5m~w1*|ZwU>yDcWWsRZ1#6hF-8u2I~`P86|PJtv^z8t#gOh!(Izm!GyD9nE) zYsF-2a2zp(Bmw#V;0CQQ`>*&<+CRTbzI95E+7_uZ^RyW2Tl9~c{1}}Jp|QFbQk_xl z9+{{ln$>b?M9Dbri%&M7xDU+kFHpU#5;%_a)&;&zvsGmZ*MeE&Ce(%x$l-Y z-M5&D-Nl&3REx}KadttuI+YTTPadm&mA9E6<1wtzMYh%y(V(+++kchGd!Ai z$Ne4WB&dYseC3l95Aa^S6L_g^1jrk~bC$3$RlOjmi6xvw+|rUTkpvzXREt)9{TPwE z9@TOjo!cUDimCd_*D<7e<(CgR5F@UTv`w884l)dOyxU#t#zR3Hma?wkRf97vMP{+e zc3)cQiTw-YiCcPs9HE(!M`d`@DQ;WG=&W$R6LYry<<-B^}amRW@2IqcC4?&XIW-sy$ zGC}rDzL>OE6`t7XU}g#YvE|tdZZ0F4i#DmSzkoZzm!=|1dOi#Elc4A)${~tn)&s$_ zmb>|XaEvGKd?$iVI<9PH@FL3`U)lIsF`O*;->rrIEUGuzYka4JvUSt%G?TCe-22*g z$I5p~Y3+X^&^}R1i3^D=)1fRWbxQLOw(0uN24=o-(`1m>dFPh{zir$?TeFO!Ra7`T zN;^AB-I?wR&D*V86(^MW&;k~H21-gecOj3C$=-6`>>uAcU0dmvkF}^TUOpzG0EXc1 zYM9mpc}Gji9I&#=p^%#2!U4nt#D}5+VC)QVvtg3~;Y2u;zbEL+biv;N68HBH65;Ga zVDPOWTYeln&M)>l$?2~GXu`*)`90a;sJVQ1YdKGL{@M8_NSsXibFGOy$afApPah!X z@6hIs&YxjRy9#n~6wJ}!~6Tyykd5=EuOGIO1ofGsiwX*`rW>u80* zscw8nv8&cdIB~}II2Wj^Hsi{p_R+;He^=6!Bs&+_o^@q2J0@@bfrluSkFT=hxu6Mq zns{tRefss?le1Qi1MG{foehv@iB&|!+I|xY?#BT83*_fn5h}vjbN)%k7;LCNs+=H2 z{s%`yWS_^ak{yIpMc0qzCX9Ufc(3Lb!_`FXw!7r=#UU2fP1crS%OS49mkkn?v#ah}9U`%H z=b3KWQc3Rx2JL~LA~ zpE(idE^7$OJ0SSAb`oC;e(D<`UPX@qZH07$!p`0Adjnbogy4KtpL4(Dt_0J!5YMD; z%`F~OhO}w?1VqZNNxd69Tg23GU3!c%aymrng`UvqyxPR|plT3ovM9^#F7HFKS}Ywd%!wZ*1# zxVWDG@K-hO#&$4m*Q_RMGp}PE+2;&PSV|hWFv&4m+BVrK>00pR@wJ=7(%O>IQkECE zQdqB*Shp6@C~@dVD+$y}lh=t)qG~NBPu_py#5RMmBCAyJLpF#HeE7sycJ>-+mG<^> zNW@mhJ;gYJ+!Kn8eC@+uNO`u7>WiL&_O^KPArhv8{0t9hc&34yu6JSDb>7+pR;Y zxi1CvSGMmB`ruu+UV$QU)H3jP_GcGT@0ksizEmJP-g<|W$>aHFN8rDgBw2cpkj2A_ z)2We%8#aayI(epIlk&o0KN7fSaF}(u7`(y+QVk%I4ILiZ{F=*oB8Dw*+Xt%#dWz&f zueyD(#(6_j%?$o*6-PIt-pC{G|McsumGsa7l7zAwoXE$d*G^Vzl_8?ec$hjf&Osh%TUACJv}2SY1QrIx0YKzWePBnadC1!dkHF{LAwf?dY#c)or@oQ&d!eTr`)wZ zB`#?Y&wl8ZHhuEXVLTgf)e?kLt58Ai7a|^emB?tXm_8Rc7`)BKhOJod5sc12Soo(F zDIv^`17&?P0p%~PMv#v7LHDNbH2p&Q3@1mmd3{z7Vu2qHKE6TD-L`oCKA}}07e*$8 zq8cg8smXJ@=X&z-@%=QBlTfGr6Kn!B*}Edh$%jm50tvR&DL$w_*pZd|(ZhN7HA9_c+5Q=_I+?uzoO!E(J z{56;dX37uYX%J?p?I`#q+4`0?sHb=0ZS>6gIyiXrn(?;x;U#e+IPzNJZ)xx|_VDY6$G2dZqz1=-M#Eup8vluFz@z+6lM3e@HZB!4JOUmk z7mbvp2BMiO_g8#cX&H6X{}a-HY2rz|`v-UFAUiHFco8m!4AIoydcT4)Ob*%>PUH6jz86*t zP?Ir!kk=L8EYP+W}m z8e2y{3k!1qDs%CdD^BCw#J}#ic)L%qeHr5-8iksm>o`g&abt-w3Nt&sN@Khc<`K z?oNIPF@Do3ntT!tIH<QL6LN~L@$cc*j*ahY<|h!#9t{#{=oqzUp0e2BwY2QH&zwF zkl)xsT3pPYvQ9wkrOeMYzcP=5G0|%U{gIp$p_nY_39}sLJsCY0OMB>g>V6IwjC8`+ zh-|n{`~AloFN>ZmkA#oN%3t4a2`va24s514x?EZ80aU=$RH%_>y^D> zOA#D*7zbyJJLEMD2!pTXC#{sRwlc;_FPp|0Av7sIQ|K|^w&YJ(;-j~z-_VY3LuO1Y z!>)z5IuuZbPVa?O-UoO2Z5{j7Va0ECLM3GH+5^daMo~576!@aaGz##fV_3NME5c1Y zG?|OVG!(-A1 zCI~O#Q8KZAl`K{^&F3@#Atm2|P#hW>E;GoR7RJ~@TlEbL4Czz!L9jM7q;L8!jn*w#p$65%xO}N}1W3+1j(Sv%w-;!Md%(+az79GPqSW#;#P8nV-#= zoB4??i~Z1$=1`_u=zW2Uuo16C|L%()g~6cqa`4Z(s`C*cvIOPQPUX@b<&!@q^e?@Z zF9|2K9b;F3t`z~X0E&e$7kFCcGRB+}*TW8PppY;Ws-`GEER z+wLZpfT(peP_?Ct$9)M1m8$nSUXkW^&pMr8S0mRS;fk{$`Y)jz^5v7~T3*hk3{LHum&r)VurTWE)O{ub54-H*P;C$CYn$gP0=_UEjUWtSmzTtA&)s#Xhq0bQ zYKi!WAQ=g-%?he~IQ&<_48n}DFb?F3E=3(*uK7en=e|+eVnWh!jDav}#nNZ&cvQIt z1AToJS;Br%)8_vl8eq`e+|bYhBlQd^Gc#LT>`4p^3_4ipe+OEX>6(CBc^x8dUNh_e zk$(mNoY%pYGR~GV@ZWD>-mUx}oIET}&*K4#i?PFHS=hvHexx1-%%&|MX*>MIaqSTE z=M@yEi8-}`X2Ng{Nr^bPNmn6A-rq<=lucS6mR^ULM5|wdGYT z(t}PYF}LK%hy(%!Rt~?zj>t8whsl?!F1!hp<8~0&C|46nU<`T^B*(q6B2F zVOUEQcB0__$m*N5fRwyEk`v&-{zqmZQ6qLd>m$sDhmS*S-&a-5`wtG^^~jjRQ@V=K zxXUR@{zWM!g)Q^+R@=9nDO5h5gk1d}TotX=5H+|Gx`X%V-~m(KiBVB(vY?=dKN2h= zECN8SrW0Hq_S_t{^>-}>Bzng@9%2ZvtHO7qZdA3{am&M4{v+@lq65s`FzQEL(s3V? zbhF+SibiPx1PXDe=dv*dGui8hZ>IshvcWz;tT?NNxp4U6evLQ_Vwce;3WxQ^$F&h2 zU%~Q3ZLs1w>`4VmfirMCU6XkhH0#s%)%a?^TR( zKJcPGeMH4R%C2Jk8%EWjIU%Z!=Yx@tk(h>#jwSEIu09JJLlx&kV%L(@%jWpF_(wVk ztkY4GAg`!l(+2b6@q^n^pJflx`#sMf#=ZNaWzS=s`%r<%dISdRJ7Z%7HUq)QajhVv zlVx4jkrxt3J=usKUcV2yE~&stj69B7*9qzbE1lLwuEqMDyoAm(<2u3fgxL5baVfSO z4fOQ6MpG>HBDZI(#tg<&aQz=o0kbRcA76p+A5-xT2Ob-jnhF7r3z1WbMpDB}9f{i& z##Qi0%OoZLPxKS!qVYej;));eO#F}s@FlQ)e2%7@CZ?sOvj%LH&ZKrI|K(Q6QQ|2b z$Bj#d0fW5F)L4p2;h3jX?y0A+oc1#%#a%Ig{^px4ICa*dgoCDxgYf+RKs%38Lqb(H z3wNAuJVkMn{tWo z^`hEt=x`Aw;0@JktYV!o!(nZl?kT=N<9+{|V(3rt5{t_EP#zv0TI$YP2APVqp?SJg zTRPWUmK2dFopHJ!vKt(`sq%`qHuln9DpunmJ!RUJGIcGQni@-$t(JyWbS0HWIx?0z zOUGHvXUiIEHUHq^HU7a7zbP>fWKxC7c!gUdHl_dFzgkMj1S^x8Zx@$Pj>m|nBAUzX zZ#H8)WLUw5;B0c6tZNIcM^Rtd1#qOMRUEjV2B%mhi!KJal1rXxBo$wpYN)SoX!~Kn znKF4lUL4yqBLcQwR(f`~v)1XK>~RJe;MAgX`?_E*nO;V0xWCPpt}&5Ptihx=PE3qv z(>q)`p7INZjS_UZ1%;A3{DT7ZpMKdkJ|60wfs}^wE3;Z>=Xdw}V?DCTwmoc-j<|rN zD%mU^2Jl5~2`!x^3^C(b+;kqBgfe>3a=Z%wrW61-UJlqBdwomsO7UDIZft7GvosK_ zqc*-+t9h6{O1&aLix$zk_UlIFU-r0G^5#Bgd#*R;&%gq^+J0 zB$_n7>+oV-D~R|-Cxy|!fFZLFpRlQcJbW9@EjdfjrPQpE}f!;=@P+_I=B;#xoR zs$i9JaqG9rHHC7u3irxX=9oinr4GMp%9YilM!Y1^NkvtX(elQP)8!~u3D>vG~B&bt-QSR@$6Dr~RK+mcIgKM>{s z__wpoKa0i?6p_&5Fwn?SmkNuk8lV!*6|W`x93bM@lB-(@uOB5un43?@wm2H7Oig9= z=Ge>^1DJnp~6BmY;@c!l0uMYV2e=ZchFSk21Bi8E7p~{tyJz_m28~3+tO8wY_nTye~&`s zjFi%u&OL;UD9bMMA&Xuq=jw0Yd>n?8q-lO!Y!IhPJEWMr-=K$*WW66wQrI0%o13VN zPq_%!P|g{&C>{Aglgld%LTpk6rLeTCiBayRI3v2;GFn;}t3=DUHwtpoO|p@2FI^=BH(H^*Jmx7TdsbtiILgNUVbr?pOrhMaUR+JTqGVCCCm`*q zr^fM7xZCb&mSsEF;3H}e3|r81>ZCVCQirGblySObu^V;8ECwridHLuk@YK|;F&beRlp15C$ zgW_O(f=LPeBe%)drMTo2aM`sIT^bWTC4k8x%_avxH%^zOd-@*L;YG&1Fez!uqC&g2QEANvl`F{Dz2v%_VucSJtMrLm&&KzLtl9&CxL`AmlebN{!A#L40Y02MNAZ!oA`Z%40VoxPqRzp1nL3Ih`z+?_2MOf zE=9F8XBN{?J{~(tmIenkO!|M-m*AVnX?*?vVN=8!RxPaa&8%;7JyHE%_6K>+Nw1r) zTAL_1#gT5q`@%l^S@qVKLtvgMO^F`&KVm46udfs4wV6i^vq`g=FOiQb4YLW8rV1e_ z!ZwfM;-U)0!+o##6h(7w6K+E@Zv$&Xnz=EUzaHf2=jvx4)&2h!9g&$Cpi{&#mF zI5E^j|GPKTFmv-LnhmO<5Q4&C_XKRIk0ChGuzJD=2+FUeKCVNpE^h|JX9L3uD^PDK&_TpmgB4srmoPrin{< zt-xTV)_4-W5#RB*OSc)!{!v$ zaXQ#+o{aL3Avi_(NWxLV6%kvSDzWVvK=+&#;2ta0#7;4UWAldfAJOyY;|-AVcmJg; zaI@PlIeb!m!5+ygiKE3kf1%Xq^%L9DKeM0xPIhC()}Q$%U(dfzx1mbHYoC?^BNhN% z>WCvUvP6@FRWJYY;c{=k1eD{)j_Qj`h#+|U<+o~L5WvQN?_&u2$4DD~hxHz=0ZI-D z|5R>=@ur~pXA1Fk+~DM-@dHX%30u0G0&_?HG`REXf~8 zu7N~cJ}!U`=WgN9AB0qP&VO9djTRVEteCU(LcDMTC_+BAP0 zI_%dd+OQ1(UB1=;-Ei^%Ds8U6VHeLV@gMVR^M7zOYQHaL(VE%{%zj>aUd(PQ|FI5j zzRpoCU!1_)PU@b9sfhHp{~)k=CWm$3gS+P(#t4c2(YD^^ZOC?l3rn!ohu!|n;o}|o z;-{``z87s{4#FR(4=0yYtCWsQ50MPdOwe=_mjN8+Kxd?~Z zl0rv9LqtJFK!tg8gZXnqz@f(FLd2%wmeMeD{hEx&Q#8LvC9Mf_qnFaykAzRlOXnV% zT0F48DWhd^eM8^0PoS>t5mvHzD=C`<{eP-J$PzGBpr=3zH0WF_1_g8JH*eOiSlC+= zSw20-K;jBHK6uUaqMyyOA=MGOQ04?DQ3bjNKH4D)cp%kBtrQHdzzp0!5(6=Jv}*U} z z_!~r|#rf?YTqNR#8+6W0gj>O{j%bX*4N6K^&3K6lpPE<29C#0LSHg`uh29bA-0b|4 zmDcWWsP4wu3A+}|qb#LBJA{0}5EMg@Sx-B%M(aaBZg;g_lXl*w{w5+76?vKzReBrq ziPCz|;~(4#Pm;$XQJ6GL$LTGjPg znAB_v`zrfRLfYa(>B+pPpi~x5OC{J0n6No%k)zZ|-GHeAN#&8MC2T_v+2=(KB}9Dp zc628lu1T7R^c5cUm4^In?rp4xb`vvn#x+SdFH$CG3}oG?jMQH|?ESX`;XEpA3|oMI zo@~x$03@-@_bzNNF-oe7nD4u6atn5+wIH)hR&?#~N0|h(@2a76YbJi~o6P#|6X8!5 z8}h1TPKe|Dq!#>CANei|?4S{*m+W(=yrMA?$7bsN-Z?lc%HDO!_4ez5rV}IX>;P1jk_Ol*W1 zp@|+=(h1U&Ao|b_6IRQpWB@umKX#iG+(UgGJ=>O>x9BV7~EBnC~Tz?t9mdEB;c< zbwLz27s8r{P1jNFGKr9E{O!ETTKPc?hCrG;E*g6(96}b}G!h2OM#Y~C*_66%M!FGg zrd2rzlAFF#&?e7}>Rgkv4v@#g29TVTydN(uP#uh4gM5Mh=<-nb4u>y^sg>sQ)jLhg zD!CQ;A-V&ujh4y{YF;=pwbD={!l^q@aT50DUz%P?_Se?(!k-TTT=cA&4)p_JRGdEm z(>nV>jk>BSV$3OcIl1E4mEL@k1X+sc3(<9g2s^(7xv4IIIGlpmCe(YuXD!Fh9bopthChrs=M)~kg(?~Uw&)Ct5 z|0U@!f-Sg)Nw==m)E0Z(sQ6QF3@T1orkav7O-b7PQOd`|`Gwg59oN#Wma~~*UM>RJ zW5^ekiF#Q5ERCXSzLM^v?;OBtKH{>lWGPxT=#eom)#@3Ip)pNVHW+Wb$}TcDz2Hva zqT0g=omrpFZX-pFF)C{<&0ST9JEc1or=}fin%8ln%4xCSa`B2a0M!!1k85c{se9}^ zw3?_}c47#N4KO=+v5is-S+!hP`U9Ge(rLh+2%NgAjUB^+!WwX$-o<+!rFB{+F}Eu_ zV0OF?!HGc+es7lLow%jDbI_`JKuh&`QzE^4`iN2u2%10af_`=diW#%i$WdJT>gp~H zcSNq6UfxXn)8jCjJdCArOJ`xf`~m;5gR_wtqj^f|P&*ZrLA##@nv(3W zeFh-S?i^a}7zvS`xI@tH1khU1y{3O7ZeH0GFPkAs z+i-5liE87RuPSY7IplmAIz7gAEqE9SraSR?Uzn#&G#xQ!r5Cn)a)drc!`)Vsk!sk= zBJJ@2Q6Ltt#M5eFpA;1Fi~N?#f{mlSL~vVBCTotuSJp{hrmf~yoU^YCjwuDWu-07s zUu2nLy-%vEOM%KQXgkF4R>-GjvABV#t=2Kvr?d0wq?D%aEt&Hk(lZYoBj9*v?tb#2 zuH*4k<>asAVLBdJt-3{E1!a-Woo<>c{nS!xw)Kce{}jg;kt#rCDANsU5oV5Oiq5Xd z3S$>ZdAoK3EfUpo<0}a0+p;njTTJ_Cy$|nHuChk5>a*RdI$ZxqIAJNm-n{O|w@2@m zInGk0L&{Js1K9Ae8#Zm~-D)GjUF7$>Jl52wh-%fcCoL{JJxtL5)x zSkGMOxf;EnLEbZQ+DJ2!@3AJ+5pqlEL$@9=NA9uxHfkL|9XT`^IZOvu6?AL1N zvBcB55u^CE7jMd<*@J$oVsF^`s@lsXZSK^5r4}Zx{HcC0>G#~eQ zNAc~G9$Tgg0tmzo?J)4oKhd$Axi>3ZpL3SSCnh=&r~d;ie<%qHEI$qZMf~19ZnL@m zyBW$a?0|XP6U^*X^MOyCMx}Av;nq{^L@nmaaq#G!cA8abKP#0RjmKI0b-1-Xb;$YD z6_VxIKWm??)n)47YQnGFEn2IdYm(`H!7m%JQIbfc@G?obZ{ zq|Ib82PDH&D+n8khNjqr?)(AWIcKkIquL3EzvxtIXhfz=(T14i4U22>crrD`MHzut z07#U@)6?C^I8p{B42&4`i@4?Y)M&rAvQJbyOJn0^R!w4#q~!Q~kLy7{nfI)bw5V%p zeaXfAn4%N)Hu-ol6Gd0=@s!xu* zk~isdxu(j>rvNK!m7ia!sLb7_=&N zUaA9CYeLMQ%+sY?u;|LmPjvwmQJkwR_f4I%6Ch52~14wHT^fO7yQnU zNIgZhYe8tO#wV5-4@Hk&KYo4=rP@83=inG)ardm=8P@QFr34u$_ zv&%~ZE9E5m&4BnMH!Y&)@HYWx*}PQ+IhDMe`lN(fu`$yuWuIHBmk!HyRpJ)k*$}3u ztT-d-EZysRyxFS}c=KtJl#&_MRULUToK;a(r_~<*pbz*~S|y`%4#B@5wpWY%)RJKO zqoh^80s7`VFW@#%JR|uP(UC61pA`1CY}X)cnZXn$OxFR7lPe2kcdZh@!|T-w-m}|) zmj(BeSpiy=YY=8vxjIoi8D^OChHd<5X~H~aJG5&mai^eQ#kf-7@N#r9pb5D~2P7ZW zx%@8ur%YQ-v_Vh*LUTB1)>emSSQcq|rKw)t+AhTwD26K{T18>sP@d+i0T8&n+H5? z@KiJ&Y>aa?Za=1t$m9xQ|2A-jsgXd_4^rmhsx9q+c~;>Z+DSWQR2uE?e3vRI*4Nf) z)1n*z$+e^(nlDdv8Xk*F8_wiO0fO*jlf&BX(G)y!x_FTD3cL zadd2PL7ZFgUGYEpAht5p6;o)W%OrwzmAOedAodBl!uYWyQuc{Go^oYA!*uCp5G$Xo z$7l2w%QitBXcs`Vv4f33lXwF?IpQeuN-2=|I01kpXnbD5Z4qn4{9S%oV6luBMK^rD z1?u^|ya4aHX%&D1oHXQoU`ozH#%I!G4>J)E8nuhsB*#~DYQ!Zwfz?^;mWM~TInZF> z=+E52ASc-BW$2qeq0HE#yBsflNQZ_1(j0ePg3?uT@m1p=B~%Md?3YE+DqXVFzB7hT zFNCi6vbgKj#=3POx2dEJKWpbn)#wpS9FB7HDw^x@9olL!fX_yy>-Bb`we8Sli>#`O zW8y`Yv-C8lJQ%jqm~!7s37vwrv@Z>`sI@|%IO{ACb`8f(Mf%A5(d~AcO_z^flcd%6 zZn~A}2erah_xeqpa^R%|*9a|3RT%kcRXyS{03zD4HlK85v%QK(g)h-{`!w?k!|Hab z)|sItc}7oA~MJzqQS|;J|TG_ zJT!$I%F61YtJ&Ww7j$-a5b8_ky4Rse@hYa_tIr?C$>iM>!vm>rAvcw-Y_Ke6`Bj18#4M>sCO_2! zpbA^XiKhv5Zq1b|))&vqA@Vyh#7vSH)tmE|Z_8GGQ*i)cv3o7avNcvKHR2(=XNlD5 z?t+Q$UD0W1zvO?({hhIQ96sMlAos_EhV1D_4)4HYoP~4tix;|GR<;k3ki9$O=4eg!l?wfMz)A5h zTV^VXObFH~=^`c%b?#T)S@|1Zds2oPv}?)61ypJC5UcKRvl$mmd)Vr5iB{!^ViQ>= zXM-U&+@9TMoup&Fl4-aiMt+5P#Hu~K2SORhRn->=&?c5!L}?DHYn~qIrbn<_X^g@D zJ?%+A`Ey#VQ3p6BIyuFCEbX3oY!Zi07O7~^7F1WzBIh?wUDh@|Vrfl$VPr-+FoRxf zUrptp?aaq0ciI;AlPff*%DW;%N?Kchb(EK7{CzP)O_o7QbtDq`Z@#ovDiU(+q*BR( znxg>WCzi8cdd96KH8#@ecUGO<%P`L4ie)V0`*JaYcO{g=ybM^VYnK^u#k^?WM&=%GpN2-di?VHcW)Zu@)fz}ow3Lbh zgltLF-HXLni5V~~3w4^hE#=)On?-k;j5kTZrRMN_I>xeBOXX|Rm3g|KrT9LjkS__( zy(qdZz>g<`>xEa8*M66Kq-4l=f9fbP_uik1e+TJ6c~3LxIgZ}>sbMacP6}ln!t1u7 zsMgGuY-F~X3d#?p+CXCEFKbB5b{KaRMgSO*Vr*EJN61z&AZ2Bx*OI+R(A5DWdp7-s0Q6FUl%x{U_$Es2B-R$ zzN}38j+2E;tm6jZXuUOjAmD={gf1ATWeqAblA;fx+V0mxXt5^H*I=w$*Y#tHcN!~0 zmxIX)an>YceJ^G^h3Wi3)NU#_2dpnyjo5;QR~5*T$ayvSzi2=yM5JkdakZtdw*RW) zNie$PQZd6Ss9`8uv@OuZNpo^J`E3*!tozZ(WH}5)X?q*jBY)y=D?AGQsCUfI-Cl4y zEho-fo#rb=+tf~6?f@Tr#hR6&RWzKMY-xmI?e3|ujj*j>>0>?rKRDj1Z2Nz3Qt$_p zgz0Vq-&d%uYL*$I@ybkhgQH*S__%wMgw?OZ* zKCb2`&4m?XvgKjzFs_fBa+McO@?=iuj~xn{lU`Wlb(_>A}LvBemn%Hl# zxYnZ(OkiC;r)~QO_hv(x(hcK%Dz%YLn5XipXO6qxSl-abJXfoTOSq3j6hbPiy3*N- zTC6qTDWJ>7oikF_^S)gx2^f<5)qcdKRb!uy+@`BKEcul>2~{Wg{nNLwQgVT#LTjY<^?R zIy&C_G35Hcr(nlWNIr-i_Nk|2REnF1Ia5xyHSlE@&B#}}8qWEI9#5L#Y*zxjFLl%* zuZqcog1W2cNvxW<+d8xr9hi=kr6<0@H(hO1oTO$6_ z>6XJ9eesGKB{7942`pLEv|?2rJ7Klt3p!?Gu8K1T#}$Oh-6}J7=KiI1JhD^p(2SHW z0_J`@k>qjIKZjKo_V1ZhZK%*`d_-Um^s=i%l z+LPPc-hi6d2<;=TTPk^1wU;H2<7(M+2;PsdbXa?_q|1yO*B?X6p(95HIswZs10^fr zfqF4>&3G|A@;IY!Iv(jZmFXQyN=JVScD>zR`W}_)u;;rtrdooWh<18~F!oJyBBz6F zV}G-0Jx^kt*H38~rSuA5g3UO4=>KLa{Q0VAh7DL=K>GX%XU%w7qD%U9*C>Ev2WgdO z-^gCi;G;hJ>;+3bne0dTu^szf55Vwx}?C_yzwm zL>b3ohK--<2j-`MqW( zUf42?SIs_z;YF~@Xe>R|K3Sg?(;8`JS{rw3pdNG!((p}0(!WH|?jo=l&`w16NB_2C zrnZ6co3n2nM-MfWy0p7$Y@or`-%;i+3is92$b)hRNjx8I-QiOcHrnG?%n5(SmEMQg@*gU02i^vDQuHKmYInfuh*WJGh$j4J`26d;cE* zyFf(0N=oiwUZo&;p*G1}3Er~@1*&=$5On2yl?w4w(uF1#q)2`by3wD_T@30oKPjx% z&uN#{43-^a#a?5!2!(YIpQ#Za(F0}aMe#@Qp*CKy??KHBC)JW5zoR=?Pskv83rE-? zbuHb}Jy!&1+tVKPlL-5+1xm?b+adB(pd041SmuxL{{Y!d@Z4Lf!Tyy@9uvK*)Zc-$ zeyq7<;Li7n{-CRa@N?bXRd7%b>l{^E3CB^J{)_P8;JjEuoS^PuiS$*$!5Z4<51Ok> zKWX5vGxmY{ESo3c{*_!9d!z&MQZVDQZ5B*NxmG1M%}KEuIJ^3nSmHK+_ELTjA#8J@ z{a5E1?wCPj;ZYplQOql^h&`(-^D#aqqxjAvY*4aE{o*rtu7^0TvKc#7Vlk&+DpuTm zmZ_Gw?`c^%Cv%j;nntHJTs_c>LqP{z)>VV<87vkIq1iY)++^Bk1wzK`va(Bfv?>4|06vJHq0BOUyYFX|Zzu9%K;|hO8m2KbpR-dBp!5Q4B$VYG88{D#)2aXKY zA<%>W0GD6ge&fYwVBqb+*I9$h{{SqqpM!qlUsauhe3d#GdY|s7M;l%0y&2ve-m;$w zpAcPO;T7RhJE#8un1A&yz5f8efAuMS;rsem_o`tLANxmltmwztjDM+r`&=Xc0Mx25 zuip5o93Xq@_~W% z`lDt$cS%r*?FeN<_&ikIV9QOS6rT;Q*w@Y7s<2gV!9@j1~^{EC}^cJQ+rLM5=QIQh+*xSj7k%B-)o zHGQ35>T0ra^n+EP=@neB^|@z5k!i!)DqS3H z_PkeG@x7zDO}Jw}X$gu#E(Y#B%Dn*|q|frKp)#r^!DLH>mE((4tU^1VF3T$@KZR8~ zLBVy73bALzEekSbO34b2b5~dh+YMy~o)Sa|4I}l3`cbzI&ibyYoR^%@u&M_;wSE*Q z%VSnsVga?pDny0F)B35~YM)9a>e*Bdh&5#QD=)#iJ)xoBRfI@XPWGu-Z^PocsE*je zJJm2;B44d&`m0W4cK*_9B*mVpeL~4?jnY>RO7lIAE3S+2uS?php^@B)>Ed%=nS8Zb z9m=kEV(VY~!``o+_Y;a>PkC1WQ1edKlB&p0164kyX2^c9slWJ@%;2SF2;j0nZ?Zg5 z7ToV@5${xMNcLs*S&+}XdzbegzVK3r{2hn&!CD3pZoZT)e$G;#LmcjgvYc7%*>BZJ zkg%AZvtil_X9JGov63p&aC|0yji*nMDo(-UvU=f@@|vVG@ktO{WbJTNh}QeVy?$`Z z+$ynNHjJN>mPgtmTATyEs59>n;yYj+>XG2jQEC3Ad_EA|N6ksy8QEa{QIjB_am`b7 z0C?K@g)r8$_6o4#L_VuX`mC1W5cW&{lCvTawY~~Yg!8weKRHQc;LrWgKTw45a|rHP zCdSgheyi~?h)wqC^-?im9mFYwGq*`+KZ2uN?!}pFWTvI}Sf9I+>f%FEHp%u1t`7HZ zf>in_p7oZj^^*1KT;0CiX{X7yV~f$mccJVTaK2XAu7$l=}=QJ)dq6pc8A zCPjOFn+jbc-L$o;k!E(ZD$P*>dm)!Cn;55AjfGt90a1@xwEqBGS5)J+mTQ#KCy zEaFugBe7IFg1DnL({k$Va8h8h7ei(>t#%z}oR{~5*@{BzBmVvU=w-OeTZvcpqjBC8 zfrdTb6fP%{s~R?kDos>!MaJN)!ohIIk4vE~9jMxag+Y&qv zKk8PXK$QZ5bJV86Xi6igZ`wN!mpU>hcWc9H2ATe>B$stp##@h_W^A9*v?lisppzmt2E zorT(Ws?prDt#6qARbG$h+^>XxW&Z$LlNjB?82+^}gSqKXf? z{c2_e0)1$5{$-fPyHkJNNU~#Nm8nE-=(fI#Haq)4{sd3`pM-_^7_i&xw!MW-%y1i8 zROe4wRhGS#o6TG~s>xgxWVT|tX8e|)hZPF^%r@*S@l?WKkJyv^$j=Rf`?ojwl`zO8 z``^~TI|_cs{c5qyL+Y`stZ?23dX>3t5XXATjTOUFn8_@gEkCLYA}F7;B|`=%pPv$6 z$g?1^yX_yM!X(!{`BMy#H}6?uDEK^8VmouJ_bG=5ZRps3icbOKQ=ty!HZso8(iQ6d z2)pop?J73TR{5_LGSfYaN}!s#QPSYErmk6v=8fw@l+QJs@xe;+r&TEwWM9ZD-^ob0 z@>iZqHCE+$R5aRE__XuKmki6)o0IIq!=PFJ03HcoJEoi>L=h{JL^~t}Ns^}uE8uQ~ z!W5#LXVPPA{Yvxu6`#ehg*Zltg*1tC2{m=eI z`N#JE08*|U7;bB2+NbyU_O;n+9VQg`+r*VAx}PPJ(jA>4RENSNh$wy60qM!USYJmj z#rj4FaC5f^Y(er_EeF~@3V(KlCw!w3@@f)oje;pZFsd8x6Me^(lsseYbb5d*)Qwgy)?e731zp? zM5ug;%GRwn3XKnCuvKGQ=ia~FP`h=avS%%?n$1^vT2<3otGKF{g((B=E1S(^klsqM zEe*>!F0gW4Yf&(wT5$743%HV{AME&7b%P0~DyFs|WH+Be)5kd@F+?n3m`)A)n+^G| z&tWh2tbasC27T{$fyT` z>f6ay1TTOpK!i6iDnW+b%nINY47{7MFfptw^NXJ%H&e*DUFF=g;?jpp5wNFqw@FUY z&R40-`8t*UMunr2(0D7qfaADxRk9Vsn$hts1B&}e^Q_{3%(DZLvMlXsMTRs;VBTT8 z1I|mL5u-)}&W!tR=Zc1YPlh|za=BHi<(fIGf~duQJJk%9eANY|sBcwz$#nND;y9^v zmDsbpRP4FuQ(A+pmQT%f2*sgj7`v99bH{SIX05A6Jsk09=ZityR*rbHX|8KelXvzx zO>4J_sR1pXMDgCHLx}ZTYviUSi#ru&uf=+-_%NTMyR#kGu2C?Ea6=RRiKz_TtmzUyDMiZv=l4(I@Fkxm`7>UKV#|E0xOSrxul@ zEh|qoqI>c9D~h>Xt4=E5sIHpLlB~Hc3jW2MuT{l4uMZ;(qbLUa3BQ zPOYn>bava+CS}P@wp$;a{{fdO8G0osRx49i%u#GdR5Bl{bbCrXH{4-sBuXSXLM|6;w4Jua1I~IGmO;+DDYG=7tgIPka z4%ihUJK|F@9Cj&Bm{*Kfg?Lwmp?KrX>{lz4h|+jG^(|yf}NW0$5nj2?**JEy=msE>2E8zO-)R;y2_F-b_0XC#--gDnMJrP zC~#VIF|t=k;_zBZxolsHgFuVz!v$IUvDAL5ko8^;K5@ zy3x){AWL8L3Zi{+0(xGR;xT&EiaY(wig-RffkfYa)>O4JM* z94BjNR4(|6@mf{qfm500s8#DJ_IIn*apCbgTJ&as4nbOg%h*A9J{x!xx80gC3a#KZ5{zq+8>1ziWwZ-s>bml zW`_

X4lMEl&tc(%lYY>8_1r+b1{D%3s6_bDl z4_JVV@K|H^ijZLTp=KTo-fKPwBi^ZmG=9x~Z~p*fEU&Z(_F#dFTb583wFK)}F6%J@ z6NFlHkqQ1(EwlsVQ@6>zUgh8YjO?vTIY>TdQoa0@N~^#-RpZDrvUn;` ztdVH#GWsFV;LzT6z4H1joGS-{WuhGZida+NMLv`^mg1H@O0FE$9Nbz8yB2`F(~m|g z)(^!<#^_er{!vzxP!f9;Srm;!?{xmjjK6LxKpOY`hfwRnkZC7~S`&bi#Wa6(hH> zszz)5s15omJ{gx*g$w^zAtx8mR9gC(haFXp`czvE%Aky=^r!nrf#|aUlN^QZRwCe#( zHiK%T-Zx2P#(x1tNWuQlQZRuV7G4c!ihCOjKt7$B%YywQ4Qxn%sIu>$(NZks>J`=w zB1=89TGnc~9qO%WtR`=gotb;Z64#Nws1X!6jbci#fbVTr1;L-}xxMIr!~PPa>=y`! zx@xSB-Qmlva30H#(E>1;PV0jF;MJVPksLd43#@MtuVMFmRq*~F6*1Aza_Zr`-tbj{ z599D&3iX_?2Zk-U@9bBFek|V9FU6!Q>aSW>m8Y7{Dvno-S05sBREOG!;SujGUPaPW z7__YASIlixvCc~s92Z;7iug|UmsH1l-YcnT-4RQmc=wvC1}7=0*ono6PSE#C&zA64 zT*WI;&jou-wNYt1-8E=Ur`0l7yzhL_Gb7(Bk!8VW&w9QcG>}yeD~APkE7j`tq^pX} zcO})hg_wD)-qmsDr(9PRnpNvZ#ILHoPGM&ZSB^?UwO>`77g@=5t378eO7)yqSg*9I zWyMb1z2(;Stn!?G^7jMgzRbQ$%{&$BKlqjXs&=~-N~41CUUvYlcfhkx5VKvuTvgTF zycNY(YrHB~a`Tr~cu7n_d3LFVl?q2y=;TQ3Q(P|JV?ORGw;4BQ=hc zSThQh9!2=CcLKV| zhqX^H-UYukdZgU5WxvSQuH>k$DvL4PmCxb|)5{(B?^+e$ZfdP~EdzqRXDM1)?pL^} ztCh;{F7t}@dcVodD>hi9uf;cD@LX@5fUU&9*=yQ&dY8@B{?5WMmk&X?JkxFck%x!*!o>#SWSeFi zm`I(7g3E%=p6|R#2bPw?I!RO>$7t3%=MaD;sz0r{-(M&)v~5ZM~#HD(k? zPg9|}zcri{;b#tN@OCYWMk*`u=~?7(Q=Tn2v!%gasCzM0S#)@9cB1biMDbLqtFh@? zRXbLFoyz&EkCjo5?0y)pSCvwHooeh_acbU;+*RPKyH=H)uUdF3%D)$eEZ)V{jxAZC zT#Th?D%oCdHBjDovr!@hQ!M}v^(bFwM~~|Xe5Qzxfc>Ea?SbNnYcQwF(ZSqM==xDf z<-&V!-l=Dbyl`7rik+B`^;ptLPZ2OU_$8aVX=QT7q~g@7X|-mF`zrCtTphhMb4XlF}8()5xI3 zr3100;pA00BQC{{Z@Na^=hEFQm-CQ5{rndl|{-XKN|f(o>nx zddvR+_0y75l2fmw>}4HQ-i)XJ0FwUz^7CK)U4QUQZ==zErGJTDWqFq_T)B+~@?{}% z=>zsG7#gEC{{V~WE@%G$Gc*4Ho`3SCNB_~_r4-9t`<^xI_S#dE#85g)^%^louRrAYx zD3~{WQTOc_MNj&oHqJxU6%pPVRbQ?Wz3W-MA+NJ&Xfwo?udr*5N9L za3%=Va;s&2#v%7E z71OSN_@jwsKyJY`VN&VuC7w1hw3twOL6^FNnmT zS0qi97MHU+M((y1dVoYsNs7nAfq6uhN0F!A>YzjjgR8F9zi zzv&j%-t_ll!zVVw(zcg?j3q+VunCPjK|B_&&eD$4Vbu!p<{zRspkrsYxkXmhtJ<0) zd8|1@I#p9&6{h!c4gRvTW(lfC#%eISdm+NZHh%-qCBl`pyK^kHYkCB6MdACIn2qL= zWxAN&ms2-SNvd8IHNf0uiDesoB7-X!eXQyUSKQCcUHT=zVlaY6M)-4Z2A84ln3CZ3 z*Yvm?{{Z+)h@rj__N|FD5w^)@z{*Iij1K&=2a%bFLuD1-u))SvQSRO<8C`G#+Wq3> zrq|Ig1%~a3NICRy1hn~>!Dtqv-EblF=v(BJj!hofr3qkv%Tb7rs`1_>lWx7RYOaO! zptE|1g=ZYT5!|iad*wBRD ztFi(PrSzS1?c*%CEB-@p9f(!qSZ(RO%xy9p z0XSzGVyB#J$_g0O<}V8Ct{_m>%ze!Zyab{VglIjbVgnSX`4=_XZZmFUY{zp_0*C<1 zjbKp?b*u4H1v<_Rb=MM>@wF3NeOVu9K5>88m`;U$SU{GwXt|uc(8s|nO_;y=iSD&_ zhstXRLoHiz))yC-=4q#1ZonbMR{l~);k0;b$4u|{kg%n-$BZ@H{0(zrTC<0C{e@6j$6)4Gm5rs+S#S?f z!5dIm;Vi*ddRss?nomg7uIw^8FyH<{6}Sh=Q423O1L__x8kHSVj4u%60OnG!l)ITP zQ5JHR_)1+a)k2VNh&eMujMrAD)}F}E+@mB?Ai_N*EbOQavlN(zO?c~2GP$7D1ey&p zMNJH&*j5H~J)#B0U(?8@4qI^MbBaIVY@!gtCD;oRM zs$J7oJF!x5622j-xz^{yHI&}8cpzybs6!r*%|`OQIh$CO1I1}({IsoBv-2{yo%lB2 zp+XlZUTf9%jhMPzQLQ>9o$O}F_aOsO6QL`lVvM2Sl&G%AqYtYM@tJf7(xZV$p;fd2 znKjoI#KrI$(rR79D%h?D4bb#d1Ca25T0R1$htZ%8V9E=P6;>Nu+Rf5u6^_j&W(Ct% z%%#Dzoug<3??j?UHwb4?YhkR+hp)104-hf!DrnyDduu$0%^?^M3MkK4EYQ02im+oj zs>)ky_NEUY8myC*XE`h_&su2>*T}LeG_@K^uJYn#ULvE=N>z<73pz_18^>%eVfu(V zlP+w}S4iJ^9OQy>Izetq*uj0`D72xn=2&FA6b~iKGe2#_l(Mo&&S{do)Xb4^bN>J( zGPbt7K(i`qtRJb1Q!S$&Vh2N$&GeR~-BQ$ZZPP+I-b_7i6`i650fZIWz?H`v((V*! zR)R*YM+~ub4TG%#9vsRC+C!P3?30&<7?UY{AH#Z75=!2{6R31nXMO;Z!cWjU>1LTf=SkhMln~Y2xoQK4&Q8nsb zf5fAyVbW@)Z5VRPUA8&kn%E`4ACjBf&$+DBAUbm`0zMQ%iKy;*hKn`5SlUp66~bsr za3!p@?Fv&bT9|TkQr~z*XuTk480`iGq|gH?0XWw(jD7EE-ejJs)c^^LXJ0ayqchb9 zpuRV}=1`=w%No<;RC$d^R@k>cynbeGiP7r@R)hSwp>JDJ9MJi32U*1nUF@00!y()j zkhQxmQY;5fghoMciuah!NtP?N%hjJil+xvOmwAP3G|-grwR;3ZOshX7pP9sg>z8WSf$(6VGa12#oLx; zyB=VeLK|s3Iz=Lo?*_%Ud1Cw#E$h-z=oTWVjH{XTL8lK`E2n>bW3+XoMf6?z6rPD; zv{oQ|LFB|@5}>5h3hGOR5{)b?8cU5`k+xbj5{Onwkcwl)3K|^`Y5obWEP3m^gvhX${3#y`^1qWr8#Locl%m;f+d; zgei0g9b+n`3f}?Nbch%nL@sJ>!~<%1gKCc`SD};1zsbo5VQ@<5nOES;Juq`QCn!^c z*6TH6=T8;Y1KP~HC;EwFQ>1i6>i2UOP++DO zhV*HEi!Orri9RnitJ*H_WhSaQ>`Y!nkLZD zifN7RYcym9%HrMq97M)k0I1uNIs!7n{r9)2DtQU5FPDS02Gg}UKy1Ury`f=f%10Ng zO=bk9RLq+d#LX)W(Okt@rY2o25PG`6)LWJWtKp$B0M5bGxK|n$8GLUmgck1B1hfvb zK@qx_3P)Z~SmgsRkpYZMC0)Vv0%EEaLC+!z!W&(aB}-Vsz9tw40j*D2nNhM5>R;F~ z$pc$%Rck3lry|6yLA4bzj!=Wv(X3_Bb=@fRK!%`F3ztmL8JZ%Rmhv%fZR6tBg8H6AdY8p*G+)D3YxvfBRDWn;pl1ETUI3s((yPH`V?TXU zSR1QcUOb$`9cWa}C8MF_()9Y2!&^wrhQJ}eSxFp^LxiYNrptGZU`yA=EQ>9m(pdzo zEskdSilK2@^{JVZrA<1^m88Y%Dr%B!Zu0}TNR}%m_ZE5bMTE1X0d$V-%@I!#0^ao> zru%OMS`P%K(>gIMsC0xWuOL(L7E@j6RWnxiFxYA~Sx(UG2Fas|nrPhX1ZW2rq;J`@ zy{}lM%5Nd|#HEO$_CdfjBT=BrBlXF(nFQ%fOeyG*yJ{R`O~h9xn4P*EAg;mSj-~5K zX<8Qwt%4xcITbUJ=W{OSE^Tq#`@$Hd2+=0+>l0c#Be;>E?G0)dRiyTf%r|4~U#Wx@ zX9c);9yKzz1zt0`Jr_F56*aiH*ayIRF#6k1VTaEP6&^W>LsF{B<^UN?DvoaDxpu&s ztDR+en}EUy+FD|w$_Dg{fv;gq7Bsp`4M4DJc-9G7jR$x4F|A<9Y~_hi?cjhG7+A<9 zkZ5S#EBjc4^mc~o;{|yVTv+lhY#s$A@e9}uIXq?sg0?x06flSim0pZ4)EmNe)@FLV zGl@`{R_(n{POz2r0a2*X;fW|VlwQNtd&l@2O%eKE5ntUbHF3GP8kE+|6J44j3KZlG z3`JenW#;Bw=~r3xfMlbJN~2F)$~sDJE#y70)tGaH6~*5WXeF|#h}L}IO_Ql86s9

y*6X`2u^-xH)Y0$I>0X8mS@lhzB6jB_~+BFmk!m=%4c7}LC^ zhG3)&<<~N_4^t3{Luv$K%_P_1Dwx%GER?!%LtUlY5MYs+vdNU{~(Miv4r7!d#h z`i#nha#d8Rc)48Ytnl$s)e}LCY@ptxVFly}v;@c+$6J}rp3y3^RTHv_S3rw3-YLxT z3_&(9UV-r_E5eVN8^*4^(1N_Y&e_&$D%gGrNDWz#lzd!03(jG8)YUs~C+Zzvou$AX zL@^TJ3u)lO9+(n~N4#Q~vR=#vY@!rhCAFzAK69)9GZe~LR+oF}8v4Nvf@#bSR|+`q zEnFjVzs)@%!K;>iA8B$V8<@%MFAxGwD82LZ(kTGm97jp1LWzwFp@H3p2T8Us#_-=m zhh5Z{8IF`XoD$x4XhJF0kK$VJ=rdEt0~oU1fG$dM>C7lP;6Ufkcu?87K`c}rMrJOf zr9?8oX>1O|p3j+c5$!+=(bfL|A`M%;j69bxMMtifuqhpk5LJQuvh}k9TTFoW5W~To`5Rh}$++TS$bg(FIKn7Ho)1T2k0wSkY%gj6_kg zmg$|bEfqKWOXEgEozjh4V>>{JZn%b$cY-E?b+T_gu356&1gw|_3}~9yPH0MAool4+ zGR;}3{oZSy5h$r~bqYx-#2^+Ck*;}xnQ3R-_ucM+XshYZiF7ER7;f7kq-9tmL|0RI zXsqJ}j@@Db<&6X85dhk2-Ek)RS#p>|U>ZF1gX9YHX|zLt!x*nwO;&g+SfFLO;XD?v zi%0+GCI+i6%^kF-q-7?CZF?s1DzBRpK9I9xOIt5wTzDo9 z_oa1+?K;q0uir>u$X%b(JHoZv^#o_7+gKxlglEP^f*6!ML<5uDI>oYtn38!I1XmRDP>h{mn9kvVmNJAr`P0bwo zu}&tvDOfE)z!OajuoN+D`>|=Lvcd)&E}7i2uo93*+^-9j5@eN~SM?DKPWb&#Mm4=j zq`BQ=4@_ca$=a`qs1BNDDqGt9;aL|8UeR5cqebRw#?^Bp)ZS^%On@UNzo>V|h#p1c z{V~u@+5%vNDuIrY%&6fJ*^7NeTg%yw2t`wAU2QbUwJ5DTz`UomsdZ3_x(}O_*)r^< z@n7r`@7I~_RS8(IF@>{0Xlv~==m1%LZ@kyYH?`uhuDZv`Ca^W4uL4sqx|O|6`FNSa z`p`UL8XPo@tc-MycQ`;{&fbahP#prfgJ7V{IFz{c5le;5BSnJP-jOb$r5Z~``A(-h z!`$0x&aF+iKJ%UQU6Zm!I;JIyp(~(N34LXGYmJ}^&zPQdS<(Wt++&o|?hI(4Y`~iZ zd%u!=E z*Y=9&qfn#PZi2NJSZ~BqQooZ;Fvr=efIz-CKBYFLyf4dHcvGWe*GX0j(y)5M2xDMV z9XlleWa1(#8UQb(dKV4n#@zeMDC-etc3>=Ti9>{-p{n(`3M86gQUHhYz zYRsD_Q!{HF$Xv=i#Vw=bC}xOO{ia2^;yR)9VP`i&;J_H~mveXj0Ap@!bY;KkFl)KP zxy0Pkj+d?!LY*#K%QbfQ5~B=Yssu?t@kKQ?2q^FfN$yGFLhb-^tC(a61u9DGmGxbr?rfNf6%Pw`BS*|u+3 z*}V;*EI8nZ(4d`95&lV+{F%0{{{Y$3eNz7b^@k36Kt@H1cFcbjD2KX~mKZAUd1bJ) zZIyoW{XBDL?81SUSnZ3ZgvU|qEY|qV*;20WS*t8N%-MbCT{?(5uoq}ms`Cvvh9S?| zben~y66U=W%=G2i ziQ<HzbCk@hFnl#OO`E$*WIOHJW3`!YOUp`3G7ge87$kaNfO+>siUD?(- z`wD8U*}vKT5JnbTmk+4jqeg|}qZ4JG;q=TiJXMd(t!$d1!fw`Jmy5*_JZ!3bMRt8o zp3`7R=L0Y`TND_4iqs8EoW#3v5iGSr=Bx*CWx_Y&rNJUjTP z07e_!I)X#ypX@3cZDSK!ysFQcYHQvjO#-~Zpino4iA}}4761;-P7D{ReiD+0Z5VWH zVU=a}xr&G;&_bhtmfIuhMrs-c)fi1&v?+SJBhY1lIz^P-02QccxDAQ!O==Z316&@_ ztYkK8l$U2)FVvK3%W(>#(?{ke7!{oD_LOK9jLkNvbCYj~cY5W-6f6%&28wA~%UZSb zX>)e?aWoN4^+Ppy0>-Uw*)NG31w`cgC>H5U&7PMq$>~e;jqZfA0{B@#y0#3(a3^%e zF&iSdxKu6N1Bg~}d|gg2)^1iy&4omeJCLL3ypsp9pqxZ{@9sd9bUTAVt@i8Zrf@N14 zCiwmj(mUkC^BPy)0;u5ff7T19i}Zmlr=VpnJXZy*Pk^&7D@R|-71x}Q(jRZiG=NX< zNqL&O#3xsu$}u#ZXn!Jw>hA=1dqYm~NFy_ut$vd>{{X8D`b1mQx##(VqTUvK%*sQL z^@6E?i&(57{+MZdH6wm6BwIcTmj00o9MBwP^Oj7}gt&qw6U8 zEmIU-bP_!=IQo+(X~OPLJel2{g*bkuk-A$5>SiG;+mM z1}h{2o*0YwkEXF%W7L6)6D8l195%qG=50rYE;R2}KG3|ci$mrv*Sk}}<#+w}a*flUUG$)%iq;<5KGJ*A6W zQkv+OQHAQtDk&W3`i-sv&INse{SkLiUN>av2N!6o{O z-Vh`59ZF{v#)BK!_+ewuFP3Via^)Mr&2{Mp>WwmYw!fIcvZ5C%4dn<6{w`XWZ$S8j zN-x&+L!0sJgxBdc_q9zu&akGi5|^u*DaI9uWTN2eUvE zRgSim@hkG1`@vJzqkcu{2VRa_n^9MD$ffea|J~^8v5=vl53!OYqFtom{eL z_CHa?Ymi6OmuMZ|Y2QHQeWl`4i&?N|^kkMiVKty(;tEQA1jqy5mU+!QgmyLZFl#{4KgZrW7Jrf;Sx_jP@FXj;t#iy-Ir}s2Cbzng8U@!+ z>nrJ+)Is<|bMTdEsk~Y#=rZkW;eg6~hu#|HxYO-7anLLtG@7)A7e4tv65j*40~>$H zEZEz+#nhanRvwVI7+Q-=&!RAPDj&TWoUOjPfNB%W_m=suj=qrjCFw!GF`WkK=g>p+ z;g3J5>!AMGwD@P*Gy&MxGOs`u`3XP^Z9cK3iaJNE0J{mVd4~2{Y3RXVx_rcA2Q$C0 z+-Gvv=!B=xgsg*h$BKI{5`bt8`IahgR(<4SMbY*!7x#^sf1dYaH8vXTdIo2AmWo4L z{=oV`4Qs*nf~%km^|m^g+&(FfO+YW%BL4s%Jffu^0{p^^^NaZSP*$gV4V z{{TK>8zWj%#HCqwS`>6WPuPy*YA@d`lY-%@U1D>i+cv**2a!utIr|K@;U{~TfUaJ6P&+}MWv|1&# z;uR5C`ePM>H{nd*T}(}YF-*K=6>C>d#1E=}fF`v>z$nM8m1N{!wmCF8{cyyuX+0QZ zd3^x`7jrMUBerK^k&ET!N$Q|WBI#rWe6P7rh-22RE59T(cNZLc<-`5A1P4Xkx57to zF0YoJ(D#nXU(ftXu?Dyo(qr;gCzcfk{L`-jj$UhF^^aKvJl>^x9mz_(KCnO4Oqrdv9sP?0_Tra{7` z^B#GBTuo7S(|5T9q`7JRV0A|i_Z6GE&G$6|_)+N!>`x*{yvT1%!paf9+)C<2*!Cvw z%GKA23wz6HNxB}fzdvb~d8L&5O#)fJJ>}bBihK!3e>4>oMx9jqN(JE@29HE*EmU>; zJ@GMuMVXd5JDxCY+=`2@-8 zOV+*Z1ZbUeFlrtTX@3KBeoE~d(!||Pl$~Cv4`qlJ$7rNv-GX7a^Bq^(3E0C;50c@K z3fGB|O>>ZT0^m25lUgyyi+E@pqV>mVjpTyne_fS&m+vS;XW(G;*P5w$?O%*oE}n^0 z0$SyMrL-N5rlgM<4`^0pxwi;s-;?-AF3agOaFVlLzFViPc4LJqkqXp7h1na zD!Q%FP*C3^{3aN#xj!=*M_c$t*P7V)n3VL0&$O$5gH-eg=Y=S0Ka_!qt1^hBZY%RJ zKXcH-J>Qr00QPI`h?G$+)`ZNgB@H97_>DQP(y_m!uxR&~tFKhBe2@oZ#cR?D^aiC; z+~BhsnCSliBZs0}?qyWc$ohyo9iPc=$9%8I=VyohN;b_HM-%{+ZU%r`v0pN%8o>Io z4=_N-&TQ#d^(p}!$9Bxw&b24nEt-a#;V4}XXUqv%1refNj0<>PGXk!zk8y;W^S*ro zP&O8D!V2O25VFAjxSQA$9pxOph9lF=qrFNEw;CR#+5@}e3T$2g-9RG0x0BjXVb9@H z$Hfn*I~$A>Nc8t`Ij;nxr04{%7c*Q|^p|2Bz&b5{Qd(+o_#h*7DIF2$Pgr_lb(NKk zoju?>s9~f7arKz3E#4A@#;v_zP*8Uw9Y?flyAe%O-dWyU-`Y6Q^qIS2-s~VhP+Mz0 zZdl^`&2;=k@m1|R%xnJuQzcjM>j-6WU*iQt1G_ayCwS@}Z}Tl+9pA}8bUhBMWlNnl`@q-BPsnh~)EDtESl^am;-wmycSlh)Mex_$AWU*h8qXxCaTG`vaoiIQ zS)PZh5a!v$Y8f%3g7!&zXfqJTh?r6w#*EqC2-Z%~K!_BrAy!)HbBU-|l)zf(OCq7d zP2tcab()t`QNHlXt0Ux}F#_+& zJ|S*BYvrWwOZI&rEXI1xIe2B5uPIC26;^`vmep0>3uQSzl8UPJg_wedln!qmu*e>u z#DB?}16?yHg6Lb?YREdu^D^A^hZ9oEB6peY^o6#r(7Qz3)l06JpYJvX<{o7`3+8PZ z#|HocfKW^Rxo?@xNP^?To4Ct(lEF+Z{6|=xu?0~U(;lxg6m`o5McydA zvnVwh=}FCdBD}X;r9D3N0+DLL_GY{e^=X4m>i~1vxvxu0*f2}dHkV$~>o3M8bbGT~ z?rIj{c4_Yxw^-rYgc{LaVbgAe$%;6rZ4lR25=~K~J`8chTZ;5dOV6ase+UlHr+W99 z9A()Rlgs`fOrhFeoArf~tU3Pxsim5?{$X)Wl=Xlw9_oF{RoUhVVU7fx2YU^aT{9Pz zE+lw{oHH11#k>#_0PiYkuZgZ5u`HE!9b522UmQ+I?J0Qpm+5Ti63y4ls`TxwH1EM3 zLDU-fW18F*kA%wZ`HR30Ev5NMhSht`YcIT_YvE>w{F#@s4JB)ajd`6}0|A1&qB?pe zTAJewss8|FjgMKJQOAg$Hi*3<`M3i>o6SNAYwtz(xFRpZZ(xl&61@)6<6+o{=eY#B zs-}X|*ohUp2e^Y_VAkVQ3VJ8hG=lM00veQQpd#w2Bak97*Qu8;TU&}$Hn+B3y~q%b z)^RCpfZ^=SYy+fd(!A8)v!pN}*Y&gssW<&)O%2@_;s(yp*YCUysnCNB`U~|EUKjjk z6bbf?_g;u$JBmRtuc|&7sz0mxmFNb#-#0XrxfHq&6u`EA}njM{^ng?{2GlRCU3-?(rm_6cN&xM$;$XmPZ9lvBZ zwcmKR2Ta4TCs|{Y%x^;7rAEbG`&m&dN}`PN)`NyDd@K8pS*!GhwKHWN<0x1=OV$I{ zT+NLEnoFL=o*rk6FK`sEr|MUytV-`fC$-HNjV#O&$?FHbLD zP(%aDx)<(C!FA+P`%0%LnEBr*0s(gW_w6XJZ(zM~Uvafznp69NX>cafG1b+dLK>?y z1T}-U8H1+W<FR)kBrOrFQl;gB}1jp z68c(}?0Zpw6OeP!hx%y)k|rxI@HJ|cxgLn>i#;{uWdv1paCFrv(}N#%$G{CYz4)9%*iia2MWLhO|cJ zJa`PZ0eHpFf6<@P>sdK}dNmtB=(PRj4J|t6j<7Z7bTgwZq_7>lp>Z5yw5&pfVCY3~%cxhw<`uB* z8&fTG-WNN~M6b3Z*rtt7aOmWZx%o%wUGhHCn%MsU)VO?8((kCyMt1;j)R{8dls={q zQMR&xFf=1eapqy;#KD{81vDNI zO;sJD_x69X`+8EQ*PKpUV#X7yRo!FS;5NOG-?Xg?ZbJG?0rti0_WhN#aiw~4=*(J` z8Wnekw7|RQipXekduA~OuUR{8e{V4_SD~5DIL-VaUZ>4pr(|wTnm%+AHv#F&~`lhTn)qj0Moi0ZQ2WbboPm`6Nr02ZlKHJ zr3D?N?y;!*hiGw`zmOFLs(6xH$E3Fl0XWTIha^tXuv?W6s1QaUA@>#f z^};@c3iWDJ6`WpANYmP7u~!qYz8U1I9_XGHK#g5-f5a&3+5CiB^(wG{Ag}3vkfl~? ze6)xx&0|;u^j}QGk)mIa$T8!Gw=wER8a@Aydrj2#4(4TN{K68I`lr1LoVf^AY zC^=*wB(WYeTBP2kd_ySXyDF;h3#zker?}4~DDa$sBSODquG!T801&Rkcn$%{ ztcz~bc2;km)fyWSs=I~#Mw+VD-g{Tvqz}4(h^#d2$!GHe z8ffArrj3gznX&GBJy3!SJC}X#tKNA07(Mec3K|8}gVy3qUl$j)$Xo6(ZkQ<>@tA@& z!SO7&^9@`qMy`9zwMyb`$4DpqMKv?C43}-~1sy7HW6LSC7hE%=FI$*akU@$&PPnqk zYH4CRGMW{7!=R4O`z~?d2h3<%S0Nn^tZNl&h~CLcauc#?Gq@va*y53nyvrOmth$Z- z5{JqiVs1g>SF~6xrC~hV?Dv!|+%S8NB03arQid4eoJ@9{wR_A@lnY~S)_7Vh{g=IQ z%Qc@SWw;)a+^#f-LmalRCFVCz8%A7LtmKqtM&7XoEz%4fs$Dm4?PF6VR{j`DRpnlB zH}wI$y2A2rSF7v4iQ~~UzG8#;N3vRD-s@w*8bvl;~gGG?=nx zmObEhDs5+f+{6i5e2@P5`t36xxnJcao`GgD*;4kC}-IM#`XxN(i z7zTMyxwEU;_{`%S9T(jHk_9CmEep6~dZ`FR!b2vVyT}xv#4p+)R$&Km$9dXOx4zzec59~by;meDvM~#7p}1LEZY7v7lw9pGrl0@ zYU$h7W}5E(eal@>vGsQfVX~k%jj8_T2A1@%RaY!K+{nhr$hNeLOP`1~x;78;BfFk; zE*D_7xHXIC2U~ZGc9jF%xT%|KS=9BEVy%C713M4H;w7fq)bwRPLO1zy0aTZ$Jk$aV zOi+7KDJ5J=uNu%;x*0R-tpJ0(>%qFE8V`3 zhtNOV>h4BHLfz?anZZ|+`Mh_T#t+zykFLLCs`r$My*tG+-(3#p7wZGC^Apgh3M{VB%BgvP z=h+r(4yXAIAmjf4h=oLGALS8+XBWR|rxp1=lFMjjxurBNk7y<=+FET+Nu3%vj=_4W z8gpkAQBSL+P|ps%?r4^#`z4m|jn(s0uHm`_(@;s{pbvHOb|?T_zx=PS+Ma2JqpY3Tqs zUChK#HEh5u3^V#8jhBlvDQX=MrB2auF-X^s)pKm*>iZQbN1dPLamWUoo-q-^dfzZn zUI~v$rYimw$3PEzh9Soc1r3$aHXiU#*aMAby0?v`Sx` znWL;V(4FH(d{K&_SFEGC-m^N(txhrN8!##w2(hpHMsbIfwCw8y0A3lJyTHqq>im-L z#ScJkCW5@N8q(qEe{&MB{&O43_^!n9mHRLX3s^2s|B97c}xt+bZ_j*$|1stTZsE88HWajka zeIjLlOTOThG@=8`5KDUOJi?uAOyJFPtgm^UF~{=iXONnqg;wo^R)$$ypQ)truSvUjY(!HX;^>b@+D5EAN?W4WBgr$})ns>Xq^pbiqw*w7 z9N5D<4IBReBKF&Y82l*CgO|jzuDbsKqV|-Q0y-~3wYN_4siGN9Kr7(%Bb*zBpAMafLJpt-?d<|+qaB0Eyn={voc`8dqd^qP!4(+5{{6IG{Z@ucq-46)i=qeb<6 zLduDHCOUngH&)jsuwczSNz*U}Z@)98JIckL=$JiOf*2G0YYVnHMLMG3gm#@OZY?To z@v#dtM`>1f~;S5eH<{!P{AQxic3 z?8NC(?cD4_%27C3dC6xH(4oZDJthwS07oRqtWH21-ezpv>tB+&&N!%tdq#dCCN|I> zg(XjPXtCcerY@BH#Bwa}6t;@>jl)c3_bUfh``oZ~CH|zQr#G$50M`Uw4@2r0&v?M# z&l<&EH-t>f&1Kw*>vE>fH-FfDWb}T@QL(4tH6(4eW1#&ibB7i-xn;7~jK<8Kd?3DOph~Ja)W{!ZxE2nl2 ziVf|IH5`6~y=|_S+!m^i74XXP^QFS;7V^ytQgAti>Xq5JqP081n_FBBF0|SK{-c3Z z?@(<;P^fIcpdOh^Flfsb+3E|1b%-fuE?Tr-GaU&_M84BGitv| z2j^mD{{X`Q`$vu?~P$`O+Z^!H56;i7e4ayo9M?z*w&!U5NiobFM!@@on2;DxbEUBmpH`8WnN{m zUGEB|n2J{D(=yGx;(lfu!k-D3!s}nWpgqck%U)44qU@TBA&q0^_8Ei`rRr)MY)1@( zuHbQfnBEg?KVhM4>l`rlxscbUkVH zIOYKYZ>(E+0wS+M3heGyVXHcG5wn=5mvj|U#T?yqnwGP?cpYtkrs;5O$hCPg~;(o_y8Z-v|agH9K= zQ7cCcYEiI{5E>1gb24WGI9$G-)B{Mt#(J{G3-78@Ka-nn^h=pe!~{!rqNag?+9t0P ziGoe?L$lHa!aHQa#);-o*&dne9F&7o{&Ns?j?}$cS*|g2s*K66FbHt^>(czgH4~>! zpdO4l6}o@fLY%PmE-q>p3T=z8rk<=FB`*0ctkq+E<>dH^DM{3?_GsVY97Yv7jk1_2 zop#y{c(|^y{1FSQsmROsnq0JFP90^x_bJqSOqzSk7fk%aP0oa4!1tkXt&nS26o>9& z>#_r#d4k>0YeqXW8fW5UbWad#u4NoR6k<~K_L&Vn$*YV~5I1UBl|#JUsAj{jB<3-t zhrA94v^~=j;>yj+4dEJVK^Hf_cIDn`lml3ki!f5zrc)QmST>*O=&3FQ{HPc>%#7-U^CPvwC#DTV&2mF@3lxY=ex@lD!Fs;UVO9rQFVqnHJ(8M&icUY=! zrO;vIoUwb>@k&2p)>`|5VO|oKevAzjr(R)AmFDH_PcC4muM|M1jH$I9XDWB7(!{^! z(5k#;^!mAd_IdvRV5ldy-zdBOF{IyV_S+?rNhFeJ09F72WERUMmH7u*rqWcD zNCHVjnJlG1INNXk0P*|e&&jBg-DN~3f=MYao_;o|_8?%WRTKB$H&4223au3?!C90HBm;X|U8? zqfuv%uYKm*Z8qEezxrQwv+=@r4FiUyF1Cb|dQHc1zn@#2^O&$*k;Qk3#-1BXkxlw6Cmk5HxaFfc!lM*3$En!`!x}*c z@eW7jQOW-Rn_$@b&lIFvVS9~Z4|6^%`~GEzHJ>bwq%ZyJLf>Rc$ZS3x3~uWHH#s9b z)B2*7tKTC#J*uSLp5Z`uzv)P*_d5Q#S*|r@Q;k)}-#tu*ZFwVs@Gj2ehOgjpVC%`* z<7aE4C-|PO7Xs_3KF=k|-J^efe5*E>#}2kVsl4cKsan%MFS;c+pta}v@6lhUV@<1w+1Ql5{678ub~>BjEKX~U)BP3K0Fw( z_s&o|c;`$7X>hHYtlc^H$M$HZjwhrL!Gc!cJb@-Ng|#XV%5NL|@bGwqFST#RF!A-> z#8_`N#CGA&I3bU@irot;6*^+5l90XDweIA7TjQ2W*G z7U>FU^%E%q{Y2%*T8Z8{q=Sqt*fzil|DLQV@1b+1VCy3}>< zz{(R7HdNDXAJS6P%Rp?t6X-k{^uL;x{#QNfgwKj<+6b90hS6o%lFB!_BTn6M08+yg zW=6r!i;rq%8FFn%fNU`&-w0MRq>3w{FG3$y#6e$PvqpZK!d^k2m~G8|@^usN^0sM& zYy4gQS!{Q|L_lyw%TPFD^1gWCbF9}8v5as?lS&xbL?T?_J7lHgEbuALLj#*r&vcHy z`4Jh?Ni%(LC2mN7=OLYV7ISOZ6+8MpF_UH(GhI2+w6-O<#t;>SDBvZEn4a{7_%2T2 z!Jg2DrVfLOSdDm*ZA`tSnN~YkFFg4fVF+l-MgY3M0x0+63Tlt0@t@{4r{sFP^!#tA z<+21z-wAhQ(Gh<`-~s%}uAZu4Qw1&HmK_van=(YQw8?}Ksb(uFQ-8*qP~XkpFzlf> z;(dZ@74lSF4;y1*D4|9LJs*q?Ptjp2xiWKJBN{KJZMy+1q4!Qzl}Wrd2QL$}QZq2^n%UN6GvHQK7Y`BV*al{;f91>|)OQsB z04zrHh!p8xV+4X}t0X&37zZ40`E)(#7SH(IRf8E2m4E*LDOd6nL^c~&xZOf24}sCY ze+}~a2j4BFAU#yUPYWKBnPG;DXdF=g05OxG9Z1O85YV~wJ6azz{+M|T`h`rDy$7JT zuvBzy0$OKOZ=e@(JW|7NgEuL(tb9qsg;)XnWY?kVh!2_8#;w!wjR5_o-Ja~NaGyfB zcci5qv0XaQ%UgX?3$U@tn^MtOSYU|F-eAjr2oVKD-i$#00RI6 z00IF70s;d8000015g{=_5K&$Nd+7lYf@~0H9yd(_hnHJ^WYk{{H~4(Z3V&pT*Jn-{F59fAP2G zc+dPQ{tf>CU+`b?dHzfI=)cFW>F=BTH}JpDf2V@{7yJzJzYF~If6?pA{Bh=fGyWld zd;Wrd4*vi?d7scjAIF~={JZ&5e;odiBt)4N8u34qejymt@y|ELKP@5=R*U%K`M2gj zMQo;5g_KewA^Q?#5s{%Hi3meVN96s9^q&vd{k$52NN6O#Ecjk0@pr?(l^l3ZDv+7_ zKV+jy4I?cQZ}LfWi_p~#(lHz~0uFfeU?hZtLLu-PAc3hK5+xdz@{tjQR9_6JtEHM{ zCOmu~=L9D-H|(pWhR5(vW#^PP4H@xY2jMkOrSREkp2VaPAEPEnPYt3mRG|zAxJ$L+ z3t|@Kn`i8_p2a;I@wMSAByemov9}O$^YBTu3--$jFWwqTo&?{9hBHOJ5;9*1ZTLvo z)#z>Pi474VqNDU=$o_>9d?yj_!6Fbj!CQn)nQ*I!N$9DSbkIXOGas-@H){-~wtnUj zVNsB>G|-z~Dj9|*kqFqRm=b&=Sfv`J6cj1ihfy2Jp}dMXV5r@)v5n+wqfL#>3u+2W zuNyIvvBgkmrW^KWgN_jN1s#cdWEj{cYHp?XId;b8(i0w_WCi+X!v3O*xX*5Ekb%Emo!TK^UBE}PE>`kJLlN)S9 zqPaqW=Yh9HvaK62gHK|e8B1C72olant45m1bK;=}pAl4|88fz5#wRN*lE2ZO0W|hv zY*t8Vku*5bXtm%_zKTO}UrF?hybU{(BPL9&indf|A!`HBnGo_P1^8mqSu#=0dL*yu zRq!&V(;og=TiIya!f1a6i{dr0I?&u~LPEB(VA1eFIY@1ZzW64d$DNNtVtqDX;$mXY zfc=l|IEK;?Z8k50%LTkLiJ`YV0tQo02<11H&)CdPv!})(4O$Cz8NsnzrJ`jif}yBp zcvrEiMXZUsBGXzmmVS(*`w6kDks+#W7}8rBqOBw8_-n-WkEo%tsfy`~e`6TmfjQCL zmU!$&G{$(WH_$A)WGLf=lB^~gQ zh&i$z&tQ{+Hu5CcjurYVYr|W=p^r${_8e-8iAgGp@?*kD7@Hd6VqFX8#Us#=%U%x~ z61*M))~xMTWAmxPN0crrU0p{7rQQ-m@m#Ic_XDh-{7J5tiWn zMk7U}%o>f6EzpE5aJWu!KgiIjFNF%2(h{;$1FOQ2m5EH6L^OWOd?aLyRy7R>Ol248 z2zv(lAZaZSc)){9=(uH2!$QL$;i0H2$@Dd3(<6-;D?(;Uterzt(Vfw&vE2zaqY_Do zmT1xYA`Pl!rQofx);7}u_7kK;f+XjPq#{S?O!6loDk*Suq9FZ^N>O?-J4S=oN%Tc% z)A5{bnew&6Ab3JL@B4|Hhp6@yOrbP}oiTD`yc(NJbdoXc9>U!ky$wNC^bx%zxNK;) z%Eyu-F(Kqa5J{n_p{oMQGo(cLB!q;}gKQ%VG-s41_9p0~Y-ws^eF>YugHo*x zQ6mx=X~ZUQ&6qtI6}*j@l(HtaW?4m>DrN>A8yC_g(N|BVII%>%3)K^CVTsYWB;7Rf zd`xc{M8zZs%Ohw-^d@*iqQ&5ZZc?Sw8(Tf0Q_LDZ2?S-dbTiZ^N+G`n>5NY-Yf1>v zi6UgIq(yR%L2ZwRc@~iZni$RUL%>K%b<=1{ze8)l-17eb0+v|*kKB3Tj|!N+$2n|R z?4q<@>SfV=4f2~SNX%f_PYJQ5VT-d9ZkF*kWrNnFS{{Y|nC#E=Th@B;L*!g92IO`QvlAAGmJ!>gFkPYr1jG?kvq_CL=p&8+w znHtd_8B7or#D}1_85(;mQF&S`VCs7@>}K&sj19(7M2Ni!7)WKa7ZYY2%bi9 z)q*XuHV96b6uA>E73^N)?0txIpG)-mp78zVAicxcPL#v5>Qka$jd&k`>7gq~_!a2% zg7_5ho}=|8aGyhG508rYVpl}UY;eO7OCCG?9ZmA#EuN9x7hE7@Bh!zuJld|i+A{=u>+p_6VS;56_n z(Rzb_MeYfnVF)z#W_j5O;t-F_WPWeqlO}vRjQPGF@V}DT35(+^rDbJ%A=B)A z4%GKQ6pyjkfxvPl;!0(SUMnkwc(@6XLvO>OUN^2=D z5(;}1=+~kX*pO=wl* zd{MHpnMy@;$yr5gqOyw41&b<`m6TPZVz^O4b1n(S@17D_n4oO-EU)SsGNCopH+WGu zM9fv_(jJ5jrd81bV`kA z$2}1VHnueSVBthR3Br^jQjX0HHb#*0KXh5>@FKm6rAo@n1}Lv36J;T`7X!eX6Y#r1 zwqIh83J)qb=|i=Va1-edVXc%Yc0J*VW~Ni=W2-J&FyPF=J+ZeK$};#em+57|rbO?nr{gx7p8fi@UvKQaEYCVB?aWkgRz zGGMJ~m5*YH897Nz@QjpAtQPE#MNdemNraGzY*V3z2yXpdG%(UI+9!yI;KR}y_KDObf_)O58u~WY1ll=7mq?z5kkVc@M89TTAovtuWRonH z(;R9S;fKL|3+QQenJf`>$taR^v!x0(rR1F+MEo?6ms2)hU-$lu41Fh8f@VVnqZ=hd zOM*e+;FQoxtT70-j3Tzk?I@q2dmhO$Z$x?;X$ik&Xl@Dg_%PX!tZpb@4;Vj0dW4vz z`c}`;X3v8r`bt5fKSX|t%#^$of)|ReZ-Z#639mw&H^NOJIioHgWo)E9=fV-gKFswO z*>8q~klQ6RQ{jFdh;AIpA}~~ncpuXaA7etB6mU@6ZJ0cmbW!w+V@uo`zKv7KDjwlm z1Tm=-W&1KgM@PejBS=zMB>hpP)H&#gX%IyEh(d(HAEPUSyApjGhaiytih3HG@SBES z$%Ev2Fp$*5Fw!Gj79a2b0As)R{{UtiL#}`1Y3hIE{RrY3 zUWjVKCPbVk(|aG>SJ=~e5^F>)zM;QGZv*&jUdseFCw4ZUM%PecJ1olQ`~Lt!LHi>< zkJ=we4Nef+;AC~8>`aRa6y&DHw0;bAi6@$1wn7tONKTC|2^dJ)6SjoT9)w`9UZNwy zC#GauC!wl|(zriFW26)@vT$l5;F*HO$koawMm*qY3neT3A*A$#iRDswOa1=$f(;$O|iFdh3SdC zmhFo0tvHIuO7`)DsuL>NThW6^fz}s>I9x$kSXb09gJ`}J_BzWlhc?9eHdB{5X)K8*>u2bfV;E6uu_Sv58e+@9trYlK zvXOMssB7o>5@6gb0)GDhSF`ZCDHN{1q%u{Q^!1ezmwBwG>*y6E+k zG}jN=452zQ`tYF*r-B&Ft&bbC!vk5vJnq7 zpJRZLrY9Y~7lLUe3W|~~{>8wxkt?1ULtGMt=-MFh;Gae;YV@lLM$k#E6?AihAyhE5 zKY}TI62T2m1GWU7h^`HESsM0E(W{}Ruw~;lwqfW@D7a>1Y4G05M&}0@8+Zz(DV4S+ z-il^fOH7_Wxj3=f2prIq;L;*6CNQ+TKhgE0I)<3NBL!HpWjIsY9&pHeQYE%W{{ZN@ zjgO~4|DyY$|k%&doHpi9? z=vSmO1x!4_x<_eUBW4(>Q6Q|05-sG)Kcn!7hx7iLQ)V zg=?hfhk-v#yoQYpLA!!SS%sE7>5jC@_E*+FVYlp25xkGu6nkZx7_VszJTi$0x`da7 zIzm}uP7XATv8jR&Au7E)lVWp&Y-#XDwp?SvSQA<{(AB-3hCg8wq+zS0VL~*J);Hc9}Bzr0`FeT(l=-!3y2`ok(8a$DtkJR-Pxq)Yb zPjIWzcJf#!>`f#Z&`lo7Y;EbWUA8>*Y4l00_)8cy`x}1A)W;8I8yyT`wqtFNTN9=b znvSwsDp?a}5-`z~H-N46Au!h#5g_6z~whg_)@Qe@KO7@Ci!oi`e<&8`cCQnBLg0_*hp)=b*b0sBv zAZfxb5x}ajCYDChV&F)Kqdp0B2FBeJV3BSieh#HdGt%6NdS**vNpxzgG5ZnhOhR1^ zE3v(sBIIzci`y|o$+wMak&SSWYUEfg*s!oSa%^{kO>F-FP)SV*{1p_kC)QL&S1f5g z7LBxIx@2pk_I}15PheH1W@LMZawMc=^g1-3Poav7qsqXwqOqp7G|?_b_;UxsXJTzL zxiV*^bHdJ2h&OO&RQ}^akx!#ZU`;_Qu_+07!r==-g`-hNJ&7%nqa-mf#?*zIWj14D zc)?p}O}AkyBV82|x_v`jUkTNCFGu1hwqc4r7+Jmz{Ttw&VoLOQF}6lW!vgdYnk7`; zDUj1xMJd@J6Sjs5h2HL@fjBS-4VDjO?<1EC2t3&X<%4bk)dJ2Y2yc)q8Ke)4x8ra(6 zU*v9)X)-s@4P!!VsS&&w8YN)(M%h)#Xvmyym=g3XlczE^ky9RWnVzv>LB0v3j)p|` zEG?HO=%mJ#BY88XV2_4)Hu>Puyp1HpjjW_xdL{CS+(K1x4$eC`p4pjOg}5 zQz&W|(aL^>dMgTe5=~@@D8+>%WcVcD!;YV^MWH4%*_l!Bouj)dS~?-^i^iT$NLpt< z32f>Xg|Y1EpvA$R3;zJKo;bX0J&R_~!4E@bdlv<t@+ z7MF>@g{CcwX2^RsG&f_>sP={8S`S0lgEV-yL)f%%S{~Wau=sG%j7Jau!~iQ100RI5 z0|5a50RjdC0RaF2009vpF%UsfVR0aVp)gR9vBB^_VDZu6|Jncu0RjO5KM?-_G=JyM z{8j!@KhAF}9B=;s7JrEU0Lz#1-}zhr0MxJWf5QI&>XZDbe}#X^Z}`*y022QI>bw5{ z*dP9N{{Sj_rg|xr{{Vu}PUv6CbG~*cJB`KuS>pcyDg4_+5|PlxQurFM-}YzJYTFi& zT27Kx=+(A#Q?Btyp3zObX9kr1s zLsp5?C!-@HTnV2=W0jGI+bVsEr3`x$Y%M8?LU54DD0S-NMjNmunyHklWql2^$|7qn z5OFbAhQzdFA>hkJ;CzpP@!%(kp6<`(9?8m#oQu%X5v;R=^kOWwvX7p&caN*od!PwQJ|%-gJkeC zjXxDpBka$yyV#=ap(5;V=$)~Cap;D78Ze2oV_g~OA%<4E5c(Uzwl4@OG8rDI%9{TG zzKz}mIv3qD;hAKFxwwE`eZHt!!Fp6J4y+=4f|qR3#3rUSucofiQBCmXm6qtZ(~PlL7~z@t%+nzc+k_`D3N4b&qX#Tq=g$hVpe{j(t=DDOc4l0(F}|&j&jDH zQ63RJjJAkXi9EPXgY5`%!V!rUNN9Rm6(HA4?$WD~qp_pVk)b2(`N4e>9tCJ{K(++D z61y9=hkFO+I6;SoNLC(~{IWnPw-JIVH!(i`59+I@>NLs{ONufZH<9CmN$H6(Ibfb4 zW*P8OEp%UK&}f9P&mlNoqi?n~cp#c+XoRe`D?!GJX$m$Ib)hohj7rhn3D_PWqIoJB zWLOeuq$EhP62Y#KH1I{xR`lUjRv$tfXzC@QyTuq_ z`)G4P(@XsY*3MJIx6Qv{bX~XmiJG)DY7(}LvY%(s3u9U#{2sy-VWZ&{_*bDMWve4d zX@qNoBqz6o&RDkVF`|i9BX0*E1hSSVIzmSJCyG?nPX@ow`C>(15QUp>_7bAZ zk7B_V(C=nQX1JY%i6s)z&5DIgs+o1cWNV=t>|+VT6tt;439iaMluRiS<`R1wL�A zmJ-19!Q3Qto(hT~IxBSuC!!deAtpeiUr{eZlr6|qU}}lhO5qU*NPb2Z$mK;fkvyVv z&WK%zcnd4wr$S>+*yjT@!W}S@?~a8*qcG6Tpz_$%V91n7WKohF!t_-eCZGNP z0Kf3(VI`bB`pY%9;u{h8yh4tL9SXLDgGpdYTPYC`RJtnB6?jI&J_Aoh@?PUhp~OVi z$d<;NOj_9tLc(P($IvBYt|5GkLtJ$1RV%4WV)`5nO(9XPi}pN{(rbym8^Ua`DcIPm zY@y_7A|Wgr3uEaLtTSD*`-F9**M8K zHre_n`VgK;5RRi>?HUBv1s!IA62v0#rGcnOY7N2UQI7**G^8Pz!>?tbAYPGMXqXw8 z30N!WPVC2A6C69R+R)!n!onzREQY9cqSCDmruZaMv|h|*!CuO4k=>q*N|uRlMF_)7 zqBpWMH=PR*_8T!Q7l|Gd(iycfyA!s?gu(k%{{VqzBB=WFV-9OMKIoNVwGh$fjF!h4 zr;zbuOobG0$3{IAdKq*oYX(NDkj{*4);4k>vQ`J7G&f{PwuFg9imFCL>qv|=^2JI* zuFSg`p{tChj0WLBsEY(Qm}Kf^tkzo}f(V%0R6|s>`Wc7O!mlDkC88RHa3|dRkJNV) zvPYZ^15+U~f}untSRoSdk4(#%UEB_%E7kc&y5qSjce2&D6ZyC{7d6TZki7tw1XXR$&uoGR>}Pj5z*5mjViJJ8z$ zXK?r{)4^DqY|Fu2!CWr3RN&pg#W+wlQq&}TJSV}lqdw|M{*GC6bW1 z3kkS3))N<&ez7j3WSxk$qM3Vtu$P*4{{XcI zR)n!DVS_dU8AQEzR8!p(HX0y6LQm*D38B}7-dljss}uzcO+i4q0@4X3)X-#JmPaoh6<~_Z;N@wXj(IcOBBhg5T=Hcj0UArC2q+b4sg#Owv zlY13Xmktjn+%eqer`VAOT4Vq63rzF2{>@O)K!$7F9VG-by-ARpy<=W;VNgHFzj`70 zAS(5=v3=6*VLO$Gpio8mKz;lUECL*iBfh0>_p5x~8fYyz@5ht0HW_-k%h}bRuM}jR*jAItqT5QN>bR;-mX5P=*E+6$_Ykc%?9X<|3P z>BRaPZb@#jl7kTt_m1=Rne{MkCne6@6Iuk_e0}OG#XK_pc+Fmbd~r=T z^43_51KnTe%Wfxf>pg)+K_@k9Y*T9W*+Uc_}TxGA3 znw|TIA9=dUm$R?Za2s>N=`q9BVf-9iscWAa8>mFzMa6E>aLJo9diDh!d+h4$>I$58I&9f`xnn?k(sM z0s-Y7PqB#K&v-L5+t%ZmX1^L}JDHu{3k}5)iu*n%_;>1uqZ4JAeatel?apXpA9;T7(k32QTtj{4&29WEePv|+>N8qOI{^=D;~!ijtAL;(b!h!>as6A+ zN4h4Nx^D>iXCo?5@B@$I1>&k#!k}|H{qx%Xe_;3(e`o4KBwV8Cf#Ism-T4+1II6A# zd#nSlIPXe)kSO~>d{r-sz2bxTnElFI>LY1*eOws#xRhs-k>dF+hL$~XAhTbGP`E(! z>8zfb_CnhT?ur|0am7(S&@T&=u^X0g$wCMj#0~05GxkTqJsY!7NTsf!EAi@C_t!;C z?ZnHxjZavWy%$Wzo3vFAFwRh!G&ZKiqlLMSg6l7m;4b4%y$`5O=w7P#Dmjw&$kLfrCiMqDXx1~z;?VQlV5feb zAA4HUrRw9mSK8o8%1UEa`SC(Wew!M68@0`{_ z3_rAc;`t0JZ`~B1=jGB)vUIp9()0|xP{Q0+XT^1DKbJBgywbApikGg`$ET^zN_>MN(56x6jh678p$K(`K8l3QXL`zI-;Q)@PPllw)^%j5Xdm1kcr zkOuQvGx&>bDvKO+DSy8#kIilU1!g~k{iB)2Kw(Fp&;e1C@F+;fwdRlQRBie65p$?$CzX4i*$kD0$eO&yls z#vhAz95MvE%Tr=VU@1lbb{zm=M@GQGSS%(vjlSxr;+pMPGtuk9#^r0d(%h$& zaUD`FmRNYf8D(T8zGq^-nSWm8I)(M>*|dDCTssol8jyGFB>Dm<=`ntPVl-!$75q<< zy*}>t!AN13n2hjT$XsMSE;pbZWJo}%A6~MwXS+F37%%_fLtq^BJ$wD`P}#T}D|0cq zpT0=<{KvC*&(l-@cvJ_kfY3vk`xlT(5;@6r=T&I56g~&znVxG8+ZkI(Jn0muQ%ing zdt$R1h_cLoCcdyA-InQIpK~!=eI-VsLWt|K34M>|VG)^2YNku1j@75lQuWZ7@A1qE zAkp51h#v(&E%X1--#;Qw|AAkz4Ewboy+cRq&=Pj(<>Wj(LwxDm6w=xuIYn5Zc^8vW zj2?$)u-VZCi&MZOHHRE})@i7$M+vOU`VSO>DqAxU6L5t>UQ2O6jNFksct0J`5`*h~a*ty2B<$5M-rVApi^^G4 za~9LL$Xc083#HE;GLw=MCjGLgw0q`_uh;^xLirPk=2#gv0{WozubyruNlQ&;}E1{-&T-{JO9#0v*p_wx*})QlT@z6HM4!xomq+ zf^YQaQ^ZlDFXO!8-0G^O&$I98o_6G@7qpUu13orR>(%sX$%2LMJ1TlQeZQ(A#Wva_ z(bH(c!o1m7_pGkbv9)N&-F^O}hPvV39N!72I#zP-x?{%!=Ix$^Z!#}0P+YLrwOrv5 z+-q#+`nd@=eS9pb8!(;9XqQd?ItaPkJ?e#cr>lj4LwZYHHDBSE!}bIBpy z8M!$#>7Pn7;aI33f&1s`#55pM>{iOD6RrP_?*T?XdFnU|_rix123JS1S9=6-YgW=$ z;xjuW0}A#XZi;^oDY}y$ELV=f1e=X5!oG=DnQmM`q>{sVjyhKkU0WyF_Uh}O^8W{b zkcuF1IVQ9Ef1)Twp)2DD{DX&Jg+!eh{K}fW>%e1H`l|Z5;=LN56mEf91)U-lb!I%h zoL{^1bk3$MRSK86fX;0?W+tz2c{!6W=~AnVQ?tf5v_YGPvdX@oonxN4Ha;1Emr(nl3$(B|?f>>a{ez==~} zRsTd+V%2&=tcS_h^UkTFf>0|aHRBYWS1ETPPN6?>p4%fnmz18uzVRN9U~zMZHw0U; zS8B5-s-X7cs*mZpzGOj)wo-*G%Veie$|*kAz3Zu)x2%Zss=^qw2T7U5bCOwGI?g$L zzOWM8nCC^%z7TUWrPFC16FaI>Tt(%FO6zM@|AWY9iDN?*tea>zOCL+uji!S;?ZtI{ z+qx2Lr+oei_AO>{YLsJ2)6V5LvhCrV#bRB0ff(udnk`bGIC|6{N0{#n7z4C}; zZ<95M)i|#U_UQ^X*uQmmzdpG7agsgv#ciY1)4U7;KUWWM&bT!j_9J(#k-%$6b4H5o zO}wIkMYl=3#XMC{oO%C-D+U7{QR>wR8+bH`@$0CxVw#dYJa&eF(wdVM@Rp+k9ckrp zo|B(}iYSJmB(+WVKydBKhgZxOwsUU?C~V>Js>M*R`-^Htx!Mc{u$W z=sY=;NSTaKWRZKHQ7}eto?~x6-%R(@~W&qidg3PeQpVbHN9rF&$>XS@rb(xr02=UmQx{c_6jcP2&p8K z^cG60EM626F^`IEF|=RsDU_z&XBc|W9nPK34_33wQgVuwA`QduL#7I??l7CY|F#RpHD0+>7zK zKd6VJ|F8V)|HCK;{C{vX;QyDOWtu;NZ63>L{a-Yi1X}+OjXokF(335@W{NpV(F`Wh zRPA?#&;3V*f}%;sCxNTkW6dk!^TKOkQlZ(BCg~FQaLeZSwS4h3NhGuW+M0A8Z7M(a zQ~D3EFHb5pV?Hnc8l6F+ei3GGPi^e{2OyOQZ7VB9GhcBkkN(m;+CDES`FKUbJr6Zc zmdpZ3BgRNKkdk?*-jb5?tw3+#bItVV0YBku=BtBiuKxfp-2E@>2L-lA*%|6NwR z@;)zld`+@ye0)P@KA_T2k?`FQl#+pWy6-q%##YVIU!u1J5-oa5Jb|1O#N zo%5e-_K_^&y1pwcIO88pDBnKL3GGpyu{$E&I(mELKfvYfmaXW#Yh}`a@~{5?0kZxB zP#>I^pPQMG%Hnb|I7hsNg|B5vMuYzYq|GQr4+KVkQ64!742a&;JTBiHA14{LD@^L> zmboh*UqaH|CA|mSgs=I6KL1iyo-zOR_kVuOXearWWC-)sj2~&`q`68qNpq4i6p17! z|1F+&-`UvNjre@keJxCSk*3>h*(EK%EShkwc*V$Beg1etlR(n;B_aN#&;GY5gJC1y z*D55Vc1ZJ%Xl_Lxm#l}5XYPh6qppr~{_nCUqBJw2;}x%FN!z)5UQ)hWew}s2cy)GV zezi-gVNW9MAp6zquckn=9_Ac@Wr0t>%X|v1TkWW4bf6CsPuESwK95m-zWWs|>+zLw zU9bDm-SnL_()QtaWw$v;&v&o6u0u(FTfLe+iB?%}S5GQXJxdJw$NoF9Acy7l2qjKv zLWePwal8EdQ26}E)&E>7du4tUOG!ER>Kf5VLtZL;J4s09nxzMKP}T;pk&?XjkvaF+ zn#)(0ID9J+9vy02XONbunU;Azc6L5e8#}SWrp@t78S_TXNT#JCOA{RLlP&g5%R~aT z$Tr6xCShFjXxx!g+UEuGE-`%afcV?};q5R|IfF7^PLkHF1D%PT<2`Iia^Z7tF6uPL zQAFtg-EPJW9r8FN-?mpjlDH6ZgOjg1@iaUj+Cfy3p6%dW!;i;f>3n(Fo0q*AIu}DK z{JJrxuH`Fe@rQ5Eh+h9%sW^5_CHmc_Hsq4)`kX7mx@R8$;n2s#a0BJEZ^O21c(2OW zqf@SbP3~O(n%(>l5ccILy7%N(jTCQ_5cx$ssCWN4& z<*4FB{H%33b{vDIrDG)%Yy7YXG&QX*5MhjS@xp$PG7y?%>1Z~eVBFRr;WoA9BT@Dh)k3*cSG8tg@F`VJSf5$i8L0&lq51d*-DSTO|LAwg^-qPCN z{X$benh6`S8hR}YuBKSqVs&V@j_H&-n+6*dLT>IdiN&I6VMtr4W6R|DvbG^Z`Fd~Ap7KA-&zu#uA6<#K3@Rw z&o_VePu%;d6s5o_FNFQC3^-bFHq!kVlA`T?F=kUUYjUE2b{QQp*T$FfD}(7TTul`e z!^ih7XO9AB`FFDbvyFYQ%nvBY)~Pbmn$EX+tJ-)whlkR<^GQPXlCvkHrY@BheIUqWk@_C0ay!Dm(y-i;cr!Th?j5H&izfeZzus)aHfkLlJm4GJhM}wx zGnIHyu5Z#TV8z5lsVTh8Jn?8z)ca$y0q8x0h<%0(97X40vB&f9m{9d4H)G*O+4JNS z%5bVp@gp8r;b*#Bp#ok0zBR@#$s=cW=>@AM-0vm7Y*J^H6Pi{ZA0f`Nrg&LW^4kB7 zbNUA>#ysu$MYv~wJ>jbcL)vJ0oBSvI9j5r>-LGOBUJo4fxeAj%)L0yR3j5|kt>CZ~@cw+et_HdjT`)m64=EstZ zLq7enJbi{WoZg5293x#j{qgb=fHq;EPK!ss!~IKc!Gv|)kygxv#8fLha+gW=aj{N6 zUl@D`@Yk?;^Lslqph5ar zj7y#6M~rwdhN=oYY-MY-BM^DC?Az|-9(y?|)m}4+nN$zy)mbj=Kauk0UCM5#YOx=d zO0Jq(?n5(kp=vNND#n%!UVKLmCzjn3=7S zN-Q$3>58MA$T@X~l{Jd;2H`OC8Rid;_Po`if#Z*cF|Y?Z;Ouu9Et`ubhLBsVCg{qNGg$XKq>I&^BJ}ZyhRvi|1Eh)sDJ|e?7=9I zpW_Xjmt8UX;5_j-BkbgOIxZV(ZoqyZl)+u0 z8WOP)<3j3F(3`>6QDajc27SMxho-Yo>&_byeAUiU9(qXwO>?AUK600r)3e=(k?$$p zh@Zy|em=`Rdd8~B@n)Z{I)8<93@yRD7N*8>WcM$QheFtf6GAN2aOM`x1`!-MER*oL zWin`t7BuzA+|+e9)l&HF?J!j>LxrN3mqlWwoKVAwq$1}H8 zrv%w503Lu-9+sqfa3dH)JILCZ6)eO_0!S)K0y;Z;R?n2iU8n}LZ6TT^V*M2+_j4eE zbA`;SSr*tgx)QI`tQu8XoVL6rsDVVT)vVSSjlZ&ds8)yt1DD{&E-NLc0*~D1-ue;3 zWo?s+krn>zKBn`X#$qTO)Ms_h*fYGusU@nG`yXH$*9s&Kq(A`6w+6^FDuFRm{%9SY z_M3EE-#b87P5CiHXW0Tb-?>l3yZ{d#%%3G-1RVsnk-7sGRX@1L z)xt80o|x@7+*anV%FX%)ME)Ia$~y?kmS%r3nC5+XihyD=xeicx<*Z$aE?0 z(WJS;<&iQD<)I>U?Ri!~XB>q$hYXm;S)Bq-vJCLPvHb*OdQ@BhjJz%)lTL>TPsa49 zUqdJK`OuYW$-vzCl!Jc)#S`jKm(IAI$ARP$RC=sdUHRb;&;&3{D154QcBVySSjT^v zlTR#skKF``t1}s)WZRP;p-1DQfBgsW7i5%&ZevE@9>DdrpZvA(7G)yRgy7rnBGdoi zDkhRSVe3B4vDw;B`G5Dcm#};yXAG*V26UuDRbJkDGaqL}ZZDdh{n`&m8yhc|PNZy! zfL0D!*zcw-m!bq>u-dY;2i!x?Lvrq!Er_8wI$)-1#Qn20r5Y=fp?2xtrN7gw2?+rP z)@e%;fE%7P0{8vS$TU*Z;1!)MEzD=}`L#dI8L$=)hNMlm>lN}OUK=&;Cu_((aW-HE zNxhVj-(6`H65z%#LM!`R0B!Z%YJmq7c0(q~!KDtD!v{?M+9GGah*;{>rUT7ENQ_Tj z#NX+HTR1)B+jS0xakvV0@j%ckaTAO;;}MlxFmdf^eXtx~C!Zia1l5d0RhkHD|7C{z z&S^a7O9EGb#7bhgF2`T#lJQk&9DfAqP!+KfU);*EN*1i)5uma5G#MrzC~E zP<@RVU3q9Wq0WA<&OX_wN%n9*#`aF7#zUoc;@*+-BwEWSTWISLB9@~{ZeQvmBvK1X z$(OeN%+&lN)BWY&K;Jd&ON2DxFWUZfGRWWjsfm{vg~h(TJvvZA{k$*Ma5*T;5uk7^ z$M&>%`qfxmGQs}JRsCszsDsOwi?O^X*52uHPb05fL%g8)Mu$8V_GQ6oq7Va4#a-ih zc16`nkuE>eA9TjdKSBH6Q(dk{I@^$Y4GuwyO&FQ4C$40R_gdWEVh#i%m`YT!1u@CX zN*7xVWZ&pM+Z_P%3SVl9bNvl8BBR-{7kg(iD^Iu!gz-APKWmZ;1+nhIDCM2P`sqrP zBr80uQh~|+zBObimXAJc+#LGz%P}|~G>GQNf|}`zNKk`TL;!0v?6!@rD@1vFAm`C;7a&7rB>n1REH2c7V0kiGhNdHyann&NFIb!=yEza!D ziUqv!VT2NtE^PP8J>$tHU*vX;GYkG#!Gv47R8M(xN;ei@CitiNuJyS2-1 zk8T4#^(zrNhy#~@_R%LZ())$yP~^o5ya?22<>uBM+yMQKr)O(Wj|s{CW|51AKWdgq zy_xjxv=1``h&7<7z&&ZCe|+n&Hx;hbfZAlS`6~@&aeOzBOT0@%M6BoVBaPx%zA&_Y zx0$U7o<}wFzGrA(m5ZudKCvUF@?G)SmP}w*g=(V8R>?iBqGElc_(R1?k1X->GY*Oe zL#qcDW7k~upF>IPlwh*yb#=`A$nVB{_76UPrjdKD{w9zh6p*b(WuX%6~cwtQy#(z;(!O5%NcNJ-#ZdxqDc^x zQxWA@(9l-1)A*BqjU%-@=hS~N&U;_{WxoQ;Vip5!^6NwUV5(O1yCTncvlowRIb!7A zs;Ig<>H$k0`(BLAJnj~M*MHym8S|iGe?;i`%dPtFSuIh=0d#9)Z#hx#9gnr3!t#0r*7j59%>Ao6&HnUIbkrET4`1RLBURY~#MSnA!77)ixlI^t{lO zki)%MKOSwmCDy+E%rzP9P6r7&R)!w24fwOSYaGkdnCxxDWj7)OAV`C(NLNawGWqS$ zh^M%kDMt1b1^vLkqXLn1?mdk6R|vH5T<;$L&bDVmv;U5XTrv!DMZs5Ycr+Zg@BB{G>Bp$O~ed?Vr&kI>(IEu4GP_CJ}A1s+LTaW!%I`i^Wt7VjyCm}8a zamHFWmUsFWt|$nw(PXHcqtqVm`R&;^^)0gI^r+|qY6i+x{xmi%A^3Oj74F{OJ9Zd) z)6U&zB@vpxKPK)<59u#h$b}JKAcqtxyE$XKQLID#;LwjOzNz# z1mmNIB|&l_Hu|5Hc?9VzLd?#GQePS}c%GU&GI)z~;ND1-=cm1-0<~vyKA~rNiyBcV z+M^ls2qqRr;(&kp)jkaknMp#2wZ<6@k!M3)ydb!$5G}pVg%o!%;$oB~^`=gJySi$f zo)Pct0gM@xaQnXS8zXy+soBn0qJc+C1VPz^Cy`1R66%r^TDeGZp!7Y4CXbq1rC98k zDY5STSq?CJlZ_`HAPDYRzHBy5Xi(TxhZWkNXy^Y>l5yQ5x9hq+VA0p)$QNIm-uo-= zffYHYLshyPh5*_p&S4?R2Y(01#dZ91D5wBX=f6k`n@xboyXTL?n&>O%jq=FDwG@}3 z4~Ru?-s&^9N?L~chB6}w!0i1DrIH&yEOUjqz{w@PfYxzl@2{N6pU#IL1V6o5lt*~& zmXbVI+-h?+kgcxc7n&uB9k=nWu`yZp8)1CMIS0$61y(-SzZxpb5wcmp0bAJ&=pGH- zs}Vs(03x@tWhHmQbC5&S%`bV+Vi~R0Y_{`VrLg2JF~Ur3#>oi2D;WyUSkkkcF+$At zTYdR2LVg^iNW~ou!&ge{cC`n?U6fAj*LD`2T2J_|23CmKqEdb12Lv)23P+uYt#_`^ zUc8Ed0pbD>qjKz_)PP&A5r+u!`jT<`uY&MzZ$oXX-V0wJbCzU`eGN%)#U2fro9OZA(5SQ?7P>Uu&si080M)XgS~SQeAMxylU<8I-%2 z_hcfHC2Q_)32lBJ@wBLaauZ_)z_3fUECJhGL~KsS;WRF@cTkoK@qSB2QTnMlgIzBA zh7X>MD+frvPSX*8QQ;B;W^(wx^*%sM{CbrMLF+;H>HSWuYOwG+^+rju>E(u0rh(LbaMrt&1>5X6sC>A9?dPM#6!VlVcdp)_DM%!*nU_nouHSWjs&Xw5F9b^mIuVT>_&&i zdXX8cbCs32#{vVJga9U=%uSxTnoGvsS>2D_t}>J{ye#qV57T14%^~AFu6WChwSTH# z23EMum%c$0%CdVq4M7E@zLJ7|?gy%w0W;vi;p6>fC>*IXBH^P|m}y8LFj0gKIjFs1 zkM5kyZLxp;@1FURRxR_LM;o>Oo^F3kAt8^4#cJE+d|H;@T2Jq3B|W9;buFWLG8<78 z+{eC4PK$)tuN9shr{incGgXHrnbu#9&ImDQAQrp)=xeLMqxD=~!_T`CFI~pisTYZ3 z5JTEht!5MTS~8HRof%CUTO2!+D5sck>C~IU3=F%5i!87oZXl|9VkP^;wS` zB&3%O$KWOk;`H7C^AO|s<2f$mTtg(D_770&L#=!xoTWp}DC&$|mbz%xGWoo&* zw7#!Ix{!zFBTc*5$D522zmbCot@ABP@iOB8L41i)q>`N*;N2a5h@m~&CN9NI>v1)G zN0R06%=vhA96kAwwBx-HE69hYiP4x5v%ARyw9_%HOa4NvMiP7SDg!zqHs0I(9M)p3 zO`R6rDj&w3V{#)H&IOW*n0}~RS-0dnmlRPK;xEou?7gkZjm3v)+VI>HQ8zDu_s#J; z#Y8rTqB<-W^f!K_ACH)e!cAQP?coDIf{eC->DnH^9|?na@4mllGo zHp<2cLh}%MW3M*z(#P(e7ARRs!0F_R#W+8emT(*F_YB7(F&>T4XwOLrhy!J;B4t!Y zg6e*hE9QdWGI)KD;I7XR*`?U}t;VxBot_s`ISeB6oFC2CX%g;vDks=Uf}C38Ehr5Ys025RVhX+6MabYCC{2A7d{9EhxN5 z(vwY$tuw$Xgt9bGHB6N39Ua<$1>YHG>;5_NfCqj3MO1u_T>3l08A!evr(u$BP80VBl~&sxYaQVzLsc=35N_D2($khKa>X?jb}Ezv z8psYEFxRrQJz8L7u=}|M4bBdn2+-Q*+B+RDR~I@m`={K)L1h5o=WJw)Gr-!dD+P!# zIg8m(OPmaJbnb2Z17kvR*S<_z=?TQXTf*AT&#M1%AB7DG3mbnfF{o%4{i>A4;Pfi@ zRN^q7RF0+e>>ti0rUJS18h73)@sK&Qx8gbZ;q3498e7y)Bs6TO^Fg5FDc9O-lID0{ zwNUTrm;llAyZWW892VRL9bBa3oXRo3oFSY4CCkX2SugNZpQXlkjHgP^US|%+5EYQH zFD2b+t{X#Zao|72xUAGtQ%LUFm|3D?>_FBdEI}z(p_6?X6lS1pr7pv+lKeZK>FEdp zmoC-R)4z}Gf!>IKT0GrvZJpSkZ;T^L47?Po(Pw5dd{O69^8QB^?3S*OJipzBh^0=V z-tWok@9g?1_JP(RDMi{^&^aVl>B=od9N2n8N613s&nAAPwm^>1m0>(q=JVSYCdWjQnidVa?s7apO)hQuE;3N z_zKzYHm>RYGnCyNLossUzM<1zP*3I7?m+&>e917cuZm{^Ae~k~`SJ6^+sf!QdCDKk zx0<|F`MkMdmcTMM0RaiXZ+`Xg?fMi1Q`shH zOlnynj{Oz#?Hm7%lI+`QL^fWd2w7q<8teA5#(pw~nb3o>MOvhyUY{0k7$2-_wEUYdR9FQ-vR7^0tT>r|FoTe+T z@P6~M4*oJvEr_WRHR9zwMhJp9U&~Ce>0xBXro1~}w@V1d5oKCmFNx9UQA<+yRL_WF zEUI40(m(7qO%#MBJ#0jC)#ErjN07}8?N|8A;XY!g&JZSH^;eEa4N(O_|0-yN!Ojb? z#{H!1ySoM}ur?#z!73n~Mh=+a`US1BLB9I6h7tL2)R+EAQ%HQ!vHDS$>{P>~URc<& zlI0?73|$;VSTJs+d}_<^kO1xzkpCy%4BQh@+O3j$&Qab?KY`%Bfprz~MUg}Gp9mef zgK7qKjnu#Q(-7#GwMsjlbP2GW(gd(#LxBGQeyTG8pugh6-i_`ewaLc^U@3p^1dtKu z_EX~ZHmtKnfugKxgNuSTRZ?8JHK9N7#~|p9fHz>}fpsR(!q^QE)vc$C~ zez%myuWDkr^Y72(9FS3C;_i&Mx0GlG=OXZE&VckR_X%lh{n4m)*{pgmJd-rfkHH&g z>=;emz56Ok={NdYQ`{{ARZ>Rch81L(`es+-AJzq02WTOZ=@Jyb0B9b|894OPCFtIm z6e%b@T}J|9vQrkdZxxi_g>&;Ce^)cJlQ2>-LQwWso{bnrkUL=W2tALv*N^SH4M(6} zRfXIG+CkiPUqrt;(^Vw{wJa>mjzZZCh6J+n0pZ>G;#|ODxL`m>Bf7n`Z z4x2S*T5sT|Fu|WlYkYq}Kg0jd04<{esoU=Y`la_jhi?1sjI>zl41YoyIi1$Kr!Pzz z0=afybqjT)nG0FTm|Z73O&*MZYusl|HK#WX1q1z)@{Sji$GfKiK24Qs)Q-F_pAt>E z*}IEBW8!FxoP5)SZ9VgPkfB<@x_fF5l^PBRsdr+nMyf|{<+!^2h;4iRBWD0A%8&tR zr>EC9rv}URS5LtjLmVo~X70Wf`}Qj+HfG0Q;Nd`{RHG9{?ssXOan;vS0*Hok1M%Jw z_8NkMfLGEiN_FoO;!IR?oZn&(W9b}ErEsW`n+XjWCi3f78vqNdVcS@0D^9+@w9pfj zgo#_cr-M?kMIPwkN4pd#_&Mzws>w2!Dmy2eAyDGGt8aAZ7x%8djwVqrB?l%aLI%3rn)7JH0NxY`boU2<%s; zn8@Ehh+>}%#ec;B`l^&Bh;j3yL3I+@N8qm4e@FD}M1AVUxhOVlTQ>}4jV|_6`lvw9 z0mjPbC4un?9A@Xz$^Fn$pw2|6MpY%vA%B4u^ zVtWtJG>J(xQgQ`NVI^N?4y0|~=^ZcHi_Xg&A-Y#Z1^_We6iY@jlL(UzQ){8=lg##G;UcA^O4Ew2o9p-JCiU;* zdFz39Wv41Am610fO|CPPmZ_rfhq?)798`2r*pu5Z=d|2j-qu05lNS#7|c=cQ%oI$1d*Ur8R339(Cd50t_vj`{{f;pOl(Tc?NjbbMKiDh(l`H> zgb0JQoS(kMqh(O8F3W$$Ehz3?6@(h$AA<^zj1Q!t#;$PXeq-TTW@zqyzikju(*dq~ z#5GP&=`Z6FdY}Jc?44#^{t~6;7uQN1I0n~i|783&83dGU80MeA_gkq|2KR!m2P1(| z&c`?>gm>e#8Pg-*ySjQS%2{>Q^o-NZ^yy3%20Yna0m{T`dHo+PTMy4IbR}l0L8n~ zJ-!p8yt^ecN49m1Ug6m|!QA-|@X9Om%;YAi$!#=eE7Csa+$E&7@I8)PQA+qM*JBXS z7h`CjOm%5~S!dofNc&s&VoWp(3+(031K3M1 z@heKejw&zEg+j?n_daY=kYXc5>xYWJBZg^Ji(k>`xkH{DdUs0YIftV$csz~ann9_% z#D{G0qX$FaW1FXw7sGX5|kJ1CKHanKat$mFT0(9mu@&fO|Lpjy{{auhO2S z+2m1R-By;p()Mb5EbAqil&jYdd!YL*@`5izeYTkFhX%jXjg#~Fz5gC$2%zwh zhNL)F8^8HS?#uWqU?xMgatyZO_2#pnI8icf=AUjXv36lK7bBIxT8kj_0}wH`KeY6= zqe5u~=*N_b!p4f7SJDah#22YbpYvN}-;}jZKJuQ3af!5vz82+h-)+EX@M|33yN}f8 z!MfDyQr`IUF4#m#A`HU1Ui+uPKu0m9x+1p?|e{$LJ=poc(0=HrHWl#b@x| zj<-fkw#4uK&-6*p5=&~G$zT>!8fM*T0F#ERT?SFFyDx5zx0BI=)M&}w|IvW}BHi?^ ztcG&tDJ8%OjF_A{UtwomAr)Er2u#N;Hk!d$lH6YRkSoTJUi6swoM8a`;hu|{kVckp z$B6V-F`f|NxZX};*T0d={i8=l0@zq-vFjZ|c9Hf>5^ z%a<+_ze74xiKZgbfbS)7RJ@Izt+(FreUI-GdEa!o+y{WucHS6cEPeiYAU%K!G82nM|_ z+-uO3ogd*Bc7^Z?UhR0MR~5>7iCheb$2ABvc)GGBV}+l*5q3V&;)}fvp+2M?ui01k z{x^dl)9^GK?6s&DwYi%jV4v_jUAFk<4<%PD6k0&!_NHkTx1yeWxXxPN6W%S|MXkl` zq)$c!f0tr09riX&%vsE0T_q=^Hk1a$3)^tMc)0{rA&M!g&>6W(nW>Zg9g=(@;(XBq zwtbg7);cun7pANr@=2{@8z$1HpSSw;%G#cTG7jG}B{MqPTUdppJrY79D*yVt+f1RA z5lGKF!@tC`)Rfu1P==5_!8p6>m-c9R)pUJRj&#c+Bcn4wz0CnBFRRSQ>xaqmr`oaO%WAezXUrL-a^EL!>{IB_;;Gg4b+RBFRb%RIkk)Xq}H!9zrUqP zIZ*G3S9`flJ7-_NGcER#oKL{MVL`BEWyT&;7(PHWkFjF@>6jt&lHC>PbO4H@nKVzB z#+kLujfaMy!uyHG-!;%^kZ0^Kd^3?n4U#Q3A3ZnVoZ8MzsxlT}1T>&GqXJuh$WX)r zM^$xyXH9hFkB%5)5e7$fGUYq!N1}(&+g<@jTWdEi2Qmw8xXzbG#sewJBsfe&U*2J$ zNUk%1!?<+#KT-qWSn4J60}EP4^39u+tHB$Q?kHcQ+u};Stz(K+-d&p5V89uxGE93j zYomCaR7Da0_=O3A4A9j7B+bFb)0Obu^)gxK6EZ=WJibc&Xj4Vq&@~rR>(eBM%pM$E zF5!&hBdF`^2;l=FCgHFYci|=;HBk#dDD2+`?kC>m#4z(Y6XgDpM(gVGP)57;?ULeC zt$aS0-?94BEc-jV&}r(x-?6;@N67@*`M9=~IF(Ax(A-W=so!S*w!K;nWxge@h{jmq z)ymKMe>{n~ZRNMFjHF4Noiu|B8h<_@kszjZvg+?%oTzIH@&eqmhz$LoEUsLY(+l&i zK$9;!Bo_Bg2#AW9OgZTp2TF9GF?X8JaH{Ax0&C6_Obo?0wnsBys#O5Jw|#7#t=8`h z|KydCv%Kv!3eV5jt2DM|cbK2-WS6J_YwP38x#@r|<6AY$Rs`@_x50bwW3vJ9kv zUj+}4`N}!yi>&GoWJ4jLf|~V8zKlw3W68p9ED=Cs9NW0$oRLg`lJo_pw~0W66g?4p zUMwuzRq7xQT`LDrFcx6cP>!pG6Ui{^x%)CmeD)sO2cKC0MpN=-J+SoH5gc*J-&2yL zu!&`G8Sc0#V#kW2k8C}0$Vecy^0d8-Iy*2fKLD%FA|$qJ*d+SE0BSlb#oG*l3iJ)Y zmFE@7NN`3=B>bc2Hsna&BVR^9pn#)=2s+`x7_E{#%FkmDsTfid%J;IyaRv^uFRW^s z!8g}K_GHHfJv=N6fj@x|$8ss2=3Gz+s`3KB%!tw&>I=h_yn{R;VP0XHA@Mu%Vi1s# zU*g^{P9NEloL#E~k2az4gj&sjK7%2pEkkd5l!yp?@{j*xQX4klzMXDxOHP-82Lez9`J1S^^+te@W}#I7B8<%GQ0>cU_TW&{LNwV*SH;THp$IeY`6K0jPsR9?~Y zS&t(lKtxY)#?FlmGu|*vAef8jM99_LD1I2qA_8R->Gg{@_>iAMD~@sL@&ZSvIRm{{Wl6SgO>d4z?M?KN>(ixLJGj2Z(Vz z4v2@`{_+GRa=Wp75CkS7zc>gk|982 z62v)3r5h}Dlk<{ksFeZM*q$OXAl0{b3kiKaL6~@Z@^*!u0)9%WBG} zceC#RUUCKA*UNb?K+=ru5zucS84%laxG+Yr$TXOJE`KLBm?Nei4sZv z5bAlz(};RIfpRq_DaY7PoGx&VG1Y+Y7L=n;;z#E#j6^H&zB~Nl3hLU4?8+;yhkHq% zI4DY*kN0HbqllkCaPXQ%B;k@2Wlg;fGTecYy)4Y+AVCxt9sv5m8Vw2Om+`R))>i)5 zNdyR{u1Ozwpplm3^G~w_moBH*53KB`9_LKX94EDvJ?UrT2r$G>-)j%78h}giNv~!P z03l299o?`o29XeLsa#-&k&A{Z> zwiVi3dz_1srINf098DO4`GlW1kARRI_CI+OI;4T+x64ZkLMEFDB4ls;1qv@I11j2&HAVBRS@iCDA z`hodi2-$8!-Cj%PZ1WCJe58LewbY!rASta;h%0xicF0bPF%K-CSQ>`HtnmK;W+r6_ zW=@EeclyH`Cgk#1`p0p#K_kqo0yZ*!A&1r;Ms9F*LZ&iqn^)n#Sp92dH$k)O9!OwY z_b%16crP@gI^e((yB5HRM@`Y;+4s788Qez4_D zCQrNp_|6iLzK=;s;#__r0QgdTU_b*v8SYaoGG=35B>l1gK~+5NlHn1CmOP0*F{HKt zLU0UH?IWbNd>D)?1Zc_OC)aF4(-QO5QTK|uQmfpoO#w;~JrrL!(}qeR<^q4ik6~6w z9}|TwGYK&0iN(Jd_|k=XgV%~eu&-$Yf*~tpm$72#O+kJ`e8wg*gpvAz^?dRf%3`si>t<9xpx>o@g_vfAs(Ko{YXWDdbkA^0%q4CD_$jH`U{>_-^ECDX4F5lo24c~Dl+b#DS8 zqGX={$Gjk=7hd>bK4&B}$W%UYh7izzBgO?PxXU(A1AG}K$BXqo{p0e>Y`7wy)A5p{ zZGt-Pz)vY8Bf)s>89ydT>C&`(&wSO)BMO*-1H+9U5D6r{$KE&+%-nqbtkK}Z6mC2G zWJr^Dza}fl8AHE}Sb!QMG|D97@R3PA6l=2OMA|x@fz=VuqWeazK)_uyoe4vN8}IpfY9zpuI}{tYq1C(hrRW9SnfJ{{Y=W26;``|O4W_?7;+74#wzcac5g*D<3fSd)0K=paAN1;}1#f!+#!5z7dqTj zakq?$jEp-#OV)8GI*V;>NO8mO4TngwxU%_NL0a!?Mz7#~VbDA!{MbA5iDF3t&zMa^ zQHD4m02d^O+zekwA{umyaGOR3z~+>X7{tdVC4bJde+(~AL~vA|VJ702I69wf#>`1RXV!iBE%@!>cIWsQI!xPIFkFLg$qCf(pNrb1x^(;;-&2|g?a=+!lLG; z_j16cl}@sw-wi}TKzdWu#sd;bL+>G^)`$+)W2@LesL>|6jz%WYl|MMpkYF3bv^dv6 z78PZ|LY$XbLMl-|jT4ZWZ7|PmGx3fi8qinEzntk&ELC{O9EG54@etr5d_s@3aF)Aq z0Qj5G=KQNt3JoMoGdG0I@&q z2O6rB}!ggaz#i5f%N&(vg) zMJUR2RL)V!!@-a1II|Hk+M$3mBWS0HtUdtRykB47M66g9AF zVT#VX!lX=3Er^S>l=X6CScHLOD5~@Vt>Xn*uS{$_+m{{9^hE4g4#&m{k_w;sk4wj+ z(_uaT0D}Th_Mz=-8NiSvPdh&u2Dbs3*nn!iZCI~^fIZQJZE~WcyEc(S1#T5lq0dBXu$%(Y(aDnNum;9?_fYX$reoo~Kw`{5x_v(u{u z_Z)%@+5{)p4RFdekYJifZ+S6rZJ*y3O+!E9T0LWVf`QUlli)Gf5XckaKhuDb>gZ!R z7HuSJrMT&WWzi&w^pxUFmQN|KtX~3)g^UbIT+jJ94F)5d$T1#}pHkzK0s$!Zk25}& zL*g=z+)4O6xWrP>ZHGgn3_#1gI?=JLNzkD4piuIjT+rLm=W2 z@?u_Q!kZ3JHyWG35DaEZTJ0vR_Z+(%^fP7NR^o*ZO5Vs7SeVAMDT9WkI(cS1in z%rx;VelH~?#U)LiH`gQt+KbQB<54^!P&wWvflz`}yke^ol^S4*L!tc)NGUW*JjPK_ z(=Frjz){thOFV1i9YQq+AL_|QgptSN2s5$CE0Gd2KuGt*LjaamK8z6x=VADkUwD9H z@s`J{6euaF2N}K!Q)8~!Fqj}C(;tz^(p8J?i}ERCT>vXeVFHH(tOgD}BD`o~q&@NY z#H>@e)&(>rxz;d+IvBnSda8Rl%gx3>AKQWq$+|*tiC_)7KCxy9IB6${{{TUVAxfy1 z>nVfc1NzF=sWKvU{{XC$V~c>ihns3-J|pgOLVBRk`pI~k1*7D|3ABk$GK84b`N%kr zAScWpjDQgn(~nrw6Wa9o&vHpVrwB^gusHHYb~n3>N`r_=KFE)U0VjfX0oE{LWeC9E z<0gvYSK*AG4q$jUiDd_FLzaXbVC!KV@;JOJkx6bql1`$J3xcv%)lbH69VxJJn(#;`pd%)FNY{DANCRw-);5LS1`=45(2lT!+D5Uf~4)}ne&u%W^v zXwk>$$OA030weQ@PaRZX00EsT`@sS;cJ$Hs!)hI=Rz(yGC>Su`JVE-!TeCC}CI^F! z4@V8LFmg^72gKwnAa*D&XAS@uEq|;r18kq1{{Y?rY$YiT@eA%TlY>JceQ==5u>jYc zya!RQISNk*vW}?-&iI@nNbX&G3{R{Y)DRoV#6%pWdW;oWv4AEnAjCv}=JM64(7d7p zf@4~QgF{9HQ92Yu=5Z*ARL)-tuOvrr7+ot_PP*?0Or4~}JH=r$d7?APILr(t0ARB6 z#0mS!c@(ECKuvM9-XuhchzHSu$tXZZOrjAZ{R~W}V5~@F5h;ggs;!)H5;w=>!lkb!1*}xYB6kGI7^@IArw(ZL z(n0lj$}d(w7mpS!NNXpVR(34>IN1`Oj-$pC!8@b@-m*gj$d5MG^BDNLd?Xe;6VWBf!6` zJe#Xe>l?>MN9u4&+TUNs3EdhP{{U|%Arr4GQYKx2C)NyH9;5RaE(tSXC$k*{L0JTZ zk?D{KqBmANtov~ighBoR+Y*?ZDKo5EAan;j&%SaS67EmIF_KjeZR)EE2FnECY_bjc zs>RQ&dz4gBWMp7bvl-K*)7ml^T4W6ik>FrzW3k-DMKud$`3zV~()aAkH^fCb;gbl!gcPBTP3H6rHF}^2xSjJ-gaQuS z&zwl4UDxp#sbrR}N;3ELEjhJ>a5ntW)AnClFea`4{IFW>SC##;E%- zFicAEJgyFMUKkh9xR5~eJQ2h5ofv%?f=O42d@c|PjwyLU1nP`O1fP>9i-$P-TpGxd zJKiILnbSbOX@8T1XP^NdAj7i>h+|$WDPX7VB8PNfd^^6M+l_uu3-9IkqJL%rt=JT@^aQG<3v3?K(Pp#L_q-Zw)b^zhWAA zmnb0Ym4J`|>Aq4v@R1{;6A*+UgG2X@jFt|(0k=+RF@_Vw zpcQn#c!u_EhCj%M)gXesgAj!&2r@^KIK>$;M6x?#^ovwP_1*;2;i2R{1DBKu2@&>S zP>EcROc-GbeTZ@?I#rTH4&?--(I8KPa9dK%po@qu05r%(L>KYEDU=npKqce=;zKTe zWAecQAx1vq^@+SEjC;!+M?C%Ex6nB-C09S&jFg}uvK>P9Vf4(ax5Vr5l7b6-s!(?% z$S}>4kFf;shF;Z)KTRTsbq^Pgr`8#iP-KgkEsptkV9=I;Z`~&~QGh`YvU3C4PWcN| zQ)^-7F9Jb43yYS6yqn=@#iLyve|WnNm$ed{N|;FK>myl!fKuVFjtn@Jc!uAD?#269&QqP zFraD`*c(fOGZ2iYZG_?E(7>R_L_V;gfe=qRfIUVg5#*}{IweE{aO!(xwDh-;!>8{Z zWbk+yl_CXDy_wP`6vTQ@`<$H?m*?nuP7NY1$>^95jDILm1*_D=cZUU}GKJ2V81mr< zLY4j;88ga)WGFV?W^N-@h7}b82a5As=L3LG!pZ}@Ffn#WYQ4z(WMDK> zvYzrwW{&A#;Cka!0E8A3bfJi>mr3W&MM^3PK1{W2k)kc-NiwBP`g*dqk>^@df5zQSM>aTtKHNbnI8+OS@h3!a%@d1w(-d&JHP zP(=eA_Pi-zSRxUu@kXpOQ13wG795lTAMIr53MSC8*v_IL)?#6H;F;VR1zN}k{#(G9 zAdsW%aX8WqC8F8ISOP`A%tqhNGYxIWI~Cegr1d=$A*J;;ZACB zig@=%Mj48W%dnM=JCxV{u%TM55pt=L9&oSePU$KjlEbUKP1;b~%=I6KC1Y4w+o3E< z;S=Ivgf~1el&mgwcdq(yKY*#sQgadqQ@2cM+l(RBPBIs96p?~yhfX^k@V!qXi^*30 z9N`z*S3vq=Qas;$9xjtV3&$p^Ztx% zw6^5=GHv4k6$=Mem%!lS05k~WnUIMP9f@Cu0?-lGau>>>+Xy!VAxCJx)=1_eUXTKO7@~5AOXO=9z*GT89x!A8 z;k4j1?0lyaA{qpyJ$~|bf>7%4%C8XW@Gi3u!si)~Ow?@n)&Q5Cq`%fOP*pAoUpWC# zs8nB*0~8JQyG*TGmTRu8rEm)R=0LU zKvLqVQ(!amI0_R|M7M{}OjFhZele0HMUm!F^*Hhnod?)AfJ!ZpyLv(27g0e6;UY=t zlK}@Q*~kD$mu3{GTq0jNd=Tm1r_O53uD&?fB$P=Pi%KEEES;#pJh*70I~fzSP7lL5 z%c7=DN~jdd;y8s$@Me)@$u5DfDnvDoAi^o7B|$ESgS?C6BMv1%G{K=Xmtj-|7B((r z#)+P>jtNbxCg=d_DjI7mgih!YNvyR9M{IrMt0ve=kQ_k1(n3a6h^WH}#Lp_%Y(}EC zaFGO-f#kO7=vcQcZbVR&X}x^Acv-Th033*F^{8sj&ORuS6n`ql=U$ZH*|F?(uw%K9OY1FK^unDvKA z5gl5vyU~4eKSrCJ+{DDN3%?jHfuQ2XO|D;paYYo-AoK#hdBc@R zL=DSA>r>|?q|&kqt%?sYc*Zkg4VRb)2bAti6WrYkxJ#F8xBv{!hq@lK&zz)CHW-ae z7@82sy+h0ZCbl*@dNx)Mfc>{$Q@gqmW?K>-oPbBk*O(LzG(gqUz7$x3pEQ|=eR zT(~)Ko|+FKJ?DtU^qv$013^PlQM?Y-LJUy|GLaHfPD?7za|c2tEJ(Y!!z*%H)TC_I zl@-$RDgeOP=+o z!Q!3v$#WL#r($pt5(cD?I>FiYr^ZrfKikAg5kG8!shz?Y2_qR@CrP} zweS^)!Qvzd+h_VR6B(H~CtiMXs?4|^OFRuHyd=g__NAJae;xIUs>u{@VtC-gPyj6W zEf{2Z=qvY#$L8cbS6pO7O^`V4cx(uXT(Uuc$;k6OWKu(S4{D460uqGvss<}efe%BG zLGgkdB&e=wnw!ZeA!ttRR}{+|g97Lz-9X&=a2P2Qq(Vwf!LFI7OA1h-aKHRq6{4G= z6S*q|m8Y!a_s>Pg-p{MIi2j~ zMi5Z&bW%h`4l+xmtV(cEtB*`d#L*x?PLF(I zvKF3V3rgodEdr6+OYW?6Ke&>NVz4UO&a15q9I0J7%_9iiYk!< zE;^!XoB%%M5bObKpMqtcDG{1t4LE<+RTPF3&A>dNs>2V!4cm z0}nQJ;|>ZA?aDtvz`%Prk_>YxlRvUudZedT}*A%W{Maj3qz6zyw1!gQIqCncMD z0h4kPM0;VS6g@{(z|?8$0s_FynBc4=0l7T>azuz7vGIh=2y)7@5rHKLpPYw)s)_l7 z?+FN944SNnLIZ^X*AwP2NMbj`@H+AM#vvs=KV$vxAc;!}K|VmqITaZkj7y;UZGY>G zE&ypHf*x%1{=5zuTp)_s_`w)EL!?ZY1Zrv(9h_E*SrDPQsqc!BFj*$iOq38y$*f9~ ziH*eo3b}~Wc_eW|C5s40aHh2ckvvI2oeEQ8ssI&MHzY?eiTSBo0<`D!~S(!&n zFnBPS1|>xv+Tt;MjL9*=0yS-f>aZwN}r?ugv8-~O_U*xFQtWa`LX zGn8>bOR}DO48N7K!EL9z$g_#2he`lu;LBNJ}bU7rB_NZ6Hp+h z55_ceP=yHCaf$R+f2I-&5K=w$#fc}$A2W({+@Oxrj8bM4XTMk$H-xHJZx}_v5!8Bd zB}L&^2B|s-%0|T#-Qe37q$4IF#o)?sZlXBY#SmX2@UIzU0)WCPTZxWW;|(PtKt%c` zaCizbBMf^nuq!gI{{UFrj5A?=FjR*PA3#nr=_vSslKH_vp$jgSUk)-mB$FNGo-q{y zs1HE`yq6E^B?{uYWb_l1pLn)R6O|pFeYr!mQvvgcg*2ESxB4)NVi`NZGr7xvHv-$; zVMR=WgvG-8wH6VA07H6S^h`K&Qpc*G{9-Z!T%?1&=^{ zq~TbH0)jZ^U+!R%P9_%;sWk8}U1u;QGP%s%WB_Zkn3g1P35?sgHM($SW2BTb+x^ov zF-58?)9dqu^&ND|*r2uZY$Ypo}F2S|R@cyp-})WN!9Ioy?PwbCM~?f_Z0VvJYGaVhn7GdN;bj zLrg@J2$Gc(uDh23hk`CX2EN9(#w<*v6-LfiO6ig$)V>B(W@ooz*gO&-S0MzcQdTD8 z4jIWLvT6nnsfSwxf@j*?2^A)4Tw-Ptts2qKI>c02C6c%1>OA+v&QOJfISm-<5-FuoR}PYLlC zFN_qDRH;5}{{ZGj6KF}FGXDU1yy3S|=S=ZEF@(7jD1I?9jToeQn`Hd-AtSSV1Zf}k zFL`DVlq2zr!HcI}aN8kuLlB&yP#?`2@XEmH(sn!xeaiLXEH;CMl_0Qp|}c+P>$2<`58hBB*=Tmbs>-lkMAach$^FfXX6zojU#{R!YZVxPjxuJprL?tZhx$1 z$rh&=&|;F(9+w_)H9)foKUInaCc*d>tP(?Ew|Lfb0P@MxlPbj6w6@})1C&H|dSl7j zXpXu*j1dW4mFXU_2q~bE(un^6STQPg+xup>3YJ&EC_){r>>oNT>-WxleokvO<`~p-Qlm%sR*=wFj4^ zSLo%yDg=WnNlLI5CRqq1B<3iCXu3`%yR#Zp5mZR8Y)Hf*6Km^CAOM9C8sdyawMd&a zbVoeDcqt=kjT2_OZzmEVVPwv!IpFEr10?0W9NL%v038t~z?2p`^mOabY$UZ%ksL`Y z)k(aAAd1PUVhD=w?ZTeM(V@o3)+v~_09c|W?ZjFwkuBW%;~@#i0OeF1^}`ih5ah(v zFFYJQFxCweI6uk%0IX7a#K{{ewDpm~WJsE+xV`HpPX@a20SWOqz#@>vS`a}uzM8BR zM<`_1G?&-w1QdFUE7r*ERjeQ1n&nb#Pg1ZE;{s+9i#bjUz6=Xgp@M~KRn_|fNX{i%wSmwjaB->E3KL*wZ|D%U{HGu zWIId>58hX(tSV#7KZ6LTLV+vY<%ndEng0M>f&yS{{z!5v7%-|xj94-x7vdVhOiu9~ z%db&|76#xY^G>?rAnUm2p7@5++SHwCtO#3JDLtUi7Vu0l{%gg;Cs3D`tWik>*eQvui^^LUvO9lt1FS!N-* zNH;pjsnFana|RJaM36MqCO3qFgT#~MW6mKEDzbnYMwdd;Y9B^7JzZMC{^CIv_yhBm z=%(tOYiEoF_dv_*oM7#u*T+AcM#kHPe!n4-;+H;t(ePM8swVH2*q#fE7Gv?xdvY6+KW;V5wO9O;j3;mioQf`qISQwnr(E;A`I zCBOmZ;MPfogwkCs)6(K=07qaw#k0JG$RV(lYSvBw0X)7n5$^*bYP;M(F_{HeJ6U`8 z*B=6#9!?L%taw_(BrmbZ!W18DY84q9bDp~GeC$4mYE*}4W$k2Df`8VBVqaq z?URAcOdWBpVDJ$IGlpXjY{O>vs;q&Q3t*4Fog*PyPa}Wh5I_Xj4S$S8q{u-BU&-~6 zQyJoS_x1V7Pz1Uw<5B+rxC|)7sZUsrl~@)*cBKwFV{8B-9vU)by$Mq85HJ86-ASH)ubRI$5wnsY8Z4~4kupos8b@2Ck&0rDV}!jXyX*K z^8@`DN`Ph*->*1ovV|;oI>dSg37xS-os15>NWtWRLI6r^TdKlhfLTT!yLIV{5;BaE zM_acacmih%qe-4oj+d7RN`qp!jlA$LSiC zSzMvMr+Ute(#-f%+`Q8ClGc_`HD_nfw>=cLz zdSwQwndMWayJ8esXsHg`#vx(M3VsOb@s3eRA$dp=)eey|<=_mg>oq_5=Pyj5YCi94 z#V+Vm;G{fasS=PWT`slb0^l@+6&`Ei@k(%LpTm?U#=`Z#_e>IYBup|86cHT1+W!FD z0))Ct%h%3MLL`Phm7N|CzcR)6;|&T^2-Vqr{<07Sk<RU%nk-Ve>3IxT^}euV6imUttO>LoI$NA8hVw+ zZy<$4`gQl40%)i;)rd;VG#!ScjG_^qMd`=;wgFqb$=`Vj2@ahHs?@(z&Pye3510*8 zdKVC+i?lD%E3PEOV3|p~t{C{oteQ$rNJMgrCwg~`!vKUt6h$ZnO zta53(U6aiJ09n$@2uV7K4spbZcpL38+Z+H|Nj*58n5jgy2e(`iAyNPyb~A~YBy9yJ zwi1gMP_jn*D-a^KcuUhqhgjCi5KPP~R^hOWWJLURdAwgX3P3CgS}%2z=th$+&Eo;fGH03qA;{BoGx~7fL4ufuIv%uQVW6Vf60x)4+jT& z?-zi`G1Ql%tzb7;s1kp?kg^s>;yO-7AZ5@v!7XD*kNd_$g9r;i_}4iw1A0?E?|$%T z5ZV$x@Fj@QAbH~$@ur+|XF?1;oM3fyMnZOsP#k7M-_{z0*I_@1{pD09QfC1M(*VMV z)F!bQgSuF3WmfdSst9$b^;ph;3t@=u&)y+&lN*R|prmQ!_5T1^WQfzJ>nS2w3;>@v zvZJaYPFRnO)3G*X??wUu7>ELG-`Do(7GMJiM_c{kvAzK+f202JSpyj$B7QI!f^;#e zsmTLE8zgp&fgo~_kDpix6)_=M2r!&85jDmRvNKSh))X^{76TDK0fdcSr32Ib&L{^K z{u10BNXJPU^{wKjp!5KxO#Id+V17xy*(20gn(vE(69;Eo;}lyTfnN>?gi)bK=CM+s z;Y;JbKmy^HYttabU6edxPIdz(yc|!~OAMsQ!7={;zOqk6LE#Fj?HkD{7L{<~KNBS& zqZALu0kIUpFO#KUFxh$~-sf1=5|nJ_In9d*t~h@f5t-2@6Y;kc$kni5dgJwkWS#Ui zdg-FFY7lk;BblpmX%NAyFAyMzfe&*r20{Q7!6<{8{{VA>+8KZvJtrKhvWHCZgrzG~ zLVIg`_05q+w7h+Pd|~E*$dt!SH5s%$)>yo=flRNlafKwE#MU{SJma7tW%+@&0Sdxd zKzTfv4|;l@+@vQ`b7!xZ?VEN{LY4@co3dUh!BFa`jio>AIiOnVAV|A|u}o4@O{FWu zy8i&)P46-&U-yJcNGfb74FWy)o^Wx17q=8F zBt~069SsI5gg|RjgtGQdo#PM?T4Z%ep6>9L&`O{$##musFZ<3Cp0a?w{{T2tI*B4b zJ#i@7(rx^chS(KFmCw&?0!T`Sp8o*ua4;x1w()U_AsP<&If@L7(>Rq1x`s?l74j(u z>~c^NWCl~|Fl;6`MXz|V0XH!wvLIsl%fWfHKNtw-($n!6qqdYo_0AQUkUXcvMo>YN z4Rym*5mFXQOTgRjBb_A!hR&Fvpc_N;kg5nIg*RA=!#XlL{baCM(6GneI7uA{w`O&b zzzB&i`K(bZ6p1h2f5t)*3#;d@1PMf(2exv@F;AvboReYOtX>Qz)5Y^3(!3E$=~A@lsO`iAL_{{3Sd}UkM~?GBMWGs6Wzq2 z4yLuQwx;`JG1(zwALl~?sw-BWio%02PQ&r6f;Gt$did8EL^p7mef5k`03^6Rrtt+z zIv5f0c#5Vbqx;5H6v-s_k}_FJm1iyC6A>guQG3iEj4NzRto&cjQ?WFm9;ZsF^PNZ{ zRL1+^2$IxoeSNsNSElXy#hr2vfrKdUExWqGlM*&%53!A${xY=jB&W!9F(ialJHGIw zNt#HX+Yg@v5X7L=C!<={MZTQ#qiBfz$!Hfk=zpUOuiD6*vYMFz`+z9VITL1))2xOPxav1#_8_s z-)g=nss}|q1xwvj204-2v5O~ zESXy|ic*88<swF&lMaSR2BBaA@ZUbz}1s#&MbR1y^tZurCiNX#tg(#|*rSs3%z zAiv@pD*RzlBIe<1x!(yAyxnD2Xq+IyWJ)@coIGlhz!_j!%BFbUD?tP@!QK63DHYn0 zCD%XWBGKnesm5CYE+oE!)+n-CS-!Q$`M@|OF;zSH$(f#lmj3+t#JdA!r=ySwu@)lx zJK+jz4Fy-<`NX7kC4hWpsz%rT>zf%KkW=mF8cOphfM~@k5hRIc*G~8)fE_YFIEf^o zgz+EuhB`>g{pp+H)tGwfoKyl52GhYpb-_v-7a)t_F&wPh0mna&SSD1SpSC+>fF44r z6TZK!p*>Ou2mLs)U4m+PBJf~LEDS3hoEUA|GbgVZHlo={6F-a!dnCUCd%~HRO!0f) zT#O_PNfMdg;{xy-fLnH+m^q!7LVhv=mLPudG(;%A&Lj`VITQo|D-SVQB8Wn+Om+Ij zfeJwgh};}`#d(dsMR)V`(c<` zB|i--{WzK>3cOc!G1mwQlPoVj@J~QWC=to;974-wG^z(GRYH(nI=T6OdGeV6Ay!0Vq>&hPQO)7$Zf`1bxkr}<`&^PF_@ zXW{+1_djNA^P4=g{N9d!WA~qw{5i4b`Ezgiw?5PLn*RXBnC<@Yh9=)Xw>AF&iqEai z-v{8${C{R>^8IGs--doq<35`M>3%bRS@|EAKgHue-^PE7&VRqYeb3RKPxjCD`Z=y& z>+e6`>7PUN{ulkcWBqQfY3m#R0KGHseolX;r6 literal 0 HcmV?d00001 diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index ab7dab7..23459f1 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -7,6 +7,9 @@ if( POLICY CMP0074 ) cmake_policy( SET CMP0074 NEW ) # find_package() uses PackageName_ROOT vars endif() +# Options +option( ENABLE_MPI_EXAMPLES "Enable examples that use MPI to setup" ON) + # By default, cmake looks in your ~/.cmake directory for information # about packages. While that should be fine for most users, we disable # it in these examples because developers often have different versions @@ -14,23 +17,57 @@ endif() set( CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY TRUE ) set( BINARY_INSTALL_DIR bin CACHE PATH "Installation directory for executables" ) +# The CMake Cray system module does not define the RPATH flags (probably +# because Cray has historically only supported static linking). Now that +# dynamic linking is supported, we define them ourselves. +if( ${CMAKE_SYSTEM_NAME} MATCHES "CrayLinuxEnvironment" ) + set(CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG "-Wl,-rpath," ) + set(CMAKE_EXECUTABLE_RUNTIME_CXX_FLAG_SEP ":" ) + set(CMAKE_EXECUTABLE_RUNTIME_C_FLAG "-Wl,-rpath," ) + set(CMAKE_EXECUTABLE_RUNTIME_C_FLAG_SEP ":" ) + + set(CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG "-Wl,-rpath," ) + set(CMAKE_SHARED_LIBRARY_RUNTIME_CXX_FLAG_SEP ":" ) + set(CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG "-Wl,-rpath," ) + set(CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG_SEP ":" ) + + set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG "-Wl,-soname," ) + set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "-Wl,-soname," ) + + set(CMAKE_EXECUTABLE_RPATH_LINK_CXX_FLAG "-Wl,-rpath-link," ) + set(CMAKE_EXECUTABLE_RPATH_LINK_C_FLAG "-Wl,-rpath-link," ) + set(CMAKE_SHARED_LIBRARY_RPATH_LINK_CXX_FLAG "-Wl,-rpath-link," ) + set(CMAKE_SHARED_LIBRARY_RPATH_LINK_C_FLAG "-Wl,-rpath-link," ) + + set(CMAKE_SKIP_INSTALL_RPATH NO ) + set(CMAKE_SKIP_RPATH NO ) +endif() find_package( Faodel CONFIG REQUIRED ) +if( ENABLE_MPI_EXAMPLES ) + find_package(MPI REQUIRED) +endif() + + + add_subdirectory( common ) add_subdirectory( whookie ) +add_subdirectory( services ) +add_subdirectory( lunasa ) +add_subdirectory( opbox ) +add_subdirectory( dirman ) +add_subdirectory( kelpie ) if( Faodel_ENABLE_MPI_SUPPORT ) - # These examples have some MPI requirements, so don't build if no mpi + # These examples have some MPI requirements, so don't build if missing mpi - add_subdirectory( services ) if( Faodel_BUILT_Faodel_NETWORK_LIBRARY STREQUAL "nnti" ) add_subdirectory( nnti ) endif() - add_subdirectory( lunasa ) - add_subdirectory( opbox ) - add_subdirectory( dirman ) - add_subdirectory( kelpie ) + + + endif() diff --git a/examples/common/CMakeLists.txt b/examples/common/CMakeLists.txt index 31596d1..a5ed036 100644 --- a/examples/common/CMakeLists.txt +++ b/examples/common/CMakeLists.txt @@ -1,8 +1,7 @@ -set( EXAMPLES_LIBS Faodel::common ) - -add_subdirectory( data_types ) -add_subdirectory( info_interface ) -add_subdirectory( logging_interface ) +add_subdirectory( data-types ) +add_subdirectory( info-interface ) +add_subdirectory( logging-interface ) +add_subdirectory( serialization ) add_subdirectory( singleton ) add_subdirectory( bootstrap ) diff --git a/examples/common/bootstrap/CMakeLists.txt b/examples/common/bootstrap/CMakeLists.txt index 1589b56..b2fbaa7 100644 --- a/examples/common/bootstrap/CMakeLists.txt +++ b/examples/common/bootstrap/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME bootstrap_example) +set(PROJECT_NAME ex_common_bootstrap) @@ -6,11 +6,16 @@ set(SOURCES bootstrap_example.cpp ) +# When doing a standalone build, use the Faodel::common imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::common ) +endif() + add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/common/bootstrap/bootstrap_example.cpp b/examples/common/bootstrap/bootstrap_example.cpp index 2592388..adcbcfd 100644 --- a/examples/common/bootstrap/bootstrap_example.cpp +++ b/examples/common/bootstrap/bootstrap_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Boostrap Example // diff --git a/examples/common/data-types/CMakeLists.txt b/examples/common/data-types/CMakeLists.txt index da3d1e6..b9970aa 100644 --- a/examples/common/data-types/CMakeLists.txt +++ b/examples/common/data-types/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME data_types) +set(PROJECT_NAME ex_common_data-types) set(SOURCES @@ -7,11 +7,15 @@ set(SOURCES main.cpp ) +# When doing a standalone build, use the Faodel::common imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::common ) +endif() add_executable( ${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties( ${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries( ${PROJECT_NAME} ${EXAMPLES_LIBS} ) +target_link_libraries( ${PROJECT_NAME} ${EXAMPLE_LIBS} ) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/common/data-types/main.cpp b/examples/common/data-types/main.cpp index d3aa12b..98a311f 100644 --- a/examples/common/data-types/main.cpp +++ b/examples/common/data-types/main.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // This is just a main that calls the actual examples // diff --git a/examples/common/data-types/nodeid_example.cpp b/examples/common/data-types/nodeid_example.cpp index f58824b..475c259 100644 --- a/examples/common/data-types/nodeid_example.cpp +++ b/examples/common/data-types/nodeid_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/common/data-types/resourceurl_example.cpp b/examples/common/data-types/resourceurl_example.cpp index a9a6e55..4ec4e95 100644 --- a/examples/common/data-types/resourceurl_example.cpp +++ b/examples/common/data-types/resourceurl_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/common/info-interface/CMakeLists.txt b/examples/common/info-interface/CMakeLists.txt index cc32888..9843a6d 100644 --- a/examples/common/info-interface/CMakeLists.txt +++ b/examples/common/info-interface/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME info_interface) +set(PROJECT_NAME ex_common_info-interface) @@ -6,11 +6,16 @@ set(SOURCES info_interface.cpp ) +# When doing a standalone build, use the Faodel::common imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::common ) +endif() + add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/common/info-interface/info_interface.cpp b/examples/common/info-interface/info_interface.cpp index e7d3b5d..75179c3 100644 --- a/examples/common/info-interface/info_interface.cpp +++ b/examples/common/info-interface/info_interface.cpp @@ -1,10 +1,10 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -// info_interface +// info-interface // -// The info_interface is a simple way to dump debug info for a class. It +// The info-interface is a simple way to dump debug info for a class. It // is meant for hierarchical class structures, and has a depth option // to allow you to control how deep a dump goes. diff --git a/examples/common/logging-interface/CMakeLists.txt b/examples/common/logging-interface/CMakeLists.txt index 630a33e..f74cece 100644 --- a/examples/common/logging-interface/CMakeLists.txt +++ b/examples/common/logging-interface/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME logging_interface) +set(PROJECT_NAME ex_common_logging-interface) set(HEADERS @@ -14,11 +14,16 @@ set(SOURCES logging_interface.cpp ) +# When doing a standalone build, use the Faodel::common imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::common ) +endif() + add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/common/logging-interface/ClassA.cpp b/examples/common/logging-interface/ClassA.cpp index 628bc1b..df075ff 100644 --- a/examples/common/logging-interface/ClassA.cpp +++ b/examples/common/logging-interface/ClassA.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/common/logging-interface/ClassA.hh b/examples/common/logging-interface/ClassA.hh index 2624b7b..52ed09b 100644 --- a/examples/common/logging-interface/ClassA.hh +++ b/examples/common/logging-interface/ClassA.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef CLASSA_HH #define CLASSA_HH diff --git a/examples/common/logging-interface/ClassB.cpp b/examples/common/logging-interface/ClassB.cpp index dd72731..14062c6 100644 --- a/examples/common/logging-interface/ClassB.cpp +++ b/examples/common/logging-interface/ClassB.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "ClassB.hh" diff --git a/examples/common/logging-interface/ClassB.hh b/examples/common/logging-interface/ClassB.hh index 5419cd7..48bfa73 100644 --- a/examples/common/logging-interface/ClassB.hh +++ b/examples/common/logging-interface/ClassB.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef CLASSB_HH #define CLASSB_HH diff --git a/examples/common/logging-interface/example1_selectiveA.cpp b/examples/common/logging-interface/example1_selectiveA.cpp index 5b34d63..73244d8 100644 --- a/examples/common/logging-interface/example1_selectiveA.cpp +++ b/examples/common/logging-interface/example1_selectiveA.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/common/logging-interface/example2_disableB.cpp b/examples/common/logging-interface/example2_disableB.cpp index e393bd7..affe71c 100644 --- a/examples/common/logging-interface/example2_disableB.cpp +++ b/examples/common/logging-interface/example2_disableB.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/common/logging-interface/logging_interface.cpp b/examples/common/logging-interface/logging_interface.cpp index 5484092..6430388 100644 --- a/examples/common/logging-interface/logging_interface.cpp +++ b/examples/common/logging-interface/logging_interface.cpp @@ -1,8 +1,8 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -// logging_interface example +// logging-interface example // // This LoggingInterface is a simple way for us to plug logging operations into // classes. In order to use it, you need to provide a name for the diff --git a/examples/common/serialization/CMakeLists.txt b/examples/common/serialization/CMakeLists.txt new file mode 100644 index 0000000..ce0aecb --- /dev/null +++ b/examples/common/serialization/CMakeLists.txt @@ -0,0 +1,23 @@ +set(PROJECT_NAME ex_common_serialization) + +# Note: Make sure to include Boost::serialization when linking + +set(SOURCES + serialization_example.cpp +) + +# When doing a standalone build, use the Faodel::common imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::common ) +endif() + + +add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) +set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) + +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS} Boost::serialization) + +install(TARGETS ${PROJECT_NAME} + EXPORT faodelExampleTargets + RUNTIME DESTINATION "${BINARY_INSTALL_DIR}" COMPONENT bin +) diff --git a/examples/common/serialization/MyContainer.hh b/examples/common/serialization/MyContainer.hh new file mode 100644 index 0000000..f478bc1 --- /dev/null +++ b/examples/common/serialization/MyContainer.hh @@ -0,0 +1,62 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_MYCONTAINER_HH +#define FAODEL_MYCONTAINER_HH + +#include +#include + +// In this example we have a MyContainer class that has a vector of MyItem +// items. To serialize this to a string, we just add a serialize template +// to both both. + +struct MyItem { + MyItem() = default; + MyItem(const std::string &item_name, double val) : item_name(item_name), x(val) {} + + std::string item_name; + double x; + + std::string str(){ + return "Name: '"+item_name+"' Value: "+std::to_string(x); + } + //Serialization hook + template + void serialize(Archive &ar, const unsigned int version){ + ar & item_name; + ar & x; + } +}; + + +class MyContainer { +public: + MyContainer() = default; + MyContainer(const std::string &name) : name(name) {} + + void append(const std::string &item_name, double val) { + items.emplace_back(MyItem(item_name, val)); + } + void dump(){ + std::cout <<"Container: "< + void serialize(Archive &ar, const unsigned int version){ + ar & name; + ar & items; + } + +private: + std::string name; + std::vector items; + +}; + + +#endif //FAODEL_MYCONTAINER_HH diff --git a/examples/common/serialization/serialization_example.cpp b/examples/common/serialization/serialization_example.cpp new file mode 100644 index 0000000..fc3be9c --- /dev/null +++ b/examples/common/serialization/serialization_example.cpp @@ -0,0 +1,49 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include + +#include "faodel-common/SerializationHelpersBoost.hh" +#include + +#include "MyContainer.hh" + +// Boost and Cereal make it easy to serialize/deserialize complex classes, +// you just add a serialize template to each class and let the library +// pack it into an archive. There are four things to remember: +// +// 1. Add a serialize() template to each class being packed +// 2. Add boost includes for STL's (eg boost/serialization/vector.hpp) +// 3. Use something like faodel's BoostPack to pack the class to bytes +// 4. Add Boost::serialization to the target_link_libraries +// +// In addition to Boost, Faodel also has support for Cereal. Cereal is +// faster and is a header-only library. Unfortunately, Faodel does not +// export cereal at installation (to avoid conflicts with other libs), +// so it is only available inside of the faodel libs. + + +using namespace std; + +int main(int argc, char **argv) { + + //Create an object and add some data to it + MyContainer bag1("big bag of stuff"); + bag1.append("thing1",101.0); + bag1.append("thing2", 102.0); + bag1.append("thing3", 103.0); + + //Use boost to pack the object into a string of bytes + string s_bytes = faodel::BoostPack(bag1); + + cout <<"Serialized size is "<(s_bytes); + + //Look at both objects + bag1.dump(); + bag2.dump(); + +} \ No newline at end of file diff --git a/examples/common/singleton/CMakeLists.txt b/examples/common/singleton/CMakeLists.txt index 937c633..3c355e1 100644 --- a/examples/common/singleton/CMakeLists.txt +++ b/examples/common/singleton/CMakeLists.txt @@ -1,30 +1,24 @@ -set(PROJECT_NAME singleton_example) +set(PROJECT_NAME ex_common_singleton) -#Note: globals can get stripped out of libraries and may require -Wl,--whole-archive wrapping -# This problem usually shows up when you don't build an executable w/ libs - -set(SERVICEA_HEADERS ServiceA.hh) -set(SERVICEA_SOURCES ServiceA.cpp) -add_library(libServiceA ${SERVICEA_SOURCES} ) -target_link_libraries( libServiceA Faodel::common ) +# When doing a standalone build, use the Faodel::common imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::common ) +endif() -set(SERVICEB_HEADERS ServiceB.hh) -set(SERVICEB_SOURCES ServiceB.cpp) -add_library(libServiceB ${SERVICEB_SOURCES} ) -target_link_libraries( libServiceB Faodel::common ) - -set(SERVICEC_HEADERS ServiceC.hh) -set(SERVICEC_SOURCES ServiceC.cpp) -add_library(libServiceC ${SERVICEC_SOURCES} ) -target_link_libraries( libServiceC Faodel::common ) +#Note: globals can get stripped out of libraries and may require -Wl,--whole-archive wrapping +# This problem usually shows up when you don't build an executable w/ libs +foreach(LIB_LETTER A B C) + add_library(ex_common_libService${LIB_LETTER} Service${LIB_LETTER}.cpp) + target_link_libraries( ex_common_libService${LIB_LETTER} ${EXAMPLE_LIBS} ) +endforeach() set(SOURCES singleton_example.cpp) add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS} libServiceA libServiceB libServiceC) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS} ex_common_libServiceA ex_common_libServiceB ex_common_libServiceC) install(TARGETS ${PROJECT_NAME} diff --git a/examples/common/singleton/ServiceA.cpp b/examples/common/singleton/ServiceA.cpp index a03563b..65eabc0 100644 --- a/examples/common/singleton/ServiceA.cpp +++ b/examples/common/singleton/ServiceA.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "ServiceA.hh" diff --git a/examples/common/singleton/ServiceA.hh b/examples/common/singleton/ServiceA.hh index aa5db1f..4a19a79 100644 --- a/examples/common/singleton/ServiceA.hh +++ b/examples/common/singleton/ServiceA.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef SERVICE_A_SERVICEA_HH #define SERVICE_A_SERVICEA_HH diff --git a/examples/common/singleton/ServiceB.cpp b/examples/common/singleton/ServiceB.cpp index 8b348de..2de5370 100644 --- a/examples/common/singleton/ServiceB.cpp +++ b/examples/common/singleton/ServiceB.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "ServiceB.hh" diff --git a/examples/common/singleton/ServiceB.hh b/examples/common/singleton/ServiceB.hh index a94a96d..71e9463 100644 --- a/examples/common/singleton/ServiceB.hh +++ b/examples/common/singleton/ServiceB.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef SERVICE_B_SERVICEB_HH #define SERVICE_B_SERVICEB_HH diff --git a/examples/common/singleton/ServiceC.cpp b/examples/common/singleton/ServiceC.cpp index a574989..7854dcc 100644 --- a/examples/common/singleton/ServiceC.cpp +++ b/examples/common/singleton/ServiceC.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "ServiceC.hh" diff --git a/examples/common/singleton/ServiceC.hh b/examples/common/singleton/ServiceC.hh index 6b36e74..8e0cb19 100644 --- a/examples/common/singleton/ServiceC.hh +++ b/examples/common/singleton/ServiceC.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef SERVICE_C_SERVICEC_HH #define SERVICE_C_SERVICEC_HH diff --git a/examples/common/singleton/singleton_example.cpp b/examples/common/singleton/singleton_example.cpp index 2b70985..d5bf201 100644 --- a/examples/common/singleton/singleton_example.cpp +++ b/examples/common/singleton/singleton_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Singleton Example // diff --git a/examples/dirman/CMakeLists.txt b/examples/dirman/CMakeLists.txt index 41aa489..eec5f14 100644 --- a/examples/dirman/CMakeLists.txt +++ b/examples/dirman/CMakeLists.txt @@ -1,15 +1,34 @@ -set(PROJECT_NAME dirman-example) +set(PROJECT_NAME ex_dirman) +if( NOT TARGET MPI::MPI_CXX ) + message( STATUS "Some Dirman Examples will not be built because MPI Support was not enabled") -add_executable( dirman1-preload-configuration dirman1-preload-configuration.cpp) -set_target_properties(dirman1-preload-configuration PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(dirman1-preload-configuration Faodel::dirman) +else() -add_executable( dirman2-runtime-define dirman2-runtime-define.cpp) -set_target_properties(dirman2-runtime-define PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(dirman2-runtime-define Faodel::dirman) -install(TARGETS dirman1-preload-configuration dirman2-runtime-define - EXPORT faodelExampleTargets - RUNTIME DESTINATION "${BINARY_INSTALL_DIR}" COMPONENT bin -) + # When doing a standalone build, use the Faodel::dirman imported target + if(NOT DEFINED EXAMPLE_LIBS) + set(EXAMPLE_LIBS Faodel::dirman) + endif() + + # We also use mpi + list(APPEND EXAMPLE_LIBS + MPI::MPI_CXX + ) + + set(TARGET ex_dirman1-preload-configuration) + add_executable( ${TARGET} dirman1-preload-configuration.cpp) + set_target_properties(${TARGET} PROPERTIES LINKER_LANGUAGE CXX ) + target_link_libraries(${TARGET} ${EXAMPLE_LIBS} ) + + set(TARGET ex_dirman2-runtime-define) + add_executable( ${TARGET} dirman2-runtime-define.cpp) + set_target_properties(${TARGET} PROPERTIES LINKER_LANGUAGE CXX ) + target_link_libraries(${TARGET} ${EXAMPLE_LIBS}) + + install(TARGETS ex_dirman1-preload-configuration ex_dirman2-runtime-define + EXPORT faodelExampleTargets + RUNTIME DESTINATION "${BINARY_INSTALL_DIR}" COMPONENT bin + ) + +endif() \ No newline at end of file diff --git a/examples/dirman/dirman1-preload-configuration.cpp b/examples/dirman/dirman1-preload-configuration.cpp index cd2653a..f220e58 100644 --- a/examples/dirman/dirman1-preload-configuration.cpp +++ b/examples/dirman/dirman1-preload-configuration.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -44,7 +44,7 @@ int main(int argc, char **argv){ if(mpi_size<4){ if(mpi_rank==0) - cout <<"This should be run with four or more ranks\n"; + cout <<"This example should be run with four or more ranks\n"; MPI_Finalize(); exit(0); } diff --git a/examples/dirman/dirman2-runtime-define.cpp b/examples/dirman/dirman2-runtime-define.cpp index 324aa41..f9a8642 100644 --- a/examples/dirman/dirman2-runtime-define.cpp +++ b/examples/dirman/dirman2-runtime-define.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // This test shows how to // 1. Use dirman to get a list of all the nodes in the system diff --git a/examples/dirman/example1.cpp b/examples/dirman/example1.cpp index 2d66a09..d63acfc 100644 --- a/examples/dirman/example1.cpp +++ b/examples/dirman/example1.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/dirman/example2.cpp b/examples/dirman/example2.cpp index f1c88da..eff9f1f 100644 --- a/examples/dirman/example2.cpp +++ b/examples/dirman/example2.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/dirman/example3.cpp b/examples/dirman/example3.cpp index bc0f412..df66872 100644 --- a/examples/dirman/example3.cpp +++ b/examples/dirman/example3.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/dirman/example4.cpp b/examples/dirman/example4.cpp index 209e7d8..2ccf7ed 100644 --- a/examples/dirman/example4.cpp +++ b/examples/dirman/example4.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/dirman/example5.cpp b/examples/dirman/example5.cpp index 73d8236..e3133fe 100644 --- a/examples/dirman/example5.cpp +++ b/examples/dirman/example5.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/faodel-cli/basic-startstop.sh b/examples/faodel-cli/basic-startstop.sh index 600fa9f..efcce8d 100755 --- a/examples/faodel-cli/basic-startstop.sh +++ b/examples/faodel-cli/basic-startstop.sh @@ -61,7 +61,7 @@ Step 1: Launch a dirman server DirMan is a service for tracking different resources in the cluster and will help us control what resources we have. We can launch a plain dirman server with the faodel tool. By default it creates a file named - '.faodel-dirm in your current directory to tell nodes where to look. Other + '.faodel-dirman in your current directory to tell nodes where to look. Other operations in the faodel tool will use this to locate the server. This example uses the .faodel-dirman file to reference dirman to simplify command line options. diff --git a/examples/faodel-cli/playback-scripts/s01_basic_local.play b/examples/faodel-cli/playback-scripts/s01_basic_local.play new file mode 100644 index 0000000..22b89fa --- /dev/null +++ b/examples/faodel-cli/playback-scripts/s01_basic_local.play @@ -0,0 +1,81 @@ +# Script 01: Basic Local ops +# +# faodel -v play s01_basic_local.play +# +# This script just works on a standalone node and makes local queries. It +# starts up dirman, registers a pool for itself, and then does the most +# basic operations. This example provides more info about different +# phases in the play script. +# +# + +# Part 1: Configuration Appends +# ============================================================================ +# You can use the "config" directive in a play script to append some info +# to the faodel Configuration that gets loaded by bootstrap. This is handy +# for when you want to hardware some resources into the script or override +# some settings (eg, kelpie.debug) so you can get more verbose messages +# without having to touch your $FAODEL_CONFIG file. The faodel command loads +# your $FAODEL_CONFIG and then appends it with anything following "config". +# +# For this example we're going to define two pools, each with their own +# bucket namespace. + +config dirman.resources[] local:/my/place +config dirman.resources[] local:[thing]/other/place +#config bootstrap.debug true + + +# Part 2: Setting some defaults +# ============================================================================ +# It gets tedious having to specify some settings over and over. The "set" +# command lets you define a default value for a particular setting. You +# can redefine the variable later on in the play script, or specify a +# different value on each command. The only defaults to set are for: +# +# rank : indicate the following commands are for a specific rank +# pool : specify the default pool + +set pool /my/place + + +# Part 3: Looking at resources +# ============================================================================ +# You can use the dirman resource tools to get more info about what resources +# are available in the system. In addition to seeing our two predefined +# resources, we should also be able to see the directory structure that gets +# setup when defining hierarchical names. + +print == Dump Some Dirman Path Info ================================== +rlist / # Should have 'my' as a member +rlist /my # Should have 'place' as a member +rlist /my/place # Should give local:/my/place +rlist /other/place # Should give local:/my/place + + +# Part 4: Working with Kelpie objects +# ============================================================================ +# For each pool, you can issue queries to list, put/get, load/store, or get +# info for different keys. Briefly: +# +# klist : Shows a list of keys/blobs matching simple search patterns +# kinfo : Get more detailed info about a key/blob +# kput : Publish an object. May generate it, load from file, or use existing +# kget : Retrieve a particular key. Will block of unavailable +# kload : Read from a saved ldo and push to pool +# ksave : Save an object to a ldo file + +print == Showing contents (expect empty) ============================= +klist *|* # Nothing here yet + +print == Create foobar =============================================== +kput -M 63k -D 2k -k foobar # generate a 63k+2k object and publish + +print == See if it's there =========================================== +kinfo -k foobar # get some info about the object + +print == Try saving the LDO ========================================== +ksave -d ./tmp -k foobar # Save object to a directory + + + diff --git a/examples/faodel-cli/playback-scripts/s02_trace_pool_example.play b/examples/faodel-cli/playback-scripts/s02_trace_pool_example.play new file mode 100644 index 0000000..205310d --- /dev/null +++ b/examples/faodel-cli/playback-scripts/s02_trace_pool_example.play @@ -0,0 +1,51 @@ +# Script 02 Simple trace pool example +# +# faodel -v play s02_trace_pool_example.txt +# +# FAODEL's trace pool provides a way to record the activities that happen +# during the lifetime of a pool. The trace for each rank's pool gets +# written out to a separate file so you can then combine results and +# replay them. +# +# The trace pool can be connected to another pool handle to ensure +# that data gets routed properly after going through the trace. By +# default this pool is the null: pool, which just drops data. You +# can override that pool though by specifying the "next_pool" option +# in the trace url. +# +# In this example, we make a few different objects land in different +# namespaces that are colocated in the local pool. The namespaces +# are squares, trianngle, secret, and default. We want to intercept +# data that's going into the squares pool and route it to the +# secret pool. +# + +# Define the resources in config to make the messages easier +config dirman.resources[] trace:[squares]/my/place&next_pool=[secret]/my/secret/place +config dirman.resources[] local:[triangles]/other/place +config dirman.resources[] local:[secret]/my/secret/place + +#config bootstrap.debug true +#config dirman.debug true + + + +print == Create Data ================================================= +kput -p [squares]/my/place -M 63k -D 2k -k put-into-trace1 +kput -p [squares]/my/place -M 63k -D 2k -k put-into-trace2 + +kput -p local:/ -M 1k -D 1k -k put-into-default +kput -p local:[squares]/ -M 1k -D 1k -k put-into-squares +kput -p local:[hexes]/ -M 1k -D 1k -k put-into-hexes + +print +print == Local should have just put-into-default ===================== +klist -p local: -k * + +print +print == Squares should only have put-into-squares =================== +klist -p local:[squares] -k * + +print +print == Secret should have put-into-trace1&2 ======================== +klist -p [secret]/my/secret/place -k * diff --git a/examples/faodel-cli/playback-scripts/s03_advanced_file_handling.play b/examples/faodel-cli/playback-scripts/s03_advanced_file_handling.play new file mode 100644 index 0000000..42b93c2 --- /dev/null +++ b/examples/faodel-cli/playback-scripts/s03_advanced_file_handling.play @@ -0,0 +1,75 @@ +# Script 03: Advanced file handling +# +# faodel -v play s03_advanced_file_handling.play +# + + +config dirman.resources[] local:/my/place +config dirman.resources[] local:[thing]/other/place +#config bootstrap.debug true + +# Set the default pool used in commands that follow. Each command +# can override this by specifying -p poolname. +set pool /my/place + +print == Dump Some Dirman Path Info ================================== +rlist / # Should have 'my' as a member +rlist /my # Should have 'place' as a member + +print == Showing Info about our Pre-registered pool ================== +rlist /my/place # Should give local:/my/place +rlist /other/place # Should give local:/my/place + +print == Showing contents (expect empty) ============================= +klist *|* # Nothing here yet + +print == Create foobar =============================================== +kput -M 63k -D 2k -k foobar + +print == See if it's there =========================================== +kinfo -k foobar + +print == Try saving the LDO ========================================== +ksave -d ./tmp -k foobar + +print == Load it into another pool==================================== +kload -d ./tmp -k foobar -p [thing]/other/place +kput -D 1k -k foobar2 -p [thing]/other/place +klist -p [thing]/other/place -k * +klist -k * + +print == Reading file foobar and pushing it as 'bozo' ================ +# KPut can read a normal file, wrap it into an ldo, and push it into +# a pool. For this script we're going to pretend like the file we +# just saved is a regular file and inject it into the system. +# +# note: This isn't the right way to load a saved ldo into a pool, +# since it creates two wrappings around your data. This is +# just an example of how to push a file. +kput -f tmp/foobar%06%00 -m ObjectThing -k bozo + + +print == Do a list of First Pool (should be bozo and foobar) ========= +# Verify this new object is in faodel. +klist -k *|* + +print == Get the item (should succeed, but no output) ================ +# Retrieve the object +kget -k bozo + +print == Get Info (should be >64KB) ================================== +kinfo bozo + +print == Push file as a few more keys (3 variants of limbo) ========== +kput -f tmp/foobar%06%00 -k limbo +kput -f tmp/foobar%06%00 -k limbo_jimbo +kput -f tmp/foobar%06%00 -k limbo_timbo + +print == List all Limbos (should be 3 limbo*) ======================== +klist limbo*|* + +print == List a subset of limbo_* (limbo_jimbo, limbo_timbo) ========= +klist limbo_*|* + +print == Make sure no visible in other bucket (foobar, foobar2) ====== +klist -p [thing]/other/place -k * \ No newline at end of file diff --git a/examples/faodel-cli/playback-scripts/s04_bucket_renaming.play b/examples/faodel-cli/playback-scripts/s04_bucket_renaming.play new file mode 100644 index 0000000..7621647 --- /dev/null +++ b/examples/faodel-cli/playback-scripts/s04_bucket_renaming.play @@ -0,0 +1,50 @@ +# Script 04: Bucket renaming +# +# faodel -v plat s04_bucket_renaming.play +# +# +# This example shows how you can append "bucket=" to the end of the url +# to change what bucket gets used for a pool. There are two places where +# this is useful: +# +# 1. Shorthand: If you add the bucket= tag to a url when you define it, +# it will use that bucket when you reference it. If you don't do this +# you need to include the bucket in the ref (ie [bob]/a/b/c/second) +# +# 2. Using the same pool configuration for different datasets: By appending +# a new bucket to the end of the reference you use in Connect, you get +# an existing pool configuration, but with a different bucket to give +# you some k/v isolation. + + +config bootstrap.debug true + +config mpisyncstart.debug true +config dirman.debug true +config dirman.root_node_mpi END + +config kelpie.ioms myiom +config kelpie.iom.myiom.type posixindividualobjects +config kelpie.iom.myiom.path ./faodel-dump + +config dirman.resources[] local:/a/b/c/first&iom=myiom +config dirman.resources[] local:[bob]/a/b/c/second&iom=myiom +config dirman.resources[] local:/a/b/c/third&bucket=frank&iom=myiom + +config dirman.resources_mpi[] dht:/a/b/c/dht&min_members=1&bucket=bongo&iom=myiom ALL + + +barrier + +kput -p /a/b/c/first -M 1k -D 1k -k wrote-to-first +kput -p [bob]/a/b/c/second -D 1k -k wrote-to-second +kput -p /a/b/c/third -D 1k -k wrote-to-third + +kput -r 0 -p /a/b/c/dht -D 1k -k wrote-to-dht0 +kput -r 1 -p /a/b/c/dht -D 1k -k wrote-to-dht1 +kput -r 0 -p /a/b/c/dht&bucket=newbucket -D 1k -k alternate-bucket + +barrier + +# Look in your faodel-dump directory. There should be multiple hash directories, +# each with different data files \ No newline at end of file diff --git a/examples/faodel-cli/playback-scripts/s05_self_contained_pool.play b/examples/faodel-cli/playback-scripts/s05_self_contained_pool.play new file mode 100644 index 0000000..43515fa --- /dev/null +++ b/examples/faodel-cli/playback-scripts/s05_self_contained_pool.play @@ -0,0 +1,36 @@ +# Script 05: Self Contained Pool +# +# salloc -N 4 --ntasks-per-node 1 +# srun faodel -v play s05_self_contained_pool.play +# + +# Purpose: This script launches everything you need to host a kelpie pool on +# a collection of nodes launched by mpi. It does the following: +# +# 1. Runs a dirman on rank 0 +# 2. Writes the dirman id to ./.faodel-dirman for cli to find +# 3. It defines a dht hosted on all mpi ranks +# 4. It launches the kelpie nodes on all ranks to host the pool +# 5. It haults so it stays live until you kill it +# +# Note: This currently uses bootstrap halt to hang. Needs a proper halt cmd +# +# +config dirman.root_node_mpi 0 +config dirman.resources_mpi[] dht:/ioss/dht ALL +config dirman.write_root.file ./.faodel-dirman + +config bootstrap.debug true +config bootstrap.halt_on_shutdown true + +print == Starting Kelpie node == + +barrier + + +# Once this is running, you should be able to put/get items from the cli. Make +# sure you use srun on the Cray platforms to prevent drc issues. +# +# echo "Hello" | srun faodel kput -p /ioss/dht -k mystuff +# srun faodel kget -p /ioss/dht -k mystuff +# diff --git a/examples/kelpie/CMakeLists.txt b/examples/kelpie/CMakeLists.txt index 0b52eec..97d04a6 100644 --- a/examples/kelpie/CMakeLists.txt +++ b/examples/kelpie/CMakeLists.txt @@ -1,9 +1,17 @@ -set(EXAMPLES_LIBS - Faodel::kelpie - Boost::program_options -) +add_subdirectory( nonet/start-finish ) +add_subdirectory( nonet/local-pool-basics ) +add_subdirectory( nonet/using-whookie ) +add_subdirectory( nonet/storing-in-iom ) +add_subdirectory( nonet/dim-sum ) -add_subdirectory( nonet ) -add_subdirectory( prod-con ) + + +if( NOT TARGET MPI::MPI_CXX ) + message( STATUS "Some Kelpie Examples will not be built because MPI Support was not enabled") + +else() + add_subdirectory( compute ) + add_subdirectory( prod-con ) +endif() \ No newline at end of file diff --git a/examples/kelpie/README.md b/examples/kelpie/README.md index 7a2a6dc..9741134 100644 --- a/examples/kelpie/README.md +++ b/examples/kelpie/README.md @@ -49,4 +49,12 @@ entities. The consumer sums up information generated by each producer. prod-con -------- The prod-con is a networked example that uses a DHT pool to move data -objects between producer and consumer ranks. \ No newline at end of file +objects between producer and consumer ranks. + +compute +------- +Starting in 1.2108.1, Kelpie allows users to create their own user-defined +functions for processing data at remote nodes. The compute examples show +how you can (1) use a built-in function to pick the column +to pick the first/last/smallest/largest item in a row and (2) define +your own function for manipulating data before it is returned. \ No newline at end of file diff --git a/examples/kelpie/compute/CMakeLists.txt b/examples/kelpie/compute/CMakeLists.txt new file mode 100644 index 0000000..b9381f6 --- /dev/null +++ b/examples/kelpie/compute/CMakeLists.txt @@ -0,0 +1,24 @@ +set(FOO ex_kelpie_compute1_pick ex_kelpie_compute2_udf) +# When doing a standalone build, use the Faodel::kelpie imported target +if( NOT DEFINED EXAMPLE_LIBS) + set(EXAMPLE_LIBS Faodel::kelpie) +endif() + +# We also use boost and mpi +list(APPEND EXAMPLE_LIBS + MPI::MPI_CXX + ) + +set(TARGET ex_kelpie_compute1_pick) +add_executable( ${TARGET} ex_compute1_pick.cpp ) +target_link_libraries( ${TARGET} ${EXAMPLE_LIBS} ) + +set(TARGET ex_kelpie_compute2_udf) +add_executable( ${TARGET} ex_compute2_udf.cpp ) +target_link_libraries( ${TARGET} ${EXAMPLE_LIBS} ) + + +install(TARGETS ${FOO} + EXPORT faodelExampleTargets + RUNTIME DESTINATION "${BINARY_INSTALL_DIR}" COMPONENT bin + ) diff --git a/examples/kelpie/compute/ex_compute1_pick.cpp b/examples/kelpie/compute/ex_compute1_pick.cpp new file mode 100644 index 0000000..b9d0437 --- /dev/null +++ b/examples/kelpie/compute/ex_compute1_pick.cpp @@ -0,0 +1,100 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +// Kelpie Compute Example: Built-in 'pick' function +// +// Kelpie now provides a basic way to perform compute operations on the +// node that owns an object within a pool. Users may register a +// user-defined function that takes one or more key/blobs as input and +// produces a single DataObject output. When a user calls the +// pool.Compute() operation, the command is dispatched to the proper +// server, data is atomically retrieved out of its local in-memory store, +// the function is applied, and the result is returned. +// +// Some useful features: +// 1. User may specify a key with a column wildcard. For example, if +// you ask for key("mything", "foo*"), you'll get all entries in +// row mything that have a column name starting with foo. +// 2. User supply a string argument for the computation. The string +// is brought to the remote node and supplied into the function. +// 3. Kelpie provides a built-in function called 'pick', which has +// four options for arguments: 'first', 'last','smallest', and +// biggest. This function returns the object that has a keyname +// that is alphabetically first or last in the wildcard list, +// or the object that is smallest/largest in size. +// +// In this example, we use the built-in 'pick' function to select an +// object from the row. This example uses mpisyncstart to simplify +// starting up a dht on all the nodes in the mpi job. Each node writes a +// string object to the same row (but different column). Rank 0 then +// uses the 'pick' function and a wildcard to read different objects. + +#include +#include + +#include "faodel-common/Common.hh" +#include "faodel-services/MPISyncStart.hh" +#include "lunasa/common/Helpers.hh" +#include "kelpie/Kelpie.hh" + +//The configuration used in this example +std::string default_config_string = R"EOF( + +# Use mpisyncstart to create a DHT that is spread across all our nodes +mpisyncstart.enable true +dirman.type centralized +dirman.root_node_mpi 0 +dirman.resources_mpi[] dht:/myplace ALL + +# Uncomment these options to get debug info for each component +bootstrap.debug true +#whookie.debug true +#opbox.debug true +#dirman.debug true +#kelpie.debug true +)EOF"; + +using namespace std; + +int main(int argc, char **argv) { + + //Initialize MPI + int provided, mpi_rank,mpi_size; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + + //Start faodel. Use mpisyncstart to setup dirman and create a dht named /myplace + faodel::mpisyncstart::bootstrap(); + faodel::bootstrap::Start(faodel::Configuration(default_config_string), kelpie::bootstrap); + + //Connect to the pool and write our rank's contribution to the row + auto pool = kelpie::Connect("/myplace"); + kelpie::Key k1("myrow",to_string(mpi_rank)); + auto ldo1 = lunasa::AllocateStringObject("This is an object from rank "+ + to_string(mpi_rank)+string(mpi_size-mpi_rank,'!')); + pool.Publish(k1, ldo1); + + //Wait for everyone to be done, then have rank issue 'pick' operations to get first/last items in row + MPI_Barrier(MPI_COMM_WORLD); + if(mpi_rank==0) { + kelpie::Key key_myrow("myrow", "*"); //Look at all row entries + lunasa::DataObject ldo_first, ldo_last, ldo_smallest, ldo_largest; + pool.Compute(key_myrow,"pick","first", &ldo_first); + pool.Compute(key_myrow,"pick","last", &ldo_last); + pool.Compute(key_myrow,"pick","smallest", &ldo_smallest); + pool.Compute(key_myrow,"pick","largest", &ldo_largest); + + cout<<"First item: "<< lunasa::UnpackStringObject(ldo_first) << endl; //Should be from rank 0 + cout<<"Last item: "<< lunasa::UnpackStringObject(ldo_last) << endl; //Should be from last rank + cout<<"Smallest item: "<< lunasa::UnpackStringObject(ldo_smallest) << endl; //Should be from last rank + cout<<"Largest item: "<< lunasa::UnpackStringObject(ldo_largest) << endl; //Should be from last rank + } + + MPI_Barrier(MPI_COMM_WORLD); + faodel::bootstrap::Finish(); + MPI_Finalize(); + return 0; +} + diff --git a/examples/kelpie/compute/ex_compute2_udf.cpp b/examples/kelpie/compute/ex_compute2_udf.cpp new file mode 100644 index 0000000..b09b291 --- /dev/null +++ b/examples/kelpie/compute/ex_compute2_udf.cpp @@ -0,0 +1,112 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +// Kelpie Compute Example: Adding a User-Defined Function +// +// Users can create their own user-defined function and register them with +// servers. All you need to do is: +// +// 1. Crate a UDF that follows the fn_compute_t API specified in faodel/common/Types.hh +// 2. Register the function after init but before start +// 3. Call pool.Compute(key, function_name, args, ldos, return_ldo) to dispatch + +#include +#include +#include + +#include "faodel-common/Common.hh" +#include "faodel-services/MPISyncStart.hh" +#include "lunasa/common/Helpers.hh" +#include "kelpie/Kelpie.hh" + +//The configuration used in this example +std::string default_config_string = R"EOF( + +# Use mpisyncstart to create a DHT that is spread across all our nodes +dirman.type centralized +dirman.root_node_mpi 0 +dirman.resources_mpi[] dht:/myplace ALL + +# Uncomment these options to get debug info for each component +bootstrap.debug true +#whookie.debug true +#opbox.debug true +#dirman.debug true +#kelpie.debug true +)EOF"; + +using namespace std; +using namespace kelpie; + +// This is an example of a user-defined function that simply cats all the objects together +// into a new string object that can be sent to the user +rc_t fn_udf_merge(faodel::bucket_t, const Key &key, const std::string &args, + std::map ldos, lunasa::DataObject *ext_ldo) { + stringstream ss; + for(auto &key_blob : ldos) { + ss<<"Key: "<< key_blob.first.str()< ldos, lunasa::DataObject *ext_ldo) { + + stringstream ss; + for(auto &key_blob : ldos) { + ss<<"Key: "<< key_blob.first.str()< #include diff --git a/examples/kelpie/nonet/dim-sum/Consumer.hh b/examples/kelpie/nonet/dim-sum/Consumer.hh index 3a42d14..89ed76b 100644 --- a/examples/kelpie/nonet/dim-sum/Consumer.hh +++ b/examples/kelpie/nonet/dim-sum/Consumer.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_DIMSUM_CONSUMER_HH #define KELPIE_DIMSUM_CONSUMER_HH diff --git a/examples/kelpie/nonet/dim-sum/DimSum.cpp b/examples/kelpie/nonet/dim-sum/DimSum.cpp index eed32b4..1f3d1b3 100644 --- a/examples/kelpie/nonet/dim-sum/DimSum.cpp +++ b/examples/kelpie/nonet/dim-sum/DimSum.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/kelpie/nonet/dim-sum/DimSum.hh b/examples/kelpie/nonet/dim-sum/DimSum.hh index 2577d93..e77ce8a 100644 --- a/examples/kelpie/nonet/dim-sum/DimSum.hh +++ b/examples/kelpie/nonet/dim-sum/DimSum.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_DIMSUM_DIMSUM_HH #define KELPIE_DIMSUM_DIMSUM_HH diff --git a/examples/kelpie/nonet/dim-sum/Producer.cpp b/examples/kelpie/nonet/dim-sum/Producer.cpp index bc4b8e3..b734a89 100644 --- a/examples/kelpie/nonet/dim-sum/Producer.cpp +++ b/examples/kelpie/nonet/dim-sum/Producer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/kelpie/nonet/dim-sum/Producer.hh b/examples/kelpie/nonet/dim-sum/Producer.hh index 50d243e..6b0af11 100644 --- a/examples/kelpie/nonet/dim-sum/Producer.hh +++ b/examples/kelpie/nonet/dim-sum/Producer.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_DIMSUM_PRODUCER_HH #define KELPIE_DIMSUM_PRODUCER_HH diff --git a/examples/kelpie/nonet/dim-sum/WorkerThread.cpp b/examples/kelpie/nonet/dim-sum/WorkerThread.cpp index a9ab819..57729f5 100644 --- a/examples/kelpie/nonet/dim-sum/WorkerThread.cpp +++ b/examples/kelpie/nonet/dim-sum/WorkerThread.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "WorkerThread.hh" diff --git a/examples/kelpie/nonet/dim-sum/WorkerThread.hh b/examples/kelpie/nonet/dim-sum/WorkerThread.hh index 1dbbcce..9ae0c9a 100644 --- a/examples/kelpie/nonet/dim-sum/WorkerThread.hh +++ b/examples/kelpie/nonet/dim-sum/WorkerThread.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_DIMSUM_WORKERTHREAD_HH #define KELPIE_DIMSUM_WORKERTHREAD_HH diff --git a/examples/kelpie/nonet/dim-sum/dim-sum.cpp b/examples/kelpie/nonet/dim-sum/dim-sum.cpp index 8b7ac82..18a7eab 100644 --- a/examples/kelpie/nonet/dim-sum/dim-sum.cpp +++ b/examples/kelpie/nonet/dim-sum/dim-sum.cpp @@ -1,11 +1,11 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -// Example: kelpie local pool basics -// Purpose: +// Example: dim-sum +// Purpose: Create a number of producer and consumer threads that pass data through a local kelpie // -// Keypoints +// Keypoints: Shows how to use Kelpie to pass events between threads #include @@ -21,6 +21,7 @@ std::string default_config_string = R"EOF( # For local testing, tell kelpie to use the nonet implementation kelpie.type nonet +dirman.type none # Uncomment these options to get debug info for each component #bootstrap.debug true @@ -53,6 +54,7 @@ int main(){ for(int i=0; iStart(); } @@ -62,9 +64,6 @@ int main(){ wptr->Join(); } - - - cout <<"Finishing..\n"; faodel::bootstrap::Finish(); diff --git a/examples/kelpie/nonet/local-pool-basics/CMakeLists.txt b/examples/kelpie/nonet/local-pool-basics/CMakeLists.txt index 90d139b..d44c667 100644 --- a/examples/kelpie/nonet/local-pool-basics/CMakeLists.txt +++ b/examples/kelpie/nonet/local-pool-basics/CMakeLists.txt @@ -1,19 +1,23 @@ -set(PROJECT_NAME local-pool-basics) +set(PROJECT_NAME ex_kelpie_local-pool-basics) set(HEADERS ) set(SOURCES - local-pool-basics.cpp - ex1_get_pool_handle.cpp - ex2_publish_need.cpp + local-pool-basics.cpp + ex1_get_pool_handle.cpp + ex2_publish_need.cpp ) +# When doing a standalone build, use the Faodel::kelpie imported target +if( NOT DEFINED EXAMPLE_LIBS) + set(EXAMPLE_LIBS Faodel::kelpie) +endif() add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS} ) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS} ) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/kelpie/nonet/local-pool-basics/ex1_get_pool_handle.cpp b/examples/kelpie/nonet/local-pool-basics/ex1_get_pool_handle.cpp index 76b2b88..4679d91 100644 --- a/examples/kelpie/nonet/local-pool-basics/ex1_get_pool_handle.cpp +++ b/examples/kelpie/nonet/local-pool-basics/ex1_get_pool_handle.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/kelpie/nonet/local-pool-basics/ex2_publish_need.cpp b/examples/kelpie/nonet/local-pool-basics/ex2_publish_need.cpp index 1742bf5..bc9c31f 100644 --- a/examples/kelpie/nonet/local-pool-basics/ex2_publish_need.cpp +++ b/examples/kelpie/nonet/local-pool-basics/ex2_publish_need.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/kelpie/nonet/local-pool-basics/local-pool-basics.cpp b/examples/kelpie/nonet/local-pool-basics/local-pool-basics.cpp index 177739b..7d26552 100644 --- a/examples/kelpie/nonet/local-pool-basics/local-pool-basics.cpp +++ b/examples/kelpie/nonet/local-pool-basics/local-pool-basics.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Example: kelpie local pool basics // Purpose: @@ -18,6 +18,7 @@ std::string default_config_string = R"EOF( # For local testing, tell kelpie to use the nonet implementation kelpie.type standard +dirman.type none # Uncomment these options to get debug info for each component #bootstrap.debug true diff --git a/examples/kelpie/nonet/start-finish/CMakeLists.txt b/examples/kelpie/nonet/start-finish/CMakeLists.txt index bc87d69..840d953 100644 --- a/examples/kelpie/nonet/start-finish/CMakeLists.txt +++ b/examples/kelpie/nonet/start-finish/CMakeLists.txt @@ -1,17 +1,21 @@ -set(PROJECT_NAME start-finish) +set(PROJECT_NAME ex_kelpie_start-finish) set(HEADERS ) set(SOURCES - start-finish.cpp + start-finish.cpp ) +# When doing a standalone build, use the Faodel::kelpie imported target +if( NOT DEFINED EXAMPLE_LIBS) + set(EXAMPLE_LIBS Faodel::kelpie) +endif() add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS} ) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS} ) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/kelpie/nonet/start-finish/start-finish.cpp b/examples/kelpie/nonet/start-finish/start-finish.cpp index 31cbadb..2eb9e48 100644 --- a/examples/kelpie/nonet/start-finish/start-finish.cpp +++ b/examples/kelpie/nonet/start-finish/start-finish.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Example: kelpie start-finish // Purpose: This is a minimal example that demonstrates how to start/finish @@ -27,6 +27,7 @@ std::string default_config_string = R"EOF( # For local testing, tell kelpie to use the nonet implementation kelpie.type nonet +dirman.type none # Uncomment these options to get debug info for each component #bootstrap.debug true diff --git a/examples/kelpie/nonet/storing-in-iom/CMakeLists.txt b/examples/kelpie/nonet/storing-in-iom/CMakeLists.txt index 2a8360e..abb0694 100644 --- a/examples/kelpie/nonet/storing-in-iom/CMakeLists.txt +++ b/examples/kelpie/nonet/storing-in-iom/CMakeLists.txt @@ -1,17 +1,22 @@ -set(PROJECT_NAME storing-in-iom) +set(PROJECT_NAME ex_kelpie_storing-in-iom) set(HEADERS - ) +) set(SOURCES - storing-in-iom.cpp - storing-in-iom.cpp) + storing-in-iom.cpp + storing-in-iom.cpp +) +# When doing a standalone build, use the Faodel::kelpie imported target +if( NOT DEFINED EXAMPLE_LIBS) + set(EXAMPLE_LIBS Faodel::kelpie) +endif() add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS} ) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS} ) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/kelpie/nonet/storing-in-iom/storing-in-iom.cpp b/examples/kelpie/nonet/storing-in-iom/storing-in-iom.cpp index 06f7c45..37194b5 100644 --- a/examples/kelpie/nonet/storing-in-iom/storing-in-iom.cpp +++ b/examples/kelpie/nonet/storing-in-iom/storing-in-iom.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. //This example shows how you can use an iom to store data persistently. // @@ -24,6 +24,7 @@ std::string default_config_string = R"EOF( # For local testing, tell kelpie to use the nonet implementation kelpie.type nonet +dirman.type none default.iom.type PosixIndividualObjects default.ioms my_iom @@ -97,16 +98,13 @@ int main(int argc, char **argv) { if(is_writer) { cout <<"Writing "< items_left(NUM_ITEMS); + kelpie::ResultCollector results(NUM_ITEMS); for (int i = 0; i < NUM_ITEMS; i++) { auto ldo = createLDO(i, "my brick_" + std::to_string(i), 1024); - piom.Publish(kelpie::Key("my_brick_" + std::to_string(i)), ldo, - [&items_left](kelpie::rc_t result, kelpie::kv_row_info_t &ri, kelpie::kv_col_info_t &ci) { - items_left--; - }); + piom.Publish(kelpie::Key("my_brick_" + std::to_string(i)), ldo, results); } //Wait until all publishes complete - while (items_left) { sched_yield(); } + results.Sync(); cout << "Done writing to " << path_name << endl; } else { @@ -124,4 +122,4 @@ int main(int argc, char **argv) { faodel::bootstrap::Finish(); return 0; -} \ No newline at end of file +} diff --git a/examples/kelpie/nonet/using-whookie/CMakeLists.txt b/examples/kelpie/nonet/using-whookie/CMakeLists.txt index 4db8e66..6d0d56a 100644 --- a/examples/kelpie/nonet/using-whookie/CMakeLists.txt +++ b/examples/kelpie/nonet/using-whookie/CMakeLists.txt @@ -1,17 +1,21 @@ -set(PROJECT_NAME using-whookie) +set(PROJECT_NAME ex_kelpie_using-whookie) set(HEADERS ) set(SOURCES - using-whookie.cpp + using-whookie.cpp ) +# When doing a standalone build, use the Faodel::kelpie imported target +if( NOT DEFINED EXAMPLE_LIBS) + set(EXAMPLE_LIBS Faodel::kelpie) +endif() add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS} ) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS} ) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/kelpie/nonet/using-whookie/using-whookie.cpp b/examples/kelpie/nonet/using-whookie/using-whookie.cpp index 6236591..4b8da4f 100644 --- a/examples/kelpie/nonet/using-whookie/using-whookie.cpp +++ b/examples/kelpie/nonet/using-whookie/using-whookie.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Example: kelpie using-whookie // Purpose: Whookie provides an easy way to check on the status of different @@ -23,6 +23,7 @@ std::string default_config_string = R"EOF( # For local testing, tell kelpie to use the nonet implementation kelpie.type nonet +dirman.type none # Uncomment these options to get debug info for each component #bootstrap.debug true diff --git a/examples/kelpie/prod-con/CMakeLists.txt b/examples/kelpie/prod-con/CMakeLists.txt index a6d2938..44f9f67 100644 --- a/examples/kelpie/prod-con/CMakeLists.txt +++ b/examples/kelpie/prod-con/CMakeLists.txt @@ -1,17 +1,33 @@ +set( Prod_Con_REQUIRED_HDF5_VERSION "1.10" ) set( HDF5_PREFER_PARALLEL TRUE ) -add_executable( prod-con prod-con.cpp Globals.cpp ) -target_link_libraries( prod-con ${EXAMPLES_LIBS} ) +set(TARGET ex_kelpie_prod-con) + +# When doing a standalone build, use the Faodel::kelpie imported target +if( NOT DEFINED EXAMPLE_LIBS) + set(EXAMPLE_LIBS Faodel::kelpie) +endif() + +# We also use boost and mpi +list(APPEND EXAMPLE_LIBS + Boost::program_options + MPI::MPI_CXX +) + +add_executable( ${TARGET} prod-con.cpp Globals.cpp ) +target_link_libraries( ${TARGET} ${EXAMPLE_LIBS} ) find_package( HDF5 COMPONENTS CXX HL ) if( HDF5_FOUND ) - target_link_libraries( prod-con ${HDF5_LIBRARIES} ) - target_include_directories( prod-con PUBLIC ${HDF5_INCLUDE_DIRS} ) - target_compile_definitions( prod-con PUBLIC ${HDF5_DEFINITIONS} -DHAVE_HDF5 ) + if( HDF5_VERSION VERSION_GREATER_EQUAL ${Prod_Con_REQUIRED_HDF5_VERSION} ) + target_link_libraries( ${TARGET} ${HDF5_LIBRARIES} ) + target_include_directories( ${TARGET} PUBLIC ${HDF5_INCLUDE_DIRS} ) + target_compile_definitions( ${TARGET} PUBLIC ${HDF5_DEFINITIONS} -DHAVE_HDF5 ) + endif() endif() -install(TARGETS prod-con +install(TARGETS ${TARGET} EXPORT faodelExampleTargets RUNTIME DESTINATION "${BINARY_INSTALL_DIR}" COMPONENT bin ) diff --git a/examples/kelpie/prod-con/Globals.cpp b/examples/kelpie/prod-con/Globals.cpp index ea2de22..a3983b8 100644 --- a/examples/kelpie/prod-con/Globals.cpp +++ b/examples/kelpie/prod-con/Globals.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/kelpie/prod-con/Globals.hh b/examples/kelpie/prod-con/Globals.hh index 23b89fc..16063b9 100644 --- a/examples/kelpie/prod-con/Globals.hh +++ b/examples/kelpie/prod-con/Globals.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef GLOBALS_HH #define GLOBALS_HH diff --git a/examples/kelpie/prod-con/prod-con.cpp b/examples/kelpie/prod-con/prod-con.cpp index 5ac67e1..9c21ba2 100644 --- a/examples/kelpie/prod-con/prod-con.cpp +++ b/examples/kelpie/prod-con/prod-con.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Warning: This example currently currently has problems at shutdown // when operations are still in flight. @@ -181,6 +181,12 @@ int main(int argc, char **argv) { return 0; } + if(G.mpi_size==1) { + std::cerr<<"This example needs to be run with multiple mpi ranks\n"; + G.StopAll(); + return -1; + } + participants = G.mpi_size; /* Enforce 2:1 ratio of producers to consumers */ diff --git a/examples/lunasa/CMakeLists.txt b/examples/lunasa/CMakeLists.txt index f0f5d2a..b2c1d63 100644 --- a/examples/lunasa/CMakeLists.txt +++ b/examples/lunasa/CMakeLists.txt @@ -1,10 +1,12 @@ +add_subdirectory( bootstrap ) +add_subdirectory( ldo_packing ) -if( NOT Faodel_BUILT_WITH_MPI ) - message( STATUS "The installed Faodel was not built with MPI, so Lunasa examples cannot be built" ) + +if( NOT TARGET MPI::MPI_CXX ) + message( STATUS "Some Lunasa Examples will not be built because MPI support was not enabled" ) else() - set( EXAMPLES_LIBS Faodel::lunasa ) - add_subdirectory( bootstrap ) + # These examples require nnti for moving data between nodes if(Faodel_NETWORK_LIBRARY STREQUAL "nnti") add_subdirectory( simple_ldo_send ) add_subdirectory( simple_user_ldo_put ) diff --git a/examples/lunasa/bootstrap/CMakeLists.txt b/examples/lunasa/bootstrap/CMakeLists.txt index b080ca4..56203c4 100644 --- a/examples/lunasa/bootstrap/CMakeLists.txt +++ b/examples/lunasa/bootstrap/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME lunasa_bootstrap_example) +set(PROJECT_NAME ex_lunasa_bootstrap) set(HEADERS ) @@ -7,11 +7,16 @@ set(SOURCES bootstrap_example.cpp ) +# When doing a standalone build, use the Faodel::lunasa imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::lunasa ) +endif() + add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS} ) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS} ) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/lunasa/bootstrap/bootstrap_example.cpp b/examples/lunasa/bootstrap/bootstrap_example.cpp index 387f6e4..ba14741 100644 --- a/examples/lunasa/bootstrap/bootstrap_example.cpp +++ b/examples/lunasa/bootstrap/bootstrap_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/lunasa/ldo_packing/CMakeLists.txt b/examples/lunasa/ldo_packing/CMakeLists.txt new file mode 100644 index 0000000..e969105 --- /dev/null +++ b/examples/lunasa/ldo_packing/CMakeLists.txt @@ -0,0 +1,19 @@ +set(PROJECT_NAME ex_lunasa_ldo-packing) + +set(HEADERS MyData.hh MyHelpers.hh) +set(SOURCES ldo_packing_example.cpp) + +# When doing a standalone build, use the Faodel::lunasa imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::lunasa ) +endif() + + +add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) +set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) + +install(TARGETS ${PROJECT_NAME} + EXPORT faodelExampleTargets + RUNTIME DESTINATION "${BINARY_INSTALL_DIR}" COMPONENT bin +) \ No newline at end of file diff --git a/examples/lunasa/ldo_packing/MyData.hh b/examples/lunasa/ldo_packing/MyData.hh new file mode 100644 index 0000000..4f04236 --- /dev/null +++ b/examples/lunasa/ldo_packing/MyData.hh @@ -0,0 +1,55 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_EXAMPLES_MYDATA_HH +#define FAODEL_EXAMPLES_MYDATA_HH + +#include +#include + +/** + * @brief Example class that contains different types of data + * @note Anything more sophisticated than this should use a real serializer + */ + +class MyData { + +public: + + MyData() : field_length(0), field1(nullptr), field2(nullptr) {} + MyData(int field_length) + : field_length(field_length) { + + prop1 = "My big property"; + prop2 = 42.0; + field1 = new float[field_length]; + field2 = new int[field_length]; + for(int i=0; i field3; + +}; + + +#endif //FAODEL_EXAMPLES_MYDATA_HH diff --git a/examples/lunasa/ldo_packing/MyHelpers.hh b/examples/lunasa/ldo_packing/MyHelpers.hh new file mode 100644 index 0000000..bd1721c --- /dev/null +++ b/examples/lunasa/ldo_packing/MyHelpers.hh @@ -0,0 +1,95 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_EXAMPLES_MYHELPERS_HH +#define FAODEL_EXAMPLES_MYHELPERS_HH + +#include +#include + +/** + * @brief Our custom type specification. User is free to encode any layout they want + */ +enum MyTypes : uint8_t { + STRING = 1, + INT = 2, + FLOAT = 3, + DOUBLE = 4 +}; + + +/** + * @brief Helper class to make it easier to gather up variables for packing + * + * This is an example helper class that shows how to use templates to gather + * up variable information so you can pack it easier. + */ +struct MyVariableGatherer { + std::vector names; + std::vector ptrs; + std::vector bytes; + std::vector types; + + template + void append(const std::string &name, const T *ptr, int num) { + names.push_back(name); + ptrs.push_back((void *) ptr); + bytes.push_back( sizeof(T)*num); + uint8_t type=0; + if( std::is_same::value) type=MyTypes::INT; + else if (std::is_same::value) type=MyTypes::FLOAT; + else if (std::is_same::value) type=MyTypes::DOUBLE; + types.push_back(type); + } + + //Make a special handler for strings + void append(const std::string &name, const std::string *ptr) { + names.push_back(name); + ptrs.push_back((void *) ptr->c_str()); + bytes.push_back(ptr->length()); + types.push_back(MyTypes::STRING); + } + +}; + +class MyVariableAccess { +public: + MyVariableAccess(lunasa::DataObjectPacker *dop) : dop(dop) {} + + std::string ExpectString(const std::string name) { + char *ptr; + size_t bytes; + uint8_t type; + int rc = dop->GetVarPointer(name,(void **)&ptr, &bytes, &type); + if((rc!=0) || (type!=MyTypes::STRING)) + throw std::runtime_error("Could not retrieve variable named "+name); + return std::string(ptr, bytes); + } + + template + T * ExpectArray(const std::string name, size_t *num_words) { + T *ptr; + size_t bytes; + uint8_t type; + int rc = dop->GetVarPointer(name, (void **)&ptr, &bytes, &type); + + size_t tmp_num_words=0; + if( ((std::is_same::value) && (type==MyTypes::INT)) || + ((std::is_same::value) && (type==MyTypes::FLOAT)) || + ((std::is_same::value) && (type==MyTypes::DOUBLE)) ){ + tmp_num_words = bytes/sizeof(T); + } + + if ( (rc) || (!tmp_num_words) ){ + ptr = nullptr; + } + if(num_words) *num_words = tmp_num_words; + return ptr; + } + +private: + lunasa::DataObjectPacker *dop; +}; + +#endif //FAODEL_EXAMPLES_MYHELPERS_HH diff --git a/examples/lunasa/ldo_packing/ldo_packing_example.cpp b/examples/lunasa/ldo_packing/ldo_packing_example.cpp new file mode 100644 index 0000000..bedde00 --- /dev/null +++ b/examples/lunasa/ldo_packing/ldo_packing_example.cpp @@ -0,0 +1,279 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +// DataObjectPacker Example +// +// Purpose: Demonstrate how DOP can be used to pack variables into an LDO +// +// Often users need a way to stuff a collection of variables into a memory +// blob that can be transported about the system. There are a few ways +// to do this, depending on how complex the data is: +// +// POD Structs: If you have plain-old-data types that can be described +// by a C struct, just cast the struct to the LDO and plug away. This +// is quick to pack/unpack and easy, but doesn't work well with +// variable data. +// +// SER/DES Libs: If you have complex data structures (hierarchical classes. +// c++ containers) or readability matters more than performance, use +// a serialization lib (eg, boost serialization, cereal, xdr, avro) and +// copy the serial stream into an ldo. +// +// DataObjectPacker: If you have a bunch of variable length arrays and you +// don't mind working w/ pointers into the LDO, use Lunasa's DOP. The DOP +// is a thin layer on top of an LDO that allocates and packs labeled +// vars into the object. It's not fancy, but it doesn't take much overhead +// and its fast. +// +// In this example a user has a class with multiple variables in it (MyData). +// A user needs to pack all of the values into an object and do something +// with the data on the receiving side. These examples show how to pack +// and unpack the items. +// +// Examples 1 and 3 show packing where we "allocate and pack" in the constructor +// Example 5 shows how to "Allocate a fixed capacity and append vars" +// +// The MyVariableGatherer and MyVariableAccess classes show how to simplify +// some of the tedious work in packing data structures. + + +#include +#include +#include + +#include "lunasa/Lunasa.hh" +#include "lunasa/common/DataObjectPacker.hh" + +#include "MyData.hh" +#include "MyHelpers.hh" + +using namespace std; + +string default_config = R"EOF( +server.mutex_type rwlock +# In this example, the default allocator is lunasa::AllocatorTcmalloc +lunasa.eager_memory_manager tcmalloc +node_role server +)EOF"; + + +//Since we often mix different data objects in FAODEL, it's useful to define +//a unique id to each packed data object so we can check to make sure its +//the thing we're looking for before we go and unpack it. This data +const uint32_t myapp_data_id = faodel::const_hash32("my data app"); + + +//Example 1: Simple up-front packing +// If you only have a few variables to pack, you can create a list of their +// stats and hand them over to the constructor. The ctor allocates the exact +// amount of space you need and then packs everything in. +// +// Note: This example only stores TWO variables. Example 3 shows how to +// simplify the packing of many vars using a helper class and templates. +lunasa::DataObject ex1_manual_upfront_pack(const MyData &src) { + + //Get info for all variables that are going to be packed + vector names; + vector ptrs; + vector bytes; + vector types; + + //Simple String + names.push_back("PROP1"); + ptrs.push_back((void *)src.prop1.c_str()); + bytes.push_back(src.prop1.length()); + types.push_back(MyTypes::STRING); + + + //Float array + names.push_back("FIELD1"); + ptrs.push_back((void *)src.field1); + bytes.push_back(sizeof(double) * src.field_length); + types.push_back( MyTypes::FLOAT); + + //Allocate an ldo and pack it + lunasa::DataObjectPacker dop(names, ptrs, bytes, types, myapp_data_id); + + return dop.GetDataObject(); + +} + + +//Example 2: Manual unpacking +// If you don't have a lot of data, it isn't hard to manually unpack things +// yourself. Just lookup the variable and convert its pointers/length. The +// pointer is a raw pointer into the DataObject so be careful not to +// write data or pass the pointer to another application. + +void ex2_manual_unpacking(lunasa::DataObject &ldo) { + + lunasa::DataObjectPacker dop(ldo); + + dop.VerifyDataType(myapp_data_id); + + void *ptr; + size_t bytes; + uint8_t type; + int rc; + + rc = dop.GetVarPointer("PROP1", &ptr, &bytes, &type); + assert(rc==0); + string prop1((char *)ptr, bytes); + cout <<"Prop1 is '"<(ptr); + size_t words = bytes/sizeof(float); + for(int i=0; i("PROP2", &num_words); + assert(p2!=nullptr); + cout <<"Prop2: num_words="<("FIELD2", &nw2); assert(f2!=nullptr); + double *f3 = access.ExpectArray("FIELD3", &nw3); assert(f3!=nullptr); + + cout <<"Field1: num_words="< names; + int rc = dop.GetVarNames(&names); + if(rc==0) { + cout<<"Received an object with the following vars:\n"; + for(int i=0; i #include @@ -232,7 +232,13 @@ int main(int argc, char **argv){ MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); /* Currently, only works for two MPI ranks: one sender and one receiver. */ - assert(2==mpi_size); + if(mpi_size==1) { + std::cerr<<"This example needs to be run with multiple mpi ranks\n"; + faodel::bootstrap::Finish(); + MPI_Finalize(); + return -1; + } + /* * The value of allows us to tell NNTI which process we want to connect to. diff --git a/examples/lunasa/simple_user_ldo_put/CMakeLists.txt b/examples/lunasa/simple_user_ldo_put/CMakeLists.txt index 66b474b..c745b6d 100644 --- a/examples/lunasa/simple_user_ldo_put/CMakeLists.txt +++ b/examples/lunasa/simple_user_ldo_put/CMakeLists.txt @@ -1,10 +1,21 @@ -set(PROJECT_NAME simple_user_ldo_put_example) +set(PROJECT_NAME ex_lunasa_ldo-put) set(SOURCES simple_user_ldo_put_example.cpp) +# When doing a standalone build, use the Faodel::lunasa imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::lunasa ) +endif() + +# We also use mpi +list(APPEND EXAMPLE_LIBS + MPI::MPI_CXX + ) + + add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/lunasa/simple_user_ldo_put/simple_user_ldo_put_example.cpp b/examples/lunasa/simple_user_ldo_put/simple_user_ldo_put_example.cpp index f3efe4f..5ee3c6a 100644 --- a/examples/lunasa/simple_user_ldo_put/simple_user_ldo_put_example.cpp +++ b/examples/lunasa/simple_user_ldo_put/simple_user_ldo_put_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -535,7 +535,12 @@ int main(int argc, char **argv){ MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); /* Currently, only works for two MPI ranks: one sender and one receiver. */ - assert(2==mpi_size); + if(mpi_size!=2) { + std::cerr<<"This example needs to be run with two mpi ranks\n"; + faodel::bootstrap::Finish(); + MPI_Finalize(); + return -1; + } /* * The value of allows us to tell NNTI which process we want to connect to. diff --git a/examples/nnti/CMakeLists.txt b/examples/nnti/CMakeLists.txt index a719a98..a3ba3e3 100644 --- a/examples/nnti/CMakeLists.txt +++ b/examples/nnti/CMakeLists.txt @@ -20,6 +20,7 @@ function( make_nnti_cpp_example item extra_cxx_flags extra_linker_flags ) add_executable( ${target} ${source} ) target_link_libraries( ${target} cppexampleutils + ZLIB::ZLIB ${extra_linker_flags} ) @@ -46,6 +47,8 @@ target_link_libraries(${cppexampleutils_target} Faodel::nnti ) set_target_properties(${cppexampleutils_target} PROPERTIES LINKER_LANGUAGE CXX ) -if (IBVerbs_FOUND) -make_nnti_cpp_example( cpp/PingPong "" "-lz -lpthread" ) -endif() +if( Faodel_HAVE_CRC32 ) +if( IBVerbs_FOUND ) +make_nnti_cpp_example( cpp/PingPong "" "" ) +endif( IBVerbs_FOUND ) +endif( Faodel_HAVE_CRC32 ) diff --git a/examples/nnti/cpp/PingPong.cpp b/examples/nnti/cpp/PingPong.cpp index 3e56fc8..e172526 100644 --- a/examples/nnti/cpp/PingPong.cpp +++ b/examples/nnti/cpp/PingPong.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" diff --git a/examples/nnti/cpp/example_utils.cpp b/examples/nnti/cpp/example_utils.cpp index 80828f7..75743ce 100644 --- a/examples/nnti/cpp/example_utils.cpp +++ b/examples/nnti/cpp/example_utils.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. //#include "nnti/nnti_pch.hpp" diff --git a/examples/nnti/cpp/example_utils.hpp b/examples/nnti/cpp/example_utils.hpp index 452c4ce..7d7d55c 100644 --- a/examples/nnti/cpp/example_utils.hpp +++ b/examples/nnti/cpp/example_utils.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef TEST_UTILS_HPP diff --git a/examples/opbox/CMakeLists.txt b/examples/opbox/CMakeLists.txt index 37b83d7..6cf9780 100644 --- a/examples/opbox/CMakeLists.txt +++ b/examples/opbox/CMakeLists.txt @@ -1,35 +1,37 @@ -include_directories( - support -) - -#-- Support Lib---------------------------------------------------------------- -add_library(mpi_support - support/Globals.hh - support/Globals.cpp - support/SimpleDataStore.hh - support/SimpleDataStore.cpp -) -set_target_properties( mpi_support PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries( mpi_support - MPI::MPI_CXX - Faodel::common ) - -set(EXAMPLES_LIBS - mpi_support - Faodel::opbox - Boost::program_options -) - -add_subdirectory( basic/my_simple_ping ) -add_subdirectory( basic/rdma_ping ) -add_subdirectory( basic/message_packing ) - -add_subdirectory( collectives/rapidfire ) -add_subdirectory( collectives/ringer ) -add_subdirectory( collectives/scatter_gather ) - -add_subdirectory( advanced/dirty_use ) -add_subdirectory( advanced/job2job ) - -add_subdirectory( benchmarks ) +if( NOT TARGET MPI::MPI_CXX ) + message( STATUS "OpBox Examples will not be built because MPI Support was not enabled") + +else() + + if(NOT DEFINED EXAMPLE_LIBS) + # A standalone project can pull in everything via Faodel::opbox + set(EXAMPLE_LIBS Faodel::opbox) + endif() + + # We use a few helpers here to simplify the examples code + add_subdirectory(opbox-example-support) + + + include_directories( + opbox-example-support + ) + + list(APPEND EXAMPLE_LIBS + opbox_example_support + Boost::program_options ) + + add_subdirectory( basic/my-simple-ping ) + add_subdirectory( basic/rdma-ping ) + add_subdirectory( basic/message-packing ) + + add_subdirectory( collectives/rapidfire ) + add_subdirectory( collectives/ringer ) + add_subdirectory( collectives/scatter-gather ) + + add_subdirectory( advanced/lingering-data ) + add_subdirectory( advanced/job2job ) + + add_subdirectory( benchmarks ) + +endif() \ No newline at end of file diff --git a/examples/opbox/advanced/CMakeLists.txt b/examples/opbox/advanced/CMakeLists.txt new file mode 100644 index 0000000..b4d45cb --- /dev/null +++ b/examples/opbox/advanced/CMakeLists.txt @@ -0,0 +1,2 @@ +add_subdirectory(lingering-data) +add_subdirectory( job2job ) \ No newline at end of file diff --git a/examples/opbox/advanced/job2job/CMakeLists.txt b/examples/opbox/advanced/job2job/CMakeLists.txt index 4bba6f2..c6b28b8 100644 --- a/examples/opbox/advanced/job2job/CMakeLists.txt +++ b/examples/opbox/advanced/job2job/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME job2job) +set(PROJECT_NAME ex_opbox_job2job) set(HEADERS OpInterjobPing.hh @@ -22,11 +22,11 @@ set(CLIENT_EXEC ${PROJECT_NAME}-client) add_executable(${SERVER_EXEC} ${HEADERS} ${SERVER_SOURCES} ${COMMON_SOURCES}) set_target_properties(${SERVER_EXEC} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${SERVER_EXEC} ${EXAMPLES_LIBS}) +target_link_libraries(${SERVER_EXEC} ${EXAMPLE_LIBS}) add_executable(${CLIENT_EXEC} ${HEADERS} ${CLIENT_SOURCES} ${COMMON_SOURCES}) set_target_properties(${CLIENT_EXEC} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${CLIENT_EXEC} ${EXAMPLES_LIBS}) +target_link_libraries(${CLIENT_EXEC} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME}-server ${PROJECT_NAME}-client EXPORT faodelExampleTargets diff --git a/examples/opbox/advanced/job2job/OpInterjobPing.cpp b/examples/opbox/advanced/job2job/OpInterjobPing.cpp index af805d2..69b29e9 100644 --- a/examples/opbox/advanced/job2job/OpInterjobPing.cpp +++ b/examples/opbox/advanced/job2job/OpInterjobPing.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include "faodel-common/Debug.hh" @@ -70,7 +70,7 @@ WaitingType OpInterjobPing::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -99,7 +99,7 @@ WaitingType OpInterjobPing::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -109,6 +109,6 @@ string OpInterjobPing::GetStateName() const { case State::snd_wait_for_reply: return "Sender-WaitForReply"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/advanced/job2job/OpInterjobPing.hh b/examples/opbox/advanced/job2job/OpInterjobPing.hh index 88e54c5..14d41ca 100644 --- a/examples/opbox/advanced/job2job/OpInterjobPing.hh +++ b/examples/opbox/advanced/job2job/OpInterjobPing.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPINTERJOBPING_HH #define OPINTERJOBPING_HH diff --git a/examples/opbox/advanced/job2job/job2job-client.cpp b/examples/opbox/advanced/job2job/job2job-client.cpp index a5be14a..c4ccab4 100644 --- a/examples/opbox/advanced/job2job/job2job-client.cpp +++ b/examples/opbox/advanced/job2job/job2job-client.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -10,7 +10,7 @@ #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpInterjobPing.hh" //The Globals class just holds basic communication vars we use in these diff --git a/examples/opbox/advanced/job2job/job2job-server.cpp b/examples/opbox/advanced/job2job/job2job-server.cpp index a2343cf..c4bb0ff 100644 --- a/examples/opbox/advanced/job2job/job2job-server.cpp +++ b/examples/opbox/advanced/job2job/job2job-server.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -10,7 +10,7 @@ #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpInterjobPing.hh" //The Globals class just holds basic communication vars we use in these diff --git a/examples/opbox/advanced/lingering-data/CMakeLists.txt b/examples/opbox/advanced/lingering-data/CMakeLists.txt index f05016b..3c67c30 100644 --- a/examples/opbox/advanced/lingering-data/CMakeLists.txt +++ b/examples/opbox/advanced/lingering-data/CMakeLists.txt @@ -1,17 +1,17 @@ -set(PROJECT_NAME lingering_data) +set(PROJECT_NAME ex_opbox_lingering-data) set(HEADERS ) set(SOURCES - lingering_data.cpp + lingering-data.cpp ) add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/advanced/lingering-data/lingering-data.cpp b/examples/opbox/advanced/lingering-data/lingering-data.cpp index 595c9cf..43bc501 100644 --- a/examples/opbox/advanced/lingering-data/lingering-data.cpp +++ b/examples/opbox/advanced/lingering-data/lingering-data.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Example: opbox dirty use: Lingering Data // Purpose: Lunasa manages memory for many components and uses lunasa data @@ -29,7 +29,7 @@ #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" using namespace std; diff --git a/examples/opbox/basic/CMakeLists.txt b/examples/opbox/basic/CMakeLists.txt new file mode 100644 index 0000000..288d1b4 --- /dev/null +++ b/examples/opbox/basic/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory(message-packing) +add_subdirectory(my-simple-ping) +add_subdirectory(rdma-ping) \ No newline at end of file diff --git a/examples/opbox/basic/message-packing/CMakeLists.txt b/examples/opbox/basic/message-packing/CMakeLists.txt index 30e099f..8900c76 100644 --- a/examples/opbox/basic/message-packing/CMakeLists.txt +++ b/examples/opbox/basic/message-packing/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME message_packing) +set(PROJECT_NAME ex_opbox_message-packing) set(HEADERS ) @@ -52,22 +52,30 @@ if( CAPNPROTO_ROOT ) endif() endif() -set( CEREAL_ROOT $ENV{CEREAL_ROOT} ) -if( CEREAL_ROOT ) - # try to include the Cereal example - find_path( CEREAL binary.hpp HINTS $ENV{CEREAL_ROOT}/include/cereal/archives ) - if( CEREAL ) - message( STATUS "Will build Cereal message-packing example using Cereal from $ENV{CEREAL_ROOT}" ) - include_directories( $ENV{CEREAL_ROOT}/include ) - set( SOURCES ${SOURCES} example7.cpp ) - add_definitions( -DUSE_CEREAL ) - endif() +#set( CEREAL_ROOT $ENV{CEREAL_ROOT} ) +#if( CEREAL_ROOT ) +# # try to include the Cereal example +# find_path( CEREAL binary.hpp HINTS $ENV{CEREAL_ROOT}/include/cereal/archives ) +# if( CEREAL ) +# message( STATUS "Will build Cereal message-packing example using Cereal from $ENV{CEREAL_ROOT}" ) +# include_directories( $ENV{CEREAL_ROOT}/include ) +# set( SOURCES ${SOURCES} example7.cpp ) +# add_definitions( -DUSE_CEREAL ) +# endif() +#endif() + +if( TARGET Faodel::tpl-cereal ) + message( STATUS "Will build Cereal message-packing example using the Cereal TPL installed with Faodel" ) + set( EXTRA_EXAMPLE_LIBS ${EXTRA_EXAMPLE_LIBS} Faodel::tpl-cereal ) + set( SOURCES ${SOURCES} example7.cpp ) + add_definitions( -DUSE_CEREAL ) endif() + add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS} ${EXTRA_EXAMPLE_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS} ${EXTRA_EXAMPLE_LIBS} ) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/basic/message-packing/example1.cpp b/examples/opbox/basic/message-packing/example1.cpp index 36cb2ef..08f0569 100644 --- a/examples/opbox/basic/message-packing/example1.cpp +++ b/examples/opbox/basic/message-packing/example1.cpp @@ -1,9 +1,10 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include +#include #include "opbox/OpBox.hh" #include "opbox/common/MessageHelpers.hh" diff --git a/examples/opbox/basic/message-packing/example2.cpp b/examples/opbox/basic/message-packing/example2.cpp index a43b8fb..fee9075 100644 --- a/examples/opbox/basic/message-packing/example2.cpp +++ b/examples/opbox/basic/message-packing/example2.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/opbox/basic/message-packing/example3.cpp b/examples/opbox/basic/message-packing/example3.cpp index 06b1f32..48e127c 100644 --- a/examples/opbox/basic/message-packing/example3.cpp +++ b/examples/opbox/basic/message-packing/example3.cpp @@ -1,8 +1,9 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include +#include #include #include "opbox/OpBox.hh" @@ -37,7 +38,7 @@ void example3_ldo_string_request_reply(){ //Extract the message and then acknowledge all-caps day string s = UnpackStringMessage(msg_request); - transform(s.begin(), s.end(), s.begin(), ::toupper); + std::transform(s.begin(), s.end(), s.begin(), ::toupper); //Allocate a reply and use the incoming message to fill in fields lunasa::DataObject ldo_reply; diff --git a/examples/opbox/basic/message-packing/example4.cpp b/examples/opbox/basic/message-packing/example4.cpp index f9f02cd..78ec2ba 100644 --- a/examples/opbox/basic/message-packing/example4.cpp +++ b/examples/opbox/basic/message-packing/example4.cpp @@ -1,26 +1,29 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include #include -#include + +#include +#include + #include "opbox/OpBox.hh" #include "opbox/common/MessageHelpers.hh" using namespace std; -// Boost's serialization library provides an easy way for us to ser/des a +// Cereal's serialization library provides an easy way for us to ser/des a // complex class. In addition to being hierarchical, it also supports // STL containers (provided you include the corresponding header file. -// eg. boost/serialization/vector.hpp). All you need to do to make a -// class boost serializable is (1) make sure there's a default constructor +// eg. cereal/serialization/vector.hpp). All you need to do to make a +// class cereal serializable is (1) make sure there's a default constructor // and (2) provide a "serialize" template that specifies which variables // are archived. // -// - Caveat 1: Boost serialization can be expensive. In addition to +// - Caveat 1: Cereal serialization can be expensive. In addition to // streaming its data as it moves along, the OpBox code // has to copy the packed string of data into the outgoing // message. @@ -88,7 +91,7 @@ class FancyPants { }; -void example4_boost_messages(){ +void example4_cereal_messages(){ //Create some fancy pants for us to pack into a message FancyPants fancy_pants("britches", "red", 4); @@ -96,7 +99,7 @@ void example4_boost_messages(){ lunasa::DataObject ldo; //Use the templated allocator to create a message. - AllocateBoostMessage( ldo, + AllocateCerealMessage( ldo, opbox::GetMyID(), opbox::GetMyID(), //Usually dst node 100, //Usually supplied by op @@ -108,10 +111,10 @@ void example4_boost_messages(){ auto msg = ldo.GetDataPtr(); cout <<"LDO Size is: " << ldo.GetDataSize()<body_len<body_len<(msg); + auto fp2 = UnpackCerealMessage(msg); //Dump out the original and unpacked classes cout <<"Original class:\n"< #include diff --git a/examples/opbox/basic/message-packing/example6.cpp b/examples/opbox/basic/message-packing/example6.cpp index 3153015..16a316a 100644 --- a/examples/opbox/basic/message-packing/example6.cpp +++ b/examples/opbox/basic/message-packing/example6.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/opbox/basic/message-packing/example7.cpp b/examples/opbox/basic/message-packing/example7.cpp index b9c1215..4c4f04d 100644 --- a/examples/opbox/basic/message-packing/example7.cpp +++ b/examples/opbox/basic/message-packing/example7.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -99,7 +99,7 @@ example7_cereal_messages() << "*********************************" << endl; - lunasa::DataObject *ldo; + lunasa::DataObject ldo; FancyPants my_pants( "britches", "red", 4 ); @@ -121,7 +121,7 @@ example7_cereal_messages() // arbitrary struct. Perhaps there's a better way to do this but I couldn't // figure it out. ldo = opbox::net::NewMessage( sizeof( message_t ) + my_cerealized_pants.size() ); - message_t *msg = reinterpret_cast< message_t* >( ldo->dataPtr() ); + message_t *msg = reinterpret_cast< message_t* >( ldo.GetDataPtr() ); // Use the size of the marshaled buffer in the FB builder as the length of the payload msg->body_len = my_cerealized_pants.size(); @@ -129,10 +129,10 @@ example7_cereal_messages() std::memcpy( msg->body, my_cerealized_pants.data(), msg->body_len ); cout << "Cerealized buffer size is " << msg->body_len << endl; - cout << "LDO size is " << ldo->dataSize() << endl; + cout << "LDO size is " << ldo.GetDataSize() << endl; // Get the message back out of the LDO, as though we'd received it from a sender - auto *msg2 = reinterpret_cast< message_t* >( ldo->dataPtr() ); + auto *msg2 = reinterpret_cast< message_t* >( ldo.GetDataPtr() ); string more_cerealized_pants( msg->body, msg->body_len ); @@ -149,6 +149,4 @@ example7_cereal_messages() // All done. Let's see what happened. cout << "Original pants:" << endl << my_pants << endl; cout << "unpacked pants:" << endl << my_other_pants; - - delete ldo; } diff --git a/examples/opbox/basic/message-packing/message_packing_example.cpp b/examples/opbox/basic/message-packing/message_packing_example.cpp index 9583725..8443cb9 100644 --- a/examples/opbox/basic/message-packing/message_packing_example.cpp +++ b/examples/opbox/basic/message-packing/message_packing_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -12,7 +12,7 @@ #include "dirman/DirMan.hh" -#include "../../support/Globals.hh" +#include "../../opbox-example-support/Globals.hh" //The Globals class just holds basic communication vars we use in these @@ -64,7 +64,7 @@ void example2_ldo_string_messages(); void example3_ldo_string_request_reply(); -void example4_boost_messages(); +void example4_cereal_messages(); #ifdef USE_FLATBUFFERS void example5_flatbuffer_messages(); @@ -94,7 +94,7 @@ int main(int argc, char **argv){ example3_ldo_string_request_reply(); - example4_boost_messages(); + example4_cereal_messages(); #ifdef USE_FLATBUFFERS example5_flatbuffer_messages(); diff --git a/examples/opbox/basic/my-simple-ping/CMakeLists.txt b/examples/opbox/basic/my-simple-ping/CMakeLists.txt index 2d9586e..76177cf 100644 --- a/examples/opbox/basic/my-simple-ping/CMakeLists.txt +++ b/examples/opbox/basic/my-simple-ping/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME my_simple_ping) +set(PROJECT_NAME ex_opbox_my-simple-ping) set(HEADERS OpMySimplePing.hh @@ -13,7 +13,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/basic/my-simple-ping/OpMySimplePing.cpp b/examples/opbox/basic/my-simple-ping/OpMySimplePing.cpp index 7540b76..05c0e65 100644 --- a/examples/opbox/basic/my-simple-ping/OpMySimplePing.cpp +++ b/examples/opbox/basic/my-simple-ping/OpMySimplePing.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include "faodel-common/Debug.hh" @@ -75,7 +75,7 @@ WaitingType OpMySimplePing::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -105,7 +105,7 @@ WaitingType OpMySimplePing::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -115,5 +115,5 @@ string OpMySimplePing::GetStateName() const { case State::snd_wait_for_reply: return "Sender-WaitForReply"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); } diff --git a/examples/opbox/basic/my-simple-ping/OpMySimplePing.hh b/examples/opbox/basic/my-simple-ping/OpMySimplePing.hh index 8bfc9fb..bb2466f 100644 --- a/examples/opbox/basic/my-simple-ping/OpMySimplePing.hh +++ b/examples/opbox/basic/my-simple-ping/OpMySimplePing.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPMYSIMPLEPING_HH #define OPMYSIMPLEPING_HH diff --git a/examples/opbox/basic/my-simple-ping/my_simple_ping.cpp b/examples/opbox/basic/my-simple-ping/my_simple_ping.cpp index 750bc3d..8f17bce 100644 --- a/examples/opbox/basic/my-simple-ping/my_simple_ping.cpp +++ b/examples/opbox/basic/my-simple-ping/my_simple_ping.cpp @@ -1,13 +1,13 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpMySimplePing.hh" //The Globals class just holds basic communication vars we use in these @@ -72,6 +72,12 @@ int main(int argc, char **argv){ G.StartAll(argc, argv, config); + if(G.mpi_size!=2) { + cerr <<"This example needs to be run with two mpi ranks.\n"; + G.StopAll(); + return -1; + } + //For this example, the master node is going to create a new Op and //launch it. The flow of operation is as follows: // Origin (user) : Create a new Op diff --git a/examples/opbox/basic/rdma-ping/CMakeLists.txt b/examples/opbox/basic/rdma-ping/CMakeLists.txt index 2f90b19..77ec05e 100644 --- a/examples/opbox/basic/rdma-ping/CMakeLists.txt +++ b/examples/opbox/basic/rdma-ping/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME rdma_ping) +set(PROJECT_NAME ex_opbox_rdma-ping) set(HEADERS OpRdmaPing.hh @@ -13,7 +13,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/basic/rdma-ping/OpRdmaPing.cpp b/examples/opbox/basic/rdma-ping/OpRdmaPing.cpp index cfcfd60..efe9865 100644 --- a/examples/opbox/basic/rdma-ping/OpRdmaPing.cpp +++ b/examples/opbox/basic/rdma-ping/OpRdmaPing.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include //for memcpy @@ -120,7 +120,7 @@ WaitingType OpRdmaPing::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -187,7 +187,7 @@ WaitingType OpRdmaPing::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -199,5 +199,5 @@ string OpRdmaPing::GetStateName() const { case State::put_wait_complete: return "PutWaitComplete"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); } diff --git a/examples/opbox/basic/rdma-ping/OpRdmaPing.hh b/examples/opbox/basic/rdma-ping/OpRdmaPing.hh index 183b94d..6582aa1 100644 --- a/examples/opbox/basic/rdma-ping/OpRdmaPing.hh +++ b/examples/opbox/basic/rdma-ping/OpRdmaPing.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPRDMAPING_HH #define OPRDMAPING_HH diff --git a/examples/opbox/basic/rdma-ping/rdma_ping.cpp b/examples/opbox/basic/rdma-ping/rdma_ping.cpp index 717d872..15498a4 100644 --- a/examples/opbox/basic/rdma-ping/rdma_ping.cpp +++ b/examples/opbox/basic/rdma-ping/rdma_ping.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include //for memcpy @@ -8,7 +8,7 @@ #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpRdmaPing.hh" //The Globals class just holds basic communication vars we use in these @@ -75,6 +75,12 @@ int main(int argc, char **argv){ G.StartAll(argc, argv, config); + if(G.mpi_size==1) { + std::cerr<<"This example needs to be run with multiple mpi ranks\n"; + G.StopAll(); + return -1; + } + //For this example, the master node is going to create a new Op and //launch it. The flow of operation is as follows: // Origin (user) : Allocate a Lunasa RDMA buffer and fill with a message diff --git a/examples/opbox/benchmarks/atomics/CMakeLists.txt b/examples/opbox/benchmarks/atomics/CMakeLists.txt index 4c29385..51c16ef 100644 --- a/examples/opbox/benchmarks/atomics/CMakeLists.txt +++ b/examples/opbox/benchmarks/atomics/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME atomics) +set(PROJECT_NAME ex_opbox_atomics) set(HEADERS OpBenchmarkCompareSwap.hh @@ -15,7 +15,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/benchmarks/atomics/OpBenchmarkCompareSwap.cpp b/examples/opbox/benchmarks/atomics/OpBenchmarkCompareSwap.cpp index 8a04b61..cbe5a2e 100644 --- a/examples/opbox/benchmarks/atomics/OpBenchmarkCompareSwap.cpp +++ b/examples/opbox/benchmarks/atomics/OpBenchmarkCompareSwap.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -123,7 +123,7 @@ WaitingType OpBenchmarkCompareSwap::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -232,7 +232,7 @@ WaitingType OpBenchmarkCompareSwap::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -242,6 +242,6 @@ string OpBenchmarkCompareSwap::GetStateName() const { case State::snd_wait_for_ack: return "Sender-WaitForAck"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/benchmarks/atomics/OpBenchmarkCompareSwap.hh b/examples/opbox/benchmarks/atomics/OpBenchmarkCompareSwap.hh index 2a1ec65..8ad0599 100644 --- a/examples/opbox/benchmarks/atomics/OpBenchmarkCompareSwap.hh +++ b/examples/opbox/benchmarks/atomics/OpBenchmarkCompareSwap.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBENCHMARKCOMPARESWAP_HH #define OPBENCHMARKCOMPARESWAP_HH diff --git a/examples/opbox/benchmarks/atomics/OpBenchmarkFetchAdd.cpp b/examples/opbox/benchmarks/atomics/OpBenchmarkFetchAdd.cpp index f097ff3..6d4dafe 100644 --- a/examples/opbox/benchmarks/atomics/OpBenchmarkFetchAdd.cpp +++ b/examples/opbox/benchmarks/atomics/OpBenchmarkFetchAdd.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -122,7 +122,7 @@ WaitingType OpBenchmarkFetchAdd::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -226,7 +226,7 @@ WaitingType OpBenchmarkFetchAdd::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -236,6 +236,6 @@ string OpBenchmarkFetchAdd::GetStateName() const { case State::snd_wait_for_ack: return "Sender-WaitForAck"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/benchmarks/atomics/OpBenchmarkFetchAdd.hh b/examples/opbox/benchmarks/atomics/OpBenchmarkFetchAdd.hh index 91e1c42..10a2d78 100644 --- a/examples/opbox/benchmarks/atomics/OpBenchmarkFetchAdd.hh +++ b/examples/opbox/benchmarks/atomics/OpBenchmarkFetchAdd.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBENCHMARKFETCHADD_HH #define OPBENCHMARKFETCHADD_HH diff --git a/examples/opbox/benchmarks/atomics/atomics.cpp b/examples/opbox/benchmarks/atomics/atomics.cpp index c169312..948138b 100644 --- a/examples/opbox/benchmarks/atomics/atomics.cpp +++ b/examples/opbox/benchmarks/atomics/atomics.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -10,7 +10,7 @@ #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpBenchmarkCompareSwap.hh" #include "OpBenchmarkFetchAdd.hh" @@ -102,6 +102,13 @@ int main(int argc, char **argv) G.StartAll(argc, argv, config); + if(G.mpi_size<2) { + cerr <<"This example should be run with at least two mpi ranks. Exiting.\n"; + G.StopAll(); + return -1; + } + + //For this example, the master node is going to create a new Op and //launch it. The flow of operation is as follows: if(G.mpi_rank==0){ diff --git a/examples/opbox/benchmarks/connect/CMakeLists.txt b/examples/opbox/benchmarks/connect/CMakeLists.txt index b210fb3..3aed2dc 100644 --- a/examples/opbox/benchmarks/connect/CMakeLists.txt +++ b/examples/opbox/benchmarks/connect/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME connect) +set(PROJECT_NAME ex_opbox_connect) set(SOURCES connect.cpp @@ -8,7 +8,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/benchmarks/connect/connect.cpp b/examples/opbox/benchmarks/connect/connect.cpp index 8faabd9..216b7e2 100644 --- a/examples/opbox/benchmarks/connect/connect.cpp +++ b/examples/opbox/benchmarks/connect/connect.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/opbox/benchmarks/msgplusrdma/CMakeLists.txt b/examples/opbox/benchmarks/msgplusrdma/CMakeLists.txt index e224a3f..608a9b0 100644 --- a/examples/opbox/benchmarks/msgplusrdma/CMakeLists.txt +++ b/examples/opbox/benchmarks/msgplusrdma/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME msgplusrdma) +set(PROJECT_NAME ex_opbox_msgplusrdma) set(HEADERS OpBenchmarkMsgRdma.hh @@ -13,7 +13,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/benchmarks/msgplusrdma/OpBenchmarkMsgRdma.cpp b/examples/opbox/benchmarks/msgplusrdma/OpBenchmarkMsgRdma.cpp index f66c84c..74d9566 100644 --- a/examples/opbox/benchmarks/msgplusrdma/OpBenchmarkMsgRdma.cpp +++ b/examples/opbox/benchmarks/msgplusrdma/OpBenchmarkMsgRdma.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -189,7 +189,7 @@ WaitingType OpBenchmarkMsgRdma::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -252,7 +252,7 @@ WaitingType OpBenchmarkMsgRdma::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -262,6 +262,6 @@ string OpBenchmarkMsgRdma::GetStateName() const { case State::snd_wait_for_ack: return "Sender-WaitForAck"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/benchmarks/msgplusrdma/OpBenchmarkMsgRdma.hh b/examples/opbox/benchmarks/msgplusrdma/OpBenchmarkMsgRdma.hh index f1e0211..8ba62ac 100644 --- a/examples/opbox/benchmarks/msgplusrdma/OpBenchmarkMsgRdma.hh +++ b/examples/opbox/benchmarks/msgplusrdma/OpBenchmarkMsgRdma.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBENCHMARKMSGRDMA_HH #define OPBENCHMARKMSGRDMA_HH diff --git a/examples/opbox/benchmarks/msgplusrdma/msgplusrdma.cpp b/examples/opbox/benchmarks/msgplusrdma/msgplusrdma.cpp index 49ec0ba..da928e3 100644 --- a/examples/opbox/benchmarks/msgplusrdma/msgplusrdma.cpp +++ b/examples/opbox/benchmarks/msgplusrdma/msgplusrdma.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -10,7 +10,7 @@ #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpBenchmarkMsgRdma.hh" //The Globals class just holds basic communication vars we use in these @@ -106,6 +106,12 @@ int main(int argc, char **argv) G.StartAll(argc, argv, config); + if(G.mpi_size<2) { + cerr<<"This example should be run with at least two mpi ranks. Exiting.\n"; + G.StopAll(); + return -1; + } + //For this example, the master node is going to create a new Op and //launch it. The flow of operation is as follows: if(G.mpi_rank==0){ diff --git a/examples/opbox/benchmarks/rdma/CMakeLists.txt b/examples/opbox/benchmarks/rdma/CMakeLists.txt index 24da3c9..e158896 100644 --- a/examples/opbox/benchmarks/rdma/CMakeLists.txt +++ b/examples/opbox/benchmarks/rdma/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME rdma) +set(PROJECT_NAME ex_opbox_rdma) set(HEADERS OpBenchmarkGet.hh @@ -15,7 +15,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/benchmarks/rdma/OpBenchmarkGet.cpp b/examples/opbox/benchmarks/rdma/OpBenchmarkGet.cpp index d20e63a..554e3ab 100644 --- a/examples/opbox/benchmarks/rdma/OpBenchmarkGet.cpp +++ b/examples/opbox/benchmarks/rdma/OpBenchmarkGet.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -16,7 +16,7 @@ const string OpBenchmarkGet::op_name = "OpBenchmarkGet"; OpBenchmarkGet::OpBenchmarkGet(opbox::net::peer_ptr_t dst, uint32_t size, uint32_t count, uint32_t max_inflight) - : faodel::LoggingInterface("Opbox", "OpBenchmarkGet"), + : faodel::LoggingInterface("opbox", "OpBenchmarkGet"), state(State::start), warmup_count(10), size(size), count(count), max_inflight(max_inflight), @@ -29,9 +29,9 @@ OpBenchmarkGet::OpBenchmarkGet(opbox::net::peer_ptr_t dst, uint32_t size, uint32 } OpBenchmarkGet::OpBenchmarkGet(op_create_as_target_t t) - : faodel::LoggingInterface("Opbox", "OpBenchmarkGet"), + : faodel::LoggingInterface("opbox", "OpBenchmarkGet"), state(State::start), - warmup_count(0), + warmup_count(10), size(0), count(0), max_inflight(0), started(0), inflight(0), completed(0), first_burst_sent(false), @@ -124,7 +124,7 @@ WaitingType OpBenchmarkGet::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -156,45 +156,70 @@ WaitingType OpBenchmarkGet::UpdateTarget(OpArgs *args) { 0, //Not expecting a reply incoming_msg->src_mailbox); + state = State::warm_up; + + // Opbox doesn't have a "Don't Wait" WaitingType that just calls + // the state machine again, so we fall through. + } + case State::warm_up: { + state = State::warm_up_wait; + for (int i=0;iat(started).start = std::chrono::high_resolution_clock::now(); opbox::net::Get(peer, &nbr, rdma_ldo, AllEventsCallback(this)); started++; inflight++; - ss.str(std::string()); - ss << "inflight=" << inflight << " started=" << started << " completed=" << completed << std::endl; - dbg(ss.str()); +// ss.str(std::string()); +// ss << "inflight=" << inflight << " started=" << started << " completed=" << completed << std::endl; +// dbg(ss.str()); } first_burst_sent = true; - ss.str(std::string()); - ss << "count=" << count << std::endl; - dbg(ss.str()); +// ss.str(std::string()); +// ss << "count=" << count << std::endl; +// dbg(ss.str()); + + state = State::execute_benchmark; return WaitingType::waiting_on_cq; } - case State::tgt_created: + case State::execute_benchmark: { timers->at(completed).end = std::chrono::high_resolution_clock::now(); completed++; inflight--; - ss.str(std::string()); - ss << "inflight=" << inflight << " max_inflight=" << max_inflight << " completed=" << completed << std::endl; - dbg(ss.str()); +// ss.str(std::string()); +// ss << "inflight=" << inflight << " max_inflight=" << max_inflight << " completed=" << completed << std::endl; +// dbg(ss.str()); while((inflightat(started).start = std::chrono::high_resolution_clock::now(); opbox::net::Get(peer, &nbr, rdma_ldo, AllEventsCallback(this)); started++; inflight++; - ss.str(std::string()); - ss << "inflight=" << inflight << " started=" << started << " completed=" << completed << std::endl; - dbg(ss.str()); +// ss.str(std::string()); +// ss << "inflight=" << inflight << " started=" << started << " completed=" << completed << std::endl; +// dbg(ss.str()); } if(completed == count) { @@ -223,12 +248,13 @@ WaitingType OpBenchmarkGet::UpdateTarget(OpArgs *args) { } return WaitingType::waiting_on_cq; - - case State::done: + } + case State::done: { return WaitingType::done_and_destroy; + } } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -238,6 +264,6 @@ string OpBenchmarkGet::GetStateName() const { case State::snd_wait_for_ack: return "Sender-WaitForAck"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/benchmarks/rdma/OpBenchmarkGet.hh b/examples/opbox/benchmarks/rdma/OpBenchmarkGet.hh index 24dccff..c4ae848 100644 --- a/examples/opbox/benchmarks/rdma/OpBenchmarkGet.hh +++ b/examples/opbox/benchmarks/rdma/OpBenchmarkGet.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBENCHMARKGET_HH #define OPBENCHMARKGET_HH @@ -22,7 +22,10 @@ class OpBenchmarkGet enum class State : int { start=0, snd_wait_for_ack, - tgt_created, + warm_up, + warm_up_wait, + start_benchmark, + execute_benchmark, done }; @@ -61,7 +64,7 @@ private: State state; opbox::net::peer_t *peer; - uint32_t warmup_count; // the number of RDMAs to do before the timer starts + uint32_t warmup_count; // the number of RDMAs to do before the timer starts uint32_t size; // the size of each RDMA uint32_t count; // the number of RDMAs this op must complete diff --git a/examples/opbox/benchmarks/rdma/OpBenchmarkPut.cpp b/examples/opbox/benchmarks/rdma/OpBenchmarkPut.cpp index bc73c51..50f23a3 100644 --- a/examples/opbox/benchmarks/rdma/OpBenchmarkPut.cpp +++ b/examples/opbox/benchmarks/rdma/OpBenchmarkPut.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -16,7 +16,7 @@ const string OpBenchmarkPut::op_name = "OpBenchmarkPut"; OpBenchmarkPut::OpBenchmarkPut(opbox::net::peer_ptr_t dst, uint32_t size, uint32_t count, uint32_t max_inflight) - : faodel::LoggingInterface("Opbox", "OpBenchmarkPut"), + : faodel::LoggingInterface("opbox", "OpBenchmarkPut"), state(State::start), warmup_count(10), size(size), count(count), max_inflight(max_inflight), @@ -29,9 +29,9 @@ OpBenchmarkPut::OpBenchmarkPut(opbox::net::peer_ptr_t dst, uint32_t size, uint32 } OpBenchmarkPut::OpBenchmarkPut(op_create_as_target_t t) - : faodel::LoggingInterface("Opbox", "OpBenchmarkPut"), + : faodel::LoggingInterface("opbox", "OpBenchmarkPut"), state(State::start), - warmup_count(0), + warmup_count(10), size(0), count(0), max_inflight(0), started(0), inflight(0), completed(0), first_burst_sent(false), @@ -124,7 +124,7 @@ WaitingType OpBenchmarkPut::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -156,46 +156,71 @@ WaitingType OpBenchmarkPut::UpdateTarget(OpArgs *args) { 0, //Not expecting a reply incoming_msg->src_mailbox); + state = State::warm_up; + + // Opbox doesn't have a "Don't Wait" WaitingType that just calls + // the state machine again, so we fall through. + } + case State::warm_up: { + state = State::warm_up_wait; + for (int i=0;iat(started).start = std::chrono::high_resolution_clock::now(); opbox::net::Put(peer, rdma_ldo, &nbr, AllEventsCallback(this)); started++; inflight++; - ss.str(std::string()); - ss << "inflight=" << inflight << " started=" << started << " completed=" << completed << std::endl; - dbg(ss.str()); +// ss.str(std::string()); +// ss << "inflight=" << inflight << " started=" << started << " completed=" << completed << std::endl; +// dbg(ss.str()); } first_burst_sent = true; - ss.str(std::string()); - ss << "count=" << count << std::endl; - dbg(ss.str()); +// ss.str(std::string()); +// ss << "count=" << count << std::endl; +// dbg(ss.str()); + + state = State::execute_benchmark; return WaitingType::waiting_on_cq; } - case State::tgt_created: + case State::execute_benchmark: //TODO: Should there be a check for args type here? timers->at(completed).end = std::chrono::high_resolution_clock::now(); completed++; inflight--; - ss.str(std::string()); - ss << "inflight=" << inflight << " max_inflight=" << max_inflight << " completed=" << completed << std::endl; - dbg(ss.str()); +// ss.str(std::string()); +// ss << "inflight=" << inflight << " max_inflight=" << max_inflight << " completed=" << completed << std::endl; +// dbg(ss.str()); while((inflightat(started).start = std::chrono::high_resolution_clock::now(); opbox::net::Put(peer, rdma_ldo, &nbr, AllEventsCallback(this)); started++; inflight++; - ss.str(std::string()); - ss << "inflight=" << inflight << " started=" << started << " completed=" << completed << std::endl; - dbg(ss.str()); +// ss.str(std::string()); +// ss << "inflight=" << inflight << " started=" << started << " completed=" << completed << std::endl; +// dbg(ss.str()); } if(completed == count) { @@ -228,7 +253,7 @@ WaitingType OpBenchmarkPut::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -238,6 +263,6 @@ string OpBenchmarkPut::GetStateName() const { case State::snd_wait_for_ack: return "Sender-WaitForAck"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/benchmarks/rdma/OpBenchmarkPut.hh b/examples/opbox/benchmarks/rdma/OpBenchmarkPut.hh index 4d5cfa2..9fb6fb3 100644 --- a/examples/opbox/benchmarks/rdma/OpBenchmarkPut.hh +++ b/examples/opbox/benchmarks/rdma/OpBenchmarkPut.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBENCHMARKPUT_HH #define OPBENCHMARKPUT_HH @@ -22,7 +22,10 @@ class OpBenchmarkPut enum class State : int { start=0, snd_wait_for_ack, - tgt_created, + warm_up, + warm_up_wait, + start_benchmark, + execute_benchmark, done }; diff --git a/examples/opbox/benchmarks/rdma/rdma.cpp b/examples/opbox/benchmarks/rdma/rdma.cpp index 686d2cc..b6727b5 100644 --- a/examples/opbox/benchmarks/rdma/rdma.cpp +++ b/examples/opbox/benchmarks/rdma/rdma.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -10,7 +10,7 @@ #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpBenchmarkGet.hh" #include "OpBenchmarkPut.hh" @@ -106,6 +106,12 @@ int main(int argc, char **argv) { G.StartAll(argc, argv, config); + if(G.mpi_size==1) { + std::cerr<<"This example needs to be run with multiple mpi ranks\n"; + G.StopAll(); + return -1; + } + //For this example, the master node is going to create a new Op and //launch it. The flow of operation is as follows: if(G.mpi_rank == 0) { diff --git a/examples/opbox/benchmarks/short_message/CMakeLists.txt b/examples/opbox/benchmarks/short_message/CMakeLists.txt index c4d42c3..f0e49a8 100644 --- a/examples/opbox/benchmarks/short_message/CMakeLists.txt +++ b/examples/opbox/benchmarks/short_message/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME short_message) +set(PROJECT_NAME ex_opbox_short-message) set(HEADERS OpBenchmarkPing.hh @@ -13,7 +13,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/benchmarks/short_message/OpBenchmarkPing.cpp b/examples/opbox/benchmarks/short_message/OpBenchmarkPing.cpp index e7a222e..b37c285 100644 --- a/examples/opbox/benchmarks/short_message/OpBenchmarkPing.cpp +++ b/examples/opbox/benchmarks/short_message/OpBenchmarkPing.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -200,7 +200,7 @@ WaitingType OpBenchmarkPing::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -210,6 +210,6 @@ string OpBenchmarkPing::GetStateName() const { case State::snd_wait_for_reply: return "Sender-WaitForReply"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/benchmarks/short_message/OpBenchmarkPing.hh b/examples/opbox/benchmarks/short_message/OpBenchmarkPing.hh index 9e617fb..adae82e 100644 --- a/examples/opbox/benchmarks/short_message/OpBenchmarkPing.hh +++ b/examples/opbox/benchmarks/short_message/OpBenchmarkPing.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBENCHMARKPING_HH #define OPBENCHMARKPING_HH diff --git a/examples/opbox/benchmarks/short_message/short_message.cpp b/examples/opbox/benchmarks/short_message/short_message.cpp index ef081d1..f9cbebb 100644 --- a/examples/opbox/benchmarks/short_message/short_message.cpp +++ b/examples/opbox/benchmarks/short_message/short_message.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -10,7 +10,7 @@ #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpBenchmarkPing.hh" //The Globals class just holds basic communication vars we use in these @@ -106,6 +106,12 @@ int main(int argc, char **argv) G.StartAll(argc, argv, config); + if(G.mpi_size==1) { + std::cerr<<"This example needs to be run with multiple mpi ranks\n"; + G.StopAll(); + return -1; + } + //For this example, the master node is going to create a new Op and //launch it. The flow of operation is as follows: if(G.mpi_rank==0){ diff --git a/examples/opbox/collectives/CMakeLists.txt b/examples/opbox/collectives/CMakeLists.txt new file mode 100644 index 0000000..67e0eda --- /dev/null +++ b/examples/opbox/collectives/CMakeLists.txt @@ -0,0 +1,3 @@ +add_subdirectory( rapidfire ) +add_subdirectory( ringer ) +add_subdirectory(scatter-gather) \ No newline at end of file diff --git a/examples/opbox/collectives/rapidfire/CMakeLists.txt b/examples/opbox/collectives/rapidfire/CMakeLists.txt index cf86598..a91e373 100644 --- a/examples/opbox/collectives/rapidfire/CMakeLists.txt +++ b/examples/opbox/collectives/rapidfire/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME rapidfire) +set(PROJECT_NAME ex_opbox_rapidfire) set(HEADERS OpRapidFire.hh @@ -13,7 +13,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/collectives/rapidfire/OpRapidFire.cpp b/examples/opbox/collectives/rapidfire/OpRapidFire.cpp index 8972ecd..6a45a01 100644 --- a/examples/opbox/collectives/rapidfire/OpRapidFire.cpp +++ b/examples/opbox/collectives/rapidfire/OpRapidFire.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -76,7 +76,7 @@ WaitingType OpRapidFire::UpdateOrigin(OpArgs *args) { switch(state){ case State::start: //Message is already packed and peer is set. Fire and forget. - KTODO("The Origin constructor sets the initial state to 'snd_OneAtATime'. Can the 'start' state be eliminated?"); + F_TODO("The Origin constructor sets the initial state to 'snd_OneAtATime'. Can the 'start' state be eliminated?"); return WaitingType::waiting_on_cq; @@ -126,7 +126,7 @@ WaitingType OpRapidFire::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -163,7 +163,7 @@ WaitingType OpRapidFire::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -173,6 +173,6 @@ string OpRapidFire::GetStateName() const { case State::snd_OneAtATime: return "Sender-One-At-A-Time"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/collectives/rapidfire/OpRapidFire.hh b/examples/opbox/collectives/rapidfire/OpRapidFire.hh index c4376f1..bbb97db 100644 --- a/examples/opbox/collectives/rapidfire/OpRapidFire.hh +++ b/examples/opbox/collectives/rapidfire/OpRapidFire.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPRAPIDFIRE_HH #define OPRAPIDFIRE_HH diff --git a/examples/opbox/collectives/rapidfire/rapidfire.cpp b/examples/opbox/collectives/rapidfire/rapidfire.cpp index 01faec6..c24e6ee 100644 --- a/examples/opbox/collectives/rapidfire/rapidfire.cpp +++ b/examples/opbox/collectives/rapidfire/rapidfire.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -9,7 +9,7 @@ #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpRapidFire.hh" //Globals holds mpi info and manages connections (see ping example for info) @@ -52,6 +52,12 @@ int main(int argc, char **argv){ G.StartAll(argc, argv, config); + if(G.mpi_size==1) { + std::cerr<<"This example needs to be run with two mpi ranks\n"; + G.StopAll(); + return -1; + } + //The master node (rank 0) creates the request and issues it if(G.mpi_rank==0){ diff --git a/examples/opbox/collectives/ringer/CMakeLists.txt b/examples/opbox/collectives/ringer/CMakeLists.txt index 182a3e7..b77b352 100644 --- a/examples/opbox/collectives/ringer/CMakeLists.txt +++ b/examples/opbox/collectives/ringer/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME ringer) +set(PROJECT_NAME ex_opbox_ringer) set(HEADERS OpRinger.hh @@ -14,7 +14,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/collectives/ringer/OpRinger.cpp b/examples/opbox/collectives/ringer/OpRinger.cpp index 874acbf..753a087 100644 --- a/examples/opbox/collectives/ringer/OpRinger.cpp +++ b/examples/opbox/collectives/ringer/OpRinger.cpp @@ -1,10 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include -#include +#include +#include + #include "opbox/common/MessageHelpers.hh" @@ -27,8 +29,8 @@ OpRinger::OpRinger(RingInfo ring_info) //We'll need to convert this to a peer_ptr_t for send command peer = opbox::net::ConvertNodeIDToPeer(dst_node); - //We can use the standard boost request function to pack this message - opbox::AllocateBoostRequestMessage(ldo_msg, + //We can use the standard cereal request function to pack this message + opbox::AllocateCerealRequestMessage(ldo_msg, dst_node, GetAssignedMailbox(), op_id, 0, ring_info); @@ -64,7 +66,7 @@ WaitingType OpRinger::UpdateOrigin(OpArgs *args) { //We expect to have received a message with RingInfo in it. Unpack //and send it back to the user via a promise auto msg = args->ExpectMessageOrDie(); - auto ring_info = UnpackBoostMessage(msg); + auto ring_info = UnpackCerealMessage(msg); ring_promise.set_value(ring_info); state=State::done; @@ -74,7 +76,7 @@ WaitingType OpRinger::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -90,7 +92,7 @@ WaitingType OpRinger::UpdateTarget(OpArgs *args) { //New message should have a RingInfo we can use for locating next node auto msg = args->ExpectMessageOrDie(); - auto ring_info = UnpackBoostMessage(msg); + auto ring_info = UnpackCerealMessage(msg); //Create a message for this node and add it to the ring int spot = ring_info.GetNumValues(); @@ -109,7 +111,7 @@ WaitingType OpRinger::UpdateTarget(OpArgs *args) { peer = opbox::net::ConvertNodeIDToPeer(next_node); //We're going to forward a request on to the next node, using origin's info - AllocateBoostMessage(ldo_msg, + AllocateCerealMessage(ldo_msg, msg->src, next_node, msg->src_mailbox, dst_mailbox, op_id, 0, @@ -128,7 +130,7 @@ WaitingType OpRinger::UpdateTarget(OpArgs *args) { cout <<"Done waiting: "< #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "OpRinger.hh" //Globals holds mpi info and manages connections (see ping example for info) @@ -47,6 +47,12 @@ int main(int argc, char **argv){ G.StartAll(argc, argv, config); + if(G.mpi_size==1) { + std::cerr<<"This example needs to be run with multiple mpi ranks\n"; + G.StopAll(); + return -1; + } + //The master node (rank 0) creates the request and issues it if(G.mpi_rank==0){ diff --git a/examples/opbox/collectives/scatter-gather/CMakeLists.txt b/examples/opbox/collectives/scatter-gather/CMakeLists.txt index 4344747..ac54a9e 100644 --- a/examples/opbox/collectives/scatter-gather/CMakeLists.txt +++ b/examples/opbox/collectives/scatter-gather/CMakeLists.txt @@ -1,4 +1,4 @@ -set(PROJECT_NAME scatter_gather) +set(PROJECT_NAME ex_opbox_scatter-gather) set(HEADERS OpScatter.hh @@ -15,7 +15,7 @@ set(SOURCES add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/opbox/collectives/scatter-gather/OpGather.cpp b/examples/opbox/collectives/scatter-gather/OpGather.cpp index 2f3e99c..3d37e79 100644 --- a/examples/opbox/collectives/scatter-gather/OpGather.cpp +++ b/examples/opbox/collectives/scatter-gather/OpGather.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -154,7 +154,7 @@ WaitingType OpGather::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -206,7 +206,7 @@ WaitingType OpGather::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -217,6 +217,6 @@ string OpGather::GetStateName() const { case State::put_wait_complete: return "PutWaitComplete"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/collectives/scatter-gather/OpGather.hh b/examples/opbox/collectives/scatter-gather/OpGather.hh index cd632ab..ef91b06 100644 --- a/examples/opbox/collectives/scatter-gather/OpGather.hh +++ b/examples/opbox/collectives/scatter-gather/OpGather.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPGATHER_HH #define OPGATHER_HH diff --git a/examples/opbox/collectives/scatter-gather/OpScatter.cpp b/examples/opbox/collectives/scatter-gather/OpScatter.cpp index cb314bc..a29e0c7 100644 --- a/examples/opbox/collectives/scatter-gather/OpScatter.cpp +++ b/examples/opbox/collectives/scatter-gather/OpScatter.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -153,7 +153,7 @@ WaitingType OpScatter::UpdateOrigin(OpArgs *args) { return WaitingType::done_and_destroy; } //Shouldn't be here - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -206,7 +206,7 @@ WaitingType OpScatter::UpdateTarget(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; } - KHALT("Missing state"); + F_HALT("Missing state"); return WaitingType::done_and_destroy; } @@ -217,6 +217,6 @@ string OpScatter::GetStateName() const { case State::get_wait_complete: return "GetWaitComplete"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); return "Unknown"; } diff --git a/examples/opbox/collectives/scatter-gather/OpScatter.hh b/examples/opbox/collectives/scatter-gather/OpScatter.hh index 7a66b57..8c9f100 100644 --- a/examples/opbox/collectives/scatter-gather/OpScatter.hh +++ b/examples/opbox/collectives/scatter-gather/OpScatter.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPSCATTER_HH #define OPSCATTER_HH diff --git a/examples/opbox/collectives/scatter-gather/scatter_gather.cpp b/examples/opbox/collectives/scatter-gather/scatter_gather.cpp index accbd30..7907f56 100644 --- a/examples/opbox/collectives/scatter-gather/scatter_gather.cpp +++ b/examples/opbox/collectives/scatter-gather/scatter_gather.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -8,7 +8,7 @@ #include "faodel-common/Common.hh" #include "opbox/OpBox.hh" -#include "Globals.hh" +#include "../../opbox-example-support/Globals.hh" #include "SimpleDataStore.hh" #include "OpScatter.hh" #include "OpGather.hh" @@ -103,6 +103,12 @@ int main(int argc, char **argv){ G.StartAll(argc, argv, config); + if(G.mpi_size==1) { + std::cerr<<"This example needs to be run with multiple mpi ranks\n"; + G.StopAll(); + return -1; + } + //For this example, the master node is going to create a new Op and //launch it. The flow of operation is as follows: // Origin (user) : Allocate a Lunasa RDMA buffer and fill with data for each target diff --git a/examples/opbox/opbox-example-support/CMakeLists.txt b/examples/opbox/opbox-example-support/CMakeLists.txt new file mode 100644 index 0000000..88cd252 --- /dev/null +++ b/examples/opbox/opbox-example-support/CMakeLists.txt @@ -0,0 +1,11 @@ + +set(TARGET opbox_example_support) + +add_library(${TARGET} + Globals.cpp + SimpleDataStore.cpp + ) +set_target_properties( ${TARGET} PROPERTIES LINKER_LANGUAGE CXX ) +target_link_libraries( ${TARGET} + MPI::MPI_CXX + ${EXAMPLE_LIBS} ) \ No newline at end of file diff --git a/examples/opbox/opbox-example-support/Globals.cpp b/examples/opbox/opbox-example-support/Globals.cpp index d86ab74..6747c4f 100644 --- a/examples/opbox/opbox-example-support/Globals.cpp +++ b/examples/opbox/opbox-example-support/Globals.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/opbox/opbox-example-support/Globals.hh b/examples/opbox/opbox-example-support/Globals.hh index 515f776..947fe3c 100644 --- a/examples/opbox/opbox-example-support/Globals.hh +++ b/examples/opbox/opbox-example-support/Globals.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_EXAMPLES_OPBOX_GLOBALS_HH #define FAODEL_EXAMPLES_OPBOX_GLOBALS_HH diff --git a/examples/opbox/opbox-example-support/SimpleDataStore.cpp b/examples/opbox/opbox-example-support/SimpleDataStore.cpp index 82f552e..ceb147f 100644 --- a/examples/opbox/opbox-example-support/SimpleDataStore.cpp +++ b/examples/opbox/opbox-example-support/SimpleDataStore.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/examples/opbox/opbox-example-support/SimpleDataStore.hh b/examples/opbox/opbox-example-support/SimpleDataStore.hh index f82edf3..4156919 100644 --- a/examples/opbox/opbox-example-support/SimpleDataStore.hh +++ b/examples/opbox/opbox-example-support/SimpleDataStore.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef SIMPLEDATASTORE_HH #define SIMPLEDATASTORE_HH diff --git a/examples/services/CMakeLists.txt b/examples/services/CMakeLists.txt index 5b8808a..6d4d9cd 100644 --- a/examples/services/CMakeLists.txt +++ b/examples/services/CMakeLists.txt @@ -1,7 +1,13 @@ -set( EXAMPLES_LIBS Faodel::services ) add_subdirectory( backburner ) -add_subdirectory( mpi_sync_start ) + +if( (NOT TARGET MPI::MPI_CXX) OR (NOT Faodel_ENABLE_MPI_SUPPORT) ) + message( STATUS "Some Service Examples will not be built because MPI Support was not enabled") + +else() + add_subdirectory( mpi-sync-start ) + +endif() diff --git a/examples/services/backburner/CMakeLists.txt b/examples/services/backburner/CMakeLists.txt index e733a96..ccdc27d 100644 --- a/examples/services/backburner/CMakeLists.txt +++ b/examples/services/backburner/CMakeLists.txt @@ -1,16 +1,18 @@ -set(PROJECT_NAME backburner_example) - - +set(PROJECT_NAME ex_service_backburner) set(SOURCES backburner_example.cpp ) +# When doing a standalone build, use the Faodel::services imported target +if( NOT DEFINED EXAMPLE_LIBS) + set(EXAMPLE_LIBS Faodel::services) +endif() + add_executable(${PROJECT_NAME} ${HEADERS} ${SOURCES}) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX ) - -target_link_libraries(${PROJECT_NAME} ${EXAMPLES_LIBS}) +target_link_libraries(${PROJECT_NAME} ${EXAMPLE_LIBS}) install(TARGETS ${PROJECT_NAME} EXPORT faodelExampleTargets diff --git a/examples/services/backburner/backburner_example.cpp b/examples/services/backburner/backburner_example.cpp index 69f59ba..9902dea 100644 --- a/examples/services/backburner/backburner_example.cpp +++ b/examples/services/backburner/backburner_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/examples/services/mpi-sync-start/CMakeLists.txt b/examples/services/mpi-sync-start/CMakeLists.txt new file mode 100644 index 0000000..68b89fa --- /dev/null +++ b/examples/services/mpi-sync-start/CMakeLists.txt @@ -0,0 +1,22 @@ +set(PROJECT_NAME ex_service_mpi-sync-start) + +# When doing a standalone build, use the Faodel::services imported target +if(NOT DEFINED EXAMPLE_LIBS) + set( EXAMPLE_LIBS Faodel::services ) +endif() + + +set( TARGET ex_service_mpi-sync-start-basic ) +add_executable(${TARGET} mpi_sync_start_basic.cpp) +set_target_properties(${TARGET} PROPERTIES LINKER_LANGUAGE CXX ) +target_link_libraries(${TARGET} ${EXAMPLE_LIBS}) + +set( TARGET ex_service_mpi-sync-start-advanced ) +add_executable(${TARGET} mpi_sync_start_advanced.cpp) +set_target_properties(${TARGET} PROPERTIES LINKER_LANGUAGE CXX ) +target_link_libraries(${TARGET} ${EXAMPLE_LIBS}) + +install(TARGETS ex_service_mpi-sync-start-basic ex_service_mpi-sync-start-advanced + EXPORT faodelExampleTargets + RUNTIME DESTINATION "${BINARY_INSTALL_DIR}" COMPONENT bin + ) diff --git a/examples/services/mpi-sync-start/mpi_sync_start_advanced.cpp b/examples/services/mpi-sync-start/mpi_sync_start_advanced.cpp index 9ea7009..12eb39c 100644 --- a/examples/services/mpi-sync-start/mpi_sync_start_advanced.cpp +++ b/examples/services/mpi-sync-start/mpi_sync_start_advanced.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // A more useful thing to do with the mpisyncstart service is use it to update // a config with runtime info. This service can currently perform two diff --git a/examples/services/mpi-sync-start/mpi_sync_start_basic.cpp b/examples/services/mpi-sync-start/mpi_sync_start_basic.cpp index ad519e1..5b570c8 100644 --- a/examples/services/mpi-sync-start/mpi_sync_start_basic.cpp +++ b/examples/services/mpi-sync-start/mpi_sync_start_basic.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // The mpisyncstart service is a simple service for starting up jobs that // use MPI. If you enable the service it will (in the least) do a barrier diff --git a/examples/whookie/CMakeLists.txt b/examples/whookie/CMakeLists.txt index 378f840..da03db9 100644 --- a/examples/whookie/CMakeLists.txt +++ b/examples/whookie/CMakeLists.txt @@ -1,5 +1,22 @@ -set( EXAMPLE_LIBS Faodel::whookie Faodel::common ) +if( NOT DEFINED EXAMPLE_LIBS ) + # A standalone project can pull in everything via Faodel::whookie and Faodel::common + set( EXAMPLE_LIBS Faodel::whookie Faodel::common ) +endif() -add_subdirectory( simple ) + +set(TARGET ex_whookie_bootstrap) +add_executable( ${TARGET} bootstrap_example.cpp ) +target_link_libraries( ${TARGET} ${EXAMPLE_LIBS} ) + + +set(TARGET ex_whookie_killit) +add_executable( ${TARGET} killit_example.cpp ) +target_link_libraries( ${TARGET} ${EXAMPLE_LIBS} ) + + +install(TARGETS ex_whookie_bootstrap ex_whookie_killit + EXPORT faodelExampleTargets + RUNTIME DESTINATION "${BINARY_INSTALL_DIR}" COMPONENT bin +) \ No newline at end of file diff --git a/examples/whookie/bootstrap_example.cpp b/examples/whookie/bootstrap_example.cpp index e614ccc..7ff0739 100644 --- a/examples/whookie/bootstrap_example.cpp +++ b/examples/whookie/bootstrap_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Whookie Bootstrap Example // @@ -37,8 +37,15 @@ whookie.debug true int main(int argc, char* argv[]) { - - //The simplest hook is just a static web page. We can encode all + + //In this example, whookie is all we need from the FAODEL + //stack. We need to tell bootstrap that it should launch whookie + //and all of its dependencies + faodel::bootstrap::Init(faodel::Configuration(default_config), + whookie::bootstrap); + + //After initialization you can start registering web page handlers with + //whookie. The simplest hook is just a static web page. We can encode all //the information needed for the page inside of a lambda. More //sophisticated handlers should call out to functions in order to make //the core more readable. @@ -46,16 +53,14 @@ int main(int argc, char* argv[]) { html::mkHeader(results, "Bob's Page"); html::mkTable(results, args, "Bobs args"); html::mkFooter(results); - }); - + }); - //In this example, whookie is all we need from the FAODEL - //stack. We need to tell bootstrap that it should launch whookie - //and all of its dependencies - faodel::bootstrap::Init(faodel::Configuration(default_config), - whookie::bootstrap); + //You should register other handlers after init, but (preferably) before start. + //The whookie service is a little different than other services because it is + //functional after Init instead of Start. This is because some services need to + //know the local node id during their init. - //Once it's started, you can retrieve our node id + //eg, we haven't run start yet, but we can get the whookie node id faodel::nodeid_t nid = whookie::Server::GetNodeID(); //You should be able to browse to the web page now. diff --git a/examples/whookie/killit_example.cpp b/examples/whookie/killit_example.cpp index aa54a61..a527fee 100644 --- a/examples/whookie/killit_example.cpp +++ b/examples/whookie/killit_example.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Killit Example // @@ -38,17 +38,15 @@ void ShutMeDown(){ int main(int argc, char* argv[]) { - - //Register a simple killit page that calls the ShutMeDown function - whookie::Server::registerHook("/killit", [] (const map &args, stringstream &results){ - ShutMeDown(); - }); - - //Startup bootstraps (should only be whookie) faodel::bootstrap::Init(faodel::Configuration(default_config), whookie::bootstrap); + //Register a simple killit page that calls the ShutMeDown function + whookie::Server::registerHook("/killit", [] (const map &args, stringstream &results){ + ShutMeDown(); + }); + //Once it's started, you can retrieve our node id faodel::nodeid_t nid = whookie::Server::GetNodeID(); cout<<"Started Webserver. Go to killit page to kill it: curl "< #include diff --git a/src/dirman/DirMan.hh b/src/dirman/DirMan.hh index cfe3275..8dd33af 100644 --- a/src/dirman/DirMan.hh +++ b/src/dirman/DirMan.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef DIRMAN_DIRMAN_HH #define DIRMAN_DIRMAN_HH diff --git a/src/dirman/README_DirMan.md b/src/dirman/README_DirMan.md index 4a584b2..b8858dd 100644 --- a/src/dirman/README_DirMan.md +++ b/src/dirman/README_DirMan.md @@ -47,7 +47,9 @@ centralized implementation currently overwrites the nodeid with the root node. Users should only populate that path and directory portions of the ResourceURL when passing a ResourceURL into dirman. -Example Use: Consider the case where an OpBox application needs to +Example Scenario +---------------- +Consider the case where an OpBox application needs to connect to a separate array of nodes that store data objects in memory. In order to simplify how nodeid information is passed between applications, we would expect three jobs to run. First, a single node @@ -62,6 +64,43 @@ provided dirman root nodeid to connect with the root, obtain the list of nodes for the array, and then start using the list. As this example illustrates, dirman provides a scalable way to share runtime information without having to write node information to files. + +DirMan Cores +------------ +There are currently three types of dirman cores that can be used: + +- **Centralized** (default): The centralized dirman core assigns one node + in the platform to store all dirman information. While this node may perform + other FAODEL functions, it must be started before the other nodes. The central + dirman node has `dirman.host_root true` specified in its configuration file. + All other nodes must have a pointer to this node specified in either + their config (eg `dirman.root_node 0x1234`), a dirman root node file, or + an environment variable (see `Root Node Resolution` for more info). +- **Static**: In the specialized scenario where all information about + resources is known at start time, users may use the `static` dirman + core and plug all the information into the configuration. +- **None**: In some test scenarios, it is useful to launch services without + a dirman service. This option disables dirman services from running, and + is not useful in normal workloads. + +Root Node Resolution +-------------------- +When operating in the `centralized` mode, users must provide information to +their services about how to find the root dirman node. FAODEL checks for +this information using the following process: + +1. `dirman.root_node` in Configuration: The easiest way to pass the root + node id is to just place it in the configuration. This technique works + especially well if you're using mpisyncstart and you specify the rank + in the job that is the root via `dirman.root_node_mpi 0`. +2. `dirman.root_node.file` in Configuration: The second option is to read + a file to learn the id of the root node. This method works well when the + root node is also configured with `dirman.write_root file`. +3. Environment variable `FAODEL_DIRMAN_ROOT_NODE_FILE`: This option allows + users to point to the root node file using an environment variable. +4. Environment variable `FAODEL_DIRMAN_ROOT_NODE`: The last option allows + users to specify the root node as an environment variable. + Build and Configuration Settings ================================ @@ -85,7 +124,25 @@ Run-Time Options When the DirMan service starts, it examines the Configuration object for the following run-time options: -| Property | Type | Default | Description | -| --------------------- | ----------- | -------- | --------------------------------------------------- | -| dirman.host_root | bool | false | When true, this node is the dirman root node | -| dirman.root_node | nodeid | "" | The nodeid of the root node (hex value) | +| Property | Type | Default | Description | +| --------------------- | ----------------------- | ----------- | ---------------------------------------------------------------------- | +| dirman.type | none,static,centralized | centralized | Static assumes all info is in config, centralized uses a single server | +| dirman.host_root | bool | false | When true, this node is the dirman root node | +| dirman.write_root | filename | "" | Instruct the root node to write its id to a file | +| dirman.root_node | nodeid | "" | Use the node id supplied here to reference the root node (hex value) | +| dirman.root_node.file | filename | "" | Read the file and use its contents to identify the root node | +| dirman.resource[] | url | "" | Define a static resource in the configuration | + +note: the `dirman.resource` command is typically used when `mpisyncstart` is +enabled, as it provides users with an easy way to define resources that the +mpi job will host. For example, a user might distribute a kelpie pool in their +job via: + +``` +mpisyncstart.enable true # Use mpisyncstart service to let us map ranks to faodel nodes +dirman.root_node_mpi 0 # Set rank 0 as the dirman root + +dirman.resources_mpi[] rft:/my/pool ALL # Make a pool across all ranks +dirman.resources_mpi[] dht:/my/meta END # Make a pool w/ just the last rank +dirman.resources_mpi[] dht:/my/first 0-3 # Make a pool on the first four ranks +``` \ No newline at end of file diff --git a/src/dirman/common/DirectoryCache.cpp b/src/dirman/common/DirectoryCache.cpp index 8344868..671a93e 100644 --- a/src/dirman/common/DirectoryCache.cpp +++ b/src/dirman/common/DirectoryCache.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -34,7 +34,7 @@ DirectoryCache::~DirectoryCache(){ } } void DirectoryCache::Init(const faodel::Configuration &config, string threading_model, string mutex_type) { - kassert(mutex==nullptr, "Initialized more than once"); + F_ASSERT(mutex == nullptr, "Initialized more than once"); ConfigureLogging(config); mutex = GenerateMutex(threading_model, mutex_type); @@ -454,8 +454,8 @@ void DirectoryCache::sstr(stringstream &ss, int depth, int indent) const { */ bool DirectoryCache::_lookup(const faodel::ResourceURL &url, DirectoryInfo **resource_info, nodeid_t *reference_node){ - kassert(resource_info!=nullptr, "Invalid resource info pointer"); - kassert(url.Valid(), "Invalid url given to DC:"+url.GetFullURL()); + F_ASSERT(resource_info != nullptr, "Invalid resource info pointer"); + F_ASSERT(url.Valid(), "Invalid url given to DC:"+url.GetFullURL()); map::iterator it; string bucket_path_name = url.GetBucketPathName(); @@ -480,7 +480,7 @@ bool DirectoryCache::_lookup(const faodel::ResourceURL &url, DirectoryInfo **res } bool DirectoryCache::_lookupRootDir(bucket_t bucket, faodel::DirectoryInfo *resource_info) { bool found=false; - kassert(resource_info!=nullptr, "Invalid resource info pointer"); + F_ASSERT(resource_info != nullptr, "Invalid resource info pointer"); //Set the dir info to root if(resource_info) { diff --git a/src/dirman/common/DirectoryCache.hh b/src/dirman/common/DirectoryCache.hh index 2eb453b..ad5ff38 100644 --- a/src/dirman/common/DirectoryCache.hh +++ b/src/dirman/common/DirectoryCache.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef DIRMAN_DIRECTORYCACHE_HH #define DIRMAN_DIRECTORYCACHE_HH diff --git a/src/dirman/common/DirectoryOwnerCache.cpp b/src/dirman/common/DirectoryOwnerCache.cpp index 329c106..9b81110 100644 --- a/src/dirman/common/DirectoryOwnerCache.cpp +++ b/src/dirman/common/DirectoryOwnerCache.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -26,7 +26,7 @@ DirectoryOwnerCache::~DirectoryOwnerCache(){ } } void DirectoryOwnerCache::Init(const faodel::Configuration &config, string threading_model, string mutex_type){ - kassert(mutex== nullptr, "Initialized more than once"); + F_ASSERT(mutex == nullptr, "Initialized more than once"); ConfigureLogging(config); mutex = GenerateMutex(threading_model, mutex_type); } @@ -158,8 +158,8 @@ void DirectoryOwnerCache::sstr(stringstream &ss, int depth, int indent) const { bool DirectoryOwnerCache::_Lookup(const faodel::ResourceURL &url, nodeid_t *node_id){ - kassert(node_id!=nullptr, "Invalid node_id"); - kassert(url.Valid(), "Invalid url given to RIC:"+url.GetFullURL()); + F_ASSERT(node_id != nullptr, "Invalid node_id"); + F_ASSERT(url.Valid(), "Invalid url given to RIC:"+url.GetFullURL()); string bucket_path_name = url.GetBucketPathName(); diff --git a/src/dirman/common/DirectoryOwnerCache.hh b/src/dirman/common/DirectoryOwnerCache.hh index 629f7f8..816b64a 100644 --- a/src/dirman/common/DirectoryOwnerCache.hh +++ b/src/dirman/common/DirectoryOwnerCache.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef DIRMAN_DIRECTORYOWNERCACHE_HH #define DIRMAN_DIRECTORYOWNERCACHE_HH diff --git a/src/dirman/core/DirManCoreBase.cpp b/src/dirman/core/DirManCoreBase.cpp index d7a5a26..96ab34b 100644 --- a/src/dirman/core/DirManCoreBase.cpp +++ b/src/dirman/core/DirManCoreBase.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include //for ifstream parsing of file @@ -92,9 +92,11 @@ DirManCoreBase::DirManCoreBase(const faodel::Configuration &config, const string // gets a url with all the members, but the non-root nodes // get zero members because mpisyncstart doesn't globally // sync everything. - if((di.url.Type()!="local") && (di.members.size()==0)) { + set ok_types={"local","trace","null"}; + bool not_local = ok_types.find(di.url.Type()) == ok_types.end(); + if((not_local) && (di.members.size()==0)) { dbg("Not adding predefined resource "+key_url.first+" because it is not local and does not have any members"); - throw runtime_error("Abort on adding "+key_url.first); + throw runtime_error("Dirman aborted adding "+key_url.first+" because it was not a local resource and didn't have predefined members"); continue; } dbg("adding predefined resource "+key_url.first+" --> "+di.url.GetFullURL()+" Num Members="+std::to_string(di.members.size())); @@ -197,14 +199,14 @@ DirManCoreBase::~DirManCoreBase() = default; bool DirManCoreBase::Locate(const faodel::ResourceURL &search_url, nodeid_t *reference_node) { bool ok = lookupLocal(search_url, nullptr, reference_node); - KWARN("Locate should be global, not just local"); + F_WARN("Locate should be global, not just local"); return ok; } /** * @brief Retrieve info about a particular resource directory entry - * @param url - Resource URL to look up + * @param search_url - Resource URL to look up * @param check_local - Check our local cache for answer first * @param check_remote - Check the remote node responsible for dir (if local not successful) * @param dir_info - The resulting directory info (set to empty value if no entry) @@ -219,7 +221,7 @@ bool DirManCoreBase::GetDirectoryInfo(const faodel::ResourceURL &search_url, boo if(ok || !check_remote) return ok; } if(check_remote){ - KWARN("GetDirectoryInfo should be global, not just local"); + F_WARN("GetDirectoryInfo should be global, not just local"); return false; } return false; @@ -251,7 +253,7 @@ bool DirManCoreBase::DefineNewDir(const DirectoryInfo &dir_info) { // Fail: This default DefineNewDir doesn't know what to do with an undefined reference node. // A core should define this and do the right thing instead. - KTODO("Default DefineNewDir does not handle unspecified reference node case. Derived class should implement."); + F_TODO("Default DefineNewDir does not handle unspecified reference node case. Derived class should implement."); return false; } @@ -312,7 +314,7 @@ bool DirManCoreBase::HostNewDir(const DirectoryInfo &dir_info){ bool ok = discoverParent(dir_info.url, &parent_node); dbg("hostresource discovered ok="+to_string(ok)+" parent was "+parent_node.GetHex()); - kassert(ok,"couldn't discover parent for "+dir_info.url.GetFullURL()); + F_ASSERT(ok, "couldn't discover parent for "+dir_info.url.GetFullURL()); if((parent_node == my_node)||(parent_node==NODE_LOCALHOST)){ dbg("hosted resource's parent available here. Joining."); return dc_mine.Join(dir_info.url); @@ -321,7 +323,7 @@ bool DirManCoreBase::HostNewDir(const DirectoryInfo &dir_info){ //Not local, we must join a resource dbg("hosted resource's parent not available here. remote joining."); ok = joinRemote(parent_node, dir_info.url); - kassert(ok,"Couldn't host resource, because couldn't join parent?"); + F_ASSERT(ok, "Couldn't host resource, because couldn't join parent?"); return true; } @@ -338,7 +340,7 @@ bool DirManCoreBase::HostNewDir(const faodel::ResourceURL &url){ //Plug in our default bucket and node id if(tmp_url.bucket==BUCKET_UNSPECIFIED) { - kassert(!strict_checking, "HostNewDir given a url with a null bucket"); + F_ASSERT(!strict_checking, "HostNewDir given a url with a null bucket"); tmp_url.bucket=default_bucket; } @@ -472,7 +474,7 @@ nodeid_t DirManCoreBase::parseConfigForRootNode(const Configuration &config) con rc_t rc; string fname, root_node_hex; - dbg("Parsing condfig for root node info"); + dbg("Parsing config for root node info"); rc = config.GetString(&root_node_hex, "dirman.root_node"); dbg("Searching for dirman.root_node gave '"+root_node_hex+"'"); if(rc==ENOENT) { @@ -571,7 +573,7 @@ bool DirManCoreBase::writeURLsToFileOrDie(const string file_name, const vector @@ -41,6 +41,8 @@ DirManCoreCentralized::DirManCoreCentralized(const faodel::Configuration &config dbg("Am hosting root"); root_id = my_node; + info("Root node link: "+root_id.GetHttpLink()); + info("Node node id: "+root_id.GetHex()); //See if we've been instructed to write to a file if(!write_root_filename.empty()) { dbg("Root is writing file "+write_root_filename); @@ -50,6 +52,7 @@ DirManCoreCentralized::DirManCoreCentralized(const faodel::Configuration &config throw std::runtime_error("dirman root node failed to write_root_filename "+write_root_filename); } f << my_node.GetHex() < @@ -30,10 +30,10 @@ const string OpDirManCentralized::op_name = "OpDirManCentralized"; OpDirManCentralized::OpDirManCentralized(RequestType req_type, faodel::nodeid_t root_id, faodel::DirectoryInfo dir_info) : Op(true), state(State::start), ldo_msg(), request_type(RequestType::HostNewDir) { - kassert(req_type == RequestType::HostNewDir, "Only supports hostnewdir now"); + F_ASSERT(req_type == RequestType::HostNewDir, "Only supports hostnewdir now"); int rc = opbox::net::Connect(&peer, root_id); //Retrieve the root's peer ptr - kassert((rc==0), "Connect failed?"); + F_ASSERT((rc == 0), "Connect failed?"); msg_dirman::AllocateRequest(ldo_msg, RequestType::HostNewDir, @@ -52,11 +52,11 @@ OpDirManCentralized::OpDirManCentralized(RequestType req_type, faodel::nodeid_t OpDirManCentralized::OpDirManCentralized(RequestType req_type, faodel::nodeid_t root_id, faodel::ResourceURL url) : Op(true), state(State::start), ldo_msg(), request_type(req_type) { - kassert((req_type == RequestType::GetInfo) || - (req_type == RequestType::JoinDir) || - (req_type == RequestType::LeaveDir) || - (req_type == RequestType::DropDir) || - (req_type == RequestType::ReturnDirInfo), "Request type not handled"); + F_ASSERT((req_type == RequestType::GetInfo) || + (req_type == RequestType::JoinDir) || + (req_type == RequestType::LeaveDir) || + (req_type == RequestType::DropDir) || + (req_type == RequestType::ReturnDirInfo), "Request type not handled"); int rc = opbox::net::Connect(&peer, root_id); //Retrieve the root's peer ptr if(rc!=0) { @@ -108,7 +108,7 @@ string OpDirManCentralized::GetStateName() const { case State::done: return "Done"; } - KFAIL(); + F_FAIL(); } } // namespace dirman \ No newline at end of file diff --git a/src/dirman/ops/OpDirManCentralized.hh b/src/dirman/ops/OpDirManCentralized.hh index 38d5cc9..4aa3285 100644 --- a/src/dirman/ops/OpDirManCentralized.hh +++ b/src/dirman/ops/OpDirManCentralized.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef DIRMAN_OPDIRMANCENTRALIZED_HH #define DIRMAN_OPDIRMANCENTRALIZED_HH diff --git a/src/dirman/ops/OpDirManCentralized_Origin.cpp b/src/dirman/ops/OpDirManCentralized_Origin.cpp index 7d74720..f96d481 100644 --- a/src/dirman/ops/OpDirManCentralized_Origin.cpp +++ b/src/dirman/ops/OpDirManCentralized_Origin.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -55,7 +55,7 @@ WaitingType OpDirManCentralized::UpdateOrigin(OpArgs *args) { case RequestType::Invalid: { //Invalid can happen. this isn't a reply message cerr << "Invalid OpDirManCentalized response at origin\n"; - KHALT("InvalidMessage") + F_HALT("InvalidMessage") } break; @@ -78,7 +78,7 @@ WaitingType OpDirManCentralized::UpdateOrigin(OpArgs *args) { case State::done: return updateState(State::done, WaitingType::done_and_destroy); - default: KTODO("Unhandled state in OpDirManCentralized origin Update"); + default: F_TODO("Unhandled state in OpDirManCentralized origin Update"); } //Shouldn't get here return updateState(State::done, WaitingType::error); diff --git a/src/dirman/ops/OpDirManCentralized_Target.cpp b/src/dirman/ops/OpDirManCentralized_Target.cpp index 856526f..9541915 100644 --- a/src/dirman/ops/OpDirManCentralized_Target.cpp +++ b/src/dirman/ops/OpDirManCentralized_Target.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -81,7 +81,7 @@ WaitingType OpDirManCentralized::UpdateTarget(OpArgs *args) { case State::done: return updateState(State::done, WaitingType::done_and_destroy); - default: KTODO("Unhandled state in OpDirManCentralized receiver Update"); + default: F_TODO("Unhandled state in OpDirManCentralized receiver Update"); } return WaitingType::error; } diff --git a/src/dirman/ops/msg_dirman.cpp b/src/dirman/ops/msg_dirman.cpp index 7b1642f..f97641a 100644 --- a/src/dirman/ops/msg_dirman.cpp +++ b/src/dirman/ops/msg_dirman.cpp @@ -1,18 +1,15 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include -#include "faodel-common/SerializationHelpers.hh" +#include +#include #include "opbox/common/MessageHelpers.hh" - - #include "dirman/ops/msg_dirman.hh" -#include - using namespace std; using namespace faodel; @@ -81,7 +78,7 @@ DirectoryInfo msg_dirman::ExtractDirInfo(message_t *hdr) { if (!(reinterpret_cast(hdr))->hasDirInfo()) { throw std::runtime_error("ExtractURL called on a message that didn't contain a URL"); } - return UnpackBoostMessage(hdr); + return UnpackCerealMessage(hdr); } /** @@ -101,7 +98,7 @@ bool msg_dirman::AllocateRequest(lunasa::DataObject &new_ldo, const DirectoryInfo &dir_info) { - return AllocateBoostRequestMessage( + return AllocateCerealRequestMessage( new_ldo, dst_node, src_mailbox, OpDirManCentralized::op_id, @@ -123,7 +120,7 @@ bool msg_dirman::AllocateReply(lunasa::DataObject &new_ldo, const message_t *request_msg, const DirectoryInfo &dir_info) { - return AllocateBoostReplyMessage( + return AllocateCerealReplyMessage( new_ldo, request_msg, static_cast(req_type), diff --git a/src/dirman/ops/msg_dirman.hh b/src/dirman/ops/msg_dirman.hh index 1620696..c21d2e1 100644 --- a/src/dirman/ops/msg_dirman.hh +++ b/src/dirman/ops/msg_dirman.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef DIRMAN_MSG_DIRMAN_HH #define DIRMAN_MSG_DIRMAN_HH @@ -16,7 +16,7 @@ namespace dirman { * @brief A message data structure used by dirman ops * * In addition to the standard opbox message header, a dirman message contains - * is a URL string or a boost-packed DirInfo. Flags are stored in the hdr_flags + * is a URL string or a cereal-packed DirInfo. Flags are stored in the hdr_flags * of the message header. * * A user is expected to call the Allocate(Request/Reply) functions to generate @@ -27,7 +27,7 @@ typedef struct msg_dirman { //Body: The body section following hdr contains either: // - a url string - // - a boost-packed DirInfo + // - a cereal-packed DirInfo // bit[4] of hdr_flags specifies which type. Make sure your op id's agree msg_dirman()=delete; diff --git a/src/faodel-common/Bootstrap.cpp b/src/faodel-common/Bootstrap.cpp index ae27266..8036efd 100644 --- a/src/faodel-common/Bootstrap.cpp +++ b/src/faodel-common/Bootstrap.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -130,7 +130,13 @@ void setNodeID(internal_use_only_t iuo, NodeID nodeid) { void Init(const Configuration &config, fn_register last_component) { string last_component_name = last_component(); //Calls registration - return BSCORE.Init(config); + if(!BSCORE.HasComponent(last_component_name)) { + throw std::runtime_error("Bootstrap did not know about "+last_component_name+". This may happen when multiple entities use FAODEL\n" + " and one starts without knowing that other services are needed. Make sure the entity that calls\n" + " bootstrap::init has all components in it.\n"); + } + BSCORE.Init(config); + return; } /** @@ -153,6 +159,11 @@ void Start() { void Start(const Configuration &config, fn_register last_component) { string last_component_name = last_component(); //Calls registration + if(!BSCORE.HasComponent(last_component_name)) { + throw std::runtime_error("Bootstrap did not know about "+last_component_name+". This may happen when multiple entities use FAODEL\n" + " and one starts without knowing that other services are needed. Make sure the entity that calls\n" + " bootstrap::start has all components in it.\n"); + } return BSCORE.Start(config); } @@ -196,6 +207,14 @@ bool IsStarted() { return BSCORE.IsStarted(); } +/** + * @brief Disaplay current number of bootstrap Initialization calls that have not yet had a Finish issued for them + * @return NumberUsers The current count + */ +int GetNumberOfUsers() { + return BSCORE.GetNumberOfUsers(); +} + /** * @brief Get the Configuration that Bootstrap is using * diff --git a/src/faodel-common/Bootstrap.hh b/src/faodel-common/Bootstrap.hh index 88bb414..7be5e6f 100644 --- a/src/faodel-common/Bootstrap.hh +++ b/src/faodel-common/Bootstrap.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_BOOTSTRAP_HH #define FAODEL_COMMON_BOOTSTRAP_HH @@ -51,6 +51,8 @@ void FinishSoft(); //Stop services, but don't clear out registration list void setNodeID(internal_use_only_t iuo, NodeID nodeid); std::string GetState(); bool IsStarted(); +int GetNumberOfUsers(); + Configuration GetConfiguration(); void dumpInfo(faodel::ReplyStream &rs); diff --git a/src/faodel-common/BootstrapImplementation.cpp b/src/faodel-common/BootstrapImplementation.cpp index 714033d..244e5ea 100644 --- a/src/faodel-common/BootstrapImplementation.cpp +++ b/src/faodel-common/BootstrapImplementation.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -12,9 +12,11 @@ #include #include - #include +#ifdef Faodel_ENABLE_MPI_SUPPORT +#include +#endif #include "faodel-common/BootstrapImplementation.hh" @@ -29,14 +31,21 @@ Bootstrap::Bootstrap() : LoggingInterface("bootstrap"), halt_on_shutdown(false), status_on_shutdown(false), + mpisyncstop_enabled(false), sleep_seconds_before_shutdown(0), my_node_id(NODE_UNSPECIFIED), + num_init_callers(0), state(State::UNINITIALIZED) { + state_mutex = GenerateMutexByTypeID(MutexWrapperTypeID::PTHREADS_LOCK); } Bootstrap::~Bootstrap() { + + state_mutex->Lock(); + num_init_callers=1; //ensure we actually shutdown if(state == State::INITIALIZED) { warn("Bootstrap was initialized but never started"); } - if(state == State::STARTED) Finish(true); + if(state == State::STARTED) finish_(true); + delete state_mutex; } /** @@ -58,8 +67,21 @@ void Bootstrap::RegisterComponent(string name, fn_fini fini_function, bool allow_overwrites) { - if(state != State::UNINITIALIZED) - throw std::runtime_error("Bootstrap RegsiterComponent: Register of " + name + " called after init"); + state_mutex->Lock(); + + //dbg("RegisteringComponent "+name); + + //See if someone is trying to register a component after bootstrap has already started. This is ok + //if the component was registered the first time around (eg, when users hide a bootstrap init of + //kelpie in their ctors). We have to fail though if they didn't register this component. + if(state != State::UNINITIALIZED) { + bool already_exists = HasComponent(name); + state_mutex->Unlock(); + if(!already_exists) { + throw std::runtime_error("Bootstrap RegisterComponent: Register of " + name + " called after init"); + } + return; + } bstrap_t bs; bs.name = name; @@ -72,14 +94,17 @@ void Bootstrap::RegisterComponent(string name, for(auto &tmp_bs : bstraps) { if(tmp_bs.name == name) { - if(!allow_overwrites) - throw std::runtime_error("Bootstrap RegsiterComponent Attempted to register " + name + " multiple times"); + if(!allow_overwrites) { + state_mutex->Unlock(); + throw std::runtime_error("Bootstrap RegisterComponent Attempted to register " + name + " multiple times"); + } tmp_bs = bs; return; } } bstraps.push_back(bs); + state_mutex->Unlock(); } /** @@ -94,6 +119,23 @@ void Bootstrap::RegisterComponent(BootstrapInterface *component, bool allow_over vector optional; component->GetBootstrapDependencies(name, requires, optional); + state_mutex->Lock(); + + //dbg("RegisteringComponent "+name); + + //See if someone is trying to register a component after bootstrap has already started. This is ok + //if the component was registered the first time around (eg, when users hide a bootstrap init of + //kelpie in their ctors). We have to fail though if they didn't register this component. + if(state != State::UNINITIALIZED) { + bool already_exists = HasComponent(name); + state_mutex->Unlock(); + if(!already_exists) { + throw std::runtime_error("Bootstrap RegisterComponent: Register of " + name + " called after init"); + } + return; + } + + //Normal registration bstrap_t bs; bs.name = name; bs.requires = requires; @@ -105,21 +147,42 @@ void Bootstrap::RegisterComponent(BootstrapInterface *component, bool allow_over for(auto &tmp_bs : bstraps) { if(tmp_bs.name == name) { - if(!allow_overwrites) + if(!allow_overwrites) { + state_mutex->Unlock(); throw std::runtime_error("Bootstrap RegisterComponent: Attempted to register " + name + " multiple times"); + } tmp_bs = bs; + state_mutex->Unlock(); return; } } bstraps.push_back(bs); + + state_mutex->Unlock(); + } /** * @brief Initialize all the pre-registered bootstrap components with a given config * @param config The configuration to pass each component + * @retval TRUE We initialized using this call (nobody else initialized before us) + * @retval FALSE Someone else had already initialized, so we used those settings * @note The Configuration may be modified in Init to fill in runtime values */ -void Bootstrap::Init(const Configuration &config) { +bool Bootstrap::Init(const Configuration &config) { + + bool exit_on_errors; //When true, we catch exceptions and then call exit + + state_mutex->Lock(); + + //Only proceed if this is the first initialization. Otherwise use previous initialization + int current_count = num_init_callers++; + if(current_count != 0) { + warn("Multiple bootstrap Init's called. Using existing initialization."); + state_mutex->Unlock(); + return false; + } + //We need to load any updates from references configuration = config; @@ -128,32 +191,66 @@ void Bootstrap::Init(const Configuration &config) { //Now we can update bootstrap's logging ConfigureLogging(configuration); - configuration.GetBool(&halt_on_shutdown, "bootstrap.halt_on_shutdown", "false"); - configuration.GetBool(&status_on_shutdown, "bootstrap.status_on_shutdown", "false"); + bool mpisyncstart_enabled=false; + configuration.GetBool(&show_config_at_init, "bootstrap.show_config", "false"); + configuration.GetBool(&exit_on_errors, "bootstrap.exit_on_errors", "true"); + configuration.GetBool(&halt_on_shutdown, "bootstrap.halt_on_shutdown", "false"); + configuration.GetBool(&status_on_shutdown, "bootstrap.status_on_shutdown", "false"); configuration.GetUInt(&sleep_seconds_before_shutdown, "bootstrap.sleep_seconds_before_shutdown", "0"); - + configuration.GetBool(&mpisyncstart_enabled, "mpisyncstart.enable", "false"); + configuration.GetBool(&mpisyncstop_enabled, "mpisyncstop.enable", "false"); dbg("Init (" + std::to_string(bstraps.size()) + " bootstraps known)"); - if(state != State::UNINITIALIZED) - throw std::runtime_error("Bootstrap Init: Init called multiple times (it can only be called once)"); + #ifndef Faodel_ENABLE_MPI_SUPPORT + if(mpisyncstart_enabled) { + warn("The mpisyncstart option was enabled, but FAODEL was not built with MPI support. Ignoring."); + } + if(mpisyncstop_enabled) { + warn("The mpisyncstop option was enabled, but FAODEL was not built with MPI support. Ignoring."); + } + #else + //cout<<"MPISYNCSTART enabled: "<Unlock(); + throw std::runtime_error("Bootstrap Init: Dependency error " + info_message); + } - //Execute each one - dbg("Init'ing all services"); + //Execute each bootstrap + try { + for(auto &bs : bstraps) { + dbg("Initializing service " + bs.name); + bs.init_function(&configuration); + } + } catch(const runtime_error &e) { + state_mutex->Unlock(); + if(exit_on_errors) { + cerr <<"Bootstrap Init Error: "<Unlock(); + return true; } /** @@ -162,10 +259,19 @@ void Bootstrap::Init(const Configuration &config) { */ void Bootstrap::Start() { - if(state != State::INITIALIZED) + state_mutex->Lock(); + if(state == State::STARTED) { + dbg("Already in started state. Continuing."); + state_mutex->Unlock(); + return; + } + + if(state != State::INITIALIZED) { + state_mutex->Unlock(); throw std::runtime_error( "Bootstrap Start: Attempted to Start() when not in the Initialized state. Call Init, Start, Finish\n" "(Current State is " + GetState() + ")"); + } dbg("Starting all services"); for(auto &bs : bstraps) { @@ -174,20 +280,43 @@ void Bootstrap::Start() { } dbg("Completed Starting services. Moved to 'started' state."); state = State::STARTED; + state_mutex->Unlock(); } /** * @brief Perform Init on all components, then perform Start on all components * @param[in] config The configuration to pass all components - * @throw runtime_error When bootstrap state is not Uninitialized + * @note If bootstrap is already started, it will reuse it. This may cause the config passed in to be ignored */ void Bootstrap::Start(const Configuration &config) { + bool we_initialized = Init(config); + if(we_initialized) Start(); +} - if(state != State::UNINITIALIZED) - throw std::runtime_error("Bootstrap Start: Attempted to Start(config) when already initialized and/or started. Call Init, Start, Finish\n" - "(Current State is " + GetState() + ")"); - Init(config); - Start(); +/** + * @brief Get the number of times bootstrap Init was called + * @return Current count + * Sometimes different applications embed bootstrap start in their code to enable servces + * to start automatically. This function tells you how many times Init was called. + */ +int Bootstrap::GetNumberOfUsers() const { + state_mutex->Lock(); + int count = num_init_callers; + state_mutex->Unlock(); + return count; +} + +/** + * @brief Determine if a bootstrap component (eg kelpie) has been registered yet + * @param component_name Name of the component to locate + * @retval TRUE This component is known + * @retval FALSE This component has not been registered yet + */ +bool Bootstrap::HasComponent(const string &component_name) const { + for(auto bs : bstraps) { + if(bs.name == component_name) return true; + } + return false; } /** @@ -214,6 +343,7 @@ void Bootstrap::dumpInfo(ReplyStream &rs) const { rs.tableBegin("Bootstrap Settings"); rs.tableTop({"Parameter", "Value"}); rs.tableRow({"Current State", GetState() }); + rs.tableRow({"MPISyncStop Enabled on Shutdown", std::to_string(mpisyncstop_enabled)}); rs.tableRow({"Status on Shutdown", std::to_string(status_on_shutdown)}); rs.tableRow({"Halt on Shutdown", std::to_string(halt_on_shutdown) }); rs.tableRow({"Sleep Seconds Before Shutdown", std::to_string(sleep_seconds_before_shutdown)}); @@ -231,13 +361,41 @@ void Bootstrap::dumpInfo(ReplyStream &rs) const { */ void Bootstrap::Finish(bool clear_list_of_bootstrap_users) { - if(state == State::UNINITIALIZED) + state_mutex->Lock(); + finish_(clear_list_of_bootstrap_users); + state_mutex->Unlock(); + +} + +void Bootstrap::finish_(bool clear_list_of_bootstrap_users) { + + if(state == State::UNINITIALIZED) { + state_mutex->Unlock(); throw std::runtime_error("Bootstrap Finish: Attempted to Finish when not Init state. Currently: " + GetState()); + } + + //dbg("Num init callers is "+std::to_string(num_init_callers)+" state is "+GetState()); + //Check number of initializers + num_init_callers--; + + if(num_init_callers>0) { + dbg("Received finish, but other entities started bootstrap. Waiting for their finish."); + return; + } + + #ifdef Faodel_ENABLE_MPI_SUPPORT + //Halt until everyone has had a chance to shutdown. This should catch any late senders + if(mpisyncstop_enabled) { + dbg("Performing mpisyncstop"); + MPI_Barrier(MPI_COMM_WORLD); + } + #endif + dbg("Finish (" + std::to_string(bstraps.size()) + " bootstraps known)"); if(halt_on_shutdown) { if(status_on_shutdown) dumpStatus(); - KHALT("Bootstrap finish called with Halt on Shutdown activated"); + F_HALT("Bootstrap finish called with Halt on Shutdown activated"); } //Some apps need a few seconds before they shutdown. Here's a nice place to do it diff --git a/src/faodel-common/BootstrapImplementation.hh b/src/faodel-common/BootstrapImplementation.hh index c2bf3df..ef513d3 100644 --- a/src/faodel-common/BootstrapImplementation.hh +++ b/src/faodel-common/BootstrapImplementation.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_BOOTSTRAPINTERNAL_HH #define FAODEL_COMMON_BOOTSTRAPINTERNAL_HH @@ -56,7 +56,7 @@ public: bool CheckDependencies(std::string *info_message=nullptr); std::vector GetStartupOrder(); - void Init(const Configuration &config); + bool Init(const Configuration &config); void Start(); void Start(const Configuration &config); //Init and Start void Finish(bool clear_list_of_bootstrap_users); @@ -65,13 +65,21 @@ public: bool IsStarted() const { return state==State::STARTED; } Configuration GetConfiguration() const { return configuration; } + int GetNumberOfUsers() const; + bool HasComponent(const std::string &component_name) const; + void dumpStatus() const; void dumpInfo(faodel::ReplyStream &rs) const; private: + + void finish_(bool clear_list_of_bootstrap_users); + Configuration configuration; + bool show_config_at_init; bool halt_on_shutdown; bool status_on_shutdown; + bool mpisyncstop_enabled; uint64_t sleep_seconds_before_shutdown; nodeid_t my_node_id; bool expandDependencies(std::map> &dep_lut, @@ -81,6 +89,9 @@ private: std::vector bstraps; + faodel::MutexWrapper *state_mutex; //!< Protects num_init_callers and State + int num_init_callers; //!< Number of entities that have called init (or start) + enum class State { UNINITIALIZED, INITIALIZED, STARTED }; State state; diff --git a/src/faodel-common/BootstrapInterface.hh b/src/faodel-common/BootstrapInterface.hh index 6962b42..aa96ba9 100644 --- a/src/faodel-common/BootstrapInterface.hh +++ b/src/faodel-common/BootstrapInterface.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_BOOTSTRAPINTERFACE_HH #define FAODEL_COMMON_BOOTSTRAPINTERFACE_HH diff --git a/src/faodel-common/Bucket.cpp b/src/faodel-common/Bucket.cpp index 0be50ca..d1ac4dd 100644 --- a/src/faodel-common/Bucket.cpp +++ b/src/faodel-common/Bucket.cpp @@ -1,9 +1,8 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include -#include #include diff --git a/src/faodel-common/Bucket.hh b/src/faodel-common/Bucket.hh index a23d544..1970cf4 100644 --- a/src/faodel-common/Bucket.hh +++ b/src/faodel-common/Bucket.hh @@ -1,11 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_BUCKET_HH #define FAODEL_COMMON_BUCKET_HH #include +#include #include "faodel-common/FaodelTypes.hh" @@ -46,18 +47,12 @@ const bucket_t BUCKET_UNSPECIFIED(0, internal_use_only); /** * @brief An exception used to deal with bad bucket string parsing */ -class BucketParseError : public std::exception { -private: - std::string msg; +class BucketParseError : public std::runtime_error { public: - BucketParseError() : msg("") {} - BucketParseError(std::string s) : msg(s) {} - - const char* what() const throw() override { - std::stringstream ss; - ss << "Format problem while parsing Bucket string: " << msg << std::endl; - return ss.str().c_str(); - } + BucketParseError() + : std::runtime_error( "Format problem while parsing Bucket string" ) {} + explicit BucketParseError( const std::string& s ) + : std::runtime_error( "Format problem while parsing Bucket string: " + s ) {} }; } // namespace faodel diff --git a/src/faodel-common/CMakeLists.txt b/src/faodel-common/CMakeLists.txt index ff0de5d..38b6ab7 100644 --- a/src/faodel-common/CMakeLists.txt +++ b/src/faodel-common/CMakeLists.txt @@ -18,7 +18,8 @@ set(HEADERS QuickHTML.hh ReplyStream.hh ResourceURL.hh - SerializationHelpers.hh + SerializationHelpersBoost.hh + SerializationHelpersCereal.hh StringHelpers.hh ) @@ -42,11 +43,17 @@ set(SOURCES StringHelpers.cpp ) +if(Faodel_ENABLE_MPI_SUPPORT) + LIST( APPEND Common_imports MPI::MPI_CXX) +endif() +LIST( APPEND Common_imports tpl-cereal ) + + add_library( common ${HEADERS} ${SOURCES} ) set_target_properties( common PROPERTIES OUTPUT_NAME "faodel-common" ) -target_link_libraries( common sbl ) +target_link_libraries( common PUBLIC sbl ${Common_imports}) diff --git a/src/faodel-common/Common.hh b/src/faodel-common/Common.hh index f398e25..61a290c 100644 --- a/src/faodel-common/Common.hh +++ b/src/faodel-common/Common.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_HH #define FAODEL_COMMON_HH diff --git a/src/faodel-common/Configuration.cpp b/src/faodel-common/Configuration.cpp index e080e52..74fa723 100644 --- a/src/faodel-common/Configuration.cpp +++ b/src/faodel-common/Configuration.cpp @@ -1,18 +1,16 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include #include #include -#include #include #include #include -#include "faodel-common/Debug.hh" #include "faodel-common/Configuration.hh" #include "faodel-common/StringHelpers.hh" @@ -22,6 +20,10 @@ using namespace std; namespace faodel { namespace configlog { namespace internal { + +//The AppendRequestedGet captures all the config.Get options that were requested in the +//application so people can learn what options are being queries. + map> config_values; } //internal void AppendRequestedGet(const string &field, const string &option_type, const string &default_value) { @@ -53,8 +55,8 @@ namespace faodel { /** * @brief Parse a user-supplied Configuration * @param[in] configuration_string A multi-line string with different config settings in it - * @param[in] env_variable_for_extra_settings Name of an envionrment variable that points to a file with additional settings - * @note While the Envionment Variable is set here, the file is NOT LOADED here. You need to call + * @param[in] env_variable_for_extra_settings Name of an environment variable that points to a file with additional settings + * @note While the Environment Variable is set here, the file is NOT LOADED here. You need to call * AppendFromReferences, or have bootstrap::Init load it. * @note The default environment variable name is FAODEL_CONFIG */ @@ -80,7 +82,7 @@ Configuration::~Configuration() = default; * @param[in] config_str One or more lines of configuration data (separated by \n) * @retval 0 Always works */ -int Configuration::Append(const string &config_str) { +rc_t Configuration::Append(const string &config_str) { addConfigToMap(config_str); return 0; } @@ -90,11 +92,22 @@ int Configuration::Append(const string &config_str) { * @param[in] val String value to set * @retval 0 Always works */ -int Configuration::Append(const string &name, const string &val) { +rc_t Configuration::Append(const string &name, const string &val) { //cout <<"Config Append '"< search_list = { node_role+"."+lname, + "default."+lname, + lname}; + for(auto &s : search_list) { + auto it=config_map.find(s); + if(it!=config_map.end()) { + if(val) *val = it->second; + return 0; + } + } + if(val) *val=default_value; + return ENOENT; +} + /** * @brief Search configuration data and return string of value, or default if not found * @param[out] val Value that was found, or default value @@ -318,37 +350,9 @@ bool Configuration::Contains(const std::string &name) const { rc_t Configuration::GetString(string *val, const string &name, const string &default_value) const { - //Convert to lowercase - string lname=ToLowercase(name); - - configlog::AppendRequestedGet(name, "string", default_value); - - map::const_iterator it; - - //First, look for something defined for our specific node_role - string name2 = node_role + "." + lname; - it = config_map.find(name2); - if(it==config_map.end()) { - - //Node role not here, try default.name - string name3 = "default."+lname; - it = config_map.find(name3); - if(it==config_map.end()) { - - //default not here, try bare name - it = config_map.find(lname); - if(it==config_map.end()) { - //Not here either, bail out with default val - if(val) *val=default_value; - return ENOENT; - } - } - } - //Pass back to the user - if(val) *val = it->second; - - return 0; + return findBestMatch(val, name, "string", default_value); } + /** * @brief Search configuration data and return lowercase string of value, or default if not found * @param[out] val Lowercase version of value that was found, or default value @@ -365,7 +369,6 @@ rc_t Configuration::GetLowercaseString(string *val, const string &name, const st return rc; } - /** * @brief Search through configuration and return value if found. Otherwise return default value @@ -378,38 +381,13 @@ rc_t Configuration::GetLowercaseString(string *val, const string &name, const st */ rc_t Configuration::GetInt(int64_t *val, const string &name, const string &default_value) const { - configlog::AppendRequestedGet(name, "int", default_value); - - string result = default_value; - int rc=ENOENT; - - //Convert to lowercase - string lname = ToLowercase(name); - map::const_iterator it; - - //See if our node_role has something specified - string name2 = node_role + "."+lname; - it = config_map.find(name2); - if(it==config_map.end()) { - - //Nope. Try default.name - string name3 = "default."+lname; - it = config_map.find(name3); - if(it==config_map.end()) { - //Nope. Try original name - it = config_map.find(lname); - } - } - - //If hit on something, get the result - if(it!=config_map.end()) { - result = it->second; - rc=0; - } - int rc2 = StringToInt64(val, result); + std::string tmp; + int rc = findBestMatch(&tmp, name, "int", default_value); + int rc2 = StringToInt64(val, tmp); if(rc2 != 0) return rc2; return rc; + } /** @@ -425,103 +403,64 @@ rc_t Configuration::GetInt(int64_t *val, const string &name, rc_t Configuration::GetUInt(uint64_t *uval, const string &name, const string &default_value) const { - configlog::AppendRequestedGet(name, "int", default_value); - - string result = default_value; - int rc=ENOENT; - - //Convert to lowercase - string lname = ToLowercase(name); - - map::const_iterator it; - - //See if our node_role has something specified - string name2 = node_role + "."+lname; - it = config_map.find(name2); - if(it==config_map.end()) { - - //Nope. Try default.name - string name3 = "default."+lname; - it = config_map.find(name3); - if(it==config_map.end()) { - //Nope. Try original name - it = config_map.find(lname); - } - } + std::string tmp; + int rc = findBestMatch(&tmp, name, "uint", default_value); - //If hit on something, get the result - if(it!=config_map.end()) { - result = it->second; - rc=0; - } - //Convert to signed integer to see if this is a negative number which is an error + //Make sure this isn't actually a negative value int64_t ival; - int rc2 = StringToInt64(&ival, result); + int rc2 = StringToInt64(&ival, tmp); if(rc2 != 0) return rc2; if (ival < 0) { if(uval) *uval = 0; return EINVAL; } //Number is positive, go ahead and do uint64 conversion - int rc3 = StringToUInt64(uval, result); + int rc3 = StringToUInt64(uval, tmp); if(rc3 != 0) return rc3; return rc; + } /** - * @brief Search through configuration and return value if found. Otherwise return default value + * @brief Search through configuration and return value if found. Otherwise + return default value + * @param[out] us_val Value that was found (in microseconds) * @param[in] name Keyword in configuration to look for - * @param[out] val Value that was found - * @param[in] default_value A default value to use if not found (parsed) + * @param[in] default_value A default value string to return if not found (parsed) * @retval 0 If Found value in configuration * @retval ENOENT Data wasn't found, using default value - * @retval EINVAL Data or default value could not be parsed to Int + * @retval EINVAL Data or default value could not be parsed to UInt */ -rc_t Configuration::GetBool(bool *val, const string &name, +rc_t Configuration::GetTimeUS(uint64_t *us_val, const string &name, const string &default_value) const { - configlog::AppendRequestedGet(name, "bool", default_value); - - string result = default_value; - rc_t rc=ENOENT; - - //Convert to lowercase - string lname = ToLowercase(name); + std::string tmp; + int rc = findBestMatch(&tmp, name, "timeUS", default_value); + int rc2 = StringToTimeUS(us_val, tmp); + if(rc2 != 0) return rc2; + return rc; - map::const_iterator it; +} - //See if our node_role has something specified - string name2 = node_role + "."+lname; - it = config_map.find(name2); - if(it==config_map.end()) { - //Nope. try default.name - string name3 = "default."+lname; - it = config_map.find(name3); - if(it==config_map.end()) { - //Nope. Try original name - it = config_map.find(lname); - } - } - - //If hit on something, get the result - if(it!=config_map.end()) { - result = it->second; - rc=0; - } else { - result = default_value; - } +/** + * @brief Search through configuration and return value if found. Otherwise return default value + * @param[in] name Keyword in configuration to look for + * @param[out] val Value that was found + * @param[in] default_value A default value to use if not found (parsed) + * @retval 0 If Found value in configuration + * @retval ENOENT Data wasn't found, using default value + * @retval EINVAL Data or default value could not be parsed to Bool + */ +rc_t Configuration::GetBool(bool *val, const string &name, + const string &default_value) const { - ToLowercaseInPlace(result); - if((result == "true") || (result == "t") || (result == "1")) { - if(val) *val = true; - } else if((result == "false") || (result == "f") || (result == "0")) { - if(val) *val = false; - } else { - //Didn't parse. - rc=EINVAL; - } + std::string tmp; + int rc = findBestMatch(&tmp, name, "bool", default_value); + int rc2 = StringToBoolean(val, tmp); + if(rc2 != 0) return rc2; return rc; + } /** @@ -538,41 +477,14 @@ rc_t Configuration::GetBool(bool *val, const string &name, rc_t Configuration::GetPtr(void **val, const string &name, void *default_value) const { - configlog::AppendRequestedGet(name, "ptr", "(todo)"); - - string result; - int rc=ENOENT; - - //Convert to lowercase - string lname = ToLowercase(name); - - map::const_iterator it; - - // If we don't find "name", fallback to "default_value". - if(val) *val = default_value; - - //See if our node_role has something specified - string name2 = node_role + "."+lname; - it = config_map.find(name2); - if(it==config_map.end()) { - - //Nope. Try default.name - string name3 = "default."+lname; - it = config_map.find(name3); - if(it==config_map.end()) { - //Nope. Try original name - it = config_map.find(lname); - } + std::string tmp; + int rc = findBestMatch(&tmp, name, "ptr", ""); + if(rc==ENOENT) { + if(val) *val=default_value; + return ENOENT; } - - //If hit on something, get the result - if(it!=config_map.end()) { - result = it->second; - rc=0; - int rc2 = StringToPtr(val, result); - if(rc2 != 0) return rc2; - } - + int rc2 = StringToPtr(val, tmp); + if(rc2 != 0) return rc2; return rc; } @@ -655,7 +567,7 @@ rc_t Configuration::GetFilename(string *fname, const string &name, if(fname) *fname = expanded; return 0; } - throw std::runtime_error("Configuration shell expension failed for "+name+".file.env_name "+ename); + throw std::runtime_error("Configuration shell expansion failed for "+name+".file.env_name "+ename); } throw std::runtime_error("Configuration required "+name+".file.env_name "+ename+" but it was not set\n"); } @@ -767,22 +679,32 @@ map Configuration::GetComponentSettings(const string &component_n * @param[out] info_enabled either component.debug or component.log.info is true * @param[out] warn_enabled either component.debug or component.log.warn is true * @param[in] component_name Name of component to look up in Configuration - * @retval 0 + * @retval 0 Always succeeds */ rc_t Configuration::GetComponentLoggingSettings(bool *dbg_enabled, bool *info_enabled, bool *warn_enabled, const string &component_name) const { + bool tmp_dbg_enabled=false, tmp_info_enabled = false; + GetBool(&tmp_dbg_enabled, component_name+".debug", "false"); + GetBool(&tmp_info_enabled, component_name+".info", "false"); - //Allow user to do "component.debug" instead of "component.log.debug" - GetBool(dbg_enabled, component_name+".debug", "false"); + string default_setting="false"; //Everything defaults to off. - string default_setting="false"; - if((dbg_enabled!=nullptr) && (*dbg_enabled)) { + //If .debug is on, use it for everyone + if(tmp_dbg_enabled) { default_setting="true"; + tmp_info_enabled=true; //Debug triggers info } - - //But still trust log.debug as an override. + //..but allow the log.debug to override GetBool(dbg_enabled, component_name+".log.debug", default_setting); + + //2. Next, do info. Force on if debug was on, or info was set to on + if(tmp_info_enabled) { + default_setting="true"; + } + //..but allow log.info to override GetBool(info_enabled, component_name+".log.info", default_setting); + + //3. Last do warn. Turn on if either the debug/info are on, but allow log.warn to override GetBool(warn_enabled, component_name+".log.warn", default_setting); return 0; } @@ -790,17 +712,17 @@ rc_t Configuration::GetComponentLoggingSettings(bool *dbg_enabled, bool *info_en /** * @brief Get entire list of configuration settings as a string table for debug * @param[out] results A vector of k/v pairs - * @returns 0 + * @retval 0 Success + * @retval EINVAL Did not provide a valid pointer for results */ rc_t Configuration::GetAllSettings(vector> *results) const { - results->push_back(pair("node_role", GetRole())); - if(results){ - for(auto &it : config_map) { - results->push_back(pair(it.first, it.second)); - } - } - return 0; + if(!results) return EINVAL; + results->push_back(pair("node_role", GetRole())); + for(auto &it : config_map) { + results->push_back(pair(it.first, it.second)); + } + return 0; } /** @@ -812,7 +734,7 @@ rc_t Configuration::GetAllSettings(vector> *results) const { vector Configuration::tokenizeLine(const string &str) { vector tokens; - size_t p0 = 0, p1 = string::npos; + size_t p0 = 0, p1; size_t endpos; endpos = str.find_first_of('#'); @@ -833,7 +755,7 @@ vector Configuration::tokenizeLine(const string &str) { * @param[in] config Input string to parse * @returns number of items found */ -int Configuration::addConfigToMap(string config) { +int Configuration::addConfigToMap(const string &config) { stringstream ss(config); int items_found=0; @@ -888,14 +810,12 @@ rc_t Configuration::GetDefaultSecurityBucket(string *bucket_name) const { for(int i=0; !names[i].empty(); i++) { map::const_iterator jj; - //cout <<"Searching for "<<*ii<second; break; } } - //cout << "found bucket name: '"< tokenizeLine(const std::string &str); //Parse a line - int addConfigToMap(std::string config); //Parse all the lines in a string + int findBestMatch(std::string *value, const std::string &name, const std::string &type_label, const std::string &default_value) const; + + static std::vector tokenizeLine(const std::string &str); //Parse a line + int addConfigToMap(const std::string &config); //Parse all the lines in a string std::map config_map; //Holds all the config k/vs diff --git a/src/faodel-common/Debug.cpp b/src/faodel-common/Debug.cpp index 81eb763..11fa021 100644 --- a/src/faodel-common/Debug.cpp +++ b/src/faodel-common/Debug.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -15,24 +15,29 @@ using namespace std; namespace faodel { -bool _kassert_dont_die=false; //For use in testing -bool _kassert_quiet=false; //For use in testing -int _kfail_count=0; //For use in testing -void _kassert(bool true_or_die, std::string message, const char *file, int line) { + +void _f_assert(bool true_or_die, const std::string &message, const char *file, int line) { + if(true_or_die) return; - _kfail_count++; - if(!_kassert_quiet) cout <<"KAssert: "< #include +#include "faodelConfig.h" + + +// In some debugging scenarios it helps to have some alternate ways of +// dealing with assertions. The motivation here is: +// +// cassert (default): just handle with standard assert macro (drops the message) +// debugHalt: Dump the message and halt. Useful for whookie postmortem +// debugExit: Dump the message and exit. +// debugWarn: Dump the message, but continue on. See what else breaks. +#if Faodel_ASSERT_METHOD_NONE +#define F_ASSERT(a,msg) {} +#elif (Faodel_ASSERT_METHOD_DEBUG_WARN || Faodel_ASSERT_METHOD_DEBUG_HALT || Faodel_ASSERT_METHOD_DEBUG_EXIT) +#define F_ASSERT(a,msg) { faodel::_f_assert(a, msg, __FILE__, __LINE__); } +#else +// default to using plain assert +#include +#define F_ASSERT(a,msg) { assert(a); } +#endif + + namespace faodel { -void _kassert(bool true_or_die, std::string message, const char *file, int line); -void _khalt(std::string message, const char *file, int line); +void _f_assert(bool true_or_die, const std::string &message, const char *file, int line); +void _f_halt(const std::string &message, const char *file, int line); + +const std::string TXT_RED = "\033[1;31m"; +const std::string TXT_WARN = "\033[1;97;44m"; +const std::string TXT_NORMAL = "\033[0m"; -// Debug: make halt spots easier to identify -#define KHALT(msg) { faodel::_khalt(msg, __FILE__, __LINE__); } -#define KFAIL(rc) { printf("Fail at %s line %d\n",__FILE__, __LINE__); exit(-1); } -#define KWARN(a) { static bool _warned=false; if(!_warned) { std::cout <<"WARNING: "<<(a)< #include @@ -31,7 +31,7 @@ DirectoryInfo::DirectoryInfo(faodel::ResourceURL new_url) : min_members(0) { if(!s_min_members.empty()) { int rc = faodel::StringToUInt32(&min_members, s_min_members); if(rc != 0) { - KWARN("DirectoryInfo had parse error when extracting 'min_members' from url '"+new_url.GetFullURL()+"'"); + F_WARN("DirectoryInfo had parse error when extracting 'min_members' from url '"+new_url.GetFullURL()+"'"); } } @@ -51,7 +51,7 @@ DirectoryInfo::DirectoryInfo(faodel::ResourceURL new_url) : min_members(0) { } } } else { - KWARN("DirectoryInfo had parse problem when extracting 'num' from url '"+new_url.GetFullURL()+"'"); + F_WARN("DirectoryInfo had parse problem when extracting 'num' from url '"+new_url.GetFullURL()+"'"); } } url=new_url; //Stripped of all imported details diff --git a/src/faodel-common/DirectoryInfo.hh b/src/faodel-common/DirectoryInfo.hh index 3773d39..f32b2b9 100644 --- a/src/faodel-common/DirectoryInfo.hh +++ b/src/faodel-common/DirectoryInfo.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_DIRECTORYINFO_HH #define OPBOX_DIRECTORYINFO_HH diff --git a/src/faodel-common/FaodelTypes.hh b/src/faodel-common/FaodelTypes.hh index d88a7fc..d896032 100644 --- a/src/faodel-common/FaodelTypes.hh +++ b/src/faodel-common/FaodelTypes.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_TYPES_HH #define FAODEL_COMMON_TYPES_HH diff --git a/src/faodel-common/InfoInterface.hh b/src/faodel-common/InfoInterface.hh index 418387f..b989ea3 100644 --- a/src/faodel-common/InfoInterface.hh +++ b/src/faodel-common/InfoInterface.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_INFOINTERFACE_HH #define FAODEL_COMMON_INFOINTERFACE_HH diff --git a/src/faodel-common/LoggingInterface.cpp b/src/faodel-common/LoggingInterface.cpp index fbcc861..27a899e 100644 --- a/src/faodel-common/LoggingInterface.cpp +++ b/src/faodel-common/LoggingInterface.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/src/faodel-common/LoggingInterface.hh b/src/faodel-common/LoggingInterface.hh index 39779e9..6cf08cb 100644 --- a/src/faodel-common/LoggingInterface.hh +++ b/src/faodel-common/LoggingInterface.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_LOGGINGITERFACE_HH #define FAODEL_COMMON_LOGGINGITERFACE_HH diff --git a/src/faodel-common/LoggingInterfaceMacros.hh b/src/faodel-common/LoggingInterfaceMacros.hh index 5e341f0..ad823ff 100644 --- a/src/faodel-common/LoggingInterfaceMacros.hh +++ b/src/faodel-common/LoggingInterfaceMacros.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_LOGGINGITERFACEMACROS_HH #define FAODEL_COMMON_LOGGINGITERFACEMACROS_HH diff --git a/src/faodel-common/MutexWrapper.cpp b/src/faodel-common/MutexWrapper.cpp index cd0c5be..602e3e7 100644 --- a/src/faodel-common/MutexWrapper.cpp +++ b/src/faodel-common/MutexWrapper.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -166,7 +166,7 @@ MutexWrapperTypeID GetMutexTypeID(std::string threading_model, std::string mutex } #endif - kassert(false, "Unable to resolve Mutex Wrapper for threading model/type "+threading_model+"/"+mutex_type+".\n" + F_ASSERT(false, "Unable to resolve Mutex Wrapper for threading model/type "+threading_model+"/"+mutex_type+".\n" " library may not have right compile flags (eg, -lpthread)\n"); return MutexWrapperTypeID::ERROR; @@ -191,8 +191,8 @@ MutexWrapper * GenerateMutexByTypeID(MutexWrapperTypeID mutex_type_id) { default: ; } //Here by mistake - kassert(false, "Unable to resolve Mutex Wrapper "+to_string(mutex_type_id)+"\n"+ - " library may not have right compile flags (eg, -lpthread)\n"); + F_ASSERT(false, "Unable to resolve Mutex Wrapper "+to_string(mutex_type_id)+"\n"+ + " library may not have right compile flags (eg, -lpthread)\n"); return nullptr; } diff --git a/src/faodel-common/MutexWrapper.hh b/src/faodel-common/MutexWrapper.hh index 6005ba5..68024a3 100644 --- a/src/faodel-common/MutexWrapper.hh +++ b/src/faodel-common/MutexWrapper.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_MUTEXWRAPPER_HH #define FAODEL_COMMON_MUTEXWRAPPER_HH diff --git a/src/faodel-common/NodeID.cpp b/src/faodel-common/NodeID.cpp index 83d3997..1ff983e 100644 --- a/src/faodel-common/NodeID.cpp +++ b/src/faodel-common/NodeID.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/src/faodel-common/NodeID.hh b/src/faodel-common/NodeID.hh index 818d97c..70e7d65 100644 --- a/src/faodel-common/NodeID.hh +++ b/src/faodel-common/NodeID.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_NODEID_HH #define FAODEL_COMMON_NODEID_HH @@ -9,6 +9,7 @@ #include #include +#include #include #include "faodel-common/FaodelTypes.hh" @@ -79,19 +80,12 @@ const nodeid_t NODE_UNSPECIFIED(0x00, internal_use_only); //!< For designating t /** * @brief An exception for capturing error info when parsing a nodeid string */ -class NodeIDParseError : public std::exception { -private: - std::string msg; +class NodeIDParseError : public std::runtime_error { public: - NodeIDParseError() : msg("") {} - - explicit NodeIDParseError(std::string s) : msg(std::move(s)) {} - - const char* what() const throw() override { - std::stringstream ss; - ss << "Format problem while parsing NodeID string: " << msg << std::endl; - return ss.str().c_str(); - } + NodeIDParseError() + : std::runtime_error( "Format problem while parsing NodeID string" ) {} + explicit NodeIDParseError( const std::string& s ) + : std::runtime_error( "Format problem while parsing NodeID string: " + s ) {} }; diff --git a/src/faodel-common/QuickHTML.cpp b/src/faodel-common/QuickHTML.cpp index 5a3fae1..1a3f463 100644 --- a/src/faodel-common/QuickHTML.cpp +++ b/src/faodel-common/QuickHTML.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "faodel-common/QuickHTML.hh" diff --git a/src/faodel-common/QuickHTML.hh b/src/faodel-common/QuickHTML.hh index 5fd355d..1bba36f 100644 --- a/src/faodel-common/QuickHTML.hh +++ b/src/faodel-common/QuickHTML.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_QUICKHTML_HH #define FAODEL_COMMON_QUICKHTML_HH diff --git a/src/faodel-common/README_Common.md b/src/faodel-common/README_Common.md index bb93e1b..20c315a 100644 --- a/src/faodel-common/README_Common.md +++ b/src/faodel-common/README_Common.md @@ -80,11 +80,17 @@ Run-Time Options The following options are parsed out of a Configuration by different components in Common: -| Property | Type | Default | Description | -| ------------------ | ----------- | ------- | -------------------------------------------- | -| bootstrap.debug | boolean | false | Display debug messages during bootstrap | -| config.additional_files | string list | "" | Load additional info for listed files | -| config.purge | boolean | false | Removes all tags and reloads current config | +| Property | Type | Default | Description | +| -------------------------------------- | ----------- | ------- | -------------------------------------------- | +| bootstrap.debug | boolean | false | Display debug messages during bootstrap | +| bootstrap.show_config_at_init | boolean | false | Display config used at init time | +| bootstrap.halt_on_shutdown | boolean | false | Do infinite-wait instead of exit at shutdown | +| bootstrap.status_on_shutdown | boolean | false | Dump an ok message on successful exit | +| bootstrap.exit_on_errors | boolean | false | Exit on errors instead of throwing exception | +| bootstrap.sleep_seconds_before_shutdown | int | 0 | Delay Finish shutdown for specified seconds | +| mpisyncstop.enable | boolean | false | Perform an mpi barrier before Finish | +| config.additional_files | string list | "" | Load additional info for listed files | +| config.purge | boolean | false | Removes all tags and reloads current config | Note: There are multiple suffixes for config.additional_files that can be used to control how additional config files are loaded: diff --git a/src/faodel-common/ReplyStream.cpp b/src/faodel-common/ReplyStream.cpp index 7e9184d..e6502b4 100644 --- a/src/faodel-common/ReplyStream.cpp +++ b/src/faodel-common/ReplyStream.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -103,6 +103,19 @@ void ReplyStream::mkText(string text) { exit(-1); } } +void ReplyStream::mkPlainText(string text) { + switch(format) { + case ReplyStreamType::TEXT: + *ss << text; + break; + case ReplyStreamType::HTML: + *ss << "

" << text << "
\n"; + break; + default: + cerr << "Unsupported format in ReplyStream\n"; + exit(-1); + } +} /** * @brief Insert a 2D table into the stream and give it a label diff --git a/src/faodel-common/ReplyStream.hh b/src/faodel-common/ReplyStream.hh index eb610e8..cff15a6 100644 --- a/src/faodel-common/ReplyStream.hh +++ b/src/faodel-common/ReplyStream.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_REPLYSTREAM_HH #define FAODEL_COMMON_REPLYSTREAM_HH @@ -34,6 +34,7 @@ public: void mkSection(std::string label, int heading_level=1); void mkText(std::string text); + void mkPlainText(std::string text); void mkTable(const std::vector> entries, std::string label="", bool highlight_top=true); void mkTable(const std::map entries, std::string label="", bool highlight_top=true); void mkTable(const std::vector> entries, std::string label="", bool highlight_top=true); @@ -51,6 +52,9 @@ public: std::string createBold(std::string text); void Finish(); + + bool IsHTML() { return (format==ReplyStreamType::HTML);} + bool IsText() { return (format==ReplyStreamType::TEXT);} private: ReplyStreamType format; diff --git a/src/faodel-common/ResourceURL.cpp b/src/faodel-common/ResourceURL.cpp index 422cadd..b603be5 100644 --- a/src/faodel-common/ResourceURL.cpp +++ b/src/faodel-common/ResourceURL.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -48,7 +48,7 @@ namespace faodel { * @param[in] url The string version of a url * @return ResourceURL */ -ResourceURL::ResourceURL(string url) +ResourceURL::ResourceURL(const string &url) : reference_node(NODE_UNSPECIFIED), bucket(BUCKET_UNSPECIFIED) { string tmp_bucket, tmp_nodeid; @@ -67,7 +67,7 @@ ResourceURL::ResourceURL(string url) * @retval 0 - Parsed fine * @retval EINVAL - todo- other error codes */ -rc_t ResourceURL::SetURL(const std::string& url) { +rc_t ResourceURL::SetURL(const std::string &url) { std::string temp_bucket, temp_nodeid; auto rc = ParseURL( url, &resource_type, &temp_bucket, &temp_nodeid, &path, &name, &options ); if( rc == 0 ) { @@ -107,7 +107,7 @@ string ResourceURL::GetURL(bool include_type, bool include_node, bool include_bu if(path!="/") ss< ops; string sname=option_name+"="; Split(ops, options, '&', true); @@ -448,7 +461,7 @@ string ResourceURL::GetOption(string option_name) const { return full_op.substr(sname.size()); } } - return ""; + return default_value; } /** @@ -491,7 +504,7 @@ vector> ResourceURL::GetOptions() const { * @param[in] option_name - The option name to remove * @return string - The value for the last option removed or "" */ -string ResourceURL::RemoveOption(string option_name) { +string ResourceURL::RemoveOption(const string &option_name) { vector ops; string removed_val=""; string sname=option_name+"="; diff --git a/src/faodel-common/ResourceURL.hh b/src/faodel-common/ResourceURL.hh index 4788a0f..5e55e10 100644 --- a/src/faodel-common/ResourceURL.hh +++ b/src/faodel-common/ResourceURL.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_RESOURCEURL_HH #define FAODEL_COMMON_RESOURCEURL_HH @@ -47,10 +47,13 @@ class ResourceURL : public InfoInterface { public: ResourceURL() = default; //Necessary for placeholders - explicit ResourceURL(std::string url); - ResourceURL(std::string resource_type, nodeid_t reference_node, + explicit ResourceURL(const std::string &url); + ResourceURL(const std::string &resource_type, + nodeid_t reference_node, bucket_t bucket, - std::string path, std::string name, std::string options) + const std::string &path, + const std::string &name, + const std::string &options) : reference_node(reference_node), bucket(bucket), path((path.empty())?"/":path), name(name), options(options), @@ -72,12 +75,14 @@ public: bool IsEmpty() const; bool IsReference() const { return resource_type.empty(); } //!< True if this is a reference to a resource (ie ref:) - rc_t SetURL( const std::string& url ); + rc_t SetURL(const std::string &url); std::string GetURL(bool include_type=false, bool include_node=false, bool include_bucket=false, bool include_options=false) const; std::string GetPathName() const { return GetURL(false,false,false,false); } //!< Get the path/name: /root/rack0/mydht std::string GetBucketPathName() const { return GetURL(false,false,true,false); } //!< Get Bucket/path name: [a23]/root/rack0/mydht std::string GetFullURL() const { return GetURL(true,true,true,true); } //!< Get full encoding "[a23]/root/rack0/mydht&min_members=2&thing=4" + std::string Dashify() const; + //Manipulate paths void PushDir(std::string next_dir); std::string PopDir(); @@ -87,9 +92,9 @@ public: int GetPathDepth() const; - void SetOption(std::string option_name, std::string value); - std::string RemoveOption(std::string option_name); - std::string GetOption(std::string option_name) const; + void SetOption(const std::string &option_name, const std::string &value); + std::string RemoveOption(const std::string &option_name); + std::string GetOption(const std::string &option_name, const std::string &default_value="") const; std::string GetSortedOptions() const; std::vector< std::pair > GetOptions() const; diff --git a/src/faodel-common/SerializationHelpersBoost.hh b/src/faodel-common/SerializationHelpersBoost.hh index a307a51..c825e17 100644 --- a/src/faodel-common/SerializationHelpersBoost.hh +++ b/src/faodel-common/SerializationHelpersBoost.hh @@ -1,17 +1,49 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_SERIALIZATION_HELPERS_HH #define FAODEL_COMMON_SERIALIZATION_HELPERS_HH #include +// These are helper functions to make it easier to use Boost or Cereal to +// serialize a class to string that can be shipped around. Boost requires +// and external library. Cereal is a header-only library included in +// faodel's tpls. Of the two, Cereal has been faster in nearly all of our +// tests. Both require the user to provide a serialize template in their +// class definition that defines the order in which items are serialized. +// +// An example class might look like this: +// +// class foo { +// public: +// vector stuff1; +// int x2; +// +// template +// void serialize(Archive &ar, const unsigned int version) { +// ar & stuff1; +// ar & x2; +// } +// }; +// +// +// Note: Make sure to include the proper boost/cereal data type header +// files for your data structure. If you don't, you'll get a lot of +// hard to figure out template errors. +// +// eg: +// boost: #include +// cereal: #include +// cereal: #include +// + + //For boostPack()/boostUnpack() #include #include - namespace faodel { template @@ -37,4 +69,6 @@ T BoostUnpack(const std::string &packed) { } // namespace faodel + + #endif // FAODEL_COMMON_SERIALIZATION_HELPERS_HH diff --git a/src/faodel-common/SerializationHelpersCereal.hh b/src/faodel-common/SerializationHelpersCereal.hh new file mode 100644 index 0000000..efd7af8 --- /dev/null +++ b/src/faodel-common/SerializationHelpersCereal.hh @@ -0,0 +1,42 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_SERIALIZATIONHELPERSCEREAL_HH +#define FAODEL_SERIALIZATIONHELPERSCEREAL_HH + + +// Note: this is separated out from the normal SerializationHelpers.hh +// because faodel does not include cereal when it does its +// installation (to avoid conflict with existing libs). + + +//For CerealPack()/CerealUnpack() +#include + +namespace faodel { + +template +std::string CerealPack(const T &t1) { + std::stringstream ss; + { //<--Important, put archive in {} to finalize last write at end + cereal::BinaryOutputArchive oarchive(ss); //Binary archive + oarchive(t1); + } + return ss.str(); +} + +template +T CerealUnpack(const std::string &packed) { + T t1; + std::istringstream astream(packed); + { + cereal::BinaryInputArchive iarchive(astream); + iarchive(t1); + } + return t1; +} + +} // namespace faodel + +#endif //FAODEL_SERIALIZATIONHELPERSCEREAL_HH diff --git a/src/faodel-common/StringHelpers.cpp b/src/faodel-common/StringHelpers.cpp index e592f1f..ec5ab36 100644 --- a/src/faodel-common/StringHelpers.cpp +++ b/src/faodel-common/StringHelpers.cpp @@ -1,13 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include #include #include #include -#include #include #include @@ -58,7 +57,7 @@ string ExpandPunycode(string const &s) { if( isxdigit(c[i]) && isxdigit(c[i+1]) ) { - unsigned char tmp=0; + unsigned char tmp; if (isdigit(c[i])) tmp = c[i] - '0'; else if(islower(c[i])) tmp = 10 + c[i] - 'a'; else tmp = 10 + c[i] - 'A'; @@ -86,14 +85,14 @@ bool IsValidIPString(const string &hostname) { bool all_digits=true; bool has_digits=false; - for(auto s : octets) { - if(s=="") return false; + for(auto &s : octets) { + if(s.empty()) return false; char *p; long val = strtol(s.c_str(), &p,10); if(*p) { //Couldn't parse.. not a digit all_digits=false; - } else if((val>=0) || (val<=255)) { + } else if((val>=0) && (val<=255)) { has_digits=true; } else { has_digits=true; @@ -109,14 +108,14 @@ bool IsValidIPString(const string &hostname) { /** * @brief Convert a numerical string (eg "100" "4K") into an int32 value (eg 100, 4096) * @param[out] val The output variable to set - * @param[in] name Input string to parse + * @param[in] token Input string to parse * @retval 0 If input could be parsed * @retval EINVAL If input string couldn't be parsed */ -int StringToInt32(int32_t *val, const std::string &name) { - kassert(val,"Null val ptr handed to StringToInt32"); +int StringToInt32(int32_t *val, const std::string &token) { + F_ASSERT(val, "Null val ptr handed to StringToInt32"); int64_t val64; - int rc = StringToInt64(&val64, name); + int rc = StringToInt64(&val64, token); if(rc!=0) return rc; *val = (val64 & 0x0FFFFFFFFL); return 0; @@ -125,14 +124,14 @@ int StringToInt32(int32_t *val, const std::string &name) { /** * @brief Convert a numerical string (eg "100" "4K") into an uint32 value (eg 100, 4096) * @param[out] val The output variable to set - * @param[in] name Input string to parse + * @param[in] token Input string to parse * @retval 0 If input could be parsed * @retval EINVAL If input string couldn't be parsed */ -int StringToUInt32(uint32_t *val, const std::string &name) { - kassert(val,"Null val ptr handed to StringToUInt32"); +int StringToUInt32(uint32_t *val, const std::string &token) { + F_ASSERT(val, "Null val ptr handed to StringToUInt32"); uint64_t val64; - int rc = StringToUInt64(&val64, name); + int rc = StringToUInt64(&val64, token); if(rc!=0) return rc; *val = (val64 & 0x0FFFFFFFFL); return 0; @@ -141,13 +140,13 @@ int StringToUInt32(uint32_t *val, const std::string &name) { /** * @brief Convert a numerical string (eg "100" "4K") into an int64 value (eg 100, 4096) * @param[out] val The output variable to set - * @param[in] name Input string to parse + * @param[in] token Input string to parse * @retval 0 If input could be parsed * @retval EINVAL If input string couldn't be parsed */ -int StringToInt64(int64_t *val, string const &name) { - kassert(val,"Null val ptr handed to StringToInt64"); - string sname = name; +int StringToInt64(int64_t *val, string const &token) { + F_ASSERT(val, "Null val ptr handed to StringToInt64"); + string sname = token; int64_t multiplier=1; char last_char = *sname.rbegin(); if(!isdigit(last_char)) { @@ -156,7 +155,7 @@ int StringToInt64(int64_t *val, string const &name) { case'm': multiplier = 1024*1024; break; case'g': multiplier = 1024*1024*1024; break; default: - cerr << "Parsing problem decyphering " << name << " as an integer string\n"; + cerr << "Parsing problem decyphering " << token << " as an integer string\n"; return EINVAL; } } @@ -169,13 +168,13 @@ int StringToInt64(int64_t *val, string const &name) { /** * @brief Convert a numerical string (eg "100" "4K") into an uint64 value (eg 100, 4096) * @param[out] val The output variable to set - * @param[in] name Input string to parse + * @param[in] token Input string to parse * @retval 0 If input could be parsed * @retval EINVAL If input string couldn't be parsed */ -int StringToUInt64(uint64_t *val, string const &name) { - kassert(val,"Null val ptr handed to StringToUInt64"); - string sname = name; +int StringToUInt64(uint64_t *val, string const &token) { + F_ASSERT(val, "Null val ptr handed to StringToUInt64"); + string sname = token; int64_t multiplier=1; char last_char = *sname.rbegin(); if(!isdigit(last_char)) { @@ -184,7 +183,7 @@ int StringToUInt64(uint64_t *val, string const &name) { case'm': multiplier = 1024*1024; break; case'g': multiplier = 1024*1024*1024; break; default: - cerr << "Parsing problem decyphering " << name << " as an unsigned integer string\n"; + cerr << "Parsing problem decyphering " << token << " as an unsigned integer string\n"; return EINVAL; } } @@ -199,18 +198,87 @@ int StringToUInt64(uint64_t *val, string const &name) { return 0; } /** - * @brief Convert a numerical string into an pointer value + * @brief Convert a numerical string into a pointer value * @param[out] val The output variable to set - * @param[in] sval Input string to parse + * @param[in] token Input string to parse * @retval 0 If input could be parsed * @retval EINVAL If input string couldn't be parsed * @note This function is **not** commonly used and can be dangerous */ -int StringToPtr(void **val, string const &sval) { - kassert(val,"Null val ptr handed to StringToPtr"); +int StringToPtr(void **val, string const &token) { + F_ASSERT(val, "Null val ptr handed to StringToPtr"); //Pass back to the user - *val = (void*)strtoull(sval.c_str(), nullptr, 16); + *val = (void*)strtoull(token.c_str(), nullptr, 16); + + return 0; +} + +/** + * @brief Convert a string with a boolean flag into a value + * @param val The output + * @param token Input string to parse (valid options: true,false,1,0,t,f) + * @retval 0 If input token could be parsed + * @retval EINVAL if input token couldn't be parsed + */ +int StringToBoolean(bool *val, const string &token) { + if(token.empty()) { if(val) *val=false; return EINVAL; } + string ltoken = ToLowercase(token); + if((ltoken=="true") || (ltoken=="1") || (ltoken=="t")) { + if(val) *val = true; + return 0; + } + if((ltoken=="false") || (ltoken=="0") || (ltoken=="f")) { + if(val) *val = false; + return 0; + } + return EINVAL; +} + +/** + * @brief Convert a time string (with us,ms,minutes,hours,seconds,s suffixes) to a uint64 microsecond value + * @param val The output in microseconds + * @param token Input string to parse + * @retval 0 If input could be parsed + * @retval EINVAL if input string couldn't be parsed + */ +int StringToTimeUS(uint64_t *val, string const &token) { + F_ASSERT(val, "Null val ptr handed to StringToTimeUS"); + if(token.empty()) { *val=0; return EINVAL; } + + vector> table = { {"us", 1}, + {"ms", 1000}, + {"minutes", 60*1000*1000ul}, + {"hours", 3600*1000*1000ul}, + {"seconds", 1000*1000ul}, + {"s", 1000*1000ul} //note:make sure this is last since others end in s + }; + uint64_t multiplier=1; //Assume us if not specified + string sname = faodel::ToLowercase(token); + for(auto &name_mult : table) { + if(StringEndsWith(sname, name_mult.first)) { + multiplier = name_mult.second; + size_t last=sname.size()-name_mult.first.size(); + while((last>0) && (sname.at(last-1)==' ')) //trim spaces + last--; + sname = sname.substr(0, last); + break; + } + } + //Check for non-numbers + bool is_good= !sname.empty(); + for(int i=0; (is_good) && (i char + { + const char charset[] = + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + const size_t max_index = (sizeof(charset) - 1); + return charset[ rand() % max_index ]; + }; + std::string str(string_length,0); + std::generate_n( str.begin(), string_length, randchar ); + return str; +} + + /** * @brief Determine if string begins with a specific prefix * @@ -297,6 +383,28 @@ bool StringEndsWith(const std::string &s, const std::string &search_suffix) { search_suffix.begin()); } +/** + * @brief Convert an integer into a zero-padded string of a specified number of digits + * @param val The number to use + * @param to_num_digits The number of string digits to return + * @return Zero-padded string (eg, 93 zero padded to 5 digits becomes "00093") + */ +std::string StringZeroPad(int val, int to_num_digits) { + string s=std::to_string(val); + if(s.size() &settings) { + string p = GetItemFromComponentSettings("path", settings); + if((!p.empty()) && (p.back()!='/')) p=p+"/"; + return p; +} +std::string GetFileFromComponentSettings(const std::map &settings) { + return GetItemFromComponentSettings("file", settings); } + + void ConvertToHexDump(const char *x, ssize_t len, int chars_per_line, int grouping_size, - string even_prefix, string even_suffix, - string odd_prefix, string odd_suffix, + const string &even_prefix, const string &even_suffix, + const string &odd_prefix, const string &odd_suffix, vector *byte_offsets, vector *hex_lines, vector *txt_lines) { @@ -471,7 +622,7 @@ void ConvertToHexDump(const char *x, ssize_t len, int chars_per_line, if(txt_part) *txt_part = ss_txt.str(); } -void ConvertToHexDump(const string s, int chars_per_line, +void ConvertToHexDump(const string &s, int chars_per_line, string *hex_part, string *txt_part) { ConvertToHexDump(s.c_str(),s.size(), chars_per_line, hex_part, txt_part); @@ -526,6 +677,10 @@ uint32_t hash_dbj2(const bucket_t &bucket, const string &s) { uint32_t hash32(const std::string &s) { return hash_dbj2(s); } +uint16_t hash16(const std::string &s) { + uint32_t h = hash32(s); + return (h>>16) ^ (h & 0x0FFFF); +} /** * @brief Unpack a hash id from a packed string. String may be the value ("0x12345678") or a string to hash ("foo") @@ -604,7 +759,7 @@ int parseIDInRange(const std::string &token, int num_nodes) { * @return Set containing all the values * @thows runtime_error When invalid input */ -std::set ExtractIDs(const std::string line, int num_nodes) { +std::set ExtractIDs(const std::string &line, int num_nodes) { string s=ToLowercase(line); @@ -626,7 +781,7 @@ std::set ExtractIDs(const std::string line, int num_nodes) { //This is some range value, like 2-4 or 2-end int a = parseIDInRange(range_val[0], num_nodes); int b = parseIDInRange(range_val[1], num_nodes); - if((a>b) || (b<0) || (a>=num_nodes) || (b>=num_nodes) ||(a==b)) + if((a>b) || (b<0) || (a>=num_nodes) || (b>=num_nodes)) // ||(a==b)) throw std::runtime_error("ExtractID Range parse problem in token '"+t+"' for '"+line+"'"); for(int i=a; i<=b; i++) diff --git a/src/faodel-common/StringHelpers.hh b/src/faodel-common/StringHelpers.hh index 4effcc8..868c5d7 100644 --- a/src/faodel-common/StringHelpers.hh +++ b/src/faodel-common/StringHelpers.hh @@ -1,12 +1,13 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_STRINGHELPERS_HH #define FAODEL_COMMON_STRINGHELPERS_HH #include #include +#include #include #include "faodel-common/FaodelTypes.hh" @@ -18,14 +19,22 @@ std::string MakePunycode(std::string const &s); std::string ExpandPunycode(std::string const &s); bool IsValidIPString(const std::string &hostname); -int StringToInt32(int32_t *val, const std::string &name); -int StringToUInt32(uint32_t *val, const std::string &name); -int StringToInt64(int64_t *val, const std::string &name); -int StringToUInt64(uint64_t *val, const std::string &name); -int StringToPtr(void **val, const std::string &sval); + +//Note: These all convert k/m/g suffixes to x1024 etc +int StringToInt32(int32_t *val, const std::string &token); +int StringToUInt32(uint32_t *val, const std::string &token); +int StringToInt64(int64_t *val, const std::string &token); +int StringToUInt64(uint64_t *val, const std::string &token); +int StringToPtr(void **val, const std::string &token); +int StringToBoolean(bool *val, const std::string &name); +int StringToTimeUS(uint64_t *val, const std::string &token); + bool StringBeginsWith(const std::string &s, const std::string &search_prefix); bool StringEndsWith(const std::string &s, const std::string &search_suffix); +std::string StringZeroPad(int val, int to_num_digits); +std::string StringCenterTitle(const std::string &s); + std::vector Split(const std::string &text, char sep, bool remove_empty=true); void Split(std::vector &tokens, const std::string &text, char sep, bool remove_empty=true); @@ -33,6 +42,7 @@ std::string Join(const std::vector &tokens, char sep); std::string ToLowercase(std::string const &s); void ToLowercaseInPlace(std::string &s); +std::string RandomString(size_t string_length); std::vector SplitPath(std::string const &s); @@ -41,21 +51,27 @@ std::string ExpandPath(std::string const &s, int flags); std::string ExpandPath(std::string const &s); std::string ExpandPathSafely(std::string const &s); +//Helpers for parsing a set of matches from Configuration +std::string GetItemFromComponentSettings(const std::string &item, const std::map &settings); +std::string GetPathFromComponentSettings(const std::map &settings); +std::string GetFileFromComponentSettings(const std::map &settings); + -std::set ExtractIDs(const std::string line, int num_nodes); +std::set ExtractIDs(const std::string &line, int num_nodes); void ConvertToHexDump(const char *x, ssize_t len, int chars_per_line, int grouping_size, - std::string even_prefix, std::string even_suffix, - std::string odd_prefix, std::string odd_suffix, + const std::string &even_prefix, const std::string &even_suffix, + const std::string &odd_prefix, const std::string &odd_suffix, std::vector *byte_offsets, std::vector *hex_lines, std::vector *txt_lines); void ConvertToHexDump(const char *x, ssize_t len, int chars_per_line, std::string *hex_part, std::string *txt_part); -void ConvertToHexDump(const std::string s, int chars_per_line, std::string *hex_part, std::string *txt_part); +void ConvertToHexDump(const std::string &s, int chars_per_line, std::string *hex_part, std::string *txt_part); uint32_t hash_dbj2(const std::string &s); uint32_t hash_dbj2(const bucket_t &bucket, const std::string &s); uint32_t hash32(const std::string &s); //Maps to hash_dbj2 +uint16_t hash16(const std::string &s); //xors top and bottom of hash32 uint32_t UnpackHash32(const std::string &s); //Either extract 0x1234 or generate a hash for a string diff --git a/src/faodel-services/BackBurner.cpp b/src/faodel-services/BackBurner.cpp index ecfb4dc..7824843 100644 --- a/src/faodel-services/BackBurner.cpp +++ b/src/faodel-services/BackBurner.cpp @@ -1,11 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include #include #include +#include #include "faodel-services/BackBurner.hh" @@ -38,7 +39,7 @@ BackBurner::~BackBurner() { } void BackBurner::RegisterPollingFunction(const string &name, uint32_t group_id, fn_backburner_work polling_function) { - kassert(!configured, "BackBurner RegisterPollingFunction called after Start called"); + F_ASSERT(!configured, "BackBurner RegisterPollingFunction called after Start called"); workers->at(group_id%worker_count).RegisterPollingFunction(name, group_id, std::move(polling_function)); } @@ -55,7 +56,7 @@ void BackBurner::DisablePollingFunction(const string &name, uint32_t group_id) { } void BackBurner::Init(const Configuration &config) { - kassert(!configured, "BackBurner Init called twice"); + F_ASSERT(!configured, "BackBurner Init called twice"); ConfigureLogging(config); @@ -77,7 +78,7 @@ void BackBurner::Start() { } void BackBurner::Finish() { - kassert(configured, "Backburner Finish called when not in configured state"); + F_ASSERT(configured, "Backburner Finish called when not in configured state"); if(workers_launched) { workers_launched=false; } @@ -118,7 +119,7 @@ void BackBurner::AddWork(uint32_t tag, vector work) { BackBurner::Worker::Worker() : - LoggingInterface("backburnerWorker"), + LoggingInterface("backburner.worker"), parent(nullptr), worker_id(0), kill_worker(false), @@ -127,7 +128,7 @@ BackBurner::Worker::Worker() : } BackBurner::Worker::Worker(BackBurner *parent) : - LoggingInterface("backburnerWorker"), + LoggingInterface("backburner.worker"), parent(parent), tasks_consumer(&tasks_a), tasks_producer(&tasks_b), producer_num(0), consumer_num(0) { @@ -137,6 +138,7 @@ BackBurner::Worker::Worker(BackBurner *parent) : BackBurner::Worker::~Worker() { kill_worker = true; + notifyNewWork(); //Some notification methods may need to be unblocked to see kill_worker th_server.join(); } @@ -144,6 +146,55 @@ void BackBurner::Worker::SetConfiguration(const Configuration &config, int id) { worker_id = id; SetSubcomponentName("["+std::to_string(id)+"]"); ConfigureLogging(config); + + string notification_method; + config.GetString(¬ification_method, "backburner.notification_method", "pipe"); + + if(notification_method == "polling") { + //This version doesn't do any special notification and will burn the cpu + notifyNewWork = []() {}; + blockUntilWork = []() { return 1; }; + + } else if (notification_method == "sleep_polling") { + //This version polls, but injects sleep time so we don't eat as much time + uint64_t time_us; + config.GetTimeUS(&time_us, "backburner.sleep_polling_time","100us"); + dbg("Notification method: sleep_polling with a delay of "+std::to_string(time_us)+" us"); + notifyNewWork = []() {}; + blockUntilWork = [=]() { + std::this_thread::sleep_for(std::chrono::microseconds(time_us)); + return 1; + }; + + } else if (notification_method == "pipe") { + dbg("Notification method: pipe"); + + int rc = pipe(notification_pipe); + F_ASSERT(!rc, "Trouble setting up notification pipe in backburner?"); + //Use non blocking writes + int flags = fcntl(notification_pipe[1], F_GETFL); + F_ASSERT(flags>=0, "Pipe flags were bad?"); + rc = fcntl(notification_pipe[1], F_SETFL, flags | O_NONBLOCK); + F_ASSERT(rc>=0, "Backburner could not set notification pipe writes to non blocking?"); + + notifyNewWork = [=] { + dbg("Notifying through pipe"); + uint32_t new_work=1; + ssize_t write_bytes = write(notification_pipe[1], &new_work, 4); + F_ASSERT(write_bytes==4, "notifyNewWork could not write full word to pipe?"); + }; + blockUntilWork = [=] { + dbg("Blocking on pipe"); + uint32_t val; + ssize_t read_size; + do { + read_size = read(notification_pipe[0], &val, 4); + F_ASSERT(read_size==4, "blockUntilWork did not get full word from pipe?"); + } while(read_size!=4); + return 1; + }; + } + th_server = thread(&Worker::server, this); } @@ -153,6 +204,7 @@ void BackBurner::Worker::AddWork(fn_backburner_work work) { tasks_producer->push(work); producer_num++; mtx.unlock(); + notifyNewWork(); } void BackBurner::Worker::AddWork(vector work) { @@ -162,10 +214,10 @@ void BackBurner::Worker::AddWork(vector work) { tasks_producer->push(w); producer_num+=work.size(); mtx.unlock(); + notifyNewWork(); } void BackBurner::Worker::RegisterPollingFunction(string name, uint32_t group_id, fn_backburner_work polling_function) { - dbg("Register polling function "+name); auto it = registered_poll_functions.find(name); if(it != registered_poll_functions.end()) { @@ -176,7 +228,6 @@ void BackBurner::Worker::RegisterPollingFunction(string name, uint32_t group_id, } void BackBurner::Worker::DisablePollingFunction(string name) { - dbg("Disabling polling function "+name); auto it = registered_poll_functions.find(name); if(it != registered_poll_functions.end()) { @@ -184,14 +235,17 @@ void BackBurner::Worker::DisablePollingFunction(string name) { } } - void BackBurner::Worker::server() { int num_bundles=0; int num=0; + bool no_extra_poll_functions = registered_poll_functions.empty(); while(!kill_worker) { - //cout <<"Backburner\n"; + dbg("Polling"); + + //If user didn't add any polling functions we can use notifications to save some cycles + if(no_extra_poll_functions) blockUntilWork(); //Only lock when there's work if(producer_num!=consumer_num) { diff --git a/src/faodel-services/BackBurner.hh b/src/faodel-services/BackBurner.hh index 6b16115..ba3c9ed 100644 --- a/src/faodel-services/BackBurner.hh +++ b/src/faodel-services/BackBurner.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_COMMON_BACKBURNER_HH #define FAODEL_COMMON_BACKBURNER_HH @@ -11,6 +11,7 @@ #include #include +#include #include #include "faodel-common/Configuration.hh" @@ -119,6 +120,10 @@ private: std::map registered_poll_functions; + int notification_pipe[2]; //For pausing thread until work + + std::function notifyNewWork; + std::function blockUntilWork; }; bool configured; diff --git a/src/faodel-services/MPISyncStart.cpp b/src/faodel-services/MPISyncStart.cpp index fe6cb09..0c65cd6 100644 --- a/src/faodel-services/MPISyncStart.cpp +++ b/src/faodel-services/MPISyncStart.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -44,7 +44,6 @@ void MPISyncStart::InitAndModifyConfiguration(Configuration *config) { int64_t dirman_root_mpi; config->GetBool(&needs_patch, "mpisyncstart.enable", "false"); - //needs_patch |= (ENOENT != config->GetInt(&dirman_root_mpi, "dirman.root_node_mpi", "-1")); needs_patch |= (ENOENT != config->GetString(&dirman_root_mpi_string, "dirman.root_node_mpi", "-1")); //Finalize in mpi section config->GetStringVector(&dirman_resources_mpi, "dirman.resources_mpi"); @@ -58,17 +57,30 @@ void MPISyncStart::InitAndModifyConfiguration(Configuration *config) { throw std::invalid_argument("Configuration contained an mpi update, but FAODEL not configured with MPI."); #else - int mpi_rank, mpi_size; - + int mpi_rank, mpi_size, initialized=-1; + MPI_Initialized(&initialized); + if(initialized != 1) { + throw std::runtime_error("Faodel requested mpisyncstart, but MPI has not been initialized. You must call MPI_Init before starting Faodel."); + } MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + config->Append("observed_mpi_info", std::to_string(mpi_rank) + "/"+std::to_string(mpi_size) ); + + //Convert to an root string to integer. User may have passed in first/middle/last token to parse - if(dirman_root_mpi_string == "-1") dirman_root_mpi = -1; + if(dirman_root_mpi_string == "-1") { + dbg("dirman_root_node_mpi was not found in config."); + dirman_root_mpi = -1; + } else { auto ids = ExtractIDs(dirman_root_mpi_string, mpi_size); - kassert(ids.size()==1, "dirman.root_node_mpi can only have one value. Observed:"+dirman_root_mpi_string); + F_ASSERT(ids.size() == 1, "dirman.root_node_mpi can only have one value. Observed:"+dirman_root_mpi_string); dirman_root_mpi = *ids.begin(); //Should be first value + if(dirman_root_mpi == mpi_rank){ + dbg("Detected rank "+std::to_string(dirman_root_mpi)+" is the dirman root. Setting to host root"); + config->Append("dirman.host_root true"); + } } @@ -76,7 +88,7 @@ void MPISyncStart::InitAndModifyConfiguration(Configuration *config) { //See if we just need a barrier if((dirman_root_mpi==-1) && (dirman_resources_mpi.empty())) { - dbg("mpi_sync_start requested, but no specific needs spedified. Performing Barrier"); + dbg("mpi_sync_start requested, but no specific needs specified. Performing Barrier"); MPI_Barrier(MPI_COMM_WORLD); dbg("Barrier completed."); } @@ -85,7 +97,7 @@ void MPISyncStart::InitAndModifyConfiguration(Configuration *config) { //An MPI rank specified for the dirman root. Find the ID of the node if(dirman_root_mpi != -1) { dbg("Dirman Root specified as rank "+std::to_string(dirman_root_mpi)+". Perform bcast to learn whookie root."); - kassert(dirman_root_mpi < mpi_size, "dirman.root_node_mpi value is larger than mpi ranks"); + F_ASSERT(dirman_root_mpi < mpi_size, "dirman.root_node_mpi value is larger than mpi ranks"); nodeid_t root_node = my_id; MPI_Bcast(&root_node, sizeof(nodeid_t), MPI_CHAR, dirman_root_mpi, MPI_COMM_WORLD); config->Append("dirman.root_node", root_node.GetHex()); @@ -93,7 +105,12 @@ void MPISyncStart::InitAndModifyConfiguration(Configuration *config) { } //One or more dirman resources were specified with mpi ranks. Update them in the config. - if(!dirman_resources_mpi.empty()){ + if(!dirman_resources_mpi.empty()) { + + if((mpi_rank == 0) && (dirman_root_mpi<0)) { + F_WARN("Faodel configuration contained "+std::to_string(dirman_resources_mpi.size())+ + " dirman.resources_mpi[], but dirman.root_node_mpi was not set. Ignoring."); + } //TODO: narrow the scope to only be necessary nodes auto nodes = new faodel::nodeid_t[mpi_size]; @@ -105,6 +122,8 @@ void MPISyncStart::InitAndModifyConfiguration(Configuration *config) { if(mpi_rank == dirman_root_mpi) { //Only load up resources on the root node for (const auto &line : dirman_resources_mpi) { + //dbg("Working on line: "+line); + //Pull out the url and list of MPI nodes vector tokens; Split(tokens, line, ' ', true); @@ -125,15 +144,12 @@ void MPISyncStart::InitAndModifyConfiguration(Configuration *config) { } //Add back to the config - config->Append("dirman.resources[]", ss.str()); - dbg("Adding new resource: " + ss.str()); + config->Append("dirman.resources[]", ss.str()); } } - - delete[] nodes; //TOOD: Could cache this? - + delete[] nodes; } #endif diff --git a/src/faodel-services/MPISyncStart.hh b/src/faodel-services/MPISyncStart.hh index 61548bf..6917d70 100644 --- a/src/faodel-services/MPISyncStart.hh +++ b/src/faodel-services/MPISyncStart.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_MPISYNCHSTART_HH #define FAODEL_MPISYNCHSTART_HH diff --git a/src/faodel-services/README_Services.md b/src/faodel-services/README_Services.md index 3567200..7d332d1 100644 --- a/src/faodel-services/README_Services.md +++ b/src/faodel-services/README_Services.md @@ -17,14 +17,27 @@ threads (each with its own work queue). The BackBurner worker thread pulls a bundle of queue operations out at a time and then processes each one in order. -BackBurner parses the Configuration object for the following settings: +BackBurner has a few mechanisms for controlling how workers are notified of +new work: +- **Pipe** (default): A pipe is used to pass notifications of new work to the + workers. While this increases latencies, it allows the kernel to make a + decision about when to schedule the worker thread. +- **Polling**: Workers continuously check their queue sizes. This method + improve latencies, but it wastes a tremendous amount of CPU time. +- **Sleep Polling**: Workers do a sleep operation before they poll for new + work. Users can adjust the polling rate by changing the sleep time. Given + that a worker still processes all the entries that are available when it + wakes up, this approach can be useful for batching up work. -| Property | Type | Default | Description | -| ------------------ | ----------- | ------- | ------------------------------------ | -| backburner.debug | boolean | false | Display debug messages | -| backburner.threads | integer | 1 | Number of worker threads to use | +BackBurner parses the Configuration object for the following settings: +| Property | Type | Default | Description | +| ------------------------------ | ---------------------------- | ------- | --------------------------------------------- | +| backburner.debug | boolean | false | Display debug messages | +| backburner.threads | integer | 1 | Number of worker threads to use | +| backburner.notification_method | pipe, polling, sleep_polling | pipe | Controls how workers are notified of new work | +| backburner.sleep_polling_time | time (us) | 100us | Set polling delay in sleep_polling mode | MPISyncStart ------------ @@ -32,6 +45,14 @@ FAODEL applications often need a simple way to pass initial configuration information between nodes. The MPISyncStart service examines a configuration and updates certain properties (ending with "_mpi") with runtime information. +For properties that allow a user to specify a list of nodes (eg +`dirman.resources_mpi`), the parser supports a few ways to build node lists: + +- Rank IDs: The actual rank numbers, such as `0` or `0,1,2` or `0-3` +- "middle": the middle rank. eg in an 8 rank job, `0-middle` would be the first half +- "end": the last rank in the job. eg in an 8 rank job, `end` would be 7. +- "all": all of the ranks, eg in an 8 rank job `all` would be 0-7. + MPISyncStart parses the Configuration object for the following settings: | Property | Type | Default | Description | @@ -39,5 +60,7 @@ MPISyncStart parses the Configuration object for the following settings: | mpisyncstart.debug | boolean | false | Display debug messages | | mpisyncstart.enable | boolean | false | Enable the service | | dirman.root_node_mpi | Rank | -1 | Specify rank for dirman root node | -| dirman.resources_mpi<> | url node | "" | Specify resource using mpi ranks | +| dirman.resources_mpi[] | url node(s) | "" | Specify resource using mpi ranks | + +Note: Bootstrap has an `mpisyncstop.enable` option that performs a barrier on shutdown. diff --git a/src/kelpie/CMakeLists.txt b/src/kelpie/CMakeLists.txt old mode 100644 new mode 100755 index ffc1a8e..d629d2d --- a/src/kelpie/CMakeLists.txt +++ b/src/kelpie/CMakeLists.txt @@ -13,12 +13,14 @@ set( HEADERS_PUBLIC ioms/IomBase.hh ioms/IomRegistry.hh localkv/LocalKV.hh + localkv/LocalKVTypes.hh localkv/LocalKVCell.hh localkv/LocalKVRow.hh pools/Pool.hh ) set( HEADERS + common/ComputeRegistry.hh common/OpArgsObjectAvailable.hh core/KelpieCoreBase.hh core/KelpieCoreNoNet.hh @@ -26,6 +28,8 @@ set( HEADERS core/KelpieCoreUnconfigured.hh core/Singleton.hh ops/direct/msg_direct.hh + ops/direct/OpKelpieCompute.hh + ops/direct/OpKelpieDrop.hh ops/direct/OpKelpieGetBounded.hh ops/direct/OpKelpieGetUnbounded.hh ops/direct/OpKelpieList.hh @@ -35,13 +39,18 @@ set( HEADERS pools/PoolBase.hh pools/LocalPool/LocalPool.hh pools/DHTPool/DHTPool.hh + pools/NullPool/NullPool.hh + pools/TracePool/TracePool.hh pools/PoolRegistry.hh - pools/UnconfiguredPool/UnconfiguredPool.hh + pools/ResultCollector.cpp + pools/UnconfiguredPool/UnconfiguredPool.hh + ) set( SOURCES Kelpie.cpp Key.cpp + common/ComputeRegistry.cpp common/KelpieInternal.cpp common/ObjectCapacities.cpp common/OpArgsObjectAvailable.cpp @@ -58,18 +67,22 @@ set( SOURCES localkv/LocalKVCell.cpp localkv/LocalKVRow.cpp ops/direct/msg_direct.cpp + ops/direct/OpKelpieCompute.cpp + ops/direct/OpKelpieDrop.cpp ops/direct/OpKelpieGetBounded.cpp ops/direct/OpKelpieGetUnbounded.cpp ops/direct/OpKelpieList.cpp ops/direct/OpKelpieMeta.cpp ops/direct/OpKelpiePublish.cpp - pools/LocalPool/LocalPool.cpp pools/DHTPool/DHTPool.cpp + pools/LocalPool/LocalPool.cpp + pools/NullPool/NullPool.cpp + pools/TracePool/TracePool.cpp pools/Pool.cpp pools/PoolBase.cpp pools/PoolRegistry.cpp + pools/ResultCollector.cpp pools/UnconfiguredPool/UnconfiguredPool.cpp - ) if( Faodel_ENABLE_MPI_SUPPORT ) @@ -132,9 +145,9 @@ install( FILES common/Types.hh common/KelpieInternal.hh common/ObjectCapacities. DESTINATION ${INCLUDE_INSTALL_DIR}/faodel/kelpie/common ) install( FILES ioms/IomBase.hh ioms/IomRegistry.hh DESTINATION ${INCLUDE_INSTALL_DIR}/faodel/kelpie/ioms ) -install( FILES localkv/LocalKV.hh localkv/LocalKVCell.hh localkv/LocalKVRow.hh +install( FILES localkv/LocalKV.hh localkv/LocalKVCell.hh localkv/LocalKVRow.hh localkv/LocalKVTypes.hh DESTINATION ${INCLUDE_INSTALL_DIR}/faodel/kelpie/localkv ) -install( FILES pools/Pool.hh +install( FILES pools/Pool.hh pools/ResultCollector.hh DESTINATION ${INCLUDE_INSTALL_DIR}/faodel/kelpie/pools ) install( FILES services/PoolServerDriver.hh DESTINATION ${INCLUDE_INSTALL_DIR}/faodel/kelpie/services ) diff --git a/src/kelpie/Kelpie.cpp b/src/kelpie/Kelpie.cpp index 36236d0..6cf4b73 100644 --- a/src/kelpie/Kelpie.cpp +++ b/src/kelpie/Kelpie.cpp @@ -1,13 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include +#include #include "kelpie/Kelpie.hh" - #include "kelpie/core/KelpieCoreBase.hh" - #include "kelpie/core/Singleton.hh" using std::string; @@ -38,6 +37,25 @@ void RegisterPoolConstructor(std::string pool_name, fn_PoolCreate_t ctor_functio pool_name, ctor_function); } +/** + * @brief Registration for users to add user-defined compute functions to Kelpie + * @param[in] compute_function_name The name of the function users will reference + * @param[in] function_pointer Function that will be called to process data at target + */ +void RegisterComputeFunction(const std::string &compute_function_name, fn_compute_t function_pointer) { + kelpie::internal::Singleton::impl.core->compute_registry.RegisterComputeFunction(compute_function_name,function_pointer); +} + + +/** + * @brief Volunteer to be a server in a pool. Blocks if pool is not registered + * @param url The pool that we are joining (details may be filled in by DirMan) + * @retval 0 joined as a server + * @retval -1 failed to join + */ +int JoinServerPool(const faodel::ResourceURL &url, const string &optional_node_name) { + return kelpie::internal::Singleton::impl.core->JoinServerPool(url, optional_node_name); +} /** @brief Establish a connection to a particular resource based on its url * @@ -62,15 +80,40 @@ Pool Connect(string url_string) { } + /** * @brief Register a new user-defined I/O Module (IOM) * * @param[in] type A name for this category of IOM * @param[in] ctor_function A function for constructing new instances of this IOM */ -void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function) { - return kelpie::internal::Singleton::impl.core->RegisterIomConstructor(type, ctor_function); +void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function, fn_IomGetValidSetting_t valid_settings_function) { + if(kelpie::internal::Singleton::impl.core->GetType() == "unconfigured") { + F_WARN("Attempted to registerIOM when Kelpie is not currently configured"); + return; + } + return kelpie::internal::Singleton::impl.core->iom_registry.RegisterIomConstructor(type, ctor_function, valid_settings_function); +} + +std::vector GetRegisteredIomTypes() { + if(kelpie::internal::Singleton::impl.core->GetType() == "unconfigured") { + return {}; + } + return kelpie::internal::Singleton::impl.core->iom_registry.RegisteredTypes(); +} + +std::vector> GetRegisteredIOMTypeParameters(const std::string &type) { + if(kelpie::internal::Singleton::impl.core->GetType() == "unconfigured") { + return {}; + } + return kelpie::internal::Singleton::impl.core->iom_registry.RegisteredTypeParameters(type); } +std::vector GetIomNames() { + if(kelpie::internal::Singleton::impl.core->GetType() == "unconfigured") { + return {}; + } + return kelpie::internal::Singleton::impl.core->iom_registry.GetIomNames(); +} } // namespace kelpie diff --git a/src/kelpie/Kelpie.hh b/src/kelpie/Kelpie.hh index f4e6746..75ae5a0 100644 --- a/src/kelpie/Kelpie.hh +++ b/src/kelpie/Kelpie.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_KELPIE_HH #define KELPIE_KELPIE_HH @@ -12,7 +12,6 @@ #include "faodel-common/ResourceURL.hh" #include "kelpie/common/Types.hh" - #include "kelpie/pools/Pool.hh" // This header provides some shortcuts for users to interact with different @@ -28,8 +27,19 @@ std::string bootstrap(); // Kelpie-specific bootstrap. Registers kelpie-specifi bool IsUnconfigured(); // Whether kelpie is currently configured //Shortcuts for... -void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function); // ..user-defined IOM driver -void RegisterPoolConstructor(std::string pool_name, fn_PoolCreate_t ctor_function); // ..user-defined pools +void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function, fn_IomGetValidSetting_t valid_settings_function); // ..user-defined IOM driver +void RegisterPoolConstructor(std::string pool_name, fn_PoolCreate_t ctor_function); // ..user-defined pools + +void RegisterComputeFunction(const std::string &compute_function_name, fn_compute_t function_pointer); + +int JoinServerPool(const faodel::ResourceURL &url, const std::string &optional_node_name=""); + +std::vector GetRegisteredIomTypes(); +std::vector> GetRegisteredIOMTypeParameters(const std::string &type); + +std::vector GetIomNames(); + + Pool Connect(const faodel::ResourceURL &resource_url); // ..connecting to a pool Pool Connect(const std::string url_string); // ..connecting to a pool diff --git a/src/kelpie/Key.cpp b/src/kelpie/Key.cpp index ce57da8..882ca03 100644 --- a/src/kelpie/Key.cpp +++ b/src/kelpie/Key.cpp @@ -1,9 +1,10 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include +#include "faodel-common/StringHelpers.hh" #include "kelpie/Key.hh" using namespace std; @@ -11,6 +12,94 @@ using namespace std; namespace kelpie { +/** + * @brief Determine if Key has a wildcard in its row (ie, ends with '*') + * @retval true It's a wildcard + * @retval false Not a wildcard + */ +bool Key::IsRowWildcard() const { + return faodel::StringEndsWith(k1, "*"); +} + +/** + * @brief Determine if Key has a wildcard in its column (ie, ends with '*') + * @retval true It's a wildcard + * @retval false Not a wildcard + */ +bool Key::IsColWildcard() const { + return faodel::StringEndsWith(k2, "*"); +} + +/** + * @brief Determine if Key has a wildcard in either its row or column (ie, ends with '*') + * @retval true It's a wildcard + * @retval false Not a wildcard + */ +bool Key::IsWildcard() const { + return IsRowWildcard() || IsColWildcard(); +} + +/** + * @brief Determine whether this key matches row/col search parameters + * @param row_is_prefix true=Allow any key that matches this prefix, false=Row name must be exact + * @param row_match The row name to match against (does NOT permit wildcards) + * @param col_is_prefix true=Allow any key that matches this prefix, false=Col name must be exact + * @param col_match The col name to match against (does NOT permit wildcards) + * @retval true The key matches + * @retval false The key does NOT match + * + * This is a power user function, as it expects that you've looked at some key, determined + * it was a wildcard or not, and removed the trailing '*' because you're searching a lot + * of keys. The Matches() function below illustrates that kind of behavior. + */ +bool Key::matchesPrefixString(bool row_is_prefix, const std::string &row_match, + bool col_is_prefix, const std::string &col_match) const { + + //Check the row first + if(row_is_prefix) { + //User originally asked for "my_row_prefix*" or "*", so row_match is "my_row_prefix" or "" + if( (!row_match.empty()) && //Empty in this case means match everything + (!faodel::StringBeginsWith(k1, row_match))) { + return false; + } + + } else { + //User is expecting an exact match + if(k1 != row_match) return false; + } + + //Check the column + if(col_is_prefix) { + //User originally asked for "my_col_prefix*" or "*", so col_match is "my_col_prefix" or "" + if( (!col_match.empty()) && //Empty in this case means match everything + (!faodel::StringBeginsWith(k2, col_match))) { + return false; + } + + } else { + //User is expecting an exact match + if(k2 != col_match) return false; + } + + return true; +} + +/** + * @brief Determine if this key matches a row/col wildcards + * @param row_wildcard The exact row name to match, or a wildcard (eg my_row*) + * @param col_wildcard The exact col name to match, or a wildcard (eg my_col*) + * @retval true This key matches the inputs + * @retval false This key does NOT match the inputs + */ +bool Key::Matches(const std::string &row_wildcard, const std::string &col_wildcard) const { + bool is_row_wildcard = faodel::StringEndsWith(row_wildcard, "*"); + bool is_col_wildcard = faodel::StringEndsWith(col_wildcard, "*"); + string row_prefix = (is_row_wildcard) ? row_wildcard.substr(0, row_wildcard.size()-1) : row_wildcard; + string col_prefix = (is_col_wildcard) ? col_wildcard.substr(0, col_wildcard.size()-1) : col_wildcard; + return matchesPrefixString(is_row_wildcard, row_prefix, is_col_wildcard, col_prefix); +} + + /** * @brief A Packing function for converting a key into a binary string * @@ -32,11 +121,11 @@ string Key::pup() const { size_t len = 2+k1.size()+k2.size(); s.resize(len); - s[0] = static_cast(k1.size() & 0x0ff); - s[1] = static_cast(k2.size() & 0x0ff); - size_t i=2; + size_t i=0; for(size_t j=0; j(k1.size() & 0x0ff); + s[i++] = static_cast(k2.size() & 0x0ff); return s; } @@ -53,15 +142,70 @@ void Key::pup(const std::string &packed_string) { int s0{0},s1{0}; if(packed_string.size()>2) { //2 bytes for length fields, which can be zero - s0=static_cast(packed_string[0]); - s1=static_cast(packed_string[1]); + int i = packed_string.size() - 1; + // WORKING backward from the end of the string + s1=static_cast(packed_string[i--]); + s0=static_cast(packed_string[i--]); } if(s0+s1+2>static_cast(packed_string.size())){ throw std::runtime_error("Error unpacking key"); } - k1 = (s0==0) ? "" : packed_string.substr(2, s0); - k2 = (s1==0) ? "" : packed_string.substr(2+s0,s1); + k1 = (s0==0) ? "" : packed_string.substr(0, s0); + k2 = (s1==0) ? "" : packed_string.substr(0+s0,s1); +} + + +string Key::str_as_args() const { + //Todo: would be nice to handle spaces via quotes + stringstream ss; + if(!k1.empty()) ss<<"-k1 "< 0); } //!< A valid key has to at least have a row name + static Key Random(size_t k1_length, size_t k2_length=0); //!< Generate a random string key of specified length + static Key Random(const std::string &k1_name, size_t k2_length=0); //!< Generate a key with a fixed row name and random column name + static Key Random(size_t k1_length, const std::string &k2_name); //!< Generate a key with a fixed column name and random row name + private: std::string k1; std::string k2; diff --git a/src/kelpie/README_Kelpie.md b/src/kelpie/README_Kelpie.md index 66a48eb..e5e9a4b 100644 --- a/src/kelpie/README_Kelpie.md +++ b/src/kelpie/README_Kelpie.md @@ -80,9 +80,32 @@ most users. Users may invoke the following operations on a pool: particular data object be available before execution can continue. A need returns the actual, local LDO to the user (while generally retaining the LDO in the LocalKV). - -Future versions of this API will include updates in order to get more -network information from the pool. +- **Drop**: A drop operation signifies that an object is no longer + needed and should be removed from the pool. + +Users can also obtain information about objects that are in a pool. These +operations return with current information and do not wait for objects +to be published. + +- **Info**: An info operation is a blocking request that retrieves + information about a single object in the pool. The info command + provides more detailed information about an object than a list + command. +- **List**: Dispatch a query that retrieves all keys that exist in a + pool that match a specific value or wildcard pattern. Wildcards + can only be suffix based (eg `foo*` but not `foo*bar`). Results + are the list of keys and the current object sizes. + +Finally, Kelpie now provides an experimental mechanism for executing +compute operations on objects via user-defined functions (UDFs). UDFs +must be registered at start time and given a string label for +referencing the function. + +- **Compute**: The compute function retrieves one or more objects from + a remote row, executes a UDF on the object(s), and then returns + a new object back to the user. See the examples/kelpie/compute + directory for more information on how to use Compute. + Connecting to a Pool -------------------- @@ -96,13 +119,45 @@ that the application can use. Pool handles are reference counted and reused in the implementation, given that user operations cannot change the pool's settings after creation. +Issuing Multiple Requests +------------------------- +Often it is desirable for users to issue a batch of asynchronous requests +and then at a later time block until all tasks are complete. While users +car create their own system for managing notifications via the pool +command's callback functions, Kelpie provides a `ResultCollector` class +that gathers results and blocks until all operations complete. + +Pool Types +---------- + +Kelpie supports multiple types of pools and can be extended with new +functionality as needed. The following are the built-in pool types: + +- **LocalPool**: Only reference the localkv store that is hosted in + this node. +- **DHTPool**: A distributed hash table (DHT) pool that spreads data + across a collection of (static) nodes. The owner of the an + object is determined by hashing the row portion of a key. This + approach ensures all objects for a row reside on the same node. +- **RFTPool**: A rank-folding table (RFT) pool is similar to a + DHT, except the modulo of the producer's rank is used to select + which node receives the data. Consumer nodes should provide + the rank they wish to access in the pool construction request. +- **TracePool**: A trace pool records information about all the + requests a client made to a particular pool. The TracePool can + be configured to relay requests to another pool. +- **NullPool**: The null pool simply drops all requests made to it. + +Users may create their own pools by extending the PoolBase. We envision +that users may wish to handle replication or perform more customized +RDMA opertions in a pool. Build and Configuration Settings ================================ -Build Depndencies ------------------ +Build Dependencies +------------------ Kelpie has a few library dependencies: @@ -127,10 +182,9 @@ Run-Time Options When started, Kelpie examines the Configuration passed to it for the following variables: -| Property | Type | Default | Description | -| ----------------------- | ----------- | -------- | ----------------------------------------------- | -| kelpie.type | string | standard | Select between different Kelpie implementations | -| kelpie.lkv.max_capacity | int | 1G | Limit amount of space the LocalKV can occupy | +| Property | Type | Default | Description | +| ----------------------- | --------------- | -------- | ------------------------------------------------------------------------------- | +| kelpie.type | standard, nonet | standard | Select between the standard networked core, or a debug option without networkin | This release provides two kelpie implementation types: diff --git a/src/kelpie/common/ComputeRegistry.cpp b/src/kelpie/common/ComputeRegistry.cpp new file mode 100644 index 0000000..7a3a302 --- /dev/null +++ b/src/kelpie/common/ComputeRegistry.cpp @@ -0,0 +1,151 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include + +#include "faodel-common/Debug.hh" +#include "kelpie/common/ComputeRegistry.hh" +#include "whookie/Server.hh" + +using namespace std; +using namespace faodel; + +namespace kelpie { +namespace internal { + + +ComputeRegistry::ComputeRegistry() + : LoggingInterface("kelpie.compute_registry"), + started(false), + default_compute_logging_level(0) { +} + +ComputeRegistry::~ComputeRegistry() { +} + +void ComputeRegistry::init(const faodel::Configuration &config) { + + ConfigureLogging(config); //Set registry's logging level + default_compute_logging_level = LoggingInterface::GetLoggingLevelFromConfiguration(config, "kelpie.compute"); + + RegisterComputeFunction("pick", fn_pick); + + + whookie::Server::updateHook("/kelpie/compute_registry", [this] (const map &args, stringstream &results) { + return HandleWhookieStatus(args, results); + }); + +} +void ComputeRegistry::start(){ + started=true; //Prevent anyone from registering new pool types after this point +} + +void ComputeRegistry::finish() { + dbg("Finishing"); + whookie::Server::deregisterHook("/kelpie/compute_registry"); + + compute_fns.clear(); + //NOTE: This may need a mutex+clear for safety + started=false; +} + +void ComputeRegistry::RegisterComputeFunction(const std::string &compute_function_name, fn_compute_t function_pointer){ + + F_ASSERT(!started, "Attempted to register compute function after bootstrap Start()."); + + dbg("Registering compute function "+compute_function_name); + + auto name_fn = compute_fns.find(compute_function_name); + F_ASSERT(name_fn == compute_fns.end(), "Attempting to overwrite existing compute function for "+compute_function_name); + compute_fns[compute_function_name]=function_pointer; +} + +rc_t ComputeRegistry::doCompute( const std::string &compute_function_name, const std::string &args, + faodel::bucket_t bucket, const Key &key, + std::map ldos, + lunasa::DataObject *ext_ldo) { + + auto name_fn = compute_fns.find(compute_function_name); + if(name_fn == compute_fns.end()) return KELPIE_EINVAL; + auto fn = name_fn->second; + return fn(bucket, key, args, ldos, ext_ldo); + +} + +void ComputeRegistry::HandleWhookieStatus(const std::map &args, std::stringstream &results) { + faodel::ReplyStream rs(args, "Kelpie Compute Function Registry", &results); + + vector> stats; + + //Just a one-column table with function names for now.. Maybe include pointers later? + vector> compute_names; + compute_names.push_back({"Registered Compute Function Names"}); + for(auto &name_fn : compute_fns){ + vector row; + row.push_back(name_fn.first); + compute_names.push_back(row); + } + rs.mkTable(compute_names, "Compute Functions"); + rs.Finish(); +} + +/** + * @brief Compute Function for selecting an output object from a list of keys based on a user constraint + * @param[in] key The key that was used in the search (can have wildcard in column) + * @param[in] args Additional pick command: first, last, largest, smallest + * @param[in] ldos A map of the key/blobs that matched the key search + * @param[out] ext_ldo The ldo to return to the user. + * @retval KELPIE_OK Was able to perform the specified pick + * @retval KELPIE_ENOENT Search didn't find any hits + * @retval KELPIE_EINVAL Pick argument didn't match anything we knew about + */ +rc_t ComputeRegistry::fn_pick(faodel::bucket_t, const Key &key, const string &args, + map ldos, + lunasa::DataObject *ext_ldo) { + int i; + string choices[]={ "first", "last", "largest", "smallest" }; + if(args.empty()) { + i=0; + } else { + for(i = 0; i < 4; i++) { + if(args == choices[i]) break; + } + } + if(i==4) return KELPIE_EINVAL; + if(ldos.empty()) return KELPIE_ENOENT; + if(!ext_ldo) return KELPIE_OK; + + switch(i) { + case 0: *ext_ldo = ldos.begin()->second; break; + case 1: *ext_ldo = ldos.rbegin()->second; break; + case 2: { + uint32_t largest_size = 0; + for(auto &name_ldo : ldos) { + if(name_ldo.second.GetUserSize() > largest_size) { + *ext_ldo = name_ldo.second; + largest_size = ext_ldo->GetUserSize(); + } + } + break; + } + case 3: { + uint32_t smallest_size=ldos.begin()->second.GetUserSize() + 1; + for(auto &name_ldo : ldos) { + if(name_ldo.second.GetUserSize() < smallest_size) { + *ext_ldo = name_ldo.second; + smallest_size = ext_ldo->GetUserSize(); + } + } + break; + } + default: ; + } + + return KELPIE_OK; +} + + +} //namespace internal +} //namespace kelpie + diff --git a/src/kelpie/common/ComputeRegistry.hh b/src/kelpie/common/ComputeRegistry.hh new file mode 100644 index 0000000..fdbd479 --- /dev/null +++ b/src/kelpie/common/ComputeRegistry.hh @@ -0,0 +1,61 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef KELPIE_COMPUTEREGISTRY_HH +#define KELPIE_COMPUTEREGISTRY_HH + +#include +#include +#include + +#include "faodel-common/FaodelTypes.hh" +#include "faodel-common/LoggingInterface.hh" + +#include "kelpie/common/Types.hh" + +namespace kelpie { +namespace internal { + +/** + * @brief An internal registry for (tracking existing/creating new) pools + * + * The ComputeRegistry stores functions that are available at the node for users + */ +class ComputeRegistry : + public faodel::LoggingInterface { + + public: + ComputeRegistry(); + ~ComputeRegistry(); + + void init(const faodel::Configuration &config); + void start(); + void finish(); + + void RegisterComputeFunction(const std::string &compute_function_name, fn_compute_t function_pointer); + + rc_t doCompute(const std::string &compute_function_name, + const std::string &args, + faodel::bucket_t bucket, const Key &key, + std::map ldos, + lunasa::DataObject *ext_ldo); + + + //Built-in UDF 'pick' selects the 'first', 'last', 'smallest', 'largest' DataObject to return + static rc_t fn_pick(faodel::bucket_t, const Key &key, const std::string &args, + std::map ldos, lunasa::DataObject *ext_ldo); + + +private: + bool started; + int default_compute_logging_level; + std::map compute_fns; + + void HandleWhookieStatus(const std::map &args, std::stringstream &results); +}; + +} // namespace internal +} // namespace kelpie + +#endif // KELPIE_COMPUTEREGISTRY_HH diff --git a/src/kelpie/common/KelpieInternal.cpp b/src/kelpie/common/KelpieInternal.cpp index 30b7ea6..2c37e66 100644 --- a/src/kelpie/common/KelpieInternal.cpp +++ b/src/kelpie/common/KelpieInternal.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "kelpie/common/KelpieInternal.hh" #include "kelpie/core/Singleton.hh" diff --git a/src/kelpie/common/KelpieInternal.hh b/src/kelpie/common/KelpieInternal.hh index 2a2ac6f..6d8307d 100644 --- a/src/kelpie/common/KelpieInternal.hh +++ b/src/kelpie/common/KelpieInternal.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_KELPIEINTERNAL_HH #define KELPIE_KELPIEINTERNAL_HH diff --git a/src/kelpie/common/ObjectCapacities.cpp b/src/kelpie/common/ObjectCapacities.cpp index 46b7ca1..bf346c6 100644 --- a/src/kelpie/common/ObjectCapacities.cpp +++ b/src/kelpie/common/ObjectCapacities.cpp @@ -1,6 +1,9 @@ -// Copyright 2021 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include #include "kelpie/common/Types.hh" @@ -8,7 +11,7 @@ using namespace std; namespace kelpie { -/// Append a key/capacity to this ObjectCapacities. Does NOT dedupe +/// Append a ket/capacity to this ObjectCapacities. Does NOT dedupe void ObjectCapacities::Append(const kelpie::Key &key, size_t capacity) { keys.emplace_back(key); capacities.emplace_back(capacity); diff --git a/src/kelpie/common/ObjectCapacities.hh b/src/kelpie/common/ObjectCapacities.hh index 2286bd8..d3de28e 100644 --- a/src/kelpie/common/ObjectCapacities.hh +++ b/src/kelpie/common/ObjectCapacities.hh @@ -1,6 +1,6 @@ -// Copyright 2021 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_OBJECTCAPACITIES_HH #define FAODEL_OBJECTCAPACITIES_HH @@ -18,8 +18,8 @@ class ObjectCapacities : public: //Note: These data structures are plain vectors because some callers need to update capacities first, then set the keys - std::vector keys; - std::vector capacities; + std::vector keys; + std::vector capacities; /// Append a ket/capacity to this ObjectCapacities. Does NOT dedupe void Append(const kelpie::Key &key, size_t capacity); @@ -31,7 +31,7 @@ public: void Wipe(); /// How many entries are here - size_t Size() { return keys.size(); } + size_t Size() const { return keys.size(); } //Serialization hook template diff --git a/src/kelpie/common/OpArgsObjectAvailable.cpp b/src/kelpie/common/OpArgsObjectAvailable.cpp index 1ea94d1..71a8a6d 100644 --- a/src/kelpie/common/OpArgsObjectAvailable.cpp +++ b/src/kelpie/common/OpArgsObjectAvailable.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/src/kelpie/common/OpArgsObjectAvailable.hh b/src/kelpie/common/OpArgsObjectAvailable.hh index a500a44..811e6e2 100644 --- a/src/kelpie/common/OpArgsObjectAvailable.hh +++ b/src/kelpie/common/OpArgsObjectAvailable.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPARGSOBJECTAVAILABLE_HH #define OPBOX_OPARGSOBJECTAVAILABLE_HH @@ -19,23 +19,20 @@ namespace kelpie { class OpArgsObjectAvailable : public opbox::OpArgs { public: - OpArgsObjectAvailable(lunasa::DataObject ldo, kv_row_info_t row_info, kv_col_info_t col_info) + OpArgsObjectAvailable(lunasa::DataObject ldo, object_info_t info) : OpArgs(opbox::UpdateType::user_trigger), - ldo(ldo), - row_info(row_info), - col_info(col_info) { + ldo(std::move(ldo)), + info(info) { } - ~OpArgsObjectAvailable() override { - } + ~OpArgsObjectAvailable() override = default; //InfoInterface void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; //ObjectAvailable-specific Variables lunasa::DataObject ldo; - kv_row_info_t row_info; - kv_col_info_t col_info; + object_info_t info; }; diff --git a/src/kelpie/common/Types.cpp b/src/kelpie/common/Types.cpp index e981707..ca6d047 100644 --- a/src/kelpie/common/Types.cpp +++ b/src/kelpie/common/Types.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "faodel-common/StringHelpers.hh" @@ -24,57 +24,23 @@ string availability_to_string(const Availability &a){ } } -/** - * @brief Clear out all data values in this data structure - */ -void kv_row_info_t::Wipe(){ - row_bytes=0; - num_cols_in_row=num_row_receiver_nodes=num_row_dependencies=0; - availability = Availability::Unavailable; -} - -string kv_row_info_t::str(){ - std::stringstream ss; - ss<<"RowInfo: " - <<" RowCols: " << num_cols_in_row - <<" RowBytes: " << row_bytes - <<" Availability: " << availability_to_string(availability) - <<" WaitingNodes: " << num_row_receiver_nodes - <<" WaitingFunctions: " << num_row_dependencies; - return ss.str(); +void object_info_t::Wipe() { + row_user_bytes=row_num_columns=col_user_bytes=col_dependencies=0; + col_availability = Availability::Unavailable; } - - -void kv_row_info_t::ChangeAvailabilityFromLocalToRemote(){ - if(availability == Availability::InLocalMemory) - availability = Availability::InRemoteMemory; -} - -/** - * @brief Clear out all data values in this data structure - */ -void kv_col_info_t::Wipe(){ - node_origin=faodel::NODE_UNSPECIFIED; - num_bytes=0; - num_col_receiver_nodes=num_col_dependencies=0; - availability = Availability::Unavailable; -} - -string kv_col_info_t::str(){ +string object_info_t::str() const { std::stringstream ss; - ss<<"ColInfo: " - <<" Origin: " << node_origin.GetHex() - <<" NumBytes: " << std::dec << num_bytes - <<" WaitingNodes: " << num_col_receiver_nodes - <<" LocalDependencies: " << num_col_dependencies - <<" Availability: " << availability_to_string(availability); + ss<<"RowBytes: "< names; - if(f & PoolBehavior:: WriteToLocal) names.push_back("WriteToLocal"); - if(f & PoolBehavior:: WriteToRemote) names.push_back("WriteToRemote"); - if(f & PoolBehavior:: WriteToIOM) names.push_back("WriteToIOM"); - if(f & PoolBehavior:: ReadToLocal) names.push_back("ReadToLocal"); - if(f & PoolBehavior:: ReadToRemote) names.push_back("ReadToRemote"); - if(f & PoolBehavior:: ReadToLocal) names.push_back("ReadToLocal"); + if(f & PoolBehavior:: WriteToLocal) names.emplace_back("WriteToLocal"); + if(f & PoolBehavior:: WriteToRemote) names.emplace_back("WriteToRemote"); + if(f & PoolBehavior:: WriteToIOM) names.emplace_back("WriteToIOM"); + if(f & PoolBehavior:: ReadToLocal) names.emplace_back("ReadToLocal"); + if(f & PoolBehavior:: ReadToRemote) names.emplace_back("ReadToRemote"); + if(f & PoolBehavior:: EnableOverwrites) names.emplace_back("EnableOverwrites"); string s = faodel::Join(names,' '); return s; diff --git a/src/kelpie/common/Types.hh b/src/kelpie/common/Types.hh index c8d977a..1fb80ba 100644 --- a/src/kelpie/common/Types.hh +++ b/src/kelpie/common/Types.hh @@ -1,6 +1,6 @@ -// Copyright 2021 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_TYPES_HH #define KELPIE_TYPES_HH @@ -10,13 +10,13 @@ #include #include - +#include "faodel-common/FaodelTypes.hh" +#include "faodel-common/Bucket.hh" #include "lunasa/DataObject.hh" #include "kelpie/Key.hh" #include "kelpie/common/ObjectCapacities.hh" -#include //For packing ObjectCapacities //Forward references @@ -39,7 +39,8 @@ namespace internal { class IomBase; } -typedef int rc_t; //!< Kelpie functions return standad return codes, plus some extras +//typedef int rc_t; //!< Kelpie functions return standard return codes, plus some extras +using rc_t = faodel::rc_t; //Ok results const rc_t KELPIE_OK = 0; //!< Function successful @@ -49,7 +50,7 @@ const rc_t KELPIE_RECHECK = 3; //!< Operation worked, but may have caveats //Fail const rc_t KELPIE_ENOENT = -2; //!< Item doesn't exist -const rc_t KELPIE_EIO = -5; //!< Input/output error +const rc_t KELPIE_EIO = -5; //!< Input/output error const rc_t KELPIE_NXIO = -6; //!< Not configured const rc_t KELPIE_EINVAL = -22; //!< Bad input const rc_t KELPIE_ETIMEDOUT = -110; //!< Timed out @@ -78,32 +79,21 @@ enum class Availability : uint8_t { }; std::string availability_to_string(const Availability &a); -/** - * @brief Information about a particular row of items - */ -struct kv_row_info_t { - ssize_t row_bytes; //!< The current total for how big the row is - uint32_t num_cols_in_row; //!< How many columns are filled in this row - uint32_t num_row_receiver_nodes; //!< How many nodes are waiting on updates to this row - uint32_t num_row_dependencies; //!< How many actions are waiting if this row has activities - Availability availability; //!< Where the row is available - void Wipe(); - std::string str(); - void ChangeAvailabilityFromLocalToRemote(); -}; /** - * @brief Information about a particular column of data + * @brief Information about a particular object stored in kelpie + * @note: these are ordered to make this pack down to 3x8 words */ -struct kv_col_info_t { - faodel::nodeid_t node_origin; //!< What node generated this item - ssize_t num_bytes; //!< How big this item is - uint32_t num_col_receiver_nodes; //!< How many nodes are waiting on updates to this column - uint32_t num_col_dependencies; //!< How many local actions are waiting on this col - Availability availability; //!< Whether item is available locally +struct object_info_t { + size_t row_user_bytes; //!< The current total for how big the row is + size_t col_user_bytes; //!< How big the requested column is (meta+data) + uint16_t row_num_columns; //!< How many columns are filled in this row + uint16_t col_dependencies; //!< How many local actions are waiting on this column + Availability col_availability; //!< Where the column is available void Wipe(); - std::string str(); + std::string str() const; void ChangeAvailabilityFromLocalToRemote(); + }; @@ -113,24 +103,28 @@ typedef uint8_t pool_behavior_t; */ struct PoolBehavior { //Individual actions - static constexpr pool_behavior_t WriteToLocal = 0x01; //!< Publish writes to local memory - static constexpr pool_behavior_t WriteToRemote = 0x02; //!< Publish writes to remote memory - static constexpr pool_behavior_t WriteToIOM = 0x04; //!< Publish writes to remote IOM - static constexpr pool_behavior_t ReadToLocal = 0x08; //!< Want/Need writes to local memory - static constexpr pool_behavior_t ReadToRemote = 0x10; //!< Want/Need writes to remote memory + static constexpr pool_behavior_t WriteToLocal = 0x01; //!< Publish writes to local memory + static constexpr pool_behavior_t WriteToRemote = 0x02; //!< Publish writes to remote memory + static constexpr pool_behavior_t WriteToIOM = 0x04; //!< Publish writes to remote IOM + static constexpr pool_behavior_t ReadToLocal = 0x08; //!< Want/Need writes to local memory + static constexpr pool_behavior_t ReadToRemote = 0x10; //!< Want/Need writes to remote memory + static constexpr pool_behavior_t EnableOverwrites = 0x80; //!< Allow a publish to overwrite an existing copy //Common labels (combine individual actions) static constexpr pool_behavior_t WriteAround = WriteToIOM; //!< Publish only to IOM (skip local/remote memory) + static constexpr pool_behavior_t WriteToMemory = WriteToLocal | WriteToRemote; //!< Only write to local/remote memory static constexpr pool_behavior_t WriteToAll = WriteToLocal | WriteToRemote | WriteToIOM; //!< Publish to all levels static constexpr pool_behavior_t ReadToNone = 0x00; //!< Want/Need isn't cached in local/remote memory static constexpr pool_behavior_t NoAction = 0x00; //!< Don't take any action static constexpr pool_behavior_t TODO = 0x00; //!< Not implemented. Should revisit //Default behaviors: assume ioms are fire and forget - static constexpr pool_behavior_t DefaultBaseClass = WriteToAll | ReadToLocal |ReadToRemote; //!< Cache everywhere + static constexpr pool_behavior_t DefaultBaseClass = WriteToMemory | ReadToLocal |ReadToRemote; //!< Cache everywhere + static constexpr pool_behavior_t DefaultLocal = WriteToLocal | ReadToLocal; //!< Cache locally + static constexpr pool_behavior_t DefaultRemote = WriteToRemote | ReadToLocal; //!< No local caching static constexpr pool_behavior_t DefaultIOM = WriteToIOM | ReadToNone; //!< Don't cache writes/reads - static constexpr pool_behavior_t DefaultLocalIOM = WriteToIOM | ReadToNone; //!< Don't cache writes/reads - static constexpr pool_behavior_t DefaultRemoteIOM = WriteToIOM | ReadToRemote; //!< Only cache reads on remote side + static constexpr pool_behavior_t DefaultLocalIOM = WriteToIOM | WriteToLocal | ReadToLocal; //!< Cache locally + static constexpr pool_behavior_t DefaultRemoteIOM = WriteToIOM | WriteToRemote | ReadToRemote; //!< Only cache reads on remote side static constexpr pool_behavior_t DefaultCachingIOM = WriteToAll | ReadToLocal | ReadToRemote; //!< Cache everywhere static pool_behavior_t ChangeRemoteToLocal(pool_behavior_t f); @@ -143,8 +137,10 @@ struct PoolBehavior { //Pool callbacks -using fn_publish_callback_t = std::function; -using fn_want_callback_t = std::function; +using fn_publish_callback_t = std::function; +using fn_want_callback_t = std::function; +using fn_drop_callback_t = std::function; +using fn_compute_callback_t = std::function; using fn_opget_result_t = std::function; //!< Lambda for passing back an op get @@ -155,11 +151,19 @@ using fn_row_op_t = std::function ( const faodel::ResourceURL &pool_url)>; //!< Lambda for creating a pool from a url +//User-defined Compute functions +using fn_compute_t = std::function ldos, + lunasa::DataObject *ext_ldo)>; + + //Iom driver constructor function pointer using fn_IomConstructor_t = std::function &settings)>; //!< Lambda for creating a new IOM driver +using fn_IomGetValidSetting_t = std::function> ()>; //!< Lambda for getting list of valid setting names/descriptions + } // namespace kelpie diff --git a/src/kelpie/core/KelpieCoreBase.cpp b/src/kelpie/core/KelpieCoreBase.cpp index e7bafc8..2febe95 100644 --- a/src/kelpie/core/KelpieCoreBase.cpp +++ b/src/kelpie/core/KelpieCoreBase.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. @@ -22,13 +22,6 @@ KelpieCoreBase::KelpieCoreBase() } KelpieCoreBase::~KelpieCoreBase() = default; -#if 0 -void KelpieCoreBase::registerPoolConstructor(string pool_name, fn_PoolCreate_t function_pointer) { - //shortcut for registration. A core uses this to register its own handles - //kelpie_core_unconfigured.pool_registry.RegisterPoolConstructor(pool_name, function_pointer); - return kelpie::internal::RegisterPoolConstructor(pool_name, function_pointer); -} -#endif } // namespace internal } // namespace kelpie diff --git a/src/kelpie/core/KelpieCoreBase.hh b/src/kelpie/core/KelpieCoreBase.hh index cd27501..6719e62 100644 --- a/src/kelpie/core/KelpieCoreBase.hh +++ b/src/kelpie/core/KelpieCoreBase.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_KELPIECOREBASE_HH #define KELPIE_KELPIECOREBASE_HH @@ -16,6 +16,7 @@ #include "kelpie/pools/PoolRegistry.hh" #include "kelpie/ioms/IomRegistry.hh" +#include "kelpie/common/ComputeRegistry.hh" namespace kelpie { namespace internal { @@ -46,11 +47,14 @@ public: //Pool Management virtual void RegisterPoolConstructor(std::string pool_name, fn_PoolCreate_t ctor_function) = 0; virtual Pool Connect(const faodel::ResourceURL &resource_url) = 0; + virtual std::vector GetRegisteredPoolTypes() const = 0; - //IOM Management - virtual void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function) = 0; - internal::IomBase * FindIOM(std::string iom_name) { return FindIOM(faodel::hash32(iom_name)); } - virtual internal::IomBase * FindIOM(iom_hash_t iom_hash) = 0; + //Pool Server + virtual int JoinServerPool(const faodel::ResourceURL &url, const std::string &optional_node_name) = 0; + + //IO Module + IomRegistry iom_registry; + ComputeRegistry compute_registry; protected: diff --git a/src/kelpie/core/KelpieCoreNoNet.cpp b/src/kelpie/core/KelpieCoreNoNet.cpp index f40534f..8848512 100644 --- a/src/kelpie/core/KelpieCoreNoNet.cpp +++ b/src/kelpie/core/KelpieCoreNoNet.cpp @@ -1,11 +1,16 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +#include #include "kelpie/common/KelpieInternal.hh" #include "kelpie/core/KelpieCoreNoNet.hh" #include "kelpie/pools/LocalPool/LocalPool.hh" +#include "kelpie/pools/NullPool/NullPool.hh" +#include "kelpie/pools/TracePool/TracePool.hh" + + using namespace std; using namespace faodel; @@ -25,15 +30,16 @@ void KelpieCoreNoNet::init(const faodel::Configuration &config) { ConfigureLogging(config); rc_t rc = lkv.Init(config); - kassert(rc==KELPIE_OK, "lkv init failed"); - - + F_ASSERT(rc == KELPIE_OK, "lkv init failed"); iom_registry.init(config); //Setup any ioms that are part of the Configuration pool_registry.init(config); //Setup logging/default bucket - + compute_registry.init(config); //Setup built-in compute functions + //Register built-in resource creators pool_registry.RegisterPoolConstructor("local", &LocalPoolCreate); pool_registry.RegisterPoolConstructor("lkv", &LocalPoolCreate); + pool_registry.RegisterPoolConstructor("null", &NullPoolCreate); + pool_registry.RegisterPoolConstructor("trace", &TracePoolCreate); //Register whookie whookie::Server::updateHook("/kelpie", [this] (const map &args, stringstream &results) { @@ -61,8 +67,27 @@ Pool KelpieCoreNoNet::Connect(const faodel::ResourceURL &resource_url) { return pool_registry.Connect(resource_url); } -void KelpieCoreNoNet::RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function) { - iom_registry.RegisterIomConstructor(type, ctor_function); +int KelpieCoreNoNet::JoinServerPool(const faodel::ResourceURL &url, const std::string &optional_node_name) { + bool ok; int rc=0; + DirectoryInfo dir_info; + if(optional_node_name.empty()) { ok = dirman::JoinDirWithoutName(url, &dir_info); + } else { ok = dirman::JoinDirWithName(url, optional_node_name, &dir_info); + } + if(ok) { + string iom_name = dir_info.url.GetOption("iom"); + if(!iom_name.empty()) { + IomBase *iom = iom_registry.Find(iom_name); + if(iom==nullptr) { + rc=iom_registry.RegisterIomFromURL(dir_info.url); + if(rc!=0) return -1; //expect an exception + } + } + } + return 0; +} + +vector KelpieCoreNoNet::GetRegisteredPoolTypes() const { + return pool_registry.GetRegisteredPoolTypes(); } void KelpieCoreNoNet::HandleWhookieStatus(const std::map &args, std::stringstream &results) { diff --git a/src/kelpie/core/KelpieCoreNoNet.hh b/src/kelpie/core/KelpieCoreNoNet.hh index d946daa..a5dd994 100644 --- a/src/kelpie/core/KelpieCoreNoNet.hh +++ b/src/kelpie/core/KelpieCoreNoNet.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_KELPIECORENONET_HH #define KELPIE_KELPIECORENONET_HH @@ -42,12 +42,12 @@ public: //Pool Management void RegisterPoolConstructor(std::string pool_name, fn_PoolCreate_t ctor_function) override; Pool Connect(const faodel::ResourceURL &resource_url) override; - - //IOM Management - void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function) override; - IomBase * FindIOM(iom_hash_t iom_hash) override { return iom_registry.Find(iom_hash); } + //Pool Server + int JoinServerPool(const faodel::ResourceURL &url, const std::string &optional_node_name) override; + std::vector GetRegisteredPoolTypes() const override; + //Whookie handling void HandleWhookieStatus(const std::map &args, std::stringstream &results); @@ -56,7 +56,6 @@ public: private: LocalKV lkv; - IomRegistry iom_registry; PoolRegistry pool_registry; }; diff --git a/src/kelpie/core/KelpieCoreStandard.cpp b/src/kelpie/core/KelpieCoreStandard.cpp index 7df7e31..8db070f 100644 --- a/src/kelpie/core/KelpieCoreStandard.cpp +++ b/src/kelpie/core/KelpieCoreStandard.cpp @@ -1,23 +1,26 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "kelpie/common/KelpieInternal.hh" #include "kelpie/core/KelpieCoreStandard.hh" #include "kelpie/pools/LocalPool/LocalPool.hh" +#include "kelpie/pools/NullPool/NullPool.hh" +#include "kelpie/pools/TracePool/TracePool.hh" #include "kelpie/pools/DHTPool/DHTPool.hh" #ifdef Faodel_ENABLE_MPI_SUPPORT #include "kelpie/pools/RFTPool/RFTPool.hh" #endif -#include "kelpie/ops/direct/OpKelpieMeta.hh" -#include "kelpie/ops/direct/OpKelpiePublish.hh" +#include "kelpie/ops/direct/OpKelpieCompute.hh" +#include "kelpie/ops/direct/OpKelpieDrop.hh" #include "kelpie/ops/direct/OpKelpieGetBounded.hh" #include "kelpie/ops/direct/OpKelpieGetUnbounded.hh" #include "kelpie/ops/direct/OpKelpieList.hh" - +#include "kelpie/ops/direct/OpKelpieMeta.hh" +#include "kelpie/ops/direct/OpKelpiePublish.hh" using namespace std; @@ -37,31 +40,38 @@ void KelpieCoreStandard::init(const faodel::Configuration &config) { ConfigureLogging(config); rc_t rc = lkv.Init(config); - kassert(rc==KELPIE_OK, "lkv init failed"); + F_ASSERT(rc == KELPIE_OK, "lkv init failed"); iom_registry.init(config); pool_registry.init(config); - + compute_registry.init(config); //Register built-in pool creators pool_registry.RegisterPoolConstructor("local", &LocalPoolCreate); pool_registry.RegisterPoolConstructor("lkv", &LocalPoolCreate); + pool_registry.RegisterPoolConstructor("null", &NullPoolCreate); pool_registry.RegisterPoolConstructor("dht", &DHTPoolCreate); + pool_registry.RegisterPoolConstructor("trace", &TracePoolCreate); #ifdef Faodel_ENABLE_MPI_SUPPORT pool_registry.RegisterPoolConstructor("rft", &RFTPoolCreate); #endif //Register OPs with opbox, program in their lkv - opbox::RegisterOp(); - opbox::RegisterOp(); + opbox::RegisterOp(); + opbox::RegisterOp(); opbox::RegisterOp(); opbox::RegisterOp(); opbox::RegisterOp(); - OpKelpieMeta::configure(faodel::internal_use_only, &lkv); - OpKelpiePublish::configure(faodel::internal_use_only, &lkv); - OpKelpieGetBounded::configure(faodel::internal_use_only, &lkv); - OpKelpieGetUnbounded::configure(faodel::internal_use_only, &lkv); - OpKelpieList::configure(faodel::internal_use_only, &config, &lkv); + opbox::RegisterOp(); + opbox::RegisterOp(); + + OpKelpieCompute::configure( faodel::internal_use_only, &config, &lkv); + OpKelpieDrop::configure( faodel::internal_use_only, &config, &lkv); + OpKelpieGetBounded::configure( faodel::internal_use_only, &config, &lkv); + OpKelpieGetUnbounded::configure(faodel::internal_use_only, &config, &lkv); + OpKelpieList::configure( faodel::internal_use_only, &config, &lkv); + OpKelpieMeta::configure( faodel::internal_use_only, &config, &lkv); + OpKelpiePublish::configure( faodel::internal_use_only, &config, &lkv); whookie::Server::updateHook("/kelpie", [this] (const map &args, stringstream &results) { @@ -77,11 +87,12 @@ void KelpieCoreStandard::start(){ void KelpieCoreStandard::finish(){ whookie::Server::deregisterHook("/kelpie"); - OpKelpieMeta::configure(faodel::internal_use_only, nullptr); //TODO: Would be nice to be a dummy lkv - OpKelpiePublish::configure(faodel::internal_use_only, nullptr); //TODO: Would be nice to be a dummy lkv - OpKelpieGetBounded::configure(faodel::internal_use_only, nullptr); //TODO: Would be nice to be a dummy lkv - OpKelpieGetUnbounded::configure(faodel::internal_use_only, nullptr); //TODO: Would be nice to be a dummy lkv + OpKelpieDrop::configure(faodel::internal_use_only, nullptr, nullptr); + OpKelpieGetBounded::configure(faodel::internal_use_only, nullptr, nullptr); + OpKelpieGetUnbounded::configure(faodel::internal_use_only, nullptr, nullptr); OpKelpieList::configure(faodel::internal_use_only, nullptr, nullptr); + OpKelpieMeta::configure(faodel::internal_use_only, nullptr, nullptr ); + OpKelpiePublish::configure(faodel::internal_use_only, nullptr, nullptr); pool_registry.finish(); iom_registry.finish(); } @@ -94,12 +105,27 @@ Pool KelpieCoreStandard::Connect(const faodel::ResourceURL &resource_url) { return pool_registry.Connect(resource_url); } -void KelpieCoreStandard::RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function) { - iom_registry.RegisterIomConstructor(type, ctor_function); +vector KelpieCoreStandard::GetRegisteredPoolTypes() const { + return pool_registry.GetRegisteredPoolTypes(); } -IomBase * KelpieCoreStandard::FindIOM(iom_hash_t iom_hash) { - return iom_registry.Find(iom_hash); +int KelpieCoreStandard::JoinServerPool(const faodel::ResourceURL &url, const std::string &optional_node_name) { + bool ok; int rc=0; + DirectoryInfo dir_info; + if(optional_node_name.empty()) { ok = dirman::JoinDirWithoutName(url, &dir_info); + } else { ok = dirman::JoinDirWithName(url, optional_node_name, &dir_info); + } + if(ok) { + string iom_name = dir_info.url.GetOption("iom"); + if(!iom_name.empty()) { + IomBase *iom = iom_registry.Find(iom_name); + if(iom==nullptr) { + rc=iom_registry.RegisterIomFromURL(dir_info.url); + if(rc!=0) return -1; //expect an exception + } + } + } + return 0; } void KelpieCoreStandard::HandleWhookieStatus(const std::map &args, std::stringstream &results) { diff --git a/src/kelpie/core/KelpieCoreStandard.hh b/src/kelpie/core/KelpieCoreStandard.hh index 819aa29..13c3515 100644 --- a/src/kelpie/core/KelpieCoreStandard.hh +++ b/src/kelpie/core/KelpieCoreStandard.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_KELPIECORESTANDARD_HH #define KELPIE_KELPIECORESTANDARD_HH @@ -38,14 +38,15 @@ public: //Pool Management void RegisterPoolConstructor(std::string pool_name, fn_PoolCreate_t ctor_function) override; Pool Connect(const faodel::ResourceURL &resource_url) override; - - //IOM Management - void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function) override; - IomBase * FindIOM(iom_hash_t iom_hash) override; + virtual std::vector GetRegisteredPoolTypes() const override; + + //Pool Server + int JoinServerPool(const faodel::ResourceURL &url, const std::string &optional_node_name) override; std::string GetType() const override { return "standard"; } void getLKV(LocalKV **localkv_ptr) override { *localkv_ptr = &lkv; } + //Whookie handling void HandleWhookieStatus(const std::map &args, std::stringstream &results); //InfoInterface function @@ -53,7 +54,6 @@ public: private: LocalKV lkv; - IomRegistry iom_registry; PoolRegistry pool_registry; }; diff --git a/src/kelpie/core/KelpieCoreUnconfigured.cpp b/src/kelpie/core/KelpieCoreUnconfigured.cpp index 2866902..bbc9092 100644 --- a/src/kelpie/core/KelpieCoreUnconfigured.cpp +++ b/src/kelpie/core/KelpieCoreUnconfigured.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "kelpie/common/KelpieInternal.hh" @@ -24,16 +24,18 @@ KelpieCoreUnconfigured::~KelpieCoreUnconfigured(){ //The unconfigured core should not be asked to do the internal start, init, or //finish calls because it is never the real module. Init should pick a real //core to create and then direct start/finish/init to it. -void KelpieCoreUnconfigured::start() { Panic("start"); } -void KelpieCoreUnconfigured::finish() { Panic("finish"); } +void KelpieCoreUnconfigured::start() { Panic("start"); } +void KelpieCoreUnconfigured::finish() { Panic("finish"); } void KelpieCoreUnconfigured::init(const faodel::Configuration &config) { Panic("init"); } -void KelpieCoreUnconfigured::getLKV(LocalKV **localkv_ptr) { Panic("getLKV"); } +void KelpieCoreUnconfigured::getLKV(LocalKV **localkv_ptr) { Panic("getLKV"); } void KelpieCoreUnconfigured::RegisterPoolConstructor(std::string pool_name, - fn_PoolCreate_t ctor_function) { Panic("RegisterPoolConstructor"); } + fn_PoolCreate_t ctor_function) { Panic("RegisterPoolConstructor"); } Pool KelpieCoreUnconfigured::Connect(const faodel::ResourceURL &resource_url) { Panic("Connect"); return Pool(); } -void KelpieCoreUnconfigured::RegisterIomConstructor(std::string type, - fn_IomConstructor_t ctor_function) { Panic("RegisterIomConstructor"); } -IomBase * KelpieCoreUnconfigured::FindIOM(iom_hash_t iom_hash) { Panic("FindIOM"); return nullptr; } +vector KelpieCoreUnconfigured::GetRegisteredPoolTypes() const { Panic("GetRegisterePoolTypes"); return {}; } + +int KelpieCoreUnconfigured::JoinServerPool(const faodel::ResourceURL & url, + const std::string & optional_node_name) { Panic("JoinServerPool"); return -1; } + void KelpieCoreUnconfigured::sstr(std::stringstream &ss, int depth, int indent) const { ss <<"Kelpie: Currently Unconfigured (call kelpie::Init(config))\n"; diff --git a/src/kelpie/core/KelpieCoreUnconfigured.hh b/src/kelpie/core/KelpieCoreUnconfigured.hh index 5cf8478..dfc8b5f 100644 --- a/src/kelpie/core/KelpieCoreUnconfigured.hh +++ b/src/kelpie/core/KelpieCoreUnconfigured.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_KELPIECOREUNCONFIGURED_HH #define KELPIE_KELPIECOREUNCONFIGURED_HH @@ -39,11 +39,11 @@ public: //Pool Management void RegisterPoolConstructor(std::string pool_name, fn_PoolCreate_t ctor_function) override; Pool Connect(const faodel::ResourceURL &resource_url) override; + std::vector GetRegisteredPoolTypes() const override; + + //Pool Server + int JoinServerPool(const faodel::ResourceURL &url, const std::string &optional_node_name) override; - //IOM Management - void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function) override; - IomBase * FindIOM(iom_hash_t iom_hash) override; - //InfoInterface function void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; diff --git a/src/kelpie/core/Singleton.cpp b/src/kelpie/core/Singleton.cpp index 9b153fe..2bf3536 100644 --- a/src/kelpie/core/Singleton.cpp +++ b/src/kelpie/core/Singleton.cpp @@ -1,7 +1,15 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. - +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include + +#include +#include +#include +#include +#include +#include #include "kelpie/core/Singleton.hh" #include "kelpie/core/KelpieCoreNoNet.hh" @@ -117,12 +125,54 @@ void SingletonImpl::Finish() { if(IsUnconfigured()){ error("Attempted to finish Kelpie that is unconfigured"); } else { + + //Often users launch kelpie ops but forget to check on their status. Do + //a quick check and make sure we don't have any ops that are stuck. Give + //up after a few tries- sometimes things can't be helped. + int num_kelpie_ops=0; + for(int i=0; i<3; i++) { + num_kelpie_ops = 0; + num_kelpie_ops += opbox::GetNumberOfActiveOps(OpKelpieDrop::op_id); + num_kelpie_ops += opbox::GetNumberOfActiveOps(OpKelpieGetBounded::op_id); + num_kelpie_ops += opbox::GetNumberOfActiveOps(OpKelpieGetUnbounded::op_id); + num_kelpie_ops += opbox::GetNumberOfActiveOps(OpKelpieList::op_id); + num_kelpie_ops += opbox::GetNumberOfActiveOps(OpKelpieMeta::op_id); + num_kelpie_ops += opbox::GetNumberOfActiveOps(OpKelpiePublish::op_id); + + dbg("Kelpie Finish detected "+std::to_string(num_kelpie_ops)+" active kelpie ops"); + + if(num_kelpie_ops>0) { + warn("Kelpie detected "+std::to_string(num_kelpie_ops)+" active ops. Delaying shutdown for 5 seconds."); + std::this_thread::sleep_for(std::chrono::seconds(5)); + } else { + break; + } + } + if (num_kelpie_ops>0) { + dbg("Kelpie Finish is charging ahead with "+std::to_string(num_kelpie_ops)+" active kelpie ops. Expect bad things."); + } + delete core; core = &unconfigured; } } +vector getCoreTypes() { return { "nonet", "standard" }; } +vector getPoolTypes() { return getKelpieCore()->GetRegisteredPoolTypes(); } +vector getIomTypes() { return getKelpieCore()->iom_registry.RegisteredTypes();} + +/** + * @brief Get a pointer to the current kelpie core (for testing) + * @return The core inside the Singleton + */ +KelpieCoreBase * getKelpieCore() { return kelpie::internal::Singleton::impl.core; } + +IomBase * FindIOM(std::string iom_name) { return Singleton::impl.core->iom_registry.Find(iom_name); } +IomBase * FindIOM(iom_hash_t iom_hash) { return Singleton::impl.core->iom_registry.Find(iom_hash); } + + + } // namespace internal /** diff --git a/src/kelpie/core/Singleton.hh b/src/kelpie/core/Singleton.hh index 163d594..d9a3566 100644 --- a/src/kelpie/core/Singleton.hh +++ b/src/kelpie/core/Singleton.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_SINGLETON_HH #define KELPIE_SINGLETON_HH @@ -70,6 +70,16 @@ public: static kelpie::internal::SingletonImpl impl; }; +/// Alias for getting at the current kelpie core +KelpieCoreBase * getKelpieCore(); + +//Internal functions +std::vector getCoreTypes(); +std::vector getPoolTypes(); +std::vector getIomTypes(); + +IomBase * FindIOM(std::string iom_name); +IomBase * FindIOM(iom_hash_t iom_hash); } // namespace internal } // namespace kelpie diff --git a/src/kelpie/ioms/IomBase.cpp b/src/kelpie/ioms/IomBase.cpp index 7c6c501..e2fbba5 100644 --- a/src/kelpie/ioms/IomBase.cpp +++ b/src/kelpie/ioms/IomBase.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -20,19 +20,19 @@ void IomBase::finish() { /** * @brief Get info about a collection of keys on this iom (simply iterates over all) - * @param bucket The bucket to search in - * @param keys Vector of keys to find - * @param col_infos Resulting info about the columns (in key order) + * @param[in] bucket The bucket to search in + * @param[in] keys Vector of keys to find + * @param[out] infos The return list of object info (in key order) * @retval KELPIE_OK All requests were successful * @retval error One or more requests were not successful. Last unsuccessful error is returned */ -rc_t IomBase::GetInfo(faodel::bucket_t bucket, const std::vector &keys, vector *col_infos) { +rc_t IomBase::GetInfo(faodel::bucket_t bucket, const std::vector &keys, vector *infos) { rc_t rc = KELPIE_OK; for(auto &k : keys) { - kv_col_info_t col_info; - rc_t rc2 = GetInfo(bucket, k, &col_info); + object_info_t info; + rc_t rc2 = GetInfo(bucket, k, &info); if(rc2!=KELPIE_OK) rc=rc2; - if(col_infos) col_infos->push_back(col_info); + if(infos) infos->emplace_back(info); } return rc; } @@ -78,8 +78,14 @@ rc_t IomBase::ReadObjects(faodel::bucket_t bucket, const vector &keys, return return_rc; } +rc_t IomBase::ListObjects(faodel::bucket_t bucket, Key key, ObjectCapacities *oc) { + // !!!! DEBUG !!!! + printf("WARNING: Current IOM (%s) does NOT support ListObjects() method\n", Name().c_str()); + return KELPIE_ENOENT; +} + /** - * @brief Get a particular seting that was passed in during creation + * @brief Get a particular setting that was passed in during creation * @param setting_name The name of the iom setting to search for * @return The value, or empty string */ diff --git a/src/kelpie/ioms/IomBase.hh b/src/kelpie/ioms/IomBase.hh index 2bc474a..a3f776b 100644 --- a/src/kelpie/ioms/IomBase.hh +++ b/src/kelpie/ioms/IomBase.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_IOMBASE_HH #define KELPIE_IOMBASE_HH @@ -35,19 +35,21 @@ public: IomBase() : LoggingInterface("kelpie.iom"), stat_wr_requests(0), stat_wr_bytes(0), - stat_rd_requests(0), stat_rd_bytes(0) {} + stat_rd_requests(0), stat_rd_bytes(0), + stat_rd_hits(0),stat_rd_misses(0) {} IomBase(std::string name, const std::map< std::string, std::string >& new_settings, const std::vector< std::string >& valid_settings ) : LoggingInterface("kelpie.iom"), name(name), stat_wr_requests(0), stat_wr_bytes(0), - stat_rd_requests(0), stat_rd_bytes(0) + stat_rd_requests(0), stat_rd_bytes(0), + stat_rd_hits(0),stat_rd_misses(0) { // Only keep settings that are valid for( auto&& s : valid_settings ) { auto found = new_settings.find( s ); if( found != new_settings.end() ) { - settings[s] = found->second; + settings[s] = found->second; } } }; @@ -56,13 +58,14 @@ public: virtual ~IomBase() override { } std::string Name() const { return name; } + iom_hash_t NameHash() const { return faodel::hash32(name); } virtual void finish(); - virtual rc_t GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col_info_t *col_info) = 0; - virtual rc_t GetInfo(faodel::bucket_t bucket, const std::vector &keys, std::vector *col_infos); + virtual rc_t GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, object_info_t *info) = 0; + virtual rc_t GetInfo(faodel::bucket_t bucket, const std::vector &keys, std::vector *infos); - virtual void WriteObject(faodel::bucket_t bucket, const kelpie::Key &key, const lunasa::DataObject &ldo) = 0; + virtual rc_t WriteObject(faodel::bucket_t bucket, const kelpie::Key &key, const lunasa::DataObject &ldo) = 0; virtual void WriteObjects(faodel::bucket_t bucket, const std::vector> &items); virtual rc_t ReadObject(faodel::bucket_t bucket, const kelpie::Key &key, lunasa::DataObject *ldo) = 0; @@ -70,19 +73,22 @@ public: std::vector> *found_objects, std::vector *missing_keys); + virtual rc_t ListObjects(faodel::bucket_t bucket, Key key, ObjectCapacities *oc); + virtual std::string Type() const = 0; virtual std::map Settings() const { return settings; } virtual std::string Setting(std::string setting_name) const; virtual void AppendWebInfo(faodel::ReplyStream rs, std::string reference_link, const std::map &args) = 0; + protected: std::map settings; std::string name; int stat_wr_requests, stat_wr_bytes; int stat_rd_requests, stat_rd_bytes; - + int stat_rd_hits, stat_rd_misses; }; } // namespace internal diff --git a/src/kelpie/ioms/IomCassandra.cpp b/src/kelpie/ioms/IomCassandra.cpp index f495329..3f2b8bf 100644 --- a/src/kelpie/ioms/IomCassandra.cpp +++ b/src/kelpie/ioms/IomCassandra.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -142,16 +142,22 @@ namespace kelpie { const cass_byte_t* payload */ - void + rc_t IomCassandra::WriteObject( faodel::bucket_t bucket, const kelpie::Key& key, - const lunasa::DataObject &ldo ) { - WriteObjects( bucket, { kvpair( key, ldo ) } ); + const lunasa::DataObject &ldo ) + { + try { + internal_WriteObject( bucket, { kvpair( key, ldo ) } ); + return 0; + } catch (std::runtime_error &e) { + return 1; + } } void - IomCassandra::WriteObjects( faodel::bucket_t bucket, + IomCassandra::internal_WriteObject( faodel::bucket_t bucket, const std::vector< kvpair >& kvpairs ) { size_t wr_amt = 0; @@ -165,8 +171,9 @@ namespace kelpie { cass_future_wait( fut ); rc = cass_future_error_code( fut ); if( rc not_eq CASS_OK ) { - std::cerr << "Cassandra batch write preparation failed: " << fut << std::endl; - return; + std::stringstream ss; + ss << "IomCassandra::Cassandra batch write preparation failed: " << fut; + throw std::runtime_error(ss.str()); } const CassPrepared* prep = cass_future_get_prepared( fut ); @@ -195,7 +202,9 @@ namespace kelpie { cass_future_wait( fut ); rc = cass_future_error_code( fut ); if( rc not_eq CASS_OK ) { - std::cerr << "Cassandra batch write failed: " << fut << std::endl; + std::stringstream ss; + ss << "IomCassandra::Cassandra batch write failed: " << fut; + throw std::runtime_error(ss.str()); } else { stat_wr_requests += kvpairs.size(); stat_wr_bytes += wr_amt; @@ -366,7 +375,7 @@ namespace kelpie { rc_t - IomCassandra::GetInfo( faodel::bucket_t bucket, const kelpie::Key& key, kv_col_info_t* col_info ) + IomCassandra::GetInfo( faodel::bucket_t bucket, const kelpie::Key& key, object_info_t *info ) { rc_t krc; CassError crc = CASS_OK; @@ -374,10 +383,10 @@ namespace kelpie { CassStatement* stmt = nullptr; std::string select_cql = "SELECT meta_size, data_size FROM " + keyspace_table_ + " WHERE bucket = ? AND key = ?"; - // currently makes zero sense to call this with a null col_info pointer, so zero the struct + // currently makes zero sense to call this with a null info pointer, so zero the struct // if the pointer is not null and bail out otherwise - if( col_info not_eq nullptr ) { - col_info->Wipe(); + if( info not_eq nullptr ) { + info->Wipe(); } else { return KELPIE_EINVAL; } @@ -398,11 +407,11 @@ namespace kelpie { const CassRow* row = cass_result_first_row( result ); cass_value_get_int64( cass_row_get_column( row, 0 ), &ms ); cass_value_get_int64( cass_row_get_column( row, 1 ), &ds ); - col_info->num_bytes = ms + ds; - col_info->availability = kelpie::Availability::InDisk; + info->col_user_bytes = ms + ds; + info->col_availability = kelpie::Availability::InDisk; krc = KELPIE_OK; } else { - col_info->availability = kelpie::Availability::Unavailable; + info->col_availability = kelpie::Availability::Unavailable; krc = KELPIE_ENOENT; } @@ -415,6 +424,7 @@ namespace kelpie { return krc; } + } } diff --git a/src/kelpie/ioms/IomCassandra.hh b/src/kelpie/ioms/IomCassandra.hh index 671f50e..e18bffd 100644 --- a/src/kelpie/ioms/IomCassandra.hh +++ b/src/kelpie/ioms/IomCassandra.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_IOMCASSANDRA_HH #define KELPIE_IOMCASSANDRA_HH @@ -28,19 +28,21 @@ namespace kelpie { virtual ~IomCassandra(); rc_t GetInfo( faodel::bucket_t bucket, - const kelpie::Key &key, - kv_col_info_t *col_info ) override; + const kelpie::Key &key, + object_info_t *info ) override; - void WriteObject( faodel::bucket_t bucket, - const kelpie::Key &key, - const lunasa::DataObject &ldo ) override; + rc_t WriteObject( faodel::bucket_t bucket, + const kelpie::Key &key, + const lunasa::DataObject &ldo ) override; - void WriteObjects( faodel::bucket_t bucket, - const std::vector< kvpair >& kvpairs ); +// void WriteObjects( faodel::bucket_t bucket, +// const std::vector< kvpair >& kvpairs ); + void internal_WriteObject( faodel::bucket_t bucket, + const std::vector &kvpairs); rc_t ReadObject( faodel::bucket_t bucket, - const kelpie::Key& key, - lunasa::DataObject* ldo ) override; + const kelpie::Key& key, + lunasa::DataObject* ldo ) override; constexpr static char type_str[] = "cassandra"; @@ -50,6 +52,18 @@ namespace kelpie { std::string reference_link, const std::map< std::string, std::string >& args ) override; + /// Return a list of all the setting names this IOM accepts at construction and provide a brief description for each + static const std::vector> ValidSettingNamesAndDescriptions() { + return { + {"endpoint", "Information necessary for connecting to a Cassandra endpoint "}, + {"keyspace", "Which keyspace to use in Cassandra"}, + {"table", "The Cassandra table to access"}, + {"teardown", "Drop the Cassandra keyspace when IOM is destroyed"}, + {"cass-replication-class", "Which Cassandra replication strategy to follow (Defaults to SimpleStrategy)"}, + {"cass-replication-factor","How many replications (defaults to 1)"} + }; + } + void sstr( std::stringstream& ss, int depth = 0, int indent = 0 ) const override; diff --git a/src/kelpie/ioms/IomHDF5.cpp b/src/kelpie/ioms/IomHDF5.cpp index 00b27da..e4544a5 100644 --- a/src/kelpie/ioms/IomHDF5.cpp +++ b/src/kelpie/ioms/IomHDF5.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -54,12 +54,18 @@ IomHDF5::~IomHDF5() { } -void IomHDF5::WriteObject(faodel::bucket_t bucket, const Key &key, const lunasa::DataObject &ldo) { - WriteObjects(bucket, {kvpair(key, ldo)}); +rc_t IomHDF5::WriteObject(faodel::bucket_t bucket, const Key &key, const lunasa::DataObject &ldo) { + try { + internal_WriteObject(bucket, {kvpair(key, ldo)}); + return 0; + } + catch (std::runtime_error &e) { + return 1; + } } -void IomHDF5::WriteObjects(faodel::bucket_t bucket, - const std::vector &kvpairs) { +void IomHDF5::internal_WriteObject(faodel::bucket_t bucket, + const std::vector &kvpairs) { hid_t ldo_dset; hvl_t dset_descriptor; @@ -135,22 +141,22 @@ void IomHDF5::WriteObjects(faodel::bucket_t bucket, } -rc_t IomHDF5::ReadObjects(faodel::bucket_t bucket, - std::vector &keys, - std::vector *found_keys, - std::vector *missing_keys) { - for(auto &&k : keys) { - lunasa::DataObject ldo; - rc_t each_rc = ReadObject(bucket, k, &ldo); - if(each_rc == KELPIE_OK) { - found_keys->push_back({kvpair(k, ldo)}); - } else { - missing_keys->push_back(k); - } - } - - return KELPIE_OK; -} +//rc_t IomHDF5::ReadObjects(faodel::bucket_t bucket, +// std::vector &keys, +// std::vector *found_keys, +// std::vector *missing_keys) { +// for(auto &&k : keys) { +// lunasa::DataObject ldo; +// rc_t each_rc = ReadObject(bucket, k, &ldo); +// if(each_rc == KELPIE_OK) { +// found_keys->push_back({kvpair(k, ldo)}); +// } else { +// missing_keys->push_back(k); +// } +// } +// +// return KELPIE_OK; +//} rc_t IomHDF5::ReadObject(faodel::bucket_t bucket, const kelpie::Key &key, @@ -347,11 +353,10 @@ void IomHDF5::AppendWebInfo(faodel::ReplyStream rs, } -rc_t IomHDF5::GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col_info_t *col_info) { +rc_t IomHDF5::GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, object_info_t *info) { rc_t rc; - if(col_info not_eq nullptr) - col_info->Wipe(); + if(info) info->Wipe(); std::string target = "/" + bucket.GetHex() + "/" + key.str(); @@ -360,8 +365,8 @@ rc_t IomHDF5::GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col_in if(meta_attr_id<0 || data_attr_id<0) { - if(col_info not_eq nullptr) { - col_info->availability = kelpie::Availability::Unavailable; + if(info not_eq nullptr) { + info->col_availability = kelpie::Availability::Unavailable; } rc = KELPIE_ENOENT; @@ -373,9 +378,9 @@ rc_t IomHDF5::GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col_in H5Aread(meta_attr_id, H5T_NATIVE_USHORT, &ldo_meta_size); H5Aread(data_attr_id, H5T_NATIVE_ULONG, &ldo_data_size); - if(col_info not_eq nullptr) { - col_info->num_bytes = ldo_meta_size + ldo_data_size; - col_info->availability = kelpie::Availability::InDisk; + if(info) { + info->col_user_bytes = ldo_meta_size + ldo_data_size; + info->col_availability = kelpie::Availability::InDisk; } rc = KELPIE_OK; @@ -388,6 +393,7 @@ rc_t IomHDF5::GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col_in } + } // namespace internal } // namespace kelpie diff --git a/src/kelpie/ioms/IomHDF5.hh b/src/kelpie/ioms/IomHDF5.hh index 874f560..ecae063 100644 --- a/src/kelpie/ioms/IomHDF5.hh +++ b/src/kelpie/ioms/IomHDF5.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_IOMHDF_HH #define KELPIE_IOMHDF_HH @@ -26,17 +26,17 @@ public: ~IomHDF5(); - rc_t GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col_info_t *col_info) override; + rc_t GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, object_info_t *info) override; - void WriteObject(faodel::bucket_t, const kelpie::Key &key, const lunasa::DataObject &ldo) override; - - void WriteObjects(faodel::bucket_t, const std::vector &kvpairs); + rc_t WriteObject(faodel::bucket_t, const kelpie::Key &key, const lunasa::DataObject &ldo) override; + void internal_WriteObject(faodel::bucket_t bucket, + const std::vector &kvpairs); rc_t ReadObject(faodel::bucket_t, const kelpie::Key &key, lunasa::DataObject *ldo) override; - rc_t ReadObjects(faodel::bucket_t, std::vector &keys, - std::vector *found_keys, - std::vector *missing_keys); +// rc_t ReadObjects(faodel::bucket_t, std::vector &keys, +// std::vector *found_keys, +// std::vector *missing_keys); constexpr static char type_str[] = "HDF5"; @@ -45,6 +45,14 @@ public: void AppendWebInfo(faodel::ReplyStream rs, std::string reference_link, const std::map &args) override; + /// Return a list of all the setting names this IOM accepts at construction and provide a brief description for each + static const std::vector> ValidSettingNamesAndDescriptions() { + return { + {"path", "The path that the IOM writer should use for storing data"}, + {"unique", "An additional marker appended to path to make this instance unique"} + }; + } + void sstr(std::stringstream &ss, int depth = 0, int indent = 0) const override; protected: diff --git a/src/kelpie/ioms/IomLevelDB.cpp b/src/kelpie/ioms/IomLevelDB.cpp index ba46df0..204cfa2 100644 --- a/src/kelpie/ioms/IomLevelDB.cpp +++ b/src/kelpie/ioms/IomLevelDB.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -69,14 +69,19 @@ leveldb::DB * IomLevelDB::bucketToDB(faodel::bucket_t &bkt) { } -void IomLevelDB::WriteObject(faodel::bucket_t bucket, - const kelpie::Key &key, - const lunasa::DataObject &ldo) { - WriteObjects(bucket, {kvpair(key, ldo)}); +rc_t IomLevelDB::WriteObject(faodel::bucket_t bucket, + const kelpie::Key &key, + const lunasa::DataObject &ldo) { + try { + internal_WriteObject(bucket, {kvpair(key, ldo)}); + return 0; + } catch (std::runtime_error &e) { + return 1; + } } -void IomLevelDB::WriteObjects(faodel::bucket_t bucket, - const std::vector &kvpairs) { +void IomLevelDB::internal_WriteObject(faodel::bucket_t bucket, + const std::vector &kvpairs) { leveldb::WriteBatch batch; size_t wr_amt = 0; @@ -104,7 +109,7 @@ void IomLevelDB::WriteObjects(faodel::bucket_t bucket, leveldb::Status status = db->Write(leveldb::WriteOptions(), &batch); if(not status.ok()) { - std::cerr << "leveldb write failed: " << status.ToString() << std::endl; + throw std::runtime_error("IomLevelDB::leveldb write failed: " + status.ToString()); } else { stat_wr_requests += kvpairs.size(); stat_wr_bytes += wr_amt; @@ -138,22 +143,22 @@ rc_t IomLevelDB::ReadObject(faodel::bucket_t bucket, return rc; } -rc_t IomLevelDB::ReadObjects(faodel::bucket_t bucket, - std::vector &keys, - std::vector *found_keys, - std::vector *missing_keys) { - for(auto &&k : keys) { - lunasa::DataObject ldo; - rc_t each_rc = ReadObject(bucket, k, &ldo); - if(each_rc == KELPIE_OK) { - found_keys->push_back({kvpair(k, ldo)}); - } else { - missing_keys->push_back(k); - } - } - - return KELPIE_OK; -} +//rc_t IomLevelDB::ReadObjects(faodel::bucket_t bucket, +// std::vector &keys, +// std::vector *found_keys, +// std::vector *missing_keys) { +// for(auto &&k : keys) { +// lunasa::DataObject ldo; +// rc_t each_rc = ReadObject(bucket, k, &ldo); +// if(each_rc == KELPIE_OK) { +// found_keys->push_back({kvpair(k, ldo)}); +// } else { +// missing_keys->push_back(k); +// } +// } +// +// return KELPIE_OK; +//} void IomLevelDB::sstr(std::stringstream &ss, int depth, int index) const { ss << std::string(index, ' ') + "IomLevelDB Path: " << path_ << std::endl; @@ -214,24 +219,23 @@ void IomLevelDB::AppendWebInfo(faodel::ReplyStream rs, } -rc_t IomLevelDB::GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col_info_t *col_info) { +rc_t IomLevelDB::GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, object_info_t *info) { rc_t rc; std::string val; - if(col_info not_eq nullptr) - col_info->Wipe(); + if(info) info->Wipe(); leveldb::DB *db = bucketToDB(bucket); leveldb::Status s = db->Get(leveldb::ReadOptions(), key.str(), &val); if(s.ok()) { - if(col_info not_eq nullptr) { - col_info->num_bytes = val.size(); - col_info->availability = kelpie::Availability::InDisk; + if(info not_eq nullptr) { + info->col_user_bytes = val.size(); + info->col_availability = kelpie::Availability::InDisk; } rc = KELPIE_OK; } else { - if(col_info not_eq nullptr) { - col_info->availability = kelpie::Availability::Unavailable; + if(info not_eq nullptr) { + info->col_availability = kelpie::Availability::Unavailable; } rc = KELPIE_ENOENT; } @@ -239,7 +243,6 @@ rc_t IomLevelDB::GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col return rc; } - } // namespace internal } // namespace kelpie diff --git a/src/kelpie/ioms/IomLevelDB.hh b/src/kelpie/ioms/IomLevelDB.hh index 77bd577..16dd89e 100644 --- a/src/kelpie/ioms/IomLevelDB.hh +++ b/src/kelpie/ioms/IomLevelDB.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_IOMLEVELDB_HH #define KELPIE_IOMLEVELDB_HH @@ -26,17 +26,16 @@ public: ~IomLevelDB(); - rc_t GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col_info_t *col_info) override; + rc_t GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, object_info_t *info) override; - void WriteObject(faodel::bucket_t, const kelpie::Key &key, const lunasa::DataObject &ldo) override; - - void WriteObjects(faodel::bucket_t, const std::vector &kvpairs); + rc_t WriteObject(faodel::bucket_t, const kelpie::Key &key, const lunasa::DataObject &ldo) override; + void internal_WriteObject(faodel::bucket_t, const std::vector &kvpairs); rc_t ReadObject(faodel::bucket_t, const kelpie::Key &key, lunasa::DataObject *ldo) override; - rc_t ReadObjects(faodel::bucket_t, std::vector &keys, - std::vector *found_keys, - std::vector *missing_keys); +// rc_t ReadObjects(faodel::bucket_t, std::vector &keys, +// std::vector *found_keys, +// std::vector *missing_keys); constexpr static char type_str[] = "LevelDB"; @@ -45,6 +44,13 @@ public: void AppendWebInfo(faodel::ReplyStream rs, std::string reference_link, const std::map &args) override; + /// Return a list of all the setting names this IOM accepts at construction and provide a brief description for each + static const std::vector> ValidSettingNamesAndDescriptions() { + return { + {"path", "The path that the IOM writer should use for storing data"}, + {"unique", "An additional marker appended to path to make this instance unique"} + }; + } void sstr(std::stringstream &ss, int depth = 0, int indent = 0) const override; protected: diff --git a/src/kelpie/ioms/IomPosixIndividualObjects.cpp b/src/kelpie/ioms/IomPosixIndividualObjects.cpp index 33c646e..8349fca 100644 --- a/src/kelpie/ioms/IomPosixIndividualObjects.cpp +++ b/src/kelpie/ioms/IomPosixIndividualObjects.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -10,6 +10,7 @@ #include #include #include +#include #include "faodel-common/StringHelpers.hh" @@ -24,18 +25,17 @@ namespace internal { constexpr char IomPosixIndividualObjects::type_str[]; IomPosixIndividualObjects::IomPosixIndividualObjects(std::string name, const map &new_settings) - : IomBase(name, new_settings, {"path"}) { + : IomBase(name, new_settings, {"path","path.env_name"}) { //Set debug info SetSubcomponentName("-pio-"+name); - auto ii = settings.find("path"); - if(ii == settings.end()) { + //Resolve the path from config settings. May go out to env var is "path.env_name" used. + path = faodel::GetPathFromComponentSettings(settings); + if(path.empty()){ throw std::runtime_error("Iom "+name+" lacked a setting for 'path'"); } - path = ii->second; - if(path.back()!='/') path=path+"/"; - + //TODO Replace this C nastiness with c++17 stuff, or throw in a lib //TODO Add a directory traversal in case iom is in writable space @@ -74,35 +74,35 @@ IomPosixIndividualObjects::IomPosixIndividualObjects(std::string name, const map throw std::runtime_error(ss.str()); } -rc_t IomPosixIndividualObjects::GetInfo(faodel::bucket_t bucket, const Key &key, kv_col_info_t *col_info) { +rc_t IomPosixIndividualObjects::GetInfo(faodel::bucket_t bucket, const Key &key, object_info_t *info) { rc_t rc; dbg("GetInfo for "+key.str()); - if(col_info) - col_info->Wipe(); + if(info) info->Wipe(); string fname = genBucketPathFile(bucket, key); struct stat sb; if((stat(fname.c_str(), &sb)==0) && (S_ISREG(sb.st_mode))){ - if(col_info){ - col_info->num_bytes = sb.st_size; - col_info->availability = Availability::InDisk; + if(info){ + info->col_user_bytes = sb.st_size - lunasa::DataObject::GetHeaderSize(); + info->col_availability = Availability::InDisk; } rc = KELPIE_OK; } else { - if(col_info) { - col_info->availability = Availability::Unavailable; + if(info) { + info->col_availability = Availability::Unavailable; } rc = KELPIE_ENOENT; } return rc; } -void IomPosixIndividualObjects::WriteObject(faodel::bucket_t bucket, const Key &key, const lunasa::DataObject &ldo) { +rc_t IomPosixIndividualObjects::WriteObject(faodel::bucket_t bucket, const Key &key, const lunasa::DataObject &ldo) { dbg("WriteObject "+key.str()); string fname = genBucketPathFile(bucket, key); - ldo.writeToFile(fname.c_str()); + uint32_t rc = ldo.writeToFile(fname.c_str()); stat_wr_requests++; stat_wr_bytes+=ldo.GetWireSize(); + return (rc==0) ? KELPIE_OK : KELPIE_EIO; } @@ -118,14 +118,100 @@ rc_t IomPosixIndividualObjects::ReadObject(faodel::bucket_t bucket, const Key &k if(ldo!=nullptr) { *ldo = lunasa::DataObject(0, sb.st_size, lunasa::DataObject::AllocatorType::eager); ldo->readFromFile(fname.c_str()); - stat_rd_bytes+=sb.st_size; } + stat_rd_bytes+=sb.st_size; + stat_rd_hits++; return KELPIE_OK; + } else { + stat_rd_misses; return KELPIE_ENOENT; } } +rc_t IomPosixIndividualObjects::ListObjects(faodel::bucket_t bucket, Key key, ObjectCapacities *oc) { + + // NOTES: + // * Only currently supported wildcard is a glob ('*') suffix + // * Wildcards are only currently supported for the column of the Key + + // HANDLE wildcard keys (wildcard characters, e.g., '*', should not be encoded into the path name) + bool k1_is_wild=key.IsRowWildcard(); + bool k2_is_wild=key.IsColWildcard(); + string k1 = key.K1(); + string k2 = key.K2(); + if(k1_is_wild) k1=k1.substr(0, k1.size()-1); + if(k2_is_wild) k2=k2.substr(0, k2.size()-1); + + + string bucket_path = genBucketPath(bucket); + + DIR *dp; + struct dirent *ep; + dp = opendir(bucket_path.c_str()); + if(dp!=NULL) { + while((ep = readdir(dp)) != NULL) { + string name = string(ep->d_name); + if(!((name == ".") || (name == ".."))) { + string pname = bucket_path + "/" + name; + struct stat sb; + if((stat(pname.c_str(), &sb) == 0) && (S_ISREG(sb.st_mode))) { + Key key; + key.pup(faodel::ExpandPunycode(name)); + + //See if actually matches + if(key.matchesPrefixString(k1_is_wild, k1, k2_is_wild, k2)) { + oc->keys.push_back(key); + oc->capacities.push_back(sb.st_size); + } + } + } + } + closedir(dp); + } + +#if 0 + bool wildcard = key.IsColWildcard(); + + cout<<"PIO-LIST: is wildcard: "<keys.push_back(k); + + cout <<"PIO-LIST: recovered key "<capacities.push_back(buffer.st_size); + } +#endif + + return KELPIE_OK; +} /** * @brief Generate a file path down to the bucket identifier (w/ trailing slash) @@ -151,10 +237,28 @@ string IomPosixIndividualObjects::genBucketPath(faodel::bucket_t bucket) { ss <<"Could not write to '"<pup(s); + return 0; +} + vector IomPosixIndividualObjects::getBucketNames(){ vector buckets; DIR *dp; @@ -211,6 +315,8 @@ void IomPosixIndividualObjects::AppendWebInfo(faodel::ReplyStream rs, string ref {"Path", path }, {"Write Requests", to_string(stat_wr_requests)}, {"Read Requests", to_string(stat_rd_requests)}, + {"Read Request Hits", to_string(stat_rd_hits)}, + {"Read Request Misses", to_string(stat_rd_misses)}, {"Write Bytes", to_string(stat_wr_bytes)}, {"Read Bytes", to_string(stat_rd_bytes)} }; @@ -257,5 +363,6 @@ void IomPosixIndividualObjects::sstr(stringstream &ss, int depth, int indent) co ss << string(indent,' ') + "IomPosixIndividualObjects Path: "< &new_settings); ~IomPosixIndividualObjects() override {} - rc_t GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, kv_col_info_t *col_info) override; - void WriteObject(faodel::bucket_t bucket, const kelpie::Key &key, const lunasa::DataObject &ldo) override; + rc_t GetInfo(faodel::bucket_t bucket, const kelpie::Key &key, object_info_t *info) override; + rc_t WriteObject(faodel::bucket_t bucket, const kelpie::Key &key, const lunasa::DataObject &ldo) override; rc_t ReadObject(faodel::bucket_t bucket, const kelpie::Key &key, lunasa::DataObject *ldo) override; //WriteObjects and ReadObjects come from base class + rc_t ListObjects(faodel::bucket_t bucket, Key key, ObjectCapacities *oc) override; + constexpr static char type_str[] = "PosixIndividualObjects"; std::string Type() const override { return IomPosixIndividualObjects::type_str; }; void AppendWebInfo(faodel::ReplyStream rs, std::string reference_link, const std::map &args) override; - + + + /// Return a list of all the setting names this IOM accepts at construction and provide a brief description for each + static const std::vector> ValidSettingNamesAndDescriptions() { + return { + {"path", "The path that the IOM writer should use for storing data"} + }; + } + //Info interface void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; @@ -45,6 +55,7 @@ private: std::string path; std::string genBucketPath(faodel::bucket_t bucket); std::string genBucketPathFile(faodel::bucket_t bucket, const kelpie::Key &key); + int getKeyFromBucketPathFile(const std::string path, Key *key); std::vector getBucketNames(); std::vector> getBucketContents(std::string bucket); }; diff --git a/src/kelpie/ioms/IomRegistry.cpp b/src/kelpie/ioms/IomRegistry.cpp index a3df245..8810b70 100644 --- a/src/kelpie/ioms/IomRegistry.cpp +++ b/src/kelpie/ioms/IomRegistry.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -9,6 +9,8 @@ #include "faodel-common/MutexWrapper.hh" #include "faodel-common/StringHelpers.hh" #include "faodel-common/LoggingInterface.hh" +#include "faodel-common/ResourceURL.hh" +#include "faodel-common/StringHelpers.hh" #include "whookie/Server.hh" #include "kelpie/ioms/IomRegistry.hh" @@ -89,12 +91,63 @@ void IomRegistry::RegisterIom(string type, string name, const map } +/** + * @brief Register an IOM based on settings encoded in a resource url + * @param url The resource url that contains iom info in its options + * @retval 0 IOM was successfully registered + * @retval -1 IOM could not be registered + * + * This is intended to be used by tools like the cli that allow users to + * define resources that are later instantiated. When the cli tool launches a + * kelpie server, the server retrieves a dirman entry that contains a resource + * url for the server. That url may include all of the info the server needs + * to setup the iom. A valid iom inside a url will have the following optins: + * + * iom_name : The name of the iom + * iom_type : What kind of iom driver to use. eg PosixIndividualObjects + * + * Additional driver-specific options may include: + * path : The path PIO should use to write objects + */ +int IomRegistry::RegisterIomFromURL(const faodel::ResourceURL &url) { + + //Convert all options + auto options_vector = url.GetOptions(); + map options_map; + string iom_name; + string iom_type; + for(auto &k_v : options_vector) { + string k = k_v.first; + faodel::ToLowercaseInPlace(k); + if(k=="iom") { iom_name = k_v.second; + } else if (k=="iom_type") { iom_type = k_v.second; + } else if (faodel::StringBeginsWith(k, "iom_")) { + k.erase(0,4); //Remove prefix to be compatible with default iom syntax + if(!k.empty()) options_map[k] = k_v.second; + } + } + if(iom_name.empty() || iom_type.empty()){ + dbg("Error: attempted register iom from url, but didn't have iom_name/iom_type defined"); + return -1; + } + try { + RegisterIom(iom_type, iom_name, options_map); + } catch(exception &e) { + dbg("Error: could not register iom due to '"+string(e.what())+"' Note: all settings must have 'iom_' prefix, which is stripped off during registration"); + return -1; + } + + return 0; +} + + /** * @brief Register a new iom driver with kelpie - * @param type The name of this driver - * @param constructor_function A constructor for building an instance of the driver + * @param[in] type The name of this driver + * @param[in] constructor_function A constructor for building an instance of the driver + * @param[in] valid_settings_function A function for validating params */ -void IomRegistry::RegisterIomConstructor(string type, fn_IomConstructor_t constructor_function) { +void IomRegistry::RegisterIomConstructor(string type, fn_IomConstructor_t constructor_function, fn_IomGetValidSetting_t valid_settings_function) { dbg("Registering iom driver for type "+type); @@ -105,9 +158,10 @@ void IomRegistry::RegisterIomConstructor(string type, fn_IomConstructor_t constr auto name_fn = iom_ctors.find(type); if(name_fn != iom_ctors.end()){ - KWARN("Overwriting iom constroctor "+type); + F_WARN("Overwriting iom constructor "+type); } iom_ctors[type] = constructor_function; + iom_valid_setting_fns[type] = valid_settings_function; } @@ -127,7 +181,7 @@ void IomRegistry::init(const faodel::Configuration &config) { fn_IomConstructor_t fn_pio = [] (string name, const map &settings) -> IomBase * { return new IomPosixIndividualObjects(name, settings); }; - RegisterIomConstructor("posixindividualobjects", fn_pio); + RegisterIomConstructor("posixindividualobjects", fn_pio, IomPosixIndividualObjects::ValidSettingNamesAndDescriptions); // @@ -137,39 +191,40 @@ void IomRegistry::init(const faodel::Configuration &config) { fn_IomConstructor_t fn_ldb = [] (string name, const map< string, string > &settings) -> IomBase * { return new IomLevelDB( name, settings ); }; - RegisterIomConstructor( "leveldb", fn_ldb ); + RegisterIomConstructor( "leveldb", fn_ldb, IomLevelDB::ValidSettingNamesAndDescriptions ); #endif #ifdef FAODEL_HAVE_HDF5 fn_IomConstructor_t fn_hdf5 = [] (string name, const map< string, string > &settings) -> IomBase * { return new IomHDF5( name, settings ); }; - RegisterIomConstructor( "hdf5", fn_hdf5 ); + RegisterIomConstructor( "hdf5", fn_hdf5, IomHDF5::ValidSettingNamesAndDescriptions ); #endif #ifdef FAODEL_HAVE_CASSANDRA fn_IomConstructor_t fn_cassandra = [] (string name, const map< string, string > &settings) -> IomBase * { return new IomCassandra( name, settings ); }; - RegisterIomConstructor( "cassandra", fn_cassandra ); + RegisterIomConstructor( "cassandra", fn_cassandra, IomCassandra::ValidSettingNamesAndDescriptions ); #endif //Get the list of Ioms this Configuration wants to use string s,role; - config.GetString(&s, "ioms"); + config.GetString(&s, "kelpie.ioms"); role = config.GetRole(); - if(s!=""){ + if(!s.empty()) { + dbg("Registering "+s); vector names = faodel::Split(s,';',true); for(auto &name : names) { - //Get all settings for this iom. Do hierarchy of default, iom.name, then role.iom.name + //Get all settings for this iom. Do hierarchy of default, kelpie.iom.name, then role.kelpie.iom.name map settings; - config.GetComponentSettings(&settings, "default.iom"); - config.GetComponentSettings(&settings, "iom."+name); - config.GetComponentSettings(&settings, role+".iom."+name); + config.GetComponentSettings(&settings, "default.kelpie.iom"); + config.GetComponentSettings(&settings, "kelpie.iom."+name); + config.GetComponentSettings(&settings, role+".kelpie.iom."+name); string emsg = ""; string type = faodel::ToLowercase(settings["type"]); @@ -184,8 +239,10 @@ void IomRegistry::init(const faodel::Configuration &config) { emsg="Iom type '"+type+"' is unknown. Deferred iom types not currently supported"; //TODO add ability to defer unknown types until start } - if(emsg!="") + if(!emsg.empty()) { + dbg("IOM Configuration error: "+emsg); throw std::runtime_error("IOM Configuration error. "+emsg); + } //Do the actual creation RegisterIom(type, name, settings); @@ -325,6 +382,47 @@ void IomRegistry::HandleWhookieStatus(const std::map &a } } +/** + * @brief Get a list of all the ioms that are available + * @return Vector of names + */ +vector IomRegistry::GetIomNames() const { + vector names; + mutex->ReaderLock(); + for(auto &hash_iom : ioms_by_hash_pre) { + names.push_back( hash_iom.second->Name() ); + } + for(auto &hash_iom : ioms_by_hash_post) { + names.push_back( hash_iom.second->Name() ); + } + mutex->Unlock(); + return names; +} + +/** + * @brief Get a list of all the registered iom types + * @return List of type names + */ +vector IomRegistry::RegisteredTypes() const { + vector names; + for(auto &name_fn : iom_ctors) + names.push_back(name_fn.first); + return names; +} + +/** + * @brief Get a list of parameters this IOM accepts when it is constructed + * @param type The IOM type to lookup + * @return Vector with the name and description of each parameter + */ +vector> IomRegistry::RegisteredTypeParameters(const string &type) const { + auto name_fn = iom_valid_setting_fns.find(type); + if( (name_fn== iom_valid_setting_fns.end()) || (name_fn->second == nullptr)) { + return {}; //No entry + } + return name_fn->second(); +} + void IomRegistry::sstr(stringstream &ss, int depth, int indent) const { if(depth<0) return; diff --git a/src/kelpie/ioms/IomRegistry.hh b/src/kelpie/ioms/IomRegistry.hh index 3aada57..2376ba0 100644 --- a/src/kelpie/ioms/IomRegistry.hh +++ b/src/kelpie/ioms/IomRegistry.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_IOMREGISTRY_HH #define KELPIE_IOMREGISTRY_HH @@ -44,11 +44,18 @@ public: void finish(); void RegisterIom(std::string type, std::string name, const std::map &settings); - void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function); + int RegisterIomFromURL(const faodel::ResourceURL &url); + + void RegisterIomConstructor(std::string type, fn_IomConstructor_t ctor_function, fn_IomGetValidSetting_t valid_settings_function=nullptr); IomBase * Find(std::string iom_name) { return Find(faodel::hash32(iom_name)); } IomBase * Find(iom_hash_t iom_hash); - + + std::vector GetIomNames() const; + + std::vector RegisteredTypes() const; + std::vector> RegisteredTypeParameters(const std::string &type) const; + //InfoInterface function void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; @@ -60,6 +67,7 @@ private: std::map ioms_by_hash_post; std::map iom_ctors; + std::map iom_valid_setting_fns; void HandleWhookieStatus(const std::map &args, std::stringstream &results); }; diff --git a/src/kelpie/localkv/LocalKV.cpp b/src/kelpie/localkv/LocalKV.cpp index 7bc8a66..4c3027f 100644 --- a/src/kelpie/localkv/LocalKV.cpp +++ b/src/kelpie/localkv/LocalKV.cpp @@ -1,16 +1,19 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -#include -#include +#include +#include #include #include "faodel-common/Debug.hh" #include "kelpie/localkv/LocalKV.hh" +#include "kelpie/ioms/IomBase.hh" +#include "kelpie/core/Singleton.hh" using namespace std; using namespace faodel; +using namespace kelpie::lkv; namespace kelpie { @@ -44,15 +47,11 @@ LocalKV::~LocalKV(){ */ rc_t LocalKV::Init(const Configuration &config){ - kassert(!configured, "Attempted to call LocalKV Init more than once"); + F_ASSERT(!configured, "Attempted to call LocalKV Init more than once"); //Enable our configuration ConfigureLogging(config); - - //Set our max capacity - config.GetInt(&max_capacity, "kelpie.lkv.max_capacity","1M"); - //Create our mutex table_mutex = config.GenerateComponentMutex("kelpie.lkv", "rwlock"); configured=true; @@ -76,9 +75,9 @@ rc_t LocalKV::Init(const Configuration &config){ * @param[in] bucket The user id that is marked as the bucket of this data * @param[in] key The key used to reference this data * @param[in] new_ldo The Lunasa Data Object that the lkv will use to reference an object - * @param[in] behavior_flags Infor about how the lkv should handle new data - * @param[out] row_info Information about the row, after update - * @param[out] col_info Information about the column, after update + * @param[in] behavior_flags Info about how the lkv should handle new data + * @param[in] iom Pointer to the iom that may be affected by this put + * @param[out] info Information about the row/column, after update * @retval KELPIE_OK Success with no triggers * @retval KELPIE_TODO Success with triggers that need to be handled * @retval KELPIE_EEXIST Entry already exists and therefore was not written @@ -88,23 +87,28 @@ rc_t LocalKV::Init(const Configuration &config){ rc_t LocalKV::put(bucket_t bucket, const Key &key, lunasa::DataObject const &new_ldo, pool_behavior_t behavior_flags, - kv_row_info_t *row_info, - kv_col_info_t *col_info){ - - kassert(key.valid(), "Put given invalid key"); - - dbg("Put "+bucket.GetHex()+"|"+key.str()+" length "+to_string(new_ldo.GetUserSize())); + internal::IomBase *iom, + object_info_t *info) { - bool create_if_missing = (behavior_flags & PoolBehavior::WriteToLocal); //Some pools just need to notify others - bool trigger_dependency_check = true; + F_ASSERT(key.valid(), "Put given invalid key"); + dbg("Put "+bucket.GetHex()+"|"+key.str()+" length "+to_string(new_ldo.GetUserSize())+" behavior: "+to_string(behavior_flags)); - rc_t rc = doColOp(bucket, key, create_if_missing, trigger_dependency_check, row_info, col_info, //Pub creates if missing and triggers deps - [&new_ldo, key] (LocalKVRow &row, LocalKVCell &col, bool previously_existed) { + //Only create if writing to local, but always see if we trigger dependencies + lambda_flags_t lambda_flags = LambdaFlags::TRIGGER_DEPENDENCIES; + if (behavior_flags & PoolBehavior::WriteToLocal) { + lambda_flags |= lkv::LambdaFlags::CREATE_IF_MISSING; + } - //TODO: update to truely use new behaviors. Currently, this ignores a WriteToLocal=0 if dependencies or entry exist - if(col.availability==Availability::InLocalMemory){ - //Existing version? ignore? + rc_t rc = doColOp(bucket, key, + lambda_flags, //Pub triggers dependencies, and may need create + info, + [&new_ldo, key, behavior_flags] (LocalKVRow &row, LocalKVCell &col, bool previously_existed) { + + //Bail out if this already exists in memory and we aren't overwriting things. + if( (col.availability==Availability::InLocalMemory) && + (!(behavior_flags & PoolBehavior::EnableOverwrites)) ) { + //Exists - ignore updates return KELPIE_EEXIST; } //New item. Entry created, we just need to fill in the data @@ -115,6 +119,19 @@ rc_t LocalKV::put(bucket_t bucket, const Key &key, return KELPIE_OK; }); + dbg("put to lkv returned "+to_string(rc)); + + //See if we need to write out to storage + if(behavior_flags & PoolBehavior::WriteToIOM) { + if(iom) { + rc_t rc2 = iom->WriteObject(bucket, key, new_ldo); + if(rc==KELPIE_OK) rc=rc2; //Capture first error code + } else { + rc=KELPIE_EIO; //Asked us to use an IOM but it didn't work + } + } + + return rc; } @@ -123,8 +140,7 @@ rc_t LocalKV::put(bucket_t bucket, const Key &key, * @param[in] bucket The user id that is marked as the bucket of this data * @param[in] key The key used to reference this data * @param[out] ext_ldo A new (reference counted) ldo that provides access to the data stored in the lkv - * @param[out] row_info Information about the row this ldo is/should live in - * @param[out] col_info Information about the column this ldo is/should live in + * @param[out] info Information about the row this ldo is/should live in * @retval KELPIE_OK Succes * @retval KELPIE_ENOENT Data not here, but node was placed on waiting list * @retval KELPIE_WAITING Data not here, but request has already been made @@ -135,15 +151,15 @@ rc_t LocalKV::put(bucket_t bucket, const Key &key, */ rc_t LocalKV::get(faodel::bucket_t bucket, const Key &key, lunasa::DataObject *ext_ldo, - kv_row_info_t *row_info, - kv_col_info_t *col_info) { + object_info_t *info) { - kassert(key.valid(), "get given invalid key"); + F_ASSERT(key.valid(), "get given invalid key"); dbg("Get "+bucket.GetHex()+"|"+key.str()); - rc_t rc = doColOp(bucket, key, false, false, //<--don't create an entry if it doesn't exist, don't trigger deps - row_info, col_info, + rc_t rc = doColOp(bucket, key, + lkv::LambdaFlags::DONT_CREATE_OR_TRIGGER, //<--Get doesn't create or trigger + info, [ext_ldo] (LocalKVRow &row, LocalKVCell &col, bool previously_existed) { if(col.availability == Availability::InLocalMemory){ @@ -156,91 +172,112 @@ rc_t LocalKV::get(faodel::bucket_t bucket, const Key &key, return rc; } -#if 0 //Do we need this anymore -/** - * @brief Get a Lunasa Data Object back for a desired key, if available - * @param[in] bucket The user id that is marked as the bucket of this data - * @param[in] key The key used to reference this data - * @param[in] requesting_node_id The node that wants the data, in cases where a node should be put on the waiting list - * @param[out] ext_ldo A new (reference counted) ldo that provides access to the data stored in the lkv - * @param[out] row_info Information about the row this ldo is/should live in - * @param[out] col_info Information about the column this ldo is/should live in - * @retval KELPIE_OK Succes - * @retval KELPIE_ENOENT Data not here, but node was placed on waiting list - * @retval KELPIE_WAITING Data not here, but request has already been made - * @retval KELPIE_TODO Needed load from disk - * @retval KELPIE_EINVAL Invalid input (eg, requesting_node_id was UNSPECIFIED) - * @note This version passes back an ldo reference to the data. Modifying the data could have side effects. - * @todo Put some logic here to disable wait lists? - */ -rc_t LocalKV::get(bucket_t bucket, const Key &key, - nodeid_t requesting_node_id_if_missing, - lunasa::DataObject *ext_ldo, - kv_row_info_t *row_info, - kv_col_info_t *col_info) { - kassert(key.valid(), "get given invalid key"); +rc_t LocalKV::getAvailable(faodel::bucket_t bucket, const Key &key, + std::map &ldos) { - dbg("Get "+bucket.GetHex()+"|"+key.str()); + F_ASSERT(key.valid(), "getAvailable given invalid key"); + F_ASSERT(!key.IsRowWildcard(), "getAvailable given a row wildcard"); - rc_t rc = doColOp(bucket, key, true, false, //<--Creates a node dependency if missing, but don't trigger dep check - row_info, col_info, - [ext_ldo, &requesting_node_id_if_missing] (LocalKVRow &row, LocalKVCell &col, bool previously_existed) { + dbg("Get "+bucket.GetHex()+"|"+key.str()); + rc_t rc=KELPIE_ENOENT; - if(col.availability == Availability::InMemory){ - if(ext_ldo) *ext_ldo = col.ldo; - return KELPIE_OK; - } - //Add to the wait list. TODO: send back waiting? - col.appendWaitingList(requesting_node_id_if_missing); - return KELPIE_ENOENT; + if(!key.IsColWildcard()) { + lunasa::DataObject ldo; + rc = get(bucket, key, &ldo, nullptr); + if(rc==KELPIE_OK) { + ldos[key] = ldo; + } + } else { - //TODO: check local for not here? - }); + //Col wildcard + rc = doRowOp(bucket, key, + lkv::LambdaFlags::DONT_CREATE_OR_TRIGGER, //<--Get doesn't create or trigger + nullptr, + [&ldos, key](LocalKVRow &row, bool previously_existed) { + + vector col_names; + row.getActiveColumnNamesCapacities(key.K2(), &col_names, nullptr); + + for(auto &name : col_names) { + auto *cell = row.getCol(name); + if((cell) && (cell->availability == Availability::InLocalMemory)) { + ldos[Key(key.K1(), name)] = cell->ldo; + } + } + return (!ldos.empty()) ? KELPIE_OK : KELPIE_ENOENT; + }); + } return rc; } -#endif + + /** * @brief Get a Lunasa Data Object back for a desired key. Leave mailbox dependency if not available * @param[in] bucket The user id that is marked as the bucket of this data * @param[in] key The key used to reference this data * @param[in] mailbox_if_missing An op mailbox to wakeup if the object isn't available + * @param[in] behavior_flags Specifies whether a load from iom gets cached here when miss and in iom + * @param[in] iom_hash The hash of the iom to consult if item is not found in memory * @param[out] ext_ldo A new (reference counted) ldo that provides access to the data stored in the lkv - * @param[out] col_info Information about the column this ldo is/should live in - * @param[out] row_info Information about the row this ldo is/should live in - * @retval KELPIE_OK Succes + * @param[out] info Information about the row/column this ldo is/should live in + * @retval KELPIE_OK Success, item found and returned in this call * @retval KELPIE_ENOENT Data not here, but node was placed on waiting list - * @retval KELPIE_WAITING Data not here, but request has already been made - * @retval KELPIE_TODO Needed load from disk - * @retval KELPIE_EINVAL Invalid input (eg, requesting_node_id was UNSPECIFIED) + * @retval KELPIE_EIO The requested iom could not be found, but op was placed in waiting list * @note This version passes back an ldo reference to the data. Modifying the data could have side effects. - * @todo Put some logic here to disable wait lists? */ -rc_t LocalKV::get(faodel::bucket_t bucket, const Key &key, +rc_t LocalKV::getForOp(faodel::bucket_t bucket, const Key &key, opbox::mailbox_t mailbox_if_missing, + pool_behavior_t behavior_flags, + iom_hash_t iom_hash, lunasa::DataObject *ext_ldo, - kv_row_info_t *row_info, - kv_col_info_t *col_info) { + object_info_t *info) { - kassert(key.valid(), "get given invalid key"); + F_ASSERT(key.valid(), "get given invalid key"); dbg("Get "+bucket.GetHex()+"|"+key.str()); - rc_t rc = doColOp(bucket, key, true, false, //<--Creates a mailbox dependency if missing, but don't trigger dep check - row_info, col_info, - [ext_ldo, mailbox_if_missing] (LocalKVRow &row, LocalKVCell &col, bool previously_existed) { + rc_t rc = doColOp(bucket, key, + LambdaFlags::CREATE_IF_MISSING, //Get: creates mailbox dependency but doesn't trigger a dependency check + info, + [bucket, key, behavior_flags, iom_hash, ext_ldo, mailbox_if_missing] (LocalKVRow &row, LocalKVCell &col, bool previously_existed) { + //See if this item is in memory if(col.availability == Availability::InLocalMemory){ if(ext_ldo) *ext_ldo = col.ldo; return KELPIE_OK; } - //Add to the wait list. TODO: send back waiting? + + //Not here. See if we need to load from disk + rc_t rc = KELPIE_ENOENT; + if(iom_hash!=0) { + auto iom = kelpie::internal::FindIOM(iom_hash); + + if(iom==nullptr) { + rc = KELPIE_EIO; //IOM not found + } else { + rc = iom->ReadObject(bucket, key, ext_ldo); + if(rc==KELPIE_OK) { + //Loaded it from disk. Do we keep a copy? + if(behavior_flags & PoolBehavior::ReadToRemote) { + col.availability = Availability::InLocalMemory; + col.ldo = *ext_ldo; + col.time_posted = col.getTime(); + } else { + //Don't cache, but keep a record of it + col.availability = Availability::InDisk; + } + return KELPIE_OK; + } + } + } + + //Not in memory and not in disk. Add op to waiting list col.appendWaitingList(mailbox_if_missing); - return KELPIE_ENOENT; + return rc; //This is KELPIE_ENOENT unless iom not found (KELPIE_EIO) - //TODO: check local for not here? }); return rc; } @@ -254,8 +291,7 @@ rc_t LocalKV::get(faodel::bucket_t bucket, const Key &key, * @param[in] mem_ptr A raw pointer to where the data should be copied * @param[in] max_size The maximum amount of data to copy * @param[out] copied_size The amount of data actually copied out - * @param[out] row_info Optional information about the row - * @param[out] col_info Optional information about the column + * @param[out] info Optional information about the row/column * @remark This only copies from the user's DATA section of an LDO (not the meta) * @retval KELPIE_OK Success * @retval KELPIE_ENOENT Data not here, but node was placed on waiting list @@ -266,19 +302,18 @@ rc_t LocalKV::get(faodel::bucket_t bucket, const Key &key, rc_t LocalKV::getData(bucket_t bucket, const Key &key, void *mem_ptr, size_t max_size, size_t *copied_size, - kv_row_info_t *row_info, - kv_col_info_t *col_info){ + object_info_t *info){ //No locks here. All handled by getRef - kassert(key.valid(), "getData given invalid key"); + F_ASSERT(key.valid(), "getData given invalid key"); lunasa::DataObject ldo_ref; size_t tmp_copied_size; tmp_copied_size = 0; - rc_t rc = get(bucket, key, &ldo_ref, row_info, col_info); + rc_t rc = get(bucket, key, &ldo_ref, info); if(rc==KELPIE_OK){ //tmp_copied_size = (max_size check_rows; //Candidate rows to inspect + vector recheck_rows; //Rows where drops happened and may be removals + + //Determine which rows to look at + if(!key_prefix.IsRowWildcard()) { + //Exact match on K1, partial match on K2 + check_rows.push_back(key_prefix.K1()); + + } else { + //Search for possible rows + string prefix = makeRowname(bucket, key_prefix.K1()); + prefix.erase(prefix.size()-1); //remove the trailing * + + table_mutex->ReaderLock(); + for(auto name_rowptr = rows.lower_bound(prefix); name_rowptr!=rows.end(); ++name_rowptr) { + if(StringBeginsWith(name_rowptr->first, prefix)) + check_rows.push_back(name_rowptr->second->rowname); + } + table_mutex->Unlock(); + } - //Try to drop the cell - LocalKVCell *cell = row.getCol(key); - if(!cell) return KELPIE_ENOENT; - if(!cell->isDroppable()) return KELPIE_WAITING; //TODO: Should be put on a queue for delete later - delete cell; + //Inspect each row that matches row key and delete matching columns. + for(auto &rowname : check_rows) { + rc_t rc = doRowOp(bucket, Key(rowname), + false, + nullptr, + [&key_prefix, &found_items](LocalKVRow &row, bool previously_existed) { - if(key.K2()=="") row.col_single=nullptr; - else row.cols.erase(key.K2()); + found_items += row.dropColumns(key_prefix.K2()); - //Check to see if whole row is empty - if(row.isDroppable()) - return KELPIE_RECHECK; + //Check to see if whole row is gone. If so, remember it for cleanup + return (row.isEmpty()) ? KELPIE_RECHECK : KELPIE_OK; + }); - return KELPIE_OK; - }); + if(rc == KELPIE_RECHECK) recheck_rows.push_back(rowname); + } - //When nothing left, we may need to recheck to get rid of the old row - if(rc==KELPIE_RECHECK){ + //Cleanup: Search for rows that can be deleted + if(!recheck_rows.empty()) { table_mutex->WriterLock(); - string fullrowname = makeRowname(bucket, key); - LocalKVRow *row = getRow(fullrowname); - if(row==nullptr){ - //Nothing to delete. Someone beat us to it? - table_mutex->Unlock(); - return KELPIE_OK; - } - if(row->isDroppable()){ - delete row; - rows.erase(fullrowname); + for(auto &rowname : recheck_rows) { + string fullrowname = makeRowname(bucket, Key(rowname)); + LocalKVRow *row = getRow(fullrowname); + if((row) && (row->isEmpty())) { + delete row; + rows.erase(fullrowname); + } } table_mutex->Unlock(); - rc=KELPIE_OK; } - return rc; + return (found_items) ? KELPIE_OK : KELPIE_ENOENT; + } /** * @brief Perform a search for keys that match a specific pattern * @param[in] bucket The bucket id we should limit the search to - * @param[in] key_prefix The key to search for. Row and/or Key may end in '*' for prefix matching - * @param[out] opject_capacities The results in this localkv that matched the key_prefix + * @param[in] key_prefix The key to search for. Row and/or Column may end in '*' for prefix matching + * @param[in] iom Pointer to iom related to this operation + * @param[out] object_capacities The results in this localkv that matched the key_prefix * @retval KELPIE_OK Found matches * @retval KELPIE_ENOENT Did not find matches */ rc_t LocalKV::list(bucket_t bucket, const Key &key_prefix, + internal::IomBase *iom, ObjectCapacities *object_capacities) { - kassert(key_prefix.valid(), "list given an invalid key"); + F_ASSERT(key_prefix.valid(), "list given an invalid key"); dbg("List "+key_prefix.str()); - bool row_wildcard = StringEndsWith(key_prefix.K1(), "*"); bool found_items=false; + bool needs_an_iom_check = (iom!=nullptr); - if(!row_wildcard) { + if(!key_prefix.IsRowWildcard()) { //Exact row known. Search on the columns vector col_names; //Exact match on K1, partial match on K2 - /*rc_t rc =*/ doRowOp(bucket, key_prefix, + doRowOp(bucket, key_prefix, false, nullptr, [&key_prefix, &col_names, &object_capacities] (LocalKVRow &row, bool previously_existed) { @@ -441,10 +487,15 @@ rc_t LocalKV::list(bucket_t bucket, const Key &key_prefix, //Convert row's column names into full keys for(auto &col_name : col_names) { - object_capacities->keys.push_back(Key(key_prefix.K1(), col_name)); + object_capacities->keys.emplace_back(Key(key_prefix.K1(), col_name)); } + + //Append any info from iom. Skip the case where we had an exact column name and we got back a result + needs_an_iom_check = needs_an_iom_check && ( key_prefix.IsColWildcard() || (col_names.size()==1)); + found_items = (col_names.size()!=0); //Only report what we found in this function + } else { //Fuzzy Row Name @@ -466,34 +517,54 @@ rc_t LocalKV::list(bucket_t bucket, const Key &key_prefix, row->unlock(); //Append all keys for each column to the output - for(auto c : col_names) { - object_capacities->keys.push_back(Key(row_name, c)); + for(auto &c : col_names) { + object_capacities->keys.emplace_back(Key(row_name, c)); } found_items |= (col_names.size()!=0); //Only report what we found in this function } } table_mutex->Unlock(); } + + if(needs_an_iom_check) { + ObjectCapacities oc2; + iom->ListObjects(bucket, key_prefix, &oc2); + object_capacities->Merge(oc2); //Only inserts items that were unknown + found_items |= (oc2.Size()>0); + } + return (found_items) ? KELPIE_OK : KELPIE_ENOENT; } +rc_t LocalKV::doCompute(const string &function_name, const string &args, + bucket_t bucket, const Key &key, + lunasa::DataObject *ext_ldo) { + + map ldos; + rc_t rc = getAvailable(bucket, key, ldos); + + kelpie::internal::Singleton::impl.core->compute_registry.doCompute( + function_name, args, + bucket, key, + ldos, + ext_ldo); + + return rc; +} /** * @brief Use a lambda to manipulate a column inside the localkv * @param[in] bucket The user id that is marked as the bucket of this data * @param[in] key The key used to reference this data - * @param[in] create_if_missing Boolean specifying whether the row/column should be created if missing - * @param[in] trigger_dependency_check Check to see if this item had waiting actions - * @param[out] row_info Optional information about the row - * @param[out] col_info Optional information about the column + * @param[in] flags Specific actions to take during lambda (eg create row if missing) + * @param[out] info Optional information about the row/column * @param[in] func The function to be applied on the column in the local k/v * @return rc_t The rc of the Lambda or KELPIE_ENOENT if it was not available when create_if_missing is false */ rc_t LocalKV::doColOp(bucket_t bucket, const Key &key, - bool create_if_missing, - bool trigger_dependency_check, - kv_row_info_t *row_info, kv_col_info_t *col_info, - fn_column_op_t func){ + lambda_flags_t flags, + object_info_t *info, + fn_column_op_t func) { table_mutex->ReaderLock(); string fullrowname = makeRowname(bucket, key); @@ -504,10 +575,9 @@ rc_t LocalKV::doColOp(bucket_t bucket, const Key &key, //Row not available. Create it or bail out table_mutex->Unlock(); - if(!create_if_missing){ - //done: bail out - if(row_info) row_info->Wipe(); - if(col_info) col_info->Wipe(); + //Didn't want us to create, so bail out + if(!(flags & LambdaFlags::CREATE_IF_MISSING)) { + if(info) info->Wipe(); return KELPIE_ENOENT; } @@ -525,10 +595,10 @@ rc_t LocalKV::doColOp(bucket_t bucket, const Key &key, table_mutex->Unlock(); //Do the user's op, which may trigger a dependency check after a cell is updated - rc_t rc = row->doColOp(key, create_if_missing, trigger_dependency_check, col_info, func); + rc_t rc = row->doColOp(key, flags, info, func); //Create an updated row info for user if requested - if(row_info) row->getInfo(row_info); + if(info) row->getInfo(key, info); row->unlock(); @@ -539,15 +609,15 @@ rc_t LocalKV::doColOp(bucket_t bucket, const Key &key, * @brief Use a lambda to manipulate a row inside the k/v * @param[in] bucket The user id that is marked as the bucket of this data * @param[in] key The key used to reference this data - * @param[in] create_if_missing Boolean specifying whether the row/column should be created if missing - * @param[out] row_info Optional information about the row + * @param[in] flags Specific actions to take during lambda (eg create row if missing) + * @param[out] info Optional information about the row * @param[in] func The function to be applied on the column in the local k/v * @return rc_t The rc of the Lambda or KELPIE_ENOENT if it was not available when create_if_missing is false */ rc_t LocalKV::doRowOp(bucket_t bucket, const Key &key, - bool create_if_missing, - kv_row_info_t *row_info, - fn_row_op_t func){ + lambda_flags_t flags, + object_info_t *info, + fn_row_op_t func) { table_mutex->ReaderLock(); string fullrowname = makeRowname(bucket, key); @@ -557,9 +627,9 @@ rc_t LocalKV::doRowOp(bucket_t bucket, const Key &key, //Row not available. Create it or bail out. table_mutex->Unlock(); - if(!create_if_missing){ - //done: bail out - if(row_info) row_info->Wipe(); + if(!(flags & LambdaFlags::CREATE_IF_MISSING)) { + //done: Didn't find and didn't want to create + if(info) info->Wipe(); return KELPIE_ENOENT; } @@ -578,49 +648,38 @@ rc_t LocalKV::doRowOp(bucket_t bucket, const Key &key, table_mutex->Unlock(); rc_t rc = func(*row, previously_existed); - if(row_info) row->getInfo(row_info); + if(info) row->getInfo(key, info); row->unlock(); return rc; } -/** - * @brief Find a particular column by key and return its information - * @param[in] bucket The user id that is marked as the bucket of this data - * @param[in] key The key used to reference this data - * @param[out] col_info Optional information about the column - * @retval KELPIE_OK if column was here - * @retval KELPIE_ENOENT if column was not here - * @retval KELPIE_WAITING we're waiting for this column to be generated - */ -rc_t LocalKV::getColInfo(bucket_t bucket, const Key &key, kv_col_info_t *col_info) { - - dbg("GetColInfo "+bucket.GetHex()+"|"+key.str()); - return doColOp(bucket, key, false, false, nullptr, col_info, - [] (LocalKVRow &row, LocalKVCell &cell, bool previously_existed) { - if(cell.availability==Availability::Requested) return KELPIE_WAITING; - return (previously_existed) ? KELPIE_OK : KELPIE_ENOENT; - }); -} /** * @brief Find a particular row by key and return its information * @param[in] bucket The user id that is marked as the bucket of this data * @param[in] key The key used to reference this data - * @param[out] row_info Optional information about the row + * @param[out] info Optional information about the item * @retval KELPIE_OK if row was here * @retval KELPIE_ENOENT if row was not here * @retval KELPIE_WAITING if row was not here, but has been requested */ -rc_t LocalKV::getRowInfo(bucket_t bucket, const Key &key, kv_row_info_t *row_info) { +rc_t LocalKV::getInfo(bucket_t bucket, const Key &key, object_info_t *info) { dbg("GetRowInfo "+bucket.GetHex()+"|"+key.str()); - return doRowOp(bucket, key, false, row_info, - [] (LocalKVRow &row, bool previously_existed) { - if(row.getAvailability()==Availability::Requested) return KELPIE_WAITING; - return (previously_existed) ? KELPIE_OK : KELPIE_ENOENT; - }); + rc_t rc = doColOp(bucket, key, + LambdaFlags::DONT_CREATE_OR_TRIGGER, + info, + [] (LocalKVRow &row, LocalKVCell &col, bool previously_existed) { + if(row.getAvailability()==Availability::Requested) return KELPIE_WAITING; + return (previously_existed) ? KELPIE_OK : KELPIE_ENOENT; + }); + if((rc==KELPIE_WAITING) || (!info)) return rc; + + //Check the findings. Wildcards stats don't get generated until after the above lambda is called + return (info->row_num_columns!=0) ? KELPIE_OK : KELPIE_ENOENT; + } @@ -637,7 +696,7 @@ string LocalKV::makeRowname(bucket_t bucket, const Key &key) { * @returns Pointer to the row * @note table must already be locked when calling this */ -LocalKVRow * LocalKV::getRow(string full_row_name){ +LocalKVRow * LocalKV::getRow(const string &full_row_name){ //printf("GetRow for bucket %d key '%s' fullrowname: '%s\n",bucket, key.str().c_str(), full_row_name.c_str() ); @@ -695,7 +754,6 @@ void LocalKV::whookieInfo(faodel::ReplyStream &rs, bool detailed){ rs.tableBegin("LocalKV"); rs.tableTop({"Parameter","Setting"}); rs.tableRow({"Configured:", (configured)?"True":"False"}); - rs.tableRow({"Max Capacity:", to_string(max_capacity)}); rs.tableRow({"Current Rows:", to_string(rows.size())}); rs.tableEnd(); @@ -706,22 +764,22 @@ void LocalKV::whookieInfo(faodel::ReplyStream &rs, bool detailed){ //Print row summaries table_mutex->ReaderLock(); rs.tableBegin("LocalKV Row Summary"); - rs.tableTop({"FullRowID","RowName","NumCols","FirstColumn", "RowBytes","Subscribers","LocalCallbacks"}); + rs.tableTop({"FullRowID","RowName","NumCols","FirstColumn", "RowBytes"}); for(auto rname_rptr : rows){ rname_rptr.second->lock(); - kv_row_info_t row_info; - rname_rptr.second->getInfo(&row_info); + object_info_t info; + rname_rptr.second->getInfo(Key(rname_rptr.first), &info); string cname=rname_rptr.second->getFirstColumnName(); - if(cname=="") cname=html::mkLink("(noname)", "/kelpie/lkv/cell&row="+rname_rptr.first); - else cname=html::mkLink(cname, "/kelpie/lkv/cell&row="+rname_rptr.first+"&col="+cname); + if(cname.empty()) cname=html::mkLink("(noname)", "/kelpie/lkv/cell&row="+rname_rptr.first); + else cname=html::mkLink(cname, "/kelpie/lkv/cell&row="+rname_rptr.first+"&col="+cname); - rs.tableRow( {rname_rptr.first, + rs.tableRow( { + rname_rptr.first, html::mkLink(rname_rptr.second->rowname,"/kelpie/lkv/row&row="+rname_rptr.first), - to_string(row_info.num_cols_in_row), + to_string(info.row_num_columns), cname, - to_string(row_info.row_bytes), - to_string(row_info.num_row_receiver_nodes), - to_string(row_info.num_row_dependencies)}); + to_string(info.row_user_bytes)} ); + rname_rptr.second->unlock(); } rs.tableEnd(); @@ -729,38 +787,36 @@ void LocalKV::whookieInfo(faodel::ReplyStream &rs, bool detailed){ } else { //Print each column in its own row - kv_col_info_t col_info; + object_info_t info; table_mutex->ReaderLock(); rs.tableBegin("LocalKV Full Details"); - rs.tableTop({"FullRowID","RowName","ColumnName","ColBytes","NodeDependencies","LocalDependencies"}); + rs.tableTop({"FullRowID","RowName","ColumnName","ColBytes","Dependencies"}); for(auto rname_rptr : rows){ rname_rptr.second->lock(); //Pull out the default, no-name column if(rname_rptr.second->col_single){ - rname_rptr.second->col_single->getInfo(&col_info); + rname_rptr.second->col_single->getInfo(&info); rs.tableRow( { rname_rptr.first, html::mkLink(rname_rptr.second->rowname,"/kelpie/lkv/row&row="+rname_rptr.first), html::mkLink("(noname)","/kelpie/lkv/cell&row="+rname_rptr.first), - to_string(col_info.num_bytes), - to_string(col_info.num_col_receiver_nodes), - to_string(col_info.num_col_dependencies)}); + to_string(info.col_user_bytes), + to_string(info.col_dependencies)}); } //Now look at all named columns - for(auto cname_cptr : rname_rptr.second->cols){ - - cname_cptr.second->getInfo(&col_info); + for(auto cname_cptr : rname_rptr.second->cols) { + info.Wipe(); + cname_cptr.second->getInfo(&info); rs.tableRow( { rname_rptr.first, html::mkLink(rname_rptr.second->rowname, "/kelpie/lkv/row&row="+rname_rptr.first), html::mkLink(cname_cptr.first, "/kelpie/lkv/cell&row="+rname_rptr.first+"&col="+cname_cptr.first), - to_string(col_info.num_bytes), - to_string(col_info.num_col_receiver_nodes), - to_string(col_info.num_col_dependencies)}); + to_string(info.col_user_bytes), + to_string(info.col_dependencies)}); } rname_rptr.second->unlock(); @@ -782,7 +838,7 @@ void LocalKV::whookieInfo(faodel::ReplyStream &rs, bool detailed){ void LocalKV::HandleWhookieRow(const std::map &args, std::stringstream &results){ faodel::ReplyStream rs(args, "Kelpie LocalKV Row", &results); - string rname=""; + string rname; auto ritr = args.find("row"); if(ritr != args.end()) rname=ritr->second; @@ -798,16 +854,13 @@ void LocalKV::HandleWhookieRow(const std::map &args, st row->lock(); table_mutex->Unlock(); - kv_row_info_t ri; - row->getInfo(&ri); + object_info_t info; + row->getInfo(Key(rname), &info); rs.tableBegin("Row Info"); rs.tableTop({"Parameter","Setting"}); rs.tableRow({"Row Name:", row->rowname}); - rs.tableRow({"Total Bytes:", to_string(ri.row_bytes)}); - rs.tableRow({"Node Dependencies:", to_string(ri.num_row_receiver_nodes)}); - rs.tableRow({"Local Dependencies:", to_string(ri.num_row_dependencies)}); - rs.tableRow({"Availability:", availability_to_string(ri.availability)}); + rs.tableRow({"Total Bytes:", to_string(info.row_user_bytes)}); rs.tableRow({"Current Columns:", to_string(row->cols.size()+((row->col_single!=nullptr)?1:0))}); rs.tableEnd(); @@ -842,8 +895,7 @@ void LocalKV::HandleWhookieRow(const std::map &args, st void LocalKV::HandleWhookieCell(const std::map &args, std::stringstream &results){ faodel::ReplyStream rs(args, "Kelpie LocalKV Cell", &results); - string rname=""; - string cname=""; + string rname, cname; auto ritr = args.find("row"); auto citr = args.find("col"); if(ritr != args.end()) rname=ritr->second; @@ -873,7 +925,7 @@ void LocalKV::HandleWhookieCell(const std::map &args, s rs.tableRow({"Row Name", html::mkLink(row->rowname, "/kelpie/lkv/row&row=" + rname)}); rs.tableRow({"Column Name", cname_txt}); rs.tableRow({"Availability:", availability_to_string(col->availability)}); - + rs.tableRow({"Dependencies", to_string(col->getNumDependencies())}); rs.tableRow({"Object Meta Size", to_string(msize)}); rs.tableRow({"Object Data Size", to_string(dsize)}); rs.tableRow({"Object User Capacity", to_string(col->ldo.GetUserCapacity())}); @@ -912,9 +964,8 @@ void LocalKV::sstr(stringstream &ss, int depth, int indent) const { ss << string(indent,' ') << "[LKV] Number of Rows: " << rows.size() <0){ - for(map::const_iterator ii=rows.begin(); ii!=rows.end(); ++ii) { - //ss << string(indent+1,' ') << ii->first << " " <<" Number Cols: "<second->getNumCols()<second->sstr(ss, depth-1, indent+1); + for(auto &name_rowptr : rows) { + name_rowptr.second->sstr(ss,depth-1, indent+1); } } } diff --git a/src/kelpie/localkv/LocalKV.hh b/src/kelpie/localkv/LocalKV.hh index d057ebf..493dc73 100644 --- a/src/kelpie/localkv/LocalKV.hh +++ b/src/kelpie/localkv/LocalKV.hh @@ -1,11 +1,11 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_LOCALKV_HH #define KELPIE_LOCALKV_HH -#include +#include #include #include #include @@ -17,6 +17,7 @@ #include "faodel-common/ReplyStream.hh" #include "kelpie/Key.hh" +#include "kelpie/localkv/LocalKVTypes.hh" #include "kelpie/localkv/LocalKVRow.hh" #include "kelpie/pools/Pool.hh" @@ -55,7 +56,6 @@ class LocalKV : public: LocalKV(); - ~LocalKV() override; //One-time configure the lkv @@ -66,70 +66,69 @@ public: rc_t put(faodel::bucket_t bucket, const Key &key, const lunasa::DataObject &new_ldo, pool_behavior_t behavior_flags, - kv_row_info_t *row_info, - kv_col_info_t *col_info); + internal::IomBase *iom, + object_info_t *info); //Get local reference. Do nothing if unavailable rc_t get(faodel::bucket_t bucket, const Key &key, lunasa::DataObject *ext_ldo, - kv_row_info_t *row_info, - kv_col_info_t *col_info); + object_info_t *info); - #if 0 - //Get local reference. Leave node dependency if unavailable - rc_t get(faodel::bucket_t bucket, const Key &key, - faodel::nodeid_t requesting_node_id_if_missing, //Use localhost unless remote - lunasa::DataObject *ext_ldo, - kv_row_info_t *row_info, - kv_col_info_t *col_info); - #endif + //Get multiple references. Do nothing if unavailable + rc_t getAvailable(faodel::bucket_t bucket, const Key &key, + std::map &ldos); //Get local reference. Leave OpBox mailbox dependency if unavailable - rc_t get(faodel::bucket_t bucket, const Key &key, + rc_t getForOp(faodel::bucket_t bucket, const Key &key, opbox::mailbox_t op_mailbox_if_missing, + pool_behavior_t behavior_flags, + iom_hash_t iom_hash, lunasa::DataObject *ext_ldo, - kv_row_info_t *row_info, - kv_col_info_t *col_info); + object_info_t *info); //Copy the data out of the store and into the new struct rc_t getData(faodel::bucket_t bucket, const Key &key, void *mem_ptr, size_t max_size, size_t *copied_size, - kv_row_info_t *row_info, - kv_col_info_t *col_info); + object_info_t *info); //Request a callback be made when an item becomes available - rc_t want(faodel::bucket_t bucket, const Key &key, - faodel::nodeid_t requesting_node_id_if_missing, + rc_t wantLocal(faodel::bucket_t bucket, const Key &key, bool caller_will_fetch_if_missing, fn_want_callback_t callback); //Drop a particular item - rc_t drop(faodel::bucket_t bucket, const Key &key); + rc_t drop(faodel::bucket_t bucket, const Key &key_prefix); //Locate info about one or more keys rc_t list(faodel::bucket_t bucket, const Key &key_prefix, + internal::IomBase *iom, ObjectCapacities *object_capacities); + + //Fetch local objects from row, perform computation, return ldo result + rc_t doCompute(const std::string &function_name, const std::string &args, + faodel::bucket_t bucket, const Key &key, + lunasa::DataObject *ext_ldo); + //Get info for a particular item - rc_t getColInfo(faodel::bucket_t bucket, const Key &key, kv_col_info_t *col_info); - rc_t getRowInfo(faodel::bucket_t bucket, const Key &key, kv_row_info_t *row_info); + rc_t getInfo(faodel::bucket_t bucket, const Key &key, object_info_t *info); //Workhorse: Does all the work of finding a column and doing a lambda on it rc_t doColOp(faodel::bucket_t bucket, const Key &key, - bool create_if_missing, - bool trigger_dependency_check, - kv_row_info_t *row_info, - kv_col_info_t *col_info, + lkv::lambda_flags_t flags, + object_info_t *info, fn_column_op_t func); //Workhorse: Does all the work of finding a row and doing a lambda on it rc_t doRowOp(faodel::bucket_t bucket, const Key &key, - bool create_if_missing, - kv_row_info_t *row_info, + lkv::lambda_flags_t flags, + object_info_t *info, fn_row_op_t func); + + //Internal: Delete everything void wipeAll(faodel::internal_use_only_t iuo); @@ -140,18 +139,18 @@ public: void whookieInfo(faodel::ReplyStream &rs, bool detailed=false); //InfoInterface function - void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; + void sstr(std::stringstream &ss, int depth, int indent) const override; private: faodel::MutexWrapperTypeID row_mutex_type_id; //!< The mutex type to use for creating new rows bool configured; //!< Whether LovalKV has been Configured yet - int64_t max_capacity; //!< Max amount of space the lkv can hold std::map rows; //!< Map for storing each row's LocalKVRow faodel::MutexWrapper *table_mutex; //!< Mutex needed for controlling single-access to the top-level row map std::string makeRowname(faodel::bucket_t bucket, const Key &key); - LocalKVRow * getRow(std::string full_row_name); + LocalKVRow * getRow(const std::string &full_row_name); + }; diff --git a/src/kelpie/localkv/LocalKVCell.cpp b/src/kelpie/localkv/LocalKVCell.cpp index 6240794..c96ca19 100644 --- a/src/kelpie/localkv/LocalKVCell.cpp +++ b/src/kelpie/localkv/LocalKVCell.cpp @@ -1,11 +1,10 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -#include -#include -#include +#include +#include #include @@ -24,7 +23,7 @@ using namespace faodel; namespace kelpie { LocalKVCell::LocalKVCell() - : availability(Availability::Unavailable), hold_until(0), + : availability(Availability::Unavailable), hold_until(0), drop_requested(false), time_offloaded(0) { time_posted = getTime(); time_accessed = time_posted; @@ -50,7 +49,7 @@ bool LocalKVCell::operator<( const LocalKVCell &x) const { * @todo implement flush to disk */ int LocalKVCell::evict(const Key &key, const Availability new_availability, lunasa::DataObject *new_owners_ldo){ - KTODO("Evict cell"); + F_TODO("Evict cell"); #if 0 if(!in_memory) return KELPIE_OK; if(!in_disk) { @@ -65,7 +64,7 @@ int LocalKVCell::evict(const Key &key, const Availability new_availability, luna } bool LocalKVCell::isDroppable(){ - if((waiting_nodes_list.size()>0) || (waiting_ops_list.size())) return false; + if( waiting_ops_list.size() + callback_list.size() > 0 ) return false; if(availability!=Availability::InLocalMemory) return false; //todo uint32_t t = getTime(); @@ -74,26 +73,36 @@ bool LocalKVCell::isDroppable(){ return true; } +#if 0 void LocalKVCell::getInfo(kv_col_info_t *col_info){ if(!col_info) return; auto num_receivers = static_cast(waiting_nodes_list.size()); if(!num_receivers) num_receivers = static_cast(waiting_ops_list.size()); - col_info->num_bytes = ldo.GetUserSize(); + col_info->num_user_bytes = ldo.GetUserSize(); col_info->num_col_receiver_nodes = num_receivers; col_info->num_col_dependencies = static_cast(waiting_nodes_list.size() + waiting_ops_list.size()); col_info->availability=availability; } +#endif -/** - * @brief Add a requesting node to the list of nodes that want this data - * @param[in] requesting_node The (non-local) node that wants the data - */ -void LocalKVCell::appendWaitingList(nodeid_t requesting_node){ - if(requesting_node!=NODE_LOCALHOST) - waiting_nodes_list.insert(requesting_node); +void LocalKVCell::getInfo(object_info_t *info) { + if(!info) return; + info->col_user_bytes = ldo.GetUserSize(); + info->col_dependencies = waiting_ops_list.size() + callback_list.size(); + info->col_availability = availability; } + +///** +// * @brief Add a requesting node to the list of nodes that want this data +// * @param[in] requesting_node The (non-local) node that wants the data +// */ +//void LocalKVCell::appendWaitingList(nodeid_t requesting_node){ +// if(requesting_node!=NODE_LOCALHOST) +// waiting_nodes_list.insert(requesting_node); +//} + /** * @brief Add an op mailbox to the list of nodes that want this data * @param[in] op_mailbox The id of an op waiting on this item @@ -116,43 +125,38 @@ void LocalKVCell::appendCallbackList(fn_want_callback_t callback){ void LocalKVCell::dispatchCallbacksAndNotifications(LocalKVRow *row, const Key &key){ //Bail out if no dependencies - if((callback_list.size()==0) && - (waiting_nodes_list.size()==0) && - (waiting_ops_list.size()==0) ) return; + if( callback_list.empty() && waiting_ops_list.empty() ) return; - //cout <<"LKV Cell: dispatch callbacks for "< backburner_work; - kv_row_info_t row_info; - kv_col_info_t col_info; - row->getInfo(&row_info); - getInfo(&col_info); + + //Gather info one time for all cbs + object_info_t object_info; + row->getInfo(key, &object_info); + getInfo(&object_info); + //Callbacks on the local node //cout <<"LKV LocalCBs. NumCBs: "<0){ + if(!waiting_ops_list.empty()){ //We need to do some trigger ops here. OpBox spools these up internally //(potentially launching on backburner), so just let them do the work - auto args = make_shared(ldo, row_info, col_info); + auto args = make_shared(ldo, object_info); for(auto &mb : waiting_ops_list) { opbox::TriggerOp(mb, std::move(args)); } @@ -161,12 +165,6 @@ void LocalKVCell::dispatchCallbacksAndNotifications(LocalKVRow *row, const Key & } - //Waiting nodes - for(auto &nd : waiting_nodes_list){ - //TODO: schedule a send - KWARN("Need to update kelpie for waiting lists") - } - //cout<<"LKV total backburner jobs adding: "< +#include #include #include #include @@ -45,6 +45,7 @@ public: Availability availability; //!< Where this data resides uint32_t hold_until; //!< Hold at least until this point in time + bool drop_requested; //!< User requested a drop, but dependencies prevented //Some usage statistics uint32_t time_posted; //!< The time this block was stored locally @@ -58,7 +59,7 @@ public: //Functions for manipulating the dependencies. - void appendWaitingList(faodel::nodeid_t); + //void appendWaitingList(faodel::nodeid_t); void appendWaitingList(opbox::mailbox_t op_mailbox); void appendCallbackList(fn_want_callback_t callback); void dispatchCallbacksAndNotifications(LocalKVRow *row, const Key &key); @@ -66,15 +67,19 @@ public: //Functions for getting state info bool isDroppable(); uint32_t getTime(); //Make a timestamp - void getInfo(kv_col_info_t *col_info); + void getInfo(object_info_t *info); + + void getWaitingInfo(size_t *num_waiting_actions) { + if(num_waiting_actions) *num_waiting_actions = getNumDependencies(); + } + size_t getNumDependencies() { return waiting_ops_list.size() + callback_list.size(); } //InfoInterface function - void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; + void sstr(std::stringstream &ss, int depth, int indent) const override; private: //Information about who generated and who wants - std::set waiting_nodes_list; //!< Nodes that want a copy of this when it becomes available std::set waiting_ops_list; //!< Ops that are stalled on this item std::vector callback_list; //!< Local callbacks to schedule when item becomes available diff --git a/src/kelpie/localkv/LocalKVRow.cpp b/src/kelpie/localkv/LocalKVRow.cpp index 347adca..392ee81 100644 --- a/src/kelpie/localkv/LocalKVRow.cpp +++ b/src/kelpie/localkv/LocalKVRow.cpp @@ -1,9 +1,8 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -#include -#include +#include #include "kelpie/localkv/LocalKVCell.hh" #include "kelpie/localkv/LocalKVRow.hh" @@ -34,25 +33,25 @@ LocalKVRow::~LocalKVRow(){ /** * @brief Do a lambda on a desired column. Create if it doesn't exist * @param[in] key Key to look for (key.K2 is only part used) - * @param[in] create_if_missing Create the column if its missing - * @param[in] trigger_dependency_check After a cell update, see if we need to trigger any callbacks - * @param[out] col_info (Optional) pass back information about the column, after op + * @param[in] flags Specific actions to take during lambda (eg create row if missing) + * @param[out] info Optional pass back information about the column, after op * @param[in] func Lambda to execute on the column * @returns return code of lambda, or KELPIE_ENOENT if * * NOTE: row_info is filled in the next layer up, at LocalKV::doColOp */ -rc_t LocalKVRow::doColOp(const Key &key, bool create_if_missing, bool trigger_dependency_check, - kv_col_info_t *col_info, fn_column_op_t func){ +rc_t LocalKVRow::doColOp(const Key &key, + lkv::lambda_flags_t flags, + object_info_t *info, fn_column_op_t func){ bool previously_existed = true; LocalKVCell *cell; - if(create_if_missing){ + if( flags & lkv::LambdaFlags::CREATE_IF_MISSING ) { cell = getOrCreateCol(key, &previously_existed); } else { cell = getCol(key); if(cell==nullptr){ - if(col_info) col_info->Wipe(); + if(info) info->Wipe(); return KELPIE_ENOENT; } } @@ -62,12 +61,12 @@ rc_t LocalKVRow::doColOp(const Key &key, bool create_if_missing, bool trigger_de //Cell is updated, so stats should be good. See if this is an operation //that needs to trigger any dependencies - if(trigger_dependency_check){ + if( flags & lkv::LambdaFlags::TRIGGER_DEPENDENCIES){ cell->dispatchCallbacksAndNotifications(this, key); } //Pass back generic cell info if requested - if(col_info) cell->getInfo(col_info); + if(info) cell->getInfo(info); cell->time_accessed = cell->getTime(); return rc; @@ -78,7 +77,7 @@ rc_t LocalKVRow::doColOp(const Key &key, bool create_if_missing, bool trigger_de * @note requires ownership of mutex * @returns number of columns */ -int LocalKVRow::getNumCols(){ +size_t LocalKVRow::getNumCols() { return cols.size() + (col_single!=nullptr); } @@ -89,10 +88,10 @@ int LocalKVRow::getNumCols(){ * @note requires ownership of the mutex * @returns Cell or nullptr */ -LocalKVCell * LocalKVRow::getCol(const Key &key){ +LocalKVCell * LocalKVRow::getCol(const Key &key) { //Empty column gets put somewhere special - if(key.K2()=="") return col_single; + if(key.K2().empty()) return col_single; //All other columns go in map map::iterator it; @@ -107,10 +106,10 @@ LocalKVCell * LocalKVRow::getCol(const Key &key){ * @note requires ownership of the mutex * @returns Cell or nullptr */ -LocalKVCell * LocalKVRow::getCol(const string colname){ +LocalKVCell * LocalKVRow::getCol(const string &colname){ //Empty column gets put somewhere special - if(colname=="") return col_single; + if(colname.empty()) return col_single; //All other columns go in map map::iterator it; @@ -148,8 +147,8 @@ LocalKVCell * LocalKVRow::getOrCreateCol(const Key &key, bool *previously_existe //Cell previously unknown. Create and add it cell = new LocalKVCell(); - if(key.K2()=="") col_single = cell; - else cols[key.K2()] = cell; + if(key.K2().empty()) col_single = cell; + else cols[key.K2()] = cell; existed=false; } if(previously_existed) *previously_existed = existed; @@ -184,6 +183,12 @@ int LocalKVRow::getActiveColumnNamesCapacities(const string &search_string, vect string prefix=search_string; prefix.erase(prefix.size()-1); + //Check no-name column first + if(prefix.empty() && (col_single)) { + if(names) names->push_back(""); + if(capacities) capacities->push_back(col_single->getUserSize()); + } + //Walk through all entries and see if we have a hit for(auto name_colptr = cols.lower_bound(prefix); name_colptr!=cols.end(); ++name_colptr) { if(name_colptr->second->getUserSize()) { //Only entries with data @@ -199,58 +204,67 @@ int LocalKVRow::getActiveColumnNamesCapacities(const string &search_string, vect } /** - * @brief Insert a node into the waiting list for a particular cell. - * @param[in] colname out column name - * @param[in] requesting_node_id id of the node that really wants data (for forwarding) - * @returns - @retval KELPIE_OK This item was added and has not been asked for yet - @retval KELPIE_EEXIST This item has already been asked for -*/ -rc_t LocalKVRow::insertWaitingList(string colname, nodeid_t requesting_node_id){ - - set node_set; - - map>::iterator it; - it=col_waiting_list.find(colname); - - if(it!=col_waiting_list.end()){ - //This entry already on the wait list - node_set=it->second; - return KELPIE_EEXIST; - } - node_set.insert(requesting_node_id); - col_waiting_list[colname] = node_set; + * @brief Remove columns that match a search string + * @param[in] search_string The column pattern we want. Does prefix match if string ends in '*' + * @return The number of columns that the drop command matched + */ +int LocalKVRow::dropColumns(const std::string &search_string) { - return KELPIE_OK; -} + bool col_wildcard = StringEndsWith(search_string, "*"); + if(!col_wildcard) { + //Find exact match + LocalKVCell *cell = getCol(search_string); + if(cell == nullptr) return 0; + if(cell->isDroppable()) { + delete cell; + if(search_string.empty()) col_single = nullptr; + else cols.erase(search_string); + + } else { + //Cell has things waiting on it. Mark as drop. + cell->drop_requested = true; + } + return 1; -/** - * @brief Return the list of nodes that have asked for this column before - * @param[in] colname String name of the column we want - * @param[out] waiting_list A set containing all of the nodes that are waiting on this entry - * @note this will clear out the column's waiting list, as it is assumed you're going to handle it - * @todo check into passing back an empty set - */ -void LocalKVRow::getWaitingList(string colname, set &waiting_list){ + } else { + + //Do a wildcard search + int num_found=0; + //Adjust the search name so it doesn't end in * + string prefix=search_string; + prefix.erase(prefix.size()-1); - map>::iterator it; - it=col_waiting_list.find(colname); + //Check no-name column first + if(prefix.empty() && (col_single)) { + num_found++; + delete col_single; + col_single = nullptr; + } - if(it!=col_waiting_list.end()){ - //Found something. Pass the list back, then clear out the entry - waiting_list = it->second; - col_waiting_list.erase(it); + //Walk through all entries and see if we have a hit + vector delete_names; + for(auto name_colptr = cols.lower_bound(prefix); name_colptr!=cols.end(); ++name_colptr) { + if(StringBeginsWith(name_colptr->first, prefix)) { + num_found++; + if(name_colptr->second->isDroppable()) { + delete name_colptr->second; + delete_names.push_back(name_colptr->first); + } else { + name_colptr->second->drop_requested = true; + } + } + } - }// else { - // //No members. Be nice and pass back an empty set - // set node_set; - // *waiting_list = node_set; - //} + //Remove all the items from the map + for(auto &name : delete_names) + cols.erase(name); - return; + return num_found; + } } + /** * @brief Drops a particular key from the table * @param[in] key The key to drop @@ -268,20 +282,33 @@ rc_t LocalKVRow::drop(const Key &key, int drop_options, size_t *dropped_bytes, b delete cell; - if(key.K2()=="") col_single=nullptr; - else cols.erase(key.K2()); + if(key.K2().empty()) col_single=nullptr; + else cols.erase(key.K2()); } *is_empty = isDroppable(); return KELPIE_OK; } /** - * @brief Reports back whether this row is empty and can be deleted - * @retval TRUE Row is empty and can be dropped - * @retval FALSE Row has columns, or a list of subscribers waiting for data + * @brief Reports back whether this row CAN be dropped (ie, is it free of dependencies) + * @retval TRUE Row does not have any dependencies and can be dropped + * @retval FALSE Row has columns with dependencies that prevent dropping */ -bool LocalKVRow::isDroppable(){ - return cols.empty() && col_waiting_list.empty(); +bool LocalKVRow::isDroppable() { + if((col_single) && (!col_single->isDroppable())) return false; + for(auto &name_col : cols) { + if(!name_col.second->isDroppable()) return false; + } + return true; +} + +/** + * @brief Determine if this row is empty (ie, no cells in single or cols) + * @retval TRUE Row doesn't have cells or waiting dependencies + * @retval FALSE Row has at least one cell in it (possibly with a dependency) + */ +bool LocalKVRow::isEmpty() { + return (col_single==nullptr) && (cols.empty()); } @@ -296,7 +323,7 @@ Availability LocalKVRow::getAvailability() { //Set the initial value if(col_single) { a=col_single->availability; - } else if (cols.size()>0) { + } else if (!cols.empty()) { a=cols.begin()->second->availability; } else { return Availability::Unavailable; @@ -314,23 +341,44 @@ Availability LocalKVRow::getAvailability() { /** * @brief Generate summary information for all the cells in this row - * @param[out] row_info Summary statistics for the row + * @param[in] key Optional info that can be used to filter columns via a wildcard + * @param[out] info Updates the ROW portion of info */ -void LocalKVRow::getInfo(kv_row_info_t *row_info){ +void LocalKVRow::getInfo(const Key &key, object_info_t *info){ + + if(!info) return; + string prefix_match; + if(key.IsColWildcard()) { + prefix_match=key.K2(); + prefix_match.erase( prefix_match.size()-1); //remove the trailing '*' + } - if(!row_info) return; + if(prefix_match.empty()) { + //No '*' provided so we match everything in row - //Sum up all the column bytes - ssize_t row_bytes=0; - if(col_single) row_bytes=col_single->ldo.GetUserSize(); - for(auto &name_cellptr : cols) - row_bytes += name_cellptr.second->ldo.GetUserSize(); + //Sum up all the column bytes + size_t row_bytes = 0; + if(col_single) row_bytes = col_single->getUserSize(); + for(auto &name_cellptr : cols) + row_bytes += name_cellptr.second->getUserSize(); - row_info->row_bytes = row_bytes; - row_info->num_cols_in_row = cols.size() + (col_single!=nullptr); - row_info->num_row_receiver_nodes = col_waiting_list.size(); - row_info->num_row_dependencies = 0; //TODO - row_info->availability = getAvailability(); + info->row_user_bytes = row_bytes; + info->row_num_columns = getNumCols(); + return; + + } else { + //User gave us a wildcard. This automatically filters out the noname column. Only look at matching cols. + + //Col Wildcard. Compute based on matches + info->row_user_bytes = 0; + info->row_num_columns = 0; + for(auto &name_cellptr : cols) { + if( StringBeginsWith(name_cellptr.first, prefix_match) ) { + info->row_user_bytes += name_cellptr.second->getUserSize(); + info->row_num_columns++; + } + } + } } /** @@ -341,7 +389,7 @@ void LocalKVRow::getInfo(kv_row_info_t *row_info){ */ void LocalKVRow::sstr(stringstream &ss, int depth, int indent) const { - int num_cols = cols.size() + (col_single!=nullptr); + //int num_cols = cols.size() + (col_single!=nullptr); ss << string(indent,' ')+"[Row] '" << rowname; if(col_single!=nullptr) { diff --git a/src/kelpie/localkv/LocalKVRow.hh b/src/kelpie/localkv/LocalKVRow.hh index ded6694..a3adeee 100644 --- a/src/kelpie/localkv/LocalKVRow.hh +++ b/src/kelpie/localkv/LocalKVRow.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LOCALKVROW_HH #define LOCALKVROW_HH @@ -14,21 +14,23 @@ #include "faodel-common/InfoInterface.hh" #include "kelpie/Key.hh" +#include "kelpie/localkv/LocalKVTypes.hh" #include "kelpie/localkv/LocalKVCell.hh" #include "lunasa/DataObject.hh" namespace kelpie { - /** - * The LocalKVRow holds multiple cells with the same row key and provides a way - * to access these items in a thread safe way. The LocalKVRow uses a single mutex for - * all members of the row and uses a map to store each item (by column name). The LocalKVRow - * also provides a waitlist structure that allows actions to be queued up until a particular - * item or set of items arrive. - * @note caller must lock/unlock the row while using it to prevent cols from changing during work - * @brief Container for multiple cells that share the same row key - */ + +/** + * The LocalKVRow holds multiple cells with the same row key and provides a way + * to access these items in a thread safe way. The LocalKVRow uses a single mutex for + * all members of the row and uses a map to store each item (by column name). The LocalKVRow + * also provides a waitlist structure that allows actions to be queued up until a particular + * item or set of items arrive. + * @note caller must lock/unlock the row while using it to prevent cols from changing during work + * @brief Container for multiple cells that share the same row key + */ class LocalKVRow : public faodel::InfoInterface { @@ -36,37 +38,36 @@ public: LocalKVRow(const std::string &rowname, faodel::MutexWrapperTypeID mutex_type); ~LocalKVRow() override; - int getNumCols(); - void getInfo(kv_row_info_t *row_info); + size_t getNumCols(); + void getInfo(const Key &key, object_info_t *row_info); Availability getAvailability(); void lock(){ row_lock->Lock(); } //!< Locks this row void unlock(){ row_lock->Unlock(); } //!< Unlocks this row - rc_t doColOp(const Key &key, bool create_if_missing, bool trigger_dependency_check, kv_col_info_t *col_info, fn_column_op_t func); + rc_t doColOp(const Key &key, kelpie::lkv::lambda_flags_t flags, object_info_t *info, fn_column_op_t func); rc_t drop(const Key &key, int drop_options, size_t *dropped_bytes, bool *is_empty); bool isDroppable(); + bool isEmpty(); //InfoInterface function - void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; + void sstr(std::stringstream &ss, int depth, int indent) const override; //The following would normally be private, but are exposed so doOp can see them std::string rowname; //!< The row name (first string in key) LocalKVCell *col_single; //!< When user hands "" for column part of key std::map cols; //!< Map for locating our cell, based on the column key - std::map > col_waiting_list; //!< All the nodes that are waiting on this item to arrive - rc_t insertWaitingList(std::string colname, faodel::nodeid_t requesting_node_id); - void getWaitingList(std::string colname, std::set &waiting_list); LocalKVCell * getCol(const Key &key); - LocalKVCell * getCol(const std::string colname); + LocalKVCell * getCol(const std::string &colname); std::string getFirstColumnName(); size_t getFirstColumnUserSize(); int getActiveColumnNamesCapacities(const std::string &search_string, std::vector *names, std::vector *capacities); + int dropColumns(const std::string &search_string); private: LocalKVCell * getOrCreateCol(const Key &key, bool *previously_existed); diff --git a/src/kelpie/localkv/LocalKVTypes.hh b/src/kelpie/localkv/LocalKVTypes.hh new file mode 100644 index 0000000..745dca1 --- /dev/null +++ b/src/kelpie/localkv/LocalKVTypes.hh @@ -0,0 +1,36 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_LOCALKVTYPES_HH +#define FAODEL_LOCALKVTYPES_HH + + +namespace kelpie { + +namespace lkv { + +typedef uint8_t lambda_flags_t; + +/** + * @brief Flags passed into the LocalKV's lambda operators + * The LocalKV has doRowOp/doColCop functions to let you write core functions at the top level (localkv) + * without having to worry about all the mutex handling and output generation on the way down. We use the + * lambda_flags option to ensure we are more descriptive about what actions should be taken in the lambda. + * Currently there are two flags: + * + * CREATE_IF_MISSING: If the key we're looking for doesn't exist, allocate space for it. Puts, Gets usually + * need this, but info functions do not + * TRIGGER_DEPENDENCIES: When done, check the entry to see if anything was waiting on this op and needs to + * be woken up. Puts need this, but gets do not. + */ +struct LambdaFlags { + static constexpr lambda_flags_t CREATE_IF_MISSING = 0x01; + static constexpr lambda_flags_t TRIGGER_DEPENDENCIES = 0x02; + static constexpr lambda_flags_t DONT_CREATE_OR_TRIGGER = 0x0; +}; + +} // namespace lkv +} // namespace kelpie + +#endif //FAODEL_LOCALKVTYPES_HH diff --git a/src/kelpie/ops/direct/OpKelpieCompute.cpp b/src/kelpie/ops/direct/OpKelpieCompute.cpp new file mode 100644 index 0000000..ece243d --- /dev/null +++ b/src/kelpie/ops/direct/OpKelpieCompute.cpp @@ -0,0 +1,235 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include + +#include "faodel-common/Debug.hh" +#include "kelpie/common/OpArgsObjectAvailable.hh" +#include "kelpie/core/Singleton.hh" + +#include "kelpie/ops/direct/OpKelpieCompute.hh" + +using namespace std; +using namespace faodel; +using namespace kelpie; + + +//Statics: Standard id/name info for an op +const unsigned int OpKelpieCompute::op_id = const_hash("OpKelpieCompute"); +const string OpKelpieCompute::op_name = "OpKelpieCompute"; +bool OpKelpieCompute::debug_enabled = false; + +//Statics: This op has a static localkv pointer. lkv lives inside KelpieCore instance +LocalKV * OpKelpieCompute::lkv = nullptr; + +/** + * @brief Internal startup command for setting static variables + * + * @param[in] iuo Designates this function is for internal use only + * @param[in] config Pointer to configuration so that kelpie.op.getunbounded settings can be retrieved + * @param[in] new_lkv A pointer to the kelpie localkv we should uses_allocator + */ +void OpKelpieCompute::configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, LocalKV *new_lkv) { + lkv = new_lkv; + if(config) { + config->GetComponentLoggingSettings(&OpKelpieCompute::debug_enabled, nullptr, nullptr, "kelpie.op.compute"); + } +} + + + +/** + * @brief Create a new remote compute operation + * + * @param[in] target_node The target node to query + * @param[in] target_ptr The target node's peer pointer + * @param[in] bucket The bucket namespace for the object + * @param[in] key The key label for the object + * @param[in] iom_hash Hash id of an IOM associated with this request + * @param[in] behavior_flags Info about how to behave in different cases + * @param[in] function_name The name of the user-defined function to call + * @param[in] function_args A string of arguments to pass to the function * + * @param[in] cb_result Callback function invoke when success/failure known + * @return OpKelpieCompute + */ +OpKelpieCompute::OpKelpieCompute( + const faodel::nodeid_t target_node, + const net::peer_ptr_t target_ptr, + const faodel::bucket_t bucket, + const Key &key, + const iom_hash_t iom_hash, + const pool_behavior_t behavior_flags, + const string &function_name, + const string &function_args, + fn_compute_callback_t cb_result) + : Op(true), state(State::orig_compute_send), + peer(target_ptr), bucket(bucket), key(key), + cb_compute_result(cb_result) { + + //bool exceeds; + + //Create the outgoing message + /*exceeds =*/ msg_direct_simple_t::Alloc(ldo_msg, op_id, + DirectFlags::CMD_COMPUTE, target_node, + GetAssignedMailbox(), opbox::MAILBOX_UNSPECIFIED, + bucket, key, iom_hash, behavior_flags, + function_name, function_args); +} + + +/** + * @brief Create target-side handler for a new OpKelpieCompute + * + * @param[in] t Marker designating this ctor is just for creatingf a target op + * @return OpKelpieCompute + */ +OpKelpieCompute::OpKelpieCompute(Op::op_create_as_target_t t) + : Op(t), state(State::trgt_compute_start), ldo_msg() { + //No work to do - done in target's state machine + peer = nullptr; + GetAssignedMailbox(); //For safety, get a mailbox. Not needed everywhere? +} + + +/** + * @brief Destructor for OpKelpieCompute + */ +OpKelpieCompute::~OpKelpieCompute(){ + //cout <<"OpKelpieCompute() dtor : state is "<ExpectMessageOrDie(&peer); + bucket = imsg->bucket; + string function_name, function_args; + imsg->ExtractComputeArgs(&key, &function_name, &function_args); + + dbg("Received new compute request for function "+function_name+" on key "+key.str()+" args "+function_args); + + //Have the lkv take care of all the fetching. We either get ok or op is queued up + rc_t rc = lkv->doCompute(function_name, function_args, + bucket, key, &ldo_data); + + dbg("lkv-compute success was "+to_string(rc)); + + + if(rc==KELPIE_OK) { + //Create an ack that includes buffer pointers + msg_direct_buffer_t::Alloc(ldo_msg, op_id, DirectFlags::CMD_STATUS_ACK, imsg->hdr.src, GetAssignedMailbox(), + imsg->hdr.src_mailbox, bucket, key, 0, PoolBehavior::NoAction, + &ldo_data); + } else { + //Create a nack (no pointer) + msg_direct_buffer_t::Alloc(ldo_msg, op_id, DirectFlags::CMD_STATUS_NACK, imsg->hdr.src, GetAssignedMailbox(), + imsg->hdr.src_mailbox, bucket, key, 0, PoolBehavior::NoAction, + nullptr); + } + net::SendMsg(peer, std::move(ldo_msg)); + return updateState(State::trgt_compute_wait_for_ack, WaitingType::waiting_on_cq); + +} + + + +//ORIGIN: Wait for the object to become available or for someone to cancel us +WaitingType OpKelpieCompute::smo_Compute_WaitInfo(OpArgs *args){ + + //Grab essential from this message for later user + auto imsg = args->ExpectMessageOrDie(&peer); + + if(DirectFlags::IsNack(&imsg->hdr)) { + //Nack, just finish the request + dbg("Origin received Nack"); + + cb_compute_result(KELPIE_ENOENT, key, ldo_data); + return updateStateDone(); + } + + if(imsg->meta_plus_data_size==0) { + dbg("Origin received Ack, but no data"); + cb_compute_result(KELPIE_OK, key, ldo_data); + return updateStateDone(); + } + + dbg("Origin got ack and info for rdma"); + + //Allocate object for the data and get it via an RDMA pull + ldo_data = lunasa::DataObject(0, imsg->meta_plus_data_size, lunasa::DataObject::AllocatorType::eager); + net::Get(peer, &imsg->net_buffer_remote, ldo_data, AllEventsCallback(this)); + + msg_direct_status_t::AllocAck(ldo_msg, &imsg->hdr); //Success changed below + + return updateState(State::orig_compute_wait_for_rdma, WaitingType::waiting_on_cq); + +} + +//ORIGIN: Wait for RDMA done and Send ACK +WaitingType OpKelpieCompute::smo_Compute_WaitRDMA(OpArgs *args) { + + dbg("Done retrieving. Notifying target we are done"); + args->VerifyTypeOrDie(UpdateType::get_success, op_name); //TODO: Add better error handling + + //TODO: swap order with SendMsg? + cb_compute_result(KELPIE_OK, key, ldo_data); //TODO: handle not success + + auto omsg = ldo_msg.GetDataPtr(); + omsg->Success(true); + net::SendMsg(peer, std::move(ldo_msg)); + + return updateStateDone(); +} + +//TARGET: Wait for ACK to release LDO +WaitingType OpKelpieCompute::smt_Compute_WaitAck(OpArgs *args) { + dbg("Received ack. Done."); + /*auto imsg =*/ args->ExpectMessageOrDie(); + return updateStateDone(); +} + + + +WaitingType OpKelpieCompute::Update(opbox::OpArgs *args) { + switch(state){ + case State::orig_compute_send: return smo_Compute_Send(); + case State::trgt_compute_start: return smt_Compute_Start(args); + case State::orig_compute_wait_for_info: return smo_Compute_WaitInfo(args); + case State::orig_compute_wait_for_rdma: return smo_Compute_WaitRDMA(args); + case State::trgt_compute_wait_for_ack: return smt_Compute_WaitAck(args); + case State::done: return updateStateDone(); + } + F_FAIL(); + return WaitingType::error; +} + +/** + * @brief Get a string name for the current state + * @retval string Human-readable name for state + */ +std::string OpKelpieCompute::GetStateName() const { + switch(state){ + case State::orig_compute_send: return "Origin-Compute-Send"; + case State::trgt_compute_start: return "Target-Compute-Start"; + case State::orig_compute_wait_for_info: return "Origin-Compute-WaitForInfo"; + case State::orig_compute_wait_for_rdma: return "Origin-Compute-WaitForRDMA"; + case State::trgt_compute_wait_for_ack: return "Target-Compute-WaitForAck"; + case State::done: return "Done"; + } + F_FAIL(); +} + + diff --git a/src/kelpie/ops/direct/OpKelpieCompute.hh b/src/kelpie/ops/direct/OpKelpieCompute.hh new file mode 100644 index 0000000..e57703e --- /dev/null +++ b/src/kelpie/ops/direct/OpKelpieCompute.hh @@ -0,0 +1,120 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef KELPIE_OPKELPIECOMPUTE_HH +#define KELPIE_OPKELPIECOMPUTE_HH + +#include "opbox/OpBox.hh" +#include "opbox/ops/OpHelpers.hh" +#include "lunasa/Lunasa.hh" +#include "lunasa/DataObject.hh" + +#include "kelpie/Kelpie.hh" +#include "kelpie/localkv/LocalKV.hh" + +#include "kelpie/ops/direct/msg_direct.hh" + +namespace kelpie { + + +/** + * @brief An OpBox state machine for getting an unknown-sized object that is the result of a remore compute op + */ +class OpKelpieCompute : public opbox::Op { + + //States + enum class State : int { + orig_compute_send=0, + trgt_compute_start, + orig_compute_wait_for_info, + orig_compute_wait_for_rdma, + trgt_compute_wait_for_ack, + done }; + +public: + + //Get item with a known size + OpKelpieCompute( + const faodel::nodeid_t target_node, + const net::peer_ptr_t target_ptr, + const faodel::bucket_t bucket, + const Key &key, + const iom_hash_t iom_hash, + const pool_behavior_t behavior_flags, + const std::string &function_name, + const std::string &function_args, + fn_compute_callback_t cb_result); + + //A target starts off the same way no matter what command + OpKelpieCompute(Op::op_create_as_target_t t); + ~OpKelpieCompute() override; + + //Unique name and id for this op + const static unsigned int op_id; + const static std::string op_name; + static bool debug_enabled; //!< Dump debug messages + + unsigned int getOpID() const override { return op_id; } + std::string getOpName() const override { return op_name; } + + WaitingType Update(OpArgs *args) override; //Combined use + WaitingType UpdateOrigin(OpArgs *args) override { return WaitingType::error; } //Remove + WaitingType UpdateTarget(OpArgs *args) override { return WaitingType::error; } //Remove + + std::string GetStateName() const override; + + static void configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, LocalKV *new_lkv); + +private: + + //todo: put this in a standard form so it can be reused + #if Faodel_LOGGINGINTERFACE_DISABLED==0 + void dbg(const std::string &s) const { + if(OpKelpieCompute::debug_enabled) { + std::cout << "\033[1;93mD " << op_name << ": ["<GetComponentLoggingSettings(&OpKelpieDrop::debug_enabled, nullptr, nullptr, "kelpie.op.drop"); + } +} + +/** + * @brief Create a new Drop Object operation + * + * @param[in] targets A list of nodes where this should be sent + * @param[in] bucket he bucket namespace for this object + * @param[in] search_key The key label for the object (may be a wildcard) + * @param[in] already_dropped_locally Whether we have already dropped item from this node + * @param[in] callback Function to call when we complete + */ +OpKelpieDrop::OpKelpieDrop( + vector > targets, + const faodel::bucket_t bucket, + const Key &search_key, + bool already_dropped_locally, + fn_drop_callback_t callback) + + : Op(false), + targets(std::move(targets)), + bucket(bucket), search_key(search_key), + callback(callback), + state(State::orig_drop_send) { + + dbg("Creating new drop"); + successful_drops = (already_dropped_locally) ? 1 : 0; +} + +OpKelpieDrop::OpKelpieDrop(Op::op_create_as_target_t t) + : Op(t), state(State::trgt_drop_start) { +} + +OpKelpieDrop::~OpKelpieDrop() { + if(state!=State::done){ + F_TODO("Dtor when state not done"); + } +} + +//Origin: Send out requests to all targets. Set counters on expected replies +WaitingType OpKelpieDrop::smo_Drop_Send() { + + dbg("Starting to send. NumTargets="+to_string(targets.size())); + + bool expects_reply = (callback!=nullptr); + mailbox_t mbox = (expects_reply) ? GetAssignedMailbox() : opbox::MAILBOX_UNSPECIFIED; + num_targets_left = (expects_reply) ? targets.size() : 0; + + //Note: assumes caller has correctly filtered this node from list + for(auto &node_peerptr : targets) { + dbg("Sending to target "+node_peerptr.first.GetHex()); + + lunasa::DataObject ldo; + msg_direct_simple_t::Alloc( + ldo, + op_id, DirectFlags::CMD_DROP, node_peerptr.first, + mbox, opbox::MAILBOX_UNSPECIFIED, + bucket, search_key, + 0, PoolBehavior::NoAction); //TODO: Update iom/behavior settings + net::SendMsg(node_peerptr.second, std::move(ldo)); + } + + if(expects_reply) { + state = State::orig_drop_wait_for_results; + return WaitingType::waiting_on_cq; + } else { + state = State::done; + return WaitingType::done_and_destroy; + } +} + +//Target: Got a request. Issue a drop and send back a reply if user expects one +WaitingType OpKelpieDrop::smt_Drop_Start(opbox::OpArgs *args) { + + dbg("Starting a new drop"); + net::peer_ptr_t peer; + auto imsg = args->ExpectMessageOrDie(&peer); + search_key = imsg->ExtractKey(); + + rc_t rc = lkv->drop(imsg->bucket, search_key); + + if(imsg->hdr.src_mailbox != MAILBOX_UNSPECIFIED) { + + //Build a reply message and send it + lunasa::DataObject ldo_reply; + auto omsg = msg_direct_status_t::AllocAck(ldo_reply, &imsg->hdr); //Success changed below + if(rc!=KELPIE_OK) + omsg->Success(false); + dbg("Sending a reply message. Dropped items: "+to_string(rc==KELPIE_OK)); + net::SendMsg(peer, std::move(ldo_reply)); + + state=State::done; + return WaitingType::done_and_destroy; + + } else { + + //No reply needed + state=State::done; + return WaitingType::done_and_destroy; + } +} + +//Origin: Count replies until we've got them all +WaitingType OpKelpieDrop::smo_Drop_WaitForResults(opbox::OpArgs *args) { + + auto imsg = args->ExpectMessageOrDie(); + if(imsg->Success()) + successful_drops++; + + num_targets_left--; + dbg("Got a reply message. Num Targets now left "+to_string(num_targets_left)); + if(num_targets_left<1) { + callback((successful_drops>0), search_key); + state=State::done; + return WaitingType::done_and_destroy; + } + //Wait for more + return WaitingType::waiting_on_cq; +} + + +WaitingType OpKelpieDrop::Update(OpArgs *args) { + //dbg("Update. Processing state "+GetStateName()); + switch(state) { + case State::orig_drop_send: return smo_Drop_Send(); + case State::trgt_drop_start: return smt_Drop_Start(args); + case State::orig_drop_wait_for_results: return smo_Drop_WaitForResults(args); + case State::done: return WaitingType::done_and_destroy; + } + F_FAIL(); + return WaitingType::error; +} + +/** + * @brief Get a string name for the current state + * @retval string Human-readable name for state + */ +std::string OpKelpieDrop::GetStateName() const { + + switch(state) { + case State::orig_drop_send: return "Origin-Drop-Send"; + case State::trgt_drop_start: return "Target-Drop-Start"; + case State::orig_drop_wait_for_results: return "Origin-Drop-Wait-for-Results"; + case State::done: return "Done"; + } + F_FAIL(); +} diff --git a/src/kelpie/ops/direct/OpKelpieDrop.hh b/src/kelpie/ops/direct/OpKelpieDrop.hh new file mode 100644 index 0000000..49ca5b6 --- /dev/null +++ b/src/kelpie/ops/direct/OpKelpieDrop.hh @@ -0,0 +1,99 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_OPKELPIEDROP_HH +#define FAODEL_OPKELPIEDROP_HH + +#include + +#include "opbox/OpBox.hh" +#include "opbox/ops/OpHelpers.hh" +#include "lunasa/Lunasa.hh" +#include "lunasa/DataObject.hh" + +#include "kelpie/Kelpie.hh" +#include "kelpie/localkv/LocalKV.hh" + +#include "kelpie/ops/direct/msg_direct.hh" + +namespace kelpie { + +/** + * @brief An ObBox state machine for dropping objects from remote nodes + */ +class OpKelpieDrop : public opbox::Op { + + //States + enum class State : int { + orig_drop_send = 0, + trgt_drop_start, + orig_drop_wait_for_results, + done + }; + +public: + + OpKelpieDrop( + std::vector> targets, + const faodel::bucket_t bucket, + const Key &search_key, + bool already_dropped_locally, + fn_drop_callback_t callback); + + explicit OpKelpieDrop(Op::op_create_as_target_t t); + + ~OpKelpieDrop() override; + + //Unique name and id for this op + const static unsigned int op_id; + const static std::string op_name; + static bool debug_enabled; //!< Dump debug messages + + unsigned int getOpID() const override { return op_id; } + std::string getOpName() const override { return op_name; } + + WaitingType Update(OpArgs *args) override; //Combined use + WaitingType UpdateOrigin(OpArgs *args) override { return WaitingType::error; } //Remove + WaitingType UpdateTarget(OpArgs *args) override { return WaitingType::error; } //Remove + + std::string GetStateName() const override; + + static void configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, LocalKV *new_lkv); + +private: + + //todo: put this in a standard form so it can be reused + #if Faodel_LOGGINGINTERFACE_DISABLED==0 + void dbg(const std::string &s) const { + if(OpKelpieDrop::debug_enabled) { + std::cout << "\033[1;93mD " << op_name << ": ["<> targets; + faodel::bucket_t bucket; + kelpie::Key search_key; + fn_drop_callback_t callback; + + std::atomic num_targets_left; + std::atomic successful_drops; + + State state; + + //Origin/Target States (in order) + WaitingType smo_Drop_Send(); + WaitingType smt_Drop_Start(opbox::OpArgs *args); + WaitingType smo_Drop_WaitForResults(opbox::OpArgs *args); + + +}; + +} // namespace kelpie + +#endif //FAODEL_OPKELPIEDROP_HH diff --git a/src/kelpie/ops/direct/OpKelpieGetBounded.cpp b/src/kelpie/ops/direct/OpKelpieGetBounded.cpp index fc454e4..7f6ba03 100644 --- a/src/kelpie/ops/direct/OpKelpieGetBounded.cpp +++ b/src/kelpie/ops/direct/OpKelpieGetBounded.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -15,6 +15,7 @@ using namespace kelpie; //Statics: Standard id/name info for an op const unsigned int OpKelpieGetBounded::op_id = const_hash("OpKelpieGetBounded"); const string OpKelpieGetBounded::op_name = "OpKelpieGetBounded"; +bool OpKelpieGetBounded::debug_enabled = false; //Statics: This op has a static localkv pointer. lkv lives inside KelpieCore instance LocalKV * OpKelpieGetBounded::lkv = nullptr; @@ -23,10 +24,14 @@ LocalKV * OpKelpieGetBounded::lkv = nullptr; * @brief Internal startup command for setting static variables * * @param[in] iuo Designates this function is for internal use only + * @param[in] config Pointer to configuration so that kelpie.op.getbounded settings can be retrieved * @param[in] new_lkv A pointer to the kelpie localkv we should uses_allocator */ -void OpKelpieGetBounded::configure(faodel::internal_use_only_t iuo, LocalKV *new_lkv) { +void OpKelpieGetBounded::configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, LocalKV *new_lkv) { lkv = new_lkv; + if(config) { + config->GetComponentLoggingSettings(&OpKelpieGetBounded::debug_enabled, nullptr, nullptr, "kelpie.op.getbounded"); + } } @@ -34,7 +39,6 @@ void OpKelpieGetBounded::configure(faodel::internal_use_only_t iuo, LocalKV *new /** * @brief Create a new Get Object operation * - * @param[in] target_node The target node to publish data to * @param[in] target_ptr The target node's peer pointer * @param[in] bucket The bucket namespace for this object @@ -62,7 +66,7 @@ OpKelpieGetBounded::OpKelpieGetBounded( bucket(bucket), key(key), cb_opget_result(cb_result) { - kassert(expected_ldo_user_size>0, "GetBounded op given a zero byte ldo?"); + F_ASSERT(expected_ldo_user_size > 0, "GetBounded op given a zero byte ldo?"); //Create space for the requested data ldo_data = lunasa::DataObject(0, expected_ldo_user_size, lunasa::DataObject::AllocatorType::eager); @@ -85,7 +89,7 @@ OpKelpieGetBounded::OpKelpieGetBounded( OpKelpieGetBounded::OpKelpieGetBounded(Op::op_create_as_target_t t) : Op(t), state(State::trgt_getbounded_start), ldo_msg() { //No work to do - done in target's state machine - peer = 0; + peer = nullptr; GetAssignedMailbox(); //For safety, get a mailbox. Not needed everywhere? } @@ -94,14 +98,14 @@ OpKelpieGetBounded::OpKelpieGetBounded(Op::op_create_as_target_t t) * @brief Destructor for OpKelpieGetBounded */ OpKelpieGetBounded::~OpKelpieGetBounded(){ - //cout <<"OpKelpieGetBounded() dtor : state is "<net_buffer_remote; //Copy the remote pointers bucket = imsg->bucket; key = imsg->ExtractKey(); - auto target_iom = imsg->iom_hash; - //cout <<"OPDHT-TRG: message is get bounded for "<hdr); //Success changed below - //Lookup data - rc_t rc = lkv->get(imsg->bucket, key, mailbox, &ldo_data, &omsg->row_info, &omsg->col_info); - - - //TODO: Provide row/col info back in ack - - //When miss, see if we need to try loading from an iom - if((rc!=0) && (target_iom!=0)){ - - auto *iom = kelpie::internal::Singleton::impl.core->FindIOM(target_iom); - if(iom==nullptr) { - throw runtime_error("OpKelpieGetBounded attempted to read key "+key.str()+" to a node with a bad iom"); - } - rc = iom->ReadObject(imsg->bucket, key, &ldo_data); - - if(rc==0) { - //TODO: We need to post this, without triggering our mailbox to be sent - auto target_behavior_flags = PoolBehavior::ChangeRemoteToLocal(imsg->behavior_flags); - lkv->put(imsg->bucket, key, ldo_data, target_behavior_flags, &omsg->row_info, &omsg->col_info); - } - } - + //Have the lkv take care of all the fetching. We either get ok, or op is queued up + rc_t rc = lkv->getForOp(bucket, key, mailbox, imsg->behavior_flags, imsg->iom_hash, + &ldo_data, &omsg->object_info); - if(rc==0){ - //cout <<"OPDHT-TRG Result is ready and am putting\n"; - //Data was in memory, get ready to send it back + if(rc==KELPIE_OK) { + dbg("Item located. Starting to send data"); //TODO: Does put check lengths at all? net::Put(peer, ldo_data, &nbr, AllEventsCallback(this)); omsg->Success(true); return updateState(State::trgt_getbounded_wait_for_rdma, WaitingType::waiting_on_cq); } else { - //cout <<"OPDHT-TRG Result not ready\n"; - - - //Option 1: Just send a nack - //omsg->Success(false); - //net::SendMsg(peer, ldo_msg); - //return updateStateDone(); - - //Option 2: Stall this op and wait for the data + dbg("Item not found locally. Stalling op and sending a nack\n"); return updateState(State::trgt_getbounded_wait_for_data, WaitingType::wait_on_user); - - //Option 3?: Stall this op and wait/timeout for the data - } } //TARGET: Data now available. Start the RDMA. WaitingType OpKelpieGetBounded::smt_GetBounded_WaitData(OpArgs *args){ - //cout <<"OPDHT-TRG: getbounded wait for data\n"; - //See if this is a timeout - if(args->type == opbox::UpdateType::timeout){ + if(args->type == opbox::UpdateType::timeout) { //Send a nack and be done. Message ready to go, just toggle the success + dbg("Timeout waiting for data. Sending nack."); auto omsg = ldo_msg.GetDataPtr(); omsg->Success(false); net::SendMsg(peer, std::move(ldo_msg)); @@ -189,6 +161,8 @@ WaitingType OpKelpieGetBounded::smt_GetBounded_WaitData(OpArgs *args){ throw std::runtime_error("OpKelpieGetBounded_Target was expecting user trigger in GetBounded_WaitData()?"); } + dbg("Data available. Sending."); + //Data now available. Setup send auto opargs = reinterpret_cast(args); ldo_data = opargs->ldo; //Transfer ownership here @@ -200,7 +174,7 @@ WaitingType OpKelpieGetBounded::smt_GetBounded_WaitData(OpArgs *args){ //TARGET: RDMA PUT is Done, send the ACK notification WaitingType OpKelpieGetBounded::smt_GetBounded_WaitRDMA(OpArgs *args) { - //cout <<"OPDHT-TRG: getbounded wait for rdma, sending ack\n"; + dbg("Data transfer done. Sending an ack"); args->VerifyTypeOrDie(UpdateType::put_success, op_name); //TODO: Add better error handling //RDMA done, now send an ACK message @@ -214,11 +188,9 @@ WaitingType OpKelpieGetBounded::smo_GetBounded_WaitAck(OpArgs *args) { auto imsg = args->ExpectMessageOrDie(); - //cout <<"OPXFER-ORG getbounded "<< (imsg->Success() ? "Success":"Fail") <str(); - + dbg("Received completion. Status is "+std::to_string(imsg->Success())); cb_opget_result(imsg->Success(), key, ldo_data); - //cout <<"OPXFER-ORG: Done with callback\n"; return updateStateDone(); } @@ -233,7 +205,7 @@ WaitingType OpKelpieGetBounded::Update(opbox::OpArgs *args) { case State::orig_getbounded_wait_for_ack: return smo_GetBounded_WaitAck(args); case State::done: return updateStateDone(); } - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -251,6 +223,6 @@ std::string OpKelpieGetBounded::GetStateName() const { case State::orig_getbounded_wait_for_ack: return "Origin-GetBounded-WaitForAck"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); } diff --git a/src/kelpie/ops/direct/OpKelpieGetBounded.hh b/src/kelpie/ops/direct/OpKelpieGetBounded.hh index aeaee6b..f0c8687 100644 --- a/src/kelpie/ops/direct/OpKelpieGetBounded.hh +++ b/src/kelpie/ops/direct/OpKelpieGetBounded.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_OPKELPIEGETBOUNDED_HH #define KELPIE_OPKELPIEGETBOUNDED_HH @@ -51,6 +51,8 @@ public: //Unique name and id for this op const static unsigned int op_id; const static std::string op_name; + static bool debug_enabled; //!< Dump debug messages + unsigned int getOpID() const override { return op_id; } std::string getOpName() const override { return op_name; } @@ -60,9 +62,21 @@ public: std::string GetStateName() const override; - static void configure(faodel::internal_use_only_t iuo, LocalKV *new_lkv); + static void configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, LocalKV *new_lkv); private: + + //todo: put this in a standard form so it can be reused + #if Faodel_LOGGINGINTERFACE_DISABLED==0 + void dbg(const std::string &s) const { + if(OpKelpieGetBounded::debug_enabled) { + std::cout << "\033[1;93mD " << op_name << ": ["< #include #include "faodel-common/Debug.hh" #include "kelpie/common/OpArgsObjectAvailable.hh" +#include "kelpie/core/Singleton.hh" #include "kelpie/ops/direct/OpKelpieGetUnbounded.hh" @@ -17,6 +18,7 @@ using namespace kelpie; //Statics: Standard id/name info for an op const unsigned int OpKelpieGetUnbounded::op_id = const_hash("OpKelpieGetUnbounded"); const string OpKelpieGetUnbounded::op_name = "OpKelpieGetUnbounded"; +bool OpKelpieGetUnbounded::debug_enabled = false; //Statics: This op has a static localkv pointer. lkv lives inside KelpieCore instance LocalKV * OpKelpieGetUnbounded::lkv = nullptr; @@ -25,10 +27,14 @@ LocalKV * OpKelpieGetUnbounded::lkv = nullptr; * @brief Internal startup command for setting static variables * * @param[in] iuo Designates this function is for internal use only + * @param[in] config Pointer to configuration so that kelpie.op.getunbounded settings can be retrieved * @param[in] new_lkv A pointer to the kelpie localkv we should uses_allocator */ -void OpKelpieGetUnbounded::configure(faodel::internal_use_only_t iuo, LocalKV *new_lkv) { +void OpKelpieGetUnbounded::configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, LocalKV *new_lkv) { lkv = new_lkv; + if(config) { + config->GetComponentLoggingSettings(&OpKelpieGetUnbounded::debug_enabled, nullptr, nullptr, "kelpie.op.getunbounded"); + } } @@ -56,18 +62,17 @@ OpKelpieGetUnbounded::OpKelpieGetUnbounded( const iom_hash_t iom_hash, const pool_behavior_t behavior_flags, fn_opget_result_t cb_result) - : state(State::orig_getunbounded_send), - Op(true), peer(target_ptr), bucket(bucket), key(key), + : Op(true), state(State::orig_getunbounded_send), + peer(target_ptr), bucket(bucket), key(key), cb_opget_result(cb_result) { - bool exceeds; + //bool exceeds; //Create the outgoing message - exceeds = msg_direct_buffer_t::Alloc(ldo_msg, op_id, + /*exceeds =*/ msg_direct_simple_t::Alloc(ldo_msg, op_id, DirectFlags::CMD_GET_UNBOUNDED, target_node, GetAssignedMailbox(), opbox::MAILBOX_UNSPECIFIED, - bucket, key, iom_hash, behavior_flags, - nullptr); + bucket, key, iom_hash, behavior_flags); } @@ -80,7 +85,7 @@ OpKelpieGetUnbounded::OpKelpieGetUnbounded( OpKelpieGetUnbounded::OpKelpieGetUnbounded(Op::op_create_as_target_t t) : Op(t), state(State::trgt_getunbounded_start), ldo_msg() { //No work to do - done in target's state machine - peer = 0; + peer = nullptr; GetAssignedMailbox(); //For safety, get a mailbox. Not needed everywhere? } @@ -91,12 +96,13 @@ OpKelpieGetUnbounded::OpKelpieGetUnbounded(Op::op_create_as_target_t t) OpKelpieGetUnbounded::~OpKelpieGetUnbounded(){ //cout <<"OpKelpieGetUnbounded() dtor : state is "<ExpectMessageOrDie(&peer); + //Grab essential from this message for later use + auto imsg = args->ExpectMessageOrDie(&peer); bucket = imsg->bucket; key = imsg->ExtractKey(); - //cout <<"OPDHT-TRG: message is get unbounded for "<hdr.src, GetAssignedMailbox(), + imsg->hdr.src_mailbox, bucket, key, 0, PoolBehavior::NoAction, nullptr); - //Lookup the item - rc_t rc = lkv->get(imsg->bucket, key, mailbox, &ldo_data, nullptr, nullptr); - if(rc==0) { - //Item is ready. Send the origin the pointers to its data. No need for IOM or behavior - msg_direct_buffer_t::Alloc(ldo_msg, op_id, DirectFlags::CMD_GET_UNBOUNDED, imsg->hdr.src, GetAssignedMailbox(), - imsg->hdr.src_mailbox, bucket, key, 0, PoolBehavior::NoAction, &ldo_data); //We have the ldo + //Have the lkv take care of all the fetching. We either get ok or op is queued up + rc_t rc = lkv->getForOp(bucket, key, mailbox, imsg->behavior_flags, imsg->iom_hash, + &ldo_data, nullptr); //todo: does not pass back row/col info - net::SendMsg(peer, std::move(ldo_msg)); + dbg("lkv-get success was "+to_string(rc)+" iom hash is "+to_string(imsg->iom_hash)); + + if(rc==KELPIE_OK) { + dbg("Item located. Sending pointers"); + auto omsg = ldo_msg.GetDataPtr(); + omsg->SetLDO(&ldo_data); + net::SendMsg(peer, std::move(ldo_msg)); return updateState(State::trgt_getunbounded_wait_for_ack, WaitingType::waiting_on_cq); } else { - //Object was not here. We need to nack or wait - - msg_direct_buffer_t::Alloc(ldo_msg, op_id, DirectFlags::CMD_GET_UNBOUNDED, imsg->hdr.src, GetAssignedMailbox(), - imsg->hdr.src_mailbox, bucket, key, 0, PoolBehavior::NoAction, nullptr); //We don't know the ldo yet - + dbg("Item Not available. Waiting for it to be published."); return updateState(State::trgt_getunbounded_wait_for_data, WaitingType::wait_on_user); - KTODO("WaitOnUser states?"); - } + } + //TARGET: Wait for the object to become available or for someone to cancel us WaitingType OpKelpieGetUnbounded::smt_GetUnbounded_WaitData(OpArgs *args){ @@ -146,13 +153,11 @@ WaitingType OpKelpieGetUnbounded::smt_GetUnbounded_WaitData(OpArgs *args){ auto opargs = reinterpret_cast(args); ldo_data = opargs->ldo; //Hold on to a copy - //Fill in the LDO's size and nbr info + dbg("Data available. Sending info."); + + //Add the buffer pointer to the outgoing message we created in previous step, then send it auto omsg = ldo_msg.GetDataPtr(); omsg->SetLDO(&ldo_data); - - - //TODO: Should col_info be placed here? - net::SendMsg(peer, std::move(ldo_msg)); return updateState(State::trgt_getunbounded_wait_for_ack, WaitingType::waiting_on_cq); } @@ -164,15 +169,16 @@ WaitingType OpKelpieGetUnbounded::smo_GetUnbounded_WaitInfo(OpArgs *args){ //Grab essential from this message for later user auto imsg = args->ExpectMessageOrDie(&peer); - //TODO: Make sure not a nack - if(imsg->meta_plus_data_size==0){ - KTODO("GetUnbounded SMO NACK not implemented"); + //Make sure not a nack. Current code does not do this, but future version may have a cancel + if(imsg->meta_plus_data_size==0) { + F_TODO("GetUnbounded SMO NACK not implemented"); } + dbg("Retrieving data"); //Create an ack TODO: move to after Get when opbox handles this right - auto omsg = msg_direct_status_t::AllocAck(ldo_msg, &imsg->hdr); //Success changed below + msg_direct_status_t::AllocAck(ldo_msg, &imsg->hdr); //Success changed below - //Allocate object and setup Get + //Allocate object for the data and get it via an RDMA pull ldo_data = lunasa::DataObject(0, imsg->meta_plus_data_size, lunasa::DataObject::AllocatorType::eager); net::Get(peer, &imsg->net_buffer_remote, ldo_data, AllEventsCallback(this)); return updateState(State::orig_getunbounded_wait_for_rdma, WaitingType::waiting_on_cq); @@ -181,11 +187,12 @@ WaitingType OpKelpieGetUnbounded::smo_GetUnbounded_WaitInfo(OpArgs *args){ //ORIGIN: Wait for RDMA done and Send ACK WaitingType OpKelpieGetUnbounded::smo_GetUnbounded_WaitRDMA(OpArgs *args) { - args->VerifyTypeOrDie(UpdateType::get_success, op_name); //TODO: Add better error handling + dbg("Done retrieving. Notifying target we are done"); + args->VerifyTypeOrDie(UpdateType::get_success, op_name); - //TODO: swap order with SendMsg? - cb_opget_result(true, key, ldo_data); //TODO: handle not success + cb_opget_result(true, key, ldo_data); //Current retrieval ends in success. Future code may have timeout/cancels + //Send back an ack to release the target auto omsg = ldo_msg.GetDataPtr(); omsg->Success(true); net::SendMsg(peer, std::move(ldo_msg)); @@ -195,17 +202,14 @@ WaitingType OpKelpieGetUnbounded::smo_GetUnbounded_WaitRDMA(OpArgs *args) { //TARGET: Wait for ACK to release LDO WaitingType OpKelpieGetUnbounded::smt_GetUnbounded_WaitAck(OpArgs *args) { - auto imsg = args->ExpectMessageOrDie(); + dbg("Received ack. Done."); + /*auto imsg =*/ args->ExpectMessageOrDie(); return updateStateDone(); } WaitingType OpKelpieGetUnbounded::Update(opbox::OpArgs *args) { -// stringstream ss; -// args->print(ss,2,1); -// cout <<"OPGETU-Update "<< GetStateName()<<" Args: "< +#include #include +#include #include "OpKelpieList.hh" using namespace std; -using namespace faodel; using namespace kelpie; //Statics: Standard id/name info for an op @@ -21,29 +24,40 @@ LocalKV * OpKelpieList::lkv = nullptr; * @brief Internal startup command for setting static variables * * @param[in] iuo Designates this function is for internal use only + * @param[in] config Pointer to configuration so that kelpie.op.list settings can be retrieved * @param[in] new_lkv A pointer to the kelpie localkv we should uses_allocator */ void kelpie::OpKelpieList::configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, kelpie::LocalKV *new_lkv) { lkv = new_lkv; if(config) { - config->GetComponentLoggingSettings(&OpKelpieList::debug_enabled, nullptr, nullptr, "OpKelpieList"); + config->GetComponentLoggingSettings(&OpKelpieList::debug_enabled, nullptr, nullptr, "kelpie.op.list"); } } - - - -kelpie::OpKelpieList::OpKelpieList( +/** + * @brief Create a new List operation + * + * @param[in] targets A list of nodes where this should be sent + * @param[in] bucket The bucket namespace for this object + * @param[in] search_key The key label for the object (may be a wildcard) + * @param[in] iom_hash The hash code of an iom to check if this pool has storage + * @param[out] object_capacities Place to return information about the found objects + * @param[out] caller_promise Notification of completion + */ +OpKelpieList::OpKelpieList( const vector> targets, const faodel::bucket_t bucket, const kelpie::Key &search_key, + const kelpie::iom_hash_t &iom_hash, ObjectCapacities *object_capacities, - condition_variable *cv, int *num_targets_left) + std::promise *caller_promise) - : Op(true), targets(targets), bucket(bucket), search_key(search_key), + : Op(true), + targets(targets), bucket(bucket), search_key(search_key), iom_hash(iom_hash), user_object_capacities(object_capacities), - user_cv(cv), user_num_targets_left(num_targets_left), + caller_promise(caller_promise), state(State::orig_list_send) { + num_targets_left = targets.size(); //Defer message creation until send state } @@ -56,15 +70,14 @@ OpKelpieList::OpKelpieList(Op::op_create_as_target_t t) kelpie::OpKelpieList::~OpKelpieList() { if(state!=State::done){ - KTODO("Dtor when state not done"); + F_TODO("Dtor when state not done"); } } WaitingType OpKelpieList::smo_List_Send() { - //Remember how many nodes we have left - *user_num_targets_left = targets.size(); - if(*user_num_targets_left<1) { + //Check node count to make sure we have work + if(num_targets_left<1) { dbg("Bail: op didn't have any targets"); return updateStateDone(); } @@ -75,13 +88,12 @@ WaitingType OpKelpieList::smo_List_Send() { dbg("Sending to target "+node_peerptr.first.GetHex()); lunasa::DataObject ldo; - msg_direct_buffer_t::Alloc( + msg_direct_simple_t::Alloc( ldo, op_id, DirectFlags::CMD_LIST, node_peerptr.first, mbox, opbox::MAILBOX_UNSPECIFIED, bucket, search_key, - 0, PoolBehavior::NoAction, //TODO: Add iom and behavior flags - nullptr); + iom_hash, PoolBehavior::NoAction); net::SendMsg(node_peerptr.second, std::move(ldo)); } return updateState(State::orig_list_wait_for_results, WaitingType::waiting_on_cq); @@ -92,19 +104,24 @@ WaitingType OpKelpieList::smt_List_Start(opbox::OpArgs *args) { net::peer_ptr_t peer; - auto imsg = args->ExpectMessageOrDie(&peer); + auto imsg = args->ExpectMessageOrDie(&peer); search_key = imsg->ExtractKey(); - dbg("Target1 received a request for "+search_key.str()); - ObjectCapacities found_object_capacities; - rc_t rc = lkv->list(imsg->bucket, search_key, &found_object_capacities); + dbg("Target received a list request for "+search_key.str()); + + // If there's an IOM, attempt to find matching objects there. + auto *iom = kelpie::internal::FindIOM(imsg->iom_hash); + + rc_t rc = lkv->list(imsg->bucket, search_key, iom, &found_object_capacities); + dbg("Target list found objects final: "+to_string(found_object_capacities.capacities.size())); + uint16_t simple_rc = (rc==KELPIE_OK) ? 0 : 1; - //Create a reply message. For simplicity just boost pack it in the payload + //Create a reply message. For simplicity just cereal pack it in the payload lunasa::DataObject ldo_out; - AllocateBoostReplyMessage(ldo_out, &imsg->hdr, simple_rc, found_object_capacities); + AllocateCerealReplyMessage(ldo_out, &imsg->hdr, simple_rc, found_object_capacities); net::SendMsg(peer, std::move(ldo_out)); @@ -117,15 +134,15 @@ WaitingType OpKelpieList::smo_List_WaitForResults(opbox::OpArgs *args) { auto imsg = args->ExpectMessageOrDie(&peer); //Annoyingly, we extract to a temp struct and then append the users - auto found_object_capacities = UnpackBoostMessage(imsg); + auto found_object_capacities = UnpackCerealMessage(imsg); user_object_capacities->Append(found_object_capacities); - (*user_num_targets_left)--; - dbg("Origing received response. num_left="+std::to_string(*user_num_targets_left)); + num_targets_left--; + dbg("Origin received response. num_left="+std::to_string(num_targets_left)); - if(*user_num_targets_left<1) { + if(num_targets_left<1) { dbg("Received last item. Notifying user of result"); - user_cv->notify_one(); + caller_promise->set_value(true); return updateStateDone(); } @@ -141,7 +158,7 @@ WaitingType OpKelpieList::Update(OpArgs *args) { case State::orig_list_wait_for_results: return smo_List_WaitForResults(args); case State::done: return updateStateDone(); } - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -157,7 +174,7 @@ std::string OpKelpieList::GetStateName() const { case State::orig_list_wait_for_results: return "Origin-List-WaitForResults"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); } diff --git a/src/kelpie/ops/direct/OpKelpieList.hh b/src/kelpie/ops/direct/OpKelpieList.hh index 323e101..8949f39 100644 --- a/src/kelpie/ops/direct/OpKelpieList.hh +++ b/src/kelpie/ops/direct/OpKelpieList.hh @@ -1,11 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_OPKELPIELIST_HH #define KELPIE_OPKELPIELIST_HH #include +#include #include "opbox/OpBox.hh" #include "opbox/ops/OpHelpers.hh" @@ -38,12 +39,12 @@ public: const std::vector> targets, const faodel::bucket_t bucket, const Key &search_key, + const iom_hash_t &iom_hash, ObjectCapacities *object_capacities, - std::condition_variable *cv, - int *num_targets_left); + std::promise *caller_promise); //A target starts off the same way no matter what command - OpKelpieList(Op::op_create_as_target_t t); + explicit OpKelpieList(Op::op_create_as_target_t t); ~OpKelpieList() override; @@ -67,7 +68,7 @@ private: //todo: put this in a standard form so it can be reused #if Faodel_LOGGINGINTERFACE_DISABLED==0 - void dbg(std::string s) const { + void dbg(const std::string &s) const { if(OpKelpieList::debug_enabled) { std::cout << "\033[1;31mD " << op_name << ":\033[0m " << (s) << std::endl; } @@ -84,10 +85,11 @@ private: std::vector> targets; faodel::bucket_t bucket; Key search_key; + iom_hash_t iom_hash; ObjectCapacities *user_object_capacities; - std::condition_variable *user_cv; //FIXME - int *user_num_targets_left; + std::promise *caller_promise; + int num_targets_left; State state; diff --git a/src/kelpie/ops/direct/OpKelpieMeta.cpp b/src/kelpie/ops/direct/OpKelpieMeta.cpp index f0480d7..d79b333 100644 --- a/src/kelpie/ops/direct/OpKelpieMeta.cpp +++ b/src/kelpie/ops/direct/OpKelpieMeta.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -14,6 +14,7 @@ using namespace kelpie; //Statics: Standard id/name info for an op const unsigned int OpKelpieMeta::op_id = const_hash("OpKelpieMeta"); const string OpKelpieMeta::op_name = "OpKelpieMeta"; +bool OpKelpieMeta::debug_enabled = false; //Statics: This op has a static localkv pointer. lkv lives inside KelpieCore instance LocalKV * OpKelpieMeta::lkv = nullptr; @@ -22,10 +23,14 @@ LocalKV * OpKelpieMeta::lkv = nullptr; * @brief Internal startup command for setting static variables * * @param[in] iuo Designates this function is for internal use only + * @param[in] config Pointer to configuration so that kelpie.op.meta settings can be retrieved * @param[in] new_lkv A pointer to the kelpie localkv we should uses_allocator */ -void OpKelpieMeta::configure(faodel::internal_use_only_t iuo, LocalKV *new_lkv) { +void OpKelpieMeta::configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, LocalKV *new_lkv) { lkv = new_lkv; + if(config) { + config->GetComponentLoggingSettings(&OpKelpieMeta::debug_enabled, nullptr, nullptr, "kelpie.op.meta"); + } } @@ -52,16 +57,12 @@ OpKelpieMeta::OpKelpieMeta( : Op(true), peer(target_ptr), - cb_info_result(cb_result) { - - msg_direct_buffer_t::Alloc(ldo_msg, op_id, xferdirect_command, target_node, GetAssignedMailbox(), - opbox::MAILBOX_UNSPECIFIED, bucket, key, iom_hash, PoolBehavior::NoAction, nullptr); - - state_after_start = State::orig_colinfo_wait_for_ack; + cb_info_result(cb_result), + state(State::orig_meta_send) { + msg_direct_simple_t::Alloc(ldo_msg, op_id, xferdirect_command, target_node, GetAssignedMailbox(), + opbox::MAILBOX_UNSPECIFIED, bucket, key, iom_hash, PoolBehavior::NoAction); - //All states start in a send - state = State::orig_meta_send; } /** @@ -74,7 +75,7 @@ OpKelpieMeta::OpKelpieMeta(Op::op_create_as_target_t t) : Op(t), state(State::trgt_meta_start), ldo_msg() { //No work to do - done in target's state machine - peer = 0; + peer = nullptr; GetAssignedMailbox(); //For safety, get a mailbox. Not needed everywhere? } @@ -83,80 +84,78 @@ OpKelpieMeta::OpKelpieMeta(Op::op_create_as_target_t t) * @brief Destructor for OpKelpieMeta */ OpKelpieMeta::~OpKelpieMeta(){ - //cout <<"OpKelpieMeta() dtor : state is "<ExpectMessageOrDie(&peer); + auto imsg = args->ExpectMessageOrDie(&peer); auto cmd = imsg->GetCommand(); //Grab essential from this message for later user - bucket = imsg->bucket; - key = imsg->ExtractKey(); + auto bucket = imsg->bucket; + auto key = imsg->ExtractKey(); auto target_iom = imsg->iom_hash; - if(cmd == DirectFlags::CMD_GET_COLINFO) { - //This is a plain get colinfo. Result goes back in a simple message + bool is_colinfo = (cmd == DirectFlags::CMD_GET_COLINFO); + bool is_rowinfo = (cmd == DirectFlags::CMD_GET_ROWINFO); + + dbg("Received meta request for "+key.str()); + if( is_colinfo || is_rowinfo) { + + //Allocate a result message for us to store our row/col info auto omsg = msg_direct_status_t::AllocAck(ldo_msg, &imsg->hdr); - omsg->remote_rc = lkv->getColInfo(bucket, key, &omsg->col_info); - if((omsg->remote_rc!=KELPIE_OK) && (target_iom!=0)) { - auto *iom = kelpie::internal::Singleton::impl.core->FindIOM(target_iom); - if(iom==nullptr) { - throw runtime_error("OpKelpieGetBounded attempted to read key "+key.str()+" to a node with a bad iom"); + rc_t tmp_rc = lkv->getInfo(bucket, key, &omsg->object_info); + + //For cols, we can dig more info out of the iom if we don't have anything in memory + if((tmp_rc != KELPIE_OK) && (target_iom != 0) && (is_colinfo)) { + auto *iom = kelpie::internal::FindIOM(target_iom); + if(iom == nullptr) { + throw runtime_error("OpKelpieMeta attempted to read key " + key.str() + " from a node with a bad iom ID"); } - omsg->remote_rc = iom->GetInfo(bucket, key, &omsg->col_info); - //cout <<" OPMeta-TRG: Went to disk and got back "<remote_rc<<" col is "<col_info.str()<GetInfo(bucket, key, &omsg->object_info); } + omsg->remote_rc = tmp_rc; - - if((omsg->remote_rc != 0) && (imsg->CanStall())){ - //Not here but user can wait - KTODO("Get Column Info"); - //return; - } else { - //Good or bad, we're sending back a result - net::SendMsg(peer, std::move(ldo_msg)); - return updateStateDone(); //TODO - } + //Send reply + net::SendMsg(peer, std::move(ldo_msg)); + return updateStateDone(); } else { //Unknown op - KTODO("Unknown op?"); + F_TODO("Unknown op?"); } } -//ORIGIN: Process a ColInfo response -WaitingType OpKelpieMeta::smo_colinfo_WaitAck(OpArgs *args) { +//ORIGIN: Process a response +WaitingType OpKelpieMeta::smo_WaitAck(OpArgs *args) { auto imsg = args->ExpectMessageOrDie(); - kassert(imsg->IsStatus(), "Expecting a Status message"); + F_ASSERT(imsg->IsStatus(), "Expecting a Status message"); - if(cb_info_result!=nullptr){ - cb_info_result(imsg->remote_rc, imsg->row_info, imsg->col_info); + dbg("Received meta info"); + if(cb_info_result!=nullptr) { + imsg->object_info.ChangeAvailabilityFromLocalToRemote(); //Convert from local to remote + cb_info_result(imsg->remote_rc, imsg->object_info); } return updateStateDone(); } - -WaitingType OpKelpieMeta::Update(opbox::OpArgs *args){ +WaitingType OpKelpieMeta::Update(opbox::OpArgs *args) { //cout <<"OPMETA-Update (in state "<< GetStateName()<<" Args: "<str(2,1) < -#include +#include "kelpie/core/Singleton.hh" #include "kelpie/ops/direct/OpKelpiePublish.hh" @@ -13,6 +13,7 @@ using namespace kelpie; //Statics: Standard id/name info for an op const unsigned int OpKelpiePublish::op_id = const_hash("OpKelpiePublish"); const string OpKelpiePublish::op_name = "OpKelpiePublish"; +bool OpKelpiePublish::debug_enabled = false; //Statics: This op has a static localkv pointer. lkv lives inside KelpieCore instance LocalKV * OpKelpiePublish::lkv = nullptr; @@ -21,10 +22,14 @@ LocalKV * OpKelpiePublish::lkv = nullptr; * @brief Internal startup command for setting static variables * * @param[in] iuo Designates this function is for internal use only + * @param[in] config Pointer to configuration so that kelpie.op.publish settings can be retrieved * @param[in] new_lkv A pointer to the kelpie localkv we should uses_allocator */ -void OpKelpiePublish::configure(faodel::internal_use_only_t iuo, LocalKV *new_lkv) { +void OpKelpiePublish::configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, LocalKV *new_lkv) { lkv = new_lkv; + if(config) { + config->GetComponentLoggingSettings(&OpKelpiePublish::debug_enabled, nullptr, nullptr, "kelpie.op.publish"); + } } @@ -56,8 +61,9 @@ OpKelpiePublish::OpKelpiePublish( ldo_data = ldo_users_data; //We need to keep a copy of the ldo until sent - bool exceeds = msg_direct_buffer_t::Alloc(ldo_msg, op_id, DirectFlags::CMD_PUBLISH, target_node, GetAssignedMailbox(), - opbox::MAILBOX_UNSPECIFIED, bucket, key, iom_hash, behavior_flags, &ldo_data); + /*bool exceeds =*/ msg_direct_buffer_t::Alloc(ldo_msg, op_id, DirectFlags::CMD_PUBLISH, target_node, GetAssignedMailbox(), + opbox::MAILBOX_UNSPECIFIED, bucket, key, iom_hash, behavior_flags, + &ldo_data); } @@ -71,7 +77,7 @@ OpKelpiePublish::OpKelpiePublish(Op::op_create_as_target_t t) : Op(t), state(State::trgt_pub_start), ldo_msg(){ //No work to do - done in target's state machine - peer = 0; + peer = nullptr; target_iom=0; target_behavior_flags=0; GetAssignedMailbox(); //For safety, get a mailbox. Not needed everywhere? @@ -94,7 +100,7 @@ OpKelpiePublish::~OpKelpiePublish(){ WaitingType OpKelpiePublish::smo_Publish_Send(){ //All origin options start here, but branch out to different states //cout <<"OPPUB-ORG: About to send message\n"; - + dbg("Sending initial Publish message"); net::SendMsg(peer, std::move(ldo_msg)); return updateState(State::orig_pub_wait_for_ack, WaitingType::waiting_on_cq); } @@ -113,15 +119,30 @@ WaitingType OpKelpiePublish::smt_Publish_Start(OpArgs *args) { target_behavior_flags = PoolBehavior::ChangeRemoteToLocal(imsg->behavior_flags); //cout <<"OPPUB-TRG: message is a publish. OMBox="<hdr.src_mailbox<<" Meta+Data Length is "<meta_plus_data_size<<" key is "<meta_plus_data_size)); + + //Create return ack message + msg_direct_status_t::AllocAck(ldo_msg, &imsg->hdr); + + //See if we can ignore this request when it the object already exists + if(!(target_behavior_flags & PoolBehavior::EnableOverwrites)) { + object_info_t info; + rc_t rc = lkv->getInfo(bucket, key, &info); + if((rc==KELPIE_OK) && (info.col_availability == Availability::InLocalMemory)) { + //The object is here. We can just send an ack back + dbg("Object already exists but we don't have permission to overwrite it. Sending an ack."); + auto omsg = ldo_msg.GetDataPtr(); + omsg->Success(true); + omsg->remote_rc = KELPIE_EEXIST; + net::SendMsg(peer, std::move(ldo_msg)); + return updateStateDone(); + } + //Item not here or in waiting state. Go ahead and fetch it. + } - //TODO: See if Kelpie already has this object //Allocate space for incoming data ldo_data = lunasa::DataObject(0, imsg->meta_plus_data_size, lunasa::DataObject::AllocatorType::eager); - //cout <<"OPPUB-TRG ldo allocated size is "<hdr); //Setup RDMA for transferring data net::Get(peer, &nbr, ldo_data, AllEventsCallback(this)); @@ -134,10 +155,11 @@ WaitingType OpKelpiePublish::smt_Publish_Start(OpArgs *args) { WaitingType OpKelpiePublish::smt_Publish_WaitRDMA(opbox::OpArgs *args){ //cout <<"OPPUB-TRG: got dma done notification. sending ack. Key is "<type == UpdateType::send_success) { - cout<<"FIXME: got send_success instead of get_success\n"; + dbg("FIXME: got send_success instead of get_success"); return WaitingType::waiting_on_cq; } //FIXME^^ @@ -146,12 +168,16 @@ WaitingType OpKelpiePublish::smt_Publish_WaitRDMA(opbox::OpArgs *args){ auto omsg = ldo_msg.GetDataPtr(); - //Got the data, now store it locally in memory - rc_t rc = lkv->put(bucket, key, ldo_data, target_behavior_flags, &omsg->row_info, &omsg->col_info); + //Got the data, use the lkv to store it. Translate iom if provided. + internal::IomBase *iom = nullptr; + if((target_iom!=0) && (target_behavior_flags & PoolBehavior::WriteToIOM)) { + iom = kelpie::internal::FindIOM(target_iom); + } + rc_t rc = lkv->put(bucket, key, ldo_data, target_behavior_flags, iom, &omsg->object_info); //See if it also goes to an iom if((target_iom!=0) && (target_behavior_flags & PoolBehavior::WriteToIOM)){ - auto *iom = kelpie::internal::Singleton::impl.core->FindIOM(target_iom); + auto *iom = kelpie::internal::FindIOM(target_iom); if(iom==nullptr) { throw runtime_error("OpKelpiePublish attempted to write key "+key.str()+" to a node with a bad iom"); } @@ -168,7 +194,8 @@ WaitingType OpKelpiePublish::smt_Publish_WaitRDMA(opbox::OpArgs *args){ //ORIGIN: Wait for an ACK WaitingType OpKelpiePublish::smo_Publish_WaitAck(opbox::OpArgs *args) { - //cout <<"OPPUB-ORG: got ack\n"; + + dbg("Received an ack"); auto imsg = args->ExpectMessageOrDie(); //TODO: better handling of unexpected arg/message @@ -176,10 +203,9 @@ WaitingType OpKelpiePublish::smo_Publish_WaitAck(opbox::OpArgs *args) { //Got a reply. We no longer have to hold on to the publish data's ldo. If //caller specified a callback, pass the result back if(cb_info_result!=nullptr){ - //cout <<"OPPUB-ORG got ack Reply. Need to extract the return code\n"; - imsg->row_info.ChangeAvailabilityFromLocalToRemote(); - imsg->col_info.ChangeAvailabilityFromLocalToRemote(); - cb_info_result(imsg->remote_rc, imsg->row_info, imsg->col_info); + dbg("Got ack Reply. Remote rc was "+to_string(imsg->remote_rc)+" success "+ to_string(imsg->Success())+"\n"); + imsg->object_info.ChangeAvailabilityFromLocalToRemote(); + cb_info_result(imsg->remote_rc, imsg->object_info); } return updateStateDone(); } @@ -193,9 +219,8 @@ WaitingType OpKelpiePublish::Update(opbox::OpArgs *args){ case State::trgt_pub_wait_for_rdma: return smt_Publish_WaitRDMA(args); case State::orig_pub_wait_for_ack: return smo_Publish_WaitAck(args); case State::done: return updateStateDone(); - default: ; } - KFAIL(); + F_FAIL(); return WaitingType::error; } @@ -211,5 +236,5 @@ std::string OpKelpiePublish::GetStateName() const { case State::orig_pub_wait_for_ack: return "Origin-Publish-WaitForAck"; case State::done: return "Done"; } - KFAIL(); + F_FAIL(); } diff --git a/src/kelpie/ops/direct/OpKelpiePublish.hh b/src/kelpie/ops/direct/OpKelpiePublish.hh index 5740e74..758037b 100644 --- a/src/kelpie/ops/direct/OpKelpiePublish.hh +++ b/src/kelpie/ops/direct/OpKelpiePublish.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_OPKELPIEPUBLISH_HH #define KELPIE_OPKELPIEPUBLISH_HH @@ -43,12 +43,14 @@ public: fn_publish_callback_t callback); //A target starts off the same way no matter what command - OpKelpiePublish(Op::op_create_as_target_t t); + explicit OpKelpiePublish(Op::op_create_as_target_t t); ~OpKelpiePublish() override; //Unique name and id for this op const static unsigned int op_id; const static std::string op_name; + static bool debug_enabled; //!< Dump debug messages + unsigned int getOpID() const override { return op_id; } std::string getOpName() const override { return op_name; } @@ -58,9 +60,22 @@ public: std::string GetStateName() const override; - static void configure(faodel::internal_use_only_t iuo, LocalKV *new_lkv); + static void configure(faodel::internal_use_only_t iuo, const faodel::Configuration *config, LocalKV *new_lkv); private: + + //todo: put this in a standard form so it can be reused + #if Faodel_LOGGINGINTERFACE_DISABLED==0 + void dbg(const std::string &s) const { + if(OpKelpiePublish::debug_enabled) { + std::cout << "\033[1;93mD " << op_name << ": ["< -#include #include #include @@ -37,28 +36,131 @@ void DirectFlags::Success(opbox::message_t *msg, bool is_success) { bool DirectFlags::Success(const opbox::message_t *msg) { return ((msg->user_flags & FLAG_IS_SUCCESS) == FLAG_IS_SUCCESS); } +bool DirectFlags::IsAck(const opbox::message_t *msg) { + return ((msg->user_flags & CMD_STATUS_ACK)==CMD_STATUS_ACK); +} +bool DirectFlags::IsNack(const opbox::message_t *msg) { + return ((msg->user_flags & CMD_STATUS_ACK)==CMD_STATUS_NACK); +} +bool +msg_direct_simple_t::Alloc(lunasa::DataObject &ldo_msg, const uint32_t op_id, const uint16_t command_and_flags, const faodel::nodeid_t dst, + const opbox::mailbox_t src_mailbox, const opbox::mailbox_t dst_mailbox, const faodel::bucket_t bucket, + const kelpie::Key &key, const kelpie::iom_hash_t iom_hash, const kelpie::pool_behavior_t behavior_flags, + const std::string &function_name, const std::string &function_args) { -void msg_direct_buffer_t::SetLDO(lunasa::DataObject *ldo_data) { - meta_plus_data_size = (ldo_data) ? (ldo_data->GetMetaSize()+ldo_data->GetDataSize()) : 0; - if(meta_plus_data_size){ - //Convert to local nbr - net::NetBufferLocal *nbl = nullptr; - net::GetRdmaPtr(ldo_data, &nbl, &net_buffer_remote); - } else { - //no reference - memset(&net_buffer_remote, 0, sizeof(net::NetBufferRemote)); + size_t string_size = key.size() + function_name.size() + function_args.size(); + + //Allocate the message + ldo_msg = net::NewMessage(sizeof(msg_direct_simple_t)+string_size); + + //Get a pointer we can work with + auto *msg = ldo_msg.GetDataPtr(); + + //Fill in identity info + msg->k1_size = key.k1_size(); + msg->k2_size = key.k2_size(); + msg->bucket = bucket; + msg->iom_hash = iom_hash; + msg->behavior_flags = behavior_flags; + msg->function_name_size = (function_name.size()) & 0x0FF; + msg->function_args_size = (function_args.size()) & 0x0FFFF; + + //Set the header now that we know our key sizes + msg->hdr.SetStandardRequest(dst, src_mailbox, op_id, command_and_flags); + msg->hdr.dst_mailbox = dst_mailbox; + + //Fix the header length + msg->hdr.body_len = (sizeof(msg_direct_simple_t) - sizeof(opbox::message_t) + string_size); + + //Append all our string data at the end + char *ptr = &msg->string_data[0]; + memcpy(ptr, key.K1().c_str(), msg->k1_size); ptr += msg->k1_size; + memcpy(ptr, key.K2().c_str(), msg->k2_size); ptr += msg->k2_size; + memcpy(ptr, function_name.c_str(), msg->function_name_size); ptr += msg->function_name_size; + memcpy(ptr, function_args.c_str(), msg->function_args_size); + + return (ldo_msg.GetWireSize() > MESSAGE_MTU); +} + +kelpie::Key msg_direct_simple_t::ExtractKey() const { + kelpie::Key key( std::string(&string_data[0], static_cast(k1_size)), + std::string(&string_data[k1_size], static_cast(k2_size)) ); + if(!key.valid()){ + throw std::invalid_argument("msg_direct_request had an invalid key"); } + return key; +} + +/** + * @brief Parse this message and extract the Key, Function Name, and Function Args + * @param[out] key Kelpie Key + * @param[out] function_name The string value of the function + * @param[out] function_args The string arguments to pass to the compute function + */ +void msg_direct_simple_t::ExtractComputeArgs(kelpie::Key *key, + string *function_name, string *function_args) const { + string s1,s2; + const char *ptr = &string_data[0]; + s1 = std::string( ptr, static_cast(k1_size)); ptr+=k1_size; + s2 = std::string( ptr, static_cast(k2_size)); ptr+=k2_size; + *key = kelpie::Key( s1, s2); + if(!key->valid()) { + throw std::invalid_argument("msg_direct_request had an invalid key"); + } + + *function_name = std::string( ptr, static_cast(function_name_size)); ptr+=function_name_size; + *function_args = std::string( ptr, static_cast(function_args_size)); + +} + +std::string msg_direct_simple_t::str() { + + kelpie::Key key; + string function_name, function_args; + ExtractComputeArgs(&key, &function_name, &function_args); + + std::stringstream ss; + ss<<"msg_direct_simple_t :" + <<"\n meta+data_size "<(); @@ -78,26 +180,38 @@ msg_direct_buffer_t::Alloc(lunasa::DataObject &ldo_msg, const uint32_t op_id, co msg->hdr.dst_mailbox = dst_mailbox; //Fix the header length - msg->hdr.body_len = (sizeof(msg_direct_buffer_t) - sizeof(opbox::message_t)) + - msg->k1_size + msg->k2_size; + msg->hdr.body_len = (sizeof(msg_direct_buffer_t) - sizeof(opbox::message_t) + string_size); - //Pack the key data into the end - memcpy(&msg->key_data[0], key.K1().c_str(), msg->k1_size); - memcpy(&msg->key_data[msg->k1_size], key.K2().c_str(), msg->k2_size); + //Append the key to the string section + char *ptr = &msg->string_data[0]; + memcpy(ptr, key.K1().c_str(), msg->k1_size); ptr += msg->k1_size; + memcpy(ptr, key.K2().c_str(), msg->k2_size); ptr += msg->k2_size; return (ldo_msg.GetWireSize() > MESSAGE_MTU); } +void msg_direct_buffer_t::SetLDO(lunasa::DataObject *ldo_data) { + meta_plus_data_size = (ldo_data) ? (ldo_data->GetMetaSize()+ldo_data->GetDataSize()) : 0; + if(meta_plus_data_size){ + //Convert to local nbr + net::NetBufferLocal *nbl = nullptr; + net::GetRdmaPtr(ldo_data, &nbl, &net_buffer_remote); + } else { + //no reference + memset(&net_buffer_remote, 0, sizeof(net::NetBufferRemote)); + } +} -kelpie::Key msg_direct_buffer_t::ExtractKey(){ - kelpie::Key key( std::string(&key_data[0], k1_size), - std::string(&key_data[k1_size], k2_size) ); +kelpie::Key msg_direct_buffer_t::ExtractKey() const { + kelpie::Key key( std::string(&string_data[0], static_cast(k1_size)), + std::string(&string_data[k1_size], static_cast(k2_size)) ); if(!key.valid()){ throw std::invalid_argument("msg_direct_request had an invalid key"); } return key; } + std::string msg_direct_buffer_t::str() { std::stringstream ss; kelpie::Key key = ExtractKey(); @@ -132,7 +246,7 @@ std::string msg_direct_status_t::str() { msg_direct_status_t * msg_direct_status_t::Alloc( lunasa::DataObject &ldo_msg, message_t *incoming_msg_hdr, - uint16_t user_flags){ + uint16_t user_flags) { //Allocate the message ldo_msg = net::NewMessage(sizeof(msg_direct_status_t)); @@ -141,7 +255,7 @@ msg_direct_status_t * msg_direct_status_t::Alloc( auto msg = ldo_msg.GetDataPtr(); //Wipe all fields.. probably just need hdr and target rc - memset(msg, 0, sizeof(msg_direct_status_t)); + memset((void *)msg, 0, sizeof(msg_direct_status_t)); //Populate header msg->hdr.SetStandardReply(incoming_msg_hdr, diff --git a/src/kelpie/ops/direct/msg_direct.hh b/src/kelpie/ops/direct/msg_direct.hh index 0a2a1cc..cbb5f9f 100644 --- a/src/kelpie/ops/direct/msg_direct.hh +++ b/src/kelpie/ops/direct/msg_direct.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_MSG_DIRECT_HH #define KELPIE_MSG_DIRECT_HH @@ -16,9 +16,10 @@ namespace kelpie { -// The Direct ops use two kinds of messages: +// The Direct ops use three kinds of messages: // - buffer : for sending a buffer handle, bucket, and key to remote // - status : for passing back ack/nacks and info about a k/v +// - listreply : custom cereal-packed message with info for reply messages // // The following messages are sent during different types of // @@ -41,13 +42,29 @@ namespace kelpie { // meta-colinfo // origin: sends a BUFFER message with CMD_GET_COLINFO // target: sends a STATUS message with col_info populated +// origin: relays back to user via callback +// +// meta-rowinfo +// origin: sends a BUFFER message with CMD_GET_ROWINFO +// target: sends a STATUS message with row_info populated +// origin: relays back to user via callback +// +// list: get a list of all objects and their lengths that match a search string +// origin: sends BUFFER message with CMD_LIST to one or more targets +// target: sends custom cereal message with all the info packed into a message +// origin: gathers all responses. Ends when all collected. +// +// drop: specify node should drop all objects matching a search string +// origin: sends BUFFER message with CMD_DROP to one or more targets +// target: optional send STATUS message with Ack/Nack info +// origin: gathers all responses. Ends when all collected. // // USER_FLAGS // OpDirect uses the message USER_FLAGS to pass along info. // 0x0080 : Command message type mask (0x0000=status msg, 0x0080=buffer msg) // 0x00F0 : Command mask (eg, CMD_PUBLISH, CMD_GET_BOUNDED, etc) // 0x0002 : Request should stall until complete (0x0002=stall, 0x0000=return immediately) -// 0x0001 : Status success mask ( 0x0001=succes, 0x0000=failure) +// 0x0001 : Status success mask ( 0x0001=success, 0x0000=failure) /** @@ -56,13 +73,16 @@ namespace kelpie { struct DirectFlags { static constexpr uint16_t CMD_MASK = 0x00F0; + static constexpr uint16_t CMD_COMPUTE = 0x0080; + static constexpr uint16_t CMD_PUBLISH = 0x0090; static constexpr uint16_t CMD_GET_BOUNDED = 0x00A0; static constexpr uint16_t CMD_GET_UNBOUNDED = 0x00B0; static constexpr uint16_t CMD_GET_COLINFO = 0x00C0; - static constexpr uint16_t CMD_DELETE = 0x00D0; + static constexpr uint16_t CMD_GET_ROWINFO = 0x00D0; static constexpr uint16_t CMD_LIST = 0x00E0; + static constexpr uint16_t CMD_DROP = 0x00F0; static constexpr uint16_t CMD_STATUS_ACK = 0x0011; static constexpr uint16_t CMD_STATUS_NACK = 0x0010; @@ -78,6 +98,8 @@ struct DirectFlags { static void CanStall(opbox::message_t *msg, bool can_stall); static void Success(opbox::message_t *msg, bool is_success); static bool Success(const opbox::message_t *msg); + static bool IsAck(const opbox::message_t *msg); + static bool IsNack(const opbox::message_t *msg); }; @@ -85,6 +107,62 @@ struct DirectFlags { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" + +/** + * @brief Message format for sending commands in Direct Messages that don't need to pass a remote_buffer_pointer + * @note Actual commands are stored in the user_flags section of the header + * @note The string section of the message may hold the key and remote function name/args + * Key data is manually packed into the key_data section at the end of + * the message, so you MUST use the Alloc() function to create a new message + */ +struct msg_direct_simple_t { + opbox::message_t hdr; //!< Standard header field + + uint64_t meta_plus_data_size; //!< Used on remote end to allocate LDO for data + uint16_t k1_size; //!< Used for serdes of key.k1 + uint16_t k2_size; //!< Used for serdes of key.k2 + faodel::bucket_t bucket; //!< Hashed bucket id + iom_hash_t iom_hash; //!< Hash of the IOM to use + pool_behavior_t behavior_flags; //!< Flags specifying actions to take + uint8_t function_name_size; //!< Length of function name for compute + uint16_t function_args_size; //!< Length of function args for compute + + char string_data[0]; //!< Place where additional packed key data is stored + + //Since key_data is variable prevent user from using an empty ctor, as + //the Set() operation would overflow. We expect Op to allocate the space + //in an LDO, so we don't have any ctors here. + msg_direct_simple_t()=delete; + + uint16_t GetCommand() const { return DirectFlags::GetCommand(&hdr); } + bool IsStatus() const { return DirectFlags::IsStatus(&hdr); } + void CanStall(bool can_stall) { return DirectFlags::CanStall(&hdr, can_stall); } //setter + bool CanStall() const { return DirectFlags::CanStall(&hdr); } //getter + + kelpie::Key ExtractKey() const; + void ExtractComputeArgs(kelpie::Key *key, std::string *function_name, std::string *function_args) const; + std::string str(); + + + static bool Alloc(lunasa::DataObject &new_ldo, //!< New LDO generated for holding this message + const uint32_t op_id, //!< Which op this is for + const uint16_t command_and_flags, //!< The command/flags users wants to set + const faodel::nodeid_t dst, //!< Where this message is going + const opbox::mailbox_t src_mailbox, //!< What our mailbox should be + const opbox::mailbox_t dst_mailbox, //!< Destination mailbox (if a response) + const faodel::bucket_t bucket, //!< Hashed bucket id for message + const kelpie::Key &key, //!< The key for this request + const kelpie::iom_hash_t iom_hash, //!< Optional IO Module Hash id associated with this op + const kelpie::pool_behavior_t behavior_flags, //!< Behavior settings for this transfer + const std::string &function_name="", //!< String name of function to use (only in Compute ops) + const std::string &function_args="" //!< String of args to pass (only in Compute ops) + ); + + //note: rdma address lookups may make ldo_data non const TODO +}; + + + /** * @brief Message format for sending commands in Direct messages involving keys or rdma pointers * @note Actual commands are stored in the user_flags section of the header @@ -102,26 +180,26 @@ struct msg_direct_buffer_t { iom_hash_t iom_hash; //!< Hash of the IOM to use pool_behavior_t behavior_flags; //!< Flags specifying actions to take - char key_data[0]; //!< Place where packed key data is stored + char string_data[0]; //!< Place where additional packed key data is stored //Since key_data is variable prevent user from using an empty ctor, as //the Set() operation would overflow. We expect Op to allocate the space //in an LDO, so we don't have any ctors here. msg_direct_buffer_t()=delete; - uint16_t GetCommand() { return DirectFlags::GetCommand(&hdr); } - bool IsStatus() { return DirectFlags::IsStatus(&hdr); } - void CanStall(bool can_stall) { return DirectFlags::CanStall(&hdr, can_stall); } - bool CanStall() { return DirectFlags::CanStall(&hdr); } + uint16_t GetCommand() const { return DirectFlags::GetCommand(&hdr); } + bool IsStatus() const { return DirectFlags::IsStatus(&hdr); } + void CanStall(bool can_stall) { return DirectFlags::CanStall(&hdr, can_stall); } //setter + bool CanStall() const { return DirectFlags::CanStall(&hdr); } //getter void SetLDO(lunasa::DataObject *ldo_data); - kelpie::Key ExtractKey(); + kelpie::Key ExtractKey() const; std::string str(); static bool Alloc(lunasa::DataObject &new_ldo, //!< New LDO generated for holding this message - const uint32_t op_id, //!< Which op this is for + const uint32_t op_id, //!< Which opbox op this is for const uint16_t command_and_flags, //!< The command/flags users wants to set const faodel::nodeid_t dst, //!< Where this message is going const opbox::mailbox_t src_mailbox, //!< What our mailbox should be @@ -137,6 +215,9 @@ struct msg_direct_buffer_t { }; + + + /** * @brief Provide a short status message back to a sender with row/col info * @note Call an Alloc function to create an appropriately-sized LDO and @@ -145,8 +226,7 @@ struct msg_direct_buffer_t { struct msg_direct_status_t { opbox::message_t hdr; //!< Standard header field int remote_rc; //!< Return code seen at the other node - kv_row_info_t row_info; //!< Statistics for this k/v's row (optional) - kv_col_info_t col_info; //!< Statistics for this k/v's col (optional) + object_info_t object_info; //!< Statistics about this object's row/column bool IsStatus() { return DirectFlags::IsStatus(&hdr); } void Success(bool is_success) { return DirectFlags::Success(&hdr, is_success); } @@ -178,23 +258,6 @@ struct msg_direct_status_t { }; - - -struct msg_direct_list_result_t { - opbox::message_t hdr; //!< Standard header field - int remote_rc; - int num_entries; - char *packed_data; - - static msg_direct_list_result_t * Alloc( - lunasa::DataObject &new_ldo_ptr, - message_t *origin_msg_hdr, - std::vector &keys, - std::vector *capacities); - -}; - - #pragma GCC diagnostic pop //For ignoring array[0] kinds of allocation } // namespace kelpie diff --git a/src/kelpie/pools/DHTPool/DHTPool.cpp b/src/kelpie/pools/DHTPool/DHTPool.cpp index d24aa89..ca2c008 100644 --- a/src/kelpie/pools/DHTPool/DHTPool.cpp +++ b/src/kelpie/pools/DHTPool/DHTPool.cpp @@ -1,22 +1,23 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. - -#include -#include -#include -#include -#include +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include #include #include #include +#include #include +#include #include "faodel-common/Common.hh" #include "faodel-common/Debug.hh" #include "kelpie/Kelpie.hh" #include "kelpie/pools/DHTPool/DHTPool.hh" +#include "kelpie/ops/direct/OpKelpieCompute.hh" #include "kelpie/ops/direct/OpKelpieGetBounded.hh" #include "kelpie/ops/direct/OpKelpieGetUnbounded.hh" #include "kelpie/ops/direct/OpKelpieMeta.hh" @@ -31,7 +32,7 @@ namespace kelpie { DHTPool::DHTPool(const ResourceURL &pool_url) - : PoolBase(pool_url) { + : PoolBase(pool_url, PoolBehavior::DefaultRemote) { bool ok = dirman::GetDirectoryInfo(pool_url, &dir_info); @@ -39,34 +40,54 @@ DHTPool::DHTPool(const ResourceURL &pool_url) throw std::runtime_error("Pool "+TypeName()+" could not get directory info for "+pool_url.str()); } + + bool we_are_in_pool=false; + //Allocate peers and connect to each one - for(size_t i=0; i(dir_info.members[i].node, peer)); + nodes.emplace_back(pair(dir_info.members[i].node, peer)); } - //Pull out iom info in order to associate the iom with this local pool. While iom option was - //parsed in base, it didn't save the name (just the hash) - string iom_option = ""; - if (pool_url.path == "/local/iom") { - iom_option = pool_url.name; - } else { - iom_option = pool_url.GetOption("iom"); + //Change default behavior when we have an iom attached + if((iom_hash) && (pool_url.GetOption("behavior").empty())) { + behavior_flags=PoolBehavior::DefaultRemoteIOM; } - if(!iom_option.empty()) { - iom = kelpie::internal::Singleton::impl.core->FindIOM(iom_option); - if(iom==nullptr) { - throw std::runtime_error("Could not find iom '"+iom_option+"' for local pool with url: "+pool_url.str()); + + if(we_are_in_pool) { + //Pull out iom info in order to associate the iom with this local pool. While iom option was + //parsed in base, it didn't save the name (just the hash) + string iom_option; + if(pool_url.path == "/local/iom") { + iom_option = pool_url.name; + } else { + iom_option = pool_url.GetOption("iom"); } - } + if(!iom_option.empty()) { + iom = kelpie::internal::Singleton::impl.core->iom_registry.Find(iom_option); + if(iom == nullptr) { + //IOM isn't known, try registering it from the url + int rc = kelpie::internal::Singleton::impl.core->iom_registry.RegisterIomFromURL(pool_url); + if(rc==0) { + //Registered ok, retry the lookup + iom = kelpie::internal::Singleton::impl.core->iom_registry.Find(iom_option); + } + if(iom == nullptr) { + throw std::runtime_error( + "Could not find iom '" + iom_option + "' for dht pool with url: " + pool_url.GetFullURL()); + } -} -DHTPool::~DHTPool(){ - //lkv was allocated in core + } + } + } } /** @@ -76,13 +97,13 @@ DHTPool::~DHTPool(){ * @retval KELPIE_ENOENT The item wasn't found locally * @retval KELPIE_OK The request was successfully launched (failures may happen in callback) */ -rc_t DHTPool::Publish(const Key &key, fn_publish_callback_t callback){ +rc_t DHTPool::Publish(const Key &key, const fn_publish_callback_t &callback){ dbg("Publish (from lkv) bucket "+default_bucket.GetHex()+" key "+key.str()); //Retrieve the item from lkv so we can send to the destination lunasa::DataObject ldo; - rc_t rc = lkv->get(default_bucket, key, &ldo, nullptr, nullptr); + rc_t rc = lkv->get(default_bucket, key, &ldo, nullptr); if(rc==KELPIE_ENOENT) return KELPIE_ENOENT; return Publish(key, ldo, callback); @@ -98,7 +119,7 @@ rc_t DHTPool::Publish(const Key &key, fn_publish_callback_t callback){ */ rc_t DHTPool::Publish(const Key &key, const lunasa::DataObject &user_ldo, - fn_publish_callback_t callback) { + const fn_publish_callback_t &callback) { //Figure out which node in our list gets the spot uint32_t spot = findNodeIndex(key); @@ -106,24 +127,17 @@ rc_t DHTPool::Publish(const Key &key, dbg("Publish ldo to dht node "+std::to_string(spot)+" for bucket "+default_bucket.GetHex()+" key "+key.str()); - //Skip ops if local + //Skip ops if we're actually the target node in the dht if(nodes[spot].first == my_nodeid) { - kv_row_info_t row_info; - kv_col_info_t col_info; - rc_t rc = lkv->put(default_bucket, key, user_ldo, behavior_flags, &row_info, &col_info); + object_info_t info; + rc_t rc = lkv->put(default_bucket, key, user_ldo, PoolBehavior::ChangeRemoteToLocal(behavior_flags), iom, &info); - if((iom!=nullptr) && (behavior_flags & PoolBehavior::WriteToIOM)){ - iom->WriteObject(default_bucket, key, user_ldo); - } - - if(callback) - callback(rc, row_info, col_info); + if(callback) callback(rc, info); return KELPIE_OK; } - - - OpKelpiePublish *op = new OpKelpiePublish( + //Send it off to the right destination for processing + auto *op = new OpKelpiePublish( nodes[spot].first, nodes[spot].second, default_bucket, key, @@ -132,11 +146,15 @@ rc_t DHTPool::Publish(const Key &key, behavior_flags, callback ); - opbox::LaunchOp(op); - //TODO: Add lkv trigger so if anything waiting on publish + //See if we also need to write it here + if(behavior_flags & PoolBehavior::WriteToLocal) { + object_info_t info; + rc_t rc = lkv->put(default_bucket, key, user_ldo, behavior_flags, nullptr, &info); //ignore iom - this is for caching + } + return KELPIE_OK; } @@ -148,23 +166,32 @@ rc_t DHTPool::Publish(const Key &key, * @param callback Function to execute when the object arrives * @retval KELPIE_OK Request placed (or detected it's already been placed) */ -rc_t DHTPool::Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callback_t callback){ +rc_t DHTPool::Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback){ dbg("Want (size="+to_string(expected_ldo_user_bytes)+") key "+key.str()); //Check the lkv to see if it already exists. If it does, have lkv do //the callback. If it doesn't, leave the callback in place so it can //trigger when the data arrives. - rc_t rc = lkv->want(default_bucket, key, NODE_LOCALHOST, true, callback); + rc_t rc = lkv->wantLocal(default_bucket, key, true, callback); if((rc==KELPIE_OK) || (rc==KELPIE_WAITING)) return KELPIE_OK; //Already requested.. - - //Figure out which node in our list gets the spot uint32_t spot = findNodeIndex(key); //See if this item belongs here, we don't send anything if(nodes[spot].first == my_nodeid) { + + //See if we can load it from disk + if((rc==KELPIE_ENOENT) && (iom!=nullptr) && (behavior_flags & PoolBehavior::WriteToIOM)) { + lunasa::DataObject ldo; + rc = iom->ReadObject(default_bucket, key, &ldo); + if(rc==KELPIE_OK){ + //We got it. Push it into lkv in order to trigger waiting callbacks + lkv->put(default_bucket, key, ldo, behavior_flags, nullptr, nullptr); + } + } + //Item belongs on this node, but we've already marked the lkv for it return KELPIE_OK; } @@ -173,15 +200,14 @@ rc_t DHTPool::Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callb if(expected_ldo_user_bytes > 0) { //We know length. Do a Bounded Get op - OpKelpieGetBounded *op = new OpKelpieGetBounded( + auto *op = new OpKelpieGetBounded( nodes[spot].first, nodes[spot].second, default_bucket, key, expected_ldo_user_bytes, iom_hash, behavior_flags, - //TODO: Can this be restricted to a smaller capture list and still work? - [=] (bool success, Key &key, lunasa::DataObject &ldo) { + [this] (bool success, Key &key, lunasa::DataObject &ldo) { //Only push into lkv if we had a successful fetch if(success) { lkv->put(default_bucket, key, ldo, behavior_flags, nullptr, nullptr); @@ -192,7 +218,7 @@ rc_t DHTPool::Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callb } else { //Don't know length. Use an Unbounded Get op - OpKelpieGetUnbounded *op = new OpKelpieGetUnbounded( + auto *op = new OpKelpieGetUnbounded( nodes[spot].first, nodes[spot].second, default_bucket, key, @@ -215,65 +241,102 @@ rc_t DHTPool::Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callb /** * @brief Blocking request for a blob from a pool * @param key The Key label for the item - * @param expected_ldo_user_bytes The expected size of the item (if known), or zero (if unknown) - * @param returned_ldo The returned object + * @param[in] expected_ldo_user_bytes The expected size of the item (if known), or zero (if unknown) + * @param[out] returned_ldo The returned object * @retval KELPIE_OK Item was located */ rc_t DHTPool::Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo){ dbg("Need (size="+to_string(expected_ldo_user_bytes)+") key "+key.str()); - std::mutex m; - std::condition_variable cv; - std::unique_lock lk(m); - bool is_found=false; + std::promise found_promise; + std::future found_future = found_promise.get_future(); rc_t rc = Want(key, expected_ldo_user_bytes, - [&returned_ldo, &cv, &is_found] (bool success, Key key, lunasa::DataObject result_ldo, - const kv_row_info_t &ri, const kv_col_info_t &c) { + [&returned_ldo, &found_promise] (bool success, const Key &key, const lunasa::DataObject &result_ldo, + const object_info_t &info) { if(success) { *returned_ldo = result_ldo; } else { + //This should not happen in current code. Leaving here in case we implement cancels throw std::runtime_error("DHT Pool could not resolve need for "+key.str()); - //TODO: better error handling } *returned_ldo = result_ldo; - is_found=true; - cv.notify_one(); + found_promise.set_value(true); }); if(rc!=KELPIE_OK){ - KTODO("DHTPool could not issue Need"); + F_TODO("DHTPool could not issue Need"); + } + + bool is_found = found_future.get(); + + // There's currently no way for is_found to be false + if( !is_found ) { + throw std::runtime_error("DHT Pool could not resolve need for "+key.str()); + } + + return KELPIE_OK; +} + +/** + * @brief Perform a computation on a remote object and return a new object (NonBlocking Version) + * @param[in] key The Key that references an object (foo,bar) or multiple objects in the same row (foo,bar*) + * @param[in] function_name The text name of the registered function to execute on the remote node + * @param[in] function_args A string of arguments that are passed to the remote node + * @param[in] callback A callback to execute when the result is returned + * @retval KELPIE_OK Operation dispatched properly and completed correctly + * @retval KELPIE_EINVAL The function name was not known at the object + */ +rc_t DHTPool::Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback) { + dbg("Compute function "+function_name+" for key "+key.str()); + F_ASSERT(!key.IsRowWildcard(), "Requested a key with a row wildcard. Only column wildcards are supported"); + + //Figure out which node in our list gets the spot + uint32_t spot = findNodeIndex(key); + rc_t rc; + + //See if this item belongs here, we don't send anything + if(nodes[spot].first == my_nodeid) { + lunasa::DataObject ldo; + rc = lkv->doCompute(function_name, function_args, default_bucket, key, &ldo); + callback(rc, key, ldo); + return KELPIE_OK; //Always launch ok.. real result is handed back in callback } - while(!is_found) - cv.wait(lk); + //Node lives elsewhere - start an op to dispatch it + auto *op = new OpKelpieCompute( + nodes[spot].first, nodes[spot].second, + default_bucket, + key, + iom_hash, + behavior_flags, + function_name, + function_args, + callback); + opbox::LaunchOp(op); return KELPIE_OK; } /** * @brief Get info about a particular key/blob. Does not wait for blob to be generated - * @param key The Key label for the blob - * @param col_info Basic statistics about the item + * @param[in] key The Key label for the blob + * @param[out] info Basic statistics about the item * @retval KELPIE_OK The item was located and info obtained * @retval KELPIE_WAITING Local node is already waiting on item, so no request made * @retval KELPIE_ENOENT Item should be on this node, but isn't and hasn't been requested */ -rc_t DHTPool::Info(const Key &key, kv_col_info_t *col_info){ +rc_t DHTPool::Info(const Key &key, object_info_t *info){ dbg("Info for key "+key.str()); //Check local first. Only issue request if we aren't already waiting - rc_t rc = lkv->getColInfo(default_bucket, key, col_info); + rc_t rc = lkv->getInfo(default_bucket, key, info); if((rc==KELPIE_OK) || (rc==KELPIE_WAITING)) return rc; - //Wasn't available locally. Prepare to do some messaging - std::mutex m; - std::condition_variable cv; - std::unique_lock lk(m); bool got_result=false; uint32_t spot = findNodeIndex(key); @@ -283,36 +346,37 @@ rc_t DHTPool::Info(const Key &key, kv_col_info_t *col_info){ //Item belongs on this node, but it's not in memory. Check on disk //if this pool has an iom associated with it. if(iom!=nullptr){ - rc = iom->GetInfo(default_bucket, key, col_info); + rc = iom->GetInfo(default_bucket, key, info); } return rc; } - + + //Wasn't available locally. Prepare to do some messaging + std::promise found_promise; + std::future found_future = found_promise.get_future(); + //Hosting node is somewhere else. Launch a new op to get the info - auto *op = new OpKelpieMeta(DirectFlags::CMD_GET_COLINFO, + opbox::LaunchOp( new OpKelpieMeta(DirectFlags::CMD_GET_COLINFO, nodes[spot].first, nodes[spot].second, default_bucket, key, iom_hash, - [col_info, &got_result, &cv, &rc](rc_t result, kv_row_info_t &ri, kv_col_info_t &ci) { + [info, &got_result, &found_promise, &rc](rc_t result, object_info_t &new_info) { rc=result; - if(col_info != nullptr){ - if(result==0){ - *col_info = ci; + if(info != nullptr) { + if(result == 0) { + *info = new_info; } else { - col_info->Wipe(); + info->Wipe(); } } - got_result=true; - cv.notify_one(); - }); - opbox::LaunchOp(op); + found_promise.set_value(true); + })); - while(!got_result) - cv.wait(lk); + bool is_found = found_future.get(); //Switch a local memory label to a remote memory label - if((col_info!=nullptr) && (rc==0)){ - col_info->ChangeAvailabilityFromLocalToRemote(); + if((info!=nullptr) && (rc==KELPIE_OK)){ + info->ChangeAvailabilityFromLocalToRemote(); } return rc; @@ -320,40 +384,117 @@ rc_t DHTPool::Info(const Key &key, kv_col_info_t *col_info){ /** * @brief Get info about a particular row. Currently only looks locally - * @todo Update to work on remotes nodes - * @param key The Key label for the blob - * @param row_info Basic statistics about the row + * @param[in] key The Key label for the blob + * @param[out] info Basic statistics about the row * @retval KELPIE_OK if info known */ -rc_t DHTPool::RowInfo(const Key &key, kv_row_info_t *row_info) { +rc_t DHTPool::RowInfo(const Key &key, object_info_t *info) { dbg("RowInfo for key "+key.str()); + //Check local first. Only issue request if we aren't already waiting + rc_t rc = lkv->getInfo(default_bucket, key, info); + if((rc==KELPIE_OK) || (rc==KELPIE_WAITING)) return rc; + + + bool got_result=false; uint32_t spot = findNodeIndex(key); + + //See if this node is supposed to be hosting the data. We've already checked, so IOM is only other option if(nodes[spot].first == my_nodeid) { - return lkv->getRowInfo(default_bucket, key, row_info); + //todo: iom check: we could check the iom here + return rc; + } + + std::promise found_promise; + std::future found_future = found_promise.get_future(); + + //Hosting node is somewhere else. Launch a new op to get the info + opbox::LaunchOp( new OpKelpieMeta(DirectFlags::CMD_GET_ROWINFO, + nodes[spot].first, nodes[spot].second, + default_bucket, key, + iom_hash, + [info, &got_result, &found_promise, &rc](rc_t result, object_info_t &new_info) { + rc=result; + if(info != nullptr) { + if(result == 0) { + *info = new_info; + } else { + info->Wipe(); + } + } + found_promise.set_value(true); + })); + + bool is_found = found_future.get(); + + //Switch a local memory label to a remote memory label + if((info!=nullptr) && (rc==KELPIE_OK)){ + info->ChangeAvailabilityFromLocalToRemote(); } - KTODO("RowInfo does not currently support remote operations"); - return KELPIE_TODO; + + return rc; } + /** * @brief Signify that an item is no longer needed - * @note This currently only works on the local node - * @todo Update state machine to trigger a drop of a value on remote side * @param key The Key label for the blob + * @param callback Optional function to call if a response is required * @retval KELPIE_OK if info known */ -rc_t DHTPool::Drop(const Key &key) { +rc_t DHTPool::Drop(const Key &key, fn_drop_callback_t callback) { dbg("Drop key "+key.str()); - uint32_t spot = findNodeIndex(key); - if(nodes[spot].first == my_nodeid) { - return lkv->drop(default_bucket, key); + //Check here first if caching + rc_t rc_local = KELPIE_ENOENT; + bool needs_local_search=(behavior_flags & (PoolBehavior::WriteToLocal | PoolBehavior::ReadToLocal)); + bool needs_external_search=false; + + vector> tmp_nodes; + + if( !key.IsRowWildcard() ) { + uint32_t spot = findNodeIndex(key); + if(nodes[spot].first == my_nodeid) { + //Special case: we're the server. + needs_local_search=true; + needs_external_search=false; + } else { + //Single external node + needs_external_search=true; + tmp_nodes.push_back(nodes[spot]); + } + } else { + //Row Wildcard. Build a list of external nodes to check + for(auto &node_peer : nodes) { + if (node_peer.first == my_nodeid) needs_local_search=true; + else { + needs_external_search=true; + tmp_nodes.push_back(node_peer); + } + } + } + + dbg("DHT-DROP: needs_local "+to_string(needs_local_search)+" needs_external "+to_string(needs_external_search)+ " num_targets: "+to_string(tmp_nodes.size())); + + //Actually do the local version + if(needs_local_search) { + rc_local = lkv->drop(default_bucket, key); + } + + dbg("DHT-DROP: Cleared local, found was "+to_string(rc_local==KELPIE_OK)+", now working on remote"); + + //See if we need to launch to external nodes + if(needs_external_search) { + opbox::LaunchOp(new OpKelpieDrop(std::move(tmp_nodes), default_bucket, key, (rc_local==KELPIE_OK), callback)); + + } else if (callback) { + callback((rc_local==KELPIE_OK), key); } - KTODO("Drop does not currently support remote operations"); - return KELPIE_TODO; + + return KELPIE_OK; + } @@ -368,19 +509,48 @@ rc_t DHTPool::List(const kelpie::Key &search_key, ObjectCapacities *object_capac dbg("List key "+search_key.str()); - //No local search - always goes out to nodes + bool needs_local_search=false; + bool needs_external_search=false; + vector> tmp_nodes; + + if( !search_key.IsRowWildcard() ) { + uint32_t spot = findNodeIndex(search_key); + if(nodes[spot].first == my_nodeid) { + //Special case: we're the server. + needs_local_search=true; + needs_external_search=false; + } else { + //Single external node + needs_external_search=true; + tmp_nodes.push_back(nodes[spot]); + } + } else { + //Row Wildcard. Build a list of external nodes to check + for(auto &node_peer : nodes) { + if (node_peer.first == my_nodeid) needs_local_search=true; + else { + needs_external_search=true; + tmp_nodes.push_back(node_peer); + } + } + } + - //Wasn't available locally. Prepare to do some messaging - std::mutex m; - std::condition_variable cv; - std::unique_lock lk(m); - int num_targets_left=nodes.size(); - opbox::LaunchOp(new OpKelpieList(nodes, default_bucket, search_key, object_capacities, - &cv, &num_targets_left)); + if(needs_local_search) { + lkv->list(default_bucket, search_key, iom, object_capacities); + } + + if(needs_external_search) { + //Start an op to go fetch results from one or more nodes + std::promise found_promise; + std::future found_future = found_promise.get_future(); + + iom_hash_t iom_hash = GetIomHash(); + opbox::LaunchOp(new OpKelpieList(tmp_nodes, default_bucket, search_key, iom_hash, + object_capacities, &found_promise)); - while(num_targets_left) { - cv.wait(lk); + bool is_done = found_future.get(); //block } return KELPIE_OK; @@ -396,7 +566,7 @@ rc_t DHTPool::List(const kelpie::Key &search_key, ObjectCapacities *object_capac */ int DHTPool::FindTargetNode(const Key &key, faodel::nodeid_t *node_id, net::peer_ptr_t *peer_ptr){ - if(nodes.size()<1) return 0; + if(nodes.empty()) return 0; uint32_t spot = findNodeIndex(key); if(node_id) *node_id = nodes[spot].first; if(peer_ptr) *peer_ptr = nodes[spot].second; diff --git a/src/kelpie/pools/DHTPool/DHTPool.hh b/src/kelpie/pools/DHTPool/DHTPool.hh index 6394249..c12421f 100644 --- a/src/kelpie/pools/DHTPool/DHTPool.hh +++ b/src/kelpie/pools/DHTPool/DHTPool.hh @@ -1,12 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_DHTPOOL_HH #define KELPIE_DHTPOOL_HH -#include -#include +#include +#include #include #include #include @@ -49,28 +49,29 @@ class DHTPool : public PoolBase { public: - DHTPool(const faodel::ResourceURL &pool_url); - ~DHTPool() override; + explicit DHTPool(const faodel::ResourceURL &pool_url); + ~DHTPool() override = default; //PoolBase functions - rc_t Publish(const Key &key, fn_publish_callback_t callback) override; //Lookup a local k/v and publish it - rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, fn_publish_callback_t callback) override; - - rc_t Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callback_t callback) override; //Notify when available + rc_t Publish(const Key &key, const fn_publish_callback_t &callback) override; //Lookup a local k/v and publish it + rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, const fn_publish_callback_t &callback) override; + rc_t Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback) override; //Notify when available rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo) override; //Block until get - rc_t Info(const Key &key, kv_col_info_t *col_info) override; - rc_t RowInfo(const Key &key, kv_row_info_t *row_info) override; - rc_t Drop(const Key &key) override; - rc_t List(const Key &search_key, ObjectCapacities *object_capacities=nullptr) override; + rc_t Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback) override; //Async compute + + rc_t Info(const Key &key, object_info_t *col_info) override; + rc_t RowInfo(const Key &key, object_info_t *row_info) override; + rc_t Drop(const Key &key, fn_drop_callback_t callback) override; + rc_t List(const Key &search_key, ObjectCapacities *object_capacities) override; - int FindTargetNode(const Key &key, faodel::nodeid_t *node_id=nullptr, net::peer_ptr_t *peer_ptr=nullptr) override; + int FindTargetNode(const Key &key, faodel::nodeid_t *node_id, net::peer_ptr_t *peer_ptr) override; std::string TypeName() const override { return "dht"; } //InfoInterface function - void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; + void sstr(std::stringstream &ss, int depth, int indent) const override; protected: std::vector> nodes; diff --git a/src/kelpie/pools/LocalPool/LocalPool.cpp b/src/kelpie/pools/LocalPool/LocalPool.cpp index 384d844..a1b1e95 100644 --- a/src/kelpie/pools/LocalPool/LocalPool.cpp +++ b/src/kelpie/pools/LocalPool/LocalPool.cpp @@ -1,10 +1,10 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -#include -#include -#include +#include +#include +#include #include #include #include @@ -23,23 +23,28 @@ using namespace faodel; namespace kelpie { LocalPool::LocalPool(const ResourceURL &pool_url) - : PoolBase(pool_url) { + : PoolBase(pool_url, PoolBehavior::DefaultLocal) { //We're a new local pool that has a label the pool registry hasn't seen //before. See if we're associated with an iom. //Pull out iom info in order to associate the iom with this local pool. While iom option was - //parsed in base, it didn't save the name (just the hash) - string iom_option = ""; + //parsed in base, it didn't save the name (just the hash). If the user didn't specify + //a behavior, we need to switch from DefaultLocal to DefaultLocalIOM. + string iom_option; if (pool_url.path == "/local/iom") { iom_option = pool_url.name; } else { iom_option = pool_url.GetOption("iom"); } if(!iom_option.empty()) { - iom = kelpie::internal::Singleton::impl.core->FindIOM(iom_option); + iom = kelpie::internal::Singleton::impl.core->iom_registry.Find(iom_option); if(iom==nullptr) { - throw std::runtime_error("Could not find iom '"+iom_option+"' for local pool with url: "+pool_url.str()); + throw std::runtime_error("Could not find iom '"+iom_option+"' for local pool with url: "+pool_url.GetFullURL()); + } + iom_hash = iom->NameHash(); + if(pool_url.GetOption("behavior").empty()) { + behavior_flags = PoolBehavior::DefaultLocalIOM; } } @@ -48,9 +53,6 @@ LocalPool::LocalPool(const ResourceURL &pool_url) } -LocalPool::~LocalPool(){ - //No extra allocates here. lkv was allocated in core -} /** * @brief Pull an item from the local store (optionally writing to an IOM) @@ -59,15 +61,14 @@ LocalPool::~LocalPool(){ * @retval KELPIE_ENOENT The item wasn't found locally * @retval KELPIE_OK The request was successfully launched (failures may happen in callback) */ -rc_t LocalPool::Publish(const Key &key, fn_publish_callback_t callback){ - kv_col_info_t col_info; - kv_row_info_t row_info; +rc_t LocalPool::Publish(const Key &key, const fn_publish_callback_t &callback){ + object_info_t info; lunasa::DataObject ldo; dbg("Publish (from lkv) bucket "+default_bucket.GetHex()+" key "+key.str()); //Get the ldo - rc_t rc = lkv->get(default_bucket, key, &ldo, &row_info, &col_info ); + rc_t rc = lkv->get(default_bucket, key, &ldo, &info ); if(rc!=KELPIE_OK) return rc; //Not found.. publish cannot proceed //Write out if we have an iom @@ -75,8 +76,8 @@ rc_t LocalPool::Publish(const Key &key, fn_publish_callback_t callback){ iom->WriteObject(default_bucket, key, ldo); } - //Found. No instructions on where to publish, so trigger the callback as successfull - if(callback) callback(KELPIE_OK, row_info, col_info); + //Found. No instructions on where to publish, so trigger the callback as successful + if(callback) callback(KELPIE_OK, info); return KELPIE_OK; } @@ -89,24 +90,18 @@ rc_t LocalPool::Publish(const Key &key, fn_publish_callback_t callback){ */ rc_t LocalPool::Publish(const Key &key, const lunasa::DataObject &user_ldo, - fn_publish_callback_t callback) { + const fn_publish_callback_t &callback) { - kv_col_info_t col_info; - kv_row_info_t row_info; + object_info_t info; rc_t rc = KELPIE_OK; dbg("Publish ldo for bucket "+default_bucket.GetHex()+" key "+key.str()); //Default to putting in the lkv - rc = lkv->put(default_bucket, key, user_ldo, behavior_flags, &row_info, &col_info); + rc = lkv->put(default_bucket, key, user_ldo, behavior_flags, iom, &info); - //Write out if we have an iom - if((iom!=nullptr) && (behavior_flags & PoolBehavior::WriteToIOM)){ - iom->WriteObject(default_bucket, key, user_ldo); - } - //Launch is always successful. Only send the rc to a callback - if(callback) callback(rc, row_info, col_info); + if(callback) callback(rc, info); return KELPIE_OK; //TODO: or should this be rc? } @@ -118,11 +113,11 @@ rc_t LocalPool::Publish(const Key &key, * @retval KELPIE_OK Request placed (or detected it's already been placed) * @note If an IOM is associated with this localpool, it will be queried if the local cache misses */ -rc_t LocalPool::Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callback_t callback) { +rc_t LocalPool::Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback) { dbg("Want (size="+to_string(expected_ldo_user_bytes)+") key "+key.str()); - int rc = lkv->want(default_bucket, key, NODE_LOCALHOST, false, callback); + int rc = lkv->wantLocal(default_bucket, key, false, callback); //See if we can load it from disk if((rc==KELPIE_ENOENT) && (iom!=nullptr)) { @@ -147,8 +142,10 @@ rc_t LocalPool::Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_cal */ rc_t LocalPool::Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo){ - kassert(returned_ldo!=nullptr, "User must provide an LDO"); - kassert(returned_ldo->internal_use_only.GetRefCount()==0, "Refuse to overwrite an LDO"); + dbg("Key is "+key.str()+" return ldo count is "+to_string(returned_ldo->internal_use_only.GetRefCount())+" expected size "+to_string(expected_ldo_user_bytes)); + F_ASSERT(returned_ldo != nullptr, "User must provide an LDO"); + F_ASSERT(returned_ldo->internal_use_only.GetRefCount() == 0, "User gave an preallocated LDO to Need. Refusing to overwrite it"); + dbg("Need (size="+to_string(expected_ldo_user_bytes)+") key "+key.str()); @@ -158,7 +155,7 @@ rc_t LocalPool::Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::Dat bool first_time=true; bool keep_going=true; do { - rc = lkv->get(default_bucket, key, returned_ldo, nullptr, nullptr); + rc = lkv->get(default_bucket, key, returned_ldo, nullptr); if(rc==KELPIE_ENOENT) { //First time: See if we can load from disk @@ -185,24 +182,34 @@ rc_t LocalPool::Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::Dat return rc; } +rc_t LocalPool::Compute(const Key &key, const std::string &function_name, const std::string &function_args, + const fn_compute_callback_t &callback) { + dbg("Key is "+key.str()+" function is "+function_name); + lunasa::DataObject ext_ldo; + rc_t rc = lkv->doCompute(function_name, function_args, default_bucket, key, &ext_ldo); + callback(rc, key, ext_ldo); + + return KELPIE_OK; +} + /** * @brief Get info about a particular key/blob. Does not wait for blob to be generated * @param key The Key label for the blob - * @param col_info Basic statistics about the item + * @param info Basic statistics about the item * @retval KELPIE_OK The item was located and info obtained * @retval KELPIE_WAITING Local node is already waiting on item, so no request made * @note This currently does NOT query the IOM for info, but will in the future */ -rc_t LocalPool::Info(const Key &key, kv_col_info_t *col_info) { +rc_t LocalPool::Info(const Key &key, object_info_t *info) { dbg("Info for key "+key.str()); - rc_t rc = lkv->getColInfo(default_bucket, key, col_info); + rc_t rc = lkv->getInfo(default_bucket, key, info); //Go out to disk if not here if((rc==KELPIE_ENOENT) && (iom!=nullptr)) { - rc = iom->GetInfo(default_bucket, key, col_info); + rc = iom->GetInfo(default_bucket, key, info); //TODO: update colinfo with InDisk info? } @@ -212,35 +219,37 @@ rc_t LocalPool::Info(const Key &key, kv_col_info_t *col_info) { /** * @brief Get info about a particular row. Currenly only looks locally * @param key The Key label for the blob - * @param row_info Basic statistics about the row + * @param info Basic statistics about the row * @retval KELPIE_OK if info known * @note This currently does NOT query the IOM for info, but will in the future */ -rc_t LocalPool::RowInfo(const Key &key, kv_row_info_t *row_info) { +rc_t LocalPool::RowInfo(const Key &key, object_info_t *info) { dbg("RowInfo for key "+key.str()); //TODO: add disk check for row info - return lkv->getRowInfo(default_bucket, key, row_info); + return lkv->getInfo(default_bucket, key, info); } /** * @brief Signify that an item is no longer needed - * @param key The Key label for the blob + * @param[in] key The Key label for the blob + * @param[in] callback Optional callback for passing back results * @retval KELPIE_OK if info known * @note This does not affect the IOM */ -rc_t LocalPool::Drop(const Key &key){ +rc_t LocalPool::Drop(const Key &key, fn_drop_callback_t callback){ dbg("Drop key "+key.str()); //Don't delete from disk - return lkv->drop(default_bucket, key); + rc_t rc = lkv->drop(default_bucket, key); + if(callback) callback(rc==KELPIE_OK, key); + return rc; } /** * @brief Perform a search for keys that match a specific pattern - * @param[in] bucket The bucket id we should limit the search to - * @param[in] key_prefix The key to search for. Row and/or Key may end in '*' for prefix matching + * @param[in] search_key The key to search for. Row and/or Key may end in '*' for prefix matching * @param[out] object_capacities Info about the objects that match this key search * @retval KELPIE_OK Found matches * @retval KELPIE_ENOENT Did not find matches @@ -248,7 +257,7 @@ rc_t LocalPool::Drop(const Key &key){ rc_t LocalPool::List(const kelpie::Key &search_key, ObjectCapacities *object_capacities) { dbg("List key "+search_key.str()); - return lkv->list(default_bucket, search_key, object_capacities); + return lkv->list(default_bucket, search_key, iom, object_capacities); } /** @@ -260,7 +269,7 @@ rc_t LocalPool::List(const kelpie::Key &search_key, ObjectCapacities *object_cap */ int LocalPool::FindTargetNode(const Key &key, faodel::nodeid_t *node_id, net::peer_ptr_t *peer_ptr){ if(node_id) *node_id=faodel::NODE_LOCALHOST; - if(peer_ptr) KTODO("peer_ptr must be null here"); //TODO: need to have a null peer_ptr + if(peer_ptr) F_TODO("peer_ptr must be null here"); //TODO: need to have a null peer_ptr return 0; } diff --git a/src/kelpie/pools/LocalPool/LocalPool.hh b/src/kelpie/pools/LocalPool/LocalPool.hh index e07136e..4857995 100644 --- a/src/kelpie/pools/LocalPool/LocalPool.hh +++ b/src/kelpie/pools/LocalPool/LocalPool.hh @@ -1,12 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_LOCALPOOL_HH #define KELPIE_LOCALPOOL_HH -#include -#include +#include +#include #include #include #include @@ -38,28 +38,29 @@ class LocalPool : public PoolBase { public: - LocalPool(const faodel::ResourceURL &pool_url); - ~LocalPool() override; + explicit LocalPool(const faodel::ResourceURL &pool_url); + ~LocalPool() override = default; //PoolBase functions - rc_t Publish(const Key &key, fn_publish_callback_t callback=nullptr) override; - rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, fn_publish_callback_t callback) override; - - rc_t Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callback_t callback) override; //Notify when available + rc_t Publish(const Key &key, const fn_publish_callback_t &callback) override; + rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, const fn_publish_callback_t &callback) override; + rc_t Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback) override; //Notify when available rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo) override; //Block until get - rc_t Info(const Key &key, kv_col_info_t *col_info) override; - rc_t RowInfo(const Key &key, kv_row_info_t *row_info) override; - rc_t Drop(const Key &key) override; - rc_t List(const Key &search_key, ObjectCapacities *object_capacities=nullptr) override; + rc_t Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback) override; //Async compute + + rc_t Info(const Key &key, object_info_t *col_info) override; + rc_t RowInfo(const Key &key, object_info_t *row_info) override; + rc_t Drop(const Key &key, fn_drop_callback_t callback) override; + rc_t List(const Key &search_key, ObjectCapacities *object_capacities) override; int FindTargetNode(const Key &key, faodel::nodeid_t *node_id, net::peer_ptr_t *peer_ptr) override; std::string TypeName() const override { return "local"; } //InfoInterface function - void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; + void sstr(std::stringstream &ss, int depth, int indent) const override; }; //LocalPool diff --git a/src/kelpie/pools/NullPool/NullPool.cpp b/src/kelpie/pools/NullPool/NullPool.cpp new file mode 100644 index 0000000..ed071a4 --- /dev/null +++ b/src/kelpie/pools/NullPool/NullPool.cpp @@ -0,0 +1,234 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include +#include +#include +#include +#include + +#include "faodel-common/Common.hh" + +#include "kelpie/Kelpie.hh" +#include "kelpie/core/Singleton.hh" +#include "kelpie/pools/NullPool/NullPool.hh" + + +using namespace std; +using namespace faodel; + +namespace kelpie { + +NullPool::NullPool(const ResourceURL &pool_url) + : PoolBase(pool_url, PoolBehavior::DefaultLocal) { + + //No real options, just drop everything + + //Set debug info + SetSubcomponentName("-Null-"+pool_url.bucket.GetHex()); + +} + + +/** + * @brief Pull an item from the local store (optionally writing to an IOM) + * @param key A global Key for referencing the blob + * @param callback Function to call when this operation completes (whether success or failure) + * @retval KELPIE_ENOENT The item wasn't found locally + * @retval KELPIE_OK The request was successfully launched (failures may happen in callback) + */ +rc_t NullPool::Publish(const Key &key, const fn_publish_callback_t &callback){ + object_info_t info; + lunasa::DataObject ldo; + + dbg("Publish (from lkv) bucket "+default_bucket.GetHex()+" key "+key.str()); + + info.Wipe(); + + //Found. No instructions on where to publish, so trigger the callback as successful + if(callback) callback(KELPIE_OK, info); + return KELPIE_OK; +} + +/** + * @brief Publish an object to the local pool (optionally writing to an IOM) + * @param key A global Key for referencing the blob + * @param user_ldo The data object to publish + * @param callback Function to call when this operation completes (whether success or failure) + * @retval KELPIE_OK The request was successfully launched (failures may happen in callback) + */ +rc_t NullPool::Publish(const Key &key, + const lunasa::DataObject &user_ldo, + const fn_publish_callback_t &callback) { + + object_info_t info; + rc_t rc = KELPIE_OK; + + dbg("Publish ldo for bucket "+default_bucket.GetHex()+" key "+key.str()); + + info.Wipe(); + + //Launch is always successful. Only send the rc to a callback + if(callback) callback(rc, info); + return KELPIE_OK; //TODO: or should this be rc? +} + +/** + * @brief Request a callback be executed when an item becomes available locally + * @param key The Key for the desired blob + * @param expected_ldo_user_bytes Not used in this pool + * @param callback Function to execute when the object arrives + * @retval KELPIE_OK Request placed (or detected it's already been placed) + * @note If an IOM is associated with this localpool, it will be queried if the local cache misses + */ +rc_t NullPool::Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback) { + + dbg("Want (size="+to_string(expected_ldo_user_bytes)+") key "+key.str()); + + lunasa::DataObject ldo; + if(expected_ldo_user_bytes>0) + ldo = lunasa::DataObject(0,expected_ldo_user_bytes,lunasa::DataObject::AllocatorType::lazy); + + object_info_t info; + info.Wipe(); + callback(true, key, ldo, info); + + return KELPIE_OK; + +} + +/** + * @brief Blocking request for a blob from the local cache + * @param key The Key label for the item + * @param expected_ldo_user_bytes Ignored + * @param returned_ldo The returned object + * @retval KELPIE_OK Item was located + * @note If an IOM is associated with this localpool, it will be queried if the local cache misses + */ +rc_t NullPool::Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo){ + + dbg("Key is "+key.str()+" return ldo count is "+to_string(returned_ldo->internal_use_only.GetRefCount())+" expected size "+to_string(expected_ldo_user_bytes)); + F_ASSERT(returned_ldo != nullptr, "User must provide an LDO"); + F_ASSERT(returned_ldo->internal_use_only.GetRefCount() == 0, "User gave an preallocated LDO to Need. Refusing to overwrite it"); + + dbg("Need (size="+to_string(expected_ldo_user_bytes)+") key "+key.str()); + + lunasa::DataObject ldo; + if(expected_ldo_user_bytes>0) + ldo = lunasa::DataObject(0,expected_ldo_user_bytes,lunasa::DataObject::AllocatorType::lazy); + + *returned_ldo = ldo; + + return KELPIE_OK; +} + +/** + * @brief Perform a computation on a remote object and return a new object (NonBlocking Version) + * @param[in] key The Key that references an object (foo,bar) or multiple objects in the same row (foo,bar*) + * @param[in] function_name The text name of the registered function to execute on the remote node + * @param[in] function_args A string of arguments that are passed to the remote node + * @param[in] callback A callback to execute when the result is returned (answer is always KELPIE_ENOENT and empty LDO) + * @retval KELPIE_OK Operation dispatched properly and completed correctly + */ +rc_t NullPool::Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback) { + dbg("Key is "+key.str()+" function is "+function_name); + lunasa::DataObject ldo; + callback(KELPIE_ENOENT,key, ldo); + return KELPIE_OK; +} + +/** + * @brief Get info about a particular key/blob. Does not wait for blob to be generated + * @param key The Key label for the blob + * @param info Basic statistics about the item + * @retval KELPIE_OK The item was located and info obtained + * @retval KELPIE_WAITING Local node is already waiting on item, so no request made + * @note This currently does NOT query the IOM for info, but will in the future + */ +rc_t NullPool::Info(const Key &key, object_info_t *info) { + + dbg("Info for key "+key.str()); + + info->Wipe(); + return KELPIE_OK; +} + +/** + * @brief Get info about a particular row. Currenly only looks locally + * @param key The Key label for the blob + * @param info Basic statistics about the row + * @retval KELPIE_OK if info known + * @note This currently does NOT query the IOM for info, but will in the future + */ +rc_t NullPool::RowInfo(const Key &key, object_info_t *info) { + + dbg("RowInfo for key "+key.str()); + info->Wipe(); + return KELPIE_OK; +} + +/** + * @brief Signify that an item is no longer needed + * @param[in] key The Key label for the blob + * @param[in] callback Optional callback for passing back results + * @retval KELPIE_OK if info known + * @note This does not affect the IOM + */ +rc_t NullPool::Drop(const Key &key, fn_drop_callback_t callback){ + dbg("Drop key "+key.str()); + return KELPIE_OK; +} + +/** + * @brief Perform a search for keys that match a specific pattern + * @param[in] search_key The key to search for. Row and/or Key may end in '*' for prefix matching + * @param[out] object_capacities Info about the objects that match this key search + * @retval KELPIE_OK Found matches + * @retval KELPIE_ENOENT Did not find matches + */ +rc_t NullPool::List(const kelpie::Key &search_key, ObjectCapacities *object_capacities) { + dbg("List key "+search_key.str()); + + return KELPIE_OK; +} + +/** + * @brief Use the key's row info to determine which node is responsible for the data + * @param key The Key label for the blob (only ROW portion used) + * @param node_id Always NODE_LOCALHOST + * @param peer_ptr Ignored + * @return KELPIE_OK + */ +int NullPool::FindTargetNode(const Key &key, faodel::nodeid_t *node_id, net::peer_ptr_t *peer_ptr){ + return 0; +} + + +/** + * @brief Write debug info into a stream stream + * @param[in] ss String Stream to append info into + * @param[in] depth How many more steps in hierarchy to go down (default=0) + * @param[in] indent How many spaces to put in front of this line (default=0) + */ +void NullPool::sstr(stringstream &ss, int depth, int indent) const { + + ss << string(indent,' ') + "NullPool " + << " Iom: "<< ((iom==nullptr) ? "None" : iom->Name()) + << endl; + lkv->sstr(ss, depth-1,indent+1); + //TODO: internals +} + +/** + * @brief Pool constructor function for creating a new NullPool via a URL + * @param pool_url The URL for the pool + * @return New NullPool + */ +shared_ptr NullPoolCreate(const ResourceURL &pool_url) { + return make_shared(pool_url); +} + +} // namespace kelpie diff --git a/src/kelpie/pools/NullPool/NullPool.hh b/src/kelpie/pools/NullPool/NullPool.hh new file mode 100644 index 0000000..b341016 --- /dev/null +++ b/src/kelpie/pools/NullPool/NullPool.hh @@ -0,0 +1,73 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef KELPIE_NULLPOOL_HH +#define KELPIE_NULLPOOL_HH + +#include +#include +#include +#include +#include +#include + +#include "faodel-common/Common.hh" + +#include "kelpie/Key.hh" +#include "kelpie/localkv/LocalKV.hh" +#include "kelpie/common/Types.hh" +#include "kelpie/pools/Pool.hh" +#include "kelpie/pools/PoolBase.hh" + +#include "kelpie/ioms/IomBase.hh" + +#include "lunasa/DataObject.hh" + +namespace kelpie { + +/** + * @brief A handle for interacting only with the node's local key/blob store + * + * A NullPool is a trivial pool handle that always drops an item or returns null data + */ +class NullPool : public PoolBase { + +public: + + explicit NullPool(const faodel::ResourceURL &pool_url); + ~NullPool() override = default; + + //PoolBase functions + rc_t Publish(const Key &key, const fn_publish_callback_t &callback) override; + rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, const fn_publish_callback_t &callback) override; + + rc_t Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback) override; //Notify when available + rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo) override; //Block until get + + rc_t Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback) override; //Async compute + + rc_t Info(const Key &key, object_info_t *col_info) override; + rc_t RowInfo(const Key &key, object_info_t *row_info) override; + rc_t Drop(const Key &key, fn_drop_callback_t callback) override; + rc_t List(const Key &search_key, ObjectCapacities *object_capacities) override; + + int FindTargetNode(const Key &key, faodel::nodeid_t *node_id, net::peer_ptr_t *peer_ptr) override; + + std::string TypeName() const override { return "null"; } + + //InfoInterface function + void sstr(std::stringstream &ss, int depth, int indent) const override; + + +}; //NullPool + + +//For use by connect +std::shared_ptr NullPoolCreate(const faodel::ResourceURL &pool_url); + + +} // namespace kelpie + +#endif // KELPIE_NULLPOOL_HH + diff --git a/src/kelpie/pools/Pool.cpp b/src/kelpie/pools/Pool.cpp index 3ca6f14..7e544b4 100644 --- a/src/kelpie/pools/Pool.cpp +++ b/src/kelpie/pools/Pool.cpp @@ -1,8 +1,8 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -#include //memcpy +#include //memcpy #include "kelpie/core/Singleton.hh" @@ -51,8 +51,6 @@ Pool::Pool(Pool &&source) { source.impl = kelpie::internal::Singleton::impl.unconfigured_pool; } -Pool::~Pool() { -} /** * @brief Shallow copies a pool. Both source and destination will point to the same implementation @@ -81,52 +79,93 @@ Pool& Pool::operator=(Pool &&source){ * @retval KELPIE_ENOENT The item wasn't found locally * @retval KELPIE_OK The request was successfully launched (failures may happen in callback) */ -rc_t Pool::Publish(const Key &key, fn_publish_callback_t callback){ +rc_t Pool::Publish(const Key &key, const fn_publish_callback_t &callback) { + if(key.IsWildcard()) { + throw std::runtime_error("Publish using a wildcard is not supported. Key: "+key.str()); + } return impl->Publish(key, callback); } /** - * @brief Asynchronously publish an object to the pool + * @brief Pull an item from the local store and asynchronously publish to the pool * @param[in] key A global Key for referencing the blob - * @param[in] user_ldo The data object to publish - * @param[in] callback Function to call when this operation completes (whether success or failure) + * @param[in] collector A result collector to capture info about the operation + * @retval KELPIE_ENOENT The item wasn't found locally * @retval KELPIE_OK The request was successfully launched (failures may happen in callback) */ -rc_t Pool::Publish(const Key &key, const lunasa::DataObject &user_ldo, fn_publish_callback_t callback){ - return impl->Publish(key, user_ldo, callback); + +rc_t Pool::Publish(const Key &key, ResultCollector &collector) { + return Publish(key, + [&collector] (kelpie::rc_t result, object_info_t new_info) { + collector.fn_publish_callback(result, new_info); + }); +} + + +/** + * @brief Blocking publish of an object to the pool + * @param[in] key A global Key for referencing the blob + * @param[in] user_ldo The data object to publish + * @retval KELPIE_OK The request was successfully executed + */ +rc_t Pool::Publish(const Key &key, const lunasa::DataObject &user_ldo) { + object_info_t info; + return Publish(key, user_ldo, &info); } /** * @brief Blocking publish of an object, returning row/column info for destination * @param[in] key A global Key for referencing the blob * @param[in] user_ldo The data object to publish - * @param[out] row_info Optional row info from the destination - * @param[out] col_info Optional column info from the destination + * @param[out] info Return optional info from the destination * @retval KELPIE_OK The request was successfully launched (failures may happen in callback) * @retval Other Additional error information from the destination may be returned */ -rc_t Pool::Publish(const Key &key, const lunasa::DataObject &user_ldo, kv_row_info_t *row_info, kv_col_info_t *col_info) { +rc_t Pool::Publish(const Key &key, const lunasa::DataObject &user_ldo, object_info_t *info) { rc_t rc1, rc2; atomic num_left(1); rc1 = Publish(key, user_ldo, - [&num_left, &rc2, row_info, col_info]( kelpie::rc_t result, kelpie::kv_row_info_t &ri, kelpie::kv_col_info_t &ci) { + [&num_left, &rc2, info]( kelpie::rc_t result, object_info_t new_info) { rc2=result; - if(row_info) *row_info = ri; - if(col_info) *col_info = ci; + if(info) *info = new_info; num_left--; }); if(rc1!=KELPIE_OK) return rc1; - while(num_left) { sched_yield(); } + while(num_left) { std::this_thread::yield(); } return rc2; } +/** + * @brief Asynchronously publish an object to the pool + * @param[in] key A global Key for referencing the blob + * @param[in] user_ldo The data object to publish + * @param[in] callback Function to call when this operation completes (whether success or failure) + * @retval KELPIE_OK The request was successfully launched (failures may happen in callback) + */ +rc_t Pool::Publish(const Key &key, const lunasa::DataObject &user_ldo, const fn_publish_callback_t &callback) { + if(key.IsWildcard()) { + throw std::runtime_error("Publish using a wildcard is not supported. Key: "+key.str()); + } + return impl->Publish(key, user_ldo, callback); +} + +rc_t Pool::Publish(const Key &key, const lunasa::DataObject &user_ldo, ResultCollector &collector) { + return Publish(key, user_ldo, + [&collector] (kelpie::rc_t result, object_info_t new_info) { + collector.fn_publish_callback(result, new_info); + }); +} + /** * @brief Asynchronously request an object of unknown size, call a callback when available * @param[in] key The Key for the desired blob * @param[in] callback Function to execute when the object arrives * @retval KELPIE_OK Request placed(or detected it's already been placed) */ -rc_t Pool::Want(const Key &key, fn_want_callback_t callback){ +rc_t Pool::Want(const Key &key, const fn_want_callback_t &callback) { + if(key.IsWildcard()) { + throw std::runtime_error("Want using a wildcard is not supported. Key: "+key.str()); + } return impl->Want(key, 0, callback); } @@ -138,10 +177,26 @@ rc_t Pool::Want(const Key &key, fn_want_callback_t callback){ * @retval KELPIE_OK Request placed(or detected it's already been placed) * @note Data is truncated if object is larger than expected. Check column info in callback */ -rc_t Pool::Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callback_t callback){ +rc_t Pool::Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback) { + if(key.IsWildcard()) { + throw std::runtime_error("Want using a wildcard is not supported. Key: "+key.str()); + } return impl->Want(key, expected_ldo_user_bytes, callback); } +rc_t Pool::Want(const Key &key, ResultCollector &collector) { + return Want(key, 0, collector); +} +rc_t Pool::Want(const Key &key, size_t expected_ldo_user_bytes, ResultCollector &collector) { + if(key.IsWildcard()) { + throw std::runtime_error("Want using a wildcard is not supported. Key: "+key.str()); + } + return Want(key, expected_ldo_user_bytes, + [&collector] (bool success, Key key, lunasa::DataObject user_ldo, const object_info_t &info) { + collector.fn_want_callback(success, key, user_ldo, info); + }); +} + /** * @brief Blocking request for an object of a known size * @param[in] key he Key for the desired blob @@ -150,44 +205,125 @@ rc_t Pool::Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callbac * @retval KELPIE_OK Request placed(or detected it's already been placed) * @note Data is truncated if object is larger than expected. Check column info in callback */ -rc_t Pool::Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo){ +rc_t Pool::Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo) { + F_ASSERT(returned_ldo != nullptr, "Need didn't have a valid returned_ldo pointer"); + F_ASSERT(returned_ldo->GetUserCapacity() == 0, "Need request attempted to use a preallocated ldo. Use deepcopy instead. Key:"+key.str()); return impl->Need(key, expected_ldo_user_bytes, returned_ldo); } +/** + * @brief Perform a computation on a remote object and return a new object (NonBlocking Version) + * @param[in] key The Key that references an object (foo,bar) or multiple objects in the same row (foo,bar*) + * @param[in] function_name The text name of the registered function to execute on the remote node + * @param[in] function_args A string of arguments that are passed to the remote node + * @param[in] callback A callback to execute when the result is returned + * @retval KELPIE_OK Operation dispatched properly and completed correctly + * @retval KELPIE_EINVAL The function name was not known at the object + */ +rc_t Pool::Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback) { + return impl->Compute(key, function_name, function_args, callback); +} + +/** + * @brief Perform a compute operation on a remote pool and return and object + * @param[in] key The Key that references an object (foo,bar) or multiple objects in the same row (foo,bar*) + * @param[in] function_name The text name of the registered function to execute on the remote node + * @param[in] function_args A string of arguments that are passed to the remote node + * @param[in] collector A result collector to capture info about the operation + * @retval KELPIE_OK The request was successfully launched (failures may happen in callback) + * @retval KELPIE_EINVAL The function name was not known at the object + */ + +rc_t Pool::Compute(const Key &key, const std::string &function_name, const std::string &function_args, ResultCollector &collector) { + return Compute(key, function_name, function_args, + [&collector] (kelpie::rc_t result, kelpie::Key key, lunasa::DataObject user_ldo) { + collector.fn_compute_callback(result, key, user_ldo); + }); +} + + + +/** + * @brief Perform a computation on a remote object and return a new object (Blocking Version) + * @param[in] key The Key that references an object (foo,bar) or multiple objects in the same row (foo,bar*) + * @param[in] function_name The text name of the registered function to execute on the remote node + * @param[in] function_args A string of arguments that are passed to the remote node + * @param[out] returned_ldo The resulting LDO from the operation (this is NOT cached in the system) + * @retval KELPIE_OK Operation dispatched properly and completed correctly + * @retval KELPIE_EINVAL The function name was not known at the object + */ +rc_t Pool::Compute(const Key &key, const std::string &function_name, const std::string &function_args, lunasa::DataObject *returned_ldo) { + rc_t rc1, rc2; + atomic num_left(1); + rc1 = Compute(key, function_name, function_args, + [&num_left, &rc2, &returned_ldo]( kelpie::rc_t result, Key key, lunasa::DataObject user_ldo) { + rc2 = result; + if(returned_ldo) *returned_ldo = user_ldo; + num_left--; + }); + if(rc1!=KELPIE_OK) return rc1; + while(num_left) { std::this_thread::yield(); } + return rc2; +} + /** * @brief Blocking request for info about a particular object. Does not wait for object to be generated - * @param[in] key The Key label for the blob - * @param[out] col_info Basic statistics about the item + * @param[in] key The Key label for the blob (must be exact key: wildcards are not supported) + * @param[out] info Return basic statistics about the item * @retval KELPIE_OK The item was located and info obtained * @retval KELPIE_WAITING Local node is already waiting on item, so no request made * @retval KELPIE_ENOENT Item should be on this node, but isn't and hasn't been requested */ -rc_t Pool::Info(const Key &key, kv_col_info_t *col_info) { - return impl->Info(key, col_info); +rc_t Pool::Info(const Key &key, object_info_t *info) { + if(key.IsRowWildcard() || key.IsColWildcard()) { + F_WARN("Kelpie Info called with wildcard key. Wildcards are not currently supported."); + } + return impl->Info(key, info); } /** - * @brief Get info about a particular row. Currently only looks locally + * @brief Get info about a particular row. * @todo Update to work on remotes nodes - * @param[in] key The Key label for the blob - * @param[out] row_info Basic statistics about the row + * @param[in] key The Key label for the blob (Wildcards only supported for Columns, not rows) + * @param[out] info Basic statistics about the row * @retval KELPIE_OK if info known */ -rc_t Pool::RowInfo(const Key &key, kv_row_info_t *row_info){ - return impl->RowInfo(key, row_info); +rc_t Pool::RowInfo(const Key &key, object_info_t *info) { + if(key.IsRowWildcard()) { + F_WARN("Kelpie RowInfo called with a row wildcard key. Wildcards are only supported for columns."); + } + return impl->RowInfo(key, info); } /** * @brief Signify that this object is no longer needed and should be released by pool - * @note This currently only works on local pools - * @todo Update pools to perform remote drops * @param[in] key The Key label for the blob + * @param[in] callback An optional callback to execute when the operation completes * @retval KELPIE_OK if info known */ -rc_t Pool::Drop(const Key &key) { - return impl->Drop(key); +rc_t Pool::Drop(const Key &key, fn_drop_callback_t callback) { + return impl->Drop(key, callback); } +/** + * @brief Do a blocking drop of a key (which can have wildcards) + * @param[in] key The item to search for (can contain row/column wildcards) + * @retval KELPIE_OK Removed one or more items + * @retval KELPIE_ENOENT Did not locate any items to drop + */ +rc_t Pool::BlockingDrop(const Key &key) { + + std::promise success_promise; + std::future success_future = success_promise.get_future(); + + Drop(key, [&success_promise](bool inner_success, const kelpie::Key &inner_key) { + success_promise.set_value(inner_success); + }); + + bool success = success_future.get(); + + return (success) ? KELPIE_OK : KELPIE_ENOENT; +} /** * @brief Perform a search for keys in this pool that match a specific pattern * @param[in] search_key The key to search for. Row and/or Key may end in '*' for prefix matching @@ -210,6 +346,31 @@ int Pool::FindTargetNode(const Key &key, faodel::nodeid_t *node_id, net::peer_pt return impl->FindTargetNode(key, node_id, peer_ptr); } +/** + * @brief Determine whether this pool initialized successfully + * @param error_message Optional information about why this pool failed to initialize + * @retval TRUE This pool is functional + * @retval FALSE This pool did not initialize ok + */ +bool Pool::Valid(std::string *error_message) { + if((impl==nullptr) || (impl->TypeName()=="unconfigured")) { + if(error_message) { + *error_message = dynamic_cast(impl.get())->error_message; + } + return false; + } + if(error_message) *error_message = ""; + return true; +} +void Pool::ValidOrDie() { + string err; + if(!Valid(&err)) { + cout<<"Pool.ValidOrDie() shutdown. Reason: "<GetURL(); } */ faodel::DirectoryInfo Pool::GetDirectoryInfo() { return impl->GetDirectoryInfo(); } +/** + * @brief Get information about how read/write actions for this pool take place + * @return Behavior A flag specifying how write/reads for local/remote/ioms take place + */ +kelpie::pool_behavior_t Pool::GetBehavior() { return impl->GetBehavior(); } + /** * @brief Get the IOM driver hash associated with this pool (if exists) * @retval 0 An IOM driver is not associated with this pool diff --git a/src/kelpie/pools/Pool.hh b/src/kelpie/pools/Pool.hh index 66fcb00..dcce893 100644 --- a/src/kelpie/pools/Pool.hh +++ b/src/kelpie/pools/Pool.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_POOL_HH #define KELPIE_POOL_HH @@ -16,6 +16,7 @@ #include "opbox/net/net.hh" #include "dirman/DirMan.hh" #include "kelpie/Kelpie.hh" +#include "kelpie/pools/ResultCollector.hh" namespace kelpie { @@ -36,35 +37,54 @@ public: Pool(faodel::internal_use_only_t iuo, std::shared_ptr base); //Creation Pool(const Pool &source); //Shallow copy Pool(Pool &&source); //Move - ~Pool() override; + ~Pool() override = default; Pool& operator=(const Pool&); //For Shallow copy Pool& operator=(Pool&&); //For Move //Core functions - rc_t Publish(const Key &key, fn_publish_callback_t callback=nullptr); //Lookup locally and publish externally - rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, fn_publish_callback_t callback=nullptr); //Async publish - rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, kv_row_info_t *row_info, kv_col_info_t *col_info); //Blocking publish + rc_t Publish(const Key &key, const fn_publish_callback_t &callback); //Lookup locally and publish externally + rc_t Publish(const Key &key, ResultCollector &collector); //Async local publish that notifies a ResultCollector - rc_t Want(const Key &key, fn_want_callback_t callback=nullptr); //Notify when available - rc_t Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callback_t callback=nullptr); //Notify when available + rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo); //Blocking publish + rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, object_info_t *info); //Blocking publish - rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo); //Block until get - rc_t Need(const Key &key, lunasa::DataObject *returned_ldo) { return Need(key, 0, returned_ldo); } //Block until get + rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, const fn_publish_callback_t &callback); //Async publish + rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, ResultCollector &collector); //Async publish that notifies a ResultCollector - rc_t Info(const Key &key, kv_col_info_t *col_info); - rc_t RowInfo(const Key &key, kv_row_info_t *row_info); + rc_t Want(const Key &key, const fn_want_callback_t &callback={}); //Notify when available + rc_t Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback={}); //Notify when available + rc_t Want(const Key &key, ResultCollector &collector); //Notify collector when available + rc_t Want(const Key &key, size_t expected_ldo_user_bytes, ResultCollector &collector); //Notify collector when available + + rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo); //Block until get + rc_t Need(const Key &key, lunasa::DataObject *returned_ldo) { return Need(key, 0, returned_ldo); } //Block until get + + rc_t Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback); //Async compute + rc_t Compute(const Key &key, const std::string &function_name, const std::string &function_args, ResultCollector &collector); //Async compute w/ ResultCollector + rc_t Compute(const Key &key, const std::string &function_name, const std::string &function_args, lunasa::DataObject *returned_ldo=nullptr); //Blocking compute + + rc_t Info(const Key &key, object_info_t *info); + rc_t RowInfo(const Key &key, object_info_t *info); + + rc_t Drop(const Key &key, fn_drop_callback_t callback=nullptr); + rc_t BlockingDrop(const Key &key); - rc_t Drop(const Key &key); rc_t List(const Key &search_key, ObjectCapacities *object_capacities=nullptr); //Locate where this pool would store the data int FindTargetNode(const Key &key, faodel::nodeid_t *node_id=nullptr, net::peer_ptr_t *peer_ptr=nullptr); + //Determine if this pool was created correctly + bool Valid(std::string *error_message=nullptr); + void ValidOrDie(); + + //Get info on Default settings faodel::bucket_t GetBucket(); faodel::ResourceURL GetURL(); faodel::DirectoryInfo GetDirectoryInfo(); + kelpie::pool_behavior_t GetBehavior(); iom_hash_t GetIomHash(); int GetRefCount(); @@ -80,7 +100,7 @@ public: //InfoInterface function - void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; + void sstr(std::stringstream &ss, int depth, int indent) const override; private: std::shared_ptr impl; diff --git a/src/kelpie/pools/PoolBase.cpp b/src/kelpie/pools/PoolBase.cpp index 9f1dc24..e1489cf 100644 --- a/src/kelpie/pools/PoolBase.cpp +++ b/src/kelpie/pools/PoolBase.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -15,10 +15,10 @@ using namespace faodel; namespace kelpie { -PoolBase::PoolBase(const faodel::ResourceURL &pool_url) +PoolBase::PoolBase(const faodel::ResourceURL &pool_url, pool_behavior_t behavior_flags_) : LoggingInterface("kelpie.pool"), pool_url(pool_url), lkv(nullptr), iom(nullptr), iom_hash(0), - behavior_flags(PoolBehavior::DefaultBaseClass) { + behavior_flags(behavior_flags_) { //Unconfigure pool is empty if(pool_url.Type() == "unconfigured") return; @@ -26,11 +26,18 @@ PoolBase::PoolBase(const faodel::ResourceURL &pool_url) kelpie::internal::getLKV(&lkv); default_bucket = pool_url.bucket; + string new_bucket = pool_url.GetOption("bucket"); + if(!new_bucket.empty()) { + default_bucket = bucket_t(new_bucket); + } + + my_nodeid = net::GetMyID(); auto iom_name = pool_url.GetOption("iom"); - if(iom_name!="") { - + if(!iom_name.empty()) { + //User has provided us with an IOM to attach to this pool. Pull settings from the url. + //IMPORTANT: /local/iom is not usually handled here. Look at LocalPool's ctor if(iom_name.compare(0,2, "0x")==0) { unsigned long val; stringstream ss; diff --git a/src/kelpie/pools/PoolBase.hh b/src/kelpie/pools/PoolBase.hh index 805a0e0..b033fc4 100644 --- a/src/kelpie/pools/PoolBase.hh +++ b/src/kelpie/pools/PoolBase.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_POOLBASE_HH #define KELPIE_POOLBASE_HH @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include "faodel-common/Common.hh" #include "faodel-common/LoggingInterface.hh" @@ -43,21 +43,22 @@ class PoolBase public faodel::LoggingInterface { public: - PoolBase(faodel::ResourceURL const &pool_url); + explicit PoolBase(faodel::ResourceURL const &pool_url, pool_behavior_t behavior_flags=PoolBehavior::DefaultBaseClass); - ~PoolBase() override { } //Fill in if creates anything + ~PoolBase() override = default; //Fill in if creates anything - virtual rc_t Publish(const Key &key, fn_publish_callback_t callback) = 0; - virtual rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, fn_publish_callback_t callback) = 0; + virtual rc_t Publish(const Key &key, const fn_publish_callback_t &callback) = 0; + virtual rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, const fn_publish_callback_t &callback) = 0; - virtual rc_t Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callback_t callback) = 0; //Notify when available + virtual rc_t Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback) = 0; //Notify when available + virtual rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo) = 0; //Block until returned - virtual rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo) = 0; //Block until get + virtual rc_t Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback) = 0; //Async compute - virtual rc_t Info(const Key &key, kv_col_info_t *col_info) = 0; - virtual rc_t RowInfo(const Key &key, kv_row_info_t *row_info) = 0; + virtual rc_t Info(const Key &key, object_info_t *col_info) = 0; + virtual rc_t RowInfo(const Key &key, object_info_t *row_info) = 0; - virtual rc_t Drop(const Key &key) = 0; + virtual rc_t Drop(const Key &key, fn_drop_callback_t callback) = 0; virtual rc_t List(const Key &search_key, ObjectCapacities *object_capacities=nullptr) = 0; diff --git a/src/kelpie/pools/PoolRegistry.cpp b/src/kelpie/pools/PoolRegistry.cpp index b1fcf78..6e8b9c3 100644 --- a/src/kelpie/pools/PoolRegistry.cpp +++ b/src/kelpie/pools/PoolRegistry.cpp @@ -1,15 +1,13 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include "kelpie/pools/Pool.hh" #include "kelpie/pools/PoolRegistry.hh" #include "kelpie/pools/PoolBase.hh" - - -//TODO: This should have locks around pool creation +#include "kelpie/pools/UnconfiguredPool/UnconfiguredPool.hh" //Built in for errors using namespace std; using namespace faodel; @@ -19,7 +17,9 @@ namespace internal { PoolRegistry::PoolRegistry() - : default_bucket(faodel::BUCKET_UNSPECIFIED), + : LoggingInterface("kelpie.pool_registry"), + started(false), + default_bucket(faodel::BUCKET_UNSPECIFIED), default_pool_logging_level(0) { mutex = faodel::GenerateMutex(); } @@ -30,6 +30,7 @@ PoolRegistry::~PoolRegistry() { void PoolRegistry::init(const faodel::Configuration &config) { + ConfigureLogging(config); //Set registry's logging level config.GetDefaultSecurityBucket(&default_bucket); default_pool_logging_level = LoggingInterface::GetLoggingLevelFromConfiguration(config, "kelpie.pool"); @@ -39,106 +40,132 @@ void PoolRegistry::init(const faodel::Configuration &config) { } void PoolRegistry::start(){ - //TODO: Add tag stuff to separate from pre-start/post-start registrations? + started=true; //Prevent anyone from registering new pool types after this point } void PoolRegistry::finish() { + dbg("Finishing"); whookie::Server::deregisterHook("/kelpie/pool_registry"); pool_create_fns.clear(); mutex->Lock(); //TODO: Remove this or find a way to replace impls with UnconfiguredPools for( auto &url_rptr : known_pools){ + dbg("Removing pool "+url_rptr.second->GetFullName()); if(url_rptr.second.use_count()>1){ cout<<"Warning: shutting down with user-space pools still open\n"; } } known_pools.clear(); mutex->Unlock(); + started=false; } -void PoolRegistry::RegisterPoolConstructor(std::string pool_name, fn_PoolCreate_t function_pointer){ - map::iterator ii = pool_create_fns.find(pool_name); - kassert(ii==pool_create_fns.end(), "Attempting to overwrite existing pool constructor for "+pool_name); +void PoolRegistry::RegisterPoolConstructor(const std::string &pool_name, fn_PoolCreate_t function_pointer){ + + F_ASSERT(!started, "Attempted to register pool constructor after bootstrap Start()."); + + dbg("Registering pool ctor "+pool_name); + + auto name_fn = pool_create_fns.find(pool_name); + F_ASSERT(name_fn == pool_create_fns.end(), "Attempting to overwrite existing pool constructor for "+pool_name); pool_create_fns[pool_name]=function_pointer; } Pool PoolRegistry::Connect(const ResourceURL &pool_url){ + dbg("Connect to "+pool_url.GetFullURL()); + //Make this url more uniform. Lookup kelpie's default bucket if not available ResourceURL src_url = pool_url; //may need to modify if(src_url.bucket == BUCKET_UNSPECIFIED){ src_url.bucket = default_bucket; } - //cout <<"Input url is '"<Lock(); - auto rname_rptr = known_pools.find(pool_key); + mutex->ReaderLock(); + auto rname_rptr = known_pools.find(pool_tag); if(rname_rptr != known_pools.end() ){ - //cout <<"Found existing pool. Using "<first; + dbg("Found existing pool. Using "+rname_rptr->first); Pool p(faodel::internal_use_only, rname_rptr->second); mutex->Unlock(); return p; } + mutex->Unlock(); //Unlock here because pool_create_fns only changes during registry + + + dbg("Existing pool instance not found. Creating."); //Allocate a new PoolBase, since this is a new Pool //Look up the allocation function in our table of creators auto name_ctor = pool_create_fns.find(src_url.Type()); if(name_ctor==pool_create_fns.end()) { - mutex->Unlock(); - throw std::runtime_error("Pool registry could not find ctor for pool "+src_url.GetURL()); + return createErrorPool("Pool registry could not find constructor for pool "+src_url.GetURL()); } - - - //cout<<"PR: ITEM MISISNG. Before adding "< rbase_ptr = name_ctor->second(src_url); rbase_ptr->SetLoggingLevel(default_pool_logging_level); - known_pools[pool_key] = rbase_ptr; - - - //cout<<"PR: ITEM MISSING. Afer adding:\n"; - //for(auto &src_ptr : known_pools) { - // cout << src_ptr.first <WriterLock(); + rname_rptr = known_pools.find(pool_tag); + if(rname_rptr != known_pools.end() ) { + //Someone beat us. Get rid of the created pool and use the existing one + dbg("Found existing pool. Using "+rname_rptr->first); + rbase_ptr = rname_rptr->second; + + } else { + //Remember the ptr we just created + known_pools[pool_tag] = rbase_ptr; + } + + //Plug this into a pool, unlock and return Pool p(faodel::internal_use_only, rbase_ptr); mutex->Unlock(); - return p; } +vector PoolRegistry::GetRegisteredPoolTypes() const { + vector types; + for(auto &name_fn : pool_create_fns) { + types.push_back(name_fn.first); + } + return types; +} +/** + * @brief Rather than throw an exception, create a pool with an error message in it for handling by user + * @param error_message What broke this pool + * @return + */ +Pool PoolRegistry::createErrorPool(const std::string &error_message) { + return Pool(internal_use_only, UnconfiguredPoolCreate(error_message)); +} void PoolRegistry::HandleWhookieStatus(const std::map &args, std::stringstream &results) { faodel::ReplyStream rs(args, "Kelpie Pool Registry", &results); diff --git a/src/kelpie/pools/PoolRegistry.hh b/src/kelpie/pools/PoolRegistry.hh index 19a10b4..ada2b08 100644 --- a/src/kelpie/pools/PoolRegistry.hh +++ b/src/kelpie/pools/PoolRegistry.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_POOLREGISTRY_HH #define KELPIE_POOLREGISTRY_HH @@ -10,6 +10,7 @@ #include #include "faodel-common/FaodelTypes.hh" +#include "faodel-common/LoggingInterface.hh" #include "faodel-common/ResourceURL.hh" #include "kelpie/common/Types.hh" @@ -27,7 +28,8 @@ namespace internal { * maintains a list of pools in the system that this application instance * has connected to. New connections can be established via Connect(). */ -class PoolRegistry { +class PoolRegistry : + public faodel::LoggingInterface { public: PoolRegistry(); @@ -38,17 +40,22 @@ class PoolRegistry { void finish(); Pool Connect(const faodel::ResourceURL &pool_url); - void RegisterPoolConstructor(std::string pool_name, fn_PoolCreate_t function_pointer); + void RegisterPoolConstructor(const std::string &pool_name, fn_PoolCreate_t function_pointer); + + std::vector GetRegisteredPoolTypes() const; private: + bool started; faodel::MutexWrapper *mutex; faodel::bucket_t default_bucket; int default_pool_logging_level; std::map pool_create_fns; // ctors for pools std::map> known_pools; - std::string makeKnownPoolKey(const faodel::ResourceURL &url) const { - return url.bucket.GetHex()+url.path+"/"+url.name+"&"+url.GetSortedOptions(); + Pool createErrorPool(const std::string &error_message); + + std::string makeKnownPoolTag(const faodel::ResourceURL &url) const { + return url.GetURL(false, false, true, true); } void HandleWhookieStatus(const std::map &args, std::stringstream &results); }; diff --git a/src/kelpie/pools/RFTPool/RFTPool.cpp b/src/kelpie/pools/RFTPool/RFTPool.cpp index 1689a1b..7c5fdd7 100644 --- a/src/kelpie/pools/RFTPool/RFTPool.cpp +++ b/src/kelpie/pools/RFTPool/RFTPool.cpp @@ -1,11 +1,11 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. - -#include -#include -#include -#include +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include +#include #include #include #include @@ -32,16 +32,13 @@ RFTPool::RFTPool(const ResourceURL &pool_url) //TODO: the DHTPool ctor makes connections to ALL resources. We only need the one string intended_rank = pool_url.GetOption("rank"); - if(intended_rank == "") { + if(intended_rank.empty()) { MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); } else { mpi_rank = std::stoi(intended_rank, nullptr, 0); } } -RFTPool::~RFTPool(){ - //lkv was allocated in core -} /** * @brief Determine the index of the RFT list that owns the data diff --git a/src/kelpie/pools/RFTPool/RFTPool.hh b/src/kelpie/pools/RFTPool/RFTPool.hh index a399c0b..85bf965 100644 --- a/src/kelpie/pools/RFTPool/RFTPool.hh +++ b/src/kelpie/pools/RFTPool/RFTPool.hh @@ -1,12 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_RFTPOOL_HH #define KELPIE_RFTPOOL_HH -#include -#include +#include +#include #include #include #include @@ -32,13 +32,13 @@ class RFTPool : public DHTPool { public: - RFTPool(const faodel::ResourceURL &pool_url); - ~RFTPool() override; + explicit RFTPool(const faodel::ResourceURL &pool_url); + ~RFTPool() override = default; std::string TypeName() const override { return "rft"; } //InfoInterface function - void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; + void sstr(std::stringstream &ss, int depth, int indent) const override; protected: uint32_t findNodeIndex(const Key &key) override; diff --git a/src/kelpie/pools/ResultCollector.cpp b/src/kelpie/pools/ResultCollector.cpp new file mode 100644 index 0000000..a5bc849 --- /dev/null +++ b/src/kelpie/pools/ResultCollector.cpp @@ -0,0 +1,61 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include + +#include "kelpie/pools/ResultCollector.hh" + +using namespace std; +using namespace faodel; + +namespace kelpie { + +/** + * @brief ResultCollector constructor + * @param[in] num_requests The number of requests that will write to this collector + */ +ResultCollector::ResultCollector(int num_requests) + : expected_items(num_requests), items_left(num_requests) { + + results = new Result[num_requests]; +} +ResultCollector::~ResultCollector() { + delete [] results; +} + +void ResultCollector::fn_publish_callback(kelpie::rc_t result, object_info_t &info ) { + int spot = expected_items - (items_left--); + results[spot].request_type = RequestType::Publish; + results[spot].rc = result; + results[spot].info = info; + if(spot==expected_items-1) items_left--; //Signal we've recorded last item +} +void ResultCollector::fn_want_callback(bool success, kelpie::Key key, lunasa::DataObject user_ldo, const object_info_t &info) { + int spot = expected_items - (items_left--); + results[spot].request_type = RequestType::Want; + results[spot].rc = (success) ? KELPIE_OK : KELPIE_EINVAL; + results[spot].info = info; + results[spot].key = key; + results[spot].ldo = user_ldo; + if(spot==expected_items-1) items_left--; //Signal we've recorded last item +} + +void ResultCollector::fn_compute_callback(kelpie::rc_t result, Key key, lunasa::DataObject user_ldo) { + int spot = expected_items - (items_left--); + results[spot].request_type = RequestType::Compute; + results[spot].rc = result; + results[spot].info = {}; + results[spot].key = key; + results[spot].ldo = user_ldo; + if(spot==expected_items-1) items_left--; //Signal we've recorded last item +} + +void ResultCollector::Sync() { + while(items_left>=0) { std::this_thread::yield(); } +} + + + +} // namespace kelpie diff --git a/src/kelpie/pools/ResultCollector.hh b/src/kelpie/pools/ResultCollector.hh new file mode 100644 index 0000000..a14894c --- /dev/null +++ b/src/kelpie/pools/ResultCollector.hh @@ -0,0 +1,64 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_RESULTCOLLECTOR_HH +#define FAODEL_RESULTCOLLECTOR_HH + +#include + +#include "kelpie/common/Types.hh" + +namespace kelpie { + +/** + * @brief A class for gathering the results of multiple asynchronous operations + * + * A ResultCollector is used when you want to launch a known-number of + * asynchronous pool operations and you want to block until all operations + * complete. Create a ResultCollector, pass it to all the asynchronous + * operations you perform with a pool, and then call Sync() to block until + * all ops complete. Each op will insert any returned information in + * a vector of results that users can query after the operation completes. + */ +class ResultCollector { +public: + ResultCollector(int num_requests); + ~ResultCollector(); + + void fn_publish_callback(kelpie::rc_t result, kelpie::object_info_t &info ); + void fn_want_callback(bool success, kelpie::Key, lunasa::DataObject user_ldo, const object_info_t &info); + void fn_compute_callback(kelpie::rc_t result, Key key, lunasa::DataObject user_ldo); + + void Sync(); + + //Define what type of operation this operation was + enum class RequestType : char { + Publish=1, + Want=2, + Compute=3 + }; + + //Structure for passing back any result information from the request + struct Result { + RequestType request_type; //!< Type of operation (publish, want, compute) + kelpie::rc_t rc; //!< The return code sent back from remote node + object_info_t info; //!< Information the remote node returned about the object + kelpie::Key key; //!< The key that was used in this request + lunasa::DataObject ldo; //!< An object returned by the remote node + }; + + Result *results; //!< A collection of results for the user to examine + + +private: + int expected_items; + std::atomic items_left; + +}; + + + +} // namespace kelpie + +#endif //FAODEL_RESULTCOLLECTOR_HH diff --git a/src/kelpie/pools/TracePool/TracePool.cpp b/src/kelpie/pools/TracePool/TracePool.cpp new file mode 100644 index 0000000..31ee9f6 --- /dev/null +++ b/src/kelpie/pools/TracePool/TracePool.cpp @@ -0,0 +1,181 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include +#include + +#include "faodelConfig.h" +#ifdef Faodel_ENABLE_MPI_SUPPORT +#include +#endif + + +#include "faodel-common/Common.hh" + +#include "kelpie/Kelpie.hh" +#include "kelpie/core/Singleton.hh" +#include "kelpie/pools/TracePool/TracePool.hh" + +using namespace std; +using namespace faodel; + +namespace kelpie { + +TracePool::TracePool(const ResourceURL &pool_url) + : PoolBase(pool_url, PoolBehavior::DefaultLocal), use_relative_time(true) { + + //Set the time vals + t_last = t_start = std::chrono::high_resolution_clock::now(); + + string dashed_pool_name=pool_url.Dashify(); + string extra_bucket = pool_url.GetOption("bucket"); + if(!extra_bucket.empty()) + dashed_pool_name += "_"+extra_bucket; + + string next_pool_name = pool_url.GetOption("next_pool", "null:"); + string fname = pool_url.GetOption("file", "trace"+dashed_pool_name); + string rank = pool_url.GetOption("rank"); + + #ifdef Faodel_ENABLE_MPI_SUPPORT + if(rank.empty()) { + int mpi_rank=0; + int is_initialized=0; + MPI_Initialized(&is_initialized); + if(is_initialized) { + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + } + rank = std::to_string(mpi_rank); + } + #endif + if(!rank.empty()) { + fname = fname+"."+rank; + rank_flag = " -r "+rank +" "; //Convert to a proper flag + } + f.open(fname, ios::out); + if(!f.is_open()) { + throw std::runtime_error("Could not open trace file "+fname); + } + + //Dump what pool this is + f << "set pool "<(t_stamp - ((use_relative_time) ? t_last : t_start)).count(); + + if(use_relative_time) { + f << "delayfor " << t_us << "us " << rank_flag << ";"; + } else { + f << "delayuntil " << t_us << "us " << rank_flag << ";"; + } + + f << cmd << rank_flag << s << endl; + t_last = t_stamp; + f_mutex.unlock(); +} + + +void TracePool::sstr(stringstream &ss, int depth, int indent) const { + + ss << string(indent,' ') + "TracePool " + << " Iom: "<< ((iom==nullptr) ? "None" : iom->Name()) + << endl; +} + + + +/** + * @brief Pool constructor function for creating a new TracePool via a URL + * @param pool_url The URL for the pool + * @return New TracePool + */ +shared_ptr TracePoolCreate(const ResourceURL &pool_url) { + return make_shared(pool_url); +} + +} // namespace kelpie diff --git a/src/kelpie/pools/TracePool/TracePool.hh b/src/kelpie/pools/TracePool/TracePool.hh new file mode 100644 index 0000000..38e6504 --- /dev/null +++ b/src/kelpie/pools/TracePool/TracePool.hh @@ -0,0 +1,91 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_TRACEPOOL_HH +#define FAODEL_TRACEPOOL_HH + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "faodel-common/Common.hh" + +#include "kelpie/Key.hh" +#include "kelpie/localkv/LocalKV.hh" +#include "kelpie/common/Types.hh" +#include "kelpie/pools/Pool.hh" +#include "kelpie/pools/PoolBase.hh" +#include "kelpie/pools/NullPool/NullPool.hh" + +#include "lunasa/DataObject.hh" + + + +namespace kelpie { + +/** + * @brief A handle for interacting only with the node's local key/blob store + * + * A TracePool is a trivial pool that only captures information about when different operations were requested + */ + +class TracePool : public PoolBase { + +public: + + explicit TracePool(const faodel::ResourceURL &pool_url); + ~TracePool() override = default; + + //PoolBase functions + rc_t Publish(const Key &key, const fn_publish_callback_t &callback) override; + rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, const fn_publish_callback_t &callback) override; + + rc_t Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback) override; //Notify when available + rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo) override; //Block until get + + rc_t Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback) override; //Async compute + + rc_t Info(const Key &key, object_info_t *col_info) override; + rc_t RowInfo(const Key &key, object_info_t *row_info) override; + rc_t Drop(const Key &key, fn_drop_callback_t callback) override; + rc_t List(const Key &search_key, ObjectCapacities *object_capacities) override; + + int FindTargetNode(const Key &key, faodel::nodeid_t *node_id, net::peer_ptr_t *peer_ptr) override; + + std::string TypeName() const override { return "trace"; } + + //InfoInterface function + void sstr(std::stringstream &ss, int depth, int indent) const override; + +private: + Pool next_pool; + + std::chrono::time_point t_start; //!< Time when this TracePool was created + std::chrono::time_point t_last; //!< Time of last operation + + + std::mutex f_mutex; + std::ofstream f; + std::string rank_flag; + bool use_relative_time; + void appendTrace(const std::string &cmd, const std::string &s); + + +}; + +//For use by connect +std::shared_ptr TracePoolCreate(const faodel::ResourceURL &pool_url); + + + +} // namespace kelpie + +#endif //FAODEL_TRACEPOOL_HH diff --git a/src/kelpie/pools/UnconfiguredPool/UnconfiguredPool.cpp b/src/kelpie/pools/UnconfiguredPool/UnconfiguredPool.cpp index 9e13c27..a08658d 100644 --- a/src/kelpie/pools/UnconfiguredPool/UnconfiguredPool.cpp +++ b/src/kelpie/pools/UnconfiguredPool/UnconfiguredPool.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -13,29 +13,37 @@ using namespace std; namespace kelpie { UnconfiguredPool::UnconfiguredPool() - : PoolBase(faodel::ResourceURL("unconfigured:/")) { + : PoolBase(faodel::ResourceURL("unconfigured:/")), + error_message("Pool accessed before initialization") { } -UnconfiguredPool::~UnconfiguredPool(){ +UnconfiguredPool::UnconfiguredPool(const std::string &error_message) + : PoolBase(faodel::ResourceURL("unconfigured:/")), + error_message(error_message) { } +UnconfiguredPool::~UnconfiguredPool()= default; -rc_t UnconfiguredPool::Publish(const Key &key, fn_publish_callback_t callback) { Panic("Publish"); return KELPIE_OK; } +rc_t UnconfiguredPool::Publish(const Key &key, const fn_publish_callback_t &callback) { Panic("Publish"); return KELPIE_OK; } rc_t UnconfiguredPool::Publish(const Key &key, const lunasa::DataObject &user_ldo, - fn_publish_callback_t callback) { Panic("Publish"); return KELPIE_OK; } + const fn_publish_callback_t &callback) { Panic("Publish"); return KELPIE_OK; } rc_t UnconfiguredPool::Want(const Key &key, size_t expected_ldo_user_bytes, - fn_want_callback_t callback) { Panic("Want"); return KELPIE_OK; } + const fn_want_callback_t &callback) { Panic("Want"); return KELPIE_OK; } rc_t UnconfiguredPool::Need(const Key &key, size_t expected_ldo_user_bytes, - lunasa::DataObject *user_ldo) { Panic("Need"); return KELPIE_OK; } -rc_t UnconfiguredPool::Info(const Key &key, kv_col_info_t *col_info) { Panic("Info"); return KELPIE_OK; } -rc_t UnconfiguredPool::RowInfo(const Key &key, kv_row_info_t *row_info) { Panic("RowInfo"); return KELPIE_OK; } -rc_t UnconfiguredPool::Drop(const Key &key) { Panic("Drop"); return KELPIE_OK; } -rc_t UnconfiguredPool::List(const Key &search_key, ObjectCapacities *capacities) { Panic("List"); return KELPIE_OK; } + lunasa::DataObject *user_ldo) { Panic("Need"); return KELPIE_OK; } + +rc_t UnconfiguredPool::Compute(const Key &key, const std::string &function_name, + const std::string &function_args, + const fn_compute_callback_t &callback) { Panic("Compute"); return KELPIE_OK;} +rc_t UnconfiguredPool::Info(const Key &key, object_info_t *col_info) { Panic("Info"); return KELPIE_OK; } +rc_t UnconfiguredPool::RowInfo(const Key &key, object_info_t *row_info) { Panic("RowInfo"); return KELPIE_OK; } +rc_t UnconfiguredPool::Drop(const Key &key, fn_drop_callback_t callback) { Panic("Drop"); return KELPIE_OK; } +rc_t UnconfiguredPool::List(const Key &search_key, ObjectCapacities *capacities) { Panic("List"); return KELPIE_OK; } int UnconfiguredPool::FindTargetNode(const Key &key, faodel::nodeid_t *node_id, - net::peer_ptr_t *peer_ptr) { Panic("FindTargetNode"); return 0; } + net::peer_ptr_t *peer_ptr) { Panic("FindTargetNode"); return 0; } -void UnconfiguredPool::Panic(std::string caller) const { +void UnconfiguredPool::Panic(const std::string &caller) const { std::stringstream ss; - ss << "Kelpie pool was not initialized before calling "< UnconfiguredPoolCreate(const std::string &error_message) { + return make_shared(error_message); +} + } // namespace kelpie diff --git a/src/kelpie/pools/UnconfiguredPool/UnconfiguredPool.hh b/src/kelpie/pools/UnconfiguredPool/UnconfiguredPool.hh index c242755..dc2589e 100644 --- a/src/kelpie/pools/UnconfiguredPool/UnconfiguredPool.hh +++ b/src/kelpie/pools/UnconfiguredPool/UnconfiguredPool.hh @@ -1,12 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_UNCONFIGUREDPOOL_HH #define KELPIE_UNCONFIGUREDPOOL_HH -#include -#include +#include +#include #include #include #include @@ -31,40 +31,49 @@ namespace kelpie { * when a user requests a pool that cannot be located. The intent is for * this pool to trigger a panic operation on any call, in order to help * identify that a bad pool request was made. + * + * Users should check the pool.Valid() option after connect to make sure + * it's valid. If they don't and make a call on a bad pool, they'll wind + * up here, and will receive an exception. */ class UnconfiguredPool : public PoolBase { public: UnconfiguredPool(); + explicit UnconfiguredPool(const std::string &error_message); ~UnconfiguredPool() override; //PoolBase functions - rc_t Publish(const Key &key, fn_publish_callback_t callback) override; - rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, fn_publish_callback_t callback) override; + rc_t Publish(const Key &key, const fn_publish_callback_t &callback) override; + rc_t Publish(const Key &key, const lunasa::DataObject &user_ldo, const fn_publish_callback_t &callback) override; + + rc_t Want(const Key &key, size_t expected_ldo_user_bytes, const fn_want_callback_t &callback) override; + rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo) override; //Block until get - rc_t Want(const Key &key, size_t expected_ldo_user_bytes, fn_want_callback_t callback) override; + rc_t Compute(const Key &key, const std::string &function_name, const std::string &function_args, const fn_compute_callback_t &callback) override; //Async compute - rc_t Need(const Key &key, size_t expected_ldo_user_bytes, lunasa::DataObject *returned_ldo); //Block until get - rc_t Info(const Key &key, kv_col_info_t *col_info) override; - rc_t RowInfo(const Key &key, kv_row_info_t *row_info) override; - rc_t Drop(const Key &key) override; - rc_t List(const Key &search_key, ObjectCapacities *object_capacities=nullptr) override; + rc_t Info(const Key &key, object_info_t *col_info) override; + rc_t RowInfo(const Key &key, object_info_t *row_info) override; + rc_t Drop(const Key &key, fn_drop_callback_t callback) override; + rc_t List(const Key &search_key, ObjectCapacities *object_capacities) override; - int FindTargetNode(const Key &key, faodel::nodeid_t *node_id=nullptr, net::peer_ptr_t *peer_ptr=nullptr) override; + int FindTargetNode(const Key &key, faodel::nodeid_t *node_id, net::peer_ptr_t *peer_ptr) override; std::string TypeName() const override { return "unconfigured"; } //InfoInterface function - void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; + void sstr(std::stringstream &ss, int depth, int indent) const override; + + std::string error_message; private: - void Panic(std::string caller) const; + void Panic(const std::string &caller) const; }; //UnconfiguredPool //For use by connect -std::shared_ptr UnconfiguredPoolCreate(const faodel::ResourceURL &pool_url); +std::shared_ptr UnconfiguredPoolCreate(const std::string &error_message); } // namespace kelpie diff --git a/src/kelpie/services/PoolServerDriver.cpp b/src/kelpie/services/PoolServerDriver.cpp index 0f0c0f7..aceb90a 100644 --- a/src/kelpie/services/PoolServerDriver.cpp +++ b/src/kelpie/services/PoolServerDriver.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/src/kelpie/services/PoolServerDriver.hh b/src/kelpie/services/PoolServerDriver.hh index 182cb9f..21e52b2 100644 --- a/src/kelpie/services/PoolServerDriver.hh +++ b/src/kelpie/services/PoolServerDriver.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef KELPIE_DHTDRIVER_HH #define KELPIE_DHTDRIVER_HH diff --git a/src/lunasa/CMakeLists.txt b/src/lunasa/CMakeLists.txt index cb32b18..6bf2cde 100644 --- a/src/lunasa/CMakeLists.txt +++ b/src/lunasa/CMakeLists.txt @@ -6,6 +6,10 @@ set(HEADERS_PUBLIC DataObject.hh Lunasa.hh common/Types.hh + common/DataObjectPacker.hh + common/Helpers.hh + common/GenericRandomDataBundle.hh + common/GenericSequentialDataBundle.hh ) @@ -17,6 +21,7 @@ set(HEADERS allocators/AllocatorMalloc.hh allocators/AllocatorUnconfigured.hh common/Allocation.hh + common/DataObjectPacker.hh common/GenericRandomDataBundle.hh common/GenericSequentialDataBundle.hh common/DataObjectTypeRegistry.hh @@ -35,6 +40,8 @@ set(SOURCES allocators/AllocatorMalloc.cpp allocators/AllocatorUnconfigured.cpp common/DataObjectTypeRegistry.cpp + common/DataObjectPacker.cpp + common/Helpers.cpp core/LunasaCoreBase.cpp core/LunasaCoreSplit.cpp core/LunasaCoreUnconfigured.cpp @@ -67,7 +74,7 @@ install(TARGETS lunasa ) install( FILES DataObject.hh Lunasa.hh DESTINATION ${INCLUDE_INSTALL_DIR}/faodel/lunasa ) -install( FILES common/Types.hh DESTINATION ${INCLUDE_INSTALL_DIR}/faodel/lunasa/common ) +install( FILES common/Types.hh common/DataObjectPacker.hh common/Helpers.hh common/GenericRandomDataBundle.hh common/GenericSequentialDataBundle.hh DESTINATION ${INCLUDE_INSTALL_DIR}/faodel/lunasa/common ) #----------------------------------------- diff --git a/src/lunasa/DataObject.cpp b/src/lunasa/DataObject.cpp index a37e166..c4c36a3 100644 --- a/src/lunasa/DataObject.cpp +++ b/src/lunasa/DataObject.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "DataObject.hh" @@ -16,7 +16,6 @@ #include #include #include -#include #include #include @@ -45,6 +44,8 @@ using AllocatorBase = internal::AllocatorBase; //Our core is hidden inside of a singleton. Use this macro to make the code easier to understand #define LCORE (internal::Singleton::impl.core) +/* SET proper alignment (multiples of 4-bytes required for RDMA GETs on Aries IC) */ +#define LDO_ALIGNMENT 4 /** * @brief Empty constructor. Expects user will copy another ldo handle to this one later @@ -75,6 +76,11 @@ DataObject::DataObject(uint32_t total_user_capacity, throw invalidArgumentError; } + /* FORCE capacity to be properly aligned (multiples of 4-bytes required for RDMA GETs on Aries IC) */ + int padding = LDO_ALIGNMENT - ((meta_capacity+data_capacity) & (LDO_ALIGNMENT-1)); + + total_user_capacity += padding; + Allocation *allocation = nullptr; if( allocator == DataObject::AllocatorType::eager ) { allocation = LCORE->AllocateEager(total_user_capacity); @@ -87,8 +93,10 @@ DataObject::DataObject(uint32_t total_user_capacity, throw invalidArgumentError; } - allocation->setHeader(1, meta_capacity, data_capacity, tag); + allocation->setHeader(1, meta_capacity, data_capacity, padding, tag); impl = allocation; + + F_ASSERT((((uint64_t)&allocation->header) & (LDO_ALIGNMENT-1)) == 0, ""); } /** @@ -137,7 +145,7 @@ DataObject::DataObject(void *userMemory, //Only allocate memory for the headers. Allocation *allocation = LCORE->AllocateEager(0); - allocation->setHeader(1, 0, 0, 0); + allocation->setHeader(1, 0, 0, 0, 0); impl = allocation; // Add our user data segment @@ -236,17 +244,20 @@ DataObject& DataObject::operator=(DataObject &&source) { DataObject& DataObject::deepcopy(const DataObject &source) { // !!!! TODO: not currently supported for user LDOs - assert(source.impl->local.user_data_segments == nullptr); + F_ASSERT(source.impl->local.user_data_segments == nullptr, "Deeo copy not supported on user LDOs"); if( impl != nullptr ) { impl->DecrRef(); impl = nullptr; } + /* FORCE capacity to be properly aligned (multiples of 4-bytes required for RDMA GETs on Aries IC) */ + int padding = LDO_ALIGNMENT - ((source.impl->header.meta_bytes+source.impl->header.data_bytes) & (LDO_ALIGNMENT-1)); + //ALLOCATE a mirror of the source's allocation. - Allocation *allocation = source.impl->local.allocator->Allocate(source.impl->header.meta_bytes + - source.impl->header.data_bytes); - allocation->setHeader(1, source.impl->header.meta_bytes, source.impl->header.data_bytes, source.impl->header.type); + size_t alloc_size = source.impl->header.meta_bytes + source.impl->header.data_bytes + padding; + Allocation *allocation = source.impl->local.allocator->Allocate(alloc_size); + allocation->setHeader(1, source.impl->header.meta_bytes, source.impl->header.data_bytes, padding, source.impl->header.type); impl = allocation; //COPY the source to the mirror @@ -284,8 +295,9 @@ uint32_t DataObject::readFromFile(const char *filename) { char *p = reinterpret_cast(internal_use_only.GetHeaderPtr()); struct stat results; - assert(stat(filename, &results) == 0); - assert(results.st_size <= GetHeaderSize() + GetMetaSize() + GetDataSize()); + int rc = stat(filename, &results); + F_ASSERT(rc == 0,"File stat failed in readFromFile"); + F_ASSERT(results.st_size <= GetHeaderSize() + GetMetaSize() + GetDataSize(), "File size mismatch in readFromFile"); f.read(p, results.st_size); f.close(); @@ -350,7 +362,7 @@ uint32_t DataObject::GetLocalHeaderSize() const { * @brief Get the size of the header that travels with the LDO (meta_tag, meta_size, data_size) * @return Size of the header */ -uint32_t DataObject::GetHeaderSize() const { +uint32_t DataObject::GetHeaderSize() { return offsetof(Allocation, user[0]) - offsetof(Allocation, header); } @@ -373,6 +385,11 @@ uint32_t DataObject::GetDataSize() const { return impl->header.data_bytes; } +uint32_t DataObject::GetPaddingSize() const { + if( impl == nullptr ) return 0; + return impl->local.padding; +} + /** * @brief Get the size of the user section (meta_size + data_size) * @return Size of meta and data sections @@ -423,14 +440,14 @@ void *DataObject::GetMetaPtr() const { } else if( impl->local.user_data_segments != nullptr ) { /* === Allocation only contains headers === */ - assert(impl->local.user_data_segments->empty() == false); - assert(impl->local.allocated_bytes == sizeof(Allocation)); + F_ASSERT(impl->local.user_data_segments->empty() == false, ""); + F_ASSERT(impl->local.allocated_bytes == sizeof(Allocation), ""); /* First data segment must contain the User Metadata segment. */ AllocationSegment &segment = impl->local.user_data_segments->front(); /* Segment must be big enough to store the entire User Metadata segment. */ - assert(segment.size > impl->header.meta_bytes); + F_ASSERT(segment.size > impl->header.meta_bytes, ""); return segment.buffer_ptr; @@ -440,7 +457,8 @@ void *DataObject::GetMetaPtr() const { return (void *)&impl->user[0]; } else { /* This can only mean that the allocation is smaller than the header. Not good. */ - assert(0); + std::cerr<<"Lunasa internal allocation: "<local.allocated_bytes<<" vs "<local.user_data_segments != nullptr ) { /* === Allocation only contains headers === */ - assert(impl->local.user_data_segments->empty() == false); - assert(impl->local.allocated_bytes == sizeof(Allocation)); + F_ASSERT(impl->local.user_data_segments->empty() == false, ""); + F_ASSERT(impl->local.allocated_bytes == sizeof(Allocation), ""); /* First data segment must contain the User Metadata segment. */ AllocationSegment &segment = impl->local.user_data_segments->front(); /* Segment must be big enough to store the entire User Metadata and User Data segments. */ - assert(segment.size > (impl->header.meta_bytes+impl->header.data_bytes)); + F_ASSERT(segment.size > (impl->header.meta_bytes+impl->header.data_bytes), ""); return (void *)((uint8_t *)segment.buffer_ptr + impl->header.meta_bytes); } else if( impl->local.allocated_bytes >= sizeof(Allocation) ) { @@ -473,7 +491,7 @@ void *DataObject::GetDataPtr() const { return (void *)(impl->user + impl->header.meta_bytes); } else { /* This can only mean that the allocation is smaller than the header. Not good. */ - assert(0); + F_ASSERT(0, "Allocation smaller than header?"); } return nullptr; @@ -487,7 +505,7 @@ int DataObject::GetBaseRdmaHandles(std::queue &rdma_segments) /* If memory is EAGERLY registered, then this handle will never be NULL. As a result, if this handle is NULL, we can assume that it's LAZILY registered. */ if( impl->local.net_buffer_handle == nullptr ) { - assert(impl->local.allocator->UsingLazyRegistration() == true); + F_ASSERT(impl->local.allocator->UsingLazyRegistration() == true, ""); impl->local.allocator->RegisterMemory(impl, impl->local.allocated_bytes, impl->local.net_buffer_handle); } @@ -499,7 +517,7 @@ int DataObject::GetBaseRdmaHandles(std::queue &rdma_segments) /* ADD user segments, if any, to the queue */ if( impl->local.user_data_segments != nullptr && impl->local.user_data_segments->empty() == false ) { /* Current assumption is that there's just one user data segment. */ - assert(impl->local.user_data_segments->size() == 1); + F_ASSERT(impl->local.user_data_segments->size() == 1, ""); AllocationSegment &allocation_segment = impl->local.user_data_segments->front(); rdma_segment_desc segment(allocation_segment.net_buffer_handle, 0, allocation_segment.size); @@ -514,7 +532,7 @@ int DataObject::GetBaseRdmaHandle(void **rdma_handle, uint32_t &offset) const { std::queue segments; GetBaseRdmaHandles(segments); - assert(segments.size() == 1); + F_ASSERT(segments.size() == 1, ""); rdma_segment_desc &segment = segments.front(); *rdma_handle = segment.net_buffer_handle; offset = segment.net_buffer_offset; @@ -535,13 +553,13 @@ int DataObject::GetLocalHeaderRdmaHandle(void **rdma_handle, uint32_t &offset) c int DataObject::GetHeaderRdmaHandles(std::queue &rdma_segments) const { - assert(rdma_segments.size() == 0); + F_ASSERT(rdma_segments.size() == 0, ""); if( impl == nullptr ) return 0; /* If memory is EAGERLY registered, then this handle will never be NULL. As a result, if this handle is NULL, we can assume that it's LAZILY registered. */ if( impl->local.net_buffer_handle == nullptr ) { - assert(impl->local.allocator->UsingLazyRegistration() == true); + F_ASSERT(impl->local.allocator->UsingLazyRegistration() == true, ""); impl->local.allocator->RegisterMemory(impl, impl->local.allocated_bytes, impl->local.net_buffer_handle); } @@ -554,7 +572,7 @@ int DataObject::GetHeaderRdmaHandles(std::queue &rdma_segment /* ADD user segments, if any, to the queue */ if( impl->local.user_data_segments != nullptr && impl->local.user_data_segments->empty() == false ) { /* Current assumption is that there's just one user data segment. */ - assert(impl->local.user_data_segments->size() == 1); + F_ASSERT(impl->local.user_data_segments->size() == 1, ""); AllocationSegment &allocation_segment = impl->local.user_data_segments->front(); rdma_segment_desc segment(allocation_segment.net_buffer_handle, 0, allocation_segment.size); @@ -567,9 +585,9 @@ int DataObject::GetHeaderRdmaHandles(std::queue &rdma_segment int DataObject::GetHeaderRdmaHandle(void **rdma_handle, uint32_t &offset) const { std::queue segments; - assert(segments.size() == 0); + F_ASSERT(segments.size() == 0, ""); GetHeaderRdmaHandles(segments); - assert(segments.size() == 1); + F_ASSERT(segments.size() == 1, ""); rdma_segment_desc &segment = segments.front(); *rdma_handle = segment.net_buffer_handle; offset = segment.net_buffer_offset; @@ -584,7 +602,7 @@ int DataObject::GetMetaRdmaHandles(std::queue &rdma_segments) /* If memory is EAGERLY registered, then this handle will never be NULL. As a result, if this handle is NULL, we can assume that it's LAZILY registered. */ if( impl->local.net_buffer_handle == nullptr ) { - assert(impl->local.allocator->UsingLazyRegistration() == true); + F_ASSERT(impl->local.allocator->UsingLazyRegistration() == true, ""); impl->local.allocator->RegisterMemory(impl, impl->local.allocated_bytes, impl->local.net_buffer_handle); @@ -598,7 +616,7 @@ int DataObject::GetMetaRdmaHandles(std::queue &rdma_segments) rdma_segments.push(header_segment); /* First data segment must contain the User Metadata section. */ - assert(impl->local.user_data_segments->size() == 1); + F_ASSERT(impl->local.user_data_segments->size() == 1, ""); AllocationSegment &allocation_segment = impl->local.user_data_segments->front(); /* Segment must be big enough to store the entire User Metadata segment. */ @@ -613,7 +631,7 @@ int DataObject::GetMetaRdmaHandles(std::queue &rdma_segments) /* Allocation must be big enough to store the entire User METADATA segment. */ uint32_t meta_offset = offsetof(Allocation, user); - assert(impl->local.allocated_bytes >= meta_offset + impl->header.meta_bytes); + F_ASSERT(impl->local.allocated_bytes >= meta_offset + impl->header.meta_bytes, ""); uint32_t total_offset = impl->local.net_buffer_offset + offsetof(Allocation, user); uint32_t size = impl->local.allocated_bytes - offsetof(Allocation, user); @@ -621,7 +639,7 @@ int DataObject::GetMetaRdmaHandles(std::queue &rdma_segments) rdma_segments.push(rdma_segment); } else { /* This can only mean that the allocation is smaller than the header. Not good. */ - assert(0); + F_ASSERT(0, "Allocation is smaller than header"); } return 0; @@ -631,7 +649,7 @@ int DataObject::GetMetaRdmaHandle(void **rdma_handle, uint32_t &offset) const { std::queue segments; GetMetaRdmaHandles(segments); - assert(segments.size() == 1); + F_ASSERT(segments.size() == 1, ""); rdma_segment_desc &segment = segments.front(); *rdma_handle = segment.net_buffer_handle; offset = segment.net_buffer_offset; @@ -645,19 +663,19 @@ int DataObject::GetDataRdmaHandles(std::queue &rdma_segments) /* If memory is EAGERLY registered, then this handle will never be NULL. As a result, if this handle is NULL, we can assume that it's LAZILY registered. */ if( impl->local.net_buffer_handle == nullptr ) { - assert(impl->local.allocator->UsingLazyRegistration() == true); + F_ASSERT(impl->local.allocator->UsingLazyRegistration() == true, ""); impl->local.allocator->RegisterMemory(impl, impl->local.allocated_bytes, impl->local.net_buffer_handle); } if( impl->local.allocated_bytes == sizeof(Allocation) ) { /* === Allocation only contains headers === */ - assert(impl->local.user_data_segments->empty() == false); + F_ASSERT(impl->local.user_data_segments->empty() == false, ""); /* First data segment must contain the User Metadata segment. */ AllocationSegment &allocation_segment = impl->local.user_data_segments->front(); /* Segment must be equal to the combined size of the User METADATA and DATA sections. */ - assert(allocation_segment.size == impl->header.meta_bytes + impl->header.data_bytes); + F_ASSERT(allocation_segment.size == impl->header.meta_bytes + impl->header.data_bytes, ""); uint32_t offset = impl->local.net_buffer_offset + offsetof(Allocation, user) + @@ -671,9 +689,23 @@ int DataObject::GetDataRdmaHandles(std::queue &rdma_segments) /* === Allocation contains User METADATA === */ /* Segment must be equal to the combined size of the User METADATA and DATA sections. */ - assert(impl->local.allocated_bytes == offsetof(Allocation, user) + + if(impl->local.allocated_bytes != offsetof(Allocation, user) + impl->header.meta_bytes + - impl->header.data_bytes); + impl->header.data_bytes + + impl->local.padding) + { + fprintf(stderr, "impl->local.allocated_bytes (%d) != offsetof(Allocation, user) (%ld) +\n" + " impl->header.meta_bytes (%d) + \n" + " impl->header.data_bytes (%d)+ \n" + " impl->local.padding (%d)", + impl->local.allocated_bytes, offsetof(Allocation, user), impl->header.meta_bytes, + impl->header.data_bytes, impl->local.padding); + fflush(stderr); + } + F_ASSERT(impl->local.allocated_bytes == offsetof(Allocation, user) + + impl->header.meta_bytes + + impl->header.data_bytes + + impl->local.padding, ""); uint32_t offset = impl->local.net_buffer_offset + offsetof(Allocation, user) + @@ -685,7 +717,7 @@ int DataObject::GetDataRdmaHandles(std::queue &rdma_segments) rdma_segments.push(rdma_segment); } else { /* This can only mean that the allocation is smaller than the header. Not good. */ - assert(0); + F_ASSERT(0, ""); } return 0; @@ -694,7 +726,7 @@ int DataObject::GetDataRdmaHandles(std::queue &rdma_segments) int DataObject::GetDataRdmaHandle(void **rdma_handle, uint32_t &offset) const { std::queue segments; GetDataRdmaHandles(segments); - assert(segments.size() == 1); + F_ASSERT(segments.size() == 1, ""); rdma_segment_desc &segment = segments.front(); *rdma_handle = segment.net_buffer_handle; offset = segment.net_buffer_offset; @@ -761,7 +793,7 @@ void DataObject::AddUserDataSegment( } void DataObject::sstr(stringstream &ss, int depth, int indent) const { - assert(0 &&"TODO"); + F_TODO("Data Object sstr missing"); #if 0 // !!!! TODO : needs to be updated to account for API changes if(depth<0) return; diff --git a/src/lunasa/DataObject.hh b/src/lunasa/DataObject.hh index c0c1e10..1039c3a 100644 --- a/src/lunasa/DataObject.hh +++ b/src/lunasa/DataObject.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file DataObject.hh @@ -141,9 +141,10 @@ public: /* GET size of LDO segments */ uint32_t GetLocalHeaderSize() const; //Size of local bookkeeping - uint32_t GetHeaderSize() const; //Header (contains tag, meta, and data lengths) + static uint32_t GetHeaderSize(); //Header (contains tag, meta, and data lengths) uint32_t GetMetaSize() const; //Meta size reported in header uint32_t GetDataSize() const; //Data size reported in header + uint32_t GetPaddingSize() const; //Number of padding bytes to ensure aligned length uint32_t GetUserSize() const; //Meta + Data uint32_t GetWireSize() const; //Header+Meta+Data uint32_t GetRawAllocationSize() const; //Local bookkeeping's allocated bytes for header+meta+data (rare) @@ -153,10 +154,10 @@ public: /* Get RDMA handles */ - /*! @brief Class describing the characterisitics of a memory segment + /*! @brief Class describing the characteristics of a memory segment * * LDOs that are created from memory that was not allocated by Lunasa will be comprised - * of two (likely) discontinous memory regions. An instance of this class will be used + * of two (likely) discontinuous memory regions. An instance of this class will be used * to describe each. * * \sa DataObject::GetXXXXRdmaHandles() */ @@ -215,7 +216,7 @@ public: int DeepCompare(const DataObject &other) const; //InfoInterface - void sstr(std::stringstream &ss, int depth=0, int indent=0) const; + void sstr(std::stringstream &ss, int depth=0, int indent=0) const override; private: // Pointer to STRUCTURE that represents the "beginning" of the data object. diff --git a/src/lunasa/Lunasa.cpp b/src/lunasa/Lunasa.cpp index 27596fa..8bd6ffc 100644 --- a/src/lunasa/Lunasa.cpp +++ b/src/lunasa/Lunasa.cpp @@ -1,13 +1,13 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include -#include #include //memcpy #include +#include #include "faodelConfig.h" @@ -76,7 +76,23 @@ void DeregisterDataObjectType(dataobject_type_t tag) { bool DumpDataObject(const DataObject &ldo, faodel::ReplyStream &rs) { return internal::Singleton::impl.dataobject_type_registry.DumpDataObject(ldo, rs); } +/** + * @brief Read a DataObject from disk and store it in a new DataObject + * @param[in] filename The name of the DataObject to load from disk + * @retval DataObject The object + * @throw runtime_error if file cannot be read + */ +DataObject LoadDataObjectFromFile(std::string filename) { + uint32_t header_size = lunasa::DataObject::GetHeaderSize(); + struct stat results; + if( (stat(filename.c_str(), &results) != 0) || (results.st_size < header_size)) { + throw std::runtime_error("Could not read Lunasa DataObject '"+filename+"'"); + } + lunasa::DataObject ldo(results.st_size - header_size); + ldo.readFromFile(filename.c_str()); + return ldo; +} // Note: The Lunasa class is just a wrapper that makes calls to the singleton diff --git a/src/lunasa/Lunasa.hh b/src/lunasa/Lunasa.hh index 8558624..01f068d 100644 --- a/src/lunasa/Lunasa.hh +++ b/src/lunasa/Lunasa.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file Lunasa.hh @@ -87,6 +87,10 @@ void RegisterDataObjectType(dataobject_type_t tag, std::string name, fn_DataObje void DeregisterDataObjectType(dataobject_type_t tag); bool DumpDataObject(const DataObject &ldo, faodel::ReplyStream &rs); + +DataObject LoadDataObjectFromFile(std::string filename); + + std::vector AvailableAllocators(); std::vector AvailableCores(); diff --git a/src/lunasa/allocators/AllocatorBase.cpp b/src/lunasa/allocators/AllocatorBase.cpp index f93f528..bd97fa6 100644 --- a/src/lunasa/allocators/AllocatorBase.cpp +++ b/src/lunasa/allocators/AllocatorBase.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "lunasa/allocators/AllocatorBase.hh" diff --git a/src/lunasa/allocators/AllocatorBase.hh b/src/lunasa/allocators/AllocatorBase.hh index aa03d6b..94248cc 100644 --- a/src/lunasa/allocators/AllocatorBase.hh +++ b/src/lunasa/allocators/AllocatorBase.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_ALLOCATORBASE_HH #define LUNASA_ALLOCATORBASE_HH diff --git a/src/lunasa/allocators/AllocatorMalloc.cpp b/src/lunasa/allocators/AllocatorMalloc.cpp index d3e9881..b9abb67 100644 --- a/src/lunasa/allocators/AllocatorMalloc.cpp +++ b/src/lunasa/allocators/AllocatorMalloc.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "lunasa/allocators/AllocatorMalloc.hh" #include "lunasa/common/Allocation.hh" @@ -51,24 +51,24 @@ Allocation *AllocatorMalloc::Allocate(uint32_t user_capacity) { return nullptr; } - int total_capacity = user_capacity + sizeof(Allocation); - Allocation *alloc = static_cast(malloc(total_capacity)); + user_capacity += sizeof(Allocation); + Allocation *alloc = static_cast(malloc(user_capacity)); /* RECORD where the allocation came from. */ alloc->local.allocator = this; alloc->local.net_buffer_handle = nullptr; alloc->local.net_buffer_offset = 0; - alloc->local.allocated_bytes = total_capacity; + alloc->local.allocated_bytes = user_capacity; alloc->local.user_data_segments = nullptr; // Pin the whole chunk when eager if(mEagerPinning) { - mPinFunc(alloc, total_capacity, alloc->local.net_buffer_handle); + mPinFunc(alloc, user_capacity, alloc->local.net_buffer_handle); } // Add to the set mutex->WriterLock(); - mTotalAllocated += total_capacity; + mTotalAllocated += user_capacity; mTotalUsed += user_capacity; allocations.insert(alloc); mutex->Unlock(); @@ -80,7 +80,7 @@ void AllocatorMalloc::Free(Allocation *allocation) { bool is_empty; - kassert(allocation!=nullptr, "Free of nullptr"); + F_ASSERT(allocation != nullptr, "Free of nullptr"); dbg("Free "+to_string(allocation->local.allocated_bytes)); mutex->WriterLock(); @@ -94,7 +94,7 @@ void AllocatorMalloc::Free(Allocation *allocation) { mTotalUsed -= allocation->local.allocated_bytes - sizeof(Allocation); free(allocation); } else { - assert(0); + F_ASSERT(0, "Allocation not found in free"); } is_empty = allocations.empty(); diff --git a/src/lunasa/allocators/AllocatorMalloc.hh b/src/lunasa/allocators/AllocatorMalloc.hh index a72b1e6..d1d76b6 100644 --- a/src/lunasa/allocators/AllocatorMalloc.hh +++ b/src/lunasa/allocators/AllocatorMalloc.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_ALLOCATORMALLOC_HH #define LUNASA_ALLOCATORMALLOC_HH 1 diff --git a/src/lunasa/allocators/AllocatorTcmalloc.cpp b/src/lunasa/allocators/AllocatorTcmalloc.cpp index 3a224b0..65e9400 100644 --- a/src/lunasa/allocators/AllocatorTcmalloc.cpp +++ b/src/lunasa/allocators/AllocatorTcmalloc.cpp @@ -1,10 +1,10 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "lunasa/allocators/AllocatorTcmalloc.hh" #include "lunasa/common/Allocation.hh" -#include "assert.h" + #if defined(HAVE_MALLOC_H) #include "malloc.h" @@ -32,9 +32,9 @@ void *AllocatorTcmalloc::TcmallocSysAllocator::Alloc(size_t size, size_t *actual bytes += size; *actual_size = size; rc = posix_memalign(&memory, 8 * 1024, size); - assert(rc == 0); + F_ASSERT(rc == 0, ""); AllocatorTcmalloc *tcmalloc = GetInstance(); - assert(tcmalloc != NULL); + F_ASSERT(tcmalloc != NULL, ""); tcmalloc->mTotalManaged += size; // if eager, this memory needs to be pinned @@ -130,7 +130,7 @@ AllocatorTcmalloc *AllocatorTcmalloc::GetInstance(const faodel::Configuration &c //instance->mMutexType != mutex_type) { //TODO: fix above checks, now that config holds mutex info. Previously passed in mutex_type and threading_model /* Requested instance doesn't match existing instance. */ - cerr << "ERROR: Attempt to create multiple instances of tcmalloc allocator\n"; + throw std::runtime_error("Lunasa configuration attempted to create multiple instaces of tcmalloc allocator (not possible)"); return NULL; } @@ -147,19 +147,19 @@ Allocation *AllocatorTcmalloc::Allocate(uint32_t user_capacity) { return nullptr; } - int total_capacity = user_capacity + sizeof(Allocation); - Allocation *alloc = static_cast(tc_malloc(total_capacity)); + user_capacity += sizeof(Allocation); + Allocation *alloc = static_cast(tc_malloc(user_capacity)); /* RECORD where the allocation came from. */ alloc->local.allocator = this; alloc->local.net_buffer_handle = GetPinnedAddr(alloc); alloc->local.net_buffer_offset = GetPinnedOffset(alloc); - alloc->local.allocated_bytes = total_capacity; + alloc->local.allocated_bytes = user_capacity; alloc->local.user_data_segments = nullptr; // Add to the set mutex->WriterLock(); - mTotalAllocated += total_capacity; + mTotalAllocated += user_capacity; mTotalUsed += user_capacity; allocations.insert(alloc); mutex->Unlock(); @@ -220,12 +220,12 @@ void *AllocatorTcmalloc::GetPinnedAddr(Allocation *allocation) { it = pinned_regions.lower_bound(allocation); if(it->first != allocation) { // We want the predecessor; the largest value that is less than - assert(it != pinned_regions.begin()); + F_ASSERT(it != pinned_regions.begin(), ""); it--; } std::pair value = it->second; - assert((char *) allocation >= (char *) it->first); - assert((char *) it->first + value.first > (char *) allocation); + F_ASSERT((char *) allocation >= (char *) it->first, ""); + F_ASSERT((char *) it->first + value.first > (char *) allocation, ""); mutex->Unlock(); return value.second; @@ -237,12 +237,12 @@ uint64_t AllocatorTcmalloc::GetPinnedOffset(Allocation *allocation) { it = pinned_regions.lower_bound(allocation); if(it->first != allocation) { // We want the predecessor; the largest value that is less than - assert(it != pinned_regions.begin()); + F_ASSERT(it != pinned_regions.begin(), ""); it--; } std::pair value = it->second; - assert((char *) allocation >= (char *) it->first); - assert((char *) it->first + value.first > (char *) allocation); + F_ASSERT((char *) allocation >= (char *) it->first, ""); + F_ASSERT((char *) it->first + value.first > (char *) allocation, ""); uint64_t offset = (uint64_t) ((char *) allocation - (char *) it->first); mutex->Unlock(); diff --git a/src/lunasa/allocators/AllocatorTcmalloc.hh b/src/lunasa/allocators/AllocatorTcmalloc.hh index baea2e5..63ab561 100644 --- a/src/lunasa/allocators/AllocatorTcmalloc.hh +++ b/src/lunasa/allocators/AllocatorTcmalloc.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_ALLOCATORTCMALLOC_HH #define LUNASA_ALLOCATORTCMALLOC_HH 1 diff --git a/src/lunasa/allocators/AllocatorUnconfigured.cpp b/src/lunasa/allocators/AllocatorUnconfigured.cpp index 4487f42..8e75d95 100644 --- a/src/lunasa/allocators/AllocatorUnconfigured.cpp +++ b/src/lunasa/allocators/AllocatorUnconfigured.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "lunasa/allocators/AllocatorUnconfigured.hh" diff --git a/src/lunasa/allocators/AllocatorUnconfigured.hh b/src/lunasa/allocators/AllocatorUnconfigured.hh index cca1767..a7a4faf 100644 --- a/src/lunasa/allocators/AllocatorUnconfigured.hh +++ b/src/lunasa/allocators/AllocatorUnconfigured.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_ALLOCATORUNCONFIGURED_HH #define LUNASA_ALLOCATORUNCONFIGURED_HH diff --git a/src/lunasa/allocators/Allocators.cpp b/src/lunasa/allocators/Allocators.cpp index 65e4c59..02898d6 100644 --- a/src/lunasa/allocators/Allocators.cpp +++ b/src/lunasa/allocators/Allocators.cpp @@ -1,8 +1,7 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -#include #include "faodel-common/Debug.hh" #include "lunasa/allocators/Allocators.hh" diff --git a/src/lunasa/allocators/Allocators.hh b/src/lunasa/allocators/Allocators.hh index 41e6dbc..018a0be 100644 --- a/src/lunasa/allocators/Allocators.hh +++ b/src/lunasa/allocators/Allocators.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_ALLOCATORS_HH #define LUNASA_ALLOCATORS_HH diff --git a/src/lunasa/common/Allocation.hh b/src/lunasa/common/Allocation.hh index cd31a6d..a15661c 100644 --- a/src/lunasa/common/Allocation.hh +++ b/src/lunasa/common/Allocation.hh @@ -1,18 +1,26 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_ALLOCATION_HH #define LUNASA_ALLOCATION_HH #include -#include #include "faodel-common/Common.hh" #include "lunasa/Lunasa.hh" #include "lunasa/allocators/AllocatorBase.hh" +//Debug: dump out info about when LDO refcount changes happen +//#define Faodel_DEBUG_LDO_REFCOUNTS +#ifdef Faodel_DEBUG_LDO_REFCOUNTS +#define DBG_REFCOUNT(s) { std::cout<<"DEBUG_LDO: "< 0 && "LunasaDataObject refcount decremented to below zero"); + F_ASSERT(local.ref_count > 0, "LunasaDataObject refcount decremented to below zero"); int num_left = --local.ref_count; + DBG_REFCOUNT("DecrRef"); + if(num_left == 0) { if( local.user_data_segments != nullptr ) { diff --git a/src/lunasa/common/DataObjectPacker.cpp b/src/lunasa/common/DataObjectPacker.cpp new file mode 100644 index 0000000..eceb3c9 --- /dev/null +++ b/src/lunasa/common/DataObjectPacker.cpp @@ -0,0 +1,422 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include //memcpy +#include "faodel-common/StringHelpers.hh" +#include "lunasa/common/DataObjectPacker.hh" + +using namespace std; + +namespace lunasa { + +//Set the constants for this DataObject so they'll be recognizes +const string DataObjectPacker::object_type_name = "DataObjectPacker"; +const uint16_t DataObjectPacker::object_type_id = faodel::const_hash16("DataObjectPacker"); + + +DataObjectPacker::DataObjectPacker(const vector &names, + const vector &ptrs, + const vector &bytes, const vector &types, + uint32_t data_type_hash, + int dop_format_version, + lunasa::DataObject::AllocatorType memory_type) + : finalized(true), version(dop_format_version) { + + bool using_null_ptrs = ptrs.empty(); //Allow user to come back and fill in data later + + //Make sure all our vectors are the same size + if ( (!using_null_ptrs && (names.size()!=ptrs.size()) ) || + (names.size() != bytes.size()) || + (bytes.size() != types.size()) ) { + throw std::runtime_error("DataObjectPacker given vectors of different sizes"); + } + + if((dop_format_version<1) || (dop_format_version>2)) + throw std::runtime_error("DataObjectPacker constructed with an invalid format version number"); + + //Figure out how much space we'll need + size_t payload_size=0; + for(int i=0; i(); + meta->num_vars = names.size(); + meta->packing_version = version; + meta->data_type_hash = data_type_hash; + + //Pack the payload (using specified version) + auto *payload = ldo.GetDataPtr(); + for(int i=0; i2)) + throw std::runtime_error("DataObjectPacker constructed with an invalid format version number"); + + + ldo = DataObject(sizeof(dop_meta_t) + max_data_capacity, sizeof(dop_meta_t), 0, memory_type, DataObjectPacker::object_type_id); + + auto *meta = ldo.GetMetaPtr(); + meta->num_vars = 0; + meta->packing_version = dop_format_version; + meta->data_type_hash = data_type_hash; + +} + + +/** + * @brief Ctor for when a user wants to parse an incoming DataObject + * @param[in] ldo The DataObject that will be parsed + */ +DataObjectPacker::DataObjectPacker(const lunasa::DataObject &ldo) + : ldo(ldo), finalized(true) { + + if(ldo.GetTypeID() != DataObjectPacker::object_type_id) { + throw std::runtime_error("DataObjectPacker asked to parse a DataObject that does not match its TypeID"); + } + auto *meta = ldo.GetMetaPtr(); + version = meta->packing_version; + if((version < 1) || (version >2)) + throw std::runtime_error("DataObjectPacker asked to parse DataObject with invalid packing version number"); + +} + +/** + * @brief Inspect the DataObject's metadata and see if the data type hash matches an exected value + * @param[in] expected_data_type_hash The hash you expect this to be (eg const_hash32("my special data")) + * @retval True Hashes match + * @retval False The hashes don't match + */ +bool DataObjectPacker::VerifyDataType(uint32_t expected_data_type_hash) { + auto *meta = ldo.GetMetaPtr(); + if(expected_data_type_hash != meta->data_type_hash) return false; + return true; +} + + + +/** + * @brief Determine how much space a variable will require (overhead+data), given a particular format version + * @param[in] dop_format_version The packing format version to use + * @param[in] name The variable name + * @param[in] data_bytes How many bytes the data is + * @return EntrySize How many bytes all of this structure will use in + */ +size_t DataObjectPacker::ComputeEntrySize(int dop_format_version, const string &name, uint32_t data_bytes) { + switch(dop_format_version) { + case 1: return sizeof(dop_entry_v1_t) + std::min(name.size(), size_t(255)) + data_bytes; + case 2: return sizeof(dop_entry_v2_t) + data_bytes; + default: + throw std::runtime_error("Attempted to pack with unknown version number: "+std::to_string(dop_format_version)); + } +} + +/** + * @brief Determine how much space is left in this allocation for additional variables + * @return bytes Raw space left (note: user must take into account Entry Overhead!) + */ +uint32_t DataObjectPacker::RemainingCapacity() { + if(finalized) return 0; + + uint32_t dspace_left = ldo.GetUserCapacity() - (ldo.GetMetaSize() + ldo.GetDataSize()); + if(dspace_left < getEntryOverhead()) return 0; + + return dspace_left; +} + +/** + * @brief Append a variable to the data section of a DataObject when Capacity is available + * @param[in] name The name of the variable + * @param[in] data_ptr Pointer to the variable to be copied in (nullptr = no copy) + * @param[in] data_bytes How many bytes of data there are for this variable + * @param[in] type The data type ID a user has assigned to this variable + * @retval 0 Success: Variable was appended successfully + * @retval -1 Failure: Either the object is finalized or there wasn't enough capacity left for the data + * @note: This function is only for GenericPackers allocated using the max capacity ctor + * @note: This function does not check to see if you inserted the same variable name multiple times + */ +int DataObjectPacker::AppendVariable(const string &name, void *data_ptr, size_t data_bytes, uint8_t type) { + + //Bail out if we don't allow modifications + if(finalized) return -1; + + //Figure out how big this is and where it goes + size_t entry_size = computeEntrySize(name, data_bytes); + uint32_t offset = ldo.GetDataSize(); + + //Try to adjust the data size. Exit if no room left + int rc = ldo.ModifyUserSizes(sizeof(dop_meta_t), offset + entry_size); + if(rc < 0) return -1; //No capacity left, ldo did not update + + auto *entry_ptr = ldo.GetDataPtr() + offset; + writeEntry(entry_ptr, name, type, data_ptr, data_bytes); + + //Update the metadata + auto *meta = ldo.GetMetaPtr(); + meta->num_vars++; + + return 0; +} + + +/** + * @brief Locate a variable by name in this ldo and return the raw pointer to its data + * @param[in] name The name of the variable to lookup + * @param[out] raw_ptr_to_data Pointer to the variable's data inside the DataObject + * @param[out] bytes How many bytes of data there are for this variable + * @param[out] type The data type ID users assigned to this variable + * @retval 0 Found + * @retval ENOENT Item not found + * @note: It may be possible for the names of two variables to collide due to truncation and hashing + */ +int DataObjectPacker::GetVarPointer(const std::string &name, void **raw_ptr_to_data, size_t *bytes, uint8_t *type) { + + //First, Hash the input string. This hash varies depending on version + uint32_t hash; + switch(version){ + case 1: hash = faodel::hash16(name); break; + case 2: hash = faodel::hash32(name); break; + } + + //Second, build the index if we haven't already done so + rebuildIndexIfNeeded(); + + + //Next, Look it up in the map + typedef std::multimap::iterator MMiterator; + std::pair result = index.equal_range(hash); + for(MMiterator it = result.first; it != result.second; ++it) { + + switch(version) { + case 1: + { + auto *ptr = reinterpret_cast(it->second); + if(ptr->MatchesName(name)) { + if(raw_ptr_to_data) *raw_ptr_to_data = reinterpret_cast(ptr->GetDataPtr()); + if(bytes) *bytes = ptr->data_length; + if(type) *type = ptr->data_type; + return 0; + } + } + break; + case 2: + { + auto *ptr = reinterpret_cast(it->second); + if(ptr->hash == hash) { + if(raw_ptr_to_data) *raw_ptr_to_data = reinterpret_cast(ptr->GetDataPtr()); + if(bytes) *bytes = ptr->data_length; + if(type) *type = ptr->data_type; + return 0; + } + } + } + } + //Not found, clear out results + if(raw_ptr_to_data) *raw_ptr_to_data = nullptr; + if(bytes) *bytes=0; + if(type) *type=0; + return ENOENT; +} + +/** + * @brief Retrieve a variable when using version 2 packing format + * @param[in] hash The hash of the variable name to be retrieved + * @param[out] raw_ptr_to_data Pointer to the variable's data inside the DataObject + * @param[out] bytes How many bytes of data there are for this variable + * @param[out] type The data type ID users assigned to this variable + * @retval 0 Found + * @retval ENOENT Item not found + * @retval EINVAL Could not search on hash because data format is not version 2 + */ +int DataObjectPacker::GetVarPointer(uint32_t hash, void **raw_ptr_to_data, size_t *bytes, uint8_t *type) { + + //Version 1 only has a 16b hash and is therefore not suitable. Bail out + if(version!=2) return EINVAL; + + //First, build the index if we haven't already done so + rebuildIndexIfNeeded(); + + //Next, Look it up in the map + typedef std::multimap::iterator MMiterator; + std::pair result = index.equal_range(hash); + for(MMiterator it = result.first; it != result.second; ++it) { + switch(version) { + case 2: + { + auto *ptr = reinterpret_cast(it->second); + if(ptr->hash == hash) { + if(raw_ptr_to_data) *raw_ptr_to_data = reinterpret_cast(ptr->GetDataPtr()); + if(bytes) *bytes = ptr->data_length; + if(type) *type = ptr->data_type; + return 0; + } + } + } + } + if(raw_ptr_to_data) *raw_ptr_to_data = nullptr; + if(bytes) *bytes=0; + if(type) *type=0; + return ENOENT; +} + + + +/** + * @brief Get a list of all (truncated) variable names from a DataObject, when using version 1 format + * @param[out] names All of the truncated names + * @retval 0 Finished without errors + * @retval EINVAL Tried to run when not using version 1 format + */ +int DataObjectPacker::GetVarNames(std::vector *names) { + if (version != 1) return EINVAL; + if(!names) return 0; + + rebuildIndexIfNeeded(); + + for(auto &mm : index) { + auto ptr = reinterpret_cast(mm.second); + names->push_back(ptr->GetName()); + } + + return 0; +} + +/** + * @brief Get the DataObject that this packer maintains + * @return DataObject + * + * Pass back a (correctly ref counted) handle to the user about the DataObject + * this packer maintains. + */ +DataObject DataObjectPacker::GetDataObject() { + return ldo; +} + +/** + * @brief Recompute an Index for looking up variables in the DataObject if needed + * + * This function will walk through the current DataObject and insert the hash/ptr for + * each item into a multimap index. + */ +void DataObjectPacker::rebuildIndexIfNeeded() { + + auto *meta = ldo.GetMetaPtr(); + + if(index.size() == meta->num_vars) return; + + //We're rebuilding the index + index.clear(); + + //Make sure we don't get out of data section. Find location of last possible entry + uint32_t max_offset = ldo.GetDataSize(); + switch(version) { + case 1: max_offset -= sizeof(dop_entry_v1_t); break; + case 2: max_offset -= sizeof(dop_entry_v2_t); break; + } + + auto *payload = ldo.GetDataPtr(); + size_t offset = 0; + for(int i=0; i < meta->num_vars; i++) { + if(offset > max_offset) { + throw std::runtime_error("Building index failed: exceeded boundaries for DataObject"); + } + switch(version) { + case 1: + { + auto *entry = reinterpret_cast(payload); + index.insert( std::pair(entry->name_hash, entry)); + payload += entry->GetTotalSize(); + break; + } + case 2: + { + auto *entry = reinterpret_cast(payload); + index.insert( std::pair(entry->hash, entry)); + payload += entry->GetTotalSize(); + } + } + + } + +} + + +/** + * @brief Return information about how much overhead is needed for each variable that is added + * @return OverheadSize The size in bytes of the overhead for the packing format being used + */ +size_t DataObjectPacker::getEntryOverhead() { + switch(version) { + case 1: return sizeof(dop_entry_v1_t); + case 2: return sizeof(dop_entry_v2_t); + } + throw std::runtime_error("Unknown GeneralPacker format version"); +} + +/** + * @brief Write entry data to a user-provided address + * @param[in] payload_ptr Where to write this entry + * @param[in] name Variable's name + * @param[in] type What type of data this is + * @param[in] data_ptr Source data for variable (nullptr = don't copy) + * @param[in] data_bytes How long the data is + * @return EntryBytes The total length of this entry (structure+data) + */ +size_t DataObjectPacker::writeEntry(uint8_t *payload_ptr, const string &name, uint8_t type, const void *data_ptr, uint32_t data_bytes) { + + switch(version) { + case 1: + { + dop_entry_v1_t *entry = reinterpret_cast(payload_ptr); + entry->name_length = std::min(name.size(), size_t(255)); + entry->data_type = type; + entry->name_hash = faodel::hash16(name); + entry->data_length = data_bytes; + memcpy(&entry->name_data[0], name.c_str(), entry->name_length); + if(data_ptr) memcpy(entry->GetDataPtr(), data_ptr, data_bytes); + return entry->GetTotalSize(); + } + break; + case 2: + { + dop_entry_v2_t *entry = reinterpret_cast(payload_ptr); + entry->hash = faodel::hash32(name); + entry->data_length = data_bytes; + entry->data_type = type; + if(data_ptr) memcpy(&entry->data[0], data_ptr, data_bytes); + return entry->GetTotalSize(); + } + break; + default: + throw std::runtime_error("Attempted to pack with unknown version number: "+std::to_string(version)); + } + +} + + + + +} //namespace lunasa diff --git a/src/lunasa/common/DataObjectPacker.hh b/src/lunasa/common/DataObjectPacker.hh new file mode 100644 index 0000000..d7506b4 --- /dev/null +++ b/src/lunasa/common/DataObjectPacker.hh @@ -0,0 +1,116 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_DATAOBJECTPACKER_HH +#define FAODEL_DATAOBJECTPACKER_HH + +#include +#include + +#include "faodel-common/StringHelpers.hh" +#include "lunasa/DataObject.hh" + +namespace lunasa { + +class DataObjectPacker { + +public: + DataObjectPacker(const std::vector &names, + const std::vector &ptrs, + const std::vector &bytes, + const std::vector &types, + uint32_t data_type_hash, + int dop_format_version=1, + DataObject::AllocatorType memory_type = DataObject::AllocatorType::eager); + + DataObjectPacker(size_t max_data_capacity, + uint32_t data_type_hash, + int dop_format_version=1, + DataObject::AllocatorType memory_type = DataObject::AllocatorType::eager); + + explicit DataObjectPacker(const DataObject &ldo); + + + static size_t GetEntryOverhead(int dop_format_version); + static size_t ComputeEntrySize(int dop_format_version, const std::string &name, uint32_t data_bytes); + size_t ComputeEntrySize(const std::string &name, uint32_t data_bytes) const { return ComputeEntrySize(version, name, data_bytes); } + + + bool VerifyDataType(uint32_t expected_data_type_hash); + + uint32_t RemainingCapacity(); + int AppendVariable(const std::string &name, void *data_ptr, size_t data_bytes, uint8_t type); + + int GetVarPointer(const std::string &name, void **raw_ptr_to_data, size_t *bytes, uint8_t *type); + int GetVarPointer(uint32_t hash, void **raw_ptr_to_data, size_t *bytes, uint8_t *type); + + int GetVarNames(std::vector *names); + + DataObject GetDataObject(); + + const static uint16_t object_type_id; + const static std::string object_type_name; + +private: + + struct dop_meta_t { + uint32_t num_vars; //!< How many variables are stored in this DataObject + uint32_t data_type_hash; //!< Unique id for this data (eg const_hash32("my special data")) + uint8_t packing_version; //!< Which format was used to pack the data + }; + + //Version 1: Keep first 255 bytes of name and a short hash + struct dop_entry_v1_t { + uint8_t name_length; + uint8_t data_type; + uint16_t name_hash; + uint32_t data_length; + uint8_t name_data[0]; //non-compliant convenience to get to name and data + + size_t GetTotalSize() const { return sizeof(dop_entry_v1_t) + name_length + data_length; } + std::string GetName() { return std::string(reinterpret_cast(name_data), name_length); } + uint8_t * GetDataPtr() { return &name_data[name_length]; } + bool MatchesName(const std::string &name) { + auto my_name = GetName(); + if(my_name == name) return true; + return ((name_length==255)&&(faodel::StringBeginsWith(name, my_name))); //Truncated string + } + }; + + //Version 2: We only send the hash (no name) + struct dop_entry_v2_t { + uint32_t hash; + uint32_t data_length; + uint8_t data_type; + uint8_t pad[3]; //Pad to force alignment. + uint8_t data[0]; //non-compliant convenience to get to name and data + + size_t GetTotalSize() const { return sizeof(dop_entry_v2_t) + data_length; } + uint8_t * GetDataPtr() { return &data[0]; } + }; + + + + bool finalized; //!< Prevent user from appending new variables + uint8_t version; //!< Which packing version is being used + DataObject ldo; //!< The object managed by this Packer. Users call GetDataObject() when done packing + + + std::multimap index; //!< Optional index for locating items + + + size_t getEntryOverhead(); + size_t computeEntrySize(const std::string &name, uint32_t data_bytes) { return ComputeEntrySize(version, name, data_bytes); } + + size_t writeEntry(uint8_t *payload_ptr, + const std::string &name, uint8_t type, + const void *data_ptr, uint32_t data_bytes); + + void rebuildIndexIfNeeded(); + +}; + +}; + +#endif //FAODEL_DATAOBJECTPACKER_HH diff --git a/src/lunasa/common/DataObjectTypeRegistry.cpp b/src/lunasa/common/DataObjectTypeRegistry.cpp index 7e0ff8d..8b0513a 100644 --- a/src/lunasa/common/DataObjectTypeRegistry.cpp +++ b/src/lunasa/common/DataObjectTypeRegistry.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/src/lunasa/common/DataObjectTypeRegistry.hh b/src/lunasa/common/DataObjectTypeRegistry.hh index 8be34f9..107789b 100644 --- a/src/lunasa/common/DataObjectTypeRegistry.hh +++ b/src/lunasa/common/DataObjectTypeRegistry.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_DATAOBJECTTYPEREGISTRY_HH #define FAODEL_DATAOBJECTTYPEREGISTRY_HH diff --git a/src/lunasa/common/GenericRandomDataBundle.hh b/src/lunasa/common/GenericRandomDataBundle.hh index 87263ed..d15ba9a 100644 --- a/src/lunasa/common/GenericRandomDataBundle.hh +++ b/src/lunasa/common/GenericRandomDataBundle.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_GENERICDATABUNDLE_HH #define FAODEL_GENERICDATABUNDLE_HH @@ -16,12 +16,17 @@ struct bundle_offsets_t { uint32_t max_payload_bytes; //How much data can be stored in the LDO's data section uint32_t current_byte_offset; //Where we are in appending to/reading from payload uint32_t current_id; //Which item we're currently at + bool read_only; //Status marker specifying this came from a const ldo - bundle_offsets_t() : max_payload_bytes(0), current_byte_offset(0), current_id(0) { + bundle_offsets_t() : max_payload_bytes(0), current_byte_offset(0), current_id(0), read_only(false) { } bundle_offsets_t(DataObject *ldo) : bundle_offsets_t() { max_payload_bytes = ldo->GetDataSize(); } + bundle_offsets_t(const DataObject *ldo) : bundle_offsets_t() { + max_payload_bytes = ldo->GetDataSize(); + read_only = true; + } }; /** diff --git a/src/lunasa/common/GenericSequentialDataBundle.hh b/src/lunasa/common/GenericSequentialDataBundle.hh index 2a0f094..9e96e88 100644 --- a/src/lunasa/common/GenericSequentialDataBundle.hh +++ b/src/lunasa/common/GenericSequentialDataBundle.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_GENERICSEQUENTIALBUNDLE_HH #define FAODEL_GENERICSEQUENTIALBUNDLE_HH @@ -31,7 +31,7 @@ namespace lunasa { template struct GenericSequentialBundle { uint32_t num_items; - uint32_t pad2; //Reserved for alignme + uint32_t pad2; //Reserved for alignment EXTRA_META_TYPE meta; unsigned char packed_data[0]; diff --git a/src/lunasa/common/Helpers.cpp b/src/lunasa/common/Helpers.cpp new file mode 100644 index 0000000..70838ab --- /dev/null +++ b/src/lunasa/common/Helpers.cpp @@ -0,0 +1,31 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include + +#include "faodel-common/StringHelpers.hh" +#include "lunasa/common/Helpers.hh" +#include "lunasa/DataObject.hh" + +using namespace std; + +namespace lunasa { + +const uint16_t string_object_type_id = faodel::const_hash16("StringObject"); + +DataObject AllocateStringObject(const std::string &s) { + DataObject ldo(s.size()); + ldo.SetTypeID(string_object_type_id); + memcpy(ldo.GetDataPtr(), s.c_str(), s.size()); + return ldo; +} + +std::string UnpackStringObject(DataObject &ldo) { + if(ldo.GetTypeID()!=string_object_type_id) return ""; + return std::string(ldo.GetDataPtr(), ldo.GetDataSize()); +} + + + +} // namespace lunasa diff --git a/src/lunasa/common/Helpers.hh b/src/lunasa/common/Helpers.hh new file mode 100644 index 0000000..2673fe2 --- /dev/null +++ b/src/lunasa/common/Helpers.hh @@ -0,0 +1,20 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef LUNASA_HELPERS_HH +#define LUNASA_HELPERS_HH + +#include +#include + +#include "lunasa/DataObject.hh" + +namespace lunasa { + +DataObject AllocateStringObject(const std::string &s); +std::string UnpackStringObject(DataObject &ldo); + +} // namespace lunasa + +#endif //FAODEL_HELPERS_HH diff --git a/src/lunasa/common/Types.hh b/src/lunasa/common/Types.hh index 25d5431..83d484c 100644 --- a/src/lunasa/common/Types.hh +++ b/src/lunasa/common/Types.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_TYPES_HH #define LUNASA_TYPES_HH diff --git a/src/lunasa/core/LunasaCoreBase.cpp b/src/lunasa/core/LunasaCoreBase.cpp index fc77126..0f42cc5 100644 --- a/src/lunasa/core/LunasaCoreBase.cpp +++ b/src/lunasa/core/LunasaCoreBase.cpp @@ -1,8 +1,7 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -#include #include #include "lunasa/core/LunasaCoreBase.hh" @@ -23,7 +22,7 @@ LunasaCoreBase::~LunasaCoreBase() void LunasaCoreBase::init(const faodel::Configuration &config) { - assert(!configured && "Attempted to Init() LunasaCore multiple times"); + F_ASSERT(!configured,"Attempted to Init() LunasaCore multiple times"); //Use tcmalloc by default, but automatically switch to malloc if no build support #ifdef Faodel_ENABLE_TCMALLOC diff --git a/src/lunasa/core/LunasaCoreBase.hh b/src/lunasa/core/LunasaCoreBase.hh index c661bc3..16f3d3d 100644 --- a/src/lunasa/core/LunasaCoreBase.hh +++ b/src/lunasa/core/LunasaCoreBase.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_LUNASACOREBASE_HH #define LUNASA_LUNASACOREBASE_HH diff --git a/src/lunasa/core/LunasaCoreSplit.cpp b/src/lunasa/core/LunasaCoreSplit.cpp index 10b8f7a..e961569 100644 --- a/src/lunasa/core/LunasaCoreSplit.cpp +++ b/src/lunasa/core/LunasaCoreSplit.cpp @@ -1,8 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. - -#include +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "lunasa/Lunasa.hh" #include "lunasa/allocators/Allocators.hh" @@ -35,6 +33,7 @@ void LunasaCoreSplit::init(string lmm_name, string emm_name, bool use_whookie, const faodel::Configuration &config) { dbg("LunasaCoreSplit init. Lazy: "+lmm_name+" Eager: "+emm_name); + //May cause throws! AllocatorBase *new_lazy_allocator = createAllocator(config, lmm_name, false); AllocatorBase *new_eager_allocator = createAllocator(config, emm_name, true); diff --git a/src/lunasa/core/LunasaCoreSplit.hh b/src/lunasa/core/LunasaCoreSplit.hh index 4564f1e..e9aa352 100644 --- a/src/lunasa/core/LunasaCoreSplit.hh +++ b/src/lunasa/core/LunasaCoreSplit.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_LUNASACORESPLIT_HH #define LUNASA_LUNASACORESPLIT_HH diff --git a/src/lunasa/core/LunasaCoreUnconfigured.cpp b/src/lunasa/core/LunasaCoreUnconfigured.cpp index d10a879..fc18aaa 100644 --- a/src/lunasa/core/LunasaCoreUnconfigured.cpp +++ b/src/lunasa/core/LunasaCoreUnconfigured.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "lunasa/core/LunasaCoreUnconfigured.hh" diff --git a/src/lunasa/core/LunasaCoreUnconfigured.hh b/src/lunasa/core/LunasaCoreUnconfigured.hh index 9a48258..0e64d4e 100644 --- a/src/lunasa/core/LunasaCoreUnconfigured.hh +++ b/src/lunasa/core/LunasaCoreUnconfigured.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_LUNASACOREUNCONFIGURED_HH #define LUNASA_LUNASACOREUNCONFIGURED_HH diff --git a/src/lunasa/core/Singleton.cpp b/src/lunasa/core/Singleton.cpp index 58f3c72..83ffaea 100644 --- a/src/lunasa/core/Singleton.cpp +++ b/src/lunasa/core/Singleton.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "lunasa/core/Singleton.hh" diff --git a/src/lunasa/core/Singleton.hh b/src/lunasa/core/Singleton.hh index 1559567..b3674fb 100644 --- a/src/lunasa/core/Singleton.hh +++ b/src/lunasa/core/Singleton.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef LUNASA_SINGLETON_HH #define LUNASA_SINGLETON_HH diff --git a/src/nnti/CMakeLists.txt b/src/nnti/CMakeLists.txt index 7087791..f1fa963 100644 --- a/src/nnti/CMakeLists.txt +++ b/src/nnti/CMakeLists.txt @@ -111,22 +111,32 @@ include(nntiProbeNetwork.cmake) include(nntiProbeThreads.cmake) include(nntiProbeTimers.cmake) include(nntiProbeXDR.cmake) -include(nntiProbeCereal.cmake) -if( Faodel_ENABLE_CEREAL ) - set( NNTI_USE_XDR 0 CACHE INTERNAL "Use XDR for NNTI serialization" ) - set( NNTI_USE_CEREAL ${NNTI_HAVE_CEREAL} CACHE INTERNAL "Use Cereal for NNTI serialization" ) +string( TOLOWER "${Faodel_NNTI_SERIALIZATION_METHOD}" lower_serializer ) +if( lower_serializer STREQUAL "xdr" AND NNTI_HAVE_XDR ) + ## if a working XDR is found, use it + set( NNTI_USE_XDR 1 ) + set( NNTI_USE_CEREAL 0 ) +elseif( lower_serializer STREQUAL "cereal" AND TARGET tpl-cereal ) + ## if the user asked for Cereal and the builtin tpl-cereal is working, use it + set( NNTI_USE_XDR 0 ) + set( NNTI_USE_CEREAL 1 ) +elseif( TARGET tpl-cereal ) + ## if the above fails and the builtin tpl-cereal is working, use it + message( STATUS "NNTI Serializer: Falling back to builtin Cereal TPL" ) + set( NNTI_USE_XDR 0 ) + set( NNTI_USE_CEREAL 1 ) else() - set( NNTI_USE_XDR ${NNTI_HAVE_XDR} CACHE INTERNAL "Use XDR for NNTI serialization" ) - if( NOT NNTI_HAVE_XDR ) - set( NNTI_USE_CEREAL ${NNTI_HAVE_CEREAL} CACHE INTERNAL "Use Cereal for NNTI serialization" ) + ## if nothing works, throw an error + SET( HAVE_CEREAL_TARGET "False" ) + if( TARGET tpl-cereal ) + SET( HAVE_CEREAL_TARGET "True" ) endif() + message( FATAL_ERROR "Couldn't figure out a serializer for NNTI: Faodel_NNTI_SERIALIZATION_METHOD=${Faodel_NNTI_SERIALIZATION_METHOD} ; NNTI_HAVE_XDR=${NNTI_HAVE_XDR} ; TARGET Faodel::tpl-cereal exists? ${HAVE_CEREAL_TARGET}" ) endif() -if( NOT NNTI_USE_XDR AND NOT NNTI_USE_CEREAL ) - message( FATAL_ERROR "Couldn't find a serailization library for NNTI." ) -endif() - +set( NNTI_USE_XDR ${NNTI_USE_XDR} CACHE INTERNAL "Use XDR for NNTI serialization" ) +set( NNTI_USE_CEREAL ${NNTI_USE_CEREAL} CACHE INTERNAL "Use Cereal for NNTI serialization" ) ## What to build ############################################################## @@ -233,7 +243,9 @@ set(SOURCES nnti_serialize.cpp nnti_threads.cpp nnti_transport.cpp + nnti_types.c nnti_url.cpp + nnti_util.cpp nnti_wid.cpp nnti_wr.cpp transports/base/base_transport.cpp @@ -291,7 +303,6 @@ if( Faodel_ENABLE_MPI_SUPPORT ) LIST( APPEND NNTI_imports MPI::MPI_CXX ) endif() LIST( APPEND NNTI_imports ${NNTI_TRANSPORT_TARGETS} ) -LIST( APPEND NNTI_imports cereal ) set_source_files_properties(transport_factory.cpp PROPERTIES COMPILE_FLAGS "-O0 -g") diff --git a/src/nnti/nnti.cpp b/src/nnti/nnti.cpp index b9337d0..6427b9f 100644 --- a/src/nnti/nnti.cpp +++ b/src/nnti/nnti.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti.cpp @@ -61,7 +61,6 @@ NNTI_result_t NNTI_init( NNTI_transport_t *trans_hdl) { NNTI_result_t rc = NNTI_OK; - const char *url = (my_url==nullptr) ? "" : my_url; setup_logging(); @@ -80,8 +79,6 @@ NNTI_result_t NNTI_initialized( const NNTI_transport_id_t trans_id, int *is_init) { - NNTI_result_t rc = NNTI_OK; - *is_init = nnti::transports::factory::exists(trans_id); if (*is_init) { nnti::transports::transport *transport = nnti::transports::factory::get_instance(trans_id); @@ -746,8 +743,6 @@ NNTI_result_t NNTI_cancelall( const uint32_t wid_count) { NNTI_result_t rc=NNTI_OK; - NNTI_transport_id_t id=NNTI_TRANSPORT_NULL; - uint32_t i=0; nnti::transports::transport *t=nullptr; @@ -834,8 +829,6 @@ NNTI_result_t NNTI_waitany( NNTI_status_t *status) { NNTI_result_t rc=NNTI_OK; - NNTI_transport_id_t id=NNTI_TRANSPORT_NULL; - uint32_t i=0; nnti::transports::transport *t=nullptr; @@ -878,8 +871,6 @@ NNTI_result_t NNTI_waitall( NNTI_status_t *status) { NNTI_result_t rc=NNTI_OK; - NNTI_transport_id_t id=NNTI_TRANSPORT_NULL; - uint32_t i=0; nnti::transports::transport *t=nullptr; diff --git a/src/nnti/nnti.h b/src/nnti/nnti.h index e111055..795838b 100644 --- a/src/nnti/nnti.h +++ b/src/nnti/nnti.h @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti.h @@ -68,6 +68,7 @@ NNTI_result_t NNTI_get_url( /** * @brief Get the process ID of this process. * + * \param[in] trans_hdl A handle to an activated transport. * \param[out] pid the process ID of this process * \return A result code (NNTI_OK or an error) * @@ -79,6 +80,7 @@ NNTI_result_t NNTI_get_pid( /** * @brief Get attributes of the transport. * + * \param[in] trans_hdl A handle to an activated transport. * \param[out] attrs the current attributes * \return A result code (NNTI_OK or an error) * @@ -116,10 +118,12 @@ NNTI_result_t NNTI_disconnect( /** * @brief Create an event queue. * - * \param[in] trans_hdl A handle to an activated transport. - * \param[in] size The number of events the queue can hold. - * \param[in] flags Control the behavior of the queue. - * \param[out] eq The new event queue. + * \param[in] trans_hdl A handle to an activated transport. + * \param[in] size The number of events the queue can hold. + * \param[in] flags Control the behavior of the queue. + * \param[in] cb A callback that gets called for events delivered to eq. + * \param[in] cb_context A blob of data that is passed to each invocation of cb. + * \param[out] eq The new event queue. * \return A result code (NNTI_OK or an error) */ NNTI_result_t NNTI_eq_create( @@ -145,6 +149,7 @@ NNTI_result_t NNTI_eq_destroy( * \param[in] eq_list A list of event queues to wait on. * \param[in] eq_count The number of event queues in the list. * \param[in] timeout The amount of time (in milliseconds) to wait. + * \param[out] which The index of the eq that received an event. * \param[out] event The details of the event. * \return A result code (NNTI_OK or an error) */ @@ -160,7 +165,7 @@ NNTI_result_t NNTI_eq_wait( * * \param[in] dst_hdl Buffer where the message is delivered. * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t NNTI_next_unexpected( @@ -171,10 +176,10 @@ NNTI_result_t NNTI_next_unexpected( /** * @brief Retrieves a specific message from the unexpected list. * - * \param[in] unexpect_event Event describing the message to retrieve. - * \param[in] dst_hdl Buffer where the message is delivered. - * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[in] unexpected_event Event describing the message to retrieve. + * \param[in] dst_hdl Buffer where the message is delivered. + * \param[in] dst_offset Offset into dst_hdl where the message is delivered. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t NNTI_get_unexpected( diff --git a/src/nnti/nntiConfig.h.in b/src/nnti/nntiConfig.h.in index bf27cb7..1023b71 100644 --- a/src/nnti/nntiConfig.h.in +++ b/src/nnti/nntiConfig.h.in @@ -113,6 +113,8 @@ /* ************************************************ * XDR configuration * ************************************************ */ +#cmakedefine NNTI_HAVE_RPC_TYPES_H 1 +#cmakedefine NNTI_HAVE_RPC_XDR_H 1 #cmakedefine NNTI_HAVE_XDR_U_INT8_T 1 #cmakedefine NNTI_HAVE_XDR_U_INT16_T 1 #cmakedefine NNTI_HAVE_XDR_U_INT32_T 1 diff --git a/src/nnti/nntiProbeXDR.cmake b/src/nnti/nntiProbeXDR.cmake index 56b0447..40d4940 100644 --- a/src/nnti/nntiProbeXDR.cmake +++ b/src/nnti/nntiProbeXDR.cmake @@ -17,6 +17,13 @@ check_c_source_compiles( SET( NNTI_HAVE_XDR 1 ) +if( NOT NNTI_HAVE_RPC_TYPES_H OR NOT NNTI_HAVE_RPC_XDR_H ) + SET( NNTI_HAVE_XDR 0 ) +endif() + if( NOT NNTI_HAVE_XDR_SIZEOF ) SET( NNTI_HAVE_XDR 0 ) endif() + +SET( NNTI_HAVE_XDR ${NNTI_HAVE_XDR} CACHE BOOL "Does XDR exist on this system?" ) + diff --git a/src/nnti/nnti_buffer.cpp b/src/nnti/nnti_buffer.cpp index fd4774c..6e625e9 100644 --- a/src/nnti/nnti_buffer.cpp +++ b/src/nnti/nnti_buffer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_buffer.cpp diff --git a/src/nnti/nnti_buffer.hpp b/src/nnti/nnti_buffer.hpp index 8a527cf..f9fc4c3 100644 --- a/src/nnti/nnti_buffer.hpp +++ b/src/nnti/nnti_buffer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_BUFFER_HPP_ @@ -75,9 +75,9 @@ class nnti_buffer nnti_buffer() : nnti_datatype(NNTI_dt_buffer), free_in_dtor_(false), - payload_(nullptr), - payload_size_(0), flags_(NNTI_BF_UNSET), + payload_size_(0), + payload_(nullptr), next_read_offset_(0), next_write_offset_(0), eq_(0), @@ -93,9 +93,9 @@ class nnti_buffer : nnti_datatype(transport, NNTI_dt_buffer), free_in_dtor_(false), - payload_(nullptr), - payload_size_(0), flags_(NNTI_BF_UNSET), + payload_size_(0), + payload_(nullptr), next_read_offset_(0), next_write_offset_(0), eq_(0), @@ -116,9 +116,9 @@ class nnti_buffer : nnti_datatype(transport, NNTI_dt_buffer), free_in_dtor_(true), - payload_(nullptr), - payload_size_(size), flags_(flags), + payload_size_(size), + payload_(nullptr), next_read_offset_(0), next_write_offset_(0), eq_(eq), @@ -142,9 +142,9 @@ class nnti_buffer : nnti_datatype(transport, NNTI_dt_buffer), free_in_dtor_(false), - payload_(buffer), - payload_size_(size), flags_(flags), + payload_size_(size), + payload_(buffer), next_read_offset_(0), next_write_offset_(0), eq_(eq), @@ -162,9 +162,9 @@ class nnti_buffer : nnti_datatype(transport, NNTI_dt_buffer), free_in_dtor_(false), - payload_(nullptr), - payload_size_(0), flags_(NNTI_BF_UNSET), + payload_size_(0), + payload_(nullptr), next_read_offset_(0), next_write_offset_(0), eq_(0), @@ -257,11 +257,9 @@ class nnti_buffer char *packed_buf, const uint64_t packed_buflen) { - NNTI_result_t rc; - packed_size_ = packed_buflen; memcpy(&packed_[0], &packed_buf[0], packed_buflen); - rc = nnti::serialize::unpack_buffer(&packable_, &packed_buf[0], packed_buflen); + nnti::serialize::unpack_buffer(&packable_, &packed_buf[0], packed_buflen); flags_ = (NNTI_buffer_flags_t)packable_.flags; return NNTI_OK; } @@ -353,21 +351,21 @@ class nnti_buffer static inline nnti_buffer* to_obj( - NNTI_buffer_t buffer_hdl) + const NNTI_buffer_t buffer_hdl) { return (nnti_buffer *)buffer_hdl; } static inline NNTI_buffer_t to_hdl( - nnti_buffer *buffer) + const nnti_buffer *buffer) { return (NNTI_buffer_t)buffer; } - std::string - toString() override { + virtual std::string + toString() const override { std::stringstream out; out << "nnti_buffer.to_hdl(this)=" << to_hdl(this) << " | nnti_buffer.id_=" << this->id_ diff --git a/src/nnti/nnti_callback.cpp b/src/nnti/nnti_callback.cpp index c86f3c9..ad01ec3 100644 --- a/src/nnti/nnti_callback.cpp +++ b/src/nnti/nnti_callback.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -27,33 +27,30 @@ class default_event_callback { }; nnti_event_callback::nnti_event_callback() -: cb_(default_event_callback()), - nnti_datatype(NNTI_dt_callback) +: nnti_datatype(NNTI_dt_callback), + cb_(default_event_callback()) { return; } nnti_event_callback::nnti_event_callback( nnti::datatype::nnti_event_callback &other) -: cb_(other.cb_), - nnti_datatype(other.transport_, - other.datatype_) +: nnti_datatype(other.transport_, other.datatype_), + cb_(other.cb_) { return; } nnti_event_callback::nnti_event_callback( nnti::transports::transport *transport) -: cb_(default_event_callback()), - nnti_datatype(transport, - NNTI_dt_callback) +: nnti_datatype(transport, NNTI_dt_callback), + cb_(default_event_callback()) { return; } nnti_event_callback::nnti_event_callback( nnti::transports::transport *transport, NNTI_event_callback_t cb) -: cb_(cb), - nnti_datatype(transport, - NNTI_dt_callback) +: nnti_datatype(transport, NNTI_dt_callback), + cb_(cb) { if (!cb_) { cb_ = default_event_callback(); @@ -63,9 +60,8 @@ nnti_event_callback::nnti_event_callback( nnti_event_callback::nnti_event_callback( nnti::transports::transport *transport, std::function< NNTI_result_t(NNTI_event_t*,void*) > cb) -: cb_(cb), - nnti_datatype(transport, - NNTI_dt_callback) +: nnti_datatype(transport, NNTI_dt_callback), + cb_(cb) { if (!cb_) { cb_ = default_event_callback(); @@ -91,7 +87,7 @@ nnti_event_callback::invoke( } std::string -nnti_event_callback::toString() { +nnti_event_callback::toString() const { std::stringstream out; out << "cb_==" << &cb_; return out.str(); diff --git a/src/nnti/nnti_callback.hpp b/src/nnti/nnti_callback.hpp index 9245729..9c21a0a 100644 --- a/src/nnti/nnti_callback.hpp +++ b/src/nnti/nnti_callback.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_callback.hpp @@ -51,8 +51,8 @@ class nnti_event_callback NNTI_result_t invoke(NNTI_event_t *event, void *context) const; - std::string - toString() override; + virtual std::string + toString() const override; }; diff --git a/src/nnti/nnti_connection.cpp b/src/nnti/nnti_connection.cpp index 0408887..a6d70d4 100644 --- a/src/nnti/nnti_connection.cpp +++ b/src/nnti/nnti_connection.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_connection.cpp diff --git a/src/nnti/nnti_connection.hpp b/src/nnti/nnti_connection.hpp index 012e7e8..1ebfa2a 100644 --- a/src/nnti/nnti_connection.hpp +++ b/src/nnti/nnti_connection.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_connection.hpp @@ -43,6 +43,7 @@ class nnti_connection { uint32_t id_; NNTI_process_id_t peer_pid_; nnti::datatype::nnti_peer *peer_; + std::string fingerprint_; public: uint32_t index; @@ -79,7 +80,7 @@ class nnti_connection { { return id_; } - virtual const NNTI_process_id_t + virtual NNTI_process_id_t peer_pid() { return peer_pid_; @@ -96,6 +97,11 @@ class nnti_connection { { return peer_; } + virtual std::string + fingerprint() const + { + return fingerprint_; + } /* * generate a string that can be added into a URL query string */ @@ -118,7 +124,11 @@ class nnti_connection { toString() { std::stringstream out; - out << "conn=" << (void*)this << " ; id_=" << id_ << " ; peer_pid_=" << peer_pid_ << " ; peer=" << (void*)peer(); + out << "conn=" << (void*)this + << " ; id_=" << id_ + << " ; peer_pid_=" << peer_pid_ + << " ; peer=" << (void*)peer() + << " ; fingerprint=" << fingerprint(); return out.str(); } }; diff --git a/src/nnti/nnti_datatype.hpp b/src/nnti/nnti_datatype.hpp index 4b2c9e6..f91f0db 100644 --- a/src/nnti/nnti_datatype.hpp +++ b/src/nnti/nnti_datatype.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_datatype.hpp @@ -69,7 +69,7 @@ class nnti_datatype { } virtual std::string - toString() + toString() const { std::stringstream out; out << "datatype_==" << datatype_; diff --git a/src/nnti/nnti_eq.hpp b/src/nnti/nnti_eq.hpp index 47c7c09..978ffd8 100644 --- a/src/nnti/nnti_eq.hpp +++ b/src/nnti/nnti_eq.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_eq.hpp @@ -57,7 +57,7 @@ class nnti_event_queue class simple_reservation_manager : public reservation_manager { private: - const uint64_t max_reservations_; + const int64_t max_reservations_; std::atomic outstanding_reservations_; public: simple_reservation_manager( @@ -105,8 +105,7 @@ class nnti_event_queue } bool return_reservation( - nnti::datatype::reservation &r) - { + nnti::datatype::reservation &r) override { return true; } }; @@ -161,10 +160,10 @@ class nnti_event_queue nnti_event_queue( bool require_reservation, uint64_t size) - : require_reservation_(require_reservation), + : nnti_datatype(NNTI_dt_event_queue), + require_reservation_(require_reservation), q_size_(size), - q_(size), - nnti_datatype(NNTI_dt_event_queue) + q_(size) { if (require_reservation) { reservation_manager_ = new simple_reservation_manager(q_size_); @@ -180,13 +179,13 @@ class nnti_event_queue bool require_reservation, uint64_t size, nnti::transports::transport *transport) - : require_reservation_(require_reservation), + : nnti_datatype(transport, + NNTI_dt_event_queue), + require_reservation_(require_reservation), q_size_(size), q_(size), cb_(transport, default_eq_callback()), - cb_context_(nullptr), - nnti_datatype(transport, - NNTI_dt_event_queue) + cb_context_(nullptr) { if (require_reservation) { reservation_manager_ = new simple_reservation_manager(q_size_); @@ -204,13 +203,13 @@ class nnti_event_queue nnti::datatype::nnti_event_callback cb, void *cb_context, nnti::transports::transport *transport) - : require_reservation_(require_reservation), + : nnti_datatype(transport, + NNTI_dt_event_queue), + require_reservation_(require_reservation), q_size_(size), q_(size), cb_(cb), - cb_context_(cb_context), - nnti_datatype(transport, - NNTI_dt_event_queue) + cb_context_(cb_context) { if (require_reservation) { reservation_manager_ = new simple_reservation_manager(q_size_); diff --git a/src/nnti/nnti_freelist.hpp b/src/nnti/nnti_freelist.hpp index f245bea..db29d98 100644 --- a/src/nnti/nnti_freelist.hpp +++ b/src/nnti/nnti_freelist.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_freelist.hpp diff --git a/src/nnti/nnti_logger.cpp b/src/nnti/nnti_logger.cpp index edbc826..516c838 100644 --- a/src/nnti/nnti_logger.cpp +++ b/src/nnti/nnti_logger.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_logger.cpp diff --git a/src/nnti/nnti_logger.h b/src/nnti/nnti_logger.h index cbe976c..c9f64cd 100644 --- a/src/nnti/nnti_logger.h +++ b/src/nnti/nnti_logger.h @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_logger.h diff --git a/src/nnti/nnti_logger.hpp b/src/nnti/nnti_logger.hpp index a94f649..dd1ba21 100644 --- a/src/nnti/nnti_logger.hpp +++ b/src/nnti/nnti_logger.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_logger.hpp diff --git a/src/nnti/nnti_op.cpp b/src/nnti/nnti_op.cpp index d96cd39..c0336a8 100644 --- a/src/nnti/nnti_op.cpp +++ b/src/nnti/nnti_op.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_op.cpp diff --git a/src/nnti/nnti_op.hpp b/src/nnti/nnti_op.hpp index 0e47ca9..2d1a27b 100644 --- a/src/nnti/nnti_op.hpp +++ b/src/nnti/nnti_op.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_OP_HPP_ diff --git a/src/nnti/nnti_pch.hpp b/src/nnti/nnti_pch.hpp index 9e87296..1d13b6a 100644 --- a/src/nnti/nnti_pch.hpp +++ b/src/nnti/nnti_pch.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_PCH_HPP_ diff --git a/src/nnti/nnti_peer.hpp b/src/nnti/nnti_peer.hpp index 176d6d1..1e00ded 100644 --- a/src/nnti/nnti_peer.hpp +++ b/src/nnti/nnti_peer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_peer.hpp diff --git a/src/nnti/nnti_pid.cpp b/src/nnti/nnti_pid.cpp index 201f64d..e051fef 100644 --- a/src/nnti/nnti_pid.cpp +++ b/src/nnti/nnti_pid.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" diff --git a/src/nnti/nnti_pid.hpp b/src/nnti/nnti_pid.hpp index ef54af7..21c6c9f 100644 --- a/src/nnti/nnti_pid.hpp +++ b/src/nnti/nnti_pid.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_PID_HPP diff --git a/src/nnti/nnti_serialize.cpp b/src/nnti/nnti_serialize.cpp index 1b34a15..e5e49ff 100644 --- a/src/nnti/nnti_serialize.cpp +++ b/src/nnti/nnti_serialize.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -101,7 +101,6 @@ unpack_peer( #if NNTI_USE_XDR == 1 XDR decode_xdrs; xdrproc_t decode_fn = (xdrproc_t)&xdr_NNTI_peer_p_t; - NNTI_datatype_t *dt = (NNTI_datatype_t*)packed_buf; uint64_t dt_size = sizeof(NNTI_peer_p_t); uint64_t bytes_left = packed_buflen; @@ -210,7 +209,6 @@ unpack_buffer( #if NNTI_USE_XDR == 1 XDR decode_xdrs; xdrproc_t decode_fn = (xdrproc_t)&xdr_NNTI_buffer_p_t; - NNTI_datatype_t *dt = (NNTI_datatype_t*)packed_buf; uint64_t dt_size = sizeof(NNTI_buffer_p_t); uint64_t bytes_left = packed_buflen; diff --git a/src/nnti/nnti_serialize.hpp b/src/nnti/nnti_serialize.hpp index 2222dca..98bfd0b 100644 --- a/src/nnti/nnti_serialize.hpp +++ b/src/nnti/nnti_serialize.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_SERIALIZE_HPP #define NNTI_SERIALIZE_HPP diff --git a/src/nnti/nnti_state_machine.hpp b/src/nnti/nnti_state_machine.hpp index 7ecb89d..fff6066 100644 --- a/src/nnti/nnti_state_machine.hpp +++ b/src/nnti/nnti_state_machine.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_STATE_MACHINE_HPP_ diff --git a/src/nnti/nnti_threads.cpp b/src/nnti/nnti_threads.cpp index b2ac383..b3e928b 100644 --- a/src/nnti/nnti_threads.cpp +++ b/src/nnti/nnti_threads.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_threads.cpp diff --git a/src/nnti/nnti_threads.h b/src/nnti/nnti_threads.h index cb636d0..12f56a4 100644 --- a/src/nnti/nnti_threads.h +++ b/src/nnti/nnti_threads.h @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_threads.h diff --git a/src/nnti/nnti_threads_types.h b/src/nnti/nnti_threads_types.h index 5cd0ce9..93f93ae 100644 --- a/src/nnti/nnti_threads_types.h +++ b/src/nnti/nnti_threads_types.h @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_threads_types.h diff --git a/src/nnti/nnti_transport.cpp b/src/nnti/nnti_transport.cpp index e48f91e..2b50a1e 100644 --- a/src/nnti/nnti_transport.cpp +++ b/src/nnti/nnti_transport.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_transport.hpp" diff --git a/src/nnti/nnti_transport.hpp b/src/nnti/nnti_transport.hpp index 11cfde0..e5b2e9e 100644 --- a/src/nnti/nnti_transport.hpp +++ b/src/nnti/nnti_transport.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_transport.hpp @@ -185,9 +185,9 @@ class transport { /** * @brief Retrieves the next message from the unexpected list. * - * \param[in] dst_hdl Buffer where the message is delivered. - * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[in] dst_hdl Buffer where the message is delivered. + * \param[in] dst_offset Offset into dst_hdl where the message is delivered. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ virtual NNTI_result_t @@ -202,7 +202,7 @@ class transport { * \param[in] unexpected_event Event describing the message to retrieve. * \param[in] dst_hdl Buffer where the message is delivered. * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ virtual NNTI_result_t diff --git a/src/nnti/nnti_types.c b/src/nnti/nnti_types.c new file mode 100644 index 0000000..adff21d --- /dev/null +++ b/src/nnti/nnti_types.c @@ -0,0 +1,32 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + + +#include + +/********** Initializers **********/ +//Note: these pragmas are here because C++ people can't cope with human readable c99 structures +//#pragma GCC diagnostic push +//#pragma GCC diagnostic ignored "-Wpedantic" +/** + * @brief A simple initializer to zero-out a work request. + */ +struct NNTI_work_request_t NNTI_WR_INITIALIZER = +{ .op = NNTI_OP_NOOP, + .flags = NNTI_OF_UNSET, + .trans_hdl = 0, + .peer = 0, + .local_hdl = 0, + .local_offset = 0, + .remote_hdl = 0, + .remote_offset = 0, + .length = 0, + .operand1 = 0, + .operand2 = 0, + .alt_eq = 0, + .callback = 0, + .cb_context = 0, + .event_context = 0 +}; +//#pragma GCC diagnostic pop diff --git a/src/nnti/nnti_types.h b/src/nnti/nnti_types.h index 08533fa..0930649 100644 --- a/src/nnti/nnti_types.h +++ b/src/nnti/nnti_types.h @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_TYPES_H_ @@ -96,6 +96,9 @@ enum NNTI_result_t { /** @brief The size of the message is larger than the supported maximum size. */ NNTI_EMSGSIZE, + /** @brief The buffer does not meet the alignment restrictions of the interconnect. */ + NNTI_EALIGN, + /** @brief The operation or process has been canceled. */ NNTI_ECANCELED, @@ -401,31 +404,6 @@ struct NNTI_work_request_t { }; typedef struct NNTI_work_request_t NNTI_work_request_t; - -/********** Initializers **********/ -//Note: these pragmas are here because C++ people can't cope with human readable c99 structures -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wpedantic" -/** - * @brief A simple initializer to zero-out a work request. - */ -static struct NNTI_work_request_t NNTI_WR_INITIALIZER = -{ .op = NNTI_OP_NOOP, - .flags = NNTI_OF_UNSET, - .trans_hdl = 0, - .peer = 0, - .local_hdl = 0, - .local_offset = 0, - .remote_hdl = 0, - .remote_offset = 0, - .length = 0, - .operand1 = 0, - .operand2 = 0, - .alt_eq = 0, - .callback = 0, - .cb_context = 0, - .event_context = 0 -}; -#pragma GCC diagnostic pop +extern struct NNTI_work_request_t NNTI_WR_INITIALIZER; #endif /* NNTI_TYPES_H_ */ diff --git a/src/nnti/nnti_url.cpp b/src/nnti/nnti_url.cpp index 018ad42..2d8c4db 100644 --- a/src/nnti/nnti_url.cpp +++ b/src/nnti/nnti_url.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * @file: nnti_url.cpp @@ -80,7 +80,6 @@ void nnti_url::hostname2addr(void) { struct hostent *host_entry; - struct sockaddr_in skin; std::lock_guard lock(hostent_mutex_); host_entry = gethostbyname(hostname_.c_str()); @@ -110,9 +109,9 @@ nnti_url::nnti_url( nnti_url::nnti_url( const NNTI_process_id_t pid) -: pid_(pid), - hostname_(""), - port_("") +: hostname_(""), + port_(""), + pid_(pid) { url_ = nnti::datatype::nnti_pid::to_url(pid); @@ -137,9 +136,7 @@ nnti_url::nnti_url( nnti_url::nnti_url( const std::string &hostname, const NNTI_tcp_port port) -: nnti_url( - hostname, - std::to_string(port)) +: nnti_url(hostname, std::to_string(port)) { pid_ = nnti::datatype::nnti_pid::to_pid(*this); } diff --git a/src/nnti/nnti_url.hpp b/src/nnti/nnti_url.hpp index 3fc6e08..44a4f63 100644 --- a/src/nnti/nnti_url.hpp +++ b/src/nnti/nnti_url.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * @file: nnti_url.hpp @@ -59,7 +59,6 @@ class nnti_url { const NNTI_process_id_t pid); nnti_url( - const std::string &hostname, const std::string &port); diff --git a/src/nnti/nnti_util.cpp b/src/nnti/nnti_util.cpp new file mode 100644 index 0000000..092c402 --- /dev/null +++ b/src/nnti/nnti_util.cpp @@ -0,0 +1,76 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +/* + * @file: nnti_util.hpp + * + * Created on: Jul 13, 2015 + * Author: thkorde + */ + +#include "nnti/nntiConfig.h" + +#include "nnti/nnti_util.hpp" + +uint32_t +nnti::util::str2uint32(const std::string &s) +{ + uint32_t i = 0; + + try { + i = boost::lexical_cast(s); + } + catch (boost::bad_lexical_cast &) { + // i is already 0. do nothing. + // i = 0; + } + + return i; +} +uint64_t +nnti::util::str2uint64(const std::string &s) +{ + uint64_t i = 0; + + try { + i = boost::lexical_cast(s); + } + catch (boost::bad_lexical_cast &) { + // i is already 0. do nothing. + // i = 0; + } + + return i; +} + +int32_t +nnti::util::str2int32(const std::string &s) +{ + int32_t i = 0; + + try { + i = boost::lexical_cast(s); + } + catch (boost::bad_lexical_cast &) { + // i is already 0. do nothing. + // i = 0; + } + + return i; +} +int64_t +nnti::util::str2int64(const std::string &s) +{ + int64_t i = 0; + + try { + i = boost::lexical_cast(s); + } + catch (boost::bad_lexical_cast &) { + // i is already 0. do nothing. + // i = 0; + } + + return i; +} diff --git a/src/nnti/nnti_util.hpp b/src/nnti/nnti_util.hpp index 0838fed..a994098 100644 --- a/src/nnti/nnti_util.hpp +++ b/src/nnti/nnti_util.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * @file: nnti_util.hpp @@ -48,69 +48,17 @@ namespace nnti { namespace util { -static uint32_t -str2uint32(const std::string &s) -{ - uint32_t i = 0; - - try { - i = boost::lexical_cast(s); - } - catch (boost::bad_lexical_cast &) { - // i is already 0. do nothing. - // i = 0; - } - - return i; -} -static uint64_t -str2uint64(const std::string &s) -{ - uint64_t i = 0; - - try { - i = boost::lexical_cast(s); - } - catch (boost::bad_lexical_cast &) { - // i is already 0. do nothing. - // i = 0; - } - - return i; -} +uint32_t +str2uint32(const std::string &s); +uint64_t +str2uint64(const std::string &s); -static int32_t -str2int32(const std::string &s) -{ - int32_t i = 0; - - try { - i = boost::lexical_cast(s); - } - catch (boost::bad_lexical_cast &) { - // i is already 0. do nothing. - // i = 0; - } - - return i; -} -static int64_t -str2int64(const std::string &s) -{ - int64_t i = 0; - - try { - i = boost::lexical_cast(s); - } - catch (boost::bad_lexical_cast &) { - // i is already 0. do nothing. - // i = 0; - } - - return i; -} +int32_t +str2int32(const std::string &s); +int64_t +str2int64(const std::string &s); -static inline int +inline int sleep( const uint64_t msec) { @@ -135,7 +83,7 @@ sleep( } /* Thomas Wang's 64 bit to 32 bit Hash Function (http://www.concentric.net/~ttwang/tech/inthash.htm) */ -static inline uint32_t +inline uint32_t hash6432shift( uint64_t key) { @@ -151,7 +99,9 @@ hash6432shift( /* * Convert a 64 bit value from big endian to host byte order. */ -static inline uint64_t betoh64(uint64_t val) +inline uint64_t +betoh64( + uint64_t val) { #if __BYTE_ORDER == __BIG_ENDIAN return val; diff --git a/src/nnti/nnti_vector.hpp b/src/nnti/nnti_vector.hpp index ce5b871..ca56157 100644 --- a/src/nnti/nnti_vector.hpp +++ b/src/nnti/nnti_vector.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_VECTOR_HPP_ diff --git a/src/nnti/nnti_wid.cpp b/src/nnti/nnti_wid.cpp index 795e127..2634054 100644 --- a/src/nnti/nnti_wid.cpp +++ b/src/nnti/nnti_wid.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_wid.cpp @@ -108,7 +108,7 @@ nnti_work_id::unlock() } std::string -nnti_work_id::toString(void) { +nnti_work_id::toString(void) const { std::stringstream out; out << "id_==" << id_; return out.str(); diff --git a/src/nnti/nnti_wid.hpp b/src/nnti/nnti_wid.hpp index c50ee34..6a0cdc4 100644 --- a/src/nnti/nnti_wid.hpp +++ b/src/nnti/nnti_wid.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_wid.hpp @@ -64,8 +64,8 @@ class nnti_work_id virtual int unlock(); - std::string - toString(void) override; + virtual std::string + toString(void) const override; virtual bool is_complete(void) const; }; diff --git a/src/nnti/nnti_wr.cpp b/src/nnti/nnti_wr.cpp index f526919..af1aa03 100644 --- a/src/nnti/nnti_wr.cpp +++ b/src/nnti/nnti_wr.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -20,34 +20,34 @@ namespace datatype { nnti_work_request::nnti_work_request( nnti::transports::transport *transport) - : cb_(transport), - nnti_datatype(transport, - NNTI_dt_work_request) + : nnti_datatype(transport, + NNTI_dt_work_request), + wr_(), + cb_(transport) + { return; } nnti_work_request::nnti_work_request( nnti::transports::transport *transport, NNTI_work_request_t &wr) - : cb_(transport, - wr.callback), - nnti_datatype(transport, - NNTI_dt_work_request) + : nnti_datatype(transport, + NNTI_dt_work_request), + wr_(wr), + cb_(transport, + wr.callback) { - wr_ = wr; - return; } nnti_work_request::nnti_work_request( nnti::transports::transport *transport, NNTI_work_request_t &wr, nnti::datatype::nnti_event_callback cb) - : cb_(cb), - nnti_datatype(transport, - NNTI_dt_work_request) + : nnti_datatype(transport, + NNTI_dt_work_request), + wr_(wr), + cb_(cb) { - wr_ = wr; - return; } @@ -67,7 +67,7 @@ namespace datatype { { return wr_.peer; } - const NNTI_process_id_t + NNTI_process_id_t nnti_work_request::peer_pid(void) const { nnti::datatype::nnti_peer *peer=(nnti::datatype::nnti_peer*)wr_.peer; diff --git a/src/nnti/nnti_wr.hpp b/src/nnti/nnti_wr.hpp index 5b625ed..9b8125c 100644 --- a/src/nnti/nnti_wr.hpp +++ b/src/nnti/nnti_wr.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_WR_HPP @@ -43,7 +43,7 @@ class nnti_work_request virtual NNTI_peer_t peer(void) const; - virtual const NNTI_process_id_t + virtual NNTI_process_id_t peer_pid(void) const; NNTI_op_t @@ -81,7 +81,7 @@ class nnti_work_request event_context(void) const; virtual std::string - toString(void) const; + toString(void) const override; }; } /* namespace datatype */ diff --git a/src/nnti/serializers/cereal/nnti_cereal.hpp b/src/nnti/serializers/cereal/nnti_cereal.hpp index 8951402..27db0dd 100644 --- a/src/nnti/serializers/cereal/nnti_cereal.hpp +++ b/src/nnti/serializers/cereal/nnti_cereal.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/serializers/cereal/nnti_packable.hpp" diff --git a/src/nnti/serializers/cereal/nnti_packable.hpp b/src/nnti/serializers/cereal/nnti_packable.hpp index cb7d620..5aeec65 100644 --- a/src/nnti/serializers/cereal/nnti_packable.hpp +++ b/src/nnti/serializers/cereal/nnti_packable.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef NNTI_PACKABLE_HPP #define NNTI_PACKABLE_HPP @@ -52,7 +52,7 @@ struct NNTI_null_process_p_t { /** * @brief Remote process identifier for IB. * - * The \ref NNTI_ib_process_t identifies a particular process + * The \ref NNTI_ib_process_p_t identifies a particular process * on a particular node. If a connection has been established to the * represented process, then that connection is identified by 'qp_num'. */ @@ -83,7 +83,7 @@ using NNTI_instance_id = uint32_t; /** * @brief Remote process identifier for Gemini. * - * The \ref NNTI_ugni_process_t identifies a particular process + * The \ref NNTI_ugni_process_p_t identifies a particular process * on a particular node. If a connection has been established to the * represented process, then that connection is identified by 'inst_id'. */ @@ -108,7 +108,7 @@ struct NNTI_ugni_process_p_t { /** * @brief Remote process identifier for MPI. * - * The \ref NNTI_mpi_process_t identifies a particular process + * The \ref NNTI_mpi_process_p_t identifies a particular process * on a particular node. */ struct NNTI_mpi_process_p_t { @@ -145,7 +145,7 @@ struct NNTI_local_process_p_t { /** * @brief A structure to represent a remote processes. * - * The NNTI_remote_process_t structure contains the + * The NNTI_remote_process_p_t structure contains the * transport specific info needed to identify a process running * on a remote node. */ diff --git a/src/nnti/serializers/xdr/nnti_xdr.h b/src/nnti/serializers/xdr/nnti_xdr.h index db3495d..9025964 100644 --- a/src/nnti/serializers/xdr/nnti_xdr.h +++ b/src/nnti/serializers/xdr/nnti_xdr.h @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_xdr.h diff --git a/src/nnti/serializers/xdr/nnti_xdr.hpp b/src/nnti/serializers/xdr/nnti_xdr.hpp index 7e81d92..8d1ea0e 100644 --- a/src/nnti/serializers/xdr/nnti_xdr.hpp +++ b/src/nnti/serializers/xdr/nnti_xdr.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/serializers/xdr/nnti_xdr.h" diff --git a/src/nnti/transport_factory.cpp b/src/nnti/transport_factory.cpp index 40ab478..e25f09a 100644 --- a/src/nnti/transport_factory.cpp +++ b/src/nnti/transport_factory.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file transport_factory.cpp @@ -15,6 +15,7 @@ #include "nnti/nnti_pch.hpp" #include +#include #include #include "faodel-common/Configuration.hh" @@ -86,8 +87,8 @@ factory::get_instance( { transport *t = nullptr; NNTI_transport_id_t trans_id = NNTI_DEFAULT_TRANSPORT; - std::string proto; - std::string trans_name; + std::string proto(""); + std::string trans_name(""); std::string id_key("nnti.transport.id"); std::string proto_key("nnti.transport.protocol"); std::string name_key("nnti.transport.name"); @@ -130,6 +131,14 @@ factory::get_instance( t = ibverbs_transport::get_instance(config); #else // ibverbs is not configured. try failing back to MPI. + std::stringstream ss; + ss<< + "------------------------------------------------------------------\n" + "The FAODEL_CONFIG 'net.transport.name' key is set to 'ibverbs'.\n" + "The 'ibverbs' transport was not configured into the Faodel network\n" + "library. The 'mpi' transport will be used instead.\n" + "------------------------------------------------------------------"; + std::cerr< #include #include "faodel-common/Configuration.hh" @@ -38,7 +39,7 @@ namespace transports { * @brief Initialize NNTI to use a specific transport. * * \param[in] trans_id The ID of the transport the client wants to use. - * \param[in] my_url A string that describes the transport parameters. + * \param[in] me An nnti_peer object that references this process. * \return A result code (NNTI_OK or an error) * */ @@ -90,7 +91,11 @@ base_transport::base_transport( me_ = nnti::datatype::nnti_peer(this, url_); + uint64_t now = std::chrono::time_point_cast(std::chrono::high_resolution_clock::now()).time_since_epoch().count(); + fingerprint_ = faodel::hash32(whookie::Server::GetNodeID().GetHex() + std::to_string(now)); + log_debug_stream("base_transport") << "me_ = " << me_.url().url(); + log_debug_stream("base_transport") << "fingerprint_ = " << fingerprint_; return; } diff --git a/src/nnti/transports/base/base_transport.hpp b/src/nnti/transports/base/base_transport.hpp index 982dba8..4d849a4 100644 --- a/src/nnti/transports/base/base_transport.hpp +++ b/src/nnti/transports/base/base_transport.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file base_transport.hpp @@ -36,14 +36,15 @@ class base_transport NNTI_transport_id_t trans_id_; nnti::core::nnti_url url_; nnti::datatype::nnti_peer me_; - faodel::Configuration config_; + uint32_t fingerprint_; + faodel::Configuration config_; public: /** * @brief Initialize NNTI to use a specific transport. * * \param[in] trans_id The ID of the transport the client wants to use. - * \param[in] my_url A string that describes the transport parameters. + * \param[in] me An nnti_peer object that references this process. * \return A result code (NNTI_OK or an error) * */ @@ -55,12 +56,13 @@ class base_transport * @brief Initialize NNTI to use a specific transport. * * \param[in] trans_id The ID of the transport the client wants to use. + * \param[in] config A Configuration object that NNTI should use to configure itself. * \return A result code (NNTI_OK or an error) * */ base_transport( const NNTI_transport_id_t trans_id, - faodel::Configuration &config); + faodel::Configuration &config); /** * @brief Deactivates a specific transport. diff --git a/src/nnti/transports/ibverbs/ibverbs_atomic_op.hpp b/src/nnti/transports/ibverbs/ibverbs_atomic_op.hpp index d54512d..8cd62e6 100644 --- a/src/nnti/transports/ibverbs/ibverbs_atomic_op.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_atomic_op.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef IBVERBS_ATOMIC_OP_HPP_ @@ -35,15 +35,13 @@ namespace core { class ibverbs_atomic_op : public nnti_op { private: - nnti::transports::ibverbs_transport *transport_; struct ibv_sge sge_; struct ibv_send_wr sq_wr_; public: ibverbs_atomic_op( nnti::transports::ibverbs_transport *transport) - : nnti_op(), - transport_(transport) + : nnti_op() { sq_wr_.wr_id = (uint64_t)this; sq_wr_.next = nullptr; @@ -57,8 +55,7 @@ class ibverbs_atomic_op ibverbs_atomic_op( nnti::transports::ibverbs_transport *transport, nnti::datatype::nnti_work_id *wid) - : nnti_op(wid), - transport_(transport) + : nnti_op(wid) { const nnti::datatype::ibverbs_work_request &wr = (const nnti::datatype::ibverbs_work_request &)wid->wr(); diff --git a/src/nnti/transports/ibverbs/ibverbs_buffer.cpp b/src/nnti/transports/ibverbs/ibverbs_buffer.cpp index 0aa47f9..749bcef 100644 --- a/src/nnti/transports/ibverbs/ibverbs_buffer.cpp +++ b/src/nnti/transports/ibverbs/ibverbs_buffer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -35,14 +35,6 @@ ibverbs_buffer::ibverbs_buffer() mr = nullptr; return; } -//ibverbs_buffer::ibverbs_buffer( -// nnti::transports::ibverbs_transport *transport, -// NNTI_buffer_p_t *packable) -//: nnti_buffer(transport, -// packable) -//{ -// return; -//} ibverbs_buffer::ibverbs_buffer( nnti::datatype::ibverbs_buffer &b) : nnti_buffer(b), @@ -229,7 +221,6 @@ ibverbs_buffer::post_receive(void) { NNTI_result_t rc = NNTI_OK; int ibv_rc; - enum ibv_access_flags access; struct ibv_recv_wr rq_wr, *bad_wr; struct ibv_sge sge; diff --git a/src/nnti/transports/ibverbs/ibverbs_buffer.hpp b/src/nnti/transports/ibverbs/ibverbs_buffer.hpp index 63111f6..a18e372 100644 --- a/src/nnti/transports/ibverbs/ibverbs_buffer.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_buffer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef IBVERBS_BUFFER_HPP_ diff --git a/src/nnti/transports/ibverbs/ibverbs_cmd_buffer.cpp b/src/nnti/transports/ibverbs/ibverbs_cmd_buffer.cpp index 64fa409..5141603 100644 --- a/src/nnti/transports/ibverbs/ibverbs_cmd_buffer.cpp +++ b/src/nnti/transports/ibverbs/ibverbs_cmd_buffer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -32,8 +32,6 @@ ibverbs_cmd_buffer::ibverbs_cmd_buffer( cmd_count_(cmd_count), cmd_offset_(0) { - int rc; - setup_command_buffer(); } ibverbs_cmd_buffer::~ibverbs_cmd_buffer() @@ -86,7 +84,7 @@ ibverbs_cmd_buffer::setup_command_buffer(void) cmd_mr_ = ibv_reg_mr(transport_->pd_, cmd_buf_, cmd_size_ * cmd_count_, ibv_flags); // } - for (int i=0;iuse_odp_) { log_debug("ibverbs_buffer", "using ODP - unregister is a no-op"); } else { log_debug("ibverbs_cmd_buffer", "deregistering ibverbs_cmd_buffer (cmd_buf_=%x)", cmd_buf_); - int ibv_rc=ibv_dereg_mr(cmd_mr_); + ibv_dereg_mr(cmd_mr_); } delete[] cmd_buf_; diff --git a/src/nnti/transports/ibverbs/ibverbs_cmd_buffer.hpp b/src/nnti/transports/ibverbs/ibverbs_cmd_buffer.hpp index 8a0c383..91ddf82 100644 --- a/src/nnti/transports/ibverbs/ibverbs_cmd_buffer.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_cmd_buffer.hpp @@ -1,11 +1,11 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** - * @file nnti_cmd_buffer.hpp + * @file ibverbs_cmd_buffer.hpp * - * @brief nnti_cmd_buffer.hpp + * @brief ibverbs_cmd_buffer.hpp * * @author Todd Kordenbrock (thkorde\@sandia.gov). * Created on: Jul 07, 2015 diff --git a/src/nnti/transports/ibverbs/ibverbs_cmd_msg.cpp b/src/nnti/transports/ibverbs/ibverbs_cmd_msg.cpp index 2c6ce02..9154aca 100644 --- a/src/nnti/transports/ibverbs/ibverbs_cmd_msg.cpp +++ b/src/nnti/transports/ibverbs/ibverbs_cmd_msg.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" diff --git a/src/nnti/transports/ibverbs/ibverbs_cmd_msg.hpp b/src/nnti/transports/ibverbs/ibverbs_cmd_msg.hpp index 15dd536..cd2508e 100644 --- a/src/nnti/transports/ibverbs/ibverbs_cmd_msg.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_cmd_msg.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef IBVERBS_CMD_MSG_HPP_ diff --git a/src/nnti/transports/ibverbs/ibverbs_cmd_op.hpp b/src/nnti/transports/ibverbs/ibverbs_cmd_op.hpp index b8757a5..918c8bf 100644 --- a/src/nnti/transports/ibverbs/ibverbs_cmd_op.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_cmd_op.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef IBVERBS_CMD_OP_HPP_ @@ -102,8 +102,8 @@ class ibverbs_cmd_op : nnti_op(wid), transport_(transport), cmd_msg_(transport, id_, wid), - cmd_mr_(nullptr), - is_ack_(false) + is_ack_(false), + cmd_mr_(nullptr) { sq_wr_.wr_id = (uint64_t)this; sq_wr_.next = nullptr; diff --git a/src/nnti/transports/ibverbs/ibverbs_connection.hpp b/src/nnti/transports/ibverbs/ibverbs_connection.hpp index 9082e26..76bbfd1 100644 --- a/src/nnti/transports/ibverbs/ibverbs_connection.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_connection.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file nnti_connection.hpp @@ -47,6 +47,9 @@ class ibverbs_connection std::string hostname; uint32_t addr; uint32_t port; + uint64_t gid1; + uint64_t gid2; + std::string fingerprint; uint32_t lid; uint32_t cmd_qpn; uint32_t rdma_qpn; @@ -61,9 +64,12 @@ class ibverbs_connection } try { - hostname = peer.at("hostname"); + hostname = peer.at("hostname"); addr = nnti::util::str2uint32(peer.at("addr")); port = nnti::util::str2uint32(peer.at("port")); + gid1 = nnti::util::str2uint64(peer.at("gid1")); + gid2 = nnti::util::str2uint64(peer.at("gid2")); + fingerprint = peer.at("fingerprint"); lid = nnti::util::str2uint32(peer.at("lid")); cmd_qpn = nnti::util::str2uint32(peer.at("cmd_qpn")); rdma_qpn = nnti::util::str2uint32(peer.at("rdma_qpn")); @@ -83,23 +89,14 @@ class ibverbs_connection connection_params peer_params_; - uint32_t cmd_msg_size_; - uint32_t cmd_msg_count_; - - nnti::core::ibverbs_cmd_buffer *cmd_buf_; - public: ibverbs_connection( nnti::transports::ibverbs_transport *transport, uint32_t cmd_msg_size, uint32_t cmd_msg_count) : nnti_connection(), - transport_(transport), - cmd_msg_size_(cmd_msg_size), - cmd_msg_count_(cmd_msg_count) + transport_(transport) { - int rc; - setup_command_qp(); setup_rdma_qp(); setup_long_get_qp(); @@ -115,18 +112,16 @@ class ibverbs_connection const std::map &peer) : nnti_connection(), transport_(transport), - cmd_msg_size_(cmd_msg_size), - cmd_msg_count_(cmd_msg_count), peer_params_(peer) { - int rc; - nnti::core::nnti_url url = nnti::core::nnti_url(peer_params_.hostname, peer_params_.port); peer_pid_ = url.pid(); peer_ = new nnti::datatype::nnti_peer(transport, url); peer_->conn(this); + fingerprint_ = peer_params_.fingerprint; + setup_command_qp(); setup_rdma_qp(); setup_long_get_qp(); @@ -138,6 +133,9 @@ class ibverbs_connection log_debug("", "hostname = %s", peer_params_.hostname.c_str()); log_debug("", "addr = %lu", peer_params_.addr); log_debug("", "port = %d", peer_params_.port); + log_debug("", "gid1 = %lu", peer_params_.gid1); + log_debug("", "gid2 = %lu", peer_params_.gid2); + log_debug("", "fingerprint = %s", peer_params_.fingerprint.c_str()); log_debug("", "lid = %d", peer_params_.lid); log_debug("", "cmd_qpn = %d", peer_params_.cmd_qpn); log_debug("", "rdma_qpn = %d", peer_params_.rdma_qpn); @@ -156,8 +154,6 @@ class ibverbs_connection peer_params( const std::map ¶ms) { - int rc; - peer_params_ = connection_params(params); nnti::core::nnti_url url = nnti::core::nnti_url(peer_params_.hostname, peer_params_.port); @@ -167,6 +163,9 @@ class ibverbs_connection log_debug("", "hostname = %s", peer_params_.hostname.c_str()); log_debug("", "addr = %lu", peer_params_.addr); log_debug("", "port = %d", peer_params_.port); + log_debug("", "gid1 = %lu", peer_params_.gid1); + log_debug("", "gid2 = %lu", peer_params_.gid2); + log_debug("", "fingerprint = %s", peer_params_.fingerprint.c_str()); log_debug("", "lid = %d", peer_params_.lid); log_debug("", "cmd_qpn = %d", peer_params_.cmd_qpn); log_debug("", "rdma_qpn = %d", peer_params_.rdma_qpn); @@ -177,7 +176,6 @@ class ibverbs_connection peer_params( const std::string ¶ms) { - int rc; std::map param_map; std::istringstream iss(params); @@ -196,6 +194,9 @@ class ibverbs_connection log_debug("", "hostname = %s", peer_params_.hostname.c_str()); log_debug("", "addr = %lu", peer_params_.addr); log_debug("", "port = %d", peer_params_.port); + log_debug("", "gid1 = %lu", peer_params_.gid1); + log_debug("", "gid2 = %lu", peer_params_.gid2); + log_debug("", "fingerprint = %s", peer_params_.fingerprint.c_str()); log_debug("", "lid = %d", peer_params_.lid); log_debug("", "cmd_qpn = %d", peer_params_.cmd_qpn); log_debug("", "rdma_qpn = %d", peer_params_.rdma_qpn); @@ -260,7 +261,6 @@ class ibverbs_connection void transition_to_ready(void) { - int rc=NNTI_OK; int min_rnr_timer; int ack_timeout; int retry_count; @@ -273,6 +273,8 @@ class ibverbs_connection transition_qp_from_reset_to_ready(cmd_qp_, peer_params_.cmd_qpn, peer_params_.lid, + peer_params_.gid1, + peer_params_.gid2, min_rnr_timer, ack_timeout, retry_count); @@ -283,6 +285,8 @@ class ibverbs_connection transition_qp_from_reset_to_ready(rdma_qp_, peer_params_.rdma_qpn, peer_params_.lid, + peer_params_.gid1, + peer_params_.gid2, min_rnr_timer, ack_timeout, retry_count); @@ -293,6 +297,8 @@ class ibverbs_connection transition_qp_from_reset_to_ready(long_get_qp_, peer_params_.long_get_qpn, peer_params_.lid, + peer_params_.gid1, + peer_params_.gid2, min_rnr_timer, ack_timeout, retry_count); @@ -425,6 +431,8 @@ class ibverbs_connection struct ibv_qp *qp, uint32_t peer_qpn, int peer_lid, + uint64_t peer_gid1, + uint64_t peer_gid2, int min_rnr_timer, int ack_timeout, int retry_count) @@ -465,12 +473,36 @@ class ibverbs_connection memset(&attr, 0, sizeof(attr)); attr.qp_state = IBV_QPS_RTR; attr.max_dest_rd_atomic = 1; - attr.ah_attr.dlid = peer_lid; - attr.ah_attr.port_num = transport_->nic_port_; attr.path_mtu = IBV_MTU_1024; attr.rq_psn = 0; attr.dest_qp_num = peer_qpn; attr.min_rnr_timer = min_rnr_timer; /* delay before sending RNR NAK */ + attr.ah_attr.dlid = peer_lid; + attr.ah_attr.port_num = transport_->nic_port_; + attr.ah_attr.is_global = 0; + + if (transport_->is_roce_) + { + ibv_gid tmp_gid; + + attr.ah_attr.is_global = 1; + *(uint64_t*)&(attr.ah_attr.grh.dgid.raw[0]) = peer_gid1; + *(uint64_t*)&(attr.ah_attr.grh.dgid.raw[8]) = peer_gid2; + attr.ah_attr.grh.flow_label = 1; + attr.ah_attr.grh.hop_limit = 16; + attr.ah_attr.grh.sgid_index = transport_->nic_gid_idx_; + attr.ah_attr.grh.traffic_class = 0; + + log_debug("ibverbs_connection", "This is RoCE - setting sgid_index to %d", attr.ah_attr.grh.sgid_index); + + ibv_ah *ah = ibv_create_ah(transport_->pd_, &attr.ah_attr); + if (ah == NULL) { + log_error("ibverbs_connection", "ibv_create_ah failed - gid not operational"); + } else { + ibv_destroy_ah(ah); + } + } + if (ibv_modify_qp(qp, &attr, mask)) { log_error("ibverbs_connection", "failed to modify qp from INIT to RTR state"); } @@ -502,6 +534,8 @@ class ibverbs_connection struct ibv_qp *qp, uint32_t peer_qpn, int peer_lid, + uint64_t peer_gid1, + uint64_t peer_gid2, int min_rnr_timer, int ack_timeout, int retry_count) @@ -523,6 +557,8 @@ class ibverbs_connection transition_qp_from_reset_to_ready(qp, peer_qpn, peer_lid, + peer_gid1, + peer_gid2, min_rnr_timer, ack_timeout, retry_count); diff --git a/src/nnti/transports/ibverbs/ibverbs_peer.hpp b/src/nnti/transports/ibverbs/ibverbs_peer.hpp index 630e320..cf81e3a 100644 --- a/src/nnti/transports/ibverbs/ibverbs_peer.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_peer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file ibverbs_peer.hpp diff --git a/src/nnti/transports/ibverbs/ibverbs_rdma_op.hpp b/src/nnti/transports/ibverbs/ibverbs_rdma_op.hpp index 4c42d06..87e89d5 100644 --- a/src/nnti/transports/ibverbs/ibverbs_rdma_op.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_rdma_op.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef IBVERBS_RDMA_OP_HPP_ @@ -35,15 +35,13 @@ namespace core { class ibverbs_rdma_op : public nnti_op { private: - nnti::transports::ibverbs_transport *transport_; struct ibv_sge sge_; struct ibv_send_wr sq_wr_; public: ibverbs_rdma_op( nnti::transports::ibverbs_transport *transport) - : nnti_op(), - transport_(transport) + : nnti_op() { sq_wr_.wr_id = (uint64_t)this; sq_wr_.next = nullptr; @@ -57,8 +55,7 @@ class ibverbs_rdma_op ibverbs_rdma_op( nnti::transports::ibverbs_transport *transport, nnti::datatype::nnti_work_id *wid) - : nnti_op(wid), - transport_(transport) + : nnti_op(wid) { const nnti::datatype::ibverbs_work_request &wr = (const nnti::datatype::ibverbs_work_request &)wid->wr(); diff --git a/src/nnti/transports/ibverbs/ibverbs_transport.cpp b/src/nnti/transports/ibverbs/ibverbs_transport.cpp index be0b28c..1760b24 100644 --- a/src/nnti/transports/ibverbs/ibverbs_transport.cpp +++ b/src/nnti/transports/ibverbs/ibverbs_transport.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file ibverbs_transport.hpp @@ -67,9 +67,7 @@ namespace transports { /** * @brief Initialize NNTI to use a specific transport. * - * \param[in] trans_id The ID of the transport the client wants to use. - * \param[in] my_url A string that describes the transport parameters. - * \param[out] trans_hdl A handle to the activated transport. + * \param[in] config A Configuration object that NNTI should use to configure itself. * \return A result code (NNTI_OK or an error) * */ @@ -142,7 +140,15 @@ ibverbs_transport::start(void) dev_list = ibv_get_device_list(&dev_count); if (select_ib_device(dev_list, dev_count, &nic_port_) == false) { log_error("ibverbs_transport", "select_ib_device failed"); - return NNTI_EIO; + return NNTI_EINVAL; + } + + // The default GID index for IB hardware is 1. + // If this is RoCE hardware, use 3 instead. + nic_gid_idx_ = 1; + if (is_roce_) { + // TODO: iterate through GID indexes to find valid GID + nic_gid_idx_ = 3; } ibv_free_device_list(dev_list); @@ -165,6 +171,12 @@ ibverbs_transport::start(void) return NNTI_EIO; } + log_debug("ibverbs_transport", "calling ibv_query_gid()"); + if (ibv_query_gid(ctx_, nic_port_, nic_gid_idx_, &gid_)) { + log_error("ibverbs_connection", "ibv_query_gid failed"); + return NNTI_EIO; + } + active_mtu_bytes_=128; for (int i=0;ipop(e); if (rc) { uint32_t dummy=0; - ssize_t bytes_read=read(eq->read_fd(), &dummy, 4); + read(eq->read_fd(), &dummy, 4); *which = i; *event = *e; @@ -612,7 +621,7 @@ ibverbs_transport::eq_wait( } } - for (int i=0;iread_fd(); poll_fds[i].events = POLLIN; @@ -647,11 +656,11 @@ ibverbs_transport::eq_wait( goto cleanup; } else { log_debug("eq_wait", "polled on %d file descriptor(s). events occurred on %d file descriptor(s).", poll_fds.size(), poll_rc); - for (int i=0;ilong_get_qp(), &sq_wr, &bad_wr)) { - log_error("ibverbs_transport", "failed to post send: %s", strerror(errno)); + log_error("next_unexpected", "failed to post send: %s", strerror(errno)); rc = NNTI_EIO; } struct ibv_wc long_get_wc; memset(&long_get_wc, 0, sizeof(struct ibv_wc)); while ((rc = poll_cq(long_get_cq_, &long_get_wc)) == NNTI_ENOENT) { - log_debug("ibverbs_transport", "long get not done yet"); + log_debug("next_unexpected", "long get not done yet"); } if (rc != NNTI_OK) { - log_error("ibverbs_transport", "long get failed"); + log_error("next_unexpected", "long get failed"); } - log_debug("poll_cmd_cqs", "sending ACK"); + log_debug("next_unexpected", "sending ACK"); nnti::core::ibverbs_cmd_op *ack_op = nullptr; rc = create_ack_op(unexpected_msg->src_op_id(), &ack_op); rc = execute_ack_op(peer, ack_op); - log_debug("poll_cmd_cqs", "ACK sent"); + log_debug("next_unexpected", "ACK sent"); NNTI_FAST_STAT(stats_->long_recvs++;) } @@ -784,10 +793,10 @@ ibverbs_transport::next_unexpected( /** * @brief Retrieves a specific message from the unexpected list. * - * \param[in] unexpect_event Event describing the message to retrieve. - * \param[in] dst_hdl Buffer where the message is delivered. - * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[in] unexpected_event Event describing the message to retrieve. + * \param[in] dst_hdl Buffer where the message is delivered. + * \param[in] dst_offset Offset into dst_hdl where the message is delivered. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -797,8 +806,6 @@ ibverbs_transport::get_unexpected( uint64_t dst_offset, NNTI_event_t *result_event) { - nnti::datatype::nnti_buffer *b = (nnti::datatype::nnti_buffer *)dst_hdl; - return NNTI_OK; } @@ -987,8 +994,8 @@ ibverbs_transport::unregister_memory( /** * @brief Convert an NNTI peer to an NNTI_process_id_t. * - * \param[in] peer A handle to a peer that can be used for network operations. - * \param[out] pid Compact binary representation of a process's location on the network. + * \param[in] peer_hdl A handle to a peer that can be used for network operations. + * \param[out] pid Compact binary representation of a process's location on the network. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -1004,8 +1011,8 @@ ibverbs_transport::dt_peer_to_pid( /** * @brief Convert an NNTI_process_id_t to an NNTI peer. * - * \param[in] pid Compact binary representation of a process's location on the network. - * \param[out] peer A handle to a peer that can be used for network operations. + * \param[in] pid Compact binary representation of a process's location on the network. + * \param[out] peer_hdl A handle to a peer that can be used for network operations. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -1224,8 +1231,6 @@ NNTI_result_t ibverbs_transport::cancel( NNTI_work_id_t wid) { - nnti::datatype::nnti_work_id *work_id = (nnti::datatype::nnti_work_id *)wid; - return NNTI_OK; } @@ -1272,8 +1277,6 @@ ibverbs_transport::wait( const int64_t timeout, NNTI_status_t *status) { - nnti::datatype::nnti_work_id *work_id = (nnti::datatype::nnti_work_id *)wid; - return NNTI_OK; } @@ -1332,14 +1335,13 @@ bool ibverbs_transport::have_odp(void) { #if (NNTI_HAVE_IBV_EXP_QUERY_DEVICE && NNTI_HAVE_IBV_EXP_DEVICE_ATTR_ODP) - int ibv_rc=0; struct ibv_exp_device_attr exp_dev_attr; if (ctx_ == nullptr) { // the IB device is not open return false; } exp_dev_attr.comp_mask = IBV_EXP_DEVICE_ATTR_ODP | IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS; - ibv_rc = ibv_exp_query_device(ctx_, &exp_dev_attr); + ibv_exp_query_device(ctx_, &exp_dev_attr); if (exp_dev_attr.exp_device_cap_flags & IBV_EXP_DEVICE_ODP) { return true; } @@ -1351,14 +1353,13 @@ bool ibverbs_transport::have_implicit_odp(void) { #if (NNTI_HAVE_IBV_EXP_QUERY_DEVICE && NNTI_HAVE_IBV_EXP_DEVICE_ATTR_ODP && NNTI_HAVE_IBV_EXP_ODP_SUPPORT_IMPLICIT) - int ibv_rc=0; struct ibv_exp_device_attr exp_dev_attr; if (ctx_ == nullptr) { // the IB device is not open return false; } exp_dev_attr.comp_mask = IBV_EXP_DEVICE_ATTR_ODP | IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS; - ibv_rc = ibv_exp_query_device(ctx_, &exp_dev_attr); + ibv_exp_query_device(ctx_, &exp_dev_attr); if ((exp_dev_attr.exp_device_cap_flags & IBV_EXP_DEVICE_ODP) && (exp_dev_attr.odp_caps.per_transport_caps.rc_odp_caps & IBV_EXP_ODP_SUPPORT_SEND)) { @@ -1797,8 +1798,15 @@ ibverbs_transport::is_port_active(struct ibv_device *dev, int port) open_ib_device(dev); ibv_rc = ibv_query_port(ctx_, port, &dev_port_attr); if (ibv_rc == 0) { - if (dev_port_attr.state == IBV_PORT_ACTIVE) { - log_debug("ibverbs_transport", "port (%d) is active", port); + if ((dev_port_attr.state == IBV_PORT_ACTIVE) && + (dev_port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND)) { + this->is_roce_ = false; + log_debug("ibverbs_transport", "found active Infiniband port (%d)", port); + rc = true; + } else if ((dev_port_attr.state == IBV_PORT_ACTIVE) && + (dev_port_attr.link_layer == IBV_LINK_LAYER_ETHERNET)) { + this->is_roce_ = true; + log_debug("ibverbs_transport", "found active Ethernet port (%d)", port); rc = true; } } else { @@ -1831,7 +1839,14 @@ ibverbs_transport::find_active_ib_device(struct ibv_device **dev_list, int dev_c if ((dev_port_attr.state == IBV_PORT_ACTIVE) && (dev_port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND)) { *port = j+1; - log_debug("ibverbs_transport", "found device (%s|%s) with active port (%d)", dev->name, dev->dev_name, *port); + this->is_roce_ = false; + log_debug("ibverbs_transport", "found Infiniband device (%s|%s) with active port (%d)", dev->name, dev->dev_name, *port); + goto done; + } else if ((dev_port_attr.state == IBV_PORT_ACTIVE) && + (dev_port_attr.link_layer == IBV_LINK_LAYER_ETHERNET)) { + *port = j+1; + this->is_roce_ = true; + log_debug("ibverbs_transport", "found Ethernet device (%s|%s) with active port (%d)", dev->name, dev->dev_name, *port); goto done; } } else { @@ -1871,7 +1886,6 @@ ibverbs_transport::select_ib_device(struct ibv_device **dev_list, int dev_count, int rc; struct stat sbuf; std::string uverbs_dev; - uint32_t ib_port; std::vector interface_device_list = faodel::Split(interface_dev_list_, ',', true); for ( std::string ifdev : interface_device_list ) { @@ -1960,11 +1974,26 @@ ibverbs_transport::connect_cb(const std::map &args, std nthread_lock(&new_connection_lock_); nnti::core::nnti_url peer_url = nnti::core::nnti_url(args.at("hostname"), args.at("port")); + std::string fingerprint = args.at("fingerprint"); - log_debug("ibverbs_transport", "Looking for connection with pid=%016lx", peer_url.pid()); + log_debug("ibverbs_transport", "Looking for connection with pid=%016lx fingerprint=%s", peer_url.pid(), fingerprint.c_str()); conn = (nnti::core::ibverbs_connection*)conn_map_.get(peer_url.pid()); if (conn != nullptr) { - log_debug("ibverbs_transport", "Found connection with pid=%016lx", peer_url.pid()); + if (conn->fingerprint() == fingerprint) { + log_debug("ibverbs_transport", "Found matching connection with pid=%016lx fingerprint=%s", peer_url.pid(), fingerprint.c_str()); + } else if (conn->fingerprint().empty()) { + log_debug("ibverbs_transport", "Found connection with matching pid=%016lx and empty fingerprint=%s", peer_url.pid(), conn->fingerprint().c_str()); + } else { + log_debug("ibverbs_transport", "Found mismatched connection with pid=%016lx fingerprint=%s", peer_url.pid(), conn->fingerprint().c_str()); + + conn_map_.remove(conn); + delete conn; + + conn = new nnti::core::ibverbs_connection(this, cmd_msg_size_, cmd_msg_count_, args); + conn_map_.insert(conn); + + conn->transition_to_ready(); + } } else { log_debug("ibverbs_transport", "Couldn't find connection with pid=%016lx", peer_url.pid()); @@ -1976,10 +2005,16 @@ ibverbs_transport::connect_cb(const std::map &args, std nthread_unlock(&new_connection_lock_); - results << "hostname=" << url_.hostname() << std::endl; - results << "addr=" << url_.addr() << std::endl; - results << "port=" << url_.port() << std::endl; - results << "lid=" << nic_lid_ << std::endl; + uint64_t *gid1 = (uint64_t *)&(gid_.raw[0]); + uint64_t *gid2 = (uint64_t *)&(gid_.raw[8]); + + results << "hostname=" << url_.hostname() << std::endl; + results << "addr=" << url_.addr() << std::endl; + results << "port=" << url_.port() << std::endl; + results << "gid1=" << *gid1 << std::endl; + results << "gid2=" << *gid2 << std::endl; + results << "fingerprint=" << fingerprint_ << std::endl; + results << "lid=" << nic_lid_ << std::endl; results << conn->reply_string(); } @@ -2053,11 +2088,17 @@ ibverbs_transport::build_whookie_path( { std::stringstream wh_url; - wh_url << "/nnti/ib/" << service; - wh_url << "&hostname=" << url_.hostname(); - wh_url << "&addr=" << url_.addr(); - wh_url << "&port=" << url_.port(); - wh_url << "&lid=" << nic_lid_; + uint64_t *gid1 = (uint64_t *)&(gid_.raw[0]); + uint64_t *gid2 = (uint64_t *)&(gid_.raw[8]); + + wh_url << "/nnti/ib/" << service; + wh_url << "&hostname=" << url_.hostname(); + wh_url << "&addr=" << url_.addr(); + wh_url << "&port=" << url_.port(); + wh_url << "&gid1=" << *gid1; + wh_url << "&gid2=" << *gid2; + wh_url << "&fingerprint=" << fingerprint_; + wh_url << "&lid=" << nic_lid_; wh_url << conn->query_string(); return wh_url.str(); @@ -2195,8 +2236,9 @@ ibverbs_transport::execute_ack_op( print_send_wr(cmd_op->sq_wr()); log_debug("ibverbs_transport", "posting cmd_op(%s)", cmd_op->toString().c_str()); - if (ibv_post_send(conn->cmd_qp(), cmd_op->sq_wr(), &bad_wr)) { - log_error("ibverbs_transport", "failed to post send: %s", strerror(errno)); + int ibv_rc = ibv_post_send(conn->cmd_qp(), cmd_op->sq_wr(), &bad_wr); + if (ibv_rc) { + log_error("ibverbs_transport", "failed to post send (ibv_rc=%d): %s", ibv_rc, strerror(errno)); rc = NNTI_EIO; } @@ -2261,8 +2303,10 @@ ibverbs_transport::execute_rdma_op( print_send_wr(rdma_op->sq_wr()); log_debug("ibverbs_transport", "posting rdma_op(%s) to rdma_qp(%p)", rdma_op->toString().c_str(), conn->rdma_qp()); - if (ibv_post_send(conn->rdma_qp(), rdma_op->sq_wr(), &bad_wr)) { - log_error("ibverbs_transport", "failed to post send: %s", strerror(errno)); + int ibv_rc = ibv_post_send(conn->rdma_qp(), rdma_op->sq_wr(), &bad_wr); + if (ibv_rc) { + log_error("ibverbs_transport", "failed to post send (ibv_rc=%d): %s", ibv_rc, strerror(errno)); + print_error_wr(rdma_op->sq_wr()); rc = NNTI_EIO; } @@ -2327,8 +2371,9 @@ ibverbs_transport::execute_atomic_op( print_send_wr(atomic_op->sq_wr()); log_debug("ibverbs_transport", "posting atomic_op(%s)", atomic_op->toString().c_str()); - if (ibv_post_send(conn->rdma_qp(), atomic_op->sq_wr(), &bad_wr)) { - log_error("ibverbs_transport", "failed to post send: %s", strerror(errno)); + int ibv_rc = ibv_post_send(conn->rdma_qp(), atomic_op->sq_wr(), &bad_wr); + if (ibv_rc) { + log_error("ibverbs_transport", "failed to post send (ibv_rc=%d): %s", ibv_rc, strerror(errno)); rc = NNTI_EIO; } @@ -2510,18 +2555,63 @@ NNTI_result_t ibverbs_transport::poll_cmd_cq(void) { NNTI_result_t nnti_rc = NNTI_OK; - int ibv_rc = 0; +// int ibv_rc = 0; struct ibv_wc wc; nnti_rc = poll_cq(cmd_cq_, &wc); if ((nnti_rc != NNTI_EIO) && (nnti_rc != NNTI_ENOENT)) { + if (wc.status != IBV_WC_SUCCESS) { + // this is an error event + // wr_id is the address of the cmd_op that issued the put + nnti::core::ibverbs_cmd_op *cmd_op = (nnti::core::ibverbs_cmd_op *)wc.wr_id; + nnti::datatype::nnti_work_request &wr = cmd_op->wid()->wr(); + nnti::datatype::nnti_event_queue *alt_q = nnti::datatype::nnti_event_queue::to_obj(wr.alt_eq()); + nnti::datatype::nnti_event_queue *buf_q = nullptr; + NNTI_event_t *e = create_event(cmd_op, nnti_rc); + bool event_complete = false; + bool release_event = true; + + if (wr.invoke_cb(e) == NNTI_OK) { + event_complete = true; + } + if (!event_complete && alt_q && alt_q->invoke_cb(e) == NNTI_OK) { + event_complete = true; + } + if (!event_complete) { + nnti::datatype::nnti_buffer *b = nnti::datatype::nnti_buffer::to_obj(wr.local_hdl()); + + buf_q = nnti::datatype::nnti_event_queue::to_obj(b->eq()); + if (buf_q && buf_q->invoke_cb(e) == NNTI_OK) { + event_complete = true; + } + } + if (!event_complete && alt_q) { + alt_q->push(e); + alt_q->notify(); + event_complete = true; + release_event = false; + } + if (!event_complete && buf_q) { + buf_q->push(e); + buf_q->notify(); + event_complete = true; + release_event = false; + } + if (release_event) { + event_freelist_->push(e); + } + + cmd_op_freelist_->push(cmd_op); + + NNTI_FAST_STAT(stats_->errors++;) + } // found a work completion if (wc.opcode & IBV_WC_RECV) { nnti::core::ibverbs_cmd_msg *cmd_msg = (nnti::core::ibverbs_cmd_msg *)wc.wr_id; cmd_msg->unpack(); if (cmd_msg->ack()) { - log_debug("poll_cmd_cqs", "ACK received"); + log_debug("poll_cmd_cq", "ACK received"); nnti::core::ibverbs_cmd_op *cmd_op = (nnti::core::ibverbs_cmd_op*)op_vector_.at(cmd_msg->src_op_id()); @@ -2532,15 +2622,15 @@ ibverbs_transport::poll_cmd_cq(void) bool event_complete = false; bool release_event = true; - log_debug("poll_cmd_cqs", "considering WR callback"); + log_debug("poll_cmd_cq", "considering WR callback"); if (wr.invoke_cb(e) == NNTI_OK) { event_complete = true; } - log_debug("poll_cmd_cqs", "considering alt EQ callback"); + log_debug("poll_cmd_cq", "considering alt EQ callback"); if (!event_complete && alt_q && alt_q->invoke_cb(e) == NNTI_OK) { event_complete = true; } - log_debug("poll_cmd_cqs", "considering buf EQ callback"); + log_debug("poll_cmd_cq", "considering buf EQ callback"); if (!event_complete) { nnti::datatype::nnti_buffer *b = nnti::datatype::nnti_buffer::to_obj(wr.local_hdl()); @@ -2549,14 +2639,14 @@ ibverbs_transport::poll_cmd_cq(void) event_complete = true; } } - log_debug("poll_cmd_cqs", "considering alt EQ push"); + log_debug("poll_cmd_cq", "considering alt EQ push"); if (!event_complete && alt_q) { alt_q->push(e); alt_q->notify(); event_complete = true; release_event = false; } - log_debug("poll_cmd_cqs", "considering buf EQ push"); + log_debug("poll_cmd_cq", "considering buf EQ push"); if (!event_complete && buf_q) { buf_q->push(e); buf_q->notify(); @@ -2580,7 +2670,7 @@ ibverbs_transport::poll_cmd_cq(void) } else { if (cmd_msg->unexpected()) { - log_debug("poll_cmd_cqs", "unexpected received"); + log_debug("poll_cmd_cq", "unexpected received"); if (unexpected_queue_ == nullptr) { // If there is no unexpected queue, then there is no way @@ -2599,7 +2689,7 @@ ibverbs_transport::poll_cmd_cq(void) NNTI_FAST_STAT(stats_->unexpected_recvs++;) } } else { - log_debug("poll_cmd_cqs", "expected received"); + log_debug("poll_cmd_cq", "expected received"); nnti::datatype::ibverbs_buffer *tgt_buf = cmd_msg->target_buffer(); assert(tgt_buf != nullptr); @@ -2609,7 +2699,7 @@ ibverbs_transport::poll_cmd_cq(void) uint64_t actual_offset = 0; if (cmd_msg->eager()) { - log_debug("poll_cmd_cqs", "expected eager received"); + log_debug("poll_cmd_cq", "expected eager received"); nnti_rc = tgt_buf->copy_in(cmd_msg->target_offset(), cmd_msg->eager_payload(), cmd_msg->payload_length(), &actual_offset); @@ -2636,7 +2726,7 @@ ibverbs_transport::poll_cmd_cq(void) cmd_msg->post_recv(); NNTI_FAST_STAT(stats_->short_recvs++;) } else { - log_debug("poll_cmd_cqs", "expected long received"); + log_debug("poll_cmd_cq", "expected long received"); nnti::datatype::ibverbs_peer *peer = cmd_msg->initiator_peer(); nnti::core::ibverbs_connection *conn = (nnti::core::ibverbs_connection *)peer->conn(); @@ -2673,11 +2763,11 @@ ibverbs_transport::poll_cmd_cq(void) log_error("ibverbs_transport", "long get failed"); } - log_debug("poll_cmd_cqs", "sending ACK"); + log_debug("poll_cmd_cq", "sending ACK"); nnti::core::ibverbs_cmd_op *ack_op = nullptr; nnti_rc = create_ack_op(cmd_msg->src_op_id(), &ack_op); nnti_rc = execute_ack_op(peer, ack_op); - log_debug("poll_cmd_cqs", "ACK sent"); + log_debug("poll_cmd_cq", "ACK sent"); e = create_event(cmd_msg, nnti_rc); if (tgt_buf->invoke_cb(e) != NNTI_OK) { @@ -2698,20 +2788,19 @@ ibverbs_transport::poll_cmd_cq(void) } } if (wc.opcode == IBV_WC_SEND) { - bool need_event = true; // wr_id is the address of the cmd_op that issued the send nnti::core::ibverbs_cmd_op *cmd_op = (nnti::core::ibverbs_cmd_op *)wc.wr_id; if (cmd_op->ack()) { - log_debug("poll_cmd_cqs", "ACK send complete"); + log_debug("poll_cmd_cq", "ACK send complete"); cmd_op_freelist_->push(cmd_op); NNTI_FAST_STAT(stats_->ack_sends++;) } else if (!cmd_op->eager()) { // this is a long send. wait for the ACK. - log_debug("poll_cmd_cqs", "long send complete"); + log_debug("poll_cmd_cq", "long send complete"); } else { - log_debug("poll_cmd_cqs", "eager send complete"); + log_debug("poll_cmd_cq", "eager send complete"); nnti::datatype::nnti_work_request &wr = cmd_op->wid()->wr(); nnti::datatype::nnti_event_queue *alt_q = nnti::datatype::nnti_event_queue::to_obj(wr.alt_eq()); nnti::datatype::nnti_event_queue *buf_q = nullptr; @@ -2763,7 +2852,7 @@ ibverbs_transport::poll_cmd_cq(void) } } if (wc.opcode == IBV_WC_RDMA_READ) { - log_debug("poll_cmd_cqs", "long send GET complete"); + log_debug("poll_cmd_cq", "long send GET complete"); // if there is a READ event on the cmd_qp, then this is a long send/recv. // wr_id is the address of the cmd_op that issued the get @@ -2797,16 +2886,58 @@ NNTI_result_t ibverbs_transport::poll_rdma_cq(void) { NNTI_result_t nnti_rc = NNTI_OK; - int ibv_rc = 0; struct ibv_wc wc; nnti_rc = poll_cq(rdma_cq_, &wc); if ((nnti_rc != NNTI_EIO) && (nnti_rc != NNTI_ENOENT)) { + if (wc.status != IBV_WC_SUCCESS) { + // this is an error event + // wr_id is the address of the rdma_op that issued the put + nnti::core::ibverbs_rdma_op *rdma_op = (nnti::core::ibverbs_rdma_op *)wc.wr_id; + nnti::datatype::nnti_work_request &wr = rdma_op->wid()->wr(); + nnti::datatype::nnti_event_queue *alt_q = nnti::datatype::nnti_event_queue::to_obj(wr.alt_eq()); + nnti::datatype::nnti_event_queue *buf_q = nullptr; + NNTI_event_t *e = create_event(rdma_op, nnti_rc); + bool event_complete = false; + bool release_event = true; + + if (wr.invoke_cb(e) == NNTI_OK) { + event_complete = true; + } + if (!event_complete && alt_q && alt_q->invoke_cb(e) == NNTI_OK) { + event_complete = true; + } + if (!event_complete) { + nnti::datatype::nnti_buffer *b = nnti::datatype::nnti_buffer::to_obj(wr.local_hdl()); + + buf_q = nnti::datatype::nnti_event_queue::to_obj(b->eq()); + if (buf_q && buf_q->invoke_cb(e) == NNTI_OK) { + event_complete = true; + } + } + if (!event_complete && alt_q) { + alt_q->push(e); + alt_q->notify(); + event_complete = true; + release_event = false; + } + if (!event_complete && buf_q) { + buf_q->push(e); + buf_q->notify(); + event_complete = true; + release_event = false; + } + if (release_event) { + event_freelist_->push(e); + } + + rdma_op_freelist_->push(rdma_op); + + NNTI_FAST_STAT(stats_->errors++;) + } // found a work completion if (wc.opcode == IBV_WC_RDMA_WRITE) { - bool need_event = true; - // wr_id is the address of the rdma_op that issued the put nnti::core::ibverbs_rdma_op *rdma_op = (nnti::core::ibverbs_rdma_op *)wc.wr_id; nnti::datatype::nnti_work_request &wr = rdma_op->wid()->wr(); @@ -2851,8 +2982,6 @@ ibverbs_transport::poll_rdma_cq(void) NNTI_FAST_STAT(stats_->puts++;) } if (wc.opcode == IBV_WC_RDMA_READ) { - bool need_event = true; - // wr_id is the address of the rdma_op that issued the get nnti::core::ibverbs_rdma_op *rdma_op = (nnti::core::ibverbs_rdma_op *)wc.wr_id; nnti::datatype::nnti_work_request &wr = rdma_op->wid()->wr(); @@ -2897,8 +3026,6 @@ ibverbs_transport::poll_rdma_cq(void) NNTI_FAST_STAT(stats_->gets++;) } if (wc.opcode == IBV_WC_FETCH_ADD) { - bool need_event = true; - // wr_id is the address of the rdma_op that issued the get nnti::core::ibverbs_atomic_op *atomic_op = (nnti::core::ibverbs_atomic_op *)wc.wr_id; nnti::datatype::nnti_work_request &wr = atomic_op->wid()->wr(); @@ -2951,8 +3078,6 @@ ibverbs_transport::poll_rdma_cq(void) NNTI_FAST_STAT(stats_->fadds++;) } if (wc.opcode == IBV_WC_COMP_SWAP) { - bool need_event = true; - // wr_id is the address of the rdma_op that issued the get nnti::core::ibverbs_atomic_op *atomic_op = (nnti::core::ibverbs_atomic_op *)wc.wr_id; nnti::datatype::nnti_work_request &wr = atomic_op->wid()->wr(); @@ -3259,5 +3384,23 @@ ibverbs_transport::print_send_wr( wr->sg_list[0].lkey); } +void +ibverbs_transport::print_error_wr( + const struct ibv_send_wr *wr) +{ + log_error("print_wr", "wr=%p, wr.opcode=%d, wr.send_flags=%d, wr.wr_id=%lx, wr.next=%p, wr.num_sge=%d, wr.rdma.remote_addr=%lu, wr.sge.rkey=%X, wr.sge.addr=%lu, wr.sge.length=%u, wr.sge.lkey=%X", + wr, + wr->opcode, + wr->send_flags, + wr->wr_id, + wr->next, + wr->num_sge, + wr->wr.rdma.remote_addr, + wr->wr.rdma.rkey, + wr->sg_list[0].addr, + wr->sg_list[0].length, + wr->sg_list[0].lkey); +} + } /* namespace transports */ } /* namespace nnti */ diff --git a/src/nnti/transports/ibverbs/ibverbs_transport.hpp b/src/nnti/transports/ibverbs/ibverbs_transport.hpp index accab6f..a1a6301 100644 --- a/src/nnti/transports/ibverbs/ibverbs_transport.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_transport.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file ibverbs_transport.hpp @@ -94,6 +94,7 @@ class ibverbs_transport std::atomic puts; std::atomic fadds; std::atomic cswaps; + std::atomic errors; whookie_stats() { @@ -111,6 +112,7 @@ class ibverbs_transport puts.store(0); fadds.store(0); cswaps.store(0); + errors.store(0); } }; @@ -128,6 +130,9 @@ class ibverbs_transport struct ibv_device *dev_; uint16_t nic_lid_; int nic_port_; + uint8_t nic_gid_idx_; + bool is_roce_; + union ibv_gid gid_; struct ibv_context *ctx_; struct ibv_pd *pd_; @@ -196,8 +201,7 @@ class ibverbs_transport /** * @brief Initialize NNTI to use a specific transport. * - * \param[in] my_url A string that describes the transport parameters. - * \param[out] trans_hdl A handle to the activated transport. + * \param[in] config A Configuration object that NNTI should use to configure itself. * \return A result code (NNTI_OK or an error) * */ @@ -221,8 +225,6 @@ class ibverbs_transport /** * @brief Indicates if a transport has been initialized. * - * \param[in] trans_id The ID of the transport to test. - * \param[out] is_init 1 if the transport is initialized, 0 otherwise. * \return A result code (NNTI_OK or an error) * */ @@ -249,7 +251,7 @@ class ibverbs_transport * */ NNTI_result_t - pid(NNTI_process_id_t *pid); + pid(NNTI_process_id_t *pid) override; /** * @brief Get attributes of the transport. @@ -315,7 +317,7 @@ class ibverbs_transport */ NNTI_result_t eq_destroy( - NNTI_event_queue_t eq); + NNTI_event_queue_t eq) override; /** * @brief Wait for an event to arrive on an event queue. @@ -323,6 +325,7 @@ class ibverbs_transport * \param[in] eq_list A list of event queues to wait on. * \param[in] eq_count The number of event queues in the list. * \param[in] timeout The amount of time (in milliseconds) to wait. + * \param[out] which The index of the EQ where the event occurred. * \param[out] event The details of the event. * \return A result code (NNTI_OK or an error) */ @@ -339,7 +342,7 @@ class ibverbs_transport * * \param[in] dst_hdl Buffer where the message is delivered. * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -351,10 +354,10 @@ class ibverbs_transport /** * @brief Retrieves a specific message from the unexpected list. * - * \param[in] unexpect_event Event describing the message to retrieve. - * \param[in] dst_hdl Buffer where the message is delivered. - * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[in] unexpected_event Event describing the message to retrieve. + * \param[in] dst_hdl Buffer where the message is delivered. + * \param[in] dst_offset Offset into dst_hdl where the message is delivered. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -408,7 +411,7 @@ class ibverbs_transport nnti::datatype::nnti_event_callback cb, void *cb_context, char **reg_ptr, - NNTI_buffer_t *reg_buf); + NNTI_buffer_t *reg_buf) override; /** * @brief Disables network operations on the block of memory and frees it. @@ -455,8 +458,8 @@ class ibverbs_transport /** * @brief Convert an NNTI peer to an NNTI_process_id_t. * - * \param[in] peer A handle to a peer that can be used for network operations. - * \param[out] pid Compact binary representation of a process's location on the network. + * \param[in] peer_hdl A handle to a peer that can be used for network operations. + * \param[out] pid Compact binary representation of a process's location on the network. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -467,8 +470,8 @@ class ibverbs_transport /** * @brief Convert an NNTI_process_id_t to an NNTI peer. * - * \param[in] pid Compact binary representation of a process's location on the network. - * \param[out] peer A handle to a peer that can be used for network operations. + * \param[in] pid Compact binary representation of a process's location on the network. + * \param[out] peer_hdl A handle to a peer that can be used for network operations. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -486,7 +489,7 @@ class ibverbs_transport NNTI_result_t send( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid); + NNTI_work_id_t *wid) override; /** * @brief Transfer data to a peer. @@ -787,6 +790,9 @@ class ibverbs_transport void print_send_wr( const struct ibv_send_wr *wr); + void + print_error_wr( + const struct ibv_send_wr *wr); }; } /* namespace transports */ diff --git a/src/nnti/transports/ibverbs/ibverbs_wr.hpp b/src/nnti/transports/ibverbs/ibverbs_wr.hpp index 5c1f55b..62b4b4a 100644 --- a/src/nnti/transports/ibverbs/ibverbs_wr.hpp +++ b/src/nnti/transports/ibverbs/ibverbs_wr.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef IBVERBS_WR_HPP diff --git a/src/nnti/transports/mpi/mpi_buffer.cpp b/src/nnti/transports/mpi/mpi_buffer.cpp index 7cd19fb..f74d676 100644 --- a/src/nnti/transports/mpi/mpi_buffer.cpp +++ b/src/nnti/transports/mpi/mpi_buffer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -185,7 +185,6 @@ mpi_buffer::register_buffer(void) packable_.buffer.NNTI_remote_addr_p_t_u.mpi.put_data_tag, packable_.buffer.NNTI_remote_addr_p_t_u.mpi.atomic_data_tag); -cleanup: return rc; } diff --git a/src/nnti/transports/mpi/mpi_buffer.hpp b/src/nnti/transports/mpi/mpi_buffer.hpp index 73bfa19..c9ea9e9 100644 --- a/src/nnti/transports/mpi/mpi_buffer.hpp +++ b/src/nnti/transports/mpi/mpi_buffer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef MPI_BUFFER_HPP_ diff --git a/src/nnti/transports/mpi/mpi_cmd_buffer.cpp b/src/nnti/transports/mpi/mpi_cmd_buffer.cpp index 142a180..a74a0ac 100644 --- a/src/nnti/transports/mpi/mpi_cmd_buffer.cpp +++ b/src/nnti/transports/mpi/mpi_cmd_buffer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -27,11 +27,8 @@ mpi_cmd_buffer::mpi_cmd_buffer( const uint32_t cmd_count) : transport_(transport), cmd_size_(cmd_size), - cmd_count_(cmd_count), - cmd_offset_(0) + cmd_count_(cmd_count) { - int rc; - setup_command_buffer(); } mpi_cmd_buffer::~mpi_cmd_buffer() @@ -58,20 +55,6 @@ mpi_cmd_buffer::cmd_msg(int index) return msgs_[index]; } -//void -//mpi_cmd_buffer::post_recv( -// nnti::core::mpi_cmd_msg *cmd_msg) -//{ -// MPI_Irecv( -// cmd_msg->buf(), -// cmd_msg->size(), -// MPI_BYTE, -// MPI_ANY_SOURCE, -// nnti::transports::mpi_transport::NNTI_MPI_CMD_TAG, -// MPI_COMM_WORLD, -// &cmd_msg->mpi_request()); -//} - void mpi_cmd_buffer::setup_command_buffer(void) { @@ -79,7 +62,7 @@ mpi_cmd_buffer::setup_command_buffer(void) cmd_buf_ = new char[cmd_size_ * cmd_count_]; - for (int i=0;icmd_request()); delete msgs_[i]; } diff --git a/src/nnti/transports/mpi/mpi_cmd_buffer.hpp b/src/nnti/transports/mpi/mpi_cmd_buffer.hpp index 53cccf0..878033f 100644 --- a/src/nnti/transports/mpi/mpi_cmd_buffer.hpp +++ b/src/nnti/transports/mpi/mpi_cmd_buffer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef MPI_CMD_BUFFER_HPP_ @@ -39,7 +39,6 @@ class mpi_cmd_buffer { uint32_t cmd_count_; char *cmd_buf_; - uint32_t cmd_offset_; std::vector msgs_; diff --git a/src/nnti/transports/mpi/mpi_cmd_msg.cpp b/src/nnti/transports/mpi/mpi_cmd_msg.cpp index e02ff41..a02c22f 100644 --- a/src/nnti/transports/mpi/mpi_cmd_msg.cpp +++ b/src/nnti/transports/mpi/mpi_cmd_msg.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -25,7 +25,6 @@ mpi_cmd_msg::mpi_cmd_msg( nnti::transports::mpi_transport *transport, const uint32_t cmd_msg_size) : transport_(transport), - cmd_buf_(nullptr), cmd_msg_buf_(nullptr), cmd_msg_size_(cmd_msg_size), free_in_dtor_(true), @@ -40,7 +39,6 @@ mpi_cmd_msg::mpi_cmd_msg( uint32_t id, nnti::datatype::nnti_work_id *wid) : transport_(transport), - cmd_buf_(nullptr), cmd_msg_buf_(nullptr), unexpected_(false) { @@ -64,7 +62,6 @@ mpi_cmd_msg::mpi_cmd_msg( char *buf, uint32_t buf_size) : transport_(transport), - cmd_buf_(cmd_buf), cmd_msg_buf_((struct cmd_msg*)buf), cmd_msg_size_(buf_size), free_in_dtor_(false), diff --git a/src/nnti/transports/mpi/mpi_cmd_msg.hpp b/src/nnti/transports/mpi/mpi_cmd_msg.hpp index 4a9c324..16975f9 100644 --- a/src/nnti/transports/mpi/mpi_cmd_msg.hpp +++ b/src/nnti/transports/mpi/mpi_cmd_msg.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef MPI_CMD_MSG_HPP_ @@ -62,7 +62,6 @@ class mpi_cmd_msg { size_t index_; nnti::transports::mpi_transport *transport_; - nnti::core::mpi_cmd_buffer *cmd_buf_; struct cmd_msg *cmd_msg_buf_; uint32_t cmd_msg_size_; diff --git a/src/nnti/transports/mpi/mpi_cmd_op.hpp b/src/nnti/transports/mpi/mpi_cmd_op.hpp index ed9ca08..1144fb6 100644 --- a/src/nnti/transports/mpi/mpi_cmd_op.hpp +++ b/src/nnti/transports/mpi/mpi_cmd_op.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef MPI_CMD_OP_HPP_ @@ -67,17 +67,13 @@ class mpi_cmd_op size_t index_; - nnti::transports::mpi_transport *transport_; nnti::core::mpi_cmd_msg cmd_msg_; - enum cmd_state state_; - public: mpi_cmd_op( nnti::transports::mpi_transport *transport, const uint32_t cmd_msg_size) : nnti_op(), - transport_(transport), cmd_msg_(transport, cmd_msg_size) { return; @@ -87,7 +83,6 @@ class mpi_cmd_op const uint32_t cmd_msg_size, nnti::datatype::nnti_work_id *wid) : nnti_op(wid), - transport_(transport), cmd_msg_(transport, cmd_msg_size) { set(wid); @@ -97,7 +92,6 @@ class mpi_cmd_op nnti::transports::mpi_transport *transport, nnti::datatype::nnti_work_id *wid) : nnti_op(wid), - transport_(transport), cmd_msg_(transport, id_, wid) { return; diff --git a/src/nnti/transports/mpi/mpi_connection.hpp b/src/nnti/transports/mpi/mpi_connection.hpp index 21085bb..da1d097 100644 --- a/src/nnti/transports/mpi/mpi_connection.hpp +++ b/src/nnti/transports/mpi/mpi_connection.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef MPI_CONNECTION_HPP_ @@ -35,6 +35,7 @@ class mpi_connection std::string hostname; uint32_t addr; uint32_t port; + std::string fingerprint; int rank; connection_params(void) { @@ -46,10 +47,11 @@ class mpi_connection } try { - hostname = peer.at("hostname"); - addr = nnti::util::str2uint32(peer.at("addr")); - port = nnti::util::str2uint32(peer.at("port")); - rank = nnti::util::str2int32(peer.at("rank")); + hostname = peer.at("hostname"); + addr = nnti::util::str2uint32(peer.at("addr")); + port = nnti::util::str2uint32(peer.at("port")); + fingerprint = peer.at("fingerprint"); + rank = nnti::util::str2int32(peer.at("rank")); } catch (const std::out_of_range& oor) { log_error_stream("connection_params") << "Out of Range error: " << oor.what(); @@ -58,55 +60,53 @@ class mpi_connection }; private: - nnti::transports::mpi_transport *transport_; connection_params peer_params_; public: mpi_connection( nnti::transports::mpi_transport *transport) - : nnti_connection(), - transport_(transport) + : nnti_connection() { } mpi_connection( nnti::transports::mpi_transport *transport, const std::map &peer) : nnti_connection(), - transport_(transport), peer_params_(peer) { - int rc; - nnti::core::nnti_url url = nnti::core::nnti_url(peer_params_.hostname, peer_params_.port); peer_pid_ = url.pid(); peer_ = new nnti::datatype::mpi_peer(transport, url, peer_params_.rank); peer_->conn(this); - log_debug("mpi_connection param_map", "hostname = %s", peer_params_.hostname.c_str()); - log_debug("mpi_connection param_map", "addr = %lu", peer_params_.addr); - log_debug("mpi_connection param_map", "port = %d", peer_params_.port); - log_debug("mpi_connection param_map", "rank = %d", peer_params_.rank); + fingerprint_ = peer_params_.fingerprint; + + log_debug("mpi_connection param_map", "hostname = %s", peer_params_.hostname.c_str()); + log_debug("mpi_connection param_map", "addr = %lu", peer_params_.addr); + log_debug("mpi_connection param_map", "port = %d", peer_params_.port); + log_debug("mpi_connection param_map", "fingerprint = %s", peer_params_.fingerprint.c_str()); + log_debug("mpi_connection param_map", "rank = %d", peer_params_.rank); } mpi_connection( nnti::transports::mpi_transport *transport, const std::string ¶ms) - : nnti_connection(), - transport_(transport) + : nnti_connection() { - int rc; - peer_params(params); nnti::core::nnti_url url = nnti::core::nnti_url(peer_params_.hostname, peer_params_.port); peer_ = new nnti::datatype::mpi_peer(transport, url, peer_params_.rank); peer_->conn(this); - log_debug("mpi_connection param_str", "hostname = %s", peer_params_.hostname.c_str()); - log_debug("mpi_connection param_str", "addr = %lu", peer_params_.addr); - log_debug("mpi_connection param_str", "port = %d", peer_params_.port); - log_debug("mpi_connection param_str", "rank = %d", peer_params_.rank); + fingerprint_ = peer_params_.fingerprint; + + log_debug("mpi_connection param_str", "hostname = %s", peer_params_.hostname.c_str()); + log_debug("mpi_connection param_str", "addr = %lu", peer_params_.addr); + log_debug("mpi_connection param_str", "port = %d", peer_params_.port); + log_debug("mpi_connection param_map", "fingerprint = %s", peer_params_.fingerprint.c_str()); + log_debug("mpi_connection param_str", "rank = %d", peer_params_.rank); } - ~mpi_connection() override { + ~mpi_connection() override { return; } @@ -114,8 +114,6 @@ class mpi_connection peer_params( const std::map ¶ms) { - int rc; - peer_params_ = connection_params(params); nnti::core::nnti_url url = nnti::core::nnti_url(peer_params_.hostname, peer_params_.port); @@ -124,17 +122,17 @@ class mpi_connection nnti::datatype::mpi_peer *mp = (nnti::datatype::mpi_peer*)peer_; mp->rank(peer_params_.rank); - log_debug("peer_params", "hostname = %s", peer_params_.hostname.c_str()); - log_debug("peer_params", "addr = %lu", peer_params_.addr); - log_debug("peer_params", "port = %d", peer_params_.port); - log_debug("peer_params", "rank = %d", peer_params_.rank); + log_debug("peer_params", "hostname = %s", peer_params_.hostname.c_str()); + log_debug("peer_params", "addr = %lu", peer_params_.addr); + log_debug("peer_params", "port = %d", peer_params_.port); + log_debug("peer_params", "fingerprint = %s", peer_params_.fingerprint.c_str()); + log_debug("peer_params", "rank = %d", peer_params_.rank); } void peer_params( const std::string ¶ms) { - int rc; std::map param_map; std::istringstream iss(params); @@ -151,10 +149,11 @@ class mpi_connection nnti::datatype::mpi_peer *mp = (nnti::datatype::mpi_peer*)peer_; mp->rank(peer_params_.rank); - log_debug("peer_params", "hostname = %s", peer_params_.hostname.c_str()); - log_debug("peer_params", "addr = %lu", peer_params_.addr); - log_debug("peer_params", "port = %d", peer_params_.port); - log_debug("peer_params", "rank = %d", peer_params_.rank); + log_debug("peer_params", "hostname = %s", peer_params_.hostname.c_str()); + log_debug("peer_params", "addr = %lu", peer_params_.addr); + log_debug("peer_params", "port = %d", peer_params_.port); + log_debug("peer_params", "fingerprint = %s", peer_params_.fingerprint.c_str()); + log_debug("peer_params", "rank = %d", peer_params_.rank); } private: diff --git a/src/nnti/transports/mpi/mpi_peer.hpp b/src/nnti/transports/mpi/mpi_peer.hpp index 405458c..dcb3c6a 100644 --- a/src/nnti/transports/mpi/mpi_peer.hpp +++ b/src/nnti/transports/mpi/mpi_peer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef MPI_PEER_HPP diff --git a/src/nnti/transports/mpi/mpi_transport.cpp b/src/nnti/transports/mpi/mpi_transport.cpp index 1abc1ee..bdf1929 100644 --- a/src/nnti/transports/mpi/mpi_transport.cpp +++ b/src/nnti/transports/mpi/mpi_transport.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -53,9 +53,7 @@ namespace transports { /** * @brief Initialize NNTI to use a specific transport. * - * \param[in] trans_id The ID of the transport the client wants to use. - * \param[in] my_url A string that describes the transport parameters. - * \param[out] trans_hdl A handle to the activated transport. + * \param[in] config A Configuration object that NNTI should use to configure itself. * \return A result code (NNTI_OK or an error) * */ @@ -207,7 +205,6 @@ mpi_transport::stop(void) MPI_Finalize(); } -cleanup: log_debug("mpi_transport", "exit"); return rc; @@ -216,8 +213,6 @@ mpi_transport::stop(void) /** * @brief Indicates if a transport has been initialized. * - * \param[in] trans_id The ID of the transport to test. - * \param[out] is_init 1 if the transport is initialized, 0 otherwise. * \return A result code (NNTI_OK or an error) * */ @@ -487,12 +482,12 @@ mpi_transport::eq_wait( log_debug("eq_wait", "enter"); - for (int i=0;ipop(e); if (rc) { uint32_t dummy=0; - ssize_t bytes_read=read(eq->read_fd(), &dummy, 4); + read(eq->read_fd(), &dummy, 4); *which = i; *event = *e; @@ -502,7 +497,7 @@ mpi_transport::eq_wait( } } - for (int i=0;iread_fd(); poll_fds[i].events = POLLIN; @@ -537,11 +532,11 @@ mpi_transport::eq_wait( goto cleanup; } else { log_debug("eq_wait", "polled on %d file descriptor(s). events occurred on %d file descriptor(s).", poll_fds.size(), poll_rc); - for (int i=0;i mpi_lock(mpi_mutex_); - mpi_rc = MPI_Irecv((char*)b->payload() + dst_offset, + MPI_Irecv((char*)b->payload() + dst_offset, unexpected_msg->payload_length(), MPI_BYTE, peer->rank(), initiator_buffer->cmd_tag(), MPI_COMM_WORLD, &req); - mpi_rc = MPI_Wait(&req, &status); + MPI_Wait(&req, &status); mpi_lock.unlock(); log_debug("mpi_transport", "unexpected long send Wait() complete"); @@ -657,10 +651,10 @@ mpi_transport::next_unexpected( /** * @brief Retrieves a specific message from the unexpected list. * - * \param[in] unexpect_event Event describing the message to retrieve. - * \param[in] dst_hdl Buffer where the message is delivered. - * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[in] unexpected_event Event describing the message to retrieve. + * \param[in] dst_hdl Buffer where the message is delivered. + * \param[in] dst_offset Offset into dst_hdl where the message is delivered. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -670,8 +664,6 @@ mpi_transport::get_unexpected( uint64_t dst_offset, NNTI_event_t *result_event) { - nnti::datatype::nnti_buffer *b = (nnti::datatype::nnti_buffer *)dst_hdl; - return NNTI_OK; } @@ -859,8 +851,8 @@ mpi_transport::unregister_memory( /** * @brief Convert an NNTI peer to an NNTI_process_id_t. * - * \param[in] peer A handle to a peer that can be used for network operations. - * \param[out] pid Compact binary representation of a process's location on the network. + * \param[in] peer_hdl A handle to a peer that can be used for network operations. + * \param[out] pid Compact binary representation of a process's location on the network. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -876,8 +868,8 @@ mpi_transport::dt_peer_to_pid( /** * @brief Convert an NNTI_process_id_t to an NNTI peer. * - * \param[in] pid Compact binary representation of a process's location on the network. - * \param[out] peer A handle to a peer that can be used for network operations. + * \param[in] pid Compact binary representation of a process's location on the network. + * \param[out] peer_hdl A handle to a peer that can be used for network operations. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -1100,8 +1092,6 @@ NNTI_result_t mpi_transport::cancel( NNTI_work_id_t wid) { - nnti::datatype::nnti_work_id *work_id = (nnti::datatype::nnti_work_id *)wid; - return NNTI_OK; } @@ -1148,8 +1138,6 @@ mpi_transport::wait( const int64_t timeout, NNTI_status_t *status) { - nnti::datatype::nnti_work_id *work_id = (nnti::datatype::nnti_work_id *)wid; - return NNTI_OK; } @@ -1368,11 +1356,24 @@ mpi_transport::connect_cb( } nnti::core::nnti_url peer_url = nnti::core::nnti_url(args.at("hostname"), args.at("port")); + std::string fingerprint = args.at("fingerprint"); - log_debug("mpi_transport", "Looking for connection with pid=%016lx", peer_url.pid()); + log_debug("mpi_transport", "Looking for connection with pid=%016lx fingerprint=%s", peer_url.pid(), fingerprint.c_str()); conn = (nnti::core::mpi_connection*)conn_map_.get(peer_url.pid()); if (conn != nullptr) { - log_debug("mpi_transport", "Found connection with pid=%016lx", peer_url.pid()); + if (conn->fingerprint() == fingerprint) { + log_debug("mpi_transport", "Found matching connection with pid=%016lx fingerprint=%s", peer_url.pid(), fingerprint.c_str()); + } else if (conn->fingerprint().empty()) { + log_debug("mpi_transport", "Found connection with matching pid=%016lx and empty fingerprint=%s", peer_url.pid(), conn->fingerprint().c_str()); + } else { + log_debug("mpi_transport", "Found mismatched connection with pid=%016lx fingerprint=%s", peer_url.pid(), conn->fingerprint().c_str()); + + conn_map_.remove(conn); + delete conn; + + conn = new nnti::core::mpi_connection(this, args); + conn_map_.insert(conn); + } } else { log_debug("mpi_transport", "Couldn't find connection with pid=%016lx", peer_url.pid()); @@ -1387,10 +1388,11 @@ mpi_transport::connect_cb( nthread_unlock(&new_connection_lock_); - results << "hostname=" << url_.hostname() << std::endl; - results << "addr=" << url_.addr() << std::endl; - results << "port=" << url_.port() << std::endl; - results << "rank=" << nnti_comm_rank_ << std::endl; + results << "hostname=" << url_.hostname() << std::endl; + results << "addr=" << url_.addr() << std::endl; + results << "port=" << url_.port() << std::endl; + results << "fingerprint=" << fingerprint_ << std::endl; + results << "rank=" << nnti_comm_rank_ << std::endl; log_debug("mpi_transport", "connect_cb - results=%s", results.str().c_str()); } @@ -1466,11 +1468,12 @@ mpi_transport::build_whookie_path( { std::stringstream wh_url; - wh_url << "/nnti/mpi/" << service; - wh_url << "&hostname=" << url_.hostname(); - wh_url << "&addr=" << url_.addr(); - wh_url << "&port=" << url_.port(); - wh_url << "&rank=" << nnti_comm_rank_; + wh_url << "/nnti/mpi/" << service; + wh_url << "&hostname=" << url_.hostname(); + wh_url << "&addr=" << url_.addr(); + wh_url << "&port=" << url_.port(); + wh_url << "&fingerprint=" << fingerprint_; + wh_url << "&rank=" << nnti_comm_rank_; return wh_url.str(); } @@ -1694,14 +1697,12 @@ mpi_transport::execute_cmd_op( nnti::core::mpi_cmd_op *cmd_op) { NNTI_result_t rc = NNTI_OK; - int mpi_rc=0; log_debug("mpi_transport", "execute_cmd_op() - enter"); log_debug("mpi_transport", "looking up connection for peer pid=%016lX", work_id->wr().peer_pid()); nnti::datatype::mpi_peer *peer = (nnti::datatype::mpi_peer *)work_id->wr().peer(); - nnti::core::mpi_connection *conn = (nnti::core::mpi_connection *)peer->conn(); nnti::datatype::mpi_buffer *local_buffer = (nnti::datatype::mpi_buffer *)work_id->wr().local_hdl(); uint64_t local_offset = work_id->wr().local_offset(); @@ -1735,7 +1736,7 @@ mpi_transport::execute_cmd_op( cmd_op->toString().c_str(), local_buffer->payload(), local_offset, op_length, peer->rank(), local_buffer->cmd_tag()); std::unique_lock mpi_lock(mpi_mutex_); - mpi_rc = MPI_Issend((char*)local_buffer->payload() + local_offset, + MPI_Issend((char*)local_buffer->payload() + local_offset, op_length, MPI_BYTE, peer->rank(), @@ -1750,7 +1751,7 @@ mpi_transport::execute_cmd_op( cmd_op->toString().c_str(), cmd_op->cmd_msg(), cmd_op->cmd_msg_size(), peer->rank(), NNTI_MPI_CMD_TAG); std::unique_lock mpi_lock(mpi_mutex_); - mpi_rc=MPI_Issend(cmd_op->cmd_msg(), + MPI_Issend(cmd_op->cmd_msg(), cmd_op->cmd_msg_size(), MPI_BYTE, peer->rank(), @@ -1821,7 +1822,6 @@ mpi_transport::execute_rdma_op( std::unique_lock mpi_lock(mpi_mutex_, std::defer_lock); nnti::datatype::mpi_peer *peer = (nnti::datatype::mpi_peer *)work_id->wr().peer(); - nnti::core::mpi_connection *conn = (nnti::core::mpi_connection *)peer->conn(); nnti::datatype::mpi_buffer *local_buffer = (nnti::datatype::mpi_buffer *)work_id->wr().local_hdl(); uint64_t local_offset = work_id->wr().local_offset(); @@ -1953,12 +1953,10 @@ mpi_transport::execute_atomic_op( nnti::core::mpi_cmd_op *atomic_op) { NNTI_result_t rc = NNTI_OK; - int mpi_rc = MPI_SUCCESS; log_debug("mpi_transport", "execute_atomic_op() - enter"); nnti::datatype::mpi_peer *peer = (nnti::datatype::mpi_peer *)work_id->wr().peer(); - nnti::core::mpi_connection *conn = (nnti::core::mpi_connection *)peer->conn(); nnti::datatype::mpi_buffer *local_buffer = (nnti::datatype::mpi_buffer *)work_id->wr().local_hdl(); uint64_t local_offset = work_id->wr().local_offset(); @@ -1985,7 +1983,7 @@ mpi_transport::execute_atomic_op( } std::unique_lock mpi_lock(mpi_mutex_); - mpi_rc = MPI_Irecv((char*)local_buffer->payload() + local_offset, + MPI_Irecv((char*)local_buffer->payload() + local_offset, sizeof(int64_t), MPI_BYTE, peer->rank(), @@ -1999,7 +1997,7 @@ mpi_transport::execute_atomic_op( log_debug("mpi_transport", "posting atomic_op(%s)", atomic_op->toString().c_str()); mpi_lock.lock(); - mpi_rc = MPI_Issend(atomic_op->cmd_msg(), + MPI_Issend(atomic_op->cmd_msg(), atomic_op->cmd_msg_size(), MPI_BYTE, peer->rank(), @@ -2020,7 +2018,6 @@ mpi_transport::execute_atomic_op( NNTI_result_t mpi_transport::complete_send_command(nnti::core::mpi_cmd_msg *cmd_msg) { - int mpi_rc = MPI_SUCCESS; NNTI_result_t nnti_rc = NNTI_OK; log_debug("mpi_transport", "complete_send_command() - enter"); @@ -2088,14 +2085,14 @@ mpi_transport::complete_send_command(nnti::core::mpi_cmd_msg *cmd_msg) log_debug("mpi_transport", "long send Irecv()"); std::unique_lock mpi_lock(mpi_mutex_); - mpi_rc = MPI_Irecv((char*)target_buffer->payload() + cmd_msg->target_offset(), + MPI_Irecv((char*)target_buffer->payload() + cmd_msg->target_offset(), cmd_msg->payload_length(), MPI_BYTE, peer->rank(), initiator_buffer->cmd_tag(), MPI_COMM_WORLD, &req); - mpi_rc = MPI_Wait(&req, &status); + MPI_Wait(&req, &status); mpi_lock.unlock(); log_debug("mpi_transport", "long send Wait() complete"); @@ -2128,7 +2125,6 @@ NNTI_result_t mpi_transport::complete_get_command(nnti::core::mpi_cmd_msg *cmd_msg) { int mpi_rc = MPI_SUCCESS; - NNTI_result_t nnti_rc = NNTI_OK; log_debug("mpi_transport", "complete_get_command() - enter"); @@ -2136,9 +2132,6 @@ mpi_transport::complete_get_command(nnti::core::mpi_cmd_msg *cmd_msg) nnti::datatype::mpi_buffer *target_buffer = cmd_msg->target_buffer(); nnti::datatype::mpi_peer *peer = cmd_msg->initiator_peer(); - MPI_Request req; - MPI_Status status; - std::unique_lock mpi_lock(mpi_mutex_); mpi_rc = MPI_Ssend((char*)target_buffer->payload() + cmd_msg->target_offset(), cmd_msg->payload_length(), @@ -2162,12 +2155,8 @@ mpi_transport::complete_get_command(nnti::core::mpi_cmd_msg *cmd_msg) NNTI_result_t mpi_transport::complete_put_command(nnti::core::mpi_cmd_msg *cmd_msg) { - int mpi_rc = MPI_SUCCESS; - NNTI_result_t nnti_rc = NNTI_OK; - log_debug("mpi_transport", "complete_put_command() - enter"); - nnti::datatype::mpi_buffer *initiator_buffer = cmd_msg->initiator_buffer(); nnti::datatype::mpi_buffer *target_buffer = cmd_msg->target_buffer(); nnti::datatype::mpi_peer *peer = cmd_msg->initiator_peer(); @@ -2175,14 +2164,14 @@ mpi_transport::complete_put_command(nnti::core::mpi_cmd_msg *cmd_msg) MPI_Status status; std::unique_lock mpi_lock(mpi_mutex_); - mpi_rc = MPI_Irecv((char*)target_buffer->payload() + cmd_msg->target_offset(), + MPI_Irecv((char*)target_buffer->payload() + cmd_msg->target_offset(), cmd_msg->payload_length(), MPI_BYTE, peer->rank(), target_buffer->put_tag(), MPI_COMM_WORLD, &req); - mpi_rc = MPI_Wait(&req, &status); + MPI_Wait(&req, &status); mpi_lock.unlock(); cmd_msg->post_recv(); @@ -2202,9 +2191,6 @@ mpi_transport::complete_fadd_command(nnti::core::mpi_cmd_msg *cmd_msg) }; struct atomic_header *h = (struct atomic_header *)cmd_msg->eager_payload(); - int mpi_rc = MPI_SUCCESS; - NNTI_result_t nnti_rc = NNTI_OK; - log_debug("mpi_transport", "complete_fadd_command() - enter"); nnti::datatype::mpi_buffer *initiator_buffer = cmd_msg->initiator_buffer(); @@ -2223,14 +2209,14 @@ mpi_transport::complete_fadd_command(nnti::core::mpi_cmd_msg *cmd_msg) log_debug("mpi_transport", "sending old value back ; current=%ld ; atomic_tag=%d", current, initiator_buffer->atomic_tag()); std::unique_lock mpi_lock(mpi_mutex_); - mpi_rc = MPI_Issend(¤t, + MPI_Issend(¤t, sizeof(int64_t), MPI_BYTE, peer->rank(), initiator_buffer->atomic_tag(), MPI_COMM_WORLD, &req); - mpi_rc = MPI_Wait(&req, &status); + MPI_Wait(&req, &status); mpi_lock.unlock(); cmd_msg->post_recv(); @@ -2252,9 +2238,6 @@ mpi_transport::complete_cswap_command(nnti::core::mpi_cmd_msg *cmd_msg) }; struct atomic_header *h = (struct atomic_header *)cmd_msg->eager_payload(); - int mpi_rc = MPI_SUCCESS; - NNTI_result_t nnti_rc = NNTI_OK; - log_debug("mpi_transport", "complete_cswap_command() - enter"); nnti::datatype::mpi_buffer *initiator_buffer = cmd_msg->initiator_buffer(); @@ -2275,14 +2258,14 @@ mpi_transport::complete_cswap_command(nnti::core::mpi_cmd_msg *cmd_msg) log_debug("mpi_transport", "sending old value back ; current=%ld ; atomic_tag=%d", current, initiator_buffer->atomic_tag()); std::unique_lock mpi_lock(mpi_mutex_); - mpi_rc = MPI_Issend(¤t, + MPI_Issend(¤t, sizeof(int64_t), MPI_BYTE, peer->rank(), initiator_buffer->atomic_tag(), MPI_COMM_WORLD, &req); - mpi_rc = MPI_Wait(&req, &status); + MPI_Wait(&req, &status); mpi_lock.unlock(); cmd_msg->post_recv(); @@ -2438,8 +2421,6 @@ mpi_transport::progress_op_requests(void) log_debug("mpi_transport", "\terror = %d", event.MPI_ERROR); log_debug("mpi_transport", "}"); - bool need_event = true; - std::unique_lock mpi_lock(mpi_mutex_, std::defer_lock); nnti::datatype::nnti_work_request &wr = cmd_op->wid()->wr(); switch (wr.op()) { diff --git a/src/nnti/transports/mpi/mpi_transport.hpp b/src/nnti/transports/mpi/mpi_transport.hpp index 7d42231..a1452a7 100644 --- a/src/nnti/transports/mpi/mpi_transport.hpp +++ b/src/nnti/transports/mpi/mpi_transport.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef MPI_TRANSPORT_HPP_ @@ -146,8 +146,7 @@ class mpi_transport /** * @brief Initialize NNTI to use a specific transport. * - * \param[in] my_url A string that describes the transport parameters. - * \param[out] trans_hdl A handle to the activated transport. + * \param[in] config A Configuration object that NNTI should use to configure itself. * \return A result code (NNTI_OK or an error) * */ @@ -171,8 +170,6 @@ class mpi_transport /** * @brief Indicates if a transport has been initialized. * - * \param[in] trans_id The ID of the transport to test. - * \param[out] is_init 1 if the transport is initialized, 0 otherwise. * \return A result code (NNTI_OK or an error) * */ @@ -199,7 +196,7 @@ class mpi_transport * */ NNTI_result_t - pid(NNTI_process_id_t *pid); + pid(NNTI_process_id_t *pid) override; /** * @brief Get attributes of the transport. @@ -265,7 +262,7 @@ class mpi_transport */ NNTI_result_t eq_destroy( - NNTI_event_queue_t eq); + NNTI_event_queue_t eq) override; /** * @brief Wait for an event to arrive on an event queue. @@ -273,6 +270,7 @@ class mpi_transport * \param[in] eq_list A list of event queues to wait on. * \param[in] eq_count The number of event queues in the list. * \param[in] timeout The amount of time (in milliseconds) to wait. + * \param[out] which The index of the EQ where the event occurred. * \param[out] event The details of the event. * \return A result code (NNTI_OK or an error) */ @@ -289,7 +287,7 @@ class mpi_transport * * \param[in] dst_hdl Buffer where the message is delivered. * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -301,10 +299,10 @@ class mpi_transport /** * @brief Retrieves a specific message from the unexpected list. * - * \param[in] unexpect_event Event describing the message to retrieve. - * \param[in] dst_hdl Buffer where the message is delivered. - * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[in] unexpected_event Event describing the message to retrieve. + * \param[in] dst_hdl Buffer where the message is delivered. + * \param[in] dst_offset Offset into dst_hdl where the message is delivered. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -358,7 +356,7 @@ class mpi_transport nnti::datatype::nnti_event_callback cb, void *cb_context, char **reg_ptr, - NNTI_buffer_t *reg_buf); + NNTI_buffer_t *reg_buf) override; /** * @brief Disables network operations on the block of memory and frees it. @@ -405,8 +403,8 @@ class mpi_transport /** * @brief Convert an NNTI peer to an NNTI_process_id_t. * - * \param[in] peer A handle to a peer that can be used for network operations. - * \param[out] pid Compact binary representation of a process's location on the network. + * \param[in] peer_hdl A handle to a peer that can be used for network operations. + * \param[out] pid Compact binary representation of a process's location on the network. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -417,8 +415,8 @@ class mpi_transport /** * @brief Convert an NNTI_process_id_t to an NNTI peer. * - * \param[in] pid Compact binary representation of a process's location on the network. - * \param[out] peer A handle to a peer that can be used for network operations. + * \param[in] pid Compact binary representation of a process's location on the network. + * \param[out] peer_hdl A handle to a peer that can be used for network operations. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -516,7 +514,7 @@ class mpi_transport * \return A result code (NNTI_OK or an error) */ NNTI_result_t - interrupt(); + interrupt() override; /** diff --git a/src/nnti/transports/null/null_transport.hpp b/src/nnti/transports/null/null_transport.hpp index 056f321..f5edf24 100644 --- a/src/nnti/transports/null/null_transport.hpp +++ b/src/nnti/transports/null/null_transport.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /** * @file null_transport.hpp @@ -56,6 +56,7 @@ class null_transport faodel::Configuration &config) { static null_transport *instance = new null_transport(config); + return instance; } @@ -64,30 +65,32 @@ class null_transport * * \return A result code (NNTI_OK or an error) */ - ~null_transport() override { + ~null_transport() override + { return; } NNTI_result_t - start(void) override { + start(void) override + { return NNTI_OK; } NNTI_result_t - stop(void) override { + stop(void) override + { return NNTI_OK; } /** * @brief Indicates if a transport has been initialized. * - * \param[in] trans_id The ID of the transport to test. - * \param[out] is_init 1 if the transport is initialized, 0 otherwise. * \return A result code (NNTI_OK or an error) * */ bool - initialized(void) override { + initialized(void) override + { return true; } @@ -101,7 +104,8 @@ class null_transport NNTI_result_t get_url( char *url, - const uint64_t maxlen) override { + const uint64_t maxlen) override + { return NNTI_OK; } @@ -113,8 +117,10 @@ class null_transport * */ NNTI_result_t - pid(NNTI_process_id_t *pid) override { + pid(NNTI_process_id_t *pid) override + { *pid = 0; + return NNTI_OK; } @@ -126,8 +132,10 @@ class null_transport * */ NNTI_result_t - attrs(NNTI_attrs_t *attrs) override { + attrs(NNTI_attrs_t *attrs) override + { memset(attrs, 0, sizeof(NNTI_attrs_t)); + return NNTI_OK; } @@ -143,10 +151,9 @@ class null_transport connect( const char *url, const int timeout, - NNTI_peer_t *peer_hdl) override { - nnti::datatype::nnti_peer *peer=nullptr; - - peer_hdl = (NNTI_peer_t *)peer; + NNTI_peer_t *peer_hdl) override + { + *peer_hdl = (NNTI_peer_t)nullptr; return NNTI_OK; } @@ -159,9 +166,8 @@ class null_transport */ NNTI_result_t disconnect( - NNTI_peer_t peer_hdl) override { - nnti::datatype::nnti_peer *peer = (nnti::datatype::nnti_peer *)peer_hdl; - + NNTI_peer_t peer_hdl) override + { return NNTI_OK; } @@ -177,9 +183,9 @@ class null_transport eq_create( uint64_t size, NNTI_eq_flags_t flags, - NNTI_event_queue_t *eq) override { + NNTI_event_queue_t *eq) override + { nnti::datatype::nnti_event_queue *new_eq = new nnti::datatype::nnti_event_queue(true, size, this); - *eq = (NNTI_event_queue_t)new_eq; return NNTI_OK; @@ -191,9 +197,9 @@ class null_transport NNTI_eq_flags_t flags, nnti::datatype::nnti_event_callback cb, void *cb_context, - NNTI_event_queue_t *eq) override { + NNTI_event_queue_t *eq) override + { nnti::datatype::nnti_event_queue *new_eq = new nnti::datatype::nnti_event_queue(true, size, cb, cb_context, this); - *eq = (NNTI_event_queue_t)new_eq; return NNTI_OK; @@ -207,7 +213,8 @@ class null_transport */ NNTI_result_t eq_destroy( - NNTI_event_queue_t eq) override { + NNTI_event_queue_t eq) override + { delete (nnti::datatype::nnti_event_queue *)eq; return NNTI_OK; @@ -219,6 +226,7 @@ class null_transport * \param[in] eq_list A list of event queues to wait on. * \param[in] eq_count The number of event queues in the list. * \param[in] timeout The amount of time (in milliseconds) to wait. + * \param[out] which The index of the EQ where the event occurred. * \param[out] event The details of the event. * \return A result code (NNTI_OK or an error) */ @@ -228,7 +236,8 @@ class null_transport const uint32_t eq_count, const int timeout, uint32_t *which, - NNTI_event_t *event) override { + NNTI_event_t *event) override + { return NNTI_OK; } @@ -237,26 +246,25 @@ class null_transport * * \param[in] dst_hdl Buffer where the message is delivered. * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t next_unexpected( NNTI_buffer_t dst_hdl, uint64_t dst_offset, - NNTI_event_t *result_event) override { - nnti::datatype::nnti_buffer *b = (nnti::datatype::nnti_buffer *)dst_hdl; - + NNTI_event_t *result_event) override + { return NNTI_OK; } /** * @brief Retrieves a specific message from the unexpected list. * - * \param[in] unexpect_event Event describing the message to retrieve. - * \param[in] dst_hdl Buffer where the message is delivered. - * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[in] unexpected_event Event describing the message to retrieve. + * \param[in] dst_hdl Buffer where the message is delivered. + * \param[in] dst_offset Offset into dst_hdl where the message is delivered. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -264,9 +272,8 @@ class null_transport NNTI_event_t *unexpected_event, NNTI_buffer_t dst_hdl, uint64_t dst_offset, - NNTI_event_t *result_event) override { - nnti::datatype::nnti_buffer *b = (nnti::datatype::nnti_buffer *)dst_hdl; - + NNTI_event_t *result_event) override + { return NNTI_OK; } @@ -278,7 +285,8 @@ class null_transport */ NNTI_result_t event_complete( - NNTI_event_t *event) override { + NNTI_event_t *event) override + { return NNTI_OK; } @@ -294,7 +302,8 @@ class null_transport dt_unpack( void *nnti_dt, char *packed_buf, - const uint64_t packed_len) override { + const uint64_t packed_len) override + { return NNTI_OK; } @@ -318,7 +327,8 @@ class null_transport nnti::datatype::nnti_event_callback cb, void *cb_context, char **reg_ptr, - NNTI_buffer_t *reg_buf) override { + NNTI_buffer_t *reg_buf) override + { nnti::datatype::nnti_buffer *b = new nnti::datatype::nnti_buffer(this, size, flags, @@ -339,9 +349,9 @@ class null_transport */ NNTI_result_t free( - NNTI_buffer_t reg_buf) override { + NNTI_buffer_t reg_buf) override + { nnti::datatype::nnti_buffer *b = (nnti::datatype::nnti_buffer *)reg_buf; - delete b; return NNTI_OK; @@ -367,7 +377,8 @@ class null_transport NNTI_event_queue_t eq, nnti::datatype::nnti_event_callback cb, void *cb_context, - NNTI_buffer_t *reg_buf) override { + NNTI_buffer_t *reg_buf) override + { nnti::datatype::nnti_buffer *b = new nnti::datatype::nnti_buffer(this, buffer, size, @@ -375,41 +386,11 @@ class null_transport eq, cb, cb_context); - *reg_buf = (NNTI_buffer_t)b; return NNTI_OK; } - /** - * @brief Prepare a list of memory buffers for network operations. - * - * \param[in] segment__list List of memory buffers. - * \param[in] segment_lengths The lengths of the buffers in segment_list. - * \param[in] segment_count The number of buffers in segment_list. - * \param[in] flags Control the behavior of this buffer. - * \param[in] eq Events occurring on the memory region are delivered to this event queue. - * \param[in] cb A callback that gets called for events delivered to eq. - * \param[in] cb_context A blob of data that is passed to each invocation of cb. - * \param[out] reg_buf A handle to a memory buffer that can be used for network operations. - * \return A result code (NNTI_OK or an error) - */ - NNTI_result_t - register_segments( - char **segment_list, - const uint64_t *segment_lengths, - const uint64_t segment_count, - const NNTI_buffer_flags_t flags, - NNTI_event_queue_t eq, - nnti::datatype::nnti_event_callback cb, - void *cb_context, - NNTI_buffer_t *reg_buf) - { - nnti::datatype::nnti_event_queue *buf_eq = (nnti::datatype::nnti_event_queue *)eq; - - return NNTI_OK; - } - /** * @brief Disables network operations on a memory buffer. * @@ -418,9 +399,9 @@ class null_transport */ NNTI_result_t unregister_memory( - NNTI_buffer_t reg_buf) override { + NNTI_buffer_t reg_buf) override + { nnti::datatype::nnti_buffer *b = (nnti::datatype::nnti_buffer *)reg_buf; - delete b; return NNTI_OK; @@ -429,28 +410,30 @@ class null_transport /** * @brief Convert an NNTI peer to an NNTI_process_id_t. * - * \param[in] peer A handle to a peer that can be used for network operations. - * \param[out] pid Compact binary representation of a process's location on the network. + * \param[in] peer_hdl A handle to a peer that can be used for network operations. + * \param[out] pid Compact binary representation of a process's location on the network. * \return A result code (NNTI_OK or an error) */ NNTI_result_t dt_peer_to_pid( NNTI_peer_t peer_hdl, - NNTI_process_id_t *pid) override { + NNTI_process_id_t *pid) override + { return NNTI_OK; } /** * @brief Convert an NNTI_process_id_t to an NNTI peer. * - * \param[in] pid Compact binary representation of a process's location on the network. - * \param[out] peer A handle to a peer that can be used for network operations. + * \param[in] pid Compact binary representation of a process's location on the network. + * \param[out] peer_hdl A handle to a peer that can be used for network operations. * \return A result code (NNTI_OK or an error) */ NNTI_result_t dt_pid_to_peer( NNTI_process_id_t pid, - NNTI_peer_t *peer_hdl) override { + NNTI_peer_t *peer_hdl) override + { return NNTI_OK; } @@ -464,9 +447,9 @@ class null_transport NNTI_result_t send( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid) override { + NNTI_work_id_t *wid) override + { nnti::datatype::nnti_work_id *work_id = new nnti::datatype::nnti_work_id(*wr); - *wid = (NNTI_work_id_t)work_id; return NNTI_OK; @@ -482,9 +465,9 @@ class null_transport NNTI_result_t put( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid) override { + NNTI_work_id_t *wid) override + { nnti::datatype::nnti_work_id *work_id = new nnti::datatype::nnti_work_id(*wr); - *wid = (NNTI_work_id_t)work_id; return NNTI_OK; @@ -500,9 +483,9 @@ class null_transport NNTI_result_t get( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid) override { + NNTI_work_id_t *wid) override + { nnti::datatype::nnti_work_id *work_id = new nnti::datatype::nnti_work_id(*wr); - *wid = (NNTI_work_id_t)work_id; return NNTI_OK; @@ -518,9 +501,9 @@ class null_transport NNTI_result_t atomic_fop( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid) override { + NNTI_work_id_t *wid) override + { nnti::datatype::nnti_work_id *work_id = new nnti::datatype::nnti_work_id(*wr); - *wid = (NNTI_work_id_t)work_id; return NNTI_OK; @@ -536,9 +519,9 @@ class null_transport NNTI_result_t atomic_cswap( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid) override { + NNTI_work_id_t *wid) override + { nnti::datatype::nnti_work_id *work_id = new nnti::datatype::nnti_work_id(*wr); - *wid = (NNTI_work_id_t)work_id; return NNTI_OK; @@ -552,10 +535,8 @@ class null_transport */ NNTI_result_t cancel( - NNTI_work_id_t wid) + NNTI_work_id_t wid) override { - nnti::datatype::nnti_work_id *work_id = (nnti::datatype::nnti_work_id *)wid; - return NNTI_OK; } @@ -570,7 +551,8 @@ class null_transport NNTI_result_t cancelall( NNTI_work_id_t *wid_list, - const uint32_t wid_count) override { + const uint32_t wid_count) override + { return NNTI_OK; } @@ -581,7 +563,8 @@ class null_transport * \return A result code (NNTI_OK or an error) */ NNTI_result_t - interrupt() override { + interrupt() override + { return NNTI_OK; } @@ -598,9 +581,8 @@ class null_transport wait( NNTI_work_id_t wid, const int64_t timeout, - NNTI_status_t *status) override { - nnti::datatype::nnti_work_id *work_id = (nnti::datatype::nnti_work_id *)wid; - + NNTI_status_t *status) override + { return NNTI_OK; } @@ -620,7 +602,8 @@ class null_transport const uint32_t wid_count, const int64_t timeout, uint32_t *which, - NNTI_status_t *status) override { + NNTI_status_t *status) override + { return NNTI_OK; } @@ -638,7 +621,8 @@ class null_transport NNTI_work_id_t *wid_list, const uint32_t wid_count, const int64_t timeout, - NNTI_status_t *status) override { + NNTI_status_t *status) override + { return NNTI_OK; } diff --git a/src/nnti/transports/ugni/ugni_atomic_op.hpp b/src/nnti/transports/ugni/ugni_atomic_op.hpp index 226872a..69e7fac 100644 --- a/src/nnti/transports/ugni/ugni_atomic_op.hpp +++ b/src/nnti/transports/ugni/ugni_atomic_op.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_ATOMIC_OP_HPP_ @@ -75,8 +75,6 @@ class ugni_atomic_op void set(nnti::datatype::nnti_work_id *wid) { - const nnti::datatype::ugni_work_request &wr = (const nnti::datatype::ugni_work_request &)wid->wr(); - id_ = next_id_.fetch_add(1); wid_ = wid; state_ = op_state::INIT; @@ -261,7 +259,7 @@ class ugni_atomic_op return op_state::CLEANUP; } - op_state + void update_stats(void) { nnti::datatype::nnti_work_request &wr = wid_->wr(); diff --git a/src/nnti/transports/ugni/ugni_buffer.cpp b/src/nnti/transports/ugni/ugni_buffer.cpp index 7401037..bd0429e 100644 --- a/src/nnti/transports/ugni/ugni_buffer.cpp +++ b/src/nnti/transports/ugni/ugni_buffer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -212,8 +212,10 @@ ugni_buffer::unregister_buffer(void) switch(rc) { case GNI_RC_SUCCESS: rc=NNTI_OK; + break; default: rc=NNTI_EIO; + break; } log_debug("ugni_buffer", "exit mem_hdl(%p)", &hdl_); diff --git a/src/nnti/transports/ugni/ugni_buffer.hpp b/src/nnti/transports/ugni/ugni_buffer.hpp index b082123..d4c5ae5 100644 --- a/src/nnti/transports/ugni/ugni_buffer.hpp +++ b/src/nnti/transports/ugni/ugni_buffer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_BUFFER_HPP_ diff --git a/src/nnti/transports/ugni/ugni_cmd_buffer.cpp b/src/nnti/transports/ugni/ugni_cmd_buffer.cpp index 8eb6ecd..4f28287 100644 --- a/src/nnti/transports/ugni/ugni_cmd_buffer.cpp +++ b/src/nnti/transports/ugni/ugni_cmd_buffer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" diff --git a/src/nnti/transports/ugni/ugni_cmd_buffer.hpp b/src/nnti/transports/ugni/ugni_cmd_buffer.hpp index 4f68bad..e1bb856 100644 --- a/src/nnti/transports/ugni/ugni_cmd_buffer.hpp +++ b/src/nnti/transports/ugni/ugni_cmd_buffer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_CMD_BUFFER_HPP_ diff --git a/src/nnti/transports/ugni/ugni_cmd_msg.cpp b/src/nnti/transports/ugni/ugni_cmd_msg.cpp index 2713073..b4b99c6 100644 --- a/src/nnti/transports/ugni/ugni_cmd_msg.cpp +++ b/src/nnti/transports/ugni/ugni_cmd_msg.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -25,12 +25,12 @@ ugni_cmd_msg::ugni_cmd_msg( nnti::transports::ugni_transport *transport, const uint32_t cmd_msg_size) : transport_(transport), - initiator_peer_(nullptr), - target_peer_(nullptr), cmd_msg_buf_(nullptr), cmd_msg_size_(cmd_msg_size), free_in_dtor_(true), - unexpected_(false) + unexpected_(false), + initiator_peer_(nullptr), + target_peer_(nullptr) { get_cmd_msg_buffer(); @@ -42,12 +42,12 @@ ugni_cmd_msg::ugni_cmd_msg( uint32_t id, nnti::datatype::nnti_work_id *wid) : transport_(transport), - initiator_peer_(nullptr), - target_peer_(nullptr), cmd_msg_buf_(nullptr), cmd_msg_size_(cmd_msg_size), free_in_dtor_(true), - unexpected_(false) + unexpected_(false), + initiator_peer_(nullptr), + target_peer_(nullptr) { get_cmd_msg_buffer(); pack(id, wid); @@ -59,12 +59,12 @@ ugni_cmd_msg::ugni_cmd_msg( uint32_t id, nnti::datatype::nnti_work_id *wid) : transport_(transport), - initiator_peer_(nullptr), - target_peer_(nullptr), cmd_msg_buf_(nullptr), cmd_msg_size_(2048), free_in_dtor_(false), - unexpected_(false) + unexpected_(false), + initiator_peer_(nullptr), + target_peer_(nullptr) { if (wid->wr().flags() & NNTI_OF_ZERO_COPY) { nnti::datatype::ugni_buffer *b = (nnti::datatype::ugni_buffer *)wid->wr().local_hdl(); @@ -85,12 +85,12 @@ ugni_cmd_msg::ugni_cmd_msg( uint32_t buf_size, bool copy_buf) : transport_(transport), - initiator_peer_(nullptr), - target_peer_(nullptr), cmd_msg_buf_(nullptr), cmd_msg_size_(buf_size), free_in_dtor_(copy_buf), - unexpected_(false) + unexpected_(false), + initiator_peer_(nullptr), + target_peer_(nullptr) { get_cmd_msg_buffer(); if (copy_buf) { @@ -309,6 +309,7 @@ ugni_cmd_msg::pack( uint32_t id, nnti::datatype::nnti_work_id *wid) { + const uint32_t MSG_ALIGNMENT = NNTI_UGNI_RDMA_ALIGNMENT; const nnti::datatype::nnti_work_request &wr = wid->wr(); nnti::datatype::ugni_buffer *buf; @@ -329,11 +330,54 @@ ugni_cmd_msg::pack( buf = (nnti::datatype::ugni_buffer *)wr.local_hdl(); buf->pack(cmd_msg_buf_->packed_initiator_hdl, packed_buffer_size_); - if (!(wid->wr().flags() & NNTI_OF_ZERO_COPY) && eager()) { - // message is small, use eager - log_debug("ugni_cmd_msg", "payload=%p offset=%lu length=%lu", - buf->payload(), cmd_msg_buf_->initiator_offset, cmd_msg_buf_->payload_length); - memcpy(cmd_msg_buf_->eager_payload, buf->payload()+cmd_msg_buf_->initiator_offset, cmd_msg_buf_->payload_length); + if (!(wid->wr().flags() & NNTI_OF_ZERO_COPY)) { + if (eager()) { + // message is small, use eager + log_debug("ugni_cmd_msg", "payload=%p offset=%lu length=%lu", + buf->payload(), cmd_msg_buf_->initiator_offset, cmd_msg_buf_->payload_length); + memcpy(cmd_msg_buf_->eager_payload, + buf->payload()+cmd_msg_buf_->initiator_offset, + cmd_msg_buf_->payload_length); + } else { + // message is not small, check for alignment + uint32_t addr_alignment = (MSG_ALIGNMENT - (((uintptr_t)buf->payload()+cmd_msg_buf_->initiator_offset)%MSG_ALIGNMENT)) % MSG_ALIGNMENT; + if (addr_alignment != 0) { + log_debug("ugni_cmd_msg", + "long send address is not 4-byte aligned (%p %% %u = %u); sending first %u bytes in eager payload.", + ((uintptr_t)buf->payload()+cmd_msg_buf_->initiator_offset), + MSG_ALIGNMENT, + ((uintptr_t)buf->payload()+cmd_msg_buf_->initiator_offset)%MSG_ALIGNMENT, + addr_alignment); + + memcpy(cmd_msg_buf_->eager_payload, + buf->payload()+cmd_msg_buf_->initiator_offset, + addr_alignment); + } + uint32_t extra = (cmd_msg_buf_->payload_length - addr_alignment) % MSG_ALIGNMENT; + if (extra != 0) { + // message length isn't 4-byte alignment, send trailing bytes in eager + log_debug("ugni_cmd_msg", + "long send length is not 4-byte aligned (%lu %% %u = %u); sending last %u bytes in eager payload.", + cmd_msg_buf_->payload_length - addr_alignment, + MSG_ALIGNMENT, + (cmd_msg_buf_->payload_length - addr_alignment)%MSG_ALIGNMENT, + extra); + + memcpy(cmd_msg_buf_->eager_payload+addr_alignment, + buf->payload()+cmd_msg_buf_->initiator_offset+cmd_msg_buf_->payload_length-extra, + extra); + } + + uint64_t aligned_local_addr = ((uintptr_t)buf->payload()+cmd_msg_buf_->initiator_offset) + addr_alignment; + uint64_t aligned_length = cmd_msg_buf_->payload_length - addr_alignment - extra; + + log_debug("ugni_cmd_msg", + "\nlong get RDMA summary:\n" + "\taligned_local_addr = %p (aligned? %c)\n" + "\taligned_length = %lu (aligned? %c)\n", + aligned_local_addr, (!(aligned_local_addr%MSG_ALIGNMENT))?'Y':'N', + aligned_length, (!(aligned_length%MSG_ALIGNMENT))?'Y':'N'); + } } } else { *(uint32_t*)cmd_msg_buf_->packed_initiator_hdl = 0; diff --git a/src/nnti/transports/ugni/ugni_cmd_msg.hpp b/src/nnti/transports/ugni/ugni_cmd_msg.hpp index b705915..9db669f 100644 --- a/src/nnti/transports/ugni/ugni_cmd_msg.hpp +++ b/src/nnti/transports/ugni/ugni_cmd_msg.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_CMD_MSG_HPP_ diff --git a/src/nnti/transports/ugni/ugni_cmd_op.cpp b/src/nnti/transports/ugni/ugni_cmd_op.cpp index ed60af9..560cd52 100644 --- a/src/nnti/transports/ugni/ugni_cmd_op.cpp +++ b/src/nnti/transports/ugni/ugni_cmd_op.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -241,7 +241,7 @@ namespace core { return op_state::CLEANUP; } - ugni_cmd_op::op_state + void ugni_cmd_op::update_stats(void) { nnti::datatype::nnti_work_request &wr = wid_->wr(); diff --git a/src/nnti/transports/ugni/ugni_cmd_op.hpp b/src/nnti/transports/ugni/ugni_cmd_op.hpp index 26b5ecc..4456db9 100644 --- a/src/nnti/transports/ugni/ugni_cmd_op.hpp +++ b/src/nnti/transports/ugni/ugni_cmd_op.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_CMD_OP_HPP_ @@ -111,7 +111,7 @@ class ugni_cmd_op create_event(void); op_state issue_send_event(void); - op_state + void update_stats(void); public: diff --git a/src/nnti/transports/ugni/ugni_cmd_tgt.hpp b/src/nnti/transports/ugni/ugni_cmd_tgt.hpp index e438cc8..683644a 100644 --- a/src/nnti/transports/ugni/ugni_cmd_tgt.hpp +++ b/src/nnti/transports/ugni/ugni_cmd_tgt.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_CMD_TGT_HPP_ @@ -76,8 +76,8 @@ class ugni_cmd_tgt bool copy_buf) : transport_(transport), cmd_msg_(transport, buf, buf_size, copy_buf), - state_(msg_state::INIT), event_(nullptr), + state_(msg_state::INIT), active_entries_(0) { sm_lock_ = faodel::GenerateMutex("pthreads"); @@ -265,7 +265,7 @@ class ugni_cmd_tgt }; msg_state state_; - faodel::MutexWrapper *sm_lock_; + faodel::MutexWrapper *sm_lock_; std::atomic active_entries_; @@ -386,7 +386,7 @@ class ugni_cmd_tgt msg_state unexpected_long_get() { - NNTI_result_t rc = NNTI_OK; + const uint32_t MSG_ALIGNMENT = NNTI_UGNI_RDMA_ALIGNMENT; int gni_rc; gni_cq_entry_t ev_data; @@ -398,29 +398,75 @@ class ugni_cmd_tgt gni_post_descriptor_t post_desc; NNTI_ugni_mem_hdl_p_t mem_hdl; + + uint32_t addr_alignment = (MSG_ALIGNMENT - (((uint64_t)init_buf->payload() + this->initiator_offset()) % MSG_ALIGNMENT)) % MSG_ALIGNMENT; + if (addr_alignment != 0) { + log_debug("ugni_cmd_tgt", + "long send address is not 4-byte aligned (%p %% %u = %u); copying first %u bytes from eager payload.", + ((uint64_t)init_buf->payload() + this->initiator_offset()), + MSG_ALIGNMENT, + ((uint64_t)init_buf->payload() + this->initiator_offset())%MSG_ALIGNMENT, + addr_alignment); + + memcpy(dst_buf->payload() + unexpected_dst_offset_, + this->eager_payload(), + addr_alignment); + } + + uint32_t extra = (this->payload_length()-addr_alignment) % MSG_ALIGNMENT; + if (extra > 0) { + // message length isn't 4-byte alignment, get trailing bytes from eager + log_debug("ugni_cmd_tgt", + "long send length is not 4-byte aligned (%lu %% %u = %u); copying last %u bytes from eager payload.", + this->payload_length()-addr_alignment, + MSG_ALIGNMENT, + (this->payload_length()-addr_alignment)%MSG_ALIGNMENT, + extra); + + memcpy(dst_buf->payload() + unexpected_dst_offset_ + this->payload_length() - extra, + this->eager_payload() + addr_alignment, + extra); + } + + uint64_t aligned_local_addr = (uint64_t)dst_buf->payload() + unexpected_dst_offset_ + addr_alignment; + uint64_t aligned_remote_addr = (uint64_t)init_buf->payload() + this->initiator_offset() + addr_alignment; + uint64_t aligned_length = this->payload_length() - addr_alignment - extra; + + log_debug("ugni_cmd_tgt", + "\nlong get RDMA summary:\n" + "\taligned_local_addr = %p (aligned? %c)\n" + "\taligned_remote_addr = %p (aligned? %c)\n" + "\taligned_length = %lu (aligned? %c)\n", + aligned_local_addr, (!(aligned_local_addr%MSG_ALIGNMENT))?'Y':'N', + aligned_remote_addr, (!(aligned_remote_addr%MSG_ALIGNMENT))?'Y':'N', + aligned_length, (!(aligned_length%MSG_ALIGNMENT))?'Y':'N'); memset(&post_desc, 0, sizeof(gni_post_descriptor_t)); // post_desc.src_cq_hndl = unexpected_cq_hdl_; mem_hdl = dst_buf->mem_hdl(); - post_desc.local_addr = (uint64_t)dst_buf->payload() + unexpected_dst_offset_; + post_desc.local_addr = aligned_local_addr; post_desc.local_mem_hndl.qword1 = mem_hdl.qword1; post_desc.local_mem_hndl.qword2 = mem_hdl.qword2; mem_hdl = init_buf->mem_hdl(); - post_desc.remote_addr = (uint64_t)init_buf->payload() + this->initiator_offset(); + post_desc.remote_addr = aligned_remote_addr; post_desc.remote_mem_hndl.qword1 = mem_hdl.qword1; post_desc.remote_mem_hndl.qword2 = mem_hdl.qword2; - post_desc.length = this->payload_length(); + post_desc.length = aligned_length; post_desc.type = GNI_POST_RDMA_GET; post_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT | GNI_CQMODE_REMOTE_EVENT; post_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; - log_debug("ugni_cmd_tgt", "calling PostRdma(rdma get ; ep_hdl(%llu) transport_global_data.ep_cq_hdl(%llu) local_mem_hdl(%llu, %llu) remote_mem_hdl(%llu, %llu))", - conn->unexpected_ep_hdl(), conn->unexpected_cq_hdl(), + this->post_desc(&post_desc); + + this->index = transport_->msg_vector_.add(this); + + log_debug("ugni_cmd_tgt", "calling PostRdma(rdma get ; ep_hdl(%llu) transport_.unexpected_long_get_ep_cq_hdl_(%llu) local_mem_hdl(%llu, %llu) remote_mem_hdl(%llu, %llu))", + conn->unexpected_ep_hdl(), transport_->unexpected_long_get_ep_cq_hdl_, post_desc.local_mem_hndl.qword1, post_desc.local_mem_hndl.qword2, post_desc.remote_mem_hndl.qword1, post_desc.remote_mem_hndl.qword2); @@ -436,18 +482,24 @@ class ugni_cmd_tgt nthread_unlock(&transport_->ugni_lock_); if (gni_rc != GNI_RC_SUCCESS) { log_error("ugni_cmd_tgt", "failed to post BTE (gni_rc=%d): %s", gni_rc, strerror(errno)); - rc=NNTI_EIO; } log_debug("ugni_cmd_tgt", "called PostRdma(rdma get)"); nthread_lock(&transport_->ugni_lock_); log_debug("ugni_cmd_tgt", "calling CqWaitEvent(unexpected_cq_hdl)"); - gni_rc=GNI_CqWaitEvent(conn->unexpected_cq_hdl(), -1, &ev_data); + gni_rc=GNI_CqWaitEvent(transport_->unexpected_long_get_ep_cq_hdl_, -1, &ev_data); log_debug("ugni_cmd_tgt", "called CqWaitEvent(unexpected_cq_hdl)"); nthread_unlock(&transport_->ugni_lock_); + if (gni_rc!=GNI_RC_SUCCESS) { + log_error("ugni_cmd_tgt", "CqWaitEvent(unexpected_cq_hdl) failed: %d", gni_rc); + } else { + log_debug("ugni_cmd_tgt", "CqWaitEvent(unexpected_cq_hdl) success"); + } + + log_debug("ugni_cmd_tgt", "got event for cmd_tgt with index %d ; my index is %d", GNI_CQ_GET_INST_ID(ev_data), this->index); nthread_lock(&transport_->ugni_lock_); - gni_rc=GNI_GetCompleted(conn->unexpected_cq_hdl(), ev_data, &post_desc_ptr); + gni_rc=GNI_GetCompleted(transport_->unexpected_long_get_ep_cq_hdl_, ev_data, &post_desc_ptr); nthread_unlock(&transport_->ugni_lock_); if (gni_rc!=GNI_RC_SUCCESS) { log_error("ugni_cmd_tgt", "GetCompleted(next_unexpected(%p)) failed: %d", post_desc_ptr, gni_rc); @@ -456,6 +508,8 @@ class ugni_cmd_tgt } // print_post_desc(post_desc_ptr); + transport_->msg_vector_.remove(this->index); + NNTI_FAST_STAT(transport_->stats_->long_recvs++;); return msg_state::UNEXPECTED_LONG_GET_COMPLETE; @@ -623,7 +677,7 @@ class ugni_cmd_tgt event_ = nullptr; } - inline msg_state state_update(msg_state new_state) { + inline void state_update(msg_state new_state) { state_ = new_state; } public: diff --git a/src/nnti/transports/ugni/ugni_connection.cpp b/src/nnti/transports/ugni/ugni_connection.cpp index 2a263d7..5e8ed85 100644 --- a/src/nnti/transports/ugni/ugni_connection.cpp +++ b/src/nnti/transports/ugni/ugni_connection.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -35,8 +35,6 @@ namespace core { cmd_msg_count_(cmd_msg_count), smsg_waitlisted(false) { - int rc; - setup_mailbox(); setup_rdma(); } @@ -47,24 +45,25 @@ namespace core { const std::map &peer) : nnti_connection(), transport_(transport), + peer_params_(peer), cmd_msg_size_(cmd_msg_size), cmd_msg_count_(cmd_msg_count), - peer_params_(peer), smsg_waitlisted(false) { - int rc; - nnti::core::nnti_url url = nnti::core::nnti_url(peer_params_.hostname, peer_params_.port); peer_pid_ = url.pid(); peer_ = new nnti::datatype::nnti_peer(transport, url); peer_->conn(this); + fingerprint_ = peer_params_.fingerprint; + setup_mailbox(); setup_rdma(); log_debug("", "hostname = %s", peer_params_.hostname.c_str()); log_debug("", "addr = %lu", peer_params_.addr); log_debug("", "port = %d", peer_params_.port); + log_debug("", "fingerprint = %s", peer_params_.fingerprint.c_str()); log_debug("", "local_addr = %d", peer_params_.local_addr); log_debug("", "instance = %d", peer_params_.instance); log_debug("", "smsg_msg_buffer = %llu", peer_params_.smsg_msg_buffer); @@ -84,16 +83,17 @@ namespace core { ugni_connection::peer_params( const std::map ¶ms) { - int rc; - peer_params_ = connection_params(params); nnti::core::nnti_url url = nnti::core::nnti_url(peer_params_.hostname, peer_params_.port); peer_pid_ = url.pid(); + fingerprint_ = peer_params_.fingerprint; + log_debug("", "hostname = %s", peer_params_.hostname.c_str()); log_debug("", "addr = %lu", peer_params_.addr); log_debug("", "port = %d", peer_params_.port); + log_debug("", "fingerprint = %s", peer_params_.fingerprint.c_str()); log_debug("", "local_addr = %d", peer_params_.local_addr); log_debug("", "instance = %d", peer_params_.instance); log_debug("", "smsg_msg_buffer = %llu", peer_params_.smsg_msg_buffer); @@ -106,7 +106,6 @@ namespace core { ugni_connection::peer_params( const std::string ¶ms) { - int rc; std::map param_map; std::istringstream iss(params); @@ -121,9 +120,12 @@ namespace core { nnti::core::nnti_url url = nnti::core::nnti_url(peer_params_.hostname, peer_params_.port); peer_pid_ = url.pid(); + fingerprint_ = peer_params_.fingerprint; + log_debug("", "hostname = %s", peer_params_.hostname.c_str()); log_debug("", "addr = %lu", peer_params_.addr); log_debug("", "port = %d", peer_params_.port); + log_debug("", "fingerprint = %s", peer_params_.fingerprint.c_str()); log_debug("", "local_addr = %d", peer_params_.local_addr); log_debug("", "instance = %d", peer_params_.instance); log_debug("", "smsg_msg_buffer = %llu", peer_params_.smsg_msg_buffer); @@ -207,11 +209,11 @@ namespace core { log_debug("ugni_connection", "rdma_ep_hdl_(%llu) bound to instance(%llu) at local_addr(%llu)", rdma_ep_hdl_, peer_params_.instance, peer_params_.local_addr); nthread_lock(&transport_->ugni_lock_); - gni_rc = GNI_CqCreate (transport_->nic_hdl_, 64, 0, GNI_CQ_BLOCKING, NULL, NULL, &unexpected_ep_cq_hdl_); - if (gni_rc != GNI_RC_SUCCESS) { - log_error("ugni_transport", "CqCreate(unexpected_ep_cq_hdl_) failed: %d", gni_rc); - } - gni_rc = GNI_EpCreate(transport_->nic_hdl_, unexpected_ep_cq_hdl_, &unexpected_ep_hdl_); +// gni_rc = GNI_CqCreate (transport_->nic_hdl_, 64, 0, GNI_CQ_BLOCKING, NULL, NULL, &unexpected_ep_cq_hdl_); +// if (gni_rc != GNI_RC_SUCCESS) { +// log_error("ugni_transport", "CqCreate(unexpected_ep_cq_hdl_) failed: %d", gni_rc); +// } + gni_rc = GNI_EpCreate(transport_->nic_hdl_, transport_->unexpected_long_get_ep_cq_hdl_, &unexpected_ep_hdl_); if (gni_rc != GNI_RC_SUCCESS) { log_error("ugni_mailbox", "EpCreate(unexpected_ep_hdl_) failed: %d", gni_rc); } @@ -258,7 +260,7 @@ namespace core { gni_cq_handle_t ugni_connection::unexpected_cq_hdl(void) { - return unexpected_ep_cq_hdl_; + return 0; //unexpected_ep_cq_hdl_; } bool @@ -273,6 +275,7 @@ namespace core { std::lock_guard lock(smsg_waitlist_lock_); smsg_waitlist_.push_back(op); smsg_waitlisted = true; + return 0; } int diff --git a/src/nnti/transports/ugni/ugni_connection.hpp b/src/nnti/transports/ugni/ugni_connection.hpp index b39a425..61cb577 100644 --- a/src/nnti/transports/ugni/ugni_connection.hpp +++ b/src/nnti/transports/ugni/ugni_connection.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_CONNECTION_HPP_ @@ -35,6 +35,7 @@ class ugni_connection std::string hostname; uint32_t addr; uint32_t port; + std::string fingerprint; uint32_t local_addr; NNTI_instance_id instance; char *smsg_msg_buffer; @@ -53,6 +54,7 @@ class ugni_connection hostname = peer.at("hostname"); addr = nnti::util::str2uint32(peer.at("addr")); port = nnti::util::str2uint32(peer.at("port")); + fingerprint = peer.at("fingerprint"); local_addr = nnti::util::str2uint32(peer.at("local_addr")); instance = nnti::util::str2uint32(peer.at("instance")); smsg_msg_buffer = (char*)nnti::util::str2uint64(peer.at("smsg_msg_buffer")); diff --git a/src/nnti/transports/ugni/ugni_mailbox.cpp b/src/nnti/transports/ugni/ugni_mailbox.cpp index aeba1c1..97ac09e 100644 --- a/src/nnti/transports/ugni/ugni_mailbox.cpp +++ b/src/nnti/transports/ugni/ugni_mailbox.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -33,15 +33,11 @@ ugni_mailbox::ugni_mailbox( cmd_count_(cmd_count), cmd_offset_(0) { - int rc; - setup_command_buffer(); } ugni_mailbox::~ugni_mailbox() { teardown_command_buffer(); - - return; } std::string diff --git a/src/nnti/transports/ugni/ugni_mailbox.hpp b/src/nnti/transports/ugni/ugni_mailbox.hpp index 7c66206..0bbef70 100644 --- a/src/nnti/transports/ugni/ugni_mailbox.hpp +++ b/src/nnti/transports/ugni/ugni_mailbox.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_MAILBOX_HPP_ diff --git a/src/nnti/transports/ugni/ugni_peer.hpp b/src/nnti/transports/ugni/ugni_peer.hpp index 1b6a50e..11e7c09 100644 --- a/src/nnti/transports/ugni/ugni_peer.hpp +++ b/src/nnti/transports/ugni/ugni_peer.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_PEER_HPP diff --git a/src/nnti/transports/ugni/ugni_rdma_op.hpp b/src/nnti/transports/ugni/ugni_rdma_op.hpp index 8db8a23..c7ff3ed 100644 --- a/src/nnti/transports/ugni/ugni_rdma_op.hpp +++ b/src/nnti/transports/ugni/ugni_rdma_op.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_RDMA_OP_HPP_ @@ -199,9 +199,21 @@ class ugni_rdma_op conn->rdma_ep_hdl(), &post_desc_); if (gni_rc != GNI_RC_SUCCESS) { - log_error("ugni_rdma_op", "failed to post BTE (gni_rc=%d): %s", gni_rc, strerror(errno)); - result_ = NNTI_EMSGSIZE; - goto error; + // log an error message and set a generic error code; if possible, set a specific code below + log_error("ugni_rdma_op", "failed to post BTE (gni_rc=%d): %s", gni_rc, strerror(errno)); + result_ = NNTI_EIO; + + if (gni_rc == GNI_RC_ALIGNMENT_ERROR) { + result_ = NNTI_EALIGN; + } else if (gni_rc == GNI_RC_ERROR_NOMEM) { + result_ = NNTI_ENOMEM; + } else if (gni_rc == GNI_RC_INVALID_PARAM) { + result_ = NNTI_EINVAL; + } else if (gni_rc == GNI_RC_PERMISSION_ERROR) { + result_ = NNTI_EPERM; + } + + goto error; } nthread_unlock(&transport_->ugni_lock_); log_debug("ugni_rdma_op", "called PostRdma()"); @@ -288,7 +300,7 @@ class ugni_rdma_op return op_state::CLEANUP; } - op_state + void update_stats(void) { nnti::datatype::nnti_work_request &wr = wid_->wr(); diff --git a/src/nnti/transports/ugni/ugni_transport.cpp b/src/nnti/transports/ugni/ugni_transport.cpp index 2e8ac70..8a7defd 100644 --- a/src/nnti/transports/ugni/ugni_transport.cpp +++ b/src/nnti/transports/ugni/ugni_transport.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -55,22 +55,19 @@ namespace transports { /** * @brief Initialize NNTI to use a specific transport. * - * \param[in] trans_id The ID of the transport the client wants to use. - * \param[in] my_url A string that describes the transport parameters. - * \param[out] trans_hdl A handle to the activated transport. + * \param[in] config A Configuration object that NNTI should use to configure itself. * \return A result code (NNTI_OK or an error) * */ ugni_transport::ugni_transport( faodel::Configuration &config) - : base_transport(NNTI_TRANSPORT_UGNI, - config), + : base_transport(NNTI_TRANSPORT_UGNI, config), started_(false), event_freelist_size_(128), cmd_op_freelist_size_(128), - cmd_tgt_freelist_size_(128), rdma_op_freelist_size_(128), - atomic_op_freelist_size_(128) + atomic_op_freelist_size_(128), + cmd_tgt_freelist_size_(128) { faodel::rc_t rc = 0; uint64_t uint_value = 0; @@ -127,7 +124,13 @@ ugni_transport::start(void) log_debug("ugni_transport", "initializing libugni"); - get_drc_info(&drc_info_); + rc = get_drc_info(&drc_info_); + if (rc != NNTI_OK) { + log_error("ugni_transport", "get_drc_info() failed: %d", rc); + goto cleanup; + } + + gni_rc = GNI_GetDeviceType(&dev_type_); gni_rc = GNI_CdmGetNicAddress (drc_info_.device_id, &nic_addr, &gni_cpu_id); if (gni_rc != GNI_RC_SUCCESS) { @@ -228,6 +231,17 @@ ugni_transport::start(void) goto cleanup; } + gni_rc = GNI_CqCreate (nic_hdl_, 8192, 0, GNI_CQ_BLOCKING, NULL, NULL, &unexpected_long_get_ep_cq_hdl_); + if (gni_rc != GNI_RC_SUCCESS) { + log_error("ugni_transport", "CqCreate(unexpected_long_get_ep_cq_hdl_) failed: %d", gni_rc); + goto cleanup; + } + gni_rc = GNI_CqCreate (nic_hdl_, 8192, 0, GNI_CQ_BLOCKING, NULL, NULL, &unexpected_long_get_mem_cq_hdl_); + if (gni_rc != GNI_RC_SUCCESS) { + log_error("ugni_transport", "CqCreate(unexpected_long_get_mem_cq_hdl_) failed: %d", gni_rc); + goto cleanup; + } + gni_rc = GNI_CqCreate (nic_hdl_, 20, 0, GNI_CQ_BLOCKING, NULL, NULL, &interrupt_mem_cq_hdl_); if (gni_rc != GNI_RC_SUCCESS) { log_error("ugni_transport", "CqCreate(interrupt_cq_hdl_) failed: %d", gni_rc); @@ -304,7 +318,7 @@ ugni_transport::start(void) cleanup: log_debug("ugni_transport", "exit"); - return NNTI_OK; + return rc; } NNTI_result_t @@ -317,6 +331,8 @@ ugni_transport::stop(void) started_ = false; + stop_progress_thread(); + // purge any remaining connections from the map // FIX: this will leak memory and IB resources - do it better nthread_lock(&new_connection_lock_); @@ -329,8 +345,6 @@ ugni_transport::stop(void) unregister_whookie_cb(); - stop_progress_thread(); - teardown_freelists(); GNI_EpUnbind (interrupt_ep_hdl_); @@ -347,10 +361,22 @@ ugni_transport::stop(void) GNI_CqDestroy (rdma_mem_cq_hdl_); GNI_CdmDestroy (cdm_hdl_); + + rc = free_drc_info(&drc_info_); + if (rc == NNTI_EPERM) { + /* + * drc_release() always fails on mutrino with -DRC_EPERM which we + * translate to NNTI_EPERM. Until this is fixed on the Cray side, + * we will ignore this error. + */ + log_debug("ugni_transport", "HACK: free_drc_info() returned bogus NNTI_EPERM. Resetting to NNTI_OK."); + rc = NNTI_OK; + } else if (rc != NNTI_OK) { + log_error("ugni_transport", "free_drc_info() failed: %d", rc); + } nthread_lock_fini(&ugni_lock_); -cleanup: log_debug("ugni_transport", "exit"); return rc; @@ -359,8 +385,6 @@ ugni_transport::stop(void) /** * @brief Indicates if a transport has been initialized. * - * \param[in] trans_id The ID of the transport to test. - * \param[out] is_init 1 if the transport is initialized, 0 otherwise. * \return A result code (NNTI_OK or an error) * */ @@ -442,6 +466,13 @@ ugni_transport::connect( conn = (nnti::core::ugni_connection*)conn_map_.get(peer->pid()); if (conn != nullptr) { log_debug("ugni_transport", "Found connection with pid=%016lx", peer->pid()); + + if (!conn->fingerprint().empty()) { + log_debug("ugni_transport", "Connection has fingerprint=%s", conn->fingerprint().c_str()); + } else { + log_debug("ugni_transport", "Connection has empty fingerprint"); + } + // reuse an existing connection *peer_hdl = (NNTI_peer_t)conn->peer(); nthread_unlock(&new_connection_lock_); @@ -465,19 +496,29 @@ ugni_transport::connect( int retries = 5; wh_rc = whookie::retrieveData(peer_url.hostname(), peer_url.port(), wh_path, &reply); while (wh_rc != 0 && --retries) { - sleep(1); + log_debug("ugni_transport", "whookie::retrieveData() failed: %d", wh_rc); + sleep(5); wh_rc = whookie::retrieveData(peer_url.hostname(), peer_url.port(), wh_path, &reply); } if (wh_rc != 0) { log_debug("ugni_transport", "connect() timed out"); return(NNTI_ETIMEDOUT); } + log_debug("ugni_transport", "connect() remaining retries is %d", retries); log_debug_stream("connect") << "reply=" << reply; - conn->peer_params(reply); - - conn->transition_to_ready(); + nthread_lock(&new_connection_lock_); + if (conn->fingerprint().empty()) { + // When this connection was created above, the peer's fingerprint was + // left empty because we didn't know it. If the fingerprint is now set, + // it means that the peer also initiated a connection back to us and + // won the race. If it's still empty, then we won and need to + // populate the peer parameters here. + conn->peer_params(reply); + conn->transition_to_ready(); + } + nthread_unlock(&new_connection_lock_); *peer_hdl = (NNTI_peer_t)peer; @@ -620,12 +661,12 @@ ugni_transport::eq_wait( log_debug("eq_wait", "enter"); - for (int i=0;ipop(e); if (rc) { uint32_t dummy=0; - ssize_t bytes_read=read(eq->read_fd(), &dummy, 4); + read(eq->read_fd(), &dummy, 4); *which = i; *event = *e; @@ -635,7 +676,7 @@ ugni_transport::eq_wait( } } - for (int i=0;iread_fd(); poll_fds[i].events = POLLIN; @@ -670,11 +711,11 @@ ugni_transport::eq_wait( goto cleanup; } else { log_debug("eq_wait", "polled on %d file descriptor(s). events occurred on %d file descriptor(s).", poll_fds.size(), poll_rc); - for (int i=0;iwr(); if (gni_wr.local_offset() + gni_wr.length() > gni_wr.local_length()) { - log_error("ugni_transport", "PUT length extends beyond the end of local buffer"); + log_error("ugni_transport", + "PUT: length extends beyond the end of local buffer (local offset=%lu, xfer length=%lu, local length=%lu)", + gni_wr.local_offset(), gni_wr.length(), gni_wr.local_length()); return NNTI_EMSGSIZE; } if (gni_wr.remote_offset() + gni_wr.length() > gni_wr.remote_length()) { - log_error("ugni_transport", "PUT length extends beyond the end of remote buffer"); + log_error("ugni_transport", + "PUT: length extends beyond the end of remote buffer (remote offset=%lu, xfer length=%lu, remote length=%lu)", + gni_wr.remote_offset(), gni_wr.length(), gni_wr.remote_length()); return NNTI_EMSGSIZE; } #endif @@ -1099,15 +1143,53 @@ ugni_transport::get( nnti::core::ugni_rdma_op *get_op = nullptr; #ifdef NNTI_ENABLE_ARGS_CHECKING + // + // Do a sanity check on base addresses, offsets and lengths. + // const nnti::datatype::ugni_work_request &gni_wr = (const nnti::datatype::ugni_work_request &)work_id->wr(); if (gni_wr.local_offset() + gni_wr.length() > gni_wr.local_length()) { - log_error("ugni_transport", "GET length extends beyond the end of local buffer"); + log_error("ugni_transport", + "GET: length extends beyond the end of local buffer (local offset=%lu, xfer length=%lu, local length=%lu)", + gni_wr.local_offset(), gni_wr.length(), gni_wr.local_length()); return NNTI_EMSGSIZE; } if (gni_wr.remote_offset() + gni_wr.length() > gni_wr.remote_length()) { - log_error("ugni_transport", "GET length extends beyond the end of remote buffer"); + log_error("ugni_transport", + "GET: length extends beyond the end of remote buffer (remote offset=%lu, xfer length=%lu, remote length=%lu)", + gni_wr.remote_offset(), gni_wr.length(), gni_wr.remote_length()); + return NNTI_EMSGSIZE; + } + if (gni_wr.length() == 0) { + log_error("ugni_transport", "GET: 0-length GETs are not supported"); return NNTI_EMSGSIZE; } + + // + // Local address alignment is a Gemini-only restriction. + // + if ((dev_type_ == GNI_DEVICE_GEMINI) && + (((uint64_t)gni_wr.local_addr()+gni_wr.local_offset())%4 != 0)) { + log_error("ugni_transport", + "GET: local address (buffer+offset) is not 4-byte aligned (addr=%lu, offset=%lu)", + gni_wr.local_addr(), gni_wr.local_offset()); + return NNTI_EALIGN; + } + + // + // The remaining alignment restrictions apply to both Gemini and Aries. + // + if (((uint64_t)gni_wr.remote_addr()+gni_wr.remote_offset())%4 != 0) { + log_error("ugni_transport", + "GET: remote address (buffer+offset) is not 4-byte aligned (addr=%lu, offset=%lu)", + gni_wr.remote_addr(), gni_wr.remote_offset()); + return NNTI_EALIGN; + } + if ((gni_wr.length())%4 != 0) { + log_error("ugni_transport", + "GET: the RDMA length is not 4-byte aligned (actual=%lu)", + gni_wr.length()); + return NNTI_EALIGN; + } #endif rc = create_get_op(work_id, &get_op); @@ -1144,6 +1226,41 @@ ugni_transport::atomic_fop( nnti::datatype::nnti_work_id *work_id = new nnti::datatype::nnti_work_id(*wr); nnti::core::ugni_atomic_op *atomic_op = nullptr; +#ifdef NNTI_ENABLE_ARGS_CHECKING + // + // Do a sanity check on lengths. + // + const nnti::datatype::ugni_work_request &gni_wr = (const nnti::datatype::ugni_work_request &)work_id->wr(); + if (gni_wr.length() != 8) { + log_error("ugni_transport", + "Fetching Atomics: length must be 8 (actual=%lu)", + gni_wr.length()); + return NNTI_EMSGSIZE; + } + + // + // Check the alignment of the addresses and length + // + if (((uint64_t)gni_wr.local_addr()+gni_wr.local_offset())%8 != 0) { + log_error("ugni_transport", + "Fetching Atomics: local address (buffer+offset) is not 8-byte aligned (addr=%lu, offset=%lu)", + gni_wr.local_addr(), gni_wr.local_offset()); + return NNTI_EALIGN; + } + if (((uint64_t)gni_wr.remote_addr()+gni_wr.remote_offset())%8 != 0) { + log_error("ugni_transport", + "Fetching Atomics: remote address (buffer+offset) is not 8-byte aligned (addr=%lu, offset=%lu)", + gni_wr.remote_addr(), gni_wr.remote_offset()); + return NNTI_EALIGN; + } + if ((gni_wr.length())%8 != 0) { + log_error("ugni_transport", + "Fetching Atomics: length is not 8-byte aligned (actual=%lu)", + gni_wr.length()); + return NNTI_EALIGN; + } +#endif + rc = create_fadd_op(work_id, &atomic_op); if (rc != NNTI_OK) { log_error("ugni_transport", "create_fadd_op() failed"); @@ -1205,8 +1322,6 @@ NNTI_result_t ugni_transport::cancel( NNTI_work_id_t wid) { - nnti::datatype::nnti_work_id *work_id = (nnti::datatype::nnti_work_id *)wid; - return NNTI_OK; } @@ -1297,8 +1412,6 @@ ugni_transport::wait( const int64_t timeout, NNTI_status_t *status) { - nnti::datatype::nnti_work_id *work_id = (nnti::datatype::nnti_work_id *)wid; - return NNTI_OK; } @@ -1610,10 +1723,7 @@ ugni_transport::teardown_freelists(void) void ugni_transport::progress(void) { - int rc = 0; gni_return_t gni_rc = GNI_RC_SUCCESS; - NNTI_result_t nnti_rc = NNTI_OK; - bool cq_empty; gni_cq_handle_t cq_list[CQ_COUNT]; @@ -1626,8 +1736,6 @@ ugni_transport::progress(void) void *header = NULL; - bool interrupted = false; - nnti::core::ugni_connection *conn = NULL; uint8_t tag = GNI_SMSG_ANY_TAG; @@ -1648,6 +1756,12 @@ ugni_transport::progress(void) log_debug("ugni_transport", "checking for events on any CQ (cq_count=%d)", cq_count); gni_rc=GNI_CqVectorMonitor(cq_list, cq_count, -1, &which_cq); + if (terminate_progress_thread_.load() == true) { + // this is probably a performance hit, but if bootstrap is shutting + // down this process then anything below will likely fail. + break; + } + /* case 1: success */ if (gni_rc == GNI_RC_SUCCESS) { @@ -1729,15 +1843,15 @@ ugni_transport::progress(void) abort(); } else if (gni_rc == GNI_RC_NOT_DONE) { log_debug("ugni_transport", "GNI_RC_NOT_DONE means the mailbox is empty - implicit credit event on CQ??"); - if (conn->waitlisted()) { - // try to send some messages from this connection's wait list - conn->waitlist_execute(); - } // cq_empty = true; } else { log_debug("ugni_transport", "SmsgGetNextWTag(smsg_ep_hdl) failed: %d", gni_rc); continue; } + if (conn->waitlisted()) { + // try to send some messages from this connection's wait list + conn->waitlist_execute(); + } log_debug("ugni_transport", "goto another_event"); } else if (gni_rc == GNI_RC_NOT_DONE) { log_debug("ugni_transport", "GNI_RC_NOT_DONE means the CQ is empty"); @@ -1852,16 +1966,12 @@ ugni_transport::progress(void) case INTERRUPT_CQ_INDEX: if (get_event(cq_list[which_cq], &ev_data) == GNI_RC_SUCCESS) { log_debug("ugni_transport", "CqVectorMonitor() interrupted by transport::interrupt()"); - interrupted=true; - nnti_rc = NNTI_EINTR; } continue; default: break; } - - nnti_rc = NNTI_OK; } else { char errstr[1024]; uint32_t recoverable=0; @@ -1872,12 +1982,9 @@ ugni_transport::progress(void) log_error("ugni_transport", "CqVectorMonitor failed (gni_rc=%d) (recoverable=%llu) : %s", gni_rc, (uint64_t)recoverable, errstr); print_cq_event(&ev_data, false); - - nnti_rc = NNTI_EIO; } } -cleanup: log_debug("ugni_transport", "exit"); return; @@ -1910,12 +2017,46 @@ ugni_transport::connect_cb(const std::map &args, std::s nthread_lock(&new_connection_lock_); nnti::core::nnti_url peer_url = nnti::core::nnti_url(args.at("hostname"), args.at("port")); + std::string fingerprint = args.at("fingerprint"); - log_debug("ugni_transport", "Looking for connection with pid=%016lx", peer_url.pid()); + log_debug("ugni_transport", "Looking for connection with pid=%016lx fingerprint=%s", peer_url.pid(), fingerprint.c_str()); conn = (nnti::core::ugni_connection*)conn_map_.get(peer_url.pid()); if (conn != nullptr) { - log_debug("ugni_transport", "Found connection with pid=%016lx", peer_url.pid()); + if (conn->fingerprint() == fingerprint) { + // The connecting peer already has a fully formed connection object in the map. + // We will reply with the properties of this connection. + log_debug("ugni_transport", "Found matching connection with pid=%016lx fingerprint=%s", peer_url.pid(), fingerprint.c_str()); + } else if (conn->fingerprint().empty()) { + // The connecting peer already has a partiallu formed connection object in the map. + // This can happened when two peers initiate a connection simultaneously. One side + // will win the race and find a connection object with no fingerprint. This connection + // object should be populated with the initiator info found in 'args' and transitioned + // to ready. + // Then we can reply with the properties of the now complete connection. + log_debug("ugni_transport", "Found connection with matching pid=%016lx and empty fingerprint=%s", peer_url.pid(), conn->fingerprint().c_str()); + conn->peer_params(args); + conn->transition_to_ready(); + } else { + // The connecting peer already has a fully formed connection object in the map, + // but the fingerprint doesn't match. This can happen if the peer is restarted + // and gets a new fingerprint. The stale connection will be destroyed and + // replaced. + // Then we can reply with the properties of the new connection. + log_debug("ugni_transport", "Found mismatched connection with pid=%016lx fingerprint=%s", peer_url.pid(), conn->fingerprint().c_str()); + + conn_map_.remove(conn); + delete conn; + + conn = new nnti::core::ugni_connection(this, cmd_msg_size_, cmd_msg_count_, args); + conn->index = conn_vector_.add(conn); + conn_map_.insert(conn); + + conn->transition_to_ready(); + } } else { + // A connection object couldn't be found for the connecting peer. We will create + // a new one. + // Then we can reply with the properties of the new connection. log_debug("ugni_transport", "Couldn't find connection with pid=%016lx", peer_url.pid()); conn = new nnti::core::ugni_connection(this, cmd_msg_size_, cmd_msg_count_, args); @@ -1927,11 +2068,12 @@ ugni_transport::connect_cb(const std::map &args, std::s nthread_unlock(&new_connection_lock_); - results << "hostname=" << url_.hostname() << std::endl; - results << "addr=" << url_.addr() << std::endl; - results << "port=" << url_.port() << std::endl; - results << "local_addr=" << drc_info_.local_addr << std::endl; - results << "instance=" << instance_ << std::endl; + results << "hostname=" << url_.hostname() << std::endl; + results << "addr=" << url_.addr() << std::endl; + results << "port=" << url_.port() << std::endl; + results << "fingerprint=" << fingerprint_ << std::endl; + results << "local_addr=" << drc_info_.local_addr << std::endl; + results << "instance=" << instance_ << std::endl; results << conn->reply_string(); log_debug("ugni_transport", "connect_cb() - exit"); @@ -2006,12 +2148,13 @@ ugni_transport::build_whookie_path( { std::stringstream wh_url; - wh_url << "/nnti/ugni/" << service; - wh_url << "&hostname=" << url_.hostname(); - wh_url << "&addr=" << url_.addr(); - wh_url << "&port=" << url_.port(); - wh_url << "&local_addr=" << drc_info_.local_addr; - wh_url << "&instance=" << instance_; + wh_url << "/nnti/ugni/" << service; + wh_url << "&hostname=" << url_.hostname(); + wh_url << "&addr=" << url_.addr(); + wh_url << "&port=" << url_.port(); + wh_url << "&fingerprint=" << fingerprint_; + wh_url << "&local_addr=" << drc_info_.local_addr; + wh_url << "&instance=" << instance_; return wh_url.str(); } @@ -2144,10 +2287,10 @@ ugni_transport::create_get_op( if (rdma_op_freelist_->pop(*rdma_op)) { (*rdma_op)->set(work_id); - (*rdma_op)->index = op_vector_.add(*rdma_op); } else { *rdma_op = new nnti::core::ugni_rdma_op(this, work_id); } + (*rdma_op)->index = op_vector_.add(*rdma_op); log_debug("ugni_transport", "create_get_op() - exit"); @@ -2165,11 +2308,10 @@ ugni_transport::create_put_op( if (rdma_op_freelist_->pop(*rdma_op)) { (*rdma_op)->set(work_id); - (*rdma_op)->index = op_vector_.add(*rdma_op); } else { *rdma_op = new nnti::core::ugni_rdma_op(this, work_id); - (*rdma_op)->index = op_vector_.add(*rdma_op); } + (*rdma_op)->index = op_vector_.add(*rdma_op); log_debug("ugni_transport", "create_put_op() - exit"); @@ -2366,14 +2508,15 @@ ugni_transport::get_drc_info( errno = 0; drc_info->device_id = (uint8_t)strtoul (ptr, (char **)NULL, 10); if (0 != errno) { + log_error("ugni_transport", "PMI_GNI_DEV_ID (%s) is invalid.", ptr); return NNTI_EINVAL; } - int64_t tmp_cred=0; - rc = config_.GetInt(&tmp_cred, "nnti.transport.credential_id", "0"); - if (rc == 0) { - drc_info->credential_id = (uint32_t)tmp_cred; + int64_t tmp_cred=-1; + rc = config_.GetInt(&tmp_cred, "nnti.transport.credential_id", "-1"); + drc_info->credential_id = (uint32_t)tmp_cred; + if (drc_info->credential_id != -1) { rc = drc_access(drc_info->credential_id, flags, &drc_info->drc_info_hdl); if (rc != DRC_SUCCESS) { // Something is wrong. Either the credential doesn't exist or you don't have permission to access it @@ -2383,6 +2526,7 @@ ugni_transport::get_drc_info( drc_info->cookie1 = drc_get_first_cookie(drc_info->drc_info_hdl); drc_info->ptag1 = GNI_FIND_ALLOC_PTAG; + log_debug("ugni_transport", "looking for ptag1 on device_id=%u that matches cookie1=%u", drc_info->device_id, drc_info->cookie1); grc = GNI_GetPtag(drc_info->device_id, drc_info->cookie1, &drc_info->ptag1); if (grc != GNI_RC_SUCCESS) { // Something is wrong. This should not happen if drc_access returned DRC_SUCCESS @@ -2427,6 +2571,37 @@ ugni_transport::get_drc_info( } +NNTI_result_t +ugni_transport::free_drc_info( + drc_info_t *drc_info) +{ + int rc; + int flags = 0; + + if (drc_info->credential_id != -1) { + rc = drc_release(drc_info->credential_id, flags); + if (rc != DRC_SUCCESS) { + /* failed to release credential */ + if (rc != -DRC_EPERM) { + log_error("ugni_transport", "drc_release() failed: %d (%s)", rc, strerror(-rc)); + } + if (rc == -DRC_EINVAL) { + return NNTI_EINVAL; + } else if (rc == -DRC_CRED_NOT_FOUND) { + return NNTI_ENOENT; + } else if (rc == -DRC_EPERM) { + return NNTI_EPERM; + } else if (rc == -DRC_ECONNREFUSED) { + return NNTI_EBADRPC; + } else { + return NNTI_EIO; + } + } + } + + return NNTI_OK; +} + //void //ugni_transport::print_wc( // const nnti_gni_work_completion_t *wc) diff --git a/src/nnti/transports/ugni/ugni_transport.hpp b/src/nnti/transports/ugni/ugni_transport.hpp index 445905e..c3c9c6a 100644 --- a/src/nnti/transports/ugni/ugni_transport.hpp +++ b/src/nnti/transports/ugni/ugni_transport.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_TRANSPORT_HPP_ @@ -67,6 +67,9 @@ namespace transports { #define NNTI_SMSG_TAG_CREDIT 2 #define NNTI_SMSG_TAG_LONG_GET_ACK 3 +// UGNI requires alignment for some RDMA ops +#define NNTI_UGNI_RDMA_ALIGNMENT 4 + class ugni_transport : public base_transport { @@ -143,6 +146,8 @@ class ugni_transport uint64_t apid_; drc_info_t drc_info_; + gni_nic_device_t dev_type_; + gni_cdm_handle_t cdm_hdl_; gni_nic_handle_t nic_hdl_; NNTI_instance_id instance_; @@ -158,6 +163,9 @@ class ugni_transport gni_cq_handle_t long_get_ep_cq_hdl_; gni_cq_handle_t long_get_mem_cq_hdl_; + gni_cq_handle_t unexpected_long_get_ep_cq_hdl_; + gni_cq_handle_t unexpected_long_get_mem_cq_hdl_; + gni_cq_handle_t rdma_ep_cq_hdl_; gni_cq_handle_t rdma_mem_cq_hdl_; @@ -204,8 +212,6 @@ class ugni_transport /** * @brief Initialize NNTI to use a specific transport. * - * \param[in] my_url A string that describes the transport parameters. - * \param[out] trans_hdl A handle to the activated transport. * \return A result code (NNTI_OK or an error) * */ @@ -218,24 +224,22 @@ class ugni_transport * * \return A result code (NNTI_OK or an error) */ - ~ugni_transport(); + ~ugni_transport() override; NNTI_result_t - start(void); + start(void) override; NNTI_result_t - stop(void); + stop(void) override; /** * @brief Indicates if a transport has been initialized. * - * \param[in] trans_id The ID of the transport to test. - * \param[out] is_init 1 if the transport is initialized, 0 otherwise. * \return A result code (NNTI_OK or an error) * */ bool - initialized(void); + initialized(void) override; /** * @brief Return the URL field of this transport. @@ -247,7 +251,7 @@ class ugni_transport NNTI_result_t get_url( char *url, - const uint64_t maxlen); + const uint64_t maxlen) override; /** * @brief Get the process ID of this process. @@ -257,7 +261,7 @@ class ugni_transport * */ NNTI_result_t - pid(NNTI_process_id_t *pid); + pid(NNTI_process_id_t *pid) override; /** * @brief Get attributes of the transport. @@ -267,7 +271,7 @@ class ugni_transport * */ NNTI_result_t - attrs(NNTI_attrs_t *attrs); + attrs(NNTI_attrs_t *attrs) override; /** * @brief Prepare for communication with the peer identified by url. @@ -281,7 +285,7 @@ class ugni_transport connect( const char *url, const int timeout, - NNTI_peer_t *peer_hdl); + NNTI_peer_t *peer_hdl) override; /** * @brief Terminate communication with this peer. @@ -291,7 +295,7 @@ class ugni_transport */ NNTI_result_t disconnect( - NNTI_peer_t peer_hdl); + NNTI_peer_t peer_hdl) override; /** * @brief Create an event queue. @@ -305,7 +309,7 @@ class ugni_transport eq_create( uint64_t size, NNTI_eq_flags_t flags, - NNTI_event_queue_t *eq); + NNTI_event_queue_t *eq) override; NNTI_result_t eq_create( @@ -313,7 +317,7 @@ class ugni_transport NNTI_eq_flags_t flags, nnti::datatype::nnti_event_callback cb, void *cb_context, - NNTI_event_queue_t *eq); + NNTI_event_queue_t *eq) override; /** * @brief Destroy an event queue. @@ -323,7 +327,7 @@ class ugni_transport */ NNTI_result_t eq_destroy( - NNTI_event_queue_t eq); + NNTI_event_queue_t eq) override; /** * @brief Wait for an event to arrive on an event queue. @@ -340,29 +344,29 @@ class ugni_transport const uint32_t eq_count, const int timeout, uint32_t *which, - NNTI_event_t *event); + NNTI_event_t *event) override; /** * @brief Retrieves the next message from the unexpected list. * * \param[in] dst_hdl Buffer where the message is delivered. * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t next_unexpected( NNTI_buffer_t dst_hdl, uint64_t dst_offset, - NNTI_event_t *result_event); + NNTI_event_t *result_event) override; /** * @brief Retrieves a specific message from the unexpected list. * - * \param[in] unexpect_event Event describing the message to retrieve. - * \param[in] dst_hdl Buffer where the message is delivered. - * \param[in] dst_offset Offset into dst_hdl where the message is delivered. - * \param[out] reseult_event Event describing the message delivered to dst_hdl. + * \param[in] unexpected_event Event describing the message to retrieve. + * \param[in] dst_hdl Buffer where the message is delivered. + * \param[in] dst_offset Offset into dst_hdl where the message is delivered. + * \param[out] result_event Event describing the message delivered to dst_hdl. * \return A result code (NNTI_OK or an error) */ NNTI_result_t @@ -370,7 +374,7 @@ class ugni_transport NNTI_event_t *unexpected_event, NNTI_buffer_t dst_hdl, uint64_t dst_offset, - NNTI_event_t *result_event); + NNTI_event_t *result_event) override; /** * @brief Marks a send operation as complete. @@ -380,7 +384,7 @@ class ugni_transport */ NNTI_result_t event_complete( - NNTI_event_t *event); + NNTI_event_t *event) override; /** * @brief Decode an array of bytes into an NNTI datatype. @@ -394,7 +398,7 @@ class ugni_transport dt_unpack( void *nnti_dt, char *packed_buf, - const uint64_t packed_len); + const uint64_t packed_len) override; /** * @brief Allocate a block of memory and prepare it for network operations. @@ -416,7 +420,7 @@ class ugni_transport nnti::datatype::nnti_event_callback cb, void *cb_context, char **reg_ptr, - NNTI_buffer_t *reg_buf); + NNTI_buffer_t *reg_buf) override; /** * @brief Disables network operations on the block of memory and frees it. @@ -426,7 +430,7 @@ class ugni_transport */ NNTI_result_t free( - NNTI_buffer_t reg_buf); + NNTI_buffer_t reg_buf) override; /** * @brief Prepare a block of memory for network operations. @@ -448,7 +452,7 @@ class ugni_transport NNTI_event_queue_t eq, nnti::datatype::nnti_event_callback cb, void *cb_context, - NNTI_buffer_t *reg_buf); + NNTI_buffer_t *reg_buf) override; /** * @brief Disables network operations on a memory buffer. @@ -458,31 +462,31 @@ class ugni_transport */ NNTI_result_t unregister_memory( - NNTI_buffer_t reg_buf); + NNTI_buffer_t reg_buf) override; /** * @brief Convert an NNTI peer to an NNTI_process_id_t. * - * \param[in] peer A handle to a peer that can be used for network operations. - * \param[out] pid Compact binary representation of a process's location on the network. + * \param[in] peer_hdl A handle to a peer that can be used for network operations. + * \param[out] pid Compact binary representation of a process's location on the network. * \return A result code (NNTI_OK or an error) */ NNTI_result_t dt_peer_to_pid( NNTI_peer_t peer_hdl, - NNTI_process_id_t *pid); + NNTI_process_id_t *pid) override; /** * @brief Convert an NNTI_process_id_t to an NNTI peer. * - * \param[in] pid Compact binary representation of a process's location on the network. - * \param[out] peer A handle to a peer that can be used for network operations. + * \param[in] pid Compact binary representation of a process's location on the network. + * \param[out] peer_hdl A handle to a peer that can be used for network operations. * \return A result code (NNTI_OK or an error) */ NNTI_result_t dt_pid_to_peer( NNTI_process_id_t pid, - NNTI_peer_t *peer_hdl); + NNTI_peer_t *peer_hdl) override; /** * @brief Send a message to a peer. @@ -494,7 +498,7 @@ class ugni_transport NNTI_result_t send( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid); + NNTI_work_id_t *wid) override; /** * @brief Transfer data to a peer. @@ -506,7 +510,7 @@ class ugni_transport NNTI_result_t put( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid); + NNTI_work_id_t *wid) override; /** * @brief Transfer data from a peer. @@ -518,7 +522,7 @@ class ugni_transport NNTI_result_t get( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid); + NNTI_work_id_t *wid) override; /** * perform a 64-bit atomic operation with GET semantics @@ -530,7 +534,7 @@ class ugni_transport NNTI_result_t atomic_fop( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid); + NNTI_work_id_t *wid) override; /** * perform a 64-bit compare-and-swap operation @@ -542,7 +546,7 @@ class ugni_transport NNTI_result_t atomic_cswap( nnti::datatype::nnti_work_request *wr, - NNTI_work_id_t *wid); + NNTI_work_id_t *wid) override; /** * @brief Attempts to cancel an NNTI operation. @@ -552,7 +556,7 @@ class ugni_transport */ NNTI_result_t cancel( - NNTI_work_id_t wid); + NNTI_work_id_t wid) override; /** @@ -565,7 +569,7 @@ class ugni_transport NNTI_result_t cancelall( NNTI_work_id_t *wid_list, - const uint32_t wid_count); + const uint32_t wid_count) override; /** @@ -574,7 +578,7 @@ class ugni_transport * \return A result code (NNTI_OK or an error) */ NNTI_result_t - interrupt(); + interrupt() override; /** @@ -589,7 +593,7 @@ class ugni_transport wait( NNTI_work_id_t wid, const int64_t timeout, - NNTI_status_t *status); + NNTI_status_t *status) override; /** * @brief Wait for any operation (wid_list) in the list to complete. @@ -607,7 +611,7 @@ class ugni_transport const uint32_t wid_count, const int64_t timeout, uint32_t *which, - NNTI_status_t *status); + NNTI_status_t *status) override; /** * @brief Waits for all the operations (wid_list) in the list to complete. @@ -623,7 +627,7 @@ class ugni_transport NNTI_work_id_t *wid_list, const uint32_t wid_count, const int64_t timeout, - NNTI_status_t *status); + NNTI_status_t *status) override; public: static ugni_transport* @@ -776,6 +780,9 @@ class ugni_transport NNTI_result_t get_drc_info( drc_info_t *drc_info); + NNTI_result_t + free_drc_info( + drc_info_t *drc_info); // void // print_wc( // const nnti_gni_work_completion_t *wc); diff --git a/src/nnti/transports/ugni/ugni_wr.hpp b/src/nnti/transports/ugni/ugni_wr.hpp index 620bbeb..8096ca2 100644 --- a/src/nnti/transports/ugni/ugni_wr.hpp +++ b/src/nnti/transports/ugni/ugni_wr.hpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef UGNI_WR_HPP diff --git a/src/opbox/CMakeLists.txt b/src/opbox/CMakeLists.txt index e04fc38..4489f3b 100644 --- a/src/opbox/CMakeLists.txt +++ b/src/opbox/CMakeLists.txt @@ -14,8 +14,9 @@ set(HEADERS common/OpRegistry.hh common/Types.hh core/Singleton.hh + core/OpTimer.hh core/OpBoxCoreBase.hh - core/OpBoxCoreStandard.hh + core/OpBoxCoreDeprecatedStandard.hh core/OpBoxCoreThreaded.hh core/OpBoxCoreUnconfigured.hh net/net.hh @@ -41,9 +42,10 @@ set(SOURCES common/Types.cpp core/Singleton.cpp core/OpBoxCoreBase.cpp - core/OpBoxCoreStandard.cpp + core/OpBoxCoreDeprecatedStandard.cpp core/OpBoxCoreThreaded.cpp core/OpBoxCoreUnconfigured.cpp + core/OpTimer.cpp ops/Op.cpp ops/OpCount.cpp ops/OpPing.cpp @@ -51,7 +53,7 @@ set(SOURCES add_subdirectory( net ) -LIST( APPEND Opbox_imports lunasa ${FaodelNetlib_TARGETS} services whookie common sbl Boost::serialization ) +LIST( APPEND Opbox_imports lunasa ${FaodelNetlib_TARGETS} services whookie common sbl ) add_library( opbox ${HEADERS} ${SOURCES} ) target_link_libraries( opbox PUBLIC ${Opbox_imports} ) diff --git a/src/opbox/OpBox.cpp b/src/opbox/OpBox.cpp index cf41722..ea91496 100644 --- a/src/opbox/OpBox.cpp +++ b/src/opbox/OpBox.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -81,6 +81,17 @@ void DeregisterOp(int id, bool ignore_lock_warning) { return opbox::internal::Singleton::impl.registry.DeregisterOp(id, ignore_lock_warning); } +/** + * @brief Consult the registry and determine the op name for a particular id + * @param[in] id The hash value of the operation name + * @retval NAME The name of the op + * @retval "" The id was not found in the registry + */ +string GetOpName(int id) { + return opbox::internal::Singleton::impl.registry.GetOpName(id); +} + + /** * @brief Take ownership of an op a user has created and begin executing it * @@ -99,6 +110,16 @@ int TriggerOp(mailbox_t mailbox, shared_ptr args) { return opbox::internal::Singleton::impl.core->TriggerOp(mailbox, args); } +/** + * @brief Determine how many ops are currently active in opbox + * @param op_id An optional Op ID that is used as a filter if nonzero + * @retval COUNT The number of ops + * @note This always returns a valid value, even if opbox is not in the started state + */ +int GetNumberOfActiveOps(unsigned int op_id) { + return opbox::internal::Singleton::impl.core->GetNumberOfActiveOps(op_id); +} + /** * @brief Return a unique identifier that opbox and others can use to reference this rank * diff --git a/src/opbox/OpBox.hh b/src/opbox/OpBox.hh index 3f06e9e..2c1151d 100644 --- a/src/opbox/OpBox.hh +++ b/src/opbox/OpBox.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPBOX_HH #define OPBOX_OPBOX_HH @@ -20,9 +20,11 @@ faodel::nodeid_t GetMyID(); //Get our ID void RegisterOp(int op_id, std::string op_name, fn_OpCreate_t func); void DeregisterOp(int op_id, bool ignore_lock_warning); +std::string GetOpName(int op_id); + int LaunchOp(opbox::Op *op, mailbox_t *mailbox_id=nullptr); //Launch a new operation int TriggerOp(mailbox_t mailbox, std::shared_ptr args); - +int GetNumberOfActiveOps(unsigned int op_id=0); /** * @brief Template for registering an Op with Opbox. Automatically extracts info from Op. diff --git a/src/opbox/common/Message.cpp b/src/opbox/common/Message.cpp index 0ea3486..08ba27a 100644 --- a/src/opbox/common/Message.cpp +++ b/src/opbox/common/Message.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -64,7 +64,7 @@ bool message_t::IsExpected(uint32_t expected_op_id, uint16_t flag_mask, uint16_t */ void message_t::SetStandardRequest(faodel::nodeid_t dst_node, mailbox_t src_mailbox, uint32_t op_id, uint16_t user_flags, - uint16_t body_len){ + uint32_t body_len){ this->src = opbox::net::GetMyID(); this->dst = dst_node; this->src_mailbox = src_mailbox; @@ -82,7 +82,7 @@ void message_t::SetStandardRequest(faodel::nodeid_t dst_node, mailbox_t src_mail * @param[in] body_len The length of the data immediately following this header */ void message_t::SetStandardReply(const message_t *hdr, uint16_t user_flags, - uint16_t body_len) { + uint32_t body_len) { this->src = opbox::net::GetMyID(); this->dst = hdr->src; this->src_mailbox = MAILBOX_UNSPECIFIED; diff --git a/src/opbox/common/Message.hh b/src/opbox/common/Message.hh index 8ec8bd1..afef5be 100644 --- a/src/opbox/common/Message.hh +++ b/src/opbox/common/Message.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_MESSAGEHEADER_HH #define OPBOX_MESSAGEHEADER_HH @@ -54,8 +54,9 @@ struct message_t { mailbox_t src_mailbox; //!< ID to use when communicating back with origin mailbox_t dst_mailbox; //!< ID to use at the target (usually 0 for most ops) uint32_t op_id; //!< The id for this type of op (a hash of its name) - uint16_t user_flags; //!< small place for user to put simple flags - uint16_t body_len; //!< Length of this message's body (should be less than MTU - sizeof(message_t)) + uint16_t user_flags; //!< Small place for user to put simple flags + uint16_t user_flags2; //!< Reserved for future use + uint32_t body_len; //!< Length of this message's body (should be less than MTU - sizeof(message_t)) //Body follows next. We use the non-compliant zero-length array here to make pointers easier char body[0]; //!< Starting point for any other op-specific data in this message @@ -68,12 +69,12 @@ struct message_t { bool IsExpected(uint32_t expected_op_id, uint16_t flag_mask, uint16_t expected_flags); //Helpers for filling in standard request/reply fields - void SetStandardRequest(faodel::nodeid_t dst, mailbox_t src_mailbox, uint32_t op_id, uint16_t user_flags=0, uint16_t body_len=0); - void SetStandardReply(const message_t *hdr, uint16_t user_flags=0, uint16_t body_len=0); + void SetStandardRequest(faodel::nodeid_t dst, mailbox_t src_mailbox, uint32_t op_id, uint16_t user_flags=0, uint32_t body_len=0); + void SetStandardReply(const message_t *hdr, uint16_t user_flags=0, uint32_t body_len=0); //Provide InfoInterface w/o inheritance (which would add an extra 8 bytes to structure) - void sstr(std::stringstream &ss, int depth=0, int indent=0) const; + void sstr(std::stringstream &ss, int depth=0, int indent=0) const; //Note: not override since not inherited std::string str(int depth=0, int indent=0) const; }; diff --git a/src/opbox/common/MessageHelpers.cpp b/src/opbox/common/MessageHelpers.cpp index aebf6c6..6a89176 100644 --- a/src/opbox/common/MessageHelpers.cpp +++ b/src/opbox/common/MessageHelpers.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -51,8 +51,8 @@ bool AllocateStringMessage(lunasa::DataObject &new_ldo, const uint32_t op_id, const uint16_t user_flags, const string user_string){ - if(user_string.size() >= 64*1024) - throw std::runtime_error("AllocteStringMessage failed because string >= 64KB"); + //if(user_string.size() >= 64*1024) + // throw std::runtime_error("AllocteStringMessage failed because string >= 64KB"); new_ldo = opbox::net::NewMessage(sizeof(message_t) + user_string.size()); @@ -63,7 +63,7 @@ bool AllocateStringMessage(lunasa::DataObject &new_ldo, msg->dst_mailbox = dst_mailbox; msg->op_id = op_id; msg->user_flags = user_flags; - msg->body_len = static_cast(user_string.size()); + msg->body_len = static_cast(user_string.size()); //Append the packed data to the end of the message user_string.copy(msg->body, user_string.size()); @@ -89,8 +89,8 @@ bool AllocateStringRequestMessage(lunasa::DataObject &new_ldo, const uint16_t user_flags, const string user_string){ - if(user_string.size() >= 64*1024) - throw std::runtime_error("AllocteStringRequestMessage failed because string >= 64KB"); + //if(user_string.size() >= 64*1024) + // throw std::runtime_error("AllocteStringRequestMessage failed because string >= 64KB"); new_ldo = opbox::net::NewMessage(sizeof(message_t) + user_string.size()); @@ -117,14 +117,14 @@ bool AllocateStringReplyMessage(lunasa::DataObject &new_ldo, const uint16_t user_flags, const string user_string){ - if(user_string.size() >= 64*1024) - throw std::runtime_error("AllocteStringReplyMessage failed because string >= 64KB"); + //if(user_string.size() >= 64*1024) + // throw std::runtime_error("AllocteStringReplyMessage failed because string >= 64KB"); new_ldo = opbox::net::NewMessage(sizeof(message_t) + user_string.size()); auto *msg = new_ldo.GetDataPtr(); - msg->SetStandardReply(request_msg, user_flags, static_cast(user_string.size())); + msg->SetStandardReply(request_msg, user_flags, static_cast(user_string.size())); user_string.copy(msg->body, user_string.size()); return (user_string.size() > MESSAGE_BODY_MTU); diff --git a/src/opbox/common/MessageHelpers.hh b/src/opbox/common/MessageHelpers.hh index fdfcbb7..85f2c1a 100644 --- a/src/opbox/common/MessageHelpers.hh +++ b/src/opbox/common/MessageHelpers.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_MESSAGE_HELPERS_HH #define OPBOX_MESSAGE_HELPERS_HH @@ -9,7 +9,7 @@ #include #include "faodel-common/NodeID.hh" -#include "faodel-common/SerializationHelpers.hh" +#include "faodel-common/SerializationHelpersCereal.hh" #include "lunasa/DataObject.hh" #include "opbox/common/Message.hh" @@ -53,23 +53,23 @@ bool AllocateStringReplyMessage(lunasa::DataObject &new_ldo, std::string UnpackStringMessage(message_t *hdr); -//This template provides a way to pack a Boost-serializable structure +//This template provides a way to pack a Cereal-serializable structure //into the body of a message. This version asks for all fields in order //to allow users to set everything (eg, forwarded message) template -bool AllocateBoostMessage(lunasa::DataObject &new_ldo, +bool AllocateCerealMessage(lunasa::DataObject &new_ldo, const faodel::nodeid_t &src_node, const faodel::nodeid_t &dst_node, const mailbox_t src_mailbox, const mailbox_t dst_mailbox, const uint32_t op_id, const uint16_t user_flags, - const T &boost_serializable_object){ + const T &cereal_serializable_object){ //Pack the object - std::string packed_object = faodel::BoostPack(boost_serializable_object); + std::string packed_object = faodel::CerealPack(cereal_serializable_object); - if(packed_object.size() >= 64*1024) - throw std::runtime_error("AllocteBoostMessage failed because string >= 64KB"); + //if(packed_object.size() >= 64*1024) + // throw std::runtime_error("AllocteCerealMessage failed because string >= 64KB"); //Allocate a new LDO. Because it is handed to us as a pointer, we won't //increase/decrease the refcount. @@ -81,7 +81,7 @@ bool AllocateBoostMessage(lunasa::DataObject &new_ldo, msg->dst_mailbox = dst_mailbox; msg->op_id = op_id; msg->user_flags = user_flags; - msg->body_len = static_cast(packed_object.size()); + msg->body_len = static_cast(packed_object.size()); //Append the packed data to the end of the message packed_object.copy(&msg->body[0], packed_object.size()); @@ -90,19 +90,19 @@ bool AllocateBoostMessage(lunasa::DataObject &new_ldo, } -//This template provides a way to pack a Boost-serializable structure +//This template provides a way to pack a Cereal-serializable structure //into the body of a message. template -bool AllocateBoostRequestMessage(lunasa::DataObject &new_ldo, +bool AllocateCerealRequestMessage(lunasa::DataObject &new_ldo, const faodel::nodeid_t &dst_node, const mailbox_t src_mailbox, const uint32_t op_id, const uint16_t user_flags, - const T &boost_serializable_object){ + const T &cereal_serializable_object){ //Pack the object - std::string packed_object = faodel::BoostPack(boost_serializable_object); + std::string packed_object = faodel::CerealPack(cereal_serializable_object); - if(packed_object.size() >= 64*1024) - throw std::runtime_error("AllocteBoostRequestMessage failed because string >= 64KB"); + //if(packed_object.size() >= 64*1024) + // throw std::runtime_error("AllocteCerealRequestMessage failed because string >= 64KB"); //Allocate a new LDO. Because it is handed to us as a pointer, we won't @@ -110,7 +110,7 @@ bool AllocateBoostRequestMessage(lunasa::DataObject &new_ldo, new_ldo = opbox::net::NewMessage(sizeof(message_t) + packed_object.size()); auto *msg = new_ldo.GetDataPtr(); msg->SetStandardRequest(dst_node, src_mailbox, op_id, user_flags, - static_cast(packed_object.size())); + static_cast(packed_object.size())); //Append the packed data to the end of the message packed_object.copy(&msg->body[0], packed_object.size()); @@ -119,26 +119,26 @@ bool AllocateBoostRequestMessage(lunasa::DataObject &new_ldo, } -//Allocate a message that passes a boost-serializable structure in the body +//Allocate a message that passes a Cereal-serializable structure in the body //and uses a request message to populate the message headers. template -bool AllocateBoostReplyMessage(lunasa::DataObject &new_ldo, +bool AllocateCerealReplyMessage(lunasa::DataObject &new_ldo, const message_t *request_msg, const uint16_t user_flags, - const T &boost_serializable_object){ + const T &cereal_serializable_object){ //Pack the object - std::string packed_object = faodel::BoostPack(boost_serializable_object); + std::string packed_object = faodel::CerealPack(cereal_serializable_object); - if(packed_object.size() >= 64*1024) - throw std::runtime_error("AllocteBoostReplyMessage failed because string >= 64KB"); + //if(packed_object.size() >= 64*1024) + // throw std::runtime_error("AllocteCerealReplyMessage failed because string >= 64KB"); //Allocate a new LDO. Because it is handed to us as a pointer, we won't //increase/decrease the refcount. new_ldo = opbox::net::NewMessage(sizeof(message_t) + packed_object.size()); auto *msg = new_ldo.GetDataPtr(); - msg->SetStandardReply(request_msg, user_flags, static_cast(packed_object.size())); + msg->SetStandardReply(request_msg, user_flags, static_cast(packed_object.size())); //Append the packed data to the end of the message packed_object.copy(&msg->body[0], packed_object.size()); @@ -146,11 +146,11 @@ bool AllocateBoostReplyMessage(lunasa::DataObject &new_ldo, return (packed_object.size() > MESSAGE_BODY_MTU); } -//Unpack a message that sent along a boost-packed data structure +//Unpack a message that sent along a Cereal-packed data structure template -T UnpackBoostMessage(const message_t *hdr){ +T UnpackCerealMessage(const message_t *hdr){ std::string s(hdr->body, hdr->body_len); - return faodel::BoostUnpack(s); + return faodel::CerealUnpack(s); } } // namespace opbox diff --git a/src/opbox/common/OpArgs.cpp b/src/opbox/common/OpArgs.cpp index 4ea81b3..5aafe87 100644 --- a/src/opbox/common/OpArgs.cpp +++ b/src/opbox/common/OpArgs.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -32,12 +32,12 @@ void OpArgs::VerifyTypeOrDie(UpdateType expected_type, string op_name){ void SanityCheck(OpArgs *args, const char *src_file, int line ) { if(args==nullptr) { - faodel::_khalt("OpArgs Sanity check fail: null pointer for args", src_file, line); + faodel::_f_halt("OpArgs Sanity check fail: null pointer for args", src_file, line); } switch(args->type) { case UpdateType::incoming_message: - //kassert((args->incoming_msg!=nullptr), "OpArgs sanity check fail: null pointer for incoming_msg"); + //F_ASSERT((args->incoming_msg!=nullptr), "OpArgs sanity check fail: null pointer for incoming_msg"); break; case UpdateType::user_trigger: break; //Special @@ -53,12 +53,12 @@ void SanityCheck(OpArgs *args, const char *src_file, int line ) { case UpdateType::get_error: case UpdateType::put_error: case UpdateType::atomic_error: - //kassert(args->incoming_msg == nullptr, "OpArgs sanity check fail: incoming_msg was not null"); + //F_ASSERT(args->incoming_msg == nullptr, "OpArgs sanity check fail: incoming_msg was not null"); break; //Unrecognized? Probably a bad type default: - faodel::_khalt("OpArgs sanity check fail: Type was not valid?", src_file, line); + faodel::_f_halt("OpArgs sanity check fail: Type was not valid?", src_file, line); } } #endif diff --git a/src/opbox/common/OpArgs.hh b/src/opbox/common/OpArgs.hh index 8df8840..d527858 100644 --- a/src/opbox/common/OpArgs.hh +++ b/src/opbox/common/OpArgs.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPARGS_HH #define OPBOX_OPARGS_HH diff --git a/src/opbox/common/OpRegistry.cpp b/src/opbox/common/OpRegistry.cpp index 483d64a..51b7752 100644 --- a/src/opbox/common/OpRegistry.cpp +++ b/src/opbox/common/OpRegistry.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -27,7 +27,7 @@ OpRegistry::OpRegistry() * @brief Signify that work has been completed and that all known ops should be discarded */ void OpRegistry::Finish(){ - kassert(finalized, "Finish attempted on OpRegistry when it has not been started"); + F_ASSERT(finalized, "Finish attempted on OpRegistry when it has not been started"); mutex->Lock(); //Get rid of all post ops @@ -94,6 +94,27 @@ void OpRegistry::RegisterOp(int op_id, string op_name, fn_OpCreate_t func){ } } +/** + * @brief Convert a registered op id number to a name + * @param[in] op_id The ID of the op + * @retval NAME The name of the registered op + * @retval "" ID was not registered + */ +string OpRegistry::GetOpName(int op_id) const { + auto search1 = op_names_pre.find(op_id); + if(search1 != op_names_pre.end()) { + return search1->second; + } + string result; + mutex->ReaderLock(); + auto search2 = op_names_post.find(op_id); + if(search2 != op_names_post.end()) { + result = search2->second; + } + mutex->Unlock(); + return result; +} + /** * @brief Remove an Op from service * @param[in] op_id The op id to remove diff --git a/src/opbox/common/OpRegistry.hh b/src/opbox/common/OpRegistry.hh index fc866f9..93a7f55 100644 --- a/src/opbox/common/OpRegistry.hh +++ b/src/opbox/common/OpRegistry.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPREGISTRY_HH #define OPBOX_OPREGISTRY_HH @@ -38,7 +38,7 @@ public: void RegisterOp(int op_id, std::string op_name, fn_OpCreate_t func); void DeregisterOp(int op_id, bool ignore_lock_warning=false); - + std::string GetOpName(int op_id) const; void Start() { finalized=true; } void Finish(); diff --git a/src/opbox/common/Types.cpp b/src/opbox/common/Types.cpp index ae9bc2b..e14bb4a 100644 --- a/src/opbox/common/Types.cpp +++ b/src/opbox/common/Types.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/src/opbox/common/Types.hh b/src/opbox/common/Types.hh index 7d248cc..589cbd9 100644 --- a/src/opbox/common/Types.hh +++ b/src/opbox/common/Types.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_TYPES_HH #define OPBOX_TYPES_HH diff --git a/src/opbox/core/OpBoxCoreBase.cpp b/src/opbox/core/OpBoxCoreBase.cpp index e267bda..676a38f 100644 --- a/src/opbox/core/OpBoxCoreBase.cpp +++ b/src/opbox/core/OpBoxCoreBase.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "opbox/core/OpBoxCoreBase.hh" diff --git a/src/opbox/core/OpBoxCoreBase.hh b/src/opbox/core/OpBoxCoreBase.hh index 2e6cb5c..26326b3 100644 --- a/src/opbox/core/OpBoxCoreBase.hh +++ b/src/opbox/core/OpBoxCoreBase.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPBOXCOREBASE_HH #define OPBOX_OPBOXCOREBASE_HH @@ -27,7 +27,7 @@ public: ~OpBoxCoreBase() override; - //Boostrap calls are handled by the singleton, which then calls these functions + //Bootstrap calls are handled by the singleton, which then calls these functions virtual void init(const faodel::Configuration &config)=0; virtual void start()=0; virtual void finish()=0; @@ -35,13 +35,10 @@ public: virtual int LaunchOp(Op *op, mailbox_t *resulting_mailbox)=0; virtual int TriggerOp(mailbox_t mailbox, std::shared_ptr args) = 0; - //virtual int QueueTriggerOp(mailbox_t mailbox, std::shared_ptr args) = 0; - - virtual int HandleIncomingMessage(opbox::net::peer_ptr_t peer, message_t *incoming_message) = 0; virtual int UpdateOp(Op *op, OpArgs *args)=0; - + virtual int GetNumberOfActiveOps(unsigned int op_id=0) = 0; @@ -54,8 +51,6 @@ protected: }; - - } // namespace internal } // namespace opbox diff --git a/src/opbox/core/OpBoxCoreDeprecatedStandard.cpp b/src/opbox/core/OpBoxCoreDeprecatedStandard.cpp index 3e2a992..ff8c736 100644 --- a/src/opbox/core/OpBoxCoreDeprecatedStandard.cpp +++ b/src/opbox/core/OpBoxCoreDeprecatedStandard.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -13,11 +13,18 @@ #include "whookie/Server.hh" #include "opbox/OpBox.hh" -#include "opbox/core/OpBoxCoreStandard.hh" +#include "opbox/core/OpBoxCoreDeprecatedStandard.hh" #include "opbox/core/OpBoxCoreUnconfigured.hh" #include "opbox/core/Singleton.hh" //for registry whookie +#if Faodel_ENABLE_DEBUG_TIMERS +#define OP_TIMER(a,b) if(op_timer) op_timer->Mark(a,b); +#define OP_TIMER_DISPATCHED(mbox) if(op_timer) op_timer->MarkDispatched(mbox); +#else +#define OP_TIMER(a,b) +#define OP_TIMER_DISPATCHED(mbox) +#endif using namespace std; using namespace opbox; @@ -25,16 +32,16 @@ using namespace opbox; namespace opbox { namespace internal { -OpBoxCoreStandard::OpBoxCoreStandard() +OpBoxCoreDeprecatedStandard::OpBoxCoreDeprecatedStandard() : LoggingInterface("opbox","Standard"), initialized(false), running(false), shutdown_requested(false) { op_mutex = faodel::GenerateMutex("pthreads","rwlock"); - } -OpBoxCoreStandard::~OpBoxCoreStandard(){ + +OpBoxCoreDeprecatedStandard::~OpBoxCoreDeprecatedStandard(){ dbg("standard dtor"); if (running){ @@ -43,10 +50,8 @@ OpBoxCoreStandard::~OpBoxCoreStandard(){ finish(); } - if(initialized){ + if(initialized) { shutdown_requested=true; - //th_progress.join(); - op_mutex->WriterLock(); for(auto op_ptr : active_ops){ delete op_ptr.second; @@ -55,7 +60,7 @@ OpBoxCoreStandard::~OpBoxCoreStandard(){ } delete op_mutex; - dbg("standard dtor done"); + dbg("OpBoxCoreStandard dtor done"); } @@ -64,15 +69,23 @@ OpBoxCoreStandard::~OpBoxCoreStandard(){ * * @param[in] config Info the user has passed to us for configuring the unit */ -void OpBoxCoreStandard::init(const faodel::Configuration &config) { +void OpBoxCoreDeprecatedStandard::init(const faodel::Configuration &config) { ConfigureLogging(config); dbg("private Init"); - kassert(!initialized, "Attempted to initialize OpBoxCoreStandard more than once"); + F_ASSERT(!initialized, "Attempted to initialize OpBoxCoreStandard more than once"); opbox::net::Init(config); + #if Faodel_ENABLE_DEBUG_TIMERS + //Only consider timers if the user compiles it in and asks for them + bool enable_timers; + config.GetBool(&enable_timers, "opbox.enable_timers", "false"); + if(enable_timers) op_timer = new OpTimer(); + #endif + + dbg("Done with opbox::net::Init()"); opbox::net::RegisterRecvCallback(opbox::internal::HandleIncomingMessage); @@ -80,8 +93,6 @@ void OpBoxCoreStandard::init(const faodel::Configuration &config) { return HandleWhookieStatus(args, results); }); - //Start thread - //th_progress = thread(&OpBoxCoreStandard::eventLoop, this); initialized=true; } @@ -90,9 +101,9 @@ void OpBoxCoreStandard::init(const faodel::Configuration &config) { * @brief Internal start function (called by OpBoxUnconfigured during bootstrap Initialize) * */ -void OpBoxCoreStandard::start() { +void OpBoxCoreDeprecatedStandard::start() { dbg("private Start"); - kassert(initialized, "Attempted to start OpBoxCoreStandard before initialization"); + F_ASSERT(initialized, "Attempted to start OpBoxCoreStandard before initialization"); opbox::net::Start(); running=true; } @@ -102,9 +113,9 @@ void OpBoxCoreStandard::start() { * @brief Internal finish function (called by OpBoxUnconfigured during bootstrap Finish) * */ -void OpBoxCoreStandard::finish() { +void OpBoxCoreDeprecatedStandard::finish() { dbg("private finish"); - kassert(initialized && running, "Attempted to finish OpBoxCoreStandard that is not started"); + F_ASSERT(initialized && running, "Attempted to finish OpBoxCoreStandard that is not started"); whookie::Server::deregisterHook("/opbox"); @@ -122,6 +133,10 @@ void OpBoxCoreStandard::finish() { op_mutex->Unlock(); initialized=false; } + if(op_timer) { + op_timer->Dump(); + delete op_timer; + } running=false; } @@ -136,11 +151,13 @@ void OpBoxCoreStandard::finish() { * @retval 0 Success * @retval -1 Failure */ -int OpBoxCoreStandard::doUpdate(mailbox_t my_mailbox, Op *op, OpArgs *args){ +int OpBoxCoreDeprecatedStandard::doAction(mailbox_t my_mailbox, Op *op, OpArgs *args){ WaitingType rc = op->Update(args); op->touch(); + OP_TIMER(op,OpTimerEvent::ActionComplete); + switch(rc){ case WaitingType::done_and_destroy: if(my_mailbox!=0){ @@ -159,7 +176,7 @@ int OpBoxCoreStandard::doUpdate(mailbox_t my_mailbox, Op *op, OpArgs *args){ default: //Op is doing something. Keep it in the active queue (or add it if new). - if(my_mailbox==0){ + if(my_mailbox==0) { addActiveOp(op); //Sets the mailbox if needed } } @@ -175,7 +192,7 @@ int OpBoxCoreStandard::doUpdate(mailbox_t my_mailbox, Op *op, OpArgs *args){ * @param incoming_message Pointer to the message (memory owned by net) * @throws runtime_error if op is unknown or no longer active */ -int OpBoxCoreStandard::HandleIncomingMessage(opbox::net::peer_ptr_t peer, message_t *incoming_message){ +int OpBoxCoreDeprecatedStandard::HandleIncomingMessage(opbox::net::peer_ptr_t peer, message_t *incoming_message){ Op *op; OpArgs args(peer, incoming_message); @@ -184,7 +201,7 @@ int OpBoxCoreStandard::HandleIncomingMessage(opbox::net::peer_ptr_t peer, messag dbg("Incoming message for mailbox "+to_string(my_mailbox)); //See if this is an unexpected message - if(my_mailbox == 0){ + if(my_mailbox == 0) { //New communication. Spin up a new op to andle it. op = opbox::internal::CreateNewTargetOp(incoming_message->op_id); } else { @@ -208,9 +225,11 @@ int OpBoxCoreStandard::HandleIncomingMessage(opbox::net::peer_ptr_t peer, messag } throw std::runtime_error(ss.str()); } + OP_TIMER(op,OpTimerEvent::Incoming); + //Call the update and deal with storing the op - return doUpdate(my_mailbox, op, &args); + return doAction(my_mailbox, op, &args); } @@ -223,10 +242,11 @@ int OpBoxCoreStandard::HandleIncomingMessage(opbox::net::peer_ptr_t peer, messag * @retval 0 Success * @retval -1 Failure */ -int OpBoxCoreStandard::UpdateOp(Op *op, OpArgs *args){ +int OpBoxCoreDeprecatedStandard::UpdateOp(Op *op, OpArgs *args){ + OP_TIMER(op, OpTimerEvent::Update); args->result = 0; - int rc = doUpdate(op->GetAssignedMailbox(), op, args); + int rc = doAction(op->GetAssignedMailbox(), op, args); return rc; } @@ -237,9 +257,9 @@ int OpBoxCoreStandard::UpdateOp(Op *op, OpArgs *args){ * @param[in] op A user-created op (ownership transferred to opbox) * @param[out] resulting_mailbox The mailbox generated for this op */ -int OpBoxCoreStandard::LaunchOp(Op *op, mailbox_t *resulting_mailbox){ - kassert(initialized && running, "Attempted to StartOp when OpBoxCoreStandard that is not running"); - kassert(op!=nullptr, "Tried starting a null op"); +int OpBoxCoreDeprecatedStandard::LaunchOp(Op *op, mailbox_t *resulting_mailbox){ + F_ASSERT(initialized && running, "Attempted to StartOp when OpBoxCoreStandard that is not running"); + F_ASSERT(op != nullptr, "Tried starting a null op"); dbg("LaunchOp "+op->getOpName()); @@ -249,6 +269,8 @@ int OpBoxCoreStandard::LaunchOp(Op *op, mailbox_t *resulting_mailbox){ OpArgs args(UpdateType::start); args.result = 0; + OP_TIMER(op,OpTimerEvent::Launch); + WaitingType rc = op->Update(&args); if(rc == WaitingType::done_and_destroy){ dbg("LaunchOp update completed w/ done+destroy"); @@ -256,7 +278,6 @@ int OpBoxCoreStandard::LaunchOp(Op *op, mailbox_t *resulting_mailbox){ if(resulting_mailbox) *resulting_mailbox = MAILBOX_UNSPECIFIED; args.result = 0; - return 0; } else { @@ -265,7 +286,6 @@ int OpBoxCoreStandard::LaunchOp(Op *op, mailbox_t *resulting_mailbox){ if(resulting_mailbox) *resulting_mailbox = op->GetAssignedMailbox(); args.result = 0; - return 0; } } @@ -279,7 +299,7 @@ int OpBoxCoreStandard::LaunchOp(Op *op, mailbox_t *resulting_mailbox){ * @retval 0 Success * @retval -1 Failure */ -int OpBoxCoreStandard::TriggerOp(mailbox_t mailbox, shared_ptr args){ +int OpBoxCoreDeprecatedStandard::TriggerOp(mailbox_t mailbox, shared_ptr args){ args->result = 0; Op *op = getActiveOp(mailbox); @@ -294,7 +314,9 @@ int OpBoxCoreStandard::TriggerOp(mailbox_t mailbox, shared_ptr args){ return -1; //Not found } - return doUpdate(mailbox, op, args.get()); + OP_TIMER(op,OpTimerEvent::Trigger); + + return doAction(mailbox, op, args.get()); } @@ -306,7 +328,7 @@ int OpBoxCoreStandard::TriggerOp(mailbox_t mailbox, shared_ptr args){ * @retval op A pointer to the op we want * @retval nullptr The op was not found */ -Op * OpBoxCoreStandard::getActiveOp(mailbox_t mailbox){ +Op * OpBoxCoreDeprecatedStandard::getActiveOp(mailbox_t mailbox){ if(mailbox==0) return nullptr; Op *op; @@ -324,10 +346,10 @@ Op * OpBoxCoreStandard::getActiveOp(mailbox_t mailbox){ * * @param[in] op The op in question (may trigger the assignment of a mailbox id) */ -void OpBoxCoreStandard::addActiveOp(Op *op){ - kassert(op!=nullptr, "Null active op"); +void OpBoxCoreDeprecatedStandard::addActiveOp(Op *op){ + F_ASSERT(op != nullptr, "Null active op"); mailbox_t mailbox = op->GetAssignedMailbox(); - kassert(mailbox!=0, "Op had a zero-value mailbox"); + F_ASSERT(mailbox != 0, "Op had a zero-value mailbox"); op_mutex->WriterLock(); active_ops[mailbox] = op; op_mutex->Unlock(); @@ -339,8 +361,8 @@ void OpBoxCoreStandard::addActiveOp(Op *op){ * * @param[in] mailbox An identifier for the op we want to delete */ -void OpBoxCoreStandard::endActiveOp(mailbox_t mailbox){ - kassert(mailbox!=0, "Op had a zero-value mailbox"); +void OpBoxCoreDeprecatedStandard::endActiveOp(mailbox_t mailbox){ + F_ASSERT(mailbox != 0, "Op had a zero-value mailbox"); Op *op; op_mutex->WriterLock(); auto it = active_ops.find(mailbox); @@ -352,6 +374,21 @@ void OpBoxCoreStandard::endActiveOp(mailbox_t mailbox){ op_mutex->Unlock(); } +/** + * @brief Count the number of active Ops that ObBox has (useful for shutdown) + * @param[in] op_id Optional op_id field to search on. If set, only include ops that match + * @retval COUNT the number of ops that are active + */ +int OpBoxCoreDeprecatedStandard::GetNumberOfActiveOps(unsigned int op_id) { + if(op_id==0) return active_ops.size(); + int count=0; + op_mutex->ReaderLock(); + for(auto &mbox_opptr : active_ops){ + if (mbox_opptr.second->getOpID() == op_id) count++; + } + op_mutex->Unlock(); + return count; +} /** * @brief HandleWhookieStatus Process a request from Whookie to get status information @@ -359,7 +396,7 @@ void OpBoxCoreStandard::endActiveOp(mailbox_t mailbox){ * @param[in] args The map of k/v parameters the user sent in this request * @param[in] results The stringstream to write results to */ -void OpBoxCoreStandard::HandleWhookieStatus( +void OpBoxCoreDeprecatedStandard::HandleWhookieStatus( const std::map &args, std::stringstream &results) { @@ -380,7 +417,7 @@ void OpBoxCoreStandard::HandleWhookieStatus( * @param[in] depth How many layers deeper into the component we should go * @param[in] indent How many spaces to indent this component */ -void OpBoxCoreStandard::sstr(stringstream &ss, int depth, int indent) const { +void OpBoxCoreDeprecatedStandard::sstr(stringstream &ss, int depth, int indent) const { if(depth<0) return; ss << string(indent,' ') << "[OpBoxCore] " << " Type: " << GetType() diff --git a/src/opbox/core/OpBoxCoreDeprecatedStandard.hh b/src/opbox/core/OpBoxCoreDeprecatedStandard.hh index 0aacb95..14404eb 100644 --- a/src/opbox/core/OpBoxCoreDeprecatedStandard.hh +++ b/src/opbox/core/OpBoxCoreDeprecatedStandard.hh @@ -1,9 +1,9 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. -#ifndef OPBOX_OPBOXCORESTANDARD_HH -#define OPBOX_OPBOXCORESTANDARD_HH +#ifndef OPBOX_OPBOXCOREDEPRECATEDSTANDARD_HH +#define OPBOX_OPBOXCOREDEPRECATEDSTANDARD_HH #include #include @@ -13,6 +13,7 @@ #include "lunasa/DataObject.hh" #include "opbox/ops/Op.hh" #include "opbox/core/OpBoxCoreBase.hh" +#include "opbox/core/OpTimer.hh" namespace opbox { namespace internal { @@ -29,15 +30,15 @@ namespace internal { * @deprecated The original, standard class is now deprecated due to * ordering issues that arise in a threaded environment. Use 'threaded' instead */ -class OpBoxCoreStandard : +class OpBoxCoreDeprecatedStandard : public OpBoxCoreBase, public faodel::LoggingInterface { public: - OpBoxCoreStandard(); - ~OpBoxCoreStandard() override; + OpBoxCoreDeprecatedStandard(); + ~OpBoxCoreDeprecatedStandard() override; - //Boostrap calls are handled by the singleton, which then calls these functions + //Bootstrap calls are handled by the singleton, which then calls these functions void init(const faodel::Configuration &config) override; void start() override; void finish() override; @@ -48,6 +49,8 @@ public: int HandleIncomingMessage(opbox::net::peer_ptr_t peer, message_t *incoming_message) override; int UpdateOp(Op *op, OpArgs *args) override; + int GetNumberOfActiveOps(unsigned int op_id=0) override; + void HandleWhookieStatus(const std::map &args, std::stringstream &results); std::string GetType() const override { return "standard"; } @@ -64,15 +67,12 @@ private: void addActiveOp(opbox::Op *op); void endActiveOp(mailbox_t mailbox); - int doUpdate(mailbox_t my_mailbox, Op *op, OpArgs *args); + int doAction(mailbox_t my_mailbox, Op *op, OpArgs *args); faodel::MutexWrapper *op_mutex; std::map active_ops; - const uint32_t recv_buf_count=10; - - //std::thread th_progress; - //void eventLoop(); + OpTimer *op_timer = nullptr; //Debug: collect info about how long each op took }; @@ -81,4 +81,4 @@ private: //extern opbox::OpBox ebox; //Global contained in OpBox.cpp -#endif // OPBOX_OPBOXCORESTANDARD_HH +#endif // OPBOX_OPBOXCOREDEPRECATEDSTANDARD_HH diff --git a/src/opbox/core/OpBoxCoreThreaded.cpp b/src/opbox/core/OpBoxCoreThreaded.cpp index 368edef..b681b7a 100644 --- a/src/opbox/core/OpBoxCoreThreaded.cpp +++ b/src/opbox/core/OpBoxCoreThreaded.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -15,10 +15,17 @@ #include "opbox/OpBox.hh" #include "opbox/core/OpBoxCoreThreaded.hh" -//#include "opbox/core/OpBoxCoreUnconfigured.hh" #include "opbox/core/Singleton.hh" //for registry whookie +#if Faodel_ENABLE_DEBUG_TIMERS +#define OP_TIMER(a,b) if(op_timer) op_timer->Mark(a,b); +#define OP_TIMER_DISPATCHED(mbox) if(op_timer) op_timer->MarkDispatched(mbox); +#else +#define OP_TIMER(a,b) +#define OP_TIMER_DISPATCHED(mbox) +#endif + using namespace std; using namespace opbox; @@ -44,12 +51,11 @@ OpBoxCoreThreaded::~OpBoxCoreThreaded(){ finish(); } - if(initialized){ + if(initialized) { shutdown_requested=true; - //th_progress.join(); op_mutex->WriterLock(); - for(auto op_ptr : active_ops){ + for(auto op_ptr : active_ops) { delete op_ptr.second; } op_mutex->Unlock(); @@ -69,12 +75,18 @@ void OpBoxCoreThreaded::init(const faodel::Configuration &config) { ConfigureLogging(config); - dbg("private Init"); - kassert(!initialized, "Attempted to initialize OpBoxCoreThreaded more than once"); + F_ASSERT(!initialized, "Attempted to initialize OpBoxCoreThreaded more than once"); opbox::net::Init(config); - dbg("Done with opbox::net::Init()"); + #if Faodel_ENABLE_DEBUG_TIMERS + //Only consider timers if the user compiles it in and asks for them + bool enable_timers; + config.GetBool(&enable_timers, "opbox.enable_timers", "false"); + if(enable_timers) op_timer = new OpTimer(); + #endif + + opbox::net::RegisterRecvCallback(opbox::internal::HandleIncomingMessage); whookie::Server::updateHook("/opbox", [this] (const map &args, stringstream &results) { @@ -97,7 +109,7 @@ void OpBoxCoreThreaded::init(const faodel::Configuration &config) { */ void OpBoxCoreThreaded::start() { dbg("private Start"); - kassert(initialized, "Attempted to start OpBoxCoreThreaded before initialization"); + F_ASSERT(initialized, "Attempted to start OpBoxCoreThreaded before initialization"); opbox::net::Start(); running=true; } @@ -109,14 +121,14 @@ void OpBoxCoreThreaded::start() { */ void OpBoxCoreThreaded::finish() { dbg("private finish"); - kassert(initialized && running, "Attempted to finish OpBoxCoreThreaded that is not started"); + F_ASSERT(initialized && running, "Attempted to finish OpBoxCoreThreaded that is not started"); whookie::Server::deregisterHook("/opbox"); whookie::Server::deregisterHook("/opbox/ops"); opbox::net::Finish(); - if(initialized){ + if(initialized) { dbg("deleting all"); shutdown_requested=true; //th_progress.join(); @@ -128,6 +140,10 @@ void OpBoxCoreThreaded::finish() { op_mutex->Unlock(); initialized=false; } + if(op_timer) { + op_timer->Dump(); + delete op_timer; + } running=false; } @@ -155,10 +171,11 @@ int OpBoxCoreThreaded::doAction(faodel::internal_use_only_t iuo, mailbox_t mailb return args->result; } - WaitingType rc = op->Update(args); op->touch(); + OP_TIMER(op,OpTimerEvent::ActionComplete); + switch(rc){ case WaitingType::done_and_destroy: if(mailbox!=0){ @@ -183,8 +200,6 @@ int OpBoxCoreThreaded::doAction(faodel::internal_use_only_t iuo, mailbox_t mailb return args->result; } - - /** * @brief Examine an incoming message and pass it to either a new or existing Op * @@ -211,10 +226,9 @@ int OpBoxCoreThreaded::HandleIncomingMessage(opbox::net::peer_ptr_t peer, messag op = opbox::internal::CreateNewTargetOp(incoming_message->op_id); if(op==nullptr) { - KHALT("Incoming message asked for an opid that was not known"); + F_HALT("Incoming message asked for an opid that was not known"); } - addActiveOp(op); //Sets the mailbox if needed my_mailbox = op->GetAssignedMailbox(); } else { @@ -241,26 +255,27 @@ int OpBoxCoreThreaded::HandleIncomingMessage(opbox::net::peer_ptr_t peer, messag } cerr <(peer,incoming_message, true); - - faodel::backburner::AddWork( op->GetAssignedMailbox(), - [this, my_mailbox, op, args] () mutable { - - return doAction(faodel::internal_use_only, - my_mailbox, - op, - args.get()); - }); - - //Call the update and deal with storing the op -// return doHandleIncomingMessage(my_mailbox, op, args); + auto args = make_shared(peer, incoming_message, true); + + faodel::backburner::AddWork(op->GetAssignedMailbox(), + [this, my_mailbox, op, args]() mutable { + return doAction(faodel::internal_use_only, + my_mailbox, + op, + args.get()); + }); + + OP_TIMER_DISPATCHED(my_mailbox); + return 0; } @@ -268,25 +283,26 @@ int OpBoxCoreThreaded::HandleIncomingMessage(opbox::net::peer_ptr_t peer, messag int OpBoxCoreThreaded::UpdateOp(Op *op, OpArgs *args){ //SanityCheckArgs(args); - args->result = 0; - faodel::backburner::AddWork( op->GetAssignedMailbox(), - [this, op, args] () { - doAction(faodel::internal_use_only, - op->GetAssignedMailbox(), - op, - args); - //TODO: Check returns - return 0; - }); - - //Call the update and deal with storing the op -// return doUpdate(op->GetAssignedMailbox(), op, args); - return 0; -} + OP_TIMER(op, OpTimerEvent::Update); + + mailbox_t my_mailbox = op->GetAssignedMailbox(); + args->result = 0; + faodel::backburner::AddWork(op->GetAssignedMailbox(), + [this, op, args, my_mailbox]() { + doAction(faodel::internal_use_only, + my_mailbox, + op, + args); + //TODO: Check returns + return 0; + }); + OP_TIMER_DISPATCHED(my_mailbox); + return 0; +} /** * @brief User mechanism for launching a new op. Op ownership is transferred to OpBox @@ -297,33 +313,35 @@ int OpBoxCoreThreaded::LaunchOp(Op *op, mailbox_t *resulting_mailbox){ dbg("LaunchOp enter"); - kassert(initialized && running, "Attempted to StartOp when OpBoxCoreThreaded that is not running"); - kassert(op!=nullptr, "Tried starting a null op"); + F_ASSERT(initialized && running, "Attempted to StartOp when OpBoxCoreThreaded that is not running"); + F_ASSERT(op != nullptr, "Tried starting a null op"); dbg("LaunchOp "+op->getOpName() +" state "+op->GetStateName()); addActiveOp(op); - if(resulting_mailbox) *resulting_mailbox = op->GetAssignedMailbox(); + mailbox_t my_mailbox = op->GetAssignedMailbox(); + if(resulting_mailbox) *resulting_mailbox = my_mailbox; - faodel::backburner::AddWork( op->GetAssignedMailbox(), //Put on the right thread - [this, op] () { - OpArgs args(UpdateType::start); - int rc = doAction(faodel::internal_use_only, - op->GetAssignedMailbox(), - op, - &args); - if(rc!=0) args.result = rc; - //TODO: Should we block here? + OP_TIMER(op,OpTimerEvent::Launch); - return 0; - }); + faodel::backburner::AddWork(op->GetAssignedMailbox(), //Put on the right thread + [this, op, my_mailbox]() { + OpArgs args(UpdateType::start); + int rc = doAction(faodel::internal_use_only, + my_mailbox, + op, + &args); + if(rc != 0) args.result = rc; + //TODO: Should we block here? + + return 0; + }); + OP_TIMER_DISPATCHED(my_mailbox); -// return doLaunchOp(op, args); return 0; } -#if 0 /** * @brief Allow user to trigger an update of an op and pass in user data * @@ -332,38 +350,6 @@ int OpBoxCoreThreaded::LaunchOp(Op *op, mailbox_t *resulting_mailbox){ * @retval 0 Success * @retval -1 Failure */ -int OpBoxCoreThreaded::TriggerOp(mailbox_t mailbox, OpArgs *args){ - dbg("TriggerOp enter mailbox "+to_string(mailbox)); - SanityCheckArgs(args); - - args->result = 0; - Op *op = getActiveOp(mailbox); - if(op==nullptr){ - if(mailbox==MAILBOX_UNSPECIFIED) { - args->result = -1; - return -1; - } - //throw std::runtime_error("Error: TriggerOp failed because mailbox is no longer active.\n"); - args->result = -1; - return -1; //Not found - } - - faodel::backburner::AddWork( op->GetAssignedMailbox(), - [this, mailbox, op, args] () { - doAction(faodel::internal_use_only, - mailbox, - op, - args); - - //TODO: should this block? - return 0; - }); - -// return doTriggerOp(mailbox, op, args); - return 0; -} -#endif - int OpBoxCoreThreaded::TriggerOp(mailbox_t mailbox, std::shared_ptr args) { dbg("TriggerOp enter mailbox "+to_string(mailbox)); SanityCheckArgs(args); @@ -381,19 +367,20 @@ int OpBoxCoreThreaded::TriggerOp(mailbox_t mailbox, std::shared_ptr args return -1; //Not found } + OP_TIMER(op,OpTimerEvent::Trigger); - faodel::backburner::AddWork( mailbox, - [this, mailbox, op, args] () { - doAction(faodel::internal_use_only, - mailbox, - op, - args.get()); + faodel::backburner::AddWork(mailbox, + [this, mailbox, op, args]() { + doAction(faodel::internal_use_only, + mailbox, + op, + args.get()); + //TODO: should this block? + return 0; + }); + OP_TIMER_DISPATCHED(mailbox); - //TODO: should this block? - return 0; - }); return 0; - } /** @@ -422,9 +409,9 @@ Op * OpBoxCoreThreaded::getActiveOp(mailbox_t mailbox){ * @param[in] op The op in question (may trigger the assignment of a mailbox id) */ void OpBoxCoreThreaded::addActiveOp(Op *op){ - kassert(op!=nullptr, "Null active op"); + F_ASSERT(op != nullptr, "Null active op"); mailbox_t mailbox = op->GetAssignedMailbox(); - kassert(mailbox!=0, "Op had a zero-value mailbox"); + F_ASSERT(mailbox != 0, "Op had a zero-value mailbox"); op_mutex->WriterLock(); active_ops[mailbox] = op; op_mutex->Unlock(); @@ -437,7 +424,7 @@ void OpBoxCoreThreaded::addActiveOp(Op *op){ * @param[in] mailbox An identifier for the op we want to delete */ void OpBoxCoreThreaded::endActiveOp(mailbox_t mailbox){ - kassert(mailbox!=0, "Op had a zero-value mailbox"); + F_ASSERT(mailbox != 0, "Op had a zero-value mailbox"); dbg("EndActiveOp for mailbox "+to_string(mailbox)); Op *op; op_mutex->WriterLock(); @@ -451,6 +438,22 @@ void OpBoxCoreThreaded::endActiveOp(mailbox_t mailbox){ op_mutex->Unlock(); } +/** + * @brief Count the number of active Ops that ObBox has (useful for shutdown) + * @param[in] op_id Optional op_id field to search on. If set, only include ops that match + * @retval COUNT the number of ops that are active + */ +int OpBoxCoreThreaded::GetNumberOfActiveOps(unsigned int op_id) { + if(op_id==0) return active_ops.size(); + int count=0; + op_mutex->ReaderLock(); + for(auto &mbox_opptr : active_ops){ + if (mbox_opptr.second->getOpID() == op_id) count++; + } + op_mutex->Unlock(); + return count; +} + /** * @brief HandleWhookieStatus Process a request from Whookie to get status information @@ -469,7 +472,6 @@ void OpBoxCoreThreaded::HandleWhookieStatus( rs.tableRow({"Core Type", GetType()}); rs.tableRow({"State", ((shutdown_requested) ? "Shutdown Requested" : ((running) ? "Running" : ((initialized) ? "Initialized" : "Uninitialized")))}); rs.tableRow({"Active Ops", to_string(active_ops.size())}); - rs.tableRow({"Receive Buffer Count", to_string(recv_buf_count)}); rs.tableEnd(); rs.mkText(html::mkLink("Current Active Ops", "/opbox/ops")); diff --git a/src/opbox/core/OpBoxCoreThreaded.hh b/src/opbox/core/OpBoxCoreThreaded.hh index 032034d..e37758d 100644 --- a/src/opbox/core/OpBoxCoreThreaded.hh +++ b/src/opbox/core/OpBoxCoreThreaded.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPBOXCORETHREADED_HH #define OPBOX_OPBOXCORETHREADED_HH @@ -13,6 +13,7 @@ #include "lunasa/DataObject.hh" #include "opbox/ops/Op.hh" #include "opbox/core/OpBoxCoreBase.hh" +#include "opbox/core/OpTimer.hh" namespace opbox { namespace internal { @@ -42,13 +43,11 @@ public: int LaunchOp(opbox::Op *op, mailbox_t *resulting_mailbox) override; int TriggerOp(mailbox_t mailbox, std::shared_ptr args) override; - //int EnueueTriggerOp(mailbox_t mailbox, std::shared_ptr args) override; - int HandleIncomingMessage(opbox::net::peer_ptr_t peer, message_t *incoming_message) override; int UpdateOp(Op *op, OpArgs *args) override; - - - + + int GetNumberOfActiveOps(unsigned int op_id=0) override; + void HandleWhookieStatus(const std::map &args, std::stringstream &results); void HandleWhookieActiveOps(const std::map &args, std::stringstream &results); std::string GetType() const override { return "threaded"; } @@ -67,18 +66,12 @@ private: void addActiveOp(opbox::Op *op); void endActiveOp(mailbox_t mailbox); - int doUpdate(mailbox_t my_mailbox, Op *op, OpArgs *args); - - int doLaunchOp(Op *op); - - int doHandleIncomingMessage(mailbox_t my_mailbox, Op *op, OpArgs *args); - int doUpdateOp(Op *op, OpArgs *args); - int doTriggerOp(mailbox_t mailbox, Op *op, OpArgs *args); faodel::MutexWrapper *op_mutex; std::map active_ops; - const uint32_t recv_buf_count=10; + OpTimer *op_timer = nullptr; //Debug: collect info about how long each op took + }; } //namespace internal diff --git a/src/opbox/core/OpBoxCoreUnconfigured.cpp b/src/opbox/core/OpBoxCoreUnconfigured.cpp index 6e8a342..886339d 100644 --- a/src/opbox/core/OpBoxCoreUnconfigured.cpp +++ b/src/opbox/core/OpBoxCoreUnconfigured.cpp @@ -1,11 +1,11 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "opbox/core/OpBoxCoreBase.hh" #include "opbox/core/OpBoxCoreUnconfigured.hh" -#include "opbox/core/OpBoxCoreStandard.hh" +#include "opbox/core/OpBoxCoreDeprecatedStandard.hh" #include "faodel-common/NodeID.hh" @@ -37,6 +37,7 @@ int OpBoxCoreUnconfigured::HandleIncomingMessage( message_t *incoming_message) { Panic("HandleIncomingMessage"); return -1; } int OpBoxCoreUnconfigured::UpdateOp( Op *op, OpArgs *args) { Panic("UpdateOp"); return -1; } +int OpBoxCoreUnconfigured::GetNumberOfActiveOps(unsigned int op_id) { return 0;} //For simple queries int OpBoxCoreUnconfigured::TriggerOp( mailbox_t mailbox, shared_ptr args) { Panic("TriggerOp"); return -1; } diff --git a/src/opbox/core/OpBoxCoreUnconfigured.hh b/src/opbox/core/OpBoxCoreUnconfigured.hh index 273ebfd..f7290e2 100644 --- a/src/opbox/core/OpBoxCoreUnconfigured.hh +++ b/src/opbox/core/OpBoxCoreUnconfigured.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPBOXCOREUNCONFIGURED_HH #define OPBOX_OPBOXCOREUNCONFIGURED_HH @@ -39,6 +39,7 @@ public: int LaunchOp(Op *op, mailbox_t *mailbox) override; int TriggerOp(mailbox_t mailbox, std::shared_ptr args) override; + int GetNumberOfActiveOps(unsigned int op_id=0) override; int HandleIncomingMessage(opbox::net::peer_ptr_t peer, message_t *incoming_message) override; int UpdateOp(Op *op, OpArgs *args) override; diff --git a/src/opbox/core/OpTimer.cpp b/src/opbox/core/OpTimer.cpp new file mode 100644 index 0000000..8139af6 --- /dev/null +++ b/src/opbox/core/OpTimer.cpp @@ -0,0 +1,87 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include "opbox/core/OpTimer.hh" + +using namespace std; + +namespace opbox { +namespace internal { + +OpTimer::OpTimer() { + mutex = faodel::GenerateMutex(); +} +OpTimer::~OpTimer() { + delete mutex; +} + +std::string OpTimerEvent::str() { + switch(value) { + case OpTimerEvent::Incoming: return "Incoming"; + case OpTimerEvent::Update: return "Update"; + case OpTimerEvent::Launch: return "Launch"; + case OpTimerEvent::Trigger: return "Trigger"; + case OpTimerEvent::Dispatched: return "Dispatched"; + case OpTimerEvent::ActionComplete: return "ActionComplete"; + } + return "Unknown?"; +} + +uint64_t OpTimer::op_timestamp_t::getGapTimeUS(const op_timestamp_t &prv) { + uint64_t t1 = std::chrono::duration_cast(time.time_since_epoch()).count(); + uint64_t t0 = std::chrono::duration_cast(prv.time.time_since_epoch()).count(); + return t1-t0; +} + + +void OpTimer::Mark(Op *op, OpTimerEvent event) { + op_timestamp_t ts(op,event); + mutex->WriterLock(); + timestamps.emplace_back(std::move(ts)); + mutex->Unlock(); +} + +void OpTimer::MarkDispatched(mailbox_t m) { + op_timestamp_t ts(m); + mutex->WriterLock(); + timestamps.emplace_back(std::move(ts)); + mutex->Unlock(); +} + +void OpTimer::Dump() { + + mutex->Lock(); + std::set visited_mboxes; + + cout<<"Time stamps ("<Unlock(); + +} + +} //namespace internal +} //namespace opbox diff --git a/src/opbox/core/OpTimer.hh b/src/opbox/core/OpTimer.hh new file mode 100644 index 0000000..e5681f8 --- /dev/null +++ b/src/opbox/core/OpTimer.hh @@ -0,0 +1,79 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef OPBOX_OPBOX_OPTIMER_HH +#define OPBOX_OPBOX_OPTIMER_HH + +#include +#include "opbox/ops/Op.hh" + +namespace opbox { +namespace internal { + +class OpTimerEvent { +public: + enum Value: int { + Incoming=0, + Update, + Launch, + Trigger, + Dispatched, + ActionComplete + }; + OpTimerEvent()=default; + std::string str(); + constexpr OpTimerEvent(Value e) :value(e) {} +private: + Value value; +}; + +/** + * @brief Timing code to help estimate how long it takes to execute ops + * + * This timer creates a trace of all the different events that are passed + * to each op. An instrumented OpBoxCore should include one of these + * structures and use Mark commands to add new events to the trace. The + * op's mailbox id is extracted from the op to provide a tag for grouping + * ops. The Dump command sorts by mailbox and shows the amount of time + * since the previous marker in this op. + */ +class OpTimer { + +public: + OpTimer(); + ~OpTimer(); + void Mark(Op *op, OpTimerEvent event); + void MarkDispatched(mailbox_t); + void Dump(); + +private: + struct op_timestamp_t { + mailbox_t mbox; + unsigned int opid; + OpTimerEvent event; + std::chrono::high_resolution_clock::time_point time; + + op_timestamp_t(Op *op, OpTimerEvent event) + : mbox(op->GetAssignedMailbox()), opid(op->getOpID()), event(event) { + time = std::chrono::high_resolution_clock::now(); + } + op_timestamp_t(mailbox_t m) + : mbox(m), opid(0), event(OpTimerEvent::Dispatched) { + time = std::chrono::high_resolution_clock::now(); + } + + uint64_t getGapTimeUS(const op_timestamp_t &prv); + }; + + +private: + + faodel::MutexWrapper *mutex; + std::vector timestamps; +}; + +} //namespace internal +} //namespace opbox + +#endif //OPBOX_OPBOX_OPTIMER_HH diff --git a/src/opbox/core/Singleton.cpp b/src/opbox/core/Singleton.cpp index 9607d13..b19d4fe 100644 --- a/src/opbox/core/Singleton.cpp +++ b/src/opbox/core/Singleton.cpp @@ -1,13 +1,13 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "faodel-services/BackBurner.hh" #include "opbox/core/Singleton.hh" #include "opbox/core/OpBoxCoreUnconfigured.hh" -#include "opbox/core/OpBoxCoreStandard.hh" +#include "opbox/core/OpBoxCoreDeprecatedStandard.hh" #include "opbox/core/OpBoxCoreThreaded.hh" @@ -81,7 +81,7 @@ void SingletonImpl::Init(const faodel::Configuration &config){ if(opbox_type == "threaded"){ core = new OpBoxCoreThreaded(); } else if(opbox_type == "standard"){ - core = new OpBoxCoreStandard(); + core = new OpBoxCoreDeprecatedStandard(); } else { error("Unknown opbox.type '"+opbox_type+"' in configuration. Choices: threaded, standard"); exit(-1); diff --git a/src/opbox/core/Singleton.hh b/src/opbox/core/Singleton.hh index f9dc66c..fe839e0 100644 --- a/src/opbox/core/Singleton.hh +++ b/src/opbox/core/Singleton.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_SINGLETON_HH #define OPBOX_SINGLETON_HH diff --git a/src/opbox/net/libfabric_wrapper/fab_transport.cpp b/src/opbox/net/libfabric_wrapper/fab_transport.cpp index fedce0a..96146d5 100644 --- a/src/opbox/net/libfabric_wrapper/fab_transport.cpp +++ b/src/opbox/net/libfabric_wrapper/fab_transport.cpp @@ -1,13 +1,12 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include #include //gethostname -#include #include "opbox/net/net.hh" #include "opbox/net/peer.hh" @@ -239,7 +238,7 @@ fab_transport::print_addr(fid_ep *ep) localaddr = malloc(fi->src_addrlen); remoteaddr = malloc(fi->src_addrlen); fi_getname (&ep->fid, localaddr, &addrlen); - assert(addrlen!=0); + F_ASSERT(addrlen!=0, ""); if(localaddr !=NULL) { src_addr = (char*) inet_ntoa(((struct sockaddr_in *) localaddr)->sin_addr); @@ -1018,7 +1017,7 @@ fab_transport::find_and_update_connection( //cout << "Inside print addr " <src_addrlen); fi_getpeer (ep, remoteaddr, &addrlen); - assert(addrlen!=0); + F_ASSERT(addrlen!=0,""); if(remoteaddr !=NULL){ src_addr = diff --git a/src/opbox/net/libfabric_wrapper/fab_transport.hh b/src/opbox/net/libfabric_wrapper/fab_transport.hh index 33e0edb..2035df6 100644 --- a/src/opbox/net/libfabric_wrapper/fab_transport.hh +++ b/src/opbox/net/libfabric_wrapper/fab_transport.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FABTRANSPORT_HH #define FABTRANSPORT_HH diff --git a/src/opbox/net/libfabric_wrapper/libfabric_wrapper.cpp b/src/opbox/net/libfabric_wrapper/libfabric_wrapper.cpp index 3cb28a9..95d1251 100644 --- a/src/opbox/net/libfabric_wrapper/libfabric_wrapper.cpp +++ b/src/opbox/net/libfabric_wrapper/libfabric_wrapper.cpp @@ -1,12 +1,11 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include #include //gethostname -#include #include "opbox/net/net.hh" #include "opbox/net/peer.hh" @@ -256,7 +255,7 @@ Start() std::cerr << "NetLibfabric -> Provider name (verbs/sockets/gni) not specified. Defaulting to 'sockets'." << std::endl;; trans_name = "sockets"; } - assert(whookie::Server::IsRunning() && "Whookie not started before fabric started"); + F_ASSERT(whookie::Server::IsRunning(), "Whookie not started before fabric started"); fabtrns->mynodeid = whookie::Server::GetNodeID(); // cout << "mynodeid = " << fabtrns->mynodeid.GetHex() << endl; if((trans_name == "gni") || (trans_name == "ugni")) { @@ -311,7 +310,7 @@ opbox::net::peer_ptr_t ConvertNodeIDToPeer( faodel::nodeid_t nodeid) { - KTODO("ConvertNodeIDToPeer"); + F_TODO("ConvertNodeIDToPeer"); } string @@ -645,7 +644,7 @@ Atomic( int64_t length, // TODO: this param is ignored because we only do 64-bit atomics lambda_net_update_t user_cb) { - KTODO("Libfabric wrapper atomic"); + F_TODO("Libfabric wrapper atomic"); return; } diff --git a/src/opbox/net/libfabric_wrapper/shared.hh b/src/opbox/net/libfabric_wrapper/shared.hh index ca0c5a0..1e6101c 100644 --- a/src/opbox/net/libfabric_wrapper/shared.hh +++ b/src/opbox/net/libfabric_wrapper/shared.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef SHARED_HH #define SHARED_HH diff --git a/src/opbox/net/nbr.hh b/src/opbox/net/nbr.hh index 18ebdb0..c254b69 100644 --- a/src/opbox/net/nbr.hh +++ b/src/opbox/net/nbr.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_NBR_HH #define OPBOX_NBR_HH diff --git a/src/opbox/net/net.cpp b/src/opbox/net/net.cpp index 0ac56f7..983cf7b 100644 --- a/src/opbox/net/net.cpp +++ b/src/opbox/net/net.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "opbox/net/net.hh" @@ -65,7 +65,7 @@ int opbox::net::GetRdmaPtr( opbox::net::NetBufferLocal **nbl, // out opbox::net::NetBufferRemote *nbr) // out { - uint32_t length = ldo->GetHeaderSize() + ldo->GetMetaSize() + ldo->GetDataSize(); + uint32_t length = ldo->GetHeaderSize() + ldo->GetMetaSize() + ldo->GetDataSize() + ldo->GetPaddingSize(); return opbox::net::GetRdmaPtr(ldo, ldo->GetLocalHeaderSize(), length, nbl, nbr); } diff --git a/src/opbox/net/net.hh b/src/opbox/net/net.hh index 8a2f2a4..272c274 100644 --- a/src/opbox/net/net.hh +++ b/src/opbox/net/net.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_NET_HH #define OPBOX_NET_HH diff --git a/src/opbox/net/net_internal.hh b/src/opbox/net/net_internal.hh index 464cbb4..9ec0f5a 100644 --- a/src/opbox/net/net_internal.hh +++ b/src/opbox/net/net_internal.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_NET_INTERNAL_HH #define OPBOX_NET_INTERNAL_HH diff --git a/src/opbox/net/nnti/NetNnti.cpp b/src/opbox/net/nnti/NetNnti.cpp index 2080aec..f27b5cd 100644 --- a/src/opbox/net/nnti/NetNnti.cpp +++ b/src/opbox/net/nnti/NetNnti.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -9,7 +9,6 @@ #include #include //gethostname -#include #include @@ -130,7 +129,7 @@ namespace NetNnti //CDU: TODO- Do we need to pass in sender, and msg? OpArgs *args = new OpArgs( event_to_update_type(event) ); - WaitingType cb_rc = user_cb_(args); + user_cb_(args); //CDU: end @@ -264,7 +263,7 @@ namespace NetNnti transport_name = "ibverbs"; } else if (transport_name == "sockets") { // NNTI doesn't have a sockets transport. warn and try MPI. - // TODO: the logger isn't iniitialized yet. use cerr until this is fixed. + // TODO: the logger isn't initialized yet. use cerr until this is fixed. std::cerr << "NetNnti -> net.transport.name has unsupported value 'sockets'. Failing back to 'mpi'." << std::endl; transport_name = "mpi"; } @@ -307,15 +306,17 @@ namespace NetNnti uint64_t base_addr; uint32_t length; + virtual ~NntiBufferLocal() {} + void makeRemoteBuffer ( - size_t remote_offset, // in - size_t remote_length, // in + size_t remote_offset, // in + size_t remote_length, // in NetBufferRemote *remote_buffer) override; // out }; void NntiBufferLocal::makeRemoteBuffer( - size_t remote_offset, // in - size_t remote_length, // in + size_t remote_offset, // in + size_t remote_length, // in NetBufferRemote *remote_buffer) { // out NntiBufferRemote *rb = (NntiBufferRemote*)remote_buffer; @@ -465,28 +466,34 @@ void Start() NNTI_result_t rc; //Whookie should be started by this point. We can get its info now. - assert(whookie::Server::IsRunning() && "Whookie not started before NetNnti started"); + F_ASSERT(whookie::Server::IsRunning(), "Whookie not started before NetNnti started"); myid_ = whookie::Server::GetNodeID(); t_ = nnti::transports::factory::get_instance(config_); - t_->start(); + rc = t_->start(); + if (rc == NNTI_EINVAL) { + throw std::runtime_error("Couldn't start the NNTI transport because the network device couldn't be initialized. " + "Is net.transport.name correct in $FAODEL_CONFIG file?"); + } else if (rc != NNTI_OK) { + throw std::runtime_error("Couldn't start the NNTI transport - reason unknown"); + } t_->attrs(&nnti_attrs_); nnti::datatype::nnti_event_callback unexpected_cb(t_, unexpected_callback()); rc = t_->eq_create(128, NNTI_EQF_UNEXPECTED, unexpected_cb, nullptr, &unexpected_eq_); - assert(rc==NNTI_OK && "couldn't create unexpected EQ"); + F_ASSERT(rc==NNTI_OK, "couldn't create unexpected EQ"); rc = t_->eq_create(128, NNTI_EQF_UNSET, &send_eq_); - assert(rc==NNTI_OK && "couldn't create unexpected EQ"); + F_ASSERT(rc==NNTI_OK, "couldn't create unexpected EQ"); rc = t_->eq_create(128, NNTI_EQF_UNSET, &recv_eq_); - assert(rc==NNTI_OK && "couldn't create unexpected EQ"); + F_ASSERT(rc==NNTI_OK, "couldn't create unexpected EQ"); rc = t_->eq_create(128, NNTI_EQF_UNSET, &rdma_eq_); - assert(rc==NNTI_OK && "couldn't create unexpected EQ"); + F_ASSERT(rc==NNTI_OK, "couldn't create unexpected EQ"); lunasa::RegisterPinUnpin(RegisterMemory, UnregisterMemory); @@ -504,8 +511,8 @@ void Start() */ void Finish() { - assert(initialized_ && "NetNnti not initialized"); - assert(started_ && "NetNnti not started"); + F_ASSERT(initialized_, "NetNnti not initialized"); + F_ASSERT(started_, "NetNnti not started"); started_=false; @@ -538,7 +545,7 @@ void RegisterRecvCallback( */ faodel::nodeid_t GetMyID() { - assert(initialized_ && "NetNnti not initialized"); + F_ASSERT(initialized_, "NetNnti not initialized"); return myid_; } @@ -673,9 +680,9 @@ int Connect( int Disconnect( peer_ptr_t peer) { - log_debug("NetNnti", "Disconnecting from %p", (void*)peer); + int rc = NNTI_OK; peer_bimap.right.erase(peer); - int rc = t_->disconnect(peer->p); + rc = t_->disconnect(peer->p); delete peer; return rc; } @@ -864,7 +871,7 @@ void SendMsg( base_wr.local_offset = msg_bl_offset; base_wr.remote_hdl = NNTI_INVALID_HANDLE; base_wr.remote_offset = 0; - base_wr.length = msg.GetMetaSize() + msg.GetDataSize(); + base_wr.length = msg.GetMetaSize() + msg.GetDataSize() + msg.GetPaddingSize(); } else { rc = msg.GetDataRdmaHandle( (void **)&msg_bl, msg_bl_offset); if (rc != 0) { @@ -880,7 +887,7 @@ void SendMsg( base_wr.local_offset = msg_bl_offset; base_wr.remote_hdl = NNTI_INVALID_HANDLE; base_wr.remote_offset = 0; - base_wr.length = msg.GetDataSize(); + base_wr.length = msg.GetDataSize() + msg.GetPaddingSize(); } base_wr.cb_context = new DataObject(msg); @@ -931,7 +938,7 @@ void SendMsg( base_wr.local_offset = msg_bl_offset; base_wr.remote_hdl = NNTI_INVALID_HANDLE; base_wr.remote_offset = 0; - base_wr.length = msg.GetMetaSize() + msg.GetDataSize(); + base_wr.length = msg.GetMetaSize() + msg.GetDataSize() + msg.GetPaddingSize();; } else { rc = msg.GetDataRdmaHandle( (void **)&msg_bl, msg_bl_offset); if (rc != 0) { @@ -947,7 +954,7 @@ void SendMsg( base_wr.local_offset = msg_bl_offset; base_wr.remote_hdl = NNTI_INVALID_HANDLE; base_wr.remote_offset = 0; - base_wr.length = msg.GetDataSize(); + base_wr.length = msg.GetDataSize() + msg.GetPaddingSize(); } user_invoking_callback *uicb = new user_invoking_callback(user_cb, new DataObject(msg)); @@ -989,7 +996,8 @@ void Get( NntiBufferLocal *local_bl; uint32_t local_bl_offset; - uint64_t local_size = local_ldo.GetHeaderSize() + local_ldo.GetMetaSize() + local_ldo.GetDataSize(); + uint64_t local_size = local_ldo.GetHeaderSize() + local_ldo.GetMetaSize() + + local_ldo.GetDataSize() + local_ldo.GetPaddingSize(); local_ldo.GetHeaderRdmaHandle( (void **)&local_bl, local_bl_offset); base_wr.op = NNTI_OP_GET; @@ -1013,6 +1021,29 @@ void Get( nnti::datatype::nnti_work_request wr(t_, base_wr, *get_cb); +#if 1 + { + bool abort_now=false; + // check that local/remote offset and length are 4-byte aligned + if (base_wr.local_offset%4 != 0) { + log_error("NetNnti", "opbox::net::Get() local_offset is not 4-byte aligned - %lu", base_wr.local_offset); + abort_now=true; + } + if (base_wr.remote_offset%4 != 0) { + log_error("NetNnti", "opbox::net::Get() remote_offset is not 4-byte aligned - %lu", base_wr.remote_offset); + abort_now=true; + } + if (base_wr.length%4 != 0) { + log_error("NetNnti", "opbox::net::Get() length is not 4-byte aligned - %lu (local_size = %ld / padding = %d)", + base_wr.length, local_size, local_ldo.GetPaddingSize()); + abort_now=true; + } + if (abort_now) { + abort(); + } + } +#endif + t_->get(&wr, &wid); } @@ -1061,7 +1092,7 @@ void Get( base_wr.local_offset = local_bl_offset + local_offset; base_wr.remote_hdl = remote_hdl; base_wr.remote_offset = nbr->offset + remote_offset; - base_wr.length = length; + base_wr.length = length+local_ldo.GetPaddingSize(); nnti::datatype::nnti_event_callback *get_cb = nullptr; if (user_cb) { @@ -1074,6 +1105,36 @@ void Get( nnti::datatype::nnti_work_request wr(t_, base_wr, *get_cb); +#if 1 + { + bool abort_now=false; + bool report_all=false; + // check that local/remote offset and length are 4-byte aligned + if (base_wr.local_offset%4 != 0) { + log_error("NetNnti", "opbox::net::Get() local_offset is not 4-byte aligned - %lu", base_wr.local_offset); + abort_now=true; + } else if (report_all) { + log_error("NetNnti", "opbox::net::Get() local_offset is 4-byte aligned - %lu", base_wr.local_offset); + } + if (base_wr.remote_offset%4 != 0) { + log_error("NetNnti", "opbox::net::Get() remote_offset is not 4-byte aligned - %lu", base_wr.remote_offset); + abort_now=true; + } else if (report_all) { + log_error("NetNnti", "opbox::net::Get() remote_offset is 4-byte aligned - %lu", base_wr.remote_offset); + } + if (base_wr.length%4 != 0) { + log_error("NetNnti", "opbox::net::Get() length is not 4-byte aligned - %lu (length = %d / padding = %d)", + base_wr.length, length, local_ldo.GetPaddingSize()); + abort_now=true; + } else if (report_all) { + log_error("NetNnti", "opbox::net::Get() length is 4-byte aligned - %lu", base_wr.length); + } + if (abort_now) { + abort(); + } + } +#endif + t_->get(&wr, &wid); } @@ -1108,7 +1169,7 @@ void Put( NntiBufferLocal *local_bl; uint32_t local_bl_offset; - uint64_t local_size = local_ldo.GetHeaderSize() + local_ldo.GetMetaSize() + local_ldo.GetDataSize(); + uint64_t local_size = local_ldo.GetHeaderSize() + local_ldo.GetMetaSize() + local_ldo.GetDataSize() + local_ldo.GetPaddingSize(); local_ldo.GetHeaderRdmaHandle( (void **)&local_bl, local_bl_offset); base_wr.op = NNTI_OP_PUT; diff --git a/src/opbox/net/peer.hh b/src/opbox/net/peer.hh index 3451aee..60b5158 100644 --- a/src/opbox/net/peer.hh +++ b/src/opbox/net/peer.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_PEER_HH #define OPBOX_PEER_HH diff --git a/src/opbox/ops/Op.cpp b/src/opbox/ops/Op.cpp index c3c8aa4..d7653fe 100644 --- a/src/opbox/ops/Op.cpp +++ b/src/opbox/ops/Op.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "opbox/ops/Op.hh" diff --git a/src/opbox/ops/Op.hh b/src/opbox/ops/Op.hh index d723bb4..d49d03f 100644 --- a/src/opbox/ops/Op.hh +++ b/src/opbox/ops/Op.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OP_HH #define OPBOX_OP_HH diff --git a/src/opbox/ops/OpCount.cpp b/src/opbox/ops/OpCount.cpp index e2dd51b..f64cadc 100644 --- a/src/opbox/ops/OpCount.cpp +++ b/src/opbox/ops/OpCount.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -35,7 +35,7 @@ WaitingType OpCount::UpdateOrigin(OpArgs *args) { switch(state){ case State::start: - kassert(args->type == UpdateType::user_trigger, "Was expecting a user trigger?"); + F_ASSERT(args->type == UpdateType::user_trigger, "Was expecting a user trigger?"); if(num_left){ num_left--; } diff --git a/src/opbox/ops/OpCount.hh b/src/opbox/ops/OpCount.hh index ffa7582..37847b5 100644 --- a/src/opbox/ops/OpCount.hh +++ b/src/opbox/ops/OpCount.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPCOUNT_HH #define OPBOX_OPCOUNT_HH diff --git a/src/opbox/ops/OpHelpers.hh b/src/opbox/ops/OpHelpers.hh index 844c145..bd669e7 100644 --- a/src/opbox/ops/OpHelpers.hh +++ b/src/opbox/ops/OpHelpers.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPHELPERS_HH #define OPBOX_OPHELPERS_HH diff --git a/src/opbox/ops/OpPing.cpp b/src/opbox/ops/OpPing.cpp index 1dfb3ec..26c6b98 100644 --- a/src/opbox/ops/OpPing.cpp +++ b/src/opbox/ops/OpPing.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -74,7 +74,7 @@ std::future OpPing::GetFuture() { */ WaitingType OpPing::smo_Start(){ - kassert(peer!=nullptr, "Didn't get a proper peer?"); + F_ASSERT(peer != nullptr, "Didn't get a proper peer?"); opbox::net::SendMsg(peer, std::move(ldo_msg), AllEventsCallback(this) ); @@ -162,7 +162,7 @@ WaitingType OpPing::UpdateOrigin(OpArgs *args) { case State::done: return WaitingType::done_and_destroy; default: ; } - KFAIL(); //Should never reach here + F_FAIL(); //Should never reach here return WaitingType::error; } @@ -178,7 +178,7 @@ WaitingType OpPing::UpdateTarget(OpArgs *args){ case State::done: return WaitingType::done_and_destroy; default: ; } - KFAIL(); //Should never reach here + F_FAIL(); //Should never reach here return WaitingType::error; } diff --git a/src/opbox/ops/OpPing.hh b/src/opbox/ops/OpPing.hh index 0c91566..25fe991 100644 --- a/src/opbox/ops/OpPing.hh +++ b/src/opbox/ops/OpPing.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef OPBOX_OPPING_HH #define OPBOX_OPPING_HH diff --git a/src/sbl/README_SBL.md b/src/sbl/README_SBL.md index 35aa489..e64d4bc 100644 --- a/src/sbl/README_SBL.md +++ b/src/sbl/README_SBL.md @@ -1,6 +1,6 @@ # SBL: A Simplified Interface to Boost.Log -Simplified Boost.Log (SBL) provides an easy to use interface to +Simplified Boost.Log (SBL) provides an easyto use interface to Boost.Log. Many key features of Boost.Log are available including multiple sinks, application created channels, per-sink severity filter and per-channel severity filters. diff --git a/src/sbl/sbl_boost_headers.hh b/src/sbl/sbl_boost_headers.hh index 8d0a8dc..ba17177 100644 --- a/src/sbl/sbl_boost_headers.hh +++ b/src/sbl/sbl_boost_headers.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef SBL_BOOST_HEADERS_HH_ diff --git a/src/sbl/sbl_logger.cpp b/src/sbl/sbl_logger.cpp index cff4e9d..e78f0ad 100644 --- a/src/sbl/sbl_logger.cpp +++ b/src/sbl/sbl_logger.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * sbl_logger.cpp diff --git a/src/sbl/sbl_logger.hh b/src/sbl/sbl_logger.hh index a0951a7..49c520f 100644 --- a/src/sbl/sbl_logger.hh +++ b/src/sbl/sbl_logger.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * sbl_logger.hh diff --git a/src/sbl/sbl_source.cpp b/src/sbl/sbl_source.cpp index 41c20ea..10289c1 100644 --- a/src/sbl/sbl_source.cpp +++ b/src/sbl/sbl_source.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * sbl_logger.cpp @@ -96,7 +96,7 @@ namespace sbl { { const char *file; char buf1[256]; - char buf2[256]; + char buf2[1024]; /* path from last '/' */ file = strrchr(file_name, '/'); diff --git a/src/sbl/sbl_source.hh b/src/sbl/sbl_source.hh index c11ae5e..8b42ded 100644 --- a/src/sbl/sbl_source.hh +++ b/src/sbl/sbl_source.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * sbl_source.hh diff --git a/src/sbl/sbl_stream.cpp b/src/sbl/sbl_stream.cpp index e0c89da..4418cff 100644 --- a/src/sbl/sbl_stream.cpp +++ b/src/sbl/sbl_stream.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * sbl_logger.cpp diff --git a/src/sbl/sbl_stream.hh b/src/sbl/sbl_stream.hh index 4e06caa..6b7ee24 100644 --- a/src/sbl/sbl_stream.hh +++ b/src/sbl/sbl_stream.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * sbl_stream.hh diff --git a/src/sbl/sbl_types.hh b/src/sbl/sbl_types.hh index 4ca2e5f..c5dce7e 100644 --- a/src/sbl/sbl_types.hh +++ b/src/sbl/sbl_types.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * sbl_types.hh diff --git a/src/whookie/Server.hh b/src/whookie/Server.hh index 37d35ae..1841110 100644 --- a/src/whookie/Server.hh +++ b/src/whookie/Server.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef WHOOKIE_SERVER_HH #define WHOOKIE_SERVER_HH diff --git a/src/whookie/Whookie.hh b/src/whookie/Whookie.hh index fcd0e68..299aee8 100644 --- a/src/whookie/Whookie.hh +++ b/src/whookie/Whookie.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef WHOOKIE_WHOOKIE_HH #define WHOOKIE_WHOOKIE_HH diff --git a/src/whookie/client/Client.cpp b/src/whookie/client/Client.cpp index 324f9d1..3c1e841 100644 --- a/src/whookie/client/Client.cpp +++ b/src/whookie/client/Client.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/src/whookie/client/Client.hh b/src/whookie/client/Client.hh index 3c4d085..c62a2b6 100644 --- a/src/whookie/client/Client.hh +++ b/src/whookie/client/Client.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef WHOOKIE_CLIENT_HH #define WHOOKIE_CLIENT_HH diff --git a/src/whookie/server/ServerBoost.cpp b/src/whookie/server/ServerBoost.cpp index 36e46b5..d365f25 100644 --- a/src/whookie/server/ServerBoost.cpp +++ b/src/whookie/server/ServerBoost.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "whookie/Server.hh" #include "whookie/server/boost/server.hpp" diff --git a/src/whookie/server/boost/request_handler.cpp b/src/whookie/server/boost/request_handler.cpp index 433cf8b..c824e1f 100644 --- a/src/whookie/server/boost/request_handler.cpp +++ b/src/whookie/server/boost/request_handler.cpp @@ -42,6 +42,10 @@ request_handler::request_handler() { dumpAbout(args, results); }); + registerHook("/proc", [this] (const map &args, stringstream &results) { + dumpProc(args, results); + }); + } void request_handler::handle_request(const request& req, reply& rep) { @@ -196,6 +200,73 @@ libraries.)"); rs.Finish(); } + +void request_handler::dumpProc(const map &args, stringstream &results){ + + vector> cmds = { + { "io", "I/O statistics (number of bytes I/O has read/written)"}, + { "limits", "Hard/soft limits for OS resources"}, + { "sched", "Scheduling stats (time is in ms)"}, + { "stat", "One-line stats about the process"}, + { "statm", "One-line stats about memory in Pages (ProgramSize,RSS,Shared,Text,0,data+stack,0)"}, + { "status", "Human version of stat and statm"} + }; + + auto myMakeLink = [](const std::string name) { + return "/proc/self/"+name+""; + }; + + //See if we're the top page + auto k_v = args.find("item"); + + if(k_v != args.end()) { + for(auto &item_info : cmds) { + if(k_v->second == item_info.first) { + auto args_mod = args; + string file_name = "/proc/self/"+k_v->second; + faodel::ReplyStream rs(args, file_name, &results); + if(rs.IsHTML()) { + rs.mkSection("Current "+file_name); + rs.mkText(item_info.second); + } + string line; + ifstream ifs(file_name); + stringstream ss; + if(ifs.is_open()) { + while(getline(ifs, line)) { + ss<Return to /proc"); + } + rs.Finish(); + return; + } + } + //Didn't find - bail out and dump main menu + } + + faodel::ReplyStream rs(args, "Process Info", &results); + rs.mkSection("Proc Info",1); + rs.mkText(R"( +The following are links to different /proc/self files that you can use to get more info +about this process's runtime. Remember to add &format=text to get the plain text.)"); + + rs.tableBegin("Proc Info"); + rs.tableTop({"Item", "About"}); + for(auto &item_info : cmds) { + rs.tableRow( { myMakeLink(item_info.first), item_info.second }); + } + rs.tableEnd(); + rs.Finish(); + return; + +} + + // note: not safe to insert hooks here? random crashes observed pair request_handler::splitString(const string &item, char delim){ diff --git a/src/whookie/server/boost/request_handler.hpp b/src/whookie/server/boost/request_handler.hpp index 2d19960..e0792db 100644 --- a/src/whookie/server/boost/request_handler.hpp +++ b/src/whookie/server/boost/request_handler.hpp @@ -63,6 +63,8 @@ class request_handler void dumpRegisteredHandles(const std::map &args, std::stringstream &results); void dumpAbout(const std::map &args, std::stringstream &results); + void dumpProc(const std::map &args, std::stringstream &results); + void dumpFile(const std::map &args, std::stringstream &results, const std::string &file_name); }; diff --git a/src/whookie/server/boost/server.cpp b/src/whookie/server/boost/server.cpp index acddc85..a92313f 100644 --- a/src/whookie/server/boost/server.cpp +++ b/src/whookie/server/boost/server.cpp @@ -46,7 +46,7 @@ server::server() server::~server(){ stop(); } -void server::Init(const faodel::Configuration &config){ +void server::InitAndModifyConfiguration(faodel::Configuration *config){ int64_t port; string interfaces; @@ -54,11 +54,11 @@ void server::Init(const faodel::Configuration &config){ string interface_address(""); //Extract config info for whookie - ConfigureLogging(config); //Grab logging - config.GetString(&app_name, "whookie.app_name", "Whookie Application"); - config.GetInt(&port, "whookie.port", "1990"); - config.GetLowercaseString(&address, "whookie.address", "0.0.0.0"); - config.GetLowercaseString(&interfaces, "whookie.interfaces", "eth,lo"); + ConfigureLogging(*config); //Grab logging + config->GetString(&app_name, "whookie.app_name", "Whookie Application"); + config->GetInt(&port, "whookie.port", "1990"); + config->GetLowercaseString(&address, "whookie.address", "0.0.0.0"); + config->GetLowercaseString(&interfaces, "whookie.interfaces", "eth,lo"); /* * How whookie finds an address to bind to: @@ -77,7 +77,8 @@ void server::Init(const faodel::Configuration &config){ requested_address = address; requested_port = port; - config.GetAllSettings(&config_entries); //Copy all of the config settings for display + //config.GetAllSettings(&config_entries); //Copy all of the config settings for display + bootstrap_config_ptr = config; //Get a reference back to bootstrap asio_ = new asio_resources(); do_await_stop(); @@ -113,8 +114,11 @@ void server::HandleWhookieConfig(const map &args, stringstream &r rs.tableRow({"Whookie Link", rs.createLink( my_nodeid.GetHttpLink(), my_nodeid.GetHttpLink(), false) }); rs.tableRow({"NodeID", rs.createLink( my_nodeid.GetHex(), my_nodeid.GetHttpLink(), false) }); // my_nodeid.GetHtmlLink()}); rs.tableEnd(); - - rs.mkTable(config_entries, "User-Supplied Configuration"); + + vector> config_entries; + bootstrap_config_ptr->GetAllSettings(&config_entries); + + rs.mkTable(config_entries, "Current Configuration"); rs.mkText(rs.createBold("Note:")+"These are the parameters provided to bootstrap. Some values " "(eg whookie.port) may have been adjusted due to conflicts\n"); diff --git a/src/whookie/server/boost/server.hpp b/src/whookie/server/boost/server.hpp index f56da2a..bba29a0 100644 --- a/src/whookie/server/boost/server.hpp +++ b/src/whookie/server/boost/server.hpp @@ -73,7 +73,9 @@ class server : ~server() override; //Bootstrap standard apis - void Init(const faodel::Configuration &config) override; + //void Init(const faodel::Configuration &config) override; + void Init(const faodel::Configuration &config) override {} + void InitAndModifyConfiguration(faodel::Configuration *config) override; void Start() override; void Finish() override; void GetBootstrapDependencies(std::string &name, @@ -118,8 +120,7 @@ class server : faodel::nodeid_t my_nodeid = faodel::NODE_UNSPECIFIED; bool configured_; - std::vector> config_entries; - + faodel::Configuration *bootstrap_config_ptr; unsigned int port_; std::mutex configured_mutex_; diff --git a/tests/common/unit/tb_common_bootstrap.cpp b/tests/common/unit/tb_common_bootstrap.cpp index dd7697b..b2e32d0 100644 --- a/tests/common/unit/tb_common_bootstrap.cpp +++ b/tests/common/unit/tb_common_bootstrap.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -9,7 +9,9 @@ #include #include -#include "gtest/gtest.h" +#include + +#include #include "faodel-common/Common.hh" @@ -278,21 +280,27 @@ TEST_F(FaodelBootstrap, missingDep) { } -class A : public bootstrap::BootstrapInterface { +class A + : public bootstrap::BootstrapInterface, + public LoggingInterface { public: int state; faodel::bootstrap::internal::Bootstrap *bs; - A(faodel::bootstrap::internal::Bootstrap *bs) : state(0),bs(bs) {} + A(faodel::bootstrap::internal::Bootstrap *bs) : LoggingInterface("A"), state(0), bs(bs) {} void Init(const Configuration &config) override { + ConfigureLogging(config); + dbg("Init"); EXPECT_EQ(0, state); state=1; }; void Start() override { + dbg("Start"); EXPECT_EQ(1, state); state=2; } void Finish() override { + dbg("Finish"); EXPECT_EQ(2, state); state=3; } @@ -308,19 +316,27 @@ class A : public bootstrap::BootstrapInterface { } }; -class B : public bootstrap::BootstrapInterface { +class B + : public bootstrap::BootstrapInterface, + public LoggingInterface { public: int state; + int num_times_init; + int num_times_finish; faodel::bootstrap::internal::Bootstrap *bs; - B(faodel::bootstrap::internal::Bootstrap *bs) : state(0), bs(bs) {} + B(faodel::bootstrap::internal::Bootstrap *bs) + : LoggingInterface("B"), state(0), num_times_init(0), num_times_finish(0), bs(bs) {} void Init(const Configuration &config) override { + ConfigureLogging(config); + num_times_init++; EXPECT_EQ(0, state); //Get a pointer to A BootstrapInterface *bsc = bs->GetComponentPointer("A"); EXPECT_NE(nullptr, bsc); if(bsc!=nullptr) { + dbg("Init got valid a pointer"); A *aptr = static_cast(bsc); int tmp = aptr->GetState(); EXPECT_EQ(1,tmp); //A should be init'd @@ -333,6 +349,7 @@ class B : public bootstrap::BootstrapInterface { state=2; } void Finish() override { + num_times_finish++; EXPECT_EQ(2, state); state=3; } @@ -349,8 +366,6 @@ class B : public bootstrap::BootstrapInterface { TEST_F(FaodelBootstrap, simpleClassInterfaces) { - - //A a; B b; B *b = new B(bs); A *a = new A(bs); @@ -362,16 +377,36 @@ TEST_F(FaodelBootstrap, simpleClassInterfaces) { vector names_exp = {"A","B"}; EXPECT_EQ(names_exp, names); - Configuration config(""); + Configuration config(""); //("bootstrap.debug true\nA.debug true\nB.debug true"); config.AppendFromReferences(); bs->Start(config); bs->Finish(true); EXPECT_EQ(3, a->state); - //EXPECT_EQ(3, b->state); + delete b; + delete a; } +TEST_F(FaodelBootstrap, allowMultipleStarts) { + A *a = new A(bs); + B *b = new B(bs); + + bs->RegisterComponent(a,true); //static_cast(&a)); + bs->RegisterComponent(b,true); + + Configuration config("bootstrap.debug true"); //("bootstrap.debug true\nA.debug true\nB.debug true"); + config.AppendFromReferences(); + + bs->Start(config); EXPECT_EQ(1, b->num_times_init); EXPECT_EQ(0, b->num_times_finish); + bs->Start(config); EXPECT_EQ(1, b->num_times_init); EXPECT_EQ(0, b->num_times_finish); + EXPECT_EQ(2, bs->GetNumberOfUsers()); + bs->Finish(true); EXPECT_EQ(1, b->num_times_init); EXPECT_EQ(0, b->num_times_finish); + bs->Finish(true); EXPECT_EQ(1, b->num_times_init); EXPECT_EQ(1, b->num_times_finish); + +} + + //============================================================================== // Modify Configuration tests: verify a bootstrap can modify the config //============================================================================== @@ -466,7 +501,7 @@ default.iom.path /this/is/path2 char fname[] = "/tmp/mytestXXXXXX"; int fd = mkstemp(fname); //cout <<"Name is "< @@ -56,9 +56,6 @@ TEST_F(FaodelConfiguration, constConfig){ s=""; c1.GetString(&s, "config.additional_files.env_name.if_defined"); EXPECT_EQ("FAODEL_CONFIG", s); s=""; c2.GetString(&s, "config.additional_files.env_name.if_defined"); EXPECT_EQ("MY_ENV_VAR", s); - - - } @@ -85,8 +82,27 @@ TEST_F(FaodelConfiguration, appendString) { EXPECT_EQ("nextval", s); EXPECT_EQ(0,rc); + //Try appending when value doesn't exist + s=""; + rc = c.GetString(&s, "nothere"); + EXPECT_EQ(ENOENT,rc); + EXPECT_EQ("",s); + rc = c.AppendIfUnset("nothere", "set-by-first-aiu"); + rc = c.GetString(&s, "nothere"); + EXPECT_EQ(0,rc); + EXPECT_EQ("set-by-first-aiu", s); + + + + //Try conditional appending when value does exist + rc = c.AppendIfUnset("nothere", "should-not-overwrite"); + rc = c.GetString(&s, "nothere"); + EXPECT_EQ(0, rc); + EXPECT_EQ("set-by-first-aiu", s); + } + TEST_F(FaodelConfiguration, tabs_vs_spaces) { Configuration c; c.Append("thing1 value1"); @@ -125,7 +141,7 @@ TEST_F(FaodelConfiguration, from_env_and_file) { memset(namebuf,0,sizeof(namebuf)); strncpy(namebuf,"/tmp/ktst-XXXXXX",16); int fd = mkstemp(namebuf); - write(fd, ss.str().c_str(), ss.str().length()); + ssize_t wlen = write(fd, ss.str().c_str(), ss.str().length()); close(fd); rc_t rc; @@ -159,7 +175,7 @@ TEST_F(FaodelConfiguration, filename_expansion1) { memset(namebuf,0,sizeof(namebuf)); strncpy(namebuf,"/tmp/ktst-XXXXXX",16); int fd = mkstemp(namebuf); - write(fd, ss.str().c_str(), ss.str().length()); + auto wlen = write(fd, ss.str().c_str(), ss.str().length()); close(fd); // put TEST_TMP in the environment @@ -197,7 +213,7 @@ TEST_F(FaodelConfiguration, filename_expansion2) { memset(namebuf,0,sizeof(namebuf)); strncpy(namebuf,"/tmp/ktst-XXXXXX",16); int fd = mkstemp(namebuf); - write(fd, ss1.str().c_str(), ss1.str().length()); + auto wlen = write(fd, ss1.str().c_str(), ss1.str().length()); close(fd); // create a Configuration that will read an external file after substituting $TEST_TMP @@ -430,8 +446,7 @@ TEST_F(FaodelConfiguration, parseStringblock) { string default_config = R"EOF( default.kelpie.core_type nonet -#lkv settings for the server -server.max_capacity 32M +server.my_capacity 32M #Client only one specified client.fake_thing bob @@ -461,7 +476,7 @@ node_role server rc = c.GetString(&val, "fake_thing","frank"); EXPECT_EQ(2,rc); EXPECT_EQ("frank", val); rc = c.GetString(&val, "client.fake_thing"); EXPECT_EQ(0,rc); EXPECT_EQ("bob", val); - rc = c.GetInt(&ival, "max_capacity"); EXPECT_EQ(0,rc); EXPECT_EQ(32*1024*1024, ival); + rc = c.GetInt(&ival, "my_capacity"); EXPECT_EQ(0,rc); EXPECT_EQ(32*1024*1024, ival); } @@ -569,7 +584,7 @@ default.iom.path /this/is/path2 char fname[] = "/tmp/mytestXXXXXX"; int fd = mkstemp(fname); //cout <<"File Name is "< @@ -97,7 +97,8 @@ void * th_burnRW(void *argsIn) { //cout <<"Doing w lock on " << key << " " << args->id << " " << i << endl; args->mutex->WriterLock(); - (*args->rw_map)[key] = key; + //(*args->rw_map)[key] = key; + (*args->rw_map)[key] = val; mutex->Unlock(); num_wr++; } else { diff --git a/tests/common/unit/tb_common_resourceurl.cpp b/tests/common/unit/tb_common_resourceurl.cpp index cda0268..6243329 100644 --- a/tests/common/unit/tb_common_resourceurl.cpp +++ b/tests/common/unit/tb_common_resourceurl.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -128,7 +128,17 @@ TEST_F(UrlTest, LocalReference) { ResourceURL l4("/local/stuff/bob"); EXPECT_EQ("local",l4.Type()); } +TEST_F(UrlTest, Dashify) { + ResourceURL u1("local:/my/thing/here&is=working¬=broken"); + EXPECT_EQ("-my-thing-here", u1.Dashify()); + ResourceURL u2("dht:/my"); + EXPECT_EQ("-my", u2.Dashify()); + + ResourceURL u3("local:"); + EXPECT_EQ("-", u3.Dashify()); + +} TEST_F(UrlTest, LocalOptions) { //Local can be a special case.. More hand tests to make sure it really works @@ -257,7 +267,7 @@ TEST_F(UrlTest, BadFormats) { ru = ResourceURL(surls_ok[i]); if(!ru.Valid()) cout <<"Ok url came up invalid? "< @@ -11,7 +11,7 @@ #include "gtest/gtest.h" #include "faodel-common/Common.hh" -#include "faodel-common/SerializationHelpers.hh" +#include "faodel-common/SerializationHelpersBoost.hh" @@ -222,3 +222,40 @@ TEST_F(SerializationTest, BlobPointer) { free(foo); } + +struct FakeKey { + FakeKey() =default; + FakeKey(int a_, int b_) : a(StringZeroPad(a_,255)), b(StringZeroPad(b_,255)) {} + string a; + string b; + bool operator== (const FakeKey &other) const { return (a == other.a) && (b == other.b); } + + template + void serialize(Archive &ar, const unsigned int version) { + ar & a; + ar & b; + } +}; + + +TEST_F(SerializationTest, LargeNames) { + + //Example to make sure serialization can handle a larger number of objects + vector bignames; + for(int i=0; i<128; i++) { + for(int j=0; j<128; j++) { + bignames.push_back( FakeKey(i,j) ); + } + } + + string packed = BoostPack>(bignames); + cout <<"Packed size is "< bignames2 = BoostUnpack>(packed); + EXPECT_EQ(bignames.size(), bignames2.size()); + if(bignames.size()==bignames2.size()){ + for(int i=0; i @@ -281,6 +281,8 @@ TEST(StringHelpers, RangeParsing) { x = ExtractIDs("2-5,4-6",8); exp={2,3,4,5,6}; EXPECT_EQ( exp, x); x = ExtractIDs("1-2,4-5,7-8",9); exp={1,2,4,5,7,8}; EXPECT_EQ( exp, x); x = ExtractIDs("4-5,1,3,8",9); exp={1,3,4,5,8}; EXPECT_EQ( exp, x); + x = ExtractIDs("0-0",2); exp={0}; EXPECT_EQ( exp, x); + //Good - with spaces x = ExtractIDs("4-5, 1, 3 , 8",9); exp={1,3,4,5,8}; EXPECT_EQ( exp, x); x = ExtractIDs("1 - 2, 4 - 5, 7- 8",9); exp={1,2,4,5,7,8}; EXPECT_EQ( exp, x); @@ -294,7 +296,7 @@ TEST(StringHelpers, RangeParsing) { x = ExtractIDs("0-middle,end",4); exp={0,1,3}; EXPECT_EQ( exp,x); x = ExtractIDs("middleplus,end",4); exp={2,3}; EXPECT_EQ( exp, x); x = ExtractIDs("all,0",4); exp={0,1,2,3}; EXPECT_EQ( exp,x); - + x = ExtractIDs("0-middle",2); exp={0}; EXPECT_EQ( exp,x); //Bad values EXPECT_ANY_THROW(ExtractIDs("1", 1)); @@ -348,3 +350,25 @@ TEST(PunyCode, RawData) { EXPECT_EQ(src,dec); EXPECT_NE(src,enc); } + +TEST(StringToNum, Basics) { + int rc; + uint64_t val; + + //Good + rc = StringToTimeUS(&val, "9"); EXPECT_EQ(0,rc); EXPECT_EQ(9, val); + rc = StringToTimeUS(&val, "100us"); EXPECT_EQ(0,rc); EXPECT_EQ(100, val); + rc = StringToTimeUS(&val, "6 us"); EXPECT_EQ(0,rc); EXPECT_EQ(6, val); + rc = StringToTimeUS(&val, "82ms"); EXPECT_EQ(0,rc); EXPECT_EQ(82000, val); + rc = StringToTimeUS(&val, "3s"); EXPECT_EQ(0,rc); EXPECT_EQ(3000000ul, val); + rc = StringToTimeUS(&val, "400 Seconds"); EXPECT_EQ(0,rc); EXPECT_EQ(400*1000000, val); + rc = StringToTimeUS(&val, "5minutes"); EXPECT_EQ(0,rc); EXPECT_EQ(5*60*1000000ul, val); + rc = StringToTimeUS(&val, "2 hours"); EXPECT_EQ(0,rc); EXPECT_EQ(2*3600*1000000ul, val); + + //Bad + rc = StringToTimeUS(&val, "hours"); EXPECT_EQ(EINVAL,rc); EXPECT_EQ(0, val); + rc = StringToTimeUS(&val, "hour"); EXPECT_EQ(EINVAL,rc); EXPECT_EQ(0, val); + rc = StringToTimeUS(&val, "9x46minutes"); EXPECT_EQ(EINVAL, rc); EXPECT_EQ(0UL, val); + + +} \ No newline at end of file diff --git a/tests/common/unit/tb_common_structs.cpp b/tests/common/unit/tb_common_structs.cpp index e3abb4c..275ec4e 100644 --- a/tests/common/unit/tb_common_structs.cpp +++ b/tests/common/unit/tb_common_structs.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // This aggregates tests for basic structures including: // @@ -78,7 +78,6 @@ TEST_F(BucketTest, Copies) { } TEST_F(BucketTest, BadHexNode) { - uint64_t x; string s; //If we already computed the hash, we can pass it around @@ -223,7 +222,7 @@ TEST_F(NodeIDTest, BadUrls) { nodeid_t node_id(urls[i]); cout < #include @@ -28,6 +28,9 @@ using namespace dirman; string default_config_string = R"EOF( +# Switch dirman to none, since we don't have a root node +dirman.type none + # IMPORTANT: This test starts/finishes bootstrap multiple times. Lunasa's # tcmalloc memory manager doesn't support this feature, so we have to # switch to malloc instead. If your external CONFIG file sets these @@ -199,7 +202,7 @@ int main(int argc, char **argv){ } //One last start/finish, this time with a real teardown - bootstrap::Start(Configuration(""), opbox::bootstrap); + bootstrap::Start(Configuration("dirman.type none"), opbox::bootstrap); bootstrap::Finish(); //Deletes the bootstrap item with all the hooks MPI_Finalize(); diff --git a/tests/dirman/component/mpi_dirman_restart.cpp b/tests/dirman/component/mpi_dirman_restart.cpp index 81a0f6a..55af644 100644 --- a/tests/dirman/component/mpi_dirman_restart.cpp +++ b/tests/dirman/component/mpi_dirman_restart.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/dirman/unit/tb_dirman_corecentralized.cpp b/tests/dirman/unit/tb_dirman_corecentralized.cpp index f211bc4..ef7a0a7 100644 --- a/tests/dirman/unit/tb_dirman_corecentralized.cpp +++ b/tests/dirman/unit/tb_dirman_corecentralized.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/dirman/unit/tb_dirman_directorycache.cpp b/tests/dirman/unit/tb_dirman_directorycache.cpp index f0f44c8..85c56eb 100644 --- a/tests/dirman/unit/tb_dirman_directorycache.cpp +++ b/tests/dirman/unit/tb_dirman_directorycache.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/dirman/unit/tb_dirman_directoryownercache.cpp b/tests/dirman/unit/tb_dirman_directoryownercache.cpp index d2c79b1..a387b7d 100644 --- a/tests/dirman/unit/tb_dirman_directoryownercache.cpp +++ b/tests/dirman/unit/tb_dirman_directoryownercache.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/kelpie/CMakeLists.txt b/tests/kelpie/CMakeLists.txt index c868fc3..69fd45a 100644 --- a/tests/kelpie/CMakeLists.txt +++ b/tests/kelpie/CMakeLists.txt @@ -1,5 +1,4 @@ include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/mpi ) -SET(EXTRA_TEST_LIBS "-lz -ldl") # Build the test support lib------------------------------------------------- if(Faodel_ENABLE_MPI_SUPPORT) @@ -27,7 +26,6 @@ set(COMMON_TEST_LIBS kelpie ${FaodelNetlib_TARGETS} Boost::serialization - ${EXTRA_TEST_LIBS} ) set(SERIAL_TEST_LIBS @@ -43,27 +41,31 @@ set(MPI_TEST_LIBS -#--------------+-----------------------------+---------------+---------+ -# Format: | Name | Directory | Autorun | -#--------------+-----------------------------+---------------+---------+ -add_serial_test( tb_kelpie_types unit true ) -add_serial_test( tb_kelpie_key unit true ) -add_serial_test( tb_kelpie_localkv unit true ) -add_serial_test( tb_kelpie_message_direct unit/messages true ) -add_serial_test( tb_kelpie_iom_pio_basic unit/ioms true ) -add_serial_test( tb_kelpie_iom_all_basic unit/ioms true ) -add_serial_test( tb_kelpie_iom_service_basic unit/ioms false ) - -if( Faodel_ENABLE_MPI_SUPPORT ) - - #TODO: the nonet has mpi stuff in it to make multiple tests run - add_serial_test( tb_kelpie_nonet_iompio component/nonet true ) +#--------------+--------------------------------+---------------+---------+ +# Format: | Name | Directory | Autorun | +#--------------+--------------------------------+---------------+---------+ +add_serial_test( tb_kelpie_types unit true ) +add_serial_test( tb_kelpie_key unit true ) +add_serial_test( tb_kelpie_localkv unit true ) +add_serial_test( tb_kelpie_message_direct unit/messages true ) +add_serial_test( tb_kelpie_multiple_hidden_inits component true ) +add_serial_test( tb_kelpie_nonet_compute component/nonet true ) +add_serial_test( tb_kelpie_nonet_iompio component/nonet true ) +add_serial_test( tb_kelpie_nonet_iom_from_url component/nonet true ) +add_serial_test( tb_kelpie_iom_pio_basic unit/ioms true ) +if( Faodel_HAVE_GTEST_TYPED_TEST_SUITE_API ) + add_serial_test( tb_kelpie_iom_all_basic unit/ioms true ) + add_serial_test( tb_kelpie_iom_service_basic unit/ioms false ) +endif() +if( Faodel_ENABLE_MPI_SUPPORT ) + add_mpi_test( mpi_kelpie_compute component 2 true) add_mpi_test( mpi_kelpie_simple_peer component 2 true) + add_mpi_test( mpi_kelpie_many_items component 2 true) add_mpi_test( mpi_kelpie_dht component 4 true) add_mpi_test( mpi_kelpie_rft component 16 true) - # add_mpi_test( mpi_kelpie_standalone component 4 true) #TODO: add harness to do network test + add_mpi_test( mpi_kelpie_behaviors component 2 true) add_mpi_test( mpi_kelpie_iom_dht component 16 true) endif() diff --git a/tests/kelpie/component/mpi_kelpie_behaviors.cpp b/tests/kelpie/component/mpi_kelpie_behaviors.cpp new file mode 100644 index 0000000..4270371 --- /dev/null +++ b/tests/kelpie/component/mpi_kelpie_behaviors.cpp @@ -0,0 +1,148 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +// This test verifies that behaviors are working right. It spins off a single dht +// node on either the +#include + +#include "gtest/gtest.h" + +#include "faodel-common/Common.hh" +#include "lunasa/Lunasa.hh" +#include "opbox/OpBox.hh" +#include "dirman/DirMan.hh" +#include "faodel-services/MPISyncStart.hh" +#include "kelpie/Kelpie.hh" + +#include "support/ExperimentLauncher.hh" + +using namespace std; +using namespace faodel; +using namespace opbox; + +const int CMD_DUMP_RESOURCES = 1; +const int CMD_WRITE_PARTICLES = 2; +const int CMD_CHECK_PARTICLES = 3; //Check a list of keys to see if they match + +bool ENABLE_DEBUG=false; + + +const int PARTICLE_BLOB_BYTES = 1024; + + +string default_config_string = R"EOF( + +# Multiple runs need to be done with malloc +lunasa.lazy_memory_manager malloc +lunasa.eager_memory_manager malloc + +# Enable all debug by labeling this node's role as debug_node +debug_node.mpisyncstart.debug true +debug_node.bootstrap.debug true +debug_node.whookie.debug true +debug_node.opbox.debug true +debug_node.dirman.debug true +debug_node.dirman.cache.mine.debug true +debug_node.dirman.cache.others true +debug_node.dirman.cache.owners true +debug_node.kelpie.debug true +debug_node.kelpie.pool.debug true +debug_node.lunasa.debug true +debug_node.lunasa.allocator.debug true + + +#bootstrap.status_on_shutdown true +#bootstrap.halt_on_shutdown true + +bootstrap.sleep_seconds_before_shutdown 0 + +# All iom work is PIO and goes to faodel_data +default.kelpie.iom.type PosixIndividualObjects +default.kelpie.iom.path ./faodel_data + +## All Tests must define any additional settings in this order: +## mpisyncstart.enable -- if mpi is filling in any info +## default.kelpie.ioms -- list of ioms everyone should have +## (kelpie.iom.iomname.path) -- a path for each iom's path, if not default +## dirman.type -- centralized or static +## dirman.root_node -- root id if you're centralized +## dirman.resources -- lists of all the dirman entries to use + + +)EOF"; + + + +class BehaviorsTest : public testing::Test { +protected: + void SetUp() override { + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + + if(mpi_size!=2) { + cerr <<"This test must be run with exactly two ranks.\n"; + exit(-1); + } + //Enable debug mode + s_config = (!ENABLE_DEBUG) ? default_config_string : default_config_string + "node_role debug_node\n"; + + char p1[] = "/tmp/gtestXXXXXX"; + string path1 = mkdtemp(p1); + + s_config += "\nkelpie.iom.myiom.path " + path1; + } + + void TearDown() override { + //cout <<"Server teardown\n"; + el_bcastCommand(CMD_TEARDOWN); + bootstrap::Finish(); + MPI_Barrier(MPI_COMM_WORLD); + } + + string s_config; + + int mpi_rank, mpi_size; + int rc; +}; + +TEST_F(BehaviorsTest, WriteRemote) { + + s_config += + R"EOF( +mpisyncstart.enable true +default.kelpie.ioms my_iom +dirman.type centralized +dirman.root_node_mpi 0 +dirman.resources_mpi[] dht:/mydht&iom=my_iom 1 + +)EOF"; + + //Share our config and start + el_bcastConfig(CMD_NEW_KELPIE_START, s_config); + mpisyncstart::bootstrap(); + bootstrap::Start(Configuration(s_config), kelpie::bootstrap); + + //The tester is also the root. It should find the pool and detect that myiom1 hash is associated with it + auto pool = kelpie::Connect("ref:/mydht"); + EXPECT_EQ(faodel::hash32("my_iom"), pool.GetIomHash()); + EXPECT_EQ(kelpie::pool_behavior_t(kelpie::PoolBehavior::DefaultRemoteIOM), pool.GetBehavior()); + + kelpie::object_info_t info; + lunasa::DataObject ldo(1024); + rc = pool.Publish(kelpie::Key("write-remote"), ldo, &info); + EXPECT_EQ(kelpie::KELPIE_OK, rc); + cout <<"row info: "<< info.str() << endl; + + + + +} + + + + + +int main(int argc, char **argv){ + + return el_defaultMain(argc, argv); +} \ No newline at end of file diff --git a/tests/kelpie/component/mpi_kelpie_compute.cpp b/tests/kelpie/component/mpi_kelpie_compute.cpp new file mode 100644 index 0000000..ed43717 --- /dev/null +++ b/tests/kelpie/component/mpi_kelpie_compute.cpp @@ -0,0 +1,170 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +// +// Test: mpi_kelpie_compute +// Purpose: Set up a simple dht to verify Compute functions to remote nodes work + + +#include + +#include "gtest/gtest.h" + +#include "faodel-common/Common.hh" +#include "lunasa/Lunasa.hh" +#include "lunasa/common/Helpers.hh" +#include "opbox/OpBox.hh" +#include "dirman/DirMan.hh" +#include "kelpie/Kelpie.hh" +#include "whookie/Server.hh" + +#include "support/Globals.hh" + +using namespace std; +using namespace faodel; +using namespace opbox; + +//Globals holds mpi info and manages connections (see ping example for info) +Globals G; + + +//Note: Additional info will automatically be pulled from FAODEL_CONFIG +string default_config_string = R"EOF( + +dirman.root_role rooter +dirman.type centralized + +target.dirman.host_root + +# MPI tests will need to have a standard networking base +kelpie.type standard + +#bootstrap.debug true +#whookie.debug true +#opbox.debug true +#dirman.debug true +#kelpie.debug true + +#kelpie.op.compute.debug true + +)EOF"; + +using namespace kelpie; + +class MPIComputeTest : public testing::Test { +protected: + virtual void SetUp(){ + ResourceURL url("dht:/mydht"); + dht = kelpie::Connect(url); + } + + void TearDown() override {} + int rc; + kelpie::Pool dht; +}; + +TEST_F(MPIComputeTest, Setup) { + + ResultCollector res(4); + for(auto &name : {"a","b1","b2","c"}) { + Key k1("Stuff",name); + lunasa::DataObject ldo = lunasa::AllocateStringObject(k1.str()); + dht.Publish(k1, ldo, res); + } + res.Sync(); + + lunasa::DataObject ldo; + + //Pick first item in list + rc = dht.Compute(Key("Stuff","*"), "pick", "first", &ldo); + EXPECT_EQ(KELPIE_OK, rc); + string s4 = lunasa::UnpackStringObject(ldo); + EXPECT_EQ("Stuff|a", s4); + + //Pick the last item in the list + rc = dht.Compute(Key("Stuff","*"), "pick", "last", &ldo); + EXPECT_EQ(KELPIE_OK, rc); + string s5 = lunasa::UnpackStringObject(ldo); + EXPECT_EQ("Stuff|c", s5); + + //Pick the largest object. b1 and b2 are the largest, should pick the first + rc = dht.Compute(Key("Stuff","*"), "pick", "largest", &ldo); + EXPECT_EQ(KELPIE_OK, rc); + string s6 = lunasa::UnpackStringObject(ldo); + EXPECT_EQ("Stuff|b1", s6); + + //Pick the smallest object. There are multiple here (a and c). It should pick the first + rc = dht.Compute(Key("Stuff","*"), "pick", "smallest", &ldo); + EXPECT_EQ(KELPIE_OK, rc); + string s7 = lunasa::UnpackStringObject(ldo); + EXPECT_EQ("Stuff|a", s7); + +} + +//Same as previous, but switch out to using a result collector +TEST_F(MPIComputeTest, Collector) { + + ResultCollector res(4); + for(auto &name : {"a","b1","b2","c"}) { + Key k1("Stuff",name); + lunasa::DataObject ldo = lunasa::AllocateStringObject(k1.str()); + dht.Publish(k1, ldo, res); + } + res.Sync(); + + lunasa::DataObject ldo; + + ResultCollector res2(4); + rc = dht.Compute(Key("Stuff","*"), "pick", "first", res2); EXPECT_EQ(KELPIE_OK, rc); + rc = dht.Compute(Key("Stuff","*"), "pick", "last", res2); EXPECT_EQ(KELPIE_OK, rc); + rc = dht.Compute(Key("Stuff","*"), "pick", "largest", res2); EXPECT_EQ(KELPIE_OK, rc); + rc = dht.Compute(Key("Stuff","*"), "pick", "smallest", res2); EXPECT_EQ(KELPIE_OK, rc); + + res2.Sync(); + + string s4 = lunasa::UnpackStringObject(res2.results[0].ldo); EXPECT_EQ("Stuff|a", s4); //First + string s5 = lunasa::UnpackStringObject(res2.results[1].ldo); EXPECT_EQ("Stuff|c", s5); //Last + string s6 = lunasa::UnpackStringObject(res2.results[2].ldo); EXPECT_EQ("Stuff|b1", s6); //Largest + string s7 = lunasa::UnpackStringObject(res2.results[3].ldo); EXPECT_EQ("Stuff|a", s7); //Smallest + +} + + + + +void targetLoop(){ + //G.dump(); +} + +int main(int argc, char **argv){ + + int rc=0; + ::testing::InitGoogleTest(&argc, argv); + + //Insert any Op registrations here + + faodel::Configuration config(default_config_string); + config.AppendFromReferences(); + G.StartAll(argc, argv, config); + + + if(G.mpi_rank==0){ + //Register the dht + DirectoryInfo dir_info("dht:/mydht","This is My DHT"); + for(int i=1; i +#include +#include #include "gtest/gtest.h" @@ -30,12 +32,13 @@ using namespace opbox; Globals G; //Parameters used in this experiment +// Note: wante more than 10 for num_rows/cols so we can wildcard on 0* and 1* struct Params { int num_rows; int num_cols; int ldo_size; }; -Params P = { 8, 10, 20*1024 }; +Params P = { 16, 16, 20*1024 }; @@ -127,33 +130,19 @@ lunasa::DataObject generateLDO(int num_words, uint32_t start_val){ return ldo; } -bool ldosEqual(const lunasa::DataObject &ldo1, lunasa::DataObject &ldo2) { - if(ldo1.GetMetaSize() != ldo2.GetMetaSize()) { cout<<"Meta size mismatch\n"; return false; } - if(ldo1.GetDataSize() != ldo2.GetDataSize()) { cout<<"Data size mismatch\n"; return false; } - - if(ldo1.GetMetaSize() > 0) { - auto *mptr1 = ldo1.GetMetaPtr(); - auto *mptr2 = ldo2.GetMetaPtr(); - for(int i=ldo1.GetMetaSize()-1; i>=0; i--) - if(mptr1[i] != mptr2[i]) { cout <<"Meta mismatch at offset "< 0) { - auto *dptr1 = ldo1.GetDataPtr(); - auto *dptr2 = ldo2.GetDataPtr(); - for(int i=ldo1.GetDataSize()-1; i>=0; i--) - if(dptr1[i] != dptr2[i]) { cout <<"Data mismatch at offset "<> generateKVs(std::string prefix) { static int ldos_generated=0; vector> items; for(int i=0; i> generateAndPublish(kelpie::Pool dh atomic num_left(kvs.size()); for(auto &kv : kvs) { rc = dht.Publish(kv.first, kv.second, - [&num_left] (kelpie::rc_t result, kelpie::kv_row_info_t &ri, kelpie::kv_col_info_t &ci) { + [&num_left] (kelpie::rc_t result, kelpie::object_info_t &ci) { EXPECT_EQ(0, result); num_left--; } ); @@ -188,15 +177,15 @@ void checkInfo(kelpie::Pool dht, const vector> &kvs) { @@ -207,7 +196,7 @@ void checkNeed(kelpie::Pool dht, const vector> &kvs) { @@ -221,10 +210,9 @@ void checkWantBounded(kelpie::Pool dht, const vector0; ) { + i--; + //cout < keys; + for(int i=0; i= 4 && "mpi_kelpie_dht needs to be 4 or larger"); if(G.mpi_rank==0){ //Register the dht diff --git a/tests/kelpie/component/mpi_kelpie_iom_dht.cpp b/tests/kelpie/component/mpi_kelpie_iom_dht.cpp index 7230e25..2ad6c78 100644 --- a/tests/kelpie/component/mpi_kelpie_iom_dht.cpp +++ b/tests/kelpie/component/mpi_kelpie_iom_dht.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -57,13 +57,13 @@ debug_node.lunasa.allocator.debug true bootstrap.sleep_seconds_before_shutdown 0 # All iom work is PIO and goes to faodel_data -default.iom.type PosixIndividualObjects -default.iom.path ./faodel_data +default.kelpie.iom.type PosixIndividualObjects +default.kelpie.iom.path ./faodel_data ## All Tests must define any additional settings in this order: ## mpisyncstart.enable -- if mpi is filling in any info -## default.ioms -- list of ioms everyone should have -## (iom.iomname.path) -- a path for each iom's path, if not default +## default.kelpie.ioms -- list of ioms everyone should have +## (kelpie.iom.iomname.path) -- a path for each iom's path, if not default ## dirman.type -- centralized or static ## dirman.root_node -- root id if you're centralized ## dirman.resources -- lists of all the dirman entries to use @@ -97,9 +97,9 @@ class IOMTest : public testing::Test { string path2 = mkdtemp(p2); string path3 = mkdtemp(p3); - s_config += "\niom.myiom1.path " + path1 + - "\niom.myiom2.path " + path2 + - "\niom.myiom3.path " + path3; + s_config += "\nkelpie.iom.myiom1.path " + path1 + + "\nkelpie.iom.myiom2.path " + path2 + + "\nkelpie.iom.myiom3.path " + path3; } void TearDown() override { @@ -127,7 +127,7 @@ TEST_F(IOMTest, SetupPools) { s_config+= R"EOF( mpisyncstart.enable true -default.ioms myiom1;myiom2;myiom3 +default.kelpie.ioms myiom1;myiom2;myiom3 dirman.type centralized dirman.root_node_mpi 0 @@ -185,7 +185,7 @@ TEST_F(IOMTest, CurrentDirectory) { s_config += R"EOF( mpisyncstart.enable false -default.ioms empire_particles;empire_fields +default.kelpie.ioms empire_particles;empire_fields # just using default iom paths dirman.type static dirman.resources[] local:/EMPIRE/particles&iom=empire_particles @@ -215,8 +215,8 @@ dirman.resources[] local:/EMPIRE/fields&iom=empire_fields //Write some data out el_bcastCommand(CMD_WRITE_PARTICLES); - int rc = writeParticles(""); - EXPECT_EQ(1032, rc); //1024+8 + int wiresize = writeParticles(""); + EXPECT_EQ(PARTICLE_BLOB_BYTES+8, wiresize); //includes header //Verify that files are there @@ -233,7 +233,7 @@ TEST_F(IOMTest, WriteIOMDHT) { s_config += + R"EOF( mpisyncstart.enable true -default.ioms my_iom +default.kelpie.ioms my_iom dirman.type centralized dirman.root_node_mpi 0 dirman.resources_mpi[] dht:/EMPIRE/particles&info=booya&iom=my_iom ALL @@ -279,15 +279,16 @@ dirman.resources_mpi[] dht:/EMPIRE/particles&info=booya&iom=my_iom ALL for(int i=0; i0; retries--) { - rc = pool.Info(key, &col_info); + rc = pool.Info(key, &info); if(rc!=kelpie::KELPIE_OK){ //cout <<"Didn't find "<0; retries--){ - rc = pool.Info(key, &col_info); + rc = pool.Info(key, &info); if(rc==kelpie::KELPIE_OK) break; sleep(2); } if(rc!=kelpie::KELPIE_OK) { sso << "fail Could not find " << key.str() << endl; bad_count++; - } else if (col_info.num_bytes-8!=expected_bytes) { //TODO: Replace 8 with ldo header size.. or fiz up stream - sso << "fail Expected length of blob was wrong. Got "< +#include + +#include +#include + +#include + +#include "gtest/gtest.h" + +#include "faodel-common/Common.hh" +#include "lunasa/Lunasa.hh" +#include "opbox/OpBox.hh" +#include "dirman/DirMan.hh" +#include "faodel-services/MPISyncStart.hh" +#include "kelpie/Kelpie.hh" +#include "kelpie/ops/direct/msg_direct.hh" + +#include "whookie/Server.hh" + +#include "support/Globals.hh" + +using namespace std; +using namespace faodel; +using namespace opbox; + + + +//Parameters used in this experiment +// Note: wante more than 10 for num_rows/cols so we can wildcard on 0* and 1* +struct Params { + int num_rows; + int num_cols; + uint32_t ldo_size; +}; +Params P = { 16, 32, 1024 }; + + + +string default_config_string = R"EOF( + + +mpisyncstart.debug true + +dirman.type centralized +dirman.root_node_mpi 0 +dirman.resources_mpi[] dht:/target 1 + + +)EOF"; + +class MPIManyItems : public testing::Test { +protected: + void SetUp() override { + target = kelpie::Connect("/target"); + my_id = net::GetMyID(); + } + + void TearDown() override { + } + kelpie::Pool target; + + faodel::nodeid_t my_id; + int rc; +}; + + +TEST_F(MPIManyItems, VerifySaneWhookieIP) { + //If whookie chose the wrong port, it can get a bad ip address + + EXPECT_TRUE( my_id.Valid() ); + EXPECT_TRUE( my_id.ValidIP() ); //Most likely to break if whookie.interfaces is bad + EXPECT_TRUE( my_id.ValidPort() ); +} + + +TEST_F(MPIManyItems, MessagePackingCheck) { + //This test is here just to verify that cereal serial packs everything correctly + + + const int NUM_ROWS=P.num_rows; + const int NUM_COLS=P.num_cols; + vector keys; + kelpie::ObjectCapacities oc; + size_t spot=0; + for(int i=0; i(); + + //Fake a reply message + AllocateCerealReplyMessage(ldo2, &imsg->hdr, 0, oc); + + //Pretend like we got the reply + auto *omsg = ldo2.GetDataPtr(); + auto found_oc = opbox::UnpackCerealMessage(&omsg->hdr); + + EXPECT_EQ(oc.Size(), found_oc.Size()); + if(oc.Size() == found_oc.Size()) { + spot=0; + for(int i=0; i(); + for(int i=0; i<1024/sizeof(int); i++) + x[i]=i; + + const int NUM_ROWS=P.num_rows; + const int NUM_COLS=P.num_cols; + + //Generate all the row/col names (just zero padded numbers) + vector expected_cols; + for(int i=0; i 0) { - auto *mptr1 = ldo1.GetMetaPtr(); - auto *mptr2 = ldo2.GetMetaPtr(); - for(int i=ldo1.GetMetaSize()-1; i>=0; i--) - if(mptr1[i] != mptr2[i]) { cout <<"Meta mismatch at offset "< 0) { - auto *dptr1 = ldo1.GetDataPtr(); - auto *dptr2 = ldo2.GetDataPtr(); - for(int i=ldo1.GetDataSize()-1; i>=0; i--) - if(dptr1[i] != dptr2[i]) { cout <<"Data mismatch at offset "<(); for(int i=0; i<64/sizeof(int); i++) x[i]=i; - - kelpie::kv_row_info_t row_info; - kelpie::kv_col_info_t col_info; + + kelpie::object_info_t info; kelpie::Key key1("single_for_back_r0"); kelpie::Key key2("single_for_back_r0", "part2"); - rc = rft_back.Publish(key1, ldo1, &row_info, &col_info); + rc = rft_back.Publish(key1, ldo1, &info); EXPECT_EQ(KELPIE_OK, rc); - EXPECT_EQ(1, row_info.num_cols_in_row); - EXPECT_EQ(64, row_info.row_bytes); - EXPECT_EQ(64, col_info.num_bytes); - EXPECT_EQ(Availability::InRemoteMemory, row_info.availability); - EXPECT_EQ(Availability::InRemoteMemory, col_info.availability); + EXPECT_EQ(1, info.row_num_columns); + EXPECT_EQ(64, info.row_user_bytes); + EXPECT_EQ(64, info.col_user_bytes); + EXPECT_EQ(Availability::InRemoteMemory, info.col_availability); - rc = rft_back.Publish(key2, ldo1, &row_info, &col_info); + rc = rft_back.Publish(key2, ldo1, &info); EXPECT_EQ(KELPIE_OK, rc); - EXPECT_EQ(2, row_info.num_cols_in_row); - EXPECT_EQ(128, row_info.row_bytes); - EXPECT_EQ(64, col_info.num_bytes); - EXPECT_EQ(Availability::InRemoteMemory, row_info.availability); - EXPECT_EQ(Availability::InRemoteMemory, col_info.availability); + EXPECT_EQ(2, info.row_num_columns); + EXPECT_EQ(128, info.row_user_bytes); + EXPECT_EQ(64, info.col_user_bytes); + EXPECT_EQ(Availability::InRemoteMemory, info.col_availability); //Verify these are NOT local - rc = local.Info(key1, &col_info); EXPECT_EQ(KELPIE_ENOENT, rc); - rc = local.Info(key2, &col_info); EXPECT_EQ(KELPIE_ENOENT, rc); + rc = local.Info(key1, &info); EXPECT_EQ(KELPIE_ENOENT, rc); + rc = local.Info(key2, &info); EXPECT_EQ(KELPIE_ENOENT, rc); //See if its on rank0 - rc = rft_back0.Info(key1, &col_info); + rc = rft_back0.Info(key1, &info); EXPECT_EQ(KELPIE_OK, rc); - EXPECT_EQ(64, col_info.num_bytes); - EXPECT_EQ(Availability::InRemoteMemory, col_info.availability); + EXPECT_EQ(64, info.col_user_bytes); + EXPECT_EQ(Availability::InRemoteMemory, info.col_availability); //See if its on rank1 - rc = rft_back1.Info(key1, &col_info); + rc = rft_back1.Info(key1, &info); EXPECT_EQ(KELPIE_ENOENT, rc); lunasa::DataObject ldo2; rc = rft_back0.Need(key1, &ldo2); EXPECT_EQ(rc, KELPIE_OK); - EXPECT_TRUE(ldosEqual(ldo1,ldo2)); + EXPECT_EQ(0, ldo1.DeepCompare(ldo2)); } @@ -273,9 +245,7 @@ int main(int argc, char **argv){ faodel::Configuration config(default_config_string); config.AppendFromReferences(); - G.StartAll(argc, argv, config); - - assert(G.mpi_size >= 3 && "mpi_kelpie_rft needs to be 3 or larger"); + G.StartAll(argc, argv, config, 3); if(G.mpi_rank==0){ diff --git a/tests/kelpie/component/mpi_kelpie_simple_peer.cpp b/tests/kelpie/component/mpi_kelpie_simple_peer.cpp index 5381218..9154c56 100644 --- a/tests/kelpie/component/mpi_kelpie_simple_peer.cpp +++ b/tests/kelpie/component/mpi_kelpie_simple_peer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // // Test: mpi_kelpie_simple_peer @@ -67,6 +67,26 @@ lunasa::DataObject generateLDO(int num_words, uint32_t start_val){ return ldo; } +TEST_F(MPISimplePeerTest, Setup) { + ResourceURL url("dht:/mydht"); + kelpie::Pool dht = kelpie::Connect(url); + EXPECT_EQ(kelpie::pool_behavior_t(kelpie::PoolBehavior::DefaultRemote), dht.GetBehavior()); + EXPECT_TRUE(dht.Valid()); + + kelpie::Pool dht2 = kelpie::Connect(ResourceURL("/mydht")); + EXPECT_EQ(kelpie::pool_behavior_t(kelpie::PoolBehavior::DefaultRemote), dht.GetBehavior()); + EXPECT_TRUE(dht2.Valid()); + + //Try connecting with something not valid. Should get an error pool + //handle back that tells us we're not valid. If we call pool function + //on an invalid pool, it should throw an exception. + kelpie::Pool dhtx = kelpie::Connect(ResourceURL("/not-here")); + EXPECT_FALSE(dhtx.Valid()); + string s; + dhtx.Valid(&s); + EXPECT_FALSE(s.empty()); + EXPECT_ANY_THROW(dhtx.Info(kelpie::Key("dummy"),nullptr)); +} TEST_F(MPISimplePeerTest, BasicDHTCreateAndPublish){ @@ -78,7 +98,7 @@ TEST_F(MPISimplePeerTest, BasicDHTCreateAndPublish){ //First: Verify remote doesn't know what this item is - kelpie::kv_col_info_t col_info; + kelpie::object_info_t col_info; rc = dht.Info(k1, &col_info); EXPECT_NE(0, rc); @@ -89,10 +109,12 @@ TEST_F(MPISimplePeerTest, BasicDHTCreateAndPublish){ EXPECT_EQ(1024*sizeof(int), ldo1.GetMetaSize()+ldo1.GetDataSize()); //Publish the object out to destination - rc = dht.Publish(k1, ldo1); + kelpie::object_info_t oi; + rc = dht.Publish(k1, ldo1, &oi); EXPECT_EQ(0, rc); EXPECT_EQ(1024*sizeof(int), ldo1.GetMetaSize()+ldo1.GetDataSize()); //Sanity check + EXPECT_EQ(kelpie::Availability::InRemoteMemory, oi.col_availability); //See if we can get the info back. Give it five tries int count=5; @@ -109,9 +131,35 @@ TEST_F(MPISimplePeerTest, BasicDHTCreateAndPublish){ EXPECT_EQ(0, rc); if(rc==0){ //cout <<"Got info :"< bad_keys = { kelpie::Key("foo*"), + kelpie::Key("foo","bar*"), + kelpie::Key("foo*","bar*") }; + + auto nop_pub = [](kelpie::rc_t result, kelpie::object_info_t &info) {}; + auto nop_want = [](bool success, kelpie::Key key, lunasa::DataObject user_ldo, const kelpie::object_info_t &info) {}; + lunasa::DataObject ldo; + kelpie::ResultCollector res(1); + kelpie::object_info_t info; + for(auto bad_key : bad_keys) { + EXPECT_ANY_THROW(dht.Publish(bad_key, nop_pub)); + EXPECT_ANY_THROW(dht.Publish(bad_key, ldo)); + EXPECT_ANY_THROW(dht.Publish(bad_key, ldo, &info)); + EXPECT_ANY_THROW(dht.Publish(bad_key, ldo, res)); + EXPECT_ANY_THROW(dht.Want(bad_key, nop_want)); + EXPECT_ANY_THROW(dht.Want(bad_key, 100, nop_want)); + EXPECT_ANY_THROW(dht.Want(bad_key, res)); + EXPECT_ANY_THROW(dht.Want(bad_key, 100, res)); } + } @@ -178,12 +226,13 @@ TEST_F(MPISimplePeerTest, BasicWantUnbounded){ int num_words=1024; kelpie::Key k4("obj4"); - kelpie::kv_col_info_t col_info; + kelpie::object_info_t col_info; ResourceURL url("dht:/mydht"); kelpie::Pool dht = kelpie::Connect(url); kelpie::Pool local = kelpie::Connect("local:"); + //First, make sure it isn't here yet rc = local.Info(k4, &col_info); EXPECT_EQ(kelpie::KELPIE_ENOENT, rc); rc = dht.Info(k4, &col_info); EXPECT_EQ(kelpie::KELPIE_ENOENT, rc); @@ -197,7 +246,8 @@ TEST_F(MPISimplePeerTest, BasicWantUnbounded){ rc = dht.Info(k4, &col_info); EXPECT_EQ(kelpie::KELPIE_WAITING, rc); - //Create an ldo and publish to DHT + + //Create an ldo and do a blocking publish to DHT auto ldo4 = generateLDO(num_words, 1); rc = dht.Publish(k4, ldo4); EXPECT_EQ(0, rc); @@ -209,8 +259,8 @@ TEST_F(MPISimplePeerTest, BasicWantUnbounded){ if(rc!=kelpie::KELPIE_OK) { sleep(1); } else { - EXPECT_EQ(1024*sizeof(int), col_info.num_bytes); - EXPECT_EQ(kelpie::Availability::InLocalMemory, col_info.availability); + EXPECT_EQ(1024*sizeof(int), col_info.col_user_bytes); + EXPECT_EQ(kelpie::Availability::InLocalMemory, col_info.col_availability); success=true; break; } diff --git a/tests/kelpie/component/nonet/tb_kelpie_nonet_compute.cpp b/tests/kelpie/component/nonet/tb_kelpie_nonet_compute.cpp new file mode 100644 index 0000000..f75f957 --- /dev/null +++ b/tests/kelpie/component/nonet/tb_kelpie_nonet_compute.cpp @@ -0,0 +1,165 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include + +#include "faodelConfig.h" +#ifdef Faodel_ENABLE_MPI_SUPPORT +#include +#endif + +#include "gtest/gtest.h" + +#include "lunasa/common/Helpers.hh" +#include "faodel-common/ResourceURL.hh" +#include "kelpie/Kelpie.hh" + +#include "kelpie/core/Singleton.hh" //For core access + +using namespace std; +using namespace faodel; +using namespace kelpie; + +//The configuration used in this example +std::string default_config_string = R"EOF( + +# For local testing, tell kelpie to use the nonet implementation +kelpie.type nonet +dirman.type none + +kelpie.debug true +kelpie.pool.debug true + +# Uncomment these options to get debug info for each component +#bootstrap.debug true +#whookie.debug true +#opbox.debug true +#dirman.debug true +#kelpie.debug true + +# We start/stop multiple times (which lunasa's tcmalloc does not like), so +# we have to switch to a plain malloc allocator +lunasa.lazy_memory_manager malloc +lunasa.eager_memory_manager malloc + +)EOF"; + + +class KelpieCompute : public testing::Test { +protected: + void SetUp() override { + config.Append(default_config_string); + bootstrap::Init(config, kelpie::bootstrap); + } + virtual void TearDown() { + bootstrap::Finish(); + } + + rc_t rc; + internal_use_only_t iuo; + Configuration config; + +}; + +rc_t fn_mystuff(faodel::bucket_t, const Key &key, const std::string &args, + std::map ldos, lunasa::DataObject *ext_ldo) { + + cout<<"Got mystuff call for key "< +#include +#include + +#include "faodelConfig.h" +#ifdef Faodel_ENABLE_MPI_SUPPORT +#include +#endif + +#include "gtest/gtest.h" + +#include "faodel-common/ResourceURL.hh" +#include "kelpie/Kelpie.hh" + +#include "kelpie/core/Singleton.hh" //For core access + +using namespace std; +using namespace faodel; +using namespace kelpie; + +//The configuration used in this example +std::string default_config_string = R"EOF( + +# For local testing, tell kelpie to use the nonet implementation +kelpie.type nonet +dirman.type none + + +# Uncomment these options to get debug info for each component +#bootstrap.debug true +#whookie.debug true +#opbox.debug true +#dirman.debug true +#kelpie.debug true + +# We start/stop multiple times (which lunasa's tcmalloc does not like), so +# we have to switch to a plain malloc allocator +lunasa.lazy_memory_manager malloc +lunasa.eager_memory_manager malloc + +)EOF"; + + +class IomFromUrl : public testing::Test { +protected: + void SetUp() override { + + + config.Append(default_config_string); + + + bootstrap::Start(config, kelpie::bootstrap); + } + virtual void TearDown() { + bootstrap::Finish(); + } + + rc_t rc; + internal_use_only_t iuo; + Configuration config; + +}; + +TEST_F(IomFromUrl, Basics) { + + string url1( "dht:/my/thing" + "&iom=foobar" + "&iom_type=PosixIndividualObjects" + "&iom_path=/tmp/zip" ); + + rc = internal::getKelpieCore()->iom_registry.RegisterIomFromURL(faodel::ResourceURL(url1)); + //rc = kelpie::RegisterIomFromURL( faodel::ResourceURL(url1) ); + EXPECT_EQ(0,rc); + + string url2( "dht:/my/other/thing" + "&iom=boston-creme" + "&iom_type=PosixIndividualObjects" + "&iom_path=/tmp/zip" ); + + rc = internal::getKelpieCore()->iom_registry.RegisterIomFromURL(faodel::ResourceURL(url2)); + //rc = kelpie::RegisterIomFromURL( faodel::ResourceURL(url2) ); + EXPECT_EQ(0,rc); + + string url3( "dht:/my/other/thing" + "&iom=honey-glaze" + "&iom_type=PosixIndividualObjects" + "&iom_path=/tmp/zip" ); + + rc = internal::getKelpieCore()->iom_registry.RegisterIomFromURL(faodel::ResourceURL(url3)); + //rc = kelpie::RegisterIomFromURL( faodel::ResourceURL(url3) ); + EXPECT_EQ(0,rc); + + + auto names = kelpie::GetIomNames(); + auto f1 = find(names.begin(), names.end(), "foobar"); EXPECT_NE(names.end(), f1); + auto f2 = find(names.begin(), names.end(), "boston-creme"); EXPECT_NE(names.end(), f2); + auto f3 = find(names.begin(), names.end(), "honey-glaze"); EXPECT_NE(names.end(), f3); + + + //for(auto &n : names) cout<iom_registry.RegisterIomFromURL(faodel::ResourceURL(url1)); + //rc = kelpie::RegisterIomFromURL( faodel::ResourceURL(url1) ); + EXPECT_EQ(0,rc); + + + string url2( "dht:/my/other/thing" + "&iom=missing-type" + "&iom_path=/tmp/zip" ); + + rc = internal::getKelpieCore()->iom_registry.RegisterIomFromURL(faodel::ResourceURL(url2)); + //rc = kelpie::RegisterIomFromURL( faodel::ResourceURL(url2) ); + EXPECT_EQ(-1,rc); + + string url3( "dht:/my/other/thing" + "&iom=missing-path" + "&iom_type=PosixIndividualObjects" ); + + rc = internal::getKelpieCore()->iom_registry.RegisterIomFromURL(faodel::ResourceURL(url3)); + //rc = kelpie::RegisterIomFromURL( faodel::ResourceURL(url3) ); + EXPECT_EQ(-1,rc); + + auto names = kelpie::GetIomNames(); + auto f1 = find(names.begin(), names.end(), "single-good-one"); EXPECT_NE(names.end(), f1); + auto f2 = find(names.begin(), names.end(), "missing-type"); EXPECT_EQ(names.end(), f2); + auto f3 = find(names.begin(), names.end(), "missing-path"); EXPECT_EQ(names.end(), f3); + + //for(auto &n : names) cout<iom_registry.RegisterIomFromURL(faodel::ResourceURL(url1)); + //rc = kelpie::RegisterIomFromURL( faodel::ResourceURL(url1) ); + EXPECT_EQ(0,rc); + + string url2( "dht:/my/other/thing" + "&iom=boston-creme" + "&type=PosixIndividualObjects" + "&iom_path=/tmp/zip" ); + + rc = internal::getKelpieCore()->iom_registry.RegisterIomFromURL(faodel::ResourceURL(url2)); + //rc = kelpie::RegisterIomFromURL( faodel::ResourceURL(url2) ); + EXPECT_EQ(-1,rc); + + string url3( "dht:/my/other/thing" + "&iom=honey-glaze" + "&iom_type=PosixIndividualObjects" + "&path=/tmp/zip" ); + + rc = internal::getKelpieCore()->iom_registry.RegisterIomFromURL(faodel::ResourceURL(url3)); + //rc = kelpie::RegisterIomFromURL( faodel::ResourceURL(url3) ); + EXPECT_EQ(-1,rc); + + + auto names = kelpie::GetIomNames(); + auto f1 = find(names.begin(), names.end(), "foobar"); EXPECT_NE(names.end(), f1); + auto f2 = find(names.begin(), names.end(), "boston-creme"); EXPECT_EQ(names.end(), f2); + auto f3 = find(names.begin(), names.end(), "honey-glaze"); EXPECT_EQ(names.end(), f3); + + //for(auto &n : names) cout< #include #include #include +#include "faodelConfig.h" +#ifdef Faodel_ENABLE_MPI_SUPPORT #include +#endif #include "gtest/gtest.h" @@ -16,6 +26,7 @@ #include "kelpie/ioms/IomPosixIndividualObjects.hh" + using namespace std; using namespace faodel; using namespace kelpie; @@ -25,9 +36,11 @@ std::string default_config_string = R"EOF( # For local testing, tell kelpie to use the nonet implementation kelpie.type nonet +dirman.type none + -default.iom.type PosixIndividualObjects -default.ioms myiom1;myiom2;myiom3 +default.kelpie.iom.type PosixIndividualObjects +default.kelpie.ioms myiom1;myiom2;myiom3;myenv1 # note: additional iom info like path is filled in during SetUp() # Uncomment these options to get debug info for each component @@ -37,11 +50,23 @@ default.ioms myiom1;myiom2;myiom3 #dirman.debug true #kelpie.debug true +#kelpie.pool.debug true +#kelpie.pool.logging_level debug +#kelpie.pool_registry.debug true +#kelpie.iom_registry.debug true +#kelpie.iom.debug true +#kelpie.lkv.debug true + + + # We start/stop multiple times (which lunasa's tcmalloc does not like), so # we have to switch to a plain malloc allocator lunasa.lazy_memory_manager malloc lunasa.eager_memory_manager malloc +# enable when debugging: +#bootstrap.halt_on_shutdown true + )EOF"; @@ -52,19 +77,26 @@ class IomPosixIOSimple : public testing::Test { char p1[] = "/tmp/gtestXXXXXX"; char p2[] = "/tmp/gtestXXXXXX"; char p3[] = "/tmp/gtestXXXXXX"; + char p4[] = "/tmp/gtextXXXXXX"; + string path1 = mkdtemp(p1); string path2 = mkdtemp(p2); string path3 = mkdtemp(p3); - + string path4 = mkdtemp(p4); + config.Append(default_config_string); - - config.Append("iom.myiom1.path", p1); - config.Append("iom.myiom2.path", p2); - config.Append("iom.myiom3.path", p3); + config.Append("kelpie.iom.myiom1.path", p1); + config.Append("kelpie.iom.myiom2.path", p2); + config.Append("kelpie.iom.myiom3.path", p3); + config.Append("kelpie.iom.myenv1.path.env_name", "MY_ENV_VAR"); + + setenv("MY_ENV_VAR", "/tmp/bogyz" /*path4.c_str()*/, 1); + bootstrap::Start(config, kelpie::bootstrap); + } virtual void TearDown() { - bootstrap::FinishSoft(); + bootstrap::Finish(); //Previously this was FinishSoft? } rc_t rc; @@ -131,7 +163,7 @@ bool checkLDO(const lunasa::DataObject &ldo, int id) { TEST_F(IomPosixIOSimple, BasicIOMWrite) { int num_items=10; - kv_col_info_t ci; + object_info_t ci; atomic replies_left; kelpie::Pool plocal0 = kelpie::Connect("local:[my_bucket0]"); @@ -141,6 +173,13 @@ TEST_F(IomPosixIOSimple, BasicIOMWrite) { kelpie::Pool piom2 = kelpie::Connect("[my_bucket2]/local/iom/myiom2"); kelpie::Pool piom3 = kelpie::Connect("[my_bucket3]/local/iom/myiom3"); + + EXPECT_TRUE(plocal0.Valid()); + EXPECT_EQ(uint8_t(PoolBehavior::DefaultLocal), plocal0.GetBehavior()); + EXPECT_EQ(uint8_t(PoolBehavior::DefaultLocalIOM), piom2.GetBehavior()); + + + //Create a bunch of k/vs vector> kvs; for(int i=0; i #include diff --git a/tests/kelpie/component/support/ExperimentLauncher.hh b/tests/kelpie/component/support/ExperimentLauncher.hh index 3a956c7..a06f7b8 100644 --- a/tests/kelpie/component/support/ExperimentLauncher.hh +++ b/tests/kelpie/component/support/ExperimentLauncher.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_EXPERIMENTLAUNCHER_H #define FAODEL_EXPERIMENTLAUNCHER_H diff --git a/tests/kelpie/component/support/Globals.cpp b/tests/kelpie/component/support/Globals.cpp index ac115fe..fc43e0f 100644 --- a/tests/kelpie/component/support/Globals.cpp +++ b/tests/kelpie/component/support/Globals.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. @@ -22,10 +22,9 @@ Globals::~Globals(){ } - // This version looks at config to guess what the ip should be. The root info // is bcast to all nodes, the config updated, and then bootstrap starts. -void Globals::StartAll(int &argc, char **argv, faodel::Configuration &config){ +void Globals::StartAll(int &argc, char **argv, faodel::Configuration &config, int minimum_ranks) { stringstream ss; string whookie_port; @@ -35,8 +34,10 @@ void Globals::StartAll(int &argc, char **argv, faodel::Configuration &config){ MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); - if(mpi_size<=1) { - cerr<<"This test needs to be run with multiple ranks via MPI\n"; + if(mpi_size +#include + +#include "faodelConfig.h" +#ifdef Faodel_ENABLE_MPI_SUPPORT +#include +#endif + +#include "gtest/gtest.h" + +#include "kelpie/localkv/LocalKV.hh" + +using namespace std; +using namespace faodel; +using namespace kelpie; + + +string default_config = R"EOF( + +#bootstrap.debug true + +# This test doesn't use dirman, thus needs to disable +dirman.type none + +# We start/stop multiple times (which lunasa's tcmalloc does not like), so +# we have to switch to a plain malloc allocator +lunasa.lazy_memory_manager malloc +lunasa.eager_memory_manager malloc + +)EOF"; + + +class MultipleHiddenInits : public testing::Test { + +protected: + virtual void SetUp() {} + void TearDown() override {} + +}; + +class MyThing { + +public: + MyThing(string name) : name(name) { + Configuration config(default_config); + bootstrap::Start(config, kelpie::bootstrap); + pool = kelpie::Connect("local:"); + } + + void put(string key_name, string val) { + lunasa::DataObject ldo(val.size()); + memcpy(ldo.GetDataPtr(), val.c_str(), val.length()); + pool.Publish(kelpie::Key(key_name), ldo); + } + + string get(string key_name) { + lunasa::DataObject ldo; + pool.Need(kelpie::Key(key_name), &ldo); + string s(ldo.GetDataPtr(), ldo.GetDataSize()); + return s; + } + ~MyThing() { + bootstrap::Finish(); + } + + string name; + kelpie::Pool pool; + + +}; + + +TEST_F(MultipleHiddenInits, basics) { + + //Each thing starts up, but only the first one should actually bootstrap + MyThing *a = new MyThing("A"); + MyThing *b = new MyThing("B"); + + EXPECT_EQ(2, bootstrap::GetNumberOfUsers()); + + a->put("thing1","mydata1"); + a->put("thing2","mydata2"); + string s1 = b->get("thing1"); + EXPECT_EQ("mydata1", s1); + + delete a; + + string s2 = b->get("thing2"); + EXPECT_EQ("mydata2", s2); + + delete b; + + EXPECT_EQ(0, bootstrap::GetNumberOfUsers()); + +} + +TEST_F(MultipleHiddenInits, classWins) { + + Configuration config(default_config); + bootstrap::Start(config, kelpie::bootstrap); + + //Each thing starts up, but neither should bootstrap since we already started + MyThing *a = new MyThing("A"); + MyThing *b = new MyThing("B"); + + EXPECT_EQ(3, bootstrap::GetNumberOfUsers()); + + a->put("thing1","mydata1"); + a->put("thing2","mydata2"); + string s1 = b->get("thing1"); + EXPECT_EQ("mydata1", s1); + + delete a; + EXPECT_EQ(2, bootstrap::GetNumberOfUsers()); + + bootstrap::Finish(); + EXPECT_EQ(1, bootstrap::GetNumberOfUsers()); + + string s2 = b->get("thing2"); + EXPECT_EQ("mydata2", s2); + + delete b; + + EXPECT_EQ(0, bootstrap::GetNumberOfUsers()); + +} + + +TEST_F(MultipleHiddenInits, missingKelpie) { + + //Throw an error if someone starts bootstrap, then tries to register/start + //higher level services + + Configuration config(default_config); + bootstrap::Start(config, lunasa::bootstrap); + + //Bootstrap was started w/ Lunasa. MyThing asks for Kelpie, which isn't there. + EXPECT_ANY_THROW( MyThing *a = new MyThing("A") ); + + EXPECT_EQ(1, bootstrap::GetNumberOfUsers()); + + bootstrap::Finish(); + +} + + +int main(int argc, char **argv) { + + int rc=0; + ::testing::InitGoogleTest(&argc, argv); + + #ifdef Faodel_ENABLE_MPI_SUPPORT + //If we don't do this, lower layers may try start/stopping mpi each time they run, causing ops after Finalize + int mpi_rank; + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + if(mpi_rank==0) + rc = RUN_ALL_TESTS(); + MPI_Finalize(); + #else + rc = RUN_ALL_TESTS(); + #endif + + return rc; +} diff --git a/tests/kelpie/unit/ioms/tb_kelpie_iom_all_basic.cpp b/tests/kelpie/unit/ioms/tb_kelpie_iom_all_basic.cpp index 99b8175..0c281f4 100644 --- a/tests/kelpie/unit/ioms/tb_kelpie_iom_all_basic.cpp +++ b/tests/kelpie/unit/ioms/tb_kelpie_iom_all_basic.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -64,7 +64,7 @@ class IomSimple : public ::testing::Test { LocalKV *lkv; }; -TYPED_TEST_CASE_P(IomSimple); +TYPED_TEST_SUITE_P(IomSimple); struct test_data_t { @@ -173,12 +173,12 @@ TYPED_TEST_P(IomSimple, UsingConfigurationByRole){ faodel::Configuration config(default_config_string); std::string s; - s = "myrole.iom.myiom1.type "; config.Append(s + TypeParam::type_str); - s = "myrole.iom.myiom2.type "; config.Append(s + TypeParam::type_str); + s = "myrole.kelpie.iom.myiom1.type "; config.Append(s + TypeParam::type_str); + s = "myrole.kelpie.iom.myiom2.type "; config.Append(s + TypeParam::type_str); - config.Append("myrole.iom.myiom1.path",p1); - config.Append("myrole.iom.myiom2.path",p2); - config.Append("myrole.ioms", "myiom1;myiom2"); + config.Append("myrole.kelpie.iom.myiom1.path",p1); + config.Append("myrole.kelpie.iom.myiom2.path",p2); + config.Append("myrole.kelpie.ioms", "myiom1;myiom2"); config.Append("node_role", "myrole"); config.AppendFromReferences(); //Normally done in bootstrap @@ -246,7 +246,7 @@ TYPED_TEST_P(IomSimple, iom_registry) { //See if we can locate each one kelpie::internal::IomBase * ioms[3]; - cout<<"About to look\n"; + //cout<<"About to look\n"; ioms[0] = registry.Find("myiom1"); ASSERT_NE(nullptr, ioms[0]); ioms[1] = registry.Find("myiom2"); ASSERT_NE(nullptr, ioms[1]); ioms[2] = registry.Find("myiom3"); ASSERT_NE(nullptr, ioms[2]); @@ -278,7 +278,7 @@ TYPED_TEST_P(IomSimple, iom_registry) { } // Must enumerate all tests in the generic fixture class here -REGISTER_TYPED_TEST_CASE_P(IomSimple, +REGISTER_TYPED_TEST_SUITE_P(IomSimple, ldo_gentest, write_direct, UsingConfigurationByRole, @@ -288,12 +288,12 @@ REGISTER_TYPED_TEST_CASE_P(IomSimple, // Must enumerate all desired IOM subclasses in this typedef typedef ::testing::Types< kelpie::internal::IomPosixIndividualObjects -#ifdef FAODEL_HAVE_LEVELDB +#if defined(FAODEL_HAVE_LEVELDB) , kelpie::internal::IomLevelDB #endif -#ifdef FAODEL_HAVE_HDF5 +#if defined(FAODEL_HAVE_HDF5) , kelpie::internal::IomHDF5 #endif > IomTypes; -INSTANTIATE_TYPED_TEST_CASE_P(IomTypesInstantiation, IomSimple, IomTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(IomTypesInstantiation, IomSimple, IomTypes); diff --git a/tests/kelpie/unit/ioms/tb_kelpie_iom_pio_basic.cpp b/tests/kelpie/unit/ioms/tb_kelpie_iom_pio_basic.cpp index 8a5a7ce..99ef2f2 100644 --- a/tests/kelpie/unit/ioms/tb_kelpie_iom_pio_basic.cpp +++ b/tests/kelpie/unit/ioms/tb_kelpie_iom_pio_basic.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -162,11 +162,11 @@ TEST_F(IomPosixIOSimple, UsingConfigurationByRole){ //Create two ioms for this node only faodel::Configuration config(default_config_string); - config.Append("myrole.iom.myiom1.type PosixIndividualObjects"); - config.Append("myrole.iom.myiom2.type PosixIndividualObjects"); - config.Append("myrole.iom.myiom1.path",p1); - config.Append("myrole.iom.myiom2.path",p2); - config.Append("myrole.ioms", "myiom1;myiom2"); + config.Append("myrole.kelpie.iom.myiom1.type PosixIndividualObjects"); + config.Append("myrole.kelpie.iom.myiom2.type PosixIndividualObjects"); + config.Append("myrole.kelpie.iom.myiom1.path",p1); + config.Append("myrole.kelpie.iom.myiom2.path",p2); + config.Append("myrole.kelpie.ioms", "myiom1;myiom2"); config.Append("node_role", "myrole"); config.AppendFromReferences(); diff --git a/tests/kelpie/unit/ioms/tb_kelpie_iom_service_basic.cpp b/tests/kelpie/unit/ioms/tb_kelpie_iom_service_basic.cpp index 144eb7b..31eef1a 100644 --- a/tests/kelpie/unit/ioms/tb_kelpie_iom_service_basic.cpp +++ b/tests/kelpie/unit/ioms/tb_kelpie_iom_service_basic.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * There is some significant overlap between this file and the iom_all_basic module. @@ -69,7 +69,7 @@ class IomSimple : public ::testing::Test { LocalKV *lkv; }; -TYPED_TEST_CASE_P(IomSimple); +TYPED_TEST_SUITE_P(IomSimple); struct test_data_t { @@ -184,14 +184,14 @@ TYPED_TEST_P(IomSimple, UsingConfigurationByRole){ faodel::Configuration config(default_config_string); std::string s; - s = "myrole.iom.myiom1.type "; config.Append(s + TypeParam::type_str); - s = "myrole.iom.myiom2.type "; config.Append(s + TypeParam::type_str); + s = "myrole.kelpie.iom.myiom1.type "; config.Append(s + TypeParam::type_str); + s = "myrole.kelpie.iom.myiom2.type "; config.Append(s + TypeParam::type_str); - config.Append("myrole.iom.myiom1.endpoint",endpoint); - config.Append("myrole.iom.myiom1.keyspace",ks1); - config.Append("myrole.iom.myiom2.endpoint",endpoint); - config.Append("myrole.iom.myiom2.keyspace",ks2); - config.Append("myrole.ioms", "myiom1;myiom2"); + config.Append("myrole.kelpie.iom.myiom1.endpoint",endpoint); + config.Append("myrole.kelpie.iom.myiom1.keyspace",ks1); + config.Append("myrole.kelpie.iom.myiom2.endpoint",endpoint); + config.Append("myrole.kelpie.iom.myiom2.keyspace",ks2); + config.Append("myrole.kelpie.ioms", "myiom1;myiom2"); config.Append("node_role", "myrole"); config.AppendFromReferences(); //Normally done in bootstrap @@ -289,7 +289,7 @@ TYPED_TEST_P(IomSimple, iom_registry) { } // Must enumerate all tests in the generic fixture class here -REGISTER_TYPED_TEST_CASE_P(IomSimple, +REGISTER_TYPED_TEST_SUITE_P(IomSimple, ldo_gentest, write_direct, UsingConfigurationByRole, @@ -298,9 +298,23 @@ REGISTER_TYPED_TEST_CASE_P(IomSimple, // Must enumerate all desired IOM subclasses in this typedef typedef ::testing::Types< - #ifdef FAODEL_HAVE_CASSANDRA +#ifdef FAODEL_HAVE_CASSANDRA kelpie::internal::IomCassandra #endif > IomTypes; -INSTANTIATE_TYPED_TEST_CASE_P(IomTypesInstantiation, IomSimple, IomTypes); +// Since we don't have an always-tested service IOM type (like the POSIX file +// IOM in the basic test suite), we need to make sure not to try to instantiate the +// test suite unless at least one service IOM test is defined. So in addition to enumerating +// the subclasses above, we have to check again here. + +// These two things are logically connected and separating their resolution as done here is +// suboptimal. I was hoping there was a template metaprogramming/SFINAE solution here but +// I couldn't figure out a way to not do the instantiation at all if there were no +// subclass tests defined (without using the preprocessor). There's a way to figure out if +// IomTypes is valid using std::is_class, but no way that I could find to avoid doing +// the instantiation at all in that case. + +#if defined(FAODEL_HAVE_CASSANDRA) // || defined(FAODEL_HAVE_ANOTHER_SERVICE_IOM) || ... +INSTANTIATE_TYPED_TEST_SUITE_P(IomTypesInstantiation, IomSimple, IomTypes); +#endif diff --git a/tests/kelpie/unit/messages/tb_kelpie_message_direct.cpp b/tests/kelpie/unit/messages/tb_kelpie_message_direct.cpp index 0165f2f..94d1216 100644 --- a/tests/kelpie/unit/messages/tb_kelpie_message_direct.cpp +++ b/tests/kelpie/unit/messages/tb_kelpie_message_direct.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -23,28 +23,99 @@ bool enable_debug=false; //True turns on logging in doc class MsgDirectTest : public testing::Test { protected: - void SetUp() override { - faodel::Configuration config; - if(enable_debug){ - config.Append("bootstrap.debug","true"); - config.Append("whookie.debug", "true"); - config.Append("lunasa.debug", "true"); - config.Append("opbox.debug","true"); - } - //Force this to an mpi implementation to make running easier - //config.Append("net.transport.name","mpi"); - bootstrap::Start(config, kelpie::bootstrap); - } + void SetUp() override {} - void TearDown() override { - bootstrap::Finish(); - } + void TearDown() override {} faodel::internal_use_only_t iuo; }; -TEST_F(MsgDirectTest, SimplePub){ +#define byte_offset(a,b) (((uint64_t) &(a->b)) - ((uint64_t) a)) + +TEST_F(MsgDirectTest, Sizes) { + bucket_t bucket(0x2112, iuo); + Key k1("x","y"); + lunasa::DataObject ldo_data(36,4000, lunasa::DataObject::AllocatorType::eager); + lunasa::DataObject ldo_msg_simple, ldo_msg_buffer; + msg_direct_simple_t::Alloc(ldo_msg_simple, OpKelpiePublish::op_id, DirectFlags::CMD_PUBLISH, NODE_LOCALHOST, + 0x2064, opbox::MAILBOX_UNSPECIFIED, bucket, k1, 0x1971, PoolBehavior::TODO, + "myfunction","myargs"); + + bool exceeds = msg_direct_buffer_t::Alloc(ldo_msg_buffer, OpKelpiePublish::op_id, DirectFlags::CMD_PUBLISH, NODE_LOCALHOST, + 0x2064, opbox::MAILBOX_UNSPECIFIED, bucket, k1, 0x1971, PoolBehavior::TODO, + &ldo_data); + + + cout <<"Size of opbox message hdr : "<(); + //Fake an argument + //OpArgs args(0, &msg->hdr); + + + auto cmd = msg->GetCommand(); + EXPECT_EQ(static_cast(DirectFlags::CMD_PUBLISH), cmd); + + Key k3; + string s1,s2; + msg->ExtractComputeArgs(&k3, &s1,&s2); + EXPECT_EQ(k1,k3); + EXPECT_EQ("cheese", s1); + EXPECT_EQ("burger", s2); + EXPECT_EQ("This is the row", k3.K1()); + EXPECT_EQ("This is the Column", k3.K2()); + EXPECT_EQ(0, msg->meta_plus_data_size); //36+4000 + EXPECT_EQ(15, msg->k1_size); + EXPECT_EQ(18, msg->k2_size); + EXPECT_EQ(0x2112, msg->bucket.bid); + string s = string(msg->string_data, 15+18); + EXPECT_EQ("This is the rowThis is the Column", s); + EXPECT_EQ(0x1971, msg->iom_hash); + EXPECT_EQ(0x00, msg->behavior_flags); //Currently TODO + EXPECT_EQ(0x90, msg->hdr.user_flags); //update if op changes + EXPECT_EQ(0x2064, msg->hdr.src_mailbox); + + cout <<"Ldo stuff "<(); - //Fake an argument - //OpArgs args(0, &msg->hdr); auto cmd = msg->GetCommand(); EXPECT_EQ(static_cast(DirectFlags::CMD_PUBLISH), cmd); - Key k2= msg->ExtractKey(); - EXPECT_EQ(k1,k2); - EXPECT_EQ("This is the row", k2.K1()); - EXPECT_EQ("This is the Column", k2.K2()); + Key k3; + string s1,s2; + k3 = msg->ExtractKey(); + EXPECT_EQ(k1,k3); + EXPECT_EQ("This is the row", k3.K1()); + EXPECT_EQ("This is the Column", k3.K2()); EXPECT_EQ(4036, msg->meta_plus_data_size); //36+4000 EXPECT_EQ(15, msg->k1_size); EXPECT_EQ(18, msg->k2_size); EXPECT_EQ(0x2112, msg->bucket.bid); - string s = string(msg->key_data, 15+18); + string s = string(msg->string_data, 15+18); EXPECT_EQ("This is the rowThis is the Column", s); EXPECT_EQ(0x1971, msg->iom_hash); EXPECT_EQ(0x00, msg->behavior_flags); //Currently TODO @@ -81,3 +152,28 @@ TEST_F(MsgDirectTest, SimplePub){ cout <<"Ldo stuff "< @@ -9,7 +9,7 @@ #include //htonl #include "faodel-common/Common.hh" -#include "faodel-common/SerializationHelpers.hh" +#include "faodel-common/SerializationHelpersBoost.hh" #include "kelpie/Key.hh" @@ -221,3 +221,54 @@ TEST_F(KeyTest, Packing){ } + +TEST_F(KeyTest, Wildcards) { + + Key k[4]; + k[0] = Key("MyRowName", "MyColName"); EXPECT_FALSE( k[0].IsRowWildcard() ); EXPECT_FALSE( k[0].IsColWildcard()); + k[1] = Key("MyRowWild*", "MyColName"); EXPECT_TRUE( k[1].IsRowWildcard() ); EXPECT_FALSE( k[1].IsColWildcard()); + k[2] = Key("MyRowName", "MyColWild*");EXPECT_FALSE( k[2].IsRowWildcard() ); EXPECT_TRUE( k[2].IsColWildcard()); + k[3] = Key("MyRowWild*", "MyColWild*");EXPECT_TRUE( k[3].IsRowWildcard() ); EXPECT_TRUE( k[3].IsColWildcard()); + Key kall("*", "*"); EXPECT_TRUE( kall.IsRowWildcard() ); EXPECT_TRUE( kall.IsColWildcard()); + Key krow("MyRow*", "*"); EXPECT_TRUE( krow.IsRowWildcard() ); EXPECT_TRUE( krow.IsColWildcard()); + + //Manual tests on a real key + EXPECT_TRUE( k[0].Matches(k[0])); //Self test + EXPECT_TRUE( k[0].Matches("MyRowName", "MyColName")); + EXPECT_TRUE( k[0].Matches("MyRowName", "MyColName*")); + EXPECT_TRUE( k[0].Matches("MyRowName", "MyCol*")); + EXPECT_TRUE( k[0].Matches("MyRowName", "*")); + EXPECT_TRUE( k[0].Matches("MyRowName*","*")); + EXPECT_TRUE( k[0].Matches("MyRow*", "MyColName")); + EXPECT_TRUE( k[0].Matches("*", "MyColName")); + EXPECT_TRUE( k[0].Matches("*", "MyCol*")); + EXPECT_TRUE( k[0].Matches("*", "*")); + + //Wrong case: tests should fail + EXPECT_FALSE( k[0].Matches("MyRowName", "myColName")); + EXPECT_FALSE( k[0].Matches("myRowName", "MyColName")); + EXPECT_FALSE( k[0].Matches("myRowName", "myColName")); + EXPECT_FALSE( k[0].Matches("MyRow*", "myColName")); + EXPECT_FALSE( k[0].Matches("MyRow*", "my*")); + EXPECT_FALSE( k[0].Matches("*", "my*")); + EXPECT_FALSE( k[0].Matches("myRow*", "MyColName")); + EXPECT_FALSE( k[0].Matches("myRowName", "*")); + + for(int i=0; i<4; i++) { + EXPECT_TRUE(k[i].Matches(Key("*", "*"))); + EXPECT_TRUE(k[i].Matches(kall)); + EXPECT_TRUE(k[i].Matches(kall.K1(), kall.K2())); + EXPECT_TRUE(k[i].Matches(krow)); //All have same rowname + } + + //Above should have hit this code, but a few just in case + EXPECT_TRUE( k[0].matchesPrefixString(false, "MyRowName", false, "MyColName")); + EXPECT_TRUE( k[0].matchesPrefixString(true, "My", false, "MyColName")); + EXPECT_TRUE( k[0].matchesPrefixString(false, "MyRowName", true, "My")); + EXPECT_TRUE( k[0].matchesPrefixString(true, "My", true, "My")); + + EXPECT_FALSE(k[0].matchesPrefixString(false, "My", false, "MyColName")); + EXPECT_FALSE(k[0].matchesPrefixString(false, "MyRowName", false, "My")); + EXPECT_FALSE(k[0].matchesPrefixString(false, "My", false, "My")); + +} \ No newline at end of file diff --git a/tests/kelpie/unit/tb_kelpie_localkv.cpp b/tests/kelpie/unit/tb_kelpie_localkv.cpp index 1f47ee0..4f3784f 100644 --- a/tests/kelpie/unit/tb_kelpie_localkv.cpp +++ b/tests/kelpie/unit/tb_kelpie_localkv.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -20,7 +20,6 @@ string default_config = R"EOF( #kelpie.core_type nonet #kelpie.debug true #kelpie.lkv.debug true -#kelpie.lkv.max_capacity 32M # We start/stop multiple times (which lunasa's tcmalloc does not like), so # we have to switch to a plain malloc allocator @@ -87,15 +86,20 @@ TEST_F(LocalKVTest, basics){ //Store things in random order random_shuffle(ids.begin(), ids.end()); - for(vector>::iterator it=ids.begin(); it!=ids.end(); ++it){ - setBuf(buf, bufsize, owner.bid, it->first, it->second); + for(auto i_j : ids) { + setBuf(buf, bufsize, owner.bid, i_j.first, i_j.second); - lunasa::DataObject ldo(0, bufsize*sizeof(int), lunasa::DataObject::AllocatorType::eager); + lunasa::DataObject ldo(bufsize*sizeof(int)); memcpy(ldo.GetDataPtr(), buf, bufsize*sizeof(int)); - Key key=KeyGen(it->first, it->second); - rc = lkv->put(owner, key, ldo, PoolBehavior::DefaultBaseClass, nullptr, nullptr); + Key key=KeyGen(i_j.first, i_j.second); + object_info_t info; + rc = lkv->put(owner, key, ldo, PoolBehavior::DefaultBaseClass, nullptr, &info); EXPECT_EQ(KELPIE_OK,rc); + EXPECT_EQ(Availability::InLocalMemory, info.col_availability); + //EXPECT_EQ(ldo.GetUserCapacity(), info.col_user_bytes); + EXPECT_EQ(bufsize*sizeof(int), info.col_user_bytes); + EXPECT_GE(ldo.GetUserCapacity(), info.col_user_bytes); //Note: ldo may be allocated more, usually for alignment } memset(buf, 4, sizeof(buf)); @@ -104,14 +108,14 @@ TEST_F(LocalKVTest, basics){ //Pull things out in a different random order random_shuffle(ids.begin(), ids.end()); - for(vector>::iterator it=ids.begin(); it!=ids.end(); ++it){ + for(auto i_j : ids) { size_t ret_size=0; - rc = lkv->getData(owner, KeyGen(it->first,it->second), (void *)buf, sizeof(buf), &ret_size, nullptr, nullptr); + rc = lkv->getData(owner, KeyGen(i_j.first,i_j.second), (void *)buf, sizeof(buf), &ret_size, nullptr); EXPECT_EQ(KELPIE_OK,rc); EXPECT_EQ(ret_size, sizeof(buf)); - rc = checkBuf(buf, bufsize, owner.bid, it->first, it->second); + rc = checkBuf(buf, bufsize, owner.bid, i_j.first, i_j.second); EXPECT_EQ(0,rc); } @@ -150,7 +154,7 @@ TEST_F(LocalKVTest, ldoRefCount){ //Get a reference from lkv That makes 4 references lunasa::DataObject ldo3_lkv; - rc = lkv->get(owner, Key("booya"), &ldo3_lkv, nullptr, nullptr); EXPECT_EQ(KELPIE_OK, rc); + rc = lkv->get(owner, Key("booya"), &ldo3_lkv, nullptr); EXPECT_EQ(KELPIE_OK, rc); int *x3 = ldo3_lkv.GetDataPtr(); ref_count = ldo1.internal_use_only.GetRefCount(); EXPECT_EQ(x3[0], x[0]); @@ -194,7 +198,7 @@ TEST_F(LocalKVTest, access){ for(int c=0; c<10; c++){ size_t ret_size=0; - rc = lkv->getData(owners[i], KeyGen(r,c), (void *)buf, sizeof(buf), &ret_size, nullptr, nullptr); + rc = lkv->getData(owners[i], KeyGen(r,c), (void *)buf, sizeof(buf), &ret_size, nullptr); EXPECT_EQ(KELPIE_OK,rc); EXPECT_EQ(ret_size, sizeof(buf)); @@ -211,7 +215,7 @@ TEST_F(LocalKVTest, access){ for(int r=0; r<10; r++) for(int c=0; c<10; c++){ size_t ret_size=0; - rc = lkv->getData(b, KeyGen(r,c), (void *)buf, sizeof(buf), &ret_size, nullptr,nullptr); + rc = lkv->getData(b, KeyGen(r,c), (void *)buf, sizeof(buf), &ret_size, nullptr); EXPECT_EQ(KELPIE_ENOENT,rc); } } @@ -229,15 +233,15 @@ TEST_F(LocalKVTest, IgnoreWrites) { //LKV Should only do a write if the WriteToLocal bit is set. It still checks dependencies, but otherwise returns w/ no entry rc = lkv->put(bucket, k1, ldo1, PoolBehavior::NoAction, nullptr, nullptr ); EXPECT_EQ(KELPIE_ENOENT, rc); rc = lkv->put(bucket, k1, ldo1, PoolBehavior::WriteToRemote, nullptr, nullptr ); EXPECT_EQ(KELPIE_ENOENT, rc); - rc = lkv->put(bucket, k1, ldo1, PoolBehavior::WriteToIOM, nullptr, nullptr ); EXPECT_EQ(KELPIE_ENOENT, rc); + rc = lkv->put(bucket, k1, ldo1, PoolBehavior::WriteToIOM, nullptr, nullptr ); EXPECT_EQ(KELPIE_EIO, rc); //Double check there's no data - rc = lkv->get(bucket, k1, &ldo_return, nullptr, nullptr); EXPECT_EQ(KELPIE_ENOENT, rc); + rc = lkv->get(bucket, k1, &ldo_return, nullptr); EXPECT_EQ(KELPIE_ENOENT, rc); //Make sure a local write works rc = lkv->put(bucket, k1, ldo1, PoolBehavior::WriteToLocal, nullptr, nullptr ); EXPECT_EQ(KELPIE_OK, rc); - rc = lkv->get(bucket, k1, &ldo_return, nullptr, nullptr); EXPECT_EQ(KELPIE_OK, rc); + rc = lkv->get(bucket, k1, &ldo_return, nullptr); EXPECT_EQ(KELPIE_OK, rc); EXPECT_EQ(1024, ldo_return.GetUserSize()); } @@ -263,7 +267,7 @@ TEST_F(LocalKVTest, ListRowStar) { { //Get thing*|"" (four rows) ObjectCapacities oc; - rc = lkv->list(bucket, kelpie::Key("thing*"), &oc); + rc = lkv->list(bucket, kelpie::Key("thing*"), nullptr, &oc); EXPECT_EQ(KELPIE_OK, rc); EXPECT_EQ(oc.keys.size(), oc.capacities.size()); EXPECT_EQ(4, oc.keys.size()); @@ -276,7 +280,7 @@ TEST_F(LocalKVTest, ListRowStar) { { //Get specific row/col: "thing3|" ObjectCapacities oc; - rc = lkv->list(bucket, kelpie::Key("thing3"), &oc); + rc = lkv->list(bucket, kelpie::Key("thing3"), nullptr, &oc); EXPECT_EQ(KELPIE_OK, rc); EXPECT_EQ(oc.keys.size(), oc.capacities.size()); EXPECT_EQ(1, oc.keys.size()); @@ -325,7 +329,7 @@ TEST_F(LocalKVTest, ListRowColStars) { exact_keys.push_back(Key("some", "thing4")); for(auto k : exact_keys) { ObjectCapacities oc; - rc = lkv->list(bucket, k, &oc); + rc = lkv->list(bucket, k, nullptr, &oc); EXPECT_EQ(KELPIE_OK, rc); EXPECT_EQ(1, oc.keys.size()); EXPECT_EQ(1, oc.capacities.size()); if((oc.keys.size()==1) && (oc.capacities.size() == 1)) { @@ -341,7 +345,7 @@ TEST_F(LocalKVTest, ListRowColStars) { missing_keys.push_back(Key("Xname", "Xthing3")); //bad row/col for(auto k : missing_keys) { ObjectCapacities oc; - rc = lkv->list(bucket, k, &oc); + rc = lkv->list(bucket, k, nullptr, &oc); EXPECT_EQ(KELPIE_ENOENT, rc); EXPECT_EQ(0, oc.keys.size()); EXPECT_EQ(0,oc.capacities.size()); } @@ -349,7 +353,7 @@ TEST_F(LocalKVTest, ListRowColStars) { { //Exact Row, Col* ObjectCapacities oc; - rc = lkv->list(bucket, Key("go", "thing*"), &oc); + rc = lkv->list(bucket, Key("go", "thing*"), nullptr, &oc); EXPECT_EQ(KELPIE_OK, rc); EXPECT_EQ(4, oc.keys.size()); EXPECT_EQ(4,oc.capacities.size()); vector found_cols; @@ -367,7 +371,7 @@ TEST_F(LocalKVTest, ListRowColStars) { { //Row*, Exact col ObjectCapacities oc; - rc = lkv->list(bucket, Key("so*", "thing3"), &oc); + rc = lkv->list(bucket, Key("so*", "thing3"), nullptr, &oc); EXPECT_EQ(KELPIE_OK, rc); EXPECT_EQ(2, oc.keys.size()); EXPECT_EQ(2,oc.capacities.size()); vector found_rows; @@ -384,7 +388,7 @@ TEST_F(LocalKVTest, ListRowColStars) { { //Row*, Col* ObjectCapacities oc; - rc = lkv->list(bucket, Key("so*", "thing*"), &oc); + rc = lkv->list(bucket, Key("so*", "thing*"), nullptr, &oc); EXPECT_EQ(KELPIE_OK, rc); EXPECT_EQ(8, oc.keys.size()); EXPECT_EQ(8,oc.capacities.size()); vector found_rows; @@ -404,5 +408,243 @@ TEST_F(LocalKVTest, ListRowColStars) { EXPECT_TRUE( (found_cols==expected_cols)); } +} + +TEST_F(LocalKVTest, DropIndividual) { + + int rc; + bucket_t bucket("bucky"); + + string row_names[] = {"some", "random", "column", "names", "go", "heree", "sowhat"}; + + string col_names[] = {"nothing1", "nothing2", "nothing3", "nothing4", + "thing1", "thing2", "thing3", + "nothing5", "nothing6", + "", + "thing4"}; + + map keymap_sizes; //records how big the ldo is + + //Insert a bunch or keys into lkv + int i = 0; + for(auto r : row_names) { + for(auto c : col_names) { + Key k(r, c); + lunasa::DataObject ldo(100 + i); + rc = lkv->put(bucket, k, ldo, PoolBehavior::WriteToLocal, nullptr, nullptr); + EXPECT_EQ(KELPIE_OK, rc); + keymap_sizes[k] = 100 + i; + i++; + } + } + + + + //Try dropping each one + for(auto &key_size : keymap_sizes) { + //Verify object is there + //cout <<"Key is "<list(bucket, key_size.first, nullptr, &oc); + + EXPECT_EQ(KELPIE_OK, rc); + EXPECT_EQ(1, oc.keys.size()); + EXPECT_EQ(1, oc.capacities.size()); + if(oc.capacities.size()==1) { + EXPECT_EQ(key_size.second, oc.capacities[0]); + } + //Drop it + rc = lkv->drop(bucket,key_size.first); + EXPECT_EQ(KELPIE_OK, rc); + + //Make sure it doesn't show up + oc.Wipe(); + rc = lkv->list(bucket, key_size.first, nullptr, &oc); + + EXPECT_EQ(KELPIE_ENOENT, rc); + EXPECT_EQ(0, oc.capacities.size()); + //cout<str(10); + + } + +} + +TEST_F(LocalKVTest, DropRow) { + + int rc; + bucket_t bucket("bucky"); + + string row_names[] = {"ignore", "random", "row1", "bob", "row2", "go","stuff1","stuff2"}; + + string col_names[] = {"nothing1", "nothing2", "nothing3", "nothing4", + "thing1", "thing2", "thing3", + "nothing5", "nothing6", + "", + "thing4"}; + + map keymap_sizes; //records how big the ldo is + + //Insert a bunch or keys into lkv + int i = 0; + for(auto r : row_names) { + for(auto c : col_names) { + Key k(r, c); + lunasa::DataObject ldo(100 + i); + rc = lkv->put(bucket, k, ldo, PoolBehavior::WriteToLocal, nullptr, nullptr); + EXPECT_EQ(KELPIE_OK, rc); + keymap_sizes[k] = 100 + i; + i++; + } + } + + //Remove some columns + { + kelpie::Key k1("bob", "thing*"); + ObjectCapacities oc; + rc = lkv->drop(bucket, k1); EXPECT_EQ(KELPIE_OK, rc); + rc = lkv->list(bucket, k1, nullptr, &oc); EXPECT_EQ(KELPIE_ENOENT, rc); + rc = lkv->list(bucket, kelpie::Key("bob","*"), nullptr, &oc); EXPECT_EQ(KELPIE_OK, rc); + EXPECT_EQ(7, oc.capacities.size()); + } + + //Remove all columns + { + kelpie::Key k1("go", "*"); + ObjectCapacities oc; + rc = lkv->drop(bucket, k1); EXPECT_EQ(KELPIE_OK, rc); + rc = lkv->list(bucket, k1, nullptr, &oc); EXPECT_EQ(KELPIE_ENOENT, rc); + rc = lkv->list(bucket, kelpie::Key("go","*"), nullptr, &oc); + EXPECT_EQ(KELPIE_ENOENT, rc); + EXPECT_EQ(0, oc.capacities.size()); + } + + //Remove some columns on multiple rows + { + kelpie::Key k1("stuff*", "thing*"); + ObjectCapacities oc; + rc = lkv->drop(bucket, k1); EXPECT_EQ(KELPIE_OK, rc); + rc = lkv->list(bucket, k1, nullptr, &oc); EXPECT_EQ(KELPIE_ENOENT, rc); + rc = lkv->list(bucket, kelpie::Key("stuff*","*"), nullptr, &oc); + EXPECT_EQ(KELPIE_OK, rc); + EXPECT_EQ(14, oc.capacities.size()); + } + + //Remove all columns on multiple rows + { + kelpie::Key k1("stuff*", "*"); + ObjectCapacities oc; + rc = lkv->drop(bucket, k1); EXPECT_EQ(KELPIE_OK, rc); + rc = lkv->list(bucket, k1, nullptr, &oc); EXPECT_EQ(KELPIE_ENOENT, rc); + } + +} + +TEST_F(LocalKVTest, GetRowStar) { + + int rc; + bucket_t bucket("bucky"); + + string row_names[] = {"some", "random", "column", "names", "go", "heree", "sowhat"}; + + string col_names[] = {"nothing1", "nothing2", "nothing3", "nothing4", + "thing1", "thing2", "thing3", + "nothing5", "nothing6", + "", + "thing4"}; + + map keymap_encodes; //records the name of the key into the ldo + + //Insert a bunch or keys into lkv + int i = 0; + for(auto r : row_names) { + for(auto c : col_names) { + Key k(r,c); + + string enc = k.pup(); + lunasa::DataObject ldo(enc.size()); + memcpy(ldo.GetDataPtr(), enc.c_str(), enc.size()); + keymap_encodes[k] = enc; + + rc = lkv->put(bucket, k, ldo, PoolBehavior::WriteToLocal, nullptr, nullptr); + EXPECT_EQ(KELPIE_OK, rc); + i++; + } + } + + //Good keys: Look for a list of exact matches + vector exact_keys; + exact_keys.push_back(Key("names", "thing3")); + exact_keys.push_back(Key("random","nothing1")); + exact_keys.push_back(Key("go", "")); + exact_keys.push_back(Key("some", "thing4")); + for(auto k : exact_keys) { + std::map ldos; + rc = lkv->getAvailable(bucket, k, ldos); + EXPECT_EQ(KELPIE_OK, rc); + EXPECT_EQ(1, ldos.size()); + if(ldos.size()==1) { + EXPECT_EQ( k, ldos.begin()->first); + auto tmp_ldo = ldos.begin()->second; + string s1(tmp_ldo.GetDataPtr(), tmp_ldo.GetDataSize()); + EXPECT_EQ(keymap_encodes[k], s1); + } + } + + { //Simple wildcard + Key kstar("names", "thing*"); + vector expected_keys; + for(int i = 1; i < 5; i++) expected_keys.push_back(Key("names", "thing"+to_string(i))); + + std::map ldos; + rc = lkv->getAvailable(bucket, kstar, ldos); + EXPECT_EQ(KELPIE_OK, rc); + EXPECT_EQ(4, ldos.size()); + if(ldos.size() == 4) { + for(auto &k : expected_keys) { + //cout << "Checking " << k.str() << "\n"; + EXPECT_EQ(1, ldos.count(k)); + if(ldos.count(k) == 1) { + auto tmp_ldo = ldos[k]; + string s1(tmp_ldo.GetDataPtr(), tmp_ldo.GetDataSize()); + EXPECT_EQ(keymap_encodes[k], s1); + } + } + } + + } + + { //Get all cols in a row + Key kstar2 = Key("names", "*"); + vector expected_keys2; + for(auto c: col_names) + expected_keys2.push_back( Key("names", c) ); + std::map ldos2; + rc = lkv->getAvailable(bucket, kstar2, ldos2); + EXPECT_EQ(KELPIE_OK, rc); + EXPECT_EQ(11, ldos2.size()); + if(ldos2.size()==11) { + for(auto &k : expected_keys2) { + //cout <<"Checking "<(), tmp_ldo.GetDataSize()); + EXPECT_EQ(keymap_encodes[k], s1); + } + } + } + } + + { //Check none + vector bogus = { Key("bogus"), Key("some","nocol"), Key("foo","bar")}; + for(auto &k : bogus) { + std::map ldos; + rc = lkv->getAvailable(bucket, k, ldos); + EXPECT_EQ(KELPIE_ENOENT, rc); + EXPECT_EQ(0, ldos.size()); + } + } + + +} -} \ No newline at end of file diff --git a/tests/kelpie/unit/tb_kelpie_types.cpp b/tests/kelpie/unit/tb_kelpie_types.cpp index f0d9421..c8c488a 100644 --- a/tests/kelpie/unit/tb_kelpie_types.cpp +++ b/tests/kelpie/unit/tb_kelpie_types.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -54,4 +54,75 @@ TEST_F(TypesTest, ActionFlags){ EXPECT_EQ( w2l|r2l, PoolBehavior::ChangeRemoteToLocal(PoolBehavior::WriteToRemote | PoolBehavior::ReadToRemote)); EXPECT_EQ( w2l|r2l|w2i, PoolBehavior::ChangeRemoteToLocal(PoolBehavior::WriteToRemote | PoolBehavior::ReadToRemote | PoolBehavior::WriteToIOM)); +} + +TEST_F(TypesTest, ObjectCapacities) { + + ObjectCapacities oc1,oc2,oc1b, oc2b; + + oc1.Append(Key("a"),1); + oc1.Append(Key("c"),2); + oc1.Append(Key("e"),3); + oc1.Append(Key("g"),4); + EXPECT_EQ(0, oc1.FindIndex(Key("a"))); + EXPECT_EQ(2, oc1.FindIndex(Key("e"))); + EXPECT_EQ(-1, oc1.FindIndex(Key("b"))); + EXPECT_EQ(4, oc1.keys.size()); + EXPECT_EQ(4, oc1.capacities.size()); + EXPECT_EQ(4, oc1.Size()); + + oc2.Append(Key("b"),5); + oc2.Append(Key("e"), -1); + oc2.Append(Key("d"),6); + oc2.Append(Key("f"),7); + EXPECT_EQ(4, oc2.Size()); + + //Make a copy + oc1b = oc1; + + oc1.Merge(oc2); //Should not have duplicate e + EXPECT_EQ(7, oc1.Size()); + + oc1b.Append(oc2); //Should have duplicate e + EXPECT_EQ(8, oc1b.Size()); + + oc1.Wipe(); + EXPECT_EQ(0, oc1.Size()); + EXPECT_TRUE(oc1.keys.empty()); + EXPECT_TRUE(oc1.capacities.empty()); + +} +TEST_F(TypesTest, StructSizes) { + + EXPECT_EQ(1, sizeof(Availability)); + EXPECT_EQ(1, sizeof(pool_behavior_t)); + + if(sizeof(object_info_t)!=24) { + F_WARN("Size of kelpie's object_info_t has changed. This is ok for this platform, but may break heterogeneous runs") + } + EXPECT_EQ(24, sizeof(object_info_t)); + +} + +TEST_F(TypesTest, ObjectInfo) { + + object_info_t oi; + oi.row_user_bytes=-1; + oi.row_num_columns=-2; + oi.col_user_bytes=-3; + oi.col_dependencies=-4; + oi.col_availability=Availability::InLocalMemory; + + oi.ChangeAvailabilityFromLocalToRemote(); + EXPECT_EQ(Availability::InRemoteMemory, oi.col_availability); + + oi.Wipe(); + EXPECT_EQ(0,oi.row_user_bytes); + EXPECT_EQ(0,oi.row_num_columns); + EXPECT_EQ(0,oi.col_user_bytes); + EXPECT_EQ(0,oi.col_dependencies); + EXPECT_EQ(Availability::Unavailable,oi.col_availability); + + //cout< @@ -266,8 +266,8 @@ class LunasaPutUserTest : public testing::Test { private: nnti::transports::transport *transport = nullptr; std::promise &recv_promise; - std::promise &put_promise; std::promise &send_promise; + std::promise &put_promise; NNTI_result_t recv_callback_func(NNTI_event_t *event, void *context) { #ifdef DEBUG @@ -514,7 +514,6 @@ class LunasaPutUserTest : public testing::Test { TEST_F(LunasaPutUserTest, basic) { int rc; - int partner_id; faodel::nodeid_t partner_nodeid; MPI_Request request; @@ -542,7 +541,6 @@ TEST_F(LunasaPutUserTest, basic) { NNTI_peer_t p; transport->connect(url.str().c_str(), 1000, &p); - int payload_length = 8; NNTI_attrs_t nnti_attrs; transport->attrs(&nnti_attrs); diff --git a/tests/lunasa/component/nnti/mpi_lunasa_nnti_register_memory.cpp b/tests/lunasa/component/nnti/mpi_lunasa_nnti_register_memory.cpp index ffba00b..fb41bc3 100644 --- a/tests/lunasa/component/nnti/mpi_lunasa_nnti_register_memory.cpp +++ b/tests/lunasa/component/nnti/mpi_lunasa_nnti_register_memory.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include @@ -188,7 +188,6 @@ int seed = clock(); TEST_F(LunasaRegistrationTest, RandomAllocationRaw) { - int i; void *memory[NUMBER]; size_t *num_bytes = new size_t[NUMBER]; @@ -229,7 +228,6 @@ TEST_F(LunasaRegistrationTest, RandomAllocationRaw) { TEST_F(LunasaRegistrationTest, RandomAllocationLunasa) { - int i; DataObject *memory = new DataObject[NUMBER]; size_t *num_bytes = new size_t[NUMBER]; @@ -289,7 +287,7 @@ int main(int argc, char *argv[]) { lunasa::RegisterPinUnpin(RegisterMemory, UnregisterMemory); assert(whookie::Server::IsRunning() && "Whookie not started before NetNnti started"); - faodel::nodeid_t nodeid = whookie::Server::GetNodeID(); + //faodel::nodeid_t nodeid = whookie::Server::GetNodeID(); transport = nnti::transports::factory::get_instance(config); transport->start(); diff --git a/tests/lunasa/component/nnti/mpi_lunasa_nnti_send.cpp b/tests/lunasa/component/nnti/mpi_lunasa_nnti_send.cpp index 0a61728..c45d6fa 100644 --- a/tests/lunasa/component/nnti/mpi_lunasa_nnti_send.cpp +++ b/tests/lunasa/component/nnti/mpi_lunasa_nnti_send.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "gtest/gtest.h" #include @@ -194,7 +194,6 @@ TEST_F(LunasaSendTest, basic) { // Currently, only works for two MPI ranks int partner_rank = mpi_rank ^0x1; - int partner_id; faodel::nodeid_t partner_nodeid; MPI_Request request; diff --git a/tests/lunasa/component/nnti/mpi_lunasa_nnti_send_user.cpp b/tests/lunasa/component/nnti/mpi_lunasa_nnti_send_user.cpp index ebdcf27..333908f 100644 --- a/tests/lunasa/component/nnti/mpi_lunasa_nnti_send_user.cpp +++ b/tests/lunasa/component/nnti/mpi_lunasa_nnti_send_user.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "gtest/gtest.h" #include @@ -199,7 +199,6 @@ TEST_F(LunasaSendTest, basic) { // Currently, only works for two MPI ranks int partner_rank = mpi_rank ^0x1; - int partner_id; faodel::nodeid_t partner_nodeid; MPI_Request request; diff --git a/tests/lunasa/component/tb_lunasa_backburner_ldo.cpp b/tests/lunasa/component/tb_lunasa_backburner_ldo.cpp index 6946f6e..2c3e67c 100644 --- a/tests/lunasa/component/tb_lunasa_backburner_ldo.cpp +++ b/tests/lunasa/component/tb_lunasa_backburner_ldo.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // This test is here just to verify that we don't screw up // refcounts when we pass ldo's through backburner. @@ -66,7 +66,7 @@ TEST_F(BackBurnerLDO, PassLDO) { lunasa::DataObject ldo1(0, 1024, lunasa::DataObject::AllocatorType::eager); int *x = (int *) ldo1.GetDataPtr(); - for(int i = 0; i<1024/sizeof(int); i++) + for(int i = 0; i < static_cast(1024/sizeof(int)); i++) x[i] = i; cout << "Launch: Ldo1 size " << ldo1.GetDataSize() << " refcount " << ldo1.internal_use_only.GetRefCount() << endl; diff --git a/tests/lunasa/component/tb_lunasa_basic_allocations.cpp b/tests/lunasa/component/tb_lunasa_basic_allocations.cpp index cec79bd..7bb4e77 100644 --- a/tests/lunasa/component/tb_lunasa_basic_allocations.cpp +++ b/tests/lunasa/component/tb_lunasa_basic_allocations.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "gtest/gtest.h" #include @@ -20,7 +20,6 @@ using namespace lunasa; string default_config = R"EOF( #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock node_role server @@ -72,12 +71,97 @@ TEST_F(LunasaBasic, simpleAlloc) { EXPECT_EQ(0, Lunasa::TotalAllocated()); } +TEST_F(LunasaBasic, saveLoad) { + + struct mymeta_t { + uint32_t a; + uint32_t b; + uint64_t c; + }; + + DataObject ldo(sizeof(mymeta_t),1024*sizeof(int), DataObject::AllocatorType::eager, 0x88); + mymeta_t *meta = ldo.GetMetaPtr(); + meta->a=2001; + meta->b=2003; + meta->c=2005; + int *data = ldo.GetDataPtr(); + for(int i=0; i<1024; i++) + data[i]=1000+i; + + //Allocate simple data and free it + size_t num_bytes = sizeof(mymeta_t) + 1024*sizeof(int); + + EXPECT_TRUE(Lunasa::SanityCheck()); + EXPECT_LE(num_bytes, Lunasa::TotalAllocated()); + EXPECT_EQ(0x88, ldo.GetTypeID()); + + const char *filename = "tb_LunasaTest2.out"; + ldo.writeToFile(filename); + + + //Load 1: Manually create the object + DataObject read_obj1(0, num_bytes, DataObject::AllocatorType::eager); + read_obj1.readFromFile(filename); + + + + EXPECT_EQ(0, memcmp(ldo.internal_use_only.GetHeaderPtr(), + read_obj1.internal_use_only.GetHeaderPtr(), + ldo.GetWireSize())); + + if(ldo.GetDataSize() == read_obj1.GetDataSize()) { + EXPECT_EQ( 0, memcmp(ldo.GetDataPtr(), read_obj1.GetDataPtr(), 1024*sizeof(int))); + } + + EXPECT_NE(0, read_obj1.GetMetaSize()); + EXPECT_EQ(ldo.GetMetaSize(), read_obj1.GetMetaSize()); + EXPECT_EQ(ldo.GetDataSize(), read_obj1.GetDataSize()); + EXPECT_EQ(ldo.GetTypeID(), read_obj1.GetTypeID()); + + auto *meta2 = read_obj1.GetMetaPtr(); + EXPECT_EQ(2001, meta2->a); + EXPECT_EQ(2003, meta2->b); + EXPECT_EQ(2005, meta2->c); + + + //Load 2: Do the automated load + auto read_obj2 = lunasa::LoadDataObjectFromFile(string(filename)); + + EXPECT_EQ(0, memcmp(ldo.internal_use_only.GetHeaderPtr(), + read_obj2.internal_use_only.GetHeaderPtr(), + ldo.GetWireSize())); + + if(ldo.GetDataSize() == read_obj2.GetDataSize()) { + EXPECT_EQ( 0, memcmp(ldo.GetDataPtr(), read_obj2.GetDataPtr(), 1024*sizeof(int))); + } + + EXPECT_NE(0, read_obj2.GetMetaSize()); + EXPECT_EQ(ldo.GetMetaSize(), read_obj2.GetMetaSize()); + EXPECT_EQ(ldo.GetDataSize(), read_obj2.GetDataSize()); + EXPECT_EQ(ldo.GetTypeID(), read_obj2.GetTypeID()); + + auto *meta3 = read_obj1.GetMetaPtr(); + EXPECT_EQ(2001, meta3->a); + EXPECT_EQ(2003, meta3->b); + EXPECT_EQ(2005, meta3->c); + + EXPECT_ANY_THROW(auto bad_obj = lunasa::LoadDataObjectFromFile("/blah/blah/blah/not/a/real/file")); + + + //Wipe out all objects so total allocated goes out + ldo = DataObject(); + read_obj1 = DataObject(); + read_obj2 = DataObject(); + EXPECT_EQ(0, Lunasa::TotalAllocated()); + +} + + TEST_F(LunasaBasic, multipleAllocs) { int sizes[] = {16, 81, 92, 48, 54, 64, 112, 3, 12}; //Small chunks, should sum < 1024 vector> mems; - int spot = 0; int tot_bytes = 0; for(auto s : sizes) { cerr << "allocating " << s << endl; @@ -109,7 +193,7 @@ int main(int argc, char **argv) { int rc = 0; ::testing::InitGoogleTest(&argc, argv); - int mpi_rank = 0, mpi_size; + //int mpi_rank = 0, mpi_size; /* INITIALIZE Lunasa */ bootstrap::Init(Configuration(default_config), lunasa::bootstrap); diff --git a/tests/lunasa/component/tb_lunasa_copy_ldo.cpp b/tests/lunasa/component/tb_lunasa_copy_ldo.cpp index c5982fe..ee38750 100644 --- a/tests/lunasa/component/tb_lunasa_copy_ldo.cpp +++ b/tests/lunasa/component/tb_lunasa_copy_ldo.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "gtest/gtest.h" diff --git a/tests/lunasa/component/tb_lunasa_data_type_registry.cpp b/tests/lunasa/component/tb_lunasa_data_type_registry.cpp index ee9f925..fd7f92c 100644 --- a/tests/lunasa/component/tb_lunasa_data_type_registry.cpp +++ b/tests/lunasa/component/tb_lunasa_data_type_registry.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "gtest/gtest.h" //#include @@ -20,7 +20,6 @@ using namespace lunasa; string default_config = R"EOF( #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock node_role server diff --git a/tests/lunasa/component/tb_lunasa_dataobjectpacker.cpp b/tests/lunasa/component/tb_lunasa_dataobjectpacker.cpp new file mode 100644 index 0000000..83e3332 --- /dev/null +++ b/tests/lunasa/component/tb_lunasa_dataobjectpacker.cpp @@ -0,0 +1,316 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include "gtest/gtest.h" +//#include + +#include "faodel-common/Common.hh" +#include "lunasa/Lunasa.hh" +#include "lunasa/DataObject.hh" + +#include "lunasa/common/DataObjectPacker.hh" + +using namespace std; +using namespace faodel; +using namespace lunasa; + +//Note: Additional configuration settings will be loaded the file specified by FAODEL_CONFIG +string default_config = R"EOF( + +# IMPORTANT: this test won't work with tcmalloc implementation because it +# starts/finishes bootstrap multiple times. + +lunasa.lazy_memory_manager malloc +lunasa.eager_memory_manager malloc + +#lkv settings for the server +server.mutex_type rwlock + +node_role server +)EOF"; + + +class LunasaDataObjectPacker : public testing::Test { +protected: + void SetUp() override { + Configuration config(default_config); + config.AppendFromReferences(); + + bootstrap::Init(config, lunasa::bootstrap); + bootstrap::Start(); + } + + void TearDown() override { + bootstrap::Finish(); + } + bool ok; + int rc; +}; + +const uint8_t T_CHAR = 1; +const uint8_t T_INT =2; +const uint8_t T_FLOAT = 3; +const uint8_t T_DOUBLE = 4; + +void compareArrays(float *org, size_t org_bytes, float *tst, size_t tst_bytes) { + EXPECT_EQ(org_bytes, tst_bytes); + if(org_bytes == tst_bytes) { + int bad_count=0; + for(int i=0; i names = { "a", "b", "c", "d", "e", "f" }; + vector bytes = { 1024, 2048, 256, 100, 2000, 8 }; + vector ptrs_flt; + vector types_flt; + for(int i=0; i names; + vector bytes; + vector ptrs_flt; + vector types_flt; + for(int i=0; i<5; i++) { + int num_words = 1024*(i+1); + float *fptr = new float[num_words]; + for(int j=0; j=0; i--) { + float *org_ptr = (float *) ptrs_flt[i]; + float *tst_ptr; + size_t tst_bytes; + uint8_t tst_type; + + //Search by name + rc = gp2.GetVarPointer(names[i], (void **) &tst_ptr, &tst_bytes, &tst_type); + EXPECT_EQ(0, rc); + EXPECT_EQ(T_FLOAT, tst_type); + compareArrays((float *) ptrs_flt[i], bytes[i], tst_ptr, tst_bytes); + + tst_ptr=nullptr; + tst_bytes=0; + tst_type=0; + + //Search by hash + uint32_t hash = faodel::hash32(names[i]); + rc = gp2.GetVarPointer(hash, (void **) &tst_ptr, &tst_bytes, &tst_type); + switch(version) { + case 1: + EXPECT_EQ(EINVAL, rc); + break; + case 2: + EXPECT_EQ(0, rc); + EXPECT_EQ(T_FLOAT, tst_type); + compareArrays((float *) ptrs_flt[i], bytes[i], tst_ptr, tst_bytes); + } + } + + //See if we can retrieve names + vector pulled_names; + rc = gp2.GetVarNames(&pulled_names); + switch(version) { + case 1: + EXPECT_EQ(0, rc); + EXPECT_EQ(names.size(), pulled_names.size()); + break; + case 2: + EXPECT_EQ(EINVAL, rc); + EXPECT_EQ(0, pulled_names.size()); + break; + } + } + +} +TEST_F(LunasaDataObjectPacker, AppendStyle) { + + int rc1,rc2; + vector names; + vector ptrs; + vector found1; + vector found2; + size_t max_bytes=1024; + + + + DataObjectPacker dop1(max_bytes, 0, 1); + DataObjectPacker dop2(max_bytes, 0, 2); + + size_t bytes_left1=max_bytes; + size_t bytes_left2=max_bytes; + for(int i=0; i<20; i++) { + + string name = "thing-"+std::to_string(i); + double *ptr = new double[8]; + for(int j=0; j<8; j++) + ptr[j]=double(i*1000+j); + names.push_back(name); + ptrs.push_back(ptr); + + rc1=dop1.AppendVariable(name, (double *)ptr,8*sizeof(double), 1); + rc2=dop2.AppendVariable(name, (double *)ptr,8*sizeof(double), 1); + + //Sizes should be our expected size, plus some rounding + size_t s1 = dop1.ComputeEntrySize(name, 8*sizeof(double)); EXPECT_EQ(s1, 8*sizeof(double)+name.length()+1+1+2+4); //Should match struct + size_t s2 = dop2.ComputeEntrySize(name, 8*sizeof(double)); EXPECT_EQ(s2, 8*sizeof(double)+4+4+1+3); //Should match struct + + if(bytes_left1 < s1) { EXPECT_EQ(-1, rc1); + } else { EXPECT_EQ( 0, rc1); bytes_left1-=s1; } + if(bytes_left2 < s2) { EXPECT_EQ(-1, rc2); + } else { EXPECT_EQ( 0, rc2); bytes_left2-=s2; } + + found1.push_back(rc1==0); + found2.push_back(rc2==0); + //cout <<"Inserted ok: "<(&(ptr)), &bytes, &type); + if(rc1==0) { + EXPECT_TRUE(found1[i]); + EXPECT_EQ(sizeof(double)*8, bytes); + EXPECT_EQ(1, type); + if(bytes==sizeof(double)*8) { + double *src_ptr = ptrs[i]; + bool ok=true; + for(int j=0; j<8; j++) + ok &= (src_ptr[j] == ptr[j]); + EXPECT_TRUE(ok); + } + } else { + EXPECT_FALSE(found1[i]); + } + + //Check 2 + rc2 = dop2.GetVarPointer(names[i], reinterpret_cast(&(ptr)), &bytes, &type); + if(rc2==0) { + EXPECT_TRUE(found2[i]); + EXPECT_EQ(sizeof(double)*8, bytes); + EXPECT_EQ(1, type); + if(bytes==sizeof(double)*8) { + double *src_ptr = ptrs[i]; + bool ok=true; + for(int j=0; j<8; j++) + ok &= (src_ptr[j] == ptr[j]); + EXPECT_TRUE(ok); + } + } else { + EXPECT_FALSE(found2[i]); + } + + delete[] ptrs[i]; + + } + + +} + +TEST_F(LunasaDataObjectPacker, RefCounts) { + + int count; + + //Version 1: Make sure 2 ref counts while packer alive + DataObjectPacker dop1(1024, 0, 1); + auto ldo1 = dop1.GetDataObject(); + count = ldo1.internal_use_only.GetRefCount(); + EXPECT_EQ(2, count); + //cout << "version1 refcount " << ldo1.internal_use_only.GetRefCount() << endl; + + //Version 2: Pull ldo out of packer, deallocate packer, make sure ldo exists + DataObject ldo2; + { + DataObjectPacker dop2(1024,0,1); + ldo2 = dop2.GetDataObject(); + count = ldo2.internal_use_only.GetRefCount(); + EXPECT_EQ(2, count); + //cout <<"version2a refcount " << ldo2.internal_use_only.GetRefCount() << endl; + } + count = ldo2.internal_use_only.GetRefCount(); + EXPECT_EQ(1, count); + //cout <<"version2b refcount " << ldo2.internal_use_only.GetRefCount() << endl; + + //Version 3: pass ldo into packer, deallocate packer + { + DataObjectPacker dop3(ldo2); + count = ldo2.internal_use_only.GetRefCount(); + EXPECT_EQ(2, count); + //cout <<"version3a refcount is "< @@ -27,7 +27,6 @@ lunasa.lazy_memory_manager malloc lunasa.eager_memory_manager malloc #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock node_role server @@ -169,8 +168,8 @@ TEST_F(LunasaGenericRandomDataBundle, PackBinData) { } TEST_F(LunasaGenericRandomDataBundle, PayloadCapacityCheck) { - int LINE_SIZE=100; - int NUM_INSERTS=10; + const int LINE_SIZE=100; + const int NUM_INSERTS=10; DataObject ldo_myevents(sizeof(bundle_myevents_t), LINE_SIZE*NUM_INSERTS, DataObject::AllocatorType::eager); auto *myevents = ldo_myevents.GetMetaPtr(); @@ -326,8 +325,8 @@ TEST_F(LunasaGenericSequentialDataBundle, PackBinData) { TEST_F(LunasaGenericSequentialDataBundle, PayloadCapacityCheck) { - int LINE_SIZE=100; - int NUM_INSERTS=10; + const int LINE_SIZE=100; + const int NUM_INSERTS=10; DataObject ldo_myevents(sizeof(seq_bundle_myevents_t), (sizeof(uint32_t)+LINE_SIZE)*NUM_INSERTS, diff --git a/tests/lunasa/component/tb_lunasa_ldo.cpp b/tests/lunasa/component/tb_lunasa_ldo.cpp index 90d7170..d1163f4 100644 --- a/tests/lunasa/component/tb_lunasa_ldo.cpp +++ b/tests/lunasa/component/tb_lunasa_ldo.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "gtest/gtest.h" //#include @@ -23,7 +23,6 @@ lunasa.lazy_memory_manager malloc lunasa.eager_memory_manager malloc #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock node_role server @@ -52,7 +51,7 @@ TEST_F(LunasaDataObjectTest, StructSanityCheck) { EXPECT_EQ(0, ldo1.GetMetaSize()); EXPECT_EQ(1024, ldo1.GetDataSize()); EXPECT_EQ(1024, ldo1.GetUserSize()); - EXPECT_EQ(1024, ldo1.GetUserCapacity()); + EXPECT_LE(1024, ldo1.GetUserCapacity()); //Capacity May be a +4 bytes for safety EXPECT_EQ(8, ldo1.GetHeaderSize()); //Header should be fixed to uin16+uint16+uint32 EXPECT_EQ(1032, ldo1.GetWireSize()); //Header+user //EXPECT_EQ(1072, ldo1.GetRawAllocationSize()); //Sanity check: copied to catch changes in alloc size @@ -93,7 +92,7 @@ TEST_F(LunasaDataObjectTest, capcityChanges) { //stored everything. DataObject ldo1(1024,64,128, lunasa::DataObject::AllocatorType::eager, 0x2112); - EXPECT_EQ(1024, ldo1.GetUserCapacity()); + EXPECT_LE(1024, ldo1.GetUserCapacity()); //May get more than asked for, fo alignment EXPECT_EQ(64, ldo1.GetMetaSize()); EXPECT_EQ(128, ldo1.GetDataSize()); EXPECT_EQ(192, ldo1.GetUserSize()); @@ -102,8 +101,8 @@ TEST_F(LunasaDataObjectTest, capcityChanges) { rc= ldo1.ModifyUserSizes(256, 512); EXPECT_EQ(0, rc); rc= ldo1.ModifyUserSizes(512, 512); EXPECT_EQ(0, rc); rc= ldo1.ModifyUserSizes(64, 102); EXPECT_EQ(0, rc); - rc= ldo1.ModifyUserSizes(512, 513); EXPECT_EQ(-1, rc); - rc= ldo1.ModifyUserSizes(513, 512); EXPECT_EQ(-1, rc); + rc= ldo1.ModifyUserSizes(512, 517); EXPECT_EQ(-1, rc); //Note: capacity may be 1024+4 for alignment, try to validate + rc= ldo1.ModifyUserSizes(517, 512); EXPECT_EQ(-1, rc); //Make sure the sizes are still set to last valid setting EXPECT_EQ(64, ldo1.GetMetaSize()); diff --git a/tests/lunasa/component/tb_lunasa_performance.cpp b/tests/lunasa/component/tb_lunasa_performance.cpp index 460ade6..2c3d81a 100644 --- a/tests/lunasa/component/tb_lunasa_performance.cpp +++ b/tests/lunasa/component/tb_lunasa_performance.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "gtest/gtest.h" #include @@ -22,7 +22,6 @@ string default_config = R"EOF( default.kelpie.core_type nonet #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock lunasa.eager_memory_manager tcmalloc @@ -260,7 +259,7 @@ TEST_F(LunasaAllocTest, PreloadedAllocations) { // Preload cout << "preloading..." << flush; - for(int j = 0; j @@ -22,7 +22,6 @@ string default_config = R"EOF( default.kelpie.core_type nonet #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock lunasa.eager_memory_manager tcmalloc @@ -87,7 +86,7 @@ TEST_F(LunasaStatisticsTest, FixedSizeAllocations) { int i = 0; DataObject *allocs = new DataObject[ops]; - char buffer[BUFFER_LENGTH]; + //char buffer[BUFFER_LENGTH]; EXPECT_EQ(0, Lunasa::TotalAllocated()); while(num_bytes[i]>0) { @@ -117,10 +116,9 @@ TEST_F(LunasaStatisticsTest, RandomSizeAllocations) { srand((int) clock()); unsigned long *num_bytes = (unsigned long *) malloc(ops*sizeof(unsigned long)); - int i = 0; DataObject *allocs = new DataObject[ops]; - char buffer[BUFFER_LENGTH]; + //char buffer[BUFFER_LENGTH]; for(int i = 0; i @@ -24,7 +24,6 @@ string default_config = R"EOF( default.kelpie.core_type nonet #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock lunasa.eager_memory_manager tcmalloc @@ -122,7 +121,7 @@ void *fixed_allocation(void *arg, unsigned long allocation_size) { map *results = (map *) arg; DataObject *allocs = new DataObject[ops]; - char buffer[BUFFER_LENGTH]; + //char buffer[BUFFER_LENGTH]; test_results_t r; while(num_bytes[i]>0) { @@ -276,7 +275,7 @@ void process_alloc_results(alloc_test_results_t &results, string prefix, string double total_time = 0.0; long total_operations = 0; long total_bytes = 0; - for(int i = 0; i @@ -16,7 +16,6 @@ string valid_config = R"EOF( default.kelpie.core_type nonet #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock node_role server diff --git a/tests/lunasa/unit/tb_lunasa_configuration2_reinit_tc.cpp b/tests/lunasa/unit/tb_lunasa_configuration2_reinit_tc.cpp index d951928..af0b4c1 100644 --- a/tests/lunasa/unit/tb_lunasa_configuration2_reinit_tc.cpp +++ b/tests/lunasa/unit/tb_lunasa_configuration2_reinit_tc.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "gtest/gtest.h" //#include @@ -20,7 +20,6 @@ string valid_config = R"EOF( default.kelpie.core_type nonet #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock node_role server @@ -28,6 +27,11 @@ lunasa.eager_memory_manager tcmalloc lunasa.debug true lunasa.allocator.debug true + +# This line causes bootstrap to throw an exception instead of handling it and erroring out! +bootstrap.exit_on_errors false + +#bootstrap.debug true )EOF"; diff --git a/tests/lunasa/unit/tb_lunasa_configuration3_double_tc.cpp b/tests/lunasa/unit/tb_lunasa_configuration3_double_tc.cpp index eab58c2..30383dd 100644 --- a/tests/lunasa/unit/tb_lunasa_configuration3_double_tc.cpp +++ b/tests/lunasa/unit/tb_lunasa_configuration3_double_tc.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "gtest/gtest.h" //#include @@ -19,7 +19,6 @@ string invalid_config = R"EOF( default.kelpie.core_type nonet #lkv settings for the server -server.max_capacity 32M server.mutex_type rwlock node_role server @@ -27,12 +26,19 @@ node_role server # Attempts to create multiple instances of tcmalloc-based allocator (should fail) lunasa.eager_memory_manager tcmalloc lunasa.lazy_memory_manager tcmalloc + + +# This line causes bootstrap to throw an exception instead of handling it and erroring out! +bootstrap.exit_on_errors false +lunasa.debug true +bootstrap.debug true + + )EOF"; class LunasaCfgTest : public testing::Test { protected: void SetUp() override {} - void TearDown() override {} }; @@ -45,8 +51,7 @@ TEST_F(LunasaCfgTest, invalidCfgTest) { << " multiple instances of tcmalloc. It's ok!\n\n"; #ifdef Faodel_ENABLE_TCMALLOC - EXPECT_THROW(bootstrap::Init(Configuration(invalid_config), lunasa::bootstrap), - lunasa::LunasaConfigurationException); + EXPECT_ANY_THROW(bootstrap::Init(Configuration(invalid_config), lunasa::bootstrap)); #else //No tcmalloc support should throw its own exception EXPECT_ANY_THROW(bootstrap::Init(Configuration(invalid_config), lunasa::bootstrap)); @@ -55,3 +60,4 @@ TEST_F(LunasaCfgTest, invalidCfgTest) { cout << "\n ^--- Expect an error message above. It's ok!! ----^\n\n"; } + diff --git a/tests/nnti/benchmarks/AtomicsWithCallback.cpp b/tests/nnti/benchmarks/AtomicsWithCallback.cpp index 0b55860..d54213a 100644 --- a/tests/nnti/benchmarks/AtomicsWithCallback.cpp +++ b/tests/nnti/benchmarks/AtomicsWithCallback.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -74,7 +74,6 @@ class test_callback { } NNTI_result_t operator() (NNTI_event_t *event, void *context) { - NNTI_result_t rc = NNTI_OK; test_context *c = (test_context*)context; @@ -125,7 +124,6 @@ NNTI_result_t runbench_fadd(bool server, uint64_t total_us = std::chrono::duration_cast(end - start).count(); float us_per_xfer = (float)total_us/(float)ppc->threshold_; - float total_sec = (float)total_us/1000000.0; log_info("AtomicsWithCallback chrono fetch_add", "%6lu %6luus %6.3fus", ppc->threshold_, @@ -145,7 +143,6 @@ NNTI_result_t runbench_fadd(bool server, uint64_t total_us = std::chrono::duration_cast(end - start).count(); float us_per_xfer = (float)total_us/(float)ppc->threshold_; - float total_sec = (float)total_us/1000000.0; log_info("AtomicsWithCallback chrono fetch_add", "%6lu %6luus %6.3fus", ppc->threshold_, @@ -173,7 +170,6 @@ NNTI_result_t runbench_cswap(bool server, uint64_t total_us = std::chrono::duration_cast(end - start).count(); float us_per_xfer = (float)total_us/(float)ppc->threshold_; - float total_sec = (float)total_us/1000000.0; log_info("AtomicsWithCallback chrono compare_swap", "%6lu %6luus %6.3fus", ppc->threshold_, @@ -193,7 +189,6 @@ NNTI_result_t runbench_cswap(bool server, uint64_t total_us = std::chrono::duration_cast(end - start).count(); float us_per_xfer = (float)total_us/(float)ppc->threshold_; - float total_sec = (float)total_us/1000000.0; log_info("AtomicsWithCallback chrono compare_swap", "%6lu %6luus %6.3fus", ppc->threshold_, @@ -211,8 +206,6 @@ int main(int argc, char *argv[]) NNTI_result_t rc; nnti::transports::transport *t=nullptr; - int i=0; - char server_url[1][NNTI_URL_LEN]; const uint32_t num_servers = 1; uint32_t num_clients; diff --git a/tests/nnti/benchmarks/CMakeLists.txt b/tests/nnti/benchmarks/CMakeLists.txt index 82beb8e..38eabc1 100644 --- a/tests/nnti/benchmarks/CMakeLists.txt +++ b/tests/nnti/benchmarks/CMakeLists.txt @@ -8,8 +8,6 @@ include_directories( ${CMAKE_SOURCE_DIR}/src/nnti ${CMAKE_BINARY_DIR}/src/nnti ) -SET(EXTRA_TEST_LIBS "-lz") - # Build the test support lib------------------------------------------------- if(Faodel_ENABLE_MPI_SUPPORT) @@ -26,9 +24,9 @@ endif(Faodel_ENABLE_MPI_SUPPORT) set(SERIAL_TEST_LIBS benchutils + ZLIB::ZLIB GTest::GTest GTest::Main - ${EXTRA_TEST_LIBS} ) set(MPI_TEST_LIBS @@ -39,11 +37,12 @@ set(MPI_TEST_LIBS #--------------+-----------------------------+----------------------------------+---------+ # Format: | Name | Directory | Autorun | #--------------+-----------------------------+----------------------------------+---------+ - +if( Faodel_HAVE_CRC32 ) if( Faodel_ENABLE_MPI_SUPPORT ) add_mpi_test( MemoryRegistration . 1 false ) add_mpi_test( AtomicsWithCallback . 2 false ) add_mpi_test( RdmaWithCallback . 2 false ) add_mpi_test( SendWithCallback . 2 false ) -endif() +endif( Faodel_ENABLE_MPI_SUPPORT ) +endif( Faodel_HAVE_CRC32 ) diff --git a/tests/nnti/benchmarks/MemoryRegistration.cpp b/tests/nnti/benchmarks/MemoryRegistration.cpp index f577eb3..2295d9c 100644 --- a/tests/nnti/benchmarks/MemoryRegistration.cpp +++ b/tests/nnti/benchmarks/MemoryRegistration.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/benchmarks/RdmaWithCallback.cpp b/tests/nnti/benchmarks/RdmaWithCallback.cpp index 6578c85..5bb51aa 100644 --- a/tests/nnti/benchmarks/RdmaWithCallback.cpp +++ b/tests/nnti/benchmarks/RdmaWithCallback.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -78,7 +78,6 @@ class test_callback { } NNTI_result_t operator() (NNTI_event_t *event, void *context) { - NNTI_result_t rc = NNTI_OK; test_context *c = (test_context*)context; @@ -138,7 +137,6 @@ NNTI_result_t runbench_get(bool server, auto end = std::chrono::high_resolution_clock::now(); - float total_bytes = ppc->length_ * ppc->threshold_; float total_megabytes = (ppc->length_ * ppc->threshold_) / (1024.0 * 1024.0); uint64_t total_us = std::chrono::duration_cast(end - start).count(); float us_per_xfer = (float)total_us/(float)ppc->threshold_; @@ -162,7 +160,6 @@ NNTI_result_t runbench_get(bool server, auto end = std::chrono::high_resolution_clock::now(); - float total_bytes = ppc->length_ * ppc->threshold_; float total_megabytes = (ppc->length_ * ppc->threshold_) / (1024.0 * 1024.0); uint64_t total_us = std::chrono::duration_cast(end - start).count(); float us_per_xfer = (float)total_us/(float)ppc->threshold_; @@ -194,7 +191,6 @@ NNTI_result_t runbench_put(bool server, auto end = std::chrono::high_resolution_clock::now(); - float total_bytes = ppc->length_ * ppc->threshold_; float total_megabytes = (ppc->length_ * ppc->threshold_) / (1024.0 * 1024.0); uint64_t total_us = std::chrono::duration_cast(end - start).count(); float us_per_xfer = (float)total_us/(float)ppc->threshold_; @@ -218,7 +214,6 @@ NNTI_result_t runbench_put(bool server, auto end = std::chrono::high_resolution_clock::now(); - float total_bytes = ppc->length_ * ppc->threshold_; float total_megabytes = (ppc->length_ * ppc->threshold_) / (1024.0 * 1024.0); uint64_t total_us = std::chrono::duration_cast(end - start).count(); float us_per_xfer = (float)total_us/(float)ppc->threshold_; @@ -242,8 +237,6 @@ int main(int argc, char *argv[]) NNTI_result_t rc; nnti::transports::transport *t=nullptr; - int i=0; - char server_url[1][NNTI_URL_LEN]; const uint32_t num_servers = 1; uint32_t num_clients; diff --git a/tests/nnti/benchmarks/SendWithCallback.cpp b/tests/nnti/benchmarks/SendWithCallback.cpp index b816622..1ca546e 100644 --- a/tests/nnti/benchmarks/SendWithCallback.cpp +++ b/tests/nnti/benchmarks/SendWithCallback.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -80,7 +80,6 @@ class test_callback { } NNTI_result_t operator() (NNTI_event_t *event, void *context) { - NNTI_result_t rc = NNTI_OK; test_context *c = (test_context*)context; @@ -102,7 +101,7 @@ class test_callback { if (c->recv_count_ < c->volley_threshold_) { // issue another send - rc = send_data_async(c->transport_, c->length_, 0, c->send_src_.hdl, c->send_target_.hdl, event->peer, *c->cb_, context); + send_data_async(c->transport_, c->length_, 0, c->send_src_.hdl, c->send_target_.hdl, event->peer, *c->cb_, context); c->recv_count_++; } else { // we're done here. return NNTI_ECANCEL to push an event on the EQ. @@ -135,7 +134,6 @@ NNTI_result_t runbench(bool server, auto end = std::chrono::high_resolution_clock::now(); - float total_bytes = ppc->length_ * ppc->volley_threshold_; // total_megabytes is the ROUNDTRIP total so multiply by 2 float total_megabytes = (ppc->length_ * ppc->volley_threshold_ * 2.0) / (1024.0 * 1024.0); uint64_t total_us = std::chrono::duration_cast(end - start).count(); @@ -158,7 +156,6 @@ NNTI_result_t runbench(bool server, auto end = std::chrono::high_resolution_clock::now(); - float total_bytes = ppc->length_ * ppc->volley_threshold_; // total_megabytes is the ROUNDTRIP total so multiply by 2 float total_megabytes = (ppc->length_ * ppc->volley_threshold_ * 2.0) / (1024.0 * 1024.0); uint64_t total_us = std::chrono::duration_cast(end - start).count(); @@ -189,8 +186,6 @@ int main(int argc, char *argv[]) NNTI_result_t rc; nnti::transports::transport *t=nullptr; - int i=0; - char server_url[1][NNTI_URL_LEN]; const uint32_t num_servers = 1; uint32_t num_clients; diff --git a/tests/nnti/benchmarks/bench_utils.cpp b/tests/nnti/benchmarks/bench_utils.cpp index 42b14f2..c4b15ce 100644 --- a/tests/nnti/benchmarks/bench_utils.cpp +++ b/tests/nnti/benchmarks/bench_utils.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -183,7 +183,7 @@ get_rank(int &my_rank) } NNTI_result_t -find_server_urls(int num_servers, +find_server_urls(uint32_t num_servers, uint32_t my_rank, uint32_t num_procs, char *my_url, @@ -197,7 +197,7 @@ find_server_urls(int num_servers, MPI_Allgather(my_url, NNTI_URL_LEN, MPI_CHAR, all_urls, NNTI_URL_LEN, MPI_CHAR, MPI_COMM_WORLD); - for (int i=0;i #include diff --git a/tests/nnti/c-api/NntiInitTest1.c b/tests/nnti/c-api/NntiInitTest1.c index 9b25969..b67c967 100644 --- a/tests/nnti/c-api/NntiInitTest1.c +++ b/tests/nnti/c-api/NntiInitTest1.c @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/nnti/c-api/NntiInitTest2.c b/tests/nnti/c-api/NntiInitTest2.c index 06fd867..47a5130 100644 --- a/tests/nnti/c-api/NntiInitTest2.c +++ b/tests/nnti/c-api/NntiInitTest2.c @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/nnti/c-api/NntiInitTest3.c b/tests/nnti/c-api/NntiInitTest3.c index b3d5fbc..e9a0655 100644 --- a/tests/nnti/c-api/NntiInitTest3.c +++ b/tests/nnti/c-api/NntiInitTest3.c @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/nnti/c-api/NntiInitTest4.c b/tests/nnti/c-api/NntiInitTest4.c index e0cc33a..82ee54a 100644 --- a/tests/nnti/c-api/NntiInitTest4.c +++ b/tests/nnti/c-api/NntiInitTest4.c @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/nnti/c-api/NntiLoggerTest1.c b/tests/nnti/c-api/NntiLoggerTest1.c index 308d1ff..7398684 100644 --- a/tests/nnti/c-api/NntiLoggerTest1.c +++ b/tests/nnti/c-api/NntiLoggerTest1.c @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/nnti/c-api/NntiLoggerTest2.c b/tests/nnti/c-api/NntiLoggerTest2.c index ed56f50..f63357b 100644 --- a/tests/nnti/c-api/NntiLoggerTest2.c +++ b/tests/nnti/c-api/NntiLoggerTest2.c @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/nnti/c-api/NntiLoggerTest3.c b/tests/nnti/c-api/NntiLoggerTest3.c index 0af78ac..9d8f806 100644 --- a/tests/nnti/c-api/NntiLoggerTest3.c +++ b/tests/nnti/c-api/NntiLoggerTest3.c @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include diff --git a/tests/nnti/c-api/test_utils.cpp b/tests/nnti/c-api/test_utils.cpp index 8c61cf2..3b7f72e 100644 --- a/tests/nnti/c-api/test_utils.cpp +++ b/tests/nnti/c-api/test_utils.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nntiConfig.h" @@ -72,7 +72,6 @@ send_target_hdl(NNTI_transport_t transport, NNTI_result_t rc=NNTI_OK; NNTI_event_t event; - NNTI_event_t result_event; uint32_t which; NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; @@ -129,7 +128,6 @@ recv_target_hdl(NNTI_transport_t transport, NNTI_event_t event; NNTI_event_t result_event; uint32_t which; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; uint32_t msgs_received=0; while (1) { @@ -153,7 +151,6 @@ recv_target_hdl(NNTI_transport_t transport, *peer_hdl = event.peer; -cleanup: return rc; } @@ -169,7 +166,6 @@ send_hdl(NNTI_transport_t transport, NNTI_result_t rc=NNTI_OK; NNTI_event_t event; - NNTI_event_t result_event; uint32_t which; NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; @@ -227,7 +223,6 @@ recv_hdl(NNTI_transport_t transport, NNTI_event_t event; NNTI_event_t result_event; uint32_t which; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; uint32_t msgs_received=0; while (1) { @@ -252,7 +247,6 @@ recv_hdl(NNTI_transport_t transport, *peer_hdl = event.peer; -cleanup: return rc; } @@ -267,7 +261,6 @@ send_ack(NNTI_transport_t transport, NNTI_result_t rc=NNTI_OK; NNTI_event_t event; - NNTI_event_t result_event; uint32_t which; NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; @@ -312,9 +305,9 @@ recv_ack(NNTI_transport_t transport, NNTI_result_t rc=NNTI_OK; NNTI_event_t event; - NNTI_event_t result_event; +// NNTI_event_t result_event; uint32_t which; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; +// NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; log_debug("test_utils", "recv_ack - enter"); @@ -332,7 +325,6 @@ recv_ack(NNTI_transport_t transport, *peer_hdl = event.peer; -cleanup: log_debug("test_utils", "recv_ack - exit"); return rc; @@ -410,7 +402,6 @@ send_data(NNTI_transport_t transport, NNTI_result_t rc=NNTI_OK; NNTI_event_t event; - NNTI_event_t result_event; uint32_t which; NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; @@ -453,9 +444,7 @@ recv_data(NNTI_transport_t transport, { NNTI_result_t rc=NNTI_OK; - NNTI_event_t result_event; uint32_t which; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; uint32_t msgs_received=0; while (1) { @@ -470,7 +459,6 @@ recv_data(NNTI_transport_t transport, } } -cleanup: return rc; } @@ -485,7 +473,6 @@ get_data(NNTI_transport_t transport, NNTI_result_t rc=NNTI_OK; NNTI_event_t event; - NNTI_event_t result_event; uint32_t which; NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; @@ -531,7 +518,6 @@ put_data(NNTI_transport_t transport, NNTI_result_t rc=NNTI_OK; NNTI_event_t event; - NNTI_event_t result_event; uint32_t which; NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; diff --git a/tests/nnti/c-api/test_utils.h b/tests/nnti/c-api/test_utils.h index 7244bfd..461d211 100644 --- a/tests/nnti/c-api/test_utils.h +++ b/tests/nnti/c-api/test_utils.h @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef TEST_UTILS_H diff --git a/tests/nnti/cpp-api/AllocFreeTest.cpp b/tests/nnti/cpp-api/AllocFreeTest.cpp index 978e9ea..7ed7a11 100644 --- a/tests/nnti/cpp-api/AllocFreeTest.cpp +++ b/tests/nnti/cpp-api/AllocFreeTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/cpp-api/AtomicOpTest.cpp b/tests/nnti/cpp-api/AtomicOpTest.cpp index ac2d3e0..26c0c4c 100644 --- a/tests/nnti/cpp-api/AtomicOpTest.cpp +++ b/tests/nnti/cpp-api/AtomicOpTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -101,13 +101,9 @@ TEST_F(NntiAtomicOpTest, start1) { if (i_am_server) { NNTI_event_queue_t eq; - NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t buf_hdl; char *buf_base=nullptr; uint32_t buf_size=3200; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); t->alloc(3200, @@ -168,16 +164,12 @@ TEST_F(NntiAtomicOpTest, start1) { } else { NNTI_event_queue_t eq; - NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t buf_hdl; NNTI_buffer_t ack_hdl; char *buf_base=nullptr; char *ack_base=nullptr; uint32_t buf_size=3200; uint32_t ack_size=320; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; // give the server a chance to startup // nnti::util::sleep(1000); @@ -201,7 +193,6 @@ TEST_F(NntiAtomicOpTest, start1) { &ack_base, &ack_hdl); - NNTI_buffer_t target_hdl; NNTI_peer_t recv_peer; rc = send_hdl(t, buf_hdl, buf_base, buf_size, peer_hdl, eq); diff --git a/tests/nnti/cpp-api/CMakeLists.txt b/tests/nnti/cpp-api/CMakeLists.txt index 722fe71..4f4332f 100644 --- a/tests/nnti/cpp-api/CMakeLists.txt +++ b/tests/nnti/cpp-api/CMakeLists.txt @@ -1,32 +1,29 @@ -# Note: to find gtest, pass in GTEST_ROOT. -# eg, cmake -DGTEST_ROOT=/my/install_dir -# where install dir has include and lib in it - include_directories( - ${CMAKE_SOURCE_DIR}/src/nnti - ${CMAKE_BINARY_DIR}/src/nnti + ${CMAKE_SOURCE_DIR}/src + ${CMAKE_BINARY_DIR}/src ) -SET(EXTRA_TEST_LIBS "-lz") - # Build the test support lib------------------------------------------------- -add_library(cpptestutils +if( Faodel_ENABLE_MPI_SUPPORT ) + add_library(cpptestutils test_utils.hpp test_utils.cpp -) -set_target_properties(cpptestutils PROPERTIES LINKER_LANGUAGE CXX ) -target_link_libraries( cpptestutils PUBLIC nnti ) + ) + set_target_properties( cpptestutils PROPERTIES LINKER_LANGUAGE CXX ) + target_link_libraries( cpptestutils PUBLIC nnti ) +endif() #---------------------------------------------------------------------------- -set(SERIAL_TEST_LIBS +set(SERIAL_TEST_LIBS cpptestutils + ZLIB::ZLIB GTest::GTest GTest::Main - ${EXTRA_TEST_LIBS} ) -set(MPI_TEST_LIBS +set(MPI_TEST_LIBS + cpptestutils ${SERIAL_TEST_LIBS} MPI::MPI_CXX ) @@ -34,16 +31,21 @@ set(MPI_TEST_LIBS #--------------+-----------------------------+----------------------------------+---------+ # Format: | Name | Directory | Autorun | #--------------+-----------------------------+----------------------------------+---------+ -add_serial_test( NntiEqClassTest . false ) -add_serial_test( NntiEventCallbackClassTest . false ) -add_serial_test( NntiFreelistClassTest . false ) -add_serial_test( NntiLoggerClassTest . false ) -add_serial_test( NntiOpClassTest . false ) -add_serial_test( NntiPidClassTest . false ) -add_serial_test( NntiUrlClassTest . false ) +if( Faodel_HAVE_CRC32 ) + # Note: Previously all serial tests were here, but the support libary + # has MPI code so they all were moved into the mpi section + #add_serial_test( NntiWidClassTest . false ) if( Faodel_ENABLE_MPI_SUPPORT ) + add_serial_test( NntiEventCallbackClassTest . false ) + add_serial_test( NntiEqClassTest . false ) + add_serial_test( NntiFreelistClassTest . false ) + add_serial_test( NntiLoggerClassTest . false ) + add_serial_test( NntiOpClassTest . false ) + add_serial_test( NntiPidClassTest . false ) + add_serial_test( NntiUrlClassTest . false ) + add_mpi_test( AllocFreeTest . 1 true ) add_mpi_test( AtomicOpTest . 2 true ) add_mpi_test( CallbackStateMachineTest . 2 true ) @@ -54,12 +56,15 @@ if( Faodel_ENABLE_MPI_SUPPORT ) add_mpi_test( PingPongCallbackTest . 2 true ) add_mpi_test( QueueSendTest1 . 2 true ) add_mpi_test( QueueSendTest2 . 2 true ) + add_mpi_test( RdmaAlignmentTest . 2 true ) add_mpi_test( RdmaLengthTest . 2 true ) add_mpi_test( RdmaOpTest . 2 true ) + add_mpi_test( SelfConnectTest . 1 true ) add_mpi_test( ShortSendTest . 2 true ) add_mpi_test( UnexpectedCallbackTest . 2 true ) add_mpi_test( UnexpectedLongSendTest . 2 true ) add_mpi_test( UnexpectedSendTest . 2 true ) add_mpi_test( UrlPidTest . 1 true ) add_mpi_test( ZeroCopySendTest . 2 true ) -endif() +endif( Faodel_ENABLE_MPI_SUPPORT ) +endif( Faodel_HAVE_CRC32 ) diff --git a/tests/nnti/cpp-api/CallbackStateMachineTest.cpp b/tests/nnti/cpp-api/CallbackStateMachineTest.cpp index a0666a9..df89966 100644 --- a/tests/nnti/cpp-api/CallbackStateMachineTest.cpp +++ b/tests/nnti/cpp-api/CallbackStateMachineTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -101,7 +101,6 @@ class state_machine_callback { } NNTI_result_t operator() (NNTI_event_t *event, void *context) { - NNTI_result_t rc = NNTI_OK; state_machine_context *c = (state_machine_context*)context; @@ -116,15 +115,15 @@ class state_machine_callback { case NNTI_EVENT_SEND: if (c->send_count_ < c->send_threshold_) { // issue another send - rc = populate_buffer(c->transport_, c->send_count_, 0, c->send_src_.hdl, c->send_src_.base, c->send_src_.size); - rc = send_data_async(c->transport_, 0, c->send_src_.hdl, c->send_target_.hdl, event->peer, *c->cb_, context); + populate_buffer(c->transport_, c->send_count_, 0, c->send_src_.hdl, c->send_src_.base, c->send_src_.size); + send_data_async(c->transport_, 0, c->send_src_.hdl, c->send_target_.hdl, event->peer, *c->cb_, context); c->send_count_++; break; } else { c->state_ = state_machine_context::state::GETTING; // issue get - rc = get_data_async(c->transport_, c->remote_rdma_.hdl, c->local_rdma_.hdl, event->peer, *c->cb_, context); + get_data_async(c->transport_, c->remote_rdma_.hdl, c->local_rdma_.hdl, event->peer, *c->cb_, context); c->get_count_++; } break; @@ -135,7 +134,7 @@ class state_machine_callback { EXPECT_TRUE(verify_buffer((char*)event->start, event->offset+(i*msg_size), msg_size*test_iters)); } // issue another get - rc = get_data_async(c->transport_, c->remote_rdma_.hdl, c->local_rdma_.hdl, event->peer, *c->cb_, context); + get_data_async(c->transport_, c->remote_rdma_.hdl, c->local_rdma_.hdl, event->peer, *c->cb_, context); c->get_count_++; break; @@ -146,21 +145,21 @@ class state_machine_callback { c->state_ = state_machine_context::state::PUTTING; // issue put for (int i=0;i<10;i++) { - rc = populate_buffer(c->transport_, 2*i, i, c->local_rdma_.hdl, c->local_rdma_.base, c->local_rdma_.size); + populate_buffer(c->transport_, 2*i, i, c->local_rdma_.hdl, c->local_rdma_.base, c->local_rdma_.size); } - rc = put_data_async(c->transport_, c->local_rdma_.hdl, c->remote_rdma_.hdl, event->peer, *c->cb_, context); + put_data_async(c->transport_, c->local_rdma_.hdl, c->remote_rdma_.hdl, event->peer, *c->cb_, context); c->put_count_++; } break; case NNTI_EVENT_PUT: if (c->put_count_ < c->put_threshold_) { // issue another put - rc = put_data_async(c->transport_, c->local_rdma_.hdl, c->remote_rdma_.hdl, event->peer, *c->cb_, context); + put_data_async(c->transport_, c->local_rdma_.hdl, c->remote_rdma_.hdl, event->peer, *c->cb_, context); c->put_count_++; break; } else { c->state_ = state_machine_context::state::DONE; - rc = send_data_async(c->transport_, 0, c->send_src_.hdl, c->send_target_.hdl, event->peer, *c->cb_, context); + send_data_async(c->transport_, 0, c->send_src_.hdl, c->send_target_.hdl, event->peer, *c->cb_, context); } break; default: @@ -235,12 +234,9 @@ TEST_F(NntiCallbackStateMachineTest, start1) { NNTI_event_queue_t eq; NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; struct buffer_properties src_buf; struct buffer_properties rdma_buf; struct buffer_properties my_q_buf; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; nnti::datatype::nnti_event_callback null_cb(t); diff --git a/tests/nnti/cpp-api/ConnectTest.cpp b/tests/nnti/cpp-api/ConnectTest.cpp index f2db09e..0777a1e 100644 --- a/tests/nnti/cpp-api/ConnectTest.cpp +++ b/tests/nnti/cpp-api/ConnectTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/cpp-api/LongSendTest.cpp b/tests/nnti/cpp-api/LongSendTest.cpp index b630894..900d61a 100644 --- a/tests/nnti/cpp-api/LongSendTest.cpp +++ b/tests/nnti/cpp-api/LongSendTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -107,12 +107,9 @@ TEST_F(NntiLongSendTest, start1) { if (i_am_server) { NNTI_event_queue_t eq; NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t buf_hdl; char *buf_base=nullptr; uint32_t buf_size=40960; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); t->alloc(buf_size, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, func_cb, nullptr, &buf_base, &buf_hdl); @@ -158,12 +155,9 @@ TEST_F(NntiLongSendTest, start1) { } else { NNTI_event_queue_t eq; NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t buf_hdl, unused_hdl; char *buf_base=nullptr, *unused_base=nullptr; uint32_t buf_size=40960; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; // give the server a chance to startup MPI_Barrier(MPI_COMM_WORLD); diff --git a/tests/nnti/cpp-api/MultiPingPongTest.cpp b/tests/nnti/cpp-api/MultiPingPongTest.cpp index b9a22fc..1fcf4ff 100644 --- a/tests/nnti/cpp-api/MultiPingPongTest.cpp +++ b/tests/nnti/cpp-api/MultiPingPongTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -90,7 +90,6 @@ class pingpong_callback { } NNTI_result_t operator() (NNTI_event_t *event, void *context) { - NNTI_result_t rc = NNTI_OK; pingpong_context *c = (pingpong_context*)context; @@ -118,10 +117,10 @@ class pingpong_callback { uint32_t seed = *(uint32_t*)(payload+4); // the salt seed++; - rc = populate_buffer(c->transport_, seed, 0, c->send_src_.hdl, c->send_src_.base, c->send_src_.size); + populate_buffer(c->transport_, seed, 0, c->send_src_.hdl, c->send_src_.base, c->send_src_.size); struct buffer_properties *send_target = c->send_target_map_[event->peer]; - rc = send_data_async(c->transport_, 0, c->send_src_.hdl, send_target->hdl, event->peer, *c->cb_, context); + send_data_async(c->transport_, 0, c->send_src_.hdl, send_target->hdl, event->peer, *c->cb_, context); c->recv_count_++; } else { @@ -265,7 +264,7 @@ TEST_F(NntiMultiPingPongTest, start1) { MPI_Barrier(MPI_COMM_WORLD); - for (int i=0;ihdl, &peer_hdl, unexpected_eq); diff --git a/tests/nnti/cpp-api/NntiEqClassTest.cpp b/tests/nnti/cpp-api/NntiEqClassTest.cpp index a838197..1704dbf 100644 --- a/tests/nnti/cpp-api/NntiEqClassTest.cpp +++ b/tests/nnti/cpp-api/NntiEqClassTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -19,11 +19,11 @@ #include "nnti/nnti_eq.hpp" -static const auto num_consumers = 3; -static const auto num_producers = 1; +static const uint32_t num_consumers = 3; +static const uint32_t num_producers = 1; static const uint64_t queue_size = 32; -static const auto events_per_producer = queue_size * 1024; -static const auto total_events = events_per_producer * num_producers; +static const uint32_t events_per_producer = queue_size * 1024; +static const uint32_t total_events = events_per_producer * num_producers; static const uint64_t NOT_PRODUCED = 0xA; // this event has not been pushed onto the EQ static const uint64_t NOT_CONSUMED = 0xB; // this event has not been popped off the EQ @@ -31,8 +31,8 @@ static const uint64_t CONSUMED = 0xC; // this event has been popped off the NNTI_event_t event_source[total_events]; -std::atomic next_event(0); -std::atomic events_consumed(0); +std::atomic next_event(0); +std::atomic events_consumed(0); struct ProducerWithReservation { nnti::datatype::nnti_event_queue *eq_; @@ -47,8 +47,8 @@ struct ProducerWithReservation { void operator()() { - for (auto i = 0; i < events_per_producer; i ++) { - auto current_event = next_event.fetch_add(1); + for (uint32_t i = 0; i < events_per_producer; i ++) { + uint32_t current_event = next_event.fetch_add(1); assert(event_source[current_event].offset == NOT_PRODUCED); event_source[current_event].offset = NOT_CONSUMED; nnti::datatype::reservation r; @@ -76,8 +76,8 @@ struct ProducerWithoutReservation { void operator()() { - for (auto i = 0; i < total_events; i += num_producers) { - auto current_event = next_event.fetch_add(1); + for (uint32_t i = 0; i < total_events; i += num_producers) { + uint32_t current_event = next_event.fetch_add(1); assert(event_source[current_event].offset == NOT_PRODUCED); event_source[i].offset = NOT_CONSUMED; NNTI_event_t *e = &event_source[i]; @@ -133,27 +133,27 @@ test_without_reservation() next_event.store(0); events_consumed.store(0); memset(event_source, 0, total_events * sizeof(NNTI_event_t)); - for (auto i = 0; i < total_events; ++i) { + for (uint32_t i = 0; i < total_events; ++i) { event_source[i].offset = NOT_PRODUCED; } struct timeval tv0, tv1; gettimeofday(&tv0, NULL); - for (auto i = 0; i < num_producers; ++i) { + for (uint32_t i = 0; i < num_producers; ++i) { producers[i] = std::thread(ProducerWithoutReservation(&eq_without_reservations)); } usleep(10 * 1000); - for (auto i = 0; i < num_consumers; ++i) { + for (uint32_t i = 0; i < num_consumers; ++i) { consumers[i] = std::thread(Consumer(&eq_without_reservations)); } - for (auto i = 0; i < num_producers; ++i) { + for (uint32_t i = 0; i < num_producers; ++i) { producers[i].join(); } - for (auto i = 0; i < num_consumers; ++i) { + for (uint32_t i = 0; i < num_consumers; ++i) { consumers[i].join(); } @@ -162,7 +162,7 @@ test_without_reservation() bool success = true; std::cout << "check results..." << std::endl; - for (auto i = 0; i < total_events; ++i) { + for (uint32_t i = 0; i < total_events; ++i) { if (event_source[i].offset == NOT_PRODUCED) { std::cout << "not produced " << i << std::endl; success = false; @@ -187,27 +187,27 @@ test_with_reservation() next_event.store(0); events_consumed.store(0); memset(event_source, 0, total_events * sizeof(NNTI_event_t)); - for (auto i = 0; i < total_events; ++i) { + for (uint32_t i = 0; i < total_events; ++i) { event_source[i].offset = NOT_PRODUCED; } struct timeval tv0, tv1; gettimeofday(&tv0, NULL); - for (auto i = 0; i < num_producers; ++i) { + for (uint32_t i = 0; i < num_producers; ++i) { producers[i] = std::thread(ProducerWithReservation(&eq_with_reservations)); } usleep(10 * 1000); - for (auto i = 0; i < num_consumers; ++i) { + for (uint32_t i = 0; i < num_consumers; ++i) { consumers[i] = std::thread(Consumer(&eq_with_reservations)); } - for (auto i = 0; i < num_producers; ++i) { + for (uint32_t i = 0; i < num_producers; ++i) { producers[i].join(); } - for (auto i = 0; i < num_consumers; ++i) { + for (uint32_t i = 0; i < num_consumers; ++i) { consumers[i].join(); } @@ -216,7 +216,7 @@ test_with_reservation() bool success = true; std::cout << "check results..." << std::endl; - for (auto i = 0; i < total_events; ++i) { + for (uint32_t i = 0; i < total_events; ++i) { if (event_source[i].offset == NOT_PRODUCED) { std::cout << "not produced " << i << std::endl; success = false; diff --git a/tests/nnti/cpp-api/NntiEventCallbackClassTest.cpp b/tests/nnti/cpp-api/NntiEventCallbackClassTest.cpp index ce6e33c..f7a66ec 100644 --- a/tests/nnti/cpp-api/NntiEventCallbackClassTest.cpp +++ b/tests/nnti/cpp-api/NntiEventCallbackClassTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/cpp-api/NntiFreelistClassTest.cpp b/tests/nnti/cpp-api/NntiFreelistClassTest.cpp index df926e0..6e0036f 100644 --- a/tests/nnti/cpp-api/NntiFreelistClassTest.cpp +++ b/tests/nnti/cpp-api/NntiFreelistClassTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/cpp-api/NntiLoggerClassTest.cpp b/tests/nnti/cpp-api/NntiLoggerClassTest.cpp index c790957..3d681e0 100644 --- a/tests/nnti/cpp-api/NntiLoggerClassTest.cpp +++ b/tests/nnti/cpp-api/NntiLoggerClassTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/cpp-api/NntiOpClassTest.cpp b/tests/nnti/cpp-api/NntiOpClassTest.cpp index 0684001..06a80fd 100644 --- a/tests/nnti/cpp-api/NntiOpClassTest.cpp +++ b/tests/nnti/cpp-api/NntiOpClassTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/cpp-api/NntiOpVectorTest.cpp b/tests/nnti/cpp-api/NntiOpVectorTest.cpp index 9289cc2..e64d514 100644 --- a/tests/nnti/cpp-api/NntiOpVectorTest.cpp +++ b/tests/nnti/cpp-api/NntiOpVectorTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -94,7 +94,7 @@ TEST_F(NntiOpVectorTest, start1) { mirror.reserve(1024); - for (auto i = 0; i < num_op; ++i) { + for (int i = 0; i < num_op; ++i) { test_op *op = new test_op(t, &wid); op_vector.add(op); mirror[i] = op; @@ -112,7 +112,7 @@ TEST_F(NntiOpVectorTest, start1) { delete victim_op; } } - for (auto i = 0; i < mirror.size(); ++i) { + for (uint32_t i = 0; i < mirror.size(); ++i) { EXPECT_EQ(op_vector.at(i), mirror[i]); op_vector.remove(i); delete mirror[i]; diff --git a/tests/nnti/cpp-api/NntiPidClassTest.cpp b/tests/nnti/cpp-api/NntiPidClassTest.cpp index dbfe946..3a2ceb5 100644 --- a/tests/nnti/cpp-api/NntiPidClassTest.cpp +++ b/tests/nnti/cpp-api/NntiPidClassTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/cpp-api/NntiUrlClassTest.cpp b/tests/nnti/cpp-api/NntiUrlClassTest.cpp index f68d449..4ff3038 100644 --- a/tests/nnti/cpp-api/NntiUrlClassTest.cpp +++ b/tests/nnti/cpp-api/NntiUrlClassTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/cpp-api/NntiWidClassTest.cpp b/tests/nnti/cpp-api/NntiWidClassTest.cpp index 08c3e54..03a0192 100644 --- a/tests/nnti/cpp-api/NntiWidClassTest.cpp +++ b/tests/nnti/cpp-api/NntiWidClassTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" diff --git a/tests/nnti/cpp-api/PingPongCallbackTest.cpp b/tests/nnti/cpp-api/PingPongCallbackTest.cpp index ab98ec8..390347d 100644 --- a/tests/nnti/cpp-api/PingPongCallbackTest.cpp +++ b/tests/nnti/cpp-api/PingPongCallbackTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -74,7 +74,6 @@ class pingpong_callback { } NNTI_result_t operator() (NNTI_event_t *event, void *context) { - NNTI_result_t rc = NNTI_OK; pingpong_context *c = (pingpong_context*)context; @@ -101,9 +100,9 @@ class pingpong_callback { uint32_t seed = *(uint32_t*)(payload+4); // the salt seed++; - rc = populate_buffer(c->transport_, seed, 0, c->send_src_.hdl, c->send_src_.base, c->send_src_.size); + populate_buffer(c->transport_, seed, 0, c->send_src_.hdl, c->send_src_.base, c->send_src_.size); - rc = send_data_async(c->transport_, 0, c->send_src_.hdl, c->send_target_.hdl, event->peer, *c->cb_, context); + send_data_async(c->transport_, 0, c->send_src_.hdl, c->send_target_.hdl, event->peer, *c->cb_, context); c->recv_count_++; } else { diff --git a/tests/nnti/cpp-api/QueueSendTest1.cpp b/tests/nnti/cpp-api/QueueSendTest1.cpp index 6854875..c496420 100644 --- a/tests/nnti/cpp-api/QueueSendTest1.cpp +++ b/tests/nnti/cpp-api/QueueSendTest1.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -105,15 +105,12 @@ TEST_F(NntiQueueSendTest1, start1) { if (i_am_server) { NNTI_event_queue_t eq; NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t src_hdl; char *src_base=nullptr; uint32_t src_size=320; NNTI_buffer_t my_q_hdl; char *my_q_base=nullptr; uint32_t my_q_size=3200; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); t->alloc(src_size, @@ -163,15 +160,12 @@ TEST_F(NntiQueueSendTest1, start1) { } else { NNTI_event_queue_t eq; NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t src_hdl; char *src_base=nullptr; uint32_t src_size=320; NNTI_buffer_t my_q_hdl; char *my_q_base=nullptr; uint32_t my_q_size=3200; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; // give the server a chance to startup // nnti::util::sleep(1000); diff --git a/tests/nnti/cpp-api/QueueSendTest2.cpp b/tests/nnti/cpp-api/QueueSendTest2.cpp index e7fddaa..b00a781 100644 --- a/tests/nnti/cpp-api/QueueSendTest2.cpp +++ b/tests/nnti/cpp-api/QueueSendTest2.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -105,15 +105,12 @@ TEST_F(NntiQueueSendTest2, start1) { if (i_am_server) { NNTI_event_queue_t eq; NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t src_hdl; char *src_base=nullptr; uint32_t src_size=320; NNTI_buffer_t my_q_hdl; char *my_q_base=nullptr; uint32_t my_q_size=3200; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); t->alloc(src_size, @@ -163,15 +160,12 @@ TEST_F(NntiQueueSendTest2, start1) { } else { NNTI_event_queue_t eq; NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t src_hdl; char *src_base=nullptr; uint32_t src_size=320; NNTI_buffer_t my_q_hdl; char *my_q_base=nullptr; uint32_t my_q_size=3200; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; // give the server a chance to startup // nnti::util::sleep(1000); diff --git a/tests/nnti/cpp-api/RdmaAlignmentTest.cpp b/tests/nnti/cpp-api/RdmaAlignmentTest.cpp new file mode 100644 index 0000000..487aa4b --- /dev/null +++ b/tests/nnti/cpp-api/RdmaAlignmentTest.cpp @@ -0,0 +1,545 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +/* + * This test only applies to the libugni transport. In addition, + * the Gemini and Aries NICs have different alignment requirements. + * We don't pretend that this is a generic test for any and all + * transports. Instead, this test will immediately return a PASS + * status if it's run with any other transport. + * + * Determining Gemini vs Aries will be done by calling GNI_GetDeviceType(). + * + */ + + + +#include "nnti/nnti_pch.hpp" + +#include + +#include "gtest/gtest.h" + +#include "nnti/nntiConfig.h" + +#if (NNTI_BUILD_UGNI == 1) +#include +#endif // NNTI_BUILD_UGNI + +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "nnti/nnti_logger.hpp" + +#include "nnti/nnti_util.hpp" + +#include "nnti/nnti_transport.hpp" +#include "nnti/nnti_buffer.hpp" +#include "nnti/nnti_wid.hpp" +#include "nnti/transport_factory.hpp" + +#include "test_utils.hpp" + +using namespace std; +using namespace faodel; + +string default_config_string = R"EOF( +# default to using mpi, but allow override in config file pointed to by CONFIG +nnti.transport.name mpi +)EOF"; + +const uint32_t blocksize=8192; + + + +class NntiRdmaAlignmentTest : public testing::Test { +protected: + Configuration config; + + nnti::transports::transport *t=nullptr; + + int mpi_rank, mpi_size; + int root_rank; + + char server_url[1][NNTI_URL_LEN]; + const uint32_t num_servers = 1; + uint32_t num_clients; + bool i_am_server = false; + + void SetUp () override { + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + root_rank = 0; + config = Configuration(default_config_string); + config.AppendFromReferences(); + + MPI_Barrier(MPI_COMM_WORLD); + + test_setup(0, + NULL, + config, + "RdmaAlignmentTest", + server_url, + mpi_size, + mpi_rank, + num_servers, + num_clients, + i_am_server, + t); + } + + void TearDown () override { + NNTI_result_t nnti_rc = NNTI_OK; + bool init; + + init = t->initialized(); + EXPECT_TRUE(init); + + if (init) { + nnti_rc = t->stop(); + EXPECT_EQ(nnti_rc, NNTI_OK); + } + + bootstrap::Finish(); + } +}; + +TEST_F(NntiRdmaAlignmentTest, AlignedLength) { + NNTI_result_t rc; + + NNTI_peer_t peer_hdl; + + nnti::datatype::nnti_event_callback null_cb(t); + nnti::datatype::nnti_event_callback func_cb(t, cb_func); + nnti::datatype::nnti_event_callback obj_cb(t, callback()); + + // + // Does the transport have alignment requirements? + // + if (t->id() != NNTI_TRANSPORT_UGNI) { + return; + } + + if (i_am_server) { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + char *buf_base=nullptr; + uint32_t buf_size=3200; + + rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + t->alloc(blocksize, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, func_cb, nullptr, &buf_base, &buf_hdl); + + MPI_Barrier(MPI_COMM_WORLD); + + NNTI_buffer_t target_hdl; + NNTI_buffer_t ack_hdl; + NNTI_peer_t peer_hdl; + + rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &target_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &ack_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + // Attempt a transfer of 'blocksize' bytes, so it should PASS. + nnti::datatype::nnti_event_callback obj_cb(t, callback()); + rc = get_data_async(t, target_hdl, 0, buf_hdl, 0, blocksize, peer_hdl, obj_cb, nullptr); + EXPECT_EQ(rc, NNTI_OK); + rc = wait_data(t, eq); + EXPECT_EQ(rc, NNTI_OK); + EXPECT_TRUE(verify_buffer(buf_base, 0, blocksize, blocksize)); + + rc = send_ack(t, buf_hdl, ack_hdl, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + } else { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + NNTI_buffer_t ack_hdl; + char *buf_base=nullptr; + char *ack_base=nullptr; + uint32_t buf_size=3200; + uint32_t ack_size=320; + + // give the server a chance to startup + MPI_Barrier(MPI_COMM_WORLD); + + rc = t->connect(server_url[0], 1000, &peer_hdl); + rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + rc = t->alloc(blocksize, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + obj_cb, + nullptr, + &buf_base, + &buf_hdl); + rc = t->alloc(320, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + obj_cb, + nullptr, + &ack_base, + &ack_hdl); + + NNTI_peer_t recv_peer; + + rc = send_hdl(t, buf_hdl, buf_base, buf_size, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = populate_buffer(t, 0, blocksize, 0, buf_hdl, buf_base, blocksize); + EXPECT_EQ(rc, NNTI_OK); + + rc = send_hdl(t, ack_hdl, ack_base, ack_size, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = recv_ack(t, ack_hdl, &recv_peer, eq); + EXPECT_EQ(rc, NNTI_OK); + + t->disconnect(peer_hdl); + } + + MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_F(NntiRdmaAlignmentTest, MisalignedLength) { + NNTI_result_t rc; + + NNTI_peer_t peer_hdl; + + nnti::datatype::nnti_event_callback null_cb(t); + nnti::datatype::nnti_event_callback func_cb(t, cb_func); + nnti::datatype::nnti_event_callback obj_cb(t, callback()); + + // + // Does the transport have alignment requirements? + // + if (t->id() != NNTI_TRANSPORT_UGNI) { + return; + } + + if (i_am_server) { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + char *buf_base=nullptr; + uint32_t buf_size=3200; + + rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + t->alloc(blocksize, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, func_cb, nullptr, &buf_base, &buf_hdl); + + MPI_Barrier(MPI_COMM_WORLD); + + NNTI_buffer_t target_hdl; + NNTI_buffer_t ack_hdl; + NNTI_peer_t peer_hdl; + + rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &target_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &ack_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + // Attempt a transfer of 'blocksize-1' bytes which is NOT 4-byte aligned. + // If NNTI_ENABLE_ARGS_CHECKING is defined, + // args checking should FAIL this test with NNTI_EALIGN. + nnti::datatype::nnti_event_callback obj_cb(t, callback()); + rc = get_data_async(t, target_hdl, 0, buf_hdl, 0, blocksize-1, peer_hdl, obj_cb, nullptr); + EXPECT_EQ(rc, NNTI_EALIGN); + + rc = send_ack(t, buf_hdl, ack_hdl, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + } else { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + NNTI_buffer_t ack_hdl; + char *buf_base=nullptr; + char *ack_base=nullptr; + uint32_t buf_size=3200; + uint32_t ack_size=320; + + // give the server a chance to startup + MPI_Barrier(MPI_COMM_WORLD); + + rc = t->connect(server_url[0], 1000, &peer_hdl); + rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + rc = t->alloc(blocksize, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + obj_cb, + nullptr, + &buf_base, + &buf_hdl); + rc = t->alloc(320, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + obj_cb, + nullptr, + &ack_base, + &ack_hdl); + + NNTI_peer_t recv_peer; + + rc = send_hdl(t, buf_hdl, buf_base, buf_size, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = populate_buffer(t, 0, blocksize, 0, buf_hdl, buf_base, blocksize); + EXPECT_EQ(rc, NNTI_OK); + + rc = send_hdl(t, ack_hdl, ack_base, ack_size, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = recv_ack(t, ack_hdl, &recv_peer, eq); + EXPECT_EQ(rc, NNTI_OK); + + t->disconnect(peer_hdl); + } + + MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_F(NntiRdmaAlignmentTest, MisalignedLocalAddress) { + NNTI_result_t rc; + + NNTI_peer_t peer_hdl; + + nnti::datatype::nnti_event_callback null_cb(t); + nnti::datatype::nnti_event_callback func_cb(t, cb_func); + nnti::datatype::nnti_event_callback obj_cb(t, callback()); + + // + // Does the transport have alignment requirements? + // + if (t->id() != NNTI_TRANSPORT_UGNI) { + return; + } + + NNTI_result_t expected_result = NNTI_OK; +#if (NNTI_BUILD_UGNI == 1) + gni_nic_device_t dev_type; + GNI_GetDeviceType(&dev_type); + if (dev_type == GNI_DEVICE_GEMINI) { + // Gemini requires 4-byte alignment of the local address + expected_result = NNTI_EALIGN; + } else { + // Aries doesn't require 4-byte alignment of the local address + expected_result = NNTI_OK; + } +#endif + + if (i_am_server) { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + char *buf_base=nullptr; + uint32_t buf_size=3200; + + rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + t->alloc(blocksize, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, func_cb, nullptr, &buf_base, &buf_hdl); + + MPI_Barrier(MPI_COMM_WORLD); + + NNTI_buffer_t target_hdl; + NNTI_buffer_t ack_hdl; + NNTI_peer_t peer_hdl; + + rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &target_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &ack_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + // Attempt a transfer with a local offset of 1 byte which is NOT 4-byte aligned. + // If NNTI_ENABLE_ARGS_CHECKING is defined, + // args checking should FAIL this test with NNTI_EALIGN. + nnti::datatype::nnti_event_callback obj_cb(t, callback()); + rc = get_data_async(t, target_hdl, 0, buf_hdl, 1, blocksize-4, peer_hdl, obj_cb, nullptr); + EXPECT_EQ(rc, expected_result); + + rc = send_ack(t, buf_hdl, ack_hdl, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + } else { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + NNTI_buffer_t ack_hdl; + char *buf_base=nullptr; + char *ack_base=nullptr; + uint32_t buf_size=3200; + uint32_t ack_size=320; + + // give the server a chance to startup + MPI_Barrier(MPI_COMM_WORLD); + + rc = t->connect(server_url[0], 1000, &peer_hdl); + rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + rc = t->alloc(blocksize, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + obj_cb, + nullptr, + &buf_base, + &buf_hdl); + rc = t->alloc(320, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + obj_cb, + nullptr, + &ack_base, + &ack_hdl); + + NNTI_peer_t recv_peer; + + rc = send_hdl(t, buf_hdl, buf_base, buf_size, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = populate_buffer(t, 0, blocksize, 0, buf_hdl, buf_base, blocksize); + EXPECT_EQ(rc, NNTI_OK); + + rc = send_hdl(t, ack_hdl, ack_base, ack_size, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = recv_ack(t, ack_hdl, &recv_peer, eq); + EXPECT_EQ(rc, NNTI_OK); + + t->disconnect(peer_hdl); + } + + MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_F(NntiRdmaAlignmentTest, MisalignedRemoteAddress) { + NNTI_result_t rc; + + NNTI_peer_t peer_hdl; + + nnti::datatype::nnti_event_callback null_cb(t); + nnti::datatype::nnti_event_callback func_cb(t, cb_func); + nnti::datatype::nnti_event_callback obj_cb(t, callback()); + + // + // Does the transport have alignment requirements? + // + if (t->id() != NNTI_TRANSPORT_UGNI) { + return; + } + + if (i_am_server) { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + char *buf_base=nullptr; + uint32_t buf_size=3200; + + rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + t->alloc(blocksize, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, func_cb, nullptr, &buf_base, &buf_hdl); + + MPI_Barrier(MPI_COMM_WORLD); + + NNTI_buffer_t target_hdl; + NNTI_buffer_t ack_hdl; + NNTI_peer_t peer_hdl; + + rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &target_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &ack_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + // Attempt a transfer with a remote offset of 1 byte which is NOT 4-byte aligned. + // If NNTI_ENABLE_ARGS_CHECKING is defined, + // args checking should FAIL this test with NNTI_EALIGN. + nnti::datatype::nnti_event_callback obj_cb(t, callback()); + rc = get_data_async(t, target_hdl, 1, buf_hdl, 0, blocksize-4, peer_hdl, obj_cb, nullptr); + EXPECT_EQ(rc, NNTI_EALIGN); + + rc = send_ack(t, buf_hdl, ack_hdl, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + } else { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + NNTI_buffer_t ack_hdl; + char *buf_base=nullptr; + char *ack_base=nullptr; + uint32_t buf_size=3200; + uint32_t ack_size=320; + + // give the server a chance to startup + MPI_Barrier(MPI_COMM_WORLD); + + rc = t->connect(server_url[0], 1000, &peer_hdl); + rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + rc = t->alloc(blocksize, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + obj_cb, + nullptr, + &buf_base, + &buf_hdl); + rc = t->alloc(320, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + obj_cb, + nullptr, + &ack_base, + &ack_hdl); + + NNTI_peer_t recv_peer; + + rc = send_hdl(t, buf_hdl, buf_base, buf_size, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = populate_buffer(t, 0, blocksize, 0, buf_hdl, buf_base, blocksize); + EXPECT_EQ(rc, NNTI_OK); + + rc = send_hdl(t, ack_hdl, ack_base, ack_size, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); + + rc = recv_ack(t, ack_hdl, &recv_peer, eq); + EXPECT_EQ(rc, NNTI_OK); + + t->disconnect(peer_hdl); + } + + MPI_Barrier(MPI_COMM_WORLD); +} + +int main(int argc, char *argv[]) +{ + int rc=0; + + ::testing::InitGoogleTest(&argc, argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + +#if (NNTI_BUILD_UGNI == 1) + int mpi_rank,mpi_size; + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + EXPECT_EQ(2, mpi_size); + assert(2==mpi_size); + + rc = RUN_ALL_TESTS(); + cout <<"Tester completed all tests.\n"; + + MPI_Barrier(MPI_COMM_WORLD); +#else + std::cerr << + "----------------------------------------------------------------------\n" + "This test only runs when NNTI_BUILD_UGNI is defined.\n" + "This test will PASS.\n" + "----------------------------------------------------------------------\n" + << std::endl; +#endif // NNTI_BUILD_UGNI + + MPI_Finalize(); + + return (rc); +} diff --git a/tests/nnti/cpp-api/RdmaLengthTest.cpp b/tests/nnti/cpp-api/RdmaLengthTest.cpp index a560f75..cb46f77 100644 --- a/tests/nnti/cpp-api/RdmaLengthTest.cpp +++ b/tests/nnti/cpp-api/RdmaLengthTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -31,6 +31,7 @@ #include "nnti/nnti_wid.hpp" #include "nnti/transport_factory.hpp" +#include "test_env.hpp" #include "test_utils.hpp" using namespace std; @@ -46,197 +47,244 @@ nnti.transport.name mpi const uint32_t blocksize=8192; -class NntiRdmaOpTest : public testing::Test { +class NntiRdmaLengthTest : public testing::Test { protected: - Configuration config; + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(NntiRdmaLengthTest, simple) { + NNTI_result_t rc; + + NNTI_peer_t peer_hdl; - nnti::transports::transport *t=nullptr; + nnti::datatype::nnti_event_callback null_cb(FaodelNntiTests::globals->t); + nnti::datatype::nnti_event_callback func_cb(FaodelNntiTests::globals->t, cb_func); + nnti::datatype::nnti_event_callback obj_cb(FaodelNntiTests::globals->t, callback()); - int mpi_rank, mpi_size; - int root_rank; + if (FaodelNntiTests::globals->i_am_server) { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + char *buf_base=nullptr; + uint32_t buf_size=3200; - char server_url[1][NNTI_URL_LEN]; - const uint32_t num_servers = 1; - uint32_t num_clients; - bool i_am_server = false; + rc = FaodelNntiTests::globals->t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); - void SetUp () override { - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); - root_rank = 0; - config = Configuration(default_config_string); - config.AppendFromReferences(); + FaodelNntiTests::globals->t->alloc(blocksize, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, func_cb, nullptr, &buf_base, &buf_hdl); MPI_Barrier(MPI_COMM_WORLD); - test_setup(0, - NULL, - config, - "RdmaOpTest", - server_url, - mpi_size, - mpi_rank, - num_servers, - num_clients, - i_am_server, - t); - } + NNTI_buffer_t target_hdl; + NNTI_buffer_t ack_hdl; + NNTI_peer_t peer_hdl; - void TearDown () override { - NNTI_result_t nnti_rc = NNTI_OK; - bool init; + rc = recv_hdl(FaodelNntiTests::globals->t, buf_hdl, buf_base, buf_size, &target_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); - init = t->initialized(); - EXPECT_TRUE(init); + rc = recv_hdl(FaodelNntiTests::globals->t, buf_hdl, buf_base, buf_size, &ack_hdl, &peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); - if (init) { - nnti_rc = t->stop(); - EXPECT_EQ(nnti_rc, NNTI_OK); - } - } -}; + // We start off with a test that transfers blocksize bytes, so it should PASS. + nnti::datatype::nnti_event_callback obj_cb(FaodelNntiTests::globals->t, callback()); + rc = get_data_async(FaodelNntiTests::globals->t, target_hdl, 0, buf_hdl, 0, blocksize, peer_hdl, obj_cb, nullptr); + EXPECT_EQ(rc, NNTI_OK); + rc = wait_data(FaodelNntiTests::globals->t, eq); + EXPECT_EQ(rc, NNTI_OK); + EXPECT_TRUE(verify_buffer(buf_base, 0, blocksize, blocksize)); -TEST_F(NntiRdmaOpTest, start1) { - NNTI_result_t rc; + rc = populate_buffer(FaodelNntiTests::globals->t, 0, blocksize, 0, buf_hdl, buf_base, blocksize); + EXPECT_EQ(rc, NNTI_OK); - NNTI_peer_t peer_hdl; + rc = put_data_async(FaodelNntiTests::globals->t, buf_hdl, 0, target_hdl, 0, blocksize, peer_hdl, obj_cb, nullptr); + EXPECT_EQ(rc, NNTI_OK); + rc = wait_data(FaodelNntiTests::globals->t, eq); + EXPECT_EQ(rc, NNTI_OK); - nnti::datatype::nnti_event_callback null_cb(t); - nnti::datatype::nnti_event_callback func_cb(t, cb_func); - nnti::datatype::nnti_event_callback obj_cb(t, callback()); + rc = send_ack(FaodelNntiTests::globals->t, buf_hdl, ack_hdl, peer_hdl, eq); + EXPECT_EQ(rc, NNTI_OK); - if (i_am_server) { + } else { NNTI_event_queue_t eq; - NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t buf_hdl; + NNTI_buffer_t ack_hdl; char *buf_base=nullptr; + char *ack_base=nullptr; uint32_t buf_size=3200; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; - - rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); - t->alloc(blocksize, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, func_cb, nullptr, &buf_base, &buf_hdl); + uint32_t ack_size=320; + // give the server a chance to startup MPI_Barrier(MPI_COMM_WORLD); - NNTI_buffer_t target_hdl; - NNTI_buffer_t ack_hdl; - NNTI_peer_t peer_hdl; + rc = FaodelNntiTests::globals->t->connect(FaodelNntiTests::globals->server_url[0], 1000, &peer_hdl); + rc = FaodelNntiTests::globals->t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + rc = FaodelNntiTests::globals->t->alloc(blocksize, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, obj_cb, nullptr, &buf_base, &buf_hdl); + rc = FaodelNntiTests::globals->t->alloc(320, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, obj_cb, nullptr, &ack_base, &ack_hdl); + + NNTI_peer_t recv_peer; - rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &target_hdl, &peer_hdl, eq); + rc = send_hdl(FaodelNntiTests::globals->t, buf_hdl, buf_base, buf_size, peer_hdl, eq); EXPECT_EQ(rc, NNTI_OK); - rc = recv_hdl(t, buf_hdl, buf_base, buf_size, &ack_hdl, &peer_hdl, eq); + rc = populate_buffer(FaodelNntiTests::globals->t, 0, blocksize, 0, buf_hdl, buf_base, blocksize); EXPECT_EQ(rc, NNTI_OK); - // We start off with a test that transfers blocksize bytes, so it should PASS. - nnti::datatype::nnti_event_callback obj_cb(t, callback()); - rc = get_data_async(t, target_hdl, 0, buf_hdl, 0, blocksize, peer_hdl, obj_cb, nullptr); + rc = send_hdl(FaodelNntiTests::globals->t, ack_hdl, ack_base, ack_size, peer_hdl, eq); EXPECT_EQ(rc, NNTI_OK); - rc = wait_data(t, eq); + + rc = recv_ack(FaodelNntiTests::globals->t, ack_hdl, &recv_peer, eq); EXPECT_EQ(rc, NNTI_OK); + EXPECT_TRUE(verify_buffer(buf_base, 0, blocksize, blocksize)); - rc = populate_buffer(t, 0, blocksize, 0, buf_hdl, buf_base, blocksize); - EXPECT_EQ(rc, NNTI_OK); + FaodelNntiTests::globals->t->disconnect(peer_hdl); + } + + MPI_Barrier(MPI_COMM_WORLD); +} + +TEST_F(NntiRdmaLengthTest, out_of_bounds) { + NNTI_result_t rc; + + NNTI_peer_t peer_hdl; + +#ifndef NNTI_ENABLE_ARGS_CHECKING + if (FaodelNntiTests::globals->t->id() == NNTI_TRANSPORT_UGNI) { + /* + * NNTI is not checking args because it is too expensive. + * If the Faodel UGNI transport was built with one of the + * hugepages modules loaded, UGNI will pin the entire hugepage + * and let you RDMA to/from any region in that hugepage even + * if that is outside the actual region given in the + * GNI_RegisterMemory() call. Therefore, we skip this + * test when args checking is disabled and the transport is + * UGNI. + */ + return; + } +#endif + + nnti::datatype::nnti_event_callback null_cb(FaodelNntiTests::globals->t); + nnti::datatype::nnti_event_callback func_cb(FaodelNntiTests::globals->t, cb_func); + nnti::datatype::nnti_event_callback obj_cb(FaodelNntiTests::globals->t, callback()); + + if (FaodelNntiTests::globals->i_am_server) { + NNTI_event_queue_t eq; + NNTI_buffer_t buf_hdl; + char *buf_base=nullptr; + uint32_t buf_size=3200; + + rc = FaodelNntiTests::globals->t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + + char *heap_buffer = (char*)malloc(8*blocksize); + FaodelNntiTests::globals->t->register_memory(heap_buffer, + blocksize, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + func_cb, + nullptr, + &buf_hdl); + + MPI_Barrier(MPI_COMM_WORLD); + + NNTI_buffer_t target_hdl; + NNTI_buffer_t ack_hdl; + NNTI_peer_t peer_hdl; - rc = put_data_async(t, buf_hdl, 0, target_hdl, 0, blocksize, peer_hdl, obj_cb, nullptr); + rc = recv_hdl(FaodelNntiTests::globals->t, buf_hdl, heap_buffer, buf_size, &target_hdl, &peer_hdl, eq); EXPECT_EQ(rc, NNTI_OK); - rc = wait_data(t, eq); + + rc = recv_hdl(FaodelNntiTests::globals->t, buf_hdl, heap_buffer, buf_size, &ack_hdl, &peer_hdl, eq); EXPECT_EQ(rc, NNTI_OK); + // We start off with a test that transfers blocksize bytes, so it should PASS. + nnti::datatype::nnti_event_callback obj_cb(FaodelNntiTests::globals->t, callback()); // Next we do a test that transfers blocksize*2 bytes. // In a debug build, args checking should FAIL this test with NNTI_EMSGSIZE. // In a release build, args checking is disabled so the failure could occur immediately or in an event. - rc = get_data_async(t, target_hdl, 0, buf_hdl, 0, blocksize*2, peer_hdl, obj_cb, nullptr); + rc = get_data_async(FaodelNntiTests::globals->t, target_hdl, 0, buf_hdl, 0, blocksize*2, peer_hdl, obj_cb, nullptr); #ifdef NNTI_ENABLE_ARGS_CHECKING + // NNTI is doing bounds checking of the RDMA arguments, so expect an immediate failure EXPECT_EQ(rc, NNTI_EMSGSIZE); #else // We can't do an EXPECT_*() test here because the failure could happen now or later in an event. -// EXPECT_EQ(rc, NNTI_OK); if (rc == NNTI_OK) { NNTI_event_t event; // get_data_async() succeeded, so we need to wait for an event. - rc = wait_data(t, eq, &event); + rc = wait_data(FaodelNntiTests::globals->t, eq, &event); // we expect the the wait() to succeed. EXPECT_EQ(rc, NNTI_OK); // we expect event.result to be !NNTI_OK. EXPECT_NE(event.result, NNTI_OK); - // - EXPECT_TRUE(verify_buffer(buf_base, 0, blocksize, blocksize)); } else { // the get() failed, so we have nothing else to do here. } #endif - rc = populate_buffer(t, 0, blocksize, 0, buf_hdl, buf_base, blocksize); + rc = populate_buffer(FaodelNntiTests::globals->t, 0, 2*blocksize, 0, buf_hdl, heap_buffer, blocksize); EXPECT_EQ(rc, NNTI_OK); - rc = put_data_async(t, buf_hdl, 0, target_hdl, 0, blocksize*2, peer_hdl, obj_cb, nullptr); + rc = put_data_async(FaodelNntiTests::globals->t, buf_hdl, 0, target_hdl, 0, blocksize*2, peer_hdl, obj_cb, nullptr); #ifdef NNTI_ENABLE_ARGS_CHECKING + // NNTI is doing bounds checking of the RDMA arguments, so expect an immediate failure EXPECT_EQ(rc, NNTI_EMSGSIZE); #else - // We can't do an EXPECT_*() test here because the failure could happen now or later in an event. -// EXPECT_EQ(rc, NNTI_OK); if (rc == NNTI_OK) { NNTI_event_t event; // put_data_async() succeeded, so we need to wait for an event. - rc = wait_data(t, eq, &event); + rc = wait_data(FaodelNntiTests::globals->t, eq, &event); // we expect the the wait() to succeed. EXPECT_EQ(rc, NNTI_OK); // we expect event.result to be !NNTI_OK. EXPECT_NE(event.result, NNTI_OK); - // - EXPECT_TRUE(verify_buffer(buf_base, 0, blocksize, blocksize)); } else { // the put() failed, so we have nothing else to do here. } #endif - rc = send_ack(t, buf_hdl, ack_hdl, peer_hdl, eq); + rc = send_ack(FaodelNntiTests::globals->t, buf_hdl, ack_hdl, peer_hdl, eq); EXPECT_EQ(rc, NNTI_OK); } else { NNTI_event_queue_t eq; - NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t buf_hdl; NNTI_buffer_t ack_hdl; char *buf_base=nullptr; char *ack_base=nullptr; uint32_t buf_size=3200; uint32_t ack_size=320; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; // give the server a chance to startup MPI_Barrier(MPI_COMM_WORLD); - rc = t->connect(server_url[0], 1000, &peer_hdl); - rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); - rc = t->alloc(blocksize, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, obj_cb, nullptr, &buf_base, &buf_hdl); - rc = t->alloc(320, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, obj_cb, nullptr, &ack_base, &ack_hdl); + rc = FaodelNntiTests::globals->t->connect(FaodelNntiTests::globals->server_url[0], 1000, &peer_hdl); + rc = FaodelNntiTests::globals->t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); + char *heap_buffer = (char*)malloc(8*blocksize); + FaodelNntiTests::globals->t->register_memory(heap_buffer, + blocksize, + (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), + eq, + obj_cb, + nullptr, + &buf_hdl); +// rc = FaodelNntiTests::globals->t->alloc(blocksize, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, obj_cb, nullptr, &buf_base, &buf_hdl); + rc = FaodelNntiTests::globals->t->alloc(320, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, obj_cb, nullptr, &ack_base, &ack_hdl); - NNTI_buffer_t target_hdl; NNTI_peer_t recv_peer; - rc = send_hdl(t, buf_hdl, buf_base, buf_size, peer_hdl, eq); + rc = send_hdl(FaodelNntiTests::globals->t, buf_hdl, heap_buffer, buf_size, peer_hdl, eq); EXPECT_EQ(rc, NNTI_OK); - rc = populate_buffer(t, 0, blocksize, 0, buf_hdl, buf_base, blocksize); + rc = populate_buffer(FaodelNntiTests::globals->t, 0, 2*blocksize, 0, buf_hdl, heap_buffer, blocksize); EXPECT_EQ(rc, NNTI_OK); - rc = send_hdl(t, ack_hdl, ack_base, ack_size, peer_hdl, eq); + rc = send_hdl(FaodelNntiTests::globals->t, ack_hdl, ack_base, ack_size, peer_hdl, eq); EXPECT_EQ(rc, NNTI_OK); - rc = recv_ack(t, ack_hdl, &recv_peer, eq); + rc = recv_ack(FaodelNntiTests::globals->t, ack_hdl, &recv_peer, eq); EXPECT_EQ(rc, NNTI_OK); - EXPECT_TRUE(verify_buffer(buf_base, 0, blocksize, blocksize)); - - t->disconnect(peer_hdl); + FaodelNntiTests::globals->t->disconnect(peer_hdl); } MPI_Barrier(MPI_COMM_WORLD); @@ -254,12 +302,13 @@ int main(int argc, char *argv[]) EXPECT_EQ(2, mpi_size); assert(2==mpi_size); + // Create and register an Environment object + testing::Environment* const faodel_env = + testing::AddGlobalTestEnvironment(new FaodelNntiTests::Environment("RdmaLengthTest", default_config_string)); + int rc = RUN_ALL_TESTS(); cout <<"Tester completed all tests.\n"; - MPI_Barrier(MPI_COMM_WORLD); - bootstrap::Finish(); - MPI_Finalize(); return (rc); diff --git a/tests/nnti/cpp-api/RdmaOpTest.cpp b/tests/nnti/cpp-api/RdmaOpTest.cpp index 9e257c1..f33878b 100644 --- a/tests/nnti/cpp-api/RdmaOpTest.cpp +++ b/tests/nnti/cpp-api/RdmaOpTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -107,13 +107,9 @@ TEST_F(NntiRdmaOpTest, start1) { if (i_am_server) { NNTI_event_queue_t eq; - NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t buf_hdl; char *buf_base=nullptr; uint32_t buf_size=3200; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); t->alloc(blocksize*inner, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, func_cb, nullptr, &buf_base, &buf_hdl); @@ -135,27 +131,27 @@ TEST_F(NntiRdmaOpTest, start1) { } nnti::datatype::nnti_event_callback obj_cb(t, callback()); - for (int j=0;jalloc(blocksize*inner, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, obj_cb, nullptr, &buf_base, &buf_hdl); rc = t->alloc(320, (NNTI_buffer_flags_t)(NNTI_BF_LOCAL_READ|NNTI_BF_LOCAL_WRITE|NNTI_BF_REMOTE_READ|NNTI_BF_REMOTE_WRITE), eq, obj_cb, nullptr, &ack_base, &ack_hdl); - NNTI_buffer_t target_hdl; NNTI_peer_t recv_peer; rc = send_hdl(t, buf_hdl, buf_base, buf_size, peer_hdl, eq); @@ -191,7 +182,7 @@ TEST_F(NntiRdmaOpTest, start1) { log_error("RdmaOpTest", "send_target_hdl() failed: %d", rc); } - for (int i=0;i + +#include "gtest/gtest.h" + +#include "faodel-common/Common.hh" + +#include "nnti/nntiConfig.h" +#include "nnti/nnti_logger.hpp" +#include "nnti/nnti_util.hpp" +#include "nnti/nnti_transport.hpp" +#include "nnti/transport_factory.hpp" + +#include "test_utils.hpp" + +using namespace std; +using namespace faodel; + +string default_config_string = R"EOF( +# default to using mpi, but allow override in config file pointed to by CONFIG +nnti.transport.name mpi +)EOF"; + + +class NntiSelfConnectTest : public testing::Test { +protected: + Configuration config; + + nnti::transports::transport *t=nullptr; + + void SetUp () override { + config = Configuration(default_config_string); + config.AppendFromReferences(); + + test_setup(0, + NULL, + config, + "SelfConnectTest", + t); + } + + void TearDown () override { + NNTI_result_t nnti_rc = NNTI_OK; + bool init; + + init = t->initialized(); + EXPECT_TRUE(init); + + if (init) { + nnti_rc = t->stop(); + EXPECT_EQ(nnti_rc, NNTI_OK); + } + } +}; + +TEST_F(NntiSelfConnectTest, ConnectLoop) { + NNTI_result_t nnti_rc = NNTI_OK; + NNTI_peer_t peer_hdl; + + char my_url[NNTI_URL_LEN]; + t->get_url(my_url, NNTI_URL_LEN); + + for (int i=0;i<10;i++) { + nnti_rc = t->connect(&my_url[0], 1000, &peer_hdl); + EXPECT_EQ(nnti_rc, NNTI_OK); + nnti_rc = t->disconnect(peer_hdl); + EXPECT_EQ(nnti_rc, NNTI_OK); + } +} + +int main(int argc, char *argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + + int mpi_rank,mpi_size; + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + EXPECT_EQ(1, mpi_size); + assert(1==mpi_size); + + int rc = RUN_ALL_TESTS(); + cout <<"Tester completed all tests.\n"; + + bootstrap::Finish(); + + MPI_Finalize(); + + return (rc); +} diff --git a/tests/nnti/cpp-api/ShortSendTest.cpp b/tests/nnti/cpp-api/ShortSendTest.cpp index 9477439..dbf9ac4 100644 --- a/tests/nnti/cpp-api/ShortSendTest.cpp +++ b/tests/nnti/cpp-api/ShortSendTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -104,12 +104,9 @@ TEST_F(NntiShortSendTest, start1) { if (i_am_server) { NNTI_event_queue_t eq; NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t buf_hdl; char *buf_base=nullptr; uint32_t buf_size=3200; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; rc = t->eq_create(128, NNTI_EQF_UNEXPECTED, &eq); t->alloc(3200, NNTI_BF_LOCAL_WRITE, eq, func_cb, nullptr, &buf_base, &buf_hdl); @@ -151,12 +148,9 @@ TEST_F(NntiShortSendTest, start1) { } else { NNTI_event_queue_t eq; NNTI_event_t event; - NNTI_event_t result_event; - uint32_t which; NNTI_buffer_t buf_hdl; char *buf_base=nullptr; uint32_t buf_size=3200; - NNTI_work_request_t base_wr = NNTI_WR_INITIALIZER; // give the server a chance to startup MPI_Barrier(MPI_COMM_WORLD); diff --git a/tests/nnti/cpp-api/UnexpectedCallbackTest.cpp b/tests/nnti/cpp-api/UnexpectedCallbackTest.cpp index 0a931a5..7baf492 100644 --- a/tests/nnti/cpp-api/UnexpectedCallbackTest.cpp +++ b/tests/nnti/cpp-api/UnexpectedCallbackTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -72,7 +72,6 @@ class pingpong_callback { } NNTI_result_t operator() (NNTI_event_t *event, void *context) { - NNTI_result_t rc = NNTI_OK; pingpong_context *c = (pingpong_context*)context; @@ -94,7 +93,7 @@ class pingpong_callback { if (c->recv_count_ < c->volley_threshold_) { NNTI_event_t result_event; - rc = c->transport_->next_unexpected(c->pingpong_buf_->hdl, 0, &result_event); + c->transport_->next_unexpected(c->pingpong_buf_->hdl, 0, &result_event); // issue another send EXPECT_TRUE(verify_buffer((char*)result_event.start, result_event.offset, result_event.length)); @@ -103,9 +102,9 @@ class pingpong_callback { uint32_t seed = *(uint32_t*)(payload+4); // the salt seed++; - rc = populate_buffer(c->transport_, seed, 0, c->pingpong_buf_->hdl, c->pingpong_buf_->base, c->pingpong_buf_->size); + populate_buffer(c->transport_, seed, 0, c->pingpong_buf_->hdl, c->pingpong_buf_->base, c->pingpong_buf_->size); - rc = send_unexpected_async(c->transport_, c->pingpong_buf_->hdl, c->pingpong_buf_->base, c->pingpong_buf_->size, event->peer, *c->cb_, context); + send_unexpected_async(c->transport_, c->pingpong_buf_->hdl, c->pingpong_buf_->base, c->pingpong_buf_->size, event->peer, *c->cb_, context); c->recv_count_++; } else { diff --git a/tests/nnti/cpp-api/UnexpectedLongSendTest.cpp b/tests/nnti/cpp-api/UnexpectedLongSendTest.cpp index e276d01..9643723 100644 --- a/tests/nnti/cpp-api/UnexpectedLongSendTest.cpp +++ b/tests/nnti/cpp-api/UnexpectedLongSendTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -98,7 +98,8 @@ TEST_F(NntiUnexpectedLongSendTest, start1) { NNTI_peer_t peer_hdl; - uint32_t msg_size = 4096; +// uint32_t msg_size = 4096; + uint32_t msg_size = 4095; // not 4-byte aligned nnti::datatype::nnti_event_callback null_cb(t, (NNTI_event_callback_t)0); diff --git a/tests/nnti/cpp-api/UnexpectedSendTest.cpp b/tests/nnti/cpp-api/UnexpectedSendTest.cpp index d2f7e80..6a19b2e 100644 --- a/tests/nnti/cpp-api/UnexpectedSendTest.cpp +++ b/tests/nnti/cpp-api/UnexpectedSendTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -115,7 +115,7 @@ TEST_F(NntiUnexpectedSendTest, start1) { MPI_Barrier(MPI_COMM_WORLD); - for (int j=0;jeq_wait(&eq, 1, 1000, &which, &event); @@ -158,7 +158,7 @@ TEST_F(NntiUnexpectedSendTest, start1) { base_wr.remote_offset = 0; base_wr.length = 320; - for (int j=0;jalloc(3200, NNTI_BF_LOCAL_WRITE, (NNTI_event_queue_t)0, null_cb, nullptr, &dst_base, &dst_buf); - for (int j=0;j + +#include "gtest/gtest.h" + +#include "nnti/nntiConfig.h" + +#include "nnti/nnti_logger.hpp" + +#include "nnti/nnti_transport.hpp" +#include "nnti/nnti_buffer.hpp" +#include "nnti/nnti_wid.hpp" +#include "nnti/transport_factory.hpp" + +#include "faodel-common/Bootstrap.hh" + +#include "test_utils.hpp" + +using namespace std; +using namespace faodel; + +// Create a namespace for the NNTI tests +namespace FaodelNntiTests { + +// Define a struct to contain the global values needed a test. +// These are the common values that all the NNTI test use to +// allocate a transport, decide which ranks are servers and +// exchange server urls. +struct Globals { + Configuration config; + + nnti::transports::transport *t=nullptr; + + int mpi_rank, mpi_size; + int root_rank; + + char server_url[1][NNTI_URL_LEN]; + const uint32_t num_servers = 1; + uint32_t num_clients; + bool i_am_server = false; +}; +// Use a pointer so we can control when this object gets instantiated. +static struct Globals *globals=nullptr; + +// Define a subclass of the googletest Environment. +// This class gets registered with googletest in main(). +// The SetUp() and TearDown() methods get called exactly +// once before and after all the tests. +class Environment : public ::testing::Environment { +protected: + std::string logfile_basename; + std::string default_config_string; +public: + Environment(std::string logfile_basename, + std::string config_string) + : logfile_basename(logfile_basename), + default_config_string(config_string) + { } + ~Environment() override {} + + // Override this to define how to set up the environment. + void SetUp() override + { + globals = new Globals(); + + MPI_Comm_rank(MPI_COMM_WORLD, &globals->mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &globals->mpi_size); + globals->root_rank = 0; + globals->config = Configuration(default_config_string); + globals->config.AppendFromReferences(); + + MPI_Barrier(MPI_COMM_WORLD); + + // test_setup() will call bootstrap::Start() + test_setup(0, + NULL, + globals->config, + this->logfile_basename.c_str(), + globals->server_url, + globals->mpi_size, + globals->mpi_rank, + globals->num_servers, + globals->num_clients, + globals->i_am_server, + globals->t); + } + + // Override this to define how to tear down the environment. + void TearDown() override + { + NNTI_result_t nnti_rc = NNTI_OK; + bool init; + + init = globals->t->initialized(); + EXPECT_TRUE(init); + + if (init) { + nnti_rc = globals->t->stop(); + EXPECT_EQ(nnti_rc, NNTI_OK); + } + + MPI_Barrier(MPI_COMM_WORLD); + bootstrap::Finish(); + } +}; +} + +#endif /* TEST_ENV_HPP */ diff --git a/tests/nnti/cpp-api/test_utils.cpp b/tests/nnti/cpp-api/test_utils.cpp index b07c085..1d82a5e 100644 --- a/tests/nnti/cpp-api/test_utils.cpp +++ b/tests/nnti/cpp-api/test_utils.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "nnti/nnti_pch.hpp" @@ -177,7 +177,7 @@ get_rank(int &my_rank) } NNTI_result_t -find_server_urls(int num_servers, +find_server_urls(uint32_t num_servers, uint32_t my_rank, uint32_t num_procs, char *my_url, @@ -191,7 +191,7 @@ find_server_urls(int num_servers, MPI_Allgather(my_url, NNTI_URL_LEN, MPI_CHAR, all_urls, NNTI_URL_LEN, MPI_CHAR, MPI_COMM_WORLD); - for (int i=0;i @@ -315,7 +315,7 @@ TEST_F(OpboxAtomicsTest, start1) { int64_t *atomic_val = (int64_t *) atomics_target.GetDataPtr(); *atomic_val = 0; - fprintf(stdout, "atomic_val=%p\n", atomic_val); + fprintf(stdout, "atomic_val=%p\n", (void*)atomic_val); int64_t *atomic_rewind=atomic_val-32; for (int iii=0;iii<128;iii++) { fprintf(stdout, "%lx\n", atomic_rewind[iii]); diff --git a/tests/opbox/component/mpi_opbox_connect.cpp b/tests/opbox/component/mpi_opbox_connect.cpp index a615b31..a697897 100644 --- a/tests/opbox/component/mpi_opbox_connect.cpp +++ b/tests/opbox/component/mpi_opbox_connect.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/tests/opbox/component/mpi_opbox_get.cpp b/tests/opbox/component/mpi_opbox_get.cpp index 6244772..a9795c3 100644 --- a/tests/opbox/component/mpi_opbox_get.cpp +++ b/tests/opbox/component/mpi_opbox_get.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/tests/opbox/component/mpi_opbox_hello.cpp b/tests/opbox/component/mpi_opbox_hello.cpp index b261ba9..e4339ff 100644 --- a/tests/opbox/component/mpi_opbox_hello.cpp +++ b/tests/opbox/component/mpi_opbox_hello.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // // Test: mpi_opbox_hello diff --git a/tests/opbox/component/mpi_opbox_long_send.cpp b/tests/opbox/component/mpi_opbox_long_send.cpp index 5946bf4..a6df3c2 100644 --- a/tests/opbox/component/mpi_opbox_long_send.cpp +++ b/tests/opbox/component/mpi_opbox_long_send.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/tests/opbox/component/mpi_opbox_ping.cpp b/tests/opbox/component/mpi_opbox_ping.cpp index 53dd111..62de26b 100644 --- a/tests/opbox/component/mpi_opbox_ping.cpp +++ b/tests/opbox/component/mpi_opbox_ping.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // // Test: mpi_opbox_ping diff --git a/tests/opbox/component/mpi_opbox_put.cpp b/tests/opbox/component/mpi_opbox_put.cpp index fdf64cc..79e2317 100644 --- a/tests/opbox/component/mpi_opbox_put.cpp +++ b/tests/opbox/component/mpi_opbox_put.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -332,5 +332,5 @@ int main(int argc, char **argv) { bootstrap::Finish(); MPI_Finalize(); - return 0; + return rc; } diff --git a/tests/opbox/component/mpi_opbox_remote_buffer.cpp b/tests/opbox/component/mpi_opbox_remote_buffer.cpp index 0219a96..47d5b48 100644 --- a/tests/opbox/component/mpi_opbox_remote_buffer.cpp +++ b/tests/opbox/component/mpi_opbox_remote_buffer.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -84,7 +84,7 @@ TEST_F(OpboxRemoteBufferTest, start4) { EXPECT_EQ(nbr.GetLength(), ldo.GetDataSize()); nbr.TrimToLength(2560); - EXPECT_EQ(nbr.GetLength(), 2560); + EXPECT_EQ(nbr.GetLength(), 2560UL); } TEST_F(OpboxRemoteBufferTest, start5) { @@ -95,16 +95,16 @@ TEST_F(OpboxRemoteBufferTest, start5) { opbox::net::GetRdmaPtr(&ldo, &nbl, &nbr); EXPECT_NE(nbl, nullptr); - EXPECT_EQ(nbr.GetLength(), ldo.GetHeaderSize() + ldo.GetMetaSize() + ldo.GetDataSize()); + EXPECT_EQ(nbr.GetLength(), ldo.GetHeaderSize() + ldo.GetMetaSize() + ldo.GetDataSize() + ldo.GetPaddingSize()); nbr.IncreaseOffset(ldo.GetHeaderSize()); - EXPECT_EQ(nbr.GetLength(), ldo.GetMetaSize() + ldo.GetDataSize()); + EXPECT_EQ(nbr.GetLength(), ldo.GetMetaSize() + ldo.GetDataSize() + ldo.GetPaddingSize()); nbr.IncreaseOffset(ldo.GetMetaSize()); - EXPECT_EQ(nbr.GetLength(), ldo.GetDataSize()); + EXPECT_EQ(nbr.GetLength(), ldo.GetDataSize() + ldo.GetPaddingSize()); nbr.TrimToLength(2560); - EXPECT_EQ(nbr.GetLength(), 2560); + EXPECT_EQ(nbr.GetLength(), 2560UL); } TEST_F(OpboxRemoteBufferTest, start6) { @@ -112,12 +112,10 @@ TEST_F(OpboxRemoteBufferTest, start6) { opbox::net::NetBufferLocal *nbl = nullptr; opbox::net::NetBufferRemote nbr; - uint32_t offset = ldo.GetHeaderSize(); - uint32_t length = ldo.GetDataSize(); opbox::net::GetRdmaPtr(&ldo, &nbl, &nbr); EXPECT_NE(nbl, nullptr); EXPECT_EQ(nbr.GetOffset(), ldo.GetLocalHeaderSize()); - EXPECT_EQ(nbr.GetLength(), ldo.GetHeaderSize() + ldo.GetMetaSize() + ldo.GetDataSize()); + EXPECT_EQ(nbr.GetLength(), ldo.GetHeaderSize() + ldo.GetMetaSize() + ldo.GetDataSize() + ldo.GetPaddingSize()); } int main(int argc, char **argv) { diff --git a/tests/opbox/component/mpi_opbox_self_send.cpp b/tests/opbox/component/mpi_opbox_self_send.cpp index f6845b7..aebdb5e 100644 --- a/tests/opbox/component/mpi_opbox_self_send.cpp +++ b/tests/opbox/component/mpi_opbox_self_send.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/tests/opbox/component/mpi_opbox_send.cpp b/tests/opbox/component/mpi_opbox_send.cpp index 221cf75..56ae71a 100644 --- a/tests/opbox/component/mpi_opbox_send.cpp +++ b/tests/opbox/component/mpi_opbox_send.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/tests/opbox/component/support/Globals.cpp b/tests/opbox/component/support/Globals.cpp index 8a6a210..0e0f894 100644 --- a/tests/opbox/component/support/Globals.cpp +++ b/tests/opbox/component/support/Globals.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/tests/opbox/component/support/Globals.hh b/tests/opbox/component/support/Globals.hh index c5eecee..68ce863 100644 --- a/tests/opbox/component/support/Globals.hh +++ b/tests/opbox/component/support/Globals.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_TESTS_OPBOX_GLOBALS_HH #define FAODEL_TESTS_OPBOX_GLOBALS_HH diff --git a/tests/opbox/ops/tb_FabConnTest.cpp b/tests/opbox/ops/tb_FabConnTest.cpp index 2c4fcc0..d40b942 100644 --- a/tests/opbox/ops/tb_FabConnTest.cpp +++ b/tests/opbox/ops/tb_FabConnTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/tests/opbox/ops/tb_OpboxOpPingFabTest.cpp b/tests/opbox/ops/tb_OpboxOpPingFabTest.cpp index 16a9850..46ee904 100644 --- a/tests/opbox/ops/tb_OpboxOpPingFabTest.cpp +++ b/tests/opbox/ops/tb_OpboxOpPingFabTest.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/tests/opbox/unit/mpi_opbox_message_helpers.cpp b/tests/opbox/unit/mpi_opbox_message_helpers.cpp index d3be0c3..2cd7dc6 100644 --- a/tests/opbox/unit/mpi_opbox_message_helpers.cpp +++ b/tests/opbox/unit/mpi_opbox_message_helpers.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Test: mpi_opbox_message_helpers.cpp // Purpose: Test helper functions that convert incoming messages to outgoing @@ -16,7 +16,7 @@ #include "gtest/gtest.h" #include "faodel-common/Common.hh" -#include "faodel-common/SerializationHelpers.hh" +#include "faodel-common/SerializationHelpersBoost.hh" #include "whookie/Server.hh" #include "lunasa/Lunasa.hh" @@ -132,14 +132,6 @@ TEST_F(OpBoxMessageHelpersTest, BigStringMessage) { EXPECT_EQ(65535, s.size()); } -TEST_F(OpBoxMessageHelpersTest, BadStringMessage) { - - lunasa::DataObject ldo; - string s1 = string((64*1024), 'x'); - - EXPECT_ANY_THROW(AllocateStringMessage(ldo, src_node, dst_node, 100, 101, 2112, 0x1234, s1)); -} - TEST_F(OpBoxMessageHelpersTest, StringRequestReply) { diff --git a/tests/opbox/unit/mpi_opbox_net_init.cpp b/tests/opbox/unit/mpi_opbox_net_init.cpp index 44ef6d7..620be0d 100644 --- a/tests/opbox/unit/mpi_opbox_net_init.cpp +++ b/tests/opbox/unit/mpi_opbox_net_init.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Test: mpi_opbox_net_init // Purpose: Basic checks on opbox net (mtus are ok, nbr will serialize) @@ -11,7 +11,7 @@ #include "gtest/gtest.h" #include "faodel-common/Common.hh" -#include "faodel-common/SerializationHelpers.hh" +#include "faodel-common/SerializationHelpersBoost.hh" #include "lunasa/Lunasa.hh" #include "opbox/OpBox.hh" @@ -41,8 +41,8 @@ TEST_F(OpboxInitTest, Constants) { opbox::net::Attrs attrs; opbox::net::GetAttrs(&attrs); - EXPECT_GT(attrs.max_eager_size, 512); //Verify net can get message-sized data - EXPECT_GT(attrs.mtu, 512); //Verify net has usable packet sizes + EXPECT_GT(attrs.max_eager_size, 512UL); //Verify net can get message-sized data + EXPECT_GT(attrs.mtu, 512UL); //Verify net has usable packet sizes } @@ -73,7 +73,7 @@ TEST_F(OpboxInitTest, NBRSimpleSerialize) { opbox::net::NetBufferRemote nbr2 = faodel::BoostUnpack(s); - for(char i = 0; i #include @@ -13,7 +13,7 @@ #include "gtest/gtest.h" #include "faodel-common/Common.hh" -#include "faodel-common/SerializationHelpers.hh" +#include "faodel-common/SerializationHelpersBoost.hh" #include "whookie/Server.hh" #include "lunasa/Lunasa.hh" @@ -68,7 +68,7 @@ class OpArgsPoke : public OpArgs { class OpTrigger1 : public opbox::Op { public: - OpTrigger1(int value) : value(value), state(0), Op(true) {} + OpTrigger1(int value) : Op(true), state(0), value(value) {} OpTrigger1(op_create_as_target_t t) : Op(t) {} @@ -160,7 +160,7 @@ WaitingType OpTrigger1::UpdateOrigin(OpArgs *args) { TEST_F(OpBoxTriggerOpTest, SimplePoke) { mailbox_t mb; - uint64_t expected_val = 5; + int64_t expected_val = 5; opbox::LaunchOp(new OpTrigger1(expected_val), &mb); diff --git a/tests/opbox/unit/support/default_config_string.hh b/tests/opbox/unit/support/default_config_string.hh index 5cfd626..277b8d2 100644 --- a/tests/opbox/unit/support/default_config_string.hh +++ b/tests/opbox/unit/support/default_config_string.hh @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #ifndef FAODEL_DEFAULT_CONFIG_STRING_HH #define FAODEL_DEFAULT_CONFIG_STRING_HH diff --git a/tests/sbl/unit/tb_sbl_compile_speed.cpp b/tests/sbl/unit/tb_sbl_compile_speed.cpp index d87063f..2585751 100644 --- a/tests/sbl/unit/tb_sbl_compile_speed.cpp +++ b/tests/sbl/unit/tb_sbl_compile_speed.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include "sbl/sbl_boost_headers.hh" diff --git a/tests/sbl/unit/tb_sbl_logger_class.cpp b/tests/sbl/unit/tb_sbl_logger_class.cpp index 2c35c5a..153d61e 100644 --- a/tests/sbl/unit/tb_sbl_logger_class.cpp +++ b/tests/sbl/unit/tb_sbl_logger_class.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * SblLoggerClassTest.c diff --git a/tests/sbl/unit/tb_sbl_source_class.cpp b/tests/sbl/unit/tb_sbl_source_class.cpp index 4ec9ff4..957bd72 100644 --- a/tests/sbl/unit/tb_sbl_source_class.cpp +++ b/tests/sbl/unit/tb_sbl_source_class.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. /* * SblSourceClassTest.c diff --git a/tests/services/component/mpi_service_mpisyncstart.cpp b/tests/services/component/mpi_service_mpisyncstart.cpp index 4ad9334..46a66fb 100644 --- a/tests/services/component/mpi_service_mpisyncstart.cpp +++ b/tests/services/component/mpi_service_mpisyncstart.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // This test tests mpisync start to see if multiple nodes @@ -63,9 +63,10 @@ class GetConfig void Init(const faodel::Configuration &config) override { ConfigureLogging(config); myconfig = config; + dbg("init called"); } - void Start() override {} - void Finish() override {} + void Start() override { dbg("start called");} + void Finish() override { dbg("finish called");} void GetBootstrapDependencies(std::string &name, std::vector &requires, std::vector &optional) const override { @@ -155,6 +156,8 @@ TEST_F(BootstrapMPITest, MPISyncStartMPI) { mpisyncstart.enable true +#getconfig.debug true #Show our config interceptor getting called + dirman.root_node_mpi 0 dirman.resources_mpi[] dht:/my/all&info="booya" ALL dirman.resources_mpi[] dht:/my/single&info="single" 0 @@ -163,15 +166,17 @@ dirman.resources_mpi[] dht:/my/double&info="single" 0-middle )EOF"; test_bcastConfig(CMD_NEW_MPISYNC_START, c1); - bootstrap::Start(Configuration(c1), test_mpisync::bootstrap /*mpisyncstart::bootstrap*/); + bootstrap::Start(Configuration(c1), test_mpisync::bootstrap ); // <-- Call our special bootstrap so we can get config! + - Configuration c = test_mpisync::get_config.myconfig; + Configuration c = test_mpisync::get_config.myconfig; // <-- Use our special bootstrap to get the config everyone saw int num1, num2; vector urls_orig, urls_mod; num1 = c.GetStringVector(&urls_orig, "dirman.resources_mpi"); num2 = c.GetStringVector(&urls_mod, "dirman.resources"); EXPECT_EQ(num1, num2); EXPECT_EQ(3, num2); + if (num2 <= 3) { ResourceURL urls[3]; DirectoryInfo dir_info[3]; @@ -185,6 +190,7 @@ dirman.resources_mpi[] dht:/my/double&info="single" 0-middle EXPECT_EQ("single", urls[1].name); EXPECT_EQ("double", urls[2].name); } + } diff --git a/tests/services/unit/tb_service_backburner.cpp b/tests/services/unit/tb_service_backburner.cpp index eecb3a1..9bd8554 100644 --- a/tests/services/unit/tb_service_backburner.cpp +++ b/tests/services/unit/tb_service_backburner.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include @@ -8,8 +8,10 @@ #include #include #include -#include "gtest/gtest.h" +#include + +#include #include "faodel-common/Common.hh" #include "faodel-services/BackBurner.hh" @@ -23,10 +25,17 @@ void sleepUS(int microseconds) { //Note: Additional configuration settings will be loaded the file specified by FAODEL_CONFIG string default_config = R"EOF( -#backburner.debug true -#backburnerWorker.debug true +backburner.debug true +#backburner.worker.debug true node_role server backburner.threads 4 + +#backburner.notification_method polling + +backburner.notification_method pipe + +#backburner.notification_method sleep_polling +#backburner.sleep_polling_time 5s )EOF"; @@ -109,3 +118,20 @@ TEST_F(FaodelBackBurnerService, tags) { while(!done) {} EXPECT_EQ(2000, count); } + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + + int mpi_rank, mpi_size; + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + + if(mpi_rank==0) cout <<"Beginning tests.\n"; + int rc = RUN_ALL_TESTS(); + + MPI_Finalize(); + if(mpi_rank==0) cout <<"All complete. Exiting.\n"; + return rc; +} diff --git a/tests/whookie/CMakeLists.txt b/tests/whookie/CMakeLists.txt index 360f108..10e90c1 100644 --- a/tests/whookie/CMakeLists.txt +++ b/tests/whookie/CMakeLists.txt @@ -9,7 +9,6 @@ set(SERIAL_TEST_LIBS common GTest::GTest GTest::Main - ${EXTRA_TEST_LIBS} ) set(MPI_TEST_LIBS diff --git a/tests/whookie/component/mpi_whookie_multiprocess_threaded.cpp b/tests/whookie/component/mpi_whookie_multiprocess_threaded.cpp index f289a09..3776435 100644 --- a/tests/whookie/component/mpi_whookie_multiprocess_threaded.cpp +++ b/tests/whookie/component/mpi_whookie_multiprocess_threaded.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include diff --git a/tests/whookie/component/mpi_whookie_restart.cpp b/tests/whookie/component/mpi_whookie_restart.cpp index a98c55a..e718e71 100644 --- a/tests/whookie/component/mpi_whookie_restart.cpp +++ b/tests/whookie/component/mpi_whookie_restart.cpp @@ -1,6 +1,6 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. // Purpose: Verify whookie works properly when sender/receiver shut down // @@ -326,4 +326,5 @@ int main(int argc, char **argv) { } MPI_Finalize(); + return rc; } \ No newline at end of file diff --git a/tests/whookie/component/tb_whookie_clientserver.cpp b/tests/whookie/component/tb_whookie_clientserver.cpp index 1dcd5fd..6a01823 100644 --- a/tests/whookie/component/tb_whookie_clientserver.cpp +++ b/tests/whookie/component/tb_whookie_clientserver.cpp @@ -1,12 +1,14 @@ -// Copyright 2018 National Technology & Engineering Solutions of Sandia, -// LLC (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. #include #include #include #include +#include + #include #include "faodel-common/Common.hh" @@ -234,14 +236,22 @@ int main(int argc, char **argv){ ::testing::InitGoogleTest(&argc, argv); + int mpi_rank, mpi_size; + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + faodel::bootstrap::Start(faodel::Configuration(default_config), whookie::bootstrap ); faodel::nodeid_t nid = whookie::Server::GetNodeID(); cout <<"Whookie address: "< +#include + +#include "faodel-common/FaodelTypes.hh" +#include "faodel-common/StringHelpers.hh" + +class ActionInterface { + +public: + + ActionInterface() {} + virtual ~ActionInterface() {} + + std::string cmd; + std::string rank; + std::string error_message; + + bool Valid() { return !cmd.empty(); } + bool HasError() { return !error_message.empty(); } + + void exitOnError() { if(HasError()) { std::cerr< remaining_args; + faodel:: rc_t setError(const std::string &err) { error_message=err; return EINVAL; } + +}; + + +#endif //FAODEL_ACTION_HH diff --git a/tools/faodel-cli/CMakeLists.txt b/tools/faodel-cli/CMakeLists.txt index 0248091..63dbc59 100644 --- a/tools/faodel-cli/CMakeLists.txt +++ b/tools/faodel-cli/CMakeLists.txt @@ -1,11 +1,18 @@ set( SOURCES + all_in_one.cpp build_info.cpp config_info.cpp dirman_server.cpp + kelpie_blast.cpp kelpie_client.cpp kelpie_server.cpp - resource.cpp + KelpieBlastParams.cpp + KelpieClientAction.cpp + play.cpp + PlayAction.cpp + resource_client.cpp + ResourceAction.cpp whookie_client.cpp faodel_cli.cpp #..insert others here diff --git a/tools/faodel-cli/KelpieBlastParams.cpp b/tools/faodel-cli/KelpieBlastParams.cpp new file mode 100644 index 0000000..49e2238 --- /dev/null +++ b/tools/faodel-cli/KelpieBlastParams.cpp @@ -0,0 +1,221 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include + +#include "faodelConfig.h" +#ifdef Faodel_ENABLE_MPI_SUPPORT +#include +#endif + + +#include "faodel-common/Configuration.hh" +#include "faodel-common/ResourceURL.hh" +#include "faodel-common/Bootstrap.hh" +#include "faodel-common/StringHelpers.hh" +#include "faodel-services/MPISyncStart.hh" + +#include "kelpie/Kelpie.hh" + +#include "faodel_cli.hh" +#include "KelpieBlastParams.hh" + +using namespace std; + +KelpieBlastParams::KelpieBlastParams(const std::vector &args) + : mpi_rank(0), mpi_size(1), + timestep(0), num_timesteps(1), delay_between_timesteps_us(1000000), + k1_length(8), k2_length(8), + reuse_memory(false), + async_pubs(false), + use_rft_keys(false), + no_barrier_before_generate(false), + verbose(0), failed(false) { + + + //Parse our args + failed = (parseArgs(args) != 0); + if(failed) return; + + + faodel::Configuration config; + if(!pool_external.empty()) { + //External pool. Just get reference from user + pool_name = pool_external; + + } else { + //Internal pool. Parse string + auto vals = faodel::Split(pool_internal,':',true); + if((vals.size()<1) || (vals.size()>2)) { + cerr << "Problem parsing internal pool '"<1) { + config.Append("kelpie.debug", "true"); + config.Append("bootstrap.debug", "true"); + } + + //Launch MPI + #ifdef Faodel_ENABLE_MPI_SUPPORT + MPI_Init(nullptr, nullptr); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + global_rank=mpi_rank; + faodel::mpisyncstart::bootstrap(); + #endif + + //Launch FAODEL + faodel::bootstrap::Start(config, kelpie::bootstrap); + +} + +KelpieBlastParams::~KelpieBlastParams() { + + if(!failed) { + + #ifdef Faodel_ENABLE_MPI_SUPPORT + MPI_Barrier(MPI_COMM_WORLD); + faodel::bootstrap::Finish(); + + MPI_Barrier(MPI_COMM_WORLD); + dbg0("Finalizing"); + MPI_Finalize(); + #else + faodel::bootstrap::Finish(); + #endif + dbg0("Exiting"); + + } +} + +void KelpieBlastParams::dumpSettings(faodel::DirectoryInfo dir) { + if((failed)||(mpi_rank!=0)) return; + + cout << "# Runtime Configuration\n" + << "# mpi_size: "< &args) { + + string s_object_sizes; + int rc=0; + + struct ArgInfo { string short_name; string long_name; bool has_argument; std::function lambda; }; + vector options { + { "-a", "--async", false, [=](const string &s) { this->async_pubs = true; return 0; } }, + { "-m", "--reuse-memory", false, [=](const string &s) { this->reuse_memory = true; return 0; } }, + { "-r", "--rank-grouping", false, [=](const string &s) { this->use_rft_keys = true; return 0; } }, + { "-s", "--skip-barrier" , false, [=](const string &s) { this->no_barrier_before_generate = true; return 0; } }, + { "-t", "--timesteps", true, [=](const string &s) { return faodel::StringToUInt64(&this->num_timesteps, s); } }, + { "-T", "--delay", true, [=](const string &s) { return faodel::StringToTimeUS(&this->delay_between_timesteps_us, s); } }, + { "-o", "--objects", true, [=,&s_object_sizes](const string &s) { s_object_sizes = s; return 0; } }, + { "-p", "--external-pool", true, [=](const string &s) { this->pool_external = s; return 0; } }, + { "-P", "--internal-pool", true, [=](const string &s) { this->pool_internal = s; return 0; } } + }; + for(int i=0; i= args.size()) { + cerr << "Not enough arguments for " << option.short_name << "/" << option.long_name << endl; + return -1; + } + } + rc = option.lambda(args[i]); + if(rc != 0) { + cerr << "Problem parsing " << option.short_name << "/" << option.long_name << endl; + return -1; + } + found = true; + break; + } + } + if(!found) { + cerr <<"Unknown option "<max_object_size) max_object_size=x; + } + } + + return 0; +} + +void KelpieBlastParams::barrier() { + #ifdef Faodel_ENABLE_MPI_SUPPORT + MPI_Barrier(MPI_COMM_WORLD); + #endif +} diff --git a/tools/faodel-cli/KelpieBlastParams.hh b/tools/faodel-cli/KelpieBlastParams.hh new file mode 100644 index 0000000..daaf453 --- /dev/null +++ b/tools/faodel-cli/KelpieBlastParams.hh @@ -0,0 +1,71 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_KELPIEBLASTPARAMS_HH +#define FAODEL_KELPIEBLASTPARAMS_HH + +#include +#include +#include + +#include "faodel-common/Common.hh" + + +class KelpieBlastParams { +public: + + KelpieBlastParams() = delete; + KelpieBlastParams(const std::vector &args); + ~KelpieBlastParams(); + + + int mpi_rank; + int mpi_size; + + std::string pool_name; + + uint64_t timestep; + uint64_t num_timesteps; + uint64_t delay_between_timesteps_us; + + int k1_length; + int k2_length; + + std::vector object_sizes_per_timestep; + uint64_t max_object_size; + uint64_t bytes_per_rank_step; + + bool reuse_memory; + bool async_pubs; + bool use_rft_keys; + bool no_barrier_before_generate; + + void barrier(); + + + void dumpSettings(faodel::DirectoryInfo dir_info); + + void sleepUS(uint64_t us) { std::this_thread::sleep_for(std::chrono::microseconds(us)); } + + void SleepForComputePhase() { sleepUS(delay_between_timesteps_us); } + + + bool IsOk() { return !failed; } + +private: + + int verbose; + bool failed; + std::string pool_external; + std::string pool_internal; + + void dumpHelp(); + + int parseArgs(const std::vector &args); + + +}; + + +#endif //FAODEL_KELPIEBLASTPARAMS_HH diff --git a/tools/faodel-cli/KelpieClientAction.cpp b/tools/faodel-cli/KelpieClientAction.cpp new file mode 100644 index 0000000..b56ddd7 --- /dev/null +++ b/tools/faodel-cli/KelpieClientAction.cpp @@ -0,0 +1,268 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include "KelpieClientAction.hh" + +using namespace std; +using namespace faodel; + + + + +KelpieClientAction::KelpieClientAction(const std::string &long_or_short_cmd) + : kget_meta_only(false), + kload_all_dir_entries(false) { + + //Go through all of the commands and plug in the shorthand + vector> command_list = { + {"kelpie-put", "kput"}, + {"kelpie-get", "kget"}, + {"kelpie-get-meta", "kget"}, + {"kgetm", "kget"}, + {"kelpie-info", "kinfo"}, + {"kelpie-list", "klist"}, + {"kls", "klist"}, + {"kelpie-save", "ksave"}, + {"kelpie-load", "kload"}, + {"kelpie-load-dir", "kload"}, + {"kloadd", "kload"} + }; + + //Convert a command to its shorthand + for(auto &big_little : command_list) { + if(( long_or_short_cmd == big_little.first) || (long_or_short_cmd==big_little.second)) { + cmd=big_little.second; + break; + } + } + + //Set values for commands that are aliases + if((cmd=="kelpie-getm") || (cmd=="kgetm")) kget_meta_only=true; + if((cmd=="kelpie-loadd") || (cmd=="kloadd")) kload_all_dir_entries=true; + + + //Tag this parse as an error + if(cmd.empty()) { + error_message="Command '"+long_or_short_cmd+"' not valid"; + } + +} + + + +faodel::rc_t KelpieClientAction::appendKey(const string &k1, const string &k2) { + keys.push_back(kelpie::Key(k1,k2)); + return 0; +} +faodel::rc_t KelpieClientAction::appendKey(const string &string_separated_by_pipe) { + auto key_parts = faodel::Split(string_separated_by_pipe, '|'); + if(key_parts.size()>2) return setError("Could not parse -k/--key argument for '"+string_separated_by_pipe+"'. Can only have one '|'"); + string k1,k2; + k1=key_parts[0]; + k2=(key_parts.size()>1) ? key_parts[1] : ""; + return appendKey(k1, k2); +} + +uint16_t KelpieClientAction::getMetaCapacity() const { + uint64_t mc = meta.size(); + if(mc==0) + mc=generate_meta_size; + return mc & 0x0FFFF; +} + + +/** + * @brief Parse next option in an args list. + * @param result The value for this option + * @param args The list of args + * @param iptr Pointer to current spot in args. Will be updated if we successfully extract option + * @param s1 The simple version of this arg (eg -k) + * @param s2 The long version of this arg (eg --key) + * @retval TRUE The item matched and there was a followup arg to look at + * @retval FALSE No match OR there was match but not enough args (error was set) + */ +bool KelpieClientAction::parseArgValue(string *result, const vector &args, size_t *iptr, const string &s1, const string &s2) { + + size_t i = *iptr; + if((args[i] == s1) || (args[i] == s2)) { + i++; + if(i>=args.size()) { + setError("Could not parse "+s1+"/"+s2+": expected additional argument"); + return false; + } + if(result) *result = args[i]; + *iptr=i; + return true; + } + return false; +} + +/** + * @brief Parse a list of kelpie client arguments, validate options, and store + * @param args The list of arguments to look at + * @param default_pool The pool to use if none specified + * @retval 0 Success + * @retval EINVAL Problems parsing (see error_message for problem) + */ +rc_t KelpieClientAction::ParseArgs(const std::vector &args, const std::string &default_pool) { + + string key1, key2, tmp_key; + string rank; + string gen_meta_size, gen_data_size; + int rc; + + struct TokenEntry { string *var; string short_name; string long_name; }; + vector two_arg_tokens = { + {&pool_name, "-p", "--pool"}, + {&rank, "-r", "--rank"}, + {&file_name, "-f", "--file"}, + {&dir_name, "-d", "--dir"}, + {&gen_meta_size, "-M", "--generate-meta-size"}, + {&gen_data_size, "-D", "--generate-data-size"}, + {&meta, "-m", "--meta"} + }; + + for(size_t i=0; i64*1024) { + return setError("The -m/--meta option needs to be a string less than 64KB in size"); + } + + //Handle meta size + if(!gen_meta_size.empty()) { + if(!meta.empty()) + return setError("Both -M/--generate-meta-size and -m/--meta were defined. Only one can be specified"); + + if(cmd!="kput") + return setError("The -M/--generate-meta-size option can only be used with a kput operation"); + + //Convert to a value + rc = StringToUInt64(&generate_meta_size, gen_meta_size); + if(rc != 0) + return setError("Could not parse -M/--generate-meta-size option '"+gen_meta_size+"'"); + if(generate_meta_size >= 64*1024) + return setError("Meta data size in -M/--generate-meta-size option must be less than 64k"); + } + + + //Handle data generation + if(!gen_data_size.empty()) { + + if(!file_name.empty()) + return setError("Both -f/--file and -D/--generate-data-size were defined. Only one can be specified."); + + if(cmd!="kput") + return setError("The -D/--generate-data-size option can only be used on a kput operation"); + + //Convert to a value + rc = StringToUInt64(&generate_data_size, gen_data_size); + if(rc != 0) + return setError("Could not parse -D/--generate-data-size option '"+gen_data_size+"'"); + + } + + //Validate we got the right number of keys + if(keys.size()>1) { + if(cmd!="kinfo") + return setError("Multiple keys supplied to "+cmd+", which only accepts one key."); + } //note: don't check for 0 keys. Some commands like list default to adding "*" key + + + //Validate we got a pool, or plug the default pool in + if(pool_name.empty()) { + if(default_pool.empty()) + return setError("No pool provided. Use -p/--pool or set env var 'FAODEL_POOL'"); + pool_name = default_pool; + } + + //Validate dir specified in kload/ksave + if((cmd=="kload") || (cmd=="ksave")) { + if(dir_name.empty()) + return setError("The -d/--dir option must be defined for the "+cmd+" command"); + + //Try creating now to catch errors early + struct stat results; + int rc = stat(dir_name.c_str(), &results); + if(rc != 0) { + //Missing. try to create + rc = mkdir(dir_name.c_str(), S_IRWXU | S_IRWXG); + if(rc != 0) + return setError("Could not create directory '"+dir_name+"' for "+cmd); + + } else if(!(results.st_mode & S_IFDIR)) { + return setError("Can't use --dir '"+dir_name+"' in command "+cmd+" because it is not a directory"); + } + + } else { + if(!dir_name.empty()) + return setError("The -d/--dir option can only be defined for the kload/ksave commands"); + } + + + //Kput/kget don't allow extra options, but + if((remaining_args.size()>0) && ((cmd=="kput")||(cmd=="kget"))) { + return setError("Had extra arguments, starting with '"+remaining_args[0]+"'"); + } + + //All done + return 0; +} \ No newline at end of file diff --git a/tools/faodel-cli/KelpieClientAction.hh b/tools/faodel-cli/KelpieClientAction.hh new file mode 100644 index 0000000..f79c085 --- /dev/null +++ b/tools/faodel-cli/KelpieClientAction.hh @@ -0,0 +1,50 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_KELPIECLIENTACTION_HH +#define FAODEL_KELPIECLIENTACTION_HH + +#include +#include "kelpie/Kelpie.hh" + +#include "ActionInterface.hh" + +class KelpieClientAction + : public ActionInterface { +public: + KelpieClientAction() : kget_meta_only(false), kload_all_dir_entries(false) {} + KelpieClientAction(const std::string &long_or_short_cmd); + + std::string pool_name; + std::string file_name; + std::string dir_name; + + uint64_t generate_meta_size; + uint64_t generate_data_size; + + std::string meta; + bool kget_meta_only; + bool kload_all_dir_entries; + + std::vector keys; + + uint16_t getMetaCapacity() const; + + faodel::rc_t ParseArgs(const std::vector &args, const std::string &default_pool); + + +private: + faodel::rc_t appendKey(const std::string &k1, const std::string &k2); + faodel::rc_t appendKey(const std::string &string_separated_by_pipe); + + bool parseArgValue(std::string *result, const std::vector &args, size_t *iptr, const std::string &s1, const std::string &s2); + + +}; + + + + + +#endif //FAODEL_KELPIECLIENTACTION_HH diff --git a/tools/faodel-cli/PlayAction.cpp b/tools/faodel-cli/PlayAction.cpp new file mode 100644 index 0000000..a392c29 --- /dev/null +++ b/tools/faodel-cli/PlayAction.cpp @@ -0,0 +1,123 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include + +#include "faodel-common/StringHelpers.hh" + +#include "PlayAction.hh" + +using namespace std; + + + +int PlayAction::emsg(const string &condition) { + error_message = condition; + return EINVAL; +} + + +int PlayAction::parseCommandLine(const string &my_rank, string *default_pool, string *default_rank, + const string &file_tag, const string &command_line) { + + + + //Remove empty lines + vector tokens = faodel::Split(command_line, ' ', true); + if(tokens.empty()) return ENOENT; //Nothing to parse. Just skip + + //Set the args + filename_line = file_tag; + command = faodel::ToLowercase(tokens[0]); + vector kargs; + if(tokens.size()>1) { + kargs = {tokens.begin()+1, tokens.end()}; + } + + //Check for rank specification + bool found_rank_option=false; + for(int i=0; i=kargs.size()) return emsg("Didn't have enough arguments for -r flag?"); + if((my_rank.empty()) || (kargs[i+1]==my_rank)) { + //Hit: just delete these two spots + kargs.erase(kargs.begin()+i, kargs.begin()+i+2 ); + found_rank_option=true; + } else { + //Miss: just bail + return ENOENT; + } + break; + } + } + + //Deal with exit + if(command=="exit") { + return EAGAIN; + } + + + //Handle set. These should just override the default settings for later actions + if(command == "set") { + if(kargs.size() != 2) return emsg("Set needs two arguments"); + if(kargs[0] == "pool") { + *default_pool = kargs[1]; + return ENOENT; + } + if(kargs[0] == "rank") { + *default_rank = kargs[1]; + return ENOENT; + } + return emsg("Did not recognize set '"+kargs[0]+"'"); + } + + + //Everyone listens to barrier + if(command == "barrier") { + args.emplace_back("barrier"); + return 0; + } + + + //If not -r specified, see if our rank matches the default rank + if((!found_rank_option) && (*default_rank != my_rank)) { + return ENOENT; + } + + //Allow printing of static text line + if(command == "print") { + args.emplace_back((command_line.length() > 5) ? command_line.substr(6) : ""); + return 0; + } + + + //Delays + if((command == "delay") || (command=="delayfor")) { + if(kargs.size() != 1) return emsg("delay needs exactly one argument"); + args.emplace_back("delayfor"); + args.emplace_back(kargs[0]); + return 0; + } + + //Check resource commands + resource_action = ResourceAction(command); + if(resource_action.Valid()) { + resource_action.ParseArgs(kargs); + if(kelpie_action.HasError()) return emsg(resource_action.error_message); + return 0; + } + + //Check kelpie commands + kelpie_action = KelpieClientAction(command); + if(kelpie_action.Valid()) { + kelpie_action.ParseArgs(kargs, *default_pool); + if(kelpie_action.HasError()) return emsg(kelpie_action.error_message); + return 0; + } + + + return emsg("Unknown command?"); +} + diff --git a/tools/faodel-cli/PlayAction.hh b/tools/faodel-cli/PlayAction.hh new file mode 100644 index 0000000..1db651f --- /dev/null +++ b/tools/faodel-cli/PlayAction.hh @@ -0,0 +1,36 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_PLAYACTION_HH +#define FAODEL_PLAYACTION_HH + +#include "ResourceAction.hh" +#include "KelpieClientAction.hh" + +class PlayAction { +public: + PlayAction() {} + + std::string filename_line; //Parse info about where command came from + std::string command; + std::string error_message; + + std::vector args; //generic info + + ResourceAction resource_action; //resource-specific info + KelpieClientAction kelpie_action; //kelpie-specific info + + + int parseCommandLine(const std::string &my_rank, + std::string *default_pool, + std::string *default_rank, + const std::string &file_tag, + const std::string &command_line); + +private: + int emsg(const std::string &condition); + +}; + +#endif //FAODEL_PLAYACTION_HH diff --git a/tools/faodel-cli/README_Faodel_Cli.md b/tools/faodel-cli/README_Faodel_Cli.md new file mode 100644 index 0000000..cda4cee --- /dev/null +++ b/tools/faodel-cli/README_Faodel_Cli.md @@ -0,0 +1,362 @@ +FAODEL-CLI: A Multipurpose Tool for Interacting with FAODEL Services +===================================================================== + +The FAODEL cli tool is an all-in-one tool for doing a wide variety of +FAODEL tasks. You can: + +- Get build and configuration info for the current installation +- Launch DirMan and Kelpie servers +- Define/remove/list the Kelpie pool metadata maintained in a DirMan server +- Put/Get/List/Save/Load objects to/from a Kelpie pool +- Launch an mpi job to blast data into a kelpie pool +- Re-play a script with different commands in it +- Query whookie services on platforms that lack wget/curl + + +The documentation for the cli tool is provided in the `help` command. + +Help +---- +``` +$ faodel help +faodel COMMAND + + options: + -v/-V or --verbose/--very-verbose : Display runtime/debug info + --dirman-node id : Override config and use id for dirman + + commands: + build-info | binfo : Display FAODEL build information + config-info | cinfo : Display the Configuration tools will use + config-options | copt : List configuration options FAODEL inspects + whookie-get | wget : Retrieve a faodel service webpage + dirman-start | dstart : Start a dirman server + dirman-stop | dstop : Stop a dirman server + resource-list | rlist : Retrieve list of known resource names + resource-define | rdef : Define new resource + resource-drop | rdrop : Remove references to resources in dirman + kelpie-start | kstart : Start a kelpie server + kelpie-stop | kstop : Stop a kelpie server + kelpie-put | kput : Publish data to kelpie + kelpie-get | kget : Retrieve an item + kelpie-get-meta | kgetm : Retrieve metadata for item + kelpie-info | kinfo : Retrieve info for different keys + kelpie-list | klist : Retrieve key names/sizes + kelpie-save | ksave : Save objects from a pool to a local dir + kelpie-load | kload : Load objects from disk and store to a pool + kelpie-blast | kblast : Run MPI job to generate kelpie traffic + all-in-one | aone : Start nodes w/ dirman and pools + play-script | play script : Execute commands specified by a script + help | help : Provide more info about specific commands + +``` + +dstart: dirman-start +-------------------- + +``` +$ faodel help dstart +faodel COMMAND + + options: + -v/-V or --verbose/--very-verbose : Display runtime/debug info + --dirman-node id : Override config and use id for dirman + + commands: + dirman-start | dstart : Start a dirman server + + +DirMan is a service for keeping track of what resources are available in a +system. A user typically launches one dirman server and then establishes +one or more resource pools for hosting data. This command launches a dirman +server and then waits for the user to issue a dstop command to stop it. + +In order to make it easier to find the dirman server in later commands, +dirman-start creates a file with its nodeid when it launches. By default this +file is located at ./.faodel-dirman. You can override this location by +setting the dirman.write_root.file value in your $FAODEL_CONFIG file, or +by passing the location in through the environment variable +FAODEL_DIRMAN_ROOT_NODE_FILE. + +Examples: + + # Start and generate ./.faodel-dirman + $ faodel dstart + $ export FAODEL_DIRMAN_ROOT_NODE_FILE=$(pwd)/.faodel-dirman + $ faodel fstop + + # Start and specify file + $ export FAODEL_DIRMAN_ROOT_NODE_FILE=~/.my-dirman + $ faodel dstart + $ faodel dstop +``` + +kstart: kelpie-start +-------------------- +``` +faodel help kstart +faodel COMMAND + + options: + -v/-V or --verbose/--very-verbose : Display runtime/debug info + --dirman-node id : Override config and use id for dirman + + commands: + kelpie-start | kstart : Start a kelpie server + + +After defining resource pools with the rdef command, users will need to start +nodes to run kelpie servers that can join as nodes in the pool. When launching +a kelpie server, a user specifies a list of all the pool urls that the server +will join. Internally, a server locates dirman and issues a Join command to +volunteer to be a part of the pool. The server will continue to run until +the user issues a kstop command for all of the pools that a sever initially +was configured to join. + +Example: + + # Start and generate ./.faodel-dirman + $ faodel dstart + $ export FAODEL_DIRMAN_ROOT_NODE_FILE=$(pwd)/.faodel-dirman + + # Define a pool with two members + $ faodel rdef "dht:/my/dht&min_members=2" + $ faodel kstart /my/dht & + $ faodel kstart /my/dht & + + # Stop the pool + $ faodel kstop /my/dht +``` +kput: kelpie-put +-------------------- +``` +$ faodel help kput +faodel COMMAND + + options: + -v/-V or --verbose/--very-verbose : Display runtime/debug info + --dirman-node id : Override config and use id for dirman + + commands: + kelpie-put | kput : Publish data to kelpie + + +kelpie-put arguments: + -p/pool pool_url : The pool to publish to (resolves w/ DirMan) + (pool may be specified in $FAODEL_POOL) + + -k1/--key1 rowname : Row part of key + -k2/--key2 colname : Optional column part of key + or + -k/--key "rowname|colname" : Specify both parts of key, separated by '|' + + -f/--file filename : Read data from file instead of stdin + -m/--meta "string" : Add optional meta data to the object + +The kelpie-put command provides a simple way to publish a data object into +a pool. A user must specify a pool and a key name for the object. If no +file argument is provided, kelpie-put will read from stdin until it +gets an EOF. This version of the command is intended for publishing a +single, contiguous object and will truncate the data if it exceeds +the kelpie.chunk_size specified in Configuration (default = 512MB). + +Examples: + + # Populate from the command line + faodel kput --pool ref:/my/dht --key bob -m "My Stuff" + type text on cmd line + here, then hit con-d con-d to end + + # Use another tool to unpack a file and pipe into an object + xzcat myfile.xz | faodel kput --pool ref:/my/dht --key1 myfile + + # Load from a file and store in row stuff, column file.txt + faodel kput --pool ref:/my/dht --file file.txt --key "stuff|file.txt" + +``` + +kget: kelpie-get +-------------------- +``` +faodel help kget +faodel COMMAND + + options: + -v/-V or --verbose/--very-verbose : Display runtime/debug info + --dirman-node id : Override config and use id for dirman + + commands: + kelpie-get | kget : Retrieve an item + + +kelpie-get arguments: + -p/pool pool_url : The pool to retrieve from (resolves w/ DirMan) + (pool may be specified in $FAODEL_POOL) + + -k1/--key1 rowname : Row part of key + -k2/--key2 colname : Optional column part of key + or + -k/--key "rowname|colname" : Specify both parts of key, separated by '|' + + -f/--file filename : Read data from file instead of stdin + -i/--meta-only : Only display the meta data for the object + +The kelpie-get command provides a simple way to retrieve an object from a +pool. A user must specify the pool and key name for an object. If no file +argument is provided, the data will be dumped to stdout. A user may also +select the meta-only option to display only the meta data section of the +object. + +Examples: + + # Dump an object to stdout and use standard unix tools + faodel kget --pool ref:/my/dht --key mything | wc -l + + # Dump an object to file + faodel kget --pool ref:/my/dht --key "stuff|file.txt" --file file2.txt +``` + +kblast: kelpie-blast +``` +$ faodel help kblast +faodel COMMAND + + options: + -v/-V or --verbose/--very-verbose : Display runtime/debug info + --dirman-node id : Override config and use id for dirman + + commands: + kelpie-blast | kblast : Run MPI job to generate kelpie traffic + + +kelpie-blast Flags: + -a/--async : send many objects per timestep before blocking + -m/--reuse-memory : allocate LDOs in advance and reuse them + -r/--rank-grouping : ensure all of rank's data lands on same server + -s/--skip-barrier : skip the barrier that happens at start of cycle + +kelpie-blast Options: + -t/--timesteps x : Run for x timesteps and then stop (default = 1) + -T/--delay x : Delay for x seconds between timesteps + + -o/--object-sizes x,y : List of object sizes to publish each timestep + + -p/--external-pool pool : Name of an external pool to write (eg '/my/dht') + -P/--internal-pool pool : Type & path of internal pool to write to + (eg 'dht:/tmp', 'local:/tmp' for disk, or 'dht:') + + +The kelpie-blast command provides a parallel data generator that can produce +a variety of traffic conditions. When you run as an mpi job, each rank will +follow a bulk-sync parallel flow where each rank sleeps, dumps a collection +of objects, and then does an optional barrier. Output is in a tab-separated +format that is easy to parse. + +Examples: + mpirun -n 4 faodel kblast -P local:/tmp -t 10 # Write 10 timesteps to /tmp + mpirun -n 4 faodel kblast -P dht:/tmp -o 1k,2M,32 # Pub 3 objects/timestep + mpirun -n 4 faodel kblast -p /my/pool # Connect to external pool + +``` + +aone: all-in-one +---------------- +``` +$ faodel help aone +faodel COMMAND + + options: + -v/-V or --verbose/--very-verbose : Display runtime/debug info + --dirman-node id : Override config and use id for dirman + + commands: + all-in-one | aone : Start nodes w/ dirman and pools + + +The all-in-one option launches an mpi job that includes a dirman server, a +collection of kelpie servers (one per rank), and any pool settings you've +defined in either your configuration or the command line. +Example: + + mpirun -N 4 faodel aone "dht:/x ALL" "rft:/y 0-middle" "dht:/z 2" + # Use 4 nodes with + # "dht:/x ALL" dht named /x on all four ranks + # "dft:/y 0-middle" rft named /y on second half of ranks + # "dht:/z 2" dht named /z just on rank 2 + +``` + +play: play-script +----------------- +``` +faodel help aone +faodel COMMAND + + options: + -v/-V or --verbose/--very-verbose : Display runtime/debug info + --dirman-node id : Override config and use id for dirman + + commands: + all-in-one | aone : Start nodes w/ dirman and pools + + +The all-in-one option launches an mpi job that includes a dirman server, a +collection of kelpie servers (one per rank), and any pool settings you've +defined in either your configuration or the command line. +Example: + + mpirun -N 4 faodel aone "dht:/x ALL" "rft:/y 0-middle" "dht:/z 2" + # Use 4 nodes with + # "dht:/x ALL" dht named /x on all four ranks + # "dft:/y 0-middle" rft named /y on second half of ranks + # "dht:/z 2" dht named /z just on rank 2 + + +cdulmer@vortex:~/projects/faodel/build$ ./tools/faodel-cli/faodel help play +faodel COMMAND + + options: + -v/-V or --verbose/--very-verbose : Display runtime/debug info + --dirman-node id : Override config and use id for dirman + + commands: + play-script | play script : Execute commands specified by a script + + +Play a series of commands that setup the FAODEL environment. A script may +contain both configuration info and (most) actions that are part of the +faodel tool. The following is a brief example that shows the basic format. +More examples can be found in faodel/examples/faodel-cli/playback-scripts. +``` +The following is an example of a play script. There are a few other examples +in the `faodel/examples/faodel-cli/playback-scripts` directory. + +``` +# Hello world play script +config bootstrap.debug true # Turn on some debug messages +config dirman.root_node_mpi 0 # Set first node as dirman root +config dirman.resources_mpi[] dht:/my/dht ALL # Create a pool on all ranks + +set pool /my/dht # Set default pool +set rank 0 # Set default rank + +barrier # Do an mpi barrier +rlist -r 0 /my/dht # Rank 0 lists info for dht + +barrier # Do an mpi barrier +kput -D 1k -k object1 # Rank 0 writes 1 KB object +kput -p local:[abc] -D 1k -k object2 # Rank 0 write to local pool +klist -p /my/dht -k * # Show everything in dht +kinfo -p /my/dht -k object1 # Get more info on object1 + +barrier +ksave -d ./tmp -k object1 # Save an object to ldo file +kload -d ./tmp -k object1 -p local:[abc] # Load and write to new pool +klist -p local:[abc] * # Show all local items + +print ...And now to grab a file and display.. # Print some text +kput --file /etc/profile -k profile_file # Send a plain file to pool +kget -k profile_file # Grab file and display +barrier + +``` \ No newline at end of file diff --git a/tools/faodel-cli/ResourceAction.cpp b/tools/faodel-cli/ResourceAction.cpp new file mode 100644 index 0000000..b343b51 --- /dev/null +++ b/tools/faodel-cli/ResourceAction.cpp @@ -0,0 +1,49 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include "ResourceAction.hh" + +using namespace std; + +ResourceAction::ResourceAction(const std::string &long_or_short_cmd) { + + //Go through all of the commands and plug in the shorthand + vector> command_list = { + {"resource-list", "rlist"}, + {"resource-listr", "rlistr"}, + {"resource-define", "rdef"}, + {"resource-drop", "rdrop"} + //Todo: resource-kill? + }; + + + //Convert a command to its shorthand + for(auto &big_little : command_list) { + if(( long_or_short_cmd == big_little.first) || (long_or_short_cmd==big_little.second)) { + cmd=big_little.second; + break; + } + } + + //Tag this parse as an error + if(cmd.empty()) { + error_message="Command '"+long_or_short_cmd+"' not valid"; + } else { + rank="0"; //Always default to run on first rank + } + +} + +faodel::rc_t ResourceAction::ParseArgs(const vector &args) { + + rargs = args; + + if(rargs.empty()) { + if((cmd=="rlist") || (cmd=="rlistr")) rargs.emplace_back("/"); + else { + return setError("Command '"+cmd+"' needs at least one argument"); + } + } + return 0; +} \ No newline at end of file diff --git a/tools/faodel-cli/ResourceAction.hh b/tools/faodel-cli/ResourceAction.hh new file mode 100644 index 0000000..1d84c08 --- /dev/null +++ b/tools/faodel-cli/ResourceAction.hh @@ -0,0 +1,28 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_RESOURCEACTION_HH +#define FAODEL_RESOURCEACTION_HH + +#include +#include + +#include "ActionInterface.hh" + + +class ResourceAction + : public ActionInterface { + +public: + ResourceAction() = default; + ResourceAction(const std::string &long_or_short_cmd); + + std::vector rargs; + + faodel::rc_t ParseArgs(const std::vector &args); + +}; + + +#endif //FAODEL_RESOURCEACTION_HH diff --git a/tools/faodel-cli/all_in_one.cpp b/tools/faodel-cli/all_in_one.cpp new file mode 100644 index 0000000..64cc2ad --- /dev/null +++ b/tools/faodel-cli/all_in_one.cpp @@ -0,0 +1,133 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include +#include + +#include "faodelConfig.h" + +#ifdef Faodel_ENABLE_MPI_SUPPORT +#include +#endif + +#include "whookie/client/Client.hh" +#include "whookie/Server.hh" +#include "kelpie/Kelpie.hh" +#include "dirman/DirMan.hh" +#include "faodel-services/MPISyncStart.hh" + +#include "faodel_cli.hh" + +using namespace std; +using namespace faodel; + +int startAllInOne(const vector &args); +bool aone_keepgoing = true; + +bool dumpHelpAllInOne(string subcommand) { + + string help_aone[5] = { + "all-in-one", "aone", "", "Start nodes w/ dirman and pools", + R"( +The all-in-one option launches an mpi job that includes a dirman server, a +collection of kelpie servers (one per rank), and any pool settings you've +defined in either your configuration or the command line. +Example: + + mpirun -N 4 faodel aone "dht:/x ALL" "rft:/y 0-middle" "dht:/z 2" + # Use 4 nodes with + # "dht:/x ALL" dht named /x on all four ranks + # "dft:/y 0-middle" rft named /y on second half of ranks + # "dht:/z 2" dht named /z just on rank 2 + +)" + }; + + bool found=false; + found |= dumpSpecificHelp(subcommand, help_aone); + + return found; +} + +int checkAllInOneCommands(const std::string &cmd, const vector &args) { + if( (cmd == "all-in-one") || (cmd == "aone")) return startAllInOne(args); + //No command + return ENOENT; +} + +void KillAoneHook() { + info("Kelpie received shutdown request"); + aone_keepgoing=false; +} + +int startAllInOne(const vector &args) { + + faodel::Configuration config; + +#ifdef Faodel_ENABLE_MPI_SUPPORT + //Create the dirman info + int mpi_rank, mpi_size; + MPI_Init(0, nullptr); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + + faodel::mpisyncstart::bootstrap(); + config.Append("mpisyncstart.enable", "true"); + config.Append("dirman.root_node_mpi", "0"); + for(auto v: args) { + config.Append("dirman.resources_mpi[]", v); + } +#else + //No mpi.. just look busy + config.Append("dirman.host_root", "true"); +#endif + + //Make sure we're using dirman + string dirman_type; + config.GetLowercaseString(&dirman_type, "dirman.type"); + if(dirman_type == "") config.Append("dirman.type", "centralized"); + + config.Append("whookie.app_name", "All-in-One Server"); + + //Dump our id to a file + string root_write_file; + if((!config.Contains("dirman.write_root.file")) || (!root_write_file.empty())) { + if(root_write_file.empty()){ + root_write_file = "./.faodel-dirman"; + } + config.Append("dirman.write_root.file", root_write_file); + } + + + //Modify for debugging settings + modifyConfigLogging(&config, + {"kelpie", "whookie", "mpisyncstart"}, + {"opbox", "dirman"}); + + aone_keepgoing = true; + + //Startup in a way that adds a shutdown hook + faodel::bootstrap::Init(config, kelpie::bootstrap); + whookie::Server::registerHook("/dirman/shutdown", [] (const map &args, stringstream &results) { + KillAoneHook(); + }); + faodel::bootstrap::Start(); + + + //Wait for someone to call our shutdown service + do { + this_thread::sleep_for(chrono::seconds(3)); + } while(aone_keepgoing); + + faodel::bootstrap::Finish(); + +#ifdef Faodel_ENABLE_MPI_SUPPORT + MPI_Finalize(); +#endif + + return 0; +} + diff --git a/tools/faodel-cli/build_info.cpp b/tools/faodel-cli/build_info.cpp index d1577fa..6b31d12 100644 --- a/tools/faodel-cli/build_info.cpp +++ b/tools/faodel-cli/build_info.cpp @@ -1,3 +1,7 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + #include #include #include diff --git a/tools/faodel-cli/build_info_ib.cpp b/tools/faodel-cli/build_info_ib.cpp index 09bbaa6..58a9efc 100644 --- a/tools/faodel-cli/build_info_ib.cpp +++ b/tools/faodel-cli/build_info_ib.cpp @@ -1,3 +1,7 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + #include #include diff --git a/tools/faodel-cli/config_info.cpp b/tools/faodel-cli/config_info.cpp index 60c844d..1dc289f 100644 --- a/tools/faodel-cli/config_info.cpp +++ b/tools/faodel-cli/config_info.cpp @@ -1,14 +1,21 @@ -#include +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include #include +#include #include + #include "faodelConfig.h" #include "faodel-common/Configuration.hh" #include "lunasa/Lunasa.hh" #include "faodel-common/Common.hh" -#include "opbox/OpBox.hh" #include "whookie/Server.hh" +#include "kelpie/Kelpie.hh" +#include "kelpie/core/Singleton.hh" #include "faodel_cli.hh" @@ -19,10 +26,12 @@ using namespace faodel; int configInfo(const vector &args); +int configOptions(const vector &args); + bool dumpHelpConfig(string subcommand) { - string help_cinfo[5] ={ + string help_cinfo[5] = { "config-info", "cinfo", "", "Display the Configuration tools will use", R"( When FAODEL tools start, they will load configuration data from a file @@ -32,12 +41,24 @@ resources, and services that should run on specific nodes). This tool will dump out the configuration that FAODEL will start with. )" }; - return dumpSpecificHelp(subcommand, help_cinfo); + string help_copt[5] = { + "config-options", "copt", "", "List configuration options FAODEL inspects", + R"( +This option dumps out all the configuration settings that were checked when +Kelpie is started. If this command fails, check to make sure your $FAODEL_CONFIG +file has the minimum state necessary for starting Kelpie (eg, remove any kelpie.ioms). +)" + }; + bool found=false; + found |= dumpSpecificHelp(subcommand, help_cinfo); + found |= dumpSpecificHelp(subcommand, help_copt); + return found; } int checkConfigCommands(const std::string &cmd, const vector &args) { - if((cmd == "config-info") || (cmd == "cinfo")) return configInfo(args); + if( (cmd == "config-info") || (cmd == "cinfo")) return configInfo(args); + else if((cmd == "config-options") || (cmd == "copt")) return configOptions(args); //No command return ENOENT; @@ -49,7 +70,7 @@ void show_Configuration() { faodel::Configuration config; std::stringstream ss; - ss << "===================Faodel Configuration Variable======================" << endl; + ss << StringCenterTitle("Faodel Configuration Variable")< longest1) longest1 = f_tv.first.size(); + if(f_tv.second[0].size() > longest2) longest2 = f_tv.second[0].size(); + } + + for(auto &f_tv : field_typevals) { + cout.width(longest1); cout << std::left << f_tv.first <<" "; + cout.width(longest2); cout << std::left << f_tv.second[0] << " "; + cout << f_tv.second[1]< &args) { show_Configuration(); + Configuration config = configGetEmptyConfig(); + config.Append("dirman.host_root", "true"); //Ensure we start + + //Now try starting things up to see if the configs are usable - bootstrap::Start(Configuration(""), opbox::bootstrap); + bootstrap::Start(config, kelpie::bootstrap); show_Common(); show_Whookie(); show_Lunasa(); + show_Kelpie(); + bootstrap::Finish(); + + return 0; +} + +int configOptions(const vector &args) { + + Configuration config = configGetEmptyConfig(); + config.Append("dirman.host_root", "true"); //Ensure we start + + bootstrap::Start(config, kelpie::bootstrap); + show_ConfigOptions(); bootstrap::Finish(); return 0; diff --git a/tools/faodel-cli/dirman_server.cpp b/tools/faodel-cli/dirman_server.cpp index 3aff7d6..e44acbd 100644 --- a/tools/faodel-cli/dirman_server.cpp +++ b/tools/faodel-cli/dirman_server.cpp @@ -1,3 +1,7 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + #include #include #include diff --git a/tools/faodel-cli/faodel_cli.cpp b/tools/faodel-cli/faodel_cli.cpp index b9c1bfd..538f250 100644 --- a/tools/faodel-cli/faodel_cli.cpp +++ b/tools/faodel-cli/faodel_cli.cpp @@ -1,3 +1,7 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + #include #include #include @@ -14,7 +18,7 @@ using namespace faodel; //Global int global_verbose_level=0; - +int global_rank=0; //Helper functions bool dumpSpecificHelp(string subcommand, const string options[5]) { @@ -48,6 +52,16 @@ void warn(const string &s){ cerr << "\033[1;31m" << "Warning:" << ":\033[0m "< &basic_service_names, const vector &very_verbose_service_names) { for(auto s : basic_service_names) { @@ -66,10 +80,12 @@ void modifyConfigLogging(Configuration *config, const vector &basic_ser int dumpHelp(string subcommand) { - cout <<"faodel COMMAND \n"; - cout <<" options:\n" + cout <<"faodel COMMAND \n" + <<"\n" + <<" options:\n" <<" -v/-V or --verbose/--very-verbose : Display runtime/debug info\n" - <<" -d id or --dirman-node id : Use hex id for dirman node (overrides env vars)\n" + <<" --dirman-node id : Override config and use id for dirman\n" + <<"\n" <<" commands:\n"; bool found=false; @@ -80,6 +96,9 @@ int dumpHelp(string subcommand) { found |= dumpHelpResource(subcommand); found |= dumpHelpKelpieServer(subcommand); found |= dumpHelpKelpieClient(subcommand); + found |= dumpHelpKelpieBlast(subcommand); + found |= dumpHelpAllInOne(subcommand); + found |= dumpHelpPlay(subcommand); string help_help[5] = { "help", "help", "", "Provide more info about specific commands\n", @@ -155,10 +174,10 @@ int main(int argc, char **argv) { if( (sarg == "-v") || (sarg == "--verbose")) global_verbose_level = 1; else if((sarg == "-V") || (sarg == "--very-verbose")) global_verbose_level = 2; else if((sarg == "-VV") || (sarg == "--very-very-verbose")) global_verbose_level = 3; - else if((sarg == "-d") || (sarg == "--dirman-node")) { + else if( /* no -d */ (sarg == "--dirman-node")) { //Note: -d is common for --dir, so don't use it i++; if(i>=argc) { - cerr <<"Error: provided -d or --dirman-node, but did not provide a node id\n"; + cerr <<"Error: provided --dirman-node, but did not provide a node id\n"; return -1; } //Change env vars so this overrides.. this does not override anything in config file @@ -187,6 +206,7 @@ int main(int argc, char **argv) { } //Figure out which command this is and process it + if(rc==ENOENT) rc = checkAllInOneCommands(cmd, args); if(rc==ENOENT) rc = checkBuildCommands(cmd, args); if(rc==ENOENT) rc = checkConfigCommands(cmd, args); if(rc==ENOENT) rc = checkWhookieClientCommands(cmd, args); @@ -194,6 +214,8 @@ int main(int argc, char **argv) { if(rc==ENOENT) rc = checkResourceCommands(cmd, args); if(rc==ENOENT) rc = checkKelpieServerCommands(cmd, args); if(rc==ENOENT) rc = checkKelpieClientCommands(cmd, args); + if(rc==ENOENT) rc = checkKelpieBlastCommands(cmd, args); + if(rc==ENOENT) rc = checkPlayCommands(cmd, args); //Help menus diff --git a/tools/faodel-cli/faodel_cli.hh b/tools/faodel-cli/faodel_cli.hh index 9925673..024cc14 100644 --- a/tools/faodel-cli/faodel_cli.hh +++ b/tools/faodel-cli/faodel_cli.hh @@ -1,3 +1,7 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + #ifndef FAODEL_FAODEL_CLI_HH #define FAODEL_FAODEL_CLI_HH @@ -5,14 +9,22 @@ #include #include "faodel-common/Configuration.hh" +#include "kelpie/Kelpie.hh" extern int global_verbose_level; - +extern int global_rank; void info(const std::string &s); void dbg(const std::string &s); void warn(const std::string &s); +//Same, but only print if global_rank is zero +void info0(const std::string &s); +void dbg0(const std::string &s); +void warn0(const std::string &s); + + + // From faodel_cli.hh bool dumpSpecificHelp(std::string subcommand, const std::string options[5]); @@ -23,22 +35,33 @@ void modifyConfigLogging(faodel::Configuration *config, // Consolidated from all the different subcommands +bool dumpHelpAllInOne(std::string subcommand); bool dumpHelpBuild(std::string subcommand); bool dumpHelpConfig(std::string subcommand); bool dumpHelpDirman(std::string subcommand); bool dumpHelpResource(std::string subcommand); bool dumpHelpKelpieServer(std::string subcommand); bool dumpHelpKelpieClient(std::string subcommand); +bool dumpHelpKelpieBlast(std::string subcommand); +bool dumpHelpPlay(std::string subcommand); bool dumpHelpWhookieClient(std::string subcommand); +int checkAllInOneCommands( const std::string &cmd, const std::vector &args); int checkBuildCommands( const std::string &cmd, const std::vector &args); int checkConfigCommands( const std::string &cmd, const std::vector &args); int checkDirmanCommands( const std::string &cmd, const std::vector &args); int checkResourceCommands( const std::string &cmd, const std::vector &args); int checkKelpieServerCommands( const std::string &cmd, const std::vector &args); int checkKelpieClientCommands( const std::string &cmd, const std::vector &args); +int checkKelpieBlastCommands( const std::string &cmd, const std::vector &args); +int checkPlayCommands( const std::string &cmd, const std::vector &args); int checkWhookieClientCommands( const std::string &cmd, const std::vector &args); +//Core functions that might be called by replay +int resourceDefine_internals(const std::vector &args); +int resourceList_internals(const std::vector &args); +int resourceDrop_internals(const std::vector &args); + #endif //FAODEL_FAODEL_CLI_HH diff --git a/tools/faodel-cli/kelpie_blast.cpp b/tools/faodel-cli/kelpie_blast.cpp new file mode 100644 index 0000000..ad31971 --- /dev/null +++ b/tools/faodel-cli/kelpie_blast.cpp @@ -0,0 +1,269 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include + +#include "faodelConfig.h" +#ifdef Faodel_ENABLE_MPI_SUPPORT +#include +#endif + +#include "faodel_cli.hh" +#include "KelpieBlastParams.hh" + +using namespace std; +using namespace faodel; + +int kelpieBlast(const vector &args); + +bool dumpHelpKelpieBlast(string subcommand) { + string help_kblast[5] = { + "kelpie-blast", "kblast", "", "Run MPI job to generate kelpie traffic", + R"( +kelpie-blast Flags: + -a/--async : send many objects per timestep before blocking + -m/--reuse-memory : allocate LDOs in advance and reuse them + -r/--rank-grouping : ensure all of rank's data lands on same server + -s/--skip-barrier : skip the barrier that happens at start of cycle + +kelpie-blast Options: + -t/--timesteps x : Run for x timesteps and then stop (default = 1) + -T/--delay x : Delay for x seconds between timesteps + + -o/--object-sizes x,y : List of object sizes to publish each timestep + + -p/--external-pool pool : Name of an external pool to write (eg '/my/dht') + -P/--internal-pool pool : Type & path of internal pool to write to + (eg 'dht:/tmp', 'local:/tmp' for disk, or 'dht:') + + +The kelpie-blast command provides a parallel data generator that can produce +a variety of traffic conditions. When you run as an mpi job, each rank will +follow a bulk-sync parallel flow where each rank sleeps, dumps a collection +of objects, and then does an optional barrier. Output is in a tab-separated +format that is easy to parse. + +Output Columns: + Step: Which timestep this is for + Rank: Which rank this stat is for + Gen: Time (US) required to generate data being published + Issue: Time (US) to issue the publish operation. Only useful in async mode + Pub: Time (US) to issue publish and receive acknowledgement from target + Gap: Time (US) between when pub completes and rank gets out of barrier + All: Time (US) to generate, publish, complete, and get through barrier + Bytes: Total user bytes sent by this node for the timestep + IssueBW: How fast the issue appeared to application, in MB/s + PubBW: How fast the publish w/ acknowledgement appeared, in MB/s + +Examples: + mpirun -n 4 faodel kblast -P local:/tmp -t 10 # Write 10 timesteps to /tmp + mpirun -n 4 faodel kblast -P dht:/tmp -o 1k,2M,32 # Pub 3 objects/timestep + mpirun -n 4 faodel kblast -p /my/pool # Connect to external pool +)" + }; + + bool found=false; + found |= dumpSpecificHelp(subcommand, help_kblast); + return found; +} + +int checkKelpieBlastCommands(const std::string &cmd, const vector &args) { + if( (cmd == "kelpie-blast") || (cmd=="kblast")) return kelpieBlast(args); + return ENOENT; +} + +uint64_t getTimeUS(std::chrono::time_point t) { + return std::chrono::duration_cast(t.time_since_epoch()).count(); +} + + + +uint64_t * KelpieBlastInitializeData(const KelpieBlastParams &p, std::vector &ldos) { + + //Allocate initial ldos and scratch space for data + for(int i=0; i &ldos, uint64_t *user_data) { + + if(!p.reuse_memory) { + ldos.clear(); + dbg("Cleared ldos.. now adding"); + for(int i=0; i +KelpieBlastPublishData(const KelpieBlastParams &p, kelpie::Pool &pool, std::vector &ldos, uint64_t *user_data) { + + + static std::chrono::time_point issued_time; + + vector keys; + for(int i=0;i &args) { + + struct io_times_t { + uint64_t generate; + uint64_t issued; + uint64_t publish; + uint64_t gap; + uint64_t all; + uint64_t bytes; + }; + + char sep='\t'; + + KelpieBlastParams p(args); + if(!p.IsOk()) return 0; + + //First initialize rng! if not, all your timestep keys are the same + srand(time(NULL)+100*p.mpi_rank); + + + //Try connecting to out pool + auto pool = kelpie::Connect(p.pool_name); + string err; + if(!pool.Valid(&err)) { + cout <<"Error connecting to pool:\n"< ldos; + user_data = KelpieBlastInitializeData(p,ldos); + + + io_times_t iot; + io_times_t io_times[p.mpi_size]; + + + for(p.timestep=0; p.timestep < p.num_timesteps; p.timestep++) { + + p.SleepForComputePhase(); + if(!p.no_barrier_before_generate) + p.barrier(); + + dbg0("Generating data"); + auto t1 = std::chrono::high_resolution_clock::now(); + KelpieBlastGenerateData(p,ldos, user_data); + + + dbg0("Publishing data"); + auto t2 = std::chrono::high_resolution_clock::now(); + auto t_issued = KelpieBlastPublishData(p, pool, ldos, user_data); + + auto t3 = std::chrono::high_resolution_clock::now(); + dbg0("Published. Now send waiting"); + p.barrier(); + + auto t4 = std::chrono::high_resolution_clock::now(); + + iot.generate = getTimeUS(t2) - getTimeUS(t1); + iot.issued = getTimeUS(t_issued) - getTimeUS(t2); //How long it took to issue all async messages + iot.publish = getTimeUS(t3) - getTimeUS(t2); //How long it took for all messages to be issued and completed + iot.gap = getTimeUS(t4) - getTimeUS(t3); + iot.all = getTimeUS(t4) - getTimeUS(t1); + iot.bytes = p.bytes_per_rank_step; + + if(0) { + cout << "--->" << " " << p.mpi_rank << " " + << iot.generate << " " + << iot.publish << " " + << iot.gap << " " + << iot.all << " " + << iot.bytes << " " + << ((double) iot.bytes / (double) iot.publish) + << "\n"; + } + + #ifdef Faodel_ENABLE_MPI_SUPPORT + int rc = MPI_Gather(&iot, sizeof(io_times_t), MPI_CHAR, + &(io_times[0]), sizeof(io_times_t), MPI_CHAR, + 0, MPI_COMM_WORLD); + if(rc!=0) { + cerr<<"Gather gave bad rc?\n"; + } + #endif + + if(p.mpi_rank==0) { + for(int i = 0; i < p.mpi_size; i++) { + cout << p.timestep << sep + << i << sep + << io_times[i].generate << sep + << io_times[i].issued << sep + << io_times[i].publish << sep + << io_times[i].gap << sep + << io_times[i].all << sep + << io_times[i].bytes << sep + << ((double) io_times[i].bytes / (double) io_times[i].issued) << sep + << ((double) io_times[i].bytes / (double) io_times[i].publish) + << "\n"; + } + } + p.barrier(); + + } + dbg0("Done"); + + + delete[] user_data; + //~KelpieBlastParams should get called as we exit here + + + return 0; +} diff --git a/tools/faodel-cli/kelpie_client.cpp b/tools/faodel-cli/kelpie_client.cpp index c24750d..d61a302 100644 --- a/tools/faodel-cli/kelpie_client.cpp +++ b/tools/faodel-cli/kelpie_client.cpp @@ -1,22 +1,31 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + #include #include +#include #include +#include #include "faodel-common/StringHelpers.hh" #include "dirman/DirMan.hh" #include "kelpie/Kelpie.hh" #include "faodel_cli.hh" +#include "kelpie_client.hh" using namespace std; using namespace faodel; -int kelpieClientPut(const vector &args); -int kelpieClientGet(bool only_meta, const vector &args); -int kelpieClientInfo(const vector &args); -int kelpieClientList(const vector &args); + +/** + * @brief Provide help info for all the kelpie client commands + * @param[in] subcommand The command we're looking for + * @return FOUND whether we found this command or not + */ bool dumpHelpKelpieClient(string subcommand) { string help_kput[5] = { @@ -68,7 +77,7 @@ kelpie-get arguments: -k/--key "rowname|colname" : Specify both parts of key, separated by '|' -f/--file filename : Read data from file instead of stdin - -m/--meta-only : Only display the meta data for the object + -i/--meta-only : Only display the meta data for the object The kelpie-get command provides a simple way to retrieve an object from a pool. A user must specify the pool and key name for an object. If no file @@ -137,6 +146,47 @@ The output is a list of keys and their corresponding user lengths )" }; + string help_ksave[5] = { + "kelpie-save", "ksave", "", "Save objects from a pool to a local dir", + R"( +kelpie-save arguments: + -p/pool pool_url : The pool to retrieve from (resolves w/ DirMan) + (pool may be specified in $FAODEL_POOL) + -d/dir directory : The directory to store objects + +The kelpie-save command provides users with a way to retrieve that +objects that are in a pool and save them to a local directory. Similar +to the list command, the user must provide a list of keys or wildcards +to retrieve (if all items are desired, use '*'). + +Note: The bucket for the bool is not saved in the directory structure + +Example: + + # Save all items to the directory "mystruff/" + faodel ksave --pool ref:/my/dht --dir mystuff "*" + +)" + }; + string help_kload[5] = { + "kelpie-load", "kload", "", "Load objects from disk and store to a pool", + R"( +kelpie-load arguments: + -p/pool pool_url : The pool to retrieve from (resolves w/ DirMan) + (pool may be specified in $FAODEL_POOL) + -d/dir directory : The directory to load objects from + +The kelpie-load command allows you load objects from disk and push them into +pool. Objects must be in Lunasa's native disk format and be named as packed +key names). + +Example: + + # Load objects that were previously ksave'd to "mystuff/" + faodel kload --pool ref:/my/dht --dir mystuff + +)" + }; bool found=false; @@ -145,392 +195,402 @@ The output is a list of keys and their corresponding user lengths found |= dumpSpecificHelp(subcommand, help_kgetm); found |= dumpSpecificHelp(subcommand, help_kinfo); found |= dumpSpecificHelp(subcommand, help_klist); + found |= dumpSpecificHelp(subcommand, help_ksave); + found |= dumpSpecificHelp(subcommand, help_kload); return found; } -int checkKelpieClientCommands(const std::string &cmd, const vector &args) { - - if( (cmd == "kelpie-put") || (cmd == "kput")) return kelpieClientPut(args); - else if((cmd == "kelpie-get") || (cmd == "kget")) return kelpieClientGet(false, args); - else if((cmd == "kelpie-get-meta") || (cmd == "kgetm")) return kelpieClientGet(true, args); - else if((cmd == "kelpie-info") || (cmd == "kinfo")) return kelpieClientInfo(args); - else if((cmd == "kelpie-list") || (cmd == "klist") || (cmd=="kls")) return kelpieClientList(args); - - //No command - return ENOENT; -} - -bool getArgValue(string *result, const vector &args, size_t &i, const string &s1, const string &s2) { - if((args[i] == s1) || (args[i] == s2)) { - i++; - if(i>=args.size()) { - cerr<<"Error parsing "< &args, - vector *remaining_args, - string *pool_name, string *file_name, - kelpie::Key *key ) { - - string key1, key2, tmp_key; - - for(size_t i = 0; i2) { - cerr << "Could not parse -k/--key argument for '" << tmp_key << "'. Must be of form 'key1' or 'key1|key2'\n"; - exit(-1); - } - key1 = keys[0]; - if(keys.size()>1) - key2 = keys[1]; - - } else { - remaining_args->push_back(args[i]); - } + uint16_t meta_capacity = action.meta.size() & 0x0FFFF; //capacity checked in parser + ifstream f; + f.open(action.file_name, ios::in|ios::binary); + if(!f.is_open()) { + cerr <<"Could not open file "<= 4*1024*1024*1024LL)){ + cerr <<"File "<empty()) { - char *env_pool = getenv("FAODEL_POOL"); - if(env_pool == nullptr) { - cerr << "No pool provided. Use -p/--pool resource_url or set env var FAODEL_POOL\n"; - exit(-1); - } - *pool_name = string(env_pool); + lunasa::DataObject ldo(meta_capacity, results.st_size,lunasa::DataObject::AllocatorType::eager); + if(meta_capacity){ + memcpy(ldo.GetMetaPtr(), action.meta.c_str(), meta_capacity); } -} + f.read(ldo.GetDataPtr(), ldo.GetDataSize()); -faodel::Configuration kelpieClientStart() { + kelpie::object_info_t info; - faodel::Configuration config; + pool.Publish(action.keys[0], ldo, &info); + return 0; +} - //Make sure we're using dirman - string dirman_type; - config.GetLowercaseString(&dirman_type, "dirman.type"); - if(dirman_type == "") config.Append("dirman.type", "centralized"); +/** + * @brief Generate a block of data and publish it to a pool + * @param pool The pool to write into + * @param action The info about what needs to be published + * @retval -1 Could not find file, or data was too big + * @retval 0 Success + * @note This only uses the first key provided in args + * @note Does not shutdown bootstrap on errors + */ +int kelpieClient_PutFromGeneratedData(kelpie::Pool &pool, const KelpieClientAction &action) { - //Modify for debugging settings - modifyConfigLogging(&config, - {"kelpie", "whookie"}, - {"opbox", "dirman"}); + dbg("Putting generated data to key "+action.keys[0].str()); - config.AppendFromReferences(); - faodel::bootstrap::Start(config, kelpie::bootstrap); + lunasa::DataObject ldo(action.getMetaCapacity(), action.generate_data_size,lunasa::DataObject::AllocatorType::eager); + if(!action.meta.empty()){ + memcpy(ldo.GetMetaPtr(), action.meta.c_str(), action.meta.size()); + } - return config; + kelpie::object_info_t info; + pool.Publish(action.keys[0], ldo, &info); + return 0; } -int kelpieClientPut(const vector &args) { +/** + * @brief Take data from stdio and publish it to a pool + * @param pool The pool to write into + * @param action The info about what needs to be published + * @param MAX_CAPACITY The maximum amount of data that can be sent + * @retval 0 Success + * @note This only uses the first key provided in args + * @note Does not shutdown bootstrap on errors + */ +int kelpieClient_PutFromStdio(kelpie::Pool &pool, const KelpieClientAction &action, uint64_t MAX_CAPACITY) { - string pool_name, file_name, meta; - kelpie::Key key; - vector custom_args; + uint16_t meta_capacity = action.meta.size() & 0x0FFFF; //meta is validated to be <64KB in parse - //Pull out basic args - kelpieClientParseBasicArgs(args, &custom_args, &pool_name, &file_name, &key); + lunasa::DataObject ldo( meta_capacity, MAX_CAPACITY, lunasa::DataObject::AllocatorType::eager ); - //Pull out put-specific args - for(size_t i=0; i(); + size_t len; + while((len = fread( &dptr[offset], 1, MAX_CAPACITY - offset, stdin )) > 0) { + if(std::ferror(stdin) && !std::feof(stdin)) + throw std::runtime_error(std::strerror(errno)); + offset += len; + if(offset==MAX_CAPACITY) break; + } + //Trim the data section to the actual length + ldo.ModifyUserSizes(meta_capacity, offset); - uint64_t MAX_CAPACITY; - config.GetUInt(&MAX_CAPACITY, "kelpie.chunk_size", "512M"); - dbg("Chunk size is "+std::to_string(MAX_CAPACITY)); + kelpie::object_info_t info; + pool.Publish(action.keys[0], ldo, &info); - if(meta.size()>=64*1024){ - cerr <<"Meta sectionn must be less than 16KB\n"; - exit(-1); + } catch(std::exception const &e) { + cerr <<"STDIO error "<= 4*1024*1024*1024LL)){ - cerr <<"File "<(), ldo.GetDataSize()); - - kelpie::kv_row_info_t row; - kelpie::kv_col_info_t col; + } else { + //Handle case (3): Take data from stdin and put it into an object. Get our max chunk size + uint64_t MAX_CAPACITY; + config.GetUInt(&MAX_CAPACITY, "kelpie.chunk_size", "512M"); + dbg("Chunk size is "+std::to_string(MAX_CAPACITY)); + rc = kelpieClient_PutFromStdio(pool, action, MAX_CAPACITY); + } + return rc; +} - pool.Publish(key, ldo, &row, &col); - //cout <<"Publish row col info is "<() : ldo.GetDataPtr(); + len = (action.kget_meta_only) ? ldo.GetMetaSize() : ldo.GetDataSize(); - if(meta_capacity){ - memcpy(ldo.GetMetaPtr(), meta.c_str(), meta_capacity); + if(!action.file_name.empty()) { + ofstream f; + f.open(action.file_name, ios::out | ios::app | ios::binary); + if(!f.is_open()) { + cerr <<"Problem opening file "<(); - size_t len; - while((len = fread( &dptr[offset], 1, MAX_CAPACITY - offset, stdin )) > 0) { - //cout <<"Pulled in chunk of len "<max_key_len) + max_key_len = key_len; + } - } catch(std::exception const &e) { - cerr <<"STDIO error "< &args) { - string pool_name, file_name, meta; - kelpie::Key key; - vector custom_args; +int kelpieClient_List(kelpie::Pool &pool, const KelpieClientAction &action) { + //Get all keys and append to one OC + kelpie::ObjectCapacities oc; + for(auto key : action.keys) { + pool.List(key, &oc); + } - //Pull out basic args - kelpieClientParseBasicArgs(args, &custom_args, &pool_name, &file_name, &key); + int max_key_len = 0; + for(size_t i = 0; imax_key_len) + max_key_len = len; + } - //Pull out put-specific args - for(size_t i=0; i() : ldo.GetDataPtr(); - len = (only_meta) ? ldo.GetMetaSize() : ldo.GetDataSize(); + //Note: dir_name vetted inside parse - if(!file_name.empty()) { - ofstream f; - f.open(file_name, ios::out | ios::app | ios::binary); - if(!f.is_open()) { - cerr <<"Problem opening file "< &args) { +/** + * @brief Read kelpie objects from a raw directory and push them to the pool + * @param pool The pool to use for this operation + * @param action The kelpie action/arguments for this command + * @return + */ - string tmp_key; - vector keys; - string pool_name; +int kelpieClient_Load(kelpie::Pool &pool, const KelpieClientAction &action) { - int max_key_len; - for(size_t i = 0; i2) { - cerr << "Could not parse -k/--key argument for '" << tmp_key << "'. Must be of form 'key1' or 'key1|key2'\n"; - exit(-1); + struct file_info_t { + string full_fname; + kelpie::Key key; + size_t disk_size; + }; + vector files; + + //Scan directory and see if there are any files that we should read + DIR *dp; + struct dirent *ep; + dp = opendir(action.dir_name.c_str()); + if(dp != NULL) { + while((ep = readdir(dp)) != NULL) { + string name(ep->d_name); + if(!((name == ".") || (name == ".."))) { + string pname = action.dir_name + "/" + name; + struct stat sb; + if((stat(pname.c_str(), &sb) == 0) && (S_ISREG(sb.st_mode))) { + file_info_t finfo; + finfo.full_fname = pname; + finfo.key.pup(faodel::ExpandPunycode(name)); + finfo.disk_size = sb.st_size; + files.push_back(finfo); + } } - string k1, k2; - k1=key_parts[0]; - k2=(key_parts.size()>1) ? key_parts[1] : ""; - kelpie::Key key(k1,k2); - int len = key.str().size(); - if(len>max_key_len) max_key_len = len; - keys.push_back(key); - } - } - - //Pull from env var is no pool name - if(pool_name.empty()) { - char *env_pool = getenv("FAODEL_POOL"); - if(env_pool == nullptr) { - cerr << "No pool provided. Use -p/--pool resource_url or set env var FAODEL_POOL\n"; - exit(-1); } - pool_name = string(env_pool); } + closedir(dp); - auto pp=ResourceURL(pool_name); + if(files.size() == 0) return 0; //Done - auto config = kelpieClientStart(); - auto pool = kelpie::Connect(pool_name); - for(auto key : keys) { - kelpie::kv_col_info_t col_info; - rc_t rc = pool.Info(key, &col_info); - cout < &args) { - +/** + * @brief Check env var to see if the default FAODEL_POOL is set + * @return A string with the FAODEL_POOL setting of empty + */ +string kelpieGetPoolFromEnv() { + char *env_pool = getenv("FAODEL_POOL"); + if(env_pool == nullptr) return ""; + return string(env_pool); +} - string tmp_key; - vector keys; - string pool_name; +/** + * @brief Launch a kelpie client. Converts some cli settings to config settings + * @return Configuration the configuration that was used at launch time + */ +faodel::Configuration kelpieClientStart() { - for(size_t i = 0; i2) { - cerr << "Could not parse -k/--key argument for '" << tmp_key << "'. Must be of form 'key1' or 'key1|key2'\n"; - exit(-1); - } - string k1, k2; - k1=key_parts[0]; - k2=(key_parts.size()>1) ? key_parts[1] : ""; - kelpie::Key key(k1,k2); - keys.push_back(key); - } - } + //Modify for debugging settings + modifyConfigLogging(&config, + {"kelpie", "whookie"}, + {"opbox", "dirman"}); - //Pull from env var is no pool name - if(pool_name.empty()) { - char *env_pool = getenv("FAODEL_POOL"); - if(env_pool == nullptr) { - cerr << "No pool provided. Use -p/--pool resource_url or set env var FAODEL_POOL\n"; - exit(-1); - } - pool_name = string(env_pool); - } - auto pp=ResourceURL(pool_name); + faodel::bootstrap::Start(config, kelpie::bootstrap); - auto config = kelpieClientStart(); - auto pool = kelpie::Connect(pool_name); + return config; +} - //Get all kets and append to one OC - kelpie::ObjectCapacities oc; - for(auto key : keys) { - pool.List(key, &oc); +/** + * @brief Launch one of the kelpie client commands + * @param pool The pool to use for this operation + * @param config The configuration (for pulling out minor constants) + * @param action The kelpie action/arguments for this command + * @return Results of the operation or EINVAL for command not found + * @note Requires kelpie to be started + */ +int kelpieClient_Dispatch(kelpie::Pool &pool, faodel::Configuration &config, const KelpieClientAction &action) { + + string cmd2 = action.cmd; + int rc=EINVAL; + if (cmd2=="kput") { rc = kelpieClient_Put(pool, config, action); + } else if(cmd2=="kget") { rc = kelpieClient_Get(pool, action); + } else if(cmd2=="kinfo") { rc = kelpieClient_Info(pool, action); + } else if(cmd2=="klist") { rc = kelpieClient_List(pool, action); + } else if(cmd2=="ksave") { rc = kelpieClient_Save(pool, action); + } else if(cmd2=="kload") { rc = kelpieClient_Load(pool, action); } + return rc; +} - int max_key_len=0; - for(size_t i=0; imax_key_len) - max_key_len = len; - } +/** + * @brief One-shot kelpieClient function. Parses args, starts kelpie command, and shutsdown + * @param[in] cmd The command that the user supplied (eg kput) + * @param[in] args The other arguments passed in on the cli to parse + * @retval 0 Found the command + * @retval ENOENT Didn't find the command + * @note This Starts and Stops kelpie + */ +int checkKelpieClientCommands(const std::string &cmd, const vector &args) { - for(size_t i=0; i #include #include @@ -132,21 +136,22 @@ int startKelpieServer(const vector &args) { //Join any resource the user has supplied for(auto p : args) { - faodel::DirectoryInfo dir; faodel::ResourceURL url; try { url = ResourceURL(p); - bool ok = dirman::JoinDirWithoutName(url, &dir); - if(ok) { + cout <<"Trying to join "< +#include +#include + +#include "faodelConfig.h" + +#ifdef Faodel_ENABLE_MPI_SUPPORT +#include +#include "faodel-services/MPISyncStart.hh" +#endif + +#include "faodel-common/StringHelpers.hh" +#include "dirman/DirMan.hh" +#include "kelpie/Kelpie.hh" + +#include "kelpie/pools/ResultCollector.hh" + +#include "kelpie_client.hh" +#include "resource_client.hh" + +#include "PlayAction.hh" +#include "faodel_cli.hh" + + +using namespace std; +using namespace faodel; + + +int playMain(const vector &args); + + + +bool dumpHelpPlay(std::string subcommand) { + string help_play[5] = { + "play-script", "play", "script", "Execute commands specified by a script", + R"( +Play a series of commands that setup the FAODEL environment. A script may +contain both configuration info and (most) actions that are part of the +faodel tool. The following is a brief example that shows the basic format. +More examples can be found in faodel/examples/faodel-cli/playback-scripts. + +# Hello world play script +config bootstrap.debug true # Turn on some debug messages +config dirman.root_node_mpi 0 # Set first node as dirman root +config dirman.resources_mpi[] dht:/my/dht ALL # Create a pool on all ranks + +set pool /my/dht # Set default pool +set rank 0 # Set default rank + +barrier # Do an mpi barrier +rlist -r 0 /my/dht # Rank 0 lists info for dht + +barrier # Do an mpi barrier +kput -D 1k -k object1 # Rank 0 writes 1 KB object +kput -p local:[abc] -D 1k -k object2 # Rank 0 write to local pool +klist -p /my/dht -k * # Show everything in dht +kinfo -p /my/dht -k object1 # Get more info on object1 + +barrier +ksave -d ./tmp -k object1 # Save an object to ldo file +kload -d ./tmp -k object1 -p local:[abc] # Load and write to new pool +klist -p local:[abc] * # Show all local items + +print ...And now to grab a file and display.. # Print some text +kput --file /etc/profile -k profile_file # Send a plain file to pool +kget -k profile_file # Grab file and display +barrier + + +)" + }; + + bool found=false; + found |= dumpSpecificHelp(subcommand, help_play); + return found; + +} + +int checkPlayCommands(const std::string &cmd, const vector &args) { + if( (cmd=="play-script") || (cmd == "play")) return playMain(args); + return ENOENT; +} + + +void playInit(faodel::Configuration *config) { + + //See if dirman details are specified + config->AppendFromReferences(); + + + //See if dirman is configured. + if(!config->Contains("dirman.type")) { + config->Append("dirman.type", "centralized"); + } + if(!( config->Contains("dirman.host_root") || + config->Contains("dirman.root_node") || + config->Contains("dirman.root_node_mpi") )){ + config->Append("dirman.host_root","true"); + } + + modifyConfigLogging(config, + {"kelpie","whookie"}, + {"opbox", "dirman"}); + + dbg("Starting config:\n"+config->str()); +#ifdef Faodel_ENABLE_MPI_SUPPORT + faodel::mpisyncstart::bootstrap(); +#endif + faodel::bootstrap::Start(*config, kelpie::bootstrap); + +} + +void playFinish() { + if(faodel::bootstrap::IsStarted()) { + faodel::bootstrap::Finish(); + } +} + +vector playExtractRankInfoFromArgs(const vector &args, string *my_rank) { + + //See if there's an override on the faodel command that specifies our rank + string tmp_my_rank; + vector my_args; + for(int i=0; i=args.size()) { + cerr << "The -r rank option did not specify a value?\n"; + exit(-1); + } + i++; + tmp_my_rank=args[i]; + + } else { + my_args.push_back(args[i]); + } + } + + //Normal case: we check mpi and use that value + #ifdef Faodel_ENABLE_MPI_SUPPORT + if(tmp_my_rank.empty()) { + int mpi_rank; + MPI_Init(nullptr, nullptr); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + tmp_my_rank = std::to_string(mpi_rank); + } + #endif + + *my_rank = tmp_my_rank; + return my_args; +} + +vector playParseScripts(const vector &args, faodel::Configuration *config) { + + vector results; //actions to pass back + string default_pool; + bool needs_iomdir_defined=false; + + //Grab default pool from env + char *env_pool = getenv("FAODEL_POOL"); + if(env_pool !=nullptr) { + default_pool = string(env_pool); + } + + string my_rank, default_rank; //my_rank is what user or mpi told us, default_rank is what "set rank" said to use + vector my_args = playExtractRankInfoFromArgs(args, &my_rank); + default_rank=my_rank; + + //Parse all the files, line by line + for(auto &fn : my_args) { + dbg("Parsing file "+fn); + ifstream ifs(fn); + if(!ifs.is_open()) { + cerr <<"Could not open script "<Append(line.substr(6)); + continue; + } + //Remove comments. Skip out if nothing left + auto cmd_comment = faodel::Split(line, '#', false); + if(cmd_comment.empty()) continue; + + //Split multiple commands stacked on one line with ';' + auto commands = faodel::Split(cmd_comment[0],';', true); + string file_tag = fn+":"+std::to_string(line_num); + + //Handle each command + for(auto &cmd : commands) { + + PlayAction play_action; + int rc = play_action.parseCommandLine(my_rank, &default_pool, &default_rank, file_tag, cmd); + if(rc==ENOENT) continue; //No data - skip + if(rc==EAGAIN) { hit_exit=true; break; } //got exit command + if(rc==EINVAL) { //Parse error - bail + cerr<<"Parse Error: "<ValidOrDie(); + dbg("Connected"); +} + +vector makeKeys(const vector &key_strings) { + vector keys; + for(auto &ks : key_strings) { + auto tokens = faodel::Split(ks,'|'); + if((tokens.size()>2) || tokens.empty()) { + cerr <<"Could not parse key: "<1) k2 = tokens[1]; + keys.emplace_back(kelpie::Key(tokens[0], k2)); + } + return keys; +} + +int playMain(const vector &args){ + + int rc; + dbg("Starting play"); + faodel::Configuration config; + config.AppendFromReferences(); + + //Build up the list of actions from the scripts we were given + auto actions = playParseScripts(args, &config); + dbg("Parsed actions and found "+to_string(actions.size())+" commands"); + + //Start up faodel + playInit(&config); + + string current_pool_name; + kelpie::Pool current_pool; + + //Step through each action + for(auto &action : actions) { + + if( action.kelpie_action.Valid() ) { + dbg("Working on kelpie action "+action.kelpie_action.cmd); + + accessPool(action.kelpie_action.pool_name, ¤t_pool_name, ¤t_pool); + rc = kelpieClient_Dispatch(current_pool, config, action.kelpie_action); + + } else if( action.resource_action.Valid() ) { + dbg("Working on resource action "+action.resource_action.cmd); + rc = resourceClient_Dispatch(action.resource_action); + + } else if(action.command == "print") { cout < #include "faodel-common/StringHelpers.hh" #include "dirman/DirMan.hh" #include "kelpie/Kelpie.hh" #include "faodel_cli.hh" +#include "ResourceAction.hh" using namespace std; using namespace faodel; -int resourceList(const vector &args); -int resourceListRecursive(const vector &args); -int resourceDefine(const vector &args); -int resourceDrop(const vector &args); -int resourceKill(const vector &args); +int resourceClient_List(const ResourceAction &action); +int resourceClient_ListRecursive(const ResourceAction &action); +int resourceClient_Define(const ResourceAction &action); +int resourceClient_Drop(const ResourceAction &action); +int resourceClient_Kill(const ResourceAction &action); bool dumpHelpResource(string subcommand) { @@ -35,7 +40,7 @@ This command connects to dirman and instructs it to define the resources specified by urls. Defining a resource is the first step in creating a resource, and should be thought of as a way to specify parameters for a resource as opposed to the actual nodes that are part of the resource. A URL -should include the type, path, name, and paramters for the resource (eg +should include the type, path, name, and parameters for the resource (eg minimum number of nodes or iom names). Example: @@ -104,53 +109,18 @@ resource and instruct them to shutdown. return found; } -int checkResourceCommands(const std::string &cmd, const vector &args) { - - if( (cmd == "resource-list") || (cmd == "rlist")) return resourceList(args); - else if((cmd == "resource-listr") || (cmd == "rlistr")) return resourceListRecursive(args); - else if((cmd == "resource-define") || (cmd == "rdef")) return resourceDefine(args); - else if((cmd == "resource-drop") || (cmd == "rdrop")) return resourceDrop(args); - //else if((cmd == "resource-kill") || (cmd == "rkill")) return resourceKill(args); - - //No command - return ENOENT; -} - -void resourceInit(faodel::Configuration *config) { - - string dirman_type; - config->GetLowercaseString(&dirman_type, "dirman.type"); - if(dirman_type == "") { - config->Append("dirman.type", "centralized"); - } - - modifyConfigLogging(config, {"dirman"}, {"dirman.cache.mine", "dirman.cache.others"}); - - faodel::bootstrap::Start(*config, dirman::bootstrap); - -} - -void resourceFinish() { - if(faodel::bootstrap::IsStarted()) { - faodel::bootstrap::Finish(); - } -} -int resourceList(const vector &args) { - auto paths = args; - if(paths.size()==0) paths.push_back("/"); - faodel::Configuration config; - resourceInit(&config); +int resourceClient_List(const ResourceAction &action) { int rc=0; bool ok; string dir_path; - for(auto p : paths) { + for(auto p : action.rargs) { DirectoryInfo dir; ResourceURL url; try { @@ -180,30 +150,24 @@ int resourceList(const vector &args) { cout<<" Min Members: "< &args) { - auto paths = args; - if(paths.size()==0) paths.push_back("/"); //Show default root +int resourceClient_ListRecursive(const ResourceAction &action) { - faodel::Configuration config; - resourceInit(&config); + auto paths = action.rargs; int rc=0; bool ok; @@ -257,24 +221,15 @@ int resourceListRecursive(const vector &args) { for(auto &path : results) cout << setw(max_slen) << std::left << path.first<<" : " << path.second< &args) { +int resourceClient_Define(const ResourceAction &action) { + int rc=0; bool ok; - if(args.empty()){ - cout <<"No resources provided. Done.\n"; - return 0; - } - - faodel::Configuration config; - resourceInit(&config); - for(auto r : args) { + for(auto r : action.rargs) { ResourceURL url; try { @@ -296,19 +251,13 @@ int resourceDefine(const vector &args) { rc=-1; } } - - resourceFinish(); return rc; } -int resourceDrop(const vector &args) { - - int rc=0; +int resourceClient_Drop(const ResourceAction &action) { bool ok; - faodel::Configuration config; - resourceInit(&config); - - for(auto r : args) { + int rc=0; + for(auto r : action.rargs) { ResourceURL url; try { @@ -322,16 +271,64 @@ int resourceDrop(const vector &args) { rc=-1; } } + return rc; +} + + +faodel::Configuration resourceClientStart() { + + faodel::Configuration config; + config.AppendFromReferences(); + + + //Make sure we're using dirman + string dirman_type; + config.GetLowercaseString(&dirman_type, "dirman.type"); + if(dirman_type.empty()) config.Append("dirman.type", "centralized"); + + //Modify for debugging settings + modifyConfigLogging(&config, + {"dirman"}, + {"dirman.cache.mine", "dirman.cache.others"}); + + + faodel::bootstrap::Start(config, kelpie::bootstrap); + return config; - resourceFinish(); +} + + +int resourceClient_Dispatch(const ResourceAction &action) { + string cmd2 = action.cmd; + int rc=EINVAL; + if(cmd2=="rlist") { rc = resourceClient_List(action); + } else if(cmd2=="rlistr") { rc = resourceClient_ListRecursive(action); + } else if(cmd2=="rdef") { rc = resourceClient_Define(action); + } else if(cmd2=="rdrop") { rc = resourceClient_Drop(action); + } return rc; } -//Note: Original plan was to do a drop and then send kills to pool members -// This is a combination of kstop and rdrop now. -int resourceKill(const vector &args) { - int rc=0; - KTODO("resourceKill"); +int checkResourceCommands(const std::string &cmd, const vector &args) { + + //Figure out what command this is. Bail out if it's not a resource command + ResourceAction action(cmd); + if(action.HasError()) return ENOENT; //Didn't match + + //Parse this command's arguments + action.ParseArgs(args); + action.exitOnError(); + action.exitOnExtraArgs(); + + //Start up + resourceClientStart(); + + int rc = resourceClient_Dispatch(action); + + //Shut down + if(faodel::bootstrap::IsStarted()) { + faodel::bootstrap::Finish(); + } return rc; } diff --git a/tools/faodel-cli/resource_client.hh b/tools/faodel-cli/resource_client.hh new file mode 100644 index 0000000..f02c032 --- /dev/null +++ b/tools/faodel-cli/resource_client.hh @@ -0,0 +1,12 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_RESOURCE_CLIENT_HH +#define FAODEL_RESOURCE_CLIENT_HH + +#include "ResourceAction.hh" + +int resourceClient_Dispatch(const ResourceAction &action); + +#endif //FAODEL_RESOURCE_CLIENT_HH diff --git a/tools/faodel-cli/whookie_client.cpp b/tools/faodel-cli/whookie_client.cpp index 0028304..d5024a1 100644 --- a/tools/faodel-cli/whookie_client.cpp +++ b/tools/faodel-cli/whookie_client.cpp @@ -1,4 +1,10 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + #include +#include +#include #include "whookie/client/Client.hh" @@ -17,6 +23,7 @@ bool dumpHelpWhookieClient(string subcommand) { whookie-get arguments: -h/--html : Return the data in html format -t/--text : Return the page in plain text + -x S : Repeat this command evert S seconds url : the url to fetch The whookie-get command provides a way for you to issue queries to a faodel @@ -39,7 +46,8 @@ int checkWhookieClientCommands(const std::string &cmd, const vector &arg void whookieClientParseBasicArgs(const vector &args, string *url_string, - string *format) { + string *format, + int *sleep_interval) { *url_string =""; @@ -47,7 +55,15 @@ void whookieClientParseBasicArgs(const vector &args, string s = args[i]; if( (s == "-h") || (s == "--html")) *format="html"; else if( (s == "-t") || (s == "--text")) *format="text"; - else if (s.at(0)=='-') { + else if( (s == "-x") ) { + i++; + if(i==args.size()) { + cerr <<"Missing a value for -x ?\n"; + exit(-1); + } + *sleep_interval = atoi(args[i].c_str()); + + } else if (s.at(0)=='-') { cerr << "Unrecognized option '" << s << "'\n"; exit(-1); } else if( *url_string != "") { @@ -69,8 +85,9 @@ int whookieClientGet(const vector &args) { string mode, url; string format="text"; + int sleep_interval=0; - whookieClientParseBasicArgs(args, &url, &format); + whookieClientParseBasicArgs(args, &url, &format, &sleep_interval); if(url.compare(0,7,"http://") || (url.size()<8)) { cerr<<"URL must begin with 'http://'. Received '"< &args) { path += "&format="+format; - string data; - whookie::retrieveData(host, port, path, &data); - cout < +#include "faodel-common/Common.hh" + +#include "Job.hh" + + +using namespace std; + +Job::Job(const faodel::Configuration &config, const std::string &job_category) + : LoggingInterface(job_category), + job_category(job_category), + num_threads(1), + initialized(false), + dump_tsv(false), + ops_completed(0) { + + ConfigureLogging(config); + + int rc; + uint64_t us; + config.GetTimeUS(&us, "faodel-stress.time_limit", "30s"); + run_time_seconds = us / 1000000; + + config.GetBool(&dump_tsv,"faodel-stress.dump_tsv", "false"); + + rc = config.GetUInt(&num_threads, "faodel-stress.job."+job_category+".num_threads", "1"); + if(rc != 0) + rc = config.GetUInt(&num_threads, "faodel-stress.num_threads", "1"); + F_ASSERT(rc == 0, "Unable to parse faodel-stress.num_threads out of config?"); +} + +void Job::DumpJobNames() { + for(const auto &job_name : job_names) { + cout << std::left<< std::setw(15)< Job::GetMatchingJobNames(const std::string &search_names) { + string search_names_lower = faodel::ToLowercase(search_names); + if((search_names_lower=="all") || (search_names_lower=="*")) return job_names; + std::vector found_names; + auto names=faodel::Split(search_names,',', true); + string job_category_lower = faodel::ToLowercase(job_category); + for(auto &search_name : names) { + string search_name_lower = faodel::ToLowercase(search_name); + if((search_name_lower==job_category_lower+":all") || + (search_name_lower==job_category_lower+":*") ) + return job_names; //All in this category + + //Check for a simple * wildcard in the search name + size_t star_spot = search_name_lower.find_first_of("*"); + bool is_wildcard = (star_spot!=string::npos); + if(is_wildcard) { + search_name_lower = search_name_lower.substr(0, star_spot); + } + //Go through all the jobs in this category and check for hits + for(auto &job_name : job_names) { + string job_name_lower = faodel::ToLowercase(job_name); + if( (search_name_lower == job_name_lower) || + (is_wildcard && faodel::StringBeginsWith(job_name_lower, search_name_lower)) ) { + found_names.push_back(job_name); + } + } + } + return found_names; +} + +int Job::Setup(const std::string &search_names) { + selected_job_names = GetMatchingJobNames(search_names); + initialized = true; + return selected_job_names.size(); +} + +int Job::ExecuteAll() { + int rc=0; + for(const auto &name: selected_job_names) { + rc |= Execute(name); + } + return rc; +} diff --git a/tools/faodel-stress/Job.hh b/tools/faodel-stress/Job.hh new file mode 100644 index 0000000..7c96cc2 --- /dev/null +++ b/tools/faodel-stress/Job.hh @@ -0,0 +1,87 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_JOB_HH +#define FAODEL_JOB_HH + +#include +#include +#include +#include +#include + + +#include "faodel-common/Configuration.hh" +#include "faodel-common/LoggingInterface.hh" +#include "faodel-common/StringHelpers.hh" + +class Job : + public faodel::LoggingInterface { +public: + + Job(const faodel::Configuration &config, const std::string &job_category); + virtual ~Job() = default; + + void DumpJobNames(); + virtual void DumpJobStats(const std::string &job_name); + + std::vector GetMatchingJobNames(const std::string &search_names); + + virtual int Init() { return 0; }; //After faodel as initialized but before start + + virtual int Setup(const std::string &search_names); + virtual int Execute(const std::string &job_name) = 0; + virtual int ExecuteAll(); + virtual void Teardown() {}; + + + template + int standardExecuteWorker(const std::string &job_name, const std::map &options) { + + auto it = options.find(job_name); + if (it==options.end()) return -1; + + dbg("Launching "+std::to_string(num_threads)+" worker threads"); + std::vector w_workers; + for(int i=0; isecond)); + + testStart(); + for(auto &w : w_workers) w.Start(); + + testSleep(); + for(auto &w : w_workers) w.Stop(); + + testStop(); + for(auto &w : w_workers) { + ops_completed += w.GetOpsCompleted(); + dbg("Thread ops completed: "+std::to_string(w.GetOpsCompleted())); + } + + DumpJobStats(job_name); + + return 0; + } + + std::string job_category; + std::vector job_names; + std::vector selected_job_names; + +protected: + uint64_t num_threads; + bool initialized; + bool dump_tsv; + uint64_t ops_completed; + uint64_t run_time_seconds; + std::chrono::time_point t_start; + std::chrono::time_point t_stop; + void testStart() {ops_completed=0; t_start=std::chrono::system_clock::now();} + void testSleep() const {std::this_thread::sleep_for(std::chrono::seconds(run_time_seconds));} + void testStop() {t_stop= std::chrono::system_clock::now();} + double getTestTimeUS() { return std::chrono::duration_cast(t_stop - t_start).count();} + double getTestMops() { return ops_completed / getTestTimeUS(); } +}; + + +#endif // FAODEL_STRESSJOB_HHH diff --git a/tools/faodel-stress/JobKeys.cpp b/tools/faodel-stress/JobKeys.cpp new file mode 100644 index 0000000..69189db --- /dev/null +++ b/tools/faodel-stress/JobKeys.cpp @@ -0,0 +1,55 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include +#include + + +#include "kelpie/Key.hh" + +#include "Worker.hh" +#include "JobKeys.hh" + +using namespace std; + +class WorkerKeys + : public Worker { + +public: + WorkerKeys() = default; + WorkerKeys(int id, JobKeys::params_t params) + : Worker(id, params.num_keys, 0,0), params(params) {} + ~WorkerKeys() = default; + + void server() { + + std::function f_keygen; + if (params.k1_len==0) f_keygen = [this]() { return kelpie::Key::Random(dummy_name, params.k2_len); }; + else if (params.k2_len==0) f_keygen = [this]() { return kelpie::Key::Random(params.k1_len, dummy_name); }; + else f_keygen = [this]() { return kelpie::Key::Random(params.k1_len, params.k2_len); }; + + vector keys(batch_size); + do { + //Generate a bundle of random keys + for(int i=0; i(job_name, options); +} \ No newline at end of file diff --git a/tools/faodel-stress/JobKeys.hh b/tools/faodel-stress/JobKeys.hh new file mode 100644 index 0000000..f1fb61d --- /dev/null +++ b/tools/faodel-stress/JobKeys.hh @@ -0,0 +1,49 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_JOBKEYS_HH +#define FAODEL_JOBKEYS_HH + +#include + + +#include "Job.hh" + +/*** + * @brief Generate and sort a number of kelpie keys + * + * Kelpie uses a simple 2-string key to track objects. These tests measure + * how quickly new keys of different sizes can be generated and sorted. + */ +class JobKeys : + public Job { + +public: + explicit JobKeys(const faodel::Configuration &config) + : Job(config, JobCategoryName()) { + for(auto &k_p : options) + job_names.push_back(k_p.first); + } + + ~JobKeys() override = default; + + int Execute(const std::string &job_name) override; + + struct params_t { uint32_t num_keys; size_t k1_len; size_t k2_len;}; + const std::map options = { + {"GenSort-ShortRowKey", {1024, 16, 0}}, + {"GenSort-ShortColKey", {1024, 0, 16}}, + {"GenSort-Short2DKey", {1024, 16, 16}}, + {"GenSort-LongRowKey", {1024, 255, 0}}, + {"GenSort-LongColKey", {1024, 0, 255}}, + {"GenSort-Long2DKey", {1024, 255, 255}} + }; + + constexpr const char* JobCategoryName() { return "keys"; } + +}; + + + +#endif // FAODEL_JOBKEYS_HH \ No newline at end of file diff --git a/tools/faodel-stress/JobLocalPool.cpp b/tools/faodel-stress/JobLocalPool.cpp new file mode 100644 index 0000000..8bb1ad6 --- /dev/null +++ b/tools/faodel-stress/JobLocalPool.cpp @@ -0,0 +1,101 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include +#include + + +#include "kelpie/Kelpie.hh" + +#include "Worker.hh" +#include "JobLocalPool.hh" + +using namespace std; + +class WorkerLocalPool + : public Worker { + +public: + WorkerLocalPool() = default; + WorkerLocalPool(int id, JobLocalPool::params_t params) + : Worker(id, params.num_kvs, 0,0), + params(params) { + + for(int i=0; i f_getobject; + if(params.allocate_ondemand) f_getobject = [this](int i) { return lunasa::DataObject(params.ldo_size);}; + else f_getobject = [this](int i) { return ldos.at(i);}; + + do { + + //Insert a bunch of objects + for(int i=0; i keys; + std::vector ldos; + + kelpie::Pool pool; + + JobLocalPool::params_t params; + const std::string dummy_name = "dummy-name"; +}; + + +JobLocalPool::JobLocalPool(const faodel::Configuration &config) + : Job(config, JobCategoryName()) { + + for(auto &name_params : options) + job_names.push_back(name_params.first); + +} + +int JobLocalPool::Execute(const std::string &job_name) { + return standardExecuteWorker(job_name, options); +} + diff --git a/tools/faodel-stress/JobLocalPool.hh b/tools/faodel-stress/JobLocalPool.hh new file mode 100644 index 0000000..4af21ea --- /dev/null +++ b/tools/faodel-stress/JobLocalPool.hh @@ -0,0 +1,46 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_JOBLOCALPOOL_HH +#define FAODEL_JOBLOCALPOOL_HH + +#include "Job.hh" + +/*** + * @brief A stress test that writes objects into the local store in different ways + * + * This stress test focuses on inserting and dropping objects into a kelpie local + * pool. This pool uses a row/column notation and uses a good bit of locking to + * ensure that multiple threads do not disturb each other. This test picks keys + * in a way to either avoid collisions or cause contention. + */ +class JobLocalPool : + public Job { + +public: + explicit JobLocalPool(const faodel::Configuration &config); + + ~JobLocalPool() override = default; + + int Execute(const std::string &job_name) override; + + + struct params_t { uint32_t num_kvs; uint64_t ldo_size; bool allocate_ondemand; int key_strategy; }; + const std::map options = { + {"PutGetDrop-1D", {1024, 1024, false, 1}}, + {"PutGetDrop-PrivateRows",{1024, 1024, false, 2}}, + {"PutGetDrop-CombinedRow",{1024, 1024, false, 3}}, + + {"AllocatePutGetDrop-1D", {1024, 1024, true, 1}}, + {"AllocatePutGetDrop-PrivateRows",{1024, 1024, true, 2}}, + {"AllocatePutGetDrop-CombinedRow",{1024, 1024, true, 3}} + + + }; + constexpr const char* JobCategoryName() { return "localpool"; } + +}; + + +#endif //FAODEL_JOBLOCALPOOL_HH diff --git a/tools/faodel-stress/JobMemoryAlloc.cpp b/tools/faodel-stress/JobMemoryAlloc.cpp new file mode 100644 index 0000000..5d38eb9 --- /dev/null +++ b/tools/faodel-stress/JobMemoryAlloc.cpp @@ -0,0 +1,84 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include +#include + +#include "lunasa/Lunasa.hh" +#include "lunasa/DataObject.hh" + +#include "Worker.hh" +#include "JobMemoryAlloc.hh" + + +using namespace std; + +class WorkerMemoryAlloc + : public Worker { + +public: + WorkerMemoryAlloc() = default; + WorkerMemoryAlloc(int id, JobMemoryAlloc::params_t params) + : Worker(id, params.num_items, params.min_ldo_size, params.max_ldo_size), + params(params) { + + //The eager allocator by default uses pinned memory, while lazy defers until use. + alloc_type = (params.netmem) ? lunasa::DataObject::AllocatorType::eager + : lunasa::DataObject::AllocatorType::lazy; + + } + ~WorkerMemoryAlloc() = default; + + void server() { + + + std::function f_createobject; + + if(params.min_ldo_size == params.max_ldo_size) { + f_createobject = [this]() { + return lunasa::DataObject(0, params.max_ldo_size, alloc_type); + }; + } else { + f_createobject = [this]() { + return lunasa::DataObject(0, prngGetRangedInteger(), alloc_type); + }; + } + + + do { + vector ldos(params.num_items); + for(int i=0; i(job_name, options); +} + diff --git a/tools/faodel-stress/JobMemoryAlloc.hh b/tools/faodel-stress/JobMemoryAlloc.hh new file mode 100644 index 0000000..6385320 --- /dev/null +++ b/tools/faodel-stress/JobMemoryAlloc.hh @@ -0,0 +1,47 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_JOBMEMORYALLOC_HH +#define FAODEL_JOBMEMORYALLOC_HH + +#include "Job.hh" + +/*** + * @brief Request different memory allocations from lunasa + * + * Faodel uses Lunasa to allocate memory for objects that are shipped over + * the network and hide the high overhead of obtaining registered memory + * from applications. This test allocates different sizes of memory and then + * releases them. Memory can either be plain memory (via the lazy allocator) + * or registered memory (via the eager allocator). + */ +class JobMemoryAlloc : + public Job { + +public: + explicit JobMemoryAlloc(const faodel::Configuration &config); + + ~JobMemoryAlloc() override = default; + + int Execute(const std::string &job_name) override; + + + struct params_t { uint32_t num_items; bool netmem; uint32_t min_ldo_size; uint32_t max_ldo_size; }; + const std::map options = { + {"PlainMem-FixedSize-1K", { 1024, false, 1024,1024}}, + {"PlainMem-FixedSize-1M", { 1024, false, 1024*1024,1024*1024}}, + {"PlainMem-RandomSize-1K", { 1024, false, 128, 1024}}, + {"PlainMem-RandomSize-1M", { 1024, false, 1024, 1024*1024}}, + + {"RegisteredMem-FixedSize-1K", { 1024, true, 1024,1024}}, + {"RegisteredMem-FixedSize-1M", { 1024, true, 1024*1024,1024*1024}}, + {"RegisteredMem-RandomSize-1K", { 1024, true, 128, 1024}}, + {"RegisteredMem-RandomSize-1M", { 1024, true, 1024, 1024*1024}} + }; + constexpr const char* JobCategoryName() { return "memalloc"; } + +}; + + +#endif //FAODEL_JOBMEMORYALLOC_HH diff --git a/tools/faodel-stress/JobSerdes.cpp b/tools/faodel-stress/JobSerdes.cpp new file mode 100644 index 0000000..74f28f2 --- /dev/null +++ b/tools/faodel-stress/JobSerdes.cpp @@ -0,0 +1,61 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include +#include + +#include "Worker.hh" +#include "JobSerdes.hh" +#include "serdes/WorkerSerdes.hh" +#include "serdes/SerdesParticleBundleObject.hh" +#include "serdes/SerdesStringObject.hh" + + +using namespace std; + +JobSerdes::JobSerdes(const faodel::Configuration &config) + : Job(config, JobCategoryName()) { + + for(auto &name_params : options) + job_names.push_back(name_params.first); + +} + +int JobSerdes::Execute(const std::string &job_name) { + + auto it = options.find(job_name); + if (it==options.end()) return -1; + dbg("Launching "+to_string(num_threads)+" worker threads"); + + vector w_serdes(num_threads); + for(int i=0; isecond.obj_type) { + case 1: w_serdes.at(i) = new WorkerSerdes(i, it->second); break; + case 2: w_serdes.at(i) = new WorkerSerdes(i, it->second); break; + default: + F_ASSERT(0, "Unknown obj_type in JobSerdes"); + } + } + testStart(); + for(auto *w : w_serdes) w->Start(); + + testSleep(); + for(auto &w : w_serdes) w->Stop(); + + testStop(); + + + for(auto *w : w_serdes) { + ops_completed += w->GetOpsCompleted(); + dbg("Thread ops completed: "+std::to_string(w->GetOpsCompleted())); + delete w; + } + + DumpJobStats(job_name); + + return 0; +} + diff --git a/tools/faodel-stress/JobSerdes.hh b/tools/faodel-stress/JobSerdes.hh new file mode 100644 index 0000000..f07c84b --- /dev/null +++ b/tools/faodel-stress/JobSerdes.hh @@ -0,0 +1,80 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_JOBSERDES_HH +#define FAODEL_JOBSERDES_HH + +#include + +#include "Job.hh" + +/*** + * @brief A stress test that serializes/deserializes different objects to/from Lunasa DataObjects + * + * Faodel allows you to use whatever serialization library you want, provided + * that object data is packed into a single, continuous buffer. There's always + * a tradeoff between how easy it is to serdes an object and how quickly it + * can be converted. This test packs a few different types of data structure + * into LDOs, using Boost or Lunasa's serdes helpers. + */ +class JobSerdes : + public Job { + +public: + explicit JobSerdes(const faodel::Configuration &config); + + ~JobSerdes() override = default; + + int Execute(const std::string &job_name) override; + + struct params_t { uint32_t num_iters; int obj_type; int method; bool pack; bool unpack; int num_items; int item_len_min; int item_len_max;}; + const std::map options = { + {"Strings-Pack-Small-Boost", {64, 1, 1, true, false,16, 4, 16}}, + {"Strings-Unpack-Small-Boost", {64, 1, 1, false,true, 16, 4, 16}}, + {"Strings-PackUnpack-Small-Boost", {64, 1, 1, true, true, 16, 4, 16}}, + + {"Strings-Pack-Small-Cereal", {64, 1, 2, true, false,16, 4, 16}}, + {"Strings-Unpack-Small-Cereal", {64, 1, 2, false,true, 16, 4, 16}}, + {"Strings-PackUnpack-Small-Cereal", {64, 1, 2, true, true, 16, 4, 16}}, + + {"Strings-Pack-Small-LDOPacker", {64, 1, 3, true, false,16, 4, 16}}, + {"Strings-Unpack-Small-LDOPacker", {64, 1, 3, false,true, 16, 4, 16}}, + {"Strings-PackUnpack-Small-LDOPacker", {64, 1, 3, true, true, 16, 4, 16}}, + + + {"Strings-Pack-Large-Boost", {64, 1, 1, true, false,256, 32, 256}}, + {"Strings-Unpack-Large-Boost", {64, 1, 1, false,true, 256, 32, 256}}, + {"Strings-PackUnpack-Large-Boost", {64, 1, 1, true, true, 256, 32, 256}}, + + {"Strings-Pack-Large-Cereal", {64, 1, 2, true, false,256, 32, 256}}, + {"Strings-Unpack-Large-Cereal", {64, 1, 2, false,true, 256, 32, 256}}, + {"Strings-PackUnpack-Large-Cereal", {64, 1, 2, true, true, 256, 32, 256}}, + + + {"Strings-Pack-Large-LDOPacker", {64, 1, 3, true, false,256, 32, 256}}, + {"Strings-Unpack-Large-LDOPacker", {64, 1, 3, false,true, 256, 32, 256}}, + {"Strings-PackUnpack-Large-LDOPacker", {64, 1, 3, true, true, 256, 32, 256}}, + + + {"Particles-Pack-Small-Boost", {64, 2, 1, true, false,1024, 0, 0}}, + {"Particles-Unpack-Small-Boost", {64, 2, 1, false,true, 1024, 0, 0}}, + {"Particles-PackUnpack-Small-Boost", {64, 2, 1, true, true, 1024, 0, 0}}, + + {"Particles-Pack-Small-Cereal", {64, 2, 2, true, false,1024, 0, 0}}, + {"Particles-Unpack-Small-Cereal", {64, 2, 2, false,true, 1024, 0, 0}}, + {"Particles-PackUnpack-Small-Cereal", {64, 2, 2, true, true, 1024, 0, 0}}, + + {"Particles-Pack-Small-LDOPacker", {64, 2, 3, true, false,1024, 0, 0}}, + {"Particles-Unpack-Small-LDOPacker", {64, 2, 3, false,true, 1024, 0, 0}}, + {"Particles-PackUnpack-Small-LDOPacker", {64, 2, 3, true, true, 1024, 0, 0}} + + }; + + constexpr const char* JobCategoryName() { return "serdes"; } + +}; + + + +#endif //FAODEL_JOBSERDES_HH diff --git a/tools/faodel-stress/JobWebClient.cpp b/tools/faodel-stress/JobWebClient.cpp new file mode 100644 index 0000000..1abafdf --- /dev/null +++ b/tools/faodel-stress/JobWebClient.cpp @@ -0,0 +1,118 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include +#include + +#include "whookie/Whookie.hh" +#include "whookie/Server.hh" + +#include "Worker.hh" +#include "JobWebClient.hh" + +using namespace std; + +class WorkerWebClient + : public Worker { + +public: + WorkerWebClient() = default; + WorkerWebClient(int id, JobWebClient::params_t params) + : Worker(id, params.batch_size, params.item_len_min, params.item_len_max), + params(params), + bytes_retrieved(0) { + + nid = whookie::Server::GetNodeID(); + path = ((params.item_len_min==0)&&(params.item_len_max==0)) ? "/stress/minimal" + : "/stress/sized"; + + } + ~WorkerWebClient() = default; + + void server() { + + std::function f_createpathsuffix; + if(params.item_len_min == params.item_len_max) { + if(params.item_len_min==0) + //Fixed, minimal length + f_createpathsuffix = []() { return ""; }; + else + //Fixed length + f_createpathsuffix = [this]() { + return "&size="+std::to_string(params.item_len_min); + }; + } else { + //Random length + f_createpathsuffix = [this]() { + return "&size="+std::to_string(prngGetRangedInteger()); + }; + } + + size_t bytes_retrieved=0; + do { + for(int i=0; i < params.batch_size; i++) { + string s; + whookie::retrieveData(nid, path+f_createpathsuffix(), &s); + bytes_retrieved += s.size(); + } + ops_completed += params.batch_size; + } while(!kill_server); + } +private: + + JobWebClient::params_t params; + int id; + + faodel::nodeid_t nid; + string path; + + size_t bytes_retrieved; + +}; + + + + + +JobWebClient::JobWebClient(const faodel::Configuration &config) + : Job(config, JobCategoryName()) { + + for(auto &name_params : options) + job_names.push_back(name_params.first); + +} + +int JobWebClient::Init() { + + //Register a simple nop + whookie::Server::registerHook("/stress/minimal", [this] (const map &args, stringstream &results) { + }); + + //Register a more complicated callback to a function that handles different sizes + whookie::Server::registerHook("/stress/sized", [this] (const map &args, stringstream &results) { + return wh_Reply(args, results); + }); + + return 0; +} + +int JobWebClient::Execute(const std::string &job_name) { + return standardExecuteWorker(job_name, options); +} + +int JobWebClient::wh_Reply(const map &args, stringstream &results) const { + + string s; //Empty + auto it = args.find("size"); + if(it!=args.end()) { + uint32_t num_bytes=1; + faodel::StringToUInt32(&num_bytes, it->second); + s=std::string(num_bytes, 'x'); + } + //cout<<"whookie reply w. string len "< + +#include "Job.hh" + +class JobWebClient : + public Job { +public: + explicit JobWebClient(const faodel::Configuration &config); + + ~JobWebClient() override = default; + + int Init() override; + + int Execute(const std::string &job_name) override; + + int wh_Reply(const std::map &args, std::stringstream &results) const; + + struct params_t { uint32_t batch_size; int item_len_min; int item_len_max;}; + const std::map options = { + {"GetEmpty", {1024, 0, 0}}, + {"GetFixed-128", {1024, 128, 128}}, + {"GetFixed-1K", {1024, 1024, 1024}}, + {"GetRandom-128", {1024, 128-64, 128+64}}, + {"GetRandom-1K", {1024, 1024-128, 1024+128}} + }; + + constexpr const char* JobCategoryName() { return "webclient"; } + +}; + + +#endif //FAODEL_JOBWEBCLIENT_HH diff --git a/tools/faodel-stress/README_Faodel_Stress.md b/tools/faodel-stress/README_Faodel_Stress.md new file mode 100644 index 0000000..6b60aca --- /dev/null +++ b/tools/faodel-stress/README_Faodel_Stress.md @@ -0,0 +1,54 @@ +Faodel-Stress: A Tool for Examining the Performance of Host Systems +===================================================================== + +When porting FAODEL to a new platform, it can be useful to run some basic +benchmarks to understand the strengths and weaknesses of the new platform. +The [stress-ng][https://wiki.ubuntu.com/Kernel/Reference/stress-ng] tool +is an excellent collection of microbenchmarks that can help you compare +a new system to an old system. + +The `faodel-stress` tool follows `stress-ng`'s example and provides a +collection of stress tests that are relevant to data management tasks. +Each test performs the same operation in a tight loop for a specified +amount of time. Users can vary the number of worker threads used in +each test to explore how well the system scales. Performance is reported +back to the user in terms of millions of operations per second. (Mops/s), +where an operation is a basic but non-trival task (eg, hashing a string). + +While it is meaningless to compare the Mops/s of one stressor to another, +users can use the reported numbers to compare different machines or +configurations. + +Stressors +--------- +The basic categories of stressors are as follows: + +- **keys:**: These stressors generate a collection of Kelpie keys and then + sort them. +- **memalloc**: Lunasa's memory allocator is used to obtain either plain + memory (ie, not registered with the nic) or registered memory (ie, + memory the nic can access). +- **localpool**: Kelpie is used to write a large number of objects into + the node's local key/blob store. These tests vary whether threads write + to the same row (ie, maximize contention) or independent rows. +- **serdes**: Multiple data structures are serialized/deserialized using + a variety of packing libraries. The Particles example mimics a particle + dataset where there are many particles with a small number of data values. + The Strings example packs a variety of variable-length strings into + an object. +- **webclient**: These stressors start a whookie server and then issue + a number of web get commands to fetch data. These operations take + place over traditional sockets. + +Recommended Settings +-------------------- +It is useful to start by running the stressors with a single thread and +a short duration (5s) to verify the system can run to completion. As +more threads and longer runtimes are used, it is likely that a few tests +will exhaust all the resources in the system and cause the test to hang. +Using a duration of 30s is generally good enough to load a system and +get usable numbers. + + + + diff --git a/tools/faodel-stress/Worker.cpp b/tools/faodel-stress/Worker.cpp new file mode 100644 index 0000000..d58dc9c --- /dev/null +++ b/tools/faodel-stress/Worker.cpp @@ -0,0 +1,40 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include "Worker.hh" + + +using namespace std; + +Worker::Worker() + : id(0), thread_launched(false), kill_server(false), batch_size(0), ops_completed(0) { +} +Worker::Worker(int id, uint32_t batch_size, uint32_t min_prng, uint32_t max_prng) + : id(id), + thread_launched(false), kill_server(false), + batch_size(batch_size), ops_completed(0), + prngGen(id), //Seed with the thread id + prngDistrib(min_prng, max_prng) { + +} + +Worker::~Worker(){ + Stop(); +} + +void Worker::Start(){ + F_ASSERT(!thread_launched, "Attempted to start work when thread already started"); + th_server = thread(&Worker::server, this); + kill_server=false; + thread_launched=true; +} + +void Worker::Stop(){ + if(thread_launched){ + kill_server=true; + th_server.join(); + thread_launched=false; + } + +} diff --git a/tools/faodel-stress/Worker.hh b/tools/faodel-stress/Worker.hh new file mode 100644 index 0000000..cdcf56c --- /dev/null +++ b/tools/faodel-stress/Worker.hh @@ -0,0 +1,58 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_WORKER_HH +#define FAODEL_WORKER_HH + +#include +#include +#include +#include +#include +#include + +#include "faodel-common/Debug.hh" + +class Worker { +public: + Worker(); + Worker(int id, uint32_t batch_size, uint32_t min_prng, uint32_t max_prng); + Worker(const Worker &p) + : id(p.id), + kill_server(p.kill_server), + batch_size(p.batch_size), + ops_completed(p.ops_completed), + prngGen(p.prngGen), + prngDistrib(p.prngDistrib), + thread_launched(false) { + F_ASSERT(!p.thread_launched, "Creating worker when it's already started?"); + } + virtual ~Worker(); + + void Start(); + void Stop(); + + uint64_t GetOpsCompleted() const { return ops_completed; } + +protected: + int id; + bool kill_server; + uint32_t batch_size; + uint64_t ops_completed; + + //Ranged pseudo random number generator + //Most workers need a prng that falls within a range. + std::mt19937 prngGen; + std::uniform_int_distribution prngDistrib; + uint32_t prngGetRangedInteger() { return prngDistrib(prngGen); } + +private: + bool thread_launched; + virtual void server() = 0; + std::thread th_server; + +}; + + +#endif //FAODEL_WORKER_HH diff --git a/tools/faodel-stress/faodel-stress.cpp b/tools/faodel-stress/faodel-stress.cpp new file mode 100644 index 0000000..290f9a9 --- /dev/null +++ b/tools/faodel-stress/faodel-stress.cpp @@ -0,0 +1,141 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include +#include + +#include "kelpie/Kelpie.hh" + +#include "Job.hh" +#include "JobKeys.hh" +#include "JobLocalPool.hh" +#include "JobMemoryAlloc.hh" +#include "JobWebClient.hh" +#include "JobSerdes.hh" + +using namespace std; + +string default_config = R"EOF( +dirman.host_root true +net.transport.name mpi + +#localpool.debug true + +)EOF"; + +void dumpHelp() { + + cout << R"( +faodel-stress + + options: + -n num_workers : Number of threads to use in each test (default: 1) + -t work_duration : Amount of time to run each test (default: 5s) + -f test1,test2... : Filter down the tests to run (default: all) + + -x : Generate tabular output (tab-separated) + -v/-V : Turn on verbose/very-verbose logging + -l : List all the available tests and exit + +This program contains a set of simple stress tests to see how quickly a +system can complete common faodel operations. These bogus numbers help users +compare different platforms or compile options. + +Note: a filter can either take a 'category:all' form (eg, 'serdes:all') or + the name of individual tests (eg, 'PutGetDrop-1D,GenSort-Long2DKey'). + +)"; +} + +#include +int main(int argc, char **argv) { + + int num_threads=1; + int verbose_level=0; + bool list_tests=false; + bool dump_tsv=false; + string duration="5s"; + string test_names="all"; + + faodel::Configuration config(default_config); + config.AppendIfUnset("faodel-stress.time_limit", duration); + config.AppendIfUnset("faodel-stress.num_threads", to_string(num_threads)); + + int c; + while((c=getopt(argc,argv,"n:t:f:lxvVh")) != -1){ + switch(c){ + case 'n': num_threads = atoi(optarg); config.Append("faodel-stress.num_threads", to_string(num_threads)); break; + case 't': duration = string(optarg); + if((!duration.empty()) && (isdigit(duration[duration.size()-1]))) + duration=duration+"s"; + config.Append("faodel-stress.time_limit", duration); + break; + case 'f': test_names = string(optarg); break; + case 'l': list_tests = true; break; + case 'x': dump_tsv = true; break; + case 'v': verbose_level=1; break; + case 'V': verbose_level=2; break; + case 'h': dumpHelp(); exit(0); break; + default: + cout << "Unknown option '-"< stressors; + stressors.push_back(new JobKeys(config)); + stressors.push_back(new JobMemoryAlloc(config)); + stressors.push_back(new JobLocalPool(config)); + stressors.push_back(new JobSerdes(config)); + stressors.push_back(new JobWebClient(config)); + + //List tests and exit if requested + if(list_tests) { + cout<<"Category Test\n" + <<"---------- ----------\n"; + for(auto *s : stressors) { + s->DumpJobNames(); + delete s; + } + exit(0); + } + + //Initialize/start faodel. Give each stress chance a chance to do pre-start ops + faodel::bootstrap::Init(config, kelpie::bootstrap); + for(auto s : stressors) { + s->Init(); + if(verbose_level>0) s->ConfigureLoggingDebug(true); + } + faodel::bootstrap::Start(); + + //Step through all stress test and run if needed + for(auto s : stressors) { + int num_jobs = s->Setup(test_names); + if(num_jobs) { + s->ExecuteAll(); + s->Teardown(); + } + } + for(auto *s : stressors) + delete s; + + faodel::bootstrap::Finish(); + +} diff --git a/tools/faodel-stress/serdes/SerdesParticleBundleObject.cpp b/tools/faodel-stress/serdes/SerdesParticleBundleObject.cpp new file mode 100644 index 0000000..ffb03a2 --- /dev/null +++ b/tools/faodel-stress/serdes/SerdesParticleBundleObject.cpp @@ -0,0 +1,100 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include + +#include "faodel-common/Debug.hh" +#include "SerdesParticleBundleObject.hh" + +using namespace std; + +SerdesParticleBundleObject::SerdesParticleBundleObject( + JobSerdes::params_t params, + std::function f_prng) { + + px.resize(params.num_items); + py.resize(params.num_items); + pz.resize(params.num_items); + + vx.resize(params.num_items); + vy.resize(params.num_items); + vz.resize(params.num_items); + + val1.resize(params.num_items); + val2.resize(params.num_items); + + for(int i=0; i names = {"px","py","pz", "vx","vy","vz", "val1", "val2"}; + + //Pointers to each component + vector ptrs = { + px.data(), py.data(), pz.data(), + vx.data(), vy.data(), vz.data(), + val1.data(), val2.data() + }; + + //Bytes for each component + size_t dvec_bytes = px.size() * sizeof(double); + vector bytes = { + dvec_bytes, dvec_bytes, dvec_bytes, + dvec_bytes, dvec_bytes, dvec_bytes, + val1.size() * sizeof(uint32_t), val2.size() * sizeof(uint32_t) + }; + + //We can use whatever label we want here. + const uint8_t T_CHAR = 1; + const uint8_t T_INT =2; + const uint8_t T_FLOAT = 3; + const uint8_t T_DOUBLE = 4; + + //Types for each component + vector types = { + T_DOUBLE, T_DOUBLE, T_DOUBLE, + T_DOUBLE, T_DOUBLE, T_DOUBLE, + T_INT, T_INT + }; + + lunasa::DataObjectPacker packer(names, ptrs, bytes, types, + faodel::const_hash32("ParticleBundle")); + + return packer.GetDataObject(); +} + + +void SerdesParticleBundleObject::pup(const lunasa::DataObject &ldo) { + + lunasa::DataObjectPacker u(ldo); + void *ptr; size_t bytes; uint8_t type; + + if(!u.VerifyDataType(faodel::const_hash32("ParticleBundle"))) { + F_ASSERT(0, "Packed DataObject did not match the expected hash"); + } + + int rc=0; + + //Dangerous if anything goes wrong, but should be fairly fast + rc|=u.GetVarPointer("px", &ptr, &bytes, &type); px.resize(bytes/sizeof(double)); std::memcpy(&px[0], ptr, bytes); + rc|=u.GetVarPointer("py", &ptr, &bytes, &type); py.resize(bytes/sizeof(double)); std::memcpy(&py[0], ptr, bytes); + rc|=u.GetVarPointer("pz", &ptr, &bytes, &type); pz.resize(bytes/sizeof(double)); std::memcpy(&pz[0], ptr, bytes); + + rc|=u.GetVarPointer("vx", &ptr, &bytes, &type); vx.resize(bytes/sizeof(double)); std::memcpy(&vx[0], ptr, bytes); + rc|=u.GetVarPointer("vy", &ptr, &bytes, &type); vy.resize(bytes/sizeof(double)); std::memcpy(&vy[0], ptr, bytes); + rc|=u.GetVarPointer("vz", &ptr, &bytes, &type); vz.resize(bytes/sizeof(double)); std::memcpy(&vz[0], ptr, bytes); + + rc|=u.GetVarPointer("val1", &ptr, &bytes, &type); val1.resize(bytes/sizeof(uint32_t)); std::memcpy(&val1[0], ptr, bytes); + rc|=u.GetVarPointer("val2", &ptr, &bytes, &type); val2.resize(bytes/sizeof(uint32_t)); std::memcpy(&val2[0], ptr, bytes); + + if(rc!=0){ std::cerr<<"SerdesParticleBundleObject::pup detected an error\n"; } +} \ No newline at end of file diff --git a/tools/faodel-stress/serdes/SerdesParticleBundleObject.hh b/tools/faodel-stress/serdes/SerdesParticleBundleObject.hh new file mode 100644 index 0000000..7f2003c --- /dev/null +++ b/tools/faodel-stress/serdes/SerdesParticleBundleObject.hh @@ -0,0 +1,52 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_SERDESPARTICLEBUNDLEOBJECT_HH +#define FAODEL_SERDESPARTICLEBUNDLEOBJECT_HH + + +#include +#include +#include +#include + +#include "lunasa/common/DataObjectPacker.hh" + +#include "../JobSerdes.hh" + +// A mock up bundle of generic particles, organized as a struct of arrays. + +class SerdesParticleBundleObject { +public: + SerdesParticleBundleObject() = default; + SerdesParticleBundleObject(JobSerdes::params_t params, std::function f_prng); + + //Serialization hook + template + void serialize(Archive &ar, const unsigned int version){ + ar & px; ar & py; ar & pz; + ar & vx; ar & vy; ar & vz; + ar & val1; ar & val2; + } + + lunasa::DataObject pup() const; + void pup(const lunasa::DataObject &ldo); + + + //Actual data + std::vector px; + std::vector py; + std::vector pz; + + std::vector vx; + std::vector vy; + std::vector vz; + + std::vector val1; + std::vector val2; + + +}; + +#endif //FAODEL_SERDESPARTICLEBUNDLEOBJECT_HH diff --git a/tools/faodel-stress/serdes/SerdesStringObject.cpp b/tools/faodel-stress/serdes/SerdesStringObject.cpp new file mode 100644 index 0000000..e5c3467 --- /dev/null +++ b/tools/faodel-stress/serdes/SerdesStringObject.cpp @@ -0,0 +1,63 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#include +#include "lunasa/common/GenericSequentialDataBundle.hh" + +#include "SerdesStringObject.hh" + + +using namespace std; + +SerdesStringObject::SerdesStringObject(JobSerdes::params_t params, + std::function f_prng) { + + //Populate our string list + for(int i=0; i bundle_t; + +lunasa::DataObject SerdesStringObject::pup() const { + + // Since this is a series of random length strings, the easiest thing to + // do is just pack them into an LDO using the GenericSequentialBundler + // class. You allocate space for all the strings, and then use a + // bundle_offsets_t to keep track of where you are in the LDO. + + + //Figure out how much space our strings need. Note: each item as a 32b length + uint32_t payload_size=0; + for(auto &s: strings) { + payload_size += s.size() + sizeof(uint32_t); + } + + //Allocate an LDO, overlay our bundle structure on it, and wipe the header + lunasa::DataObject ldo(sizeof(bundle_t), payload_size,lunasa::DataObject::AllocatorType::eager); + auto *msg = ldo.GetMetaPtr(); + msg->Init(); + + //Use the offsets to track where we are so we don't overflow + lunasa::bundle_offsets_t counters(&ldo); + for(auto &s : strings) { + bool ok=msg->AppendBack(counters, s); + if(!ok) std::cerr<<"Serialization problems in SerdesStringObject\n"; + } + + return ldo; +} +void SerdesStringObject::pup(const lunasa::DataObject &ldo) { + + auto *msg = ldo.GetMetaPtr(); + lunasa::bundle_offsets_t counters(&ldo); + + strings.resize(0); + string s; + while(msg->GetNext(counters, &s)) { + strings.emplace_back(s); + } +} \ No newline at end of file diff --git a/tools/faodel-stress/serdes/SerdesStringObject.hh b/tools/faodel-stress/serdes/SerdesStringObject.hh new file mode 100644 index 0000000..42a7316 --- /dev/null +++ b/tools/faodel-stress/serdes/SerdesStringObject.hh @@ -0,0 +1,39 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_SERDESSTRINGOBJECT_HH +#define FAODEL_SERDESSTRINGOBJECT_HH + +#include +#include +#include +#include + +#include "lunasa/common/DataObjectPacker.hh" + +#include "../JobSerdes.hh" + +class SerdesStringObject { +public: + SerdesStringObject() = default; + SerdesStringObject(JobSerdes::params_t params, std::function f_prng); + + //Serialization hook + template + void serialize(Archive &ar, const unsigned int version){ + ar & strings; + } + + //Lunasa's DataObject Packer hooks + lunasa::DataObject pup() const; + void pup(const lunasa::DataObject &ldo); + + + //Actual data + std::vector strings; + +}; + + +#endif //FAODEL_SERDESSTRINGOBJECT_HH diff --git a/tools/faodel-stress/serdes/WorkerSerdes.hh b/tools/faodel-stress/serdes/WorkerSerdes.hh new file mode 100644 index 0000000..a585307 --- /dev/null +++ b/tools/faodel-stress/serdes/WorkerSerdes.hh @@ -0,0 +1,128 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + +#ifndef FAODEL_WORKERSERDES_HH +#define FAODEL_WORKERSERDES_HH + +#include +#include +#include +#include +#include + +#include "faodel-common/SerializationHelpersBoost.hh" +#include "faodel-common/SerializationHelpersCereal.hh" +#include "lunasa/DataObject.hh" + +#include + +#include +#include +#include +#include + +#include "../Worker.hh" + + + +// Boilerplate code to pack/unpack a bunch of objects. The T class should +// be one of the Serdes objects, which should have (1) a simple ctor, +// (2) a templated serialize function for boost serialization, and +// (3) pup functions for manually packing/unpacking using lunasa helpers. + + +template +class WorkerSerdes + : public Worker { + +public: + WorkerSerdes() = default; + + WorkerSerdes(int id, JobSerdes::params_t params) + : Worker(id, params.num_iters, params.item_len_min, params.item_len_max), + params(params), + pack_objects(params.pack), + unpack_objects(params.unpack) { + + if(params.method==1) { //Boost Serialization + f_pack = [](const T &obj) { + //Have boost make a string that we then copy into an LDO + std::string s=faodel::BoostPack(obj); + lunasa::DataObject ldo(s.length()); + std::memcpy(ldo.GetDataPtr(), s.c_str(), s.length()); + return ldo; + }; + f_unpack = [](const lunasa::DataObject &ldo) { + //Pull the string out and let boost do the hard work + std::string s(ldo.GetDataPtr(), ldo.GetDataSize()); + return faodel::BoostUnpack(s); + }; + } else if (params.method==2) { //Cereal Serialization + f_pack = [](const T &obj) { + //Have Cereal make a string that we then copy into an LDO + std::string s=faodel::CerealPack(obj); + lunasa::DataObject ldo(s.length()); + std::memcpy(ldo.GetDataPtr(), s.c_str(), s.length()); + return ldo; + }; + f_unpack = [](const lunasa::DataObject &ldo) { + //Pull the string out and let boost do the hard work + std::string s(ldo.GetDataPtr(), ldo.GetDataSize()); + return faodel::CerealUnpack(s); + }; + } else if (params.method==3) { //LDO Serialization + //Use the pack/unpack function built into each class to serialize + f_pack = [](const T &obj) { return obj.pup(); }; + f_unpack = [](const lunasa::DataObject &ldo) { + T obj; + obj.pup(ldo); + return obj; + }; + } else { + F_ASSERT(0, "Unknown method passed to WorkerSerdes?"); + } + + //Create initial objects and their encoded values. Lets us do pack/unpack/pack+unpack + objs.resize(batch_size); + packed_objs.resize(batch_size); + for(int i=0; i f_pack; + std::function f_unpack; + + bool pack_objects; + bool unpack_objects; + + std::vector objs; + std::vector packed_objs; + JobSerdes::params_t params; +}; + + +#endif //FAODEL_WORKERSERDES_HH diff --git a/tools/kelpie-server/kelpie_server_main.cpp b/tools/kelpie-server/kelpie_server_main.cpp index b6b4b99..1ef64ef 100644 --- a/tools/kelpie-server/kelpie_server_main.cpp +++ b/tools/kelpie-server/kelpie_server_main.cpp @@ -1,3 +1,7 @@ +// Copyright 2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. + #include #include diff --git a/tpl/cereal/CMakeLists.txt b/tpl/cereal/CMakeLists.txt index 1d46ffa..8c3a579 100644 --- a/tpl/cereal/CMakeLists.txt +++ b/tpl/cereal/CMakeLists.txt @@ -28,10 +28,13 @@ endif() SET( CEREAL_INCLUDES "${CMAKE_CURRENT_SOURCE_DIR}/include" PARENT_SCOPE ) -add_library(cereal INTERFACE) -target_include_directories(cereal INTERFACE +add_library(tpl-cereal INTERFACE) +target_include_directories(tpl-cereal INTERFACE $ ) -install(TARGETS cereal +install(TARGETS tpl-cereal EXPORT FaodelTargets - DESTINATION lib) # ignored + INCLUDES DESTINATION "${INCLUDE_INSTALL_DIR}/faodel/tpl-cereal" ) +install( DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/include/cereal + DESTINATION ${INCLUDE_INSTALL_DIR}/faodel/tpl-cereal) + diff --git a/tpl/cereal/include/cereal/external/rapidxml/license.txt b/tpl/cereal/include/cereal/external/rapidxml/license.txt index 0095bc7..1409831 100644 --- a/tpl/cereal/include/cereal/external/rapidxml/license.txt +++ b/tpl/cereal/include/cereal/external/rapidxml/license.txt @@ -1,52 +1,52 @@ -Use of this software is granted under one of the following two licenses, -to be chosen freely by the user. - -1. Boost Software License - Version 1.0 - August 17th, 2003 -=============================================================================== - -Copyright (c) 2006, 2007 Marcin Kalicinski - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -2. The MIT License -=============================================================================== - -Copyright (c) 2006, 2007 Marcin Kalicinski - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. +Use of this software is granted under one of the following two licenses, +to be chosen freely by the user. + +1. Boost Software License - Version 1.0 - August 17th, 2003 +=============================================================================== + +Copyright (c) 2006, 2007 Marcin Kalicinski + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +2. The MIT License +=============================================================================== + +Copyright (c) 2006, 2007 Marcin Kalicinski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/tpl/cereal/include/cereal/external/rapidxml/manual.html b/tpl/cereal/include/cereal/external/rapidxml/manual.html index 4e6a5f1..2c42270 100644 --- a/tpl/cereal/include/cereal/external/rapidxml/manual.html +++ b/tpl/cereal/include/cereal/external/rapidxml/manual.html @@ -1,406 +1,406 @@ -

RAPIDXML Manual

Version 1.13

Copyright (C) 2006, 2009 Marcin Kalicinski
See accompanying file license.txt for license information.

Table of Contents

1. What is RapidXml?
1.1 Dependencies And Compatibility
1.2 Character Types And Encodings
1.3 Error Handling
1.4 Memory Allocation
1.5 W3C Compliance
1.6 API Design
1.7 Reliability
1.8 Acknowledgements
2. Two Minute Tutorial
2.1 Parsing
2.2 Accessing The DOM Tree
2.3 Modifying The DOM Tree
2.4 Printing XML
3. Differences From Regular XML Parsers
3.1 Lifetime Of Source Text
3.2 Ownership Of Strings
3.3 Destructive Vs Non-Destructive Mode
4. Performance
4.1 Comparison With Other Parsers
5. Reference

1. What is RapidXml?

RapidXml is an attempt to create the fastest XML DOM parser possible, while retaining useability, portability and reasonable W3C compatibility. It is an in-situ parser written in C++, with parsing speed approaching that of strlen() function executed on the same data.

- Entire parser is contained in a single header file, so no building or linking is neccesary. To use it you just need to copy rapidxml.hpp file to a convenient place (such as your project directory), and include it where needed. You may also want to use printing functions contained in header rapidxml_print.hpp.

1.1 Dependencies And Compatibility

RapidXml has no dependencies other than a very small subset of standard C++ library (<cassert>, <cstdlib>, <new> and <exception>, unless exceptions are disabled). It should compile on any reasonably conformant compiler, and was tested on Visual C++ 2003, Visual C++ 2005, Visual C++ 2008, gcc 3, gcc 4, and Comeau 4.3.3. Care was taken that no warnings are produced on these compilers, even with highest warning levels enabled.

1.2 Character Types And Encodings

RapidXml is character type agnostic, and can work both with narrow and wide characters. Current version does not fully support UTF-16 or UTF-32, so use of wide characters is somewhat incapacitated. However, it should succesfully parse wchar_t strings containing UTF-16 or UTF-32 if endianness of the data matches that of the machine. UTF-8 is fully supported, including all numeric character references, which are expanded into appropriate UTF-8 byte sequences (unless you enable parse_no_utf8 flag).

- Note that RapidXml performs no decoding - strings returned by name() and value() functions will contain text encoded using the same encoding as source file. Rapidxml understands and expands the following character references: &apos; &amp; &quot; &lt; &gt; &#...; Other character references are not expanded.

1.3 Error Handling

By default, RapidXml uses C++ exceptions to report errors. If this behaviour is undesirable, RAPIDXML_NO_EXCEPTIONS can be defined to suppress exception code. See parse_error class and parse_error_handler() function for more information.

1.4 Memory Allocation

RapidXml uses a special memory pool object to allocate nodes and attributes, because direct allocation using new operator would be far too slow. Underlying memory allocations performed by the pool can be customized by use of memory_pool::set_allocator() function. See class memory_pool for more information.

1.5 W3C Compliance

RapidXml is not a W3C compliant parser, primarily because it ignores DOCTYPE declarations. There is a number of other, minor incompatibilities as well. Still, it can successfully parse and produce complete trees of all valid XML files in W3C conformance suite (over 1000 files specially designed to find flaws in XML processors). In destructive mode it performs whitespace normalization and character entity substitution for a small set of built-in entities.

1.6 API Design

RapidXml API is minimalistic, to reduce code size as much as possible, and facilitate use in embedded environments. Additional convenience functions are provided in separate headers: rapidxml_utils.hpp and rapidxml_print.hpp. Contents of these headers is not an essential part of the library, and is currently not documented (otherwise than with comments in code).

1.7 Reliability

RapidXml is very robust and comes with a large harness of unit tests. Special care has been taken to ensure stability of the parser no matter what source text is thrown at it. One of the unit tests produces 100,000 randomly corrupted variants of XML document, which (when uncorrupted) contains all constructs recognized by RapidXml. RapidXml passes this test when it correctly recognizes that errors have been introduced, and does not crash or loop indefinitely.

- Another unit test puts RapidXml head-to-head with another, well estabilished XML parser, and verifies that their outputs match across a wide variety of small and large documents.

- Yet another test feeds RapidXml with over 1000 test files from W3C compliance suite, and verifies that correct results are obtained. There are also additional tests that verify each API function separately, and test that various parsing modes work as expected.

1.8 Acknowledgements

I would like to thank Arseny Kapoulkine for his work on pugixml, which was an inspiration for this project. Additional thanks go to Kristen Wegner for creating pugxml, from which pugixml was derived. Janusz Wohlfeil kindly ran RapidXml speed tests on hardware that I did not have access to, allowing me to expand performance comparison table.

2. Two Minute Tutorial

2.1 Parsing

The following code causes RapidXml to parse a zero-terminated string named text: 
using namespace rapidxml;
-xml_document<> doc;    // character type defaults to char
-doc.parse<0>(text);    // 0 means default parse flags
-
doc object is now a root of DOM tree containing representation of the parsed XML. Because all RapidXml interface is contained inside namespace rapidxml, users must either bring contents of this namespace into scope, or fully qualify all the names. Class xml_document represents a root of the DOM hierarchy. By means of public inheritance, it is also an xml_node and a memory_pool. Template parameter of xml_document::parse() function is used to specify parsing flags, with which you can fine-tune behaviour of the parser. Note that flags must be a compile-time constant.

2.2 Accessing The DOM Tree

To access the DOM tree, use methods of xml_node and xml_attribute classes: 
cout << "Name of my first node is: " << doc.first_node()->name() << "\n";
-xml_node<> *node = doc.first_node("foobar");
-cout << "Node foobar has value " << node->value() << "\n";
-for (xml_attribute<> *attr = node->first_attribute();
-     attr; attr = attr->next_attribute())
-{
-    cout << "Node foobar has attribute " << attr->name() << " ";
-    cout << "with value " << attr->value() << "\n";
-}
-

2.3 Modifying The DOM Tree

DOM tree produced by the parser is fully modifiable. Nodes and attributes can be added/removed, and their contents changed. The below example creates a HTML document, whose sole contents is a link to google.com website: 
xml_document<> doc;
-xml_node<> *node = doc.allocate_node(node_element, "a", "Google");
-doc.append_node(node);
-xml_attribute<> *attr = doc.allocate_attribute("href", "google.com");
-node->append_attribute(attr);
-
One quirk is that nodes and attributes do not own the text of their names and values. This is because normally they only store pointers to the source text. So, when assigning a new name or value to the node, care must be taken to ensure proper lifetime of the string. The easiest way to achieve it is to allocate the string from the xml_document memory pool. In the above example this is not necessary, because we are only assigning character constants. But the code below uses memory_pool::allocate_string() function to allocate node name (which will have the same lifetime as the document), and assigns it to a new node: 
xml_document<> doc;
-char *node_name = doc.allocate_string(name);        // Allocate string and copy name into it
-xml_node<> *node = doc.allocate_node(node_element, node_name);  // Set node name to node_name
-
Check Reference section for description of the entire interface.

2.4 Printing XML

You can print xml_document and xml_node objects into an XML string. Use print() function or operator <<, which are defined in rapidxml_print.hpp header. 
using namespace rapidxml;
-xml_document<> doc;    // character type defaults to char
-// ... some code to fill the document
-
-// Print to stream using operator <<
-std::cout << doc;   
-
-// Print to stream using print function, specifying printing flags
-print(std::cout, doc, 0);   // 0 means default printing flags
-
-// Print to string using output iterator
-std::string s;
-print(std::back_inserter(s), doc, 0);
-
-// Print to memory buffer using output iterator
-char buffer[4096];                      // You are responsible for making the buffer large enough!
-char *end = print(buffer, doc, 0);      // end contains pointer to character after last printed character
-*end = 0;                               // Add string terminator after XML
-

3. Differences From Regular XML Parsers

RapidXml is an in-situ parser, which allows it to achieve very high parsing speed. In-situ means that parser does not make copies of strings. Instead, it places pointers to the source text in the DOM hierarchy.

3.1 Lifetime Of Source Text

In-situ parsing requires that source text lives at least as long as the document object. If source text is destroyed, names and values of nodes in DOM tree will become destroyed as well. Additionally, whitespace processing, character entity translation, and zero-termination of strings require that source text be modified during parsing (but see non-destructive mode). This makes the text useless for further processing once it was parsed by RapidXml.

- In many cases however, these are not serious issues.

3.2 Ownership Of Strings

Nodes and attributes produced by RapidXml do not own their name and value strings. They merely hold the pointers to them. This means you have to be careful when setting these values manually, by using xml_base::name(const Ch *) or xml_base::value(const Ch *) functions. Care must be taken to ensure that lifetime of the string passed is at least as long as lifetime of the node/attribute. The easiest way to achieve it is to allocate the string from memory_pool owned by the document. Use memory_pool::allocate_string() function for this purpose.

3.3 Destructive Vs Non-Destructive Mode

By default, the parser modifies source text during the parsing process. This is required to achieve character entity translation, whitespace normalization, and zero-termination of strings.

- In some cases this behaviour may be undesirable, for example if source text resides in read only memory, or is mapped to memory directly from file. By using appropriate parser flags (parse_non_destructive), source text modifications can be disabled. However, because RapidXml does in-situ parsing, it obviously has the following side-effects:

4. Performance

RapidXml achieves its speed through use of several techniques:
  • In-situ parsing. When building DOM tree, RapidXml does not make copies of string data, such as node names and values. Instead, it stores pointers to interior of the source text.
  • Use of template metaprogramming techniques. This allows it to move much of the work to compile time. Through magic of the templates, C++ compiler generates a separate copy of parsing code for any combination of parser flags you use. In each copy, all possible decisions are made at compile time and all unused code is omitted.
  • Extensive use of lookup tables for parsing.
  • Hand-tuned C++ with profiling done on several most popular CPUs.
This results in a very small and fast code: a parser which is custom tailored to exact needs with each invocation.

4.1 Comparison With Other Parsers

The table below compares speed of RapidXml to some other parsers, and to strlen() function executed on the same data. On a modern CPU (as of 2007), you can expect parsing throughput to be close to 1 GB/s. As a rule of thumb, parsing speed is about 50-100x faster than Xerces DOM, 30-60x faster than TinyXml, 3-12x faster than pugxml, and about 5% - 30% faster than pugixml, the fastest XML parser I know of.
  • The test file is a real-world, 50kB large, moderately dense XML file.
  • All timing is done by using RDTSC instruction present in Pentium-compatible CPUs.
  • No profile-guided optimizations are used.
  • All parsers are running in their fastest modes.
  • The results are given in CPU cycles per character, so frequency of CPUs is irrelevant.
  • The results are minimum values from a large number of runs, to minimize effects of operating system activity, task switching, interrupt handling etc.
  • A single parse of the test file takes about 1/10th of a millisecond, so with large number of runs there is a good chance of hitting at least one no-interrupt streak, and obtaining undisturbed results.
Platform
Compiler
strlen() RapidXml pugixml 0.3 pugxml TinyXml
Pentium 4
MSVC 8.0
2.5
5.4
7.0
61.7
298.8
Pentium 4
gcc 4.1.1
0.8
6.1
9.5
67.0
413.2
Core 2
MSVC 8.0
1.0
4.5
5.0
24.6
154.8
Core 2
gcc 4.1.1
0.6
4.6
5.4
28.3
229.3
Athlon XP
MSVC 8.0
3.1
7.7
8.0
25.5
182.6
Athlon XP
gcc 4.1.1
0.9
8.2
9.2
33.7
265.2
Pentium 3
MSVC 8.0
2.0
6.3
7.0
30.9
211.9
Pentium 3
gcc 4.1.1
1.0
6.7
8.9
35.3
316.0
(*) All results are in CPU cycles per character of source text

5. Reference

This section lists all classes, functions, constants etc. and describes them in detail.
class - template - rapidxml::memory_pool
- constructor - memory_pool()
- destructor - ~memory_pool()
function allocate_node(node_type type, const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0)
function allocate_attribute(const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0)
function allocate_string(const Ch *source=0, std::size_t size=0)
function clone_node(const xml_node< Ch > *source, xml_node< Ch > *result=0)
function clear()
function set_allocator(alloc_func *af, free_func *ff)

class rapidxml::parse_error
- constructor - parse_error(const char *what, void *where)
function what() const
function where() const

class - template - rapidxml::xml_attribute
- constructor - xml_attribute()
function document() const
function previous_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function next_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const

class - template - rapidxml::xml_base
- constructor - xml_base()
function name() const
function name_size() const
function value() const
function value_size() const
function name(const Ch *name, std::size_t size)
function name(const Ch *name)
function value(const Ch *value, std::size_t size)
function value(const Ch *value)
function parent() const

class - template - rapidxml::xml_document
- constructor - xml_document()
function parse(Ch *text)
function clear()

class - template - rapidxml::xml_node
- constructor - xml_node(node_type type)
function type() const
function document() const
function first_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function last_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function previous_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function next_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function first_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function last_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function type(node_type type)
function prepend_node(xml_node< Ch > *child)
function append_node(xml_node< Ch > *child)
function insert_node(xml_node< Ch > *where, xml_node< Ch > *child)
function remove_first_node()
function remove_last_node()
function remove_node(xml_node< Ch > *where)
function remove_all_nodes()
function prepend_attribute(xml_attribute< Ch > *attribute)
function append_attribute(xml_attribute< Ch > *attribute)
function insert_attribute(xml_attribute< Ch > *where, xml_attribute< Ch > *attribute)
function remove_first_attribute()
function remove_last_attribute()
function remove_attribute(xml_attribute< Ch > *where)
function remove_all_attributes()

namespace rapidxml
enum node_type
function parse_error_handler(const char *what, void *where)
function print(OutIt out, const xml_node< Ch > &node, int flags=0)
function print(std::basic_ostream< Ch > &out, const xml_node< Ch > &node, int flags=0)
function operator<<(std::basic_ostream< Ch > &out, const xml_node< Ch > &node)
- constant - parse_no_data_nodes
- constant - parse_no_element_values
- constant - parse_no_string_terminators
- constant - parse_no_entity_translation
- constant - parse_no_utf8
- constant - parse_declaration_node
- constant - parse_comment_nodes
- constant - parse_doctype_node
- constant - parse_pi_nodes
- constant - parse_validate_closing_tags
- constant - parse_trim_whitespace
- constant - parse_normalize_whitespace
- constant - parse_default
- constant - parse_non_destructive
- constant - parse_fastest
- constant - parse_full
- constant - print_no_indenting

class - template - rapidxml::memory_pool

- - Defined in rapidxml.hpp
- Base class for - xml_document

Description

This class is used by the parser to create new nodes and attributes, without overheads of dynamic memory allocation. In most cases, you will not need to use this class directly. However, if you need to create nodes manually or modify names/values of nodes, you are encouraged to use memory_pool of relevant xml_document to allocate the memory. Not only is this faster than allocating them by using new operator, but also their lifetime will be tied to the lifetime of document, possibly simplyfing memory management.

- Call allocate_node() or allocate_attribute() functions to obtain new nodes or attributes from the pool. You can also call allocate_string() function to allocate strings. Such strings can then be used as names or values of nodes without worrying about their lifetime. Note that there is no free() function -- all allocations are freed at once when clear() function is called, or when the pool is destroyed.

- It is also possible to create a standalone memory_pool, and use it to allocate nodes, whose lifetime will not be tied to any document.

- Pool maintains RAPIDXML_STATIC_POOL_SIZE bytes of statically allocated memory. Until static memory is exhausted, no dynamic memory allocations are done. When static memory is exhausted, pool allocates additional blocks of memory of size RAPIDXML_DYNAMIC_POOL_SIZE each, by using global new[] and delete[] operators. This behaviour can be changed by setting custom allocation routines. Use set_allocator() function to set them.

- Allocations for nodes, attributes and strings are aligned at RAPIDXML_ALIGNMENT bytes. This value defaults to the size of pointer on target architecture.

- To obtain absolutely top performance from the parser, it is important that all nodes are allocated from a single, contiguous block of memory. Otherwise, cache misses when jumping between two (or more) disjoint blocks of memory can slow down parsing quite considerably. If required, you can tweak RAPIDXML_STATIC_POOL_SIZE, RAPIDXML_DYNAMIC_POOL_SIZE and RAPIDXML_ALIGNMENT to obtain best wasted memory to performance compromise. To do it, define their values before rapidxml.hpp file is included.

Parameters

Ch
Character type of created nodes.

- constructor - memory_pool::memory_pool

Synopsis

memory_pool(); -

Description

Constructs empty pool with default allocator functions.

- destructor - memory_pool::~memory_pool

Synopsis

~memory_pool(); -

Description

Destroys pool and frees all the memory. This causes memory occupied by nodes allocated by the pool to be freed. Nodes allocated from the pool are no longer valid.

function memory_pool::allocate_node

Synopsis

xml_node<Ch>* allocate_node(node_type type, const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0); -

Description

Allocates a new node from the pool, and optionally assigns name and value to it. If the allocation request cannot be accomodated, this function will throw std::bad_alloc. If exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function will call rapidxml::parse_error_handler() function.

Parameters

type
Type of node to create.
name
Name to assign to the node, or 0 to assign no name.
value
Value to assign to the node, or 0 to assign no value.
name_size
Size of name to assign, or 0 to automatically calculate size from name string.
value_size
Size of value to assign, or 0 to automatically calculate size from value string.

Returns

Pointer to allocated node. This pointer will never be NULL.

function memory_pool::allocate_attribute

Synopsis

xml_attribute<Ch>* allocate_attribute(const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0); -

Description

Allocates a new attribute from the pool, and optionally assigns name and value to it. If the allocation request cannot be accomodated, this function will throw std::bad_alloc. If exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function will call rapidxml::parse_error_handler() function.

Parameters

name
Name to assign to the attribute, or 0 to assign no name.
value
Value to assign to the attribute, or 0 to assign no value.
name_size
Size of name to assign, or 0 to automatically calculate size from name string.
value_size
Size of value to assign, or 0 to automatically calculate size from value string.

Returns

Pointer to allocated attribute. This pointer will never be NULL.

function memory_pool::allocate_string

Synopsis

Ch* allocate_string(const Ch *source=0, std::size_t size=0); -

Description

Allocates a char array of given size from the pool, and optionally copies a given string to it. If the allocation request cannot be accomodated, this function will throw std::bad_alloc. If exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function will call rapidxml::parse_error_handler() function.

Parameters

source
String to initialize the allocated memory with, or 0 to not initialize it.
size
Number of characters to allocate, or zero to calculate it automatically from source string length; if size is 0, source string must be specified and null terminated.

Returns

Pointer to allocated char array. This pointer will never be NULL.

function memory_pool::clone_node

Synopsis

xml_node<Ch>* clone_node(const xml_node< Ch > *source, xml_node< Ch > *result=0); -

Description

Clones an xml_node and its hierarchy of child nodes and attributes. Nodes and attributes are allocated from this memory pool. Names and values are not cloned, they are shared between the clone and the source. Result node can be optionally specified as a second parameter, in which case its contents will be replaced with cloned source node. This is useful when you want to clone entire document.

Parameters

source
Node to clone.
result
Node to put results in, or 0 to automatically allocate result node

Returns

Pointer to cloned node. This pointer will never be NULL.

function memory_pool::clear

Synopsis

void clear(); -

Description

Clears the pool. This causes memory occupied by nodes allocated by the pool to be freed. Any nodes or strings allocated from the pool will no longer be valid.

function memory_pool::set_allocator

Synopsis

void set_allocator(alloc_func *af, free_func *ff); -

Description

Sets or resets the user-defined memory allocation functions for the pool. This can only be called when no memory is allocated from the pool yet, otherwise results are undefined. Allocation function must not return invalid pointer on failure. It should either throw, stop the program, or use longjmp() function to pass control to other place of program. If it returns invalid pointer, results are undefined.

- User defined allocation functions must have the following forms:

-void *allocate(std::size_t size);
-void free(void *pointer);

Parameters

af
Allocation function, or 0 to restore default function
ff
Free function, or 0 to restore default function

class rapidxml::parse_error

- - Defined in rapidxml.hpp

Description

Parse error exception. This exception is thrown by the parser when an error occurs. Use what() function to get human-readable error message. Use where() function to get a pointer to position within source text where error was detected.

- If throwing exceptions by the parser is undesirable, it can be disabled by defining RAPIDXML_NO_EXCEPTIONS macro before rapidxml.hpp is included. This will cause the parser to call rapidxml::parse_error_handler() function instead of throwing an exception. This function must be defined by the user.

- This class derives from std::exception class.

- constructor - parse_error::parse_error

Synopsis

parse_error(const char *what, void *where); -

Description

Constructs parse error.

function parse_error::what

Synopsis

virtual const char* what() const; -

Description

Gets human readable description of error.

Returns

Pointer to null terminated description of the error.

function parse_error::where

Synopsis

Ch* where() const; -

Description

Gets pointer to character data where error happened. Ch should be the same as char type of xml_document that produced the error.

Returns

Pointer to location within the parsed string where error occured.

class - template - rapidxml::xml_attribute

- - Defined in rapidxml.hpp
- Inherits from - xml_base

Description

Class representing attribute node of XML document. Each attribute has name and value strings, which are available through name() and value() functions (inherited from xml_base). Note that after parse, both name and value of attribute will point to interior of source text used for parsing. Thus, this text must persist in memory for the lifetime of attribute.

Parameters

Ch
Character type to use.

- constructor - xml_attribute::xml_attribute

Synopsis

xml_attribute(); -

Description

Constructs an empty attribute with the specified type. Consider using memory_pool of appropriate xml_document if allocating attributes manually.

function xml_attribute::document

Synopsis

xml_document<Ch>* document() const; -

Description

Gets document of which attribute is a child.

Returns

Pointer to document that contains this attribute, or 0 if there is no parent document.

function xml_attribute::previous_attribute

Synopsis

xml_attribute<Ch>* previous_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; -

Description

Gets previous attribute, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return previous attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

function xml_attribute::next_attribute

Synopsis

xml_attribute<Ch>* next_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; -

Description

Gets next attribute, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return next attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

class - template - rapidxml::xml_base

- - Defined in rapidxml.hpp
- Base class for - xml_attribute xml_node

Description

Base class for xml_node and xml_attribute implementing common functions: name(), name_size(), value(), value_size() and parent().

Parameters

Ch
Character type to use

- constructor - xml_base::xml_base

Synopsis

xml_base(); -

function xml_base::name

Synopsis

Ch* name() const; -

Description

Gets name of the node. Interpretation of name depends on type of node. Note that name will not be zero-terminated if rapidxml::parse_no_string_terminators option was selected during parse.

- Use name_size() function to determine length of the name.

Returns

Name of node, or empty string if node has no name.

function xml_base::name_size

Synopsis

std::size_t name_size() const; -

Description

Gets size of node name, not including terminator character. This function works correctly irrespective of whether name is or is not zero terminated.

Returns

Size of node name, in characters.

function xml_base::value

Synopsis

Ch* value() const; -

Description

Gets value of node. Interpretation of value depends on type of node. Note that value will not be zero-terminated if rapidxml::parse_no_string_terminators option was selected during parse.

- Use value_size() function to determine length of the value.

Returns

Value of node, or empty string if node has no value.

function xml_base::value_size

Synopsis

std::size_t value_size() const; -

Description

Gets size of node value, not including terminator character. This function works correctly irrespective of whether value is or is not zero terminated.

Returns

Size of node value, in characters.

function xml_base::name

Synopsis

void name(const Ch *name, std::size_t size); -

Description

Sets name of node to a non zero-terminated string. See Ownership Of Strings .

- Note that node does not own its name or value, it only stores a pointer to it. It will not delete or otherwise free the pointer on destruction. It is reponsibility of the user to properly manage lifetime of the string. The easiest way to achieve it is to use memory_pool of the document to allocate the string - on destruction of the document the string will be automatically freed.

- Size of name must be specified separately, because name does not have to be zero terminated. Use name(const Ch *) function to have the length automatically calculated (string must be zero terminated).

Parameters

name
Name of node to set. Does not have to be zero terminated.
size
Size of name, in characters. This does not include zero terminator, if one is present.

function xml_base::name

Synopsis

void name(const Ch *name); -

Description

Sets name of node to a zero-terminated string. See also Ownership Of Strings and xml_node::name(const Ch *, std::size_t).

Parameters

name
Name of node to set. Must be zero terminated.

function xml_base::value

Synopsis

void value(const Ch *value, std::size_t size); -

Description

Sets value of node to a non zero-terminated string. See Ownership Of Strings .

- Note that node does not own its name or value, it only stores a pointer to it. It will not delete or otherwise free the pointer on destruction. It is reponsibility of the user to properly manage lifetime of the string. The easiest way to achieve it is to use memory_pool of the document to allocate the string - on destruction of the document the string will be automatically freed.

- Size of value must be specified separately, because it does not have to be zero terminated. Use value(const Ch *) function to have the length automatically calculated (string must be zero terminated).

- If an element has a child node of type node_data, it will take precedence over element value when printing. If you want to manipulate data of elements using values, use parser flag rapidxml::parse_no_data_nodes to prevent creation of data nodes by the parser.

Parameters

value
value of node to set. Does not have to be zero terminated.
size
Size of value, in characters. This does not include zero terminator, if one is present.

function xml_base::value

Synopsis

void value(const Ch *value); -

Description

Sets value of node to a zero-terminated string. See also Ownership Of Strings and xml_node::value(const Ch *, std::size_t).

Parameters

value
Vame of node to set. Must be zero terminated.

function xml_base::parent

Synopsis

xml_node<Ch>* parent() const; -

Description

Gets node parent.

Returns

Pointer to parent node, or 0 if there is no parent.

class - template - rapidxml::xml_document

- - Defined in rapidxml.hpp
- Inherits from - xml_node memory_pool

Description

This class represents root of the DOM hierarchy. It is also an xml_node and a memory_pool through public inheritance. Use parse() function to build a DOM tree from a zero-terminated XML text string. parse() function allocates memory for nodes and attributes by using functions of xml_document, which are inherited from memory_pool. To access root node of the document, use the document itself, as if it was an xml_node.

Parameters

Ch
Character type to use.

- constructor - xml_document::xml_document

Synopsis

xml_document(); -

Description

Constructs empty XML document.

function xml_document::parse

Synopsis

void parse(Ch *text); -

Description

Parses zero-terminated XML string according to given flags. Passed string will be modified by the parser, unless rapidxml::parse_non_destructive flag is used. The string must persist for the lifetime of the document. In case of error, rapidxml::parse_error exception will be thrown.

- If you want to parse contents of a file, you must first load the file into the memory, and pass pointer to its beginning. Make sure that data is zero-terminated.

- Document can be parsed into multiple times. Each new call to parse removes previous nodes and attributes (if any), but does not clear memory pool.

Parameters

text
XML data to parse; pointer is non-const to denote fact that this data may be modified by the parser.

function xml_document::clear

Synopsis

void clear(); -

Description

Clears the document by deleting all nodes and clearing the memory pool. All nodes owned by document pool are destroyed.

class - template - rapidxml::xml_node

- - Defined in rapidxml.hpp
- Inherits from - xml_base
- Base class for - xml_document

Description

Class representing a node of XML document. Each node may have associated name and value strings, which are available through name() and value() functions. Interpretation of name and value depends on type of the node. Type of node can be determined by using type() function.

- Note that after parse, both name and value of node, if any, will point interior of source text used for parsing. Thus, this text must persist in the memory for the lifetime of node.

Parameters

Ch
Character type to use.

- constructor - xml_node::xml_node

Synopsis

xml_node(node_type type); -

Description

Constructs an empty node with the specified type. Consider using memory_pool of appropriate document to allocate nodes manually.

Parameters

type
Type of node to construct.

function xml_node::type

Synopsis

node_type type() const; -

Description

Gets type of node.

Returns

Type of node.

function xml_node::document

Synopsis

xml_document<Ch>* document() const; -

Description

Gets document of which node is a child.

Returns

Pointer to document that contains this node, or 0 if there is no parent document.

function xml_node::first_node

Synopsis

xml_node<Ch>* first_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; -

Description

Gets first child node, optionally matching node name.

Parameters

name
Name of child to find, or 0 to return first child regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found child, or 0 if not found.

function xml_node::last_node

Synopsis

xml_node<Ch>* last_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; -

Description

Gets last child node, optionally matching node name. Behaviour is undefined if node has no children. Use first_node() to test if node has children.

Parameters

name
Name of child to find, or 0 to return last child regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found child, or 0 if not found.

function xml_node::previous_sibling

Synopsis

xml_node<Ch>* previous_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; -

Description

Gets previous sibling node, optionally matching node name. Behaviour is undefined if node has no parent. Use parent() to test if node has a parent.

Parameters

name
Name of sibling to find, or 0 to return previous sibling regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found sibling, or 0 if not found.

function xml_node::next_sibling

Synopsis

xml_node<Ch>* next_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; -

Description

Gets next sibling node, optionally matching node name. Behaviour is undefined if node has no parent. Use parent() to test if node has a parent.

Parameters

name
Name of sibling to find, or 0 to return next sibling regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found sibling, or 0 if not found.

function xml_node::first_attribute

Synopsis

xml_attribute<Ch>* first_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; -

Description

Gets first attribute of node, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return first attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

function xml_node::last_attribute

Synopsis

xml_attribute<Ch>* last_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; -

Description

Gets last attribute of node, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return last attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

function xml_node::type

Synopsis

void type(node_type type); -

Description

Sets type of node.

Parameters

type
Type of node to set.

function xml_node::prepend_node

Synopsis

void prepend_node(xml_node< Ch > *child); -

Description

Prepends a new child node. The prepended child becomes the first child, and all existing children are moved one position back.

Parameters

child
Node to prepend.

function xml_node::append_node

Synopsis

void append_node(xml_node< Ch > *child); -

Description

Appends a new child node. The appended child becomes the last child.

Parameters

child
Node to append.

function xml_node::insert_node

Synopsis

void insert_node(xml_node< Ch > *where, xml_node< Ch > *child); -

Description

Inserts a new child node at specified place inside the node. All children after and including the specified node are moved one position back.

Parameters

where
Place where to insert the child, or 0 to insert at the back.
child
Node to insert.

function xml_node::remove_first_node

Synopsis

void remove_first_node(); -

Description

Removes first child node. If node has no children, behaviour is undefined. Use first_node() to test if node has children.

function xml_node::remove_last_node

Synopsis

void remove_last_node(); -

Description

Removes last child of the node. If node has no children, behaviour is undefined. Use first_node() to test if node has children.

function xml_node::remove_node

Synopsis

void remove_node(xml_node< Ch > *where); -

Description

Removes specified child from the node.

function xml_node::remove_all_nodes

Synopsis

void remove_all_nodes(); -

Description

Removes all child nodes (but not attributes).

function xml_node::prepend_attribute

Synopsis

void prepend_attribute(xml_attribute< Ch > *attribute); -

Description

Prepends a new attribute to the node.

Parameters

attribute
Attribute to prepend.

function xml_node::append_attribute

Synopsis

void append_attribute(xml_attribute< Ch > *attribute); -

Description

Appends a new attribute to the node.

Parameters

attribute
Attribute to append.

function xml_node::insert_attribute

Synopsis

void insert_attribute(xml_attribute< Ch > *where, xml_attribute< Ch > *attribute); -

Description

Inserts a new attribute at specified place inside the node. All attributes after and including the specified attribute are moved one position back.

Parameters

where
Place where to insert the attribute, or 0 to insert at the back.
attribute
Attribute to insert.

function xml_node::remove_first_attribute

Synopsis

void remove_first_attribute(); -

Description

Removes first attribute of the node. If node has no attributes, behaviour is undefined. Use first_attribute() to test if node has attributes.

function xml_node::remove_last_attribute

Synopsis

void remove_last_attribute(); -

Description

Removes last attribute of the node. If node has no attributes, behaviour is undefined. Use first_attribute() to test if node has attributes.

function xml_node::remove_attribute

Synopsis

void remove_attribute(xml_attribute< Ch > *where); -

Description

Removes specified attribute from node.

Parameters

where
Pointer to attribute to be removed.

function xml_node::remove_all_attributes

Synopsis

void remove_all_attributes(); -

Description

Removes all attributes of node.

enum node_type

Description

Enumeration listing all node types produced by the parser. Use xml_node::type() function to query node type.

Values

node_document
A document node. Name and value are empty.
node_element
An element node. Name contains element name. Value contains text of first data node.
node_data
A data node. Name is empty. Value contains data text.
node_cdata
A CDATA node. Name is empty. Value contains data text.
node_comment
A comment node. Name is empty. Value contains comment text.
node_declaration
A declaration node. Name and value are empty. Declaration parameters (version, encoding and standalone) are in node attributes.
node_doctype
A DOCTYPE node. Name is empty. Value contains DOCTYPE text.
node_pi
A PI node. Name contains target. Value contains instructions.

function parse_error_handler

Synopsis

void rapidxml::parse_error_handler(const char *what, void *where); -

Description

When exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function is called to notify user about the error. It must be defined by the user.

- This function cannot return. If it does, the results are undefined.

- A very simple definition might look like that: - void rapidxml::parse_error_handler(const char *what, void *where) - { - std::cout << "Parse error: " << what << "\n"; - std::abort(); - } -

Parameters

what
Human readable description of the error.
where
Pointer to character data where error was detected.

function print

Synopsis

OutIt rapidxml::print(OutIt out, const xml_node< Ch > &node, int flags=0); -

Description

Prints XML to given output iterator.

Parameters

out
Output iterator to print to.
node
Node to be printed. Pass xml_document to print entire document.
flags
Flags controlling how XML is printed.

Returns

Output iterator pointing to position immediately after last character of printed text.

function print

Synopsis

std::basic_ostream<Ch>& rapidxml::print(std::basic_ostream< Ch > &out, const xml_node< Ch > &node, int flags=0); -

Description

Prints XML to given output stream.

Parameters

out
Output stream to print to.
node
Node to be printed. Pass xml_document to print entire document.
flags
Flags controlling how XML is printed.

Returns

Output stream.

function operator<<

Synopsis

std::basic_ostream<Ch>& rapidxml::operator<<(std::basic_ostream< Ch > &out, const xml_node< Ch > &node); -

Description

Prints formatted XML to given output stream. Uses default printing flags. Use print() function to customize printing process.

Parameters

out
Output stream to print to.
node
Node to be printed.

Returns

Output stream.

- constant - parse_no_data_nodes

Synopsis

const int parse_no_data_nodes - = 0x1; -

Description

Parse flag instructing the parser to not create data nodes. Text of first data node will still be placed in value of parent element, unless rapidxml::parse_no_element_values flag is also specified. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_no_element_values

Synopsis

const int parse_no_element_values - = 0x2; -

Description

Parse flag instructing the parser to not use text of first data node as a value of parent element. Can be combined with other flags by use of | operator. Note that child data nodes of element node take precendence over its value when printing. That is, if element has one or more child data nodes and a value, the value will be ignored. Use rapidxml::parse_no_data_nodes flag to prevent creation of data nodes if you want to manipulate data using values of elements.

- See xml_document::parse() function.

- constant - parse_no_string_terminators

Synopsis

const int parse_no_string_terminators - = 0x4; -

Description

Parse flag instructing the parser to not place zero terminators after strings in the source text. By default zero terminators are placed, modifying source text. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_no_entity_translation

Synopsis

const int parse_no_entity_translation - = 0x8; -

Description

Parse flag instructing the parser to not translate entities in the source text. By default entities are translated, modifying source text. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_no_utf8

Synopsis

const int parse_no_utf8 - = 0x10; -

Description

Parse flag instructing the parser to disable UTF-8 handling and assume plain 8 bit characters. By default, UTF-8 handling is enabled. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_declaration_node

Synopsis

const int parse_declaration_node - = 0x20; -

Description

Parse flag instructing the parser to create XML declaration node. By default, declaration node is not created. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_comment_nodes

Synopsis

const int parse_comment_nodes - = 0x40; -

Description

Parse flag instructing the parser to create comments nodes. By default, comment nodes are not created. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_doctype_node

Synopsis

const int parse_doctype_node - = 0x80; -

Description

Parse flag instructing the parser to create DOCTYPE node. By default, doctype node is not created. Although W3C specification allows at most one DOCTYPE node, RapidXml will silently accept documents with more than one. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_pi_nodes

Synopsis

const int parse_pi_nodes - = 0x100; -

Description

Parse flag instructing the parser to create PI nodes. By default, PI nodes are not created. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_validate_closing_tags

Synopsis

const int parse_validate_closing_tags - = 0x200; -

Description

Parse flag instructing the parser to validate closing tag names. If not set, name inside closing tag is irrelevant to the parser. By default, closing tags are not validated. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_trim_whitespace

Synopsis

const int parse_trim_whitespace - = 0x400; -

Description

Parse flag instructing the parser to trim all leading and trailing whitespace of data nodes. By default, whitespace is not trimmed. This flag does not cause the parser to modify source text. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_normalize_whitespace

Synopsis

const int parse_normalize_whitespace - = 0x800; -

Description

Parse flag instructing the parser to condense all whitespace runs of data nodes to a single space character. Trimming of leading and trailing whitespace of data is controlled by rapidxml::parse_trim_whitespace flag. By default, whitespace is not normalized. If this flag is specified, source text will be modified. Can be combined with other flags by use of | operator.

- See xml_document::parse() function.

- constant - parse_default

Synopsis

const int parse_default - = 0; -

Description

Parse flags which represent default behaviour of the parser. This is always equal to 0, so that all other flags can be simply ored together. Normally there is no need to inconveniently disable flags by anding with their negated (~) values. This also means that meaning of each flag is a negation of the default setting. For example, if flag name is rapidxml::parse_no_utf8, it means that utf-8 is enabled by default, and using the flag will disable it.

- See xml_document::parse() function.

- constant - parse_non_destructive

Synopsis

const int parse_non_destructive - = parse_no_string_terminators | parse_no_entity_translation; -

Description

A combination of parse flags that forbids any modifications of the source text. This also results in faster parsing. However, note that the following will occur:
  • names and values of nodes will not be zero terminated, you have to use xml_base::name_size() and xml_base::value_size() functions to determine where name and value ends
  • entities will not be translated
  • whitespace will not be normalized
-See xml_document::parse() function.

- constant - parse_fastest

Synopsis

const int parse_fastest - = parse_non_destructive | parse_no_data_nodes; -

Description

A combination of parse flags resulting in fastest possible parsing, without sacrificing important data.

- See xml_document::parse() function.

- constant - parse_full

Synopsis

const int parse_full - = parse_declaration_node | parse_comment_nodes | parse_doctype_node | parse_pi_nodes | parse_validate_closing_tags; -

Description

A combination of parse flags resulting in largest amount of data being extracted. This usually results in slowest parsing.

- See xml_document::parse() function.

- constant - print_no_indenting

Synopsis

const int print_no_indenting - = 0x1; +

RAPIDXML Manual

Version 1.13

Copyright (C) 2006, 2009 Marcin Kalicinski
See accompanying file license.txt for license information.

Table of Contents

1. What is RapidXml?
1.1 Dependencies And Compatibility
1.2 Character Types And Encodings
1.3 Error Handling
1.4 Memory Allocation
1.5 W3C Compliance
1.6 API Design
1.7 Reliability
1.8 Acknowledgements
2. Two Minute Tutorial
2.1 Parsing
2.2 Accessing The DOM Tree
2.3 Modifying The DOM Tree
2.4 Printing XML
3. Differences From Regular XML Parsers
3.1 Lifetime Of Source Text
3.2 Ownership Of Strings
3.3 Destructive Vs Non-Destructive Mode
4. Performance
4.1 Comparison With Other Parsers
5. Reference

1. What is RapidXml?

RapidXml is an attempt to create the fastest XML DOM parser possible, while retaining useability, portability and reasonable W3C compatibility. It is an in-situ parser written in C++, with parsing speed approaching that of strlen() function executed on the same data.

+ Entire parser is contained in a single header file, so no building or linking is neccesary. To use it you just need to copy rapidxml.hpp file to a convenient place (such as your project directory), and include it where needed. You may also want to use printing functions contained in header rapidxml_print.hpp.

1.1 Dependencies And Compatibility

RapidXml has no dependencies other than a very small subset of standard C++ library (<cassert>, <cstdlib>, <new> and <exception>, unless exceptions are disabled). It should compile on any reasonably conformant compiler, and was tested on Visual C++ 2003, Visual C++ 2005, Visual C++ 2008, gcc 3, gcc 4, and Comeau 4.3.3. Care was taken that no warnings are produced on these compilers, even with highest warning levels enabled.

1.2 Character Types And Encodings

RapidXml is character type agnostic, and can work both with narrow and wide characters. Current version does not fully support UTF-16 or UTF-32, so use of wide characters is somewhat incapacitated. However, it should succesfully parse wchar_t strings containing UTF-16 or UTF-32 if endianness of the data matches that of the machine. UTF-8 is fully supported, including all numeric character references, which are expanded into appropriate UTF-8 byte sequences (unless you enable parse_no_utf8 flag).

+ Note that RapidXml performs no decoding - strings returned by name() and value() functions will contain text encoded using the same encoding as source file. Rapidxml understands and expands the following character references: &apos; &amp; &quot; &lt; &gt; &#...; Other character references are not expanded.

1.3 Error Handling

By default, RapidXml uses C++ exceptions to report errors. If this behaviour is undesirable, RAPIDXML_NO_EXCEPTIONS can be defined to suppress exception code. See parse_error class and parse_error_handler() function for more information.

1.4 Memory Allocation

RapidXml uses a special memory pool object to allocate nodes and attributes, because direct allocation using new operator would be far too slow. Underlying memory allocations performed by the pool can be customized by use of memory_pool::set_allocator() function. See class memory_pool for more information.

1.5 W3C Compliance

RapidXml is not a W3C compliant parser, primarily because it ignores DOCTYPE declarations. There is a number of other, minor incompatibilities as well. Still, it can successfully parse and produce complete trees of all valid XML files in W3C conformance suite (over 1000 files specially designed to find flaws in XML processors). In destructive mode it performs whitespace normalization and character entity substitution for a small set of built-in entities.

1.6 API Design

RapidXml API is minimalistic, to reduce code size as much as possible, and facilitate use in embedded environments. Additional convenience functions are provided in separate headers: rapidxml_utils.hpp and rapidxml_print.hpp. Contents of these headers is not an essential part of the library, and is currently not documented (otherwise than with comments in code).

1.7 Reliability

RapidXml is very robust and comes with a large harness of unit tests. Special care has been taken to ensure stability of the parser no matter what source text is thrown at it. One of the unit tests produces 100,000 randomly corrupted variants of XML document, which (when uncorrupted) contains all constructs recognized by RapidXml. RapidXml passes this test when it correctly recognizes that errors have been introduced, and does not crash or loop indefinitely.

+ Another unit test puts RapidXml head-to-head with another, well estabilished XML parser, and verifies that their outputs match across a wide variety of small and large documents.

+ Yet another test feeds RapidXml with over 1000 test files from W3C compliance suite, and verifies that correct results are obtained. There are also additional tests that verify each API function separately, and test that various parsing modes work as expected.

1.8 Acknowledgements

I would like to thank Arseny Kapoulkine for his work on pugixml, which was an inspiration for this project. Additional thanks go to Kristen Wegner for creating pugxml, from which pugixml was derived. Janusz Wohlfeil kindly ran RapidXml speed tests on hardware that I did not have access to, allowing me to expand performance comparison table.

2. Two Minute Tutorial

2.1 Parsing

The following code causes RapidXml to parse a zero-terminated string named text: 
using namespace rapidxml;
+xml_document<> doc;    // character type defaults to char
+doc.parse<0>(text);    // 0 means default parse flags
+
doc object is now a root of DOM tree containing representation of the parsed XML. Because all RapidXml interface is contained inside namespace rapidxml, users must either bring contents of this namespace into scope, or fully qualify all the names. Class xml_document represents a root of the DOM hierarchy. By means of public inheritance, it is also an xml_node and a memory_pool. Template parameter of xml_document::parse() function is used to specify parsing flags, with which you can fine-tune behaviour of the parser. Note that flags must be a compile-time constant.

2.2 Accessing The DOM Tree

To access the DOM tree, use methods of xml_node and xml_attribute classes: 
cout << "Name of my first node is: " << doc.first_node()->name() << "\n";
+xml_node<> *node = doc.first_node("foobar");
+cout << "Node foobar has value " << node->value() << "\n";
+for (xml_attribute<> *attr = node->first_attribute();
+     attr; attr = attr->next_attribute())
+{
+    cout << "Node foobar has attribute " << attr->name() << " ";
+    cout << "with value " << attr->value() << "\n";
+}
+

2.3 Modifying The DOM Tree

DOM tree produced by the parser is fully modifiable. Nodes and attributes can be added/removed, and their contents changed. The below example creates a HTML document, whose sole contents is a link to google.com website: 
xml_document<> doc;
+xml_node<> *node = doc.allocate_node(node_element, "a", "Google");
+doc.append_node(node);
+xml_attribute<> *attr = doc.allocate_attribute("href", "google.com");
+node->append_attribute(attr);
+
One quirk is that nodes and attributes do not own the text of their names and values. This is because normally they only store pointers to the source text. So, when assigning a new name or value to the node, care must be taken to ensure proper lifetime of the string. The easiest way to achieve it is to allocate the string from the xml_document memory pool. In the above example this is not necessary, because we are only assigning character constants. But the code below uses memory_pool::allocate_string() function to allocate node name (which will have the same lifetime as the document), and assigns it to a new node: 
xml_document<> doc;
+char *node_name = doc.allocate_string(name);        // Allocate string and copy name into it
+xml_node<> *node = doc.allocate_node(node_element, node_name);  // Set node name to node_name
+
Check Reference section for description of the entire interface.

2.4 Printing XML

You can print xml_document and xml_node objects into an XML string. Use print() function or operator <<, which are defined in rapidxml_print.hpp header. 
using namespace rapidxml;
+xml_document<> doc;    // character type defaults to char
+// ... some code to fill the document
+
+// Print to stream using operator <<
+std::cout << doc;   
+
+// Print to stream using print function, specifying printing flags
+print(std::cout, doc, 0);   // 0 means default printing flags
+
+// Print to string using output iterator
+std::string s;
+print(std::back_inserter(s), doc, 0);
+
+// Print to memory buffer using output iterator
+char buffer[4096];                      // You are responsible for making the buffer large enough!
+char *end = print(buffer, doc, 0);      // end contains pointer to character after last printed character
+*end = 0;                               // Add string terminator after XML
+

3. Differences From Regular XML Parsers

RapidXml is an in-situ parser, which allows it to achieve very high parsing speed. In-situ means that parser does not make copies of strings. Instead, it places pointers to the source text in the DOM hierarchy.

3.1 Lifetime Of Source Text

In-situ parsing requires that source text lives at least as long as the document object. If source text is destroyed, names and values of nodes in DOM tree will become destroyed as well. Additionally, whitespace processing, character entity translation, and zero-termination of strings require that source text be modified during parsing (but see non-destructive mode). This makes the text useless for further processing once it was parsed by RapidXml.

+ In many cases however, these are not serious issues.

3.2 Ownership Of Strings

Nodes and attributes produced by RapidXml do not own their name and value strings. They merely hold the pointers to them. This means you have to be careful when setting these values manually, by using xml_base::name(const Ch *) or xml_base::value(const Ch *) functions. Care must be taken to ensure that lifetime of the string passed is at least as long as lifetime of the node/attribute. The easiest way to achieve it is to allocate the string from memory_pool owned by the document. Use memory_pool::allocate_string() function for this purpose.

3.3 Destructive Vs Non-Destructive Mode

By default, the parser modifies source text during the parsing process. This is required to achieve character entity translation, whitespace normalization, and zero-termination of strings.

+ In some cases this behaviour may be undesirable, for example if source text resides in read only memory, or is mapped to memory directly from file. By using appropriate parser flags (parse_non_destructive), source text modifications can be disabled. However, because RapidXml does in-situ parsing, it obviously has the following side-effects:

4. Performance

RapidXml achieves its speed through use of several techniques:
  • In-situ parsing. When building DOM tree, RapidXml does not make copies of string data, such as node names and values. Instead, it stores pointers to interior of the source text.
  • Use of template metaprogramming techniques. This allows it to move much of the work to compile time. Through magic of the templates, C++ compiler generates a separate copy of parsing code for any combination of parser flags you use. In each copy, all possible decisions are made at compile time and all unused code is omitted.
  • Extensive use of lookup tables for parsing.
  • Hand-tuned C++ with profiling done on several most popular CPUs.
This results in a very small and fast code: a parser which is custom tailored to exact needs with each invocation.

4.1 Comparison With Other Parsers

The table below compares speed of RapidXml to some other parsers, and to strlen() function executed on the same data. On a modern CPU (as of 2007), you can expect parsing throughput to be close to 1 GB/s. As a rule of thumb, parsing speed is about 50-100x faster than Xerces DOM, 30-60x faster than TinyXml, 3-12x faster than pugxml, and about 5% - 30% faster than pugixml, the fastest XML parser I know of.
  • The test file is a real-world, 50kB large, moderately dense XML file.
  • All timing is done by using RDTSC instruction present in Pentium-compatible CPUs.
  • No profile-guided optimizations are used.
  • All parsers are running in their fastest modes.
  • The results are given in CPU cycles per character, so frequency of CPUs is irrelevant.
  • The results are minimum values from a large number of runs, to minimize effects of operating system activity, task switching, interrupt handling etc.
  • A single parse of the test file takes about 1/10th of a millisecond, so with large number of runs there is a good chance of hitting at least one no-interrupt streak, and obtaining undisturbed results.
Platform
Compiler
strlen() RapidXml pugixml 0.3 pugxml TinyXml
Pentium 4
MSVC 8.0
2.5
5.4
7.0
61.7
298.8
Pentium 4
gcc 4.1.1
0.8
6.1
9.5
67.0
413.2
Core 2
MSVC 8.0
1.0
4.5
5.0
24.6
154.8
Core 2
gcc 4.1.1
0.6
4.6
5.4
28.3
229.3
Athlon XP
MSVC 8.0
3.1
7.7
8.0
25.5
182.6
Athlon XP
gcc 4.1.1
0.9
8.2
9.2
33.7
265.2
Pentium 3
MSVC 8.0
2.0
6.3
7.0
30.9
211.9
Pentium 3
gcc 4.1.1
1.0
6.7
8.9
35.3
316.0
(*) All results are in CPU cycles per character of source text

5. Reference

This section lists all classes, functions, constants etc. and describes them in detail.
class + template + rapidxml::memory_pool
+ constructor + memory_pool()
+ destructor + ~memory_pool()
function allocate_node(node_type type, const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0)
function allocate_attribute(const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0)
function allocate_string(const Ch *source=0, std::size_t size=0)
function clone_node(const xml_node< Ch > *source, xml_node< Ch > *result=0)
function clear()
function set_allocator(alloc_func *af, free_func *ff)

class rapidxml::parse_error
+ constructor + parse_error(const char *what, void *where)
function what() const
function where() const

class + template + rapidxml::xml_attribute
+ constructor + xml_attribute()
function document() const
function previous_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function next_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const

class + template + rapidxml::xml_base
+ constructor + xml_base()
function name() const
function name_size() const
function value() const
function value_size() const
function name(const Ch *name, std::size_t size)
function name(const Ch *name)
function value(const Ch *value, std::size_t size)
function value(const Ch *value)
function parent() const

class + template + rapidxml::xml_document
+ constructor + xml_document()
function parse(Ch *text)
function clear()

class + template + rapidxml::xml_node
+ constructor + xml_node(node_type type)
function type() const
function document() const
function first_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function last_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function previous_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function next_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function first_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function last_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const
function type(node_type type)
function prepend_node(xml_node< Ch > *child)
function append_node(xml_node< Ch > *child)
function insert_node(xml_node< Ch > *where, xml_node< Ch > *child)
function remove_first_node()
function remove_last_node()
function remove_node(xml_node< Ch > *where)
function remove_all_nodes()
function prepend_attribute(xml_attribute< Ch > *attribute)
function append_attribute(xml_attribute< Ch > *attribute)
function insert_attribute(xml_attribute< Ch > *where, xml_attribute< Ch > *attribute)
function remove_first_attribute()
function remove_last_attribute()
function remove_attribute(xml_attribute< Ch > *where)
function remove_all_attributes()

namespace rapidxml
enum node_type
function parse_error_handler(const char *what, void *where)
function print(OutIt out, const xml_node< Ch > &node, int flags=0)
function print(std::basic_ostream< Ch > &out, const xml_node< Ch > &node, int flags=0)
function operator<<(std::basic_ostream< Ch > &out, const xml_node< Ch > &node)
+ constant + parse_no_data_nodes
+ constant + parse_no_element_values
+ constant + parse_no_string_terminators
+ constant + parse_no_entity_translation
+ constant + parse_no_utf8
+ constant + parse_declaration_node
+ constant + parse_comment_nodes
+ constant + parse_doctype_node
+ constant + parse_pi_nodes
+ constant + parse_validate_closing_tags
+ constant + parse_trim_whitespace
+ constant + parse_normalize_whitespace
+ constant + parse_default
+ constant + parse_non_destructive
+ constant + parse_fastest
+ constant + parse_full
+ constant + print_no_indenting

class + template + rapidxml::memory_pool

+ + Defined in rapidxml.hpp
+ Base class for + xml_document

Description

This class is used by the parser to create new nodes and attributes, without overheads of dynamic memory allocation. In most cases, you will not need to use this class directly. However, if you need to create nodes manually or modify names/values of nodes, you are encouraged to use memory_pool of relevant xml_document to allocate the memory. Not only is this faster than allocating them by using new operator, but also their lifetime will be tied to the lifetime of document, possibly simplyfing memory management.

+ Call allocate_node() or allocate_attribute() functions to obtain new nodes or attributes from the pool. You can also call allocate_string() function to allocate strings. Such strings can then be used as names or values of nodes without worrying about their lifetime. Note that there is no free() function -- all allocations are freed at once when clear() function is called, or when the pool is destroyed.

+ It is also possible to create a standalone memory_pool, and use it to allocate nodes, whose lifetime will not be tied to any document.

+ Pool maintains RAPIDXML_STATIC_POOL_SIZE bytes of statically allocated memory. Until static memory is exhausted, no dynamic memory allocations are done. When static memory is exhausted, pool allocates additional blocks of memory of size RAPIDXML_DYNAMIC_POOL_SIZE each, by using global new[] and delete[] operators. This behaviour can be changed by setting custom allocation routines. Use set_allocator() function to set them.

+ Allocations for nodes, attributes and strings are aligned at RAPIDXML_ALIGNMENT bytes. This value defaults to the size of pointer on target architecture.

+ To obtain absolutely top performance from the parser, it is important that all nodes are allocated from a single, contiguous block of memory. Otherwise, cache misses when jumping between two (or more) disjoint blocks of memory can slow down parsing quite considerably. If required, you can tweak RAPIDXML_STATIC_POOL_SIZE, RAPIDXML_DYNAMIC_POOL_SIZE and RAPIDXML_ALIGNMENT to obtain best wasted memory to performance compromise. To do it, define their values before rapidxml.hpp file is included.

Parameters

Ch
Character type of created nodes.

+ constructor + memory_pool::memory_pool

Synopsis

memory_pool(); +

Description

Constructs empty pool with default allocator functions.

+ destructor + memory_pool::~memory_pool

Synopsis

~memory_pool(); +

Description

Destroys pool and frees all the memory. This causes memory occupied by nodes allocated by the pool to be freed. Nodes allocated from the pool are no longer valid.

function memory_pool::allocate_node

Synopsis

xml_node<Ch>* allocate_node(node_type type, const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0); +

Description

Allocates a new node from the pool, and optionally assigns name and value to it. If the allocation request cannot be accomodated, this function will throw std::bad_alloc. If exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function will call rapidxml::parse_error_handler() function.

Parameters

type
Type of node to create.
name
Name to assign to the node, or 0 to assign no name.
value
Value to assign to the node, or 0 to assign no value.
name_size
Size of name to assign, or 0 to automatically calculate size from name string.
value_size
Size of value to assign, or 0 to automatically calculate size from value string.

Returns

Pointer to allocated node. This pointer will never be NULL.

function memory_pool::allocate_attribute

Synopsis

xml_attribute<Ch>* allocate_attribute(const Ch *name=0, const Ch *value=0, std::size_t name_size=0, std::size_t value_size=0); +

Description

Allocates a new attribute from the pool, and optionally assigns name and value to it. If the allocation request cannot be accomodated, this function will throw std::bad_alloc. If exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function will call rapidxml::parse_error_handler() function.

Parameters

name
Name to assign to the attribute, or 0 to assign no name.
value
Value to assign to the attribute, or 0 to assign no value.
name_size
Size of name to assign, or 0 to automatically calculate size from name string.
value_size
Size of value to assign, or 0 to automatically calculate size from value string.

Returns

Pointer to allocated attribute. This pointer will never be NULL.

function memory_pool::allocate_string

Synopsis

Ch* allocate_string(const Ch *source=0, std::size_t size=0); +

Description

Allocates a char array of given size from the pool, and optionally copies a given string to it. If the allocation request cannot be accomodated, this function will throw std::bad_alloc. If exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function will call rapidxml::parse_error_handler() function.

Parameters

source
String to initialize the allocated memory with, or 0 to not initialize it.
size
Number of characters to allocate, or zero to calculate it automatically from source string length; if size is 0, source string must be specified and null terminated.

Returns

Pointer to allocated char array. This pointer will never be NULL.

function memory_pool::clone_node

Synopsis

xml_node<Ch>* clone_node(const xml_node< Ch > *source, xml_node< Ch > *result=0); +

Description

Clones an xml_node and its hierarchy of child nodes and attributes. Nodes and attributes are allocated from this memory pool. Names and values are not cloned, they are shared between the clone and the source. Result node can be optionally specified as a second parameter, in which case its contents will be replaced with cloned source node. This is useful when you want to clone entire document.

Parameters

source
Node to clone.
result
Node to put results in, or 0 to automatically allocate result node

Returns

Pointer to cloned node. This pointer will never be NULL.

function memory_pool::clear

Synopsis

void clear(); +

Description

Clears the pool. This causes memory occupied by nodes allocated by the pool to be freed. Any nodes or strings allocated from the pool will no longer be valid.

function memory_pool::set_allocator

Synopsis

void set_allocator(alloc_func *af, free_func *ff); +

Description

Sets or resets the user-defined memory allocation functions for the pool. This can only be called when no memory is allocated from the pool yet, otherwise results are undefined. Allocation function must not return invalid pointer on failure. It should either throw, stop the program, or use longjmp() function to pass control to other place of program. If it returns invalid pointer, results are undefined.

+ User defined allocation functions must have the following forms:

+void *allocate(std::size_t size);
+void free(void *pointer);

Parameters

af
Allocation function, or 0 to restore default function
ff
Free function, or 0 to restore default function

class rapidxml::parse_error

+ + Defined in rapidxml.hpp

Description

Parse error exception. This exception is thrown by the parser when an error occurs. Use what() function to get human-readable error message. Use where() function to get a pointer to position within source text where error was detected.

+ If throwing exceptions by the parser is undesirable, it can be disabled by defining RAPIDXML_NO_EXCEPTIONS macro before rapidxml.hpp is included. This will cause the parser to call rapidxml::parse_error_handler() function instead of throwing an exception. This function must be defined by the user.

+ This class derives from std::exception class.

+ constructor + parse_error::parse_error

Synopsis

parse_error(const char *what, void *where); +

Description

Constructs parse error.

function parse_error::what

Synopsis

virtual const char* what() const; +

Description

Gets human readable description of error.

Returns

Pointer to null terminated description of the error.

function parse_error::where

Synopsis

Ch* where() const; +

Description

Gets pointer to character data where error happened. Ch should be the same as char type of xml_document that produced the error.

Returns

Pointer to location within the parsed string where error occured.

class + template + rapidxml::xml_attribute

+ + Defined in rapidxml.hpp
+ Inherits from + xml_base

Description

Class representing attribute node of XML document. Each attribute has name and value strings, which are available through name() and value() functions (inherited from xml_base). Note that after parse, both name and value of attribute will point to interior of source text used for parsing. Thus, this text must persist in memory for the lifetime of attribute.

Parameters

Ch
Character type to use.

+ constructor + xml_attribute::xml_attribute

Synopsis

xml_attribute(); +

Description

Constructs an empty attribute with the specified type. Consider using memory_pool of appropriate xml_document if allocating attributes manually.

function xml_attribute::document

Synopsis

xml_document<Ch>* document() const; +

Description

Gets document of which attribute is a child.

Returns

Pointer to document that contains this attribute, or 0 if there is no parent document.

function xml_attribute::previous_attribute

Synopsis

xml_attribute<Ch>* previous_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets previous attribute, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return previous attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

function xml_attribute::next_attribute

Synopsis

xml_attribute<Ch>* next_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets next attribute, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return next attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

class + template + rapidxml::xml_base

+ + Defined in rapidxml.hpp
+ Base class for + xml_attribute xml_node

Description

Base class for xml_node and xml_attribute implementing common functions: name(), name_size(), value(), value_size() and parent().

Parameters

Ch
Character type to use

+ constructor + xml_base::xml_base

Synopsis

xml_base(); +

function xml_base::name

Synopsis

Ch* name() const; +

Description

Gets name of the node. Interpretation of name depends on type of node. Note that name will not be zero-terminated if rapidxml::parse_no_string_terminators option was selected during parse.

+ Use name_size() function to determine length of the name.

Returns

Name of node, or empty string if node has no name.

function xml_base::name_size

Synopsis

std::size_t name_size() const; +

Description

Gets size of node name, not including terminator character. This function works correctly irrespective of whether name is or is not zero terminated.

Returns

Size of node name, in characters.

function xml_base::value

Synopsis

Ch* value() const; +

Description

Gets value of node. Interpretation of value depends on type of node. Note that value will not be zero-terminated if rapidxml::parse_no_string_terminators option was selected during parse.

+ Use value_size() function to determine length of the value.

Returns

Value of node, or empty string if node has no value.

function xml_base::value_size

Synopsis

std::size_t value_size() const; +

Description

Gets size of node value, not including terminator character. This function works correctly irrespective of whether value is or is not zero terminated.

Returns

Size of node value, in characters.

function xml_base::name

Synopsis

void name(const Ch *name, std::size_t size); +

Description

Sets name of node to a non zero-terminated string. See Ownership Of Strings .

+ Note that node does not own its name or value, it only stores a pointer to it. It will not delete or otherwise free the pointer on destruction. It is reponsibility of the user to properly manage lifetime of the string. The easiest way to achieve it is to use memory_pool of the document to allocate the string - on destruction of the document the string will be automatically freed.

+ Size of name must be specified separately, because name does not have to be zero terminated. Use name(const Ch *) function to have the length automatically calculated (string must be zero terminated).

Parameters

name
Name of node to set. Does not have to be zero terminated.
size
Size of name, in characters. This does not include zero terminator, if one is present.

function xml_base::name

Synopsis

void name(const Ch *name); +

Description

Sets name of node to a zero-terminated string. See also Ownership Of Strings and xml_node::name(const Ch *, std::size_t).

Parameters

name
Name of node to set. Must be zero terminated.

function xml_base::value

Synopsis

void value(const Ch *value, std::size_t size); +

Description

Sets value of node to a non zero-terminated string. See Ownership Of Strings .

+ Note that node does not own its name or value, it only stores a pointer to it. It will not delete or otherwise free the pointer on destruction. It is reponsibility of the user to properly manage lifetime of the string. The easiest way to achieve it is to use memory_pool of the document to allocate the string - on destruction of the document the string will be automatically freed.

+ Size of value must be specified separately, because it does not have to be zero terminated. Use value(const Ch *) function to have the length automatically calculated (string must be zero terminated).

+ If an element has a child node of type node_data, it will take precedence over element value when printing. If you want to manipulate data of elements using values, use parser flag rapidxml::parse_no_data_nodes to prevent creation of data nodes by the parser.

Parameters

value
value of node to set. Does not have to be zero terminated.
size
Size of value, in characters. This does not include zero terminator, if one is present.

function xml_base::value

Synopsis

void value(const Ch *value); +

Description

Sets value of node to a zero-terminated string. See also Ownership Of Strings and xml_node::value(const Ch *, std::size_t).

Parameters

value
Vame of node to set. Must be zero terminated.

function xml_base::parent

Synopsis

xml_node<Ch>* parent() const; +

Description

Gets node parent.

Returns

Pointer to parent node, or 0 if there is no parent.

class + template + rapidxml::xml_document

+ + Defined in rapidxml.hpp
+ Inherits from + xml_node memory_pool

Description

This class represents root of the DOM hierarchy. It is also an xml_node and a memory_pool through public inheritance. Use parse() function to build a DOM tree from a zero-terminated XML text string. parse() function allocates memory for nodes and attributes by using functions of xml_document, which are inherited from memory_pool. To access root node of the document, use the document itself, as if it was an xml_node.

Parameters

Ch
Character type to use.

+ constructor + xml_document::xml_document

Synopsis

xml_document(); +

Description

Constructs empty XML document.

function xml_document::parse

Synopsis

void parse(Ch *text); +

Description

Parses zero-terminated XML string according to given flags. Passed string will be modified by the parser, unless rapidxml::parse_non_destructive flag is used. The string must persist for the lifetime of the document. In case of error, rapidxml::parse_error exception will be thrown.

+ If you want to parse contents of a file, you must first load the file into the memory, and pass pointer to its beginning. Make sure that data is zero-terminated.

+ Document can be parsed into multiple times. Each new call to parse removes previous nodes and attributes (if any), but does not clear memory pool.

Parameters

text
XML data to parse; pointer is non-const to denote fact that this data may be modified by the parser.

function xml_document::clear

Synopsis

void clear(); +

Description

Clears the document by deleting all nodes and clearing the memory pool. All nodes owned by document pool are destroyed.

class + template + rapidxml::xml_node

+ + Defined in rapidxml.hpp
+ Inherits from + xml_base
+ Base class for + xml_document

Description

Class representing a node of XML document. Each node may have associated name and value strings, which are available through name() and value() functions. Interpretation of name and value depends on type of the node. Type of node can be determined by using type() function.

+ Note that after parse, both name and value of node, if any, will point interior of source text used for parsing. Thus, this text must persist in the memory for the lifetime of node.

Parameters

Ch
Character type to use.

+ constructor + xml_node::xml_node

Synopsis

xml_node(node_type type); +

Description

Constructs an empty node with the specified type. Consider using memory_pool of appropriate document to allocate nodes manually.

Parameters

type
Type of node to construct.

function xml_node::type

Synopsis

node_type type() const; +

Description

Gets type of node.

Returns

Type of node.

function xml_node::document

Synopsis

xml_document<Ch>* document() const; +

Description

Gets document of which node is a child.

Returns

Pointer to document that contains this node, or 0 if there is no parent document.

function xml_node::first_node

Synopsis

xml_node<Ch>* first_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets first child node, optionally matching node name.

Parameters

name
Name of child to find, or 0 to return first child regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found child, or 0 if not found.

function xml_node::last_node

Synopsis

xml_node<Ch>* last_node(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets last child node, optionally matching node name. Behaviour is undefined if node has no children. Use first_node() to test if node has children.

Parameters

name
Name of child to find, or 0 to return last child regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found child, or 0 if not found.

function xml_node::previous_sibling

Synopsis

xml_node<Ch>* previous_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets previous sibling node, optionally matching node name. Behaviour is undefined if node has no parent. Use parent() to test if node has a parent.

Parameters

name
Name of sibling to find, or 0 to return previous sibling regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found sibling, or 0 if not found.

function xml_node::next_sibling

Synopsis

xml_node<Ch>* next_sibling(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets next sibling node, optionally matching node name. Behaviour is undefined if node has no parent. Use parent() to test if node has a parent.

Parameters

name
Name of sibling to find, or 0 to return next sibling regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found sibling, or 0 if not found.

function xml_node::first_attribute

Synopsis

xml_attribute<Ch>* first_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets first attribute of node, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return first attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

function xml_node::last_attribute

Synopsis

xml_attribute<Ch>* last_attribute(const Ch *name=0, std::size_t name_size=0, bool case_sensitive=true) const; +

Description

Gets last attribute of node, optionally matching attribute name.

Parameters

name
Name of attribute to find, or 0 to return last attribute regardless of its name; this string doesn't have to be zero-terminated if name_size is non-zero
name_size
Size of name, in characters, or 0 to have size calculated automatically from string
case_sensitive
Should name comparison be case-sensitive; non case-sensitive comparison works properly only for ASCII characters

Returns

Pointer to found attribute, or 0 if not found.

function xml_node::type

Synopsis

void type(node_type type); +

Description

Sets type of node.

Parameters

type
Type of node to set.

function xml_node::prepend_node

Synopsis

void prepend_node(xml_node< Ch > *child); +

Description

Prepends a new child node. The prepended child becomes the first child, and all existing children are moved one position back.

Parameters

child
Node to prepend.

function xml_node::append_node

Synopsis

void append_node(xml_node< Ch > *child); +

Description

Appends a new child node. The appended child becomes the last child.

Parameters

child
Node to append.

function xml_node::insert_node

Synopsis

void insert_node(xml_node< Ch > *where, xml_node< Ch > *child); +

Description

Inserts a new child node at specified place inside the node. All children after and including the specified node are moved one position back.

Parameters

where
Place where to insert the child, or 0 to insert at the back.
child
Node to insert.

function xml_node::remove_first_node

Synopsis

void remove_first_node(); +

Description

Removes first child node. If node has no children, behaviour is undefined. Use first_node() to test if node has children.

function xml_node::remove_last_node

Synopsis

void remove_last_node(); +

Description

Removes last child of the node. If node has no children, behaviour is undefined. Use first_node() to test if node has children.

function xml_node::remove_node

Synopsis

void remove_node(xml_node< Ch > *where); +

Description

Removes specified child from the node.

function xml_node::remove_all_nodes

Synopsis

void remove_all_nodes(); +

Description

Removes all child nodes (but not attributes).

function xml_node::prepend_attribute

Synopsis

void prepend_attribute(xml_attribute< Ch > *attribute); +

Description

Prepends a new attribute to the node.

Parameters

attribute
Attribute to prepend.

function xml_node::append_attribute

Synopsis

void append_attribute(xml_attribute< Ch > *attribute); +

Description

Appends a new attribute to the node.

Parameters

attribute
Attribute to append.

function xml_node::insert_attribute

Synopsis

void insert_attribute(xml_attribute< Ch > *where, xml_attribute< Ch > *attribute); +

Description

Inserts a new attribute at specified place inside the node. All attributes after and including the specified attribute are moved one position back.

Parameters

where
Place where to insert the attribute, or 0 to insert at the back.
attribute
Attribute to insert.

function xml_node::remove_first_attribute

Synopsis

void remove_first_attribute(); +

Description

Removes first attribute of the node. If node has no attributes, behaviour is undefined. Use first_attribute() to test if node has attributes.

function xml_node::remove_last_attribute

Synopsis

void remove_last_attribute(); +

Description

Removes last attribute of the node. If node has no attributes, behaviour is undefined. Use first_attribute() to test if node has attributes.

function xml_node::remove_attribute

Synopsis

void remove_attribute(xml_attribute< Ch > *where); +

Description

Removes specified attribute from node.

Parameters

where
Pointer to attribute to be removed.

function xml_node::remove_all_attributes

Synopsis

void remove_all_attributes(); +

Description

Removes all attributes of node.

enum node_type

Description

Enumeration listing all node types produced by the parser. Use xml_node::type() function to query node type.

Values

node_document
A document node. Name and value are empty.
node_element
An element node. Name contains element name. Value contains text of first data node.
node_data
A data node. Name is empty. Value contains data text.
node_cdata
A CDATA node. Name is empty. Value contains data text.
node_comment
A comment node. Name is empty. Value contains comment text.
node_declaration
A declaration node. Name and value are empty. Declaration parameters (version, encoding and standalone) are in node attributes.
node_doctype
A DOCTYPE node. Name is empty. Value contains DOCTYPE text.
node_pi
A PI node. Name contains target. Value contains instructions.

function parse_error_handler

Synopsis

void rapidxml::parse_error_handler(const char *what, void *where); +

Description

When exceptions are disabled by defining RAPIDXML_NO_EXCEPTIONS, this function is called to notify user about the error. It must be defined by the user.

+ This function cannot return. If it does, the results are undefined.

+ A very simple definition might look like that: + void rapidxml::parse_error_handler(const char *what, void *where) + { + std::cout << "Parse error: " << what << "\n"; + std::abort(); + } +

Parameters

what
Human readable description of the error.
where
Pointer to character data where error was detected.

function print

Synopsis

OutIt rapidxml::print(OutIt out, const xml_node< Ch > &node, int flags=0); +

Description

Prints XML to given output iterator.

Parameters

out
Output iterator to print to.
node
Node to be printed. Pass xml_document to print entire document.
flags
Flags controlling how XML is printed.

Returns

Output iterator pointing to position immediately after last character of printed text.

function print

Synopsis

std::basic_ostream<Ch>& rapidxml::print(std::basic_ostream< Ch > &out, const xml_node< Ch > &node, int flags=0); +

Description

Prints XML to given output stream.

Parameters

out
Output stream to print to.
node
Node to be printed. Pass xml_document to print entire document.
flags
Flags controlling how XML is printed.

Returns

Output stream.

function operator<<

Synopsis

std::basic_ostream<Ch>& rapidxml::operator<<(std::basic_ostream< Ch > &out, const xml_node< Ch > &node); +

Description

Prints formatted XML to given output stream. Uses default printing flags. Use print() function to customize printing process.

Parameters

out
Output stream to print to.
node
Node to be printed.

Returns

Output stream.

+ constant + parse_no_data_nodes

Synopsis

const int parse_no_data_nodes + = 0x1; +

Description

Parse flag instructing the parser to not create data nodes. Text of first data node will still be placed in value of parent element, unless rapidxml::parse_no_element_values flag is also specified. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_no_element_values

Synopsis

const int parse_no_element_values + = 0x2; +

Description

Parse flag instructing the parser to not use text of first data node as a value of parent element. Can be combined with other flags by use of | operator. Note that child data nodes of element node take precendence over its value when printing. That is, if element has one or more child data nodes and a value, the value will be ignored. Use rapidxml::parse_no_data_nodes flag to prevent creation of data nodes if you want to manipulate data using values of elements.

+ See xml_document::parse() function.

+ constant + parse_no_string_terminators

Synopsis

const int parse_no_string_terminators + = 0x4; +

Description

Parse flag instructing the parser to not place zero terminators after strings in the source text. By default zero terminators are placed, modifying source text. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_no_entity_translation

Synopsis

const int parse_no_entity_translation + = 0x8; +

Description

Parse flag instructing the parser to not translate entities in the source text. By default entities are translated, modifying source text. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_no_utf8

Synopsis

const int parse_no_utf8 + = 0x10; +

Description

Parse flag instructing the parser to disable UTF-8 handling and assume plain 8 bit characters. By default, UTF-8 handling is enabled. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_declaration_node

Synopsis

const int parse_declaration_node + = 0x20; +

Description

Parse flag instructing the parser to create XML declaration node. By default, declaration node is not created. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_comment_nodes

Synopsis

const int parse_comment_nodes + = 0x40; +

Description

Parse flag instructing the parser to create comments nodes. By default, comment nodes are not created. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_doctype_node

Synopsis

const int parse_doctype_node + = 0x80; +

Description

Parse flag instructing the parser to create DOCTYPE node. By default, doctype node is not created. Although W3C specification allows at most one DOCTYPE node, RapidXml will silently accept documents with more than one. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_pi_nodes

Synopsis

const int parse_pi_nodes + = 0x100; +

Description

Parse flag instructing the parser to create PI nodes. By default, PI nodes are not created. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_validate_closing_tags

Synopsis

const int parse_validate_closing_tags + = 0x200; +

Description

Parse flag instructing the parser to validate closing tag names. If not set, name inside closing tag is irrelevant to the parser. By default, closing tags are not validated. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_trim_whitespace

Synopsis

const int parse_trim_whitespace + = 0x400; +

Description

Parse flag instructing the parser to trim all leading and trailing whitespace of data nodes. By default, whitespace is not trimmed. This flag does not cause the parser to modify source text. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_normalize_whitespace

Synopsis

const int parse_normalize_whitespace + = 0x800; +

Description

Parse flag instructing the parser to condense all whitespace runs of data nodes to a single space character. Trimming of leading and trailing whitespace of data is controlled by rapidxml::parse_trim_whitespace flag. By default, whitespace is not normalized. If this flag is specified, source text will be modified. Can be combined with other flags by use of | operator.

+ See xml_document::parse() function.

+ constant + parse_default

Synopsis

const int parse_default + = 0; +

Description

Parse flags which represent default behaviour of the parser. This is always equal to 0, so that all other flags can be simply ored together. Normally there is no need to inconveniently disable flags by anding with their negated (~) values. This also means that meaning of each flag is a negation of the default setting. For example, if flag name is rapidxml::parse_no_utf8, it means that utf-8 is enabled by default, and using the flag will disable it.

+ See xml_document::parse() function.

+ constant + parse_non_destructive

Synopsis

const int parse_non_destructive + = parse_no_string_terminators | parse_no_entity_translation; +

Description

A combination of parse flags that forbids any modifications of the source text. This also results in faster parsing. However, note that the following will occur:
  • names and values of nodes will not be zero terminated, you have to use xml_base::name_size() and xml_base::value_size() functions to determine where name and value ends
  • entities will not be translated
  • whitespace will not be normalized
+See xml_document::parse() function.

+ constant + parse_fastest

Synopsis

const int parse_fastest + = parse_non_destructive | parse_no_data_nodes; +

Description

A combination of parse flags resulting in fastest possible parsing, without sacrificing important data.

+ See xml_document::parse() function.

+ constant + parse_full

Synopsis

const int parse_full + = parse_declaration_node | parse_comment_nodes | parse_doctype_node | parse_pi_nodes | parse_validate_closing_tags; +

Description

A combination of parse flags resulting in largest amount of data being extracted. This usually results in slowest parsing.

+ See xml_document::parse() function.

+ constant + print_no_indenting

Synopsis

const int print_no_indenting + = 0x1;

Description

Printer flag instructing the printer to suppress indenting of XML. See print() function.

\ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox/sandbox.vcxproj b/tpl/cereal/vs2013/sandbox/sandbox.vcxproj index 89da192..36efde6 100644 --- a/tpl/cereal/vs2013/sandbox/sandbox.vcxproj +++ b/tpl/cereal/vs2013/sandbox/sandbox.vcxproj @@ -1,256 +1,256 @@ - - - - - DEBUG_VS2015 - Win32 - - - DEBUG_VS2015 - x64 - - - Debug - Win32 - - - Debug - x64 - - - Release_VS2015 - Win32 - - - Release_VS2015 - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - {DAC23DBB-630B-46B9-B115-F1449697898A} - sandbox - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - C:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - C:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - C:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - C:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - false - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - false - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - false - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - false - - - true - true - true - - - - - + + + + + DEBUG_VS2015 + Win32 + + + DEBUG_VS2015 + x64 + + + Debug + Win32 + + + Debug + x64 + + + Release_VS2015 + Win32 + + + Release_VS2015 + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + {DAC23DBB-630B-46B9-B115-F1449697898A} + sandbox + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + C:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + C:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + C:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + C:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + false + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + false + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + false + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + false + + + true + true + true + + + + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox/sandbox.vcxproj.filters b/tpl/cereal/vs2013/sandbox/sandbox.vcxproj.filters index 90e0f7d..c67aa80 100644 --- a/tpl/cereal/vs2013/sandbox/sandbox.vcxproj.filters +++ b/tpl/cereal/vs2013/sandbox/sandbox.vcxproj.filters @@ -1,22 +1,22 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hh;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox_json/sandbox_json.vcxproj b/tpl/cereal/vs2013/sandbox_json/sandbox_json.vcxproj index fc18258..1255547 100644 --- a/tpl/cereal/vs2013/sandbox_json/sandbox_json.vcxproj +++ b/tpl/cereal/vs2013/sandbox_json/sandbox_json.vcxproj @@ -1,248 +1,248 @@ - - - - - DEBUG_VS2015 - Win32 - - - DEBUG_VS2015 - x64 - - - Debug - Win32 - - - Debug - x64 - - - Release_VS2015 - Win32 - - - Release_VS2015 - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - {52E96EC3-125A-4525-BC72-F3C524F24640} - sandbox_json - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - + + + + + DEBUG_VS2015 + Win32 + + + DEBUG_VS2015 + x64 + + + Debug + Win32 + + + Debug + x64 + + + Release_VS2015 + Win32 + + + Release_VS2015 + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + {52E96EC3-125A-4525-BC72-F3C524F24640} + sandbox_json + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox_json/sandbox_json.vcxproj.filters b/tpl/cereal/vs2013/sandbox_json/sandbox_json.vcxproj.filters index 3edef0a..1186410 100644 --- a/tpl/cereal/vs2013/sandbox_json/sandbox_json.vcxproj.filters +++ b/tpl/cereal/vs2013/sandbox_json/sandbox_json.vcxproj.filters @@ -1,22 +1,22 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hh;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox_rtti/sandbox_rtti.vcxproj b/tpl/cereal/vs2013/sandbox_rtti/sandbox_rtti.vcxproj index 035a8ca..7be4b4d 100644 --- a/tpl/cereal/vs2013/sandbox_rtti/sandbox_rtti.vcxproj +++ b/tpl/cereal/vs2013/sandbox_rtti/sandbox_rtti.vcxproj @@ -1,252 +1,252 @@ - - - - - DEBUG_VS2015 - Win32 - - - DEBUG_VS2015 - x64 - - - Debug - Win32 - - - Debug - x64 - - - Release_VS2015 - Win32 - - - Release_VS2015 - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - {C81EF3F9-7849-4506-898C-85D0D564CD6A} - sandbox_rtti - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - C:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - C:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - C:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) - C:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - + + + + + DEBUG_VS2015 + Win32 + + + DEBUG_VS2015 + x64 + + + Debug + Win32 + + + Debug + x64 + + + Release_VS2015 + Win32 + + + Release_VS2015 + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + {C81EF3F9-7849-4506-898C-85D0D564CD6A} + sandbox_rtti + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + C:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + C:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + C:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_54;$(IncludePath) + C:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + $(SolutionDir)\..\include;C:\Boost\include\boost-1_55;$(IncludePath) + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox_rtti/sandbox_rtti.vcxproj.filters b/tpl/cereal/vs2013/sandbox_rtti/sandbox_rtti.vcxproj.filters index 813de5f..8704a74 100644 --- a/tpl/cereal/vs2013/sandbox_rtti/sandbox_rtti.vcxproj.filters +++ b/tpl/cereal/vs2013/sandbox_rtti/sandbox_rtti.vcxproj.filters @@ -1,22 +1,22 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hh;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox_vs/sandbox_vs.vcxproj b/tpl/cereal/vs2013/sandbox_vs/sandbox_vs.vcxproj index 4615581..ca7d629 100644 --- a/tpl/cereal/vs2013/sandbox_vs/sandbox_vs.vcxproj +++ b/tpl/cereal/vs2013/sandbox_vs/sandbox_vs.vcxproj @@ -1,266 +1,266 @@ - - - - - DEBUG_VS2015 - Win32 - - - DEBUG_VS2015 - x64 - - - Debug - Win32 - - - Debug - x64 - - - Release_VS2015 - Win32 - - - Release_VS2015 - x64 - - - Release - Win32 - - - Release - x64 - - - - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734} - sandbox_vs - - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) - $(SolutionDir)\Debug;E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) - $(SolutionDir)\Debug;E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) - $(SolutionDir)\x64\Debug;$(LibraryPath) - - - $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) - $(SolutionDir)\x64\Debug;$(LibraryPath) - - - $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) - $(SolutionDir)\Release;E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) - $(SolutionDir)\Release;E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) - $(SolutionDir)\x64\Release;$(LibraryPath) - - - $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) - $(SolutionDir)\x64\Release;$(LibraryPath) - - - - Level4 - Disabled - true - false - - - true - sandbox_vs_dll.lib;%(AdditionalDependencies) - - - - - Level4 - Disabled - true - false - - - true - sandbox_vs_dll.lib;%(AdditionalDependencies) - - - - - Level4 - Disabled - true - - - true - sandbox_vs_dll.lib;%(AdditionalDependencies) - - - - - Level4 - Disabled - true - - - true - sandbox_vs_dll.lib;%(AdditionalDependencies) - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - sandbox_vs_dll.lib;%(AdditionalDependencies) - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - sandbox_vs_dll.lib;%(AdditionalDependencies) - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - sandbox_vs_dll.lib;%(AdditionalDependencies) - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - sandbox_vs_dll.lib;%(AdditionalDependencies) - - - - - - - - + + + + + DEBUG_VS2015 + Win32 + + + DEBUG_VS2015 + x64 + + + Debug + Win32 + + + Debug + x64 + + + Release_VS2015 + Win32 + + + Release_VS2015 + x64 + + + Release + Win32 + + + Release + x64 + + + + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734} + sandbox_vs + + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) + $(SolutionDir)\Debug;E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) + $(SolutionDir)\Debug;E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) + $(SolutionDir)\x64\Debug;$(LibraryPath) + + + $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) + $(SolutionDir)\x64\Debug;$(LibraryPath) + + + $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) + $(SolutionDir)\Release;E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) + $(SolutionDir)\Release;E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) + $(SolutionDir)\x64\Release;$(LibraryPath) + + + $(SolutionDir)\sandbox_vs_dll;$(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(SolutionDir)\..\sandbox\sandbox_shared_lib;$(IncludePath) + $(SolutionDir)\x64\Release;$(LibraryPath) + + + + Level4 + Disabled + true + false + + + true + sandbox_vs_dll.lib;%(AdditionalDependencies) + + + + + Level4 + Disabled + true + false + + + true + sandbox_vs_dll.lib;%(AdditionalDependencies) + + + + + Level4 + Disabled + true + + + true + sandbox_vs_dll.lib;%(AdditionalDependencies) + + + + + Level4 + Disabled + true + + + true + sandbox_vs_dll.lib;%(AdditionalDependencies) + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + sandbox_vs_dll.lib;%(AdditionalDependencies) + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + sandbox_vs_dll.lib;%(AdditionalDependencies) + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + sandbox_vs_dll.lib;%(AdditionalDependencies) + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + sandbox_vs_dll.lib;%(AdditionalDependencies) + + + + + + + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox_vs/sandbox_vs.vcxproj.filters b/tpl/cereal/vs2013/sandbox_vs/sandbox_vs.vcxproj.filters index 70ecf39..c04e905 100644 --- a/tpl/cereal/vs2013/sandbox_vs/sandbox_vs.vcxproj.filters +++ b/tpl/cereal/vs2013/sandbox_vs/sandbox_vs.vcxproj.filters @@ -1,22 +1,22 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hh;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox_vs_dll/sandbox_vs_dll.vcxproj b/tpl/cereal/vs2013/sandbox_vs_dll/sandbox_vs_dll.vcxproj index a335650..8aaee5c 100755 --- a/tpl/cereal/vs2013/sandbox_vs_dll/sandbox_vs_dll.vcxproj +++ b/tpl/cereal/vs2013/sandbox_vs_dll/sandbox_vs_dll.vcxproj @@ -1,261 +1,261 @@ - - - - - DEBUG_VS2015 - Win32 - - - DEBUG_VS2015 - x64 - - - Debug - Win32 - - - Debug - x64 - - - Release_VS2015 - Win32 - - - Release_VS2015 - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - - - - - - {95D6662F-4166-40D2-87D7-CABFB60C199A} - sandbox_vs_dll - - - - DynamicLibrary - true - v120 - MultiByte - - - DynamicLibrary - true - v120 - MultiByte - - - DynamicLibrary - true - v120 - MultiByte - - - DynamicLibrary - true - v120 - MultiByte - - - DynamicLibrary - false - v120 - true - MultiByte - - - DynamicLibrary - false - v120 - true - MultiByte - - - DynamicLibrary - false - v120 - true - MultiByte - - - DynamicLibrary - false - v120 - true - MultiByte - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - Disabled - true - - - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - Level4 - MaxSpeed - true - true - true - - - true - true - true - - - - - + + + + + DEBUG_VS2015 + Win32 + + + DEBUG_VS2015 + x64 + + + Debug + Win32 + + + Debug + x64 + + + Release_VS2015 + Win32 + + + Release_VS2015 + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + + + + + {95D6662F-4166-40D2-87D7-CABFB60C199A} + sandbox_vs_dll + + + + DynamicLibrary + true + v120 + MultiByte + + + DynamicLibrary + true + v120 + MultiByte + + + DynamicLibrary + true + v120 + MultiByte + + + DynamicLibrary + true + v120 + MultiByte + + + DynamicLibrary + false + v120 + true + MultiByte + + + DynamicLibrary + false + v120 + true + MultiByte + + + DynamicLibrary + false + v120 + true + MultiByte + + + DynamicLibrary + false + v120 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + Disabled + true + + + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + Level4 + MaxSpeed + true + true + true + + + true + true + true + + + + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/sandbox_vs_dll/sandbox_vs_dll.vcxproj.filters b/tpl/cereal/vs2013/sandbox_vs_dll/sandbox_vs_dll.vcxproj.filters index d61909d..5b6bcf3 100755 --- a/tpl/cereal/vs2013/sandbox_vs_dll/sandbox_vs_dll.vcxproj.filters +++ b/tpl/cereal/vs2013/sandbox_vs_dll/sandbox_vs_dll.vcxproj.filters @@ -1,33 +1,33 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hh;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - Source Files - - - - - Source Files - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + + + Source Files + + + Source Files + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/unittests/unittests.vcxproj b/tpl/cereal/vs2013/unittests/unittests.vcxproj index d3c82ca..536f918 100644 --- a/tpl/cereal/vs2013/unittests/unittests.vcxproj +++ b/tpl/cereal/vs2013/unittests/unittests.vcxproj @@ -1,327 +1,327 @@ - - - - - DEBUG_VS2015 - Win32 - - - DEBUG_VS2015 - x64 - - - Debug - Win32 - - - Debug - x64 - - - Release_VS2015 - Win32 - - - Release_VS2015 - x64 - - - Release - Win32 - - - Release - x64 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7} - unittests - - - - Application - true - v120 - MultiByte - - - Application - true - v140 - MultiByte - - - Application - true - v120 - MultiByte - - - Application - true - v140 - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v140 - true - MultiByte - - - Application - false - v120 - true - MultiByte - - - Application - false - v140 - true - MultiByte - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib\x64;$(LibraryPath) - - - $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib\x64;$(LibraryPath) - - - $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib;$(LibraryPath) - - - $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib\x64;$(LibraryPath) - false - - - $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) - E:\Boost\lib\x64;$(LibraryPath) - false - - - - Level4 - Disabled - true - true - /bigobj %(AdditionalOptions) - true - - - true - %(AdditionalDependencies) - NotSet - Console - - - - - Level4 - Disabled - true - true - /bigobj %(AdditionalOptions) - true - - - true - %(AdditionalDependencies) - NotSet - Console - - - - - Level4 - Disabled - true - /bigobj %(AdditionalOptions) - true - - - true - Console - - - - - Level4 - Disabled - true - /bigobj %(AdditionalOptions) - true - - - true - Console - - - - - Level4 - MaxSpeed - true - true - true - /bigobj %(AdditionalOptions) - true - - - true - true - true - %(AdditionalDependencies) - Console - - - - - Level4 - MaxSpeed - true - true - true - /bigobj %(AdditionalOptions) - true - - - true - true - true - %(AdditionalDependencies) - Console - - - - - Level4 - MaxSpeed - true - true - true - /bigobj %(AdditionalOptions) - false - true - - - true - true - true - Console - - - - - Level4 - MaxSpeed - true - true - true - /bigobj %(AdditionalOptions) - false - true - - - true - true - true - Console - - - - - + + + + + DEBUG_VS2015 + Win32 + + + DEBUG_VS2015 + x64 + + + Debug + Win32 + + + Debug + x64 + + + Release_VS2015 + Win32 + + + Release_VS2015 + x64 + + + Release + Win32 + + + Release + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7} + unittests + + + + Application + true + v120 + MultiByte + + + Application + true + v140 + MultiByte + + + Application + true + v120 + MultiByte + + + Application + true + v140 + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v140 + true + MultiByte + + + Application + false + v120 + true + MultiByte + + + Application + false + v140 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib\x64;$(LibraryPath) + + + $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib\x64;$(LibraryPath) + + + $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib;$(LibraryPath) + + + $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib\x64;$(LibraryPath) + false + + + $(SolutionDir)\..\include;$(SolutionDir)\..\unittests;E:\Boost\include\boost-1_55;$(IncludePath) + E:\Boost\lib\x64;$(LibraryPath) + false + + + + Level4 + Disabled + true + true + /bigobj %(AdditionalOptions) + true + + + true + %(AdditionalDependencies) + NotSet + Console + + + + + Level4 + Disabled + true + true + /bigobj %(AdditionalOptions) + true + + + true + %(AdditionalDependencies) + NotSet + Console + + + + + Level4 + Disabled + true + /bigobj %(AdditionalOptions) + true + + + true + Console + + + + + Level4 + Disabled + true + /bigobj %(AdditionalOptions) + true + + + true + Console + + + + + Level4 + MaxSpeed + true + true + true + /bigobj %(AdditionalOptions) + true + + + true + true + true + %(AdditionalDependencies) + Console + + + + + Level4 + MaxSpeed + true + true + true + /bigobj %(AdditionalOptions) + true + + + true + true + true + %(AdditionalDependencies) + Console + + + + + Level4 + MaxSpeed + true + true + true + /bigobj %(AdditionalOptions) + false + true + + + true + true + true + Console + + + + + Level4 + MaxSpeed + true + true + true + /bigobj %(AdditionalOptions) + false + true + + + true + true + true + Console + + + + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/unittests/unittests.vcxproj.filters b/tpl/cereal/vs2013/unittests/unittests.vcxproj.filters index 902b535..b5c45f0 100644 --- a/tpl/cereal/vs2013/unittests/unittests.vcxproj.filters +++ b/tpl/cereal/vs2013/unittests/unittests.vcxproj.filters @@ -1,127 +1,127 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hh;hpp;hxx;hm;inl;inc;xsd - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + \ No newline at end of file diff --git a/tpl/cereal/vs2013/vs2013.sln b/tpl/cereal/vs2013/vs2013.sln index d0ed26a..c4e19fc 100644 --- a/tpl/cereal/vs2013/vs2013.sln +++ b/tpl/cereal/vs2013/vs2013.sln @@ -1,147 +1,147 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.24720.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox_vs", "sandbox_vs\sandbox_vs.vcxproj", "{A67E36D2-32BE-4D4D-BA2E-8FD59378E734}" - ProjectSection(ProjectDependencies) = postProject - {95D6662F-4166-40D2-87D7-CABFB60C199A} = {95D6662F-4166-40D2-87D7-CABFB60C199A} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox_json", "sandbox_json\sandbox_json.vcxproj", "{52E96EC3-125A-4525-BC72-F3C524F24640}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox", "sandbox\sandbox.vcxproj", "{DAC23DBB-630B-46B9-B115-F1449697898A}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox_rtti", "sandbox_rtti\sandbox_rtti.vcxproj", "{C81EF3F9-7849-4506-898C-85D0D564CD6A}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "unittests", "unittests\unittests.vcxproj", "{D447E13A-97A4-4907-9F61-A9BCCDB91EF7}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "performance", "performance\performance.vcxproj", "{2F374733-FCA8-4CBB-91A0-2B0B34393D86}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox_vs_dll", "sandbox_vs_dll\sandbox_vs_dll.vcxproj", "{95D6662F-4166-40D2-87D7-CABFB60C199A}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug_VS2013|Win32 = Debug_VS2013|Win32 - Debug_VS2013|x64 = Debug_VS2013|x64 - Debug_VS2015|Win32 = Debug_VS2015|Win32 - Debug_VS2015|x64 = Debug_VS2015|x64 - Release_VS2013|Win32 = Release_VS2013|Win32 - Release_VS2013|x64 = Release_VS2013|x64 - Release_VS2015|Win32 = Release_VS2015|Win32 - Release_VS2015|x64 = Release_VS2015|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2013|Win32.Build.0 = Debug|Win32 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2013|x64.ActiveCfg = Debug|x64 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2013|x64.Build.0 = Debug|x64 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2013|Win32.ActiveCfg = Release|Win32 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2013|Win32.Build.0 = Release|Win32 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2013|x64.ActiveCfg = Release|x64 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2013|x64.Build.0 = Release|x64 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 - {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2013|Win32.Build.0 = Debug|Win32 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2013|x64.ActiveCfg = Debug|x64 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2013|x64.Build.0 = Debug|x64 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2013|Win32.ActiveCfg = Release|Win32 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2013|Win32.Build.0 = Release|Win32 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2013|x64.ActiveCfg = Release|x64 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2013|x64.Build.0 = Release|x64 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 - {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2013|Win32.Build.0 = Debug|Win32 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2013|x64.ActiveCfg = Debug|x64 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2013|x64.Build.0 = Debug|x64 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2013|Win32.ActiveCfg = Release|Win32 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2013|Win32.Build.0 = Release|Win32 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2013|x64.ActiveCfg = Release|x64 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2013|x64.Build.0 = Release|x64 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 - {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2013|Win32.Build.0 = Debug|Win32 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2013|x64.ActiveCfg = Debug|x64 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2013|x64.Build.0 = Debug|x64 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2013|Win32.ActiveCfg = Release|Win32 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2013|Win32.Build.0 = Release|Win32 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2013|x64.ActiveCfg = Release|x64 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2013|x64.Build.0 = Release|x64 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 - {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2013|Win32.Build.0 = Debug|Win32 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2013|x64.ActiveCfg = Debug|x64 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2013|x64.Build.0 = Debug|x64 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2013|Win32.ActiveCfg = Release|Win32 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2013|Win32.Build.0 = Release|Win32 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2013|x64.ActiveCfg = Release|x64 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2013|x64.Build.0 = Release|x64 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 - {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2013|Win32.Build.0 = Debug|Win32 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2013|x64.ActiveCfg = Debug|x64 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2013|Win32.ActiveCfg = Release|Win32 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2013|Win32.Build.0 = Release|Win32 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2013|x64.ActiveCfg = Release|x64 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 - {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2013|Win32.Build.0 = Debug|Win32 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2013|x64.ActiveCfg = Debug|x64 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2013|Win32.ActiveCfg = Release|Win32 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2013|Win32.Build.0 = Release|Win32 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2013|x64.ActiveCfg = Release|x64 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 - {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.24720.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox_vs", "sandbox_vs\sandbox_vs.vcxproj", "{A67E36D2-32BE-4D4D-BA2E-8FD59378E734}" + ProjectSection(ProjectDependencies) = postProject + {95D6662F-4166-40D2-87D7-CABFB60C199A} = {95D6662F-4166-40D2-87D7-CABFB60C199A} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox_json", "sandbox_json\sandbox_json.vcxproj", "{52E96EC3-125A-4525-BC72-F3C524F24640}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox", "sandbox\sandbox.vcxproj", "{DAC23DBB-630B-46B9-B115-F1449697898A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox_rtti", "sandbox_rtti\sandbox_rtti.vcxproj", "{C81EF3F9-7849-4506-898C-85D0D564CD6A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "unittests", "unittests\unittests.vcxproj", "{D447E13A-97A4-4907-9F61-A9BCCDB91EF7}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "performance", "performance\performance.vcxproj", "{2F374733-FCA8-4CBB-91A0-2B0B34393D86}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "sandbox_vs_dll", "sandbox_vs_dll\sandbox_vs_dll.vcxproj", "{95D6662F-4166-40D2-87D7-CABFB60C199A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug_VS2013|Win32 = Debug_VS2013|Win32 + Debug_VS2013|x64 = Debug_VS2013|x64 + Debug_VS2015|Win32 = Debug_VS2015|Win32 + Debug_VS2015|x64 = Debug_VS2015|x64 + Release_VS2013|Win32 = Release_VS2013|Win32 + Release_VS2013|x64 = Release_VS2013|x64 + Release_VS2015|Win32 = Release_VS2015|Win32 + Release_VS2015|x64 = Release_VS2015|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2013|Win32.Build.0 = Debug|Win32 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2013|x64.ActiveCfg = Debug|x64 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2013|x64.Build.0 = Debug|x64 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2013|Win32.ActiveCfg = Release|Win32 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2013|Win32.Build.0 = Release|Win32 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2013|x64.ActiveCfg = Release|x64 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2013|x64.Build.0 = Release|x64 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 + {A67E36D2-32BE-4D4D-BA2E-8FD59378E734}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2013|Win32.Build.0 = Debug|Win32 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2013|x64.ActiveCfg = Debug|x64 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2013|x64.Build.0 = Debug|x64 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2013|Win32.ActiveCfg = Release|Win32 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2013|Win32.Build.0 = Release|Win32 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2013|x64.ActiveCfg = Release|x64 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2013|x64.Build.0 = Release|x64 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 + {52E96EC3-125A-4525-BC72-F3C524F24640}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2013|Win32.Build.0 = Debug|Win32 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2013|x64.ActiveCfg = Debug|x64 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2013|x64.Build.0 = Debug|x64 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2013|Win32.ActiveCfg = Release|Win32 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2013|Win32.Build.0 = Release|Win32 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2013|x64.ActiveCfg = Release|x64 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2013|x64.Build.0 = Release|x64 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 + {DAC23DBB-630B-46B9-B115-F1449697898A}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2013|Win32.Build.0 = Debug|Win32 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2013|x64.ActiveCfg = Debug|x64 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2013|x64.Build.0 = Debug|x64 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2013|Win32.ActiveCfg = Release|Win32 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2013|Win32.Build.0 = Release|Win32 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2013|x64.ActiveCfg = Release|x64 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2013|x64.Build.0 = Release|x64 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 + {C81EF3F9-7849-4506-898C-85D0D564CD6A}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2013|Win32.Build.0 = Debug|Win32 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2013|x64.ActiveCfg = Debug|x64 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2013|x64.Build.0 = Debug|x64 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2013|Win32.ActiveCfg = Release|Win32 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2013|Win32.Build.0 = Release|Win32 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2013|x64.ActiveCfg = Release|x64 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2013|x64.Build.0 = Release|x64 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 + {D447E13A-97A4-4907-9F61-A9BCCDB91EF7}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2013|Win32.Build.0 = Debug|Win32 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2013|x64.ActiveCfg = Debug|x64 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2013|Win32.ActiveCfg = Release|Win32 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2013|Win32.Build.0 = Release|Win32 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2013|x64.ActiveCfg = Release|x64 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 + {2F374733-FCA8-4CBB-91A0-2B0B34393D86}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2013|Win32.ActiveCfg = Debug|Win32 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2013|Win32.Build.0 = Debug|Win32 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2013|x64.ActiveCfg = Debug|x64 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2015|Win32.ActiveCfg = DEBUG_VS2015|Win32 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2015|Win32.Build.0 = DEBUG_VS2015|Win32 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2015|x64.ActiveCfg = DEBUG_VS2015|x64 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Debug_VS2015|x64.Build.0 = DEBUG_VS2015|x64 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2013|Win32.ActiveCfg = Release|Win32 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2013|Win32.Build.0 = Release|Win32 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2013|x64.ActiveCfg = Release|x64 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2015|Win32.ActiveCfg = Release_VS2015|Win32 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2015|Win32.Build.0 = Release_VS2015|Win32 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2015|x64.ActiveCfg = Release_VS2015|x64 + {95D6662F-4166-40D2-87D7-CABFB60C199A}.Release_VS2015|x64.Build.0 = Release_VS2015|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/tpl/gperftools/README_windows.txt b/tpl/gperftools/README_windows.txt index 7bba122..406bd01 100644 --- a/tpl/gperftools/README_windows.txt +++ b/tpl/gperftools/README_windows.txt @@ -1,120 +1,120 @@ ---- COMPILING - -This project has begun being ported to Windows, only tcmalloc_minimal -is supported at this time. A working solution file exists in this -directory: - gperftools.sln - -You can load this solution file into VC++ 7.1 (Visual Studio 2003) or -later -- in the latter case, it will automatically convert the files -to the latest format for you. - -When you build the solution, it will create a number of unittests, -which you can run by hand (or, more easily, under the Visual Studio -debugger) to make sure everything is working properly on your system. -The binaries will end up in a directory called "debug" or "release" in -the top-level directory (next to the .sln file). It will also create -two binaries, nm-pdb and addr2line-pdb, which you should install in -the same directory you install the 'pprof' perl script. - -I don't know very much about how to install DLLs on Windows, so you'll -have to figure out that part for yourself. If you choose to just -re-use the existing .sln, make sure you set the IncludeDir's -appropriately! Look at the properties for libtcmalloc_minimal.dll. - -Note that these systems are set to build in Debug mode by default. -You may want to change them to Release mode. - -To use tcmalloc_minimal in your own projects, you should only need to -build the dll and install it someplace, so you can link it into -further binaries. To use the dll, you need to add the following to -the linker line of your executable: - "libtcmalloc_minimal.lib" /INCLUDE:"__tcmalloc" - -Here is how to accomplish this in Visual Studio 2005 (VC8): - -1) Have your executable depend on the tcmalloc library by selecting - "Project Dependencies..." from the "Project" menu. Your executable - should depend on "libtcmalloc_minimal". - -2) Have your executable depend on a tcmalloc symbol -- this is - necessary so the linker doesn't "optimize out" the libtcmalloc - dependency -- by right-clicking on your executable's project (in - the solution explorer), selecting Properties from the pull-down - menu, then selecting "Configuration Properties" -> "Linker" -> - "Input". Then, in the "Force Symbol References" field, enter the - text "__tcmalloc" (without the quotes). Be sure to do this for both - debug and release modes! - -You can also link tcmalloc code in statically -- see the example -project tcmalloc_minimal_unittest-static, which does this. For this -to work, you'll need to add "/D PERFTOOLS_DLL_DECL=" to the compile -line of every perftools .cc file. You do not need to depend on the -tcmalloc symbol in this case (that is, you don't need to do either -step 1 or step 2 from above). - -An alternative to all the above is to statically link your application -with libc, and then replace its malloc with tcmalloc. This allows you -to just build and link your program normally; the tcmalloc support -comes in a post-processing step. This is more reliable than the above -technique (which depends on run-time patching, which is inherently -fragile), though more work to set up. For details, see - https://groups.google.com/group/google-perftools/browse_thread/thread/41cd3710af85e57b - - ---- THE HEAP-PROFILER - -The heap-profiler has had a preliminary port to Windows but does not -build on Windows by default. It has not been well tested, and -probably does not work at all when Frame Pointer Optimization (FPO) is -enabled -- that is, in release mode. The other features of perftools, -such as the cpu-profiler and leak-checker, have not yet been ported to -Windows at all. - - ---- WIN64 - -The function-patcher has to disassemble code, and is very -x86-specific. However, the rest of perftools should work fine for -both x86 and x64. In particular, if you use the 'statically link with -libc, and replace its malloc with tcmalloc' approach, mentioned above, -it should be possible to use tcmalloc with 64-bit windows. - -As of perftools 1.10, there is some support for disassembling x86_64 -instructions, for work with win64. This work is preliminary, but the -test file preamble_patcher_test.cc is provided to play around with -that a bit. preamble_patcher_test will not compile on win32. - - ---- ISSUES - -NOTE FOR WIN2K USERS: According to reports -(http://code.google.com/p/gperftools/issues/detail?id=127) -the stack-tracing necessary for the heap-profiler does not work on -Win2K. The best workaround is, if you are building on a Win2k system -is to add "/D NO_TCMALLOC_SAMPLES=" to your build, to turn off the -stack-tracing. You will not be able to use the heap-profiler if you -do this. - -NOTE ON _MSIZE and _RECALLOC: The tcmalloc version of _msize returns -the size of the region tcmalloc allocated for you -- which is at least -as many bytes you asked for, but may be more. (btw, these *are* bytes -you own, even if you didn't ask for all of them, so it's correct code -to access all of them if you want.) Unfortunately, the Windows CRT -_recalloc() routine assumes that _msize returns exactly as many bytes -as were requested. As a result, _recalloc() may not zero out new -bytes correctly. IT'S SAFEST NOT TO USE _RECALLOC WITH TCMALLOC. -_recalloc() is a tricky routine to use in any case (it's not safe to -use with realloc, for instance). - - -I have little experience with Windows programming, so there may be -better ways to set this up than I've done! If you run across any -problems, please post to the google-perftools Google Group, or report -them on the gperftools Google Code site: - http://groups.google.com/group/google-perftools - http://code.google.com/p/gperftools/issues/list - --- craig - -Last modified: 2 February 2012 +--- COMPILING + +This project has begun being ported to Windows, only tcmalloc_minimal +is supported at this time. A working solution file exists in this +directory: + gperftools.sln + +You can load this solution file into VC++ 7.1 (Visual Studio 2003) or +later -- in the latter case, it will automatically convert the files +to the latest format for you. + +When you build the solution, it will create a number of unittests, +which you can run by hand (or, more easily, under the Visual Studio +debugger) to make sure everything is working properly on your system. +The binaries will end up in a directory called "debug" or "release" in +the top-level directory (next to the .sln file). It will also create +two binaries, nm-pdb and addr2line-pdb, which you should install in +the same directory you install the 'pprof' perl script. + +I don't know very much about how to install DLLs on Windows, so you'll +have to figure out that part for yourself. If you choose to just +re-use the existing .sln, make sure you set the IncludeDir's +appropriately! Look at the properties for libtcmalloc_minimal.dll. + +Note that these systems are set to build in Debug mode by default. +You may want to change them to Release mode. + +To use tcmalloc_minimal in your own projects, you should only need to +build the dll and install it someplace, so you can link it into +further binaries. To use the dll, you need to add the following to +the linker line of your executable: + "libtcmalloc_minimal.lib" /INCLUDE:"__tcmalloc" + +Here is how to accomplish this in Visual Studio 2005 (VC8): + +1) Have your executable depend on the tcmalloc library by selecting + "Project Dependencies..." from the "Project" menu. Your executable + should depend on "libtcmalloc_minimal". + +2) Have your executable depend on a tcmalloc symbol -- this is + necessary so the linker doesn't "optimize out" the libtcmalloc + dependency -- by right-clicking on your executable's project (in + the solution explorer), selecting Properties from the pull-down + menu, then selecting "Configuration Properties" -> "Linker" -> + "Input". Then, in the "Force Symbol References" field, enter the + text "__tcmalloc" (without the quotes). Be sure to do this for both + debug and release modes! + +You can also link tcmalloc code in statically -- see the example +project tcmalloc_minimal_unittest-static, which does this. For this +to work, you'll need to add "/D PERFTOOLS_DLL_DECL=" to the compile +line of every perftools .cc file. You do not need to depend on the +tcmalloc symbol in this case (that is, you don't need to do either +step 1 or step 2 from above). + +An alternative to all the above is to statically link your application +with libc, and then replace its malloc with tcmalloc. This allows you +to just build and link your program normally; the tcmalloc support +comes in a post-processing step. This is more reliable than the above +technique (which depends on run-time patching, which is inherently +fragile), though more work to set up. For details, see + https://groups.google.com/group/google-perftools/browse_thread/thread/41cd3710af85e57b + + +--- THE HEAP-PROFILER + +The heap-profiler has had a preliminary port to Windows but does not +build on Windows by default. It has not been well tested, and +probably does not work at all when Frame Pointer Optimization (FPO) is +enabled -- that is, in release mode. The other features of perftools, +such as the cpu-profiler and leak-checker, have not yet been ported to +Windows at all. + + +--- WIN64 + +The function-patcher has to disassemble code, and is very +x86-specific. However, the rest of perftools should work fine for +both x86 and x64. In particular, if you use the 'statically link with +libc, and replace its malloc with tcmalloc' approach, mentioned above, +it should be possible to use tcmalloc with 64-bit windows. + +As of perftools 1.10, there is some support for disassembling x86_64 +instructions, for work with win64. This work is preliminary, but the +test file preamble_patcher_test.cc is provided to play around with +that a bit. preamble_patcher_test will not compile on win32. + + +--- ISSUES + +NOTE FOR WIN2K USERS: According to reports +(http://code.google.com/p/gperftools/issues/detail?id=127) +the stack-tracing necessary for the heap-profiler does not work on +Win2K. The best workaround is, if you are building on a Win2k system +is to add "/D NO_TCMALLOC_SAMPLES=" to your build, to turn off the +stack-tracing. You will not be able to use the heap-profiler if you +do this. + +NOTE ON _MSIZE and _RECALLOC: The tcmalloc version of _msize returns +the size of the region tcmalloc allocated for you -- which is at least +as many bytes you asked for, but may be more. (btw, these *are* bytes +you own, even if you didn't ask for all of them, so it's correct code +to access all of them if you want.) Unfortunately, the Windows CRT +_recalloc() routine assumes that _msize returns exactly as many bytes +as were requested. As a result, _recalloc() may not zero out new +bytes correctly. IT'S SAFEST NOT TO USE _RECALLOC WITH TCMALLOC. +_recalloc() is a tricky routine to use in any case (it's not safe to +use with realloc, for instance). + + +I have little experience with Windows programming, so there may be +better ways to set this up than I've done! If you run across any +problems, please post to the google-perftools Google Group, or report +them on the gperftools Google Code site: + http://groups.google.com/group/google-perftools + http://code.google.com/p/gperftools/issues/list + +-- craig + +Last modified: 2 February 2012 diff --git a/tpl/gperftools/src/gperftools/malloc_extension.h b/tpl/gperftools/src/gperftools/malloc_extension.h index 1be138a..8e2d45c 100644 --- a/tpl/gperftools/src/gperftools/malloc_extension.h +++ b/tpl/gperftools/src/gperftools/malloc_extension.h @@ -277,7 +277,7 @@ class PERFTOOLS_DLL_DECL MallocExtension { // system allocators. virtual void SetSystemAllocator(SysAllocator *a); - // Try to release num_bytes of free memory back to the operating + // Try to release num_user_bytes of free memory back to the operating // system for reuse. Use this extension with caution -- to get this // memory back may require faulting pages back in by the OS, and // that may be slow. (Currently only implemented in tcmalloc.) diff --git a/tpl/gperftools/src/page_heap.h b/tpl/gperftools/src/page_heap.h index 27b515a..e54b13f 100644 --- a/tpl/gperftools/src/page_heap.h +++ b/tpl/gperftools/src/page_heap.h @@ -233,7 +233,7 @@ class PERFTOOLS_DLL_DECL PageHeap { size_t GetMinSystemAlloc(void) {return kMinSystemAlloc;} void SetMinSystemAlloc(size_t min_system_alloc) { if( min_system_alloc > kMaxPages ) { - printf("ERROR: Minimum system allocation must be less than or equal to %d bytes\n", (kMaxPages << kPageShift)); + printf("ERROR: Minimum system allocation must be less than or equal to %ld bytes\n", (kMaxPages << kPageShift)); } else { min_system_alloc_ = min_system_alloc; } diff --git a/tpl/gperftools/src/tcmalloc.cc b/tpl/gperftools/src/tcmalloc.cc index 6f304a4..a526f66 100644 --- a/tpl/gperftools/src/tcmalloc.cc +++ b/tpl/gperftools/src/tcmalloc.cc @@ -892,7 +892,7 @@ class TCMallocImplementation : public MallocExtension { return; } num_bytes = num_bytes - extra_bytes_released_; - // num_bytes might be less than one page. If we pass zero to + // num_user_bytes might be less than one page. If we pass zero to // ReleaseAtLeastNPages, it won't do anything, so we release a whole // page now and let extra_bytes_released_ smooth it out over time. Length num_pages = max(num_bytes >> kPageShift, 1); @@ -901,7 +901,7 @@ class TCMallocImplementation : public MallocExtension { if (bytes_released > num_bytes) { extra_bytes_released_ = bytes_released - num_bytes; } else { - // The PageHeap wasn't able to release num_bytes. Don't try to + // The PageHeap wasn't able to release num_user_bytes. Don't try to // compensate with a big release next time. Specifically, // ReleaseFreeMemory() calls ReleaseToSystem(LONG_MAX). extra_bytes_released_ = 0; diff --git a/tpl/gperftools/src/windows/shortproc.asm b/tpl/gperftools/src/windows/shortproc.asm index 7e8e3d7..10b0b54 100644 --- a/tpl/gperftools/src/windows/shortproc.asm +++ b/tpl/gperftools/src/windows/shortproc.asm @@ -30,140 +30,140 @@ ; --- ; Author: Scott Francis ; -; Unit tests for PreamblePatcher - -.MODEL small - -.CODE - -TooShortFunction PROC - ret -TooShortFunction ENDP - -JumpShortCondFunction PROC - test cl, 1 - jnz jumpspot - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 - int 3 -jumpspot: - nop - nop - nop - nop - mov rax, 1 - ret -JumpShortCondFunction ENDP - -JumpNearCondFunction PROC - test cl, 1 - jnz jumpspot - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H -jumpspot: - nop - nop - mov rax, 1 - ret -JumpNearCondFunction ENDP - -JumpAbsoluteFunction PROC - test cl, 1 - jmp jumpspot - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H -jumpspot: - nop - nop - mov rax, 1 - ret -JumpAbsoluteFunction ENDP - -CallNearRelativeFunction PROC - test cl, 1 - call TooShortFunction - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - mov rdx, 0ffff1111H - nop - nop - nop - ret -CallNearRelativeFunction ENDP - -END +; Unit tests for PreamblePatcher + +.MODEL small + +.CODE + +TooShortFunction PROC + ret +TooShortFunction ENDP + +JumpShortCondFunction PROC + test cl, 1 + jnz jumpspot + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 + int 3 +jumpspot: + nop + nop + nop + nop + mov rax, 1 + ret +JumpShortCondFunction ENDP + +JumpNearCondFunction PROC + test cl, 1 + jnz jumpspot + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H +jumpspot: + nop + nop + mov rax, 1 + ret +JumpNearCondFunction ENDP + +JumpAbsoluteFunction PROC + test cl, 1 + jmp jumpspot + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H +jumpspot: + nop + nop + mov rax, 1 + ret +JumpAbsoluteFunction ENDP + +CallNearRelativeFunction PROC + test cl, 1 + call TooShortFunction + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + mov rdx, 0ffff1111H + nop + nop + nop + ret +CallNearRelativeFunction ENDP + +END