Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve fabtest latency tests #10394

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+226 −113
Open

Improve fabtest latency tests #10394

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
18a30fb
fabtests: Move pingpong logic into pre-posted func
iziemba Sep 18, 2024
f86ac48
fabtests: Define common run pingpong function
iziemba Sep 18, 2024
b46d826
fabtests: Split out ft_sync logic
iziemba Sep 19, 2024
353d62a
fabtests: Support no prepost RX pingpong test
iziemba Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 155 additions & 21 deletions fabtests/benchmarks/benchmark_shared.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ void ft_parse_benchmark_opts(int op, char *optarg)
case 'W':
opts.window_size = atoi(optarg);
break;
case 'r':
opts.options |= FT_OPT_NO_PRE_POSTED_RX;
break;
default:
break;
}
Expand All @@ -84,43 +87,83 @@ void ft_benchmark_usage(void)
"* The following condition is required to have at least "
"one window\nsize # of messsages to be sent: "
"# of iterations > window size");
FT_PRINT_OPTS_USAGE("-r", "Do not pre post RX buffers");
FT_PRINT_OPTS_USAGE("", "Only the following tests support this option for now:");
FT_PRINT_OPTS_USAGE("", "\tfi_rdm_tagged_pingpong");
FT_PRINT_OPTS_USAGE("", "\tfi_rdm_pingpong");
}

int pingpong(void)
/* Pingpong latency test with pre-posted receive buffers. */
iziemba marked this conversation as resolved.
Show resolved Hide resolved
static int pingpong_pre_posted_rx(size_t inject_size)
iziemba marked this conversation as resolved.
Show resolved Hide resolved
{
int ret, i;
size_t inject_size = fi->tx_attr->inject_size;

ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_MSG_SIZE,
&inject_size, &(size_t){sizeof inject_size});
if (ret && ret != -FI_ENOPROTOOPT) {
FT_PRINTERR("fi_getopt(FI_OPT_INJECT_MSG_SIZE)", ret);
return ret;
}
if (opts.dst_addr) {
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
if (i == opts.warmup_iterations)
ft_start();

if (inject_size_set)
inject_size = opts.inject_size;
if (opts.transfer_size <= inject_size)
ret = ft_inject(ep, remote_fi_addr,
opts.transfer_size);
else
ret = ft_tx(ep, remote_fi_addr,
opts.transfer_size, &tx_ctx);
if (ret)
return ret;

if (opts.options & FT_OPT_ENABLE_HMEM)
inject_size = 0;
ret = ft_rx(ep, opts.transfer_size);
if (ret)
return ret;
}
} else {
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
if (i == opts.warmup_iterations)
ft_start();

ret = ft_sync();
if (ret)
return ret;
ret = ft_rx(ep, opts.transfer_size);
if (ret)
return ret;

if (opts.transfer_size <= inject_size)
ret = ft_inject(ep, remote_fi_addr,
opts.transfer_size);
else
ret = ft_tx(ep, remote_fi_addr,
opts.transfer_size, &tx_ctx);
if (ret)
return ret;
}
}
ft_stop();

return FI_SUCCESS;
}

/* Pingpong latency test without pre-posted receive buffers. */
static int pingpong_no_pre_posted_rx(size_t inject_size)
{
int ret, i;

if (opts.dst_addr) {
for (i = 0; i < opts.iterations + opts.warmup_iterations; i++) {
if (i == opts.warmup_iterations)
ft_start();

if (opts.transfer_size <= inject_size)
ret = ft_inject(ep, remote_fi_addr, opts.transfer_size);
ret = ft_inject(ep, remote_fi_addr,
opts.transfer_size);
else
ret = ft_tx(ep, remote_fi_addr, opts.transfer_size, &tx_ctx);
ret = ft_tx(ep, remote_fi_addr,
opts.transfer_size, &tx_ctx);
if (ret)
return ret;

ret = ft_rx(ep, opts.transfer_size);
ret = ft_post_rx(ep, opts.transfer_size, &rx_ctx);
if (ret)
return ret;

ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;
}
Expand All @@ -129,20 +172,83 @@ int pingpong(void)
if (i == opts.warmup_iterations)
ft_start();

ret = ft_rx(ep, opts.transfer_size);
ret = ft_post_rx(ep, opts.transfer_size, &rx_ctx);
if (ret)
return ret;

ret = ft_get_rx_comp(rx_seq);
if (ret)
return ret;

if (ft_check_opts(FT_OPT_VERIFY_DATA | FT_OPT_ACTIVE)) {
ret = ft_check_buf((char *) rx_buf + ft_rx_prefix_size(),
opts.transfer_size);
if (ret)
return ret;
}

if (opts.transfer_size <= inject_size)
ret = ft_inject(ep, remote_fi_addr, opts.transfer_size);
ret = ft_inject(ep, remote_fi_addr,
opts.transfer_size);
else
ret = ft_tx(ep, remote_fi_addr, opts.transfer_size, &tx_ctx);
ret = ft_tx(ep, remote_fi_addr,
opts.transfer_size, &tx_ctx);
if (ret)
return ret;
}
}
ft_stop();

return FI_SUCCESS;
}

int pingpong(void)
{
int ret;
size_t inject_size = fi->tx_attr->inject_size;

ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_INJECT_MSG_SIZE,
&inject_size, &(size_t){sizeof inject_size});
if (ret && ret != -FI_ENOPROTOOPT) {
FT_PRINTERR("fi_getopt(FI_OPT_INJECT_MSG_SIZE)", ret);
return ret;
}

if (inject_size_set)
inject_size = opts.inject_size;

if (opts.options & FT_OPT_ENABLE_HMEM)
inject_size = 0;

if (ft_check_opts(FT_OPT_NO_PRE_POSTED_RX)) {
if (ft_check_opts(FT_OPT_OOB_SYNC)) {
ret = ft_sync_oob();
if (ret)
return ret;
} else {
/* Repost RX buffers to support inband sync. */
ret = ft_post_rx(ep, rx_size, &rx_ctx);
if (ret)
return ret;

ret = ft_sync_inband(false);
if (ret)
return ret;
}

ret = pingpong_no_pre_posted_rx(inject_size);
if (ret)
return ret;
} else {
ret = ft_sync();
if (ret)
return ret;

ret = pingpong_pre_posted_rx(inject_size);
if (ret)
return ret;
}

if (opts.machr)
show_perf_mr(opts.transfer_size, opts.iterations, &start, &end, 2,
opts.argc, opts.argv);
Expand All @@ -152,6 +258,34 @@ int pingpong(void)
return 0;
}

int run_pingpong(void)
{
int i, ret = 0;

ret = ft_init_fabric();
if (ret)
return ret;

if (!(opts.options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (!ft_use_size(i, opts.sizes_enabled))
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
return ret;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
return ret;
}

return ft_finalize();
}

int pingpong_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote)
{
int ret, i;
Expand Down
3 changes: 2 additions & 1 deletion fabtests/benchmarks/benchmark_shared.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,13 @@ extern "C" {

#include <rdma/fi_rma.h>

#define BENCHMARK_OPTS "vkj:W:"
#define BENCHMARK_OPTS "rvkj:W:"
#define FT_BENCHMARK_MAX_MSG_SIZE (test_size[TEST_CNT - 1].size)

void ft_parse_benchmark_opts(int op, char *optarg);
void ft_benchmark_usage(void);
int pingpong(void);
int run_pingpong(void);
int bandwidth(void);
int pingpong_rma(enum ft_rma_opcodes rma_op, struct fi_rma_iov *remote);
int bandwidth_rma(enum ft_rma_opcodes op, struct fi_rma_iov *remote);
Expand Down
30 changes: 1 addition & 29 deletions fabtests/benchmarks/rdm_pingpong.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,34 +36,6 @@
#include "shared.h"
#include "benchmark_shared.h"

static int run(void)
{
int i, ret = 0;

ret = ft_init_fabric();
if (ret)
return ret;

if (!(opts.options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (!ft_use_size(i, opts.sizes_enabled))
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
return ret;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
return ret;
}

return ft_finalize();
}

int main(int argc, char **argv)
{
int op, ret;
Expand Down Expand Up @@ -106,7 +78,7 @@ int main(int argc, char **argv)
hints->tx_attr->tclass = FI_TC_LOW_LATENCY;
hints->addr_format = opts.address_format;

ret = run();
ret = run_pingpong();

ft_free_res();
return ft_exit_code(ret);
Expand Down
32 changes: 1 addition & 31 deletions fabtests/benchmarks/rdm_tagged_pingpong.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,36 +36,6 @@
#include <shared.h>
#include "benchmark_shared.h"

static int run(void)
{
int i, ret = 0;

ret = ft_init_fabric();
if (ret)
return ret;

if (!(opts.options & FT_OPT_SIZE)) {
for (i = 0; i < TEST_CNT; i++) {
if (!ft_use_size(i, opts.sizes_enabled))
continue;
opts.transfer_size = test_size[i].size;
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
goto out;
}
} else {
init_test(&opts, test_name, sizeof(test_name));
ret = pingpong();
if (ret)
goto out;
}

ft_finalize();
out:
return ret;
}

int main(int argc, char **argv)
{
int op, ret;
Expand Down Expand Up @@ -108,7 +78,7 @@ int main(int argc, char **argv)
hints->tx_attr->tclass = FI_TC_LOW_LATENCY;
hints->addr_format = opts.address_format;

ret = run();
ret = run_pingpong();

ft_free_res();
return ft_exit_code(ret);
Expand Down
Loading
Loading