From 15704d4f1d03a696490c77f80af45c3bd18c57cc Mon Sep 17 00:00:00 2001 From: Fan Yong Date: Tue, 12 Dec 2023 09:05:44 +0800 Subject: [PATCH] DAOS-14105 object: latency metrics for collective punch For locating the performance bottle neck. Signed-off-by: Fan Yong --- src/include/daos_srv/dtx_srv.h | 3 +- src/object/obj_rpc.h | 2 + src/object/obj_tx.c | 6 + src/object/srv_internal.h | 69 +++++------ src/object/srv_mod.c | 27 +++-- src/object/srv_obj.c | 146 ++++++++++++++++++++++-- src/tests/ftest/util/telemetry_utils.py | 63 ++++++++-- 7 files changed, 249 insertions(+), 67 deletions(-) diff --git a/src/include/daos_srv/dtx_srv.h b/src/include/daos_srv/dtx_srv.h index d955426eea9..86ae0a59c8b 100644 --- a/src/include/daos_srv/dtx_srv.h +++ b/src/include/daos_srv/dtx_srv.h @@ -152,7 +152,8 @@ struct dtx_coll_entry { uint8_t *dce_hints; uint8_t *dce_bitmap; uint32_t dce_hint_sz; - uint32_t dce_bitmap_sz; + uint16_t dce_bitmap_sz; + uint16_t dce_tgt_nr; }; struct dtx_leader_handle; diff --git a/src/object/obj_rpc.h b/src/object/obj_rpc.h index ccb4db351ff..ff5547553f7 100644 --- a/src/object/obj_rpc.h +++ b/src/object/obj_rpc.h @@ -185,6 +185,8 @@ enum obj_rpc_flags { ORF_REBUILDING_IO = (1 << 23), /* 'sgls' is NULL, for update sub-request of CPD RPC. */ ORF_EMPTY_SGL = (1 << 24), + /* Internal transaction for punch object. */ + ORF_INTERNAL_PUNCH = (1 << 25), }; /* common for update/fetch */ diff --git a/src/object/obj_tx.c b/src/object/obj_tx.c index f2ddfd34cb1..f27e5983897 100644 --- a/src/object/obj_tx.c +++ b/src/object/obj_tx.c @@ -84,6 +84,7 @@ struct dc_tx { tx_retry:1, /** Retry the commit RPC. */ tx_set_resend:1, /** Set 'resend' flag. */ tx_for_convert:1, + tx_internal_punch:1, tx_has_cond:1, tx_renew:1, tx_closed:1, @@ -1918,6 +1919,9 @@ dc_tx_commit_prepare(struct dc_tx *tx, tse_task_t *task) if (rc != 0) goto out; } + + if (tx->tx_for_convert) + tx->tx_internal_punch = 1; } else { grp_idx = obj_dkey2grpidx(obj, dcsr->dcsr_dkey_hash, tx->tx_pm_ver); @@ -2311,6 +2315,8 @@ dc_tx_commit_trigger(tse_task_t *task, struct dc_tx *tx, daos_tx_commit_t *args) tx->tx_renew = 0; if (tx->tx_reintegrating) oci->oci_flags |= ORF_REINTEGRATING_IO; + if (tx->tx_internal_punch) + oci->oci_flags |= ORF_INTERNAL_PUNCH; oci->oci_sub_heads.ca_arrays = &tx->tx_head; oci->oci_sub_heads.ca_count = 1; diff --git a/src/object/srv_internal.h b/src/object/srv_internal.h index f586875b642..21852aee381 100644 --- a/src/object/srv_internal.h +++ b/src/object/srv_internal.h @@ -160,6 +160,16 @@ struct obj_tls { struct d_tm_node_t *ot_update_bio_lat[NR_LATENCY_BUCKETS]; struct d_tm_node_t *ot_fetch_bio_lat[NR_LATENCY_BUCKETS]; + + struct d_tm_node_t *ot_coll_punch_map_lat[NR_LATENCY_BUCKETS]; + struct d_tm_node_t *ot_coll_punch_layout_lat[NR_LATENCY_BUCKETS]; + struct d_tm_node_t *ot_coll_punch_parse_lat[NR_LATENCY_BUCKETS]; + struct d_tm_node_t *ot_coll_punch_lexec_lat[NR_LATENCY_BUCKETS]; + struct d_tm_node_t *ot_coll_punch_texec_lat[NR_LATENCY_BUCKETS]; + + struct d_tm_node_t *ot_cpd_punch_lexec_lat[NR_LATENCY_BUCKETS]; + struct d_tm_node_t *ot_cpd_punch_texec_lat[NR_LATENCY_BUCKETS]; + struct d_tm_node_t *ot_cpd_others_lat[NR_LATENCY_BUCKETS]; }; static inline struct obj_tls * @@ -186,55 +196,28 @@ lat_bucket(uint64_t size) return 56 - nr; } +static inline int +lat_log_bucket(uint64_t size) +{ + int i; + + for (i = 0; size > 0; i++) + size >>= 1; + + return i >= NR_LATENCY_BUCKETS ? NR_LATENCY_BUCKETS - 1 : i; +} + enum latency_type { BULK_LATENCY, BIO_LATENCY, VOS_LATENCY, + FIND_MAP, + GEN_LAYOUT, + COLL_PARSE, + LOCAL_EXEC, + TOTAL_EXEC, }; -static inline void -obj_update_latency(uint32_t opc, uint32_t type, uint64_t latency, uint64_t io_size) -{ - struct obj_tls *tls = obj_tls_get(); - struct d_tm_node_t *lat; - - latency >>= 10; /* convert to micro seconds */ - - if (opc == DAOS_OBJ_RPC_FETCH) { - switch (type) { - case BULK_LATENCY: - lat = tls->ot_fetch_bulk_lat[lat_bucket(io_size)]; - break; - case BIO_LATENCY: - lat = tls->ot_fetch_bio_lat[lat_bucket(io_size)]; - break; - case VOS_LATENCY: - lat = tls->ot_fetch_vos_lat[lat_bucket(io_size)]; - break; - default: - D_ASSERT(0); - } - } else if (opc == DAOS_OBJ_RPC_UPDATE || opc == DAOS_OBJ_RPC_TGT_UPDATE) { - switch (type) { - case BULK_LATENCY: - lat = tls->ot_update_bulk_lat[lat_bucket(io_size)]; - break; - case BIO_LATENCY: - lat = tls->ot_update_bio_lat[lat_bucket(io_size)]; - break; - case VOS_LATENCY: - lat = tls->ot_update_vos_lat[lat_bucket(io_size)]; - break; - default: - D_ASSERT(0); - } - } else { - /* Ignore other ops for the moment */ - return; - } - d_tm_set_gauge(lat, latency); -} - struct ds_obj_exec_arg { crt_rpc_t *rpc; struct obj_io_context *ioc; diff --git a/src/object/srv_mod.c b/src/object/srv_mod.c index 94099dc3f02..82940f24d70 100644 --- a/src/object/srv_mod.c +++ b/src/object/srv_mod.c @@ -125,10 +125,6 @@ obj_tls_init(int tags, int xs_id, int tgt_id) D_INIT_LIST_HEAD(&tls->ot_pool_list); - if (tgt_id < 0) - /** skip sensor setup on system xstreams */ - return tls; - /** register different per-opcode sensors */ for (opc = 0; opc < OBJ_PROTO_CLI_COUNT; opc++) { /** Start with number of active requests, of type gauge */ @@ -140,9 +136,8 @@ obj_tls_init(int tags, int xs_id, int tgt_id) D_WARN("Failed to create active counter: "DF_RC"\n", DP_RC(rc)); - if (opc == DAOS_OBJ_RPC_UPDATE || - opc == DAOS_OBJ_RPC_TGT_UPDATE || - opc == DAOS_OBJ_RPC_FETCH) + if (opc == DAOS_OBJ_RPC_UPDATE || opc == DAOS_OBJ_RPC_TGT_UPDATE || + opc == DAOS_OBJ_RPC_FETCH || opc == DAOS_OBJ_RPC_COLL_PUNCH) /** See below, latency reported per size for those */ continue; @@ -184,6 +179,24 @@ obj_tls_init(int tags, int xs_id, int tgt_id) obj_latency_tm_init(DAOS_OBJ_RPC_FETCH, tgt_id, tls->ot_fetch_bio_lat, "bio_fetch", "BIO fetch processing time"); + obj_latency_tm_init(DAOS_OBJ_RPC_COLL_PUNCH, tgt_id, tls->ot_coll_punch_map_lat, + "coll_punch_map", "Find pool map processing time"); + obj_latency_tm_init(DAOS_OBJ_RPC_COLL_PUNCH, tgt_id, tls->ot_coll_punch_layout_lat, + "coll_punch_layout", "Generate object layout processing time"); + obj_latency_tm_init(DAOS_OBJ_RPC_COLL_PUNCH, tgt_id, tls->ot_coll_punch_parse_lat, + "coll_punch_parse", "Parse layout processing time"); + obj_latency_tm_init(DAOS_OBJ_RPC_COLL_PUNCH, tgt_id, tls->ot_coll_punch_lexec_lat, + "coll_punch_lexec", "Local collective exec processing time"); + obj_latency_tm_init(DAOS_OBJ_RPC_COLL_PUNCH, tgt_id, tls->ot_coll_punch_texec_lat, + "coll_punch_texec", "Total collective exec processing time"); + + obj_latency_tm_init(DAOS_OBJ_RPC_CPD, tgt_id, tls->ot_cpd_punch_lexec_lat, + "cpd_punch_lexec", "Local cpd punch processing time"); + obj_latency_tm_init(DAOS_OBJ_RPC_CPD, tgt_id, tls->ot_cpd_punch_texec_lat, + "cpd_punch_texec", "Total cpd punch processing time"); + obj_latency_tm_init(DAOS_OBJ_RPC_CPD, tgt_id, tls->ot_cpd_others_lat, + "cpd_others", "Other CPD operation processing time"); + return tls; } diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index 0204a8d627c..3d5146ad699 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -113,6 +113,91 @@ obj_gen_dtx_mbs(uint32_t flags, uint32_t *tgt_cnt, struct daos_shard_tgt **p_tgt return 0; } +static void +obj_update_latency(uint32_t opc, uint32_t type, uint64_t latency, uint64_t size) +{ + struct obj_tls *tls = obj_tls_get(); + struct d_tm_node_t *lat; + + latency >>= 10; /* convert to micro seconds */ + + switch (opc) { + case DAOS_OBJ_RPC_FETCH: + switch (type) { + case BULK_LATENCY: + lat = tls->ot_fetch_bulk_lat[lat_bucket(size)]; + break; + case BIO_LATENCY: + lat = tls->ot_fetch_bio_lat[lat_bucket(size)]; + break; + case VOS_LATENCY: + lat = tls->ot_fetch_vos_lat[lat_bucket(size)]; + break; + default: + D_ASSERT(0); + } + break; + case DAOS_OBJ_RPC_UPDATE: + case DAOS_OBJ_RPC_TGT_UPDATE: + switch (type) { + case BULK_LATENCY: + lat = tls->ot_update_bulk_lat[lat_bucket(size)]; + break; + case BIO_LATENCY: + lat = tls->ot_update_bio_lat[lat_bucket(size)]; + break; + case VOS_LATENCY: + lat = tls->ot_update_vos_lat[lat_bucket(size)]; + break; + default: + D_ASSERT(0); + } + break; + case DAOS_OBJ_RPC_CPD: + switch (type) { + case BULK_LATENCY: + return; + case LOCAL_EXEC: + lat = tls->ot_cpd_punch_lexec_lat[d_rand() % NR_LATENCY_BUCKETS]; + break; + case TOTAL_EXEC: + lat = tls->ot_cpd_punch_texec_lat[lat_log_bucket(size)]; + break; + default: + D_ASSERT(0); + } + break; + case DAOS_OBJ_RPC_COLL_PUNCH: + switch (type) { + case FIND_MAP: + lat = tls->ot_coll_punch_map_lat[d_rand() % NR_LATENCY_BUCKETS]; + break; + case GEN_LAYOUT: + lat = tls->ot_coll_punch_layout_lat[lat_log_bucket(size)]; + break; + case COLL_PARSE: + lat = tls->ot_coll_punch_parse_lat[lat_log_bucket(size)]; + break; + case LOCAL_EXEC: + if (size >= NR_LATENCY_BUCKETS) + size = NR_LATENCY_BUCKETS - 1; + lat = tls->ot_coll_punch_lexec_lat[size]; + break; + case TOTAL_EXEC: + lat = tls->ot_coll_punch_texec_lat[lat_log_bucket(size)]; + break; + default: + D_ASSERT(0); + } + break; + default: + /* Ignore other ops for the moment */ + return; + } + + d_tm_set_gauge(lat, latency); +} + /** * After bulk finish, let's send reply, then release the resource. */ @@ -4565,10 +4650,14 @@ ds_cpd_handle_one(crt_rpc_t *rpc, struct daos_cpd_sub_head *dcsh, struct daos_cp if (rc != 0) goto out; } else { - daos_key_t *dkey; + struct obj_cpd_in *oci = crt_req_get(rpc); + daos_key_t *dkey; + uint64_t time = 0; if (dcsr->dcsr_opc == DCSO_PUNCH_OBJ) { dkey = NULL; + if (oci->oci_flags & ORF_INTERNAL_PUNCH) + time = daos_get_ntime(); } else if (dcsr->dcsr_opc == DCSO_PUNCH_DKEY || dcsr->dcsr_opc == DCSO_PUNCH_AKEY) { dkey = &dcsr->dcsr_dkey; @@ -4593,6 +4682,10 @@ ds_cpd_handle_one(crt_rpc_t *rpc, struct daos_cpd_sub_head *dcsh, struct daos_cp dcsr->dcsr_punch.dcp_akeys : NULL, dth); if (rc != 0) goto out; + + if (time != 0) + obj_update_latency(DAOS_OBJ_RPC_CPD, LOCAL_EXEC, + daos_get_ntime() - time, 0); } } @@ -4863,6 +4956,7 @@ ds_obj_dtx_leader(struct daos_cpd_args *dca) struct daos_cpd_disp_ent *dcde; struct daos_cpd_sub_req *dcsrs = NULL; struct daos_shard_tgt *tgts; + uint64_t time = 0; uint32_t flags = 0; uint32_t dtx_flags = DTX_DIST; int tgt_cnt = 0; @@ -4983,9 +5077,16 @@ ds_obj_dtx_leader(struct daos_cpd_args *dca) exec_arg.args = dca; exec_arg.flags = flags; + if (oci->oci_flags & ORF_INTERNAL_PUNCH) + time = daos_get_ntime(); + /* Execute the operation on all targets */ rc = dtx_leader_exec_ops(dlh, obj_obj_dtx_leader, NULL, 0, &exec_arg); + if (rc == 0 && time != 0) + obj_update_latency(DAOS_OBJ_RPC_CPD, TOTAL_EXEC, daos_get_ntime() - time, + dlh->dlh_normal_sub_cnt + 1); + /* Stop the distribute transaction */ rc = dtx_leader_end(dlh, dca->dca_ioc->ioc_coh, rc); @@ -5483,13 +5584,14 @@ obj_coll_tgt_punch(void *args) typedef int (*obj_coll_func_t)(void *args); static int -obj_coll_local(crt_rpc_t *rpc, struct daos_coll_shard *shards, uint8_t *bitmap, uint32_t bitmap_sz, - uint32_t *version, struct obj_io_context *ioc, struct dtx_handle *dth, void *args, - obj_coll_func_t func) +obj_coll_local(crt_rpc_t *rpc, struct daos_coll_shard *shards, uint32_t shard_nr, + uint8_t *bitmap, uint32_t bitmap_sz, uint32_t *version, struct obj_io_context *ioc, + struct dtx_handle *dth, void *args, obj_coll_func_t func) { struct obj_coll_tgt_args octa = { 0 }; struct dss_coll_ops coll_ops = { 0 }; struct dss_coll_args coll_args = { 0 }; + uint64_t time = daos_get_ntime(); uint32_t size = bitmap_sz << 3; int rc = 0; int i; @@ -5518,6 +5620,9 @@ obj_coll_local(crt_rpc_t *rpc, struct daos_coll_shard *shards, uint8_t *bitmap, coll_args.ca_tgt_bitmap_sz = bitmap_sz; rc = dss_thread_collective_reduce(&coll_ops, &coll_args, DSS_USE_CURRENT_ULT); + if (rc == 0) + obj_update_latency(opc_get(rpc->cr_opc), LOCAL_EXEC, + daos_get_ntime() - time, shard_nr); out: if (octa.octa_versions != NULL) { @@ -5543,9 +5648,10 @@ obj_coll_punch_disp(struct dtx_leader_handle *dlh, void *arg, int idx, dtx_sub_c return ds_obj_coll_punch_remote(dlh, arg, idx, comp_cb); /* Local punch on the leader rank, including the leader target. */ - rc = obj_coll_local(rpc, exec_arg->shards, dlh->dlh_coll_entry->dce_bitmap, - dlh->dlh_coll_entry->dce_bitmap_sz, NULL, exec_arg->ioc, - &dlh->dlh_handle, dlh->dlh_handle.dth_mbs, obj_coll_tgt_punch); + rc = obj_coll_local(rpc, exec_arg->shards, dlh->dlh_coll_entry->dce_tgt_nr, + dlh->dlh_coll_entry->dce_bitmap, dlh->dlh_coll_entry->dce_bitmap_sz, + NULL, exec_arg->ioc, &dlh->dlh_handle, dlh->dlh_handle.dth_mbs, + obj_coll_tgt_punch); DL_CDEBUG(rc == 0 || rc == -DER_INPROGRESS || rc == -DER_TX_RESTART, DB_IO, DLOG_ERR, rc, "Collective punch obj "DF_UOID" with "DF_DTI" on rank (leader) %u", @@ -5572,6 +5678,8 @@ obj_coll_punch_prep(struct obj_coll_punch_in *ocpi, struct dtx_coll_entry **p_dc struct dtx_daos_target *ddt; struct dtx_target_group *dtg; struct daos_obj_md md = { 0 }; + uint64_t time1 = 0; + uint64_t time2 = 0; uint32_t *tmp; uint32_t rank_nr; uint32_t tgt_nr; @@ -5594,6 +5702,8 @@ obj_coll_punch_prep(struct obj_coll_punch_in *ocpi, struct dtx_coll_entry **p_dc dce->dce_ver = ocpi->ocpi_map_ver; dce->dce_refs = 1; + time1 = daos_get_ntime(); + map = pl_map_find(ocpi->ocpi_po_uuid, ocpi->ocpi_oid.id_pub); if (map == NULL) { D_ERROR("Failed to find valid placement map for "DF_UUID"\n", @@ -5601,6 +5711,9 @@ obj_coll_punch_prep(struct obj_coll_punch_in *ocpi, struct dtx_coll_entry **p_dc D_GOTO(out, rc = -DER_INVAL); } + time2 = daos_get_ntime(); + obj_update_latency(DAOS_OBJ_RPC_COLL_PUNCH, FIND_MAP, time2 - time1, 0); + md.omd_id = ocpi->ocpi_oid.id_pub; md.omd_ver = ocpi->ocpi_map_ver; md.omd_fdom_lvl = ocpi->ocpi_fdom_lvl; @@ -5614,6 +5727,9 @@ obj_coll_punch_prep(struct obj_coll_punch_in *ocpi, struct dtx_coll_entry **p_dc goto out; } + time1 = daos_get_ntime(); + obj_update_latency(DAOS_OBJ_RPC_COLL_PUNCH, GEN_LAYOUT, time1 - time2, layout->ol_nr); + length = pool_map_node_nr(map->pl_poolmap); D_ALLOC_ARRAY(dcts, length + 1); @@ -5838,6 +5954,7 @@ obj_coll_punch_prep(struct obj_coll_punch_in *ocpi, struct dtx_coll_entry **p_dc */ D_ASSERT(dct->dct_bitmap != NULL); + dce->dce_tgt_nr = dct->dct_tgt_nr; dce->dce_bitmap = dct->dct_bitmap; dce->dce_bitmap_sz = dct->dct_bitmap_sz; dct->dct_bitmap = NULL; @@ -5852,6 +5969,9 @@ obj_coll_punch_prep(struct obj_coll_punch_in *ocpi, struct dtx_coll_entry **p_dc mbs = NULL; *p_dce = dtx_coll_entry_get(dce); + + obj_update_latency(DAOS_OBJ_RPC_COLL_PUNCH, COLL_PARSE, + daos_get_ntime() - time1, layout->ol_nr); } D_FREE(mbs); @@ -5887,6 +6007,7 @@ ds_obj_coll_punch_handler(crt_rpc_t *rpc) uint32_t max_ver = 0; struct dtx_epoch epoch; daos_epoch_t tmp; + uint64_t time; int rc; int rc1; bool need_abort = false; @@ -5938,8 +6059,9 @@ ds_obj_coll_punch_handler(crt_rpc_t *rpc) goto out; if (!(ocpi->ocpi_flags & ORF_LEADER)) { - rc = obj_coll_local(rpc, shards, dce->dce_bitmap, dce->dce_bitmap_sz, &version, - &ioc, NULL, mbs, obj_coll_tgt_punch); + rc = obj_coll_local(rpc, shards, dce->dce_tgt_nr, dce->dce_bitmap, + dce->dce_bitmap_sz, &version, &ioc, NULL, mbs, + obj_coll_tgt_punch); goto out; } @@ -6000,9 +6122,15 @@ ds_obj_coll_punch_handler(crt_rpc_t *rpc) exec_arg.shards = shards; exec_arg.flags = flags; + time = daos_get_ntime(); + /* Execute the operation on all shards */ rc = dtx_leader_exec_ops(dlh, obj_coll_punch_disp, NULL, 0, &exec_arg); + if (rc == 0) + obj_update_latency(DAOS_OBJ_RPC_COLL_PUNCH, TOTAL_EXEC, daos_get_ntime() - time, + dce->dce_ranks != NULL ? dce->dce_ranks->rl_nr + 1 : 1); + if (max_ver < dlh->dlh_rmt_ver) max_ver = dlh->dlh_rmt_ver; diff --git a/src/tests/ftest/util/telemetry_utils.py b/src/tests/ftest/util/telemetry_utils.py index abccc976a91..a0439b12020 100644 --- a/src/tests/ftest/util/telemetry_utils.py +++ b/src/tests/ftest/util/telemetry_utils.py @@ -200,6 +200,54 @@ class TelemetryUtils(): "engine_io_latency_bio_fetch_mean", "engine_io_latency_bio_fetch_min", "engine_io_latency_bio_fetch_stddev"] + ENGINE_IO_LATENCY_COLL_PUNCH_LAYOUT_METRICS = [ + "engine_io_latency_coll_punch_layout", + "engine_io_latency_coll_punch_layout_max", + "engine_io_latency_coll_punch_layout_mean", + "engine_io_latency_coll_punch_layout_min", + "engine_io_latency_coll_punch_layout_stddev"] + ENGINE_IO_LATENCY_COLL_PUNCH_LEXEC_METRICS = [ + "engine_io_latency_coll_punch_lexec", + "engine_io_latency_coll_punch_lexec_max", + "engine_io_latency_coll_punch_lexec_mean", + "engine_io_latency_coll_punch_lexec_min", + "engine_io_latency_coll_punch_lexec_stddev"] + ENGINE_IO_LATENCY_COLL_PUNCH_MAP_METRICS = [ + "engine_io_latency_coll_punch_map", + "engine_io_latency_coll_punch_map_max", + "engine_io_latency_coll_punch_map_mean", + "engine_io_latency_coll_punch_map_min", + "engine_io_latency_coll_punch_map_stddev"] + ENGINE_IO_LATENCY_COLL_PUNCH_PARSE_METRICS = [ + "engine_io_latency_coll_punch_parse", + "engine_io_latency_coll_punch_parse_max", + "engine_io_latency_coll_punch_parse_mean", + "engine_io_latency_coll_punch_parse_min", + "engine_io_latency_coll_punch_parse_stddev"] + ENGINE_IO_LATENCY_COLL_PUNCH_TEXEC_METRICS = [ + "engine_io_latency_coll_punch_texec", + "engine_io_latency_coll_punch_texec_max", + "engine_io_latency_coll_punch_texec_mean", + "engine_io_latency_coll_punch_texec_min", + "engine_io_latency_coll_punch_texec_stddev"] + ENGINE_IO_LATENCY_CPD_OTHERS_METRICS = [ + "engine_io_latency_cpd_others", + "engine_io_latency_cpd_others_max", + "engine_io_latency_cpd_others_mean", + "engine_io_latency_cpd_others_min", + "engine_io_latency_cpd_others_stddev"] + ENGINE_IO_LATENCY_CPD_PUNCH_LEXEC_METRICS = [ + "engine_io_latency_cpd_punch_lexec", + "engine_io_latency_cpd_punch_lexec_max", + "engine_io_latency_cpd_punch_lexec_mean", + "engine_io_latency_cpd_punch_lexec_min", + "engine_io_latency_cpd_punch_lexec_stddev"] + ENGINE_IO_LATENCY_CPD_PUNCH_TEXEC_METRICS = [ + "engine_io_latency_cpd_punch_texec", + "engine_io_latency_cpd_punch_texec_max", + "engine_io_latency_cpd_punch_texec_mean", + "engine_io_latency_cpd_punch_texec_min", + "engine_io_latency_cpd_punch_texec_stddev"] ENGINE_IO_LATENCY_UPDATE_METRICS = [ "engine_io_latency_update", "engine_io_latency_update_max", @@ -362,12 +410,6 @@ class TelemetryUtils(): "engine_io_ops_obj_coll_punch_active_mean", "engine_io_ops_obj_coll_punch_active_min", "engine_io_ops_obj_coll_punch_active_stddev"] - ENGINE_IO_OPS_OBJ_COLL_PUNCH_LATENCY_METRICS = [ - "engine_io_ops_obj_coll_punch_latency", - "engine_io_ops_obj_coll_punch_latency_max", - "engine_io_ops_obj_coll_punch_latency_mean", - "engine_io_ops_obj_coll_punch_latency_min", - "engine_io_ops_obj_coll_punch_latency_stddev"] ENGINE_IO_OPS_OBJ_ENUM_ACTIVE_METRICS = [ "engine_io_ops_obj_enum_active", "engine_io_ops_obj_enum_active_max", @@ -470,6 +512,14 @@ class TelemetryUtils(): ENGINE_IO_LATENCY_BULK_FETCH_METRICS +\ ENGINE_IO_LATENCY_VOS_FETCH_METRICS +\ ENGINE_IO_LATENCY_BIO_FETCH_METRICS +\ + ENGINE_IO_LATENCY_COLL_PUNCH_LAYOUT_METRICS +\ + ENGINE_IO_LATENCY_COLL_PUNCH_LEXEC_METRICS +\ + ENGINE_IO_LATENCY_COLL_PUNCH_MAP_METRICS +\ + ENGINE_IO_LATENCY_COLL_PUNCH_PARSE_METRICS +\ + ENGINE_IO_LATENCY_COLL_PUNCH_TEXEC_METRICS +\ + ENGINE_IO_LATENCY_CPD_OTHERS_METRICS +\ + ENGINE_IO_LATENCY_CPD_PUNCH_LEXEC_METRICS +\ + ENGINE_IO_LATENCY_CPD_PUNCH_TEXEC_METRICS +\ ENGINE_IO_LATENCY_UPDATE_METRICS +\ ENGINE_IO_LATENCY_TGT_UPDATE_METRICS +\ ENGINE_IO_LATENCY_BULK_UPDATE_METRICS +\ @@ -497,7 +547,6 @@ class TelemetryUtils(): ENGINE_IO_OPS_MIGRATE_ACTIVE_METRICS +\ ENGINE_IO_OPS_MIGRATE_LATENCY_METRICS +\ ENGINE_IO_OPS_OBJ_COLL_PUNCH_ACTIVE_METRICS +\ - ENGINE_IO_OPS_OBJ_COLL_PUNCH_LATENCY_METRICS +\ ENGINE_IO_OPS_OBJ_ENUM_ACTIVE_METRICS +\ ENGINE_IO_OPS_OBJ_ENUM_LATENCY_METRICS +\ ENGINE_IO_OPS_OBJ_PUNCH_ACTIVE_METRICS +\