tikv-client raising "PD cluster failed to respond" error while connect with tls

[Xuanwo]

Reproduce

For tikv setup: https://github.com/apache/incubator-opendal/blob/a7c046f55cdbee16bb6392ff0173b85bfb16b1c8/.github/services/tikv/tikv_tls/action.yml

git clone https://github.com/apache/incubator-opendal
cd incubator-opendal

export OPENDAL_TIKV_ENDPOINTS=https://127.0.0.1:2379
export OPENDAL_TIKV_INSECURE=false
export OPENDAL_TIKV_CA_PATH=/tmp/tikv/ssl/ca.pem
export OPENDAL_TIKV_CERT_PATH=/tmp/tikv/ssl/client.pem
export OPENDAL_TIKV_KEY_PATH=/tmp/tikv/ssl/client-key.pem
export OPENDAL_TEST=tikv

cargo test behavior --features=tests,services-tikv

Log

---- behavior::test_fuzz_offset_reader ----
test panicked: write must succeed: ConfigInvalid (permanent) at Writer::close => invalid configuration

Context:
   service: tikv
   service: tikv
   path: a36b094c-6c23-449b-adcf-07f555c7fc54

Source:
   [/home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tikv-client-0.3.0/src/pd/cluster.rs:189]: PD cluster failed to respond

Backtrace:
   0: opendal::types::error::Error::new
             at ./src/types/error.rs:338:24
   1: opendal::services::tikv::backend::parse_tikv_config_error
             at ./src/services/tikv/backend.rs:264:5
   2: core::ops::function::FnOnce::call_once
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/ops/function.rs:250:5
   3: core::result::Result<T,E>::map_err
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/result.rs:829:27
   4: opendal::services::tikv::backend::Adapter::get_connection::{{closure}}
             at ./src/services/tikv/backend.rs:204:13
   5: <opendal::services::tikv::backend::Adapter as opendal::raw::adapters::kv::api::Adapter>::set::{{closure}}
             at ./src/services/tikv/backend.rs:244:14
   6: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/future/future.rs:125:9
   7: <opendal::raw::adapters::kv::backend::KvWriter<S> as opendal::raw::oio::write::api::Write>::poll_close::{{closure}}
             at ./src/raw/adapters/kv/backend.rs:399:64
   8: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/future/future.rs:125:9
   9: futures_util::future::future::FutureExt::poll_unpin
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.29/src/future/future/mod.rs:558:9
  10: <opendal::raw::adapters::kv::backend::KvWriter<S> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/raw/adapters/kv/backend.rs:383:38
  11: <opendal::layers::error_context::ErrorContextWrapper<T> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/layers/error_context.rs:419:9
  12: <opendal::layers::complete::CompleteWriter<W> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/layers/complete.rs:818:16
  13: <opendal::raw::oio::write::compose_write::TwoWaysWriter<ONE,TWO> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/raw/oio/write/compose_write.rs:70:31
  14: <alloc::boxed::Box<T> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/raw/oio/write/api.rs:125:9
  15: <opendal::layers::logging::LoggingWriter<W> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/layers/logging.rs:1336:22
  16: <alloc::boxed::Box<T> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/raw/oio/write/api.rs:125:9
  17: <opendal::layers::timeout::TimeoutWrapper<R> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/layers/timeout.rs:408:15
  18: <alloc::boxed::Box<T> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/raw/oio/write/api.rs:125:9
  19: <opendal::layers::retry::RetryWrapper<R,I> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/layers/retry.rs:973:22
  20: <alloc::boxed::Box<T> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/raw/oio/write/api.rs:125:9
  21: <alloc::boxed::Box<T> as opendal::raw::oio::write::api::Write>::poll_close
             at ./src/raw/oio/write/api.rs:125:9
  22: <opendal::raw::oio::write::api::CloseFuture<W> as core::future::future::Future>::poll
             at ./src/raw/oio/write/api.rs:205:9
  23: opendal::types::operator::operator::Operator::write_with::{{closure}}::{{closure}}
             at ./src/types/operator/operator.rs:792:31
  24: <opendal::types::operator::operator_futures::OperatorFuture<T,F> as core::future::future::Future>::poll
             at ./src/types/operator/operator_futures.rs:106:52
  25: futures_util::future::future::FutureExt::poll_unpin
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.29/src/future/future/mod.rs:558:9
  26: <opendal::types::operator::operator_futures::FutureWrite as core::future::future::Future>::poll
             at ./src/types/operator/operator_futures.rs:429:9
  27: opendal::types::operator::operator::Operator::write::{{closure}}
             at ./src/types/operator/operator.rs:544:35
  28: behavior::write::test_fuzz_offset_reader::{{closure}}
             at ./tests/behavior/write.rs:805:10
  29: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.34.0/src/runtime/park.rs:282:63
  30: tokio::runtime::coop::with_budget
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.34.0/src/runtime/coop.rs:107:5
  31: tokio::runtime::coop::budget
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.34.0/src/runtime/coop.rs:73:5
  32: tokio::runtime::park::CachedParkThread::block_on
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.34.0/src/runtime/park.rs:282:31
  33: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.34.0/src/runtime/context/blocking.rs:66:9
  34: tokio::runtime::handle::Handle::block_on::{{closure}}
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.34.0/src/runtime/handle.rs:310:13
  35: tokio::runtime::context::runtime::enter_runtime
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.34.0/src/runtime/context/runtime.rs:65:16
  36: tokio::runtime::handle::Handle::block_on
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.34.0/src/runtime/handle.rs:309:9
  37: behavior::utils::build_async_trial::{{closure}}
             at ./tests/behavior/utils.rs:77:9
  38: libtest_mimic::Trial::test::{{closure}}
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libtest-mimic-0.6.1/src/lib.rs:112:54
  39: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/ops/function.rs:250:5
  40: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/alloc/src/boxed.rs:2007:9
  41: libtest_mimic::run_single::{{closure}}
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libtest-mimic-0.6.1/src/lib.rs:505:43
  42: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/panic/unwind_safe.rs:271:9
  43: std::panicking::try::do_call
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panicking.rs:504:40
  44: __rust_try
  45: std::panicking::try
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panicking.rs:468:19
  46: std::panic::catch_unwind
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panic.rs:142:14
  47: libtest_mimic::run_single
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libtest-mimic-0.6.1/src/lib.rs:505:5
  48: libtest_mimic::run::{{closure}}
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/libtest-mimic-0.6.1/src/lib.rs:476:35
  49: <F as threadpool::FnBox>::call_box
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/threadpool-1.8.1/src/lib.rs:95:9
  50: threadpool::spawn_in_pool::{{closure}}
             at /home/runner/.cargo/registry/src/index.crates.io-6f17d22bba15001f/threadpool-1.8.1/src/lib.rs:769:17
  51: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/sys_common/backtrace.rs:154:18
  52: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/thread/mod.rs:529:17
  53: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/panic/unwind_safe.rs:271:9
  54: std::panicking::try::do_call
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panicking.rs:504:40
  55: __rust_try
  56: std::panicking::try
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panicking.rs:468:19
  57: std::panic::catch_unwind
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panic.rs:142:14
  58: std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/thread/mod.rs:528:30
  59: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/ops/function.rs:250:5
  60: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/alloc/src/boxed.rs:2007:9
  61: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/alloc/src/boxed.rs:2007:9
  62: std::sys::unix::thread::Thread::new::thread_start
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/sys/unix/thread.rs:108:17
  63: <unknown>
  64: <unknown>