diff --git a/apisix/balancer.lua b/apisix/balancer.lua
index a9ee8ede22c3a..00544052c0451 100644
--- a/apisix/balancer.lua
+++ b/apisix/balancer.lua
@@ -303,6 +303,21 @@ do
             local requests = keepalive_pool.requests
 
             pool_opt.pool_size = size
+
+            local pool = server.host .. "#" .. server.port
+            local scheme = up_conf.scheme
+            -- other TLS schemes don't use http balancer keepalive
+            if (scheme == "https" or scheme == "grpcs") then
+                local sni = ctx.var.upstream_host
+                pool = pool .. "#" .. sni
+
+                if up_conf.tls and up_conf.tls.client_cert then
+                    pool = pool .. "#" .. up_conf.tls.client_cert
+                end
+            end
+
+            pool_opt.pool = pool
+
             local ok, err = balancer.set_current_peer(server.host, server.port,
                                                       pool_opt)
             if not ok then
diff --git a/t/node/upstream-keepalive-pool.t b/t/node/upstream-keepalive-pool.t
index 1818eccaf6540..4d70d99e7e149 100644
--- a/t/node/upstream-keepalive-pool.t
+++ b/t/node/upstream-keepalive-pool.t
@@ -275,3 +275,364 @@ lua balancer: keepalive saving connection \S+, cpool: \S+, connections: 1
 lua balancer: keepalive reusing connection \S+, requests: 2, cpool: \S+
 lua balancer: keepalive saving connection \S+, cpool: \S+, connections: 1
 $/
+
+
+
+=== TEST 8: upstreams with different client cert
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin")
+            local test = require("lib.test_admin").test
+            local json = require("toolkit.json")
+            local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+            local ssl_key = t.read_file("t/certs/mtls_client.key")
+            local ssl_cert2 = t.read_file("t/certs/apisix.crt")
+            local ssl_key2 = t.read_file("t/certs/apisix.key")
+
+            local code, body = test('/apisix/admin/upstreams/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "scheme": "https",
+                    "type": "roundrobin",
+                    "nodes": {
+                        "127.0.0.1:1983": 1
+                    },
+                    "keepalive_pool": {
+                        "size": 4
+                    }
+                }]]
+            )
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+
+            local data = {
+                scheme = "https",
+                type = "roundrobin",
+                nodes = {
+                    ["127.0.0.1:1983"] = 1,
+                },
+                tls = {
+                    client_cert = ssl_cert,
+                    client_key = ssl_key,
+                },
+                keepalive_pool = {
+                    size = 8
+                }
+            }
+            local code, body = test('/apisix/admin/upstreams/2',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+
+            local data = {
+                scheme = "https",
+                type = "roundrobin",
+                nodes = {
+                    ["127.0.0.1:1983"] = 1,
+                },
+                tls = {
+                    client_cert = ssl_cert2,
+                    client_key = ssl_key2,
+                },
+                keepalive_pool = {
+                    size = 16
+                }
+            }
+            local code, body = test('/apisix/admin/upstreams/3',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+
+            for i = 1, 3 do
+                local code, body = test('/apisix/admin/routes/' .. i,
+                    ngx.HTTP_PUT,
+                    [[{
+                        "uri":"/hello/]] .. i .. [[",
+                        "plugins": {
+                            "proxy-rewrite": {
+                                "uri": "/hello"
+                            }
+                        },
+                        "upstream_id": ]] .. i .. [[
+                    }]])
+                if code >= 300 then
+                    ngx.status = code
+                    ngx.print(body)
+                    return
+                end
+            end
+        }
+    }
+--- response_body
+
+
+
+=== TEST 9: hit
+--- upstream_server_config
+    ssl_client_certificate ../../certs/mtls_ca.crt;
+    ssl_verify_client on;
+--- config
+    location /t {
+        content_by_lua_block {
+            local http = require "resty.http"
+            local uri = "http://127.0.0.1:" .. ngx.var.server_port
+
+            for i = 1, 12 do
+                local idx = (i % 3) + 1
+                local httpc = http.new()
+                local res, err = httpc:request_uri(uri .. "/hello/" .. idx)
+                if not res then
+                    ngx.say(err)
+                    return
+                end
+
+                if idx == 2 then
+                    assert(res.status == 200)
+                else
+                    assert(res.status == 400)
+                end
+            end
+        }
+    }
+
+
+
+=== TEST 10: upstreams with different client cert (without pool)
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin")
+            local test = require("lib.test_admin").test
+            local json = require("toolkit.json")
+            local ssl_cert = t.read_file("t/certs/mtls_client.crt")
+            local ssl_key = t.read_file("t/certs/mtls_client.key")
+            local ssl_cert2 = t.read_file("t/certs/apisix.crt")
+            local ssl_key2 = t.read_file("t/certs/apisix.key")
+
+            local code, body = test('/apisix/admin/upstreams/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "scheme": "https",
+                    "type": "roundrobin",
+                    "nodes": {
+                        "127.0.0.1:1983": 1
+                    }
+                }]]
+            )
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+
+            local data = {
+                scheme = "https",
+                type = "roundrobin",
+                nodes = {
+                    ["127.0.0.1:1983"] = 1,
+                },
+                tls = {
+                    client_cert = ssl_cert,
+                    client_key = ssl_key,
+                }
+            }
+            local code, body = test('/apisix/admin/upstreams/2',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+
+            local data = {
+                scheme = "https",
+                type = "roundrobin",
+                nodes = {
+                    ["127.0.0.1:1983"] = 1,
+                },
+                tls = {
+                    client_cert = ssl_cert2,
+                    client_key = ssl_key2,
+                }
+            }
+            local code, body = test('/apisix/admin/upstreams/3',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+
+            for i = 1, 3 do
+                local code, body = test('/apisix/admin/routes/' .. i,
+                    ngx.HTTP_PUT,
+                    [[{
+                        "uri":"/hello/]] .. i .. [[",
+                        "plugins": {
+                            "proxy-rewrite": {
+                                "uri": "/hello"
+                            }
+                        },
+                        "upstream_id": ]] .. i .. [[
+                    }]])
+                if code >= 300 then
+                    ngx.status = code
+                    ngx.print(body)
+                    return
+                end
+            end
+        }
+    }
+--- response_body
+
+
+
+=== TEST 11: hit
+--- upstream_server_config
+    ssl_client_certificate ../../certs/mtls_ca.crt;
+    ssl_verify_client on;
+--- config
+    location /t {
+        content_by_lua_block {
+            local http = require "resty.http"
+            local uri = "http://127.0.0.1:" .. ngx.var.server_port
+
+            for i = 1, 12 do
+                local idx = i % 3
+                local httpc = http.new()
+                local res, err = httpc:request_uri(uri .. "/hello/" .. idx)
+                if not res then
+                    ngx.say(err)
+                    return
+                end
+
+                if idx == 2 then
+                    assert(res.status == 200)
+                else
+                    assert(res.status == 400)
+                end
+            end
+        }
+    }
+
+
+
+=== TEST 12: upstreams with different SNI
+--- FIRST
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin")
+            local test = require("lib.test_admin").test
+            local json = require("toolkit.json")
+
+            local code, body = test('/apisix/admin/upstreams/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "scheme": "https",
+                    "type": "roundrobin",
+                    "nodes": {
+                        "127.0.0.1:1983": 1
+                    },
+                    "pass_host": "rewrite",
+                    "upstream_host": "a.com",
+                    "keepalive_pool": {
+                        "size": 4
+                    }
+                }]]
+            )
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+
+            local data = {
+                scheme = "https",
+                type = "roundrobin",
+                nodes = {
+                    ["127.0.0.1:1983"] = 1,
+                },
+                pass_host = "rewrite",
+                upstream_host = "b.com",
+                keepalive_pool = {
+                    size = 8
+                }
+            }
+            local code, body = test('/apisix/admin/upstreams/2',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+            if code >= 300 then
+                ngx.status = code
+                ngx.print(body)
+                return
+            end
+
+            for i = 1, 2 do
+                local code, body = test('/apisix/admin/routes/' .. i,
+                    ngx.HTTP_PUT,
+                    [[{
+                        "uri":"/hello/]] .. i .. [[",
+                        "plugins": {
+                            "proxy-rewrite": {
+                                "uri": "/hello"
+                            }
+                        },
+                        "upstream_id": ]] .. i .. [[
+                    }]])
+                if code >= 300 then
+                    ngx.status = code
+                    ngx.print(body)
+                    return
+                end
+            end
+        }
+    }
+--- response_body
+
+
+
+=== TEST 13: hit
+--- config
+    location /t {
+        content_by_lua_block {
+            local http = require "resty.http"
+            local uri = "http://127.0.0.1:" .. ngx.var.server_port
+            for i = 1, 4 do
+                local idx = i % 2 + 1
+                local httpc = http.new()
+                local res, err = httpc:request_uri(uri .. "/hello/" .. idx)
+                local res, err = httpc:request_uri(uri)
+                if not res then
+                    ngx.say(err)
+                    return
+                end
+                ngx.print(res.body)
+            end
+        }
+    }
+--- grep_error_log eval
+qr/lua balancer: keepalive create pool, .*/
+--- grep_error_log_out eval
+qr/^lua balancer: keepalive create pool, crc32: \S+, size: 8
+lua balancer: keepalive create pool, crc32: \S+, size: 4
+$/