Merge pull request #42994 from Shawyeok/features/full-tls-etcd-cluster

Automatic merge from submit-queue Centos provider: generate SSL certificates for etcd cluster. **What this PR does / why we need it**: Support secure etcd cluster for centos provider, generate SSL certificates for etcd in default. Running it w/o SSL is exposing cluster data to everyone and is not recommended. [#39462](#39462 (comment)) /cc @jszczepkowski @zmerlynn **Release note**: ```release-note Support secure etcd cluster for centos provider. ```
kubernetes · Mar 28, 2017 · be4452c · be4452c
2 parents 3c5c1da + c692b55
commit be4452c
Show file tree

Hide file tree

Showing 10 changed files with 238 additions and 62 deletions.
diff --git a/cluster/centos/.gitignore b/cluster/centos/.gitignore
@@ -1,5 +1,6 @@
 binaries
 ca-cert
+etcd-cert
 
 master/bin/etcd
 master/bin/etcdctl

diff --git a/cluster/centos/config-default.sh b/cluster/centos/config-default.sh
@@ -71,7 +71,7 @@ function concat-etcd-servers() {
     if [ -n "$etcd_servers" ]; then
       prefix="${etcd_servers},"
     fi
-    etcd_servers="${prefix}http://${master_ip}:2379"
+    etcd_servers="${prefix}https://${master_ip}:2379"
   done
 
   echo "$etcd_servers"
@@ -89,15 +89,18 @@ function concat-etcd-initial-cluster() {
     if [ -n "$etcd_initial_cluster" ]; then
       etcd_initial_cluster+=","
     fi
-    etcd_initial_cluster+="infra${num_infra}=http://${master_ip}:2380"
+    etcd_initial_cluster+="infra${num_infra}=https://${master_ip}:2380"
     let ++num_infra
   done
 
   echo "$etcd_initial_cluster"
 }
 export ETCD_INITIAL_CLUSTER="$(concat-etcd-initial-cluster)"
 
-export CERT_DIR="${CERT_DIR:-$(cd "${root}/ca-cert" && pwd)}"
+CERT_DIR="${CERT_DIR:-${root}/ca-cert}"
+mkdir -p "${CERT_DIR}"
+# CERT_DIR path must be absolute.
+export CERT_DIR="$(cd "${CERT_DIR}"; pwd)"
 
 # define the IP range used for service cluster IPs.
 # according to rfc 1918 ref: https://tools.ietf.org/html/rfc1918 choose a private ip range here.
@@ -117,7 +120,7 @@ export FLANNEL_NET=${FLANNEL_NET:-"172.16.0.0/16"}
 
 # Admission Controllers to invoke prior to persisting objects in cluster
 # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely.
-export ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,ResourceQuota
+export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,ResourceQuota"}
 
 # Extra options to set on the Docker command line.
 # This is useful for setting --insecure-registry for local registries.

diff --git a/cluster/centos/master/scripts/apiserver.sh b/cluster/centos/master/scripts/apiserver.sh
@@ -31,6 +31,15 @@ KUBE_LOG_LEVEL="--v=4"
 # comma separated. Mutually exclusive with -etcd-config
 KUBE_ETCD_SERVERS="--etcd-servers=${ETCD_SERVERS}"
 
+# --etcd-cafile="": SSL Certificate Authority file used to secure etcd communication.
+KUBE_ETCD_CAFILE="--etcd-cafile=/srv/kubernetes/etcd/ca.pem"
+
+# --etcd-certfile="": SSL certification file used to secure etcd communication.
+KUBE_ETCD_CERTFILE="--etcd-certfile=/srv/kubernetes/etcd/client.pem"
+
+# --etcd-keyfile="": key file used to secure etcd communication.
+KUBE_ETCD_KEYFILE="--etcd-keyfile=/srv/kubernetes/etcd/client-key.pem"
+
 # --insecure-bind-address=127.0.0.1: The IP address on which to serve the --insecure-port.
 KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
 
@@ -77,6 +86,9 @@ EOF
 KUBE_APISERVER_OPTS="   \${KUBE_LOGTOSTDERR}         \\
                         \${KUBE_LOG_LEVEL}           \\
                         \${KUBE_ETCD_SERVERS}        \\
+                        \${KUBE_ETCD_CAFILE}         \\
+                        \${KUBE_ETCD_CERTFILE}       \\
+                        \${KUBE_ETCD_KEYFILE}        \\
                         \${KUBE_API_ADDRESS}         \\
                         \${KUBE_API_PORT}            \\
                         \${NODE_PORT}                \\

diff --git a/cluster/centos/master/scripts/etcd.sh b/cluster/centos/master/scripts/etcd.sh
@@ -31,20 +31,20 @@ ETCD_DATA_DIR="${etcd_data_dir}/default.etcd"
 #ETCD_SNAPSHOT_COUNTER="10000"
 #ETCD_HEARTBEAT_INTERVAL="100"
 #ETCD_ELECTION_TIMEOUT="1000"
-ETCD_LISTEN_PEER_URLS="http://${ETCD_LISTEN_IP}:2380"
-ETCD_LISTEN_CLIENT_URLS="http://${ETCD_LISTEN_IP}:2379,http://127.0.0.1:2379"
+ETCD_LISTEN_PEER_URLS="https://${ETCD_LISTEN_IP}:2380"
+ETCD_LISTEN_CLIENT_URLS="https://${ETCD_LISTEN_IP}:2379,https://127.0.0.1:2379"
 #ETCD_MAX_SNAPSHOTS="5"
 #ETCD_MAX_WALS="5"
 #ETCD_CORS=""
 #
 #[cluster]
-ETCD_INITIAL_ADVERTISE_PEER_URLS="http://${ETCD_LISTEN_IP}:2380"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="https://${ETCD_LISTEN_IP}:2380"
 # if you use different ETCD_NAME (e.g. test),
 # set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
 ETCD_INITIAL_CLUSTER="${ETCD_INITIAL_CLUSTER}"
 ETCD_INITIAL_CLUSTER_STATE="new"
 ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
-ETCD_ADVERTISE_CLIENT_URLS="http://${ETCD_LISTEN_IP}:2379"
+ETCD_ADVERTISE_CLIENT_URLS="https://${ETCD_LISTEN_IP}:2379"
 #ETCD_DISCOVERY=""
 #ETCD_DISCOVERY_SRV=""
 #ETCD_DISCOVERY_FALLBACK="proxy"
@@ -54,12 +54,14 @@ ETCD_ADVERTISE_CLIENT_URLS="http://${ETCD_LISTEN_IP}:2379"
 #ETCD_PROXY="off"
 #
 #[security]
-#ETCD_CA_FILE=""
-#ETCD_CERT_FILE=""
-#ETCD_KEY_FILE=""
-#ETCD_PEER_CA_FILE=""
-#ETCD_PEER_CERT_FILE=""
-#ETCD_PEER_KEY_FILE=""
+CLIENT_CERT_AUTH="true"
+ETCD_CA_FILE="/srv/kubernetes/etcd/ca.pem"
+ETCD_CERT_FILE="/srv/kubernetes/etcd/server-${ETCD_NAME}.pem"
+ETCD_KEY_FILE="/srv/kubernetes/etcd/server-${ETCD_NAME}-key.pem"
+PEER_CLIENT_CERT_AUTH="true"
+ETCD_PEER_CA_FILE="/srv/kubernetes/etcd/ca.pem"
+ETCD_PEER_CERT_FILE="/srv/kubernetes/etcd/peer-${ETCD_NAME}.pem"
+ETCD_PEER_KEY_FILE="/srv/kubernetes/etcd/peer-${ETCD_NAME}-key.pem"
 EOF
 
 cat <<EOF >//usr/lib/systemd/system/etcd.service

diff --git a/cluster/centos/master/scripts/flannel.sh b/cluster/centos/master/scripts/flannel.sh
@@ -18,10 +18,16 @@
 ETCD_SERVERS=${1:-"http://8.8.8.18:4001"}
 FLANNEL_NET=${2:-"172.16.0.0/16"}
 
+CA_FILE="/srv/kubernetes/etcd/ca.pem"
+CERT_FILE="/srv/kubernetes/etcd/client.pem"
+KEY_FILE="/srv/kubernetes/etcd/client-key.pem"
 
 cat <<EOF >/opt/kubernetes/cfg/flannel
 FLANNEL_ETCD="-etcd-endpoints=${ETCD_SERVERS}"
 FLANNEL_ETCD_KEY="-etcd-prefix=/coreos.com/network"
+FLANNEL_ETCD_CAFILE="--etcd-cafile=${CA_FILE}"
+FLANNEL_ETCD_CERTFILE="--etcd-certfile=${CERT_FILE}"
+FLANNEL_ETCD_KEYFILE="--etcd-keyfile=${KEY_FILE}"
 EOF
 
 cat <<EOF >/usr/lib/systemd/system/flannel.service
@@ -31,7 +37,7 @@ After=network.target
 
 [Service]
 EnvironmentFile=-/opt/kubernetes/cfg/flannel
-ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY}
+ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY} \${FLANNEL_ETCD_CAFILE} \${FLANNEL_ETCD_CERTFILE} \${FLANNEL_ETCD_KEYFILE}
 
 Type=notify
 
@@ -42,7 +48,8 @@ EOF
 # Store FLANNEL_NET to etcd.
 attempt=0
 while true; do
-  /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \
+  /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \
+    --no-sync -C ${ETCD_SERVERS} \
     get /coreos.com/network/config >/dev/null 2>&1
   if [[ "$?" == 0 ]]; then
     break
@@ -52,7 +59,8 @@ while true; do
       exit 2
     fi
 
-    /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \
+    /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \
+      --no-sync -C ${ETCD_SERVERS} \
       mk /coreos.com/network/config "{\"Network\":\"${FLANNEL_NET}\"}" >/dev/null 2>&1
     attempt=$((attempt+1))
     sleep 3

diff --git a/cluster/centos/node/scripts/flannel.sh b/cluster/centos/node/scripts/flannel.sh
@@ -18,10 +18,16 @@
 ETCD_SERVERS=${1:-"http://8.8.8.18:2379"}
 FLANNEL_NET=${2:-"172.16.0.0/16"}
 
+CA_FILE="/srv/kubernetes/etcd/ca.pem"
+CERT_FILE="/srv/kubernetes/etcd/client.pem"
+KEY_FILE="/srv/kubernetes/etcd/client-key.pem"
 
 cat <<EOF >/opt/kubernetes/cfg/flannel
 FLANNEL_ETCD="-etcd-endpoints=${ETCD_SERVERS}"
 FLANNEL_ETCD_KEY="-etcd-prefix=/coreos.com/network"
+FLANNEL_ETCD_CAFILE="--etcd-cafile=${CA_FILE}"
+FLANNEL_ETCD_CERTFILE="--etcd-certfile=${CERT_FILE}"
+FLANNEL_ETCD_KEYFILE="--etcd-keyfile=${KEY_FILE}"
 EOF
 
 cat <<EOF >/usr/lib/systemd/system/flannel.service
@@ -33,7 +39,7 @@ Before=docker.service
 [Service]
 EnvironmentFile=-/opt/kubernetes/cfg/flannel
 ExecStartPre=/opt/kubernetes/bin/remove-docker0.sh
-ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY}
+ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \${FLANNEL_ETCD} \${FLANNEL_ETCD_KEY} \${FLANNEL_ETCD_CAFILE} \${FLANNEL_ETCD_CERTFILE} \${FLANNEL_ETCD_KEYFILE}
 ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -d /run/flannel/docker
 
 Type=notify
@@ -46,7 +52,8 @@ EOF
 # Store FLANNEL_NET to etcd.
 attempt=0
 while true; do
-  /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \
+  /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \
+    --no-sync -C ${ETCD_SERVERS} \
     get /coreos.com/network/config >/dev/null 2>&1
   if [[ "$?" == 0 ]]; then
     break
@@ -56,7 +63,8 @@ while true; do
       exit 2
     fi
 
-    /opt/kubernetes/bin/etcdctl --no-sync -C ${ETCD_SERVERS} \
+    /opt/kubernetes/bin/etcdctl --ca-file ${CA_FILE} --cert-file ${CERT_FILE} --key-file ${KEY_FILE} \
+      --no-sync -C ${ETCD_SERVERS} \
       mk /coreos.com/network/config "{\"Network\":\"${FLANNEL_NET}\"}" >/dev/null 2>&1
     attempt=$((attempt+1))
     sleep 3

diff --git a/cluster/centos/util.sh b/cluster/centos/util.sh
@@ -208,6 +208,7 @@ echo "[INFO] tear-down-master on $1"
         fi"
   done
   kube-ssh "${1}" "sudo rm -rf /opt/kubernetes"
+  kube-ssh "${1}" "sudo rm -rf /srv/kubernetes"
   kube-ssh "${1}" "sudo rm -rf ${KUBE_TEMP}"
   kube-ssh "${1}" "sudo rm -rf /var/lib/etcd"
 }
@@ -226,6 +227,7 @@ echo "[INFO] tear-down-node on $1"
   done
   kube-ssh "$1" "sudo rm -rf /run/flannel"
   kube-ssh "$1" "sudo rm -rf /opt/kubernetes"
+  kube-ssh "$1" "sudo rm -rf /srv/kubernetes"
   kube-ssh "$1" "sudo rm -rf ${KUBE_TEMP}"
 }
 
@@ -239,6 +241,7 @@ function make-ca-cert() {
 #
 # Assumed vars:
 #   $1 (master)
+#   $2 (etcd_name)
 #   KUBE_TEMP
 #   ETCD_SERVERS
 #   ETCD_INITIAL_CLUSTER
@@ -250,12 +253,21 @@ function provision-master() {
   local master_ip="${master#*@}"
   local etcd_name="$2"
   ensure-setup-dir "${master}"
+  ensure-etcd-cert "${etcd_name}" "${master_ip}"
 
   kube-scp "${master}" "${ROOT}/ca-cert ${ROOT}/binaries/master ${ROOT}/master ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}"
+  kube-scp "${master}" "${ROOT}/etcd-cert/ca.pem \
+    ${ROOT}/etcd-cert/client.pem \
+    ${ROOT}/etcd-cert/client-key.pem \
+    ${ROOT}/etcd-cert/server-${etcd_name}.pem \
+    ${ROOT}/etcd-cert/server-${etcd_name}-key.pem \
+    ${ROOT}/etcd-cert/peer-${etcd_name}.pem \
+    ${ROOT}/etcd-cert/peer-${etcd_name}-key.pem" "${KUBE_TEMP}/etcd-cert"
   kube-ssh "${master}" " \
     sudo rm -rf /opt/kubernetes/bin; \
     sudo cp -r ${KUBE_TEMP}/master/bin /opt/kubernetes; \
-    sudo mkdir -p /srv/kubernetes; sudo cp -f ${KUBE_TEMP}/ca-cert/* /srv/kubernetes; \
+    sudo mkdir -p /srv/kubernetes/; sudo cp -f ${KUBE_TEMP}/ca-cert/* /srv/kubernetes/; \
+    sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \
     sudo chmod -R +x /opt/kubernetes/bin; \
     sudo ln -sf /opt/kubernetes/bin/* /usr/local/bin/; \
     sudo bash ${KUBE_TEMP}/master/scripts/etcd.sh ${etcd_name} ${master_ip} ${ETCD_INITIAL_CLUSTER}; \
@@ -298,12 +310,17 @@ function provision-node() {
   local dns_domain=${DNS_DOMAIN#*@}
   ensure-setup-dir ${node}
 
-  kube-scp ${node} "${ROOT}/binaries/node ${ROOT}/node ${ROOT}/config-default.sh ${ROOT}/util.sh" ${KUBE_TEMP}
+  kube-scp "${node}" "${ROOT}/binaries/node ${ROOT}/node ${ROOT}/config-default.sh ${ROOT}/util.sh" "${KUBE_TEMP}"
+  kube-scp "${node}" "${ROOT}/etcd-cert/ca.pem \
+    ${ROOT}/etcd-cert/client.pem \
+    ${ROOT}/etcd-cert/client-key.pem" "${KUBE_TEMP}/etcd-cert"
   kube-ssh "${node}" " \
     rm -rf /opt/kubernetes/bin; \
     sudo cp -r ${KUBE_TEMP}/node/bin /opt/kubernetes; \
     sudo chmod -R +x /opt/kubernetes/bin; \
+    sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \
     sudo ln -s /opt/kubernetes/bin/* /usr/local/bin/; \
+    sudo mkdir -p /srv/kubernetes/etcd; sudo cp -f ${KUBE_TEMP}/etcd-cert/* /srv/kubernetes/etcd/; \
     sudo bash ${KUBE_TEMP}/node/scripts/flannel.sh ${ETCD_SERVERS} ${FLANNEL_NET}; \
     sudo bash ${KUBE_TEMP}/node/scripts/docker.sh \"${DOCKER_OPTS}\"; \
     sudo bash ${KUBE_TEMP}/node/scripts/kubelet.sh ${MASTER_ADVERTISE_ADDRESS} ${node_ip} ${dns_ip} ${dns_domain}; \
@@ -316,10 +333,29 @@ function provision-node() {
 #   KUBE_TEMP
 function ensure-setup-dir() {
   kube-ssh "${1}" "mkdir -p ${KUBE_TEMP}; \
+                   mkdir -p ${KUBE_TEMP}/etcd-cert; \
                    sudo mkdir -p /opt/kubernetes/bin; \
                    sudo mkdir -p /opt/kubernetes/cfg"
 }
 
+# Generate certificates for etcd cluster
+#
+# Assumed vars:
+#   $1 (etcd member name)
+#   $2 (master ip)
+function ensure-etcd-cert() {
+  local etcd_name="$1"
+  local master_ip="$2"
+  local cert_dir="${ROOT}/etcd-cert"
+
+  if [[ ! -r "${cert_dir}/client.pem" || ! -r "${cert_dir}/client-key.pem" ]]; then
+    generate-etcd-cert "${cert_dir}" "${master_ip}" "client" "client"
+  fi
+
+  generate-etcd-cert "${cert_dir}" "${master_ip}" "server" "server-${etcd_name}"
+  generate-etcd-cert "${cert_dir}" "${master_ip}" "peer" "peer-${etcd_name}"
+}
+
 # Run command over ssh
 function kube-ssh() {
   local host="$1"