diff --git a/.gitignore b/.gitignore index 46ce63e..ccf3eeb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /charts/proxmox-cloud-controller-manager/values-dev.yaml /proxmox-cloud-controller-manager* /kubeconfig +/kubeconfig* /proxmox-config.yaml # diff --git a/charts/proxmox-cloud-controller-manager/Chart.yaml b/charts/proxmox-cloud-controller-manager/Chart.yaml index 88cd13c..bcddf68 100644 --- a/charts/proxmox-cloud-controller-manager/Chart.yaml +++ b/charts/proxmox-cloud-controller-manager/Chart.yaml @@ -14,7 +14,7 @@ maintainers: # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.11 +version: 0.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. diff --git a/charts/proxmox-cloud-controller-manager/README.md b/charts/proxmox-cloud-controller-manager/README.md index b34a592..fa14dc6 100644 --- a/charts/proxmox-cloud-controller-manager/README.md +++ b/charts/proxmox-cloud-controller-manager/README.md @@ -1,6 +1,6 @@ # proxmox-cloud-controller-manager -![Version: 0.1.11](https://img.shields.io/badge/Version-0.1.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.4.0](https://img.shields.io/badge/AppVersion-v0.4.0-informational?style=flat-square) +![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.4.0](https://img.shields.io/badge/AppVersion-v0.4.0-informational?style=flat-square) A Helm chart for Kubernetes @@ -73,6 +73,8 @@ helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ | config | object | `{"clusters":[]}` | Proxmox cluster config. | | serviceAccount | object | `{"annotations":{},"create":true,"name":""}` | Pods Service Account. ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ | | priorityClassName | string | `"system-cluster-critical"` | CCM pods' priorityClassName. | +| initContainers | list | `[]` | Add additional init containers to the CCM pods. ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ | +| hostAliases | list | `[]` | hostAliases Deployment pod host aliases ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ | | podAnnotations | object | `{}` | Annotations for data pods. ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ | | podSecurityContext | object | `{"fsGroup":10258,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":10258,"runAsNonRoot":true,"runAsUser":10258}` | Pods Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | | securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"seccompProfile":{"type":"RuntimeDefault"}}` | Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | diff --git a/charts/proxmox-cloud-controller-manager/templates/deployment.yaml b/charts/proxmox-cloud-controller-manager/templates/deployment.yaml index bf5080a..b9ec5c3 100644 --- a/charts/proxmox-cloud-controller-manager/templates/deployment.yaml +++ b/charts/proxmox-cloud-controller-manager/templates/deployment.yaml @@ -35,6 +35,9 @@ spec: serviceAccountName: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + hostAliases: + {{- toYaml .Values.hostAliases | nindent 8 }} + initContainers: {{- toYaml .Values.initContainers | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: @@ -73,9 +76,18 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} affinity: + {{- with .Values.affinity }} {{- toYaml . | nindent 8 }} + {{- else }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 20 }} + topologyKey: topology.kubernetes.io/zone + weight: 1 {{- end }} {{- with .Values.tolerations }} tolerations: diff --git a/charts/proxmox-cloud-controller-manager/values.yaml b/charts/proxmox-cloud-controller-manager/values.yaml index 4d02003..5d40f47 100644 --- a/charts/proxmox-cloud-controller-manager/values.yaml +++ b/charts/proxmox-cloud-controller-manager/values.yaml @@ -61,6 +61,36 @@ serviceAccount: # -- CCM pods' priorityClassName. priorityClassName: system-cluster-critical +# -- Add additional init containers to the CCM pods. +# ref: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ +initContainers: [] + # - name: loadbalancer + # restartPolicy: Always + # image: ghcr.io/sergelogvinov/haproxy:2.8.3-alpine3.18 + # imagePullPolicy: IfNotPresent + # env: + # - name: SVC + # value: "proxmox.domain.com" + # - name: PORT + # value: "8006" + # securityContext: + # runAsUser: 99 + # runAsGroup: 99 + # resources: + # limits: + # cpu: 50m + # memory: 64Mi + # requests: + # cpu: 50m + # memory: 32Mi + +# -- hostAliases Deployment pod host aliases +# ref: https://kubernetes.io/docs/tasks/network/customize-hosts-file-for-pods/ +hostAliases: [] + # - ip: 127.0.0.1 + # hostnames: + # - proxmox.domain.com + # -- Annotations for data pods. # ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ podAnnotations: {} diff --git a/docs/deploy/cloud-controller-manager-talos.yml b/docs/deploy/cloud-controller-manager-talos.yml index c22b4a7..885172f 100644 --- a/docs/deploy/cloud-controller-manager-talos.yml +++ b/docs/deploy/cloud-controller-manager-talos.yml @@ -5,7 +5,7 @@ kind: ServiceAccount metadata: name: proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.1.11 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -18,7 +18,7 @@ kind: ClusterRole metadata: name: system:proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.1.11 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -106,7 +106,7 @@ kind: Deployment metadata: name: proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.1.11 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -137,6 +137,10 @@ spec: runAsGroup: 10258 runAsNonRoot: true runAsUser: 10258 + hostAliases: + [] + initContainers: + [] containers: - name: proxmox-cloud-controller-manager securityContext: diff --git a/docs/deploy/cloud-controller-manager.yml b/docs/deploy/cloud-controller-manager.yml index 92a150d..7b52a1f 100644 --- a/docs/deploy/cloud-controller-manager.yml +++ b/docs/deploy/cloud-controller-manager.yml @@ -5,7 +5,7 @@ kind: ServiceAccount metadata: name: proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.1.11 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -18,7 +18,7 @@ kind: ClusterRole metadata: name: system:proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.1.11 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -106,7 +106,7 @@ kind: Deployment metadata: name: proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.1.11 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -137,6 +137,10 @@ spec: runAsGroup: 10258 runAsNonRoot: true runAsUser: 10258 + hostAliases: + [] + initContainers: + [] containers: - name: proxmox-cloud-controller-manager securityContext: diff --git a/docs/loadbalancer.md b/docs/loadbalancer.md new file mode 100644 index 0000000..1419040 --- /dev/null +++ b/docs/loadbalancer.md @@ -0,0 +1,82 @@ +# Loadbalancer on top of the Proxmox cluster + +Set up a load balancer to distribute traffic across multiple proxmox nodes. +We use the [haproxy](https://hub.docker.com/_/haproxy) image to create a simple load balancer on top of the proxmox cluster. +First, we need to create a headless service and set endpoints. + +```yaml +# proxmox-service.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: proxmox + namespace: kube-system +spec: + clusterIP: None + ports: + - name: https + protocol: TCP + port: 8006 + targetPort: 8006 +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: proxmox + namespace: kube-system +subsets: + - addresses: + - ip: 192.168.0.1 + - ip: 192.168.0.2 + ports: + - port: 8006 +``` + +Apply the configuration to the cluster. + +```bash +kubectl apply -f proxmox-service.yaml +``` + +Second, we need to deploy proxmox CCM with sidecar load balancer. +Haproxy will resolve the `proxmox.kube-system.svc.cluster.local` service and uses IPs from the endpoints to distribute traffic. +Proxmox CCM will use the `proxmox.domain.com` domain to connect to the proxmox cluster wich is resolved to the load balancer IP (127.0.0.1). + +```yaml +# Helm Chart values + +config: + clusters: + - region: cluster + url: https://proxmox.domain.com:8006/api2/json + insecure: true + token_id: kubernetes@pve!ccm + token_secret: 11111111-1111-1111-1111-111111111111 + +hostAliases: + - ip: 127.0.0.1 + hostnames: + - proxmox.domain.com + +initContainers: + - name: loadbalancer + restartPolicy: Always + image: ghcr.io/sergelogvinov/haproxy:2.8.6-alpine3.19 + imagePullPolicy: IfNotPresent + env: + - name: SVC + value: proxmox.kube-system.svc.cluster.local + - name: PORT + value: "8006" + securityContext: + runAsUser: 99 + runAsGroup: 99 + resources: + limits: + cpu: 50m + memory: 64Mi + requests: + cpu: 50m + memory: 32Mi +```