From 7d6cc879d4a4a37630124d5ba3067c964bc81030 Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Fri, 20 Sep 2024 05:06:09 -0700 Subject: [PATCH 1/5] Upgrading k8s lib --- paasta_tools/kubernetes_tools.py | 22 ++++++++-------------- requirements-minimal.txt | 2 +- requirements.txt | 2 +- tests/test_kubernetes_tools.py | 10 +++++----- 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/paasta_tools/kubernetes_tools.py b/paasta_tools/kubernetes_tools.py index 9f8730c7a0..108e0f474e 100644 --- a/paasta_tools/kubernetes_tools.py +++ b/paasta_tools/kubernetes_tools.py @@ -51,8 +51,6 @@ from kubernetes.client import models from kubernetes.client import V1Affinity from kubernetes.client import V1AWSElasticBlockStoreVolumeSource -from kubernetes.client import V1beta1CustomResourceDefinition -from kubernetes.client import V1beta1CustomResourceDefinitionList from kubernetes.client import V1beta1PodDisruptionBudget from kubernetes.client import V1beta1PodDisruptionBudgetSpec from kubernetes.client import V1Capabilities @@ -70,12 +68,12 @@ from kubernetes.client import V1EnvVar from kubernetes.client import V1EnvVarSource from kubernetes.client import V1ExecAction -from kubernetes.client import V1Handler from kubernetes.client import V1HostPathVolumeSource from kubernetes.client import V1HTTPGetAction from kubernetes.client import V1KeyToPath from kubernetes.client import V1LabelSelector from kubernetes.client import V1Lifecycle +from kubernetes.client import V1LifecycleHandler from kubernetes.client import V1LimitRange from kubernetes.client import V1LimitRangeItem from kubernetes.client import V1LimitRangeSpec @@ -1112,7 +1110,7 @@ def get_hacheck_sidecar_container( return V1Container( image=system_paasta_config.get_hacheck_sidecar_image_url(), lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction( command=[ "/bin/sh", @@ -1155,7 +1153,7 @@ def get_gunicorn_exporter_sidecar_container( env=self.get_kubernetes_environment(), ports=[V1ContainerPort(container_port=9117)], lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction( command=[ "/bin/sh", @@ -1464,20 +1462,20 @@ def get_readiness_probe( else: return self.get_liveness_probe(service_namespace_config) - def get_kubernetes_container_termination_action(self) -> V1Handler: + def get_kubernetes_container_termination_action(self) -> V1LifecycleHandler: command = self.config_dict.get("lifecycle", KubeLifecycleDict({})).get( "pre_stop_command", [] ) # default pre stop hook for the container if not command: - return V1Handler( + return V1LifecycleHandler( _exec=V1ExecAction( command=["/bin/sh", "-c", f"sleep {DEFAULT_PRESTOP_SLEEP_SECONDS}"] ) ) if isinstance(command, str): command = [command] - return V1Handler(_exec=V1ExecAction(command=command)) + return V1LifecycleHandler(_exec=V1ExecAction(command=command)) def get_pod_volumes( self, @@ -4155,12 +4153,8 @@ def mode_to_int(mode: Optional[Union[str, int]]) -> Optional[int]: def update_crds( kube_client: KubeClient, - desired_crds: Collection[ - Union[V1CustomResourceDefinition, V1beta1CustomResourceDefinition] - ], - existing_crds: Union[ - V1CustomResourceDefinitionList, V1beta1CustomResourceDefinitionList - ], + desired_crds: Collection[Union[V1CustomResourceDefinition]], + existing_crds: Union[V1CustomResourceDefinitionList], ) -> bool: for desired_crd in desired_crds: existing_crd = None diff --git a/requirements-minimal.txt b/requirements-minimal.txt index 9e981f1bb0..8f91dfdc41 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -29,7 +29,7 @@ kazoo >= 2.0.0 # that we can use across our different clusters (e.g, if X.0.0 removes an API version that we use # in any cluster, this upper-bound should be < X.0.0) # we should probably also be better at setting a correct lower-bound, but that's less likely to cause issues. -kubernetes >= 18.20.0, < 22.0.0 +kubernetes >= 18.20.0, < 26.0.0 ldap3 manhole mypy-extensions >= 0.3.0 diff --git a/requirements.txt b/requirements.txt index fc86a1d482..6819f2c9b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ jmespath==0.9.3 jsonref==0.1 jsonschema==2.5.1 kazoo==2.8.0 -kubernetes==21.7.0 +kubernetes==24.2.0 ldap3==2.6 manhole==1.5.0 MarkupSafe==1.1.1 diff --git a/tests/test_kubernetes_tools.py b/tests/test_kubernetes_tools.py index 93fb8fa72b..f5d6e7fb40 100644 --- a/tests/test_kubernetes_tools.py +++ b/tests/test_kubernetes_tools.py @@ -26,12 +26,12 @@ from kubernetes.client import V1EnvVar from kubernetes.client import V1EnvVarSource from kubernetes.client import V1ExecAction -from kubernetes.client import V1Handler from kubernetes.client import V1HostPathVolumeSource from kubernetes.client import V1HTTPGetAction from kubernetes.client import V1KeyToPath from kubernetes.client import V1LabelSelector from kubernetes.client import V1Lifecycle +from kubernetes.client import V1LifecycleHandler from kubernetes.client import V1NodeAffinity from kubernetes.client import V1NodeSelector from kubernetes.client import V1NodeSelectorRequirement @@ -603,7 +603,7 @@ def test_get_sidecar_containers(self): ], image="some-docker-image", lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction( command=[ "/bin/sh", @@ -649,7 +649,7 @@ def test_get_sidecar_containers(self): ], image="some-docker-image", lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction( command=[ "/bin/sh", @@ -929,7 +929,7 @@ def test_get_kubernetes_containers(self, prometheus_port, expected_ports): resources=mock_get_resource_requirements.return_value, image=mock_get_docker_url.return_value, lifecycle=V1Lifecycle( - pre_stop=V1Handler( + pre_stop=V1LifecycleHandler( _exec=V1ExecAction(command=["/bin/sh", "-c", "sleep 30"]) ) ), @@ -2201,7 +2201,7 @@ def test_kubernetes_container_termination_action( self.deployment.config_dict["lifecycle"] = { "pre_stop_command": termination_action } - handler = V1Handler(_exec=V1ExecAction(command=expected)) + handler = V1LifecycleHandler(_exec=V1ExecAction(command=expected)) assert self.deployment.get_kubernetes_container_termination_action() == handler @pytest.mark.parametrize( From ac39d5c45b7b9c2bbc9bc0188d93fe23b553e933 Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Fri, 20 Sep 2024 05:16:32 -0700 Subject: [PATCH 2/5] Fixing tests --- paasta_tools/kubernetes_tools.py | 11 ++++------- tests/test_kubernetes_tools.py | 4 ++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/paasta_tools/kubernetes_tools.py b/paasta_tools/kubernetes_tools.py index 108e0f474e..25c0b01b4b 100644 --- a/paasta_tools/kubernetes_tools.py +++ b/paasta_tools/kubernetes_tools.py @@ -608,9 +608,9 @@ def __init__( # We need to support apiextensions /v1 and /v1beta1 in order # to make our upgrade to k8s 1.22 smooth, otherwise # updating the CRDs make this script fail - self.apiextensions_v1_beta1 = kube_client.ApiextensionsV1beta1Api( - self.api_client - ) + # self.apiextensions_v1_beta1 = kube_client.ApiextensionsV1beta1Api( + # self.api_client + # ) self.custom = kube_client.CustomObjectsApi(self.api_client) self.autoscaling = kube_client.AutoscalingV2beta2Api(self.api_client) self.rbac = kube_client.RbacAuthorizationV1Api(self.api_client) @@ -4164,10 +4164,7 @@ def update_crds( break try: - if "apiextensions.k8s.io/v1beta1" == desired_crd.api_version: - apiextensions = kube_client.apiextensions_v1_beta1 - else: - apiextensions = kube_client.apiextensions + apiextensions = kube_client.apiextensions if existing_crd: desired_crd.metadata[ diff --git a/tests/test_kubernetes_tools.py b/tests/test_kubernetes_tools.py index f5d6e7fb40..a43c1ba0a1 100644 --- a/tests/test_kubernetes_tools.py +++ b/tests/test_kubernetes_tools.py @@ -4281,7 +4281,7 @@ def test_warning_big_bounce(): job_config.format_kubernetes_app().spec.template.metadata.labels[ "paasta.yelp.com/config_sha" ] - == "configd2fd7b15" + == "config3bd814d2" ), "If this fails, just change the constant in this test, but be aware that deploying this change will cause every service to bounce!" @@ -4327,7 +4327,7 @@ def test_warning_big_bounce_routable_pod(): job_config.format_kubernetes_app().spec.template.metadata.labels[ "paasta.yelp.com/config_sha" ] - == "configa2ea39be" + == "configf23a3edb" ), "If this fails, just change the constant in this test, but be aware that deploying this change will cause every smartstack-registered service to bounce!" From 5ce76781e5210ad74b10964055b8404ddecc9da2 Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Fri, 20 Sep 2024 07:20:43 -0700 Subject: [PATCH 3/5] fixing setup crd --- paasta_tools/setup_kubernetes_crd.py | 44 +++++----------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/paasta_tools/setup_kubernetes_crd.py b/paasta_tools/setup_kubernetes_crd.py index eabc667c06..9c8b1e0494 100644 --- a/paasta_tools/setup_kubernetes_crd.py +++ b/paasta_tools/setup_kubernetes_crd.py @@ -27,9 +27,7 @@ from typing import Sequence import service_configuration_lib -from kubernetes.client import V1beta1CustomResourceDefinition from kubernetes.client import V1CustomResourceDefinition -from kubernetes.client.exceptions import ApiException from paasta_tools.kubernetes_tools import KubeClient from paasta_tools.kubernetes_tools import paasta_prefixed @@ -106,22 +104,7 @@ def setup_kube_crd( label_selector=paasta_prefixed("service") ) - # This step can fail in k8s 1.22 since this version is not existing anymore - # we need to support this for the transition - try: - existing_crds_v1_beta1 = ( - kube_client.apiextensions_v1_beta1.list_custom_resource_definition( - label_selector=paasta_prefixed("service") - ) - ) - except ApiException: - existing_crds_v1_beta1 = [] - log.debug( - "Listing CRDs with apiextensions/v1beta1 not supported on this cluster, falling back to v1" - ) - desired_crds = [] - desired_crds_v1_beta1 = [] for service in services: crd_config = service_configuration_lib.read_extra_service_information( service, f"crd-{cluster}", soa_dir=soa_dir @@ -136,31 +119,18 @@ def setup_kube_crd( metadata["labels"]["yelp.com/paasta_service"] = service metadata["labels"][paasta_prefixed("service")] = service - if "apiextensions.k8s.io/v1beta1" == crd_config["apiVersion"]: - desired_crd = V1beta1CustomResourceDefinition( - api_version=crd_config.get("apiVersion"), - kind=crd_config.get("kind"), - metadata=metadata, - spec=crd_config.get("spec"), - ) - desired_crds_v1_beta1.append(desired_crd) - else: - desired_crd = V1CustomResourceDefinition( - api_version=crd_config.get("apiVersion"), - kind=crd_config.get("kind"), - metadata=metadata, - spec=crd_config.get("spec"), - ) - desired_crds.append(desired_crd) + desired_crd = V1CustomResourceDefinition( + api_version=crd_config.get("apiVersion"), + kind=crd_config.get("kind"), + metadata=metadata, + spec=crd_config.get("spec"), + ) + desired_crds.append(desired_crd) return update_crds( kube_client=kube_client, desired_crds=desired_crds, existing_crds=existing_crds, - ) and update_crds( - kube_client=kube_client, - desired_crds=desired_crds_v1_beta1, - existing_crds=existing_crds_v1_beta1, ) From cfbea323bb304c2ffe28475744a71047c3b66c59 Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Fri, 20 Sep 2024 07:58:20 -0700 Subject: [PATCH 4/5] fixing setup CR --- paasta_tools/setup_kubernetes_cr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/paasta_tools/setup_kubernetes_cr.py b/paasta_tools/setup_kubernetes_cr.py index 5972b4da72..0d0e810368 100644 --- a/paasta_tools/setup_kubernetes_cr.py +++ b/paasta_tools/setup_kubernetes_cr.py @@ -166,7 +166,6 @@ def setup_all_custom_resources( # we need to try both possibilities for apiextension in [ kube_client.apiextensions, - kube_client.apiextensions_v1_beta1, ]: try: @@ -175,7 +174,7 @@ def setup_all_custom_resources( ).items except ApiException: log.debug( - "Listing CRDs with apiextensions/v1 not supported on this cluster, falling back to v1beta1" + "Listing CRDs with apiextensions/v1 not supported on this cluster" ) crds_list = [] From 5265d56da48eaf8efaba3c77a710c14e75ffe638 Mon Sep 17 00:00:00 2001 From: Wilmer Bandres Date: Fri, 20 Sep 2024 08:20:36 -0700 Subject: [PATCH 5/5] Removing dead code --- paasta_tools/kubernetes_tools.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/paasta_tools/kubernetes_tools.py b/paasta_tools/kubernetes_tools.py index 25c0b01b4b..d5344191f1 100644 --- a/paasta_tools/kubernetes_tools.py +++ b/paasta_tools/kubernetes_tools.py @@ -605,12 +605,6 @@ def __init__( self.policy = kube_client.PolicyV1beta1Api(self.api_client) self.apiextensions = kube_client.ApiextensionsV1Api(self.api_client) - # We need to support apiextensions /v1 and /v1beta1 in order - # to make our upgrade to k8s 1.22 smooth, otherwise - # updating the CRDs make this script fail - # self.apiextensions_v1_beta1 = kube_client.ApiextensionsV1beta1Api( - # self.api_client - # ) self.custom = kube_client.CustomObjectsApi(self.api_client) self.autoscaling = kube_client.AutoscalingV2beta2Api(self.api_client) self.rbac = kube_client.RbacAuthorizationV1Api(self.api_client)