Skip to content

Commit

Permalink
Fix splitting cluster on Rolling Update
Browse files Browse the repository at this point in the history
  • Loading branch information
Sebastian Laskawiec committed Oct 11, 2019
1 parent 93d3ff4 commit 0c37709
Show file tree
Hide file tree
Showing 5 changed files with 1,126 additions and 12 deletions.
16 changes: 12 additions & 4 deletions src/main/java/org/jgroups/protocols/kubernetes/Client.java
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ protected String fetchFromKubernetes(String op, String namespace, String labels,


public List<Pod> getPods(String namespace, String labels, boolean dump_requests) throws Exception {
String result=fetchFromKubernetes("pods", namespace, labels, dump_requests);
String result = fetchFromKubernetes("pods", namespace, labels, dump_requests);
if(result == null)
return Collections.emptyList();
return parseJsonResult(result, namespace, labels);
Expand All @@ -107,20 +107,29 @@ String getPodGroup(Json pod) {
Json labels = Optional.ofNullable(meta)
.map(podMetadata -> podMetadata.at("labels"))
.orElse(null);

// This works for Deployment Config
String group = Optional.ofNullable(labels)
.map(l -> l.at("pod-template-hash"))
.map(Json::asString)
.orElse(null);

if (group == null) {
log.warn("metadata.labels.pod-template-hash not found in pod json. Impossible to reliably determine pod group during Rolling Update");
// keep backward-compatible behavior
// Ok, maybe, it's a Deployment and has a valid deployment flag?
group = Optional.ofNullable(labels)
.map(l -> l.at("deployment"))
.map(Json::asString)
.orElse(null);
}

if (group == null) {
// Final check, maybe it's a StatefulSet?
group = Optional.ofNullable(labels)
.map(l -> l.at("controller-revision-hash"))
.map(Json::asString)
.orElse(null);
}

log.debug("pod %s, group %s", Optional.ofNullable(meta)
.map(m -> m.at("name"))
.map(Json::asString)
Expand All @@ -146,7 +155,6 @@ protected List<Pod> parseJsonResult(String input, String namespace, String label
List<Pod> pods=new ArrayList<>();
for(Json obj: items) {
String parentDeployment = getPodGroup(obj);

String name = Optional.ofNullable(obj.at("metadata"))
.map(podMetadata -> podMetadata.at("name"))
.map(Json::asString)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,11 @@ public void findMembers(List<Address> members, boolean initial_discovery, Respon
if(physical_addr != null) {
String senderIp = ((IpAddress)physical_addr).getIpAddress().getHostAddress();
// Please note we search for sender parent group through all pods, ever not ready. It's because JGroup discovery is performed
// before Wildfly can respond to http liveness probe.
// before Wildfly can respond to http readiness probe.
hosts.stream()
.filter(p -> p.getPodGroup() == null)
.forEach(p -> log.warn("Pod %s doesn't have group assigned. Impossible to reliably determine pod group during Rolling Update."));

String senderPodGroup = hosts.stream()
.filter(pod -> senderIp.contains(pod.getIp()))
.map(Pod::getPodGroup)
Expand Down
12 changes: 5 additions & 7 deletions src/main/java/org/jgroups/protocols/kubernetes/Pod.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.jgroups.protocols.kubernetes;

import java.util.Objects;

public class Pod {

private final String name;
Expand All @@ -15,10 +17,6 @@ public Pod(String name, String ip, String podGroup, boolean isReady) {
this.isReady = isReady;
}

public Pod(String name, String ip, String podGroup) {
this(name, ip, podGroup, false);
}

public String getName() {
return name;
}
Expand Down Expand Up @@ -51,9 +49,9 @@ public boolean equals(Object o) {

Pod pod = (Pod) o;

if (name != null ? !name.equals(pod.name) : pod.name != null) return false;
if (ip != null ? !ip.equals(pod.ip) : pod.ip != null) return false;
return podGroup != null ? podGroup.equals(pod.podGroup) : pod.podGroup == null;
if (!Objects.equals(name, pod.name)) return false;
if (!Objects.equals(ip, pod.ip)) return false;
return Objects.equals(podGroup, pod.podGroup);
}

@Override
Expand Down
13 changes: 13 additions & 0 deletions src/test/java/org/jgroups/ping/kube/test/RollingUpdateTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ public void testPutOnlyNodesWithTheSameParentDuringRollingUpdateOpenShift() thro
KUBE_PING_FOR_TESTING testedProtocol = new KUBE_PING_FOR_TESTING("/openshift_rolling_update.json");
testedProtocol.setValue("split_clusters_during_rolling_update", true);

//when //then
testPutOnlyNodesWithTheSameParentDuringRollingUpdate(testedProtocol);
}

Expand All @@ -61,6 +62,18 @@ public void testPutOnlyNodesWithTheSameParentDuringRollingUpdateReplicaSet() thr
//given
KUBE_PING_FOR_TESTING testedProtocol = new KUBE_PING_FOR_TESTING("/replicaset_rolling_update.json");
testedProtocol.setValue("split_clusters_during_rolling_update", true);

//when //then
testPutOnlyNodesWithTheSameParentDuringRollingUpdate(testedProtocol);
}

@Test
public void testPutOnlyNodesWithTheSameParentDuringRollingUpdateStatefulSet() throws Exception {
//given
KUBE_PING_FOR_TESTING testedProtocol = new KUBE_PING_FOR_TESTING("/statefulset_rolling_update.json");
testedProtocol.setValue("split_clusters_during_rolling_update", true);

//when //then
testPutOnlyNodesWithTheSameParentDuringRollingUpdate(testedProtocol);
}

Expand Down
Loading

0 comments on commit 0c37709

Please sign in to comment.