From e9987159b8f16b50600508002443b0738aea4cb6 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Hernandez Date: Wed, 3 Feb 2021 12:19:03 +0100 Subject: [PATCH 1/2] Avoid deadlock in DatadogQueueListener --- .../listeners/DatadogQueueListener.java | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogQueueListener.java b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogQueueListener.java index a8196c08c..db66f5de4 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogQueueListener.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogQueueListener.java @@ -6,12 +6,19 @@ import hudson.model.queue.QueueListener; import org.datadog.jenkins.plugins.datadog.model.FlowNodeQueueData; import org.datadog.jenkins.plugins.datadog.model.PipelineQueueInfoAction; +import org.datadog.jenkins.plugins.datadog.util.SuppressFBWarnings; import org.jenkinsci.plugins.workflow.flow.FlowExecution; import org.jenkinsci.plugins.workflow.graph.FlowNode; import org.jenkinsci.plugins.workflow.support.steps.ExecutorStepExecution; import javax.annotation.CheckForNull; import java.io.IOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.logging.Logger; @Extension @@ -34,7 +41,9 @@ public void onEnterBuildable(Queue.BuildableItem item) { } final ExecutorStepExecution.PlaceholderTask placeholderTask = (ExecutorStepExecution.PlaceholderTask) task; - final FlowNode flowNode = placeholderTask.getNode(); + // Use async method to avoid deadlock. + // It fixes https://github.com/jenkinsci/datadog-plugin/issues/170 + final FlowNode flowNode = getNodeAsync(placeholderTask, 5000); if(flowNode == null) { logger.fine("onEnterBuildable PlaceholderTask: " + placeholderTask + ", FlowNode: is null"); @@ -65,7 +74,6 @@ public void onEnterBuildable(Queue.BuildableItem item) { } catch (Exception e){ logger.severe("Error onEnterBuildable: item:" + item + ", exception: " + e); } - } @Override @@ -83,7 +91,9 @@ public void onLeaveBuildable(Queue.BuildableItem item) { } final ExecutorStepExecution.PlaceholderTask placeholderTask = (ExecutorStepExecution.PlaceholderTask) task; - final FlowNode flowNode = placeholderTask.getNode(); + // Use async method to avoid deadlock. + // It fixes https://github.com/jenkinsci/datadog-plugin/issues/170 + final FlowNode flowNode = getNodeAsync(placeholderTask, 5000); if(flowNode == null) { logger.fine("onLeaveBuildable PlaceholderTask: " + placeholderTask + ", FlowNode: is null"); return; @@ -110,6 +120,32 @@ public void onLeaveBuildable(Queue.BuildableItem item) { } } + /** + * Gets the FlowNode from the PlaceholderTask asynchronous. + * + * This method is needed because there is a deadlock when a job is being executed and + * the Jenkins instances is killed. After restarting it, the placeholderTask is + * trying to obtain the FlowNode forever. + * + * Related to: [BUG] https://issues.jenkins.io/browse/JENKINS-64688 + * Related to: [BUG] https://github.com/jenkinsci/datadog-plugin/issues/170 + * + * @param placeholderTask + * @param timeoutMs + * @return FlowNode or null + */ + @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_BAD_PRACTICE") + private FlowNode getNodeAsync(ExecutorStepExecution.PlaceholderTask placeholderTask, int timeoutMs) { + try { + final CompletableFuture f = new CompletableFuture<>(); + Executors.newCachedThreadPool().submit(() -> f.complete(placeholderTask.getNode())); + return f.get(timeoutMs, TimeUnit.MILLISECONDS); + } catch (Exception ex){ + logger.severe("Error getNodeAsync for task:"+placeholderTask+", exception: " + ex); + return null; + } + } + /** * Gets the jenkins run object of the specified executing workflow. * From 58294d8554e47755680594d560f8f0694a7b93a4 Mon Sep 17 00:00:00 2001 From: Daniel Rodriguez Hernandez Date: Wed, 3 Feb 2021 15:34:36 +0100 Subject: [PATCH 2/2] Using log.fine --- .../jenkins/plugins/datadog/listeners/DatadogQueueListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogQueueListener.java b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogQueueListener.java index db66f5de4..1b0b0fa3a 100644 --- a/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogQueueListener.java +++ b/src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogQueueListener.java @@ -141,7 +141,7 @@ private FlowNode getNodeAsync(ExecutorStepExecution.PlaceholderTask placeholderT Executors.newCachedThreadPool().submit(() -> f.complete(placeholderTask.getNode())); return f.get(timeoutMs, TimeUnit.MILLISECONDS); } catch (Exception ex){ - logger.severe("Error getNodeAsync for task:"+placeholderTask+", exception: " + ex); + logger.fine("Error getNodeAsync for task:"+placeholderTask+", exception: " + ex); return null; } }