refactor configs

README.md

@@ -11,47 +11,48 @@ Currently, the plugin is tracking the following data.
 List of events:
 * Started build
 * Finished build
+* SCM Checkout
 
-List of metrics sent via the Datadog API:
-
-| Metric Name          | Description                                         |
-|----------------------|-----------------------------------------------------|
-| jenkins.job.duration | Build duration (in seconds)                         |
-| jenkins.job.waiting  | Time spent waiting for job to run (in milliseconds) |
-
-
-List of metrics sent via Dogstatsd:
-Requires the Datadog Agent to be installed on the Jenkins host.
 
 | Metric Name              | Description                                                   |
 |--------------------------|---------------------------------------------------------------|
-| jenkins.job.completed    | Jobs Completed                                                |
+| jenkins.job.completed    | Jobs Completed rate                                           |
+| jenkins.job.started      | Jobs Started rate                                             |
 | jenkins.job.leadtime     | Lead Time                                                     |
 | jenkins.job.cycletime    | Cycle Time                                                    |
 | jenkins.job.mttr         | MTTR: time between last failed job and current successful job |
 | jenkins.job.feedbacktime | Feedback time from code commit to job failure                 |
 | jenkins.job.mtbf         | MTBF, time between last successful job and current failed job |
+| jenkins.job.duration     | Build duration (in seconds)                                   |
+| jenkins.job.waiting      | Time spent waiting for job to run (in milliseconds)           |
+| jenkins.queue.size       | Queue Size                                                    |
+| jenkins.scm.checkout     | SCM checkout rate                                             |
+
 
 List of service checks:
 * Build status (jenkins.job.status)
 
 All events, metrics, and service checks include the following tags, if they are available:
 * `job`
 * `result`
-* (git) `branch` (Available when using the [Git Plugin](https://wiki.jenkins.io/display/JENKINS/Git+Plugin))
+* (Git Branch, SVN revision or CVS branch) `branch` 
+  * Git Branch available when using the [Git Plugin](https://wiki.jenkins.io/display/JENKINS/Git+Plugin)
+* `node`
 
-Optional tags, included in events, metrics, and service checks. (Toggle from `Manage Jenkins -> Configure System`)
-* `node` (disabled by default)
 
 ## Customization
 From the global configuration page, at `Manage Jenkins -> Configure System`.
 * Blacklisted Jobs
-	* A comma-separated list of job names that should not monitored. (eg: susans-job,johns-job,prod_folder/prod_release).
+	* A regex comma-separated list of job names that should not be monitored. (eg: susans-job,johns-.*,prod_folder/prod_release).
+* Whitelisted Jobs
+	* A regex comma-separated list of job names that should be monitored. (eg: susans-job,johns-.*,prod_folder/prod_release).
+* Global Tags
+	* A list of jobs and comma-separated list of tags that should be applied for that job (eg: (.*?)_job_(*?)_release, owner:$1, release_env:$2, optional:Tag3).
 
 From a job specific configuration page
 * Custom tags
-	* Added from a file in the job workspace (not compatible with Pipeline jobs), or
-	* Added as text directly from the configuration page
+	* From a file in the job workspace (not compatible with Pipeline jobs).
+	* As text properties directly from the configuration page.
 
 ## Installation
 _This plugin requires [Jenkins 1.580.1](http://updates.jenkins-ci.org/download/war/1.580.1/jenkins.war) or newer._
@@ -70,7 +71,7 @@ import jenkins.model.*
 import org.datadog.jenkins.plugins.datadog.DatadogBuildListener
 
 def j = Jenkins.getInstance()
-def d = j.getDescriptor("org.datadog.jenkins.plugins.datadog.DatadogBuildListener")
+def d = j.getDescriptor("org.datadog.jenkins.plugins.datadog.listeners.DatadogBuildListener")
 d.setHostname('https://your-jenkins.com:8080')
 d.setTagNode(true)
 d.setApiKey('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')
@@ -81,7 +82,7 @@ d.save()
 Configuring the plugin this way might be useful if you're running your Jenkins Master in a Docker container using the [Official Jenkins Docker Image](https://github.com/jenkinsci/docker) or any derivative that supports plugins.txt and Groovy init scripts.
 
 ### Logging
-Logging is done by utilizing the java.util.Logger, which follows the [best logging practices for Jenkins](https://wiki.jenkins-ci.org/display/JENKINS/Logging). In order to obtain logs, follow the directions listed [here](https://wiki.jenkins-ci.org/display/JENKINS/Logging). When adding a Logger, all Datadog plugin functions start with `org.datadog.jenkins.plugins.datadog.` and the function name you're after should autopopulate. As of this writing, the only function available was `org.datadog.jenkins.plugins.datadog.DatadogBuildListener`.
+Logging is done by utilizing the java.util.Logger, which follows the [best logging practices for Jenkins](https://wiki.jenkins-ci.org/display/JENKINS/Logging). In order to obtain logs, follow the directions listed [here](https://wiki.jenkins-ci.org/display/JENKINS/Logging). When adding a Logger, all Datadog plugin functions start with `org.datadog.jenkins.plugins.datadog.` and the function name you're after should autopopulate. As of this writing, the only function available was `org.datadog.jenkins.plugins.datadog.listeners.DatadogBuildListener`.
 
 ## Release Process
 ### Overview

src/main/java/org/datadog/jenkins/plugins/datadog/DatadogJobProperty.java

@@ -5,6 +5,7 @@
 import hudson.model.*;
 import net.sf.json.JSONObject;
 import org.apache.commons.lang.StringUtils;
+import org.datadog.jenkins.plugins.datadog.listeners.DatadogBuildListener;
 import org.kohsuke.stapler.DataBoundConstructor;
 import org.kohsuke.stapler.DataBoundSetter;
 import org.kohsuke.stapler.StaplerRequest;
@@ -58,6 +59,25 @@ public void setTagProperties(final String tagProperties) {
         this.tagProperties = tagProperties;
     }
 
+    /**
+     * Gets the tagFile set in the job configuration.
+     *
+     * @return a String representing the relative path to a tagFile
+     */
+    public String getTagFile() {
+        return tagFile;
+    }
+
+    /**
+     * Sets the tagFile set in the job configration.
+     *
+     * @param tagFile - a String representing the relative path to a tagFile
+     */
+    @DataBoundSetter
+    public void setTagFile(String tagFile) {
+        this.tagFile = tagFile;
+    }
+
     /**
      * This method is called whenever the Job form is saved. We use the 'on' property
      * to determine if the controls are selected.
@@ -87,25 +107,6 @@ public JobProperty<?> reconfigure(StaplerRequest req, @Nonnull JSONObject form)
         return prop;
     }
 
-    /**
-     * Gets the tagFile set in the job configuration.
-     *
-     * @return a String representing the relative path to a tagFile
-     */
-    public String getTagFile() {
-        return tagFile;
-    }
-
-    /**
-     * Sets the tagFile set in the job configration.
-     *
-     * @param tagFile - a String representing the relative path to a tagFile
-     */
-    @DataBoundSetter
-    public void setTagFile(String tagFile) {
-        this.tagFile = tagFile;
-    }
-
     /**
      * Checks if tagFile was set in the job configuration.
      *
@@ -156,7 +157,7 @@ public String readTagFile(Run r) {
             //invoked, the workspace has not yet been established, so this check is necessary.
             FilePath workspace = r.getExecutor().getCurrentWorkspace();
             if (workspace != null) {
-                FilePath path = new FilePath(workspace, tagFile);
+                FilePath path = new FilePath(workspace, getTagFile());
                 if (path.exists()) {
                     s = path.readToString();
                 }

src/main/java/org/datadog/jenkins/plugins/datadog/DatadogUtilities.java

@@ -27,7 +27,7 @@ public class DatadogUtilities {
     /**
      * @return - The descriptor for the Datadog plugin. In this case the global configuration.
      */
-    public static DatadogGlobalConfiguration getDatadogDescriptor() {
+    public static DatadogGlobalConfiguration getDatadogGlobalDescriptor() {
         Jenkins jenkins = Jenkins.getInstance();
         if (jenkins == null) {
             return null;
@@ -39,7 +39,7 @@ public static DatadogGlobalConfiguration getDatadogDescriptor() {
      * @return - The descriptor for the Datadog plugin. In this case the global configuration.
      */
     public static DatadogClient getDatadogClient() {
-        DatadogGlobalConfiguration descriptor = getDatadogDescriptor();
+        DatadogGlobalConfiguration descriptor = getDatadogGlobalDescriptor();
         return DatadogHttpClient.getInstance(descriptor.getTargetMetricURL(), descriptor.getApiKey());
     }
 
@@ -53,7 +53,7 @@ public static DatadogClient getDatadogClient() {
     public static Map<String, Set<String>> buildExtraTags(Run run, TaskListener listener) {
         Map<String, Set<String>> result = new HashMap<>();
         String jobName = run.getParent().getFullName();
-        final String globalJobTags = getDatadogDescriptor().getGlobalJobTags();
+        final String globalJobTags = getDatadogGlobalDescriptor().getGlobalJobTags();
         final DatadogJobProperty property = DatadogJobProperty.retrieveProperty(run);
         final String workspaceTagFile = property.readTagFile(run);
         try {
@@ -150,7 +150,7 @@ private static Map<String, Set<String>> getTagsFromGlobalJobTags(String jobName,
      * @return a boolean to signify if the jobName is or is not blacklisted.
      */
     private static boolean isJobBlacklisted(final String jobName) {
-        final String blacklistProp = getDatadogDescriptor().getBlacklist();
+        final String blacklistProp = getDatadogGlobalDescriptor().getBlacklist();
         List<String> blacklist = cstrToList(blacklistProp);
         for (String blacklistedJob : blacklist){
             Pattern blacklistedJobPattern = Pattern.compile(blacklistedJob);
@@ -170,7 +170,7 @@ private static boolean isJobBlacklisted(final String jobName) {
      * @return a boolean to signify if the jobName is or is not whitelisted.
      */
     private static boolean isJobWhitelisted(final String jobName) {
-        final String whitelistProp = getDatadogDescriptor().getWhitelist();
+        final String whitelistProp = getDatadogGlobalDescriptor().getWhitelist();
         final List<String> whitelist = cstrToList(whitelistProp);
         for (String whitelistedJob : whitelist){
             Pattern whitelistedJobPattern = Pattern.compile(whitelistedJob);
@@ -222,9 +222,20 @@ private static List<String> convertRegexStringtoList(final String str, String re
     }
 
     public static Map<String, Set<String>> merge(Map<String, Set<String>> dest, Map<String, Set<String>> orig) {
+        if (dest == null) {
+            dest = new HashMap<>();
+        }
+        if (orig == null) {
+            orig = new HashMap<>();
+        }
         for (String o: orig.keySet()){
             Set<String> values = dest.containsKey(o) ? dest.get(o) : new HashSet<String>();
-            values.addAll(orig.get(o));
+            if (values == null) {
+                values = new HashSet<>();
+            }
+            if (orig.get(o) != null) {
+                values.addAll(orig.get(o));
+            }
             dest.put(o, values);
         }
         return dest;
@@ -269,7 +280,7 @@ public static String getHostname(String envVarHostname) {
         // Check hostname configuration from Jenkins
         String hostname = null;
         try {
-            hostname = getDatadogDescriptor().getHostname();
+            hostname = getDatadogGlobalDescriptor().getHostname();
         } catch (NullPointerException e){
             // noop
         }

src/main/java/org/datadog/jenkins/plugins/datadog/clients/DatadogHttpClient.java

@@ -137,7 +137,7 @@ public boolean gauge(String name, long value, String hostname, JSONArray tags) {
 
     private boolean postMetric(String name, float value, String hostname, JSONArray tags, String type) {
         int INTERVAL = 10;
-        
+
         logger.fine(String.format("Sending metric '%s' with value %s", name, String.valueOf(value)));
 
         // Setup data point, of type [<unix_timestamp>, <value>]

src/main/java/org/datadog/jenkins/plugins/datadog/DatadogBuildListener.java → src/main/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogBuildListener.java

@@ -1,13 +1,16 @@
-package org.datadog.jenkins.plugins.datadog;
+package org.datadog.jenkins.plugins.datadog.listeners;
 
 import hudson.Extension;
 import hudson.model.*;
 import hudson.model.listeners.RunListener;
 import net.sf.json.JSONArray;
+import org.datadog.jenkins.plugins.datadog.DatadogClient;
+import org.datadog.jenkins.plugins.datadog.DatadogEvent;
+import org.datadog.jenkins.plugins.datadog.DatadogGlobalConfiguration;
+import org.datadog.jenkins.plugins.datadog.DatadogUtilities;
 import org.datadog.jenkins.plugins.datadog.events.BuildFinishedEventImpl;
 import org.datadog.jenkins.plugins.datadog.events.BuildStartedEventImpl;
 import org.datadog.jenkins.plugins.datadog.model.BuildData;
-import org.kohsuke.stapler.StaplerRequest;
 
 import javax.annotation.Nonnull;
 import java.io.IOException;

src/test/java/org/datadog/jenkins/plugins/datadog/DatadogUtilitiesTest.java

@@ -3,8 +3,7 @@
 import org.junit.Assert;
 import org.junit.Test;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 public class DatadogUtilitiesTest {
 
@@ -49,6 +48,90 @@ public void testLinesToList(){
 
     @Test
     public void testMerge(){
+        Map emptyTags = new HashMap<>();
+        Assert.assertTrue(DatadogUtilities.merge(null, null).equals(new HashMap<String, Set<String>>()));
+        Assert.assertTrue(DatadogUtilities.merge(emptyTags, null).equals(new HashMap<String, Set<String>>()));
+        Assert.assertTrue(DatadogUtilities.merge(null, emptyTags).equals(new HashMap<String, Set<String>>()));
+
+        Map assertionTags = new HashMap<String, Set<String>>();
+        assertionTags.put("name1", new HashSet<String>());
+        Map nullTagValue = new HashMap<>();
+        nullTagValue.put("name1", null);
+        Assert.assertTrue(DatadogUtilities.merge(null, nullTagValue).equals(assertionTags));
+        Assert.assertTrue(DatadogUtilities.merge(nullTagValue, nullTagValue).equals(assertionTags));
+        Assert.assertTrue(DatadogUtilities.merge(nullTagValue, null).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(nullTagValue, emptyTags).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(emptyTags, nullTagValue).toString().equals(assertionTags.toString()));
+        Map emptyTagValue = new HashMap<>();
+        emptyTagValue.put("name1", new HashSet<String>());
+        Assert.assertTrue(DatadogUtilities.merge(emptyTagValue, null).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(null, emptyTagValue).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(emptyTagValue, emptyTags).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(emptyTags, emptyTagValue).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(nullTagValue, emptyTagValue).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(emptyTagValue, nullTagValue).toString().equals(assertionTags.toString()));
+        Map n1v1Tag = new HashMap<>();
+        Set v1 = new HashSet<>();
+        v1.add("value1");
+        n1v1Tag.put("name1", v1);
+        assertionTags = new HashMap<String, Set<String>>();
+        Set assertionV1 = new HashSet<>();
+        assertionV1.add("value1");
+        assertionTags.put("name1", assertionV1);
+        Assert.assertTrue(DatadogUtilities.merge(n1v1Tag, null).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(null, n1v1Tag).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(n1v1Tag, emptyTags).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(emptyTags, n1v1Tag).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(nullTagValue, n1v1Tag).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(n1v1Tag, nullTagValue).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(n1v1Tag, emptyTagValue).toString().equals(assertionTags.toString()));
+        Assert.assertTrue(DatadogUtilities.merge(emptyTagValue, n1v1Tag).toString().equals(assertionTags.toString()));
+
+        Map n1v1TagCopy = new HashMap<>();
+        Set v1Copy = new HashSet<>();
+        v1Copy.add("value1");
+        n1v1TagCopy.put("name1", v1Copy);
+        Assert.assertTrue(DatadogUtilities.merge(n1v1TagCopy, n1v1Tag).toString().equals(assertionTags.toString()));
+
+        Map n1v2Tag = new HashMap<>();
+        Set v2 = new HashSet<>();
+        v2.add("value2");
+        n1v2Tag.put("name1", v2);
+        assertionTags = new HashMap<String, Set<String>>();
+        Set assertionValues = new HashSet<>();
+        assertionValues.add("value1");
+        assertionValues.add("value2");
+        assertionTags.put("name1", assertionValues);
+        Assert.assertTrue(DatadogUtilities.merge(n1v2Tag, n1v1Tag).toString().equals(assertionTags.toString()));
+
+        Map n2v1Tag = new HashMap<>();
+        v1 = new HashSet<>();
+        v1.add("value1");
+        n2v1Tag.put("name2", v1);
+        assertionTags = new HashMap<String, Set<String>>();
+        assertionV1 = new HashSet<>();
+        assertionV1.add("value1");
+        assertionTags.put("name1", assertionV1);
+        assertionV1.add("value1");
+        assertionTags.put("name2", assertionV1);
+        Assert.assertTrue(DatadogUtilities.merge(n2v1Tag, n1v1Tag).toString().equals(assertionTags.toString()));
+
+        n2v1Tag = new HashMap<>();
+        v1 = new HashSet<>();
+        v1.add("value1");
+        n2v1Tag.put("name2", v1);
+        Map n2v1v2Tag = new HashMap<>();
+        Set v1v2 = new HashSet<>();
+        v1v2.add("value1");
+        v1v2.add("value2");
+        n2v1v2Tag.put("name2", v1v2);
+        assertionTags = new HashMap<>();
+        assertionValues = new HashSet<>();
+        assertionValues.add("value1");
+        assertionValues.add("value2");
+        assertionTags.put("name2", assertionValues);
+        Assert.assertTrue(DatadogUtilities.merge(n2v1Tag, n2v1v2Tag).toString() + " - "+ assertionTags.toString(),
+                DatadogUtilities.merge(n2v1Tag, n2v1v2Tag).toString().equals(assertionTags.toString()));
 
     }
 

src/test/java/org/datadog/jenkins/plugins/datadog/DatadogBuildListenerTest.java → src/test/java/org/datadog/jenkins/plugins/datadog/listeners/DatadogBuildListenerTest.java

@@ -1,9 +1,11 @@
-package org.datadog.jenkins.plugins.datadog;
+package org.datadog.jenkins.plugins.datadog.listeners;
 
 import hudson.EnvVars;
 import hudson.model.*;
 import jenkins.model.Jenkins;
+import org.datadog.jenkins.plugins.datadog.DatadogUtilities;
 import org.datadog.jenkins.plugins.datadog.clients.DatadogClientStub;
+import org.datadog.jenkins.plugins.datadog.listeners.DatadogBuildListener;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;