Simplify exporting dashboards for Devs

CHANGELOG-developer.asciidoc

@@ -65,3 +65,4 @@ The list below covers the major changes between 6.3.0 and master only.
 - Add `mage.AddPlatforms` to allow to specify dependent platforms when building a beat. {pull}8889[8889]
 - Add `cfgwarn.CheckRemoved6xSetting(s)` to display a warning for options removed in 7.0. {pull}8909[8909]
 - Add docker image building to `mage.Package`. {pull}8898[8898]
+- Simplified exporting of dashboards. {pull}7730[7730]

auditbeat/magefile.go

@@ -103,6 +103,15 @@ func Fields() error {
 	return mage.GenerateFieldsYAML("module")
 }
 
+// ExportDashboard exports a dashboard and writes it into the correct directory
+//
+// Required ENV variables:
+// * MODULE: Name of the module
+// * ID: Dashboard id
+func ExportDashboard() error {
+	return mage.ExportDashboard()
+}
+
 // GoTestUnit executes the Go unit tests.
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.

dev-tools/cmd/dashboards/export_dashboards.go

@@ -89,6 +89,7 @@ func Export(client *http.Client, conn string, spaceID string, dashboard string,
 	for _, obj := range objects {
 		o := obj.(common.MapStr)
 
+		// All fields are optional, so errors are not catched
 		decodeValue(o, "attributes.uiStateJSON")
 		decodeValue(o, "attributes.visState")
 		decodeValue(o, "attributes.optionsJSON")
@@ -104,23 +105,27 @@ func Export(client *http.Client, conn string, spaceID string, dashboard string,
 		return err
 	}
 
-	err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0666)
+	err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0644)
 	if !quiet {
 		log.Printf("The dashboard %s was exported under the %s file\n", dashboard, out)
 	}
 	return err
 }
 
-func decodeValue(data common.MapStr, key string) {
+func decodeValue(data common.MapStr, key string) error {
 	v, err := data.GetValue(key)
 	if err != nil {
-		return
+		return err
 	}
 	s := v.(string)
 	var d interface{}
-	json.Unmarshal([]byte(s), &d)
+	err = json.Unmarshal([]byte(s), &d)
+	if err != nil {
+		return fmt.Errorf("error decoding %s: %v", key, err)
+	}
 
 	data.Put(key, d)
+	return nil
 }
 
 func ReadManifest(file string) ([]map[string]string, error) {

dev-tools/mage/dashboard.go

@@ -0,0 +1,53 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/magefile/mage/sh"
+)
+
+// ExportDashboard exports a dashboard from Kibana and writes it into the given module.
+func ExportDashboard() error {
+	module := EnvOr("MODULE", "")
+	if module == "" {
+		return fmt.Errorf("MODULE must be specified")
+	}
+
+	id := EnvOr("ID", "")
+	if id == "" {
+		return fmt.Errorf("Dashboad ID must be specified")
+	}
+
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		return err
+	}
+
+	// TODO: This is currently hardcoded for KB 6, we need to figure out what we do for KB 7
+	file := CWD("module", module, "_meta/kibana/6/dashboard", id+".json")
+
+	dashboardCmd := sh.RunCmd("go", "run",
+		filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"),
+		"-output", file, "-dashboard", id,
+	)
+
+	return dashboardCmd()
+}

docs/devguide/newdashboards.asciidoc

@@ -213,44 +213,24 @@ https://github.com/elastic/beats/tree/master/dev-tools/cmd/dashboards[dev-tools]
 for exporting Kibana 5.x dashboards. See the dev-tools
 https://github.com/elastic/beats/tree/master/dev-tools/README.md[readme] for more info.
 
-/////////////////////
-NOT YET IMPLEMENTED
-NOTE: You can make use of the Makefile from the Beat GitHub repository to export all the Kibana dashboards for a Beat
-from your Elasticsearch. If Elasticsearch is running on localhost, then you just need to run the following command from the Beat repository:
-
-[source,shell]
------------------------------
-make export-dashboards
------------------------------
-
-If Elasticsearch is running on a different host, then you can use the `ES_URL` variable:
-
-[source,shell]
-----------------------------
-ES_URL="http://192.168.3.206:9200" make export-dashboards
-----------------------------
-
-/////////////////////
-
 ==== Exporting Kibana 6.0 dashboards and newer
 
 The `dev-tools/cmd/export_dashboards.go` script helps you export your customized Kibana 6.0 dashboards and newer. You might need to export a single dashboard or all the dashboards available for a module or Beat.
 
-
 ===== Export a single Kibana dashboard
 
-You can export a single dashboard by passing the dashboard ID in the `-dashboard` flag.
-
-NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is
-`app/kibana#/dashboard/7fea2930-478e-11e7-b1f0-cb29bac6bf8b?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `7fea2930-478e-11e7-b1f0-cb29bac6bf8b`.
+To export a single dashboard for a module you can use the following command inside a Beat with modules:
 
 [source,shell]
 ---------------
-cd filebeat/module/redis/_meta/kibana/default/dashboard
-go run ../../../../../../../dev-tools/cmd/dashboards/export_dashboards.go -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b -output Filebeat-redis.json
+MODULE=redis ID=AV4REOpp5NkDleZmzKkE mage exportDashboard
 ---------------
 
-This generates the `Filebeat-redis.json` file that contains the dashboard for the Redis module of Filebeat, including the dependencies (visualizations and searches).
+This generates a `AV4REOpp5NkDleZmzKkE.json` file inside dashboard directory in the redis module.
+It contains all dependencies like visualizations and searches.
+
+NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is
+`app/kibana#/dashboard/AV4REOpp5NkDleZmzKkE?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `AV4REOpp5NkDleZmzKkE`.
 
 ===== Export all module/Beat dashboards
 

filebeat/magefile.go

@@ -125,6 +125,15 @@ func GoTestIntegration(ctx context.Context) error {
 	return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs())
 }
 
+// ExportDashboard exports a dashboard and writes it into the correct directory
+//
+// Required ENV variables:
+// * MODULE: Name of the module
+// * ID: Dashboard id
+func ExportDashboard() error {
+	return mage.ExportDashboard()
+}
+
 // -----------------------------------------------------------------------------
 // Customizations specific to Filebeat.
 // - Include modules directory in packages (minus _meta and test files).

metricbeat/magefile.go

@@ -117,6 +117,15 @@ func GoTestIntegration(ctx context.Context) error {
 	return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs())
 }
 
+// ExportDashboard exports a dashboard and writes it into the correct directory
+//
+// Required ENV variables:
+// * MODULE: Name of the module
+// * ID: Dashboard id
+func ExportDashboard() error {
+	return mage.ExportDashboard()
+}
+
 // -----------------------------------------------------------------------------
 // Customizations specific to Metricbeat.
 // - Include modules.d directory in packages.