implement merger

.travis.yml

@@ -0,0 +1,30 @@
+sudo: required
+
+language: go
+
+services:
+  - docker
+
+script:
+- docker build -t exporter-merger --no-cache .
+- >
+  docker run
+  --name exporter-merger
+  --entrypoint "sh"
+  -e CGO_ENABLED=0
+  --workdir "/go/src/github.com/rebuy-de/exporter-merger"
+  exporter-merger
+  -euxc "make xc && mkdir releases && mv exporter-merger-* releases"
+- docker cp -L exporter-merger:/go/src/github.com/rebuy-de/exporter-merger/releases ./releases
+- ls -l *
+
+deploy:
+  provider: releases
+  api_key: $GITHUB_TOKEN
+  file_glob: true
+  file: releases/*
+  skip_cleanup: true
+  on:
+    repo: rebuy-de/exporter-merger
+    tags: true
+

Gopkg.toml

@@ -1,29 +1,11 @@
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
 [[constraint]]
   name = "github.com/sirupsen/logrus"
   version = "1.0.4"
 
 [[constraint]]
   branch = "master"
   name = "github.com/spf13/cobra"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/prometheus/common"

LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 reBuy reCommerce GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

README.md

@@ -0,0 +1,37 @@
+# exporter-merger
+
+[![Build Status](https://travis-ci.org/rebuy-de/exporter-merger.svg?branch=master)](https://travis-ci.org/rebuy-de/exporter-merger)
+[![license](https://img.shields.io/github/license/rebuy-de/exporter-merger.svg)]()
+[![GitHub release](https://img.shields.io/github/release/rebuy-de/exporter-merger.svg)]()
+
+Merges Prometheus metrics from multiple sources.
+
+> **Development Status** *exporter-merger* is in an early development phase.
+> Expect incompatible changes and abandoment at any time.
+
+## But Why?!
+
+> [prometheus/prometheus#3756](https://github.com/prometheus/prometheus/issues/3756)
+
+## Usage
+
+*exporter-merger* needs a configuration file. Currently, nothing but URLs are accepted:
+
+```yaml
+exporters:
+- url: http://localhost:9100/metrics
+- url: http://localhost:9101/metrics
+```
+
+To start the exporter:
+
+```
+exporter-merger --config-path merger.yaml --listen-port 8080
+```
+
+## Planned Features
+
+* Allow transforming of metrics from backend exporters.
+  * eg add a prefix to the metric names
+  * eg add labels to the metrics
+* Allow dynamic adding of exporters.

cmd/command.go

@@ -1,20 +1,56 @@
 package cmd
 
 import (
+	"fmt"
+	"net/http"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
 func NewRootCommand() *cobra.Command {
+	app := new(App)
+
 	cmd := &cobra.Command{
 		Use:   "exporter-merger",
 		Short: "merges Prometheus metrics from multiple sources",
+		Run:   app.run,
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			log.SetLevel(log.DebugLevel)
 		},
 	}
 
+	cmd.PersistentFlags().StringVarP(
+		&app.configPath, "config-path", "c", "./merger.yaml",
+		"Path to the configuration file.")
+	cmd.PersistentFlags().IntVar(
+		&app.port, "listen-port", 8080,
+		"Listen port for the HTTP server.")
+
 	cmd.AddCommand(NewVersionCommand())
 
 	return cmd
 }
+
+type App struct {
+	configPath string
+	port       int
+}
+
+func (app *App) run(cmd *cobra.Command, args []string) {
+	config, err := ReadConfig(app.configPath)
+	if err != nil {
+		log.WithField("error", err).Error("failed to load config")
+		return
+	}
+
+	http.Handle("/metrics", Handler{
+		Config: *config,
+	})
+
+	log.Infof("starting HTTP server on port %d", app.port)
+	err = http.ListenAndServe(fmt.Sprintf(":%d", app.port), nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

cmd/config.go

@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+type Config struct {
+	Exporters []Exporter
+}
+
+type Exporter struct {
+	URL string
+}
+
+func ReadConfig(path string) (*Config, error) {
+	var err error
+
+	raw, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	config := new(Config)
+	err = yaml.Unmarshal(raw, config)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse %s", path)
+	}
+
+	log.WithFields(log.Fields{
+		"content": fmt.Sprintf("%#v", config),
+		"path":    path,
+	}).Debug("loaded config file")
+
+	return config, nil
+}

cmd/handler.go

@@ -0,0 +1,72 @@
+package cmd
+
+import (
+	"io"
+	"net/http"
+	"sort"
+
+	prom "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+	log "github.com/sirupsen/logrus"
+)
+
+type Handler struct {
+	Config Config
+}
+
+func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	log.WithFields(log.Fields{
+		"RequestURI": r.RequestURI,
+		"UserAgent":  r.UserAgent(),
+	}).Debug("handling new request")
+	err := h.Merge(w)
+	if err != nil {
+		log.Error(err)
+		w.WriteHeader(500)
+	}
+}
+
+func (h Handler) Merge(w io.Writer) error {
+	mfs := map[string]*prom.MetricFamily{}
+	tp := new(expfmt.TextParser)
+
+	for _, e := range h.Config.Exporters {
+		resp, err := http.Get(e.URL)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+
+		part, err := tp.TextToMetricFamilies(resp.Body)
+		if err != nil {
+			return err
+		}
+
+		for n, mf := range part {
+			mfo, ok := mfs[n]
+			if ok {
+				mfo.Metric = append(mfo.Metric, mf.Metric...)
+			} else {
+				mfs[n] = mf
+			}
+
+		}
+	}
+
+	names := []string{}
+	for n := range mfs {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+
+	enc := expfmt.NewEncoder(w, expfmt.FmtText)
+	for _, n := range names {
+		err := enc.Encode(mfs[n])
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+
+}