From 696e3d143817d2893ecf0fa24f266c66329ccb91 Mon Sep 17 00:00:00 2001 From: Sven Walter Date: Sat, 10 Feb 2018 19:57:44 +0100 Subject: [PATCH 1/2] setup cmd --- .gitignore | 1 + Dockerfile | 11 ++--------- Gopkg.lock | 45 +++++++++++++++++++++++++++++++++++++++++++++ Gopkg.toml | 29 +++++++++++++++++++++++++++++ cmd/command.go | 20 ++++++++++++++++++++ cmd/version.go | 29 +++++++++++++++++++++++++++++ glide.lock | 4 ---- glide.yaml | 2 -- golang.mk | 11 ++++++----- main.go | 9 +++++++-- 10 files changed, 139 insertions(+), 22 deletions(-) create mode 100644 Gopkg.lock create mode 100644 Gopkg.toml create mode 100644 cmd/command.go create mode 100644 cmd/version.go delete mode 100644 glide.lock delete mode 100644 glide.yaml diff --git a/.gitignore b/.gitignore index a177b45..97098fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /exporter-merger* +/vendor/ diff --git a/Dockerfile b/Dockerfile index 40ef38e..ee3a7a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Source: https://github.com/rebuy-de/golang-template # Version: 1.3.1 -FROM golang:1.8-alpine +FROM golang:1.9-alpine RUN apk add --no-cache git make @@ -12,14 +12,7 @@ RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin # Install Go Tools RUN go get -u github.com/golang/lint/golint - -# Install Glide -RUN go get -u github.com/Masterminds/glide/... - -WORKDIR /go/src/github.com/Masterminds/glide - -RUN git checkout v0.12.3 -RUN go install +RUN go get -u github.com/golang/dep/cmd/dep COPY . /go/src/github.com/rebuy-de/exporter-merger WORKDIR /go/src/github.com/rebuy-de/exporter-merger diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 0000000..140d929 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,45 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + +[[projects]] + name = "github.com/sirupsen/logrus" + packages = ["."] + revision = "d682213848ed68c0a260ca37d6dd5ace8423f5ba" + version = "v1.0.4" + +[[projects]] + branch = "master" + name = "github.com/spf13/cobra" + packages = ["."] + revision = "fd32f09af19efc9b1279c54e0d8ed23f66232a15" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = ["ssh/terminal"] + revision = "5119cf507ed5294cc409c092980c7497ee5d6fd2" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix","windows"] + revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "4a8b1e278eee5c448b8d64f808268c35b839e139019ec23d518d9fa3401011be" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 0000000..1611a6c --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,29 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" + + +[[constraint]] + name = "github.com/sirupsen/logrus" + version = "1.0.4" + +[[constraint]] + branch = "master" + name = "github.com/spf13/cobra" diff --git a/cmd/command.go b/cmd/command.go new file mode 100644 index 0000000..a819c64 --- /dev/null +++ b/cmd/command.go @@ -0,0 +1,20 @@ +package cmd + +import ( + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +func NewRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "exporter-merger", + Short: "merges Prometheus metrics from multiple sources", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + log.SetLevel(log.DebugLevel) + }, + } + + cmd.AddCommand(NewVersionCommand()) + + return cmd +} diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 0000000..7e8cbe2 --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,29 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var ( + BuildVersion = "unknown" + BuildDate = "unknown" + BuildHash = "unknown" + BuildEnvironment = "unknown" +) + +func NewVersionCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "version", + Short: "shows version of this application", + Run: func(cmd *cobra.Command, args []string) { + fmt.Printf("version: %s\n", BuildVersion) + fmt.Printf("build date: %s\n", BuildDate) + fmt.Printf("scm hash: %s\n", BuildHash) + fmt.Printf("environment: %s\n", BuildEnvironment) + }, + } + + return cmd +} diff --git a/glide.lock b/glide.lock deleted file mode 100644 index ff3fa27..0000000 --- a/glide.lock +++ /dev/null @@ -1,4 +0,0 @@ -hash: d52863f30988744afe8c4f157173344e3e437669380e3cb21fb135662a09db2a -updated: 2018-02-10T19:24:30.457100154+01:00 -imports: [] -testImports: [] diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index f41fdeb..0000000 --- a/glide.yaml +++ /dev/null @@ -1,2 +0,0 @@ -package: github.com/rebuy-de/exporter-merger -import: [] diff --git a/golang.mk b/golang.mk index 1240d7b..42b9cf9 100644 --- a/golang.mk +++ b/golang.mk @@ -21,15 +21,16 @@ BUILD_FLAGS=-ldflags "\ " GOFILES=$(shell find . -type f -name '*.go' -not -path "./vendor/*") -GOPKGS=$(shell glide nv) +GOPKGS=$(shell go list ./...) default: build -glide.lock: glide.yaml - glide update +Gopkg.lock: Gopkg.toml + dep ensure + touch Gopkg.lock -vendor: glide.lock glide.yaml - glide install +vendor: Gopkg.lock Gopkg.toml + dep ensure touch vendor format: diff --git a/main.go b/main.go index 89490cd..3f0d5b3 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,12 @@ package main -import "fmt" +import ( + "github.com/rebuy-de/exporter-merger/cmd" + log "github.com/sirupsen/logrus" +) func main() { - fmt.Println("test") + if err := cmd.NewRootCommand().Execute(); err != nil { + log.Fatal(err) + } } From 0c202271c9fa980aab4d329fa80abd83baa51b0f Mon Sep 17 00:00:00 2001 From: Sven Walter Date: Sat, 10 Feb 2018 21:15:34 +0100 Subject: [PATCH 2/2] implement merger --- .travis.yml | 30 +++++++++++++++++ Gopkg.lock | 42 ++++++++++++++++++++++-- Gopkg.toml | 26 +++------------ LICENSE | 22 +++++++++++++ README.md | 37 +++++++++++++++++++++ cmd/command.go | 36 ++++++++++++++++++++ cmd/config.go | 40 +++++++++++++++++++++++ cmd/handler.go | 72 ++++++++++++++++++++++++++++++++++++++++ cmd/handler_test.go | 80 +++++++++++++++++++++++++++++++++++++++++++++ merger.yaml | 3 ++ 10 files changed, 363 insertions(+), 25 deletions(-) create mode 100644 .travis.yml create mode 100644 LICENSE create mode 100644 README.md create mode 100644 cmd/config.go create mode 100644 cmd/handler.go create mode 100644 cmd/handler_test.go create mode 100644 merger.yaml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ba9e4fb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,30 @@ +sudo: required + +language: go + +services: + - docker + +script: +- docker build -t exporter-merger --no-cache . +- > + docker run + --name exporter-merger + --entrypoint "sh" + -e CGO_ENABLED=0 + --workdir "/go/src/github.com/rebuy-de/exporter-merger" + exporter-merger + -euxc "make xc && mkdir releases && mv exporter-merger-* releases" +- docker cp -L exporter-merger:/go/src/github.com/rebuy-de/exporter-merger/releases ./releases +- ls -l * + +deploy: + provider: releases + api_key: $GITHUB_TOKEN + file_glob: true + file: releases/* + skip_cleanup: true + on: + repo: rebuy-de/exporter-merger + tags: true + diff --git a/Gopkg.lock b/Gopkg.lock index 140d929..32050bc 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1,12 +1,42 @@ # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. +[[projects]] + name = "github.com/golang/protobuf" + packages = ["proto"] + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" + [[projects]] name = "github.com/inconshreveable/mousetrap" packages = ["."] revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" version = "v1.0" +[[projects]] + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + revision = "3247c84500bff8d9fb6d579d800f20b3e091582c" + version = "v1.0.0" + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + branch = "master" + name = "github.com/prometheus/client_model" + packages = ["go"] + revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" + +[[projects]] + branch = "master" + name = "github.com/prometheus/common" + packages = ["expfmt","internal/bitbucket.org/ww/goautoneg","model"] + revision = "89604d197083d4781071d3c65855d24ecfb0a563" + [[projects]] name = "github.com/sirupsen/logrus" packages = ["."] @@ -17,7 +47,7 @@ branch = "master" name = "github.com/spf13/cobra" packages = ["."] - revision = "fd32f09af19efc9b1279c54e0d8ed23f66232a15" + revision = "be77323fc05148ef091e83b3866c0d47c8e74a8b" [[projects]] name = "github.com/spf13/pflag" @@ -29,7 +59,7 @@ branch = "master" name = "golang.org/x/crypto" packages = ["ssh/terminal"] - revision = "5119cf507ed5294cc409c092980c7497ee5d6fd2" + revision = "9de5f2eaf759b4c4550b3db39fed2e9e5f86f45c" [[projects]] branch = "master" @@ -37,9 +67,15 @@ packages = ["unix","windows"] revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" +[[projects]] + branch = "v2" + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4" + [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "4a8b1e278eee5c448b8d64f808268c35b839e139019ec23d518d9fa3401011be" + inputs-digest = "3abd0cc431decbb566d1be7247d1fd98dec30b86b826abc1b84c45dafc92c190" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 1611a6c..ff66c4e 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,25 +1,3 @@ -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - - [[constraint]] name = "github.com/sirupsen/logrus" version = "1.0.4" @@ -27,3 +5,7 @@ [[constraint]] branch = "master" name = "github.com/spf13/cobra" + +[[constraint]] + branch = "master" + name = "github.com/prometheus/common" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..91672c3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2018 reBuy reCommerce GmbH + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..fec52fe --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# exporter-merger + +[![Build Status](https://travis-ci.org/rebuy-de/exporter-merger.svg?branch=master)](https://travis-ci.org/rebuy-de/exporter-merger) +[![license](https://img.shields.io/github/license/rebuy-de/exporter-merger.svg)]() +[![GitHub release](https://img.shields.io/github/release/rebuy-de/exporter-merger.svg)]() + +Merges Prometheus metrics from multiple sources. + +> **Development Status** *exporter-merger* is in an early development phase. +> Expect incompatible changes and abandoment at any time. + +## But Why?! + +> [prometheus/prometheus#3756](https://github.com/prometheus/prometheus/issues/3756) + +## Usage + +*exporter-merger* needs a configuration file. Currently, nothing but URLs are accepted: + +```yaml +exporters: +- url: http://localhost:9100/metrics +- url: http://localhost:9101/metrics +``` + +To start the exporter: + +``` +exporter-merger --config-path merger.yaml --listen-port 8080 +``` + +## Planned Features + +* Allow transforming of metrics from backend exporters. + * eg add a prefix to the metric names + * eg add labels to the metrics +* Allow dynamic adding of exporters. diff --git a/cmd/command.go b/cmd/command.go index a819c64..c26f8b3 100644 --- a/cmd/command.go +++ b/cmd/command.go @@ -1,20 +1,56 @@ package cmd import ( + "fmt" + "net/http" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) func NewRootCommand() *cobra.Command { + app := new(App) + cmd := &cobra.Command{ Use: "exporter-merger", Short: "merges Prometheus metrics from multiple sources", + Run: app.run, PersistentPreRun: func(cmd *cobra.Command, args []string) { log.SetLevel(log.DebugLevel) }, } + cmd.PersistentFlags().StringVarP( + &app.configPath, "config-path", "c", "./merger.yaml", + "Path to the configuration file.") + cmd.PersistentFlags().IntVar( + &app.port, "listen-port", 8080, + "Listen port for the HTTP server.") + cmd.AddCommand(NewVersionCommand()) return cmd } + +type App struct { + configPath string + port int +} + +func (app *App) run(cmd *cobra.Command, args []string) { + config, err := ReadConfig(app.configPath) + if err != nil { + log.WithField("error", err).Error("failed to load config") + return + } + + http.Handle("/metrics", Handler{ + Config: *config, + }) + + log.Infof("starting HTTP server on port %d", app.port) + err = http.ListenAndServe(fmt.Sprintf(":%d", app.port), nil) + if err != nil { + log.Fatal(err) + } +} diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 0000000..2bbd224 --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "fmt" + "io/ioutil" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +type Config struct { + Exporters []Exporter +} + +type Exporter struct { + URL string +} + +func ReadConfig(path string) (*Config, error) { + var err error + + raw, err := ioutil.ReadFile(path) + if err != nil { + return nil, errors.WithStack(err) + } + + config := new(Config) + err = yaml.Unmarshal(raw, config) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %s", path) + } + + log.WithFields(log.Fields{ + "content": fmt.Sprintf("%#v", config), + "path": path, + }).Debug("loaded config file") + + return config, nil +} diff --git a/cmd/handler.go b/cmd/handler.go new file mode 100644 index 0000000..af98a09 --- /dev/null +++ b/cmd/handler.go @@ -0,0 +1,72 @@ +package cmd + +import ( + "io" + "net/http" + "sort" + + prom "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + log "github.com/sirupsen/logrus" +) + +type Handler struct { + Config Config +} + +func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + log.WithFields(log.Fields{ + "RequestURI": r.RequestURI, + "UserAgent": r.UserAgent(), + }).Debug("handling new request") + err := h.Merge(w) + if err != nil { + log.Error(err) + w.WriteHeader(500) + } +} + +func (h Handler) Merge(w io.Writer) error { + mfs := map[string]*prom.MetricFamily{} + tp := new(expfmt.TextParser) + + for _, e := range h.Config.Exporters { + resp, err := http.Get(e.URL) + if err != nil { + return err + } + defer resp.Body.Close() + + part, err := tp.TextToMetricFamilies(resp.Body) + if err != nil { + return err + } + + for n, mf := range part { + mfo, ok := mfs[n] + if ok { + mfo.Metric = append(mfo.Metric, mf.Metric...) + } else { + mfs[n] = mf + } + + } + } + + names := []string{} + for n := range mfs { + names = append(names, n) + } + sort.Strings(names) + + enc := expfmt.NewEncoder(w, expfmt.FmtText) + for _, n := range names { + err := enc.Encode(mfs[n]) + if err != nil { + return err + } + } + + return nil + +} diff --git a/cmd/handler_test.go b/cmd/handler_test.go new file mode 100644 index 0000000..270bc01 --- /dev/null +++ b/cmd/handler_test.go @@ -0,0 +1,80 @@ +package cmd_test + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/rebuy-de/exporter-merger/cmd" + log "github.com/sirupsen/logrus" +) + +func testExporter(t testing.TB, content string) (cmd.Exporter, func()) { + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, content) + })) + + return cmd.Exporter{ + URL: ts.URL, + }, ts.Close +} + +func TestHandler(t *testing.T) { + log.SetLevel(log.DebugLevel) + + te1, deferrer := testExporter(t, + "foo{} 1\nconflict 2\nshared{meh=\"a\"} 3") + defer deferrer() + + te2, deferrer := testExporter(t, + "bar{} 4\nconflict 5\nshared{meh=\"b\"} 6") + defer deferrer() + + config := cmd.Config{ + Exporters: []cmd.Exporter{ + te1, + te2, + }, + } + + server := httptest.NewServer(cmd.Handler{ + Config: config, + }) + defer server.Close() + + resp, err := http.Get(server.URL) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + t.Fatalf("Received non-200 response: %d\n", resp.StatusCode) + } + + want := `# TYPE bar untyped +bar 4 +# TYPE conflict untyped +conflict 2 +conflict 5 +# TYPE foo untyped +foo 1 +# TYPE shared untyped +shared{meh="a"} 3 +shared{meh="b"} 6 +` + have, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + if want != string(have) { + t.Error("Got wrong response.") + t.Error("Want:") + t.Error(want) + t.Error("Have:") + t.Error(string(have)) + } +} diff --git a/merger.yaml b/merger.yaml new file mode 100644 index 0000000..2ed4f94 --- /dev/null +++ b/merger.yaml @@ -0,0 +1,3 @@ +exporters: +- url: http://localhost:9100/metrics +- url: http://localhost:9101/metrics