Skip to content

Commit

Permalink
Revert "[Agent] Remove heartbeat from bundled beats in agent." (#22695)
Browse files Browse the repository at this point in the history
Co-authored-by: Andrew Cholakian <andrew@andrewvc.com>
  • Loading branch information
shahzad31 and andrewvc committed Feb 8, 2021
1 parent 35e2e99 commit f87eac2
Show file tree
Hide file tree
Showing 5 changed files with 204 additions and 14 deletions.
41 changes: 41 additions & 0 deletions dev-tools/packaging/packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,16 @@ shared:
source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
mode: 0644
skip_on_missing: true
/var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
/var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
mode: 0644
/var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
mode: 0644
skip_on_missing: true
/var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
Expand Down Expand Up @@ -179,6 +189,17 @@ shared:
source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
mode: 0644
skip_on_missing: true
/etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
/etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
mode: 0644
skip_on_missing: true
/etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
mode: 0644
skip_on_missing: true
/etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
Expand Down Expand Up @@ -273,6 +294,16 @@ shared:
source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
mode: 0644
skip_on_missing: true
'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
mode: 0644
'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
mode: 0644
skip_on_missing: true
'data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
mode: 0644
Expand Down Expand Up @@ -336,6 +367,16 @@ shared:
source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
mode: 0644
skip_on_missing: true
'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
mode: 0644
'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512'
mode: 0644
'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
mode: 0644
skip_on_missing: true
'data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
mode: 0644
Expand Down
78 changes: 65 additions & 13 deletions heartbeat/monitors/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
package monitors

import (
"fmt"

"github.com/elastic/beats/v7/heartbeat/scheduler"
"github.com/elastic/beats/v7/libbeat/beat"
"github.com/elastic/beats/v7/libbeat/cfgfile"
Expand Down Expand Up @@ -50,9 +52,16 @@ type publishSettings struct {
KeepNull bool `config:"keep_null"`

// Output meta data settings
Pipeline string `config:"pipeline"` // ES Ingest pipeline name
Index fmtstr.EventFormatString `config:"index"` // ES output index pattern
DataSet string `config:"dataset"`
Pipeline string `config:"pipeline"` // ES Ingest pipeline name
Index fmtstr.EventFormatString `config:"index"` // ES output index pattern
DataStream *datastream `config:"data_stream"`
DataSet string `config:"dataset"`
}

type datastream struct {
Namespace string `config:"namespace"`
Dataset string `config:"dataset"`
Type string `config:"type"`
}

// NewFactory takes a scheduler and creates a RunnerFactory that can create cfgfile.Runner(Monitor) objects.
Expand Down Expand Up @@ -83,25 +92,24 @@ func newCommonPublishConfigs(info beat.Info, cfg *common.Config) (pipetool.Confi
return nil, err
}

var indexProcessor processors.Processor
if !settings.Index.IsEmpty() {
staticFields := fmtstr.FieldsForBeat(info.Beat, info.Version)
timestampFormat, err :=
fmtstr.NewTimestampFormatString(&settings.Index, staticFields)
if err != nil {
return nil, err
}
indexProcessor = add_formatted_index.New(timestampFormat)
indexProcessor, err := setupIndexProcessor(info, settings)
if err != nil {
return nil, err
}

userProcessors, err := processors.New(settings.Processors)
if err != nil {
return nil, err
}

// TODO: Remove this logic in the 8.0/master branch, preserve only in 7.x
dataset := settings.DataSet
if dataset == "" {
dataset = "uptime"
if settings.DataStream != nil && settings.DataStream.Dataset != "" {
dataset = settings.DataStream.Dataset
} else {
dataset = "uptime"
}
}

return func(clientCfg beat.ClientConfig) (beat.ClientConfig, error) {
Expand Down Expand Up @@ -140,3 +148,47 @@ func newCommonPublishConfigs(info beat.Info, cfg *common.Config) (pipetool.Confi
return clientCfg, nil
}, nil
}

func setupIndexProcessor(info beat.Info, settings publishSettings) (processors.Processor, error) {
var indexProcessor processors.Processor
if settings.DataStream != nil {
namespace := settings.DataStream.Namespace
if namespace == "" {
namespace = "default"
}
typ := settings.DataStream.Type
if typ == "" {
typ = "synthetics"
}

dataset := settings.DataStream.Dataset
if dataset == "" {
dataset = "generic"
}

index := fmt.Sprintf(
"%s-%s-%s",
typ,
dataset,
namespace,
)
compiled, err := fmtstr.CompileEvent(index)
if err != nil {
return nil, fmt.Errorf("could not compile datastream: '%s', this should never happen: %w", index, err)
} else {
settings.Index = *compiled
}
}

if !settings.Index.IsEmpty() {
staticFields := fmtstr.FieldsForBeat(info.Beat, info.Version)

timestampFormat, err :=
fmtstr.NewTimestampFormatString(&settings.Index, staticFields)
if err != nil {
return nil, err
}
indexProcessor = add_formatted_index.New(timestampFormat)
}
return indexProcessor, nil
}
96 changes: 96 additions & 0 deletions heartbeat/monitors/factory_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package monitors

import (
"testing"

"github.com/stretchr/testify/require"

"github.com/elastic/beats/v7/libbeat/beat"
"github.com/elastic/beats/v7/libbeat/beat/events"
"github.com/elastic/beats/v7/libbeat/common"
"github.com/elastic/beats/v7/libbeat/common/fmtstr"
)

func TestSetupIndexProcessor(t *testing.T) {
binfo := beat.Info{
Beat: "heartbeat",
IndexPrefix: "heartbeat",
Version: "8.0.0",
}
tests := map[string]struct {
settings publishSettings
expectedIndex string
wantProc bool
wantErr bool
}{
"no settings should yield no processor": {
publishSettings{},
"",
false,
false,
},
"exact index should be used exactly": {
publishSettings{Index: *fmtstr.MustCompileEvent("test")},
"test",
true,
false,
},
"data stream should be type-namespace-dataset": {
publishSettings{
DataStream: &datastream{
Type: "myType",
Dataset: "myDataset",
Namespace: "myNamespace",
},
},
"myType-myDataset-myNamespace",
true,
false,
},
"data stream should use defaults": {
publishSettings{
DataStream: &datastream{},
},
"synthetics-generic-default",
true,
false,
},
}

for name, tt := range tests {
t.Run(name, func(t *testing.T) {
e := beat.Event{Meta: common.MapStr{}, Fields: common.MapStr{}}
proc, err := setupIndexProcessor(binfo, tt.settings)
if tt.wantErr == true {
require.Error(t, err)
return
}
require.NoError(t, err)

if !tt.wantProc {
require.Nil(t, proc)
return
}

_, err = proc.Run(&e)
require.Equal(t, tt.expectedIndex, e.Meta[events.FieldMetaRawIndex])
})
}
}
1 change: 1 addition & 0 deletions x-pack/elastic-agent/CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
- Improved version CLI {pull}20359[20359]
- Enroll CLI now restarts running daemon {pull}20359[20359]
- Add restart CLI cmd {pull}20359[20359]
- Add new `synthetics/*` inputs to run Heartbeat {pull}20387[20387]
- Users of the Docker image can now pass `FLEET_ENROLL_INSECURE=1` to include the `--insecure` flag with the `elastic-agent enroll` command {issue}20312[20312] {pull}20713[20713]
- Add `docker` composable dynamic provider. {pull}20842[20842]
- Add support for dynamic inputs with providers and `{{variable|"default"}}` substitution. {pull}20839[20839]
Expand Down
2 changes: 1 addition & 1 deletion x-pack/elastic-agent/magefile.go
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,7 @@ func packageAgent(requiredPackages []string, packagingFn func()) {
defer os.RemoveAll(dropPath)
defer os.Unsetenv(agentDropPath)

packedBeats := []string{"filebeat", "metricbeat"}
packedBeats := []string{"filebeat", "heartbeat", "metricbeat"}

for _, b := range packedBeats {
pwd, err := filepath.Abs(filepath.Join("..", b))
Expand Down

0 comments on commit f87eac2

Please sign in to comment.